[root@oak-hn01 ~]# clush -w oak-io[1-6]-s[1-2] -b lnetctl peer show --nid 10.51.0.131@o2ib3 -v 4 |& tee -a /tmp/peer_show_131 oak-io1-s1: show: oak-io1-s1: - peer: oak-io1-s1: errno: -2 oak-io1-s1: descr: "cannot get peer information: No such file or directory" --------------- oak-io1-s2 --------------- peer: - primary nid: 10.51.0.131@o2ib3 Multi-Rail: True peer state: 75 peer ni: - nid: 10.0.2.216@o2ib5 state: up max_ni_tx_credits: 8 available_tx_credits: 8 min_tx_credits: -462 tx_q_num_of_buf: 0 available_rtr_credits: 8 min_rtr_credits: 8 refcount: 4 statistics: send_count: 188905530 recv_count: 204350919 drop_count: 0 sent_stats: put: 188786475 get: 119055 reply: 0 ack: 0 hello: 0 received_stats: put: 183881807 get: 469 reply: 118548 ack: 20350095 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 1 timeout: 0 error: 0 network timeout: 0 - nid: 10.51.0.131@o2ib3 state: NA max_ni_tx_credits: 0 available_tx_credits: 0 min_tx_credits: 0 tx_q_num_of_buf: 0 available_rtr_credits: 0 min_rtr_credits: 0 refcount: 2 statistics: send_count: 0 recv_count: 0 drop_count: 0 sent_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 received_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 0 timeout: 0 error: 0 network timeout: 0 --------------- oak-io2-s1 --------------- peer: - primary nid: 10.51.0.131@o2ib3 Multi-Rail: True peer state: 75 peer ni: - nid: 10.0.2.216@o2ib5 state: up max_ni_tx_credits: 8 available_tx_credits: 8 min_tx_credits: -6326 tx_q_num_of_buf: 0 available_rtr_credits: 8 min_rtr_credits: 8 refcount: 4 statistics: send_count: 632647323 recv_count: 680164458 drop_count: 0 sent_stats: put: 619130266 get: 13517057 reply: 0 ack: 0 hello: 0 received_stats: put: 549345568 get: 469 reply: 14631667 ack: 116186754 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 23 timeout: 0 error: 0 network timeout: 0 - nid: 10.51.0.131@o2ib3 state: NA max_ni_tx_credits: 0 available_tx_credits: 0 min_tx_credits: 0 tx_q_num_of_buf: 0 available_rtr_credits: 0 min_rtr_credits: 0 refcount: 2 statistics: send_count: 0 recv_count: 0 drop_count: 0 sent_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 received_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 0 timeout: 0 error: 0 network timeout: 0 --------------- oak-io2-s2 --------------- peer: - primary nid: 10.51.0.131@o2ib3 Multi-Rail: True peer state: 75 peer ni: - nid: 10.0.2.216@o2ib5 state: up max_ni_tx_credits: 8 available_tx_credits: 8 min_tx_credits: -4130 tx_q_num_of_buf: 0 available_rtr_credits: 8 min_rtr_credits: 8 refcount: 4 statistics: send_count: 754009629 recv_count: 805348729 drop_count: 0 sent_stats: put: 739906071 get: 14103558 reply: 0 ack: 0 hello: 0 received_stats: put: 572701879 get: 476 reply: 15171205 ack: 217475169 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 10 timeout: 0 error: 0 network timeout: 0 - nid: 10.51.0.131@o2ib3 state: NA max_ni_tx_credits: 0 available_tx_credits: 0 min_tx_credits: 0 tx_q_num_of_buf: 0 available_rtr_credits: 0 min_rtr_credits: 0 refcount: 2 statistics: send_count: 0 recv_count: 0 drop_count: 0 sent_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 received_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 0 timeout: 0 error: 0 network timeout: 0 --------------- oak-io3-s1 --------------- peer: - primary nid: 10.51.0.131@o2ib3 Multi-Rail: True peer state: 75 peer ni: - nid: 10.0.2.216@o2ib5 state: up max_ni_tx_credits: 8 available_tx_credits: 8 min_tx_credits: -7166 tx_q_num_of_buf: 0 available_rtr_credits: 8 min_rtr_credits: 8 refcount: 4 statistics: send_count: 700796384 recv_count: 750200078 drop_count: 0 sent_stats: put: 685769144 get: 15027240 reply: 0 ack: 0 hello: 0 received_stats: put: 602202477 get: 464 reply: 16025993 ack: 131971144 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 9 timeout: 0 error: 0 network timeout: 0 - nid: 10.51.0.131@o2ib3 state: NA max_ni_tx_credits: 0 available_tx_credits: 0 min_tx_credits: 0 tx_q_num_of_buf: 0 available_rtr_credits: 0 min_rtr_credits: 0 refcount: 2 statistics: send_count: 0 recv_count: 0 drop_count: 0 sent_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 received_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 0 timeout: 0 error: 0 network timeout: 0 --------------- oak-io3-s2 --------------- peer: - primary nid: 10.51.0.131@o2ib3 Multi-Rail: True peer state: 75 peer ni: - nid: 10.0.2.216@o2ib5 state: up max_ni_tx_credits: 8 available_tx_credits: 8 min_tx_credits: -9039 tx_q_num_of_buf: 0 available_rtr_credits: 8 min_rtr_credits: 8 refcount: 4 statistics: send_count: 684253589 recv_count: 734468476 drop_count: 0 sent_stats: put: 667060527 get: 17193062 reply: 0 ack: 0 hello: 0 received_stats: put: 631079462 get: 473 reply: 18480592 ack: 84907949 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 3 timeout: 0 error: 0 network timeout: 0 - nid: 10.51.0.131@o2ib3 state: NA max_ni_tx_credits: 0 available_tx_credits: 0 min_tx_credits: 0 tx_q_num_of_buf: 0 available_rtr_credits: 0 min_rtr_credits: 0 refcount: 2 statistics: send_count: 0 recv_count: 0 drop_count: 0 sent_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 received_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 0 timeout: 0 error: 0 network timeout: 0 --------------- oak-io4-s1 --------------- peer: - primary nid: 10.51.0.131@o2ib3 Multi-Rail: True peer state: 75 peer ni: - nid: 10.0.2.216@o2ib5 state: up max_ni_tx_credits: 8 available_tx_credits: 8 min_tx_credits: -9228 tx_q_num_of_buf: 0 available_rtr_credits: 8 min_rtr_credits: 8 refcount: 4 statistics: send_count: 836301246 recv_count: 882180165 drop_count: 0 sent_stats: put: 808877249 get: 27423997 reply: 0 ack: 0 hello: 0 received_stats: put: 722360782 get: 478 reply: 28814363 ack: 131004542 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 28 timeout: 0 error: 0 network timeout: 0 - nid: 10.51.0.131@o2ib3 state: NA max_ni_tx_credits: 0 available_tx_credits: 0 min_tx_credits: 0 tx_q_num_of_buf: 0 available_rtr_credits: 0 min_rtr_credits: 0 refcount: 2 statistics: send_count: 0 recv_count: 0 drop_count: 0 sent_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 received_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 0 timeout: 0 error: 0 network timeout: 0 --------------- oak-io4-s2 --------------- peer: - primary nid: 10.51.0.131@o2ib3 Multi-Rail: True peer state: 75 peer ni: - nid: 10.0.2.216@o2ib5 state: up max_ni_tx_credits: 8 available_tx_credits: 8 min_tx_credits: -9427 tx_q_num_of_buf: 0 available_rtr_credits: 8 min_rtr_credits: 8 refcount: 4 statistics: send_count: 847469572 recv_count: 892397439 drop_count: 0 sent_stats: put: 820103487 get: 27366085 reply: 0 ack: 0 hello: 0 received_stats: put: 735149117 get: 499 reply: 28760047 ack: 128487776 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 63 timeout: 0 error: 0 network timeout: 0 - nid: 10.51.0.131@o2ib3 state: NA max_ni_tx_credits: 0 available_tx_credits: 0 min_tx_credits: 0 tx_q_num_of_buf: 0 available_rtr_credits: 0 min_rtr_credits: 0 refcount: 2 statistics: send_count: 0 recv_count: 0 drop_count: 0 sent_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 received_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 0 timeout: 0 error: 0 network timeout: 0 --------------- oak-io5-s1 --------------- peer: - primary nid: 10.51.0.131@o2ib3 Multi-Rail: True peer state: 75 peer ni: - nid: 10.0.2.216@o2ib5 state: up max_ni_tx_credits: 8 available_tx_credits: 8 min_tx_credits: -5698 tx_q_num_of_buf: 0 available_rtr_credits: 8 min_rtr_credits: 8 refcount: 4 statistics: send_count: 2051822340 recv_count: 2425177513 drop_count: 0 sent_stats: put: 1980911998 get: 70910342 reply: 0 ack: 0 hello: 0 received_stats: put: 1810013167 get: 462 reply: 85622765 ack: 529541119 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 7 timeout: 0 error: 0 network timeout: 0 - nid: 10.51.0.131@o2ib3 state: NA max_ni_tx_credits: 0 available_tx_credits: 0 min_tx_credits: 0 tx_q_num_of_buf: 0 available_rtr_credits: 0 min_rtr_credits: 0 refcount: 2 statistics: send_count: 0 recv_count: 0 drop_count: 0 sent_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 received_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 0 timeout: 0 error: 0 network timeout: 0 --------------- oak-io5-s2 --------------- peer: - primary nid: 10.51.0.131@o2ib3 Multi-Rail: True peer state: 75 peer ni: - nid: 10.0.2.216@o2ib5 state: up max_ni_tx_credits: 8 available_tx_credits: 8 min_tx_credits: -3377 tx_q_num_of_buf: 0 available_rtr_credits: 8 min_rtr_credits: 8 refcount: 4 statistics: send_count: 2091263413 recv_count: 2477908118 drop_count: 0 sent_stats: put: 2019818719 get: 71444694 reply: 0 ack: 0 hello: 0 received_stats: put: 1837631444 get: 475 reply: 86248565 ack: 554027634 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 5 timeout: 0 error: 0 network timeout: 0 - nid: 10.51.0.131@o2ib3 state: NA max_ni_tx_credits: 0 available_tx_credits: 0 min_tx_credits: 0 tx_q_num_of_buf: 0 available_rtr_credits: 0 min_rtr_credits: 0 refcount: 2 statistics: send_count: 0 recv_count: 0 drop_count: 0 sent_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 received_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 0 timeout: 0 error: 0 network timeout: 0 --------------- oak-io6-s1 --------------- peer: - primary nid: 10.51.0.131@o2ib3 Multi-Rail: True peer state: 75 peer ni: - nid: 10.0.2.216@o2ib5 state: up max_ni_tx_credits: 8 available_tx_credits: 8 min_tx_credits: -10146 tx_q_num_of_buf: 0 available_rtr_credits: 8 min_rtr_credits: 8 refcount: 4 statistics: send_count: 2186979686 recv_count: 2646124637 drop_count: 0 sent_stats: put: 2035301501 get: 151678185 reply: 0 ack: 0 hello: 0 received_stats: put: 1730418264 get: 481 reply: 187338923 ack: 728366969 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 6 timeout: 0 error: 0 network timeout: 0 - nid: 10.51.0.131@o2ib3 state: NA max_ni_tx_credits: 0 available_tx_credits: 0 min_tx_credits: 0 tx_q_num_of_buf: 0 available_rtr_credits: 0 min_rtr_credits: 0 refcount: 2 statistics: send_count: 0 recv_count: 0 drop_count: 0 sent_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 received_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 clush: oak-io1-s1: exited with exit code 254 hello: 0 health stats: health value: 1000 dropped: 0 timeout: 0 error: 0 network timeout: 0 --------------- oak-io6-s2 --------------- peer: - primary nid: 10.51.0.131@o2ib3 Multi-Rail: True peer state: 75 peer ni: - nid: 10.0.2.216@o2ib5 state: up max_ni_tx_credits: 8 available_tx_credits: 8 min_tx_credits: -10715 tx_q_num_of_buf: 0 available_rtr_credits: 8 min_rtr_credits: 8 refcount: 4 statistics: send_count: 1993324314 recv_count: 2391000811 drop_count: 0 sent_stats: put: 1841625884 get: 151698430 reply: 0 ack: 0 hello: 0 received_stats: put: 1560619040 get: 472 reply: 184304546 ack: 646076753 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 3 timeout: 0 error: 0 network timeout: 0 - nid: 10.51.0.131@o2ib3 state: NA max_ni_tx_credits: 0 available_tx_credits: 0 min_tx_credits: 0 tx_q_num_of_buf: 0 available_rtr_credits: 0 min_rtr_credits: 0 refcount: 2 statistics: send_count: 0 recv_count: 0 drop_count: 0 sent_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 received_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 0 timeout: 0 error: 0 network timeout: 0