[root@oak-hn01 ~]# clush -w oak-io[1-6]-s[1-2] -b lnetctl peer show --nid 10.51.0.132@o2ib3 -v 4 |& tee -a /tmp/peer_show_132 oak-io1-s1: show: oak-io1-s1: - peer: oak-io1-s1: errno: -2 oak-io1-s1: descr: "cannot get peer information: No such file or directory" --------------- oak-io1-s2 --------------- peer: - primary nid: 10.51.0.132@o2ib3 Multi-Rail: True peer state: 75 peer ni: - nid: 10.0.2.217@o2ib5 state: up max_ni_tx_credits: 8 available_tx_credits: 8 min_tx_credits: -333 tx_q_num_of_buf: 0 available_rtr_credits: 8 min_rtr_credits: 8 refcount: 4 statistics: send_count: 197512173 recv_count: 182196149 drop_count: 0 sent_stats: put: 197393198 get: 118975 reply: 0 ack: 0 hello: 0 received_stats: put: 163400666 get: 477 reply: 118532 ack: 18676474 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 70 timeout: 0 error: 0 network timeout: 0 - nid: 10.51.0.132@o2ib3 state: NA max_ni_tx_credits: 0 available_tx_credits: 0 min_tx_credits: 0 tx_q_num_of_buf: 0 available_rtr_credits: 0 min_rtr_credits: 0 refcount: 2 statistics: send_count: 0 recv_count: 0 drop_count: 0 sent_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 received_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 0 timeout: 0 error: 0 network timeout: 0 --------------- oak-io2-s1 --------------- peer: - primary nid: 10.51.0.132@o2ib3 Multi-Rail: True peer state: 75 peer ni: - nid: 10.0.2.217@o2ib5 state: up max_ni_tx_credits: 8 available_tx_credits: 8 min_tx_credits: -3881 tx_q_num_of_buf: 0 available_rtr_credits: 8 min_rtr_credits: 8 refcount: 4 statistics: send_count: 647957939 recv_count: 600942086 drop_count: 0 sent_stats: put: 635540332 get: 12417607 reply: 0 ack: 0 hello: 0 received_stats: put: 481822885 get: 476 reply: 11256852 ack: 107861873 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 1341 timeout: 0 error: 0 network timeout: 0 - nid: 10.51.0.132@o2ib3 state: NA max_ni_tx_credits: 0 available_tx_credits: 0 min_tx_credits: 0 tx_q_num_of_buf: 0 available_rtr_credits: 0 min_rtr_credits: 0 refcount: 2 statistics: send_count: 0 recv_count: 0 drop_count: 0 sent_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 received_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 0 timeout: 0 error: 0 network timeout: 0 --------------- oak-io2-s2 --------------- peer: - primary nid: 10.51.0.132@o2ib3 Multi-Rail: True peer state: 75 peer ni: - nid: 10.0.2.217@o2ib5 state: up max_ni_tx_credits: 8 available_tx_credits: 8 min_tx_credits: -4129 tx_q_num_of_buf: 0 available_rtr_credits: 8 min_rtr_credits: 8 refcount: 4 statistics: send_count: 780997387 recv_count: 730144003 drop_count: 0 sent_stats: put: 768447023 get: 12550364 reply: 0 ack: 0 hello: 0 received_stats: put: 508054425 get: 471 reply: 11450421 ack: 210638686 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 686 timeout: 0 error: 0 network timeout: 0 - nid: 10.51.0.132@o2ib3 state: NA max_ni_tx_credits: 0 available_tx_credits: 0 min_tx_credits: 0 tx_q_num_of_buf: 0 available_rtr_credits: 0 min_rtr_credits: 0 refcount: 2 statistics: send_count: 0 recv_count: 0 drop_count: 0 sent_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 received_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 0 timeout: 0 error: 0 network timeout: 0 --------------- oak-io3-s1 --------------- peer: - primary nid: 10.51.0.132@o2ib3 Multi-Rail: True peer state: 75 peer ni: - nid: 10.0.2.217@o2ib5 state: up max_ni_tx_credits: 8 available_tx_credits: 8 min_tx_credits: -7164 tx_q_num_of_buf: 0 available_rtr_credits: 8 min_rtr_credits: 8 refcount: 4 statistics: send_count: 718863540 recv_count: 670015915 drop_count: 0 sent_stats: put: 706079654 get: 12783886 reply: 0 ack: 0 hello: 0 received_stats: put: 534240302 get: 481 reply: 11744529 ack: 124030603 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 1070 timeout: 0 error: 0 network timeout: 0 - nid: 10.51.0.132@o2ib3 state: NA max_ni_tx_credits: 0 available_tx_credits: 0 min_tx_credits: 0 tx_q_num_of_buf: 0 available_rtr_credits: 0 min_rtr_credits: 0 refcount: 2 statistics: send_count: 0 recv_count: 0 drop_count: 0 sent_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 received_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 0 timeout: 0 error: 0 network timeout: 0 --------------- oak-io3-s2 --------------- peer: - primary nid: 10.51.0.132@o2ib3 Multi-Rail: True peer state: 75 peer ni: - nid: 10.0.2.217@o2ib5 state: up max_ni_tx_credits: 8 available_tx_credits: 8 min_tx_credits: -9039 tx_q_num_of_buf: 0 available_rtr_credits: 8 min_rtr_credits: 8 refcount: 4 statistics: send_count: 699525621 recv_count: 649854984 drop_count: 0 sent_stats: put: 684133102 get: 15392519 reply: 0 ack: 0 hello: 0 received_stats: put: 559463167 get: 476 reply: 14069901 ack: 76321440 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 814 timeout: 0 error: 0 network timeout: 0 - nid: 10.51.0.132@o2ib3 state: NA max_ni_tx_credits: 0 available_tx_credits: 0 min_tx_credits: 0 tx_q_num_of_buf: 0 available_rtr_credits: 0 min_rtr_credits: 0 refcount: 2 statistics: send_count: 0 recv_count: 0 drop_count: 0 sent_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 received_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 0 timeout: 0 error: 0 network timeout: 0 --------------- oak-io4-s1 --------------- peer: - primary nid: 10.51.0.132@o2ib3 Multi-Rail: True peer state: 75 peer ni: - nid: 10.0.2.217@o2ib5 state: up max_ni_tx_credits: 8 available_tx_credits: -2 min_tx_credits: -9227 tx_q_num_of_buf: 8422504 available_rtr_credits: 8 min_rtr_credits: 8 refcount: 15 statistics: send_count: 818553121 recv_count: 773332734 drop_count: 0 sent_stats: put: 796595370 get: 21957751 reply: 0 ack: 0 hello: 0 received_stats: put: 634979443 get: 470 reply: 20464046 ack: 117888775 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 5176 timeout: 0 error: 0 network timeout: 0 - nid: 10.51.0.132@o2ib3 state: NA max_ni_tx_credits: 0 available_tx_credits: 0 min_tx_credits: 0 tx_q_num_of_buf: 0 available_rtr_credits: 0 min_rtr_credits: 0 refcount: 2 statistics: send_count: 0 recv_count: 0 drop_count: 0 sent_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 received_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 0 timeout: 0 error: 0 network timeout: 0 --------------- oak-io4-s2 --------------- peer: - primary nid: 10.51.0.132@o2ib3 Multi-Rail: True peer state: 75 peer ni: - nid: 10.0.2.217@o2ib5 state: up max_ni_tx_credits: 8 available_tx_credits: 8 min_tx_credits: -10411 tx_q_num_of_buf: 0 available_rtr_credits: 8 min_rtr_credits: 8 refcount: 4 statistics: send_count: 830960985 recv_count: 786683219 drop_count: 0 sent_stats: put: 808295779 get: 22665206 reply: 0 ack: 0 hello: 0 received_stats: put: 650254584 get: 461 reply: 21179806 ack: 115248368 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 5825 timeout: 0 error: 0 network timeout: 0 - nid: 10.51.0.132@o2ib3 state: NA max_ni_tx_credits: 0 available_tx_credits: 0 min_tx_credits: 0 tx_q_num_of_buf: 0 available_rtr_credits: 0 min_rtr_credits: 0 refcount: 2 statistics: send_count: 0 recv_count: 0 drop_count: 0 sent_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 received_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 0 timeout: 0 error: 0 network timeout: 0 --------------- oak-io5-s1 --------------- peer: - primary nid: 10.51.0.132@o2ib3 Multi-Rail: True peer state: 75 peer ni: - nid: 10.0.2.217@o2ib5 state: up max_ni_tx_credits: 8 available_tx_credits: 8 min_tx_credits: -9635 tx_q_num_of_buf: 0 available_rtr_credits: 8 min_rtr_credits: 8 refcount: 4 statistics: send_count: 2568057991 recv_count: 2193853428 drop_count: 0 sent_stats: put: 2481446118 get: 86611873 reply: 0 ack: 0 hello: 0 received_stats: put: 1644656824 get: 483 reply: 71868574 ack: 477327547 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 24 timeout: 0 error: 0 network timeout: 0 - nid: 10.51.0.132@o2ib3 state: NA max_ni_tx_credits: 0 available_tx_credits: 0 min_tx_credits: 0 tx_q_num_of_buf: 0 available_rtr_credits: 0 min_rtr_credits: 0 refcount: 2 statistics: send_count: 0 recv_count: 0 drop_count: 0 sent_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 received_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 0 timeout: 0 error: 0 network timeout: 0 --------------- oak-io5-s2 --------------- peer: - primary nid: 10.51.0.132@o2ib3 Multi-Rail: True peer state: 75 peer ni: - nid: 10.0.2.217@o2ib5 state: up max_ni_tx_credits: 8 available_tx_credits: 8 min_tx_credits: -9812 tx_q_num_of_buf: 0 available_rtr_credits: 8 min_rtr_credits: 8 refcount: 4 statistics: send_count: 2628182848 recv_count: 2240460148 drop_count: 0 sent_stats: put: 2541087703 get: 87095145 reply: 0 ack: 0 hello: 0 received_stats: put: 1669356512 get: 470 reply: 72259547 ack: 498843619 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 42 timeout: 0 error: 0 network timeout: 0 - nid: 10.51.0.132@o2ib3 state: NA max_ni_tx_credits: 0 available_tx_credits: 0 min_tx_credits: 0 tx_q_num_of_buf: 0 available_rtr_credits: 0 min_rtr_credits: 0 refcount: 2 statistics: send_count: 0 recv_count: 0 drop_count: 0 sent_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 received_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 0 timeout: 0 error: 0 network timeout: 0 --------------- oak-io6-s1 --------------- peer: - primary nid: 10.51.0.132@o2ib3 Multi-Rail: True peer state: 75 peer ni: - nid: 10.0.2.217@o2ib5 state: up max_ni_tx_credits: 8 available_tx_credits: 8 min_tx_credits: -10145 tx_q_num_of_buf: 0 available_rtr_credits: 8 min_rtr_credits: 8 refcount: 4 statistics: send_count: 2860397710 recv_count: 2401763004 drop_count: 0 sent_stats: put: 2667702805 get: 192694905 reply: 0 ack: 0 hello: 0 received_stats: put: 1573661998 get: 464 reply: 156964937 ack: 671135605 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 28 timeout: 0 error: 0 network timeout: 0 - nid: 10.51.0.132@o2ib3 state: NA max_ni_tx_credits: 0 available_tx_credits: 0 min_tx_credits: 0 tx_q_num_of_buf: 0 available_rtr_credits: 0 min_rtr_credits: 0 refcount: 2 statistics: send_count: 0 recv_count: 0 drop_count: 0 sent_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 received_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 clush: oak-io1-s1: exited with exit code 254 ack: 0 hello: 0 health stats: health value: 1000 dropped: 0 timeout: 0 error: 0 network timeout: 0 --------------- oak-io6-s2 --------------- peer: - primary nid: 10.51.0.132@o2ib3 Multi-Rail: True peer state: 75 peer ni: - nid: 10.0.2.217@o2ib5 state: up max_ni_tx_credits: 8 available_tx_credits: 8 min_tx_credits: -10713 tx_q_num_of_buf: 0 available_rtr_credits: 8 min_rtr_credits: 8 refcount: 4 statistics: send_count: 2555301907 recv_count: 2158095715 drop_count: 0 sent_stats: put: 2367911708 get: 187390199 reply: 0 ack: 0 hello: 0 received_stats: put: 1411110410 get: 473 reply: 154716589 ack: 592268243 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 54 timeout: 0 error: 0 network timeout: 0 - nid: 10.51.0.132@o2ib3 state: NA max_ni_tx_credits: 0 available_tx_credits: 0 min_tx_credits: 0 tx_q_num_of_buf: 0 available_rtr_credits: 0 min_rtr_credits: 0 refcount: 2 statistics: send_count: 0 recv_count: 0 drop_count: 0 sent_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 received_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 dropped_stats: put: 0 get: 0 reply: 0 ack: 0 hello: 0 health stats: health value: 1000 dropped: 0 timeout: 0 error: 0 network timeout: 0