00000002:00080000:97.0F:1470702854.063766:3088:29330:0:(mdc_request.c:782:mdc_close()) @@@ matched open req@c000001b982f7e00 x1542132002412464/t38654719391(38654719391) o101->sultan-MDT0000-mdc-c000000ff0323000@10.37.248.155@o2ib1:12/10 lens 2120/544 e 0 to 0 dl 1470702861 ref 1 fl Complete:RP/4/0 rc 301/301 00010000:00080000:0.0F:1470702858.259365:4176:29070:0:(ldlm_request.c:1330:ldlm_cli_update_pool()) @@@ Zero SLV or Limit found (SLV: 0, Limit: 4823400) req@c0000005736b5700 x1542132002412624/t0(0) o41->sultan-MDT0001-mdc-c000000ff0323000@10.37.248.156@o2ib1:12/10 lens 224/336 e 0 to 0 dl 1470702865 ref 2 fl Rpc:R/0/0 rc 0/0 00010000:00080000:0.0:1470702858.260380:4176:29070:0:(ldlm_request.c:1330:ldlm_cli_update_pool()) @@@ Zero SLV or Limit found (SLV: 0, Limit: 4823400) req@c0000005736b5700 x1542132002412640/t0(0) o41->sultan-MDT0002-mdc-c000000ff0323000@10.37.248.157@o2ib1:12/10 lens 224/336 e 0 to 0 dl 1470702865 ref 2 fl Rpc:R/0/0 rc 0/0 00010000:00080000:0.0:1470702858.260775:4176:29070:0:(ldlm_request.c:1330:ldlm_cli_update_pool()) @@@ Zero SLV or Limit found (SLV: 0, Limit: 4823400) req@c0000005736b5700 x1542132002412656/t0(0) o41->sultan-MDT0003-mdc-c000000ff0323000@10.37.248.158@o2ib1:12/10 lens 224/336 e 0 to 0 dl 1470702865 ref 2 fl Rpc:R/0/0 rc 0/0 00010000:00080000:0.0:1470702858.261134:4176:29070:0:(ldlm_request.c:1330:ldlm_cli_update_pool()) @@@ Zero SLV or Limit found (SLV: 0, Limit: 4823400) req@c0000005736b5700 x1542132002412672/t0(0) o41->sultan-MDT0004-mdc-c000000ff0323000@10.37.248.159@o2ib1:12/10 lens 224/336 e 0 to 0 dl 1470702865 ref 2 fl Rpc:R/0/0 rc 0/0 00010000:00080000:0.0:1470702858.261495:4176:29070:0:(ldlm_request.c:1330:ldlm_cli_update_pool()) @@@ Zero SLV or Limit found (SLV: 0, Limit: 4823400) req@c0000005736b5700 x1542132002412688/t0(0) o41->sultan-MDT0005-mdc-c000000ff0323000@10.37.248.160@o2ib1:12/10 lens 224/336 e 0 to 0 dl 1470702865 ref 2 fl Rpc:R/0/0 rc 0/0 00010000:00080000:0.0:1470702858.261796:4176:29070:0:(ldlm_request.c:1330:ldlm_cli_update_pool()) @@@ Zero SLV or Limit found (SLV: 0, Limit: 4823400) req@c0000005736b5700 x1542132002412704/t0(0) o41->sultan-MDT0006-mdc-c000000ff0323000@10.37.248.161@o2ib1:12/10 lens 224/336 e 0 to 0 dl 1470702865 ref 2 fl Rpc:R/0/0 rc 0/0 00010000:00080000:0.0:1470702858.262184:4176:29070:0:(ldlm_request.c:1330:ldlm_cli_update_pool()) @@@ Zero SLV or Limit found (SLV: 0, Limit: 4823400) req@c0000005736b5700 x1542132002412720/t0(0) o41->sultan-MDT0007-mdc-c000000ff0323000@10.37.248.162@o2ib1:12/10 lens 224/336 e 0 to 0 dl 1470702865 ref 2 fl Rpc:R/0/0 rc 0/0 00010000:00080000:0.0:1470702858.262567:4176:29070:0:(ldlm_request.c:1330:ldlm_cli_update_pool()) @@@ Zero SLV or Limit found (SLV: 0, Limit: 4823400) req@c0000005736b5700 x1542132002412736/t0(0) o41->sultan-MDT0008-mdc-c000000ff0323000@10.37.248.163@o2ib1:12/10 lens 224/336 e 0 to 0 dl 1470702865 ref 2 fl Rpc:R/0/0 rc 0/0 00010000:00080000:0.0:1470702858.270041:4176:29070:0:(ldlm_request.c:1330:ldlm_cli_update_pool()) @@@ Zero SLV or Limit found (SLV: 0, Limit: 4823400) req@c0000005736b5700 x1542132002412752/t0(0) o41->sultan-MDT0009-mdc-c000000ff0323000@10.37.248.164@o2ib1:12/10 lens 224/336 e 0 to 0 dl 1470702865 ref 2 fl Rpc:R/0/0 rc 0/0 00010000:00080000:0.0:1470702858.270422:4176:29070:0:(ldlm_request.c:1330:ldlm_cli_update_pool()) @@@ Zero SLV or Limit found (SLV: 0, Limit: 4823400) req@c0000005736b5700 x1542132002412768/t0(0) o41->sultan-MDT000a-mdc-c000000ff0323000@10.37.248.165@o2ib1:12/10 lens 224/336 e 0 to 0 dl 1470702865 ref 2 fl Rpc:R/0/0 rc 0/0 00010000:00080000:0.0:1470702858.270809:4176:29070:0:(ldlm_request.c:1330:ldlm_cli_update_pool()) @@@ Zero SLV or Limit found (SLV: 0, Limit: 4823400) req@c0000005736b5700 x1542132002412784/t0(0) o41->sultan-MDT000b-mdc-c000000ff0323000@10.37.248.166@o2ib1:12/10 lens 224/336 e 0 to 0 dl 1470702865 ref 2 fl Rpc:R/0/0 rc 0/0 00010000:00080000:0.0:1470702858.271187:4176:29070:0:(ldlm_request.c:1330:ldlm_cli_update_pool()) @@@ Zero SLV or Limit found (SLV: 0, Limit: 4823400) req@c0000005736b5700 x1542132002412800/t0(0) o41->sultan-MDT000c-mdc-c000000ff0323000@10.37.248.167@o2ib1:12/10 lens 224/336 e 0 to 0 dl 1470702865 ref 2 fl Rpc:R/0/0 rc 0/0 00010000:00080000:0.0:1470702858.271554:4176:29070:0:(ldlm_request.c:1330:ldlm_cli_update_pool()) @@@ Zero SLV or Limit found (SLV: 0, Limit: 4823400) req@c0000005736b5700 x1542132002412816/t0(0) o41->sultan-MDT000d-mdc-c000000ff0323000@10.37.248.168@o2ib1:12/10 lens 224/336 e 0 to 0 dl 1470702865 ref 2 fl Rpc:R/0/0 rc 0/0 00010000:00080000:0.0:1470702858.271917:4176:29070:0:(ldlm_request.c:1330:ldlm_cli_update_pool()) @@@ Zero SLV or Limit found (SLV: 0, Limit: 4823400) req@c0000005736b5700 x1542132002412832/t0(0) o41->sultan-MDT000e-mdc-c000000ff0323000@10.37.248.169@o2ib1:12/10 lens 224/336 e 0 to 0 dl 1470702865 ref 2 fl Rpc:R/0/0 rc 0/0 00010000:00080000:0.0:1470702858.272269:4176:29070:0:(ldlm_request.c:1330:ldlm_cli_update_pool()) @@@ Zero SLV or Limit found (SLV: 0, Limit: 4823400) req@c0000005736b5700 x1542132002412848/t0(0) o41->sultan-MDT000f-mdc-c000000ff0323000@10.37.248.170@o2ib1:12/10 lens 224/336 e 0 to 0 dl 1470702865 ref 2 fl Rpc:R/0/0 rc 0/0 00000800:00020000:45.0F:1470702858.330894:3040:7695:0:(o2iblnd_cb.c:892:kiblnd_post_tx_locked()) Error -12 posting transmit to 10.37.248.83@o2ib1 00000800:00020000:49.0F:1470702858.330915:3040:7692:0:(o2iblnd_cb.c:892:kiblnd_post_tx_locked()) Error -12 posting transmit to 10.37.248.83@o2ib1 00000800:00000100:49.0:1470702858.330921:3280:7692:0:(o2iblnd_cb.c:1918:kiblnd_close_conn_locked()) Closing conn to 10.37.248.83@o2ib1: error -12(waiting) 00000100:00020000:49.0:1470702858.330929:3984:7692:0:(events.c:203:client_bulk_callback()) event type 1, status -12, desc c000000408d02200 00000100:00000400:140.0F:1470702858.330949:2832:7834:0:(client.c:2113:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1470702858/real 1470702858] req@c000000f06283300 x1542132002415760/t0(0) o4->sultan-OST001e-osc-c000000ff0323000@10.37.248.83@o2ib1:6/4 lens 608/448 e 0 to 1 dl 1470702902 ref 2 fl Rpc:eX/0/ffffffff rc 0/-1 00000100:02000400:140.0:1470702858.330968:2672:7834:0:(import.c:182:ptlrpc_set_import_discon()) sultan-OST001e-osc-c000000ff0323000: Connection to sultan-OST001e (at 10.37.248.83@o2ib1) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:140.0:1470702858.330971:2672:7834:0:(import.c:191:ptlrpc_set_import_discon()) c00000078d356800 sultan-OST001e_UUID: changing import state from FULL to DISCONN 00000100:00080000:140.0:1470702858.330974:2592:7834:0:(import.c:417:ptlrpc_pinger_force()) sultan-OST001e_UUID: waking up pinger s:DISCONN 00000800:00020000:94.0F:1470702858.331004:3040:7715:0:(o2iblnd_cb.c:892:kiblnd_post_tx_locked()) Error -12 posting transmit to 10.37.248.77@o2ib1 00000100:00080000:50.0F:1470702858.331006:1488:7934:0:(pinger.c:221:ptlrpc_pinger_process_import()) 085fb3a6-8dec-ef07-c6e6-193e4c60f94c->sultan-OST001e_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:94.0:1470702858.331010:3280:7715:0:(o2iblnd_cb.c:1918:kiblnd_close_conn_locked()) Closing conn to 10.37.248.77@o2ib1: error -12(waiting) 00000100:00080000:50.0:1470702858.331011:1648:7934:0:(recover.c:62:ptlrpc_initiate_recovery()) sultan-OST001e_UUID: starting recovery 00000100:00020000:94.0:1470702858.331012:3984:7715:0:(events.c:203:client_bulk_callback()) event type 1, status -12, desc c000001abacaf400 00000100:00080000:50.0:1470702858.331013:1936:7934:0:(import.c:679:ptlrpc_connect_import()) c00000078d356800 sultan-OST001e_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:50.0:1470702858.331018:2192:7934:0:(import.c:523:import_select_connection()) sultan-OST001e-osc-c000000ff0323000: connect to NID 10.37.248.83@o2ib1 last attempt 4296225008 00000100:00000400:85.0F:1470702858.331025:2832:7897:0:(client.c:2113:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1470702858/real 1470702858] req@c000001cfa96a800 x1542132002416320/t0(0) o4->sultan-OST0031-osc-c000000ff0323000@10.37.248.77@o2ib1:6/4 lens 608/448 e 0 to 1 dl 1470702902 ref 2 fl Rpc:eX/0/ffffffff rc 0/-1 00000100:00080000:50.0:1470702858.331026:2192:7934:0:(import.c:601:import_select_connection()) sultan-OST001e-osc-c000000ff0323000: import c00000078d356800 using connection 10.37.248.83@o2ib1/10.37.248.83@o2ib1 00000100:02000400:85.0:1470702858.331035:2672:7897:0:(import.c:182:ptlrpc_set_import_discon()) sultan-OST0031-osc-c000000ff0323000: Connection to sultan-OST0031 (at 10.37.248.77@o2ib1) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:85.0:1470702858.331037:2672:7897:0:(import.c:191:ptlrpc_set_import_discon()) c00000078c29c800 sultan-OST0031_UUID: changing import state from FULL to DISCONN 00000100:00080000:85.0:1470702858.331039:2592:7897:0:(import.c:417:ptlrpc_pinger_force()) sultan-OST0031_UUID: waking up pinger s:DISCONN 00000100:00080000:50.0:1470702858.331048:1488:7934:0:(pinger.c:221:ptlrpc_pinger_process_import()) 085fb3a6-8dec-ef07-c6e6-193e4c60f94c->sultan-OST0031_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:50.0:1470702858.331051:1648:7934:0:(recover.c:62:ptlrpc_initiate_recovery()) sultan-OST0031_UUID: starting recovery 00000100:00080000:50.0:1470702858.331052:1936:7934:0:(import.c:679:ptlrpc_connect_import()) c00000078c29c800 sultan-OST0031_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:50.0:1470702858.331055:2192:7934:0:(import.c:523:import_select_connection()) sultan-OST0031-osc-c000000ff0323000: connect to NID 10.37.248.77@o2ib1 last attempt 4296217011 00000100:00080000:50.0:1470702858.331059:2192:7934:0:(import.c:601:import_select_connection()) sultan-OST0031-osc-c000000ff0323000: import c00000078c29c800 using connection 10.37.248.77@o2ib1/10.37.248.77@o2ib1 00000100:00080000:50.0:1470702858.331078:1488:7934:0:(pinger.c:221:ptlrpc_pinger_process_import()) 085fb3a6-8dec-ef07-c6e6-193e4c60f94c->sultan-OST0031_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 1 00000100:00080000:50.0:1470702858.331080:1488:7934:0:(pinger.c:234:ptlrpc_pinger_process_import()) 085fb3a6-8dec-ef07-c6e6-193e4c60f94c->sultan-OST0031_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000800:00000100:44.0F:1470702858.331439:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000800:00000100:44.0:1470702858.331540:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000800:00000100:44.0:1470702858.331542:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000800:00000100:44.0:1470702858.331545:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000800:00000100:49.0:1470702858.331558:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000800:00000100:49.0:1470702858.331560:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000800:00000100:44.0:1470702858.331564:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000800:00000100:48.0F:1470702858.331577:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000800:00000100:48.0:1470702858.331580:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000800:00000100:48.0:1470702858.331582:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000800:00000100:49.0:1470702858.331600:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000800:00000100:49.0:1470702858.331602:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000800:00000100:49.0:1470702858.331604:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000800:00000100:48.0:1470702858.331618:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000800:00000100:48.0:1470702858.331621:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000800:00000100:48.0:1470702858.331623:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000800:00000100:49.0:1470702858.331634:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000800:00000100:49.0:1470702858.331636:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000800:00000100:48.0:1470702858.331651:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000800:00000100:48.0:1470702858.331653:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000800:00000100:49.0:1470702858.331664:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000800:00000100:49.0:1470702858.331667:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000800:00000100:48.0:1470702858.331675:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000800:00000100:44.0:1470702858.331692:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000800:00000100:44.0:1470702858.331694:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0909dd0) failed: 5 00000100:00020000:49.0:1470702858.331697:2480:7692:0:(events.c:203:client_bulk_callback()) event type 1, status -5, desc c000000feef6a800 00000100:00000400:0.0:1470702858.331718:2832:7781:0:(client.c:2113:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1470702858/real 1470702858] req@c0000017ce562a00 x1542132002415616/t0(0) o4->sultan-OST0022-osc-c000000ff0323000@10.37.248.83@o2ib1:6/4 lens 608/448 e 0 to 1 dl 1470702902 ref 2 fl Rpc:eX/0/ffffffff rc 0/-1 00000800:00000100:49.0:1470702858.331720:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.331722:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.331732:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000100:02000400:0.0:1470702858.331734:2672:7781:0:(import.c:182:ptlrpc_set_import_discon()) sultan-OST0022-osc-c000000ff0323000: Connection to sultan-OST0022 (at 10.37.248.83@o2ib1) was lost; in progress operations using this service will wait for recovery to complete 00000800:00000100:48.0:1470702858.331736:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000100:00080000:0.0:1470702858.331736:2672:7781:0:(import.c:191:ptlrpc_set_import_discon()) c00000078d35e800 sultan-OST0022_UUID: changing import state from FULL to DISCONN 00000100:00080000:0.0:1470702858.331739:2592:7781:0:(import.c:417:ptlrpc_pinger_force()) sultan-OST0022_UUID: waking up pinger s:DISCONN 00000800:00000100:49.0:1470702858.331750:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.331753:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000100:00080000:50.0:1470702858.331754:1488:7934:0:(pinger.c:221:ptlrpc_pinger_process_import()) 085fb3a6-8dec-ef07-c6e6-193e4c60f94c->sultan-OST0022_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:49.0:1470702858.331756:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000100:00080000:50.0:1470702858.331757:1648:7934:0:(recover.c:62:ptlrpc_initiate_recovery()) sultan-OST0022_UUID: starting recovery 00000100:00080000:50.0:1470702858.331759:1936:7934:0:(import.c:679:ptlrpc_connect_import()) c00000078d35e800 sultan-OST0022_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:50.0:1470702858.331762:2192:7934:0:(import.c:523:import_select_connection()) sultan-OST0022-osc-c000000ff0323000: connect to NID 10.37.248.83@o2ib1 last attempt 4296225008 00000100:00080000:50.0:1470702858.331765:2192:7934:0:(import.c:601:import_select_connection()) sultan-OST0022-osc-c000000ff0323000: import c00000078d35e800 using connection 10.37.248.83@o2ib1/10.37.248.83@o2ib1 00000800:00000100:44.0:1470702858.331767:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.331770:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.331793:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.331795:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.331796:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.331798:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.331799:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.331815:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.331817:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.331817:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.331833:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.331835:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.331837:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.331855:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.331857:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:97.0:1470702858.331861:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.331869:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.331872:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.331874:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.331889:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.331892:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.331893:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.331911:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.331913:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.331923:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.331925:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:97.0:1470702858.332044:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.332046:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.332055:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.332056:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.332058:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:93.0F:1470702858.332067:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.332070:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.332072:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:44.0:1470702858.332072:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.332075:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.332076:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:97.0:1470702858.332087:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.332088:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:49.0:1470702858.332090:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.332093:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:94.0:1470702858.332101:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:94.0:1470702858.332103:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:44.0:1470702858.332104:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.332107:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.332108:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0F:1470702858.332115:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.332118:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.332119:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:44.0:1470702858.332126:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.332128:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.332129:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.332133:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.332135:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:44.0:1470702858.332143:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.332146:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:93.0:1470702858.332147:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.332148:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:44.0:1470702858.332149:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:97.0:1470702858.332151:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.332163:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.332165:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:93.0:1470702858.332167:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:44.0:1470702858.332167:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:93.0:1470702858.332169:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.332170:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:49.0:1470702858.332182:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:97.0:1470702858.332184:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:49.0:1470702858.332184:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:97.0:1470702858.332185:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:49.0:1470702858.332187:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:94.0:1470702858.332197:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:94.0:1470702858.332199:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.332201:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.332203:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.332206:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.332210:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.332212:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:44.0:1470702858.332217:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.332220:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:94.0:1470702858.332223:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:94.0:1470702858.332225:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.332237:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.332238:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:49.0:1470702858.332242:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.332245:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.332247:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:93.0:1470702858.332250:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.332252:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.332259:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.332261:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:94.0:1470702858.332263:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.332265:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000100:00020000:45.0:1470702858.332265:3984:7695:0:(events.c:203:client_bulk_callback()) event type 1, status -12, desc c00000064a09d400 00000800:00000100:49.0:1470702858.332272:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.332274:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:93.0:1470702858.332276:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:94.0:1470702858.332278:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.332285:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.332287:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.332290:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.332294:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.332296:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.332299:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:44.0:1470702858.332306:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.332309:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.332310:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:93.0:1470702858.332313:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:94.0:1470702858.332315:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:49.0:1470702858.332323:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:97.0:1470702858.332324:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:49.0:1470702858.332325:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:97.0:1470702858.332326:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:44.0:1470702858.332337:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.332339:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:44.0:1470702858.332340:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.332341:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:94.0:1470702858.332351:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.332351:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:94.0:1470702858.332353:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.332354:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.332358:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:93.0:1470702858.332365:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.332367:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:44.0:1470702858.332370:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.332373:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:94.0:1470702858.332377:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:44.0:1470702858.332377:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000100:00000400:143.0F:1470702858.332378:2832:7842:0:(client.c:2113:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1470702858/real 1470702858] req@c000000bf734f600 x1542132002415744/t0(0) o4->sultan-OST000e-osc-c000000ff0323000@10.37.248.83@o2ib1:6/4 lens 608/448 e 0 to 1 dl 1470702902 ref 2 fl Rpc:eX/0/ffffffff rc 0/-1 00000800:00000100:97.0:1470702858.332379:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000100:02000400:143.0:1470702858.332386:2672:7842:0:(import.c:182:ptlrpc_set_import_discon()) sultan-OST000e-osc-c000000ff0323000: Connection to sultan-OST000e (at 10.37.248.83@o2ib1) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:143.0:1470702858.332387:2672:7842:0:(import.c:191:ptlrpc_set_import_discon()) c00000078d196800 sultan-OST000e_UUID: changing import state from FULL to DISCONN 00000100:00080000:143.0:1470702858.332388:2592:7842:0:(import.c:417:ptlrpc_pinger_force()) sultan-OST000e_UUID: waking up pinger s:DISCONN 00000800:00000100:91.0:1470702858.332392:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:45.0:1470702858.332392:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.332394:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:45.0:1470702858.332394:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.332397:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000100:00080000:50.0:1470702858.332399:1488:7934:0:(pinger.c:221:ptlrpc_pinger_process_import()) 085fb3a6-8dec-ef07-c6e6-193e4c60f94c->sultan-OST000e_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:50.0:1470702858.332402:1648:7934:0:(recover.c:62:ptlrpc_initiate_recovery()) sultan-OST000e_UUID: starting recovery 00000100:00080000:50.0:1470702858.332403:1936:7934:0:(import.c:679:ptlrpc_connect_import()) c00000078d196800 sultan-OST000e_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:93.0:1470702858.332405:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000100:00080000:50.0:1470702858.332406:2192:7934:0:(import.c:523:import_select_connection()) sultan-OST000e-osc-c000000ff0323000: connect to NID 10.37.248.83@o2ib1 last attempt 4296225008 00000800:00000100:93.0:1470702858.332408:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000100:00080000:50.0:1470702858.332409:2192:7934:0:(import.c:601:import_select_connection()) sultan-OST000e-osc-c000000ff0323000: import c00000078d196800 using connection 10.37.248.83@o2ib1/10.37.248.83@o2ib1 00000800:00000100:49.0:1470702858.332409:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.332410:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:49.0:1470702858.332412:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:45.0:1470702858.332414:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:94.0:1470702858.332425:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.332427:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.332429:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.332430:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.332435:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:45.0:1470702858.332436:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:94.0:1470702858.332442:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.332445:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:49.0:1470702858.332448:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.332451:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.332453:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.332455:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.332457:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.332467:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.332468:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:97.0:1470702858.332469:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.332470:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:45.0:1470702858.332475:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.332483:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.332485:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.332488:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.332492:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.332494:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:93.0:1470702858.332497:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.332499:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.332501:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:44.0:1470702858.332508:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:93.0:1470702858.332509:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:44.0:1470702858.332510:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.332512:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:97.0:1470702858.332521:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.332522:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:44.0:1470702858.332528:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.332530:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.332532:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:93.0:1470702858.332535:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:94.0:1470702858.332537:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.332546:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.332549:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.332549:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.332551:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:45.0:1470702858.332561:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:97.0:1470702858.332562:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.332563:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:44.0:1470702858.332564:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.332566:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.332577:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.332579:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:45.0:1470702858.332580:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:45.0:1470702858.332582:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:94.0:1470702858.332589:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:94.0:1470702858.332592:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:49.0:1470702858.332597:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.332600:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:93.0:1470702858.332601:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.332603:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:45.0:1470702858.332603:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:45.0:1470702858.332605:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:94.0:1470702858.332615:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:94.0:1470702858.332617:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.332619:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.332621:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:93.0:1470702858.332628:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.332630:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:49.0:1470702858.332632:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.332635:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.332641:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.332643:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:45.0:1470702858.332645:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:45.0:1470702858.332648:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:94.0:1470702858.332655:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:94.0:1470702858.332657:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.332659:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.332661:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:97.0:1470702858.332667:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.332668:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:49.0:1470702858.332673:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.332675:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.332681:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:20.0F:1470702858.332683:2544:1380:0:(o2iblnd_cb.c:2625:kiblnd_check_reconnect()) 10.37.248.83@o2ib1: reconnect (conn race), 12, 12, msg_size: 4096, queue_depth: 63/63, max_frags: 16/256 00000800:00000100:91.0:1470702858.332684:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:45.0:1470702858.332684:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.332687:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:94.0:1470702858.332694:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.332696:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:49.0:1470702858.332697:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.332700:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.332709:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.332710:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:44.0:1470702858.332712:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:45.0:1470702858.332714:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:93.0:1470702858.332722:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:94.0:1470702858.332724:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.332724:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.332726:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.332736:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.332738:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:49.0:1470702858.332738:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.332742:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:97.0:1470702858.332751:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.332753:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:45.0:1470702858.332755:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:45.0:1470702858.332757:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.332766:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.332769:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:49.0:1470702858.332769:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:97.0:1470702858.332771:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:49.0:1470702858.332771:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.332780:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.332781:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.332784:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:97.0:1470702858.332792:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.332793:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:45.0:1470702858.332795:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.332798:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.332800:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.332806:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.332808:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:45.0:1470702858.332819:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.332822:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:93.0:1470702858.332823:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:94.0:1470702858.332825:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.332828:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:49.0:1470702858.332834:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.332837:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.332841:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:93.0:1470702858.332843:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.332845:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.332847:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:45.0:1470702858.332853:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:45.0:1470702858.332855:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.332858:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:94.0:1470702858.332862:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:94.0:1470702858.332864:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:45.0:1470702858.332865:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:94.0:1470702858.332867:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.332878:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:97.0:1470702858.332880:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.332880:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:97.0:1470702858.332882:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:45.0:1470702858.332891:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:45.0:1470702858.332893:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.332894:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.332896:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:44.0:1470702858.332904:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.332907:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:94.0:1470702858.332909:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.332911:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.332913:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.332918:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.332920:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:94.0:1470702858.332921:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:49.0:1470702858.332923:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:97.0:1470702858.332934:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.332935:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.332937:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.332939:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.332947:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.332949:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:45.0:1470702858.332950:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:45.0:1470702858.332952:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:94.0:1470702858.332961:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.332963:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:49.0:1470702858.332963:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.332966:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:49.0:1470702858.332966:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.332969:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.332977:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:94.0:1470702858.332981:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.332983:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.332985:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:44.0:1470702858.332989:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.332991:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:94.0:1470702858.333000:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:45.0:1470702858.333002:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:93.0:1470702858.333003:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.333005:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:45.0:1470702858.333005:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:94.0:1470702858.333014:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:49.0:1470702858.333016:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.333019:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.333022:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.333027:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.333029:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:49.0:1470702858.333036:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.333038:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:94.0:1470702858.333040:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.333041:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:97.0:1470702858.333042:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.333053:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.333055:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:45.0:1470702858.333055:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:45.0:1470702858.333057:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:93.0:1470702858.333066:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:44.0:1470702858.333069:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.333071:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.333077:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.333079:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.333082:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.333084:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.333087:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:93.0:1470702858.333091:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.333092:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:49.0:1470702858.333096:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.333104:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.333106:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:48.0:1470702858.333108:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:94.0:1470702858.333118:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.333120:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:45.0:1470702858.333121:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:45.0:1470702858.333124:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.333126:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.333132:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.333133:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.333143:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.333144:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.333157:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:94.0:1470702858.333160:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.333162:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.333176:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:94.0:1470702858.333179:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:94.0:1470702858.333182:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.333195:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.333198:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:94.0:1470702858.333208:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:94.0:1470702858.333210:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.333221:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.333223:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.333234:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.333236:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:94.0:1470702858.333252:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:94.0:1470702858.333254:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:94.0:1470702858.333256:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.333268:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.333269:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.333281:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.333283:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.333285:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.333303:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.333305:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.333308:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.333322:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.333324:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.333336:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:91.0:1470702858.333338:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.333349:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.333351:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.333354:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.333368:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:97.0:1470702858.333369:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000800:00000100:93.0:1470702858.333382:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a9b540) failed: 5 00000100:00020000:97.0:1470702858.333386:2480:7713:0:(events.c:203:client_bulk_callback()) event type 1, status -5, desc c000001b987c1c00 00000800:00000100:93.0:1470702858.333401:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.333413:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.333415:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.333426:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.333428:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.333430:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333445:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333448:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.333450:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333465:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.333468:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.333470:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.333484:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.333487:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.333489:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000100:00000400:119.0F:1470702858.333501:2832:7924:0:(client.c:2113:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1470702858/real 1470702858] req@c000001e27b7d200 x1542132002416224/t0(0) o4->sultan-OST002d-osc-c000000ff0323000@10.37.248.77@o2ib1:6/4 lens 608/448 e 0 to 1 dl 1470702902 ref 2 fl Rpc:eX/0/ffffffff rc 0/-1 00000800:00000100:93.0:1470702858.333504:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333506:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.333508:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000100:02000400:119.0:1470702858.333510:2672:7924:0:(import.c:182:ptlrpc_set_import_discon()) sultan-OST002d-osc-c000000ff0323000: Connection to sultan-OST002d (at 10.37.248.77@o2ib1) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:119.0:1470702858.333511:2672:7924:0:(import.c:191:ptlrpc_set_import_discon()) c00000078c294800 sultan-OST002d_UUID: changing import state from FULL to DISCONN 00000100:00080000:119.0:1470702858.333513:2592:7924:0:(import.c:417:ptlrpc_pinger_force()) sultan-OST002d_UUID: waking up pinger s:DISCONN 00000800:00000100:94.0:1470702858.333523:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.333525:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.333528:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000100:00080000:50.0:1470702858.333528:1488:7934:0:(pinger.c:221:ptlrpc_pinger_process_import()) 085fb3a6-8dec-ef07-c6e6-193e4c60f94c->sultan-OST002d_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:50.0:1470702858.333531:1648:7934:0:(recover.c:62:ptlrpc_initiate_recovery()) sultan-OST002d_UUID: starting recovery 00000100:00080000:50.0:1470702858.333532:1936:7934:0:(import.c:679:ptlrpc_connect_import()) c00000078c294800 sultan-OST002d_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:50.0:1470702858.333536:2192:7934:0:(import.c:523:import_select_connection()) sultan-OST002d-osc-c000000ff0323000: connect to NID 10.37.248.77@o2ib1 last attempt 4296212666 00000100:00080000:50.0:1470702858.333540:2192:7934:0:(import.c:601:import_select_connection()) sultan-OST002d-osc-c000000ff0323000: import c00000078c294800 using connection 10.37.248.77@o2ib1/10.37.248.77@o2ib1 00000800:00000100:93.0:1470702858.333543:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.333546:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.333548:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333563:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333566:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.333569:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.333582:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.333585:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333586:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.333602:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.333604:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.333606:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.333621:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.333623:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.333625:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.333641:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.333643:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333654:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.333656:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.333658:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333674:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.333676:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.333678:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333693:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.333696:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.333698:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333713:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333715:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.333717:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.333733:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333735:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.333737:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.333752:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333755:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.333757:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.333772:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333775:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.333777:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.333792:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333795:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333798:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.333810:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.333813:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.333815:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.333830:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.333831:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333844:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.333846:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.333848:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333864:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333866:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.333867:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.333883:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.333885:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.333887:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333903:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333905:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.333906:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.333922:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.333924:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.333926:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333942:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333944:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.333945:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.333962:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.333964:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.333965:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333981:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.333983:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.333985:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.334000:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334001:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:93.0:1470702858.334003:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334004:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334005:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:45.0:1470702858.334007:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.334009:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:45.0:1470702858.334011:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.334013:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:45.0:1470702858.334014:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.334017:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:45.0:1470702858.334019:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.334020:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334021:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.334022:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:93.0:1470702858.334023:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334024:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334024:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:45.0:1470702858.334025:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.334027:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.334029:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.334031:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:45.0:1470702858.334032:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.334034:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:45.0:1470702858.334035:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.334037:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.334039:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:44.0:1470702858.334039:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.334041:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:93.0:1470702858.334042:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:44.0:1470702858.334042:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:97.0:1470702858.334043:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334044:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.334045:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.334047:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.334048:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.334051:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.334052:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.334054:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:48.0:1470702858.334056:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.334057:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.334059:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334059:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.334060:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.334061:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334062:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:97.0:1470702858.334063:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:44.0:1470702858.334063:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.334065:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.334066:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.334068:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:45.0:1470702858.334070:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.334072:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:45.0:1470702858.334072:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:49.0:1470702858.334074:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.334077:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:93.0:1470702858.334078:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:45.0:1470702858.334079:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:44.0:1470702858.334079:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000800:00000100:91.0:1470702858.334080:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:45.0:1470702858.334080:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908ef8) failed: 5 00000100:00020000:44.0:1470702858.334083:2480:7693:0:(events.c:203:client_bulk_callback()) event type 1, status -5, desc c000001772710c00 00000800:00000100:45.0:1470702858.334087:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334088:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334089:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334091:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334092:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:45.0:1470702858.334092:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334094:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334094:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334095:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:45.0:1470702858.334095:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334096:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334096:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334097:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334099:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334100:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334101:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334103:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334105:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334107:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334108:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:97.0:1470702858.334110:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334110:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:97.0:1470702858.334111:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334112:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334113:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334115:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334117:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334118:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334118:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334121:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334122:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334122:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334123:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334123:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:97.0:1470702858.334125:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334126:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334126:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334127:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334130:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334131:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334131:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334134:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334134:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334135:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334136:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334136:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:91.0:1470702858.334137:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334137:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:91.0:1470702858.334139:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334139:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334139:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:97.0:1470702858.334141:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334142:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334143:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334145:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334147:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334149:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334151:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334153:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334156:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334156:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334157:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:91.0:1470702858.334158:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334158:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334158:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334159:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334161:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334164:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334166:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334167:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334169:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334169:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334170:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334171:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334172:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334173:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334173:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334174:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334176:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334177:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334179:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334185:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334187:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334187:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334189:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334189:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334190:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334191:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334193:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334193:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334195:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334196:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334197:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334197:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334200:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:91.0:1470702858.334202:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:45.0:1470702858.334202:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334203:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:91.0:1470702858.334204:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334206:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334206:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334206:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334209:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334209:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334212:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334214:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334216:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334218:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334221:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334221:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334222:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.334223:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334223:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334224:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334225:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334226:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334226:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334228:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334229:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334231:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334232:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:97.0:1470702858.334233:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334234:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334236:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334238:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334240:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334243:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334243:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334245:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334247:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334248:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334249:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334250:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334250:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:97.0:1470702858.334252:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334252:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334253:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334254:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334256:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334257:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334257:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334259:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334260:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334263:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334264:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334266:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334267:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:45.0:1470702858.334267:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334269:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334270:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334271:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:97.0:1470702858.334272:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:45.0:1470702858.334273:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334274:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334275:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000100:00000400:49.0:1470702858.334286:2832:7862:0:(client.c:2113:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1470702858/real 1470702858] req@c0000013dd005700 x1542132002415984/t0(0) o4->sultan-OST002a-osc-c000000ff0323000@10.37.248.83@o2ib1:6/4 lens 608/448 e 0 to 1 dl 1470702902 ref 2 fl Rpc:eX/0/ffffffff rc 0/-1 00000800:00000100:93.0:1470702858.334287:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.334289:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334291:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334293:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334297:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000100:02000400:49.0:1470702858.334299:2672:7862:0:(import.c:182:ptlrpc_set_import_discon()) sultan-OST002a-osc-c000000ff0323000: Connection to sultan-OST002a (at 10.37.248.83@o2ib1) was lost; in progress operations using this service will wait for recovery to complete 00000800:00000100:48.0:1470702858.334300:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000100:00080000:49.0:1470702858.334301:2672:7862:0:(import.c:191:ptlrpc_set_import_discon()) c00000078c28e800 sultan-OST002a_UUID: changing import state from FULL to DISCONN 00000800:00000100:48.0:1470702858.334303:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000100:00080000:49.0:1470702858.334305:2592:7862:0:(import.c:417:ptlrpc_pinger_force()) sultan-OST002a_UUID: waking up pinger s:DISCONN 00000800:00000100:93.0:1470702858.334307:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.334310:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334312:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000100:00080000:50.0:1470702858.334317:1488:7934:0:(pinger.c:221:ptlrpc_pinger_process_import()) 085fb3a6-8dec-ef07-c6e6-193e4c60f94c->sultan-OST002a_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:49.0:1470702858.334318:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000100:00080000:50.0:1470702858.334319:1648:7934:0:(recover.c:62:ptlrpc_initiate_recovery()) sultan-OST002a_UUID: starting recovery 00000100:00080000:50.0:1470702858.334320:1936:7934:0:(import.c:679:ptlrpc_connect_import()) c00000078c28e800 sultan-OST002a_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:49.0:1470702858.334321:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000100:00080000:50.0:1470702858.334324:2192:7934:0:(import.c:523:import_select_connection()) sultan-OST002a-osc-c000000ff0323000: connect to NID 10.37.248.83@o2ib1 last attempt 4296225008 00000800:00000100:49.0:1470702858.334324:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334327:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000100:00080000:50.0:1470702858.334328:2192:7934:0:(import.c:601:import_select_connection()) sultan-OST002a-osc-c000000ff0323000: import c00000078c28e800 using connection 10.37.248.83@o2ib1/10.37.248.83@o2ib1 00000800:00000100:94.0:1470702858.334329:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334331:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334340:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334344:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334346:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334347:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334348:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334349:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334351:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:44.0:1470702858.334364:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334366:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:44.0:1470702858.334366:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334369:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334369:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:97.0:1470702858.334371:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:44.0:1470702858.334384:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334386:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:45.0:1470702858.334386:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:44.0:1470702858.334388:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334389:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334391:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:45.0:1470702858.334404:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334406:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334408:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.334410:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334410:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:97.0:1470702858.334412:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:44.0:1470702858.334423:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334424:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334425:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:93.0:1470702858.334427:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334429:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334437:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334440:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334443:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334444:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334446:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334447:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334448:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:45.0:1470702858.334462:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334464:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:45.0:1470702858.334465:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334466:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334468:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334468:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334481:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334482:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:45.0:1470702858.334483:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334485:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:45.0:1470702858.334485:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:91.0:1470702858.334498:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.334500:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334502:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334503:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:44.0:1470702858.334506:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334507:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334517:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.334519:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334521:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334522:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334525:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:44.0:1470702858.334527:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334536:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.334538:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334540:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334541:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334544:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334547:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334550:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:91.0:1470702858.334556:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.334557:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334560:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:44.0:1470702858.334566:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:44.0:1470702858.334568:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334570:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334574:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.334576:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334578:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:45.0:1470702858.334585:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:44.0:1470702858.334587:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:91.0:1470702858.334594:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.334596:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334598:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334599:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334602:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:44.0:1470702858.334604:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334614:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.334616:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334618:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334619:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334622:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:44.0:1470702858.334623:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:91.0:1470702858.334633:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.334635:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334637:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334639:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334642:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:44.0:1470702858.334644:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334653:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.334655:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334658:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334659:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334662:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:44.0:1470702858.334663:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:91.0:1470702858.334673:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.334674:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:91.0:1470702858.334676:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334678:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334680:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:44.0:1470702858.334682:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334692:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334694:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334695:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334697:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334700:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:44.0:1470702858.334701:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334712:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334713:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334715:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334718:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334721:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334726:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334731:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334732:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:91.0:1470702858.334733:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334735:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334735:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334750:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334750:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334752:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:45.0:1470702858.334754:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334755:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334768:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334768:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334770:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334771:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:97.0:1470702858.334772:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334773:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:45.0:1470702858.334774:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334790:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334792:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334793:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334794:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334795:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:97.0:1470702858.334796:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:44.0:1470702858.334797:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334809:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334809:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334811:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334812:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334814:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334826:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334828:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334830:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334831:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:94.0:1470702858.334834:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334836:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334847:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334848:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334849:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334851:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334852:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334852:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:48.0:1470702858.334866:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334868:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334868:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334870:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:44.0:1470702858.334870:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:97.0:1470702858.334872:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334886:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334887:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334888:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334890:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334890:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:97.0:1470702858.334891:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334905:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334907:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334907:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334909:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334910:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334912:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334924:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334926:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334927:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334928:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334929:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.334930:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334944:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:97.0:1470702858.334945:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:97.0:1470702858.334946:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334947:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:44.0:1470702858.334949:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:91.0:1470702858.334963:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:49.0:1470702858.334963:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:97.0:1470702858.334965:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334965:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:91.0:1470702858.334967:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a96bd0) failed: 5 00000800:00000100:48.0:1470702858.334968:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000100:00020000:97.0:1470702858.334979:2480:7713:0:(events.c:203:client_bulk_callback()) event type 1, status -5, desc c000000f48135a00 00000800:00000100:49.0:1470702858.334983:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.334985:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:45.0:1470702858.334987:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.334998:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:93.0:1470702858.335000:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335002:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:91.0:1470702858.335003:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335005:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.335007:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:93.0:1470702858.335017:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:91.0:1470702858.335019:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335021:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:91.0:1470702858.335022:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335024:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:49.0:1470702858.335026:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000800:00000100:94.0:1470702858.335037:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:94.0:1470702858.335039:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:93.0:1470702858.335041:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335042:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f090b3b0) failed: 5 00000100:00020000:48.0:1470702858.335046:2480:7694:0:(events.c:203:client_bulk_callback()) event type 1, status -5, desc c000001738c04e00 00000800:00000100:94.0:1470702858.335056:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335058:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335059:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000100:00000400:33.0F:1470702858.335059:2832:7813:0:(client.c:2113:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1470702858/real 1470702858] req@c000000785385400 x1542132002415728/t0(0) o4->sultan-OST002e-osc-c000000ff0323000@10.37.248.83@o2ib1:6/4 lens 608/448 e 0 to 1 dl 1470702902 ref 2 fl Rpc:eX/0/ffffffff rc 0/-1 00000800:00000100:91.0:1470702858.335061:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000100:02000400:33.0:1470702858.335068:2672:7813:0:(import.c:182:ptlrpc_set_import_discon()) sultan-OST002e-osc-c000000ff0323000: Connection to sultan-OST002e (at 10.37.248.83@o2ib1) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:33.0:1470702858.335069:2672:7813:0:(import.c:191:ptlrpc_set_import_discon()) c00000078c296800 sultan-OST002e_UUID: changing import state from FULL to DISCONN 00000800:00000100:49.0:1470702858.335070:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000100:00080000:33.0:1470702858.335070:2592:7813:0:(import.c:417:ptlrpc_pinger_force()) sultan-OST002e_UUID: waking up pinger s:DISCONN 00000800:00000100:49.0:1470702858.335072:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335073:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:94.0:1470702858.335075:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000100:00080000:50.0:1470702858.335083:1488:7934:0:(pinger.c:221:ptlrpc_pinger_process_import()) 085fb3a6-8dec-ef07-c6e6-193e4c60f94c->sultan-OST002e_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:50.0:1470702858.335085:1648:7934:0:(recover.c:62:ptlrpc_initiate_recovery()) sultan-OST002e_UUID: starting recovery 00000100:00080000:50.0:1470702858.335086:1936:7934:0:(import.c:679:ptlrpc_connect_import()) c00000078c296800 sultan-OST002e_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:45.0:1470702858.335087:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.335088:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000100:00080000:50.0:1470702858.335089:2192:7934:0:(import.c:523:import_select_connection()) sultan-OST002e-osc-c000000ff0323000: connect to NID 10.37.248.83@o2ib1 last attempt 4296224966 00000800:00000100:93.0:1470702858.335090:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:45.0:1470702858.335090:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335093:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000100:00080000:50.0:1470702858.335093:2192:7934:0:(import.c:601:import_select_connection()) sultan-OST002e-osc-c000000ff0323000: import c00000078c296800 using connection 10.37.248.83@o2ib1/10.37.248.83@o2ib1 00000800:00000100:91.0:1470702858.335095:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:44.0:1470702858.335106:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335107:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:45.0:1470702858.335107:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335110:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:91.0:1470702858.335112:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335120:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.335123:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.335125:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335127:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:93.0:1470702858.335129:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:91.0:1470702858.335132:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:44.0:1470702858.335139:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.335140:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.335142:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335147:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:94.0:1470702858.335149:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:91.0:1470702858.335151:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335152:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.335164:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335166:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:45.0:1470702858.335167:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335168:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335169:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335170:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335185:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335186:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:94.0:1470702858.335188:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335188:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:91.0:1470702858.335190:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:45.0:1470702858.335190:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.335204:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335206:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335206:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:91.0:1470702858.335208:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335209:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:91.0:1470702858.335210:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335224:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335226:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335227:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335228:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:44.0:1470702858.335228:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335230:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335244:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335245:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335246:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335247:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335249:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:91.0:1470702858.335250:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335263:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335264:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335265:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335266:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:94.0:1470702858.335267:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:45.0:1470702858.335267:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.335283:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335284:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335284:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335286:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335287:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335288:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335302:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335303:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335304:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335305:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:94.0:1470702858.335306:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:44.0:1470702858.335307:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.335321:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335323:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335324:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:91.0:1470702858.335326:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335327:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:91.0:1470702858.335328:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335341:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:97.0:1470702858.335342:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:97.0:1470702858.335343:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335343:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.335345:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000100:00000400:107.0F:1470702858.335349:2832:7916:0:(client.c:2113:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1470702858/real 1470702858] req@c000001a3617c900 x1542132002416240/t0(0) o4->sultan-OST0029-osc-c000000ff0323000@10.37.248.77@o2ib1:6/4 lens 608/448 e 0 to 1 dl 1470702902 ref 2 fl Rpc:eX/0/ffffffff rc 0/-1 00000100:02000400:107.0:1470702858.335358:2672:7916:0:(import.c:182:ptlrpc_set_import_discon()) sultan-OST0029-osc-c000000ff0323000: Connection to sultan-OST0029 (at 10.37.248.77@o2ib1) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:107.0:1470702858.335359:2672:7916:0:(import.c:191:ptlrpc_set_import_discon()) c00000078c28c800 sultan-OST0029_UUID: changing import state from FULL to DISCONN 00000800:00000100:94.0:1470702858.335360:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335360:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000100:00080000:107.0:1470702858.335361:2592:7916:0:(import.c:417:ptlrpc_pinger_force()) sultan-OST0029_UUID: waking up pinger s:DISCONN 00000800:00000100:97.0:1470702858.335362:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335362:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335363:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335363:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000100:00080000:50.0:1470702858.335373:1488:7934:0:(pinger.c:221:ptlrpc_pinger_process_import()) 085fb3a6-8dec-ef07-c6e6-193e4c60f94c->sultan-OST0029_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:97.0:1470702858.335375:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000100:00080000:50.0:1470702858.335375:1648:7934:0:(recover.c:62:ptlrpc_initiate_recovery()) sultan-OST0029_UUID: starting recovery 00000800:00000100:97.0:1470702858.335376:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000100:00080000:50.0:1470702858.335376:1936:7934:0:(import.c:679:ptlrpc_connect_import()) c00000078c28c800 sultan-OST0029_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:50.0:1470702858.335379:2192:7934:0:(import.c:523:import_select_connection()) sultan-OST0029-osc-c000000ff0323000: connect to NID 10.37.248.77@o2ib1 last attempt 4296217010 00000800:00000100:49.0:1470702858.335380:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000100:00080000:50.0:1470702858.335382:2192:7934:0:(import.c:601:import_select_connection()) sultan-OST0029-osc-c000000ff0323000: import c00000078c28c800 using connection 10.37.248.77@o2ib1/10.37.248.77@o2ib1 00000800:00000100:49.0:1470702858.335383:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.335385:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335388:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:93.0:1470702858.335390:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:97.0:1470702858.335392:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335402:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.335404:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.335406:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335407:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:93.0:1470702858.335409:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:97.0:1470702858.335411:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335419:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.335422:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.335425:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335428:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:93.0:1470702858.335430:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:97.0:1470702858.335432:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335439:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.335441:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.335445:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335447:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:93.0:1470702858.335449:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:97.0:1470702858.335451:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335456:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.335458:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335465:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:93.0:1470702858.335467:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:45.0:1470702858.335468:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.335471:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:91.0:1470702858.335478:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:91.0:1470702858.335480:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:44.0:1470702858.335482:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.335484:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335492:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:93.0:1470702858.335494:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335496:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:91.0:1470702858.335497:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335498:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335506:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335509:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.335511:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:91.0:1470702858.335518:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:97.0:1470702858.335520:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:44.0:1470702858.335521:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.335522:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335531:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:94.0:1470702858.335533:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335534:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.335536:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335545:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:93.0:1470702858.335547:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335548:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.335550:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:97.0:1470702858.335557:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:97.0:1470702858.335558:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:45.0:1470702858.335561:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.335563:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335570:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:93.0:1470702858.335572:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335575:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.335576:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335583:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:91.0:1470702858.335585:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335587:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.335591:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335596:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:93.0:1470702858.335598:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:44.0:1470702858.335600:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.335601:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:97.0:1470702858.335608:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:97.0:1470702858.335609:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335613:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.335615:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:91.0:1470702858.335622:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:91.0:1470702858.335624:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335627:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.335628:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335636:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:93.0:1470702858.335638:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:45.0:1470702858.335639:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:91.0:1470702858.335641:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:44.0:1470702858.335641:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335649:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335654:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.335656:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.335658:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:91.0:1470702858.335661:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:97.0:1470702858.335663:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335671:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335673:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335673:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:91.0:1470702858.335675:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335684:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335686:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335686:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335689:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:45.0:1470702858.335696:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:97.0:1470702858.335699:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:44.0:1470702858.335699:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:97.0:1470702858.335700:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335710:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:91.0:1470702858.335712:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335712:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:97.0:1470702858.335715:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:44.0:1470702858.335724:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335726:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:45.0:1470702858.335726:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:91.0:1470702858.335729:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335737:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.335739:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335740:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:93.0:1470702858.335742:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:93.0:1470702858.335745:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335749:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.335751:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335758:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:91.0:1470702858.335760:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:45.0:1470702858.335761:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.335764:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335771:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:94.0:1470702858.335773:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335775:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.335777:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:97.0:1470702858.335783:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:97.0:1470702858.335784:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:44.0:1470702858.335788:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.335790:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:91.0:1470702858.335797:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:97.0:1470702858.335799:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:45.0:1470702858.335801:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.335803:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335811:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:94.0:1470702858.335813:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335814:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.335816:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335823:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:93.0:1470702858.335825:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:45.0:1470702858.335827:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.335829:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:97.0:1470702858.335835:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:97.0:1470702858.335836:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:44.0:1470702858.335840:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.335842:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:93.0:1470702858.335849:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:97.0:1470702858.335851:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335853:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.335855:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:94.0:1470702858.335862:1728:7715:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:93.0:1470702858.335864:1728:7714:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335866:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.335868:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:91.0:1470702858.335875:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:91.0:1470702858.335877:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:45.0:1470702858.335879:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.335881:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:97.0:1470702858.335887:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:97.0:1470702858.335888:1728:7713:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:48.0:1470702858.335892:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.335894:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:91.0:1470702858.335901:1728:7712:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c000001e45a98020) failed: 5 00000800:00000100:49.0:1470702858.335906:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.335907:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.335918:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.335920:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.335931:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.335933:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.335944:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.335946:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.335958:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.335959:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.335970:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.335972:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.335984:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.335986:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.336139:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.336141:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336152:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336154:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.336167:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336170:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336172:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.336193:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336194:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.336196:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336198:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.336215:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.336217:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.336219:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336252:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336253:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.336255:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.336256:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336258:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.336260:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336271:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.336274:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336286:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.336288:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336479:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336481:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336492:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336494:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.336505:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.336506:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336518:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336520:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.336531:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.336533:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336544:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336546:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336558:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336560:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.336571:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.336573:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336585:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336586:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336598:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336600:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.336610:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.336612:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.336624:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.336625:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336637:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336640:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336651:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336653:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336664:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336666:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.336678:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.336680:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336694:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.336696:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336707:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336710:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.336719:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.336721:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.336732:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.336734:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336748:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.336750:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.336752:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336766:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.336768:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336779:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336781:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.336790:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.336799:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.336801:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336813:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.336816:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336825:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336828:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.336838:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.336840:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336852:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336854:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336865:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.336867:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.336879:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:45.0:1470702858.336880:1728:7695:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336893:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336895:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336906:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.336908:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336919:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336921:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.336931:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:44.0:1470702858.336932:1728:7693:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336944:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:48.0:1470702858.336946:1728:7694:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336958:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000800:00000100:49.0:1470702858.336960:1728:7692:0:(o2iblnd_cb.c:3431:kiblnd_complete()) RDMA (tx: c0000017f0908d68) failed: 5 00000100:00020000:48.0:1470702858.336973:2480:7694:0:(events.c:203:client_bulk_callback()) event type 1, status -5, desc c000000bf7637600 00000100:00000400:2.0F:1470702858.337180:2832:7782:0:(client.c:2113:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1470702858/real 1470702858] req@c000000788c18100 x1542132002415920/t0(0) o4->sultan-OST0036-osc-c000000ff0323000@10.37.248.83@o2ib1:6/4 lens 608/448 e 0 to 1 dl 1470702902 ref 2 fl Rpc:eX/0/ffffffff rc 0/-1 00000100:02000400:2.0:1470702858.337188:2672:7782:0:(import.c:182:ptlrpc_set_import_discon()) sultan-OST0036-osc-c000000ff0323000: Connection to sultan-OST0036 (at 10.37.248.83@o2ib1) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:2.0:1470702858.337189:2672:7782:0:(import.c:191:ptlrpc_set_import_discon()) c00000078a806800 sultan-OST0036_UUID: changing import state from FULL to DISCONN 00000100:00080000:2.0:1470702858.337191:2592:7782:0:(import.c:417:ptlrpc_pinger_force()) sultan-OST0036_UUID: waking up pinger s:DISCONN 00000100:00080000:105.0F:1470702858.337359:1488:7934:0:(pinger.c:221:ptlrpc_pinger_process_import()) 085fb3a6-8dec-ef07-c6e6-193e4c60f94c->sultan-OST0036_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:105.0:1470702858.337362:1648:7934:0:(recover.c:62:ptlrpc_initiate_recovery()) sultan-OST0036_UUID: starting recovery 00000100:00080000:105.0:1470702858.337363:1936:7934:0:(import.c:679:ptlrpc_connect_import()) c00000078a806800 sultan-OST0036_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:105.0:1470702858.337365:2192:7934:0:(import.c:523:import_select_connection()) sultan-OST0036-osc-c000000ff0323000: connect to NID 10.37.248.83@o2ib1 last attempt 4296224910 00000100:00080000:105.0:1470702858.337369:2192:7934:0:(import.c:601:import_select_connection()) sultan-OST0036-osc-c000000ff0323000: import c00000078a806800 using connection 10.37.248.83@o2ib1/10.37.248.83@o2ib1 00000100:00080000:129.0F:1470702858.345696:4912:29116:0:(client.c:1563:ptlrpc_send_new_req()) @@@ req from PID 29116 waiting for recovery: (FULL != CONNECTING) req@c000000e273ea500 x1542132002417168/t0(0) o101->sultan-OST0029-osc-c000000ff0323000@10.37.248.77@o2ib1:28/4 lens 328/400 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 00000100:00080000:50.0:1470702858.347913:4912:29081:0:(client.c:1563:ptlrpc_send_new_req()) @@@ req from PID 29081 waiting for recovery: (FULL != CONNECTING) req@c0000017d7d83600 x1542132002417216/t0(0) o101->sultan-OST002d-osc-c000000ff0323000@10.37.248.77@o2ib1:28/4 lens 328/400 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 00000100:00080000:26.0F:1470702858.348127:4912:29111:0:(client.c:1563:ptlrpc_send_new_req()) @@@ req from PID 29111 waiting for recovery: (FULL != CONNECTING) req@c0000006cf91cc00 x1542132002417232/t0(0) o101->sultan-OST0031-osc-c000000ff0323000@10.37.248.77@o2ib1:28/4 lens 328/400 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 00000100:00080000:11.0F:1470702858.430019:4912:29071:0:(client.c:1563:ptlrpc_send_new_req()) @@@ req from PID 29071 waiting for recovery: (FULL != CONNECTING) req@c0000003851ff300 x1542132002417328/t0(0) o101->sultan-OST000e-osc-c000000ff0323000@10.37.248.83@o2ib1:28/4 lens 328/400 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 00000100:00080000:92.0F:1470702858.430039:4912:29086:0:(client.c:1563:ptlrpc_send_new_req()) @@@ req from PID 29086 waiting for recovery: (FULL != CONNECTING) req@c000001e22b08100 x1542132002417376/t0(0) o101->sultan-OST0036-osc-c000000ff0323000@10.37.248.83@o2ib1:28/4 lens 328/400 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 00000100:00080000:51.0F:1470702858.506452:2416:7773:0:(import.c:1047:ptlrpc_connect_interpret()) sultan-OST0029-osc-c000000ff0323000: connect to target with instance 9 00000080:00080000:51.0:1470702858.506464:3296:7773:0:(lcommon_misc.c:75:cl_init_ea_size()) updating def/max_easize: 144/1392 00000100:00080000:51.0:1470702858.506468:2416:7773:0:(import.c:1215:ptlrpc_connect_interpret()) reconnected to sultan-OST0029_UUID@10.37.248.77@o2ib1 after partition 00000100:00080000:51.0:1470702858.506469:2416:7773:0:(import.c:1233:ptlrpc_connect_interpret()) c00000078c28c800 sultan-OST0029_UUID: changing import state from CONNECTING to RECOVER 00000100:00080000:51.0:1470702858.506472:2624:7773:0:(import.c:1541:ptlrpc_import_recovery_state_machine()) c00000078c28c800 sultan-OST0029_UUID: changing import state from RECOVER to FULL 00000080:00080000:51.0:1470702858.506478:3680:7773:0:(lcommon_misc.c:75:cl_init_ea_size()) updating def/max_easize: 144/1392 00000100:02000000:51.0:1470702858.506480:2624:7773:0:(import.c:1547:ptlrpc_import_recovery_state_machine()) sultan-OST0029-osc-c000000ff0323000: Connection restored to 10.37.248.77@o2ib1 (at 10.37.248.77@o2ib1) 00000100:00080000:51.0:1470702858.506574:3392:7773:0:(recover.c:243:ptlrpc_wake_delayed()) @@@ waking (set c000001e41380180): req@c000001a3617c900 x1542132002416240/t0(0) o4->sultan-OST0029-osc-c000000ff0323000@10.37.248.77@o2ib1:6/4 lens 608/448 e 0 to 1 dl 1470702902 ref 2 fl Rpc:eX/0/ffffffff rc 0/-1 00000100:00080000:51.0:1470702858.506580:3392:7773:0:(recover.c:243:ptlrpc_wake_delayed()) @@@ waking (set c000000fed0ae940): req@c000000e273ea500 x1542132002417168/t0(0) o101->sultan-OST0029-osc-c000000ff0323000@10.37.248.77@o2ib1:28/4 lens 328/400 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 00000100:00080000:107.0:1470702858.506587:2544:7916:0:(client.c:3263:ptlrpc_set_bulk_mbits()) resend bulk old x1542132002416240 new x1542132002417856 00000100:00080000:51.0:1470702858.506590:2416:7773:0:(import.c:1047:ptlrpc_connect_interpret()) sultan-OST0031-osc-c000000ff0323000: connect to target with instance 9 00000080:00080000:51.0:1470702858.506594:3296:7773:0:(lcommon_misc.c:75:cl_init_ea_size()) updating def/max_easize: 144/1392 00000100:00080000:51.0:1470702858.506595:2416:7773:0:(import.c:1215:ptlrpc_connect_interpret()) reconnected to sultan-OST0031_UUID@10.37.248.77@o2ib1 after partition 00000100:00080000:51.0:1470702858.506596:2416:7773:0:(import.c:1233:ptlrpc_connect_interpret()) c00000078c29c800 sultan-OST0031_UUID: changing import state from CONNECTING to RECOVER 00000100:00080000:51.0:1470702858.506597:2624:7773:0:(import.c:1541:ptlrpc_import_recovery_state_machine()) c00000078c29c800 sultan-OST0031_UUID: changing import state from RECOVER to FULL 00000080:00080000:51.0:1470702858.506600:3680:7773:0:(lcommon_misc.c:75:cl_init_ea_size()) updating def/max_easize: 144/1392 00000100:02000000:51.0:1470702858.506602:2624:7773:0:(import.c:1547:ptlrpc_import_recovery_state_machine()) sultan-OST0031-osc-c000000ff0323000: Connection restored to 10.37.248.77@o2ib1 (at 10.37.248.77@o2ib1) 00000100:00080000:51.0:1470702858.506604:3392:7773:0:(recover.c:243:ptlrpc_wake_delayed()) @@@ waking (set c000001e44b4fe40): req@c000001cfa96a800 x1542132002416320/t0(0) o4->sultan-OST0031-osc-c000000ff0323000@10.37.248.77@o2ib1:6/4 lens 608/448 e 0 to 1 dl 1470702902 ref 2 fl Rpc:eX/0/ffffffff rc 0/-1 00000100:00080000:51.0:1470702858.506611:3392:7773:0:(recover.c:243:ptlrpc_wake_delayed()) @@@ waking (set c0000007e8745040): req@c0000006cf91cc00 x1542132002417232/t0(0) o101->sultan-OST0031-osc-c000000ff0323000@10.37.248.77@o2ib1:28/4 lens 328/400 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 00000100:00080000:85.0:1470702858.506620:2544:7897:0:(client.c:3263:ptlrpc_set_bulk_mbits()) resend bulk old x1542132002416320 new x1542132002417872 00000100:00080000:51.0:1470702858.506812:2416:7773:0:(import.c:1047:ptlrpc_connect_interpret()) sultan-OST002d-osc-c000000ff0323000: connect to target with instance 9 00000080:00080000:51.0:1470702858.506816:3296:7773:0:(lcommon_misc.c:75:cl_init_ea_size()) updating def/max_easize: 144/1392 00000100:00080000:51.0:1470702858.506818:2416:7773:0:(import.c:1215:ptlrpc_connect_interpret()) reconnected to sultan-OST002d_UUID@10.37.248.77@o2ib1 after partition 00000100:00080000:51.0:1470702858.506819:2416:7773:0:(import.c:1233:ptlrpc_connect_interpret()) c00000078c294800 sultan-OST002d_UUID: changing import state from CONNECTING to RECOVER 00000100:00080000:51.0:1470702858.506820:2624:7773:0:(import.c:1541:ptlrpc_import_recovery_state_machine()) c00000078c294800 sultan-OST002d_UUID: changing import state from RECOVER to FULL 00000080:00080000:51.0:1470702858.506825:3680:7773:0:(lcommon_misc.c:75:cl_init_ea_size()) updating def/max_easize: 144/1392 00000100:02000000:51.0:1470702858.506828:2624:7773:0:(import.c:1547:ptlrpc_import_recovery_state_machine()) sultan-OST002d-osc-c000000ff0323000: Connection restored to 10.37.248.77@o2ib1 (at 10.37.248.77@o2ib1) 00000100:00080000:51.0:1470702858.506830:3392:7773:0:(recover.c:243:ptlrpc_wake_delayed()) @@@ waking (set c000001e413b0000): req@c000001e27b7d200 x1542132002416224/t0(0) o4->sultan-OST002d-osc-c000000ff0323000@10.37.248.77@o2ib1:6/4 lens 608/448 e 0 to 1 dl 1470702902 ref 2 fl Rpc:eX/0/ffffffff rc 0/-1 00000100:00080000:51.0:1470702858.506836:3392:7773:0:(recover.c:243:ptlrpc_wake_delayed()) @@@ waking (set c0000017ef9e5dc0): req@c0000017d7d83600 x1542132002417216/t0(0) o101->sultan-OST002d-osc-c000000ff0323000@10.37.248.77@o2ib1:28/4 lens 328/400 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 00000100:00080000:119.0:1470702858.506844:2544:7924:0:(client.c:3263:ptlrpc_set_bulk_mbits()) resend bulk old x1542132002416224 new x1542132002417888 00000100:00080000:159.0F:1470702858.510115:4912:29079:0:(client.c:1563:ptlrpc_send_new_req()) @@@ req from PID 29079 waiting for recovery: (FULL != CONNECTING) req@c000001c9f5dba00 x1542132002418064/t0(0) o101->sultan-OST001e-osc-c000000ff0323000@10.37.248.83@o2ib1:28/4 lens 328/400 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 00000100:00080000:109.0F:1470702858.511328:4912:29088:0:(client.c:1563:ptlrpc_send_new_req()) @@@ req from PID 29088 waiting for recovery: (FULL != CONNECTING) req@c000001a36fe2100 x1542132002418320/t0(0) o101->sultan-OST002a-osc-c000000ff0323000@10.37.248.83@o2ib1:28/4 lens 328/400 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 00000100:00080000:114.0F:1470702858.513205:4912:29089:0:(client.c:1563:ptlrpc_send_new_req()) @@@ req from PID 29089 waiting for recovery: (FULL != CONNECTING) req@c000001be1d04800 x1542132002418400/t0(0) o101->sultan-OST002e-osc-c000000ff0323000@10.37.248.83@o2ib1:28/4 lens 328/400 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 00000100:00080000:134.0F:1470702858.517597:4912:29076:0:(client.c:1563:ptlrpc_send_new_req()) @@@ req from PID 29076 waiting for recovery: (FULL != CONNECTING) req@c000001e1c114800 x1542132002418448/t0(0) o101->sultan-OST0022-osc-c000000ff0323000@10.37.248.83@o2ib1:28/4 lens 328/400 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 00000100:00080000:129.0:1470702858.518675:4912:29116:0:(client.c:1563:ptlrpc_send_new_req()) @@@ req from PID 29116 waiting for recovery: (FULL != CONNECTING) req@c000000e273ea500 x1542132002418496/t0(0) o101->sultan-OST002a-osc-c000000ff0323000@10.37.248.83@o2ib1:28/4 lens 328/400 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 00000100:00080000:55.0F:1470702858.518746:4912:29126:0:(client.c:1563:ptlrpc_send_new_req()) @@@ req from PID 29126 waiting for recovery: (FULL != CONNECTING) req@c00000150a80ba00 x1542132002418528/t0(0) o101->sultan-OST0036-osc-c000000ff0323000@10.37.248.83@o2ib1:28/4 lens 328/400 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 00000100:00080000:50.0:1470702858.519435:4912:29081:0:(client.c:1563:ptlrpc_send_new_req()) @@@ req from PID 29081 waiting for recovery: (FULL != CONNECTING) req@c0000017d7d83600 x1542132002418544/t0(0) o101->sultan-OST002e-osc-c000000ff0323000@10.37.248.83@o2ib1:28/4 lens 328/400 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 00000100:00080000:51.0:1470702858.527319:2416:7773:0:(import.c:1047:ptlrpc_connect_interpret()) sultan-OST002a-osc-c000000ff0323000: connect to target with instance 9 00000080:00080000:51.0:1470702858.527325:3296:7773:0:(lcommon_misc.c:75:cl_init_ea_size()) updating def/max_easize: 144/1392 00000100:00080000:51.0:1470702858.527327:2416:7773:0:(import.c:1215:ptlrpc_connect_interpret()) reconnected to sultan-OST002a_UUID@10.37.248.83@o2ib1 after partition 00000100:00080000:51.0:1470702858.527328:2416:7773:0:(import.c:1233:ptlrpc_connect_interpret()) c00000078c28e800 sultan-OST002a_UUID: changing import state from CONNECTING to RECOVER 00000100:00080000:51.0:1470702858.527331:2624:7773:0:(import.c:1541:ptlrpc_import_recovery_state_machine()) c00000078c28e800 sultan-OST002a_UUID: changing import state from RECOVER to FULL 00000080:00080000:51.0:1470702858.527336:3680:7773:0:(lcommon_misc.c:75:cl_init_ea_size()) updating def/max_easize: 144/1392 00000100:02000000:51.0:1470702858.527339:2624:7773:0:(import.c:1547:ptlrpc_import_recovery_state_machine()) sultan-OST002a-osc-c000000ff0323000: Connection restored to 10.37.248.83@o2ib1 (at 10.37.248.83@o2ib1) 00000100:00080000:51.0:1470702858.527341:3392:7773:0:(recover.c:243:ptlrpc_wake_delayed()) @@@ waking (set c0000017eee15280): req@c0000013dd005700 x1542132002415984/t0(0) o4->sultan-OST002a-osc-c000000ff0323000@10.37.248.83@o2ib1:6/4 lens 608/448 e 0 to 1 dl 1470702902 ref 2 fl Rpc:eX/0/ffffffff rc 0/-1 00000100:00080000:51.0:1470702858.527347:3392:7773:0:(recover.c:243:ptlrpc_wake_delayed()) @@@ waking (set c000001e30f7ac80): req@c000001a36fe2100 x1542132002418320/t0(0) o101->sultan-OST002a-osc-c000000ff0323000@10.37.248.83@o2ib1:28/4 lens 328/400 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 00000100:00080000:51.0:1470702858.527352:3392:7773:0:(recover.c:243:ptlrpc_wake_delayed()) @@@ waking (set c000000fed0ae940): req@c000000e273ea500 x1542132002418496/t0(0) o101->sultan-OST002a-osc-c000000ff0323000@10.37.248.83@o2ib1:28/4 lens 328/400 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 00000100:00080000:49.0:1470702858.527361:2544:7862:0:(client.c:3263:ptlrpc_set_bulk_mbits()) resend bulk old x1542132002415984 new x1542132002418752 00000100:00080000:51.0:1470702858.527364:2416:7773:0:(import.c:1047:ptlrpc_connect_interpret()) sultan-OST0022-osc-c000000ff0323000: connect to target with instance 9 00000080:00080000:51.0:1470702858.527370:3296:7773:0:(lcommon_misc.c:75:cl_init_ea_size()) updating def/max_easize: 144/1392 00000100:00080000:51.0:1470702858.527371:2416:7773:0:(import.c:1215:ptlrpc_connect_interpret()) reconnected to sultan-OST0022_UUID@10.37.248.83@o2ib1 after partition 00000100:00080000:51.0:1470702858.527372:2416:7773:0:(import.c:1233:ptlrpc_connect_interpret()) c00000078d35e800 sultan-OST0022_UUID: changing import state from CONNECTING to RECOVER 00000100:00080000:51.0:1470702858.527373:2624:7773:0:(import.c:1541:ptlrpc_import_recovery_state_machine()) c00000078d35e800 sultan-OST0022_UUID: changing import state from RECOVER to FULL 00000080:00080000:51.0:1470702858.527378:3680:7773:0:(lcommon_misc.c:75:cl_init_ea_size()) updating def/max_easize: 144/1392 00000100:02000000:51.0:1470702858.527380:2624:7773:0:(import.c:1547:ptlrpc_import_recovery_state_machine()) sultan-OST0022-osc-c000000ff0323000: Connection restored to 10.37.248.83@o2ib1 (at 10.37.248.83@o2ib1) 00000100:00080000:51.0:1470702858.527383:3392:7773:0:(recover.c:243:ptlrpc_wake_delayed()) @@@ waking (set c000000fe87b8100): req@c000001e1c114800 x1542132002418448/t0(0) o101->sultan-OST0022-osc-c000000ff0323000@10.37.248.83@o2ib1:28/4 lens 328/400 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 00000100:00080000:51.0:1470702858.527392:3392:7773:0:(recover.c:243:ptlrpc_wake_delayed()) @@@ waking (set c0000007a6d97e00): req@c0000017ce562a00 x1542132002415616/t0(0) o4->sultan-OST0022-osc-c000000ff0323000@10.37.248.83@o2ib1:6/4 lens 608/448 e 0 to 1 dl 1470702902 ref 2 fl Rpc:eX/0/ffffffff rc 0/-1 00000100:00080000:0.0:1470702858.527404:2544:7781:0:(client.c:3263:ptlrpc_set_bulk_mbits()) resend bulk old x1542132002415616 new x1542132002418768 00000100:00080000:51.0:1470702858.527412:2416:7773:0:(import.c:1047:ptlrpc_connect_interpret()) sultan-OST0036-osc-c000000ff0323000: connect to target with instance 9 00000080:00080000:51.0:1470702858.527417:3296:7773:0:(lcommon_misc.c:75:cl_init_ea_size()) updating def/max_easize: 144/1392 00000100:00080000:51.0:1470702858.527418:2416:7773:0:(import.c:1215:ptlrpc_connect_interpret()) reconnected to sultan-OST0036_UUID@10.37.248.83@o2ib1 after partition 00000100:00080000:51.0:1470702858.527419:2416:7773:0:(import.c:1233:ptlrpc_connect_interpret()) c00000078a806800 sultan-OST0036_UUID: changing import state from CONNECTING to RECOVER 00000100:00080000:51.0:1470702858.527421:2624:7773:0:(import.c:1541:ptlrpc_import_recovery_state_machine()) c00000078a806800 sultan-OST0036_UUID: changing import state from RECOVER to FULL 00000080:00080000:51.0:1470702858.527426:3680:7773:0:(lcommon_misc.c:75:cl_init_ea_size()) updating def/max_easize: 144/1392 00000100:02000000:51.0:1470702858.527429:2624:7773:0:(import.c:1547:ptlrpc_import_recovery_state_machine()) sultan-OST0036-osc-c000000ff0323000: Connection restored to 10.37.248.83@o2ib1 (at 10.37.248.83@o2ib1) 00000100:00080000:51.0:1470702858.527431:3392:7773:0:(recover.c:243:ptlrpc_wake_delayed()) @@@ waking (set c000001be10d0600): req@c000001e22b08100 x1542132002417376/t0(0) o101->sultan-OST0036-osc-c000000ff0323000@10.37.248.83@o2ib1:28/4 lens 328/400 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 00000100:00080000:51.0:1470702858.527439:3392:7773:0:(recover.c:243:ptlrpc_wake_delayed()) @@@ waking (set c0000007a6d97d40): req@c000000788c18100 x1542132002415920/t0(0) o4->sultan-OST0036-osc-c000000ff0323000@10.37.248.83@o2ib1:6/4 lens 608/448 e 0 to 1 dl 1470702902 ref 2 fl Rpc:eX/0/ffffffff rc 0/-1 00000100:00080000:51.0:1470702858.527445:3392:7773:0:(recover.c:243:ptlrpc_wake_delayed()) @@@ waking (set c0000017e67af0c0): req@c00000150a80ba00 x1542132002418528/t0(0) o101->sultan-OST0036-osc-c000000ff0323000@10.37.248.83@o2ib1:28/4 lens 328/400 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 00000100:00080000:2.0:1470702858.527458:2544:7782:0:(client.c:3263:ptlrpc_set_bulk_mbits()) resend bulk old x1542132002415920 new x1542132002418784 00000100:00080000:51.0:1470702858.527460:2416:7773:0:(import.c:1047:ptlrpc_connect_interpret()) sultan-OST002e-osc-c000000ff0323000: connect to target with instance 9 00000080:00080000:51.0:1470702858.527465:3296:7773:0:(lcommon_misc.c:75:cl_init_ea_size()) updating def/max_easize: 144/1392 00000100:00080000:51.0:1470702858.527467:2416:7773:0:(import.c:1215:ptlrpc_connect_interpret()) reconnected to sultan-OST002e_UUID@10.37.248.83@o2ib1 after partition 00000100:00080000:51.0:1470702858.527469:2416:7773:0:(import.c:1233:ptlrpc_connect_interpret()) c00000078c296800 sultan-OST002e_UUID: changing import state from CONNECTING to RECOVER 00000100:00080000:51.0:1470702858.527470:2624:7773:0:(import.c:1541:ptlrpc_import_recovery_state_machine()) c00000078c296800 sultan-OST002e_UUID: changing import state from RECOVER to FULL 00000080:00080000:51.0:1470702858.527474:3680:7773:0:(lcommon_misc.c:75:cl_init_ea_size()) updating def/max_easize: 144/1392 00000100:02000000:51.0:1470702858.527475:2624:7773:0:(import.c:1547:ptlrpc_import_recovery_state_machine()) sultan-OST002e-osc-c000000ff0323000: Connection restored to 10.37.248.83@o2ib1 (at 10.37.248.83@o2ib1) 00000100:00080000:51.0:1470702858.527477:3392:7773:0:(recover.c:243:ptlrpc_wake_delayed()) @@@ waking (set c0000007e4b20fc0): req@c000000785385400 x1542132002415728/t0(0) o4->sultan-OST002e-osc-c000000ff0323000@10.37.248.83@o2ib1:6/4 lens 608/448 e 0 to 1 dl 1470702902 ref 2 fl Rpc:eX/0/ffffffff rc 0/-1 00000100:00080000:51.0:1470702858.527482:3392:7773:0:(recover.c:243:ptlrpc_wake_delayed()) @@@ waking (set c000001e13146240): req@c000001be1d04800 x1542132002418400/t0(0) o101->sultan-OST002e-osc-c000000ff0323000@10.37.248.83@o2ib1:28/4 lens 328/400 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 00000100:00080000:51.0:1470702858.527486:3392:7773:0:(recover.c:243:ptlrpc_wake_delayed()) @@@ waking (set c0000017ef9e5dc0): req@c0000017d7d83600 x1542132002418544/t0(0) o101->sultan-OST002e-osc-c000000ff0323000@10.37.248.83@o2ib1:28/4 lens 328/400 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 00000100:00080000:33.0:1470702858.527500:2544:7813:0:(client.c:3263:ptlrpc_set_bulk_mbits()) resend bulk old x1542132002415728 new x1542132002418800 00000100:00080000:51.0:1470702858.527503:2416:7773:0:(import.c:1047:ptlrpc_connect_interpret()) sultan-OST000e-osc-c000000ff0323000: connect to target with instance 9 00000080:00080000:51.0:1470702858.527509:3296:7773:0:(lcommon_misc.c:75:cl_init_ea_size()) updating def/max_easize: 144/1392 00000100:00080000:51.0:1470702858.527511:2416:7773:0:(import.c:1215:ptlrpc_connect_interpret()) reconnected to sultan-OST000e_UUID@10.37.248.83@o2ib1 after partition 00000100:00080000:51.0:1470702858.527512:2416:7773:0:(import.c:1233:ptlrpc_connect_interpret()) c00000078d196800 sultan-OST000e_UUID: changing import state from CONNECTING to RECOVER 00000100:00080000:51.0:1470702858.527513:2624:7773:0:(import.c:1541:ptlrpc_import_recovery_state_machine()) c00000078d196800 sultan-OST000e_UUID: changing import state from RECOVER to FULL 00000080:00080000:51.0:1470702858.527516:3680:7773:0:(lcommon_misc.c:75:cl_init_ea_size()) updating def/max_easize: 144/1392 00000100:02000000:51.0:1470702858.527517:2624:7773:0:(import.c:1547:ptlrpc_import_recovery_state_machine()) sultan-OST000e-osc-c000000ff0323000: Connection restored to 10.37.248.83@o2ib1 (at 10.37.248.83@o2ib1) 00000100:00080000:51.0:1470702858.527520:3392:7773:0:(recover.c:243:ptlrpc_wake_delayed()) @@@ waking (set c0000007e9ee4ec0): req@c0000003851ff300 x1542132002417328/t0(0) o101->sultan-OST000e-osc-c000000ff0323000@10.37.248.83@o2ib1:28/4 lens 328/400 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 00000100:00080000:51.0:1470702858.527527:3392:7773:0:(recover.c:243:ptlrpc_wake_delayed()) @@@ waking (set c000000fed5acb40): req@c000000bf734f600 x1542132002415744/t0(0) o4->sultan-OST000e-osc-c000000ff0323000@10.37.248.83@o2ib1:6/4 lens 608/448 e 0 to 1 dl 1470702902 ref 2 fl Rpc:eX/0/ffffffff rc 0/-1 00000100:00080000:143.0:1470702858.527542:2544:7842:0:(client.c:3263:ptlrpc_set_bulk_mbits()) resend bulk old x1542132002415744 new x1542132002418816 00000800:00020000:49.0:1470702858.528978:3040:7692:0:(o2iblnd_cb.c:892:kiblnd_post_tx_locked()) Error -12 posting transmit to 10.37.248.83@o2ib1 00000800:00000100:49.0:1470702858.528982:3280:7692:0:(o2iblnd_cb.c:1918:kiblnd_close_conn_locked()) Closing conn to 10.37.248.83@o2ib1: error -12(waiting) 00000100:00020000:49.0:1470702858.528987:3984:7692:0:(events.c:203:client_bulk_callback()) event type 1, status -12, desc c000000ff0580400 00000100:00000400:145.0F:1470702858.529122:2832:7843:0:(client.c:2113:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1470702858/real 1470702858] req@c000000fd8dedb00 x1542132002418640/t0(0) o4->sultan-OST001a-osc-c000000ff0323000@10.37.248.83@o2ib1:6/4 lens 608/448 e 0 to 1 dl 1470702902 ref 2 fl Rpc:eX/0/ffffffff rc 0/-1 00000100:02000400:145.0:1470702858.529131:2672:7843:0:(import.c:182:ptlrpc_set_import_discon()) sultan-OST001a-osc-c000000ff0323000: Connection to sultan-OST001a (at 10.37.248.83@o2ib1) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:145.0:1470702858.529133:2672:7843:0:(import.c:191:ptlrpc_set_import_discon()) c00000078d34e800 sultan-OST001a_UUID: changing import state from FULL to DISCONN 00000100:00080000:145.0:1470702858.529134:2592:7843:0:(import.c:417:ptlrpc_pinger_force()) sultan-OST001a_UUID: waking up pinger s:DISCONN 00000100:00080000:105.0:1470702858.529153:1488:7934:0:(pinger.c:221:ptlrpc_pinger_process_import()) 085fb3a6-8dec-ef07-c6e6-193e4c60f94c->sultan-OST001a_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:105.0:1470702858.529156:1648:7934:0:(recover.c:62:ptlrpc_initiate_recovery()) sultan-OST001a_UUID: starting recovery 00000100:00080000:105.0:1470702858.529156:1936:7934:0:(import.c:679:ptlrpc_connect_import()) c00000078d34e800 sultan-OST001a_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:105.0:1470702858.529159:2192:7934:0:(import.c:523:import_select_connection()) sultan-OST001a-osc-c000000ff0323000: connect to NID 10.37.248.83@o2ib1 last attempt 4296224806 00000100:00080000:105.0:1470702858.529162:2192:7934:0:(import.c:601:import_select_connection()) sultan-OST001a-osc-c000000ff0323000: import c00000078d34e800 using connection 10.37.248.83@o2ib1/10.37.248.83@o2ib1 00000100:00080000:51.0:1470702858.535670:2416:7773:0:(import.c:1047:ptlrpc_connect_interpret()) sultan-OST001a-osc-c000000ff0323000: connect to target with instance 9 00000080:00080000:51.0:1470702858.535675:3296:7773:0:(lcommon_misc.c:75:cl_init_ea_size()) updating def/max_easize: 144/1392 00000100:00080000:51.0:1470702858.535676:2416:7773:0:(import.c:1215:ptlrpc_connect_interpret()) reconnected to sultan-OST001a_UUID@10.37.248.83@o2ib1 after partition 00000100:00080000:51.0:1470702858.535677:2416:7773:0:(import.c:1233:ptlrpc_connect_interpret()) c00000078d34e800 sultan-OST001a_UUID: changing import state from CONNECTING to RECOVER 00000100:00080000:51.0:1470702858.535680:3552:7773:0:(client.c:2762:ptlrpc_resend_req()) @@@ going to resend req@c000000f06283000 x1542132002416368/t0(0) o4->sultan-OST001a-osc-c000000ff0323000@10.37.248.83@o2ib1:6/4 lens 608/448 e 0 to 0 dl 1470702902 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000100:00080000:51.0:1470702858.535684:2624:7773:0:(import.c:1541:ptlrpc_import_recovery_state_machine()) c00000078d34e800 sultan-OST001a_UUID: changing import state from RECOVER to FULL 00000080:00080000:51.0:1470702858.535687:3680:7773:0:(lcommon_misc.c:75:cl_init_ea_size()) updating def/max_easize: 144/1392 00000100:02000000:51.0:1470702858.535689:2624:7773:0:(import.c:1547:ptlrpc_import_recovery_state_machine()) sultan-OST001a-osc-c000000ff0323000: Connection restored to 10.37.248.83@o2ib1 (at 10.37.248.83@o2ib1) 00000100:00080000:51.0:1470702858.535691:3392:7773:0:(recover.c:243:ptlrpc_wake_delayed()) @@@ waking (set c000000fed5aca80): req@c000000fd8dedb00 x1542132002418640/t0(0) o4->sultan-OST001a-osc-c000000ff0323000@10.37.248.83@o2ib1:6/4 lens 608/448 e 0 to 1 dl 1470702902 ref 2 fl Rpc:eX/0/ffffffff rc 0/-1 00000100:00080000:140.0:1470702858.535697:2544:7834:0:(client.c:3263:ptlrpc_set_bulk_mbits()) resend bulk old x1542132002416368 new x1542132002419408 00000100:00080000:145.0:1470702858.535702:2544:7843:0:(client.c:3263:ptlrpc_set_bulk_mbits()) resend bulk old x1542132002418640 new x1542132002419424 00000800:00020000:45.0:1470702858.536543:3040:7695:0:(o2iblnd_cb.c:892:kiblnd_post_tx_locked()) Error -12 posting transmit to 10.37.248.83@o2ib1 00000800:00000100:45.0:1470702858.536546:3280:7695:0:(o2iblnd_cb.c:1918:kiblnd_close_conn_locked()) Closing conn to 10.37.248.83@o2ib1: error -12(waiting) 00000100:00020000:45.0:1470702858.536549:3984:7695:0:(events.c:203:client_bulk_callback()) event type 1, status -12, desc c00000143cab2c00 00000100:00020000:49.0:1470702858.536647:3984:7692:0:(events.c:203:client_bulk_callback()) event type 1, status -103, desc c000000e2a0be200 00000100:00000400:68.0F:1470702858.536653:2832:7876:0:(client.c:2113:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1470702858/real 1470702858] req@c00000150aae2d00 x1542132002418928/t0(0) o4->sultan-OST0016-osc-c000000ff0323000@10.37.248.83@o2ib1:6/4 lens 608/448 e 0 to 1 dl 1470702902 ref 2 fl Rpc:eX/0/ffffffff rc 0/-1 00000100:02000400:68.0:1470702858.536758:2672:7876:0:(import.c:182:ptlrpc_set_import_discon()) sultan-OST0016-osc-c000000ff0323000: Connection to sultan-OST0016 (at 10.37.248.83@o2ib1) was lost; in progress operations using this service will wait for recovery to complete 00000100:00000400:122.0F:1470702858.536759:2832:7822:0:(client.c:2113:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1470702858/real 1470702858] req@c000001e2771e400 x1542132002418992/t0(0) o4->sultan-OST0016-osc-c000000ff0323000@10.37.248.83@o2ib1:6/4 lens 608/448 e 0 to 1 dl 1470702902 ref 2 fl Rpc:eX/0/ffffffff rc 0/-1 00000100:00080000:68.0:1470702858.536759:2672:7876:0:(import.c:191:ptlrpc_set_import_discon()) c00000078d346800 sultan-OST0016_UUID: changing import state from FULL to DISCONN 00000100:00080000:68.0:1470702858.536761:2592:7876:0:(import.c:417:ptlrpc_pinger_force()) sultan-OST0016_UUID: waking up pinger s:DISCONN 00000100:00080000:122.0:1470702858.536768:2672:7822:0:(import.c:206:ptlrpc_set_import_discon()) osc: import c00000078d346800 already not connected (conn 5, was 5): DISCONN 00000100:00080000:105.0:1470702858.536771:1488:7934:0:(pinger.c:221:ptlrpc_pinger_process_import()) 085fb3a6-8dec-ef07-c6e6-193e4c60f94c->sultan-OST0016_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:105.0:1470702858.536773:1648:7934:0:(recover.c:62:ptlrpc_initiate_recovery()) sultan-OST0016_UUID: starting recovery 00000100:00080000:105.0:1470702858.536774:1936:7934:0:(import.c:679:ptlrpc_connect_import()) c00000078d346800 sultan-OST0016_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:105.0:1470702858.536776:2192:7934:0:(import.c:523:import_select_connection()) sultan-OST0016-osc-c000000ff0323000: connect to NID 10.37.248.83@o2ib1 last attempt 4296224842 00000100:00080000:105.0:1470702858.536778:2192:7934:0:(import.c:601:import_select_connection()) sultan-OST0016-osc-c000000ff0323000: import c00000078d346800 using connection 10.37.248.83@o2ib1/10.37.248.83@o2ib1 00000100:00080000:51.0:1470702858.542471:2416:7773:0:(import.c:1047:ptlrpc_connect_interpret()) sultan-OST0016-osc-c000000ff0323000: connect to target with instance 9 00000080:00080000:51.0:1470702858.542475:3296:7773:0:(lcommon_misc.c:75:cl_init_ea_size()) updating def/max_easize: 144/1392 00000100:00080000:51.0:1470702858.542476:2416:7773:0:(import.c:1215:ptlrpc_connect_interpret()) reconnected to sultan-OST0016_UUID@10.37.248.83@o2ib1 after partition 00000100:00080000:51.0:1470702858.542477:2416:7773:0:(import.c:1233:ptlrpc_connect_interpret()) c00000078d346800 sultan-OST0016_UUID: changing import state from CONNECTING to RECOVER 00000100:00080000:51.0:1470702858.542478:2624:7773:0:(import.c:1541:ptlrpc_import_recovery_state_machine()) c00000078d346800 sultan-OST0016_UUID: changing import state from RECOVER to FULL 00000080:00080000:51.0:1470702858.542481:3680:7773:0:(lcommon_misc.c:75:cl_init_ea_size()) updating def/max_easize: 144/1392 00000100:02000000:51.0:1470702858.542483:2624:7773:0:(import.c:1547:ptlrpc_import_recovery_state_machine()) sultan-OST0016-osc-c000000ff0323000: Connection restored to 10.37.248.83@o2ib1 (at 10.37.248.83@o2ib1) 00000100:00080000:51.0:1470702858.542484:3392:7773:0:(recover.c:243:ptlrpc_wake_delayed()) @@@ waking (set c0000017ed4ab1c0): req@c00000150aae2d00 x1542132002418928/t0(0) o4->sultan-OST0016-osc-c000000ff0323000@10.37.248.83@o2ib1:6/4 lens 608/448 e 0 to 1 dl 1470702902 ref 2 fl Rpc:eX/0/ffffffff rc 0/-1 00000100:00080000:51.0:1470702858.542489:3392:7773:0:(recover.c:243:ptlrpc_wake_delayed()) @@@ waking (set c000000fefdd5b80): req@c000001e2771e400 x1542132002418992/t0(0) o4->sultan-OST0016-osc-c000000ff0323000@10.37.248.83@o2ib1:6/4 lens 608/448 e 0 to 1 dl 1470702902 ref 2 fl Rpc:eX/0/ffffffff rc 0/-1 00000100:00080000:68.0:1470702858.542494:2544:7876:0:(client.c:3263:ptlrpc_set_bulk_mbits()) resend bulk old x1542132002418928 new x1542132002419472 00000100:00080000:122.0:1470702858.542499:2544:7822:0:(client.c:3263:ptlrpc_set_bulk_mbits()) resend bulk old x1542132002418992 new x1542132002419488 00000800:00000400:105.0:1470702858.560708:1664:7659:0:(o2iblnd_cb.c:1360:kiblnd_reconnect_peer()) Abort reconnection of 10.37.248.83@o2ib1: closed 00000100:00080000:51.0:1470702858.576512:2416:7773:0:(import.c:1047:ptlrpc_connect_interpret()) sultan-OST001e-osc-c000000ff0323000: connect to target with instance 9 00000080:00080000:51.0:1470702858.576516:3296:7773:0:(lcommon_misc.c:75:cl_init_ea_size()) updating def/max_easize: 144/1392 00000100:00080000:51.0:1470702858.576517:2416:7773:0:(import.c:1215:ptlrpc_connect_interpret()) reconnected to sultan-OST001e_UUID@10.37.248.83@o2ib1 after partition 00000100:00080000:51.0:1470702858.576518:2416:7773:0:(import.c:1233:ptlrpc_connect_interpret()) c00000078d356800 sultan-OST001e_UUID: changing import state from CONNECTING to RECOVER 00000100:00080000:51.0:1470702858.576519:2624:7773:0:(import.c:1541:ptlrpc_import_recovery_state_machine()) c00000078d356800 sultan-OST001e_UUID: changing import state from RECOVER to FULL 00000080:00080000:51.0:1470702858.576522:3680:7773:0:(lcommon_misc.c:75:cl_init_ea_size()) updating def/max_easize: 144/1392 00000100:02000000:51.0:1470702858.576524:2624:7773:0:(import.c:1547:ptlrpc_import_recovery_state_machine()) sultan-OST001e-osc-c000000ff0323000: Connection restored to 10.37.248.83@o2ib1 (at 10.37.248.83@o2ib1) 00000100:00080000:51.0:1470702858.576526:3392:7773:0:(recover.c:243:ptlrpc_wake_delayed()) @@@ waking (set c000000feab78a00): req@c000001c9f5dba00 x1542132002418064/t0(0) o101->sultan-OST001e-osc-c000000ff0323000@10.37.248.83@o2ib1:28/4 lens 328/400 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1 00000100:00080000:51.0:1470702858.576531:3392:7773:0:(recover.c:243:ptlrpc_wake_delayed()) @@@ waking (set c000000fed5ad140): req@c000000f06283300 x1542132002415760/t0(0) o4->sultan-OST001e-osc-c000000ff0323000@10.37.248.83@o2ib1:6/4 lens 608/448 e 0 to 1 dl 1470702902 ref 2 fl Rpc:eX/0/ffffffff rc 0/-1 00000100:00080000:140.0:1470702858.576540:2544:7834:0:(client.c:3263:ptlrpc_set_bulk_mbits()) resend bulk old x1542132002415760 new x1542132002419632 Debug log: 1600 lines, 1600 kept, 0 dropped, 0 bad.