00000400:02000000:1.0F:1547252926.759532:0:235131:0:(libcfs_cpu.c:1227:cfs_cpu_init()) HW NUMA nodes: 2, HW CPU cores: 48, npartitions: 2
00000001:01000000:3.0F:1547252927.010424:0:235131:0:(linux-crypto.c:390:cfs_crypto_performance_test()) Crypto hash algorithm adler32 speed = 2611 MB/s
00000001:01000000:3.0:1547252927.260473:0:235131:0:(linux-crypto.c:390:cfs_crypto_performance_test()) Crypto hash algorithm crc32 speed = 9819 MB/s
00000001:01000000:3.0:1547252927.510448:0:235131:0:(linux-crypto.c:390:cfs_crypto_performance_test()) Crypto hash algorithm crc32c speed = 11679 MB/s
00000400:00000200:3.0:1547252945.109471:0:235149:0:(api-ni.c:779:lnet_register_lnd()) lo LND registered
00000020:02000000:3.0:1547252945.126395:0:235149:0:(class_obd.c:630:obdclass_init()) Lustre: Build Version: 2.12.50_46_g07f4b9f
00000020:01200004:2.0F:1547252945.145137:0:235147:0:(obd_mount.c:1542:lustre_fill_super()) VFS Op: sb ffff9208d990e000
00000020:01000004:2.0:1547252945.145178:0:235147:0:(obd_mount.c:930:lmd_print())   mount data:
00000020:01000004:2.0:1547252945.145178:0:235147:0:(obd_mount.c:932:lmd_print()) profile: cache1-client
00000020:01000004:2.0:1547252945.145179:0:235147:0:(obd_mount.c:933:lmd_print()) device:  10.0.11.90@o2ib10:/cache1
00000020:01000004:2.0:1547252945.145179:0:235147:0:(obd_mount.c:934:lmd_print()) flags:   2
00000020:01000004:2.0:1547252945.145180:0:235147:0:(obd_mount.c:1567:lustre_fill_super()) Mounting client cache1-client
00000100:00000200:3.0:1547252945.186057:0:235163:0:(events.c:594:ptlrpc_ni_init()) My pid is: 3039
00000400:00000200:8.0F:1547252945.205405:0:235218:0:(api-ni.c:779:lnet_register_lnd()) o2ib LND registered
00000800:02000000:16.0F:1547252945.206069:0:235163:0:(o2iblnd.c:2567:kiblnd_hdev_get_attr()) Using FastReg for registration
00000800:00000200:16.0:1547252945.206324:0:235163:0:(o2iblnd.c:2032:kiblnd_init_pool()) Initialize TX pool
00000800:00000200:16.0:1547252945.209366:0:235163:0:(o2iblnd.c:2032:kiblnd_init_pool()) Initialize TX pool
00000400:02000000:16.0:1547252945.214287:0:235163:0:(api-ni.c:2132:lnet_startup_lndni()) Added LNI 10.0.10.184@o2ib10 [8/256/0/180]
00000400:00000200:16.0:1547252945.214330:0:235163:0:(api-ni.c:1776:lnet_push_target_resize()) nnis 16 success
00000400:00000200:16.0:1547252945.214349:0:235163:0:(peer.c:3220:lnet_peer_discovery_start()) discovery start: 0
00000400:00000200:1.0:1547252945.214351:0:235246:0:(peer.c:3060:lnet_peer_discovery()) started
00000020:01000004:2.0:1547252945.239494:0:235147:0:(obd_mount.c:339:lustre_start_mgc()) Start MGC 'MGC10.0.11.90@o2ib10'
00000020:00000080:2.0:1547252945.239497:0:235147:0:(obd_config.c:1128:class_process_config()) processing cmd: cf005
00000020:00000080:2.0:1547252945.239499:0:235147:0:(obd_config.c:1139:class_process_config()) adding mapping from uuid MGC10.0.11.90@o2ib10_0 to nid 0x5000a0a000b5a (10.0.11.90@o2ib10)
00000020:01000004:2.0:1547252945.239505:0:235147:0:(obd_mount.c:193:lustre_start_simple()) Starting obd MGC10.0.11.90@o2ib10 (typ=mgc)
00000020:00000080:2.0:1547252945.239506:0:235147:0:(obd_config.c:1128:class_process_config()) processing cmd: cf001
00000020:00000080:2.0:1547252945.242617:0:235147:0:(genops.c:451:class_newdev()) Allocate new device MGC10.0.11.90@o2ib10 (ffff91e3c04f0000)
00000020:00000080:2.0:1547252945.242669:0:235147:0:(obd_config.c:431:class_attach()) OBD: dev 0 attached type mgc with refcount 1
00000020:00000080:2.0:1547252945.242670:0:235147:0:(obd_config.c:1128:class_process_config()) processing cmd: cf003
00000400:00000200:2.0:1547252945.242727:0:235147:0:(lib-ptl.c:904:LNetSetLazyPortal()) Setting portal 15 lazy
00000100:00000200:2.0:1547252945.242753:0:235147:0:(service.c:832:ptlrpc_register_service()) ldlm_cbd: Started, listening on portal 15
00000100:00000200:1.0:1547252945.242792:0:235335:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 15
00000100:00000200:2.0:1547252945.242824:0:235335:0:(service.c:2590:ptlrpc_main()) service thread 0 (#1) started
00000100:00000200:1.0:1547252945.242838:0:235337:0:(service.c:2590:ptlrpc_main()) service thread 1 (#2) started
00000100:00000200:24.0F:1547252945.242876:0:235338:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 15
00000100:00000200:24.0:1547252945.242889:0:235338:0:(service.c:2590:ptlrpc_main()) service thread 0 (#1) started
00000100:00000200:24.0:1547252945.242923:0:235339:0:(service.c:2590:ptlrpc_main()) service thread 1 (#2) started
00000100:00000200:3.0:1547252945.242976:0:235147:0:(events.c:543:ptlrpc_uuid_to_peer()) MGC10.0.11.90@o2ib10_0->12345-10.0.11.90@o2ib10
00000400:00000200:3.0:1547252945.242983:0:235147:0:(peer.c:268:lnet_peer_alloc()) ffff91f0709fc540 nid 10.0.11.90@o2ib10
00000400:00000200:3.0:1547252945.242985:0:235147:0:(peer.c:219:lnet_peer_net_alloc()) ffff920899fd2b00 net o2ib10
00000400:00000200:3.0:1547252945.242994:0:235147:0:(peer.c:201:lnet_peer_ni_alloc()) ffff91db9fba8000 nid 10.0.11.90@o2ib10
00000400:00000200:3.0:1547252945.242997:0:235147:0:(peer.c:1202:lnet_peer_attach_peer_ni()) peer 10.0.11.90@o2ib10 NID 10.0.11.90@o2ib10 flags 0x0
00000400:00000200:3.0:1547252945.243000:0:235147:0:(peer.c:1767:lnet_peer_queue_for_discovery()) Queue peer 10.0.11.90@o2ib10: 0
00000400:00000200:1.0:1547252945.243003:0:235246:0:(peer.c:3007:lnet_peer_discovery_wait_for_work()) woken: 0
00000400:00000200:1.0:1547252945.243005:0:235246:0:(peer.c:3109:lnet_peer_discovery()) peer 10.0.11.90@o2ib10 state 0x20
00000400:00000200:1.0:1547252945.243009:0:235246:0:(lib-move.c:4783:LNetGet()) LNetGet -> 12345-10.0.11.90@o2ib10
00000400:00000200:1.0:1547252945.243013:0:235246:0:(lib-move.c:2450:lnet_handle_send_case_locked()) Source ANY to NMR:  10.0.11.90@o2ib10 local destination
00000400:00000200:1.0:1547252945.243016:0:235246:0:(lib-move.c:1510:lnet_get_best_ni()) compare ni 10.0.10.184@o2ib10 [c:256, d:21, s:0] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1547252945.243018:0:235246:0:(lib-move.c:1553:lnet_get_best_ni()) selected best_ni 10.0.10.184@o2ib10
00000400:00000200:1.0:1547252945.243020:0:235246:0:(lib-move.c:1861:lnet_set_non_mr_pref_nid()) Setting preferred local NID 10.0.10.184@o2ib10 on NMR peer 10.0.11.90@o2ib10
00000400:00000200:1.0:1547252945.243022:0:235246:0:(peer.c:872:lnet_peer_ni_set_non_mr_pref_nid()) peer 10.0.11.90@o2ib10 nid 10.0.10.184@o2ib10: 0
00000400:00000200:1.0:1547252945.243024:0:235246:0:(lib-move.c:1714:lnet_handle_send()) rspt_next_hop_nid = 10.0.11.90@o2ib10
00000400:00000200:1.0:1547252945.243028:0:235246:0:(lib-move.c:1728:lnet_handle_send()) TRACE: 10.0.10.184@o2ib10(10.0.10.184@o2ib10:<?>) -> 10.0.11.90@o2ib10(10.0.11.90@o2ib10:10.0.11.90@o2ib10) : GET try# 0
00000800:00000200:1.0:1547252945.243031:0:235246:0:(o2iblnd_cb.c:1636:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.11.90@o2ib10
00000800:00000200:1.0:1547252945.243035:0:235246:0:(o2iblnd_cb.c:1604:kiblnd_launch_tx()) peer_ni[ffff91f07b11ad80] -> 10.0.11.90@o2ib10 (1)++
00000800:00000200:1.0:1547252945.243038:0:235246:0:(o2iblnd_cb.c:1409:kiblnd_connect_peer()) peer_ni[ffff91f07b11ad80] -> 10.0.11.90@o2ib10 (2)++
00000800:00000200:1.0:1547252945.243050:0:235246:0:(o2iblnd_cb.c:1362:kiblnd_resolve_addr()) bound to port 1023
00000800:00000200:1.0:1547252945.243051:0:235246:0:(o2iblnd_cb.c:1611:kiblnd_launch_tx()) peer_ni[ffff91f07b11ad80] -> 10.0.11.90@o2ib10 (3)--
00000400:00000200:1.0:1547252945.243053:0:235246:0:(peer.c:2764:lnet_peer_send_ping()) peer 10.0.11.90@o2ib10
00000400:00000200:1.0:1547252945.243054:0:235246:0:(peer.c:3130:lnet_peer_discovery()) peer 10.0.11.90@o2ib10 state 0x120 rc 0
00000800:00000200:28.0F:1547252945.243061:0:232995:0:(o2iblnd_cb.c:3156:kiblnd_cm_callback()) 10.0.11.90@o2ib10 Addr resolved: 0
00000800:00000200:28.0:1547252945.243079:0:232995:0:(o2iblnd_cb.c:3173:kiblnd_cm_callback()) 10.0.11.90@o2ib10: connection bound to ib0:10.0.10.184:mlx5_0
00000800:00000200:28.0:1547252945.243551:0:232995:0:(o2iblnd_cb.c:3197:kiblnd_cm_callback()) 10.0.11.90@o2ib10 Route resolved: 0
00000800:00000200:28.0:1547252945.243574:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 0: ffff91f0791a1000 0x17791a1000(0x17791a1000)
00000800:00000200:28.0:1547252945.243576:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 1: ffff91f070dc8000 0x1770dc8000(0x1770dc8000)
00000800:00000200:28.0:1547252945.243577:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 2: ffff91f0d6e8b000 0x17d6e8b000(0x17d6e8b000)
00000800:00000200:28.0:1547252945.243578:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 3: ffff91f078767000 0x1778767000(0x1778767000)
00000800:00000200:28.0:1547252945.243578:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 4: ffff91f0d66c8000 0x17d66c8000(0x17d66c8000)
00000800:00000200:28.0:1547252945.243579:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 5: ffff91f0cfdf6000 0x17cfdf6000(0x17cfdf6000)
00000800:00000200:28.0:1547252945.243580:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 6: ffff91ef7c4cf000 0x167c4cf000(0x167c4cf000)
00000800:00000200:28.0:1547252945.243581:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 7: ffff91f0761de000 0x17761de000(0x17761de000)
00000800:00000200:28.0:1547252945.243581:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 8: ffff91f078760000 0x1778760000(0x1778760000)
00000800:00000200:28.0:1547252945.243582:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 9: ffff91f0d8e50000 0x17d8e50000(0x17d8e50000)
00000800:00000200:28.0:1547252945.243583:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 10: ffff91f017809000 0x1717809000(0x1717809000)
00000800:00000200:28.0:1547252945.243584:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 11: ffff91f0d06f3000 0x17d06f3000(0x17d06f3000)
00000800:00000200:28.0:1547252945.243585:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 12: ffff91db9d2f0000 0x29d2f0000(0x29d2f0000)
00000800:00000200:28.0:1547252945.243585:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 13: ffff91f07f5f7000 0x177f5f7000(0x177f5f7000)
00000800:00000200:28.0:1547252945.243586:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 14: ffff91f0db347000 0x17db347000(0x17db347000)
00000800:00000200:28.0:1547252945.243587:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 15: ffff91f0b97af000 0x17b97af000(0x17b97af000)
00000800:00000200:28.0:1547252945.243588:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 16: ffff91f0d7e39000 0x17d7e39000(0x17d7e39000)
00000800:00000200:28.0:1547252945.243589:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 17: ffff91f066327000 0x1766327000(0x1766327000)
00000800:00000200:16.0:1547252945.244175:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f09beb8000] (19)++
00000800:00000200:16.0:1547252945.244176:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f09beb8000] (20)--
00000800:00000200:16.0:1547252945.244177:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f09beb8000] (19)++
00000800:00000200:16.0:1547252945.244178:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f09beb8000] (20)--
00000800:00000200:16.0:1547252945.244179:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f09beb8000] (19)++
00000800:00000200:16.0:1547252945.244179:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f09beb8000] (20)--
00000800:00000200:16.0:1547252945.244180:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f09beb8000] (19)++
00000800:00000200:16.0:1547252945.244180:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f09beb8000] (20)--
00000800:00000200:16.0:1547252945.244181:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f09beb8000] (19)++
00000800:00000200:16.0:1547252945.244182:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f09beb8000] (20)--
00000800:00000200:16.0:1547252945.244182:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f09beb8000] (19)++
00000800:00000200:16.0:1547252945.244183:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f09beb8000] (20)--
00000800:00000200:16.0:1547252945.244183:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f09beb8000] (19)++
00000800:00000200:16.0:1547252945.244184:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f09beb8000] (20)--
00000800:00000200:16.0:1547252945.244185:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f09beb8000] (19)++
00000800:00000200:16.0:1547252945.244185:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f09beb8000] (20)--
00000800:00000200:16.0:1547252945.244186:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f09beb8000] (19)++
00000800:00000200:16.0:1547252945.244186:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f09beb8000] (20)--
00000800:00000200:16.0:1547252945.244187:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f09beb8000] (19)++
00000800:00000200:16.0:1547252945.244187:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f09beb8000] (20)--
00000800:00000200:16.0:1547252945.244188:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f09beb8000] (19)++
00000800:00000200:16.0:1547252945.244188:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f09beb8000] (20)--
00000800:00000200:16.0:1547252945.244189:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f09beb8000] (19)++
00000800:00000200:16.0:1547252945.244190:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f09beb8000] (20)--
00000800:00000200:16.0:1547252945.244190:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f09beb8000] (19)++
00000800:00000200:16.0:1547252945.244191:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f09beb8000] (20)--
00000800:00000200:16.0:1547252945.244191:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f09beb8000] (19)++
00000800:00000200:16.0:1547252945.244192:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f09beb8000] (20)--
00000800:00000200:16.0:1547252945.244192:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f09beb8000] (19)++
00000800:00000200:16.0:1547252945.244193:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f09beb8000] (20)--
00000800:00000200:16.0:1547252945.244193:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f09beb8000] (19)++
00000800:00000200:16.0:1547252945.244194:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f09beb8000] (20)--
00000800:00000200:16.0:1547252945.244195:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f09beb8000] (19)++
00000800:00000200:16.0:1547252945.244196:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f09beb8000] (20)--
00000800:00000200:16.0:1547252945.244196:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f09beb8000] (19)++
00000800:00000200:16.0:1547252945.244197:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f09beb8000] (20)--
00000800:00000200:12.0F:1547252945.245818:0:205094:0:(o2iblnd_cb.c:3264:kiblnd_cm_callback()) ESTABLISHED(active): 10.0.11.90@o2ib10
00000800:00000200:12.0:1547252945.245822:0:205094:0:(o2iblnd_cb.c:2281:kiblnd_connreq_done()) 10.0.11.90@o2ib10: active(1), version(12), status(0)
00000800:00000200:12.0:1547252945.245824:0:205094:0:(o2iblnd_cb.c:2308:kiblnd_connreq_done()) conn[ffff91f09beb8000] (19)++
00000800:00000200:12.0:1547252945.245825:0:205094:0:(o2iblnd_cb.c:2351:kiblnd_connreq_done()) conn[ffff91f09beb8000] (20)++
00000800:00000200:12.0:1547252945.245826:0:205094:0:(o2iblnd_cb.c:1290:kiblnd_queue_tx_locked()) conn[ffff91f09beb8000] (21)++
00000800:00000200:12.0:1547252945.245828:0:205094:0:(o2iblnd_cb.c:2371:kiblnd_connreq_done()) conn[ffff91f09beb8000] (22)--
00000800:00000200:10.2F:1547252945.245838:0:0:0:(o2iblnd_cb.c:3710:kiblnd_cq_completion()) conn[ffff91f09beb8000] (21)++
00000800:00000200:10.0F:1547252945.245850:0:235236:0:(o2iblnd_cb.c:3832:kiblnd_scheduler()) conn[ffff91f09beb8000] (22)++
00000800:00000200:10.0:1547252945.245853:0:235236:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91f09beb8000] (23)--
00000400:00000200:10.0:1547252945.245855:0:235236:0:(peer.c:2345:lnet_discovery_event_handler()) Received event: 5
00000800:00000200:1.0:1547252945.245856:0:235237:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f09beb8000] (22)--
00000400:00000200:10.0:1547252945.245857:0:235236:0:(peer.c:2297:lnet_discovery_event_send()) Ping Send to 10.0.11.90@o2ib10: 0
00000400:00000200:10.0:1547252945.245859:0:235236:0:(lib-msg.c:816:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1547252945.245862:0:235236:0:(lib-msg.c:630:lnet_health_check()) health check: 10.0.10.184@o2ib10->10.0.11.90@o2ib10: GET: OK
00000800:00000200:10.0:1547252945.245865:0:235236:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f09beb8000] (21)--
00000800:00000200:10.2:1547252945.246000:0:0:0:(o2iblnd_cb.c:3710:kiblnd_cq_completion()) conn[ffff91f09beb8000] (20)++
00000800:00000200:10.0:1547252945.246011:0:235238:0:(o2iblnd_cb.c:3832:kiblnd_scheduler()) conn[ffff91f09beb8000] (21)++
00000800:00000200:10.0:1547252945.246017:0:235238:0:(o2iblnd_cb.c:338:kiblnd_handle_rx()) Received d1[1] from 10.0.11.90@o2ib10
00000800:00000200:1.0:1547252945.246019:0:235239:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f09beb8000] (22)--
00000400:00000200:10.0:1547252945.246026:0:235238:0:(lib-move.c:4114:lnet_parse()) TRACE: 10.0.10.184@o2ib10(10.0.10.184@o2ib10) <- 10.0.11.90@o2ib10 : REPLY - for me
00000400:00000200:10.0:1547252945.246037:0:235238:0:(lib-move.c:3881:lnet_parse_reply()) 10.0.10.184@o2ib10: Reply from 12345-10.0.11.90@o2ib10 of length 64/64 into md 0x19
00000400:00000200:10.0:1547252945.246043:0:235238:0:(peer.c:2345:lnet_discovery_event_handler()) Received event: 3
00000400:00000200:10.0:1547252945.246047:0:235238:0:(peer.c:897:lnet_peer_ni_clr_non_mr_pref_nid()) peer 10.0.11.90@o2ib10: 0
00000400:00000200:10.0:1547252945.246051:0:235238:0:(peer.c:2256:lnet_discovery_event_reply()) peer 10.0.11.90@o2ib10 data present 1
00000400:00000200:10.0:1547252945.246057:0:235238:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff91f094675a90
00000400:00000200:10.0:1547252945.246060:0:235238:0:(lib-msg.c:816:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1547252945.246060:0:235246:0:(peer.c:3007:lnet_peer_discovery_wait_for_work()) woken: 0
00000800:00000200:10.0:1547252945.246064:0:235238:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f09beb8000] (21)++
00000400:00000200:1.0:1547252945.246064:0:235246:0:(peer.c:3109:lnet_peer_discovery()) peer 10.0.11.90@o2ib10 state 0x61
00000800:00000200:10.0:1547252945.246067:0:235238:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f09beb8000] (22)--
00000800:00000200:10.0:1547252945.246069:0:235238:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f09beb8000] (21)--
00000400:00000200:1.0:1547252945.246076:0:235246:0:(peer.c:201:lnet_peer_ni_alloc()) ffff91f09bee0000 nid 10.1.11.90@o2ib10
00000400:00000200:1.0:1547252945.246082:0:235246:0:(peer.c:1202:lnet_peer_attach_peer_ni()) peer 10.0.11.90@o2ib10 NID 10.1.11.90@o2ib10 flags 0x9
00000400:00000200:1.0:1547252945.246086:0:235246:0:(peer.c:2496:lnet_peer_merge_data()) peer 10.0.11.90@o2ib10: 0
00000400:00000200:1.0:1547252945.246090:0:235246:0:(peer.c:2656:lnet_peer_data_present()) peer 10.0.11.90@o2ib10: 0
00000400:00000200:1.0:1547252945.246093:0:235246:0:(peer.c:3130:lnet_peer_discovery()) peer 10.0.11.90@o2ib10 state 0xa1 rc 1
00000400:00000200:1.0:1547252945.246097:0:235246:0:(peer.c:3109:lnet_peer_discovery()) peer 10.0.11.90@o2ib10 state 0xa1
00000400:00000200:1.0:1547252945.246103:0:235246:0:(lib-move.c:4562:LNetPut()) LNetPut -> 12345-10.0.11.90@o2ib10
00000400:00000200:1.0:1547252945.246108:0:235246:0:(lib-move.c:2450:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.11.90@o2ib10 local destination
00000400:00000200:1.0:1547252945.246113:0:235246:0:(lib-move.c:1510:lnet_get_best_ni()) compare ni 10.0.10.184@o2ib10 [c:256, d:21, s:1] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1547252945.246120:0:235246:0:(lib-move.c:1553:lnet_get_best_ni()) selected best_ni 10.0.10.184@o2ib10
00000400:00000200:1.0:1547252945.246123:0:235246:0:(lib-move.c:1766:lnet_select_peer_ni()) 10.0.10.184@o2ib10 ni_is_pref = 0
00000400:00000200:1.0:1547252945.246127:0:235246:0:(lib-move.c:1766:lnet_select_peer_ni()) 10.0.10.184@o2ib10 ni_is_pref = 0
00000400:00000200:1.0:1547252945.246130:0:235246:0:(lib-move.c:1772:lnet_select_peer_ni()) 10.1.11.90@o2ib10 c:[8, 8], s:[0, 1]
00000400:00000200:1.0:1547252945.246134:0:235246:0:(lib-move.c:1822:lnet_select_peer_ni()) sd_best_lpni = 10.1.11.90@o2ib10
00000400:00000200:1.0:1547252945.246138:0:235246:0:(lib-move.c:1714:lnet_handle_send()) rspt_next_hop_nid = 10.1.11.90@o2ib10
00000400:00000200:1.0:1547252945.246148:0:235246:0:(lib-move.c:1728:lnet_handle_send()) TRACE: 10.0.10.184@o2ib10(10.0.10.184@o2ib10:<?>) -> 10.1.11.90@o2ib10(10.0.11.90@o2ib10:10.1.11.90@o2ib10) : PUT try# 0
00000800:00000200:1.0:1547252945.246155:0:235246:0:(o2iblnd_cb.c:1636:kiblnd_send()) sending 48 bytes in 1 frags to 12345-10.1.11.90@o2ib10
00000800:00000200:1.0:1547252945.246161:0:235246:0:(o2iblnd_cb.c:1604:kiblnd_launch_tx()) peer_ni[ffff91f07b11ad00] -> 10.1.11.90@o2ib10 (1)++
00000800:00000200:1.0:1547252945.246167:0:235246:0:(o2iblnd_cb.c:1409:kiblnd_connect_peer()) peer_ni[ffff91f07b11ad00] -> 10.1.11.90@o2ib10 (2)++
00000800:00000200:1.0:1547252945.246177:0:235246:0:(o2iblnd_cb.c:1362:kiblnd_resolve_addr()) bound to port 1023
00000800:00000200:1.0:1547252945.246180:0:235246:0:(o2iblnd_cb.c:1611:kiblnd_launch_tx()) peer_ni[ffff91f07b11ad00] -> 10.1.11.90@o2ib10 (3)--
00000400:00000200:1.0:1547252945.246184:0:235246:0:(peer.c:2867:lnet_peer_send_push()) peer 10.0.11.90@o2ib10
00000800:00000200:16.0:1547252945.246185:0:232995:0:(o2iblnd_cb.c:3156:kiblnd_cm_callback()) 10.1.11.90@o2ib10 Addr resolved: 0
00000400:00000200:1.0:1547252945.246187:0:235246:0:(peer.c:3130:lnet_peer_discovery()) peer 10.0.11.90@o2ib10 state 0x2a1 rc 0
00000800:00000200:16.0:1547252945.246199:0:232995:0:(o2iblnd_cb.c:3173:kiblnd_cm_callback()) 10.1.11.90@o2ib10: connection bound to ib0:10.0.10.184:mlx5_0
00000800:00000200:16.0:1547252945.246629:0:232995:0:(o2iblnd_cb.c:3197:kiblnd_cm_callback()) 10.1.11.90@o2ib10 Route resolved: 0
00000800:00000200:16.0:1547252945.246658:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 0: ffff91f07518a000 0x177518a000(0x177518a000)
00000800:00000200:16.0:1547252945.246662:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 1: ffff91f093b5b000 0x1793b5b000(0x1793b5b000)
00000800:00000200:16.0:1547252945.246664:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 2: ffff91f093b5a000 0x1793b5a000(0x1793b5a000)
00000800:00000200:16.0:1547252945.246666:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 3: ffff91f0d574d000 0x17d574d000(0x17d574d000)
00000800:00000200:16.0:1547252945.246668:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 4: ffff91f0d574c000 0x17d574c000(0x17d574c000)
00000800:00000200:16.0:1547252945.246671:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 5: ffff91f0986db000 0x17986db000(0x17986db000)
00000800:00000200:16.0:1547252945.246673:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 6: ffff91f0986da000 0x17986da000(0x17986da000)
00000800:00000200:16.0:1547252945.246675:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 7: ffff91f091a2b000 0x1791a2b000(0x1791a2b000)
00000800:00000200:16.0:1547252945.246677:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 8: ffff91f091a2a000 0x1791a2a000(0x1791a2a000)
00000800:00000200:16.0:1547252945.246679:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 9: ffff91f061bf1000 0x1761bf1000(0x1761bf1000)
00000800:00000200:16.0:1547252945.246681:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 10: ffff91f061bf0000 0x1761bf0000(0x1761bf0000)
00000800:00000200:16.0:1547252945.246684:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 11: ffff91db1aa15000 0x21aa15000(0x21aa15000)
00000800:00000200:16.0:1547252945.246686:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 12: ffff91db1aa14000 0x21aa14000(0x21aa14000)
00000800:00000200:16.0:1547252945.246688:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 13: ffff91ef7c4f9000 0x167c4f9000(0x167c4f9000)
00000800:00000200:16.0:1547252945.246691:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 14: ffff91ef7c4f8000 0x167c4f8000(0x167c4f8000)
00000800:00000200:16.0:1547252945.246693:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 15: ffff91f09af19000 0x179af19000(0x179af19000)
00000800:00000200:16.0:1547252945.246695:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 16: ffff91f0d3ad0000 0x17d3ad0000(0x17d3ad0000)
00000800:00000200:16.0:1547252945.246697:0:232995:0:(o2iblnd.c:1340:kiblnd_map_rx_descs()) rx 17: ffff91f0d8f70000 0x17d8f70000(0x17d8f70000)
00000800:00000200:16.0:1547252945.247349:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f06e3c8000] (19)++
00000800:00000200:16.0:1547252945.247351:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f06e3c8000] (20)--
00000800:00000200:16.0:1547252945.247353:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f06e3c8000] (19)++
00000800:00000200:16.0:1547252945.247355:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f06e3c8000] (20)--
00000800:00000200:16.0:1547252945.247356:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f06e3c8000] (19)++
00000800:00000200:16.0:1547252945.247358:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f06e3c8000] (20)--
00000800:00000200:16.0:1547252945.247359:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f06e3c8000] (19)++
00000800:00000200:16.0:1547252945.247361:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f06e3c8000] (20)--
00000800:00000200:16.0:1547252945.247363:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f06e3c8000] (19)++
00000800:00000200:16.0:1547252945.247365:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f06e3c8000] (20)--
00000800:00000200:16.0:1547252945.247367:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f06e3c8000] (19)++
00000800:00000200:16.0:1547252945.247368:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f06e3c8000] (20)--
00000800:00000200:16.0:1547252945.247370:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f06e3c8000] (19)++
00000800:00000200:16.0:1547252945.247371:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f06e3c8000] (20)--
00000800:00000200:16.0:1547252945.247373:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f06e3c8000] (19)++
00000800:00000200:16.0:1547252945.247375:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f06e3c8000] (20)--
00000800:00000200:16.0:1547252945.247376:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f06e3c8000] (19)++
00000800:00000200:16.0:1547252945.247378:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f06e3c8000] (20)--
00000800:00000200:16.0:1547252945.247379:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f06e3c8000] (19)++
00000800:00000200:16.0:1547252945.247381:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f06e3c8000] (20)--
00000800:00000200:16.0:1547252945.247383:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f06e3c8000] (19)++
00000800:00000200:16.0:1547252945.247384:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f06e3c8000] (20)--
00000800:00000200:16.0:1547252945.247386:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f06e3c8000] (19)++
00000800:00000200:16.0:1547252945.247387:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f06e3c8000] (20)--
00000800:00000200:16.0:1547252945.247389:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f06e3c8000] (19)++
00000800:00000200:16.0:1547252945.247391:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f06e3c8000] (20)--
00000800:00000200:16.0:1547252945.247392:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f06e3c8000] (19)++
00000800:00000200:16.0:1547252945.247394:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f06e3c8000] (20)--
00000800:00000200:16.0:1547252945.247395:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f06e3c8000] (19)++
00000800:00000200:16.0:1547252945.247403:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f06e3c8000] (20)--
00000800:00000200:16.0:1547252945.247404:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f06e3c8000] (19)++
00000800:00000200:16.0:1547252945.247406:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f06e3c8000] (20)--
00000800:00000200:16.0:1547252945.247408:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f06e3c8000] (19)++
00000800:00000200:16.0:1547252945.247409:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f06e3c8000] (20)--
00000800:00000200:16.0:1547252945.247411:0:232995:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f06e3c8000] (19)++
00000800:00000200:16.0:1547252945.247412:0:232995:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f06e3c8000] (20)--
00000800:00000200:12.0:1547252945.249024:0:205094:0:(o2iblnd_cb.c:3264:kiblnd_cm_callback()) ESTABLISHED(active): 10.1.11.90@o2ib10
00000800:00000200:12.0:1547252945.249028:0:205094:0:(o2iblnd_cb.c:2281:kiblnd_connreq_done()) 10.1.11.90@o2ib10: active(1), version(12), status(0)
00000800:00000200:12.0:1547252945.249031:0:205094:0:(o2iblnd_cb.c:2308:kiblnd_connreq_done()) conn[ffff91f06e3c8000] (19)++
00000800:00000200:12.0:1547252945.249033:0:205094:0:(o2iblnd_cb.c:2351:kiblnd_connreq_done()) conn[ffff91f06e3c8000] (20)++
00000800:00000200:12.0:1547252945.249034:0:205094:0:(o2iblnd_cb.c:1290:kiblnd_queue_tx_locked()) conn[ffff91f06e3c8000] (21)++
00000800:00000200:12.0:1547252945.249038:0:205094:0:(o2iblnd_cb.c:2371:kiblnd_connreq_done()) conn[ffff91f06e3c8000] (22)--
00000800:00000200:13.2F:1547252945.249050:0:0:0:(o2iblnd_cb.c:3710:kiblnd_cq_completion()) conn[ffff91f06e3c8000] (21)++
00000800:00000200:1.0:1547252945.249066:0:235237:0:(o2iblnd_cb.c:3832:kiblnd_scheduler()) conn[ffff91f06e3c8000] (22)++
00000800:00000200:1.0:1547252945.249071:0:235237:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91f06e3c8000] (23)--
00000400:00000200:1.0:1547252945.249074:0:235237:0:(peer.c:2345:lnet_discovery_event_handler()) Received event: 5
00000800:00000200:10.0:1547252945.249076:0:235236:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f06e3c8000] (22)--
00000400:00000200:1.0:1547252945.249078:0:235237:0:(peer.c:2297:lnet_discovery_event_send()) Push Send to 10.0.11.90@o2ib10: 0
00000400:00000200:1.0:1547252945.249080:0:235237:0:(lib-msg.c:816:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1547252945.249087:0:235237:0:(lib-msg.c:630:lnet_health_check()) health check: 10.0.10.184@o2ib10->10.1.11.90@o2ib10: PUT: OK
00000800:00000200:1.0:1547252945.249091:0:235237:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f06e3c8000] (21)--
00000400:00000100:3.0:1547253020.213520:0:235245:0:(lib-move.c:2772:lnet_finalize_expired_responses()) Response timed out: md = ffff91f094674f68: nid = 10.1.11.90@o2ib10
00000400:00000200:3.0:1547253020.213522:0:235245:0:(peer.c:2345:lnet_discovery_event_handler()) Received event: 6
00000400:00000200:3.0:1547253020.213523:0:235245:0:(peer.c:2325:lnet_discovery_event_unlink()) Push Unlink for message to peer 10.0.11.90@o2ib10
00000400:00000200:3.0:1547253020.213526:0:235245:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff91f094674f68
00000400:00000200:4.0F:1547253020.213573:0:235246:0:(peer.c:3007:lnet_peer_discovery_wait_for_work()) woken: 0
00000400:00000200:4.0:1547253020.213590:0:235246:0:(peer.c:3109:lnet_peer_discovery()) peer 10.0.11.90@o2ib10 state 0x8a1
00000400:00000200:4.0:1547253020.213592:0:235246:0:(peer.c:2803:lnet_peer_push_failed()) peer 10.0.11.90@o2ib10
00000400:00000200:4.0:1547253020.213593:0:235246:0:(peer.c:3130:lnet_peer_discovery()) peer 10.0.11.90@o2ib10 state 0xa1 rc -110
00000400:00000200:4.0:1547253020.213595:0:235246:0:(peer.c:2896:lnet_peer_discovery_error()) Discovery error 10.0.11.90@o2ib10: -110
00000400:00000200:4.0:1547253020.213596:0:235246:0:(peer.c:1785:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 10.0.11.90@o2ib10
00000400:00000200:3.0:1547253020.213605:0:235147:0:(peer.c:2108:lnet_discover_peer_locked()) peer 10.0.11.90@o2ib10 NID 10.0.11.90@o2ib10: -110. discovery complete
00000400:00000200:3.0:1547253020.213607:0:235147:0:(peer.c:1106:LNetPrimaryNID()) NID 10.0.11.90@o2ib10 primary NID 10.0.11.90@o2ib10 rc -110
00010000:00080000:3.0:1547253020.213612:0:235147:0:(ldlm_lib.c:113:import_set_conn()) imp ffff920894951800@MGC10.0.11.90@o2ib10: add connection MGC10.0.11.90@o2ib10_0 at head
00000040:01000000:3.0:1547253020.213642:0:235147:0:(llog_obd.c:214:llog_setup()) obd MGC10.0.11.90@o2ib10 ctxt 1 is initialized
10000000:01000000:1.0:1547253020.213865:0:235343:0:(mgc_request.c:622:mgc_requeue_thread()) Starting requeue thread
00000020:00000080:3.0:1547253020.213871:0:235147:0:(obd_config.c:538:class_setup()) finished setup of obd MGC10.0.11.90@o2ib10 (uuid 8534aa65-7ac8-198c-0718-3e6f83d57705)
00000020:00000080:3.0:1547253020.213877:0:235147:0:(genops.c:1419:class_connect()) connect: client 8534aa65-7ac8-198c-0718-3e6f83d57705, cookie 0xf0b66cac3fff399
00000100:00080000:3.0:1547253020.213879:0:235147:0:(import.c:677:ptlrpc_connect_import()) ffff920894951800 MGS: changing import state from NEW to CONNECTING
00000100:00080000:3.0:1547253020.213880:0:235147:0:(import.c:523:import_select_connection()) MGC10.0.11.90@o2ib10: connect to NID 10.0.11.90@o2ib10 last attempt 0
00000100:00080000:3.0:1547253020.213882:0:235147:0:(import.c:599:import_select_connection()) MGC10.0.11.90@o2ib10: import ffff920894951800 using connection MGC10.0.11.90@o2ib10_0/10.0.11.90@o2ib10
00000100:00080000:3.0:1547253020.213895:0:235147:0:(pinger.c:388:ptlrpc_pinger_add_import()) adding pingable import 8534aa65-7ac8-198c-0718-3e6f83d57705->MGS
00000080:01000000:3.0:1547253020.213910:0:235147:0:(llite_lib.c:106:ll_init_sbi()) generated uuid: 661d9c4d-c98c-da60-6aa7-63326856983b
00000100:00000200:1.0:1547253020.213921:0:235247:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1622412304056336, portal 25
00000100:00000200:1.0:1547253020.213924:0:235247:0:(niobuf.c:85:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1622412304056336, offset 0
00000400:00000200:1.0:1547253020.213927:0:235247:0:(lib-move.c:4562:LNetPut()) LNetPut -> 12345-10.0.11.90@o2ib10
00000400:00000200:1.0:1547253020.213932:0:235247:0:(peer.c:1767:lnet_peer_queue_for_discovery()) Queue peer 10.0.11.90@o2ib10: 0
00000400:00000200:1.0:1547253020.213933:0:235247:0:(peer.c:2101:lnet_discover_peer_locked()) non-blocking discovery
00000400:00000200:1.0:1547253020.213935:0:235247:0:(peer.c:2108:lnet_discover_peer_locked()) peer 10.0.11.90@o2ib10 NID 10.0.11.90@o2ib10: 0. pending discovery
00000400:00000200:4.0:1547253020.213936:0:235246:0:(peer.c:3007:lnet_peer_discovery_wait_for_work()) woken: 0
00000400:00000200:1.0:1547253020.213937:0:235247:0:(lib-move.c:2574:lnet_select_pathway()) 10.0.11.90@o2ib10 pending discovery
00000400:00000200:4.0:1547253020.213938:0:235246:0:(peer.c:3109:lnet_peer_discovery()) peer 10.0.11.90@o2ib10 state 0xb1
00000400:00000200:4.0:1547253020.213942:0:235246:0:(lib-move.c:4562:LNetPut()) LNetPut -> 12345-10.0.11.90@o2ib10
00000400:00000200:4.0:1547253020.213945:0:235246:0:(lib-move.c:2450:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.11.90@o2ib10 local destination
00000400:00000200:4.0:1547253020.213947:0:235246:0:(lib-move.c:1510:lnet_get_best_ni()) compare ni 10.0.10.184@o2ib10 [c:256, d:21, s:2] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:4.0:1547253020.213949:0:235246:0:(lib-move.c:1553:lnet_get_best_ni()) selected best_ni 10.0.10.184@o2ib10
00000400:00000200:4.0:1547253020.213950:0:235246:0:(lib-move.c:1766:lnet_select_peer_ni()) 10.0.10.184@o2ib10 ni_is_pref = 0
00000400:00000200:4.0:1547253020.213951:0:235246:0:(lib-move.c:1766:lnet_select_peer_ni()) 10.0.10.184@o2ib10 ni_is_pref = 0
00000400:00000200:4.0:1547253020.213952:0:235246:0:(lib-move.c:1772:lnet_select_peer_ni()) 10.1.11.90@o2ib10 c:[8, 8], s:[1, 1]
00000400:00000200:4.0:1547253020.213954:0:235246:0:(lib-move.c:1822:lnet_select_peer_ni()) sd_best_lpni = 10.0.11.90@o2ib10
00000400:00000200:4.0:1547253020.213955:0:235246:0:(lib-move.c:1714:lnet_handle_send()) rspt_next_hop_nid = 10.0.11.90@o2ib10
00000400:00000200:4.0:1547253020.213958:0:235246:0:(lib-move.c:1728:lnet_handle_send()) TRACE: 10.0.10.184@o2ib10(10.0.10.184@o2ib10:<?>) -> 10.0.11.90@o2ib10(10.0.11.90@o2ib10:10.0.11.90@o2ib10) : PUT try# 0
00000800:00000200:4.0:1547253020.213961:0:235246:0:(o2iblnd_cb.c:1636:kiblnd_send()) sending 48 bytes in 1 frags to 12345-10.0.11.90@o2ib10
00000800:00000200:4.0:1547253020.213964:0:235246:0:(o2iblnd.c:403:kiblnd_find_peer_locked()) got peer_ni [ffff91f07b11ad80] -> 10.0.11.90@o2ib10 (2) version: 12
00000800:00000200:4.0:1547253020.213965:0:235246:0:(o2iblnd_cb.c:1516:kiblnd_launch_tx()) conn[ffff91f09beb8000] (20)++
00000800:00000200:4.0:1547253020.213966:0:235246:0:(o2iblnd_cb.c:1290:kiblnd_queue_tx_locked()) conn[ffff91f09beb8000] (21)++
00000800:00000200:4.0:1547253020.213969:0:235246:0:(o2iblnd_cb.c:1522:kiblnd_launch_tx()) conn[ffff91f09beb8000] (22)--
00000400:00000200:4.0:1547253020.213970:0:235246:0:(peer.c:2867:lnet_peer_send_push()) peer 10.0.11.90@o2ib10
00000400:00000200:4.0:1547253020.213971:0:235246:0:(peer.c:3130:lnet_peer_discovery()) peer 10.0.11.90@o2ib10 state 0x2b1 rc 0
00000800:00000200:10.2:1547253020.214016:0:0:0:(o2iblnd_cb.c:3710:kiblnd_cq_completion()) conn[ffff91f09beb8000] (21)++
10000000:01000000:3.0:1547253020.214028:0:235147:0:(mgc_request.c:2205:mgc_process_config()) parse_log cache1-client from 0
10000000:01000000:3.0:1547253020.214029:0:235147:0:(mgc_request.c:329:config_log_add()) add config log cache1-client-ffff9208d990e000
10000000:01000000:3.0:1547253020.214030:0:235147:0:(mgc_request.c:211:do_config_log_add()) do adding config log cache1-sptlrpc-ffff91e3c04f0000
10000000:01000000:3.0:1547253020.214031:0:235147:0:(mgc_request.c:90:mgc_name2resid()) log cache1-sptlrpc to resid 0x316568636163/0x0 (cache1)
10000000:01000000:3.0:1547253020.214032:0:235147:0:(mgc_request.c:2059:mgc_process_log()) Process log cache1-sptlrpc-ffff91e3c04f0000 from 1
10000000:01000000:3.0:1547253020.214035:0:235147:0:(mgc_request.c:1129:mgc_enqueue()) Enqueue for cache1-sptlrpc (res 0x316568636163)
00000800:00000200:10.0:1547253020.214041:0:235239:0:(o2iblnd_cb.c:3832:kiblnd_scheduler()) conn[ffff91f09beb8000] (22)++
00000800:00000200:10.0:1547253020.214045:0:235239:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91f09beb8000] (23)--
00000400:00000200:10.0:1547253020.214047:0:235239:0:(peer.c:2345:lnet_discovery_event_handler()) Received event: 5
00000400:00000200:10.0:1547253020.214049:0:235239:0:(peer.c:2297:lnet_discovery_event_send()) Push Send to 10.0.11.90@o2ib10: 0
00000400:00000200:10.0:1547253020.214050:0:235239:0:(lib-msg.c:816:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1547253020.214052:0:235239:0:(lib-msg.c:630:lnet_health_check()) health check: 10.0.10.184@o2ib10->10.0.11.90@o2ib10: PUT: OK
00000800:00000200:10.0:1547253020.214054:0:235239:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f09beb8000] (22)--
00000100:00080000:3.0:1547253020.214055:0:235147:0:(client.c:1575:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING)  req@ffff91d95e2f0300 x1622412304056352/t0(0) o101->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1
00000800:00000200:10.0:1547253020.214056:0:235239:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f09beb8000] (21)--
00000800:00000200:10.2:1547253020.214139:0:0:0:(o2iblnd_cb.c:3710:kiblnd_cq_completion()) conn[ffff91f09beb8000] (20)++
00000800:00000200:10.0:1547253020.214144:0:235236:0:(o2iblnd_cb.c:3832:kiblnd_scheduler()) conn[ffff91f09beb8000] (21)++
00000800:00000200:10.0:1547253020.214148:0:235236:0:(o2iblnd_cb.c:338:kiblnd_handle_rx()) Received d1[0] from 10.0.11.90@o2ib10
00000400:00000200:10.0:1547253020.214151:0:235236:0:(lib-move.c:4114:lnet_parse()) TRACE: 10.0.10.184@o2ib10(10.0.10.184@o2ib10) <- 10.0.11.90@o2ib10 : ACK - for me
00000800:00000200:1.0:1547253020.214152:0:235237:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f09beb8000] (22)--
00000400:00000200:10.0:1547253020.214154:0:235236:0:(lib-move.c:3934:lnet_parse_ack()) 10.0.10.184@o2ib10: ACK from 12345-10.0.11.90@o2ib10 into md 0x39
00000400:00000200:10.0:1547253020.214156:0:235236:0:(peer.c:2345:lnet_discovery_event_handler()) Received event: 4
00000400:00000200:10.0:1547253020.214157:0:235236:0:(peer.c:2130:lnet_discovery_event_ack()) peer 10.0.11.90@o2ib10 ev->status 0
00000400:00000200:10.0:1547253020.214160:0:235236:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff91e2cac5eb28
00000400:00000200:10.0:1547253020.214163:0:235236:0:(lib-msg.c:816:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:4.0:1547253020.214163:0:235246:0:(peer.c:3007:lnet_peer_discovery_wait_for_work()) woken: 0
00000800:00000200:10.0:1547253020.214165:0:235236:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f09beb8000] (21)++
00000400:00000200:4.0:1547253020.214165:0:235246:0:(peer.c:3109:lnet_peer_discovery()) peer 10.0.11.90@o2ib10 state 0xb1
00000400:00000200:4.0:1547253020.214166:0:235246:0:(peer.c:2915:lnet_peer_discovered()) peer 10.0.11.90@o2ib10
00000800:00000200:10.0:1547253020.214167:0:235236:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f09beb8000] (22)--
00000800:00000200:10.0:1547253020.214167:0:235236:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f09beb8000] (21)--
00000400:00000200:4.0:1547253020.214167:0:235246:0:(peer.c:3130:lnet_peer_discovery()) peer 10.0.11.90@o2ib10 state 0x89 rc 0
00000400:00000200:4.0:1547253020.214168:0:235246:0:(peer.c:1785:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 10.0.11.90@o2ib10
00000400:00000200:4.0:1547253020.214170:0:235246:0:(peer.c:1803:lnet_peer_discovery_complete()) sending pending message PUT to target 12345-10.0.11.90@o2ib10
00000400:00000200:4.0:1547253020.214171:0:235246:0:(lib-move.c:2450:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.11.90@o2ib10 local destination
00000400:00000200:4.0:1547253020.214173:0:235246:0:(lib-move.c:1510:lnet_get_best_ni()) compare ni 10.0.10.184@o2ib10 [c:256, d:21, s:3] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:4.0:1547253020.214174:0:235246:0:(lib-move.c:1553:lnet_get_best_ni()) selected best_ni 10.0.10.184@o2ib10
00000400:00000200:4.0:1547253020.214176:0:235246:0:(lib-move.c:1766:lnet_select_peer_ni()) 10.0.10.184@o2ib10 ni_is_pref = 0
00000400:00000200:4.0:1547253020.214177:0:235246:0:(lib-move.c:1766:lnet_select_peer_ni()) 10.0.10.184@o2ib10 ni_is_pref = 0
00000400:00000200:4.0:1547253020.214178:0:235246:0:(lib-move.c:1772:lnet_select_peer_ni()) 10.1.11.90@o2ib10 c:[8, 8], s:[1, 2]
00000400:00000200:4.0:1547253020.214180:0:235246:0:(lib-move.c:1822:lnet_select_peer_ni()) sd_best_lpni = 10.1.11.90@o2ib10
00000400:00000200:4.0:1547253020.214183:0:235246:0:(lib-move.c:1728:lnet_handle_send()) TRACE: 10.0.10.184@o2ib10(10.0.10.184@o2ib10:<?>) -> 10.1.11.90@o2ib10(10.0.11.90@o2ib10:10.1.11.90@o2ib10) : PUT try# 0
00000800:00000200:4.0:1547253020.214185:0:235246:0:(o2iblnd_cb.c:1636:kiblnd_send()) sending 520 bytes in 1 frags to 12345-10.1.11.90@o2ib10
00000800:00000200:4.0:1547253020.214187:0:235246:0:(o2iblnd.c:403:kiblnd_find_peer_locked()) got peer_ni [ffff91f07b11ad00] -> 10.1.11.90@o2ib10 (2) version: 12
00000800:00000200:4.0:1547253020.214188:0:235246:0:(o2iblnd_cb.c:1516:kiblnd_launch_tx()) conn[ffff91f06e3c8000] (20)++
00000800:00000200:4.0:1547253020.214189:0:235246:0:(o2iblnd_cb.c:1290:kiblnd_queue_tx_locked()) conn[ffff91f06e3c8000] (21)++
00000800:00000200:4.0:1547253020.214191:0:235246:0:(o2iblnd_cb.c:1522:kiblnd_launch_tx()) conn[ffff91f06e3c8000] (22)--
00000800:00000200:13.2:1547253020.214240:0:0:0:(o2iblnd_cb.c:3710:kiblnd_cq_completion()) conn[ffff91f06e3c8000] (21)++
00000800:00000200:13.0F:1547253020.214267:0:235239:0:(o2iblnd_cb.c:3832:kiblnd_scheduler()) conn[ffff91f06e3c8000] (22)++
00000800:00000200:13.0:1547253020.214273:0:235239:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91f06e3c8000] (23)--
00000100:00000200:13.0:1547253020.214277:0:235239:0:(events.c:57:request_out_callback()) @@@ type 5, status 0  req@ffff91d95e2f0000 x1622412304056336/t0(0) o250->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 520/544 e 0 to 0 dl 1547253025 ref 2 fl Rpc:N/0/ffffffff rc 0/-1
00000800:00000200:11.0F:1547253020.214277:0:235238:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f06e3c8000] (22)--
00000400:00000200:13.0:1547253020.214282:0:235239:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff91f094674770
00000400:00000200:13.0:1547253020.214283:0:235239:0:(lib-msg.c:816:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1547253020.214286:0:235239:0:(lib-msg.c:630:lnet_health_check()) health check: 10.0.10.184@o2ib10->10.1.11.90@o2ib10: PUT: OK
00000800:00000200:13.0:1547253020.214288:0:235239:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f06e3c8000] (21)--
00000100:00000200:1.0:1547253025.213526:0:235247:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0  req@ffff91d95e2f0000 x1622412304056336/t0(0) o250->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 520/544 e 0 to 1 dl 1547253025 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1
00000100:00000200:1.0:1547253025.213542:0:235247:0:(events.c:114:reply_in_callback()) @@@ unlink  req@ffff91d95e2f0000 x1622412304056336/t0(0) o250->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 520/544 e 0 to 1 dl 1547253025 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1
00000400:00000200:1.0:1547253025.213567:0:235247:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff91f094675188
00000100:00080000:1.0:1547253025.213570:0:235247:0:(import.c:1276:ptlrpc_connect_interpret()) ffff920894951800 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1547253025.213571:0:235247:0:(import.c:1323:ptlrpc_connect_interpret()) recovery of MGS on MGC10.0.11.90@o2ib10_0 failed (-110)
00000100:00080000:5.0F:1547253026.213510:0:235147:0:(client.c:1179:ptlrpc_import_delay_req()) @@@ send limit expired   req@ffff91d95e2f0300 x1622412304056352/t0(0) o101->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1
10000000:01000000:5.0:1547253026.213537:0:235147:0:(mgc_request.c:1021:mgc_blocking_ast()) Lock res [0x316568636163:0x0:0x0].0x0 (cache1)
10000000:01000000:5.0:1547253026.213550:0:235147:0:(mgc_request.c:2072:mgc_process_log()) Can't get cfg lock: -110
10000000:01000000:5.0:1547253026.213559:0:235147:0:(mgc_request.c:2139:mgc_process_log()) MGC10.0.11.90@o2ib10: configuration from log 'cache1-sptlrpc' failed (-5).
10000000:00020000:5.0:1547253026.213562:0:235147:0:(mgc_request.c:249:do_config_log_add()) MGC10.0.11.90@o2ib10: failed processing log, type 1: rc = -5
10000000:01000000:5.0:1547253026.215567:0:235147:0:(mgc_request.c:211:do_config_log_add()) do adding config log params-ffff9208d990e000
10000000:01000000:5.0:1547253026.215569:0:235147:0:(mgc_request.c:90:mgc_name2resid()) log params to resid 0x736d61726170/0x3 (params)
10000000:01000000:5.0:1547253026.215570:0:235147:0:(mgc_request.c:211:do_config_log_add()) do adding config log cache1-client-ffff9208d990e000
10000000:01000000:5.0:1547253026.215571:0:235147:0:(mgc_request.c:90:mgc_name2resid()) log cache1-client to resid 0x316568636163/0x0 (cache1)
10000000:01000000:5.0:1547253026.215572:0:235147:0:(mgc_request.c:211:do_config_log_add()) do adding config log cache1-cliir-ffff9208d990e000
10000000:01000000:5.0:1547253026.215573:0:235147:0:(mgc_request.c:90:mgc_name2resid()) log cache1-cliir to resid 0x316568636163/0x2 (cache1)
10000000:01000000:5.0:1547253026.215574:0:235147:0:(mgc_request.c:2059:mgc_process_log()) Process log cache1-client-ffff9208d990e000 from 1
10000000:01000000:5.0:1547253026.215574:0:235147:0:(mgc_request.c:1129:mgc_enqueue()) Enqueue for cache1-client (res 0x316568636163)
00000100:00080000:5.0:1547253026.215589:0:235147:0:(client.c:1575:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN)  req@ffff91f072b88000 x1622412304056368/t0(0) o101->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1
00000100:00080000:3.0:1547253045.253555:0:303:0:(pinger.c:248:ptlrpc_pinger_process_import()) 8534aa65-7ac8-198c-0718-3e6f83d57705->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 0 suppress 0
00000100:00080000:3.0:1547253045.253558:0:303:0:(recover.c:58:ptlrpc_initiate_recovery()) MGS: starting recovery
00000100:00080000:3.0:1547253045.253559:0:303:0:(import.c:677:ptlrpc_connect_import()) ffff920894951800 MGS: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1547253045.253561:0:303:0:(import.c:523:import_select_connection()) MGC10.0.11.90@o2ib10: connect to NID 10.0.11.90@o2ib10 last attempt 392493
00000100:00080000:3.0:1547253045.253562:0:303:0:(import.c:565:import_select_connection()) MGC10.0.11.90@o2ib10: tried all connections, increasing latency to 5s
00000100:00080000:3.0:1547253045.253564:0:303:0:(import.c:599:import_select_connection()) MGC10.0.11.90@o2ib10: import ffff920894951800 using connection MGC10.0.11.90@o2ib10_0/10.0.11.90@o2ib10
00000100:00000200:1.0:1547253045.253638:0:235247:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1622412304056384, portal 25
00000100:00000200:1.0:1547253045.253641:0:235247:0:(niobuf.c:85:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1622412304056384, offset 0
00000400:00000200:1.0:1547253045.253643:0:235247:0:(lib-move.c:4562:LNetPut()) LNetPut -> 12345-10.0.11.90@o2ib10
00000400:00000200:1.0:1547253045.253646:0:235247:0:(lib-move.c:2450:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.11.90@o2ib10 local destination
00000400:00000200:1.0:1547253045.253649:0:235247:0:(lib-move.c:1510:lnet_get_best_ni()) compare ni 10.0.10.184@o2ib10 [c:256, d:21, s:4] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1547253045.253650:0:235247:0:(lib-move.c:1553:lnet_get_best_ni()) selected best_ni 10.0.10.184@o2ib10
00000400:00000200:1.0:1547253045.253652:0:235247:0:(lib-move.c:1766:lnet_select_peer_ni()) 10.0.10.184@o2ib10 ni_is_pref = 0
00000400:00000200:1.0:1547253045.253653:0:235247:0:(lib-move.c:1766:lnet_select_peer_ni()) 10.0.10.184@o2ib10 ni_is_pref = 0
00000400:00000200:1.0:1547253045.253653:0:235247:0:(lib-move.c:1772:lnet_select_peer_ni()) 10.1.11.90@o2ib10 c:[8, 8], s:[2, 2]
00000400:00000200:1.0:1547253045.253655:0:235247:0:(lib-move.c:1822:lnet_select_peer_ni()) sd_best_lpni = 10.0.11.90@o2ib10
00000400:00000200:1.0:1547253045.253658:0:235247:0:(lib-move.c:1728:lnet_handle_send()) TRACE: 10.0.10.184@o2ib10(10.0.10.184@o2ib10:<?>) -> 10.0.11.90@o2ib10(10.0.11.90@o2ib10:10.0.11.90@o2ib10) : PUT try# 0
00000800:00000200:1.0:1547253045.253660:0:235247:0:(o2iblnd_cb.c:1636:kiblnd_send()) sending 520 bytes in 1 frags to 12345-10.0.11.90@o2ib10
00000800:00000200:1.0:1547253045.253663:0:235247:0:(o2iblnd.c:403:kiblnd_find_peer_locked()) got peer_ni [ffff91f07b11ad80] -> 10.0.11.90@o2ib10 (2) version: 12
00000800:00000200:1.0:1547253045.253664:0:235247:0:(o2iblnd_cb.c:1516:kiblnd_launch_tx()) conn[ffff91f09beb8000] (20)++
00000800:00000200:1.0:1547253045.253665:0:235247:0:(o2iblnd_cb.c:1290:kiblnd_queue_tx_locked()) conn[ffff91f09beb8000] (21)++
00000800:00000200:1.0:1547253045.253668:0:235247:0:(o2iblnd_cb.c:1522:kiblnd_launch_tx()) conn[ffff91f09beb8000] (22)--
00000800:00000200:10.2:1547253045.253719:0:0:0:(o2iblnd_cb.c:3710:kiblnd_cq_completion()) conn[ffff91f09beb8000] (21)++
00000800:00000200:10.0:1547253045.253745:0:235237:0:(o2iblnd_cb.c:3832:kiblnd_scheduler()) conn[ffff91f09beb8000] (22)++
00000800:00000200:10.0:1547253045.253748:0:235237:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91f09beb8000] (23)--
00000100:00000200:10.0:1547253045.253752:0:235237:0:(events.c:57:request_out_callback()) @@@ type 5, status 0  req@ffff91d95e2f0600 x1622412304056384/t0(0) o250->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 520/544 e 0 to 0 dl 1547253055 ref 2 fl Rpc:N/0/ffffffff rc 0/-1
00000400:00000200:10.0:1547253045.253757:0:235237:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff91f094675c28
00000400:00000200:10.0:1547253045.253758:0:235237:0:(lib-msg.c:816:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1547253045.253760:0:235237:0:(lib-msg.c:630:lnet_health_check()) health check: 10.0.10.184@o2ib10->10.0.11.90@o2ib10: PUT: OK
00000800:00000200:10.0:1547253045.253762:0:235237:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f09beb8000] (22)--
00000800:00000200:10.0:1547253045.253764:0:235237:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f09beb8000] (21)--
00000800:00000200:10.2:1547253045.253925:0:0:0:(o2iblnd_cb.c:3710:kiblnd_cq_completion()) conn[ffff91f09beb8000] (20)++
00000800:00000200:10.0:1547253045.253936:0:235238:0:(o2iblnd_cb.c:3832:kiblnd_scheduler()) conn[ffff91f09beb8000] (21)++
00000800:00000200:10.0:1547253045.253943:0:235238:0:(o2iblnd_cb.c:338:kiblnd_handle_rx()) Received d1[2] from 10.0.11.90@o2ib10
00000400:00000200:10.0:1547253045.253952:0:235238:0:(lib-move.c:4114:lnet_parse()) TRACE: 10.0.10.184@o2ib10(10.0.10.184@o2ib10) <- 10.0.11.90@o2ib10 : PUT - for me
00000400:00000200:10.0:1547253045.253961:0:235238:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.11.90@o2ib10 of length 384 into portal 25 MB=0x5c3934d100040
00000400:00000200:10.0:1547253045.253969:0:235238:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 19 from 12345-10.0.11.90@o2ib10 of length 384/384 into md 0x41 [1] + 0
00000100:00000200:10.0:1547253045.253978:0:235238:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0  req@ffff91d95e2f0600 x1622412304056384/t0(0) o250->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 520/544 e 0 to 0 dl 1547253055 ref 1 fl Rpc:N/0/ffffffff rc 0/-1
00000400:00000200:10.0:1547253045.253993:0:235238:0:(lib-msg.c:816:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1547253045.253997:0:235238:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f09beb8000] (22)++
00000800:00000200:13.0:1547253045.253999:0:235239:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f09beb8000] (22)--
00000800:00000200:10.0:1547253045.254000:0:235238:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f09beb8000] (23)--
00000100:00000200:1.0:1547253045.254004:0:235247:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0  req@ffff91d95e2f0600 x1622412304056384/t0(0) o250->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 520/544 e 0 to 0 dl 1547253055 ref 1 fl Rpc:RN/0/ffffffff rc 0/-1
00000800:00000200:10.0:1547253045.254005:0:235238:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f09beb8000] (21)--
00000100:00000200:1.0:1547253045.254018:0:235247:0:(events.c:114:reply_in_callback()) @@@ unlink  req@ffff91d95e2f0600 x1622412304056384/t0(0) o250->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 520/544 e 0 to 0 dl 1547253055 ref 1 fl Rpc:RN/0/ffffffff rc 0/-1
00000400:00000200:1.0:1547253045.254029:0:235247:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff91f094675188
00000100:00080000:1.0:1547253045.254039:0:235247:0:(import.c:1027:ptlrpc_connect_interpret()) MGC10.0.11.90@o2ib10: connect to target with instance 0
00000100:00080000:1.0:1547253045.254045:0:235247:0:(import.c:887:ptlrpc_connect_set_flags()) MGC10.0.11.90@o2ib10: Resetting ns_connect_flags to server flags: 0x2000011005002020
10000000:01000000:1.0:1547253045.254049:0:235247:0:(mgc_request.c:1363:mgc_import_event()) import event 0x808005
00000100:00080000:1.0:1547253045.254051:0:235247:0:(import.c:1132:ptlrpc_connect_interpret()) ffff920894951800 MGS: changing import state from CONNECTING to FULL
10000000:01000000:1.0:1547253045.254054:0:235247:0:(mgc_request.c:1363:mgc_import_event()) import event 0x808004
00000100:00080000:1.0:1547253045.254060:0:235247:0:(pinger.c:203:ptlrpc_pinger_ir_up()) IR up
00000100:00080000:1.0:1547253045.254068:0:235247:0:(recover.c:223:ptlrpc_wake_delayed()) @@@ waking (set ffff91f09abf8780):  req@ffff91f072b88000 x1622412304056368/t0(0) o101->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:W/0/ffffffff rc 0/-1
00000100:00000200:6.0F:1547253045.254155:0:235147:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1622412304056368, portal 25
00000100:00000200:6.0:1547253045.254163:0:235147:0:(niobuf.c:85:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1622412304056368, offset 0
00000400:00000200:6.0:1547253045.254171:0:235147:0:(lib-move.c:4562:LNetPut()) LNetPut -> 12345-10.0.11.90@o2ib10
00000400:00000200:6.0:1547253045.254179:0:235147:0:(lib-move.c:2450:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.11.90@o2ib10 local destination
00000400:00000200:6.0:1547253045.254185:0:235147:0:(lib-move.c:1510:lnet_get_best_ni()) compare ni 10.0.10.184@o2ib10 [c:256, d:21, s:5] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1547253045.254192:0:235147:0:(lib-move.c:1553:lnet_get_best_ni()) selected best_ni 10.0.10.184@o2ib10
00000400:00000200:6.0:1547253045.254196:0:235147:0:(lib-move.c:1766:lnet_select_peer_ni()) 10.0.10.184@o2ib10 ni_is_pref = 0
00000400:00000200:6.0:1547253045.254199:0:235147:0:(lib-move.c:1766:lnet_select_peer_ni()) 10.0.10.184@o2ib10 ni_is_pref = 0
00000400:00000200:6.0:1547253045.254203:0:235147:0:(lib-move.c:1772:lnet_select_peer_ni()) 10.1.11.90@o2ib10 c:[8, 8], s:[2, 3]
00000400:00000200:6.0:1547253045.254207:0:235147:0:(lib-move.c:1822:lnet_select_peer_ni()) sd_best_lpni = 10.1.11.90@o2ib10
00000400:00000200:6.0:1547253045.254219:0:235147:0:(lib-move.c:1728:lnet_handle_send()) TRACE: 10.0.10.184@o2ib10(10.0.10.184@o2ib10:<?>) -> 10.1.11.90@o2ib10(10.0.11.90@o2ib10:10.1.11.90@o2ib10) : PUT try# 0
00000800:00000200:6.0:1547253045.254227:0:235147:0:(o2iblnd_cb.c:1636:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.1.11.90@o2ib10
00000800:00000200:6.0:1547253045.254234:0:235147:0:(o2iblnd.c:403:kiblnd_find_peer_locked()) got peer_ni [ffff91f07b11ad00] -> 10.1.11.90@o2ib10 (2) version: 12
00000800:00000200:6.0:1547253045.254238:0:235147:0:(o2iblnd_cb.c:1516:kiblnd_launch_tx()) conn[ffff91f06e3c8000] (20)++
00000800:00000200:6.0:1547253045.254241:0:235147:0:(o2iblnd_cb.c:1290:kiblnd_queue_tx_locked()) conn[ffff91f06e3c8000] (21)++
00000800:00000200:6.0:1547253045.254247:0:235147:0:(o2iblnd_cb.c:1522:kiblnd_launch_tx()) conn[ffff91f06e3c8000] (22)--
00000800:00000200:13.2:1547253045.254257:0:0:0:(o2iblnd_cb.c:3710:kiblnd_cq_completion()) conn[ffff91f06e3c8000] (21)++
00000800:00000200:13.0:1547253045.254275:0:235237:0:(o2iblnd_cb.c:3832:kiblnd_scheduler()) conn[ffff91f06e3c8000] (22)++
00000800:00000200:13.0:1547253045.254281:0:235237:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91f06e3c8000] (23)--
00000800:00000200:11.0:1547253045.254288:0:235236:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f06e3c8000] (22)--
00000100:00000200:13.0:1547253045.254290:0:235237:0:(events.c:57:request_out_callback()) @@@ type 5, status 0  req@ffff91f072b88000 x1622412304056368/t0(0) o101->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 328/344 e 0 to 0 dl 1547253057 ref 3 fl Rpc:/0/ffffffff rc 0/-1
00000400:00000200:13.0:1547253045.254304:0:235237:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff91e2275cf430
00000400:00000200:13.0:1547253045.254307:0:235237:0:(lib-msg.c:816:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1547253045.254314:0:235237:0:(lib-msg.c:630:lnet_health_check()) health check: 10.0.10.184@o2ib10->10.1.11.90@o2ib10: PUT: OK
00000800:00000200:13.0:1547253045.254320:0:235237:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f06e3c8000] (21)--
00000800:00000200:13.2:1547253045.920569:0:0:0:(o2iblnd_cb.c:3710:kiblnd_cq_completion()) conn[ffff91f06e3c8000] (20)++
00000800:00000200:13.0:1547253045.920593:0:235239:0:(o2iblnd_cb.c:3832:kiblnd_scheduler()) conn[ffff91f06e3c8000] (21)++
00000800:00000200:13.0:1547253045.920602:0:235239:0:(o2iblnd_cb.c:338:kiblnd_handle_rx()) Received d0[3] from 10.1.11.90@o2ib10
00000800:00000200:13.0:1547253045.920606:0:235239:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f06e3c8000] (22)++
00000800:00000200:13.0:1547253045.920609:0:235239:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f06e3c8000] (23)--
00000800:00000200:13.0:1547253045.920610:0:235239:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f06e3c8000] (22)--
00000800:00000200:13.0:1547253045.920613:0:235239:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f06e3c8000] (21)--
10000000:01000000:2.0:1547253050.393492:0:235343:0:(mgc_request.c:596:do_requeue()) updating log cache1-sptlrpc
10000000:01000000:2.0:1547253050.393496:0:235343:0:(mgc_request.c:2059:mgc_process_log()) Process log cache1-sptlrpc-ffff91e3c04f0000 from 1
10000000:01000000:2.0:1547253050.393499:0:235343:0:(mgc_request.c:1129:mgc_enqueue()) Enqueue for cache1-sptlrpc (res 0x316568636163)
00000100:00000200:2.0:1547253050.393536:0:235343:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1622412304056400, portal 25
00000100:00000200:2.0:1547253050.393541:0:235343:0:(niobuf.c:85:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1622412304056400, offset 0
00000400:00000200:2.0:1547253050.393549:0:235343:0:(lib-move.c:4562:LNetPut()) LNetPut -> 12345-10.0.11.90@o2ib10
00000400:00000200:2.0:1547253050.393556:0:235343:0:(lib-move.c:2450:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.11.90@o2ib10 local destination
00000400:00000200:2.0:1547253050.393562:0:235343:0:(lib-move.c:1510:lnet_get_best_ni()) compare ni 10.0.10.184@o2ib10 [c:256, d:10, s:6] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1547253050.393569:0:235343:0:(lib-move.c:1553:lnet_get_best_ni()) selected best_ni 10.0.10.184@o2ib10
00000400:00000200:2.0:1547253050.393572:0:235343:0:(lib-move.c:1766:lnet_select_peer_ni()) 10.0.10.184@o2ib10 ni_is_pref = 0
00000400:00000200:2.0:1547253050.393576:0:235343:0:(lib-move.c:1766:lnet_select_peer_ni()) 10.0.10.184@o2ib10 ni_is_pref = 0
00000400:00000200:2.0:1547253050.393579:0:235343:0:(lib-move.c:1772:lnet_select_peer_ni()) 10.1.11.90@o2ib10 c:[8, 8], s:[3, 3]
00000400:00000200:2.0:1547253050.393584:0:235343:0:(lib-move.c:1822:lnet_select_peer_ni()) sd_best_lpni = 10.0.11.90@o2ib10
00000400:00000200:2.0:1547253050.393596:0:235343:0:(lib-move.c:1728:lnet_handle_send()) TRACE: 10.0.10.184@o2ib10(10.0.10.184@o2ib10:<?>) -> 10.0.11.90@o2ib10(10.0.11.90@o2ib10:10.0.11.90@o2ib10) : PUT try# 0
00000800:00000200:2.0:1547253050.393603:0:235343:0:(o2iblnd_cb.c:1636:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.11.90@o2ib10
00000800:00000200:2.0:1547253050.393610:0:235343:0:(o2iblnd.c:403:kiblnd_find_peer_locked()) got peer_ni [ffff91f07b11ad80] -> 10.0.11.90@o2ib10 (2) version: 12
00000800:00000200:2.0:1547253050.393614:0:235343:0:(o2iblnd_cb.c:1516:kiblnd_launch_tx()) conn[ffff91f09beb8000] (20)++
00000800:00000200:2.0:1547253050.393617:0:235343:0:(o2iblnd_cb.c:1290:kiblnd_queue_tx_locked()) conn[ffff91f09beb8000] (21)++
00000800:00000200:2.0:1547253050.393623:0:235343:0:(o2iblnd_cb.c:1522:kiblnd_launch_tx()) conn[ffff91f09beb8000] (22)--
00000800:00000200:10.2:1547253050.393678:0:0:0:(o2iblnd_cb.c:3710:kiblnd_cq_completion()) conn[ffff91f09beb8000] (21)++
00000800:00000200:10.0:1547253050.393703:0:235236:0:(o2iblnd_cb.c:3832:kiblnd_scheduler()) conn[ffff91f09beb8000] (22)++
00000800:00000200:10.0:1547253050.393711:0:235236:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91f09beb8000] (23)--
00000100:00000200:10.0:1547253050.393719:0:235236:0:(events.c:57:request_out_callback()) @@@ type 5, status 0  req@ffff91da594c8000 x1622412304056400/t0(0) o101->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 328/344 e 0 to 0 dl 1547253057 ref 3 fl Rpc:/0/ffffffff rc 0/-1
00000400:00000200:10.0:1547253050.393733:0:235236:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff91f07f0d9100
00000400:00000200:10.0:1547253050.393736:0:235236:0:(lib-msg.c:816:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1547253050.393744:0:235236:0:(lib-msg.c:630:lnet_health_check()) health check: 10.0.10.184@o2ib10->10.0.11.90@o2ib10: PUT: OK
00000800:00000200:10.0:1547253050.393749:0:235236:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f09beb8000] (22)--
00000800:00000200:10.0:1547253050.393752:0:235236:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f09beb8000] (21)--
00000800:00000200:10.2:1547253050.393869:0:0:0:(o2iblnd_cb.c:3710:kiblnd_cq_completion()) conn[ffff91f09beb8000] (20)++
00000800:00000200:10.0:1547253050.393874:0:235239:0:(o2iblnd_cb.c:3832:kiblnd_scheduler()) conn[ffff91f09beb8000] (21)++
00000800:00000200:10.0:1547253050.393878:0:235239:0:(o2iblnd_cb.c:338:kiblnd_handle_rx()) Received d1[1] from 10.0.11.90@o2ib10
00000400:00000200:10.0:1547253050.393881:0:235239:0:(lib-move.c:4114:lnet_parse()) TRACE: 10.0.10.184@o2ib10(10.0.10.184@o2ib10) <- 10.0.11.90@o2ib10 : PUT - for me
00000400:00000200:10.0:1547253050.393884:0:235239:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.11.90@o2ib10 of length 312 into portal 25 MB=0x5c3934d100050
00000400:00000200:10.0:1547253050.393887:0:235239:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 19 from 12345-10.0.11.90@o2ib10 of length 312/312 into md 0x61 [1] + 192
00000100:00000200:10.0:1547253050.393891:0:235239:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0  req@ffff91da594c8000 x1622412304056400/t0(0) o101->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 328/344 e 0 to 0 dl 1547253057 ref 2 fl Rpc:/0/ffffffff rc 0/-1
00000400:00000200:10.0:1547253050.393896:0:235239:0:(lib-msg.c:816:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1547253050.393897:0:235239:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f09beb8000] (22)++
00000800:00000200:10.0:1547253050.393915:0:235239:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f09beb8000] (23)--
00000800:00000200:10.0:1547253050.393916:0:235239:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f09beb8000] (22)--
00000800:00000200:10.0:1547253050.393917:0:235239:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f09beb8000] (21)--
00000100:00000200:2.0:1547253050.393918:0:235343:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0  req@ffff91da594c8000 x1622412304056400/t0(0) o101->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 328/344 e 0 to 0 dl 1547253057 ref 2 fl Rpc:R/0/ffffffff rc 0/-1
00000100:00000200:2.0:1547253050.393922:0:235343:0:(events.c:114:reply_in_callback()) @@@ unlink  req@ffff91da594c8000 x1622412304056400/t0(0) o101->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 328/344 e 0 to 0 dl 1547253057 ref 2 fl Rpc:R/0/ffffffff rc 0/-1
00000400:00000200:2.0:1547253050.393926:0:235343:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff91f07f0d9e48
00000100:00000200:2.0:1547253050.393939:0:235343:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1622412304056416, portal 25
00000100:00000200:2.0:1547253050.393940:0:235343:0:(niobuf.c:85:ptl_send_buf()) Sending 296 bytes to portal 26, xid 1622412304056416, offset 0
00000400:00000200:2.0:1547253050.393942:0:235343:0:(lib-move.c:4562:LNetPut()) LNetPut -> 12345-10.0.11.90@o2ib10
00000400:00000200:2.0:1547253050.393944:0:235343:0:(lib-move.c:2450:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.11.90@o2ib10 local destination
00000400:00000200:2.0:1547253050.393945:0:235343:0:(lib-move.c:1510:lnet_get_best_ni()) compare ni 10.0.10.184@o2ib10 [c:256, d:10, s:7] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1547253050.393947:0:235343:0:(lib-move.c:1553:lnet_get_best_ni()) selected best_ni 10.0.10.184@o2ib10
00000400:00000200:2.0:1547253050.393948:0:235343:0:(lib-move.c:1766:lnet_select_peer_ni()) 10.0.10.184@o2ib10 ni_is_pref = 0
00000400:00000200:2.0:1547253050.393949:0:235343:0:(lib-move.c:1766:lnet_select_peer_ni()) 10.0.10.184@o2ib10 ni_is_pref = 0
00000400:00000200:2.0:1547253050.393950:0:235343:0:(lib-move.c:1772:lnet_select_peer_ni()) 10.1.11.90@o2ib10 c:[8, 8], s:[3, 4]
00000400:00000200:2.0:1547253050.393951:0:235343:0:(lib-move.c:1822:lnet_select_peer_ni()) sd_best_lpni = 10.1.11.90@o2ib10
00000400:00000200:2.0:1547253050.393955:0:235343:0:(lib-move.c:1728:lnet_handle_send()) TRACE: 10.0.10.184@o2ib10(10.0.10.184@o2ib10:<?>) -> 10.1.11.90@o2ib10(10.0.11.90@o2ib10:10.1.11.90@o2ib10) : PUT try# 0
00000800:00000200:2.0:1547253050.393957:0:235343:0:(o2iblnd_cb.c:1636:kiblnd_send()) sending 296 bytes in 1 frags to 12345-10.1.11.90@o2ib10
00000800:00000200:2.0:1547253050.393959:0:235343:0:(o2iblnd.c:403:kiblnd_find_peer_locked()) got peer_ni [ffff91f07b11ad00] -> 10.1.11.90@o2ib10 (2) version: 12
00000800:00000200:2.0:1547253050.393960:0:235343:0:(o2iblnd_cb.c:1516:kiblnd_launch_tx()) conn[ffff91f06e3c8000] (20)++
00000800:00000200:2.0:1547253050.393960:0:235343:0:(o2iblnd_cb.c:1290:kiblnd_queue_tx_locked()) conn[ffff91f06e3c8000] (21)++
00000800:00000200:2.0:1547253050.393962:0:235343:0:(o2iblnd_cb.c:1522:kiblnd_launch_tx()) conn[ffff91f06e3c8000] (22)--
00000800:00000200:13.2:1547253050.393969:0:0:0:(o2iblnd_cb.c:3710:kiblnd_cq_completion()) conn[ffff91f06e3c8000] (21)++
00000800:00000200:13.0:1547253050.393979:0:235236:0:(o2iblnd_cb.c:3832:kiblnd_scheduler()) conn[ffff91f06e3c8000] (22)++
00000800:00000200:13.0:1547253050.393982:0:235236:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91f06e3c8000] (23)--
00000100:00000200:13.0:1547253050.393986:0:235236:0:(events.c:57:request_out_callback()) @@@ type 5, status 0  req@ffff91da594c8000 x1622412304056416/t0(0) o501->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 296/272 e 0 to 0 dl 1547253057 ref 3 fl Rpc:/0/ffffffff rc 0/-1
00000400:00000200:13.0:1547253050.393991:0:235236:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff91f07f0d8220
00000400:00000200:13.0:1547253050.393992:0:235236:0:(lib-msg.c:816:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1547253050.393995:0:235236:0:(lib-msg.c:630:lnet_health_check()) health check: 10.0.10.184@o2ib10->10.1.11.90@o2ib10: PUT: OK
00000800:00000200:13.0:1547253050.393997:0:235236:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f06e3c8000] (22)--
00000800:00000200:13.0:1547253050.393999:0:235236:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f06e3c8000] (21)--
00000100:00000400:6.0:1547253057.253528:0:235147:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1547253045/real 1547253045]  req@ffff91f072b88000 x1622412304056368/t0(0) o101->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 328/344 e 0 to 1 dl 1547253057 ref 2 fl Rpc:X/0/ffffffff rc 0/-1
00000100:00000200:6.0:1547253057.253559:0:235147:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0  req@ffff91f072b88000 x1622412304056368/t0(0) o101->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 328/344 e 0 to 1 dl 1547253057 ref 2 fl Rpc:X/0/ffffffff rc 0/-1
00000100:00000200:6.0:1547253057.253573:0:235147:0:(events.c:114:reply_in_callback()) @@@ unlink  req@ffff91f072b88000 x1622412304056368/t0(0) o101->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 328/344 e 0 to 1 dl 1547253057 ref 2 fl Rpc:X/0/ffffffff rc 0/-1
00000400:00000200:6.0:1547253057.253584:0:235147:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff91e2275ceff0
00000100:02020000:6.0:1547253057.253590:0:235147:0:(import.c:184:ptlrpc_set_import_discon()) 166-1: MGC10.0.11.90@o2ib10: Connection to MGS (at 10.0.11.90@o2ib10) was lost; in progress operations using this service will fail
00000100:00080000:6.0:1547253057.255661:0:235147:0:(import.c:186:ptlrpc_set_import_discon()) ffff920894951800 MGS: changing import state from FULL to DISCONN
10000000:01000000:6.0:1547253057.255663:0:235147:0:(mgc_request.c:1363:mgc_import_event()) import event 0x808001
00000100:00080000:6.0:1547253057.255664:0:235147:0:(pinger.c:210:ptlrpc_pinger_ir_down()) IR down
00000100:00080000:6.0:1547253057.255665:0:235147:0:(import.c:440:ptlrpc_fail_import()) import MGS@MGC10.0.11.90@o2ib10_0 for MGC10.0.11.90@o2ib10 not replayable, auto-deactivating
00000100:00080000:6.0:1547253057.255666:0:235147:0:(import.c:213:ptlrpc_deactivate_and_unlock_import()) setting import MGS INVALID
10000000:01000000:6.0:1547253057.255668:0:235147:0:(mgc_request.c:1363:mgc_import_event()) import event 0x808002
00000100:00080000:6.0:1547253057.255668:0:235147:0:(import.c:417:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN
00000100:00080000:6.0:1547253057.255676:0:410:0:(pinger.c:248:ptlrpc_pinger_process_import()) 8534aa65-7ac8-198c-0718-3e6f83d57705->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:6.0:1547253057.255678:0:410:0:(recover.c:58:ptlrpc_initiate_recovery()) MGS: starting recovery
00000100:00080000:6.0:1547253057.255678:0:410:0:(import.c:677:ptlrpc_connect_import()) ffff920894951800 MGS: changing import state from DISCONN to CONNECTING
00000100:00080000:6.0:1547253057.255680:0:410:0:(import.c:523:import_select_connection()) MGC10.0.11.90@o2ib10: connect to NID 10.0.11.90@o2ib10 last attempt 392518
00000100:00080000:6.0:1547253057.255682:0:410:0:(import.c:599:import_select_connection()) MGC10.0.11.90@o2ib10: import ffff920894951800 using connection MGC10.0.11.90@o2ib10_0/10.0.11.90@o2ib10
10000000:01000000:6.0:1547253057.255724:0:235147:0:(mgc_request.c:1021:mgc_blocking_ast()) Lock res [0x316568636163:0x0:0x0].0x0 (cache1)
10000000:01000000:6.0:1547253057.255738:0:235147:0:(mgc_request.c:2072:mgc_process_log()) Can't get cfg lock: -5
10000000:01000000:6.0:1547253057.255742:0:235147:0:(mgc_request.c:2139:mgc_process_log()) MGC10.0.11.90@o2ib10: configuration from log 'cache1-client' failed (-5).
00000020:02020000:6.0:1547253057.255744:0:235147:0:(obd_mount.c:115:lustre_process_log()) 15c-8: MGC10.0.11.90@o2ib10: The configuration from log 'cache1-client' failed (-5). This may be the result of communication errors between this node and the MGS, a bad configuration, or other errors. See the syslog for more information.
00000100:00000200:3.0:1547253057.255747:0:235343:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0  req@ffff91da594c8000 x1622412304056416/t0(0) o501->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 296/272 e 0 to 0 dl 1547253057 ref 2 fl Interpret:E/0/ffffffff rc -5/-1
00000100:00000200:3.0:1547253057.255752:0:235343:0:(events.c:114:reply_in_callback()) @@@ unlink  req@ffff91da594c8000 x1622412304056416/t0(0) o501->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 296/272 e 0 to 0 dl 1547253057 ref 2 fl Interpret:E/0/ffffffff rc -5/-1
00000400:00000200:3.0:1547253057.255756:0:235343:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff91f07f0d9e48
00000100:00000200:1.0:1547253057.255772:0:235247:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1622412304056432, portal 25
10000000:01000000:3.0:1547253057.255773:0:235343:0:(mgc_request.c:2139:mgc_process_log()) MGC10.0.11.90@o2ib10: configuration from log 'cache1-sptlrpc' failed (-5).
00000100:00000200:1.0:1547253057.255775:0:235247:0:(niobuf.c:85:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1622412304056432, offset 0
10000000:00020000:3.0:1547253057.255776:0:235343:0:(mgc_request.c:599:do_requeue()) failed processing log: -5
00000400:00000200:1.0:1547253057.255778:0:235247:0:(lib-move.c:4562:LNetPut()) LNetPut -> 12345-10.0.11.90@o2ib10
00000400:00000200:1.0:1547253057.255781:0:235247:0:(lib-move.c:2450:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.11.90@o2ib10 local destination
00000400:00000200:1.0:1547253057.255783:0:235247:0:(lib-move.c:1510:lnet_get_best_ni()) compare ni 10.0.10.184@o2ib10 [c:256, d:10, s:8] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1547253057.255785:0:235247:0:(lib-move.c:1553:lnet_get_best_ni()) selected best_ni 10.0.10.184@o2ib10
00000400:00000200:1.0:1547253057.255786:0:235247:0:(lib-move.c:1766:lnet_select_peer_ni()) 10.0.10.184@o2ib10 ni_is_pref = 0
00000400:00000200:1.0:1547253057.255787:0:235247:0:(lib-move.c:1766:lnet_select_peer_ni()) 10.0.10.184@o2ib10 ni_is_pref = 0
00000400:00000200:1.0:1547253057.255788:0:235247:0:(lib-move.c:1772:lnet_select_peer_ni()) 10.1.11.90@o2ib10 c:[8, 8], s:[4, 4]
00000400:00000200:1.0:1547253057.255790:0:235247:0:(lib-move.c:1822:lnet_select_peer_ni()) sd_best_lpni = 10.0.11.90@o2ib10
00000400:00000200:1.0:1547253057.255793:0:235247:0:(lib-move.c:1728:lnet_handle_send()) TRACE: 10.0.10.184@o2ib10(10.0.10.184@o2ib10:<?>) -> 10.0.11.90@o2ib10(10.0.11.90@o2ib10:10.0.11.90@o2ib10) : PUT try# 0
00000800:00000200:1.0:1547253057.255796:0:235247:0:(o2iblnd_cb.c:1636:kiblnd_send()) sending 520 bytes in 1 frags to 12345-10.0.11.90@o2ib10
00000800:00000200:1.0:1547253057.255798:0:235247:0:(o2iblnd.c:403:kiblnd_find_peer_locked()) got peer_ni [ffff91f07b11ad80] -> 10.0.11.90@o2ib10 (2) version: 12
00000800:00000200:1.0:1547253057.255800:0:235247:0:(o2iblnd_cb.c:1516:kiblnd_launch_tx()) conn[ffff91f09beb8000] (20)++
00000800:00000200:1.0:1547253057.255800:0:235247:0:(o2iblnd_cb.c:1290:kiblnd_queue_tx_locked()) conn[ffff91f09beb8000] (21)++
00000800:00000200:1.0:1547253057.255803:0:235247:0:(o2iblnd_cb.c:1522:kiblnd_launch_tx()) conn[ffff91f09beb8000] (22)--
00000800:00000200:10.2:1547253057.255856:0:0:0:(o2iblnd_cb.c:3710:kiblnd_cq_completion()) conn[ffff91f09beb8000] (21)++
00000800:00000200:10.0:1547253057.255881:0:235239:0:(o2iblnd_cb.c:3832:kiblnd_scheduler()) conn[ffff91f09beb8000] (22)++
00000800:00000200:10.0:1547253057.255884:0:235239:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91f09beb8000] (23)--
00000100:00000200:10.0:1547253057.255888:0:235239:0:(events.c:57:request_out_callback()) @@@ type 5, status 0  req@ffff91f083358000 x1622412304056432/t0(0) o250->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 520/544 e 0 to 0 dl 1547253063 ref 2 fl Rpc:N/0/ffffffff rc 0/-1
00000400:00000200:10.0:1547253057.255893:0:235239:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff91f094675d38
00000400:00000200:10.0:1547253057.255894:0:235239:0:(lib-msg.c:816:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1547253057.255897:0:235239:0:(lib-msg.c:630:lnet_health_check()) health check: 10.0.10.184@o2ib10->10.0.11.90@o2ib10: PUT: OK
00000800:00000200:10.0:1547253057.255899:0:235239:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f09beb8000] (22)--
00000800:00000200:10.0:1547253057.255900:0:235239:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f09beb8000] (21)--
00000800:00000200:10.2:1547253057.256066:0:0:0:(o2iblnd_cb.c:3710:kiblnd_cq_completion()) conn[ffff91f09beb8000] (20)++
00000800:00000200:10.0:1547253057.256070:0:235236:0:(o2iblnd_cb.c:3832:kiblnd_scheduler()) conn[ffff91f09beb8000] (21)++
00000800:00000200:10.0:1547253057.256073:0:235236:0:(o2iblnd_cb.c:338:kiblnd_handle_rx()) Received d1[1] from 10.0.11.90@o2ib10
00000400:00000200:10.0:1547253057.256076:0:235236:0:(lib-move.c:4114:lnet_parse()) TRACE: 10.0.10.184@o2ib10(10.0.10.184@o2ib10) <- 10.0.11.90@o2ib10 : PUT - for me
00000400:00000200:10.0:1547253057.256079:0:235236:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.11.90@o2ib10 of length 384 into portal 25 MB=0x5c3934d100070
00000400:00000200:10.0:1547253057.256083:0:235236:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 19 from 12345-10.0.11.90@o2ib10 of length 384/384 into md 0x81 [1] + 0
00000100:00000200:10.0:1547253057.256086:0:235236:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0  req@ffff91f083358000 x1622412304056432/t0(0) o250->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 520/544 e 0 to 0 dl 1547253063 ref 1 fl Rpc:N/0/ffffffff rc 0/-1
00000400:00000200:10.0:1547253057.256091:0:235236:0:(lib-msg.c:816:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1547253057.256092:0:235236:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91f09beb8000] (22)++
00000800:00000200:10.0:1547253057.256094:0:235236:0:(o2iblnd_cb.c:234:kiblnd_post_rx()) conn[ffff91f09beb8000] (23)--
00000800:00000200:10.0:1547253057.256094:0:235236:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f09beb8000] (22)--
00000100:00000200:1.0:1547253057.256096:0:235247:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0  req@ffff91f083358000 x1622412304056432/t0(0) o250->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 520/544 e 0 to 0 dl 1547253063 ref 1 fl Rpc:RN/0/ffffffff rc 0/-1
00000800:00000200:10.0:1547253057.256097:0:235236:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f09beb8000] (21)--
00000100:00000200:1.0:1547253057.256101:0:235247:0:(events.c:114:reply_in_callback()) @@@ unlink  req@ffff91f083358000 x1622412304056432/t0(0) o250->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 520/544 e 0 to 0 dl 1547253063 ref 1 fl Rpc:RN/0/ffffffff rc 0/-1
00000400:00000200:1.0:1547253057.256104:0:235247:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff91f094675188
00000100:00080000:1.0:1547253057.256108:0:235247:0:(import.c:1027:ptlrpc_connect_interpret()) MGC10.0.11.90@o2ib10: connect to target with instance 0
10000000:01000000:1.0:1547253057.256111:0:235247:0:(mgc_request.c:1363:mgc_import_event()) import event 0x808005
00000100:00080000:1.0:1547253057.256112:0:235247:0:(import.c:1198:ptlrpc_connect_interpret()) reconnected to MGS@MGC10.0.11.90@o2ib10_0 after partition
00000100:00080000:1.0:1547253057.256113:0:235247:0:(import.c:1203:ptlrpc_connect_interpret()) MGC10.0.11.90@o2ib10: reconnected but import is invalid; marking evicted
00000100:00080000:1.0:1547253057.256114:0:235247:0:(import.c:1204:ptlrpc_connect_interpret()) ffff920894951800 MGS: changing import state from CONNECTING to EVICTED
00000100:00080000:1.0:1547253057.256115:0:235247:0:(import.c:1467:ptlrpc_import_recovery_state_machine()) evicted from MGS@MGC10.0.11.90@o2ib10_0; invalidating
00000100:00080000:1.0:1547253057.256266:0:235353:0:(import.c:1408:ptlrpc_invalidate_import_thread()) thread invalidate import MGC10.0.11.90@o2ib10 to MGS@MGC10.0.11.90@o2ib10_0
10000000:01000000:1.0:1547253057.256267:0:235353:0:(mgc_request.c:1363:mgc_import_event()) import event 0x808003
10000000:01000000:1.0:1547253057.256278:0:235353:0:(mgc_request.c:1021:mgc_blocking_ast()) Lock res [0x316568636163:0x0:0x0].0x0 (cache1)
00000100:00080000:1.0:1547253057.256286:0:235353:0:(import.c:1417:ptlrpc_invalidate_import_thread()) ffff920894951800 MGS: changing import state from EVICTED to RECOVER
00000100:00080000:1.0:1547253057.256287:0:235353:0:(import.c:1528:ptlrpc_import_recovery_state_machine()) ffff920894951800 MGS: changing import state from RECOVER to FULL
10000000:01000000:1.0:1547253057.256288:0:235353:0:(mgc_request.c:1363:mgc_import_event()) import event 0x808004
00000100:00080000:1.0:1547253057.256290:0:235353:0:(pinger.c:203:ptlrpc_pinger_ir_up()) IR up
00000100:02000000:1.0:1547253057.256291:0:235353:0:(import.c:1534:ptlrpc_import_recovery_state_machine()) MGC10.0.11.90@o2ib10: Connection restored to MGC10.0.11.90@o2ib10_0 (at 10.0.11.90@o2ib10)
10000000:01000000:6.0:1547253057.259915:0:235147:0:(mgc_request.c:151:config_log_put()) dropping config log cache1-cliir
10000000:01000000:6.0:1547253057.259917:0:235147:0:(mgc_request.c:151:config_log_put()) dropping config log params
10000000:01000000:6.0:1547253057.259918:0:235147:0:(mgc_request.c:151:config_log_put()) dropping config log cache1-sptlrpc
10000000:01000000:6.0:1547253057.259920:0:235147:0:(mgc_request.c:151:config_log_put()) dropping config log cache1-client
10000000:01000000:6.0:1547253057.259920:0:235147:0:(mgc_request.c:535:config_log_end()) end config log cache1-client (0)
00000080:02000400:6.0:1547253057.260007:0:235147:0:(llite_lib.c:1205:ll_put_super()) Unmounted cache1-client
00000020:01000000:6.0:1547253057.260010:0:235147:0:(obd_config.c:889:class_del_profile()) Del profile cache1-client
00000020:01000004:6.0:1547253057.260379:0:235147:0:(obd_mount.c:906:lustre_common_put_super()) dropping sb ffff9208d990e000
00000100:00080000:6.0:1547253057.260381:0:235147:0:(pinger.c:413:ptlrpc_pinger_del_import()) removing pingable import 8534aa65-7ac8-198c-0718-3e6f83d57705->MGS
00000100:00080000:6.0:1547253057.260389:0:235147:0:(import.c:1586:ptlrpc_disconnect_prep_req()) ffff920894951800 MGS: changing import state from FULL to CONNECTING
00000100:00000200:6.0:1547253057.260406:0:235147:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1622412304056448, portal 25
00000100:00000200:6.0:1547253057.260408:0:235147:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1622412304056448, offset 0
00000400:00000200:6.0:1547253057.260411:0:235147:0:(lib-move.c:4562:LNetPut()) LNetPut -> 12345-10.0.11.90@o2ib10
00000400:00000200:6.0:1547253057.260414:0:235147:0:(lib-move.c:2450:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.11.90@o2ib10 local destination
00000400:00000200:6.0:1547253057.260417:0:235147:0:(lib-move.c:1510:lnet_get_best_ni()) compare ni 10.0.10.184@o2ib10 [c:256, d:21, s:9] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1547253057.260419:0:235147:0:(lib-move.c:1553:lnet_get_best_ni()) selected best_ni 10.0.10.184@o2ib10
00000400:00000200:6.0:1547253057.260421:0:235147:0:(lib-move.c:1766:lnet_select_peer_ni()) 10.0.10.184@o2ib10 ni_is_pref = 0
00000400:00000200:6.0:1547253057.260422:0:235147:0:(lib-move.c:1766:lnet_select_peer_ni()) 10.0.10.184@o2ib10 ni_is_pref = 0
00000400:00000200:6.0:1547253057.260424:0:235147:0:(lib-move.c:1772:lnet_select_peer_ni()) 10.1.11.90@o2ib10 c:[8, 8], s:[4, 5]
00000400:00000200:6.0:1547253057.260425:0:235147:0:(lib-move.c:1822:lnet_select_peer_ni()) sd_best_lpni = 10.1.11.90@o2ib10
00000400:00000200:6.0:1547253057.260430:0:235147:0:(lib-move.c:1728:lnet_handle_send()) TRACE: 10.0.10.184@o2ib10(10.0.10.184@o2ib10:<?>) -> 10.1.11.90@o2ib10(10.0.11.90@o2ib10:10.1.11.90@o2ib10) : PUT try# 0
00000800:00000200:6.0:1547253057.260434:0:235147:0:(o2iblnd_cb.c:1636:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.1.11.90@o2ib10
00000800:00000200:6.0:1547253057.260437:0:235147:0:(o2iblnd.c:403:kiblnd_find_peer_locked()) got peer_ni [ffff91f07b11ad00] -> 10.1.11.90@o2ib10 (2) version: 12
00000800:00000200:6.0:1547253057.260439:0:235147:0:(o2iblnd_cb.c:1516:kiblnd_launch_tx()) conn[ffff91f06e3c8000] (20)++
00000800:00000200:6.0:1547253057.260440:0:235147:0:(o2iblnd_cb.c:1290:kiblnd_queue_tx_locked()) conn[ffff91f06e3c8000] (21)++
00000800:00000200:6.0:1547253057.260443:0:235147:0:(o2iblnd_cb.c:1522:kiblnd_launch_tx()) conn[ffff91f06e3c8000] (22)--
00000800:00000200:13.2:1547253057.260451:0:0:0:(o2iblnd_cb.c:3710:kiblnd_cq_completion()) conn[ffff91f06e3c8000] (21)++
00000800:00000200:13.0:1547253057.260462:0:235239:0:(o2iblnd_cb.c:3832:kiblnd_scheduler()) conn[ffff91f06e3c8000] (22)++
00000800:00000200:13.0:1547253057.260466:0:235239:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91f06e3c8000] (23)--
00000100:00000200:13.0:1547253057.260471:0:235239:0:(events.c:57:request_out_callback()) @@@ type 5, status 0  req@ffff91f083358300 x1622412304056448/t0(0) o251->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 224/224 e 0 to 0 dl 1547253063 ref 3 fl Rpc:N/0/ffffffff rc 0/-1
00000800:00000200:11.0:1547253057.260472:0:235238:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f06e3c8000] (22)--
00000400:00000200:13.0:1547253057.260482:0:235239:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff91e2275cf760
00000400:00000200:13.0:1547253057.260483:0:235239:0:(lib-msg.c:816:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1547253057.260486:0:235239:0:(lib-msg.c:630:lnet_health_check()) health check: 10.0.10.184@o2ib10->10.1.11.90@o2ib10: PUT: OK
00000800:00000200:13.0:1547253057.260488:0:235239:0:(o2iblnd_cb.c:3848:kiblnd_scheduler()) conn[ffff91f06e3c8000] (21)--
00000100:00000400:7.0F:1547253063.260517:0:235147:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1547253057/real 1547253057]  req@ffff91f083358300 x1622412304056448/t0(0) o251->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 224/224 e 0 to 1 dl 1547253063 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1
00000100:00000200:7.0:1547253063.260530:0:235147:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0  req@ffff91f083358300 x1622412304056448/t0(0) o251->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 224/224 e 0 to 1 dl 1547253063 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1
00000100:00000200:7.0:1547253063.260534:0:235147:0:(events.c:114:reply_in_callback()) @@@ unlink  req@ffff91f083358300 x1622412304056448/t0(0) o251->MGC10.0.11.90@o2ib10@10.0.11.90@o2ib10:26/25 lens 224/224 e 0 to 1 dl 1547253063 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1
00000400:00000200:7.0:1547253063.260538:0:235147:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff91e2275ceff0
00000100:00080000:7.0:1547253063.260544:0:235147:0:(import.c:1652:ptlrpc_disconnect_import()) ffff920894951800 MGS: changing import state from CONNECTING to CLOSED
00000100:00080000:7.0:1547253063.260545:0:235147:0:(import.c:213:ptlrpc_deactivate_and_unlock_import()) setting import MGS INVALID
10000000:01000000:7.0:1547253063.260546:0:235147:0:(mgc_request.c:1363:mgc_import_event()) import event 0x808002
10000000:01000000:7.0:1547253063.260547:0:235147:0:(mgc_request.c:1363:mgc_import_event()) import event 0x808003
00000020:00000080:7.0:1547253063.260549:0:235147:0:(genops.c:1503:class_disconnect()) disconnect: cookie 0xf0b66cac3fff399
00000020:00000080:7.0:1547253063.260551:0:235147:0:(genops.c:1024:class_export_put()) final put ffff91fa153dc800/8534aa65-7ac8-198c-0718-3e6f83d57705
00000020:00000080:7.0:1547253063.260556:0:419:0:(genops.c:974:class_export_destroy()) destroying export ffff91fa153dc800/8534aa65-7ac8-198c-0718-3e6f83d57705 for MGC10.0.11.90@o2ib10
00000020:01000000:7.0:1547253063.260560:0:235147:0:(obd_config.c:2128:class_manual_cleanup()) Manual cleanup of MGC10.0.11.90@o2ib10 (flags='')
00000020:00000080:7.0:1547253063.260561:0:235147:0:(obd_config.c:1128:class_process_config()) processing cmd: cf004
00000020:00000080:7.0:1547253063.260562:0:235147:0:(obd_config.c:659:class_cleanup()) MGC10.0.11.90@o2ib10: forcing exports to disconnect: 0/1
00000020:00080000:7.0:1547253063.260563:0:235147:0:(genops.c:1592:class_disconnect_exports()) OBD device 0 (ffff91e3c04f0000) has no exports
10000000:01000000:3.0:1547253063.260614:0:235343:0:(mgc_request.c:702:mgc_requeue_thread()) Ending requeue thread
10000000:01000000:7.0:1547253063.260616:0:235147:0:(obd_class.h:628:obd_cleanup_client_import()) MGC10.0.11.90@o2ib10: client import never connected
00000100:00080000:7.0:1547253063.260617:0:235147:0:(import.c:213:ptlrpc_deactivate_and_unlock_import()) setting import MGS INVALID
00000020:00000080:7.0:1547253063.260629:0:419:0:(genops.c:1224:obd_zombie_import_free()) destroying import ffff920894951800 for MGC10.0.11.90@o2ib10
00000020:00000080:7.0:1547253063.260640:0:235147:0:(obd_config.c:1128:class_process_config()) processing cmd: cf002
00000020:00000080:7.0:1547253063.260641:0:235147:0:(obd_config.c:589:class_detach()) detach on obd MGC10.0.11.90@o2ib10 (uuid 8534aa65-7ac8-198c-0718-3e6f83d57705)
00000020:00000080:7.0:1547253063.260642:0:235147:0:(genops.c:1024:class_export_put()) final put ffff9208d9c9c800/8534aa65-7ac8-198c-0718-3e6f83d57705
00000020:00000080:7.0:1547253063.260642:0:235147:0:(genops.c:974:class_export_destroy()) destroying export ffff9208d9c9c800/8534aa65-7ac8-198c-0718-3e6f83d57705 for MGC10.0.11.90@o2ib10
00000020:01000000:7.0:1547253063.260643:0:235147:0:(genops.c:481:class_free_dev()) finishing cleanup of obd MGC10.0.11.90@o2ib10 (8534aa65-7ac8-198c-0718-3e6f83d57705)
00000100:00000200:7.0:1547253063.260983:0:235147:0:(service.c:3280:ptlrpc_unregister_service()) ldlm_cbd: tearing down
00000400:00000200:7.0:1547253063.261198:0:235147:0:(lib-ptl.c:952:lnet_clear_lazy_portal()) clearing portal 15 lazy
00000100:00000200:7.0:1547253063.261200:0:235147:0:(events.c:305:request_in_callback()) event type 6, status 0, service ldlm_cbd
00000400:00000200:7.0:1547253063.261201:0:235147:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff91f094675e48
00000100:00000200:7.0:1547253063.261204:0:235147:0:(events.c:305:request_in_callback()) event type 6, status 0, service ldlm_cbd
00000400:00000200:7.0:1547253063.261206:0:235147:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff91f82e7d7078
00000020:00000080:7.0:1547253063.261476:0:235147:0:(obd_config.c:1128:class_process_config()) processing cmd: cf006
00000020:00000080:7.0:1547253063.261477:0:235147:0:(obd_config.c:1147:class_process_config()) removing mappings for uuid MGC10.0.11.90@o2ib10_0
00000020:01000004:7.0:1547253063.261478:0:235147:0:(obd_mount.c:661:lustre_put_lsi()) put ffff9208d990e000 1
00000020:01000004:7.0:1547253063.261479:0:235147:0:(obd_mount.c:605:lustre_free_lsi()) Freeing lsi ffff9208d9c9a000
00000020:00020000:7.0:1547253063.261490:0:235147:0:(obd_mount.c:1608:lustre_fill_super()) Unable to mount  (-5)
Debug log: 670 lines, 670 kept, 0 dropped, 0 bad.