00000400:00000001:25.0:1506127895.169486:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127895.169490:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000020:00000010:25.0:1506127895.169494:0:3044:0:(lprocfs_status.c:1248:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4416 at ffff880819216000 (tot 194134492). 00000100:00000001:25.0:1506127895.169500:0:3044:0:(service.c:1859:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:25.0:1506127895.169502:0:3044:0:(sec.c:2052:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:25.0:1506127895.169503:0:3044:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:25.0:1506127895.169505:0:3044:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:25.0:1506127895.169512:0:3044:0:(sec.c:2099:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:25.0:1506127895.169514:0:3044:0:(service.c:1939:ptlrpc_server_handle_req_in()) got req x1579283258778320 00000020:00000001:25.0:1506127895.169516:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:25.0:1506127895.169517:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x0 00000020:00000001:25.0:1506127895.169519:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000001:25.0:1506127895.169520:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127895.169522:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:25.0:1506127895.169525:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88080f68f400. 00000020:00000010:25.0:1506127895.169540:0:3044:0:(lu_ucred.c:49:lu_ucred_key_init()) kmalloced 'value': 104 at ffff88080f760a80. 00000020:00000010:25.0:1506127895.169543:0:3044:0:(tgt_main.c:382:tgt_ses_key_init()) kmalloced 'value': 160 at ffff880811c69500. 00000100:00000001:25.0:1506127895.169546:0:3044:0:(service.c:1656:ptlrpc_server_request_add()) Process entered 00000100:00000001:25.0:1506127895.169547:0:3044:0:(service.c:1580:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:25.0:1506127895.169548:0:3044:0:(service.c:1607:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127895.169551:0:3044:0:(service.c:1695:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127895.169558:0:3044:0:(service.c:2003:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:25.0:1506127895.169561:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127895.169563:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127895.169564:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127895.169565:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127895.169567:0:3044:0:(service.c:2028:ptlrpc_server_handle_request()) Process entered 00000100:00000001:25.0:1506127895.169568:0:3044:0:(service.c:1809:ptlrpc_server_request_get()) Process entered 00000100:00100000:25.0:1506127895.169572:0:3044:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.1.108@o2ib, seq: 0 00000100:00000001:25.0:1506127895.169574:0:3044:0:(service.c:1842:ptlrpc_server_request_get()) Process leaving (rc=18446612167001139280 : -131906708412336 : ffff880813806850) 00000100:00000040:25.0:1506127895.169578:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "New" -> "Interpret" req@ffff880813806850 x1579283258778320/t0(0) o38->@:0/0 lens 520/0 e 0 to 0 dl 1506127995 ref 1 fl New:/0/ffffffff rc 0/-1 00000100:00100000:25.0:1506127895.169588:0:3044:0:(service.c:2089:ptlrpc_server_handle_request()) Handling RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:0+-99:2041:x1579283258778320:12345-192.168.1.108@o2ib:38 00000100:00000200:25.0:1506127895.169591:0:3044:0:(service.c:2094:ptlrpc_server_handle_request()) got req 1579283258778320 00000020:00000001:25.0:1506127895.169593:0:3044:0:(tgt_handler.c:672:tgt_request_handle()) Process entered 00000004:00000010:25.0:1506127895.169595:0:3044:0:(lod_dev.c:1826:lod_key_init()) kmalloced 'value': 928 at ffff88080f506c00. 00000004:00000010:25.0:1506127895.169598:0:3044:0:(mdd_device.c:1719:mdd_key_init()) kmalloced 'value': 1480 at ffff88080f7a7800. 00000004:00000010:25.0:1506127895.169600:0:3044:0:(osp_dev.c:1852:osp_key_init()) kmalloced 'value': 600 at ffff88080f507000. 00010000:00000001:25.0:1506127895.169603:0:3044:0:(ldlm_lib.c:966:target_handle_connect()) Process entered 00010000:02020000:25.0:1506127895.169748:0:3044:0:(ldlm_lib.c:987:target_handle_connect()) 137-5: soaked-MDT0002_UUID: not available for connect from 192.168.1.108@o2ib (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00000001:25.0:1506127895.169750:0:3044:0:(ldlm_lib.c:988:target_handle_connect()) Process leaving via out (rc=18446744073709551597 : -19 : 0xffffffffffffffed) 00010000:00000001:25.0:1506127895.169753:0:3044:0:(ldlm_lib.c:1452:target_handle_connect()) Process leaving (rc=18446744073709551597 : -19 : ffffffffffffffed) 00000100:00000001:25.0:1506127895.169755:0:3044:0:(niobuf.c:660:ptlrpc_send_error()) Process entered 00000100:00000001:25.0:1506127895.169756:0:3044:0:(pack_generic.c:334:lustre_pack_reply_v2()) Process entered 02000000:00000001:25.0:1506127895.169758:0:3044:0:(sec.c:2112:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:25.0:1506127895.169760:0:3044:0:(sec_null.c:318:null_alloc_rs()) kmalloced 'rs': 480 at ffff88080f68f000. 02000000:00000001:25.0:1506127895.169761:0:3044:0:(sec.c:2148:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127895.169763:0:3044:0:(pack_generic.c:368:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127895.169766:0:3044:0:(ldlm_lib.c:2895:target_pack_pool_reply()) Process entered 00010000:00000001:25.0:1506127895.169768:0:3044:0:(ldlm_lib.c:2903:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:25.0:1506127895.169770:0:3044:0:(niobuf.c:550:ptlrpc_at_set_reply()) No early reply support: flags=0x0 req_flags=0x0 magic=bd00bd3/bd00bd3 len=192 00000100:00000001:25.0:1506127895.169772:0:3044:0:(connection.c:48:ptlrpc_connection_get()) Process entered 00000400:00000200:25.0:1506127895.169777:0:3044:0:(peer.c:624:LNetPrimaryNID()) NID 192.168.1.108@o2ib primary NID 192.168.1.108@o2ib rc 0 00000100:00000001:25.0:1506127895.169781:0:3044:0:(connection.c:53:ptlrpc_connection_get()) Process leaving via out (rc=18446612167460263808 : -131906249287808 : 0xffff88082ede1780) 00000100:00000040:25.0:1506127895.169784:0:3044:0:(connection.c:84:ptlrpc_connection_get()) conn=ffff88082ede1780 refcount 12 to 192.168.1.108@o2ib 02000000:00000001:25.0:1506127895.169787:0:3044:0:(sec.c:2161:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:25.0:1506127895.169788:0:3044:0:(sec.c:2172:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127895.169790:0:3044:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:25.0:1506127895.169792:0:3044:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-192.168.1.108@o2ib 00000400:00000010:25.0:1506127895.169794:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f760c00. 00000100:00000200:25.0:1506127895.169797:0:3044:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 4, xid 1579283258778320, offset 0 00000400:00000010:25.0:1506127895.169799:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88080f68e600 (tot 194134892). 00000400:00000200:25.0:1506127895.169803:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.108@o2ib 00000400:00000200:25.0:1506127895.169813:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.108@o2ib(192.168.1.108@o2ib:192.168.1.108@o2ib) : PUT 00000800:00000200:25.0:1506127895.169818:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 192 bytes in 1 frags to 12345-192.168.1.108@o2ib 00000800:00000200:25.0:1506127895.169824:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082cb07e00] -> 192.168.1.108@o2ib (2) version: 12 00000800:00000200:25.0:1506127895.169827:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082e03de00] (20)++ 00000800:00000200:25.0:1506127895.169829:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082e03de00] (21)++ 00000800:00000200:25.0:1506127895.169834:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082e03de00] (22)-- 00000100:00000001:25.0:1506127895.169836:0:3044:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127895.169837:0:3044:0:(connection.c:91:ptlrpc_connection_put()) Process entered 00000100:00000040:25.0:1506127895.169839:0:3044:0:(connection.c:119:ptlrpc_connection_put()) PUT conn=ffff88082ede1780 refcount 11 to 192.168.1.108@o2ib 00000100:00000001:25.0:1506127895.169841:0:3044:0:(connection.c:121:ptlrpc_connection_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127895.169843:0:3044:0:(niobuf.c:677:ptlrpc_send_error()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127895.169844:0:3044:0:(tgt_handler.c:690:tgt_request_handle()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:25.0:1506127895.169847:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff880813806850 x1579283258778320/t0(0) o38->@:0/0 lens 520/192 e 0 to 0 dl 1506127995 ref 1 fl Interpret:/0/0 rc -19/-19 00000100:00100000:25.0:1506127895.169856:0:3044:0:(service.c:2135:ptlrpc_server_handle_request()) Handled RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:0+-99:2041:x1579283258778320:12345-192.168.1.108@o2ib:38 Request procesed in 268us (402us total) trans 0 rc -19/-19 00000100:00100000:25.0:1506127895.169862:0:3044:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.1.108@o2ib, seq: 0 00000100:00000001:25.0:1506127895.169865:0:3044:0:(service.c:1613:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:25.0:1506127895.169866:0:3044:0:(service.c:1624:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:25.0:1506127895.169868:0:3044:0:(lu_ucred.c:49:lu_ucred_key_fini()) kfreed 'info': 104 at ffff88080f760a80. 00000020:00000010:25.0:1506127895.169870:0:3044:0:(tgt_main.c:382:tgt_ses_key_fini()) kfreed 'info': 160 at ffff880811c69500. 00000020:00000010:25.0:1506127895.169872:0:3044:0:(lu_object.c:1595:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff88080f68f400. 00000100:00000001:25.0:1506127895.169875:0:3044:0:(service.c:2155:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:25.0:1506127895.169878:0:3044:0:(niobuf.c:953:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000400:00000010:25.0:1506127895.169881:0:3044:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88080fccbea0. 00000400:00000010:25.0:1506127895.169883:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f760a80. 00000400:00000001:25.0:1506127895.169886:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127895.169887:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127896.390583:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127896.390588:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127896.390590:0:3044:0:(service.c:1859:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:25.0:1506127896.390591:0:3044:0:(sec.c:2052:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:25.0:1506127896.390593:0:3044:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:25.0:1506127896.390594:0:3044:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:25.0:1506127896.390597:0:3044:0:(sec.c:2099:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:25.0:1506127896.390613:0:3044:0:(service.c:1939:ptlrpc_server_handle_req_in()) got req x1579288666076464 00000020:00000001:25.0:1506127896.390615:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:25.0:1506127896.390616:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x0 00000020:00000001:25.0:1506127896.390618:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000001:25.0:1506127896.390619:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127896.390621:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:25.0:1506127896.390624:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88080f68f400. 00000020:00000010:25.0:1506127896.390627:0:3044:0:(lu_ucred.c:49:lu_ucred_key_init()) kmalloced 'value': 104 at ffff88080f760c80. 00000020:00000010:25.0:1506127896.390630:0:3044:0:(tgt_main.c:382:tgt_ses_key_init()) kmalloced 'value': 160 at ffff880811c69500. 00000100:00000001:25.0:1506127896.390632:0:3044:0:(service.c:1656:ptlrpc_server_request_add()) Process entered 00000100:00000001:25.0:1506127896.390633:0:3044:0:(service.c:1580:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:25.0:1506127896.390635:0:3044:0:(service.c:1607:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127896.390638:0:3044:0:(service.c:1695:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127896.390645:0:3044:0:(service.c:2003:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:25.0:1506127896.390648:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127896.390649:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127896.390650:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127896.390651:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127896.390653:0:3044:0:(service.c:2028:ptlrpc_server_handle_request()) Process entered 00000100:00000001:25.0:1506127896.390654:0:3044:0:(service.c:1809:ptlrpc_server_request_get()) Process entered 00000100:00100000:25.0:1506127896.390659:0:3044:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.1.109@o2ib, seq: 1 00000100:00000001:25.0:1506127896.390661:0:3044:0:(service.c:1842:ptlrpc_server_request_get()) Process leaving (rc=18446612167001138256 : -131906708413360 : ffff880813806450) 00000100:00000040:25.0:1506127896.390665:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "New" -> "Interpret" req@ffff880813806450 x1579288666076464/t0(0) o38->@:0/0 lens 520/0 e 0 to 0 dl 1506127996 ref 1 fl New:/0/ffffffff rc 0/-1 00000100:00100000:25.0:1506127896.390681:0:3044:0:(service.c:2089:ptlrpc_server_handle_request()) Handling RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:0+-99:2127:x1579288666076464:12345-192.168.1.109@o2ib:38 00000100:00000200:25.0:1506127896.390684:0:3044:0:(service.c:2094:ptlrpc_server_handle_request()) got req 1579288666076464 00000020:00000001:25.0:1506127896.390686:0:3044:0:(tgt_handler.c:672:tgt_request_handle()) Process entered 00010000:00000001:25.0:1506127896.390688:0:3044:0:(ldlm_lib.c:966:target_handle_connect()) Process entered 00010000:02020000:25.0:1506127896.390833:0:3044:0:(ldlm_lib.c:987:target_handle_connect()) 137-5: soaked-MDT0002_UUID: not available for connect from 192.168.1.109@o2ib (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00000001:25.0:1506127896.390835:0:3044:0:(ldlm_lib.c:988:target_handle_connect()) Process leaving via out (rc=18446744073709551597 : -19 : 0xffffffffffffffed) 00010000:00000001:25.0:1506127896.390838:0:3044:0:(ldlm_lib.c:1452:target_handle_connect()) Process leaving (rc=18446744073709551597 : -19 : ffffffffffffffed) 00000100:00000001:25.0:1506127896.390840:0:3044:0:(niobuf.c:660:ptlrpc_send_error()) Process entered 00000100:00000001:25.0:1506127896.390841:0:3044:0:(pack_generic.c:334:lustre_pack_reply_v2()) Process entered 02000000:00000001:25.0:1506127896.390842:0:3044:0:(sec.c:2112:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:25.0:1506127896.390845:0:3044:0:(sec_null.c:318:null_alloc_rs()) kmalloced 'rs': 480 at ffff88080f68e400. 02000000:00000001:25.0:1506127896.390846:0:3044:0:(sec.c:2148:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127896.390848:0:3044:0:(pack_generic.c:368:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127896.390851:0:3044:0:(ldlm_lib.c:2895:target_pack_pool_reply()) Process entered 00010000:00000001:25.0:1506127896.390852:0:3044:0:(ldlm_lib.c:2903:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:25.0:1506127896.390854:0:3044:0:(niobuf.c:550:ptlrpc_at_set_reply()) No early reply support: flags=0x0 req_flags=0x0 magic=bd00bd3/bd00bd3 len=192 00000100:00000001:25.0:1506127896.390857:0:3044:0:(connection.c:48:ptlrpc_connection_get()) Process entered 00000400:00000200:25.0:1506127896.390862:0:3044:0:(peer.c:624:LNetPrimaryNID()) NID 192.168.1.109@o2ib primary NID 192.168.1.109@o2ib rc 0 00000100:00000001:25.0:1506127896.390866:0:3044:0:(connection.c:53:ptlrpc_connection_get()) Process leaving via out (rc=18446612167460263520 : -131906249288096 : 0xffff88082ede1660) 00000100:00000040:25.0:1506127896.390869:0:3044:0:(connection.c:84:ptlrpc_connection_get()) conn=ffff88082ede1660 refcount 8 to 192.168.1.109@o2ib 02000000:00000001:25.0:1506127896.390871:0:3044:0:(sec.c:2161:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:25.0:1506127896.390872:0:3044:0:(sec.c:2172:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127896.390874:0:3044:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:25.0:1506127896.390876:0:3044:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-192.168.1.109@o2ib 00000400:00000010:25.0:1506127896.390878:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f760d00. 00000100:00000200:25.0:1506127896.390880:0:3044:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 4, xid 1579288666076464, offset 0 00000400:00000010:25.0:1506127896.390883:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88080f68de00 (tot 194134892). 00000400:00000200:25.0:1506127896.390886:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.109@o2ib 00000400:00000200:25.0:1506127896.390897:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.109@o2ib(192.168.1.109@o2ib:192.168.1.109@o2ib) : PUT 00000800:00000200:25.0:1506127896.390901:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 192 bytes in 1 frags to 12345-192.168.1.109@o2ib 00000800:00000200:25.0:1506127896.390907:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082edb4980] -> 192.168.1.109@o2ib (2) version: 12 00000800:00000200:25.0:1506127896.390910:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082dfb7200] (20)++ 00000800:00000200:25.0:1506127896.390912:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082dfb7200] (21)++ 00000800:00000200:25.0:1506127896.390917:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082dfb7200] (22)-- 00000100:00000001:25.0:1506127896.390919:0:3044:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127896.390920:0:3044:0:(connection.c:91:ptlrpc_connection_put()) Process entered 00000100:00000040:25.0:1506127896.390922:0:3044:0:(connection.c:119:ptlrpc_connection_put()) PUT conn=ffff88082ede1660 refcount 7 to 192.168.1.109@o2ib 00000100:00000001:25.0:1506127896.390924:0:3044:0:(connection.c:121:ptlrpc_connection_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127896.390925:0:3044:0:(niobuf.c:677:ptlrpc_send_error()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127896.390927:0:3044:0:(tgt_handler.c:690:tgt_request_handle()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:25.0:1506127896.390930:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff880813806450 x1579288666076464/t0(0) o38->@:0/0 lens 520/192 e 0 to 0 dl 1506127996 ref 1 fl Interpret:/0/0 rc -19/-19 00000100:00100000:25.0:1506127896.390939:0:3044:0:(service.c:2135:ptlrpc_server_handle_request()) Handled RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:0+-99:2127:x1579288666076464:12345-192.168.1.109@o2ib:38 Request procesed in 258us (413us total) trans 0 rc -19/-19 00000100:00100000:25.0:1506127896.390945:0:3044:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.1.109@o2ib, seq: 1 00000100:00000001:25.0:1506127896.390947:0:3044:0:(service.c:1613:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:25.0:1506127896.390948:0:3044:0:(service.c:1624:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:25.0:1506127896.390950:0:3044:0:(lu_ucred.c:49:lu_ucred_key_fini()) kfreed 'info': 104 at ffff88080f760c80. 00000020:00000010:25.0:1506127896.390952:0:3044:0:(tgt_main.c:382:tgt_ses_key_fini()) kfreed 'info': 160 at ffff880811c69500. 00000020:00000010:25.0:1506127896.390955:0:3044:0:(lu_object.c:1595:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff88080f68f400. 00000100:00000001:25.0:1506127896.390957:0:3044:0:(service.c:2155:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:25.0:1506127896.390960:0:3044:0:(niobuf.c:953:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000400:00000010:25.0:1506127896.390963:0:3044:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88080fccbf00. 00000400:00000010:25.0:1506127896.390965:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f760c80. 00000400:00000001:25.0:1506127896.390968:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127896.390969:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127920.169712:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127920.169715:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127920.169718:0:3044:0:(service.c:1859:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:25.0:1506127920.169720:0:3044:0:(sec.c:2052:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:25.0:1506127920.169721:0:3044:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:25.0:1506127920.169723:0:3044:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:25.0:1506127920.169726:0:3044:0:(sec.c:2099:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:25.0:1506127920.169729:0:3044:0:(service.c:1939:ptlrpc_server_handle_req_in()) got req x1579283258778784 00000020:00000001:25.0:1506127920.169731:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:25.0:1506127920.169732:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x0 00000020:00000001:25.0:1506127920.169734:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000001:25.0:1506127920.169736:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.169738:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:25.0:1506127920.169740:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88082e03be00. 00000020:00000010:25.0:1506127920.169744:0:3044:0:(lu_ucred.c:49:lu_ucred_key_init()) kmalloced 'value': 104 at ffff88080f761200. 00000020:00000010:25.0:1506127920.169746:0:3044:0:(tgt_main.c:382:tgt_ses_key_init()) kmalloced 'value': 160 at ffff88080f8a8e40. 00000100:00000001:25.0:1506127920.169749:0:3044:0:(service.c:1656:ptlrpc_server_request_add()) Process entered 00000100:00000001:25.0:1506127920.169750:0:3044:0:(service.c:1580:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:25.0:1506127920.169751:0:3044:0:(service.c:1607:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.169754:0:3044:0:(service.c:1695:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.169761:0:3044:0:(service.c:2003:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:25.0:1506127920.169764:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127920.169765:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127920.169767:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127920.169768:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127920.169769:0:3044:0:(service.c:2028:ptlrpc_server_handle_request()) Process entered 00000100:00000001:25.0:1506127920.169770:0:3044:0:(service.c:1809:ptlrpc_server_request_get()) Process entered 00000100:00100000:25.0:1506127920.169775:0:3044:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.1.108@o2ib, seq: 2 00000100:00000001:25.0:1506127920.169778:0:3044:0:(service.c:1842:ptlrpc_server_request_get()) Process leaving (rc=18446612167001137232 : -131906708414384 : ffff880813806050) 00000100:00000040:25.0:1506127920.169781:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "New" -> "Interpret" req@ffff880813806050 x1579283258778784/t0(0) o38->@:0/0 lens 520/0 e 0 to 0 dl 1506128020 ref 1 fl New:/0/ffffffff rc 0/-1 00000100:00100000:25.0:1506127920.169791:0:3044:0:(service.c:2089:ptlrpc_server_handle_request()) Handling RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:0+-99:2041:x1579283258778784:12345-192.168.1.108@o2ib:38 00000100:00000200:25.0:1506127920.169795:0:3044:0:(service.c:2094:ptlrpc_server_handle_request()) got req 1579283258778784 00000020:00000001:25.0:1506127920.169797:0:3044:0:(tgt_handler.c:672:tgt_request_handle()) Process entered 00010000:00000001:25.0:1506127920.169799:0:3044:0:(ldlm_lib.c:966:target_handle_connect()) Process entered 00000020:00000040:25.0:1506127920.169803:0:3044:0:(obd_config.c:753:class_incref()) incref soaked-MDT0003 (ffff880826bd4d58) now 98 00000100:00000001:25.0:1506127920.169807:0:3044:0:(pack_generic.c:334:lustre_pack_reply_v2()) Process entered 02000000:00000001:25.0:1506127920.169809:0:3044:0:(sec.c:2112:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:25.0:1506127920.169811:0:3044:0:(sec_null.c:318:null_alloc_rs()) kmalloced 'rs': 672 at ffff88080f503800. 02000000:00000001:25.0:1506127920.169813:0:3044:0:(sec.c:2148:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.169815:0:3044:0:(pack_generic.c:368:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00010000:00080000:25.0:1506127920.169821:0:3044:0:(ldlm_lib.c:1239:target_handle_connect()) soaked-MDT0003: connection from soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib recovering/t0 exp (null) cur 1506127920 last 0 00010000:00000001:25.0:1506127920.169825:0:3044:0:(obd_class.h:825:obd_connect()) Process entered 00000004:00000001:25.0:1506127920.169827:0:3044:0:(mdt_handler.c:5578:mdt_obd_connect()) Process entered 00000020:00000001:25.0:1506127920.169828:0:3044:0:(genops.c:1167:class_connect()) Process entered 00000020:00000001:25.0:1506127920.169830:0:3044:0:(genops.c:842:class_new_export()) Process entered 00000020:00000010:25.0:1506127920.169831:0:3044:0:(genops.c:844:class_new_export()) kmalloced 'export': 968 at ffff88080f507800. 00000020:00000001:25.0:1506127920.169833:0:3044:0:(lustre_handles.c:64:class_handle_hash()) Process entered 00000020:00000040:25.0:1506127920.169835:0:3044:0:(lustre_handles.c:98:class_handle_hash()) added object ffff88080f507800 with handle 0x2f3e107735f0c1c to hash 00000020:00000001:25.0:1506127920.169836:0:3044:0:(lustre_handles.c:99:class_handle_hash()) Process leaving 00000020:00000001:25.0:1506127920.169838:0:3044:0:(obd_class.h:998:obd_init_export()) Process entered 00000004:00000001:25.0:1506127920.169839:0:3044:0:(mdt_handler.c:5681:mdt_init_export()) Process entered 00000001:00000001:25.0:1506127920.169840:0:3044:0:(tgt_lastrcvd.c:386:tgt_client_alloc()) Process entered 00000001:00000010:25.0:1506127920.169841:0:3044:0:(tgt_lastrcvd.c:392:tgt_client_alloc()) kmalloced 'exp->u.eu_target_data.ted_lcd': 128 at ffff88080f761280. 00000001:00000001:25.0:1506127920.169843:0:3044:0:(tgt_lastrcvd.c:399:tgt_client_alloc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127920.169844:0:3044:0:(ldlm_lockd.c:2889:ldlm_init_export()) Process entered 00000001:00000001:25.0:1506127920.169846:0:3044:0:(hash.c:1005:cfs_hash_create()) Process entered 00000001:00000010:25.0:1506127920.169847:0:3044:0:(hash.c:1032:cfs_hash_create()) alloc '(hs)': 144 at ffff88080f8a9680 (tot 194331580). 00000001:00000010:25.0:1506127920.169850:0:3044:0:(hash.c:893:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at ffff880822a87b20 (tot 194331612). 00000001:00000010:25.0:1506127920.169852:0:3044:0:(hash.c:906:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 280 at ffff88082e03ba00 (tot 194331892). 00000001:00000010:25.0:1506127920.169855:0:3044:0:(hash.c:906:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 280 at ffff88082e03bc00 (tot 194332172). 00000001:00000010:25.0:1506127920.169857:0:3044:0:(hash.c:906:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 280 at ffff88082e03fa00 (tot 194332452). 00000001:00000010:25.0:1506127920.169859:0:3044:0:(hash.c:906:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 280 at ffff88082e038400 (tot 194332732). 00000001:00000001:25.0:1506127920.169861:0:3044:0:(hash.c:1005:cfs_hash_create()) Process entered 00000001:00000010:25.0:1506127920.169862:0:3044:0:(hash.c:1032:cfs_hash_create()) alloc '(hs)': 144 at ffff88080f8a95c0 (tot 194332876). 00000001:00000010:25.0:1506127920.169864:0:3044:0:(hash.c:893:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at ffff880822a87b00 (tot 194332908). 00000001:00000010:25.0:1506127920.169866:0:3044:0:(hash.c:906:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 280 at ffff88082e03fe00 (tot 194333188). 00000001:00000010:25.0:1506127920.169868:0:3044:0:(hash.c:906:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 280 at ffff88082e03f600 (tot 194333468). 00000001:00000010:25.0:1506127920.169871:0:3044:0:(hash.c:906:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 280 at ffff88082e03f800 (tot 194333748). 00000001:00000010:25.0:1506127920.169873:0:3044:0:(hash.c:906:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 280 at ffff88082e03f200 (tot 194334028). 00010000:00000001:25.0:1506127920.169875:0:3044:0:(ldlm_flock.c:939:ldlm_init_flock_export()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127920.169877:0:3044:0:(ldlm_lockd.c:2908:ldlm_init_export()) Process leaving (rc=0 : 0 : 0) 00000004:00000001:25.0:1506127920.169878:0:3044:0:(mdt_handler.c:5702:mdt_init_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.169880:0:3044:0:(obd_class.h:1002:obd_init_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:25.0:1506127920.169883:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507800 : new refcount 3 00000020:00000040:25.0:1506127920.169885:0:3044:0:(obd_config.c:753:class_incref()) incref soaked-MDT0003 (ffff880826bd4d58) now 99 00000020:00000001:25.0:1506127920.169887:0:3044:0:(genops.c:917:class_new_export()) Process leaving (rc=18446612166930888704 : -131906778662912 : ffff88080f507800) 00000020:00000040:25.0:1506127920.169890:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507800 : new refcount 2 00000020:00000080:25.0:1506127920.169891:0:3044:0:(genops.c:1177:class_connect()) connect: client soaked-MDT0000-mdtlov_UUID, cookie 0x2f3e107735f0c1c 00000020:00000001:25.0:1506127920.169893:0:3044:0:(genops.c:1178:class_connect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.169894:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:25.0:1506127920.169895:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x2f3e107735f0c1c 00000020:00000001:25.0:1506127920.169896:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000040:25.0:1506127920.169897:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507800 : new refcount 3 00000020:00000001:25.0:1506127920.169898:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612166930888704 : -131906778662912 : ffff88080f507800) 00000020:00000001:25.0:1506127920.169900:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=18446612166930888704 : -131906778662912 : ffff88080f507800) 00000001:00000001:25.0:1506127920.169902:0:3044:0:(nodemap_handler.c:385:nodemap_add_member()) Process entered 00000001:00000001:25.0:1506127920.169904:0:3044:0:(nodemap_handler.c:264:nodemap_classify_nid()) Process entered 00000001:00000001:25.0:1506127920.169905:0:3044:0:(interval_tree.c:629:interval_search()) Process entered 00000001:00000001:25.0:1506127920.169907:0:3044:0:(interval_tree.c:676:interval_search()) Process leaving (rc=1 : 1 : 1) 00000001:00000040:25.0:1506127920.169908:0:3044:0:(nodemap_handler.c:97:nodemap_getref()) GETting nodemap default(p=ffff880813872000) : new refcount 50 00000001:00000001:25.0:1506127920.169910:0:3044:0:(nodemap_handler.c:290:nodemap_classify_nid()) Process leaving (rc=18446612167001579520 : -131906707972096 : ffff880813872000) 00000001:00000001:25.0:1506127920.169912:0:3044:0:(nodemap_member.c:99:nm_member_add()) Process entered 00000020:00000040:25.0:1506127920.169913:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507800 : new refcount 4 00000001:00000040:25.0:1506127920.169915:0:3044:0:(nodemap_handler.c:97:nodemap_getref()) GETting nodemap default(p=ffff880813872000) : new refcount 51 00000001:00000001:25.0:1506127920.169916:0:3044:0:(nodemap_member.c:136:nm_member_add()) Process leaving (rc=0 : 0 : 0) 00000001:00000040:25.0:1506127920.169918:0:3044:0:(nodemap_handler.c:113:nodemap_putref()) PUTting nodemap default(p=ffff880813872000) : new refcount 50 00000001:00000001:25.0:1506127920.169919:0:3044:0:(nodemap_handler.c:406:nodemap_add_member()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.169922:0:3044:0:(tgt_lastrcvd.c:949:tgt_client_new()) Process entered 00000001:00000040:25.0:1506127920.169924:0:3044:0:(tgt_lastrcvd.c:1008:tgt_client_new()) soaked-MDT0003: new client at index 0 (8192) with UUID 'soaked-MDT0000-mdtlov_UUID' generation 0 00000001:00000001:25.0:1506127920.169928:0:3044:0:(tgt_lastrcvd.c:576:tgt_client_data_update()) Process entered 00080000:00000001:25.0:1506127920.169931:0:3044:0:(osd_handler.c:1612:osd_trans_create()) Process entered 00080000:00000010:25.0:1506127920.169934:0:3044:0:(osd_handler.c:1627:osd_trans_create()) kmalloced 'oh': 240 at ffff880822756b00. 00080000:00000001:25.0:1506127920.169936:0:3044:0:(osd_handler.c:1650:osd_trans_create()) Process leaving (rc=18446612167252077312 : -131906457474304 : ffff880822756b00) 00000001:00000001:25.0:1506127920.169940:0:3044:0:(osd_io.c:1576:osd_declare_write()) Process entered 00000001:00000001:25.0:1506127920.169943:0:3044:0:(osd_quota.c:630:osd_declare_inode_qid()) Process entered 00000001:00000001:25.0:1506127920.169945:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127920.169948:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127920.169950:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.169951:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.169953:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127920.169954:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127920.169955:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.169956:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.169957:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127920.169958:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127920.169959:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.169960:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.169962:0:3044:0:(osd_quota.c:677:osd_declare_inode_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.169964:0:3044:0:(osd_io.c:1650:osd_declare_write()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:25.0:1506127920.169966:0:3044:0:(osd_handler.c:1713:osd_trans_start()) Process entered 00000020:00000010:25.0:1506127920.169973:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88082e03f400. 00000020:00000010:25.0:1506127920.169976:0:3044:0:(update_records.c:1217:update_key_init()) kmalloced 'value': 416 at ffff88082e03ee00. 00000004:00000010:25.0:1506127920.169979:0:3044:0:(osp_dev.c:1869:osp_txn_key_init()) kmalloced 'value': 8 at ffff880825e00190. 00080000:00000001:25.0:1506127920.169981:0:3044:0:(osd_handler.c:1780:osd_trans_start()) Process leaving (rc=0 : 0 : 0) 00000001:00000010:25.0:1506127920.169983:0:3044:0:(tgt_lastrcvd.c:545:tgt_new_client_cb_add()) kmalloced 'ccb': 80 at ffff8808138a1cc0. 00000001:00000040:25.0:1506127920.169985:0:3044:0:(tgt_lastrcvd.c:549:tgt_new_client_cb_add()) callback GETting export ffff88080f507800 : new cb_count 1 00000020:00000040:25.0:1506127920.169987:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507800 : new refcount 5 00000001:00000001:25.0:1506127920.170008:0:3044:0:(tgt_lastrcvd.c:625:tgt_client_data_update()) Process leaving 00080000:00000001:25.0:1506127920.170010:0:3044:0:(osd_handler.c:1834:osd_trans_stop()) Process entered 00040000:00000001:25.0:1506127920.170014:0:3044:0:(qsd_handler.c:1080:qsd_op_end()) Process entered 00040000:00000001:25.0:1506127920.170016:0:3044:0:(qsd_handler.c:1092:qsd_op_end()) Process leaving 00080000:00000001:25.0:1506127920.170017:0:3044:0:(osd_handler.c:1905:osd_trans_stop()) Process leaving (rc=0 : 0 : 0) 00000001:00000040:25.0:1506127920.170019:0:3044:0:(tgt_lastrcvd.c:630:tgt_client_data_update()) soaked-MDT0003: update last_rcvd client data for UUID = soaked-MDT0003_UUID, last_transno = 30064771072: rc = 0 00000001:00000001:25.0:1506127920.170022:0:3044:0:(tgt_lastrcvd.c:1018:tgt_client_new()) Process leaving (rc=0 : 0 : 0) 00000004:00000001:25.0:1506127920.170024:0:3044:0:(mdt_fs.c:68:mdt_export_stats_init()) Process entered 00000020:00000001:25.0:1506127920.170026:0:3044:0:(lprocfs_status_server.c:319:lprocfs_exp_setup()) Process entered 00000020:01000000:25.0:1506127920.170028:0:3044:0:(lprocfs_status_server.c:342:lprocfs_exp_setup()) using hash ffff880813836d80 00000020:00000010:25.0:1506127920.170030:0:3044:0:(lprocfs_status_server.c:344:lprocfs_exp_setup()) kmalloced 'new_stat': 80 at ffff8808138a1060. 00000020:00000040:25.0:1506127920.170033:0:3044:0:(lprocfs_status_server.c:356:lprocfs_exp_setup()) Found stats ffff8808138a1060 for nid 192.168.1.108@o2ib - ref 2 00000020:00000001:25.0:1506127920.170044:0:3044:0:(lprocfs_status_server.c:425:lprocfs_exp_setup()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:25.0:1506127920.170046:0:3044:0:(lprocfs_status.c:1293:lprocfs_alloc_stats()) alloc '(stats)': 32 at ffff880822a87b40 (tot 194334060). 00000020:00000010:25.0:1506127920.170048:0:3044:0:(lprocfs_status.c:1303:lprocfs_alloc_stats()) alloc '(stats->ls_cnt_header)': 384 at ffff88082e03f000 (tot 194334444). 00000020:00000010:25.0:1506127920.170051:0:3044:0:(lprocfs_status.c:1310:lprocfs_alloc_stats()) alloc '(stats->ls_percpu[0])': 640 at ffff88080f505400 (tot 194335084). 00000020:00000010:25.0:1506127920.170055:0:3044:0:(lprocfs_status.c:1293:lprocfs_alloc_stats()) alloc '(stats)': 32 at ffff880822a87b60 (tot 194335116). 00000020:00000010:25.0:1506127920.170057:0:3044:0:(lprocfs_status.c:1303:lprocfs_alloc_stats()) alloc '(stats->ls_cnt_header)': 168 at ffff88080f8a9200 (tot 194335284). 00000020:00000010:25.0:1506127920.170059:0:3044:0:(lprocfs_status.c:1310:lprocfs_alloc_stats()) alloc '(stats->ls_percpu[0])': 280 at ffff88082e03ea00 (tot 194335564). 00000020:00000001:25.0:1506127920.170062:0:3044:0:(lprocfs_status.c:2201:lprocfs_seq_create()) Process entered 00000020:00000001:25.0:1506127920.170064:0:3044:0:(lprocfs_status.c:2211:lprocfs_seq_create()) Process leaving (rc=0 : 0 : 0) 00000004:00000001:25.0:1506127920.170066:0:3044:0:(mdt_fs.c:103:mdt_export_stats_init()) Process leaving (rc=0 : 0 : 0) 00000004:00000001:25.0:1506127920.170068:0:3044:0:(mdt_handler.c:5647:mdt_obd_connect()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127920.170069:0:3044:0:(obd_class.h:835:obd_connect()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:25.0:1506127920.170115:0:3044:0:(genops.c:1056:class_new_import()) kmalloced 'imp': 1480 at ffff8808138f9800. 00000020:00000040:25.0:1506127920.170118:0:3044:0:(obd_config.c:753:class_incref()) incref soaked-MDT0003 (ffff880826bd4d58) now 100 00000020:00000001:25.0:1506127920.170120:0:3044:0:(lustre_handles.c:64:class_handle_hash()) Process entered 00000020:00000040:25.0:1506127920.170121:0:3044:0:(lustre_handles.c:98:class_handle_hash()) added object ffff8808138f9800 with handle 0x2f3e107735f0c23 to hash 00000020:00000001:25.0:1506127920.170123:0:3044:0:(lustre_handles.c:99:class_handle_hash()) Process leaving 00000020:00000001:25.0:1506127920.170124:0:3044:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:25.0:1506127920.170125:0:3044:0:(genops.c:1026:class_import_put()) import ffff8808138f9800 refcount=1 obd=soaked-MDT0003 00000020:00000001:25.0:1506127920.170127:0:3044:0:(genops.c:1035:class_import_put()) Process leaving 00000020:00000040:25.0:1506127920.170129:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507800 : new refcount 6 00000100:00000040:25.0:1506127920.170131:0:3044:0:(service.c:999:ptlrpc_request_change_export()) RPC GETting export ffff88080f507800 : new rpc_count 1 00000100:00000001:25.0:1506127920.170133:0:3044:0:(connection.c:48:ptlrpc_connection_get()) Process entered 00000400:00000200:25.0:1506127920.170139:0:3044:0:(peer.c:624:LNetPrimaryNID()) NID 192.168.1.108@o2ib primary NID 192.168.1.108@o2ib rc 0 00000100:00000001:25.0:1506127920.170142:0:3044:0:(connection.c:53:ptlrpc_connection_get()) Process leaving via out (rc=18446612167460263808 : -131906249287808 : 0xffff88082ede1780) 00000100:00000040:25.0:1506127920.170146:0:3044:0:(connection.c:84:ptlrpc_connection_get()) conn=ffff88082ede1780 refcount 12 to 192.168.1.108@o2ib 00000020:00000040:25.0:1506127920.170148:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507800 : new refcount 7 00000100:00080000:25.0:1506127920.170150:0:3044:0:(import.c:104:ptlrpc_import_enter_resend()) ffff8808138f9800 HR½&ˆÿÿHR½&ˆÿÿ: changing import state from NEW to RECOVER 00000100:00000001:25.0:1506127920.170154:0:3044:0:(connection.c:127:ptlrpc_connection_addref()) Process entered 00000100:00000040:25.0:1506127920.170156:0:3044:0:(connection.c:132:ptlrpc_connection_addref()) conn=ffff88082ede1780 refcount 13 to 192.168.1.108@o2ib 00000100:00000001:25.0:1506127920.170158:0:3044:0:(connection.c:134:ptlrpc_connection_addref()) Process leaving (rc=18446612167460263808 : -131906249287808 : ffff88082ede1780) 02000000:00000001:25.0:1506127920.170160:0:3044:0:(sec.c:1434:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:25.0:1506127920.170162:0:3044:0:(sec.c:1331:sptlrpc_sec_create()) Process entered 02000000:08000000:25.0:1506127920.170163:0:3044:0:(sec.c:1339:sptlrpc_sec_create()) mdt soaked-MDT0003: reverse sec using flavor null 02000000:00000001:25.0:1506127920.170165:0:3044:0:(sec.c:1370:sptlrpc_sec_create()) Process leaving (rc=18446744072655015392 : -1054536224 : ffffffffc1250de0) 02000000:00000001:25.0:1506127920.170167:0:3044:0:(sec.c:1505:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.170169:0:3044:0:(import.c:1460:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:25.0:1506127920.170170:0:3044:0:(recover.c:179:ptlrpc_resend()) Process entered 00000100:00000001:25.0:1506127920.170171:0:3044:0:(recover.c:207:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:25.0:1506127920.170173:0:3044:0:(import.c:1536:ptlrpc_import_recovery_state_machine()) ffff8808138f9800 HR½&ˆÿÿHR½&ˆÿÿ: changing import state from RECOVER to FULL 00000100:00000001:25.0:1506127920.170175:0:3044:0:(obd_class.h:1168:obd_import_event()) Process entered 00000100:00000001:25.0:1506127920.170176:0:3044:0:(obd_class.h:1178:obd_import_event()) Process leaving 00000100:02000000:25.0:1506127920.170178:0:3044:0:(import.c:1542:ptlrpc_import_recovery_state_machine()) soaked-MDT0003: Connection restored to 192.168.1.108@o2ib (at 192.168.1.108@o2ib) 00000100:00000001:25.0:1506127920.170181:0:3044:0:(import.c:1551:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:25.0:1506127920.170186:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507800 : new refcount 6 00000020:00000040:25.0:1506127920.170188:0:3044:0:(obd_config.c:770:class_decref()) Decref soaked-MDT0003 (ffff880826bd4d58) now 99 00010000:00000001:25.0:1506127920.170190:0:3044:0:(ldlm_lib.c:1452:target_handle_connect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.170192:0:3044:0:(tgt_handler.c:584:tgt_handler_find_check()) Process entered 00000020:00000001:25.0:1506127920.170193:0:3044:0:(tgt_handler.c:611:tgt_handler_find_check()) Process leaving (rc=18446744072658299920 : -1051251696 : ffffffffc1572c10) 00000020:00000001:25.0:1506127920.170196:0:3044:0:(tgt_handler.c:523:tgt_handle_recovery()) Process entered 00000020:00000001:25.0:1506127920.170197:0:3044:0:(tgt_handler.c:532:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127920.170198:0:3044:0:(tgt_handler.c:382:tgt_handle_request0()) Process entered 00000020:00000001:25.0:1506127920.170200:0:3044:0:(tgt_handler.c:311:tgt_request_preprocess()) Process entered 00000020:00000001:25.0:1506127920.170201:0:3044:0:(tgt_handler.c:366:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.170203:0:3044:0:(tgt_handler.c:935:tgt_connect()) Process entered 00000020:00000001:25.0:1506127920.170205:0:3044:0:(tgt_handler.c:955:tgt_connect()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:25.0:1506127920.170207:0:3044:0:(ldlm_lib.c:2884:target_committed_to_req()) last_committed 0, transno 0, xid 1579283258778784 00010000:00000001:25.0:1506127920.170209:0:3044:0:(ldlm_lib.c:2948:target_send_reply()) Process entered 00010000:00000200:25.0:1506127920.170212:0:3044:0:(ldlm_lib.c:2936:target_send_reply_msg()) @@@ sending reply req@ffff880813806050 x1579283258778784/t0(0) o38->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:0/0 lens 520/416 e 0 to 0 dl 1506128020 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000001:25.0:1506127920.170220:0:3044:0:(ldlm_lib.c:2895:target_pack_pool_reply()) Process entered 00010000:00000001:25.0:1506127920.170222:0:3044:0:(ldlm_lib.c:2903:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:25.0:1506127920.170224:0:3044:0:(import.c:1678:at_measured()) add 1 to ffff88081629fd50 time=1506127920 v=10 (0 0 0 0) 00000100:00001000:25.0:1506127920.170227:0:3044:0:(import.c:1737:at_measured()) AT ffff88081629fd50 change: old=10 new=1 delta=-9 (val=1) hist 1 0 0 0 00000100:00001000:25.0:1506127920.170231:0:3044:0:(niobuf.c:512:ptlrpc_at_set_reply()) @@@ svc mdt_out changed estimate from 10 to 1 req@ffff880813806050 x1579283258778784/t0(0) o38->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:0/0 lens 520/384 e 0 to 0 dl 1506128020 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00001000:25.0:1506127920.170239:0:3044:0:(niobuf.c:550:ptlrpc_at_set_reply()) No early reply support: flags=0x1 req_flags=0x0 magic=bd00bd3/bd00bd3 len=384 00000100:00000001:25.0:1506127920.170241:0:3044:0:(connection.c:127:ptlrpc_connection_addref()) Process entered 00000100:00000040:25.0:1506127920.170243:0:3044:0:(connection.c:132:ptlrpc_connection_addref()) conn=ffff88082ede1780 refcount 14 to 192.168.1.108@o2ib 00000100:00000001:25.0:1506127920.170245:0:3044:0:(connection.c:134:ptlrpc_connection_addref()) Process leaving (rc=18446612167460263808 : -131906249287808 : ffff88082ede1780) 02000000:00000001:25.0:1506127920.170247:0:3044:0:(sec.c:2161:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:25.0:1506127920.170248:0:3044:0:(sec.c:2172:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.170250:0:3044:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:25.0:1506127920.170253:0:3044:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-192.168.1.108@o2ib 00000400:00000010:25.0:1506127920.170254:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761300. 00000100:00000200:25.0:1506127920.170257:0:3044:0:(niobuf.c:85:ptl_send_buf()) Sending 384 bytes to portal 4, xid 1579283258778784, offset 0 00000400:00000010:25.0:1506127920.170260:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88082e03ec00 (tot 194335964). 00000400:00000200:25.0:1506127920.170263:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.108@o2ib 00000400:00000200:25.0:1506127920.170274:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.108@o2ib(192.168.1.108@o2ib:192.168.1.108@o2ib) : PUT 00000800:00000200:25.0:1506127920.170279:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 384 bytes in 1 frags to 12345-192.168.1.108@o2ib 00000800:00000200:25.0:1506127920.170285:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082cb07e00] -> 192.168.1.108@o2ib (2) version: 12 00000800:00000200:25.0:1506127920.170288:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082e03de00] (20)++ 00000800:00000200:25.0:1506127920.170290:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082e03de00] (21)++ 00000800:00000200:25.0:1506127920.170295:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082e03de00] (22)-- 00000100:00000001:25.0:1506127920.170297:0:3044:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.170298:0:3044:0:(connection.c:91:ptlrpc_connection_put()) Process entered 00000100:00000040:25.0:1506127920.170300:0:3044:0:(connection.c:119:ptlrpc_connection_put()) PUT conn=ffff88082ede1780 refcount 13 to 192.168.1.108@o2ib 00000100:00000001:25.0:1506127920.170302:0:3044:0:(connection.c:121:ptlrpc_connection_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127920.170304:0:3044:0:(ldlm_lib.c:2960:target_send_reply()) Process leaving 00000020:00000001:25.0:1506127920.170305:0:3044:0:(tgt_handler.c:475:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.170306:0:3044:0:(tgt_handler.c:780:tgt_request_handle()) Process leaving 00000100:00000040:25.0:1506127920.170309:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff880813806050 x1579283258778784/t0(0) o38->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:0/0 lens 520/384 e 0 to 0 dl 1506128020 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00100000:25.0:1506127920.170317:0:3044:0:(service.c:2135:ptlrpc_server_handle_request()) Handled RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0000-mdtlov_UUID+6:2041:x1579283258778784:12345-192.168.1.108@o2ib:38 Request procesed in 526us (622us total) trans 0 rc 0/0 00000100:00100000:25.0:1506127920.170324:0:3044:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.1.108@o2ib, seq: 2 00000100:00000040:25.0:1506127920.170326:0:3044:0:(service.c:1034:ptlrpc_server_finish_active_request()) RPC PUTting export ffff88080f507800 : new rpc_count 0 00000100:00000001:25.0:1506127920.170328:0:3044:0:(service.c:1613:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:25.0:1506127920.170330:0:3044:0:(service.c:1624:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:25.0:1506127920.170332:0:3044:0:(lu_ucred.c:49:lu_ucred_key_fini()) kfreed 'info': 104 at ffff88080f761200. 00000020:00000010:25.0:1506127920.170334:0:3044:0:(tgt_main.c:382:tgt_ses_key_fini()) kfreed 'info': 160 at ffff88080f8a8e40. 00000020:00000010:25.0:1506127920.170336:0:3044:0:(lu_object.c:1595:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff88082e03be00. 00000020:00000040:25.0:1506127920.170338:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507800 : new refcount 5 00000100:00000001:25.0:1506127920.170341:0:3044:0:(service.c:2155:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:25.0:1506127920.170344:0:3044:0:(niobuf.c:953:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000400:00000010:25.0:1506127920.170347:0:3044:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff8808138a1300. 00000400:00000010:25.0:1506127920.170350:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761200. 00000400:00000001:25.0:1506127920.170354:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127920.170356:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127920.172014:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127920.172017:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127920.172020:0:3044:0:(service.c:1859:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:25.0:1506127920.172022:0:3044:0:(sec.c:2052:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:25.0:1506127920.172024:0:3044:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:25.0:1506127920.172026:0:3044:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:25.0:1506127920.172030:0:3044:0:(sec.c:2099:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:25.0:1506127920.172033:0:3044:0:(service.c:1939:ptlrpc_server_handle_req_in()) got req x1579283258779232 00000020:00000001:25.0:1506127920.172036:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:25.0:1506127920.172037:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x2f3e107735f0c1c 00000020:00000001:25.0:1506127920.172040:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000040:25.0:1506127920.172041:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507800 : new refcount 6 00000020:00000001:25.0:1506127920.172043:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612166930888704 : -131906778662912 : ffff88080f507800) 00000020:00000001:25.0:1506127920.172045:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=18446612166930888704 : -131906778662912 : ffff88080f507800) 00000100:00000001:25.0:1506127920.172048:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127920.172050:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:25.0:1506127920.172052:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88082e03be00. 00000020:00000010:25.0:1506127920.172055:0:3044:0:(lu_ucred.c:49:lu_ucred_key_init()) kmalloced 'value': 104 at ffff88080f761380. 00000020:00000010:25.0:1506127920.172057:0:3044:0:(tgt_main.c:382:tgt_ses_key_init()) kmalloced 'value': 160 at ffff88080f8a8e40. 00000100:00000040:25.0:1506127920.172061:0:3044:0:(service.c:1184:ptlrpc_at_set_timer()) armed mdt_out at +1s 00000100:00000001:25.0:1506127920.172063:0:3044:0:(service.c:1656:ptlrpc_server_request_add()) Process entered 00000100:00000001:25.0:1506127920.172064:0:3044:0:(service.c:1580:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:25.0:1506127920.172065:0:3044:0:(service.c:1607:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.172068:0:3044:0:(service.c:1695:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.172090:0:3044:0:(service.c:2003:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:25.0:1506127920.172093:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127920.172095:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127920.172096:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127920.172097:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127920.172099:0:3044:0:(service.c:2028:ptlrpc_server_handle_request()) Process entered 00000100:00000001:25.0:1506127920.172100:0:3044:0:(service.c:1809:ptlrpc_server_request_get()) Process entered 00000100:00100000:25.0:1506127920.172104:0:3044:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.1.108@o2ib, seq: 3 00000100:00000040:25.0:1506127920.172106:0:3044:0:(service.c:1840:ptlrpc_server_request_get()) RPC GETting export ffff88080f507800 : new rpc_count 1 00000100:00000001:25.0:1506127920.172107:0:3044:0:(service.c:1842:ptlrpc_server_request_get()) Process leaving (rc=18446612167001136208 : -131906708415408 : ffff880813805c50) 00000100:00000040:25.0:1506127920.172111:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "New" -> "Interpret" req@ffff880813805c50 x1579283258779232/t0(0) o400->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 224/0 e 0 to 0 dl 1506127926 ref 1 fl New:/c0/ffffffff rc 0/-1 00000100:00000001:25.0:1506127920.172120:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127920.172121:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000100:00100000:25.0:1506127920.172123:0:3044:0:(service.c:2089:ptlrpc_server_handle_request()) Handling RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0000-mdtlov_UUID+6:2041:x1579283258779232:12345-192.168.1.108@o2ib:400 00000100:00000200:25.0:1506127920.172126:0:3044:0:(service.c:2094:ptlrpc_server_handle_request()) got req 1579283258779232 00000020:00000001:25.0:1506127920.172128:0:3044:0:(tgt_handler.c:672:tgt_request_handle()) Process entered 00000020:00000001:25.0:1506127920.172130:0:3044:0:(tgt_handler.c:617:process_req_last_xid()) Process entered 00000020:00000001:25.0:1506127920.172132:0:3044:0:(tgt_handler.c:657:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.172133:0:3044:0:(tgt_handler.c:584:tgt_handler_find_check()) Process entered 00000020:00000001:25.0:1506127920.172135:0:3044:0:(tgt_handler.c:611:tgt_handler_find_check()) Process leaving (rc=18446744072654874048 : -1054677568 : ffffffffc122e5c0) 00000020:00000001:25.0:1506127920.172137:0:3044:0:(tgt_handler.c:523:tgt_handle_recovery()) Process entered 00000020:00000040:25.0:1506127920.172140:0:3044:0:(tgt_handler.c:562:tgt_handle_recovery()) @@@ Got new replay req@ffff880813805c50 x1579283258779232/t0(0) o400->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 224/0 e 0 to 0 dl 1506127926 ref 1 fl Interpret:/c0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127920.172147:0:3044:0:(ldlm_lib.c:2705:target_queue_recovery_request()) Process entered 00010000:00000040:25.0:1506127920.172149:0:3044:0:(ldlm_lib.c:1496:target_request_copy_get()) RPC GETting export ffff88080f507800 : new rpc_count 2 00010000:00080000:25.0:1506127920.172152:0:3044:0:(ldlm_lib.c:2718:target_queue_recovery_request()) @@@ queue final req req@ffff880813805c50 x1579283258779232/t0(0) o400->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 224/0 e 0 to 0 dl 1506127926 ref 2 fl Interpret:/c0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127920.172160:0:3044:0:(ldlm_lib.c:2730:target_queue_recovery_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.172162:0:3044:0:(tgt_handler.c:506:tgt_filter_recovery_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.172164:0:3044:0:(tgt_handler.c:566:tgt_handle_recovery()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.172166:0:3044:0:(tgt_handler.c:780:tgt_request_handle()) Process leaving 00000100:00000040:25.0:1506127920.172168:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff880813805c50 x1579283258779232/t0(0) o400->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 224/0 e 0 to 0 dl 1506127926 ref 2 fl Interpret:/c0/ffffffff rc 0/-1 00000100:00100000:25.0:1506127920.172176:0:3044:0:(service.c:2135:ptlrpc_server_handle_request()) Handled RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0000-mdtlov_UUID+6:2041:x1579283258779232:12345-192.168.1.108@o2ib:400 Request procesed in 54us (179us total) trans 0 rc 0/-999 00000100:00100000:25.0:1506127920.172182:0:3044:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.1.108@o2ib, seq: 3 00000100:00000040:25.0:1506127920.172184:0:3044:0:(service.c:1034:ptlrpc_server_finish_active_request()) RPC PUTting export ffff88080f507800 : new rpc_count 1 00000100:00000001:25.0:1506127920.172186:0:3044:0:(service.c:1613:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:25.0:1506127920.172187:0:3044:0:(service.c:1624:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:25.0:1506127920.172188:0:3044:0:(service.c:2155:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:25.0:1506127920.172190:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127920.172191:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127920.187207:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127920.187211:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127920.187213:0:3044:0:(service.c:1859:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:25.0:1506127920.187214:0:3044:0:(sec.c:2052:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:25.0:1506127920.187216:0:3044:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:25.0:1506127920.187217:0:3044:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:25.0:1506127920.187221:0:3044:0:(sec.c:2099:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:25.0:1506127920.187223:0:3044:0:(service.c:1939:ptlrpc_server_handle_req_in()) got req x1579283258778176 00000020:00000001:25.0:1506127920.187226:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:25.0:1506127920.187227:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x2f3e107735f0c1c 00000020:00000001:25.0:1506127920.187228:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000040:25.0:1506127920.187230:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507800 : new refcount 7 00000020:00000001:25.0:1506127920.187232:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612166930888704 : -131906778662912 : ffff88080f507800) 00000020:00000001:25.0:1506127920.187235:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=18446612166930888704 : -131906778662912 : ffff88080f507800) 00000100:00000001:25.0:1506127920.187238:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127920.187239:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:25.0:1506127920.187242:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88082e038a00. 00000020:00000010:25.0:1506127920.187245:0:3044:0:(lu_ucred.c:49:lu_ucred_key_init()) kmalloced 'value': 104 at ffff88080f761400. 00000020:00000010:25.0:1506127920.187247:0:3044:0:(tgt_main.c:382:tgt_ses_key_init()) kmalloced 'value': 160 at ffff88080f8a9500. 00000100:00000001:25.0:1506127920.187250:0:3044:0:(service.c:1656:ptlrpc_server_request_add()) Process entered 00000100:00000001:25.0:1506127920.187251:0:3044:0:(service.c:1580:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:25.0:1506127920.187253:0:3044:0:(service.c:1607:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.187256:0:3044:0:(service.c:1695:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.187262:0:3044:0:(service.c:2003:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:25.0:1506127920.187265:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127920.187267:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127920.187268:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127920.187269:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127920.187270:0:3044:0:(service.c:2028:ptlrpc_server_handle_request()) Process entered 00000100:00000001:25.0:1506127920.187272:0:3044:0:(service.c:1809:ptlrpc_server_request_get()) Process entered 00000100:00100000:25.0:1506127920.187276:0:3044:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.1.108@o2ib, seq: 4 00000100:00000040:25.0:1506127920.187279:0:3044:0:(service.c:1840:ptlrpc_server_request_get()) RPC GETting export ffff88080f507800 : new rpc_count 2 00000100:00000001:25.0:1506127920.187280:0:3044:0:(service.c:1842:ptlrpc_server_request_get()) Process leaving (rc=18446612167001135184 : -131906708416432 : ffff880813805850) 00000100:00000040:25.0:1506127920.187284:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "New" -> "Interpret" req@ffff880813805850 x1579283258778176/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:326/0 lens 304/0 e 0 to 0 dl 1506127931 ref 1 fl New:/0/ffffffff rc 0/-1 00000100:00000001:25.0:1506127920.187292:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127920.187294:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000100:00100000:25.0:1506127920.187296:0:3044:0:(service.c:2089:ptlrpc_server_handle_request()) Handling RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0000-mdtlov_UUID+7:4031:x1579283258778176:12345-192.168.1.108@o2ib:1000 00000100:00000200:25.0:1506127920.187299:0:3044:0:(service.c:2094:ptlrpc_server_handle_request()) got req 1579283258778176 00000020:00000001:25.0:1506127920.187301:0:3044:0:(tgt_handler.c:672:tgt_request_handle()) Process entered 00000020:00000001:25.0:1506127920.187303:0:3044:0:(tgt_handler.c:617:process_req_last_xid()) Process entered 00000020:00000001:25.0:1506127920.187305:0:3044:0:(tgt_handler.c:657:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.187307:0:3044:0:(tgt_handler.c:584:tgt_handler_find_check()) Process entered 00000020:00000001:25.0:1506127920.187308:0:3044:0:(tgt_handler.c:611:tgt_handler_find_check()) Process leaving (rc=18446744072654881248 : -1054670368 : ffffffffc12301e0) 00000020:00000001:25.0:1506127920.187310:0:3044:0:(tgt_handler.c:523:tgt_handle_recovery()) Process entered 00000020:00000040:25.0:1506127920.187313:0:3044:0:(tgt_handler.c:562:tgt_handle_recovery()) @@@ Got new replay req@ffff880813805850 x1579283258778176/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:326/0 lens 304/0 e 0 to 0 dl 1506127931 ref 1 fl Interpret:/0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127920.187319:0:3044:0:(ldlm_lib.c:2705:target_queue_recovery_request()) Process entered 00010000:00080000:25.0:1506127920.187322:0:3044:0:(ldlm_lib.c:2788:target_queue_recovery_request()) @@@ not queueing req@ffff880813805850 x1579283258778176/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:326/0 lens 304/0 e 0 to 0 dl 1506127931 ref 1 fl Interpret:/0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127920.187328:0:3044:0:(ldlm_lib.c:2789:target_queue_recovery_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127920.187329:0:3044:0:(tgt_handler.c:506:tgt_filter_recovery_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.187331:0:3044:0:(tgt_handler.c:573:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127920.187333:0:3044:0:(tgt_handler.c:382:tgt_handle_request0()) Process entered 00000020:00000001:25.0:1506127920.187334:0:3044:0:(tgt_handler.c:311:tgt_request_preprocess()) Process entered 00000020:00000001:25.0:1506127920.187336:0:3044:0:(tgt_handler.c:366:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.187339:0:3044:0:(out_handler.c:917:out_handle()) Process entered 00000020:00000010:25.0:1506127920.187343:0:3044:0:(out_handler.c:940:out_handle()) kmalloced 'update_bufs': 8 at ffff880825e00188. 00000100:00000001:25.0:1506127920.187346:0:3044:0:(pack_generic.c:334:lustre_pack_reply_v2()) Process entered 02000000:00000001:25.0:1506127920.187347:0:3044:0:(sec.c:2112:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:25.0:1506127920.187350:0:3044:0:(sec_null.c:318:null_alloc_rs()) kmalloced 'rs': 4576 at ffff880811c16000. 02000000:00000001:25.0:1506127920.187352:0:3044:0:(sec.c:2148:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.187353:0:3044:0:(pack_generic.c:368:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.187363:0:3044:0:(out_handler.c:205:out_attr_get()) Process entered 00000020:00000040:25.0:1506127920.187369:0:3044:0:(out_handler.c:233:out_attr_get()) soaked-MDT0003: insert attr get reply ffff880811c161e0 index 0: rc = 0 00000020:00000001:25.0:1506127920.187372:0:3044:0:(out_handler.c:238:out_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000002:25.0:1506127920.187375:0:3044:0:(lu_object.c:157:lu_object_put()) Add ffff88081f236f58 to site lru. hash: ffff88082b7a7080, bkt: ffffc900159f2018, lru_len: 1 00000020:00000010:25.0:1506127920.187379:0:3044:0:(out_handler.c:1169:out_handle()) kfreed 'update_bufs': 8 at ffff880825e00188. 00000020:00000001:25.0:1506127920.187381:0:3044:0:(out_handler.c:1175:out_handle()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:25.0:1506127920.187383:0:3044:0:(ldlm_lib.c:2884:target_committed_to_req()) last_committed 0, transno 0, xid 1579283258778176 00010000:00000001:25.0:1506127920.187385:0:3044:0:(ldlm_lib.c:2948:target_send_reply()) Process entered 00010000:00000200:25.0:1506127920.187388:0:3044:0:(ldlm_lib.c:2936:target_send_reply_msg()) @@@ sending reply req@ffff880813805850 x1579283258778176/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:326/0 lens 304/4320 e 0 to 0 dl 1506127931 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000001:25.0:1506127920.187395:0:3044:0:(ldlm_lib.c:2895:target_pack_pool_reply()) Process entered 00010000:00000001:25.0:1506127920.187397:0:3044:0:(ldlm_lib.c:2903:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:25.0:1506127920.187399:0:3044:0:(import.c:1678:at_measured()) add 1 to ffff88081629fd50 time=0 v=1 (1 0 0 0) 00000100:00000001:25.0:1506127920.187402:0:3044:0:(connection.c:127:ptlrpc_connection_addref()) Process entered 00000100:00000040:25.0:1506127920.187404:0:3044:0:(connection.c:132:ptlrpc_connection_addref()) conn=ffff88082ede1780 refcount 14 to 192.168.1.108@o2ib 00000100:00000001:25.0:1506127920.187406:0:3044:0:(connection.c:134:ptlrpc_connection_addref()) Process leaving (rc=18446612167460263808 : -131906249287808 : ffff88082ede1780) 02000000:00000001:25.0:1506127920.187408:0:3044:0:(sec.c:2161:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:25.0:1506127920.187409:0:3044:0:(sec.c:2172:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.187411:0:3044:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:25.0:1506127920.187413:0:3044:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-192.168.1.108@o2ib 00000400:00000010:25.0:1506127920.187415:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761480. 00000100:00000200:25.0:1506127920.187419:0:3044:0:(niobuf.c:85:ptl_send_buf()) Sending 4288 bytes to portal 4, xid 1579283258778176, offset 192 00000400:00000010:25.0:1506127920.187421:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88082e03e800 (tot 194335964). 00000400:00000200:25.0:1506127920.187425:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.108@o2ib 00000400:00000200:25.0:1506127920.187436:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.108@o2ib(192.168.1.108@o2ib:192.168.1.108@o2ib) : PUT 00000800:00000200:25.0:1506127920.187440:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 4288 bytes in 1 frags to 12345-192.168.1.108@o2ib 00000800:00000200:25.0:1506127920.187453:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082cb07e00] -> 192.168.1.108@o2ib (2) version: 12 00000800:00000200:25.0:1506127920.187456:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082e03de00] (20)++ 00000800:00000200:25.0:1506127920.187458:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082e03de00] (21)++ 00000800:00000200:25.0:1506127920.187463:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082e03de00] (22)-- 00000100:00000001:25.0:1506127920.187464:0:3044:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.187466:0:3044:0:(connection.c:91:ptlrpc_connection_put()) Process entered 00000100:00000040:25.0:1506127920.187468:0:3044:0:(connection.c:119:ptlrpc_connection_put()) PUT conn=ffff88082ede1780 refcount 13 to 192.168.1.108@o2ib 00000100:00000001:25.0:1506127920.187470:0:3044:0:(connection.c:121:ptlrpc_connection_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127920.187471:0:3044:0:(ldlm_lib.c:2960:target_send_reply()) Process leaving 00000020:00000001:25.0:1506127920.187472:0:3044:0:(tgt_handler.c:475:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.187474:0:3044:0:(tgt_handler.c:780:tgt_request_handle()) Process leaving 00000100:00000040:25.0:1506127920.187476:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff880813805850 x1579283258778176/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:326/0 lens 304/4288 e 0 to 0 dl 1506127931 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00100000:25.0:1506127920.187484:0:3044:0:(service.c:2135:ptlrpc_server_handle_request()) Handled RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0000-mdtlov_UUID+7:4031:x1579283258778176:12345-192.168.1.108@o2ib:1000 Request procesed in 191us (329us total) trans 0 rc 0/0 00000100:00100000:25.0:1506127920.187491:0:3044:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.1.108@o2ib, seq: 4 00000100:00000040:25.0:1506127920.187493:0:3044:0:(service.c:1034:ptlrpc_server_finish_active_request()) RPC PUTting export ffff88080f507800 : new rpc_count 1 00000100:00000001:25.0:1506127920.187495:0:3044:0:(service.c:1613:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:25.0:1506127920.187496:0:3044:0:(service.c:1624:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:25.0:1506127920.187498:0:3044:0:(lu_ucred.c:49:lu_ucred_key_fini()) kfreed 'info': 104 at ffff88080f761400. 00000020:00000010:25.0:1506127920.187500:0:3044:0:(tgt_main.c:382:tgt_ses_key_fini()) kfreed 'info': 160 at ffff88080f8a9500. 00000020:00000010:25.0:1506127920.187502:0:3044:0:(lu_object.c:1595:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff88082e038a00. 00000020:00000040:25.0:1506127920.187505:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507800 : new refcount 6 00000100:00000001:25.0:1506127920.187507:0:3044:0:(service.c:2155:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:25.0:1506127920.187510:0:3044:0:(niobuf.c:953:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000400:00000010:25.0:1506127920.187512:0:3044:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff8808138a1f60. 00000400:00000010:25.0:1506127920.187515:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761400. 00000400:00000001:25.0:1506127920.187517:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127920.187519:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127920.188653:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127920.188657:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127920.188659:0:3044:0:(service.c:1859:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:25.0:1506127920.188661:0:3044:0:(sec.c:2052:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:25.0:1506127920.188662:0:3044:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:25.0:1506127920.188664:0:3044:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:25.0:1506127920.188667:0:3044:0:(sec.c:2099:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:25.0:1506127920.188670:0:3044:0:(service.c:1939:ptlrpc_server_handle_req_in()) got req x1579283258779280 00000020:00000001:25.0:1506127920.188672:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:25.0:1506127920.188674:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x2f3e107735f0c1c 00000020:00000001:25.0:1506127920.188675:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000040:25.0:1506127920.188677:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507800 : new refcount 7 00000020:00000001:25.0:1506127920.188679:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612166930888704 : -131906778662912 : ffff88080f507800) 00000020:00000001:25.0:1506127920.188681:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=18446612166930888704 : -131906778662912 : ffff88080f507800) 00000100:00000001:25.0:1506127920.188685:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127920.188686:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:25.0:1506127920.188688:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88082e038a00. 00000020:00000010:25.0:1506127920.188691:0:3044:0:(lu_ucred.c:49:lu_ucred_key_init()) kmalloced 'value': 104 at ffff88080f761500. 00000020:00000010:25.0:1506127920.188693:0:3044:0:(tgt_main.c:382:tgt_ses_key_init()) kmalloced 'value': 160 at ffff88080f8a9500. 00000100:00000001:25.0:1506127920.188696:0:3044:0:(service.c:1656:ptlrpc_server_request_add()) Process entered 00000100:00000001:25.0:1506127920.188697:0:3044:0:(service.c:1580:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:25.0:1506127920.188698:0:3044:0:(service.c:1607:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.188701:0:3044:0:(service.c:1695:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.188708:0:3044:0:(service.c:2003:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:25.0:1506127920.188711:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127920.188712:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127920.188714:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127920.188715:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127920.188716:0:3044:0:(service.c:2028:ptlrpc_server_handle_request()) Process entered 00000100:00000001:25.0:1506127920.188717:0:3044:0:(service.c:1809:ptlrpc_server_request_get()) Process entered 00000100:00100000:25.0:1506127920.188722:0:3044:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.1.108@o2ib, seq: 5 00000100:00000040:25.0:1506127920.188724:0:3044:0:(service.c:1840:ptlrpc_server_request_get()) RPC GETting export ffff88080f507800 : new rpc_count 2 00000100:00000001:25.0:1506127920.188726:0:3044:0:(service.c:1842:ptlrpc_server_request_get()) Process leaving (rc=18446612167001134160 : -131906708417456 : ffff880813805450) 00000100:00000040:25.0:1506127920.188730:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "New" -> "Interpret" req@ffff880813805450 x1579283258779280/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 336/0 e 0 to 0 dl 1506127926 ref 1 fl New:/0/ffffffff rc 0/-1 00000100:00000001:25.0:1506127920.188738:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127920.188739:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000100:00100000:25.0:1506127920.188742:0:3044:0:(service.c:2089:ptlrpc_server_handle_request()) Handling RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0000-mdtlov_UUID+7:4031:x1579283258779280:12345-192.168.1.108@o2ib:1000 00000100:00000200:25.0:1506127920.188745:0:3044:0:(service.c:2094:ptlrpc_server_handle_request()) got req 1579283258779280 00000020:00000001:25.0:1506127920.188747:0:3044:0:(tgt_handler.c:672:tgt_request_handle()) Process entered 00000020:00000001:25.0:1506127920.188749:0:3044:0:(tgt_handler.c:617:process_req_last_xid()) Process entered 00000020:00000001:25.0:1506127920.188751:0:3044:0:(tgt_handler.c:657:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.188753:0:3044:0:(tgt_handler.c:584:tgt_handler_find_check()) Process entered 00000020:00000001:25.0:1506127920.188754:0:3044:0:(tgt_handler.c:611:tgt_handler_find_check()) Process leaving (rc=18446744072654881248 : -1054670368 : ffffffffc12301e0) 00000020:00000001:25.0:1506127920.188756:0:3044:0:(tgt_handler.c:523:tgt_handle_recovery()) Process entered 00000020:00000040:25.0:1506127920.188759:0:3044:0:(tgt_handler.c:562:tgt_handle_recovery()) @@@ Got new replay req@ffff880813805450 x1579283258779280/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 336/0 e 0 to 0 dl 1506127926 ref 1 fl Interpret:/0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127920.188766:0:3044:0:(ldlm_lib.c:2705:target_queue_recovery_request()) Process entered 00010000:00080000:25.0:1506127920.188768:0:3044:0:(ldlm_lib.c:2788:target_queue_recovery_request()) @@@ not queueing req@ffff880813805450 x1579283258779280/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 336/0 e 0 to 0 dl 1506127926 ref 1 fl Interpret:/0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127920.188774:0:3044:0:(ldlm_lib.c:2789:target_queue_recovery_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127920.188775:0:3044:0:(tgt_handler.c:506:tgt_filter_recovery_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.188777:0:3044:0:(tgt_handler.c:573:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127920.188778:0:3044:0:(tgt_handler.c:382:tgt_handle_request0()) Process entered 00000020:00000001:25.0:1506127920.188779:0:3044:0:(tgt_handler.c:311:tgt_request_preprocess()) Process entered 00000020:00000001:25.0:1506127920.188781:0:3044:0:(tgt_handler.c:366:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.188783:0:3044:0:(out_handler.c:917:out_handle()) Process entered 00000020:00000010:25.0:1506127920.188786:0:3044:0:(out_handler.c:940:out_handle()) kmalloced 'update_bufs': 8 at ffff880825e00188. 00000100:00000001:25.0:1506127920.188788:0:3044:0:(pack_generic.c:334:lustre_pack_reply_v2()) Process entered 02000000:00000001:25.0:1506127920.188790:0:3044:0:(sec.c:2112:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:25.0:1506127920.188793:0:3044:0:(sec_null.c:318:null_alloc_rs()) kmalloced 'rs': 4576 at ffff880811c16000. 02000000:00000001:25.0:1506127920.188794:0:3044:0:(sec.c:2148:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.188796:0:3044:0:(pack_generic.c:368:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.188802:0:3044:0:(out_handler.c:630:out_read()) Process entered 00000020:00000010:25.0:1506127920.188804:0:3044:0:(out_handler.c:660:out_read()) kmalloced 'rdbuf': 24 at ffff880822a87b80. 00000020:00000010:25.0:1506127920.188807:0:3044:0:(out_handler.c:671:out_read()) kmalloced 'rdbuf->rb_bufs[i].lb_buf': 32 at ffff880822a87ba0. 00000020:00000001:25.0:1506127920.188815:0:3044:0:(tgt_handler.c:1008:tgt_send_buffer()) Process entered 00000100:00000001:25.0:1506127920.188817:0:3044:0:(niobuf.c:130:ptlrpc_prep_bulk_exp()) Process entered 00000100:00000010:25.0:1506127920.188819:0:3044:0:(client.c:129:ptlrpc_new_bulk()) kmalloced 'desc': 280 at ffff88082e039000. 00000100:00000010:25.0:1506127920.188822:0:3044:0:(client.c:139:ptlrpc_new_bulk()) kmalloced '((desc)->bd_u.bd_kvec.bd_kvec)': 16 at ffff88082cbbd5a0. 00000020:00000040:25.0:1506127920.188824:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507800 : new refcount 8 00000100:00000001:25.0:1506127920.188826:0:3044:0:(client.c:235:ptlrpc_prep_bulk_frag()) Process entered 00000100:00000001:25.0:1506127920.188828:0:3044:0:(client.c:251:ptlrpc_prep_bulk_frag()) Process leaving (rc=32 : 32 : 20) 00010000:00000001:25.0:1506127920.188830:0:3044:0:(ldlm_lib.c:3167:target_bulk_io()) Process entered 00000100:00000001:25.0:1506127920.188833:0:3044:0:(niobuf.c:164:ptlrpc_start_bulk_transfer()) Process entered 00000400:00000010:25.0:1506127920.188835:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761580. 00000400:00000010:25.0:1506127920.188839:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88082e03e000 (tot 194340380). 00000400:00000200:25.0:1506127920.188843:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.108@o2ib 00000400:00000200:25.0:1506127920.188854:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.108@o2ib(192.168.1.108@o2ib:192.168.1.108@o2ib) : PUT 00000800:00000200:25.0:1506127920.188859:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 32 bytes in 1 frags to 12345-192.168.1.108@o2ib 00000800:00000200:25.0:1506127920.188864:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082cb07e00] -> 192.168.1.108@o2ib (2) version: 12 00000800:00000200:25.0:1506127920.188867:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082e03de00] (20)++ 00000800:00000200:25.0:1506127920.188869:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082e03de00] (21)++ 00000800:00000200:25.0:1506127920.188874:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082e03de00] (22)-- 00000100:00000200:25.0:1506127920.188878:0:3044:0:(niobuf.c:262:ptlrpc_start_bulk_transfer()) Transferring 1 pages 32 bytes via portal 14 id 12345-192.168.1.108@o2ib mbits 0x59c5989a3a690-0x59c5989a3a690 00000100:00000001:25.0:1506127920.188881:0:3044:0:(niobuf.c:264:ptlrpc_start_bulk_transfer()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127920.189345:0:3044:0:(ldlm_lib.c:3268:target_bulk_io()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.189349:0:3044:0:(client.c:257:ptlrpc_free_bulk()) Process entered 00000020:00000040:25.0:1506127920.189351:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507800 : new refcount 7 00000100:00000010:25.0:1506127920.189355:0:3044:0:(client.c:281:ptlrpc_free_bulk()) kfreed '((desc)->bd_u.bd_kvec.bd_kvec)': 16 at ffff88082cbbd5a0. 00000100:00000010:25.0:1506127920.189358:0:3044:0:(client.c:282:ptlrpc_free_bulk()) kfreed 'desc': 280 at ffff88082e039000. 00000100:00000001:25.0:1506127920.189359:0:3044:0:(client.c:283:ptlrpc_free_bulk()) Process leaving 00000020:00000001:25.0:1506127920.189361:0:3044:0:(tgt_handler.c:1023:tgt_send_buffer()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:25.0:1506127920.189363:0:3044:0:(out_handler.c:702:out_read()) kfreed 'rdbuf->rb_bufs[i].lb_buf': 32 at ffff880822a87ba0. 00000020:00000010:25.0:1506127920.189365:0:3044:0:(out_handler.c:706:out_read()) kfreed 'rdbuf': 24 at ffff880822a87b80. 00000020:00000001:25.0:1506127920.189367:0:3044:0:(out_handler.c:712:out_read()) Process leaving (rc=0 : 0 : 0) 00000020:00000002:25.0:1506127920.189372:0:3044:0:(lu_object.c:157:lu_object_put()) Add ffff88081f236f58 to site lru. hash: ffff88082b7a7080, bkt: ffffc900159f2018, lru_len: 1 00000020:00000010:25.0:1506127920.189374:0:3044:0:(out_handler.c:1169:out_handle()) kfreed 'update_bufs': 8 at ffff880825e00188. 00000020:00000001:25.0:1506127920.189376:0:3044:0:(out_handler.c:1175:out_handle()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:25.0:1506127920.189379:0:3044:0:(ldlm_lib.c:2884:target_committed_to_req()) last_committed 0, transno 0, xid 1579283258779280 00010000:00000001:25.0:1506127920.189381:0:3044:0:(ldlm_lib.c:2948:target_send_reply()) Process entered 00010000:00000200:25.0:1506127920.189386:0:3044:0:(ldlm_lib.c:2936:target_send_reply_msg()) @@@ sending reply req@ffff880813805450 x1579283258779280/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 336/4320 e 0 to 0 dl 1506127926 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000001:25.0:1506127920.189395:0:3044:0:(ldlm_lib.c:2895:target_pack_pool_reply()) Process entered 00010000:00000001:25.0:1506127920.189396:0:3044:0:(ldlm_lib.c:2903:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:25.0:1506127920.189399:0:3044:0:(import.c:1678:at_measured()) add 1 to ffff88081629fd50 time=0 v=1 (1 0 0 0) 00000100:00000001:25.0:1506127920.189402:0:3044:0:(connection.c:127:ptlrpc_connection_addref()) Process entered 00000100:00000040:25.0:1506127920.189405:0:3044:0:(connection.c:132:ptlrpc_connection_addref()) conn=ffff88082ede1780 refcount 14 to 192.168.1.108@o2ib 00000100:00000001:25.0:1506127920.189407:0:3044:0:(connection.c:134:ptlrpc_connection_addref()) Process leaving (rc=18446612167460263808 : -131906249287808 : ffff88082ede1780) 02000000:00000001:25.0:1506127920.189409:0:3044:0:(sec.c:2161:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:25.0:1506127920.189411:0:3044:0:(sec.c:2172:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.189412:0:3044:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:25.0:1506127920.189415:0:3044:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-192.168.1.108@o2ib 00000400:00000010:25.0:1506127920.189417:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761600. 00000100:00000200:25.0:1506127920.189421:0:3044:0:(niobuf.c:85:ptl_send_buf()) Sending 4288 bytes to portal 4, xid 1579283258779280, offset 192 00000400:00000010:25.0:1506127920.189423:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88082e039000 (tot 194340524). 00000400:00000200:25.0:1506127920.189427:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.108@o2ib 00000400:00000200:25.0:1506127920.189438:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.108@o2ib(192.168.1.108@o2ib:192.168.1.108@o2ib) : PUT 00000800:00000200:25.0:1506127920.189443:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 4288 bytes in 1 frags to 12345-192.168.1.108@o2ib 00000800:00000200:25.0:1506127920.189451:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082cb07e00] -> 192.168.1.108@o2ib (2) version: 12 00000800:00000200:25.0:1506127920.189453:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082e03de00] (20)++ 00000800:00000200:25.0:1506127920.189455:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082e03de00] (21)++ 00000800:00000200:25.0:1506127920.189460:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082e03de00] (22)-- 00000100:00000001:25.0:1506127920.189462:0:3044:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.189464:0:3044:0:(connection.c:91:ptlrpc_connection_put()) Process entered 00000100:00000040:25.0:1506127920.189466:0:3044:0:(connection.c:119:ptlrpc_connection_put()) PUT conn=ffff88082ede1780 refcount 13 to 192.168.1.108@o2ib 00000100:00000001:25.0:1506127920.189467:0:3044:0:(connection.c:121:ptlrpc_connection_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127920.189469:0:3044:0:(ldlm_lib.c:2960:target_send_reply()) Process leaving 00000020:00000001:25.0:1506127920.189470:0:3044:0:(tgt_handler.c:475:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.189472:0:3044:0:(tgt_handler.c:780:tgt_request_handle()) Process leaving 00000100:00000040:25.0:1506127920.189475:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff880813805450 x1579283258779280/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 336/4288 e 0 to 0 dl 1506127926 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00100000:25.0:1506127920.189483:0:3044:0:(service.c:2135:ptlrpc_server_handle_request()) Handled RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0000-mdtlov_UUID+7:4031:x1579283258779280:12345-192.168.1.108@o2ib:1000 Request procesed in 744us (880us total) trans 0 rc 0/0 00000100:00100000:25.0:1506127920.189491:0:3044:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.1.108@o2ib, seq: 5 00000100:00000040:25.0:1506127920.189493:0:3044:0:(service.c:1034:ptlrpc_server_finish_active_request()) RPC PUTting export ffff88080f507800 : new rpc_count 1 00000100:00000001:25.0:1506127920.189495:0:3044:0:(service.c:1613:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:25.0:1506127920.189497:0:3044:0:(service.c:1624:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:25.0:1506127920.189499:0:3044:0:(lu_ucred.c:49:lu_ucred_key_fini()) kfreed 'info': 104 at ffff88080f761500. 00000020:00000010:25.0:1506127920.189501:0:3044:0:(tgt_main.c:382:tgt_ses_key_fini()) kfreed 'info': 160 at ffff88080f8a9500. 00000020:00000010:25.0:1506127920.189503:0:3044:0:(lu_object.c:1595:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff88082e038a00. 00000020:00000040:25.0:1506127920.189505:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507800 : new refcount 6 00000100:00000001:25.0:1506127920.189508:0:3044:0:(service.c:2155:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:25.0:1506127920.189511:0:3044:0:(niobuf.c:953:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000400:00000010:25.0:1506127920.189514:0:3044:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff8808138a1c60. 00000400:00000010:25.0:1506127920.189516:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761500. 00000400:00000001:25.0:1506127920.189519:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127920.189520:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127920.191553:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127920.191555:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127920.191558:0:3044:0:(service.c:1859:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:25.0:1506127920.191559:0:3044:0:(sec.c:2052:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:25.0:1506127920.191561:0:3044:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:25.0:1506127920.191562:0:3044:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:25.0:1506127920.191566:0:3044:0:(sec.c:2099:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:25.0:1506127920.191568:0:3044:0:(service.c:1939:ptlrpc_server_handle_req_in()) got req x1579283258779360 00000020:00000001:25.0:1506127920.191570:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:25.0:1506127920.191571:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x2f3e107735f0c1c 00000020:00000001:25.0:1506127920.191573:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000040:25.0:1506127920.191575:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507800 : new refcount 7 00000020:00000001:25.0:1506127920.191577:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612166930888704 : -131906778662912 : ffff88080f507800) 00000020:00000001:25.0:1506127920.191579:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=18446612166930888704 : -131906778662912 : ffff88080f507800) 00000100:00000001:25.0:1506127920.191582:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127920.191583:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:25.0:1506127920.191586:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88082e038a00. 00000020:00000010:25.0:1506127920.191588:0:3044:0:(lu_ucred.c:49:lu_ucred_key_init()) kmalloced 'value': 104 at ffff88080f761680. 00000020:00000010:25.0:1506127920.191591:0:3044:0:(tgt_main.c:382:tgt_ses_key_init()) kmalloced 'value': 160 at ffff88080f8a9500. 00000100:00000001:25.0:1506127920.191593:0:3044:0:(service.c:1656:ptlrpc_server_request_add()) Process entered 00000100:00000001:25.0:1506127920.191594:0:3044:0:(service.c:1580:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:25.0:1506127920.191595:0:3044:0:(service.c:1607:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.191599:0:3044:0:(service.c:1695:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.191605:0:3044:0:(service.c:2003:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:25.0:1506127920.191608:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127920.191609:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127920.191611:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127920.191612:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127920.191613:0:3044:0:(service.c:2028:ptlrpc_server_handle_request()) Process entered 00000100:00000001:25.0:1506127920.191614:0:3044:0:(service.c:1809:ptlrpc_server_request_get()) Process entered 00000100:00100000:25.0:1506127920.191619:0:3044:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.1.108@o2ib, seq: 6 00000100:00000040:25.0:1506127920.191621:0:3044:0:(service.c:1840:ptlrpc_server_request_get()) RPC GETting export ffff88080f507800 : new rpc_count 2 00000100:00000001:25.0:1506127920.191623:0:3044:0:(service.c:1842:ptlrpc_server_request_get()) Process leaving (rc=18446612167001133136 : -131906708418480 : ffff880813805050) 00000100:00000040:25.0:1506127920.191627:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "New" -> "Interpret" req@ffff880813805050 x1579283258779360/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 304/0 e 0 to 0 dl 1506127926 ref 1 fl New:/0/ffffffff rc 0/-1 00000100:00000001:25.0:1506127920.191636:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127920.191637:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000100:00100000:25.0:1506127920.191640:0:3044:0:(service.c:2089:ptlrpc_server_handle_request()) Handling RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0000-mdtlov_UUID+7:4031:x1579283258779360:12345-192.168.1.108@o2ib:1000 00000100:00000200:25.0:1506127920.191643:0:3044:0:(service.c:2094:ptlrpc_server_handle_request()) got req 1579283258779360 00000020:00000001:25.0:1506127920.191645:0:3044:0:(tgt_handler.c:672:tgt_request_handle()) Process entered 00000020:00000001:25.0:1506127920.191647:0:3044:0:(tgt_handler.c:617:process_req_last_xid()) Process entered 00000020:00000001:25.0:1506127920.191648:0:3044:0:(tgt_handler.c:657:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.191650:0:3044:0:(tgt_handler.c:584:tgt_handler_find_check()) Process entered 00000020:00000001:25.0:1506127920.191651:0:3044:0:(tgt_handler.c:611:tgt_handler_find_check()) Process leaving (rc=18446744072654881248 : -1054670368 : ffffffffc12301e0) 00000020:00000001:25.0:1506127920.191653:0:3044:0:(tgt_handler.c:523:tgt_handle_recovery()) Process entered 00000020:00000040:25.0:1506127920.191656:0:3044:0:(tgt_handler.c:562:tgt_handle_recovery()) @@@ Got new replay req@ffff880813805050 x1579283258779360/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 304/0 e 0 to 0 dl 1506127926 ref 1 fl Interpret:/0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127920.191663:0:3044:0:(ldlm_lib.c:2705:target_queue_recovery_request()) Process entered 00010000:00080000:25.0:1506127920.191665:0:3044:0:(ldlm_lib.c:2788:target_queue_recovery_request()) @@@ not queueing req@ffff880813805050 x1579283258779360/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 304/0 e 0 to 0 dl 1506127926 ref 1 fl Interpret:/0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127920.191671:0:3044:0:(ldlm_lib.c:2789:target_queue_recovery_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127920.191672:0:3044:0:(tgt_handler.c:506:tgt_filter_recovery_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.191674:0:3044:0:(tgt_handler.c:573:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127920.191675:0:3044:0:(tgt_handler.c:382:tgt_handle_request0()) Process entered 00000020:00000001:25.0:1506127920.191677:0:3044:0:(tgt_handler.c:311:tgt_request_preprocess()) Process entered 00000020:00000001:25.0:1506127920.191678:0:3044:0:(tgt_handler.c:366:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.191680:0:3044:0:(out_handler.c:917:out_handle()) Process entered 00000020:00000010:25.0:1506127920.191683:0:3044:0:(out_handler.c:940:out_handle()) kmalloced 'update_bufs': 8 at ffff880825e00188. 00000100:00000001:25.0:1506127920.191685:0:3044:0:(pack_generic.c:334:lustre_pack_reply_v2()) Process entered 02000000:00000001:25.0:1506127920.191687:0:3044:0:(sec.c:2112:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:25.0:1506127920.191696:0:3044:0:(sec_null.c:318:null_alloc_rs()) kmalloced 'rs': 4576 at ffff88080f0a8000. 02000000:00000001:25.0:1506127920.191698:0:3044:0:(sec.c:2148:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.191700:0:3044:0:(pack_generic.c:368:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.191706:0:3044:0:(out_handler.c:205:out_attr_get()) Process entered 00000020:00000040:25.0:1506127920.191709:0:3044:0:(out_handler.c:233:out_attr_get()) soaked-MDT0003: insert attr get reply ffff88080f0a81e0 index 0: rc = 0 00000020:00000001:25.0:1506127920.191712:0:3044:0:(out_handler.c:238:out_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000002:25.0:1506127920.191714:0:3044:0:(lu_object.c:157:lu_object_put()) Add ffff88081f236970 to site lru. hash: ffff88082b7a7080, bkt: ffffc90015a74018, lru_len: 1 00000020:00000010:25.0:1506127920.191717:0:3044:0:(out_handler.c:1169:out_handle()) kfreed 'update_bufs': 8 at ffff880825e00188. 00000020:00000001:25.0:1506127920.191719:0:3044:0:(out_handler.c:1175:out_handle()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:25.0:1506127920.191721:0:3044:0:(ldlm_lib.c:2884:target_committed_to_req()) last_committed 0, transno 0, xid 1579283258779360 00010000:00000001:25.0:1506127920.191723:0:3044:0:(ldlm_lib.c:2948:target_send_reply()) Process entered 00010000:00000200:25.0:1506127920.191726:0:3044:0:(ldlm_lib.c:2936:target_send_reply_msg()) @@@ sending reply req@ffff880813805050 x1579283258779360/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 304/4320 e 0 to 0 dl 1506127926 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000001:25.0:1506127920.191733:0:3044:0:(ldlm_lib.c:2895:target_pack_pool_reply()) Process entered 00010000:00000001:25.0:1506127920.191734:0:3044:0:(ldlm_lib.c:2903:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:25.0:1506127920.191736:0:3044:0:(import.c:1678:at_measured()) add 1 to ffff88081629fd50 time=0 v=1 (1 0 0 0) 00000100:00000001:25.0:1506127920.191740:0:3044:0:(connection.c:127:ptlrpc_connection_addref()) Process entered 00000100:00000040:25.0:1506127920.191742:0:3044:0:(connection.c:132:ptlrpc_connection_addref()) conn=ffff88082ede1780 refcount 14 to 192.168.1.108@o2ib 00000100:00000001:25.0:1506127920.191744:0:3044:0:(connection.c:134:ptlrpc_connection_addref()) Process leaving (rc=18446612167460263808 : -131906249287808 : ffff88082ede1780) 02000000:00000001:25.0:1506127920.191746:0:3044:0:(sec.c:2161:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:25.0:1506127920.191747:0:3044:0:(sec.c:2172:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.191748:0:3044:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:25.0:1506127920.191751:0:3044:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-192.168.1.108@o2ib 00000400:00000010:25.0:1506127920.191753:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761780. 00000100:00000200:25.0:1506127920.191756:0:3044:0:(niobuf.c:85:ptl_send_buf()) Sending 4288 bytes to portal 4, xid 1579283258779360, offset 192 00000400:00000010:25.0:1506127920.191759:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88082e039200 (tot 194344940). 00000400:00000200:25.0:1506127920.191762:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.108@o2ib 00000400:00000200:25.0:1506127920.191774:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.108@o2ib(192.168.1.108@o2ib:192.168.1.108@o2ib) : PUT 00000800:00000200:25.0:1506127920.191778:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 4288 bytes in 1 frags to 12345-192.168.1.108@o2ib 00000800:00000200:25.0:1506127920.191788:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082cb07e00] -> 192.168.1.108@o2ib (2) version: 12 00000800:00000200:25.0:1506127920.191790:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082e03de00] (20)++ 00000800:00000200:25.0:1506127920.191792:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082e03de00] (21)++ 00000800:00000200:25.0:1506127920.191797:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082e03de00] (22)-- 00000100:00000001:25.0:1506127920.191799:0:3044:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.191800:0:3044:0:(connection.c:91:ptlrpc_connection_put()) Process entered 00000100:00000040:25.0:1506127920.191802:0:3044:0:(connection.c:119:ptlrpc_connection_put()) PUT conn=ffff88082ede1780 refcount 13 to 192.168.1.108@o2ib 00000100:00000001:25.0:1506127920.191804:0:3044:0:(connection.c:121:ptlrpc_connection_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127920.191805:0:3044:0:(ldlm_lib.c:2960:target_send_reply()) Process leaving 00000020:00000001:25.0:1506127920.191806:0:3044:0:(tgt_handler.c:475:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.191808:0:3044:0:(tgt_handler.c:780:tgt_request_handle()) Process leaving 00000100:00000040:25.0:1506127920.191810:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff880813805050 x1579283258779360/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 304/4288 e 0 to 0 dl 1506127926 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00100000:25.0:1506127920.191824:0:3044:0:(service.c:2135:ptlrpc_server_handle_request()) Handled RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0000-mdtlov_UUID+7:4031:x1579283258779360:12345-192.168.1.108@o2ib:1000 Request procesed in 186us (288us total) trans 0 rc 0/0 00000100:00100000:25.0:1506127920.191831:0:3044:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.1.108@o2ib, seq: 6 00000100:00000040:25.0:1506127920.191833:0:3044:0:(service.c:1034:ptlrpc_server_finish_active_request()) RPC PUTting export ffff88080f507800 : new rpc_count 1 00000100:00000001:25.0:1506127920.191835:0:3044:0:(service.c:1613:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:25.0:1506127920.191836:0:3044:0:(service.c:1624:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:25.0:1506127920.191838:0:3044:0:(lu_ucred.c:49:lu_ucred_key_fini()) kfreed 'info': 104 at ffff88080f761680. 00000020:00000010:25.0:1506127920.191840:0:3044:0:(tgt_main.c:382:tgt_ses_key_fini()) kfreed 'info': 160 at ffff88080f8a9500. 00000020:00000010:25.0:1506127920.191842:0:3044:0:(lu_object.c:1595:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff88082e038a00. 00000020:00000040:25.0:1506127920.191844:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507800 : new refcount 6 00000100:00000001:25.0:1506127920.191847:0:3044:0:(service.c:2155:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:25.0:1506127920.191849:0:3044:0:(niobuf.c:953:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000400:00000010:25.0:1506127920.191852:0:3044:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff8808138a1000. 00000400:00000010:25.0:1506127920.191854:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761680. 00000400:00000001:25.0:1506127920.191857:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127920.191858:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127920.192711:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127920.192714:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127920.192717:0:3044:0:(service.c:1859:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:25.0:1506127920.192719:0:3044:0:(sec.c:2052:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:25.0:1506127920.192721:0:3044:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:25.0:1506127920.192723:0:3044:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:25.0:1506127920.192727:0:3044:0:(sec.c:2099:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:25.0:1506127920.192730:0:3044:0:(service.c:1939:ptlrpc_server_handle_req_in()) got req x1579283258779424 00000020:00000001:25.0:1506127920.192733:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:25.0:1506127920.192734:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x2f3e107735f0c1c 00000020:00000001:25.0:1506127920.192737:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000040:25.0:1506127920.192738:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507800 : new refcount 7 00000020:00000001:25.0:1506127920.192740:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612166930888704 : -131906778662912 : ffff88080f507800) 00000020:00000001:25.0:1506127920.192742:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=18446612166930888704 : -131906778662912 : ffff88080f507800) 00000100:00000001:25.0:1506127920.192746:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127920.192747:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:25.0:1506127920.192749:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88082e038a00. 00000020:00000010:25.0:1506127920.192752:0:3044:0:(lu_ucred.c:49:lu_ucred_key_init()) kmalloced 'value': 104 at ffff88080f761800. 00000020:00000010:25.0:1506127920.192755:0:3044:0:(tgt_main.c:382:tgt_ses_key_init()) kmalloced 'value': 160 at ffff88080f8a9500. 00000100:00000001:25.0:1506127920.192758:0:3044:0:(service.c:1656:ptlrpc_server_request_add()) Process entered 00000100:00000001:25.0:1506127920.192759:0:3044:0:(service.c:1580:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:25.0:1506127920.192760:0:3044:0:(service.c:1607:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.192763:0:3044:0:(service.c:1695:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.192767:0:3044:0:(service.c:2003:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:25.0:1506127920.192770:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127920.192772:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127920.192773:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127920.192774:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127920.192776:0:3044:0:(service.c:2028:ptlrpc_server_handle_request()) Process entered 00000100:00000001:25.0:1506127920.192777:0:3044:0:(service.c:1809:ptlrpc_server_request_get()) Process entered 00000100:00100000:25.0:1506127920.192780:0:3044:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.1.108@o2ib, seq: 7 00000100:00000040:25.0:1506127920.192783:0:3044:0:(service.c:1840:ptlrpc_server_request_get()) RPC GETting export ffff88080f507800 : new rpc_count 2 00000100:00000001:25.0:1506127920.192784:0:3044:0:(service.c:1842:ptlrpc_server_request_get()) Process leaving (rc=18446612167001132112 : -131906708419504 : ffff880813804c50) 00000100:00000040:25.0:1506127920.192788:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "New" -> "Interpret" req@ffff880813804c50 x1579283258779424/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 632/0 e 0 to 0 dl 1506127926 ref 1 fl New:/0/ffffffff rc 0/-1 00000100:00000001:25.0:1506127920.192796:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127920.192797:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000100:00100000:25.0:1506127920.192800:0:3044:0:(service.c:2089:ptlrpc_server_handle_request()) Handling RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0000-mdtlov_UUID+7:4031:x1579283258779424:12345-192.168.1.108@o2ib:1000 00000100:00000200:25.0:1506127920.192803:0:3044:0:(service.c:2094:ptlrpc_server_handle_request()) got req 1579283258779424 00000020:00000001:25.0:1506127920.192804:0:3044:0:(tgt_handler.c:672:tgt_request_handle()) Process entered 00000020:00000001:25.0:1506127920.192807:0:3044:0:(tgt_handler.c:617:process_req_last_xid()) Process entered 00000020:00000001:25.0:1506127920.192808:0:3044:0:(tgt_handler.c:657:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.192810:0:3044:0:(tgt_handler.c:584:tgt_handler_find_check()) Process entered 00000020:00000001:25.0:1506127920.192811:0:3044:0:(tgt_handler.c:611:tgt_handler_find_check()) Process leaving (rc=18446744072654881248 : -1054670368 : ffffffffc12301e0) 00000020:00000001:25.0:1506127920.192813:0:3044:0:(tgt_handler.c:523:tgt_handle_recovery()) Process entered 00000020:00000040:25.0:1506127920.192816:0:3044:0:(tgt_handler.c:562:tgt_handle_recovery()) @@@ Got new replay req@ffff880813804c50 x1579283258779424/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 632/0 e 0 to 0 dl 1506127926 ref 1 fl Interpret:/0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127920.192822:0:3044:0:(ldlm_lib.c:2705:target_queue_recovery_request()) Process entered 00010000:00080000:25.0:1506127920.192825:0:3044:0:(ldlm_lib.c:2788:target_queue_recovery_request()) @@@ not queueing req@ffff880813804c50 x1579283258779424/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 632/0 e 0 to 0 dl 1506127926 ref 1 fl Interpret:/0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127920.192831:0:3044:0:(ldlm_lib.c:2789:target_queue_recovery_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127920.192833:0:3044:0:(tgt_handler.c:506:tgt_filter_recovery_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.192834:0:3044:0:(tgt_handler.c:573:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127920.192836:0:3044:0:(tgt_handler.c:382:tgt_handle_request0()) Process entered 00000020:00000001:25.0:1506127920.192837:0:3044:0:(tgt_handler.c:311:tgt_request_preprocess()) Process entered 00000020:00000001:25.0:1506127920.192838:0:3044:0:(tgt_handler.c:366:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.192840:0:3044:0:(out_handler.c:917:out_handle()) Process entered 00000020:00000010:25.0:1506127920.192843:0:3044:0:(out_handler.c:940:out_handle()) kmalloced 'update_bufs': 8 at ffff880825e00188. 00000100:00000001:25.0:1506127920.192846:0:3044:0:(pack_generic.c:334:lustre_pack_reply_v2()) Process entered 02000000:00000001:25.0:1506127920.192847:0:3044:0:(sec.c:2112:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:25.0:1506127920.192851:0:3044:0:(sec_null.c:318:null_alloc_rs()) kmalloced 'rs': 4576 at ffff88080f0aa000. 02000000:00000001:25.0:1506127920.192852:0:3044:0:(sec.c:2148:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.192854:0:3044:0:(pack_generic.c:368:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.192857:0:3044:0:(lu_object.c:243:lu_object_alloc()) Process entered 00000004:00000001:25.0:1506127920.192860:0:3044:0:(mdt_handler.c:5144:mdt_object_alloc()) Process entered 00000004:00000010:25.0:1506127920.192862:0:3044:0:(mdt_handler.c:5146:mdt_object_alloc()) slab-alloced 'mo': 216 at ffff8800ad2a56c8. 00000004:00000001:25.0:1506127920.192865:0:3044:0:(mdt_handler.c:5161:mdt_object_alloc()) Process leaving (rc=18446612135219451672 : -131938490099944 : ffff8800ad2a5718) 00000004:00000001:25.0:1506127920.192868:0:3044:0:(mdt_handler.c:5173:mdt_object_init()) Process entered 00000004:00000040:25.0:1506127920.192869:0:3044:0:(mdt_handler.c:5176:mdt_object_init()) object init, fid = [0x2c0003ab1:0x1:0x0] 00000004:00000010:25.0:1506127920.192872:0:3044:0:(mdd_object.c:120:mdd_object_alloc()) slab-alloced 'mdd_obj': 80 at ffff88042c24e230. 00000004:00000001:25.0:1506127920.192874:0:3044:0:(mdt_handler.c:5185:mdt_object_init()) Process leaving (rc=0 : 0 : 0) 00000004:00000001:25.0:1506127920.192876:0:3044:0:(mdd_object.c:143:mdd_object_init()) Process entered 00000004:00000001:25.0:1506127920.192878:0:3044:0:(lod_dev.c:210:lod_object_alloc()) Process entered 00000004:00000010:25.0:1506127920.192880:0:3044:0:(lod_dev.c:212:lod_object_alloc()) slab-alloced 'lod_obj': 104 at ffff88082a262f08. 00000004:00000001:25.0:1506127920.192881:0:3044:0:(lod_dev.c:221:lod_object_alloc()) Process leaving (rc=18446612167381102344 : -131906328449272 : ffff88082a262f08) 00000004:00000001:25.0:1506127920.192883:0:3044:0:(mdd_object.c:153:mdd_object_init()) Process leaving (rc=0 : 0 : 0) 00000004:00000001:25.0:1506127920.192885:0:3044:0:(lod_object.c:5108:lod_object_init()) Process entered 00000004:00000001:25.0:1506127920.192887:0:3044:0:(lod_dev.c:127:lod_fld_lookup()) Process entered 80000000:00000001:25.0:1506127920.192889:0:3044:0:(fld_handler.c:242:fld_server_lookup()) Process entered 80000000:00000001:25.0:1506127920.192891:0:3044:0:(fld_handler.c:207:fld_local_lookup()) Process entered 80000000:00000001:25.0:1506127920.192893:0:3044:0:(fld_cache.c:530:fld_cache_lookup()) Process entered 80000000:00000001:25.0:1506127920.192896:0:3044:0:(fld_cache.c:549:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:25.0:1506127920.192897:0:3044:0:(fld_handler.c:224:fld_local_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:25.0:1506127920.192899:0:3044:0:(fld_handler.c:246:fld_server_lookup()) Process leaving (rc=0 : 0 : 0) 00000004:00000040:25.0:1506127920.192901:0:3044:0:(lod_dev.c:168:lod_fld_lookup()) soaked-MDT0003-mdtlov: got tgt 3 for sequence: 0x2c0003ab1 00000004:00000001:25.0:1506127920.192903:0:3044:0:(lod_dev.c:170:lod_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00080000:00000010:25.0:1506127920.192905:0:3044:0:(osd_handler.c:332:osd_object_alloc()) kmalloced 'mo': 200 at ffff880822754000. 00000004:00000001:25.0:1506127920.192907:0:3044:0:(lod_object.c:5159:lod_object_init()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:25.0:1506127920.192911:0:3044:0:(osd_handler.c:994:osd_fid_lookup()) Process entered 00080000:00000001:25.0:1506127920.192913:0:3044:0:(osd_handler.c:1016:osd_fid_lookup()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:25.0:1506127920.192916:0:3044:0:(lu_object.c:295:lu_object_alloc()) Process leaving (rc=18446612135219451672 : -131938490099944 : ffff8800ad2a5718) 00080000:00000001:25.0:1506127920.192920:0:3044:0:(osd_handler.c:1612:osd_trans_create()) Process entered 00080000:00000010:25.0:1506127920.192921:0:3044:0:(osd_handler.c:1627:osd_trans_create()) kmalloced 'oh': 240 at ffff880822757d00. 00080000:00000001:25.0:1506127920.192923:0:3044:0:(osd_handler.c:1650:osd_trans_create()) Process leaving (rc=18446612167252081920 : -131906457469696 : ffff880822757d00) 00000020:00000001:25.0:1506127920.192926:0:3044:0:(out_handler.c:116:out_create()) Process entered 00080000:00000001:25.0:1506127920.192930:0:3044:0:(osd_handler.c:3101:osd_declare_create()) Process entered 00000001:00000001:25.0:1506127920.192932:0:3044:0:(osd_quota.c:630:osd_declare_inode_qid()) Process entered 00000001:00000001:25.0:1506127920.192933:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127920.192934:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127920.192936:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.192937:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.192938:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127920.192939:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127920.192940:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.192941:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.192943:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127920.192943:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127920.192944:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.192946:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.192947:0:3044:0:(osd_quota.c:677:osd_declare_inode_qid()) Process leaving (rc=0 : 0 : 0) 00080000:00000010:25.0:1506127920.192951:0:3044:0:(osd_handler.c:213:osd_idc_add()) kmalloced 'idc': 320 at ffff88082e03ca00. 00080000:00000001:25.0:1506127920.192953:0:3044:0:(osd_handler.c:3133:osd_declare_create()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:25.0:1506127920.192956:0:3044:0:(out_lib.c:432:tx_extend_args()) kmalloced 'new_ta': 64 at ffff880416fe3a00. 00000020:00000010:25.0:1506127920.192958:0:3044:0:(out_lib.c:441:tx_extend_args()) kmalloced 'new_ta[i]': 192 at ffff88080f8a8b40. 00000020:00000010:25.0:1506127920.192959:0:3044:0:(out_lib.c:441:tx_extend_args()) kmalloced 'new_ta[i]': 192 at ffff88080f8a8a80. 00000020:00000010:25.0:1506127920.192961:0:3044:0:(out_lib.c:441:tx_extend_args()) kmalloced 'new_ta[i]': 192 at ffff88080f8a89c0. 00000020:00000010:25.0:1506127920.192962:0:3044:0:(out_lib.c:441:tx_extend_args()) kmalloced 'new_ta[i]': 192 at ffff88080f8a8900. 00000020:00000010:25.0:1506127920.192964:0:3044:0:(out_lib.c:441:tx_extend_args()) kmalloced 'new_ta[i]': 192 at ffff88080f8a8840. 00000020:00000010:25.0:1506127920.192965:0:3044:0:(out_lib.c:441:tx_extend_args()) kmalloced 'new_ta[i]': 192 at ffff88080f8a8780. 00000020:00000010:25.0:1506127920.192967:0:3044:0:(out_lib.c:441:tx_extend_args()) kmalloced 'new_ta[i]': 192 at ffff88080f8a86c0. 00000020:00000010:25.0:1506127920.192968:0:3044:0:(out_lib.c:441:tx_extend_args()) kmalloced 'new_ta[i]': 192 at ffff88080f8a8600. 00000020:00000001:25.0:1506127920.192970:0:3044:0:(out_handler.c:155:out_create()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.192975:0:3044:0:(out_handler.c:473:out_index_insert()) Process entered 00080000:00000001:25.0:1506127920.192979:0:3044:0:(osd_handler.c:5333:osd_index_declare_ea_insert()) Process entered 00000001:00000001:25.0:1506127920.192982:0:3044:0:(osd_quota.c:630:osd_declare_inode_qid()) Process entered 00000001:00000001:25.0:1506127920.192982:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127920.192983:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127920.192984:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.192985:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.192987:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127920.192988:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127920.192988:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.192989:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.192991:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127920.192991:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127920.192992:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.192993:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.192995:0:3044:0:(osd_quota.c:677:osd_declare_inode_qid()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:25.0:1506127920.192996:0:3044:0:(osd_handler.c:5374:osd_index_declare_ea_insert()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.192998:0:3044:0:(out_handler.c:516:out_index_insert()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.193001:0:3044:0:(out_handler.c:829:out_tx_end()) Process entered 00080000:00000001:25.0:1506127920.193002:0:3044:0:(osd_handler.c:1713:osd_trans_start()) Process entered 00000001:00000001:25.0:1506127920.193005:0:3044:0:(osd_io.c:1576:osd_declare_write()) Process entered 00000001:00000001:25.0:1506127920.193007:0:3044:0:(osd_quota.c:630:osd_declare_inode_qid()) Process entered 00000001:00000001:25.0:1506127920.193007:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127920.193008:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127920.193009:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.193010:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.193011:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127920.193012:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127920.193013:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.193014:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.193015:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127920.193016:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127920.193017:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.193018:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.193019:0:3044:0:(osd_quota.c:677:osd_declare_inode_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.193021:0:3044:0:(osd_io.c:1650:osd_declare_write()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:25.0:1506127920.193025:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88082e03c400. 00000020:00000010:25.0:1506127920.193027:0:3044:0:(update_records.c:1217:update_key_init()) kmalloced 'value': 416 at ffff88082e03c800. 00000004:00000010:25.0:1506127920.193029:0:3044:0:(osp_dev.c:1869:osp_txn_key_init()) kmalloced 'value': 8 at ffff880825e00180. 00080000:00000001:25.0:1506127920.193031:0:3044:0:(osd_handler.c:1780:osd_trans_start()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:25.0:1506127920.193033:0:3044:0:(out_lib.c:538:out_tx_create_exec()) soaked-MDT0003-osd: create [0x2c0003ab1:0x1:0x0]: dof 0, mode 100000 00080000:00000001:25.0:1506127920.193038:0:3044:0:(osd_handler.c:3561:osd_create()) Process entered 00080000:00000001:25.0:1506127920.193116:0:3044:0:(osd_oi.c:545:fid_is_on_ost()) Process entered 80000000:00000001:25.0:1506127920.193118:0:3044:0:(fld_handler.c:207:fld_local_lookup()) Process entered 80000000:00000001:25.0:1506127920.193119:0:3044:0:(fld_cache.c:530:fld_cache_lookup()) Process entered 80000000:00000001:25.0:1506127920.193121:0:3044:0:(fld_cache.c:549:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:25.0:1506127920.193123:0:3044:0:(fld_handler.c:224:fld_local_lookup()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:25.0:1506127920.193124:0:3044:0:(osd_oi.c:583:fid_is_on_ost()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:25.0:1506127920.193126:0:3044:0:(osd_handler.c:3253:osd_ea_fid_set()) Process entered 00080000:00000001:25.0:1506127920.193148:0:3044:0:(osd_handler.c:3307:osd_ea_fid_set()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:25.0:1506127920.193150:0:3044:0:(osd_oi.c:545:fid_is_on_ost()) Process entered 80000000:00000001:25.0:1506127920.193150:0:3044:0:(fld_handler.c:207:fld_local_lookup()) Process entered 80000000:00000001:25.0:1506127920.193151:0:3044:0:(fld_cache.c:530:fld_cache_lookup()) Process entered 80000000:00000001:25.0:1506127920.193152:0:3044:0:(fld_cache.c:549:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:25.0:1506127920.193153:0:3044:0:(fld_handler.c:224:fld_local_lookup()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:25.0:1506127920.193155:0:3044:0:(osd_oi.c:583:fid_is_on_ost()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:25.0:1506127920.193157:0:3044:0:(osd_oi.c:647:osd_oi_iam_refresh()) Process entered 00000001:00000010:25.0:1506127920.193165:0:3044:0:(osd_dynlocks.c:109:dynlock_lock()) slab-alloced 'nhl': 72 at ffff88042c259870. 00000001:00000010:25.0:1506127920.193398:0:3044:0:(osd_dynlocks.c:193:dynlock_unlock()) slab-freed 'hl': 72 at ffff88042c259870. 00080000:00000001:25.0:1506127920.193402:0:3044:0:(osd_oi.c:668:osd_oi_iam_refresh()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:25.0:1506127920.193405:0:3044:0:(osd_handler.c:3614:osd_create()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:25.0:1506127920.193407:0:3044:0:(out_lib.c:547:out_tx_create_exec()) soaked-MDT0003-osd: insert create reply ffff88080f0aa1e0 index 0: rc = 0 00000020:00000040:25.0:1506127920.193409:0:3044:0:(out_handler.c:863:out_tx_end()) soaked-MDT0003-osd: executed 0/2: rc = 0 00000020:00000040:25.0:1506127920.193412:0:3044:0:(out_lib.c:1057:out_obj_index_insert()) soaked-MDT0003-osd: index insert [0x20000000a:0x3:0x0] name: [0x2c0003ab1:0x1:0x0] fid [0x2c0003ab1:0x1:0x0], type 32768 00080000:00000001:25.0:1506127920.193416:0:3044:0:(osd_handler.c:5402:osd_index_ea_insert()) Process entered 00080000:00000010:25.0:1506127920.193419:0:3044:0:(osd_handler.c:5465:osd_index_ea_insert()) kmalloced 'lii': 1072 at ffff8808138fc800. 00080000:00000001:25.0:1506127920.193437:0:3044:0:(osd_handler.c:4853:__osd_ea_add_rec()) Process leaving (rc=0 : 0 : 0) 00080000:00000002:25.0:1506127920.193440:0:3044:0:(osd_handler.c:5478:osd_index_ea_insert()) parent 78643201 insert [0x2c0003ab1:0x1:0x0]:184 rc = 0 00080000:00000001:25.0:1506127920.193443:0:3044:0:(osd_handler.c:5484:osd_index_ea_insert()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:25.0:1506127920.193444:0:3044:0:(out_lib.c:1104:out_tx_index_insert_exec()) soaked-MDT0003-osd: insert idx insert reply ffff88080f0aa1e0 index 1: rc = 0 00000020:00000040:25.0:1506127920.193446:0:3044:0:(out_handler.c:863:out_tx_end()) soaked-MDT0003-osd: executed 1/2: rc = 0 00080000:00000001:25.0:1506127920.193448:0:3044:0:(osd_handler.c:1834:osd_trans_stop()) Process entered 00000001:00000001:25.0:1506127920.193451:0:3044:0:(tgt_lastrcvd.c:1204:tgt_last_rcvd_update()) Process entered 00000001:00000002:25.0:1506127920.193453:0:3044:0:(tgt_lastrcvd.c:1245:tgt_last_rcvd_update()) transno = 30064771073, last_committed = 30064771072 00000001:00000010:25.0:1506127920.193457:0:3044:0:(tgt_lastrcvd.c:909:tgt_last_commit_cb_add()) kmalloced 'ccb': 96 at ffff8808138a1360. 00000001:00000040:25.0:1506127920.193459:0:3044:0:(tgt_lastrcvd.c:914:tgt_last_commit_cb_add()) callback GETting export ffff88080f507800 : new cb_count 2 00000020:00000040:25.0:1506127920.193461:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507800 : new refcount 8 00000001:00000001:25.0:1506127920.193466:0:3044:0:(tgt_lastrcvd.c:1387:tgt_last_rcvd_update()) Process leaving (rc=0 : 0 : 0) 00040000:00000001:25.0:1506127920.194570:0:3044:0:(qsd_handler.c:1080:qsd_op_end()) Process entered 00040000:00000001:25.0:1506127920.194573:0:3044:0:(qsd_handler.c:1092:qsd_op_end()) Process leaving 00080000:00000001:25.0:1506127920.194575:0:3044:0:(osd_handler.c:1905:osd_trans_stop()) Process leaving (rc=0 : 0 : 0) 00000020:00000002:25.0:1506127920.194594:0:3044:0:(lu_object.c:157:lu_object_put()) Add ffff8800ad2a56f0 to site lru. hash: ffff88082b7a7080, bkt: ffffc9001ae3e018, lru_len: 1 00000020:00000002:25.0:1506127920.194598:0:3044:0:(lu_object.c:157:lu_object_put()) Add ffff88081f236970 to site lru. hash: ffff88082b7a7080, bkt: ffffc90015a74018, lru_len: 1 00000020:00000001:25.0:1506127920.194600:0:3044:0:(out_handler.c:881:out_tx_end()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:25.0:1506127920.194602:0:3044:0:(out_handler.c:1169:out_handle()) kfreed 'update_bufs': 8 at ffff880825e00188. 00000020:00000001:25.0:1506127920.194604:0:3044:0:(out_handler.c:1175:out_handle()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:25.0:1506127920.194607:0:3044:0:(ldlm_lib.c:2884:target_committed_to_req()) last_committed 30064771073, transno 30064771073, xid 1579283258779424 00010000:00000001:25.0:1506127920.194609:0:3044:0:(ldlm_lib.c:2948:target_send_reply()) Process entered 00010000:00000200:25.0:1506127920.194614:0:3044:0:(ldlm_lib.c:2936:target_send_reply_msg()) @@@ sending reply req@ffff880813804c50 x1579283258779424/t30064771073(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 632/4320 e 0 to 0 dl 1506127926 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000001:25.0:1506127920.194623:0:3044:0:(ldlm_lib.c:2895:target_pack_pool_reply()) Process entered 00010000:00000001:25.0:1506127920.194624:0:3044:0:(ldlm_lib.c:2903:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:25.0:1506127920.194626:0:3044:0:(import.c:1678:at_measured()) add 1 to ffff88081629fd50 time=0 v=1 (1 0 0 0) 00000100:00000001:25.0:1506127920.194629:0:3044:0:(connection.c:127:ptlrpc_connection_addref()) Process entered 00000100:00000040:25.0:1506127920.194632:0:3044:0:(connection.c:132:ptlrpc_connection_addref()) conn=ffff88082ede1780 refcount 14 to 192.168.1.108@o2ib 00000100:00000001:25.0:1506127920.194633:0:3044:0:(connection.c:134:ptlrpc_connection_addref()) Process leaving (rc=18446612167460263808 : -131906249287808 : ffff88082ede1780) 02000000:00000001:25.0:1506127920.194636:0:3044:0:(sec.c:2161:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:25.0:1506127920.194637:0:3044:0:(sec.c:2172:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.194639:0:3044:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:25.0:1506127920.194641:0:3044:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-192.168.1.108@o2ib 00000400:00000010:25.0:1506127920.194643:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761880. 00000100:00000200:25.0:1506127920.194646:0:3044:0:(niobuf.c:85:ptl_send_buf()) Sending 4288 bytes to portal 4, xid 1579283258779424, offset 192 00000400:00000010:25.0:1506127920.194649:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88082e03b400 (tot 194349356). 00000400:00000200:25.0:1506127920.194652:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.108@o2ib 00000400:00000200:25.0:1506127920.194662:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.108@o2ib(192.168.1.108@o2ib:192.168.1.108@o2ib) : PUT 00000800:00000200:25.0:1506127920.194667:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 4288 bytes in 1 frags to 12345-192.168.1.108@o2ib 00000800:00000200:25.0:1506127920.194676:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082cb07e00] -> 192.168.1.108@o2ib (2) version: 12 00000800:00000200:25.0:1506127920.194678:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082e03de00] (20)++ 00000800:00000200:25.0:1506127920.194680:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082e03de00] (21)++ 00000800:00000200:25.0:1506127920.194685:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082e03de00] (22)-- 00000100:00000001:25.0:1506127920.194686:0:3044:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.194688:0:3044:0:(connection.c:91:ptlrpc_connection_put()) Process entered 00000100:00000040:25.0:1506127920.194689:0:3044:0:(connection.c:119:ptlrpc_connection_put()) PUT conn=ffff88082ede1780 refcount 13 to 192.168.1.108@o2ib 00000100:00000001:25.0:1506127920.194691:0:3044:0:(connection.c:121:ptlrpc_connection_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127920.194693:0:3044:0:(ldlm_lib.c:2960:target_send_reply()) Process leaving 00000020:00000001:25.0:1506127920.194693:0:3044:0:(tgt_handler.c:475:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.194695:0:3044:0:(tgt_handler.c:780:tgt_request_handle()) Process leaving 00000100:00000040:25.0:1506127920.194698:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff880813804c50 x1579283258779424/t30064771073(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 632/4288 e 0 to 0 dl 1506127926 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00100000:25.0:1506127920.194706:0:3044:0:(service.c:2135:ptlrpc_server_handle_request()) Handled RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0000-mdtlov_UUID+6:4031:x1579283258779424:12345-192.168.1.108@o2ib:1000 Request procesed in 1908us (2012us total) trans 30064771073 rc 0/0 00000100:00100000:25.0:1506127920.194713:0:3044:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.1.108@o2ib, seq: 7 00000100:00000040:25.0:1506127920.194715:0:3044:0:(service.c:1034:ptlrpc_server_finish_active_request()) RPC PUTting export ffff88080f507800 : new rpc_count 1 00000100:00000001:25.0:1506127920.194717:0:3044:0:(service.c:1613:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:25.0:1506127920.194718:0:3044:0:(service.c:1624:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:25.0:1506127920.194720:0:3044:0:(lu_ucred.c:49:lu_ucred_key_fini()) kfreed 'info': 104 at ffff88080f761800. 00000020:00000010:25.0:1506127920.194722:0:3044:0:(tgt_main.c:382:tgt_ses_key_fini()) kfreed 'info': 160 at ffff88080f8a9500. 00000020:00000010:25.0:1506127920.194724:0:3044:0:(lu_object.c:1595:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff88082e038a00. 00000020:00000040:25.0:1506127920.194726:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507800 : new refcount 5 00000100:00000001:25.0:1506127920.194729:0:3044:0:(service.c:2155:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:25.0:1506127920.194732:0:3044:0:(niobuf.c:953:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000400:00000010:25.0:1506127920.194734:0:3044:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff8808138a19c0. 00000400:00000010:25.0:1506127920.194736:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761800. 00000400:00000001:25.0:1506127920.194739:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127920.194740:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127920.195728:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127920.195731:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127920.195733:0:3044:0:(service.c:1859:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:25.0:1506127920.195735:0:3044:0:(sec.c:2052:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:25.0:1506127920.195736:0:3044:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:25.0:1506127920.195738:0:3044:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:25.0:1506127920.195741:0:3044:0:(sec.c:2099:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:25.0:1506127920.195744:0:3044:0:(service.c:1939:ptlrpc_server_handle_req_in()) got req x1579283258779456 00000020:00000001:25.0:1506127920.195746:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:25.0:1506127920.195747:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x2f3e107735f0c1c 00000020:00000001:25.0:1506127920.195749:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000040:25.0:1506127920.195770:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507800 : new refcount 6 00000020:00000001:25.0:1506127920.195771:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612166930888704 : -131906778662912 : ffff88080f507800) 00000020:00000001:25.0:1506127920.195774:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=18446612166930888704 : -131906778662912 : ffff88080f507800) 00000100:00000001:25.0:1506127920.195777:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127920.195778:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:25.0:1506127920.195780:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88082e038a00. 00000020:00000010:25.0:1506127920.195782:0:3044:0:(lu_ucred.c:49:lu_ucred_key_init()) kmalloced 'value': 104 at ffff88080f761900. 00000020:00000010:25.0:1506127920.195785:0:3044:0:(tgt_main.c:382:tgt_ses_key_init()) kmalloced 'value': 160 at ffff88080f8a9500. 00000100:00000001:25.0:1506127920.195787:0:3044:0:(service.c:1656:ptlrpc_server_request_add()) Process entered 00000100:00000001:25.0:1506127920.195788:0:3044:0:(service.c:1580:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:25.0:1506127920.195789:0:3044:0:(service.c:1607:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.195792:0:3044:0:(service.c:1695:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.195798:0:3044:0:(service.c:2003:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:25.0:1506127920.195801:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127920.195802:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127920.195803:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127920.195804:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127920.195805:0:3044:0:(service.c:2028:ptlrpc_server_handle_request()) Process entered 00000100:00000001:25.0:1506127920.195807:0:3044:0:(service.c:1809:ptlrpc_server_request_get()) Process entered 00000100:00100000:25.0:1506127920.195811:0:3044:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.1.108@o2ib, seq: 8 00000100:00000040:25.0:1506127920.195813:0:3044:0:(service.c:1840:ptlrpc_server_request_get()) RPC GETting export ffff88080f507800 : new rpc_count 2 00000100:00000001:25.0:1506127920.195814:0:3044:0:(service.c:1842:ptlrpc_server_request_get()) Process leaving (rc=18446612167001131088 : -131906708420528 : ffff880813804850) 00000100:00000040:25.0:1506127920.195818:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "New" -> "Interpret" req@ffff880813804850 x1579283258779456/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 304/0 e 0 to 0 dl 1506127926 ref 1 fl New:/0/ffffffff rc 0/-1 00000100:00000001:25.0:1506127920.195826:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127920.195827:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000100:00100000:25.0:1506127920.195830:0:3044:0:(service.c:2089:ptlrpc_server_handle_request()) Handling RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0000-mdtlov_UUID+6:4031:x1579283258779456:12345-192.168.1.108@o2ib:1000 00000100:00000200:25.0:1506127920.195832:0:3044:0:(service.c:2094:ptlrpc_server_handle_request()) got req 1579283258779456 00000020:00000001:25.0:1506127920.195834:0:3044:0:(tgt_handler.c:672:tgt_request_handle()) Process entered 00000020:00000001:25.0:1506127920.195836:0:3044:0:(tgt_handler.c:617:process_req_last_xid()) Process entered 00000020:00000001:25.0:1506127920.195837:0:3044:0:(tgt_handler.c:657:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.195839:0:3044:0:(tgt_handler.c:584:tgt_handler_find_check()) Process entered 00000020:00000001:25.0:1506127920.195841:0:3044:0:(tgt_handler.c:611:tgt_handler_find_check()) Process leaving (rc=18446744072654881248 : -1054670368 : ffffffffc12301e0) 00000020:00000001:25.0:1506127920.195843:0:3044:0:(tgt_handler.c:523:tgt_handle_recovery()) Process entered 00000020:00000040:25.0:1506127920.195846:0:3044:0:(tgt_handler.c:562:tgt_handle_recovery()) @@@ Got new replay req@ffff880813804850 x1579283258779456/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 304/0 e 0 to 0 dl 1506127926 ref 1 fl Interpret:/0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127920.195852:0:3044:0:(ldlm_lib.c:2705:target_queue_recovery_request()) Process entered 00010000:00080000:25.0:1506127920.195854:0:3044:0:(ldlm_lib.c:2788:target_queue_recovery_request()) @@@ not queueing req@ffff880813804850 x1579283258779456/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 304/0 e 0 to 0 dl 1506127926 ref 1 fl Interpret:/0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127920.195859:0:3044:0:(ldlm_lib.c:2789:target_queue_recovery_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127920.195861:0:3044:0:(tgt_handler.c:506:tgt_filter_recovery_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.195862:0:3044:0:(tgt_handler.c:573:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127920.195863:0:3044:0:(tgt_handler.c:382:tgt_handle_request0()) Process entered 00000020:00000001:25.0:1506127920.195864:0:3044:0:(tgt_handler.c:311:tgt_request_preprocess()) Process entered 00000020:00000001:25.0:1506127920.195866:0:3044:0:(tgt_handler.c:366:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.195867:0:3044:0:(out_handler.c:917:out_handle()) Process entered 00000020:00000010:25.0:1506127920.195870:0:3044:0:(out_handler.c:940:out_handle()) kmalloced 'update_bufs': 8 at ffff880825e00188. 00000100:00000001:25.0:1506127920.195872:0:3044:0:(pack_generic.c:334:lustre_pack_reply_v2()) Process entered 02000000:00000001:25.0:1506127920.195873:0:3044:0:(sec.c:2112:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:25.0:1506127920.195877:0:3044:0:(sec_null.c:318:null_alloc_rs()) kmalloced 'rs': 4576 at ffff88080f0ac000. 02000000:00000001:25.0:1506127920.195878:0:3044:0:(sec.c:2148:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.195880:0:3044:0:(pack_generic.c:368:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.195885:0:3044:0:(out_handler.c:205:out_attr_get()) Process entered 00000020:00000040:25.0:1506127920.195888:0:3044:0:(out_handler.c:233:out_attr_get()) soaked-MDT0003: insert attr get reply ffff88080f0ac1e0 index 0: rc = 0 00000020:00000001:25.0:1506127920.195890:0:3044:0:(out_handler.c:238:out_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000002:25.0:1506127920.195892:0:3044:0:(lu_object.c:157:lu_object_put()) Add ffff8800ad2a56f0 to site lru. hash: ffff88082b7a7080, bkt: ffffc9001ae3e018, lru_len: 1 00000020:00000010:25.0:1506127920.195894:0:3044:0:(out_handler.c:1169:out_handle()) kfreed 'update_bufs': 8 at ffff880825e00188. 00000020:00000001:25.0:1506127920.195896:0:3044:0:(out_handler.c:1175:out_handle()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:25.0:1506127920.195898:0:3044:0:(ldlm_lib.c:2884:target_committed_to_req()) last_committed 30064771073, transno 0, xid 1579283258779456 00010000:00000001:25.0:1506127920.195900:0:3044:0:(ldlm_lib.c:2948:target_send_reply()) Process entered 00010000:00000200:25.0:1506127920.195902:0:3044:0:(ldlm_lib.c:2936:target_send_reply_msg()) @@@ sending reply req@ffff880813804850 x1579283258779456/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 304/4320 e 0 to 0 dl 1506127926 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000001:25.0:1506127920.195910:0:3044:0:(ldlm_lib.c:2895:target_pack_pool_reply()) Process entered 00010000:00000001:25.0:1506127920.195911:0:3044:0:(ldlm_lib.c:2903:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:25.0:1506127920.195913:0:3044:0:(import.c:1678:at_measured()) add 1 to ffff88081629fd50 time=0 v=1 (1 0 0 0) 00000100:00000001:25.0:1506127920.195916:0:3044:0:(connection.c:127:ptlrpc_connection_addref()) Process entered 00000100:00000040:25.0:1506127920.195918:0:3044:0:(connection.c:132:ptlrpc_connection_addref()) conn=ffff88082ede1780 refcount 14 to 192.168.1.108@o2ib 00000100:00000001:25.0:1506127920.195920:0:3044:0:(connection.c:134:ptlrpc_connection_addref()) Process leaving (rc=18446612167460263808 : -131906249287808 : ffff88082ede1780) 02000000:00000001:25.0:1506127920.195922:0:3044:0:(sec.c:2161:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:25.0:1506127920.195923:0:3044:0:(sec.c:2172:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.195924:0:3044:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:25.0:1506127920.195926:0:3044:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-192.168.1.108@o2ib 00000400:00000010:25.0:1506127920.195928:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761980. 00000100:00000200:25.0:1506127920.195931:0:3044:0:(niobuf.c:85:ptl_send_buf()) Sending 4288 bytes to portal 4, xid 1579283258779456, offset 192 00000400:00000010:25.0:1506127920.195933:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88082e03d200 (tot 194349356). 00000400:00000200:25.0:1506127920.195937:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.108@o2ib 00000400:00000200:25.0:1506127920.195947:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.108@o2ib(192.168.1.108@o2ib:192.168.1.108@o2ib) : PUT 00000800:00000200:25.0:1506127920.195951:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 4288 bytes in 1 frags to 12345-192.168.1.108@o2ib 00000800:00000200:25.0:1506127920.195960:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082cb07e00] -> 192.168.1.108@o2ib (2) version: 12 00000800:00000200:25.0:1506127920.195962:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082e03de00] (20)++ 00000800:00000200:25.0:1506127920.195964:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082e03de00] (21)++ 00000800:00000200:25.0:1506127920.195968:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082e03de00] (22)-- 00000100:00000001:25.0:1506127920.195970:0:3044:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.195971:0:3044:0:(connection.c:91:ptlrpc_connection_put()) Process entered 00000100:00000040:25.0:1506127920.195973:0:3044:0:(connection.c:119:ptlrpc_connection_put()) PUT conn=ffff88082ede1780 refcount 13 to 192.168.1.108@o2ib 00000100:00000001:25.0:1506127920.195975:0:3044:0:(connection.c:121:ptlrpc_connection_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127920.195976:0:3044:0:(ldlm_lib.c:2960:target_send_reply()) Process leaving 00000020:00000001:25.0:1506127920.195977:0:3044:0:(tgt_handler.c:475:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.195978:0:3044:0:(tgt_handler.c:780:tgt_request_handle()) Process leaving 00000100:00000040:25.0:1506127920.195981:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff880813804850 x1579283258779456/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 304/4288 e 0 to 0 dl 1506127926 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00100000:25.0:1506127920.195988:0:3044:0:(service.c:2135:ptlrpc_server_handle_request()) Handled RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0000-mdtlov_UUID+6:4031:x1579283258779456:12345-192.168.1.108@o2ib:1000 Request procesed in 160us (277us total) trans 0 rc 0/0 00000100:00100000:25.0:1506127920.195994:0:3044:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.1.108@o2ib, seq: 8 00000100:00000040:25.0:1506127920.195996:0:3044:0:(service.c:1034:ptlrpc_server_finish_active_request()) RPC PUTting export ffff88080f507800 : new rpc_count 1 00000100:00000001:25.0:1506127920.195998:0:3044:0:(service.c:1613:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:25.0:1506127920.195999:0:3044:0:(service.c:1624:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:25.0:1506127920.196001:0:3044:0:(lu_ucred.c:49:lu_ucred_key_fini()) kfreed 'info': 104 at ffff88080f761900. 00000020:00000010:25.0:1506127920.196003:0:3044:0:(tgt_main.c:382:tgt_ses_key_fini()) kfreed 'info': 160 at ffff88080f8a9500. 00000020:00000010:25.0:1506127920.196005:0:3044:0:(lu_object.c:1595:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff88082e038a00. 00000020:00000040:25.0:1506127920.196007:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507800 : new refcount 5 00000100:00000001:25.0:1506127920.196010:0:3044:0:(service.c:2155:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:25.0:1506127920.196012:0:3044:0:(niobuf.c:953:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000400:00000010:25.0:1506127920.196014:0:3044:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff8808138a1c00. 00000400:00000010:25.0:1506127920.196016:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761900. 00000400:00000001:25.0:1506127920.196019:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127920.196020:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127920.196945:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127920.196948:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127920.196950:0:3044:0:(service.c:1859:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:25.0:1506127920.196952:0:3044:0:(sec.c:2052:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:25.0:1506127920.196954:0:3044:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:25.0:1506127920.196956:0:3044:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:25.0:1506127920.196960:0:3044:0:(sec.c:2099:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:25.0:1506127920.196964:0:3044:0:(service.c:1939:ptlrpc_server_handle_req_in()) got req x1579283258779504 00000020:00000001:25.0:1506127920.196966:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:25.0:1506127920.196968:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x2f3e107735f0c1c 00000020:00000001:25.0:1506127920.196983:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000040:25.0:1506127920.196985:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507800 : new refcount 6 00000020:00000001:25.0:1506127920.196986:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612166930888704 : -131906778662912 : ffff88080f507800) 00000020:00000001:25.0:1506127920.196989:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=18446612166930888704 : -131906778662912 : ffff88080f507800) 00000100:00000001:25.0:1506127920.196991:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127920.196993:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:25.0:1506127920.196995:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88082e038a00. 00000020:00000010:25.0:1506127920.196997:0:3044:0:(lu_ucred.c:49:lu_ucred_key_init()) kmalloced 'value': 104 at ffff88080f761a00. 00000020:00000010:25.0:1506127920.196999:0:3044:0:(tgt_main.c:382:tgt_ses_key_init()) kmalloced 'value': 160 at ffff88080f8a9500. 00000100:00000001:25.0:1506127920.197002:0:3044:0:(service.c:1656:ptlrpc_server_request_add()) Process entered 00000100:00000001:25.0:1506127920.197003:0:3044:0:(service.c:1580:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:25.0:1506127920.197004:0:3044:0:(service.c:1607:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.197007:0:3044:0:(service.c:1695:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.197011:0:3044:0:(service.c:2003:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:25.0:1506127920.197014:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127920.197015:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127920.197017:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127920.197018:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127920.197019:0:3044:0:(service.c:2028:ptlrpc_server_handle_request()) Process entered 00000100:00000001:25.0:1506127920.197020:0:3044:0:(service.c:1809:ptlrpc_server_request_get()) Process entered 00000100:00100000:25.0:1506127920.197023:0:3044:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.1.108@o2ib, seq: 9 00000100:00000040:25.0:1506127920.197025:0:3044:0:(service.c:1840:ptlrpc_server_request_get()) RPC GETting export ffff88080f507800 : new rpc_count 2 00000100:00000001:25.0:1506127920.197027:0:3044:0:(service.c:1842:ptlrpc_server_request_get()) Process leaving (rc=18446612167001130064 : -131906708421552 : ffff880813804450) 00000100:00000040:25.0:1506127920.197030:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "New" -> "Interpret" req@ffff880813804450 x1579283258779504/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 360/0 e 0 to 0 dl 1506127926 ref 1 fl New:/0/ffffffff rc 0/-1 00000100:00000001:25.0:1506127920.197038:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127920.197039:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000100:00100000:25.0:1506127920.197041:0:3044:0:(service.c:2089:ptlrpc_server_handle_request()) Handling RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0000-mdtlov_UUID+6:3974:x1579283258779504:12345-192.168.1.108@o2ib:1000 00000100:00000200:25.0:1506127920.197044:0:3044:0:(service.c:2094:ptlrpc_server_handle_request()) got req 1579283258779504 00000020:00000001:25.0:1506127920.197045:0:3044:0:(tgt_handler.c:672:tgt_request_handle()) Process entered 00000020:00000001:25.0:1506127920.197047:0:3044:0:(tgt_handler.c:617:process_req_last_xid()) Process entered 00000020:00000001:25.0:1506127920.197049:0:3044:0:(tgt_handler.c:657:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.197050:0:3044:0:(tgt_handler.c:584:tgt_handler_find_check()) Process entered 00000020:00000001:25.0:1506127920.197052:0:3044:0:(tgt_handler.c:611:tgt_handler_find_check()) Process leaving (rc=18446744072654881248 : -1054670368 : ffffffffc12301e0) 00000020:00000001:25.0:1506127920.197054:0:3044:0:(tgt_handler.c:523:tgt_handle_recovery()) Process entered 00000020:00000040:25.0:1506127920.197056:0:3044:0:(tgt_handler.c:562:tgt_handle_recovery()) @@@ Got new replay req@ffff880813804450 x1579283258779504/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 360/0 e 0 to 0 dl 1506127926 ref 1 fl Interpret:/0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127920.197062:0:3044:0:(ldlm_lib.c:2705:target_queue_recovery_request()) Process entered 00010000:00080000:25.0:1506127920.197064:0:3044:0:(ldlm_lib.c:2788:target_queue_recovery_request()) @@@ not queueing req@ffff880813804450 x1579283258779504/t0(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 360/0 e 0 to 0 dl 1506127926 ref 1 fl Interpret:/0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127920.197070:0:3044:0:(ldlm_lib.c:2789:target_queue_recovery_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127920.197087:0:3044:0:(tgt_handler.c:506:tgt_filter_recovery_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.197089:0:3044:0:(tgt_handler.c:573:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127920.197090:0:3044:0:(tgt_handler.c:382:tgt_handle_request0()) Process entered 00000020:00000001:25.0:1506127920.197091:0:3044:0:(tgt_handler.c:311:tgt_request_preprocess()) Process entered 00000020:00000001:25.0:1506127920.197093:0:3044:0:(tgt_handler.c:366:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.197094:0:3044:0:(out_handler.c:917:out_handle()) Process entered 00000020:00000010:25.0:1506127920.197097:0:3044:0:(out_handler.c:940:out_handle()) kmalloced 'update_bufs': 8 at ffff880825e00188. 00000100:00000001:25.0:1506127920.197100:0:3044:0:(pack_generic.c:334:lustre_pack_reply_v2()) Process entered 02000000:00000001:25.0:1506127920.197101:0:3044:0:(sec.c:2112:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:25.0:1506127920.197105:0:3044:0:(sec_null.c:318:null_alloc_rs()) kmalloced 'rs': 4576 at ffff88080f0ae000. 02000000:00000001:25.0:1506127920.197106:0:3044:0:(sec.c:2148:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.197108:0:3044:0:(pack_generic.c:368:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:25.0:1506127920.197114:0:3044:0:(osd_handler.c:1612:osd_trans_create()) Process entered 00080000:00000010:25.0:1506127920.197116:0:3044:0:(osd_handler.c:1627:osd_trans_create()) kmalloced 'oh': 240 at ffff880822756700. 00080000:00000001:25.0:1506127920.197118:0:3044:0:(osd_handler.c:1650:osd_trans_create()) Process leaving (rc=18446612167252076288 : -131906457475328 : ffff880822756700) 00000020:00000001:25.0:1506127920.197120:0:3044:0:(out_handler.c:583:out_write()) Process entered 00000001:00000001:25.0:1506127920.197122:0:3044:0:(osd_io.c:1576:osd_declare_write()) Process entered 00000001:00000001:25.0:1506127920.197123:0:3044:0:(osd_quota.c:630:osd_declare_inode_qid()) Process entered 00000001:00000001:25.0:1506127920.197125:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127920.197126:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127920.197127:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.197128:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.197130:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127920.197131:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127920.197131:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.197132:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.197134:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127920.197134:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127920.197135:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.197136:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.197138:0:3044:0:(osd_quota.c:677:osd_declare_inode_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.197139:0:3044:0:(osd_io.c:1650:osd_declare_write()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.197141:0:3044:0:(out_handler.c:609:out_write()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.197143:0:3044:0:(out_handler.c:829:out_tx_end()) Process entered 00080000:00000001:25.0:1506127920.197144:0:3044:0:(osd_handler.c:1713:osd_trans_start()) Process entered 00000001:00000001:25.0:1506127920.197146:0:3044:0:(osd_io.c:1576:osd_declare_write()) Process entered 00000001:00000001:25.0:1506127920.197147:0:3044:0:(osd_quota.c:630:osd_declare_inode_qid()) Process entered 00000001:00000001:25.0:1506127920.197148:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127920.197148:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127920.197149:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.197150:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.197151:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127920.197152:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127920.197153:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.197154:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.197155:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127920.197156:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127920.197157:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.197158:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.197159:0:3044:0:(osd_quota.c:677:osd_declare_inode_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127920.197160:0:3044:0:(osd_io.c:1650:osd_declare_write()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:25.0:1506127920.197166:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88082e03b000. 00000020:00000010:25.0:1506127920.197168:0:3044:0:(update_records.c:1217:update_key_init()) kmalloced 'value': 416 at ffff88082e03ae00. 00000004:00000010:25.0:1506127920.197170:0:3044:0:(osp_dev.c:1869:osp_txn_key_init()) kmalloced 'value': 8 at ffff880825e00170. 00080000:00000001:25.0:1506127920.197172:0:3044:0:(osd_handler.c:1780:osd_trans_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:25.0:1506127920.197174:0:3044:0:(out_lib.c:681:out_tx_write_exec()) write [0x200000009:0x3:0x0] pos 0 buf ffff880812a00130, len 32 00000020:00000040:25.0:1506127920.197185:0:3044:0:(out_handler.c:863:out_tx_end()) soaked-MDT0003-osd: executed 0/1: rc = 0 00080000:00000001:25.0:1506127920.197187:0:3044:0:(osd_handler.c:1834:osd_trans_stop()) Process entered 00000004:00000040:25.0:1506127920.197189:0:3044:0:(mdt_recovery.c:87:mdt_txn_stop_cb()) More than one transaction 00000001:00000001:25.0:1506127920.197191:0:3044:0:(tgt_lastrcvd.c:1204:tgt_last_rcvd_update()) Process entered 00000001:00000002:25.0:1506127920.197192:0:3044:0:(tgt_lastrcvd.c:1245:tgt_last_rcvd_update()) transno = 30064771074, last_committed = 30064771073 00000001:00000010:25.0:1506127920.197194:0:3044:0:(tgt_lastrcvd.c:909:tgt_last_commit_cb_add()) kmalloced 'ccb': 96 at ffff8808138a1960. 00000001:00000040:25.0:1506127920.197196:0:3044:0:(tgt_lastrcvd.c:914:tgt_last_commit_cb_add()) callback GETting export ffff88080f507800 : new cb_count 1 00000020:00000040:25.0:1506127920.197198:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507800 : new refcount 7 00000001:00000001:25.0:1506127920.197202:0:3044:0:(tgt_lastrcvd.c:1387:tgt_last_rcvd_update()) Process leaving (rc=0 : 0 : 0) 00040000:00000001:25.0:1506127920.197940:0:3044:0:(qsd_handler.c:1080:qsd_op_end()) Process entered 00040000:00000001:25.0:1506127920.197941:0:3044:0:(qsd_handler.c:1092:qsd_op_end()) Process leaving 00080000:00000001:25.0:1506127920.197943:0:3044:0:(osd_handler.c:1905:osd_trans_stop()) Process leaving (rc=0 : 0 : 0) 00000020:00000002:25.0:1506127920.197946:0:3044:0:(lu_object.c:157:lu_object_put()) Add ffff88081f236f58 to site lru. hash: ffff88082b7a7080, bkt: ffffc900159f2018, lru_len: 1 00000020:00000001:25.0:1506127920.197948:0:3044:0:(out_handler.c:881:out_tx_end()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:25.0:1506127920.197950:0:3044:0:(out_handler.c:1169:out_handle()) kfreed 'update_bufs': 8 at ffff880825e00188. 00000020:00000001:25.0:1506127920.197952:0:3044:0:(out_handler.c:1175:out_handle()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:25.0:1506127920.197954:0:3044:0:(ldlm_lib.c:2884:target_committed_to_req()) last_committed 30064771074, transno 30064771074, xid 1579283258779504 00010000:00000001:25.0:1506127920.197969:0:3044:0:(ldlm_lib.c:2948:target_send_reply()) Process entered 00010000:00000200:25.0:1506127920.197973:0:3044:0:(ldlm_lib.c:2936:target_send_reply_msg()) @@@ sending reply req@ffff880813804450 x1579283258779504/t30064771074(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 360/4320 e 0 to 0 dl 1506127926 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000001:25.0:1506127920.197980:0:3044:0:(ldlm_lib.c:2895:target_pack_pool_reply()) Process entered 00010000:00000001:25.0:1506127920.197982:0:3044:0:(ldlm_lib.c:2903:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:25.0:1506127920.197983:0:3044:0:(import.c:1678:at_measured()) add 1 to ffff88081629fd50 time=0 v=1 (1 0 0 0) 00000100:00000001:25.0:1506127920.197987:0:3044:0:(connection.c:127:ptlrpc_connection_addref()) Process entered 00000100:00000040:25.0:1506127920.197989:0:3044:0:(connection.c:132:ptlrpc_connection_addref()) conn=ffff88082ede1780 refcount 14 to 192.168.1.108@o2ib 00000100:00000001:25.0:1506127920.197991:0:3044:0:(connection.c:134:ptlrpc_connection_addref()) Process leaving (rc=18446612167460263808 : -131906249287808 : ffff88082ede1780) 02000000:00000001:25.0:1506127920.197993:0:3044:0:(sec.c:2161:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:25.0:1506127920.197994:0:3044:0:(sec.c:2172:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.197996:0:3044:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:25.0:1506127920.197998:0:3044:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-192.168.1.108@o2ib 00000400:00000010:25.0:1506127920.197999:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761a80. 00000100:00000200:25.0:1506127920.198002:0:3044:0:(niobuf.c:85:ptl_send_buf()) Sending 4288 bytes to portal 4, xid 1579283258779504, offset 192 00000400:00000010:25.0:1506127920.198005:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88082e03b000 (tot 194349356). 00000400:00000200:25.0:1506127920.198008:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.108@o2ib 00000400:00000200:25.0:1506127920.198018:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.108@o2ib(192.168.1.108@o2ib:192.168.1.108@o2ib) : PUT 00000800:00000200:25.0:1506127920.198023:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 4288 bytes in 1 frags to 12345-192.168.1.108@o2ib 00000800:00000200:25.0:1506127920.198032:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082cb07e00] -> 192.168.1.108@o2ib (2) version: 12 00000800:00000200:25.0:1506127920.198034:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082e03de00] (20)++ 00000800:00000200:25.0:1506127920.198036:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082e03de00] (21)++ 00000800:00000200:25.0:1506127920.198040:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082e03de00] (22)-- 00000100:00000001:25.0:1506127920.198042:0:3044:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127920.198043:0:3044:0:(connection.c:91:ptlrpc_connection_put()) Process entered 00000100:00000040:25.0:1506127920.198045:0:3044:0:(connection.c:119:ptlrpc_connection_put()) PUT conn=ffff88082ede1780 refcount 13 to 192.168.1.108@o2ib 00000100:00000001:25.0:1506127920.198046:0:3044:0:(connection.c:121:ptlrpc_connection_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127920.198048:0:3044:0:(ldlm_lib.c:2960:target_send_reply()) Process leaving 00000020:00000001:25.0:1506127920.198049:0:3044:0:(tgt_handler.c:475:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127920.198050:0:3044:0:(tgt_handler.c:780:tgt_request_handle()) Process leaving 00000100:00000040:25.0:1506127920.198052:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff880813804450 x1579283258779504/t30064771074(0) o1000->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 360/4288 e 0 to 0 dl 1506127926 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00100000:25.0:1506127920.198060:0:3044:0:(service.c:2135:ptlrpc_server_handle_request()) Handled RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0000-mdtlov_UUID+6:3974:x1579283258779504:12345-192.168.1.108@o2ib:1000 Request procesed in 1021us (1132us total) trans 30064771074 rc 0/0 00000100:00100000:25.0:1506127920.198066:0:3044:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.1.108@o2ib, seq: 9 00000100:00000040:25.0:1506127920.198069:0:3044:0:(service.c:1034:ptlrpc_server_finish_active_request()) RPC PUTting export ffff88080f507800 : new rpc_count 1 00000100:00000001:25.0:1506127920.198070:0:3044:0:(service.c:1613:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:25.0:1506127920.198082:0:3044:0:(service.c:1624:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:25.0:1506127920.198085:0:3044:0:(lu_ucred.c:49:lu_ucred_key_fini()) kfreed 'info': 104 at ffff88080f761a00. 00000020:00000010:25.0:1506127920.198088:0:3044:0:(tgt_main.c:382:tgt_ses_key_fini()) kfreed 'info': 160 at ffff88080f8a9500. 00000020:00000010:25.0:1506127920.198092:0:3044:0:(lu_object.c:1595:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff88082e038a00. 00000020:00000040:25.0:1506127920.198095:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507800 : new refcount 5 00000100:00000001:25.0:1506127920.198098:0:3044:0:(service.c:2155:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:25.0:1506127920.198102:0:3044:0:(niobuf.c:953:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000400:00000010:25.0:1506127920.198105:0:3044:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff8808138a13c0. 00000400:00000010:25.0:1506127920.198108:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761a00. 00000400:00000001:25.0:1506127920.198112:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127920.198113:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127921.174184:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127921.174187:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127921.174189:0:3044:0:(service.c:1437:ptlrpc_at_check_timed()) Process entered 00000100:00001000:25.0:1506127921.174191:0:3044:0:(service.c:1501:ptlrpc_at_check_timed()) timeout in +5s, asking for 30 secs on 1 early replies 00000100:00000001:25.0:1506127921.174194:0:3044:0:(service.c:1275:ptlrpc_at_send_early_reply()) Process entered 00000100:00001000:25.0:1506127921.174199:0:3044:0:(service.c:1288:ptlrpc_at_send_early_reply()) @@@ sending early reply (deadline +5s, margin +4s) for 1+30 req@ffff880813805c50 x1579283258779232/t0(0) o400->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:321/0 lens 224/0 e 0 to 0 dl 1506127926 ref 2 fl Complete:/c0/ffffffff rc 0/-1 00000100:00000010:25.0:1506127921.174209:0:3044:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 768 at ffff880819225d00. 00000100:00000010:25.0:1506127921.174212:0:3044:0:(service.c:1353:ptlrpc_at_send_early_reply()) kmalloced 'reqmsg': 224 at ffff880822757c00. 00000020:00000001:25.0:1506127921.174228:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:25.0:1506127921.174229:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x2f3e107735f0c1c 00000020:00000001:25.0:1506127921.174231:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000040:25.0:1506127921.174232:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507800 : new refcount 6 00000020:00000001:25.0:1506127921.174234:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612166930888704 : -131906778662912 : ffff88080f507800) 00000020:00000001:25.0:1506127921.174236:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=18446612166930888704 : -131906778662912 : ffff88080f507800) 00000100:00000040:25.0:1506127921.174238:0:3044:0:(service.c:1391:ptlrpc_at_send_early_reply()) RPC GETting export ffff88080f507800 : new rpc_count 2 00000100:00000001:25.0:1506127921.174240:0:3044:0:(pack_generic.c:334:lustre_pack_reply_v2()) Process entered 02000000:00000001:25.0:1506127921.174242:0:3044:0:(sec.c:2112:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:25.0:1506127921.174244:0:3044:0:(sec_null.c:318:null_alloc_rs()) kmalloced 'rs': 480 at ffff88082e038a00. 02000000:00000001:25.0:1506127921.174245:0:3044:0:(sec.c:2148:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.174247:0:3044:0:(pack_generic.c:368:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127921.174250:0:3044:0:(ldlm_lib.c:2895:target_pack_pool_reply()) Process entered 00010000:00000001:25.0:1506127921.174252:0:3044:0:(ldlm_lib.c:2903:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.174254:0:3044:0:(connection.c:127:ptlrpc_connection_addref()) Process entered 00000100:00000040:25.0:1506127921.174256:0:3044:0:(connection.c:132:ptlrpc_connection_addref()) conn=ffff88082ede1780 refcount 14 to 192.168.1.108@o2ib 00000100:00000001:25.0:1506127921.174258:0:3044:0:(connection.c:134:ptlrpc_connection_addref()) Process leaving (rc=18446612167460263808 : -131906249287808 : ffff88082ede1780) 02000000:00000001:25.0:1506127921.174261:0:3044:0:(sec.c:2161:sptlrpc_svc_wrap_reply()) Process entered 00000001:00000040:25.0:1506127921.174268:0:3044:0:(linux-crypto.c:118:cfs_crypto_hash_alloc()) Using crypto hash: crc32 (crc32-pclmul) speed 2143 MB/s 02000000:00000001:25.0:1506127921.174275:0:3044:0:(sec.c:2172:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.174277:0:3044:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:25.0:1506127921.174279:0:3044:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-192.168.1.108@o2ib 00000400:00000010:25.0:1506127921.174281:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761b80. 00000100:00000200:25.0:1506127921.174285:0:3044:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 4, xid 1579283258779232, offset 0 00000400:00000010:25.0:1506127921.174288:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88082e03ae00 (tot 194349356). 00000400:00000200:25.0:1506127921.174291:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.108@o2ib 00000400:00000200:25.0:1506127921.174303:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.108@o2ib(192.168.1.108@o2ib:192.168.1.108@o2ib) : PUT 00000800:00000200:25.0:1506127921.174307:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 192 bytes in 1 frags to 12345-192.168.1.108@o2ib 00000800:00000200:25.0:1506127921.174313:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082cb07e00] -> 192.168.1.108@o2ib (2) version: 12 00000800:00000200:25.0:1506127921.174315:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082e03de00] (20)++ 00000800:00000200:25.0:1506127921.174317:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082e03de00] (21)++ 00000800:00000200:25.0:1506127921.174322:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082e03de00] (22)-- 00000100:00000001:25.0:1506127921.174324:0:3044:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.174325:0:3044:0:(connection.c:91:ptlrpc_connection_put()) Process entered 00000100:00000040:25.0:1506127921.174327:0:3044:0:(connection.c:119:ptlrpc_connection_put()) PUT conn=ffff88082ede1780 refcount 13 to 192.168.1.108@o2ib 00000100:00000001:25.0:1506127921.174329:0:3044:0:(connection.c:121:ptlrpc_connection_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:25.0:1506127921.174331:0:3044:0:(service.c:1415:ptlrpc_at_send_early_reply()) RPC PUTting export ffff88080f507800 : new rpc_count 1 00000020:00000040:25.0:1506127921.174333:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507800 : new refcount 5 00000100:00000010:25.0:1506127921.174335:0:3044:0:(service.c:1419:ptlrpc_at_send_early_reply()) kfreed 'reqmsg': 224 at ffff880822757c00. 00000100:00000010:25.0:1506127921.174337:0:3044:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 768 at ffff880819225d00. 00000100:00000001:25.0:1506127921.174339:0:3044:0:(service.c:1422:ptlrpc_at_send_early_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:25.0:1506127921.174342:0:3044:0:(service.c:1184:ptlrpc_at_set_timer()) armed mdt_out at +25s 00000100:00000001:25.0:1506127921.174344:0:3044:0:(service.c:1527:ptlrpc_at_check_timed()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:25.0:1506127921.174346:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127921.174347:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127921.391987:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127921.392004:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127921.392006:0:3044:0:(service.c:1859:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:25.0:1506127921.392008:0:3044:0:(sec.c:2052:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:25.0:1506127921.392010:0:3044:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:25.0:1506127921.392011:0:3044:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:25.0:1506127921.392015:0:3044:0:(sec.c:2099:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:25.0:1506127921.392018:0:3044:0:(service.c:1939:ptlrpc_server_handle_req_in()) got req x1579288666076944 00000020:00000001:25.0:1506127921.392020:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:25.0:1506127921.392021:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x0 00000020:00000001:25.0:1506127921.392023:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000001:25.0:1506127921.392024:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.392026:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:25.0:1506127921.392029:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88082e03d200. 00000020:00000010:25.0:1506127921.392032:0:3044:0:(lu_ucred.c:49:lu_ucred_key_init()) kmalloced 'value': 104 at ffff88080f761b00. 00000020:00000010:25.0:1506127921.392034:0:3044:0:(tgt_main.c:382:tgt_ses_key_init()) kmalloced 'value': 160 at ffff88080f8a9500. 00000100:00000001:25.0:1506127921.392037:0:3044:0:(service.c:1656:ptlrpc_server_request_add()) Process entered 00000100:00000001:25.0:1506127921.392038:0:3044:0:(service.c:1580:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:25.0:1506127921.392039:0:3044:0:(service.c:1607:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.392042:0:3044:0:(service.c:1695:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.392049:0:3044:0:(service.c:2003:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:25.0:1506127921.392053:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127921.392054:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127921.392055:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127921.392056:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127921.392058:0:3044:0:(service.c:2028:ptlrpc_server_handle_request()) Process entered 00000100:00000001:25.0:1506127921.392059:0:3044:0:(service.c:1809:ptlrpc_server_request_get()) Process entered 00000100:00100000:25.0:1506127921.392063:0:3044:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.1.109@o2ib, seq: 10 00000100:00000001:25.0:1506127921.392066:0:3044:0:(service.c:1842:ptlrpc_server_request_get()) Process leaving (rc=18446612167001129040 : -131906708422576 : ffff880813804050) 00000100:00000040:25.0:1506127921.392070:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "New" -> "Interpret" req@ffff880813804050 x1579288666076944/t0(0) o38->@:0/0 lens 520/0 e 0 to 0 dl 1506128021 ref 1 fl New:/0/ffffffff rc 0/-1 00000100:00100000:25.0:1506127921.392101:0:3044:0:(service.c:2089:ptlrpc_server_handle_request()) Handling RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:0+-99:2127:x1579288666076944:12345-192.168.1.109@o2ib:38 00000100:00000200:25.0:1506127921.392104:0:3044:0:(service.c:2094:ptlrpc_server_handle_request()) got req 1579288666076944 00000020:00000001:25.0:1506127921.392119:0:3044:0:(tgt_handler.c:672:tgt_request_handle()) Process entered 00010000:00000001:25.0:1506127921.392121:0:3044:0:(ldlm_lib.c:966:target_handle_connect()) Process entered 00000020:00000040:25.0:1506127921.392125:0:3044:0:(obd_config.c:753:class_incref()) incref soaked-MDT0003 (ffff880826bd4d58) now 100 00000100:00000001:25.0:1506127921.392129:0:3044:0:(pack_generic.c:334:lustre_pack_reply_v2()) Process entered 02000000:00000001:25.0:1506127921.392130:0:3044:0:(sec.c:2112:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:25.0:1506127921.392132:0:3044:0:(sec_null.c:318:null_alloc_rs()) kmalloced 'rs': 672 at ffff88080f503800. 02000000:00000001:25.0:1506127921.392134:0:3044:0:(sec.c:2148:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.392136:0:3044:0:(pack_generic.c:368:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00010000:00080000:25.0:1506127921.392141:0:3044:0:(ldlm_lib.c:1239:target_handle_connect()) soaked-MDT0003: connection from soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib recovering/t0 exp (null) cur 1506127921 last 0 00010000:00000001:25.0:1506127921.392145:0:3044:0:(obd_class.h:825:obd_connect()) Process entered 00000004:00000001:25.0:1506127921.392146:0:3044:0:(mdt_handler.c:5578:mdt_obd_connect()) Process entered 00000020:00000001:25.0:1506127921.392148:0:3044:0:(genops.c:1167:class_connect()) Process entered 00000020:00000001:25.0:1506127921.392149:0:3044:0:(genops.c:842:class_new_export()) Process entered 00000020:00000010:25.0:1506127921.392150:0:3044:0:(genops.c:844:class_new_export()) kmalloced 'export': 968 at ffff88080f507400. 00000020:00000001:25.0:1506127921.392152:0:3044:0:(lustre_handles.c:64:class_handle_hash()) Process entered 00000020:00000040:25.0:1506127921.392153:0:3044:0:(lustre_handles.c:98:class_handle_hash()) added object ffff88080f507400 with handle 0x2f3e107735f0c2a to hash 00000020:00000001:25.0:1506127921.392155:0:3044:0:(lustre_handles.c:99:class_handle_hash()) Process leaving 00000020:00000001:25.0:1506127921.392156:0:3044:0:(obd_class.h:998:obd_init_export()) Process entered 00000004:00000001:25.0:1506127921.392157:0:3044:0:(mdt_handler.c:5681:mdt_init_export()) Process entered 00000001:00000001:25.0:1506127921.392158:0:3044:0:(tgt_lastrcvd.c:386:tgt_client_alloc()) Process entered 00000001:00000010:25.0:1506127921.392160:0:3044:0:(tgt_lastrcvd.c:392:tgt_client_alloc()) kmalloced 'exp->u.eu_target_data.ted_lcd': 128 at ffff88080f761c00. 00000001:00000001:25.0:1506127921.392161:0:3044:0:(tgt_lastrcvd.c:399:tgt_client_alloc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127921.392163:0:3044:0:(ldlm_lockd.c:2889:ldlm_init_export()) Process entered 00000001:00000001:25.0:1506127921.392164:0:3044:0:(hash.c:1005:cfs_hash_create()) Process entered 00000001:00000010:25.0:1506127921.392165:0:3044:0:(hash.c:1032:cfs_hash_create()) alloc '(hs)': 144 at ffff88080f8a8540 (tot 194349100). 00000001:00000010:25.0:1506127921.392181:0:3044:0:(hash.c:893:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at ffff880822a87b80 (tot 194349132). 00000001:00000010:25.0:1506127921.392183:0:3044:0:(hash.c:906:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 280 at ffff88082e03b400 (tot 194349412). 00000001:00000010:25.0:1506127921.392186:0:3044:0:(hash.c:906:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 280 at ffff88082e03f400 (tot 194349692). 00000001:00000010:25.0:1506127921.392188:0:3044:0:(hash.c:906:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 280 at ffff88082e03ee00 (tot 194349972). 00000001:00000010:25.0:1506127921.392190:0:3044:0:(hash.c:906:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 280 at ffff88082e03c400 (tot 194350252). 00000001:00000001:25.0:1506127921.392192:0:3044:0:(hash.c:1005:cfs_hash_create()) Process entered 00000001:00000010:25.0:1506127921.392193:0:3044:0:(hash.c:1032:cfs_hash_create()) alloc '(hs)': 144 at ffff88080f8a8480 (tot 194350396). 00000001:00000010:25.0:1506127921.392195:0:3044:0:(hash.c:893:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at ffff880822a87ba0 (tot 194350428). 00000001:00000010:25.0:1506127921.392197:0:3044:0:(hash.c:906:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 280 at ffff88082e03c800 (tot 194350708). 00000001:00000010:25.0:1506127921.392199:0:3044:0:(hash.c:906:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 280 at ffff88082e039200 (tot 194350988). 00000001:00000010:25.0:1506127921.392201:0:3044:0:(hash.c:906:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 280 at ffff88082e039c00 (tot 194351268). 00000001:00000010:25.0:1506127921.392203:0:3044:0:(hash.c:906:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 280 at ffff88082e039000 (tot 194351548). 00010000:00000001:25.0:1506127921.392205:0:3044:0:(ldlm_flock.c:939:ldlm_init_flock_export()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127921.392207:0:3044:0:(ldlm_lockd.c:2908:ldlm_init_export()) Process leaving (rc=0 : 0 : 0) 00000004:00000001:25.0:1506127921.392208:0:3044:0:(mdt_handler.c:5702:mdt_init_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.392210:0:3044:0:(obd_class.h:1002:obd_init_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:25.0:1506127921.392213:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507400 : new refcount 3 00000020:00000040:25.0:1506127921.392215:0:3044:0:(obd_config.c:753:class_incref()) incref soaked-MDT0003 (ffff880826bd4d58) now 101 00000020:00000001:25.0:1506127921.392217:0:3044:0:(genops.c:917:class_new_export()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000020:00000040:25.0:1506127921.392219:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507400 : new refcount 2 00000020:00000080:25.0:1506127921.392221:0:3044:0:(genops.c:1177:class_connect()) connect: client soaked-MDT0001-mdtlov_UUID, cookie 0x2f3e107735f0c2a 00000020:00000001:25.0:1506127921.392222:0:3044:0:(genops.c:1178:class_connect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.392223:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:25.0:1506127921.392224:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x2f3e107735f0c2a 00000020:00000001:25.0:1506127921.392225:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000040:25.0:1506127921.392227:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507400 : new refcount 3 00000020:00000001:25.0:1506127921.392228:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000020:00000001:25.0:1506127921.392230:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000001:00000001:25.0:1506127921.392232:0:3044:0:(nodemap_handler.c:385:nodemap_add_member()) Process entered 00000001:00000001:25.0:1506127921.392233:0:3044:0:(nodemap_handler.c:264:nodemap_classify_nid()) Process entered 00000001:00000001:25.0:1506127921.392235:0:3044:0:(interval_tree.c:629:interval_search()) Process entered 00000001:00000001:25.0:1506127921.392236:0:3044:0:(interval_tree.c:676:interval_search()) Process leaving (rc=1 : 1 : 1) 00000001:00000040:25.0:1506127921.392238:0:3044:0:(nodemap_handler.c:97:nodemap_getref()) GETting nodemap default(p=ffff880813872000) : new refcount 51 00000001:00000001:25.0:1506127921.392240:0:3044:0:(nodemap_handler.c:290:nodemap_classify_nid()) Process leaving (rc=18446612167001579520 : -131906707972096 : ffff880813872000) 00000001:00000001:25.0:1506127921.392242:0:3044:0:(nodemap_member.c:99:nm_member_add()) Process entered 00000020:00000040:25.0:1506127921.392243:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507400 : new refcount 4 00000001:00000040:25.0:1506127921.392244:0:3044:0:(nodemap_handler.c:97:nodemap_getref()) GETting nodemap default(p=ffff880813872000) : new refcount 52 00000001:00000001:25.0:1506127921.392246:0:3044:0:(nodemap_member.c:136:nm_member_add()) Process leaving (rc=0 : 0 : 0) 00000001:00000040:25.0:1506127921.392248:0:3044:0:(nodemap_handler.c:113:nodemap_putref()) PUTting nodemap default(p=ffff880813872000) : new refcount 51 00000001:00000001:25.0:1506127921.392249:0:3044:0:(nodemap_handler.c:406:nodemap_add_member()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.392252:0:3044:0:(tgt_lastrcvd.c:949:tgt_client_new()) Process entered 00000001:00000040:25.0:1506127921.392253:0:3044:0:(tgt_lastrcvd.c:1008:tgt_client_new()) soaked-MDT0003: new client at index 1 (8320) with UUID 'soaked-MDT0001-mdtlov_UUID' generation 0 00000001:00000001:25.0:1506127921.392256:0:3044:0:(tgt_lastrcvd.c:576:tgt_client_data_update()) Process entered 00080000:00000001:25.0:1506127921.392258:0:3044:0:(osd_handler.c:1612:osd_trans_create()) Process entered 00080000:00000010:25.0:1506127921.392260:0:3044:0:(osd_handler.c:1627:osd_trans_create()) kmalloced 'oh': 240 at ffff880822757c00. 00080000:00000001:25.0:1506127921.392262:0:3044:0:(osd_handler.c:1650:osd_trans_create()) Process leaving (rc=18446612167252081664 : -131906457469952 : ffff880822757c00) 00000001:00000001:25.0:1506127921.392264:0:3044:0:(osd_io.c:1576:osd_declare_write()) Process entered 00000001:00000001:25.0:1506127921.392266:0:3044:0:(osd_quota.c:630:osd_declare_inode_qid()) Process entered 00000001:00000001:25.0:1506127921.392267:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127921.392269:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127921.392270:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.392272:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.392273:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127921.392274:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127921.392275:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.392276:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.392277:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127921.392278:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127921.392279:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.392280:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.392281:0:3044:0:(osd_quota.c:677:osd_declare_inode_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.392283:0:3044:0:(osd_io.c:1650:osd_declare_write()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:25.0:1506127921.392285:0:3044:0:(osd_handler.c:1713:osd_trans_start()) Process entered 00000020:00000010:25.0:1506127921.392290:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88082e03e000. 00000020:00000010:25.0:1506127921.392292:0:3044:0:(update_records.c:1217:update_key_init()) kmalloced 'value': 416 at ffff88082e03e800. 00000004:00000010:25.0:1506127921.392294:0:3044:0:(osp_dev.c:1869:osp_txn_key_init()) kmalloced 'value': 8 at ffff880825e00188. 00080000:00000001:25.0:1506127921.392296:0:3044:0:(osd_handler.c:1780:osd_trans_start()) Process leaving (rc=0 : 0 : 0) 00000001:00000010:25.0:1506127921.392298:0:3044:0:(tgt_lastrcvd.c:545:tgt_new_client_cb_add()) kmalloced 'ccb': 80 at ffff8808138a1900. 00000001:00000040:25.0:1506127921.392300:0:3044:0:(tgt_lastrcvd.c:549:tgt_new_client_cb_add()) callback GETting export ffff88080f507400 : new cb_count 1 00000020:00000040:25.0:1506127921.392301:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507400 : new refcount 5 00000001:00000001:25.0:1506127921.392311:0:3044:0:(tgt_lastrcvd.c:625:tgt_client_data_update()) Process leaving 00080000:00000001:25.0:1506127921.392312:0:3044:0:(osd_handler.c:1834:osd_trans_stop()) Process entered 00040000:00000001:25.0:1506127921.392315:0:3044:0:(qsd_handler.c:1080:qsd_op_end()) Process entered 00040000:00000001:25.0:1506127921.392316:0:3044:0:(qsd_handler.c:1092:qsd_op_end()) Process leaving 00080000:00000001:25.0:1506127921.392317:0:3044:0:(osd_handler.c:1905:osd_trans_stop()) Process leaving (rc=0 : 0 : 0) 00000001:00000040:25.0:1506127921.392318:0:3044:0:(tgt_lastrcvd.c:630:tgt_client_data_update()) soaked-MDT0003: update last_rcvd client data for UUID = soaked-MDT0003_UUID, last_transno = 30064771072: rc = 0 00000001:00000001:25.0:1506127921.392321:0:3044:0:(tgt_lastrcvd.c:1018:tgt_client_new()) Process leaving (rc=0 : 0 : 0) 00000004:00000001:25.0:1506127921.392322:0:3044:0:(mdt_fs.c:68:mdt_export_stats_init()) Process entered 00000020:00000001:25.0:1506127921.392324:0:3044:0:(lprocfs_status_server.c:319:lprocfs_exp_setup()) Process entered 00000020:01000000:25.0:1506127921.392327:0:3044:0:(lprocfs_status_server.c:342:lprocfs_exp_setup()) using hash ffff880813836d80 00000020:00000010:25.0:1506127921.392328:0:3044:0:(lprocfs_status_server.c:344:lprocfs_exp_setup()) kmalloced 'new_stat': 80 at ffff8808138a1ba0. 00000020:00000040:25.0:1506127921.392331:0:3044:0:(lprocfs_status_server.c:356:lprocfs_exp_setup()) Found stats ffff8808138a1ba0 for nid 192.168.1.109@o2ib - ref 2 00000020:00000001:25.0:1506127921.392342:0:3044:0:(lprocfs_status_server.c:425:lprocfs_exp_setup()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:25.0:1506127921.392344:0:3044:0:(lprocfs_status.c:1293:lprocfs_alloc_stats()) alloc '(stats)': 32 at ffff880822a87bc0 (tot 194351580). 00000020:00000010:25.0:1506127921.392347:0:3044:0:(lprocfs_status.c:1303:lprocfs_alloc_stats()) alloc '(stats->ls_cnt_header)': 384 at ffff88082e03ec00 (tot 194351964). 00000020:00000010:25.0:1506127921.392349:0:3044:0:(lprocfs_status.c:1310:lprocfs_alloc_stats()) alloc '(stats->ls_percpu[0])': 640 at ffff88080f507c00 (tot 194352604). 00000020:00000010:25.0:1506127921.392354:0:3044:0:(lprocfs_status.c:1293:lprocfs_alloc_stats()) alloc '(stats)': 32 at ffff880822a87be0 (tot 194352636). 00000020:00000010:25.0:1506127921.392356:0:3044:0:(lprocfs_status.c:1303:lprocfs_alloc_stats()) alloc '(stats->ls_cnt_header)': 168 at ffff88080f8a8180 (tot 194352804). 00000020:00000010:25.0:1506127921.392358:0:3044:0:(lprocfs_status.c:1310:lprocfs_alloc_stats()) alloc '(stats->ls_percpu[0])': 280 at ffff88082e03b200 (tot 194353084). 00000020:00000001:25.0:1506127921.392361:0:3044:0:(lprocfs_status.c:2201:lprocfs_seq_create()) Process entered 00000020:00000001:25.0:1506127921.392363:0:3044:0:(lprocfs_status.c:2211:lprocfs_seq_create()) Process leaving (rc=0 : 0 : 0) 00000004:00000001:25.0:1506127921.392365:0:3044:0:(mdt_fs.c:103:mdt_export_stats_init()) Process leaving (rc=0 : 0 : 0) 00000004:00000001:25.0:1506127921.392366:0:3044:0:(mdt_handler.c:5647:mdt_obd_connect()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127921.392368:0:3044:0:(obd_class.h:835:obd_connect()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:25.0:1506127921.392370:0:3044:0:(genops.c:1056:class_new_import()) kmalloced 'imp': 1480 at ffff8808138fe800. 00000020:00000040:25.0:1506127921.392372:0:3044:0:(obd_config.c:753:class_incref()) incref soaked-MDT0003 (ffff880826bd4d58) now 102 00000020:00000001:25.0:1506127921.392374:0:3044:0:(lustre_handles.c:64:class_handle_hash()) Process entered 00000020:00000040:25.0:1506127921.392375:0:3044:0:(lustre_handles.c:98:class_handle_hash()) added object ffff8808138fe800 with handle 0x2f3e107735f0c31 to hash 00000020:00000001:25.0:1506127921.392377:0:3044:0:(lustre_handles.c:99:class_handle_hash()) Process leaving 00000020:00000001:25.0:1506127921.392378:0:3044:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:25.0:1506127921.392379:0:3044:0:(genops.c:1026:class_import_put()) import ffff8808138fe800 refcount=1 obd=soaked-MDT0003 00000020:00000001:25.0:1506127921.392381:0:3044:0:(genops.c:1035:class_import_put()) Process leaving 00000020:00000040:25.0:1506127921.392383:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507400 : new refcount 6 00000100:00000040:25.0:1506127921.392385:0:3044:0:(service.c:999:ptlrpc_request_change_export()) RPC GETting export ffff88080f507400 : new rpc_count 1 00000100:00000001:25.0:1506127921.392387:0:3044:0:(connection.c:48:ptlrpc_connection_get()) Process entered 00000400:00000200:25.0:1506127921.392392:0:3044:0:(peer.c:624:LNetPrimaryNID()) NID 192.168.1.109@o2ib primary NID 192.168.1.109@o2ib rc 0 00000100:00000001:25.0:1506127921.392395:0:3044:0:(connection.c:53:ptlrpc_connection_get()) Process leaving via out (rc=18446612167460263520 : -131906249288096 : 0xffff88082ede1660) 00000100:00000040:25.0:1506127921.392398:0:3044:0:(connection.c:84:ptlrpc_connection_get()) conn=ffff88082ede1660 refcount 8 to 192.168.1.109@o2ib 00000020:00000040:25.0:1506127921.392401:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507400 : new refcount 7 00000100:00080000:25.0:1506127921.392403:0:3044:0:(import.c:104:ptlrpc_import_enter_resend()) ffff8808138fe800 HR½&ˆÿÿHR½&ˆÿÿ: changing import state from NEW to RECOVER 00000100:00000001:25.0:1506127921.392406:0:3044:0:(connection.c:127:ptlrpc_connection_addref()) Process entered 00000100:00000040:25.0:1506127921.392408:0:3044:0:(connection.c:132:ptlrpc_connection_addref()) conn=ffff88082ede1660 refcount 9 to 192.168.1.109@o2ib 00000100:00000001:25.0:1506127921.392409:0:3044:0:(connection.c:134:ptlrpc_connection_addref()) Process leaving (rc=18446612167460263520 : -131906249288096 : ffff88082ede1660) 02000000:00000001:25.0:1506127921.392412:0:3044:0:(sec.c:1434:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:25.0:1506127921.392413:0:3044:0:(sec.c:1331:sptlrpc_sec_create()) Process entered 02000000:08000000:25.0:1506127921.392415:0:3044:0:(sec.c:1339:sptlrpc_sec_create()) mdt soaked-MDT0003: reverse sec using flavor null 02000000:00000001:25.0:1506127921.392416:0:3044:0:(sec.c:1370:sptlrpc_sec_create()) Process leaving (rc=18446744072655015392 : -1054536224 : ffffffffc1250de0) 02000000:00000001:25.0:1506127921.392419:0:3044:0:(sec.c:1505:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.392420:0:3044:0:(import.c:1460:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:25.0:1506127921.392422:0:3044:0:(recover.c:179:ptlrpc_resend()) Process entered 00000100:00000001:25.0:1506127921.392423:0:3044:0:(recover.c:207:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:25.0:1506127921.392425:0:3044:0:(import.c:1536:ptlrpc_import_recovery_state_machine()) ffff8808138fe800 HR½&ˆÿÿHR½&ˆÿÿ: changing import state from RECOVER to FULL 00000100:00000001:25.0:1506127921.392427:0:3044:0:(obd_class.h:1168:obd_import_event()) Process entered 00000100:00000001:25.0:1506127921.392428:0:3044:0:(obd_class.h:1178:obd_import_event()) Process leaving 00000100:02000000:25.0:1506127921.392430:0:3044:0:(import.c:1542:ptlrpc_import_recovery_state_machine()) soaked-MDT0003: Connection restored to 192.168.1.109@o2ib (at 192.168.1.109@o2ib) 00000100:00000001:25.0:1506127921.392432:0:3044:0:(import.c:1551:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:25.0:1506127921.392438:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507400 : new refcount 6 00000020:00000040:25.0:1506127921.392440:0:3044:0:(obd_config.c:770:class_decref()) Decref soaked-MDT0003 (ffff880826bd4d58) now 101 00010000:00000001:25.0:1506127921.392441:0:3044:0:(ldlm_lib.c:1452:target_handle_connect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.392444:0:3044:0:(tgt_handler.c:584:tgt_handler_find_check()) Process entered 00000020:00000001:25.0:1506127921.392445:0:3044:0:(tgt_handler.c:611:tgt_handler_find_check()) Process leaving (rc=18446744072658299920 : -1051251696 : ffffffffc1572c10) 00000020:00000001:25.0:1506127921.392447:0:3044:0:(tgt_handler.c:523:tgt_handle_recovery()) Process entered 00000020:00000001:25.0:1506127921.392448:0:3044:0:(tgt_handler.c:532:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127921.392450:0:3044:0:(tgt_handler.c:382:tgt_handle_request0()) Process entered 00000020:00000001:25.0:1506127921.392451:0:3044:0:(tgt_handler.c:311:tgt_request_preprocess()) Process entered 00000020:00000001:25.0:1506127921.392452:0:3044:0:(tgt_handler.c:366:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.392454:0:3044:0:(tgt_handler.c:935:tgt_connect()) Process entered 00000020:00000001:25.0:1506127921.392456:0:3044:0:(tgt_handler.c:955:tgt_connect()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:25.0:1506127921.392458:0:3044:0:(ldlm_lib.c:2884:target_committed_to_req()) last_committed 0, transno 0, xid 1579288666076944 00010000:00000001:25.0:1506127921.392460:0:3044:0:(ldlm_lib.c:2948:target_send_reply()) Process entered 00010000:00000200:25.0:1506127921.392463:0:3044:0:(ldlm_lib.c:2936:target_send_reply_msg()) @@@ sending reply req@ffff880813804050 x1579288666076944/t0(0) o38->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:0/0 lens 520/416 e 0 to 0 dl 1506128021 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000001:25.0:1506127921.392471:0:3044:0:(ldlm_lib.c:2895:target_pack_pool_reply()) Process entered 00010000:00000001:25.0:1506127921.392473:0:3044:0:(ldlm_lib.c:2903:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:25.0:1506127921.392474:0:3044:0:(import.c:1678:at_measured()) add 1 to ffff88081629fd50 time=1 v=1 (1 0 0 0) 00000100:00001000:25.0:1506127921.392478:0:3044:0:(niobuf.c:550:ptlrpc_at_set_reply()) No early reply support: flags=0x1 req_flags=0x0 magic=bd00bd3/bd00bd3 len=384 00000100:00000001:25.0:1506127921.392480:0:3044:0:(connection.c:127:ptlrpc_connection_addref()) Process entered 00000100:00000040:25.0:1506127921.392482:0:3044:0:(connection.c:132:ptlrpc_connection_addref()) conn=ffff88082ede1660 refcount 10 to 192.168.1.109@o2ib 00000100:00000001:25.0:1506127921.392484:0:3044:0:(connection.c:134:ptlrpc_connection_addref()) Process leaving (rc=18446612167460263520 : -131906249288096 : ffff88082ede1660) 02000000:00000001:25.0:1506127921.392486:0:3044:0:(sec.c:2161:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:25.0:1506127921.392488:0:3044:0:(sec.c:2172:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.392489:0:3044:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:25.0:1506127921.392492:0:3044:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-192.168.1.109@o2ib 00000400:00000010:25.0:1506127921.392494:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761c80. 00000100:00000200:25.0:1506127921.392496:0:3044:0:(niobuf.c:85:ptl_send_buf()) Sending 384 bytes to portal 4, xid 1579288666076944, offset 0 00000400:00000010:25.0:1506127921.392499:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88082e03aa00 (tot 194353484). 00000400:00000200:25.0:1506127921.392502:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.109@o2ib 00000400:00000200:25.0:1506127921.392513:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.109@o2ib(192.168.1.109@o2ib:192.168.1.109@o2ib) : PUT 00000800:00000200:25.0:1506127921.392517:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 384 bytes in 1 frags to 12345-192.168.1.109@o2ib 00000800:00000200:25.0:1506127921.392523:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082edb4980] -> 192.168.1.109@o2ib (2) version: 12 00000800:00000200:25.0:1506127921.392525:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082dfb7200] (20)++ 00000800:00000200:25.0:1506127921.392528:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082dfb7200] (21)++ 00000800:00000200:25.0:1506127921.392532:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082dfb7200] (22)-- 00000100:00000001:25.0:1506127921.392534:0:3044:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.392536:0:3044:0:(connection.c:91:ptlrpc_connection_put()) Process entered 00000100:00000040:25.0:1506127921.392537:0:3044:0:(connection.c:119:ptlrpc_connection_put()) PUT conn=ffff88082ede1660 refcount 9 to 192.168.1.109@o2ib 00000100:00000001:25.0:1506127921.392539:0:3044:0:(connection.c:121:ptlrpc_connection_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127921.392541:0:3044:0:(ldlm_lib.c:2960:target_send_reply()) Process leaving 00000020:00000001:25.0:1506127921.392542:0:3044:0:(tgt_handler.c:475:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.392543:0:3044:0:(tgt_handler.c:780:tgt_request_handle()) Process leaving 00000100:00000040:25.0:1506127921.392546:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff880813804050 x1579288666076944/t0(0) o38->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:0/0 lens 520/384 e 0 to 0 dl 1506128021 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00100000:25.0:1506127921.392554:0:3044:0:(service.c:2135:ptlrpc_server_handle_request()) Handled RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+6:2127:x1579288666076944:12345-192.168.1.109@o2ib:38 Request procesed in 454us (612us total) trans 0 rc 0/0 00000100:00100000:25.0:1506127921.392561:0:3044:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.1.109@o2ib, seq: 10 00000100:00000040:25.0:1506127921.392563:0:3044:0:(service.c:1034:ptlrpc_server_finish_active_request()) RPC PUTting export ffff88080f507400 : new rpc_count 0 00000100:00000001:25.0:1506127921.392565:0:3044:0:(service.c:1613:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:25.0:1506127921.392566:0:3044:0:(service.c:1624:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:25.0:1506127921.392568:0:3044:0:(lu_ucred.c:49:lu_ucred_key_fini()) kfreed 'info': 104 at ffff88080f761b00. 00000020:00000010:25.0:1506127921.392571:0:3044:0:(tgt_main.c:382:tgt_ses_key_fini()) kfreed 'info': 160 at ffff88080f8a9500. 00000020:00000010:25.0:1506127921.392573:0:3044:0:(lu_object.c:1595:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff88082e03d200. 00000020:00000040:25.0:1506127921.392575:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507400 : new refcount 5 00000100:00000001:25.0:1506127921.392577:0:3044:0:(service.c:2155:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:25.0:1506127921.392580:0:3044:0:(niobuf.c:953:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000400:00000010:25.0:1506127921.392583:0:3044:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff8808138a18a0. 00000400:00000010:25.0:1506127921.392585:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761b00. 00000400:00000001:25.0:1506127921.392588:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127921.392589:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127921.394224:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127921.394228:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127921.394230:0:3044:0:(service.c:1859:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:25.0:1506127921.394232:0:3044:0:(sec.c:2052:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:25.0:1506127921.394233:0:3044:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:25.0:1506127921.394235:0:3044:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:25.0:1506127921.394238:0:3044:0:(sec.c:2099:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:25.0:1506127921.394241:0:3044:0:(service.c:1939:ptlrpc_server_handle_req_in()) got req x1579288666077392 00000020:00000001:25.0:1506127921.394243:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:25.0:1506127921.394244:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x2f3e107735f0c2a 00000020:00000001:25.0:1506127921.394246:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000040:25.0:1506127921.394248:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507400 : new refcount 6 00000020:00000001:25.0:1506127921.394250:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000020:00000001:25.0:1506127921.394252:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000100:00000001:25.0:1506127921.394256:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127921.394257:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:25.0:1506127921.394260:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88082e03d200. 00000020:00000010:25.0:1506127921.394262:0:3044:0:(lu_ucred.c:49:lu_ucred_key_init()) kmalloced 'value': 104 at ffff88080f761d00. 00000020:00000010:25.0:1506127921.394265:0:3044:0:(tgt_main.c:382:tgt_ses_key_init()) kmalloced 'value': 160 at ffff88080f8a9500. 00000100:00000040:25.0:1506127921.394269:0:3044:0:(service.c:1184:ptlrpc_at_set_timer()) armed mdt_out at +1s 00000100:00000001:25.0:1506127921.394271:0:3044:0:(service.c:1656:ptlrpc_server_request_add()) Process entered 00000100:00000001:25.0:1506127921.394272:0:3044:0:(service.c:1580:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:25.0:1506127921.394273:0:3044:0:(service.c:1607:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.394276:0:3044:0:(service.c:1695:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.394282:0:3044:0:(service.c:2003:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:25.0:1506127921.394286:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127921.394287:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127921.394288:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127921.394289:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127921.394290:0:3044:0:(service.c:2028:ptlrpc_server_handle_request()) Process entered 00000100:00000001:25.0:1506127921.394292:0:3044:0:(service.c:1809:ptlrpc_server_request_get()) Process entered 00000100:00100000:25.0:1506127921.394296:0:3044:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.1.109@o2ib, seq: 11 00000100:00000040:25.0:1506127921.394298:0:3044:0:(service.c:1840:ptlrpc_server_request_get()) RPC GETting export ffff88080f507400 : new rpc_count 1 00000100:00000001:25.0:1506127921.394300:0:3044:0:(service.c:1842:ptlrpc_server_request_get()) Process leaving (rc=18446612167001128016 : -131906708423600 : ffff880813803c50) 00000100:00000040:25.0:1506127921.394304:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "New" -> "Interpret" req@ffff880813803c50 x1579288666077392/t0(0) o400->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 224/0 e 0 to 0 dl 1506127927 ref 1 fl New:/c0/ffffffff rc 0/-1 00000100:00000001:25.0:1506127921.394312:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127921.394313:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000100:00100000:25.0:1506127921.394316:0:3044:0:(service.c:2089:ptlrpc_server_handle_request()) Handling RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+6:2127:x1579288666077392:12345-192.168.1.109@o2ib:400 00000100:00000200:25.0:1506127921.394319:0:3044:0:(service.c:2094:ptlrpc_server_handle_request()) got req 1579288666077392 00000020:00000001:25.0:1506127921.394321:0:3044:0:(tgt_handler.c:672:tgt_request_handle()) Process entered 00000020:00000001:25.0:1506127921.394323:0:3044:0:(tgt_handler.c:617:process_req_last_xid()) Process entered 00000020:00000001:25.0:1506127921.394324:0:3044:0:(tgt_handler.c:657:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.394326:0:3044:0:(tgt_handler.c:584:tgt_handler_find_check()) Process entered 00000020:00000001:25.0:1506127921.394327:0:3044:0:(tgt_handler.c:611:tgt_handler_find_check()) Process leaving (rc=18446744072654874048 : -1054677568 : ffffffffc122e5c0) 00000020:00000001:25.0:1506127921.394329:0:3044:0:(tgt_handler.c:523:tgt_handle_recovery()) Process entered 00000020:00000040:25.0:1506127921.394332:0:3044:0:(tgt_handler.c:562:tgt_handle_recovery()) @@@ Got new replay req@ffff880813803c50 x1579288666077392/t0(0) o400->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 224/0 e 0 to 0 dl 1506127927 ref 1 fl Interpret:/c0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127921.394363:0:3044:0:(ldlm_lib.c:2705:target_queue_recovery_request()) Process entered 00010000:00000040:25.0:1506127921.394365:0:3044:0:(ldlm_lib.c:1496:target_request_copy_get()) RPC GETting export ffff88080f507400 : new rpc_count 2 00010000:00080000:25.0:1506127921.394367:0:3044:0:(ldlm_lib.c:2718:target_queue_recovery_request()) @@@ queue final req req@ffff880813803c50 x1579288666077392/t0(0) o400->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 224/0 e 0 to 0 dl 1506127927 ref 2 fl Interpret:/c0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127921.394375:0:3044:0:(ldlm_lib.c:2730:target_queue_recovery_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.394376:0:3044:0:(tgt_handler.c:506:tgt_filter_recovery_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.394378:0:3044:0:(tgt_handler.c:566:tgt_handle_recovery()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.394379:0:3044:0:(tgt_handler.c:780:tgt_request_handle()) Process leaving 00000100:00000040:25.0:1506127921.394381:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff880813803c50 x1579288666077392/t0(0) o400->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 224/0 e 0 to 0 dl 1506127927 ref 2 fl Interpret:/c0/ffffffff rc 0/-1 00000100:00100000:25.0:1506127921.394388:0:3044:0:(service.c:2135:ptlrpc_server_handle_request()) Handled RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+6:2127:x1579288666077392:12345-192.168.1.109@o2ib:400 Request procesed in 75us (231us total) trans 0 rc 0/-999 00000100:00100000:25.0:1506127921.394394:0:3044:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.1.109@o2ib, seq: 11 00000100:00000040:25.0:1506127921.394396:0:3044:0:(service.c:1034:ptlrpc_server_finish_active_request()) RPC PUTting export ffff88080f507400 : new rpc_count 1 00000100:00000001:25.0:1506127921.394398:0:3044:0:(service.c:1613:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:25.0:1506127921.394399:0:3044:0:(service.c:1624:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:25.0:1506127921.394400:0:3044:0:(service.c:2155:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:25.0:1506127921.394402:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127921.394403:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127921.424164:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127921.424167:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127921.424170:0:3044:0:(service.c:1859:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:25.0:1506127921.424172:0:3044:0:(sec.c:2052:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:25.0:1506127921.424173:0:3044:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:25.0:1506127921.424176:0:3044:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:25.0:1506127921.424180:0:3044:0:(sec.c:2099:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:25.0:1506127921.424183:0:3044:0:(service.c:1939:ptlrpc_server_handle_req_in()) got req x1579288666076400 00000020:00000001:25.0:1506127921.424186:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:25.0:1506127921.424187:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x2f3e107735f0c2a 00000020:00000001:25.0:1506127921.424190:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000040:25.0:1506127921.424191:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507400 : new refcount 7 00000020:00000001:25.0:1506127921.424193:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000020:00000001:25.0:1506127921.424195:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000100:00000001:25.0:1506127921.424198:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127921.424200:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:25.0:1506127921.424202:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88082e03ce00. 00000020:00000010:25.0:1506127921.424205:0:3044:0:(lu_ucred.c:49:lu_ucred_key_init()) kmalloced 'value': 104 at ffff88080f761d80. 00000020:00000010:25.0:1506127921.424207:0:3044:0:(tgt_main.c:382:tgt_ses_key_init()) kmalloced 'value': 160 at ffff88080f8a9ec0. 00000100:00000001:25.0:1506127921.424210:0:3044:0:(service.c:1656:ptlrpc_server_request_add()) Process entered 00000100:00000001:25.0:1506127921.424211:0:3044:0:(service.c:1580:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:25.0:1506127921.424212:0:3044:0:(service.c:1607:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.424215:0:3044:0:(service.c:1695:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.424219:0:3044:0:(service.c:2003:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:25.0:1506127921.424222:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127921.424224:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127921.424225:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127921.424226:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127921.424227:0:3044:0:(service.c:2028:ptlrpc_server_handle_request()) Process entered 00000100:00000001:25.0:1506127921.424229:0:3044:0:(service.c:1809:ptlrpc_server_request_get()) Process entered 00000100:00100000:25.0:1506127921.424232:0:3044:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.1.109@o2ib, seq: 12 00000100:00000040:25.0:1506127921.424234:0:3044:0:(service.c:1840:ptlrpc_server_request_get()) RPC GETting export ffff88080f507400 : new rpc_count 2 00000100:00000001:25.0:1506127921.424236:0:3044:0:(service.c:1842:ptlrpc_server_request_get()) Process leaving (rc=18446612167001126992 : -131906708424624 : ffff880813803850) 00000100:00000040:25.0:1506127921.424240:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "New" -> "Interpret" req@ffff880813803850 x1579288666076400/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:327/0 lens 304/0 e 0 to 0 dl 1506127932 ref 1 fl New:/0/ffffffff rc 0/-1 00000100:00000001:25.0:1506127921.424248:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127921.424249:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000100:00100000:25.0:1506127921.424252:0:3044:0:(service.c:2089:ptlrpc_server_handle_request()) Handling RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+7:3057:x1579288666076400:12345-192.168.1.109@o2ib:1000 00000100:00000200:25.0:1506127921.424255:0:3044:0:(service.c:2094:ptlrpc_server_handle_request()) got req 1579288666076400 00000020:00000001:25.0:1506127921.424257:0:3044:0:(tgt_handler.c:672:tgt_request_handle()) Process entered 00000020:00000001:25.0:1506127921.424259:0:3044:0:(tgt_handler.c:617:process_req_last_xid()) Process entered 00000020:00000001:25.0:1506127921.424260:0:3044:0:(tgt_handler.c:657:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.424262:0:3044:0:(tgt_handler.c:584:tgt_handler_find_check()) Process entered 00000020:00000001:25.0:1506127921.424263:0:3044:0:(tgt_handler.c:611:tgt_handler_find_check()) Process leaving (rc=18446744072654881248 : -1054670368 : ffffffffc12301e0) 00000020:00000001:25.0:1506127921.424265:0:3044:0:(tgt_handler.c:523:tgt_handle_recovery()) Process entered 00000020:00000040:25.0:1506127921.424268:0:3044:0:(tgt_handler.c:562:tgt_handle_recovery()) @@@ Got new replay req@ffff880813803850 x1579288666076400/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:327/0 lens 304/0 e 0 to 0 dl 1506127932 ref 1 fl Interpret:/0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127921.424288:0:3044:0:(ldlm_lib.c:2705:target_queue_recovery_request()) Process entered 00010000:00080000:25.0:1506127921.424291:0:3044:0:(ldlm_lib.c:2788:target_queue_recovery_request()) @@@ not queueing req@ffff880813803850 x1579288666076400/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:327/0 lens 304/0 e 0 to 0 dl 1506127932 ref 1 fl Interpret:/0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127921.424296:0:3044:0:(ldlm_lib.c:2789:target_queue_recovery_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127921.424297:0:3044:0:(tgt_handler.c:506:tgt_filter_recovery_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.424299:0:3044:0:(tgt_handler.c:573:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127921.424300:0:3044:0:(tgt_handler.c:382:tgt_handle_request0()) Process entered 00000020:00000001:25.0:1506127921.424301:0:3044:0:(tgt_handler.c:311:tgt_request_preprocess()) Process entered 00000020:00000001:25.0:1506127921.424303:0:3044:0:(tgt_handler.c:366:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.424304:0:3044:0:(out_handler.c:917:out_handle()) Process entered 00000020:00000010:25.0:1506127921.424307:0:3044:0:(out_handler.c:940:out_handle()) kmalloced 'update_bufs': 8 at ffff880825e00178. 00000100:00000001:25.0:1506127921.424310:0:3044:0:(pack_generic.c:334:lustre_pack_reply_v2()) Process entered 02000000:00000001:25.0:1506127921.424311:0:3044:0:(sec.c:2112:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:25.0:1506127921.424313:0:3044:0:(sec_null.c:318:null_alloc_rs()) kmalloced 'rs': 4576 at ffff88080f0ae000. 02000000:00000001:25.0:1506127921.424315:0:3044:0:(sec.c:2148:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.424317:0:3044:0:(pack_generic.c:368:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.424322:0:3044:0:(out_handler.c:205:out_attr_get()) Process entered 00000020:00000040:25.0:1506127921.424324:0:3044:0:(out_handler.c:233:out_attr_get()) soaked-MDT0003: insert attr get reply ffff88080f0ae1e0 index 0: rc = 0 00000020:00000001:25.0:1506127921.424326:0:3044:0:(out_handler.c:238:out_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000002:25.0:1506127921.424329:0:3044:0:(lu_object.c:157:lu_object_put()) Add ffff88081f236f58 to site lru. hash: ffff88082b7a7080, bkt: ffffc900159f2018, lru_len: 1 00000020:00000010:25.0:1506127921.424331:0:3044:0:(out_handler.c:1169:out_handle()) kfreed 'update_bufs': 8 at ffff880825e00178. 00000020:00000001:25.0:1506127921.424333:0:3044:0:(out_handler.c:1175:out_handle()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:25.0:1506127921.424335:0:3044:0:(ldlm_lib.c:2884:target_committed_to_req()) last_committed 0, transno 0, xid 1579288666076400 00010000:00000001:25.0:1506127921.424337:0:3044:0:(ldlm_lib.c:2948:target_send_reply()) Process entered 00010000:00000200:25.0:1506127921.424353:0:3044:0:(ldlm_lib.c:2936:target_send_reply_msg()) @@@ sending reply req@ffff880813803850 x1579288666076400/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:327/0 lens 304/4320 e 0 to 0 dl 1506127932 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000001:25.0:1506127921.424360:0:3044:0:(ldlm_lib.c:2895:target_pack_pool_reply()) Process entered 00010000:00000001:25.0:1506127921.424362:0:3044:0:(ldlm_lib.c:2903:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:25.0:1506127921.424364:0:3044:0:(import.c:1678:at_measured()) add 1 to ffff88081629fd50 time=1 v=1 (1 0 0 0) 00000100:00000001:25.0:1506127921.424367:0:3044:0:(connection.c:127:ptlrpc_connection_addref()) Process entered 00000100:00000040:25.0:1506127921.424369:0:3044:0:(connection.c:132:ptlrpc_connection_addref()) conn=ffff88082ede1660 refcount 10 to 192.168.1.109@o2ib 00000100:00000001:25.0:1506127921.424371:0:3044:0:(connection.c:134:ptlrpc_connection_addref()) Process leaving (rc=18446612167460263520 : -131906249288096 : ffff88082ede1660) 02000000:00000001:25.0:1506127921.424373:0:3044:0:(sec.c:2161:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:25.0:1506127921.424374:0:3044:0:(sec.c:2172:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.424376:0:3044:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:25.0:1506127921.424378:0:3044:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-192.168.1.109@o2ib 00000400:00000010:25.0:1506127921.424380:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761e00. 00000100:00000200:25.0:1506127921.424383:0:3044:0:(niobuf.c:85:ptl_send_buf()) Sending 4288 bytes to portal 4, xid 1579288666076400, offset 192 00000400:00000010:25.0:1506127921.424385:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88082e03a400 (tot 194353484). 00000400:00000200:25.0:1506127921.424389:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.109@o2ib 00000400:00000200:25.0:1506127921.424400:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.109@o2ib(192.168.1.109@o2ib:192.168.1.109@o2ib) : PUT 00000800:00000200:25.0:1506127921.424404:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 4288 bytes in 1 frags to 12345-192.168.1.109@o2ib 00000800:00000200:25.0:1506127921.424412:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082edb4980] -> 192.168.1.109@o2ib (2) version: 12 00000800:00000200:25.0:1506127921.424414:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082dfb7200] (20)++ 00000800:00000200:25.0:1506127921.424416:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082dfb7200] (21)++ 00000800:00000200:25.0:1506127921.424421:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082dfb7200] (22)-- 00000100:00000001:25.0:1506127921.424422:0:3044:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.424424:0:3044:0:(connection.c:91:ptlrpc_connection_put()) Process entered 00000100:00000040:25.0:1506127921.424426:0:3044:0:(connection.c:119:ptlrpc_connection_put()) PUT conn=ffff88082ede1660 refcount 9 to 192.168.1.109@o2ib 00000100:00000001:25.0:1506127921.424427:0:3044:0:(connection.c:121:ptlrpc_connection_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127921.424429:0:3044:0:(ldlm_lib.c:2960:target_send_reply()) Process leaving 00000020:00000001:25.0:1506127921.424430:0:3044:0:(tgt_handler.c:475:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.424432:0:3044:0:(tgt_handler.c:780:tgt_request_handle()) Process leaving 00000100:00000040:25.0:1506127921.424434:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff880813803850 x1579288666076400/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:327/0 lens 304/4288 e 0 to 0 dl 1506127932 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00100000:25.0:1506127921.424443:0:3044:0:(service.c:2135:ptlrpc_server_handle_request()) Handled RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+7:3057:x1579288666076400:12345-192.168.1.109@o2ib:1000 Request procesed in 193us (297us total) trans 0 rc 0/0 00000100:00100000:25.0:1506127921.424449:0:3044:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.1.109@o2ib, seq: 12 00000100:00000040:25.0:1506127921.424451:0:3044:0:(service.c:1034:ptlrpc_server_finish_active_request()) RPC PUTting export ffff88080f507400 : new rpc_count 1 00000100:00000001:25.0:1506127921.424453:0:3044:0:(service.c:1613:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:25.0:1506127921.424455:0:3044:0:(service.c:1624:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:25.0:1506127921.424456:0:3044:0:(lu_ucred.c:49:lu_ucred_key_fini()) kfreed 'info': 104 at ffff88080f761d80. 00000020:00000010:25.0:1506127921.424459:0:3044:0:(tgt_main.c:382:tgt_ses_key_fini()) kfreed 'info': 160 at ffff88080f8a9ec0. 00000020:00000010:25.0:1506127921.424461:0:3044:0:(lu_object.c:1595:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff88082e03ce00. 00000020:00000040:25.0:1506127921.424463:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507400 : new refcount 6 00000100:00000001:25.0:1506127921.424465:0:3044:0:(service.c:2155:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:25.0:1506127921.424468:0:3044:0:(niobuf.c:953:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000400:00000010:25.0:1506127921.424471:0:3044:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff8808138a1420. 00000400:00000010:25.0:1506127921.424473:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761d80. 00000400:00000001:25.0:1506127921.424476:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127921.424477:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127921.425369:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127921.425372:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127921.425374:0:3044:0:(service.c:1859:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:25.0:1506127921.425376:0:3044:0:(sec.c:2052:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:25.0:1506127921.425378:0:3044:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:25.0:1506127921.425381:0:3044:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:25.0:1506127921.425384:0:3044:0:(sec.c:2099:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:25.0:1506127921.425388:0:3044:0:(service.c:1939:ptlrpc_server_handle_req_in()) got req x1579288666077440 00000020:00000001:25.0:1506127921.425391:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:25.0:1506127921.425392:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x2f3e107735f0c2a 00000020:00000001:25.0:1506127921.425394:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000040:25.0:1506127921.425396:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507400 : new refcount 7 00000020:00000001:25.0:1506127921.425398:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000020:00000001:25.0:1506127921.425401:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000100:00000001:25.0:1506127921.425404:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127921.425405:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:25.0:1506127921.425408:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88082e03ce00. 00000020:00000010:25.0:1506127921.425411:0:3044:0:(lu_ucred.c:49:lu_ucred_key_init()) kmalloced 'value': 104 at ffff88080f761e80. 00000020:00000010:25.0:1506127921.425413:0:3044:0:(tgt_main.c:382:tgt_ses_key_init()) kmalloced 'value': 160 at ffff88080f8a9ec0. 00000100:00000001:25.0:1506127921.425416:0:3044:0:(service.c:1656:ptlrpc_server_request_add()) Process entered 00000100:00000001:25.0:1506127921.425417:0:3044:0:(service.c:1580:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:25.0:1506127921.425418:0:3044:0:(service.c:1607:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.425421:0:3044:0:(service.c:1695:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.425426:0:3044:0:(service.c:2003:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:25.0:1506127921.425429:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127921.425430:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127921.425432:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127921.425433:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127921.425434:0:3044:0:(service.c:2028:ptlrpc_server_handle_request()) Process entered 00000100:00000001:25.0:1506127921.425435:0:3044:0:(service.c:1809:ptlrpc_server_request_get()) Process entered 00000100:00100000:25.0:1506127921.425439:0:3044:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.1.109@o2ib, seq: 13 00000100:00000040:25.0:1506127921.425441:0:3044:0:(service.c:1840:ptlrpc_server_request_get()) RPC GETting export ffff88080f507400 : new rpc_count 2 00000100:00000001:25.0:1506127921.425443:0:3044:0:(service.c:1842:ptlrpc_server_request_get()) Process leaving (rc=18446612167001125968 : -131906708425648 : ffff880813803450) 00000100:00000040:25.0:1506127921.425447:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "New" -> "Interpret" req@ffff880813803450 x1579288666077440/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 336/0 e 0 to 0 dl 1506127927 ref 1 fl New:/0/ffffffff rc 0/-1 00000100:00000001:25.0:1506127921.425455:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127921.425456:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000100:00100000:25.0:1506127921.425459:0:3044:0:(service.c:2089:ptlrpc_server_handle_request()) Handling RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+7:3057:x1579288666077440:12345-192.168.1.109@o2ib:1000 00000100:00000200:25.0:1506127921.425462:0:3044:0:(service.c:2094:ptlrpc_server_handle_request()) got req 1579288666077440 00000020:00000001:25.0:1506127921.425464:0:3044:0:(tgt_handler.c:672:tgt_request_handle()) Process entered 00000020:00000001:25.0:1506127921.425466:0:3044:0:(tgt_handler.c:617:process_req_last_xid()) Process entered 00000020:00000001:25.0:1506127921.425467:0:3044:0:(tgt_handler.c:657:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.425469:0:3044:0:(tgt_handler.c:584:tgt_handler_find_check()) Process entered 00000020:00000001:25.0:1506127921.425470:0:3044:0:(tgt_handler.c:611:tgt_handler_find_check()) Process leaving (rc=18446744072654881248 : -1054670368 : ffffffffc12301e0) 00000020:00000001:25.0:1506127921.425473:0:3044:0:(tgt_handler.c:523:tgt_handle_recovery()) Process entered 00000020:00000040:25.0:1506127921.425490:0:3044:0:(tgt_handler.c:562:tgt_handle_recovery()) @@@ Got new replay req@ffff880813803450 x1579288666077440/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 336/0 e 0 to 0 dl 1506127927 ref 1 fl Interpret:/0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127921.425496:0:3044:0:(ldlm_lib.c:2705:target_queue_recovery_request()) Process entered 00010000:00080000:25.0:1506127921.425499:0:3044:0:(ldlm_lib.c:2788:target_queue_recovery_request()) @@@ not queueing req@ffff880813803450 x1579288666077440/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 336/0 e 0 to 0 dl 1506127927 ref 1 fl Interpret:/0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127921.425504:0:3044:0:(ldlm_lib.c:2789:target_queue_recovery_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127921.425506:0:3044:0:(tgt_handler.c:506:tgt_filter_recovery_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.425507:0:3044:0:(tgt_handler.c:573:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127921.425509:0:3044:0:(tgt_handler.c:382:tgt_handle_request0()) Process entered 00000020:00000001:25.0:1506127921.425510:0:3044:0:(tgt_handler.c:311:tgt_request_preprocess()) Process entered 00000020:00000001:25.0:1506127921.425511:0:3044:0:(tgt_handler.c:366:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.425513:0:3044:0:(out_handler.c:917:out_handle()) Process entered 00000020:00000010:25.0:1506127921.425515:0:3044:0:(out_handler.c:940:out_handle()) kmalloced 'update_bufs': 8 at ffff880825e00178. 00000100:00000001:25.0:1506127921.425518:0:3044:0:(pack_generic.c:334:lustre_pack_reply_v2()) Process entered 02000000:00000001:25.0:1506127921.425519:0:3044:0:(sec.c:2112:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:25.0:1506127921.425521:0:3044:0:(sec_null.c:318:null_alloc_rs()) kmalloced 'rs': 4576 at ffff88080f0ac000. 02000000:00000001:25.0:1506127921.425523:0:3044:0:(sec.c:2148:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.425524:0:3044:0:(pack_generic.c:368:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.425529:0:3044:0:(out_handler.c:630:out_read()) Process entered 00000020:00000010:25.0:1506127921.425531:0:3044:0:(out_handler.c:660:out_read()) kmalloced 'rdbuf': 24 at ffff880822a87c00. 00000020:00000010:25.0:1506127921.425532:0:3044:0:(out_handler.c:671:out_read()) kmalloced 'rdbuf->rb_bufs[i].lb_buf': 32 at ffff880822a87c20. 00000020:00000001:25.0:1506127921.425537:0:3044:0:(tgt_handler.c:1008:tgt_send_buffer()) Process entered 00000100:00000001:25.0:1506127921.425551:0:3044:0:(niobuf.c:130:ptlrpc_prep_bulk_exp()) Process entered 00000100:00000010:25.0:1506127921.425553:0:3044:0:(client.c:129:ptlrpc_new_bulk()) kmalloced 'desc': 280 at ffff88082e03fc00. 00000100:00000010:25.0:1506127921.425555:0:3044:0:(client.c:139:ptlrpc_new_bulk()) kmalloced '((desc)->bd_u.bd_kvec.bd_kvec)': 16 at ffff88082cbbd5a0. 00000020:00000040:25.0:1506127921.425557:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507400 : new refcount 8 00000100:00000001:25.0:1506127921.425559:0:3044:0:(client.c:235:ptlrpc_prep_bulk_frag()) Process entered 00000100:00000001:25.0:1506127921.425560:0:3044:0:(client.c:251:ptlrpc_prep_bulk_frag()) Process leaving (rc=32 : 32 : 20) 00010000:00000001:25.0:1506127921.425562:0:3044:0:(ldlm_lib.c:3167:target_bulk_io()) Process entered 00000100:00000001:25.0:1506127921.425563:0:3044:0:(niobuf.c:164:ptlrpc_start_bulk_transfer()) Process entered 00000400:00000010:25.0:1506127921.425566:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761f00. 00000400:00000010:25.0:1506127921.425569:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88082e03ac00 (tot 194353484). 00000400:00000200:25.0:1506127921.425573:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.109@o2ib 00000400:00000200:25.0:1506127921.425584:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.109@o2ib(192.168.1.109@o2ib:192.168.1.109@o2ib) : PUT 00000800:00000200:25.0:1506127921.425588:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 32 bytes in 1 frags to 12345-192.168.1.109@o2ib 00000800:00000200:25.0:1506127921.425594:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082edb4980] -> 192.168.1.109@o2ib (2) version: 12 00000800:00000200:25.0:1506127921.425596:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082dfb7200] (20)++ 00000800:00000200:25.0:1506127921.425598:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082dfb7200] (21)++ 00000800:00000200:25.0:1506127921.425603:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082dfb7200] (22)-- 00000100:00000200:25.0:1506127921.425606:0:3044:0:(niobuf.c:262:ptlrpc_start_bulk_transfer()) Transferring 1 pages 32 bytes via portal 14 id 12345-192.168.1.109@o2ib mbits 0x59c5acbf07900-0x59c5acbf07900 00000100:00000001:25.0:1506127921.425609:0:3044:0:(niobuf.c:264:ptlrpc_start_bulk_transfer()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127921.425825:0:3044:0:(ldlm_lib.c:3268:target_bulk_io()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.425829:0:3044:0:(client.c:257:ptlrpc_free_bulk()) Process entered 00000020:00000040:25.0:1506127921.425831:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507400 : new refcount 7 00000100:00000010:25.0:1506127921.425847:0:3044:0:(client.c:281:ptlrpc_free_bulk()) kfreed '((desc)->bd_u.bd_kvec.bd_kvec)': 16 at ffff88082cbbd5a0. 00000100:00000010:25.0:1506127921.425850:0:3044:0:(client.c:282:ptlrpc_free_bulk()) kfreed 'desc': 280 at ffff88082e03fc00. 00000100:00000001:25.0:1506127921.425852:0:3044:0:(client.c:283:ptlrpc_free_bulk()) Process leaving 00000020:00000001:25.0:1506127921.425853:0:3044:0:(tgt_handler.c:1023:tgt_send_buffer()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:25.0:1506127921.425855:0:3044:0:(out_handler.c:702:out_read()) kfreed 'rdbuf->rb_bufs[i].lb_buf': 32 at ffff880822a87c20. 00000020:00000010:25.0:1506127921.425857:0:3044:0:(out_handler.c:706:out_read()) kfreed 'rdbuf': 24 at ffff880822a87c00. 00000020:00000001:25.0:1506127921.425858:0:3044:0:(out_handler.c:712:out_read()) Process leaving (rc=0 : 0 : 0) 00000020:00000002:25.0:1506127921.425863:0:3044:0:(lu_object.c:157:lu_object_put()) Add ffff88081f236f58 to site lru. hash: ffff88082b7a7080, bkt: ffffc900159f2018, lru_len: 1 00000020:00000010:25.0:1506127921.425866:0:3044:0:(out_handler.c:1169:out_handle()) kfreed 'update_bufs': 8 at ffff880825e00178. 00000020:00000001:25.0:1506127921.425868:0:3044:0:(out_handler.c:1175:out_handle()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:25.0:1506127921.425870:0:3044:0:(ldlm_lib.c:2884:target_committed_to_req()) last_committed 0, transno 0, xid 1579288666077440 00010000:00000001:25.0:1506127921.425872:0:3044:0:(ldlm_lib.c:2948:target_send_reply()) Process entered 00010000:00000200:25.0:1506127921.425877:0:3044:0:(ldlm_lib.c:2936:target_send_reply_msg()) @@@ sending reply req@ffff880813803450 x1579288666077440/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 336/4320 e 0 to 0 dl 1506127927 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000001:25.0:1506127921.425887:0:3044:0:(ldlm_lib.c:2895:target_pack_pool_reply()) Process entered 00010000:00000001:25.0:1506127921.425888:0:3044:0:(ldlm_lib.c:2903:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:25.0:1506127921.425890:0:3044:0:(import.c:1678:at_measured()) add 1 to ffff88081629fd50 time=1 v=1 (1 0 0 0) 00000100:00000001:25.0:1506127921.425894:0:3044:0:(connection.c:127:ptlrpc_connection_addref()) Process entered 00000100:00000040:25.0:1506127921.425896:0:3044:0:(connection.c:132:ptlrpc_connection_addref()) conn=ffff88082ede1660 refcount 10 to 192.168.1.109@o2ib 00000100:00000001:25.0:1506127921.425898:0:3044:0:(connection.c:134:ptlrpc_connection_addref()) Process leaving (rc=18446612167460263520 : -131906249288096 : ffff88082ede1660) 02000000:00000001:25.0:1506127921.425901:0:3044:0:(sec.c:2161:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:25.0:1506127921.425902:0:3044:0:(sec.c:2172:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.425904:0:3044:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:25.0:1506127921.425907:0:3044:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-192.168.1.109@o2ib 00000400:00000010:25.0:1506127921.425909:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761f80. 00000100:00000200:25.0:1506127921.425913:0:3044:0:(niobuf.c:85:ptl_send_buf()) Sending 4288 bytes to portal 4, xid 1579288666077440, offset 192 00000400:00000010:25.0:1506127921.425915:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88082e03fc00 (tot 194353484). 00000400:00000200:25.0:1506127921.425919:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.109@o2ib 00000400:00000200:25.0:1506127921.425932:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.109@o2ib(192.168.1.109@o2ib:192.168.1.109@o2ib) : PUT 00000800:00000200:25.0:1506127921.425936:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 4288 bytes in 1 frags to 12345-192.168.1.109@o2ib 00000800:00000200:25.0:1506127921.425944:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082edb4980] -> 192.168.1.109@o2ib (2) version: 12 00000800:00000200:25.0:1506127921.425947:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082dfb7200] (20)++ 00000800:00000200:25.0:1506127921.425949:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082dfb7200] (21)++ 00000800:00000200:25.0:1506127921.425954:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082dfb7200] (22)-- 00000100:00000001:25.0:1506127921.425956:0:3044:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.425958:0:3044:0:(connection.c:91:ptlrpc_connection_put()) Process entered 00000100:00000040:25.0:1506127921.425960:0:3044:0:(connection.c:119:ptlrpc_connection_put()) PUT conn=ffff88082ede1660 refcount 9 to 192.168.1.109@o2ib 00000100:00000001:25.0:1506127921.425961:0:3044:0:(connection.c:121:ptlrpc_connection_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127921.425963:0:3044:0:(ldlm_lib.c:2960:target_send_reply()) Process leaving 00000020:00000001:25.0:1506127921.425964:0:3044:0:(tgt_handler.c:475:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.425966:0:3044:0:(tgt_handler.c:780:tgt_request_handle()) Process leaving 00000100:00000040:25.0:1506127921.425968:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff880813803450 x1579288666077440/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 336/4288 e 0 to 0 dl 1506127927 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00100000:25.0:1506127921.425977:0:3044:0:(service.c:2135:ptlrpc_server_handle_request()) Handled RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+7:3057:x1579288666077440:12345-192.168.1.109@o2ib:1000 Request procesed in 521us (626us total) trans 0 rc 0/0 00000100:00100000:25.0:1506127921.425985:0:3044:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.1.109@o2ib, seq: 13 00000100:00000040:25.0:1506127921.425987:0:3044:0:(service.c:1034:ptlrpc_server_finish_active_request()) RPC PUTting export ffff88080f507400 : new rpc_count 1 00000100:00000001:25.0:1506127921.425989:0:3044:0:(service.c:1613:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:25.0:1506127921.425991:0:3044:0:(service.c:1624:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:25.0:1506127921.425993:0:3044:0:(lu_ucred.c:49:lu_ucred_key_fini()) kfreed 'info': 104 at ffff88080f761e80. 00000020:00000010:25.0:1506127921.425995:0:3044:0:(tgt_main.c:382:tgt_ses_key_fini()) kfreed 'info': 160 at ffff88080f8a9ec0. 00000020:00000010:25.0:1506127921.425997:0:3044:0:(lu_object.c:1595:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff88082e03ce00. 00000020:00000040:25.0:1506127921.426000:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507400 : new refcount 6 00000100:00000001:25.0:1506127921.426002:0:3044:0:(service.c:2155:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:25.0:1506127921.426006:0:3044:0:(niobuf.c:953:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000400:00000010:25.0:1506127921.426008:0:3044:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff8808138a1840. 00000400:00000010:25.0:1506127921.426010:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761e80. 00000400:00000001:25.0:1506127921.426013:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127921.426015:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127921.429213:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127921.429216:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127921.429219:0:3044:0:(service.c:1859:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:25.0:1506127921.429220:0:3044:0:(sec.c:2052:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:25.0:1506127921.429222:0:3044:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:25.0:1506127921.429223:0:3044:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:25.0:1506127921.429227:0:3044:0:(sec.c:2099:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:25.0:1506127921.429229:0:3044:0:(service.c:1939:ptlrpc_server_handle_req_in()) got req x1579288666077584 00000020:00000001:25.0:1506127921.429232:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:25.0:1506127921.429233:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x2f3e107735f0c2a 00000020:00000001:25.0:1506127921.429235:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000040:25.0:1506127921.429236:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507400 : new refcount 7 00000020:00000001:25.0:1506127921.429239:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000020:00000001:25.0:1506127921.429241:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000100:00000001:25.0:1506127921.429244:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127921.429245:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:25.0:1506127921.429248:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88082e03ce00. 00000020:00000010:25.0:1506127921.429250:0:3044:0:(lu_ucred.c:49:lu_ucred_key_init()) kmalloced 'value': 104 at ffff88080f761f80. 00000020:00000010:25.0:1506127921.429253:0:3044:0:(tgt_main.c:382:tgt_ses_key_init()) kmalloced 'value': 160 at ffff88080f8a9ec0. 00000100:00000001:25.0:1506127921.429256:0:3044:0:(service.c:1656:ptlrpc_server_request_add()) Process entered 00000100:00000001:25.0:1506127921.429257:0:3044:0:(service.c:1580:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:25.0:1506127921.429258:0:3044:0:(service.c:1607:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.429261:0:3044:0:(service.c:1695:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.429268:0:3044:0:(service.c:2003:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:25.0:1506127921.429271:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127921.429272:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127921.429274:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127921.429275:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127921.429276:0:3044:0:(service.c:2028:ptlrpc_server_handle_request()) Process entered 00000100:00000001:25.0:1506127921.429277:0:3044:0:(service.c:1809:ptlrpc_server_request_get()) Process entered 00000100:00100000:25.0:1506127921.429282:0:3044:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.1.109@o2ib, seq: 14 00000100:00000040:25.0:1506127921.429284:0:3044:0:(service.c:1840:ptlrpc_server_request_get()) RPC GETting export ffff88080f507400 : new rpc_count 2 00000100:00000001:25.0:1506127921.429286:0:3044:0:(service.c:1842:ptlrpc_server_request_get()) Process leaving (rc=18446612167001124944 : -131906708426672 : ffff880813803050) 00000100:00000040:25.0:1506127921.429290:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "New" -> "Interpret" req@ffff880813803050 x1579288666077584/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 304/0 e 0 to 0 dl 1506127927 ref 1 fl New:/0/ffffffff rc 0/-1 00000100:00000001:25.0:1506127921.429299:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127921.429300:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000100:00100000:25.0:1506127921.429302:0:3044:0:(service.c:2089:ptlrpc_server_handle_request()) Handling RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+7:3057:x1579288666077584:12345-192.168.1.109@o2ib:1000 00000100:00000200:25.0:1506127921.429305:0:3044:0:(service.c:2094:ptlrpc_server_handle_request()) got req 1579288666077584 00000020:00000001:25.0:1506127921.429307:0:3044:0:(tgt_handler.c:672:tgt_request_handle()) Process entered 00000020:00000001:25.0:1506127921.429309:0:3044:0:(tgt_handler.c:617:process_req_last_xid()) Process entered 00000020:00000001:25.0:1506127921.429311:0:3044:0:(tgt_handler.c:657:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.429313:0:3044:0:(tgt_handler.c:584:tgt_handler_find_check()) Process entered 00000020:00000001:25.0:1506127921.429314:0:3044:0:(tgt_handler.c:611:tgt_handler_find_check()) Process leaving (rc=18446744072654881248 : -1054670368 : ffffffffc12301e0) 00000020:00000001:25.0:1506127921.429329:0:3044:0:(tgt_handler.c:523:tgt_handle_recovery()) Process entered 00000020:00000040:25.0:1506127921.429332:0:3044:0:(tgt_handler.c:562:tgt_handle_recovery()) @@@ Got new replay req@ffff880813803050 x1579288666077584/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 304/0 e 0 to 0 dl 1506127927 ref 1 fl Interpret:/0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127921.429338:0:3044:0:(ldlm_lib.c:2705:target_queue_recovery_request()) Process entered 00010000:00080000:25.0:1506127921.429341:0:3044:0:(ldlm_lib.c:2788:target_queue_recovery_request()) @@@ not queueing req@ffff880813803050 x1579288666077584/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 304/0 e 0 to 0 dl 1506127927 ref 1 fl Interpret:/0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127921.429346:0:3044:0:(ldlm_lib.c:2789:target_queue_recovery_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127921.429348:0:3044:0:(tgt_handler.c:506:tgt_filter_recovery_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.429349:0:3044:0:(tgt_handler.c:573:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127921.429350:0:3044:0:(tgt_handler.c:382:tgt_handle_request0()) Process entered 00000020:00000001:25.0:1506127921.429352:0:3044:0:(tgt_handler.c:311:tgt_request_preprocess()) Process entered 00000020:00000001:25.0:1506127921.429353:0:3044:0:(tgt_handler.c:366:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.429355:0:3044:0:(out_handler.c:917:out_handle()) Process entered 00000020:00000010:25.0:1506127921.429357:0:3044:0:(out_handler.c:940:out_handle()) kmalloced 'update_bufs': 8 at ffff880825e00178. 00000100:00000001:25.0:1506127921.429360:0:3044:0:(pack_generic.c:334:lustre_pack_reply_v2()) Process entered 02000000:00000001:25.0:1506127921.429361:0:3044:0:(sec.c:2112:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:25.0:1506127921.429363:0:3044:0:(sec_null.c:318:null_alloc_rs()) kmalloced 'rs': 4576 at ffff88080f0aa000. 02000000:00000001:25.0:1506127921.429365:0:3044:0:(sec.c:2148:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.429366:0:3044:0:(pack_generic.c:368:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.429372:0:3044:0:(out_handler.c:205:out_attr_get()) Process entered 00000020:00000040:25.0:1506127921.429374:0:3044:0:(out_handler.c:233:out_attr_get()) soaked-MDT0003: insert attr get reply ffff88080f0aa1e0 index 0: rc = 0 00000020:00000001:25.0:1506127921.429376:0:3044:0:(out_handler.c:238:out_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000002:25.0:1506127921.429392:0:3044:0:(lu_object.c:157:lu_object_put()) Add ffff88081f236970 to site lru. hash: ffff88082b7a7080, bkt: ffffc90015a74018, lru_len: 1 00000020:00000010:25.0:1506127921.429395:0:3044:0:(out_handler.c:1169:out_handle()) kfreed 'update_bufs': 8 at ffff880825e00178. 00000020:00000001:25.0:1506127921.429396:0:3044:0:(out_handler.c:1175:out_handle()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:25.0:1506127921.429398:0:3044:0:(ldlm_lib.c:2884:target_committed_to_req()) last_committed 0, transno 0, xid 1579288666077584 00010000:00000001:25.0:1506127921.429401:0:3044:0:(ldlm_lib.c:2948:target_send_reply()) Process entered 00010000:00000200:25.0:1506127921.429403:0:3044:0:(ldlm_lib.c:2936:target_send_reply_msg()) @@@ sending reply req@ffff880813803050 x1579288666077584/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 304/4320 e 0 to 0 dl 1506127927 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000001:25.0:1506127921.429411:0:3044:0:(ldlm_lib.c:2895:target_pack_pool_reply()) Process entered 00010000:00000001:25.0:1506127921.429412:0:3044:0:(ldlm_lib.c:2903:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:25.0:1506127921.429414:0:3044:0:(import.c:1678:at_measured()) add 1 to ffff88081629fd50 time=1 v=1 (1 0 0 0) 00000100:00000001:25.0:1506127921.429418:0:3044:0:(connection.c:127:ptlrpc_connection_addref()) Process entered 00000100:00000040:25.0:1506127921.429420:0:3044:0:(connection.c:132:ptlrpc_connection_addref()) conn=ffff88082ede1660 refcount 10 to 192.168.1.109@o2ib 00000100:00000001:25.0:1506127921.429422:0:3044:0:(connection.c:134:ptlrpc_connection_addref()) Process leaving (rc=18446612167460263520 : -131906249288096 : ffff88082ede1660) 02000000:00000001:25.0:1506127921.429423:0:3044:0:(sec.c:2161:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:25.0:1506127921.429425:0:3044:0:(sec.c:2172:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.429426:0:3044:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:25.0:1506127921.429429:0:3044:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-192.168.1.109@o2ib 00000400:00000010:25.0:1506127921.429431:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761e00. 00000100:00000200:25.0:1506127921.429434:0:3044:0:(niobuf.c:85:ptl_send_buf()) Sending 4288 bytes to portal 4, xid 1579288666077584, offset 192 00000400:00000010:25.0:1506127921.429437:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88082e039e00 (tot 194353484). 00000400:00000200:25.0:1506127921.429441:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.109@o2ib 00000400:00000200:25.0:1506127921.429452:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.109@o2ib(192.168.1.109@o2ib:192.168.1.109@o2ib) : PUT 00000800:00000200:25.0:1506127921.429457:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 4288 bytes in 1 frags to 12345-192.168.1.109@o2ib 00000800:00000200:25.0:1506127921.429465:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082edb4980] -> 192.168.1.109@o2ib (2) version: 12 00000800:00000200:25.0:1506127921.429467:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082dfb7200] (20)++ 00000800:00000200:25.0:1506127921.429469:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082dfb7200] (21)++ 00000800:00000200:25.0:1506127921.429474:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082dfb7200] (22)-- 00000100:00000001:25.0:1506127921.429476:0:3044:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.429478:0:3044:0:(connection.c:91:ptlrpc_connection_put()) Process entered 00000100:00000040:25.0:1506127921.429479:0:3044:0:(connection.c:119:ptlrpc_connection_put()) PUT conn=ffff88082ede1660 refcount 9 to 192.168.1.109@o2ib 00000100:00000001:25.0:1506127921.429481:0:3044:0:(connection.c:121:ptlrpc_connection_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127921.429483:0:3044:0:(ldlm_lib.c:2960:target_send_reply()) Process leaving 00000020:00000001:25.0:1506127921.429484:0:3044:0:(tgt_handler.c:475:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.429485:0:3044:0:(tgt_handler.c:780:tgt_request_handle()) Process leaving 00000100:00000040:25.0:1506127921.429488:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff880813803050 x1579288666077584/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 304/4288 e 0 to 0 dl 1506127927 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00100000:25.0:1506127921.429496:0:3044:0:(service.c:2135:ptlrpc_server_handle_request()) Handled RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+7:3057:x1579288666077584:12345-192.168.1.109@o2ib:1000 Request procesed in 196us (350us total) trans 0 rc 0/0 00000100:00100000:25.0:1506127921.429502:0:3044:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.1.109@o2ib, seq: 14 00000100:00000040:25.0:1506127921.429505:0:3044:0:(service.c:1034:ptlrpc_server_finish_active_request()) RPC PUTting export ffff88080f507400 : new rpc_count 1 00000100:00000001:25.0:1506127921.429507:0:3044:0:(service.c:1613:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:25.0:1506127921.429508:0:3044:0:(service.c:1624:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:25.0:1506127921.429510:0:3044:0:(lu_ucred.c:49:lu_ucred_key_fini()) kfreed 'info': 104 at ffff88080f761f80. 00000020:00000010:25.0:1506127921.429512:0:3044:0:(tgt_main.c:382:tgt_ses_key_fini()) kfreed 'info': 160 at ffff88080f8a9ec0. 00000020:00000010:25.0:1506127921.429514:0:3044:0:(lu_object.c:1595:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff88082e03ce00. 00000020:00000040:25.0:1506127921.429517:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507400 : new refcount 6 00000100:00000001:25.0:1506127921.429519:0:3044:0:(service.c:2155:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:25.0:1506127921.429522:0:3044:0:(niobuf.c:953:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000400:00000010:25.0:1506127921.429524:0:3044:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff8808138a1b40. 00000400:00000010:25.0:1506127921.429526:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761f80. 00000400:00000001:25.0:1506127921.429529:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127921.429531:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127921.430510:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127921.430514:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127921.430516:0:3044:0:(service.c:1859:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:25.0:1506127921.430530:0:3044:0:(sec.c:2052:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:25.0:1506127921.430531:0:3044:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:25.0:1506127921.430533:0:3044:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:25.0:1506127921.430536:0:3044:0:(sec.c:2099:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:25.0:1506127921.430539:0:3044:0:(service.c:1939:ptlrpc_server_handle_req_in()) got req x1579288666077616 00000020:00000001:25.0:1506127921.430541:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:25.0:1506127921.430543:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x2f3e107735f0c2a 00000020:00000001:25.0:1506127921.430544:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000040:25.0:1506127921.430546:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507400 : new refcount 7 00000020:00000001:25.0:1506127921.430548:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000020:00000001:25.0:1506127921.430550:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000100:00000001:25.0:1506127921.430554:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127921.430555:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:25.0:1506127921.430557:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88082e03ce00. 00000020:00000010:25.0:1506127921.430560:0:3044:0:(lu_ucred.c:49:lu_ucred_key_init()) kmalloced 'value': 104 at ffff88080f761c80. 00000020:00000010:25.0:1506127921.430562:0:3044:0:(tgt_main.c:382:tgt_ses_key_init()) kmalloced 'value': 160 at ffff88080f8a9ec0. 00000100:00000001:25.0:1506127921.430565:0:3044:0:(service.c:1656:ptlrpc_server_request_add()) Process entered 00000100:00000001:25.0:1506127921.430566:0:3044:0:(service.c:1580:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:25.0:1506127921.430567:0:3044:0:(service.c:1607:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.430570:0:3044:0:(service.c:1695:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.430577:0:3044:0:(service.c:2003:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:25.0:1506127921.430580:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127921.430581:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127921.430583:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127921.430584:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127921.430585:0:3044:0:(service.c:2028:ptlrpc_server_handle_request()) Process entered 00000100:00000001:25.0:1506127921.430586:0:3044:0:(service.c:1809:ptlrpc_server_request_get()) Process entered 00000100:00100000:25.0:1506127921.430591:0:3044:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.1.109@o2ib, seq: 15 00000100:00000040:25.0:1506127921.430593:0:3044:0:(service.c:1840:ptlrpc_server_request_get()) RPC GETting export ffff88080f507400 : new rpc_count 2 00000100:00000001:25.0:1506127921.430595:0:3044:0:(service.c:1842:ptlrpc_server_request_get()) Process leaving (rc=18446612167001123920 : -131906708427696 : ffff880813802c50) 00000100:00000040:25.0:1506127921.430599:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "New" -> "Interpret" req@ffff880813802c50 x1579288666077616/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 632/0 e 0 to 0 dl 1506127927 ref 1 fl New:/0/ffffffff rc 0/-1 00000100:00000001:25.0:1506127921.430607:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127921.430609:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000100:00100000:25.0:1506127921.430612:0:3044:0:(service.c:2089:ptlrpc_server_handle_request()) Handling RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+7:3057:x1579288666077616:12345-192.168.1.109@o2ib:1000 00000100:00000200:25.0:1506127921.430615:0:3044:0:(service.c:2094:ptlrpc_server_handle_request()) got req 1579288666077616 00000020:00000001:25.0:1506127921.430617:0:3044:0:(tgt_handler.c:672:tgt_request_handle()) Process entered 00000020:00000001:25.0:1506127921.430619:0:3044:0:(tgt_handler.c:617:process_req_last_xid()) Process entered 00000020:00000001:25.0:1506127921.430620:0:3044:0:(tgt_handler.c:657:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.430635:0:3044:0:(tgt_handler.c:584:tgt_handler_find_check()) Process entered 00000020:00000001:25.0:1506127921.430636:0:3044:0:(tgt_handler.c:611:tgt_handler_find_check()) Process leaving (rc=18446744072654881248 : -1054670368 : ffffffffc12301e0) 00000020:00000001:25.0:1506127921.430638:0:3044:0:(tgt_handler.c:523:tgt_handle_recovery()) Process entered 00000020:00000040:25.0:1506127921.430641:0:3044:0:(tgt_handler.c:562:tgt_handle_recovery()) @@@ Got new replay req@ffff880813802c50 x1579288666077616/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 632/0 e 0 to 0 dl 1506127927 ref 1 fl Interpret:/0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127921.430647:0:3044:0:(ldlm_lib.c:2705:target_queue_recovery_request()) Process entered 00010000:00080000:25.0:1506127921.430649:0:3044:0:(ldlm_lib.c:2788:target_queue_recovery_request()) @@@ not queueing req@ffff880813802c50 x1579288666077616/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 632/0 e 0 to 0 dl 1506127927 ref 1 fl Interpret:/0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127921.430654:0:3044:0:(ldlm_lib.c:2789:target_queue_recovery_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127921.430656:0:3044:0:(tgt_handler.c:506:tgt_filter_recovery_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.430657:0:3044:0:(tgt_handler.c:573:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127921.430659:0:3044:0:(tgt_handler.c:382:tgt_handle_request0()) Process entered 00000020:00000001:25.0:1506127921.430660:0:3044:0:(tgt_handler.c:311:tgt_request_preprocess()) Process entered 00000020:00000001:25.0:1506127921.430661:0:3044:0:(tgt_handler.c:366:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.430663:0:3044:0:(out_handler.c:917:out_handle()) Process entered 00000020:00000010:25.0:1506127921.430665:0:3044:0:(out_handler.c:940:out_handle()) kmalloced 'update_bufs': 8 at ffff880825e00178. 00000100:00000001:25.0:1506127921.430668:0:3044:0:(pack_generic.c:334:lustre_pack_reply_v2()) Process entered 02000000:00000001:25.0:1506127921.430669:0:3044:0:(sec.c:2112:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:25.0:1506127921.430671:0:3044:0:(sec_null.c:318:null_alloc_rs()) kmalloced 'rs': 4576 at ffff88080f0a8000. 02000000:00000001:25.0:1506127921.430673:0:3044:0:(sec.c:2148:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.430675:0:3044:0:(pack_generic.c:368:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.430677:0:3044:0:(lu_object.c:243:lu_object_alloc()) Process entered 00000004:00000001:25.0:1506127921.430678:0:3044:0:(mdt_handler.c:5144:mdt_object_alloc()) Process entered 00000004:00000010:25.0:1506127921.430680:0:3044:0:(mdt_handler.c:5146:mdt_object_alloc()) slab-alloced 'mo': 216 at ffff8800ad2a55f0. 00000004:00000001:25.0:1506127921.430682:0:3044:0:(mdt_handler.c:5161:mdt_object_alloc()) Process leaving (rc=18446612135219451456 : -131938490100160 : ffff8800ad2a5640) 00000004:00000001:25.0:1506127921.430697:0:3044:0:(mdt_handler.c:5173:mdt_object_init()) Process entered 00000004:00000040:25.0:1506127921.430698:0:3044:0:(mdt_handler.c:5176:mdt_object_init()) object init, fid = [0x2c0003ab2:0x1:0x0] 00000004:00000010:25.0:1506127921.430700:0:3044:0:(mdd_object.c:120:mdd_object_alloc()) slab-alloced 'mdd_obj': 80 at ffff88042c24e190. 00000004:00000001:25.0:1506127921.430702:0:3044:0:(mdt_handler.c:5185:mdt_object_init()) Process leaving (rc=0 : 0 : 0) 00000004:00000001:25.0:1506127921.430704:0:3044:0:(mdd_object.c:143:mdd_object_init()) Process entered 00000004:00000001:25.0:1506127921.430705:0:3044:0:(lod_dev.c:210:lod_object_alloc()) Process entered 00000004:00000010:25.0:1506127921.430707:0:3044:0:(lod_dev.c:212:lod_object_alloc()) slab-alloced 'lod_obj': 104 at ffff88082a262d00. 00000004:00000001:25.0:1506127921.430708:0:3044:0:(lod_dev.c:221:lod_object_alloc()) Process leaving (rc=18446612167381101824 : -131906328449792 : ffff88082a262d00) 00000004:00000001:25.0:1506127921.430710:0:3044:0:(mdd_object.c:153:mdd_object_init()) Process leaving (rc=0 : 0 : 0) 00000004:00000001:25.0:1506127921.430712:0:3044:0:(lod_object.c:5108:lod_object_init()) Process entered 00000004:00000001:25.0:1506127921.430713:0:3044:0:(lod_dev.c:127:lod_fld_lookup()) Process entered 80000000:00000001:25.0:1506127921.430714:0:3044:0:(fld_handler.c:242:fld_server_lookup()) Process entered 80000000:00000001:25.0:1506127921.430715:0:3044:0:(fld_handler.c:207:fld_local_lookup()) Process entered 80000000:00000001:25.0:1506127921.430716:0:3044:0:(fld_cache.c:530:fld_cache_lookup()) Process entered 80000000:00000001:25.0:1506127921.430718:0:3044:0:(fld_cache.c:549:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:25.0:1506127921.430719:0:3044:0:(fld_handler.c:224:fld_local_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:25.0:1506127921.430720:0:3044:0:(fld_handler.c:246:fld_server_lookup()) Process leaving (rc=0 : 0 : 0) 00000004:00000040:25.0:1506127921.430722:0:3044:0:(lod_dev.c:168:lod_fld_lookup()) soaked-MDT0003-mdtlov: got tgt 3 for sequence: 0x2c0003ab2 00000004:00000001:25.0:1506127921.430724:0:3044:0:(lod_dev.c:170:lod_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00080000:00000010:25.0:1506127921.430726:0:3044:0:(osd_handler.c:332:osd_object_alloc()) kmalloced 'mo': 200 at ffff880822755600. 00000004:00000001:25.0:1506127921.430728:0:3044:0:(lod_object.c:5159:lod_object_init()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:25.0:1506127921.430730:0:3044:0:(osd_handler.c:994:osd_fid_lookup()) Process entered 00080000:00000001:25.0:1506127921.430731:0:3044:0:(osd_handler.c:1016:osd_fid_lookup()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:25.0:1506127921.430734:0:3044:0:(lu_object.c:295:lu_object_alloc()) Process leaving (rc=18446612135219451456 : -131938490100160 : ffff8800ad2a5640) 00080000:00000001:25.0:1506127921.430738:0:3044:0:(osd_handler.c:1612:osd_trans_create()) Process entered 00080000:00000010:25.0:1506127921.430740:0:3044:0:(osd_handler.c:1627:osd_trans_create()) kmalloced 'oh': 240 at ffff880822756d00. 00080000:00000001:25.0:1506127921.430741:0:3044:0:(osd_handler.c:1650:osd_trans_create()) Process leaving (rc=18446612167252077824 : -131906457473792 : ffff880822756d00) 00000020:00000001:25.0:1506127921.430744:0:3044:0:(out_handler.c:116:out_create()) Process entered 00080000:00000001:25.0:1506127921.430746:0:3044:0:(osd_handler.c:3101:osd_declare_create()) Process entered 00000001:00000001:25.0:1506127921.430747:0:3044:0:(osd_quota.c:630:osd_declare_inode_qid()) Process entered 00000001:00000001:25.0:1506127921.430748:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127921.430750:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127921.430751:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.430752:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.430754:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127921.430755:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127921.430756:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.430757:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.430758:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127921.430759:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127921.430760:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.430761:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.430762:0:3044:0:(osd_quota.c:677:osd_declare_inode_qid()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:25.0:1506127921.430764:0:3044:0:(osd_handler.c:3133:osd_declare_create()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.430766:0:3044:0:(out_handler.c:155:out_create()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.430770:0:3044:0:(out_handler.c:473:out_index_insert()) Process entered 00080000:00000001:25.0:1506127921.430772:0:3044:0:(osd_handler.c:5333:osd_index_declare_ea_insert()) Process entered 00000001:00000001:25.0:1506127921.430774:0:3044:0:(osd_quota.c:630:osd_declare_inode_qid()) Process entered 00000001:00000001:25.0:1506127921.430775:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127921.430776:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127921.430777:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.430778:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.430779:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127921.430780:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127921.430781:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.430782:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.430783:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127921.430784:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127921.430785:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.430786:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.430787:0:3044:0:(osd_quota.c:677:osd_declare_inode_qid()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:25.0:1506127921.430789:0:3044:0:(osd_handler.c:5374:osd_index_declare_ea_insert()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.430790:0:3044:0:(out_handler.c:516:out_index_insert()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.430792:0:3044:0:(out_handler.c:829:out_tx_end()) Process entered 00080000:00000001:25.0:1506127921.430793:0:3044:0:(osd_handler.c:1713:osd_trans_start()) Process entered 00000001:00000001:25.0:1506127921.430796:0:3044:0:(osd_io.c:1576:osd_declare_write()) Process entered 00000001:00000001:25.0:1506127921.430797:0:3044:0:(osd_quota.c:630:osd_declare_inode_qid()) Process entered 00000001:00000001:25.0:1506127921.430798:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127921.430798:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127921.430799:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.430800:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.430802:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127921.430802:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127921.430803:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.430804:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.430806:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127921.430806:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127921.430807:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.430808:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.430809:0:3044:0:(osd_quota.c:677:osd_declare_inode_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.430811:0:3044:0:(osd_io.c:1650:osd_declare_write()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:25.0:1506127921.430815:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88082e03b800. 00000020:00000010:25.0:1506127921.430817:0:3044:0:(update_records.c:1217:update_key_init()) kmalloced 'value': 416 at ffff88082e03c000. 00000004:00000010:25.0:1506127921.430819:0:3044:0:(osp_dev.c:1869:osp_txn_key_init()) kmalloced 'value': 8 at ffff880825e00198. 00080000:00000001:25.0:1506127921.430821:0:3044:0:(osd_handler.c:1780:osd_trans_start()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:25.0:1506127921.430823:0:3044:0:(out_lib.c:538:out_tx_create_exec()) soaked-MDT0003-osd: create [0x2c0003ab2:0x1:0x0]: dof 0, mode 100000 00080000:00000001:25.0:1506127921.430826:0:3044:0:(osd_handler.c:3561:osd_create()) Process entered 00080000:00000001:25.0:1506127921.430860:0:3044:0:(osd_oi.c:545:fid_is_on_ost()) Process entered 80000000:00000001:25.0:1506127921.430862:0:3044:0:(fld_handler.c:207:fld_local_lookup()) Process entered 80000000:00000001:25.0:1506127921.430863:0:3044:0:(fld_cache.c:530:fld_cache_lookup()) Process entered 80000000:00000001:25.0:1506127921.430864:0:3044:0:(fld_cache.c:549:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:25.0:1506127921.430866:0:3044:0:(fld_handler.c:224:fld_local_lookup()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:25.0:1506127921.430867:0:3044:0:(osd_oi.c:583:fid_is_on_ost()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:25.0:1506127921.430869:0:3044:0:(osd_handler.c:3253:osd_ea_fid_set()) Process entered 00080000:00000001:25.0:1506127921.430874:0:3044:0:(osd_handler.c:3307:osd_ea_fid_set()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:25.0:1506127921.430876:0:3044:0:(osd_oi.c:545:fid_is_on_ost()) Process entered 80000000:00000001:25.0:1506127921.430876:0:3044:0:(fld_handler.c:207:fld_local_lookup()) Process entered 80000000:00000001:25.0:1506127921.430877:0:3044:0:(fld_cache.c:530:fld_cache_lookup()) Process entered 80000000:00000001:25.0:1506127921.430878:0:3044:0:(fld_cache.c:549:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:25.0:1506127921.430879:0:3044:0:(fld_handler.c:224:fld_local_lookup()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:25.0:1506127921.430881:0:3044:0:(osd_oi.c:583:fid_is_on_ost()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:25.0:1506127921.430882:0:3044:0:(osd_oi.c:647:osd_oi_iam_refresh()) Process entered 00000001:00000010:25.0:1506127921.430887:0:3044:0:(osd_dynlocks.c:109:dynlock_lock()) slab-alloced 'nhl': 72 at ffff88042c259870. 00000001:00000010:25.0:1506127921.431106:0:3044:0:(osd_dynlocks.c:193:dynlock_unlock()) slab-freed 'hl': 72 at ffff88042c259870. 00080000:00000001:25.0:1506127921.431110:0:3044:0:(osd_oi.c:668:osd_oi_iam_refresh()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:25.0:1506127921.431113:0:3044:0:(osd_handler.c:3614:osd_create()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:25.0:1506127921.431115:0:3044:0:(out_lib.c:547:out_tx_create_exec()) soaked-MDT0003-osd: insert create reply ffff88080f0a81e0 index 0: rc = 0 00000020:00000040:25.0:1506127921.431117:0:3044:0:(out_handler.c:863:out_tx_end()) soaked-MDT0003-osd: executed 0/2: rc = 0 00000020:00000040:25.0:1506127921.431120:0:3044:0:(out_lib.c:1057:out_obj_index_insert()) soaked-MDT0003-osd: index insert [0x20000000a:0x3:0x0] name: [0x2c0003ab2:0x1:0x0] fid [0x2c0003ab2:0x1:0x0], type 32768 00080000:00000001:25.0:1506127921.431124:0:3044:0:(osd_handler.c:5402:osd_index_ea_insert()) Process entered 00080000:00000001:25.0:1506127921.431135:0:3044:0:(osd_handler.c:4853:__osd_ea_add_rec()) Process leaving (rc=0 : 0 : 0) 00080000:00000002:25.0:1506127921.431137:0:3044:0:(osd_handler.c:5478:osd_index_ea_insert()) parent 78643201 insert [0x2c0003ab2:0x1:0x0]:186 rc = 0 00080000:00000001:25.0:1506127921.431140:0:3044:0:(osd_handler.c:5484:osd_index_ea_insert()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:25.0:1506127921.431141:0:3044:0:(out_lib.c:1104:out_tx_index_insert_exec()) soaked-MDT0003-osd: insert idx insert reply ffff88080f0a81e0 index 1: rc = 0 00000020:00000040:25.0:1506127921.431144:0:3044:0:(out_handler.c:863:out_tx_end()) soaked-MDT0003-osd: executed 1/2: rc = 0 00080000:00000001:25.0:1506127921.431146:0:3044:0:(osd_handler.c:1834:osd_trans_stop()) Process entered 00000004:00000040:25.0:1506127921.431147:0:3044:0:(mdt_recovery.c:87:mdt_txn_stop_cb()) More than one transaction 00000001:00000001:25.0:1506127921.431149:0:3044:0:(tgt_lastrcvd.c:1204:tgt_last_rcvd_update()) Process entered 00000001:00000002:25.0:1506127921.431151:0:3044:0:(tgt_lastrcvd.c:1245:tgt_last_rcvd_update()) transno = 30064771075, last_committed = 30064771074 00000001:00000010:25.0:1506127921.431154:0:3044:0:(tgt_lastrcvd.c:909:tgt_last_commit_cb_add()) kmalloced 'ccb': 96 at ffff8808138a17e0. 00000001:00000040:25.0:1506127921.431155:0:3044:0:(tgt_lastrcvd.c:914:tgt_last_commit_cb_add()) callback GETting export ffff88080f507400 : new cb_count 2 00000020:00000040:25.0:1506127921.431157:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507400 : new refcount 8 00000001:00000001:25.0:1506127921.431163:0:3044:0:(tgt_lastrcvd.c:1387:tgt_last_rcvd_update()) Process leaving (rc=0 : 0 : 0) 00040000:00000001:25.0:1506127921.431975:0:3044:0:(qsd_handler.c:1080:qsd_op_end()) Process entered 00040000:00000001:25.0:1506127921.431976:0:3044:0:(qsd_handler.c:1092:qsd_op_end()) Process leaving 00080000:00000001:25.0:1506127921.431978:0:3044:0:(osd_handler.c:1905:osd_trans_stop()) Process leaving (rc=0 : 0 : 0) 00000020:00000002:25.0:1506127921.431981:0:3044:0:(lu_object.c:157:lu_object_put()) Add ffff8800ad2a5618 to site lru. hash: ffff88082b7a7080, bkt: ffffc9001aec0018, lru_len: 1 00000020:00000002:25.0:1506127921.431984:0:3044:0:(lu_object.c:157:lu_object_put()) Add ffff88081f236970 to site lru. hash: ffff88082b7a7080, bkt: ffffc90015a74018, lru_len: 1 00000020:00000001:25.0:1506127921.431986:0:3044:0:(out_handler.c:881:out_tx_end()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:25.0:1506127921.431988:0:3044:0:(out_handler.c:1169:out_handle()) kfreed 'update_bufs': 8 at ffff880825e00178. 00000020:00000001:25.0:1506127921.431990:0:3044:0:(out_handler.c:1175:out_handle()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:25.0:1506127921.431992:0:3044:0:(ldlm_lib.c:2884:target_committed_to_req()) last_committed 30064771075, transno 30064771075, xid 1579288666077616 00010000:00000001:25.0:1506127921.431995:0:3044:0:(ldlm_lib.c:2948:target_send_reply()) Process entered 00010000:00000200:25.0:1506127921.431998:0:3044:0:(ldlm_lib.c:2936:target_send_reply_msg()) @@@ sending reply req@ffff880813802c50 x1579288666077616/t30064771075(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 632/4320 e 0 to 0 dl 1506127927 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000001:25.0:1506127921.432007:0:3044:0:(ldlm_lib.c:2895:target_pack_pool_reply()) Process entered 00010000:00000001:25.0:1506127921.432008:0:3044:0:(ldlm_lib.c:2903:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:25.0:1506127921.432010:0:3044:0:(import.c:1678:at_measured()) add 1 to ffff88081629fd50 time=1 v=1 (1 0 0 0) 00000100:00000001:25.0:1506127921.432014:0:3044:0:(connection.c:127:ptlrpc_connection_addref()) Process entered 00000100:00000040:25.0:1506127921.432016:0:3044:0:(connection.c:132:ptlrpc_connection_addref()) conn=ffff88082ede1660 refcount 10 to 192.168.1.109@o2ib 00000100:00000001:25.0:1506127921.432018:0:3044:0:(connection.c:134:ptlrpc_connection_addref()) Process leaving (rc=18446612167460263520 : -131906249288096 : ffff88082ede1660) 02000000:00000001:25.0:1506127921.432020:0:3044:0:(sec.c:2161:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:25.0:1506127921.432021:0:3044:0:(sec.c:2172:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.432023:0:3044:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:25.0:1506127921.432025:0:3044:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-192.168.1.109@o2ib 00000400:00000010:25.0:1506127921.432027:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761b80. 00000100:00000200:25.0:1506127921.432030:0:3044:0:(niobuf.c:85:ptl_send_buf()) Sending 4288 bytes to portal 4, xid 1579288666077616, offset 192 00000400:00000010:25.0:1506127921.432033:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88082e03a600 (tot 194353484). 00000400:00000200:25.0:1506127921.432036:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.109@o2ib 00000400:00000200:25.0:1506127921.432048:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.109@o2ib(192.168.1.109@o2ib:192.168.1.109@o2ib) : PUT 00000800:00000200:25.0:1506127921.432053:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 4288 bytes in 1 frags to 12345-192.168.1.109@o2ib 00000800:00000200:25.0:1506127921.432060:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082edb4980] -> 192.168.1.109@o2ib (2) version: 12 00000800:00000200:25.0:1506127921.432063:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082dfb7200] (20)++ 00000800:00000200:25.0:1506127921.432065:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082dfb7200] (21)++ 00000800:00000200:25.0:1506127921.432070:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082dfb7200] (22)-- 00000100:00000001:25.0:1506127921.432080:0:3044:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.432082:0:3044:0:(connection.c:91:ptlrpc_connection_put()) Process entered 00000100:00000040:25.0:1506127921.432084:0:3044:0:(connection.c:119:ptlrpc_connection_put()) PUT conn=ffff88082ede1660 refcount 9 to 192.168.1.109@o2ib 00000100:00000001:25.0:1506127921.432085:0:3044:0:(connection.c:121:ptlrpc_connection_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127921.432087:0:3044:0:(ldlm_lib.c:2960:target_send_reply()) Process leaving 00000020:00000001:25.0:1506127921.432088:0:3044:0:(tgt_handler.c:475:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.432090:0:3044:0:(tgt_handler.c:780:tgt_request_handle()) Process leaving 00000100:00000040:25.0:1506127921.432092:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff880813802c50 x1579288666077616/t30064771075(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 632/4288 e 0 to 0 dl 1506127927 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00100000:25.0:1506127921.432101:0:3044:0:(service.c:2135:ptlrpc_server_handle_request()) Handled RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+6:3057:x1579288666077616:12345-192.168.1.109@o2ib:1000 Request procesed in 1492us (1652us total) trans 30064771075 rc 0/0 00000100:00100000:25.0:1506127921.432108:0:3044:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.1.109@o2ib, seq: 15 00000100:00000040:25.0:1506127921.432110:0:3044:0:(service.c:1034:ptlrpc_server_finish_active_request()) RPC PUTting export ffff88080f507400 : new rpc_count 1 00000100:00000001:25.0:1506127921.432112:0:3044:0:(service.c:1613:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:25.0:1506127921.432113:0:3044:0:(service.c:1624:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:25.0:1506127921.432115:0:3044:0:(lu_ucred.c:49:lu_ucred_key_fini()) kfreed 'info': 104 at ffff88080f761c80. 00000020:00000010:25.0:1506127921.432117:0:3044:0:(tgt_main.c:382:tgt_ses_key_fini()) kfreed 'info': 160 at ffff88080f8a9ec0. 00000020:00000010:25.0:1506127921.432120:0:3044:0:(lu_object.c:1595:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff88082e03ce00. 00000020:00000040:25.0:1506127921.432123:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507400 : new refcount 5 00000100:00000001:25.0:1506127921.432126:0:3044:0:(service.c:2155:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:25.0:1506127921.432130:0:3044:0:(niobuf.c:953:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000400:00000010:25.0:1506127921.432133:0:3044:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff8808138a1480. 00000400:00000010:25.0:1506127921.432136:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761c80. 00000400:00000001:25.0:1506127921.432140:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127921.432142:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127921.433046:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127921.433047:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127921.433049:0:3044:0:(service.c:1859:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:25.0:1506127921.433050:0:3044:0:(sec.c:2052:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:25.0:1506127921.433052:0:3044:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:25.0:1506127921.433053:0:3044:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:25.0:1506127921.433055:0:3044:0:(sec.c:2099:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:25.0:1506127921.433057:0:3044:0:(service.c:1939:ptlrpc_server_handle_req_in()) got req x1579288666077664 00000020:00000001:25.0:1506127921.433059:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:25.0:1506127921.433059:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x2f3e107735f0c2a 00000020:00000001:25.0:1506127921.433061:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000040:25.0:1506127921.433062:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507400 : new refcount 6 00000020:00000001:25.0:1506127921.433064:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000020:00000001:25.0:1506127921.433066:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000100:00000001:25.0:1506127921.433068:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127921.433069:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:25.0:1506127921.433070:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88082e03ce00. 00000020:00000010:25.0:1506127921.433085:0:3044:0:(lu_ucred.c:49:lu_ucred_key_init()) kmalloced 'value': 104 at ffff88080f761a80. 00000020:00000010:25.0:1506127921.433086:0:3044:0:(tgt_main.c:382:tgt_ses_key_init()) kmalloced 'value': 160 at ffff88080f8a9ec0. 00000100:00000001:25.0:1506127921.433089:0:3044:0:(service.c:1656:ptlrpc_server_request_add()) Process entered 00000100:00000001:25.0:1506127921.433090:0:3044:0:(service.c:1580:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:25.0:1506127921.433090:0:3044:0:(service.c:1607:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.433093:0:3044:0:(service.c:1695:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.433097:0:3044:0:(service.c:2003:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:25.0:1506127921.433099:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127921.433100:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127921.433101:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127921.433102:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127921.433103:0:3044:0:(service.c:2028:ptlrpc_server_handle_request()) Process entered 00000100:00000001:25.0:1506127921.433104:0:3044:0:(service.c:1809:ptlrpc_server_request_get()) Process entered 00000100:00100000:25.0:1506127921.433107:0:3044:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.1.109@o2ib, seq: 16 00000100:00000040:25.0:1506127921.433109:0:3044:0:(service.c:1840:ptlrpc_server_request_get()) RPC GETting export ffff88080f507400 : new rpc_count 2 00000100:00000001:25.0:1506127921.433110:0:3044:0:(service.c:1842:ptlrpc_server_request_get()) Process leaving (rc=18446612167001122896 : -131906708428720 : ffff880813802850) 00000100:00000040:25.0:1506127921.433113:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "New" -> "Interpret" req@ffff880813802850 x1579288666077664/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 304/0 e 0 to 0 dl 1506127927 ref 1 fl New:/0/ffffffff rc 0/-1 00000100:00000001:25.0:1506127921.433120:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127921.433121:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000100:00100000:25.0:1506127921.433123:0:3044:0:(service.c:2089:ptlrpc_server_handle_request()) Handling RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+6:3057:x1579288666077664:12345-192.168.1.109@o2ib:1000 00000100:00000200:25.0:1506127921.433126:0:3044:0:(service.c:2094:ptlrpc_server_handle_request()) got req 1579288666077664 00000020:00000001:25.0:1506127921.433127:0:3044:0:(tgt_handler.c:672:tgt_request_handle()) Process entered 00000020:00000001:25.0:1506127921.433129:0:3044:0:(tgt_handler.c:617:process_req_last_xid()) Process entered 00000020:00000001:25.0:1506127921.433130:0:3044:0:(tgt_handler.c:657:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.433131:0:3044:0:(tgt_handler.c:584:tgt_handler_find_check()) Process entered 00000020:00000001:25.0:1506127921.433132:0:3044:0:(tgt_handler.c:611:tgt_handler_find_check()) Process leaving (rc=18446744072654881248 : -1054670368 : ffffffffc12301e0) 00000020:00000001:25.0:1506127921.433134:0:3044:0:(tgt_handler.c:523:tgt_handle_recovery()) Process entered 00000020:00000040:25.0:1506127921.433137:0:3044:0:(tgt_handler.c:562:tgt_handle_recovery()) @@@ Got new replay req@ffff880813802850 x1579288666077664/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 304/0 e 0 to 0 dl 1506127927 ref 1 fl Interpret:/0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127921.433142:0:3044:0:(ldlm_lib.c:2705:target_queue_recovery_request()) Process entered 00010000:00080000:25.0:1506127921.433145:0:3044:0:(ldlm_lib.c:2788:target_queue_recovery_request()) @@@ not queueing req@ffff880813802850 x1579288666077664/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 304/0 e 0 to 0 dl 1506127927 ref 1 fl Interpret:/0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127921.433149:0:3044:0:(ldlm_lib.c:2789:target_queue_recovery_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127921.433151:0:3044:0:(tgt_handler.c:506:tgt_filter_recovery_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.433152:0:3044:0:(tgt_handler.c:573:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127921.433153:0:3044:0:(tgt_handler.c:382:tgt_handle_request0()) Process entered 00000020:00000001:25.0:1506127921.433155:0:3044:0:(tgt_handler.c:311:tgt_request_preprocess()) Process entered 00000020:00000001:25.0:1506127921.433156:0:3044:0:(tgt_handler.c:366:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.433157:0:3044:0:(out_handler.c:917:out_handle()) Process entered 00000020:00000010:25.0:1506127921.433159:0:3044:0:(out_handler.c:940:out_handle()) kmalloced 'update_bufs': 8 at ffff880825e00178. 00000100:00000001:25.0:1506127921.433161:0:3044:0:(pack_generic.c:334:lustre_pack_reply_v2()) Process entered 02000000:00000001:25.0:1506127921.433162:0:3044:0:(sec.c:2112:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:25.0:1506127921.433165:0:3044:0:(sec_null.c:318:null_alloc_rs()) kmalloced 'rs': 4576 at ffff88080f0a8000. 02000000:00000001:25.0:1506127921.433166:0:3044:0:(sec.c:2148:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.433181:0:3044:0:(pack_generic.c:368:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.433186:0:3044:0:(out_handler.c:205:out_attr_get()) Process entered 00000020:00000040:25.0:1506127921.433189:0:3044:0:(out_handler.c:233:out_attr_get()) soaked-MDT0003: insert attr get reply ffff88080f0a81e0 index 0: rc = 0 00000020:00000001:25.0:1506127921.433191:0:3044:0:(out_handler.c:238:out_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000002:25.0:1506127921.433193:0:3044:0:(lu_object.c:157:lu_object_put()) Add ffff8800ad2a5618 to site lru. hash: ffff88082b7a7080, bkt: ffffc9001aec0018, lru_len: 1 00000020:00000010:25.0:1506127921.433195:0:3044:0:(out_handler.c:1169:out_handle()) kfreed 'update_bufs': 8 at ffff880825e00178. 00000020:00000001:25.0:1506127921.433197:0:3044:0:(out_handler.c:1175:out_handle()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:25.0:1506127921.433198:0:3044:0:(ldlm_lib.c:2884:target_committed_to_req()) last_committed 30064771075, transno 0, xid 1579288666077664 00010000:00000001:25.0:1506127921.433201:0:3044:0:(ldlm_lib.c:2948:target_send_reply()) Process entered 00010000:00000200:25.0:1506127921.433203:0:3044:0:(ldlm_lib.c:2936:target_send_reply_msg()) @@@ sending reply req@ffff880813802850 x1579288666077664/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 304/4320 e 0 to 0 dl 1506127927 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000001:25.0:1506127921.433210:0:3044:0:(ldlm_lib.c:2895:target_pack_pool_reply()) Process entered 00010000:00000001:25.0:1506127921.433211:0:3044:0:(ldlm_lib.c:2903:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:25.0:1506127921.433213:0:3044:0:(import.c:1678:at_measured()) add 1 to ffff88081629fd50 time=1 v=1 (1 0 0 0) 00000100:00000001:25.0:1506127921.433216:0:3044:0:(connection.c:127:ptlrpc_connection_addref()) Process entered 00000100:00000040:25.0:1506127921.433218:0:3044:0:(connection.c:132:ptlrpc_connection_addref()) conn=ffff88082ede1660 refcount 10 to 192.168.1.109@o2ib 00000100:00000001:25.0:1506127921.433220:0:3044:0:(connection.c:134:ptlrpc_connection_addref()) Process leaving (rc=18446612167460263520 : -131906249288096 : ffff88082ede1660) 02000000:00000001:25.0:1506127921.433222:0:3044:0:(sec.c:2161:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:25.0:1506127921.433223:0:3044:0:(sec.c:2172:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.433224:0:3044:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:25.0:1506127921.433226:0:3044:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-192.168.1.109@o2ib 00000400:00000010:25.0:1506127921.433228:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761980. 00000100:00000200:25.0:1506127921.433230:0:3044:0:(niobuf.c:85:ptl_send_buf()) Sending 4288 bytes to portal 4, xid 1579288666077664, offset 192 00000400:00000010:25.0:1506127921.433232:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88082e03c600 (tot 194353484). 00000400:00000200:25.0:1506127921.433235:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.109@o2ib 00000400:00000200:25.0:1506127921.433245:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.109@o2ib(192.168.1.109@o2ib:192.168.1.109@o2ib) : PUT 00000800:00000200:25.0:1506127921.433249:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 4288 bytes in 1 frags to 12345-192.168.1.109@o2ib 00000800:00000200:25.0:1506127921.433255:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082edb4980] -> 192.168.1.109@o2ib (2) version: 12 00000800:00000200:25.0:1506127921.433257:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082dfb7200] (20)++ 00000800:00000200:25.0:1506127921.433259:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082dfb7200] (21)++ 00000800:00000200:25.0:1506127921.433263:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082dfb7200] (22)-- 00000100:00000001:25.0:1506127921.433264:0:3044:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.433266:0:3044:0:(connection.c:91:ptlrpc_connection_put()) Process entered 00000100:00000040:25.0:1506127921.433268:0:3044:0:(connection.c:119:ptlrpc_connection_put()) PUT conn=ffff88082ede1660 refcount 9 to 192.168.1.109@o2ib 00000100:00000001:25.0:1506127921.433269:0:3044:0:(connection.c:121:ptlrpc_connection_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127921.433271:0:3044:0:(ldlm_lib.c:2960:target_send_reply()) Process leaving 00000020:00000001:25.0:1506127921.433272:0:3044:0:(tgt_handler.c:475:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.433273:0:3044:0:(tgt_handler.c:780:tgt_request_handle()) Process leaving 00000100:00000040:25.0:1506127921.433276:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff880813802850 x1579288666077664/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 304/4288 e 0 to 0 dl 1506127927 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00100000:25.0:1506127921.433283:0:3044:0:(service.c:2135:ptlrpc_server_handle_request()) Handled RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+6:3057:x1579288666077664:12345-192.168.1.109@o2ib:1000 Request procesed in 162us (248us total) trans 0 rc 0/0 00000100:00100000:25.0:1506127921.433292:0:3044:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.1.109@o2ib, seq: 16 00000100:00000040:25.0:1506127921.433294:0:3044:0:(service.c:1034:ptlrpc_server_finish_active_request()) RPC PUTting export ffff88080f507400 : new rpc_count 1 00000100:00000001:25.0:1506127921.433296:0:3044:0:(service.c:1613:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:25.0:1506127921.433297:0:3044:0:(service.c:1624:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:25.0:1506127921.433299:0:3044:0:(lu_ucred.c:49:lu_ucred_key_fini()) kfreed 'info': 104 at ffff88080f761a80. 00000020:00000010:25.0:1506127921.433301:0:3044:0:(tgt_main.c:382:tgt_ses_key_fini()) kfreed 'info': 160 at ffff88080f8a9ec0. 00000020:00000010:25.0:1506127921.433303:0:3044:0:(lu_object.c:1595:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff88082e03ce00. 00000020:00000040:25.0:1506127921.433305:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507400 : new refcount 5 00000100:00000001:25.0:1506127921.433307:0:3044:0:(service.c:2155:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:25.0:1506127921.433310:0:3044:0:(niobuf.c:953:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000400:00000010:25.0:1506127921.433312:0:3044:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff8808138a1780. 00000400:00000010:25.0:1506127921.433314:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761a80. 00000400:00000001:25.0:1506127921.433316:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127921.433318:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127921.434044:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127921.434045:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127921.434046:0:3044:0:(service.c:1859:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:25.0:1506127921.434048:0:3044:0:(sec.c:2052:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:25.0:1506127921.434049:0:3044:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:25.0:1506127921.434050:0:3044:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:25.0:1506127921.434052:0:3044:0:(sec.c:2099:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:25.0:1506127921.434054:0:3044:0:(service.c:1939:ptlrpc_server_handle_req_in()) got req x1579288666077712 00000020:00000001:25.0:1506127921.434055:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:25.0:1506127921.434056:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x2f3e107735f0c2a 00000020:00000001:25.0:1506127921.434057:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000040:25.0:1506127921.434059:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507400 : new refcount 6 00000020:00000001:25.0:1506127921.434060:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000020:00000001:25.0:1506127921.434062:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000100:00000001:25.0:1506127921.434064:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127921.434065:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:25.0:1506127921.434066:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88082e03ce00. 00000020:00000010:25.0:1506127921.434068:0:3044:0:(lu_ucred.c:49:lu_ucred_key_init()) kmalloced 'value': 104 at ffff88080f761880. 00000020:00000010:25.0:1506127921.434070:0:3044:0:(tgt_main.c:382:tgt_ses_key_init()) kmalloced 'value': 160 at ffff88080f8a9ec0. 00000100:00000001:25.0:1506127921.434081:0:3044:0:(service.c:1656:ptlrpc_server_request_add()) Process entered 00000100:00000001:25.0:1506127921.434085:0:3044:0:(service.c:1580:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:25.0:1506127921.434086:0:3044:0:(service.c:1607:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.434088:0:3044:0:(service.c:1695:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.434113:0:3044:0:(service.c:2003:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:25.0:1506127921.434116:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127921.434117:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:25.0:1506127921.434118:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:25.0:1506127921.434119:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:25.0:1506127921.434120:0:3044:0:(service.c:2028:ptlrpc_server_handle_request()) Process entered 00000100:00000001:25.0:1506127921.434121:0:3044:0:(service.c:1809:ptlrpc_server_request_get()) Process entered 00000100:00100000:25.0:1506127921.434125:0:3044:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.1.109@o2ib, seq: 17 00000100:00000040:25.0:1506127921.434126:0:3044:0:(service.c:1840:ptlrpc_server_request_get()) RPC GETting export ffff88080f507400 : new rpc_count 2 00000100:00000001:25.0:1506127921.434128:0:3044:0:(service.c:1842:ptlrpc_server_request_get()) Process leaving (rc=18446612167001121872 : -131906708429744 : ffff880813802450) 00000100:00000040:25.0:1506127921.434131:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "New" -> "Interpret" req@ffff880813802450 x1579288666077712/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 360/0 e 0 to 0 dl 1506127927 ref 1 fl New:/0/ffffffff rc 0/-1 00000100:00000001:25.0:1506127921.434137:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:25.0:1506127921.434138:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000100:00100000:25.0:1506127921.434141:0:3044:0:(service.c:2089:ptlrpc_server_handle_request()) Handling RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+6:3000:x1579288666077712:12345-192.168.1.109@o2ib:1000 00000100:00000200:25.0:1506127921.434143:0:3044:0:(service.c:2094:ptlrpc_server_handle_request()) got req 1579288666077712 00000020:00000001:25.0:1506127921.434144:0:3044:0:(tgt_handler.c:672:tgt_request_handle()) Process entered 00000020:00000001:25.0:1506127921.434146:0:3044:0:(tgt_handler.c:617:process_req_last_xid()) Process entered 00000020:00000001:25.0:1506127921.434147:0:3044:0:(tgt_handler.c:657:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.434149:0:3044:0:(tgt_handler.c:584:tgt_handler_find_check()) Process entered 00000020:00000001:25.0:1506127921.434150:0:3044:0:(tgt_handler.c:611:tgt_handler_find_check()) Process leaving (rc=18446744072654881248 : -1054670368 : ffffffffc12301e0) 00000020:00000001:25.0:1506127921.434152:0:3044:0:(tgt_handler.c:523:tgt_handle_recovery()) Process entered 00000020:00000040:25.0:1506127921.434154:0:3044:0:(tgt_handler.c:562:tgt_handle_recovery()) @@@ Got new replay req@ffff880813802450 x1579288666077712/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 360/0 e 0 to 0 dl 1506127927 ref 1 fl Interpret:/0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127921.434160:0:3044:0:(ldlm_lib.c:2705:target_queue_recovery_request()) Process entered 00010000:00080000:25.0:1506127921.434162:0:3044:0:(ldlm_lib.c:2788:target_queue_recovery_request()) @@@ not queueing req@ffff880813802450 x1579288666077712/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 360/0 e 0 to 0 dl 1506127927 ref 1 fl Interpret:/0/ffffffff rc 0/-1 00010000:00000001:25.0:1506127921.434168:0:3044:0:(ldlm_lib.c:2789:target_queue_recovery_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127921.434169:0:3044:0:(tgt_handler.c:506:tgt_filter_recovery_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.434171:0:3044:0:(tgt_handler.c:573:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:25.0:1506127921.434172:0:3044:0:(tgt_handler.c:382:tgt_handle_request0()) Process entered 00000020:00000001:25.0:1506127921.434174:0:3044:0:(tgt_handler.c:311:tgt_request_preprocess()) Process entered 00000020:00000001:25.0:1506127921.434175:0:3044:0:(tgt_handler.c:366:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.434176:0:3044:0:(out_handler.c:917:out_handle()) Process entered 00000020:00000010:25.0:1506127921.434178:0:3044:0:(out_handler.c:940:out_handle()) kmalloced 'update_bufs': 8 at ffff880825e00178. 00000100:00000001:25.0:1506127921.434180:0:3044:0:(pack_generic.c:334:lustre_pack_reply_v2()) Process entered 02000000:00000001:25.0:1506127921.434181:0:3044:0:(sec.c:2112:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:25.0:1506127921.434183:0:3044:0:(sec_null.c:318:null_alloc_rs()) kmalloced 'rs': 4576 at ffff88080f0aa000. 02000000:00000001:25.0:1506127921.434185:0:3044:0:(sec.c:2148:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.434186:0:3044:0:(pack_generic.c:368:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:25.0:1506127921.434191:0:3044:0:(osd_handler.c:1612:osd_trans_create()) Process entered 00080000:00000010:25.0:1506127921.434192:0:3044:0:(osd_handler.c:1627:osd_trans_create()) kmalloced 'oh': 240 at ffff880822757500. 00080000:00000001:25.0:1506127921.434194:0:3044:0:(osd_handler.c:1650:osd_trans_create()) Process leaving (rc=18446612167252079872 : -131906457471744 : ffff880822757500) 00000020:00000001:25.0:1506127921.434196:0:3044:0:(out_handler.c:583:out_write()) Process entered 00000001:00000001:25.0:1506127921.434198:0:3044:0:(osd_io.c:1576:osd_declare_write()) Process entered 00000001:00000001:25.0:1506127921.434199:0:3044:0:(osd_quota.c:630:osd_declare_inode_qid()) Process entered 00000001:00000001:25.0:1506127921.434201:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127921.434202:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127921.434203:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.434204:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.434206:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127921.434207:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127921.434208:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.434209:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.434210:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127921.434211:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127921.434212:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.434213:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.434214:0:3044:0:(osd_quota.c:677:osd_declare_inode_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.434216:0:3044:0:(osd_io.c:1650:osd_declare_write()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.434218:0:3044:0:(out_handler.c:609:out_write()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.434219:0:3044:0:(out_handler.c:829:out_tx_end()) Process entered 00080000:00000001:25.0:1506127921.434220:0:3044:0:(osd_handler.c:1713:osd_trans_start()) Process entered 00000001:00000001:25.0:1506127921.434222:0:3044:0:(osd_io.c:1576:osd_declare_write()) Process entered 00000001:00000001:25.0:1506127921.434223:0:3044:0:(osd_quota.c:630:osd_declare_inode_qid()) Process entered 00000001:00000001:25.0:1506127921.434224:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127921.434225:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127921.434225:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.434227:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.434228:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127921.434229:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127921.434230:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.434231:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.434232:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:25.0:1506127921.434233:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:25.0:1506127921.434234:0:3044:0:(qsd_handler.c:844:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.434235:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.434236:0:3044:0:(osd_quota.c:677:osd_declare_inode_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:25.0:1506127921.434238:0:3044:0:(osd_io.c:1650:osd_declare_write()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:25.0:1506127921.434243:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88082e03c200. 00000020:00000010:25.0:1506127921.434245:0:3044:0:(update_records.c:1217:update_key_init()) kmalloced 'value': 416 at ffff88082e03b600. 00000004:00000010:25.0:1506127921.434247:0:3044:0:(osp_dev.c:1869:osp_txn_key_init()) kmalloced 'value': 8 at ffff880825e001a0. 00080000:00000001:25.0:1506127921.434249:0:3044:0:(osd_handler.c:1780:osd_trans_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:25.0:1506127921.434250:0:3044:0:(out_lib.c:681:out_tx_write_exec()) write [0x200000009:0x3:0x0] pos 32 buf ffff880813200130, len 32 00000020:00000040:25.0:1506127921.434258:0:3044:0:(out_handler.c:863:out_tx_end()) soaked-MDT0003-osd: executed 0/1: rc = 0 00080000:00000001:25.0:1506127921.434260:0:3044:0:(osd_handler.c:1834:osd_trans_stop()) Process entered 00000004:00000040:25.0:1506127921.434262:0:3044:0:(mdt_recovery.c:87:mdt_txn_stop_cb()) More than one transaction 00000001:00000001:25.0:1506127921.434263:0:3044:0:(tgt_lastrcvd.c:1204:tgt_last_rcvd_update()) Process entered 00000001:00000002:25.0:1506127921.434265:0:3044:0:(tgt_lastrcvd.c:1245:tgt_last_rcvd_update()) transno = 30064771076, last_committed = 30064771075 00000001:00000010:25.0:1506127921.434267:0:3044:0:(tgt_lastrcvd.c:909:tgt_last_commit_cb_add()) kmalloced 'ccb': 96 at ffff8808138a1ae0. 00000001:00000040:25.0:1506127921.434269:0:3044:0:(tgt_lastrcvd.c:914:tgt_last_commit_cb_add()) callback GETting export ffff88080f507400 : new cb_count 1 00000020:00000040:25.0:1506127921.434270:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507400 : new refcount 7 00000001:00000001:25.0:1506127921.434274:0:3044:0:(tgt_lastrcvd.c:1387:tgt_last_rcvd_update()) Process leaving (rc=0 : 0 : 0) 00040000:00000001:25.0:1506127921.434938:0:3044:0:(qsd_handler.c:1080:qsd_op_end()) Process entered 00040000:00000001:25.0:1506127921.434939:0:3044:0:(qsd_handler.c:1092:qsd_op_end()) Process leaving 00080000:00000001:25.0:1506127921.434940:0:3044:0:(osd_handler.c:1905:osd_trans_stop()) Process leaving (rc=0 : 0 : 0) 00000020:00000002:25.0:1506127921.434942:0:3044:0:(lu_object.c:157:lu_object_put()) Add ffff88081f236f58 to site lru. hash: ffff88082b7a7080, bkt: ffffc900159f2018, lru_len: 1 00000020:00000001:25.0:1506127921.434945:0:3044:0:(out_handler.c:881:out_tx_end()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:25.0:1506127921.434947:0:3044:0:(out_handler.c:1169:out_handle()) kfreed 'update_bufs': 8 at ffff880825e00178. 00000020:00000001:25.0:1506127921.434948:0:3044:0:(out_handler.c:1175:out_handle()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:25.0:1506127921.434950:0:3044:0:(ldlm_lib.c:2884:target_committed_to_req()) last_committed 30064771076, transno 30064771076, xid 1579288666077712 00010000:00000001:25.0:1506127921.434952:0:3044:0:(ldlm_lib.c:2948:target_send_reply()) Process entered 00010000:00000200:25.0:1506127921.434955:0:3044:0:(ldlm_lib.c:2936:target_send_reply_msg()) @@@ sending reply req@ffff880813802450 x1579288666077712/t30064771076(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 360/4320 e 0 to 0 dl 1506127927 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000001:25.0:1506127921.434963:0:3044:0:(ldlm_lib.c:2895:target_pack_pool_reply()) Process entered 00010000:00000001:25.0:1506127921.434964:0:3044:0:(ldlm_lib.c:2903:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:25.0:1506127921.434966:0:3044:0:(import.c:1678:at_measured()) add 1 to ffff88081629fd50 time=1 v=1 (1 0 0 0) 00000100:00000001:25.0:1506127921.434969:0:3044:0:(connection.c:127:ptlrpc_connection_addref()) Process entered 00000100:00000040:25.0:1506127921.434971:0:3044:0:(connection.c:132:ptlrpc_connection_addref()) conn=ffff88082ede1660 refcount 10 to 192.168.1.109@o2ib 00000100:00000001:25.0:1506127921.434972:0:3044:0:(connection.c:134:ptlrpc_connection_addref()) Process leaving (rc=18446612167460263520 : -131906249288096 : ffff88082ede1660) 02000000:00000001:25.0:1506127921.434974:0:3044:0:(sec.c:2161:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:25.0:1506127921.434976:0:3044:0:(sec.c:2172:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.434977:0:3044:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:25.0:1506127921.434979:0:3044:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-192.168.1.109@o2ib 00000400:00000010:25.0:1506127921.434981:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761780. 00000100:00000200:25.0:1506127921.434983:0:3044:0:(niobuf.c:85:ptl_send_buf()) Sending 4288 bytes to portal 4, xid 1579288666077712, offset 192 00000400:00000010:25.0:1506127921.434986:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88082e03cc00 (tot 194353484). 00000400:00000200:25.0:1506127921.434989:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.109@o2ib 00000400:00000200:25.0:1506127921.434998:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.109@o2ib(192.168.1.109@o2ib:192.168.1.109@o2ib) : PUT 00000800:00000200:25.0:1506127921.435002:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 4288 bytes in 1 frags to 12345-192.168.1.109@o2ib 00000800:00000200:25.0:1506127921.435008:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082edb4980] -> 192.168.1.109@o2ib (2) version: 12 00000800:00000200:25.0:1506127921.435010:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082dfb7200] (20)++ 00000800:00000200:25.0:1506127921.435012:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082dfb7200] (21)++ 00000800:00000200:25.0:1506127921.435016:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082dfb7200] (22)-- 00000100:00000001:25.0:1506127921.435018:0:3044:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:25.0:1506127921.435019:0:3044:0:(connection.c:91:ptlrpc_connection_put()) Process entered 00000100:00000040:25.0:1506127921.435021:0:3044:0:(connection.c:119:ptlrpc_connection_put()) PUT conn=ffff88082ede1660 refcount 9 to 192.168.1.109@o2ib 00000100:00000001:25.0:1506127921.435023:0:3044:0:(connection.c:121:ptlrpc_connection_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:25.0:1506127921.435024:0:3044:0:(ldlm_lib.c:2960:target_send_reply()) Process leaving 00000020:00000001:25.0:1506127921.435026:0:3044:0:(tgt_handler.c:475:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:25.0:1506127921.435027:0:3044:0:(tgt_handler.c:780:tgt_request_handle()) Process leaving 00000100:00000040:25.0:1506127921.435030:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff880813802450 x1579288666077712/t30064771076(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 360/4288 e 0 to 0 dl 1506127927 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00100000:25.0:1506127921.435037:0:3044:0:(service.c:2135:ptlrpc_server_handle_request()) Handled RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+6:3000:x1579288666077712:12345-192.168.1.109@o2ib:1000 Request procesed in 898us (1004us total) trans 30064771076 rc 0/0 00000100:00100000:25.0:1506127921.435046:0:3044:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.1.109@o2ib, seq: 17 00000100:00000040:25.0:1506127921.435048:0:3044:0:(service.c:1034:ptlrpc_server_finish_active_request()) RPC PUTting export ffff88080f507400 : new rpc_count 1 00000100:00000001:25.0:1506127921.435051:0:3044:0:(service.c:1613:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:25.0:1506127921.435053:0:3044:0:(service.c:1624:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:25.0:1506127921.435055:0:3044:0:(lu_ucred.c:49:lu_ucred_key_fini()) kfreed 'info': 104 at ffff88080f761880. 00000020:00000010:25.0:1506127921.435058:0:3044:0:(tgt_main.c:382:tgt_ses_key_fini()) kfreed 'info': 160 at ffff88080f8a9ec0. 00000020:00000010:25.0:1506127921.435061:0:3044:0:(lu_object.c:1595:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff88082e03ce00. 00000020:00000040:25.0:1506127921.435064:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507400 : new refcount 5 00000100:00000001:25.0:1506127921.435067:0:3044:0:(service.c:2155:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:25.0:1506127921.435080:0:3044:0:(niobuf.c:953:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000400:00000010:25.0:1506127921.435083:0:3044:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff8808138a1720. 00000400:00000010:25.0:1506127921.435086:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88080f761880. 00000400:00000001:25.0:1506127921.435089:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:25.0:1506127921.435091:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:10.0:1506127922.396274:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:10.0:1506127922.396278:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:10.0:1506127922.396281:0:3044:0:(service.c:1437:ptlrpc_at_check_timed()) Process entered 00000100:00000040:10.0:1506127922.396284:0:3044:0:(service.c:1184:ptlrpc_at_set_timer()) armed mdt_out at +24s 00000100:00001000:10.0:1506127922.396287:0:3044:0:(service.c:1501:ptlrpc_at_check_timed()) timeout in +5s, asking for 30 secs on 1 early replies 00000100:00000001:10.0:1506127922.396289:0:3044:0:(service.c:1275:ptlrpc_at_send_early_reply()) Process entered 00000100:00001000:10.0:1506127922.396306:0:3044:0:(service.c:1288:ptlrpc_at_send_early_reply()) @@@ sending early reply (deadline +5s, margin +4s) for 1+30 req@ffff880813803c50 x1579288666077392/t0(0) o400->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:322/0 lens 224/0 e 0 to 0 dl 1506127927 ref 2 fl Complete:/c0/ffffffff rc 0/-1 00000100:00000010:10.0:1506127922.396317:0:3044:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 768 at ffff8808198f2a00. 00000100:00000010:10.0:1506127922.396319:0:3044:0:(service.c:1353:ptlrpc_at_send_early_reply()) kmalloced 'reqmsg': 224 at ffff88042bf1c200. 00000020:00000001:10.0:1506127922.396322:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:10.0:1506127922.396323:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x2f3e107735f0c2a 00000020:00000001:10.0:1506127922.396324:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000040:10.0:1506127922.396326:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507400 : new refcount 6 00000020:00000001:10.0:1506127922.396327:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000020:00000001:10.0:1506127922.396329:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000100:00000040:10.0:1506127922.396331:0:3044:0:(service.c:1391:ptlrpc_at_send_early_reply()) RPC GETting export ffff88080f507400 : new rpc_count 2 00000100:00000001:10.0:1506127922.396333:0:3044:0:(pack_generic.c:334:lustre_pack_reply_v2()) Process entered 02000000:00000001:10.0:1506127922.396334:0:3044:0:(sec.c:2112:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:10.0:1506127922.396336:0:3044:0:(sec_null.c:318:null_alloc_rs()) kmalloced 'rs': 480 at ffff88080f602400. 02000000:00000001:10.0:1506127922.396337:0:3044:0:(sec.c:2148:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127922.396339:0:3044:0:(pack_generic.c:368:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:10.0:1506127922.396342:0:3044:0:(ldlm_lib.c:2895:target_pack_pool_reply()) Process entered 00010000:00000001:10.0:1506127922.396344:0:3044:0:(ldlm_lib.c:2903:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127922.396346:0:3044:0:(connection.c:127:ptlrpc_connection_addref()) Process entered 00000100:00000040:10.0:1506127922.396348:0:3044:0:(connection.c:132:ptlrpc_connection_addref()) conn=ffff88082ede1660 refcount 10 to 192.168.1.109@o2ib 00000100:00000001:10.0:1506127922.396350:0:3044:0:(connection.c:134:ptlrpc_connection_addref()) Process leaving (rc=18446612167460263520 : -131906249288096 : ffff88082ede1660) 02000000:00000001:10.0:1506127922.396352:0:3044:0:(sec.c:2161:sptlrpc_svc_wrap_reply()) Process entered 00000001:00000040:10.0:1506127922.396359:0:3044:0:(linux-crypto.c:118:cfs_crypto_hash_alloc()) Using crypto hash: crc32 (crc32-pclmul) speed 2143 MB/s 02000000:00000001:10.0:1506127922.396365:0:3044:0:(sec.c:2172:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127922.396367:0:3044:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:10.0:1506127922.396369:0:3044:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-192.168.1.109@o2ib 00000400:00000010:10.0:1506127922.396371:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88082bec7a00. 00000100:00000200:10.0:1506127922.396374:0:3044:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 4, xid 1579288666077392, offset 0 00000400:00000010:10.0:1506127922.396377:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88080f601e00 (tot 194353484). 00000400:00000200:10.0:1506127922.396380:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.109@o2ib 00000400:00000200:10.0:1506127922.396391:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.109@o2ib(192.168.1.109@o2ib:192.168.1.109@o2ib) : PUT 00000800:00000200:10.0:1506127922.396395:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 192 bytes in 1 frags to 12345-192.168.1.109@o2ib 00000800:00000200:10.0:1506127922.396400:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082edb4980] -> 192.168.1.109@o2ib (2) version: 12 00000800:00000200:10.0:1506127922.396403:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082dfb7200] (20)++ 00000800:00000200:10.0:1506127922.396405:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082dfb7200] (21)++ 00000800:00000200:10.0:1506127922.396409:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082dfb7200] (22)-- 00000100:00000001:10.0:1506127922.396411:0:3044:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127922.396412:0:3044:0:(connection.c:91:ptlrpc_connection_put()) Process entered 00000100:00000040:10.0:1506127922.396414:0:3044:0:(connection.c:119:ptlrpc_connection_put()) PUT conn=ffff88082ede1660 refcount 9 to 192.168.1.109@o2ib 00000100:00000001:10.0:1506127922.396416:0:3044:0:(connection.c:121:ptlrpc_connection_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:10.0:1506127922.396417:0:3044:0:(service.c:1415:ptlrpc_at_send_early_reply()) RPC PUTting export ffff88080f507400 : new rpc_count 1 00000020:00000040:10.0:1506127922.396419:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507400 : new refcount 5 00000100:00000010:10.0:1506127922.396421:0:3044:0:(service.c:1419:ptlrpc_at_send_early_reply()) kfreed 'reqmsg': 224 at ffff88042bf1c200. 00000100:00000010:10.0:1506127922.396423:0:3044:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 768 at ffff8808198f2a00. 00000100:00000001:10.0:1506127922.396425:0:3044:0:(service.c:1422:ptlrpc_at_send_early_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127922.396427:0:3044:0:(service.c:1527:ptlrpc_at_check_timed()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:10.0:1506127922.396428:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:10.0:1506127922.396429:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:10.0:1506127946.450202:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:10.0:1506127946.450206:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:10.0:1506127946.450209:0:3044:0:(service.c:1437:ptlrpc_at_check_timed()) Process entered 00000100:00000001:10.0:1506127946.450210:0:3044:0:(service.c:1449:ptlrpc_at_check_timed()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:10.0:1506127946.450214:0:3044:0:(niobuf.c:953:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000400:00000010:10.0:1506127946.450219:0:3044:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88081f3fc180. 00000400:00000010:10.0:1506127946.450223:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88042c0c5680. 00000100:00000200:10.0:1506127946.450227:0:3044:0:(niobuf.c:953:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000400:00000010:10.0:1506127946.450228:0:3044:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88081f3fc1e0. 00000400:00000010:10.0:1506127946.450230:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88042c0c4c00. 00000400:00000001:10.0:1506127946.450231:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:10.0:1506127946.450233:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:10.0:1506127961.236166:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:10.0:1506127961.236171:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:10.0:1506127961.236174:0:3044:0:(service.c:1859:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:10.0:1506127961.236175:0:3044:0:(sec.c:2052:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:10.0:1506127961.236177:0:3044:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:10.0:1506127961.236179:0:3044:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:10.0:1506127961.236183:0:3044:0:(sec.c:2099:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:10.0:1506127961.236186:0:3044:0:(service.c:1939:ptlrpc_server_handle_req_in()) got req x1579288666082112 00000020:00000001:10.0:1506127961.236188:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:10.0:1506127961.236189:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x2f3e107735f0c2a 00000020:00000001:10.0:1506127961.236191:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000040:10.0:1506127961.236194:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507400 : new refcount 5 00000020:00000001:10.0:1506127961.236196:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000020:00000001:10.0:1506127961.236198:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000100:00000001:10.0:1506127961.236202:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:10.0:1506127961.236203:0:3044:0:(service.c:1074:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:10.0:1506127961.236206:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88080f601400. 00000020:00000010:10.0:1506127961.236209:0:3044:0:(lu_ucred.c:49:lu_ucred_key_init()) kmalloced 'value': 104 at ffff88042c0c4300. 00000020:00000010:10.0:1506127961.236212:0:3044:0:(tgt_main.c:382:tgt_ses_key_init()) kmalloced 'value': 160 at ffff88080f7b32c0. 00000100:00000040:10.0:1506127961.236215:0:3044:0:(service.c:1184:ptlrpc_at_set_timer()) armed mdt_out at +1s 00000100:00000001:10.0:1506127961.236217:0:3044:0:(service.c:1656:ptlrpc_server_request_add()) Process entered 00000100:00000001:10.0:1506127961.236219:0:3044:0:(service.c:1580:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:10.0:1506127961.236220:0:3044:0:(service.c:1607:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.236223:0:3044:0:(service.c:1695:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.236232:0:3044:0:(service.c:2003:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:10.0:1506127961.236235:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:10.0:1506127961.236237:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:10.0:1506127961.236238:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:10.0:1506127961.236239:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:10.0:1506127961.236241:0:3044:0:(service.c:2028:ptlrpc_server_handle_request()) Process entered 00000100:00000001:10.0:1506127961.236242:0:3044:0:(service.c:1809:ptlrpc_server_request_get()) Process entered 00000100:00100000:10.0:1506127961.236247:0:3044:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.1.109@o2ib, seq: 18 00000100:00000040:10.0:1506127961.236249:0:3044:0:(service.c:1840:ptlrpc_server_request_get()) RPC GETting export ffff88080f507400 : new rpc_count 1 00000100:00000001:10.0:1506127961.236251:0:3044:0:(service.c:1842:ptlrpc_server_request_get()) Process leaving (rc=18446612167001120848 : -131906708430768 : ffff880813802050) 00000100:00000040:10.0:1506127961.236255:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "New" -> "Interpret" req@ffff880813802050 x1579288666082112/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:362/0 lens 376/0 e 0 to 0 dl 1506127967 ref 1 fl New:/0/ffffffff rc 0/-1 00000100:00000001:10.0:1506127961.236264:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:10.0:1506127961.236265:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000100:00100000:10.0:1506127961.236268:0:3044:0:(service.c:2089:ptlrpc_server_handle_request()) Handling RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+5:3000:x1579288666082112:12345-192.168.1.109@o2ib:1000 00000100:00000200:10.0:1506127961.236271:0:3044:0:(service.c:2094:ptlrpc_server_handle_request()) got req 1579288666082112 00000020:00000001:10.0:1506127961.236273:0:3044:0:(tgt_handler.c:672:tgt_request_handle()) Process entered 00000020:00000001:10.0:1506127961.236275:0:3044:0:(tgt_handler.c:617:process_req_last_xid()) Process entered 00000020:00000001:10.0:1506127961.236277:0:3044:0:(tgt_handler.c:657:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.236279:0:3044:0:(tgt_handler.c:584:tgt_handler_find_check()) Process entered 00000020:00000001:10.0:1506127961.236280:0:3044:0:(tgt_handler.c:611:tgt_handler_find_check()) Process leaving (rc=18446744072654881248 : -1054670368 : ffffffffc12301e0) 00000020:00000001:10.0:1506127961.236283:0:3044:0:(tgt_handler.c:523:tgt_handle_recovery()) Process entered 00000020:00000001:10.0:1506127961.236285:0:3044:0:(tgt_handler.c:573:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:10.0:1506127961.236287:0:3044:0:(tgt_handler.c:382:tgt_handle_request0()) Process entered 00000020:00000001:10.0:1506127961.236288:0:3044:0:(tgt_handler.c:311:tgt_request_preprocess()) Process entered 00000020:00000001:10.0:1506127961.236290:0:3044:0:(tgt_handler.c:366:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.236293:0:3044:0:(out_handler.c:917:out_handle()) Process entered 00000020:00000010:10.0:1506127961.236297:0:3044:0:(out_handler.c:940:out_handle()) kmalloced 'update_bufs': 24 at ffff880826a71b60. 00000100:00000001:10.0:1506127961.236301:0:3044:0:(niobuf.c:130:ptlrpc_prep_bulk_exp()) Process entered 00000100:00000010:10.0:1506127961.236303:0:3044:0:(client.c:129:ptlrpc_new_bulk()) kmalloced 'desc': 280 at ffff88080f601c00. 00000100:00000010:10.0:1506127961.236305:0:3044:0:(client.c:139:ptlrpc_new_bulk()) kmalloced '((desc)->bd_u.bd_kvec.bd_kvec)': 48 at ffff88042b7fc340. 00000020:00000040:10.0:1506127961.236307:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507400 : new refcount 6 00000020:00000010:10.0:1506127961.236311:0:3044:0:(out_handler.c:967:out_handle()) kmalloced 'update_bufs[i]': 4096 at ffff88080f54f000. 00000100:00000001:10.0:1506127961.236312:0:3044:0:(client.c:235:ptlrpc_prep_bulk_frag()) Process entered 00000100:00000001:10.0:1506127961.236314:0:3044:0:(client.c:251:ptlrpc_prep_bulk_frag()) Process leaving (rc=4096 : 4096 : 1000) 00000020:00000010:10.0:1506127961.236340:0:3044:0:(out_handler.c:967:out_handle()) kmalloced 'update_bufs[i]': 32841 at ffff88080f180000. 00000100:00000001:10.0:1506127961.236341:0:3044:0:(client.c:235:ptlrpc_prep_bulk_frag()) Process entered 00000100:00000001:10.0:1506127961.236342:0:3044:0:(client.c:251:ptlrpc_prep_bulk_frag()) Process leaving (rc=36937 : 36937 : 9049) 00000020:00000010:10.0:1506127961.236346:0:3044:0:(out_handler.c:967:out_handle()) kmalloced 'update_bufs[i]': 4096 at ffff88083fcb1000. 00000100:00000001:10.0:1506127961.236347:0:3044:0:(client.c:235:ptlrpc_prep_bulk_frag()) Process entered 00000100:00000001:10.0:1506127961.236348:0:3044:0:(client.c:251:ptlrpc_prep_bulk_frag()) Process leaving (rc=41033 : 41033 : a049) 00010000:00000001:10.0:1506127961.236351:0:3044:0:(ldlm_lib.c:3167:target_bulk_io()) Process entered 00000100:00000001:10.0:1506127961.236353:0:3044:0:(niobuf.c:164:ptlrpc_start_bulk_transfer()) Process entered 00000400:00000010:10.0:1506127961.236356:0:3044:0:(lib-lnet.h:229:lnet_md_alloc()) alloc '(md)': 160 at ffff88080f7b3380 (tot 195264012). 00000400:00000010:10.0:1506127961.236361:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88080f601a00 (tot 195264412). 00000400:00000200:10.0:1506127961.236365:0:3044:0:(lib-move.c:2998:LNetGet()) LNetGet -> 12345-192.168.1.109@o2ib 00000400:00000200:10.0:1506127961.236377:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.109@o2ib(192.168.1.109@o2ib:192.168.1.109@o2ib) : GET 00000800:00000200:10.0:1506127961.236382:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 0 bytes in 0 frags to 12345-192.168.1.109@o2ib 00000400:00000010:10.0:1506127961.236395:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88080f601e00 (tot 195264812). 00000400:00000200:10.0:1506127961.236400:0:3044:0:(lib-move.c:2877:lnet_create_reply_msg()) 192.168.1.111@o2ib: Reply from 12345-192.168.1.109@o2ib md ffff88080f7b3380 00000800:00000200:10.0:1506127961.236404:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082edb4980] -> 192.168.1.109@o2ib (2) version: 12 00000800:00000200:10.0:1506127961.236407:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082dfb7200] (20)++ 00000800:00000200:10.0:1506127961.236409:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082dfb7200] (21)++ 00000800:00000200:10.0:1506127961.236414:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082dfb7200] (22)-- 00000100:00000200:10.0:1506127961.236418:0:3044:0:(niobuf.c:262:ptlrpc_start_bulk_transfer()) Transferring 3 pages 41033 bytes via portal 14 id 12345-192.168.1.109@o2ib mbits 0x59c5acbf08b40-0x59c5acbf08b40 00000100:00000001:10.0:1506127961.236422:0:3044:0:(niobuf.c:264:ptlrpc_start_bulk_transfer()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:10.0:1506127961.236738:0:3044:0:(ldlm_lib.c:3268:target_bulk_io()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.236746:0:3044:0:(pack_generic.c:334:lustre_pack_reply_v2()) Process entered 02000000:00000001:10.0:1506127961.236747:0:3044:0:(sec.c:2112:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:10.0:1506127961.236753:0:3044:0:(sec_null.c:318:null_alloc_rs()) kmalloced 'rs': 4576 at ffff88080f142000. 02000000:00000001:10.0:1506127961.236756:0:3044:0:(sec.c:2148:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.236758:0:3044:0:(pack_generic.c:368:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.236763:0:3044:0:(lu_object.c:243:lu_object_alloc()) Process entered 00000004:00000001:10.0:1506127961.236766:0:3044:0:(mdt_handler.c:5144:mdt_object_alloc()) Process entered 00000004:00000010:10.0:1506127961.236768:0:3044:0:(mdt_handler.c:5146:mdt_object_alloc()) slab-alloced 'mo': 216 at ffff8808263ef878. 00000004:00000001:10.0:1506127961.236771:0:3044:0:(mdt_handler.c:5161:mdt_object_alloc()) Process leaving (rc=18446612167315617992 : -131906393933624 : ffff8808263ef8c8) 00000004:00000001:10.0:1506127961.236774:0:3044:0:(mdt_handler.c:5173:mdt_object_init()) Process entered 00000004:00000040:10.0:1506127961.236776:0:3044:0:(mdt_handler.c:5176:mdt_object_init()) object init, fid = [0x2c0003ab2:0x3:0x0] 00000004:00000010:10.0:1506127961.236780:0:3044:0:(mdd_object.c:120:mdd_object_alloc()) slab-alloced 'mdd_obj': 80 at ffff88042cbec370. 00000004:00000001:10.0:1506127961.236782:0:3044:0:(mdt_handler.c:5185:mdt_object_init()) Process leaving (rc=0 : 0 : 0) 00000004:00000001:10.0:1506127961.236784:0:3044:0:(mdd_object.c:143:mdd_object_init()) Process entered 00000004:00000001:10.0:1506127961.236786:0:3044:0:(lod_dev.c:210:lod_object_alloc()) Process entered 00000004:00000010:10.0:1506127961.236788:0:3044:0:(lod_dev.c:212:lod_object_alloc()) slab-alloced 'lod_obj': 104 at ffff88082aa63750. 00000004:00000001:10.0:1506127961.236790:0:3044:0:(lod_dev.c:221:lod_object_alloc()) Process leaving (rc=18446612167389493072 : -131906320058544 : ffff88082aa63750) 00000004:00000001:10.0:1506127961.236792:0:3044:0:(mdd_object.c:153:mdd_object_init()) Process leaving (rc=0 : 0 : 0) 00000004:00000001:10.0:1506127961.236794:0:3044:0:(lod_object.c:5108:lod_object_init()) Process entered 00000004:00000001:10.0:1506127961.236796:0:3044:0:(lod_dev.c:127:lod_fld_lookup()) Process entered 80000000:00000001:10.0:1506127961.236799:0:3044:0:(fld_handler.c:242:fld_server_lookup()) Process entered 80000000:00000001:10.0:1506127961.236801:0:3044:0:(fld_handler.c:207:fld_local_lookup()) Process entered 80000000:00000001:10.0:1506127961.236803:0:3044:0:(fld_cache.c:530:fld_cache_lookup()) Process entered 80000000:00000001:10.0:1506127961.236806:0:3044:0:(fld_cache.c:549:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:10.0:1506127961.236808:0:3044:0:(fld_handler.c:224:fld_local_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:10.0:1506127961.236810:0:3044:0:(fld_handler.c:246:fld_server_lookup()) Process leaving (rc=0 : 0 : 0) 00000004:00000040:10.0:1506127961.236811:0:3044:0:(lod_dev.c:168:lod_fld_lookup()) soaked-MDT0003-mdtlov: got tgt 3 for sequence: 0x2c0003ab2 00000004:00000001:10.0:1506127961.236814:0:3044:0:(lod_dev.c:170:lod_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00080000:00000010:10.0:1506127961.236818:0:3044:0:(osd_handler.c:332:osd_object_alloc()) kmalloced 'mo': 200 at ffff88042bf1ea00. 00000004:00000001:10.0:1506127961.236820:0:3044:0:(lod_object.c:5159:lod_object_init()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:10.0:1506127961.236823:0:3044:0:(osd_handler.c:994:osd_fid_lookup()) Process entered 00080000:00000001:10.0:1506127961.236825:0:3044:0:(osd_handler.c:1016:osd_fid_lookup()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:10.0:1506127961.236829:0:3044:0:(lu_object.c:295:lu_object_alloc()) Process leaving (rc=18446612167315617992 : -131906393933624 : ffff8808263ef8c8) 00080000:00000001:10.0:1506127961.236836:0:3044:0:(osd_handler.c:1612:osd_trans_create()) Process entered 00080000:00000010:10.0:1506127961.236838:0:3044:0:(osd_handler.c:1627:osd_trans_create()) kmalloced 'oh': 240 at ffff88042bf1fa00. 00080000:00000001:10.0:1506127961.236841:0:3044:0:(osd_handler.c:1650:osd_trans_create()) Process leaving (rc=18446612150231366144 : -131923478185472 : ffff88042bf1fa00) 00000020:00000001:10.0:1506127961.236844:0:3044:0:(out_handler.c:116:out_create()) Process entered 00080000:00000001:10.0:1506127961.236848:0:3044:0:(osd_handler.c:3101:osd_declare_create()) Process entered 00000001:00000001:10.0:1506127961.236851:0:3044:0:(osd_quota.c:630:osd_declare_inode_qid()) Process entered 00000001:00000001:10.0:1506127961.236854:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:10.0:1506127961.236856:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:10.0:1506127961.236859:0:3044:0:(qsd_handler.c:859:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:10.0:1506127961.236860:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:10.0:1506127961.236862:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:10.0:1506127961.236863:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:10.0:1506127961.236864:0:3044:0:(qsd_handler.c:859:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:10.0:1506127961.236865:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:10.0:1506127961.236866:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:10.0:1506127961.236867:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:10.0:1506127961.236868:0:3044:0:(qsd_handler.c:859:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:10.0:1506127961.236869:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:10.0:1506127961.236871:0:3044:0:(osd_quota.c:677:osd_declare_inode_qid()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:10.0:1506127961.236874:0:3044:0:(osd_handler.c:3133:osd_declare_create()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.236877:0:3044:0:(out_handler.c:155:out_create()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.236884:0:3044:0:(out_handler.c:473:out_index_insert()) Process entered 00080000:00000001:10.0:1506127961.236889:0:3044:0:(osd_handler.c:5333:osd_index_declare_ea_insert()) Process entered 00000001:00000001:10.0:1506127961.236891:0:3044:0:(osd_quota.c:630:osd_declare_inode_qid()) Process entered 00000001:00000001:10.0:1506127961.236892:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:10.0:1506127961.236893:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:10.0:1506127961.236894:0:3044:0:(qsd_handler.c:851:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:10.0:1506127961.236895:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:10.0:1506127961.236897:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:10.0:1506127961.236897:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:10.0:1506127961.236898:0:3044:0:(qsd_handler.c:851:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:10.0:1506127961.236899:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:10.0:1506127961.236901:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:10.0:1506127961.236901:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:10.0:1506127961.236902:0:3044:0:(qsd_handler.c:851:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:10.0:1506127961.236904:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:10.0:1506127961.236905:0:3044:0:(osd_quota.c:677:osd_declare_inode_qid()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:10.0:1506127961.236906:0:3044:0:(osd_handler.c:5374:osd_index_declare_ea_insert()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.236909:0:3044:0:(out_handler.c:516:out_index_insert()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.236913:0:3044:0:(out_handler.c:583:out_write()) Process entered 00000001:00000001:10.0:1506127961.236917:0:3044:0:(osd_io.c:1576:osd_declare_write()) Process entered 00000001:00000001:10.0:1506127961.236919:0:3044:0:(osd_quota.c:630:osd_declare_inode_qid()) Process entered 00000001:00000001:10.0:1506127961.236920:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:10.0:1506127961.236921:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:10.0:1506127961.236921:0:3044:0:(qsd_handler.c:851:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:10.0:1506127961.236923:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:10.0:1506127961.236924:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:10.0:1506127961.236925:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:10.0:1506127961.236926:0:3044:0:(qsd_handler.c:851:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:10.0:1506127961.236927:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:10.0:1506127961.236928:0:3044:0:(osd_quota.c:536:osd_declare_qid()) Process entered 00040000:00000001:10.0:1506127961.236929:0:3044:0:(qsd_handler.c:832:qsd_op_begin()) Process entered 00040000:00000001:10.0:1506127961.236929:0:3044:0:(qsd_handler.c:851:qsd_op_begin()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:10.0:1506127961.236931:0:3044:0:(osd_quota.c:601:osd_declare_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:10.0:1506127961.236932:0:3044:0:(osd_quota.c:677:osd_declare_inode_qid()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:10.0:1506127961.236934:0:3044:0:(osd_io.c:1650:osd_declare_write()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.236936:0:3044:0:(out_handler.c:609:out_write()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.236937:0:3044:0:(out_handler.c:583:out_write()) Process entered 00000020:00020000:10.0:1506127961.236938:0:3044:0:(out_handler.c:597:out_write()) soaked-MDT0003: empty or wrong size 0 pos: rc = -61 00000020:00000001:10.0:1506127961.252079:0:3044:0:(out_handler.c:598:out_write()) Process leaving (rc=18446744073709551555 : -61 : ffffffffffffffc3) 00000020:00000001:10.0:1506127961.252083:0:3044:0:(out_handler.c:1150:out_handle()) Process leaving via out (rc=18446744073709551555 : -61 : 0xffffffffffffffc3) 00000020:00000001:10.0:1506127961.252087:0:3044:0:(out_handler.c:829:out_tx_end()) Process entered 00000020:00000001:10.0:1506127961.252088:0:3044:0:(out_handler.c:835:out_tx_end()) Process leaving via stop (rc=18446744073709551555 : -61 : 0xffffffffffffffc3) 00080000:00000001:10.0:1506127961.252091:0:3044:0:(osd_handler.c:1834:osd_trans_stop()) Process entered 00080000:00000010:10.0:1506127961.252093:0:3044:0:(osd_handler.c:1879:osd_trans_stop()) kfreed 'oh': 240 at ffff88042bf1fa00. 00040000:00000001:10.0:1506127961.252097:0:3044:0:(qsd_handler.c:1080:qsd_op_end()) Process entered 00040000:00000001:10.0:1506127961.252099:0:3044:0:(qsd_handler.c:1111:qsd_op_end()) Process leaving 00080000:00000001:10.0:1506127961.252101:0:3044:0:(osd_handler.c:1905:osd_trans_stop()) Process leaving (rc=0 : 0 : 0) 00080000:00000010:10.0:1506127961.252107:0:3044:0:(osd_handler.c:1451:osd_object_free()) kfreed 'obj': 200 at ffff88042bf1ea00. 00000004:00000010:10.0:1506127961.252110:0:3044:0:(lod_object.c:5228:lod_object_free()) slab-freed '(lo)': 104 at ffff88082aa63750. 00000004:00000010:10.0:1506127961.252113:0:3044:0:(mdd_object.c:175:mdd_object_free()) slab-freed '(mdd)': 80 at ffff88042cbec370. 00000004:00000001:10.0:1506127961.252116:0:3044:0:(mdt_handler.c:5192:mdt_object_free()) Process entered 00000004:00000040:10.0:1506127961.252117:0:3044:0:(mdt_handler.c:5196:mdt_object_free()) object free, fid = [0x2c0003ab2:0x3:0x0] 00000004:00000010:10.0:1506127961.252120:0:3044:0:(mdt_handler.c:5203:mdt_object_free()) slab-freed '(mo)': 216 at ffff8808263ef878. 00000004:00000001:10.0:1506127961.252121:0:3044:0:(mdt_handler.c:5205:mdt_object_free()) Process leaving 00000020:00000002:10.0:1506127961.252124:0:3044:0:(lu_object.c:157:lu_object_put()) Add ffff88081f236970 to site lru. hash: ffff88082b7a7080, bkt: ffffc90015a74018, lru_len: 1 00000020:00000002:10.0:1506127961.252127:0:3044:0:(lu_object.c:157:lu_object_put()) Add ffff8800ad2a5618 to site lru. hash: ffff88082b7a7080, bkt: ffffc9001aec0018, lru_len: 1 00000020:00000001:10.0:1506127961.252129:0:3044:0:(out_handler.c:881:out_tx_end()) Process leaving (rc=18446744073709551555 : -61 : ffffffffffffffc3) 00000020:00000010:10.0:1506127961.252131:0:3044:0:(out_handler.c:1165:out_handle()) kfreed 'update_bufs[i]': 4096 at ffff88080f54f000. 00000020:00000010:10.0:1506127961.252135:0:3044:0:(out_handler.c:1165:out_handle()) kfreed 'update_bufs[i]': 32841 at ffff88080f180000. 00000020:00000010:10.0:1506127961.252148:0:3044:0:(out_handler.c:1165:out_handle()) kfreed 'update_bufs[i]': 4096 at ffff88083fcb1000. 00000020:00000010:10.0:1506127961.252150:0:3044:0:(out_handler.c:1169:out_handle()) kfreed 'update_bufs': 24 at ffff880826a71b60. 00000100:00000001:10.0:1506127961.252152:0:3044:0:(client.c:257:ptlrpc_free_bulk()) Process entered 00000020:00000040:10.0:1506127961.252154:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507400 : new refcount 5 00000100:00000010:10.0:1506127961.252156:0:3044:0:(client.c:281:ptlrpc_free_bulk()) kfreed '((desc)->bd_u.bd_kvec.bd_kvec)': 48 at ffff88042b7fc340. 00000100:00000010:10.0:1506127961.252159:0:3044:0:(client.c:282:ptlrpc_free_bulk()) kfreed 'desc': 280 at ffff88080f601c00. 00000100:00000001:10.0:1506127961.252160:0:3044:0:(client.c:283:ptlrpc_free_bulk()) Process leaving 00000020:00000001:10.0:1506127961.252161:0:3044:0:(out_handler.c:1175:out_handle()) Process leaving (rc=18446744073709551555 : -61 : ffffffffffffffc3) 00010000:00000040:10.0:1506127961.252164:0:3044:0:(ldlm_lib.c:2884:target_committed_to_req()) last_committed 30064771076, transno 0, xid 1579288666082112 00010000:00000001:10.0:1506127961.252167:0:3044:0:(ldlm_lib.c:2948:target_send_reply()) Process entered 00010000:00000200:10.0:1506127961.252172:0:3044:0:(ldlm_lib.c:2936:target_send_reply_msg()) @@@ sending reply req@ffff880813802050 x1579288666082112/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:362/0 lens 376/4320 e 0 to 0 dl 1506127967 ref 1 fl Interpret:/0/0 rc -61/0 00010000:00000001:10.0:1506127961.252182:0:3044:0:(ldlm_lib.c:2895:target_pack_pool_reply()) Process entered 00010000:00000001:10.0:1506127961.252183:0:3044:0:(ldlm_lib.c:2903:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:10.0:1506127961.252185:0:3044:0:(import.c:1678:at_measured()) add 1 to ffff88081629fd50 time=41 v=1 (1 0 0 0) 00000100:00000001:10.0:1506127961.252189:0:3044:0:(connection.c:127:ptlrpc_connection_addref()) Process entered 00000100:00000040:10.0:1506127961.252192:0:3044:0:(connection.c:132:ptlrpc_connection_addref()) conn=ffff88082ede1660 refcount 10 to 192.168.1.109@o2ib 00000100:00000001:10.0:1506127961.252194:0:3044:0:(connection.c:134:ptlrpc_connection_addref()) Process leaving (rc=18446612167460263520 : -131906249288096 : ffff88082ede1660) 02000000:00000001:10.0:1506127961.252197:0:3044:0:(sec.c:2161:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:10.0:1506127961.252198:0:3044:0:(sec.c:2172:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.252200:0:3044:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:10.0:1506127961.252203:0:3044:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-192.168.1.109@o2ib 00000400:00000010:10.0:1506127961.252205:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88042c0c5600. 00000100:00000200:10.0:1506127961.252209:0:3044:0:(niobuf.c:85:ptl_send_buf()) Sending 4288 bytes to portal 4, xid 1579288666082112, offset 192 00000400:00000010:10.0:1506127961.252211:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88080f601c00 (tot 195264252). 00000400:00000200:10.0:1506127961.252215:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.109@o2ib 00000400:00000200:10.0:1506127961.252227:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.109@o2ib(192.168.1.109@o2ib:192.168.1.109@o2ib) : PUT 00000800:00000200:10.0:1506127961.252231:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 4288 bytes in 1 frags to 12345-192.168.1.109@o2ib 00000800:00000200:10.0:1506127961.252241:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082edb4980] -> 192.168.1.109@o2ib (2) version: 12 00000800:00000200:10.0:1506127961.252244:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082dfb7200] (20)++ 00000800:00000200:10.0:1506127961.252246:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082dfb7200] (21)++ 00000800:00000200:10.0:1506127961.252250:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082dfb7200] (22)-- 00000100:00000001:10.0:1506127961.252252:0:3044:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.252254:0:3044:0:(connection.c:91:ptlrpc_connection_put()) Process entered 00000100:00000040:10.0:1506127961.252256:0:3044:0:(connection.c:119:ptlrpc_connection_put()) PUT conn=ffff88082ede1660 refcount 9 to 192.168.1.109@o2ib 00000100:00000001:10.0:1506127961.252258:0:3044:0:(connection.c:121:ptlrpc_connection_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:10.0:1506127961.252260:0:3044:0:(ldlm_lib.c:2960:target_send_reply()) Process leaving 00000020:00000001:10.0:1506127961.252261:0:3044:0:(tgt_handler.c:475:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.252263:0:3044:0:(tgt_handler.c:780:tgt_request_handle()) Process leaving 00000100:00000040:10.0:1506127961.252266:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff880813802050 x1579288666082112/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:362/0 lens 376/4288 e 0 to 0 dl 1506127967 ref 1 fl Interpret:/0/0 rc -61/-61 00000100:00100000:10.0:1506127961.252275:0:3044:0:(service.c:2135:ptlrpc_server_handle_request()) Handled RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+5:3000:x1579288666082112:12345-192.168.1.109@o2ib:1000 Request procesed in 16009us (16158us total) trans 0 rc -61/-61 00000100:00100000:10.0:1506127961.252282:0:3044:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.1.109@o2ib, seq: 18 00000100:00000040:10.0:1506127961.252285:0:3044:0:(service.c:1034:ptlrpc_server_finish_active_request()) RPC PUTting export ffff88080f507400 : new rpc_count 0 00000100:00000001:10.0:1506127961.252287:0:3044:0:(service.c:1613:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:10.0:1506127961.252288:0:3044:0:(service.c:1624:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:10.0:1506127961.252291:0:3044:0:(lu_ucred.c:49:lu_ucred_key_fini()) kfreed 'info': 104 at ffff88042c0c4300. 00000020:00000010:10.0:1506127961.252293:0:3044:0:(tgt_main.c:382:tgt_ses_key_fini()) kfreed 'info': 160 at ffff88080f7b32c0. 00000020:00000010:10.0:1506127961.252296:0:3044:0:(lu_object.c:1595:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff88080f601400. 00000020:00000040:10.0:1506127961.252298:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507400 : new refcount 4 00000100:00000001:10.0:1506127961.252300:0:3044:0:(service.c:2155:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:10.0:1506127961.252304:0:3044:0:(niobuf.c:953:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000400:00000010:10.0:1506127961.252307:0:3044:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88081f3fc2a0. 00000400:00000010:10.0:1506127961.252309:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88042c0c4300. 00000400:00000001:10.0:1506127961.252312:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:10.0:1506127961.252314:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:10.0:1506127961.256443:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:10.0:1506127961.256447:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:10.0:1506127961.256449:0:3044:0:(service.c:1859:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:10.0:1506127961.256451:0:3044:0:(sec.c:2052:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:10.0:1506127961.256452:0:3044:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:10.0:1506127961.256454:0:3044:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:10.0:1506127961.256458:0:3044:0:(sec.c:2099:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:10.0:1506127961.256461:0:3044:0:(service.c:1939:ptlrpc_server_handle_req_in()) got req x1579288666082160 00000020:00000001:10.0:1506127961.256463:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:10.0:1506127961.256464:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x2f3e107735f0c2a 00000020:00000001:10.0:1506127961.256466:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000040:10.0:1506127961.256468:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507400 : new refcount 5 00000020:00000001:10.0:1506127961.256470:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000020:00000001:10.0:1506127961.256473:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000100:00000001:10.0:1506127961.256476:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:10.0:1506127961.256477:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:10.0:1506127961.256479:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88080f602400. 00000020:00000010:10.0:1506127961.256482:0:3044:0:(lu_ucred.c:49:lu_ucred_key_init()) kmalloced 'value': 104 at ffff88042c0c4180. 00000020:00000010:10.0:1506127961.256484:0:3044:0:(tgt_main.c:382:tgt_ses_key_init()) kmalloced 'value': 160 at ffff88080f7b32c0. 00000100:00000040:10.0:1506127961.256488:0:3044:0:(service.c:1184:ptlrpc_at_set_timer()) armed mdt_out at +1s 00000100:00000001:10.0:1506127961.256490:0:3044:0:(service.c:1656:ptlrpc_server_request_add()) Process entered 00000100:00000001:10.0:1506127961.256491:0:3044:0:(service.c:1580:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:10.0:1506127961.256493:0:3044:0:(service.c:1607:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.256496:0:3044:0:(service.c:1695:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.256502:0:3044:0:(service.c:2003:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:10.0:1506127961.256505:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:10.0:1506127961.256507:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:10.0:1506127961.256508:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:10.0:1506127961.256509:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:10.0:1506127961.256510:0:3044:0:(service.c:2028:ptlrpc_server_handle_request()) Process entered 00000100:00000001:10.0:1506127961.256511:0:3044:0:(service.c:1809:ptlrpc_server_request_get()) Process entered 00000100:00100000:10.0:1506127961.256516:0:3044:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.1.109@o2ib, seq: 19 00000100:00000040:10.0:1506127961.256518:0:3044:0:(service.c:1840:ptlrpc_server_request_get()) RPC GETting export ffff88080f507400 : new rpc_count 1 00000100:00000001:10.0:1506127961.256520:0:3044:0:(service.c:1842:ptlrpc_server_request_get()) Process leaving (rc=18446612167001119824 : -131906708431792 : ffff880813801c50) 00000100:00000040:10.0:1506127961.256524:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "New" -> "Interpret" req@ffff880813801c50 x1579288666082160/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:362/0 lens 304/0 e 0 to 0 dl 1506127967 ref 1 fl New:/0/ffffffff rc 0/-1 00000100:00000001:10.0:1506127961.256532:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:10.0:1506127961.256534:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000100:00100000:10.0:1506127961.256536:0:3044:0:(service.c:2089:ptlrpc_server_handle_request()) Handling RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+5:3069:x1579288666082160:12345-192.168.1.109@o2ib:1000 00000100:00000200:10.0:1506127961.256541:0:3044:0:(service.c:2094:ptlrpc_server_handle_request()) got req 1579288666082160 00000020:00000001:10.0:1506127961.256543:0:3044:0:(tgt_handler.c:672:tgt_request_handle()) Process entered 00000020:00000001:10.0:1506127961.256546:0:3044:0:(tgt_handler.c:617:process_req_last_xid()) Process entered 00000020:00000001:10.0:1506127961.256547:0:3044:0:(tgt_handler.c:657:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.256549:0:3044:0:(tgt_handler.c:584:tgt_handler_find_check()) Process entered 00000020:00000001:10.0:1506127961.256550:0:3044:0:(tgt_handler.c:611:tgt_handler_find_check()) Process leaving (rc=18446744072654881248 : -1054670368 : ffffffffc12301e0) 00000020:00000001:10.0:1506127961.256552:0:3044:0:(tgt_handler.c:523:tgt_handle_recovery()) Process entered 00000020:00000001:10.0:1506127961.256554:0:3044:0:(tgt_handler.c:573:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:10.0:1506127961.256555:0:3044:0:(tgt_handler.c:382:tgt_handle_request0()) Process entered 00000020:00000001:10.0:1506127961.256557:0:3044:0:(tgt_handler.c:311:tgt_request_preprocess()) Process entered 00000020:00000001:10.0:1506127961.256558:0:3044:0:(tgt_handler.c:366:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.256560:0:3044:0:(out_handler.c:917:out_handle()) Process entered 00000020:00000010:10.0:1506127961.256563:0:3044:0:(out_handler.c:940:out_handle()) kmalloced 'update_bufs': 8 at ffff88042cbe8818. 00000100:00000001:10.0:1506127961.256565:0:3044:0:(pack_generic.c:334:lustre_pack_reply_v2()) Process entered 02000000:00000001:10.0:1506127961.256566:0:3044:0:(sec.c:2112:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:10.0:1506127961.256570:0:3044:0:(sec_null.c:318:null_alloc_rs()) kmalloced 'rs': 4576 at ffff88080f144000. 02000000:00000001:10.0:1506127961.256572:0:3044:0:(sec.c:2148:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.256574:0:3044:0:(pack_generic.c:368:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.256580:0:3044:0:(out_handler.c:205:out_attr_get()) Process entered 00000020:00000040:10.0:1506127961.256584:0:3044:0:(out_handler.c:233:out_attr_get()) soaked-MDT0003: insert attr get reply ffff88080f1441e0 index 0: rc = 0 00000020:00000001:10.0:1506127961.256587:0:3044:0:(out_handler.c:238:out_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000002:10.0:1506127961.256589:0:3044:0:(lu_object.c:157:lu_object_put()) Add ffff8800ad2a5618 to site lru. hash: ffff88082b7a7080, bkt: ffffc9001aec0018, lru_len: 1 00000020:00000010:10.0:1506127961.256592:0:3044:0:(out_handler.c:1169:out_handle()) kfreed 'update_bufs': 8 at ffff88042cbe8818. 00000020:00000001:10.0:1506127961.256594:0:3044:0:(out_handler.c:1175:out_handle()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:10.0:1506127961.256596:0:3044:0:(ldlm_lib.c:2884:target_committed_to_req()) last_committed 30064771076, transno 0, xid 1579288666082160 00010000:00000001:10.0:1506127961.256598:0:3044:0:(ldlm_lib.c:2948:target_send_reply()) Process entered 00010000:00000200:10.0:1506127961.256601:0:3044:0:(ldlm_lib.c:2936:target_send_reply_msg()) @@@ sending reply req@ffff880813801c50 x1579288666082160/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:362/0 lens 304/4320 e 0 to 0 dl 1506127967 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000001:10.0:1506127961.256610:0:3044:0:(ldlm_lib.c:2895:target_pack_pool_reply()) Process entered 00010000:00000001:10.0:1506127961.256611:0:3044:0:(ldlm_lib.c:2903:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:10.0:1506127961.256613:0:3044:0:(import.c:1678:at_measured()) add 1 to ffff88081629fd50 time=41 v=1 (1 0 0 0) 00000100:00000001:10.0:1506127961.256616:0:3044:0:(connection.c:127:ptlrpc_connection_addref()) Process entered 00000100:00000040:10.0:1506127961.256618:0:3044:0:(connection.c:132:ptlrpc_connection_addref()) conn=ffff88082ede1660 refcount 10 to 192.168.1.109@o2ib 00000100:00000001:10.0:1506127961.256620:0:3044:0:(connection.c:134:ptlrpc_connection_addref()) Process leaving (rc=18446612167460263520 : -131906249288096 : ffff88082ede1660) 02000000:00000001:10.0:1506127961.256623:0:3044:0:(sec.c:2161:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:10.0:1506127961.256624:0:3044:0:(sec.c:2172:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.256626:0:3044:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:10.0:1506127961.256628:0:3044:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-192.168.1.109@o2ib 00000400:00000010:10.0:1506127961.256630:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88042c0c5580. 00000100:00000200:10.0:1506127961.256633:0:3044:0:(niobuf.c:85:ptl_send_buf()) Sending 4288 bytes to portal 4, xid 1579288666082160, offset 192 00000400:00000010:10.0:1506127961.256636:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88080f602000 (tot 195264252). 00000400:00000200:10.0:1506127961.256639:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.109@o2ib 00000400:00000200:10.0:1506127961.256651:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.109@o2ib(192.168.1.109@o2ib:192.168.1.109@o2ib) : PUT 00000800:00000200:10.0:1506127961.256656:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 4288 bytes in 1 frags to 12345-192.168.1.109@o2ib 00000800:00000200:10.0:1506127961.256665:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082edb4980] -> 192.168.1.109@o2ib (2) version: 12 00000800:00000200:10.0:1506127961.256668:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082dfb7200] (20)++ 00000800:00000200:10.0:1506127961.256670:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082dfb7200] (21)++ 00000800:00000200:10.0:1506127961.256674:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082dfb7200] (22)-- 00000100:00000001:10.0:1506127961.256676:0:3044:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.256678:0:3044:0:(connection.c:91:ptlrpc_connection_put()) Process entered 00000100:00000040:10.0:1506127961.256680:0:3044:0:(connection.c:119:ptlrpc_connection_put()) PUT conn=ffff88082ede1660 refcount 9 to 192.168.1.109@o2ib 00000100:00000001:10.0:1506127961.256681:0:3044:0:(connection.c:121:ptlrpc_connection_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:10.0:1506127961.256683:0:3044:0:(ldlm_lib.c:2960:target_send_reply()) Process leaving 00000020:00000001:10.0:1506127961.256684:0:3044:0:(tgt_handler.c:475:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.256685:0:3044:0:(tgt_handler.c:780:tgt_request_handle()) Process leaving 00000100:00000040:10.0:1506127961.256688:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff880813801c50 x1579288666082160/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:362/0 lens 304/4288 e 0 to 0 dl 1506127967 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00100000:10.0:1506127961.256696:0:3044:0:(service.c:2135:ptlrpc_server_handle_request()) Handled RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+5:3069:x1579288666082160:12345-192.168.1.109@o2ib:1000 Request procesed in 162us (298us total) trans 0 rc 0/0 00000100:00100000:10.0:1506127961.256703:0:3044:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.1.109@o2ib, seq: 19 00000100:00000040:10.0:1506127961.256705:0:3044:0:(service.c:1034:ptlrpc_server_finish_active_request()) RPC PUTting export ffff88080f507400 : new rpc_count 0 00000100:00000001:10.0:1506127961.256707:0:3044:0:(service.c:1613:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:10.0:1506127961.256708:0:3044:0:(service.c:1624:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:10.0:1506127961.256710:0:3044:0:(lu_ucred.c:49:lu_ucred_key_fini()) kfreed 'info': 104 at ffff88042c0c4180. 00000020:00000010:10.0:1506127961.256712:0:3044:0:(tgt_main.c:382:tgt_ses_key_fini()) kfreed 'info': 160 at ffff88080f7b32c0. 00000020:00000010:10.0:1506127961.256714:0:3044:0:(lu_object.c:1595:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff88080f602400. 00000020:00000040:10.0:1506127961.256716:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507400 : new refcount 4 00000100:00000001:10.0:1506127961.256719:0:3044:0:(service.c:2155:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:10.0:1506127961.256721:0:3044:0:(niobuf.c:953:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000400:00000010:10.0:1506127961.256724:0:3044:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88081f3fc300. 00000400:00000010:10.0:1506127961.256726:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88042c0c4180. 00000400:00000001:10.0:1506127961.256729:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:10.0:1506127961.256730:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:10.0:1506127961.258005:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:10.0:1506127961.258009:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:10.0:1506127961.258011:0:3044:0:(service.c:1859:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:10.0:1506127961.258013:0:3044:0:(sec.c:2052:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:10.0:1506127961.258014:0:3044:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:10.0:1506127961.258016:0:3044:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:10.0:1506127961.258019:0:3044:0:(sec.c:2099:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:10.0:1506127961.258022:0:3044:0:(service.c:1939:ptlrpc_server_handle_req_in()) got req x1579288666082176 00000020:00000001:10.0:1506127961.258024:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:10.0:1506127961.258026:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x2f3e107735f0c2a 00000020:00000001:10.0:1506127961.258027:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000040:10.0:1506127961.258029:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507400 : new refcount 5 00000020:00000001:10.0:1506127961.258031:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000020:00000001:10.0:1506127961.258033:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000100:00000001:10.0:1506127961.258037:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:10.0:1506127961.258038:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:10.0:1506127961.258040:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88080f602400. 00000020:00000010:10.0:1506127961.258043:0:3044:0:(lu_ucred.c:49:lu_ucred_key_init()) kmalloced 'value': 104 at ffff88042c0c5980. 00000020:00000010:10.0:1506127961.258046:0:3044:0:(tgt_main.c:382:tgt_ses_key_init()) kmalloced 'value': 160 at ffff88080f7b32c0. 00000100:00000040:10.0:1506127961.258049:0:3044:0:(service.c:1184:ptlrpc_at_set_timer()) armed mdt_out at +1s 00000100:00000001:10.0:1506127961.258051:0:3044:0:(service.c:1656:ptlrpc_server_request_add()) Process entered 00000100:00000001:10.0:1506127961.258052:0:3044:0:(service.c:1580:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:10.0:1506127961.258053:0:3044:0:(service.c:1607:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.258056:0:3044:0:(service.c:1695:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.258063:0:3044:0:(service.c:2003:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:10.0:1506127961.258066:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:10.0:1506127961.258068:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:10.0:1506127961.258069:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:10.0:1506127961.258070:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:10.0:1506127961.258086:0:3044:0:(service.c:2028:ptlrpc_server_handle_request()) Process entered 00000100:00000001:10.0:1506127961.258088:0:3044:0:(service.c:1809:ptlrpc_server_request_get()) Process entered 00000100:00100000:10.0:1506127961.258092:0:3044:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.1.109@o2ib, seq: 20 00000100:00000040:10.0:1506127961.258095:0:3044:0:(service.c:1840:ptlrpc_server_request_get()) RPC GETting export ffff88080f507400 : new rpc_count 1 00000100:00000001:10.0:1506127961.258096:0:3044:0:(service.c:1842:ptlrpc_server_request_get()) Process leaving (rc=18446612167001118800 : -131906708432816 : ffff880813801850) 00000100:00000040:10.0:1506127961.258100:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "New" -> "Interpret" req@ffff880813801850 x1579288666082176/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:362/0 lens 392/0 e 0 to 0 dl 1506127967 ref 1 fl New:/0/ffffffff rc 0/-1 00000100:00000001:10.0:1506127961.258135:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:10.0:1506127961.258136:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000100:00100000:10.0:1506127961.258139:0:3044:0:(service.c:2089:ptlrpc_server_handle_request()) Handling RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+5:3000:x1579288666082176:12345-192.168.1.109@o2ib:1000 00000100:00000200:10.0:1506127961.258142:0:3044:0:(service.c:2094:ptlrpc_server_handle_request()) got req 1579288666082176 00000020:00000001:10.0:1506127961.258144:0:3044:0:(tgt_handler.c:672:tgt_request_handle()) Process entered 00000020:00000001:10.0:1506127961.258146:0:3044:0:(tgt_handler.c:617:process_req_last_xid()) Process entered 00000020:00000001:10.0:1506127961.258147:0:3044:0:(tgt_handler.c:657:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.258149:0:3044:0:(tgt_handler.c:584:tgt_handler_find_check()) Process entered 00000020:00000001:10.0:1506127961.258150:0:3044:0:(tgt_handler.c:611:tgt_handler_find_check()) Process leaving (rc=18446744072654881248 : -1054670368 : ffffffffc12301e0) 00000020:00000001:10.0:1506127961.258152:0:3044:0:(tgt_handler.c:523:tgt_handle_recovery()) Process entered 00000020:00000001:10.0:1506127961.258153:0:3044:0:(tgt_handler.c:573:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:10.0:1506127961.258155:0:3044:0:(tgt_handler.c:382:tgt_handle_request0()) Process entered 00000020:00000001:10.0:1506127961.258156:0:3044:0:(tgt_handler.c:311:tgt_request_preprocess()) Process entered 00000020:00000001:10.0:1506127961.258157:0:3044:0:(tgt_handler.c:366:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.258159:0:3044:0:(out_handler.c:917:out_handle()) Process entered 00000020:00000010:10.0:1506127961.258162:0:3044:0:(out_handler.c:940:out_handle()) kmalloced 'update_bufs': 40 at ffff88042b7fc340. 00000100:00000001:10.0:1506127961.258164:0:3044:0:(niobuf.c:130:ptlrpc_prep_bulk_exp()) Process entered 00000100:00000010:10.0:1506127961.258165:0:3044:0:(client.c:129:ptlrpc_new_bulk()) kmalloced 'desc': 280 at ffff88080f602200. 00000100:00000010:10.0:1506127961.258168:0:3044:0:(client.c:139:ptlrpc_new_bulk()) kmalloced '((desc)->bd_u.bd_kvec.bd_kvec)': 80 at ffff88081f3fc360. 00000020:00000040:10.0:1506127961.258169:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507400 : new refcount 6 00000020:00000010:10.0:1506127961.258171:0:3044:0:(out_handler.c:967:out_handle()) kmalloced 'update_bufs[i]': 4096 at ffff88083fcb1000. 00000100:00000001:10.0:1506127961.258173:0:3044:0:(client.c:235:ptlrpc_prep_bulk_frag()) Process entered 00000100:00000001:10.0:1506127961.258174:0:3044:0:(client.c:251:ptlrpc_prep_bulk_frag()) Process leaving (rc=4096 : 4096 : 1000) 00000020:00000010:10.0:1506127961.258201:0:3044:0:(out_handler.c:967:out_handle()) kmalloced 'update_bufs[i]': 32841 at ffff88080f180000. 00000100:00000001:10.0:1506127961.258202:0:3044:0:(client.c:235:ptlrpc_prep_bulk_frag()) Process entered 00000100:00000001:10.0:1506127961.258203:0:3044:0:(client.c:251:ptlrpc_prep_bulk_frag()) Process leaving (rc=36937 : 36937 : 9049) 00000020:00000010:10.0:1506127961.258206:0:3044:0:(out_handler.c:967:out_handle()) kmalloced 'update_bufs[i]': 4096 at ffff88080f54f000. 00000100:00000001:10.0:1506127961.258207:0:3044:0:(client.c:235:ptlrpc_prep_bulk_frag()) Process entered 00000100:00000001:10.0:1506127961.258208:0:3044:0:(client.c:251:ptlrpc_prep_bulk_frag()) Process leaving (rc=41033 : 41033 : a049) 00000020:00000010:10.0:1506127961.258230:0:3044:0:(out_handler.c:967:out_handle()) kmalloced 'update_bufs[i]': 32841 at ffff88080f190000. 00000100:00000001:10.0:1506127961.258232:0:3044:0:(client.c:235:ptlrpc_prep_bulk_frag()) Process entered 00000100:00000001:10.0:1506127961.258232:0:3044:0:(client.c:251:ptlrpc_prep_bulk_frag()) Process leaving (rc=73874 : 73874 : 12092) 00000020:00000010:10.0:1506127961.258238:0:3044:0:(out_handler.c:967:out_handle()) kmalloced 'update_bufs[i]': 4096 at ffff880817a78000. 00000100:00000001:10.0:1506127961.258239:0:3044:0:(client.c:235:ptlrpc_prep_bulk_frag()) Process entered 00000100:00000001:10.0:1506127961.258240:0:3044:0:(client.c:251:ptlrpc_prep_bulk_frag()) Process leaving (rc=77970 : 77970 : 13092) 00010000:00000001:10.0:1506127961.258242:0:3044:0:(ldlm_lib.c:3167:target_bulk_io()) Process entered 00000100:00000001:10.0:1506127961.258244:0:3044:0:(niobuf.c:164:ptlrpc_start_bulk_transfer()) Process entered 00000400:00000010:10.0:1506127961.258246:0:3044:0:(lib-lnet.h:229:lnet_md_alloc()) alloc '(md)': 192 at ffff88080f7b3440 (tot 195264044). 00000400:00000010:10.0:1506127961.258251:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88080f602600 (tot 195264444). 00000400:00000200:10.0:1506127961.258255:0:3044:0:(lib-move.c:2998:LNetGet()) LNetGet -> 12345-192.168.1.109@o2ib 00000400:00000200:10.0:1506127961.258267:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.109@o2ib(192.168.1.109@o2ib:192.168.1.109@o2ib) : GET 00000800:00000200:10.0:1506127961.258271:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 0 bytes in 0 frags to 12345-192.168.1.109@o2ib 00000400:00000010:10.0:1506127961.258281:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88080f602800 (tot 195264844). 00000400:00000200:10.0:1506127961.258286:0:3044:0:(lib-move.c:2877:lnet_create_reply_msg()) 192.168.1.111@o2ib: Reply from 12345-192.168.1.109@o2ib md ffff88080f7b3440 00000800:00000200:10.0:1506127961.258290:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082edb4980] -> 192.168.1.109@o2ib (2) version: 12 00000800:00000200:10.0:1506127961.258292:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082dfb7200] (20)++ 00000800:00000200:10.0:1506127961.258294:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082dfb7200] (21)++ 00000800:00000200:10.0:1506127961.258299:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082dfb7200] (22)-- 00000100:00000200:10.0:1506127961.258302:0:3044:0:(niobuf.c:262:ptlrpc_start_bulk_transfer()) Transferring 5 pages 77970 bytes via portal 14 id 12345-192.168.1.109@o2ib mbits 0x59c5acbf08b80-0x59c5acbf08b80 00000100:00000001:10.0:1506127961.258306:0:3044:0:(niobuf.c:264:ptlrpc_start_bulk_transfer()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:10.0:1506127961.258595:0:3044:0:(ldlm_lib.c:3268:target_bulk_io()) Process leaving (rc=0 : 0 : 0) 00000020:00020000:10.0:1506127961.258600:0:3044:0:(out_handler.c:1000:out_handle()) soaked-MDT0003: invalid update buffer magic 0 expect bdde0002: rc = -71 00000020:00000001:10.0:1506127961.279044:0:3044:0:(out_handler.c:1001:out_handle()) Process leaving via out_free (rc=18446744073709547449 : -4167 : 0xffffffffffffefb9) 00000020:00000010:10.0:1506127961.279048:0:3044:0:(out_handler.c:1165:out_handle()) kfreed 'update_bufs[i]': 4096 at ffff88083fcb1000. 00000020:00000010:10.0:1506127961.279066:0:3044:0:(out_handler.c:1165:out_handle()) kfreed 'update_bufs[i]': 32841 at ffff88080f180000. 00000020:00000010:10.0:1506127961.279086:0:3044:0:(out_handler.c:1165:out_handle()) kfreed 'update_bufs[i]': 4096 at ffff88080f54f000. 00000020:00000010:10.0:1506127961.279090:0:3044:0:(out_handler.c:1165:out_handle()) kfreed 'update_bufs[i]': 32841 at ffff88080f190000. 00000020:00000010:10.0:1506127961.279115:0:3044:0:(out_handler.c:1165:out_handle()) kfreed 'update_bufs[i]': 4096 at ffff880817a78000. 00000020:00000010:10.0:1506127961.279117:0:3044:0:(out_handler.c:1169:out_handle()) kfreed 'update_bufs': 40 at ffff88042b7fc340. 00000100:00000001:10.0:1506127961.279119:0:3044:0:(client.c:257:ptlrpc_free_bulk()) Process entered 00000020:00000040:10.0:1506127961.279120:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507400 : new refcount 5 00000100:00000010:10.0:1506127961.279122:0:3044:0:(client.c:281:ptlrpc_free_bulk()) kfreed '((desc)->bd_u.bd_kvec.bd_kvec)': 80 at ffff88081f3fc360. 00000100:00000010:10.0:1506127961.279124:0:3044:0:(client.c:282:ptlrpc_free_bulk()) kfreed 'desc': 280 at ffff88080f602200. 00000100:00000001:10.0:1506127961.279125:0:3044:0:(client.c:283:ptlrpc_free_bulk()) Process leaving 00000020:00000001:10.0:1506127961.279126:0:3044:0:(out_handler.c:1175:out_handle()) Process leaving (rc=18446744073709547449 : -4167 : ffffffffffffefb9) 00010000:00000001:10.0:1506127961.279128:0:3044:0:(ldlm_lib.c:2948:target_send_reply()) Process entered 00010000:00000200:10.0:1506127961.279132:0:3044:0:(ldlm_lib.c:2932:target_send_reply_msg()) @@@ processing error (-71) req@ffff880813801850 x1579288666082176/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:362/0 lens 392/0 e 0 to 0 dl 1506127967 ref 1 fl Interpret:/0/ffffffff rc -71/-1 00000100:00000001:10.0:1506127961.279140:0:3044:0:(niobuf.c:660:ptlrpc_send_error()) Process entered 00000100:00000001:10.0:1506127961.279141:0:3044:0:(pack_generic.c:334:lustre_pack_reply_v2()) Process entered 02000000:00000001:10.0:1506127961.279142:0:3044:0:(sec.c:2112:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:10.0:1506127961.279144:0:3044:0:(sec_null.c:318:null_alloc_rs()) kmalloced 'rs': 480 at ffff88080f602200. 02000000:00000001:10.0:1506127961.279146:0:3044:0:(sec.c:2148:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.279148:0:3044:0:(pack_generic.c:368:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:10.0:1506127961.279150:0:3044:0:(ldlm_lib.c:2895:target_pack_pool_reply()) Process entered 00010000:00000001:10.0:1506127961.279151:0:3044:0:(ldlm_lib.c:2903:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.279154:0:3044:0:(connection.c:127:ptlrpc_connection_addref()) Process entered 00000100:00000040:10.0:1506127961.279156:0:3044:0:(connection.c:132:ptlrpc_connection_addref()) conn=ffff88082ede1660 refcount 10 to 192.168.1.109@o2ib 00000100:00000001:10.0:1506127961.279158:0:3044:0:(connection.c:134:ptlrpc_connection_addref()) Process leaving (rc=18446612167460263520 : -131906249288096 : ffff88082ede1660) 02000000:00000001:10.0:1506127961.279160:0:3044:0:(sec.c:2161:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:10.0:1506127961.279161:0:3044:0:(sec.c:2172:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.279162:0:3044:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:10.0:1506127961.279165:0:3044:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-192.168.1.109@o2ib 00000400:00000010:10.0:1506127961.279167:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88042c0c4500. 00000100:00000200:10.0:1506127961.279169:0:3044:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 4, xid 1579288666082176, offset 192 00000400:00000010:10.0:1506127961.279172:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88080f602a00 (tot 195264252). 00000400:00000200:10.0:1506127961.279175:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.109@o2ib 00000400:00000200:10.0:1506127961.279184:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.109@o2ib(192.168.1.109@o2ib:192.168.1.109@o2ib) : PUT 00000800:00000200:10.0:1506127961.279188:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 192 bytes in 1 frags to 12345-192.168.1.109@o2ib 00000800:00000200:10.0:1506127961.279193:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082edb4980] -> 192.168.1.109@o2ib (2) version: 12 00000800:00000200:10.0:1506127961.279195:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082dfb7200] (20)++ 00000800:00000200:10.0:1506127961.279196:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082dfb7200] (21)++ 00000800:00000200:10.0:1506127961.279201:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082dfb7200] (22)-- 00000100:00000001:10.0:1506127961.279202:0:3044:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.279204:0:3044:0:(connection.c:91:ptlrpc_connection_put()) Process entered 00000100:00000040:10.0:1506127961.279205:0:3044:0:(connection.c:119:ptlrpc_connection_put()) PUT conn=ffff88082ede1660 refcount 9 to 192.168.1.109@o2ib 00000100:00000001:10.0:1506127961.279207:0:3044:0:(connection.c:121:ptlrpc_connection_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.279209:0:3044:0:(niobuf.c:677:ptlrpc_send_error()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:10.0:1506127961.279210:0:3044:0:(ldlm_lib.c:2960:target_send_reply()) Process leaving 00000020:00000001:10.0:1506127961.279211:0:3044:0:(tgt_handler.c:475:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.279213:0:3044:0:(tgt_handler.c:780:tgt_request_handle()) Process leaving 00000100:00000040:10.0:1506127961.279215:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff880813801850 x1579288666082176/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:362/0 lens 392/192 e 0 to 0 dl 1506127967 ref 1 fl Interpret:/0/0 rc -71/-71 00000100:00100000:10.0:1506127961.279223:0:3044:0:(service.c:2135:ptlrpc_server_handle_request()) Handled RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+5:3000:x1579288666082176:12345-192.168.1.109@o2ib:1000 Request procesed in 21087us (21264us total) trans 0 rc -71/-71 00000100:00100000:10.0:1506127961.279230:0:3044:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.1.109@o2ib, seq: 20 00000100:00000040:10.0:1506127961.279232:0:3044:0:(service.c:1034:ptlrpc_server_finish_active_request()) RPC PUTting export ffff88080f507400 : new rpc_count 0 00000100:00000001:10.0:1506127961.279234:0:3044:0:(service.c:1613:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:10.0:1506127961.279235:0:3044:0:(service.c:1624:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:10.0:1506127961.279237:0:3044:0:(lu_ucred.c:49:lu_ucred_key_fini()) kfreed 'info': 104 at ffff88042c0c5980. 00000020:00000010:10.0:1506127961.279239:0:3044:0:(tgt_main.c:382:tgt_ses_key_fini()) kfreed 'info': 160 at ffff88080f7b32c0. 00000020:00000010:10.0:1506127961.279241:0:3044:0:(lu_object.c:1595:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff88080f602400. 00000020:00000040:10.0:1506127961.279243:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507400 : new refcount 4 00000100:00000001:10.0:1506127961.279245:0:3044:0:(service.c:2155:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:10.0:1506127961.279248:0:3044:0:(niobuf.c:953:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000400:00000010:10.0:1506127961.279250:0:3044:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88081f3fc360. 00000400:00000010:10.0:1506127961.279252:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88042c0c5980. 00000400:00000001:10.0:1506127961.279255:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:10.0:1506127961.279256:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:10.0:1506127961.296216:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:10.0:1506127961.296220:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:10.0:1506127961.296222:0:3044:0:(service.c:1859:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:10.0:1506127961.296223:0:3044:0:(sec.c:2052:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:10.0:1506127961.296225:0:3044:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:10.0:1506127961.296227:0:3044:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:10.0:1506127961.296230:0:3044:0:(sec.c:2099:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:10.0:1506127961.296232:0:3044:0:(service.c:1939:ptlrpc_server_handle_req_in()) got req x1579288666082336 00000020:00000001:10.0:1506127961.296234:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:10.0:1506127961.296236:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x2f3e107735f0c2a 00000020:00000001:10.0:1506127961.296237:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000040:10.0:1506127961.296239:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507400 : new refcount 5 00000020:00000001:10.0:1506127961.296241:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000020:00000001:10.0:1506127961.296243:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000100:00000001:10.0:1506127961.296246:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:10.0:1506127961.296247:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:10.0:1506127961.296249:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88080f602400. 00000020:00000010:10.0:1506127961.296252:0:3044:0:(lu_ucred.c:49:lu_ucred_key_init()) kmalloced 'value': 104 at ffff88042c0c4b80. 00000020:00000010:10.0:1506127961.296254:0:3044:0:(tgt_main.c:382:tgt_ses_key_init()) kmalloced 'value': 160 at ffff88080f7b32c0. 00000100:00000040:10.0:1506127961.296258:0:3044:0:(service.c:1184:ptlrpc_at_set_timer()) armed mdt_out at +1s 00000100:00000001:10.0:1506127961.296260:0:3044:0:(service.c:1656:ptlrpc_server_request_add()) Process entered 00000100:00000001:10.0:1506127961.296261:0:3044:0:(service.c:1580:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:10.0:1506127961.296262:0:3044:0:(service.c:1607:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.296265:0:3044:0:(service.c:1695:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.296271:0:3044:0:(service.c:2003:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:10.0:1506127961.296274:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:10.0:1506127961.296275:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:10.0:1506127961.296276:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:10.0:1506127961.296277:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:10.0:1506127961.296278:0:3044:0:(service.c:2028:ptlrpc_server_handle_request()) Process entered 00000100:00000001:10.0:1506127961.296279:0:3044:0:(service.c:1809:ptlrpc_server_request_get()) Process entered 00000100:00100000:10.0:1506127961.296283:0:3044:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.1.109@o2ib, seq: 21 00000100:00000040:10.0:1506127961.296285:0:3044:0:(service.c:1840:ptlrpc_server_request_get()) RPC GETting export ffff88080f507400 : new rpc_count 1 00000100:00000001:10.0:1506127961.296287:0:3044:0:(service.c:1842:ptlrpc_server_request_get()) Process leaving (rc=18446612167001117776 : -131906708433840 : ffff880813801450) 00000100:00000040:10.0:1506127961.296291:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "New" -> "Interpret" req@ffff880813801450 x1579288666082336/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:362/0 lens 304/0 e 0 to 0 dl 1506127967 ref 1 fl New:/0/ffffffff rc 0/-1 00000100:00000001:10.0:1506127961.296299:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:10.0:1506127961.296300:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000100:00100000:10.0:1506127961.296302:0:3044:0:(service.c:2089:ptlrpc_server_handle_request()) Handling RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+5:3098:x1579288666082336:12345-192.168.1.109@o2ib:1000 00000100:00000200:10.0:1506127961.296305:0:3044:0:(service.c:2094:ptlrpc_server_handle_request()) got req 1579288666082336 00000020:00000001:10.0:1506127961.296307:0:3044:0:(tgt_handler.c:672:tgt_request_handle()) Process entered 00000020:00000001:10.0:1506127961.296309:0:3044:0:(tgt_handler.c:617:process_req_last_xid()) Process entered 00000020:00000001:10.0:1506127961.296310:0:3044:0:(tgt_handler.c:657:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.296312:0:3044:0:(tgt_handler.c:584:tgt_handler_find_check()) Process entered 00000020:00000001:10.0:1506127961.296313:0:3044:0:(tgt_handler.c:611:tgt_handler_find_check()) Process leaving (rc=18446744072654881248 : -1054670368 : ffffffffc12301e0) 00000020:00000001:10.0:1506127961.296315:0:3044:0:(tgt_handler.c:523:tgt_handle_recovery()) Process entered 00000020:00000001:10.0:1506127961.296317:0:3044:0:(tgt_handler.c:573:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:10.0:1506127961.296318:0:3044:0:(tgt_handler.c:382:tgt_handle_request0()) Process entered 00000020:00000001:10.0:1506127961.296319:0:3044:0:(tgt_handler.c:311:tgt_request_preprocess()) Process entered 00000020:00000001:10.0:1506127961.296321:0:3044:0:(tgt_handler.c:366:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.296322:0:3044:0:(out_handler.c:917:out_handle()) Process entered 00000020:00000010:10.0:1506127961.296325:0:3044:0:(out_handler.c:940:out_handle()) kmalloced 'update_bufs': 8 at ffff88042cbe8818. 00000100:00000001:10.0:1506127961.296327:0:3044:0:(pack_generic.c:334:lustre_pack_reply_v2()) Process entered 02000000:00000001:10.0:1506127961.296328:0:3044:0:(sec.c:2112:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:10.0:1506127961.296332:0:3044:0:(sec_null.c:318:null_alloc_rs()) kmalloced 'rs': 4576 at ffff88080f146000. 02000000:00000001:10.0:1506127961.296333:0:3044:0:(sec.c:2148:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.296335:0:3044:0:(pack_generic.c:368:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.296340:0:3044:0:(out_handler.c:205:out_attr_get()) Process entered 00000020:00000040:10.0:1506127961.296342:0:3044:0:(out_handler.c:233:out_attr_get()) soaked-MDT0003: insert attr get reply ffff88080f1461e0 index 0: rc = 0 00000020:00000001:10.0:1506127961.296344:0:3044:0:(out_handler.c:238:out_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000002:10.0:1506127961.296347:0:3044:0:(lu_object.c:157:lu_object_put()) Add ffff8800ad2a5618 to site lru. hash: ffff88082b7a7080, bkt: ffffc9001aec0018, lru_len: 1 00000020:00000010:10.0:1506127961.296349:0:3044:0:(out_handler.c:1169:out_handle()) kfreed 'update_bufs': 8 at ffff88042cbe8818. 00000020:00000001:10.0:1506127961.296351:0:3044:0:(out_handler.c:1175:out_handle()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:10.0:1506127961.296353:0:3044:0:(ldlm_lib.c:2884:target_committed_to_req()) last_committed 30064771076, transno 0, xid 1579288666082336 00010000:00000001:10.0:1506127961.296355:0:3044:0:(ldlm_lib.c:2948:target_send_reply()) Process entered 00010000:00000200:10.0:1506127961.296357:0:3044:0:(ldlm_lib.c:2936:target_send_reply_msg()) @@@ sending reply req@ffff880813801450 x1579288666082336/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:362/0 lens 304/4320 e 0 to 0 dl 1506127967 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000001:10.0:1506127961.296365:0:3044:0:(ldlm_lib.c:2895:target_pack_pool_reply()) Process entered 00010000:00000001:10.0:1506127961.296367:0:3044:0:(ldlm_lib.c:2903:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:10.0:1506127961.296368:0:3044:0:(import.c:1678:at_measured()) add 1 to ffff88081629fd50 time=41 v=1 (1 0 0 0) 00000100:00000001:10.0:1506127961.296371:0:3044:0:(connection.c:127:ptlrpc_connection_addref()) Process entered 00000100:00000040:10.0:1506127961.296373:0:3044:0:(connection.c:132:ptlrpc_connection_addref()) conn=ffff88082ede1660 refcount 10 to 192.168.1.109@o2ib 00000100:00000001:10.0:1506127961.296375:0:3044:0:(connection.c:134:ptlrpc_connection_addref()) Process leaving (rc=18446612167460263520 : -131906249288096 : ffff88082ede1660) 02000000:00000001:10.0:1506127961.296377:0:3044:0:(sec.c:2161:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:10.0:1506127961.296378:0:3044:0:(sec.c:2172:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.296380:0:3044:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:10.0:1506127961.296382:0:3044:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-192.168.1.109@o2ib 00000400:00000010:10.0:1506127961.296384:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88042c0c5000. 00000100:00000200:10.0:1506127961.296387:0:3044:0:(niobuf.c:85:ptl_send_buf()) Sending 4288 bytes to portal 4, xid 1579288666082336, offset 192 00000400:00000010:10.0:1506127961.296389:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88080f603200 (tot 195264252). 00000400:00000200:10.0:1506127961.296393:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.109@o2ib 00000400:00000200:10.0:1506127961.296403:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.109@o2ib(192.168.1.109@o2ib:192.168.1.109@o2ib) : PUT 00000800:00000200:10.0:1506127961.296407:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 4288 bytes in 1 frags to 12345-192.168.1.109@o2ib 00000800:00000200:10.0:1506127961.296416:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082edb4980] -> 192.168.1.109@o2ib (2) version: 12 00000800:00000200:10.0:1506127961.296418:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082dfb7200] (20)++ 00000800:00000200:10.0:1506127961.296420:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082dfb7200] (21)++ 00000800:00000200:10.0:1506127961.296425:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082dfb7200] (22)-- 00000100:00000001:10.0:1506127961.296426:0:3044:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.296428:0:3044:0:(connection.c:91:ptlrpc_connection_put()) Process entered 00000100:00000040:10.0:1506127961.296429:0:3044:0:(connection.c:119:ptlrpc_connection_put()) PUT conn=ffff88082ede1660 refcount 9 to 192.168.1.109@o2ib 00000100:00000001:10.0:1506127961.296431:0:3044:0:(connection.c:121:ptlrpc_connection_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:10.0:1506127961.296432:0:3044:0:(ldlm_lib.c:2960:target_send_reply()) Process leaving 00000020:00000001:10.0:1506127961.296433:0:3044:0:(tgt_handler.c:475:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.296435:0:3044:0:(tgt_handler.c:780:tgt_request_handle()) Process leaving 00000100:00000040:10.0:1506127961.296437:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff880813801450 x1579288666082336/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:362/0 lens 304/4288 e 0 to 0 dl 1506127967 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00100000:10.0:1506127961.296445:0:3044:0:(service.c:2135:ptlrpc_server_handle_request()) Handled RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+5:3098:x1579288666082336:12345-192.168.1.109@o2ib:1000 Request procesed in 144us (276us total) trans 0 rc 0/0 00000100:00100000:10.0:1506127961.296451:0:3044:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.1.109@o2ib, seq: 21 00000100:00000040:10.0:1506127961.296453:0:3044:0:(service.c:1034:ptlrpc_server_finish_active_request()) RPC PUTting export ffff88080f507400 : new rpc_count 0 00000100:00000001:10.0:1506127961.296454:0:3044:0:(service.c:1613:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:10.0:1506127961.296455:0:3044:0:(service.c:1624:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:10.0:1506127961.296457:0:3044:0:(lu_ucred.c:49:lu_ucred_key_fini()) kfreed 'info': 104 at ffff88042c0c4b80. 00000020:00000010:10.0:1506127961.296459:0:3044:0:(tgt_main.c:382:tgt_ses_key_fini()) kfreed 'info': 160 at ffff88080f7b32c0. 00000020:00000010:10.0:1506127961.296461:0:3044:0:(lu_object.c:1595:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff88080f602400. 00000020:00000040:10.0:1506127961.296463:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507400 : new refcount 4 00000100:00000001:10.0:1506127961.296465:0:3044:0:(service.c:2155:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:10.0:1506127961.296468:0:3044:0:(niobuf.c:953:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000400:00000010:10.0:1506127961.296470:0:3044:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88081f3fc3c0. 00000400:00000010:10.0:1506127961.296472:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88042c0c4b80. 00000400:00000001:10.0:1506127961.296475:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:10.0:1506127961.296476:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:10.0:1506127961.297516:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:10.0:1506127961.297519:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:10.0:1506127961.297521:0:3044:0:(service.c:1859:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:10.0:1506127961.297522:0:3044:0:(sec.c:2052:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:10.0:1506127961.297524:0:3044:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:10.0:1506127961.297525:0:3044:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:10.0:1506127961.297528:0:3044:0:(sec.c:2099:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:10.0:1506127961.297531:0:3044:0:(service.c:1939:ptlrpc_server_handle_req_in()) got req x1579288666082352 00000020:00000001:10.0:1506127961.297533:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:10.0:1506127961.297534:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x2f3e107735f0c2a 00000020:00000001:10.0:1506127961.297536:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000040:10.0:1506127961.297537:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507400 : new refcount 5 00000020:00000001:10.0:1506127961.297539:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000020:00000001:10.0:1506127961.297542:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=18446612166930887680 : -131906778663936 : ffff88080f507400) 00000100:00000001:10.0:1506127961.297545:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:10.0:1506127961.297546:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:10.0:1506127961.297548:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88080f602400. 00000020:00000010:10.0:1506127961.297551:0:3044:0:(lu_ucred.c:49:lu_ucred_key_init()) kmalloced 'value': 104 at ffff88042c0c4680. 00000020:00000010:10.0:1506127961.297553:0:3044:0:(tgt_main.c:382:tgt_ses_key_init()) kmalloced 'value': 160 at ffff88080f7b32c0. 00000100:00000040:10.0:1506127961.297556:0:3044:0:(service.c:1184:ptlrpc_at_set_timer()) armed mdt_out at +1s 00000100:00000001:10.0:1506127961.297558:0:3044:0:(service.c:1656:ptlrpc_server_request_add()) Process entered 00000100:00000001:10.0:1506127961.297559:0:3044:0:(service.c:1580:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:10.0:1506127961.297560:0:3044:0:(service.c:1607:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.297563:0:3044:0:(service.c:1695:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.297569:0:3044:0:(service.c:2003:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:10.0:1506127961.297572:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:10.0:1506127961.297573:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:10.0:1506127961.297574:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:10.0:1506127961.297575:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:10.0:1506127961.297576:0:3044:0:(service.c:2028:ptlrpc_server_handle_request()) Process entered 00000100:00000001:10.0:1506127961.297577:0:3044:0:(service.c:1809:ptlrpc_server_request_get()) Process entered 00000100:00100000:10.0:1506127961.297581:0:3044:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.1.109@o2ib, seq: 22 00000100:00000040:10.0:1506127961.297584:0:3044:0:(service.c:1840:ptlrpc_server_request_get()) RPC GETting export ffff88080f507400 : new rpc_count 1 00000100:00000001:10.0:1506127961.297585:0:3044:0:(service.c:1842:ptlrpc_server_request_get()) Process leaving (rc=18446612167001116752 : -131906708434864 : ffff880813801050) 00000100:00000040:10.0:1506127961.297589:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "New" -> "Interpret" req@ffff880813801050 x1579288666082352/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:362/0 lens 304/0 e 0 to 0 dl 1506127967 ref 1 fl New:/0/ffffffff rc 0/-1 00000100:00000001:10.0:1506127961.297597:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:10.0:1506127961.297598:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000100:00100000:10.0:1506127961.297601:0:3044:0:(service.c:2089:ptlrpc_server_handle_request()) Handling RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+5:3098:x1579288666082352:12345-192.168.1.109@o2ib:1000 00000100:00000200:10.0:1506127961.297603:0:3044:0:(service.c:2094:ptlrpc_server_handle_request()) got req 1579288666082352 00000020:00000001:10.0:1506127961.297605:0:3044:0:(tgt_handler.c:672:tgt_request_handle()) Process entered 00000020:00000001:10.0:1506127961.297607:0:3044:0:(tgt_handler.c:617:process_req_last_xid()) Process entered 00000020:00000001:10.0:1506127961.297608:0:3044:0:(tgt_handler.c:657:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.297610:0:3044:0:(tgt_handler.c:584:tgt_handler_find_check()) Process entered 00000020:00000001:10.0:1506127961.297611:0:3044:0:(tgt_handler.c:611:tgt_handler_find_check()) Process leaving (rc=18446744072654881248 : -1054670368 : ffffffffc12301e0) 00000020:00000001:10.0:1506127961.297613:0:3044:0:(tgt_handler.c:523:tgt_handle_recovery()) Process entered 00000020:00000001:10.0:1506127961.297614:0:3044:0:(tgt_handler.c:573:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:10.0:1506127961.297616:0:3044:0:(tgt_handler.c:382:tgt_handle_request0()) Process entered 00000020:00000001:10.0:1506127961.297617:0:3044:0:(tgt_handler.c:311:tgt_request_preprocess()) Process entered 00000020:00000001:10.0:1506127961.297618:0:3044:0:(tgt_handler.c:366:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.297620:0:3044:0:(out_handler.c:917:out_handle()) Process entered 00000020:00000010:10.0:1506127961.297622:0:3044:0:(out_handler.c:940:out_handle()) kmalloced 'update_bufs': 8 at ffff88042cbe8818. 00000100:00000001:10.0:1506127961.297625:0:3044:0:(pack_generic.c:334:lustre_pack_reply_v2()) Process entered 02000000:00000001:10.0:1506127961.297626:0:3044:0:(sec.c:2112:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:10.0:1506127961.297629:0:3044:0:(sec_null.c:318:null_alloc_rs()) kmalloced 'rs': 4576 at ffff88080f146000. 02000000:00000001:10.0:1506127961.297630:0:3044:0:(sec.c:2148:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.297632:0:3044:0:(pack_generic.c:368:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.297636:0:3044:0:(lu_object.c:243:lu_object_alloc()) Process entered 00000004:00000001:10.0:1506127961.297637:0:3044:0:(mdt_handler.c:5144:mdt_object_alloc()) Process entered 00000004:00000010:10.0:1506127961.297638:0:3044:0:(mdt_handler.c:5146:mdt_object_alloc()) slab-alloced 'mo': 216 at ffff8808263ef878. 00000004:00000001:10.0:1506127961.297640:0:3044:0:(mdt_handler.c:5161:mdt_object_alloc()) Process leaving (rc=18446612167315617992 : -131906393933624 : ffff8808263ef8c8) 00000004:00000001:10.0:1506127961.297642:0:3044:0:(mdt_handler.c:5173:mdt_object_init()) Process entered 00000004:00000040:10.0:1506127961.297643:0:3044:0:(mdt_handler.c:5176:mdt_object_init()) object init, fid = [0x2c0003ab2:0x3:0x0] 00000004:00000010:10.0:1506127961.297645:0:3044:0:(mdd_object.c:120:mdd_object_alloc()) slab-alloced 'mdd_obj': 80 at ffff88042cbec370. 00000004:00000001:10.0:1506127961.297647:0:3044:0:(mdt_handler.c:5185:mdt_object_init()) Process leaving (rc=0 : 0 : 0) 00000004:00000001:10.0:1506127961.297648:0:3044:0:(mdd_object.c:143:mdd_object_init()) Process entered 00000004:00000001:10.0:1506127961.297649:0:3044:0:(lod_dev.c:210:lod_object_alloc()) Process entered 00000004:00000010:10.0:1506127961.297651:0:3044:0:(lod_dev.c:212:lod_object_alloc()) slab-alloced 'lod_obj': 104 at ffff88082aa63750. 00000004:00000001:10.0:1506127961.297652:0:3044:0:(lod_dev.c:221:lod_object_alloc()) Process leaving (rc=18446612167389493072 : -131906320058544 : ffff88082aa63750) 00000004:00000001:10.0:1506127961.297654:0:3044:0:(mdd_object.c:153:mdd_object_init()) Process leaving (rc=0 : 0 : 0) 00000004:00000001:10.0:1506127961.297656:0:3044:0:(lod_object.c:5108:lod_object_init()) Process entered 00000004:00000001:10.0:1506127961.297657:0:3044:0:(lod_dev.c:127:lod_fld_lookup()) Process entered 80000000:00000001:10.0:1506127961.297658:0:3044:0:(fld_handler.c:242:fld_server_lookup()) Process entered 80000000:00000001:10.0:1506127961.297659:0:3044:0:(fld_handler.c:207:fld_local_lookup()) Process entered 80000000:00000001:10.0:1506127961.297660:0:3044:0:(fld_cache.c:530:fld_cache_lookup()) Process entered 80000000:00000001:10.0:1506127961.297662:0:3044:0:(fld_cache.c:549:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:10.0:1506127961.297663:0:3044:0:(fld_handler.c:224:fld_local_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:10.0:1506127961.297664:0:3044:0:(fld_handler.c:246:fld_server_lookup()) Process leaving (rc=0 : 0 : 0) 00000004:00000040:10.0:1506127961.297666:0:3044:0:(lod_dev.c:168:lod_fld_lookup()) soaked-MDT0003-mdtlov: got tgt 3 for sequence: 0x2c0003ab2 00000004:00000001:10.0:1506127961.297667:0:3044:0:(lod_dev.c:170:lod_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00080000:00000010:10.0:1506127961.297669:0:3044:0:(osd_handler.c:332:osd_object_alloc()) kmalloced 'mo': 200 at ffff88042bf1ea00. 00000004:00000001:10.0:1506127961.297671:0:3044:0:(lod_object.c:5159:lod_object_init()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:10.0:1506127961.297673:0:3044:0:(osd_handler.c:994:osd_fid_lookup()) Process entered 00080000:00000001:10.0:1506127961.297676:0:3044:0:(osd_oi.c:545:fid_is_on_ost()) Process entered 80000000:00000001:10.0:1506127961.297677:0:3044:0:(fld_handler.c:207:fld_local_lookup()) Process entered 80000000:00000001:10.0:1506127961.297678:0:3044:0:(fld_cache.c:530:fld_cache_lookup()) Process entered 80000000:00000001:10.0:1506127961.297679:0:3044:0:(fld_cache.c:549:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:10.0:1506127961.297680:0:3044:0:(fld_handler.c:224:fld_local_lookup()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:10.0:1506127961.297682:0:3044:0:(osd_oi.c:583:fid_is_on_ost()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:10.0:1506127961.297684:0:3044:0:(osd_oi.c:515:osd_oi_iam_lookup()) Process entered 00000001:00000010:10.0:1506127961.297692:0:3044:0:(osd_dynlocks.c:109:dynlock_lock()) slab-alloced 'nhl': 72 at ffff88042cbef708. 00000001:00000010:10.0:1506127961.297705:0:3044:0:(osd_dynlocks.c:193:dynlock_unlock()) slab-freed 'hl': 72 at ffff88042cbef708. 00080000:00000001:10.0:1506127961.297708:0:3044:0:(osd_oi.c:537:osd_oi_iam_lookup()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:10.0:1506127961.297710:0:3044:0:(osd_oi.c:545:fid_is_on_ost()) Process entered 80000000:00000001:10.0:1506127961.297711:0:3044:0:(fld_handler.c:207:fld_local_lookup()) Process entered 80000000:00000001:10.0:1506127961.297711:0:3044:0:(fld_cache.c:530:fld_cache_lookup()) Process entered 80000000:00000001:10.0:1506127961.297712:0:3044:0:(fld_cache.c:549:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:10.0:1506127961.297714:0:3044:0:(fld_handler.c:224:fld_local_lookup()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:10.0:1506127961.297715:0:3044:0:(osd_oi.c:583:fid_is_on_ost()) Process leaving (rc=0 : 0 : 0) 00080000:00000001:10.0:1506127961.297716:0:3044:0:(osd_handler.c:1043:osd_fid_lookup()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:10.0:1506127961.297719:0:3044:0:(lu_object.c:295:lu_object_alloc()) Process leaving (rc=18446612167315617992 : -131906393933624 : ffff8808263ef8c8) 00000020:00000001:10.0:1506127961.297722:0:3044:0:(out_handler.c:205:out_attr_get()) Process entered 00000020:00000001:10.0:1506127961.297723:0:3044:0:(out_handler.c:217:out_attr_get()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00080000:00000010:10.0:1506127961.297727:0:3044:0:(osd_handler.c:1451:osd_object_free()) kfreed 'obj': 200 at ffff88042bf1ea00. 00000004:00000010:10.0:1506127961.297729:0:3044:0:(lod_object.c:5228:lod_object_free()) slab-freed '(lo)': 104 at ffff88082aa63750. 00000004:00000010:10.0:1506127961.297731:0:3044:0:(mdd_object.c:175:mdd_object_free()) slab-freed '(mdd)': 80 at ffff88042cbec370. 00000004:00000001:10.0:1506127961.297732:0:3044:0:(mdt_handler.c:5192:mdt_object_free()) Process entered 00000004:00000040:10.0:1506127961.297733:0:3044:0:(mdt_handler.c:5196:mdt_object_free()) object free, fid = [0x2c0003ab2:0x3:0x0] 00000004:00000010:10.0:1506127961.297735:0:3044:0:(mdt_handler.c:5203:mdt_object_free()) slab-freed '(mo)': 216 at ffff8808263ef878. 00000004:00000001:10.0:1506127961.297737:0:3044:0:(mdt_handler.c:5205:mdt_object_free()) Process leaving 00000020:00000001:10.0:1506127961.297738:0:3044:0:(out_handler.c:1150:out_handle()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000010:10.0:1506127961.297740:0:3044:0:(out_handler.c:1169:out_handle()) kfreed 'update_bufs': 8 at ffff88042cbe8818. 00000020:00000001:10.0:1506127961.297741:0:3044:0:(out_handler.c:1175:out_handle()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00010000:00000040:10.0:1506127961.297743:0:3044:0:(ldlm_lib.c:2884:target_committed_to_req()) last_committed 30064771076, transno 0, xid 1579288666082352 00010000:00000001:10.0:1506127961.297746:0:3044:0:(ldlm_lib.c:2948:target_send_reply()) Process entered 00010000:00000200:10.0:1506127961.297748:0:3044:0:(ldlm_lib.c:2936:target_send_reply_msg()) @@@ sending reply req@ffff880813801050 x1579288666082352/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:362/0 lens 304/4320 e 0 to 0 dl 1506127967 ref 1 fl Interpret:/0/0 rc -2/0 00010000:00000001:10.0:1506127961.297757:0:3044:0:(ldlm_lib.c:2895:target_pack_pool_reply()) Process entered 00010000:00000001:10.0:1506127961.297758:0:3044:0:(ldlm_lib.c:2903:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:10.0:1506127961.297760:0:3044:0:(import.c:1678:at_measured()) add 1 to ffff88081629fd50 time=41 v=1 (1 0 0 0) 00000100:00000001:10.0:1506127961.297763:0:3044:0:(connection.c:127:ptlrpc_connection_addref()) Process entered 00000100:00000040:10.0:1506127961.297765:0:3044:0:(connection.c:132:ptlrpc_connection_addref()) conn=ffff88082ede1660 refcount 10 to 192.168.1.109@o2ib 00000100:00000001:10.0:1506127961.297767:0:3044:0:(connection.c:134:ptlrpc_connection_addref()) Process leaving (rc=18446612167460263520 : -131906249288096 : ffff88082ede1660) 02000000:00000001:10.0:1506127961.297769:0:3044:0:(sec.c:2161:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:10.0:1506127961.297770:0:3044:0:(sec.c:2172:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.297771:0:3044:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:10.0:1506127961.297774:0:3044:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-192.168.1.109@o2ib 00000400:00000010:10.0:1506127961.297775:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88042c0c4700. 00000100:00000200:10.0:1506127961.297778:0:3044:0:(niobuf.c:85:ptl_send_buf()) Sending 4288 bytes to portal 4, xid 1579288666082352, offset 192 00000400:00000010:10.0:1506127961.297781:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88080f602e00 (tot 195264252). 00000400:00000200:10.0:1506127961.297784:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.109@o2ib 00000400:00000200:10.0:1506127961.297795:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.109@o2ib(192.168.1.109@o2ib:192.168.1.109@o2ib) : PUT 00000800:00000200:10.0:1506127961.297799:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 4288 bytes in 1 frags to 12345-192.168.1.109@o2ib 00000800:00000200:10.0:1506127961.297806:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082edb4980] -> 192.168.1.109@o2ib (2) version: 12 00000800:00000200:10.0:1506127961.297808:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082dfb7200] (20)++ 00000800:00000200:10.0:1506127961.297810:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082dfb7200] (21)++ 00000800:00000200:10.0:1506127961.297815:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082dfb7200] (22)-- 00000100:00000001:10.0:1506127961.297817:0:3044:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127961.297818:0:3044:0:(connection.c:91:ptlrpc_connection_put()) Process entered 00000100:00000040:10.0:1506127961.297820:0:3044:0:(connection.c:119:ptlrpc_connection_put()) PUT conn=ffff88082ede1660 refcount 9 to 192.168.1.109@o2ib 00000100:00000001:10.0:1506127961.297822:0:3044:0:(connection.c:121:ptlrpc_connection_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:10.0:1506127961.297823:0:3044:0:(ldlm_lib.c:2960:target_send_reply()) Process leaving 00000020:00000001:10.0:1506127961.297824:0:3044:0:(tgt_handler.c:475:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127961.297825:0:3044:0:(tgt_handler.c:780:tgt_request_handle()) Process leaving 00000100:00000040:10.0:1506127961.297828:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff880813801050 x1579288666082352/t0(0) o1000->soaked-MDT0001-mdtlov_UUID@192.168.1.109@o2ib:362/0 lens 304/4288 e 0 to 0 dl 1506127967 ref 1 fl Interpret:/0/0 rc -2/-2 00000100:00100000:10.0:1506127961.297836:0:3044:0:(service.c:2135:ptlrpc_server_handle_request()) Handled RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0001-mdtlov_UUID+5:3098:x1579288666082352:12345-192.168.1.109@o2ib:1000 Request procesed in 237us (378us total) trans 0 rc -2/-2 00000100:00100000:10.0:1506127961.297842:0:3044:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.1.109@o2ib, seq: 22 00000100:00000040:10.0:1506127961.297844:0:3044:0:(service.c:1034:ptlrpc_server_finish_active_request()) RPC PUTting export ffff88080f507400 : new rpc_count 0 00000100:00000001:10.0:1506127961.297846:0:3044:0:(service.c:1613:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:10.0:1506127961.297847:0:3044:0:(service.c:1624:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:10.0:1506127961.297848:0:3044:0:(lu_ucred.c:49:lu_ucred_key_fini()) kfreed 'info': 104 at ffff88042c0c4680. 00000020:00000010:10.0:1506127961.297850:0:3044:0:(tgt_main.c:382:tgt_ses_key_fini()) kfreed 'info': 160 at ffff88080f7b32c0. 00000020:00000010:10.0:1506127961.297852:0:3044:0:(lu_object.c:1595:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff88080f602400. 00000020:00000040:10.0:1506127961.297854:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507400 : new refcount 4 00000100:00000001:10.0:1506127961.297856:0:3044:0:(service.c:2155:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:10.0:1506127961.297859:0:3044:0:(niobuf.c:953:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000400:00000010:10.0:1506127961.297861:0:3044:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88081f3fc420. 00000400:00000010:10.0:1506127961.297863:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88042c0c4680. 00000400:00000001:10.0:1506127961.297866:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:10.0:1506127961.297867:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:10.0:1506127962.298147:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:10.0:1506127962.298151:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:10.0:1506127962.298153:0:3044:0:(service.c:1437:ptlrpc_at_check_timed()) Process entered 00000100:00000001:10.0:1506127962.298155:0:3044:0:(service.c:1449:ptlrpc_at_check_timed()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:10.0:1506127962.298158:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:10.0:1506127962.298159:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:10.0:1506127970.169504:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:10.0:1506127970.169508:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:10.0:1506127970.169510:0:3044:0:(service.c:1859:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:10.0:1506127970.169512:0:3044:0:(sec.c:2052:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:10.0:1506127970.169514:0:3044:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:10.0:1506127970.169515:0:3044:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:10.0:1506127970.169519:0:3044:0:(sec.c:2099:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:10.0:1506127970.169521:0:3044:0:(service.c:1939:ptlrpc_server_handle_req_in()) got req x1579283258788912 00000020:00000001:10.0:1506127970.169523:0:3044:0:(genops.c:711:class_conn2export()) Process entered 00000020:00000040:10.0:1506127970.169525:0:3044:0:(genops.c:723:class_conn2export()) looking for export cookie 0x2f3e107735f0c1c 00000020:00000001:10.0:1506127970.169526:0:3044:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000040:10.0:1506127970.169528:0:3044:0:(genops.c:807:class_export_get()) GETting export ffff88080f507800 : new refcount 5 00000020:00000001:10.0:1506127970.169531:0:3044:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612166930888704 : -131906778662912 : ffff88080f507800) 00000020:00000001:10.0:1506127970.169533:0:3044:0:(genops.c:725:class_conn2export()) Process leaving (rc=18446612166930888704 : -131906778662912 : ffff88080f507800) 00000100:00000001:10.0:1506127970.169536:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:10.0:1506127970.169538:0:3044:0:(service.c:1074:ptlrpc_update_export_timer()) Process leaving 00000020:00000010:10.0:1506127970.169540:0:3044:0:(lu_object.c:1678:keys_init()) kmalloced 'ctx->lc_value': 320 at ffff88080f602e00. 00000020:00000010:10.0:1506127970.169543:0:3044:0:(lu_ucred.c:49:lu_ucred_key_init()) kmalloced 'value': 104 at ffff88042c0c4780. 00000020:00000010:10.0:1506127970.169545:0:3044:0:(tgt_main.c:382:tgt_ses_key_init()) kmalloced 'value': 160 at ffff88080f7b3680. 00000100:00000040:10.0:1506127970.169549:0:3044:0:(service.c:1184:ptlrpc_at_set_timer()) armed mdt_out at +1s 00000100:00000001:10.0:1506127970.169551:0:3044:0:(service.c:1656:ptlrpc_server_request_add()) Process entered 00000100:00000001:10.0:1506127970.169552:0:3044:0:(service.c:1580:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:10.0:1506127970.169553:0:3044:0:(service.c:1607:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127970.169556:0:3044:0:(service.c:1695:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127970.169563:0:3044:0:(service.c:2003:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:10.0:1506127970.169566:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:10.0:1506127970.169567:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:10.0:1506127970.169569:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:10.0:1506127970.169570:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:10.0:1506127970.169571:0:3044:0:(service.c:2028:ptlrpc_server_handle_request()) Process entered 00000100:00000001:10.0:1506127970.169572:0:3044:0:(service.c:1809:ptlrpc_server_request_get()) Process entered 00000100:00100000:10.0:1506127970.169576:0:3044:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.1.108@o2ib, seq: 23 00000100:00000040:10.0:1506127970.169579:0:3044:0:(service.c:1840:ptlrpc_server_request_get()) RPC GETting export ffff88080f507800 : new rpc_count 1 00000100:00000001:10.0:1506127970.169580:0:3044:0:(service.c:1842:ptlrpc_server_request_get()) Process leaving (rc=18446612167001115728 : -131906708435888 : ffff880813800c50) 00000100:00000040:10.0:1506127970.169584:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "New" -> "Interpret" req@ffff880813800c50 x1579283258788912/t0(0) o400->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:371/0 lens 224/0 e 0 to 0 dl 1506127976 ref 1 fl New:/0/ffffffff rc 0/-1 00000100:00000001:10.0:1506127970.169593:0:3044:0:(service.c:1049:ptlrpc_update_export_timer()) Process entered 00000100:00000001:10.0:1506127970.169594:0:3044:0:(service.c:1062:ptlrpc_update_export_timer()) Process leaving 00000100:00100000:10.0:1506127970.169597:0:3044:0:(service.c:2089:ptlrpc_server_handle_request()) Handling RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0000-mdtlov_UUID+5:2068:x1579283258788912:12345-192.168.1.108@o2ib:400 00000100:00000200:10.0:1506127970.169600:0:3044:0:(service.c:2094:ptlrpc_server_handle_request()) got req 1579283258788912 00000020:00000001:10.0:1506127970.169602:0:3044:0:(tgt_handler.c:672:tgt_request_handle()) Process entered 00000020:00000001:10.0:1506127970.169604:0:3044:0:(tgt_handler.c:617:process_req_last_xid()) Process entered 00000020:00000001:10.0:1506127970.169605:0:3044:0:(tgt_handler.c:657:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127970.169607:0:3044:0:(tgt_handler.c:584:tgt_handler_find_check()) Process entered 00000020:00000001:10.0:1506127970.169609:0:3044:0:(tgt_handler.c:611:tgt_handler_find_check()) Process leaving (rc=18446744072654874048 : -1054677568 : ffffffffc122e5c0) 00000020:00000001:10.0:1506127970.169611:0:3044:0:(tgt_handler.c:523:tgt_handle_recovery()) Process entered 00000020:00000001:10.0:1506127970.169612:0:3044:0:(tgt_handler.c:573:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:10.0:1506127970.169614:0:3044:0:(tgt_handler.c:382:tgt_handle_request0()) Process entered 00000020:00000001:10.0:1506127970.169615:0:3044:0:(tgt_handler.c:311:tgt_request_preprocess()) Process entered 00000020:00000001:10.0:1506127970.169616:0:3044:0:(tgt_handler.c:366:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127970.169618:0:3044:0:(tgt_handler.c:983:tgt_obd_ping()) Process entered 00010000:00000001:10.0:1506127970.169619:0:3044:0:(obd_class.h:917:obd_ping()) Process entered 00010000:00000001:10.0:1506127970.169620:0:3044:0:(obd_class.h:919:obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127970.169622:0:3044:0:(pack_generic.c:334:lustre_pack_reply_v2()) Process entered 02000000:00000001:10.0:1506127970.169623:0:3044:0:(sec.c:2112:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:10.0:1506127970.169626:0:3044:0:(sec_null.c:318:null_alloc_rs()) kmalloced 'rs': 480 at ffff88080f603200. 02000000:00000001:10.0:1506127970.169627:0:3044:0:(sec.c:2148:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127970.169629:0:3044:0:(pack_generic.c:368:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127970.169631:0:3044:0:(tgt_handler.c:989:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:10.0:1506127970.169632:0:3044:0:(ldlm_lib.c:2884:target_committed_to_req()) last_committed 30064771074, transno 0, xid 1579283258788912 00010000:00000001:10.0:1506127970.169635:0:3044:0:(ldlm_lib.c:2948:target_send_reply()) Process entered 00010000:00000200:10.0:1506127970.169637:0:3044:0:(ldlm_lib.c:2936:target_send_reply_msg()) @@@ sending reply req@ffff880813800c50 x1579283258788912/t0(0) o400->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:371/0 lens 224/224 e 0 to 0 dl 1506127976 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000001:10.0:1506127970.169645:0:3044:0:(ldlm_lib.c:2895:target_pack_pool_reply()) Process entered 00010000:00000001:10.0:1506127970.169646:0:3044:0:(ldlm_lib.c:2903:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:10.0:1506127970.169648:0:3044:0:(import.c:1678:at_measured()) add 1 to ffff88081629fd50 time=50 v=1 (1 0 0 0) 00000100:00000001:10.0:1506127970.169651:0:3044:0:(connection.c:127:ptlrpc_connection_addref()) Process entered 00000100:00000040:10.0:1506127970.169654:0:3044:0:(connection.c:132:ptlrpc_connection_addref()) conn=ffff88082ede1780 refcount 14 to 192.168.1.108@o2ib 00000100:00000001:10.0:1506127970.169656:0:3044:0:(connection.c:134:ptlrpc_connection_addref()) Process leaving (rc=18446612167460263808 : -131906249287808 : ffff88082ede1780) 02000000:00000001:10.0:1506127970.169658:0:3044:0:(sec.c:2161:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:10.0:1506127970.169659:0:3044:0:(sec.c:2172:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127970.169661:0:3044:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:10.0:1506127970.169663:0:3044:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-192.168.1.108@o2ib 00000400:00000010:10.0:1506127970.169665:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88042c0c5700. 00000100:00000200:10.0:1506127970.169668:0:3044:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 4, xid 1579283258788912, offset 192 00000400:00000010:10.0:1506127970.169671:0:3044:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88080f602a00 (tot 195432892). 00000400:00000200:10.0:1506127970.169674:0:3044:0:(lib-move.c:2796:LNetPut()) LNetPut -> 12345-192.168.1.108@o2ib 00000400:00000200:10.0:1506127970.169686:0:3044:0:(lib-move.c:1930:lnet_select_pathway()) TRACE: 192.168.1.111@o2ib(192.168.1.111@o2ib:192.168.1.111@o2ib) -> 192.168.1.108@o2ib(192.168.1.108@o2ib:192.168.1.108@o2ib) : PUT 00000800:00000200:10.0:1506127970.169690:0:3044:0:(o2iblnd_cb.c:1505:kiblnd_send()) sending 192 bytes in 1 frags to 12345-192.168.1.108@o2ib 00000800:00000200:10.0:1506127970.169696:0:3044:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88082cb07e00] -> 192.168.1.108@o2ib (2) version: 12 00000800:00000200:10.0:1506127970.169699:0:3044:0:(o2iblnd_cb.c:1386:kiblnd_launch_tx()) conn[ffff88082e03de00] (20)++ 00000800:00000200:10.0:1506127970.169701:0:3044:0:(o2iblnd_cb.c:1156:kiblnd_queue_tx_locked()) conn[ffff88082e03de00] (21)++ 00000800:00000200:10.0:1506127970.169706:0:3044:0:(o2iblnd_cb.c:1392:kiblnd_launch_tx()) conn[ffff88082e03de00] (22)-- 00000100:00000001:10.0:1506127970.169708:0:3044:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:10.0:1506127970.169709:0:3044:0:(connection.c:91:ptlrpc_connection_put()) Process entered 00000100:00000040:10.0:1506127970.169711:0:3044:0:(connection.c:119:ptlrpc_connection_put()) PUT conn=ffff88082ede1780 refcount 13 to 192.168.1.108@o2ib 00000100:00000001:10.0:1506127970.169713:0:3044:0:(connection.c:121:ptlrpc_connection_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:10.0:1506127970.169714:0:3044:0:(ldlm_lib.c:2960:target_send_reply()) Process leaving 00000020:00000001:10.0:1506127970.169715:0:3044:0:(tgt_handler.c:475:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:10.0:1506127970.169717:0:3044:0:(tgt_handler.c:780:tgt_request_handle()) Process leaving 00000100:00000040:10.0:1506127970.169719:0:3044:0:(lustre_net.h:2481:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff880813800c50 x1579283258788912/t0(0) o400->soaked-MDT0000-mdtlov_UUID@192.168.1.108@o2ib:371/0 lens 224/192 e 0 to 0 dl 1506127976 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00100000:10.0:1506127970.169727:0:3044:0:(service.c:2135:ptlrpc_server_handle_request()) Handled RPC pname:cluuid+ref:pid:xid:nid:opc mdt_out01_001:soaked-MDT0000-mdtlov_UUID+5:2068:x1579283258788912:12345-192.168.1.108@o2ib:400 Request procesed in 133us (273us total) trans 0 rc 0/0 00000100:00100000:10.0:1506127970.169734:0:3044:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.1.108@o2ib, seq: 23 00000100:00000040:10.0:1506127970.169736:0:3044:0:(service.c:1034:ptlrpc_server_finish_active_request()) RPC PUTting export ffff88080f507800 : new rpc_count 0 00000100:00000001:10.0:1506127970.169738:0:3044:0:(service.c:1613:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:10.0:1506127970.169739:0:3044:0:(service.c:1624:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:10.0:1506127970.169741:0:3044:0:(lu_ucred.c:49:lu_ucred_key_fini()) kfreed 'info': 104 at ffff88042c0c4780. 00000020:00000010:10.0:1506127970.169743:0:3044:0:(tgt_main.c:382:tgt_ses_key_fini()) kfreed 'info': 160 at ffff88080f7b3680. 00000020:00000010:10.0:1506127970.169745:0:3044:0:(lu_object.c:1595:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff88080f602e00. 00000020:00000040:10.0:1506127970.169748:0:3044:0:(genops.c:817:class_export_put()) PUTting export ffff88080f507800 : new refcount 4 00000100:00000001:10.0:1506127970.169750:0:3044:0:(service.c:2155:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:10.0:1506127970.169753:0:3044:0:(niobuf.c:953:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000400:00000010:10.0:1506127970.169756:0:3044:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88081f3fc600. 00000400:00000010:10.0:1506127970.169758:0:3044:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff88042c0c4780. 00000400:00000001:10.0:1506127970.169760:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:10.0:1506127970.169762:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving 00000400:00000001:10.0:1506127971.170170:0:3044:0:(watchdog.c:412:lc_watchdog_touch()) Process entered 00000400:00000001:10.0:1506127971.170174:0:3044:0:(watchdog.c:422:lc_watchdog_touch()) Process leaving 00000100:00000001:10.0:1506127971.170176:0:3044:0:(service.c:1437:ptlrpc_at_check_timed()) Process entered 00000100:00000001:10.0:1506127971.170178:0:3044:0:(service.c:1449:ptlrpc_at_check_timed()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:10.0:1506127971.170181:0:3044:0:(watchdog.c:428:lc_watchdog_disable()) Process entered 00000400:00000001:10.0:1506127971.170182:0:3044:0:(watchdog.c:436:lc_watchdog_disable()) Process leaving