00000400:00000010:3.0F:1506921872.564204:0:38872:0:(module.c:155:libcfs_ioctl()) kfreed 'hdr': 128 at ffff880035deaf80 (tot 59423468). 00000400:00000001:3.0:1506921872.564207:0:38872:0:(module.c:156:libcfs_ioctl()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0F:1506921872.584782:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.31@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921872.584789:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.32@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921872.584792:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.41@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921872.584795:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.42@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000100:00000001:14.0F:1506921872.611767:0:3910:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0F:1506921872.611768:0:3902:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0F:1506921872.611770:0:3905:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:14.0:1506921872.611771:0:3910:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921872.611772:0:3902:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921872.611774:0:3905:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921872.611775:0:3902:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:14.0:1506921872.611775:0:3910:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921872.611776:0:3902:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921872.611776:0:3910:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921872.611777:0:3905:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921872.611778:0:3905:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921872.611778:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921872.611778:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921872.611780:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921872.611782:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921872.611782:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0F:1506921872.611782:0:3911:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:4.0F:1506921872.611782:0:3903:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921872.611783:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921872.611783:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921872.611784:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921872.611785:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921872.611785:0:3903:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921872.611786:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921872.611786:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921872.611786:0:3911:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921872.611787:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921872.611788:0:3903:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:6.0:1506921872.611789:0:3911:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:4.0:1506921872.611789:0:3903:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921872.611790:0:3911:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921872.611791:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921872.611792:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921872.611795:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921872.611796:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921872.611796:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921872.611797:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921872.611799:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921872.611799:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921872.612767:0:3909:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921872.612767:0:3907:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921872.612769:0:3909:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921872.612769:0:3907:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921872.612771:0:3909:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921872.612772:0:3909:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921872.612772:0:3907:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921872.612772:0:3907:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921872.612774:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921872.612774:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921872.612777:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921872.612778:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921872.612779:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921872.612779:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0F:1506921872.612779:0:3906:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921872.612781:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921872.612782:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921872.612783:0:3906:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:8.0:1506921872.612786:0:3906:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:8.0:1506921872.612787:0:3906:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921872.612788:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921872.612791:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921872.612793:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921872.612794:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:12.0F:1506921872.613760:0:3908:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:14.0:1506921872.613762:0:3904:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:14.0:1506921872.613764:0:3904:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:12.0:1506921872.613764:0:3908:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:14.0:1506921872.613767:0:3904:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:12.0:1506921872.613767:0:3908:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:14.0:1506921872.613768:0:3904:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:12.0:1506921872.613768:0:3908:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:12.0:1506921872.613769:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921872.613770:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921872.613773:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:12.0:1506921872.613773:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921872.613774:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:12.0:1506921872.613775:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921872.613777:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:12.0:1506921872.613777:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921873.035788:0:3901:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921873.035790:0:3901:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921873.035793:0:3901:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921873.035794:0:3901:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921873.035795:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921873.035798:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921873.035799:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921873.035801:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0F:1506921873.040766:0:3917:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:7.0F:1506921873.040766:0:3913:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:7.0:1506921873.040770:0:3913:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:17.0:1506921873.040773:0:3917:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:7.0:1506921873.040773:0:3913:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:7.0:1506921873.040774:0:3913:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921873.040776:0:3917:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:7.0:1506921873.040776:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921873.040777:0:3917:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0F:1506921873.040778:0:3918:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:17.0:1506921873.040779:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0F:1506921873.040779:0:3912:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:7.0:1506921873.040779:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921873.040781:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921873.040782:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921873.040782:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921873.040782:0:3918:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:17.0:1506921873.040783:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921873.040783:0:3912:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:17.0:1506921873.040785:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921873.040785:0:3918:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921873.040786:0:3912:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921873.040786:0:3918:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921873.040787:0:3912:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921873.040788:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921873.040788:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921873.040791:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921873.040792:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921873.040793:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921873.040793:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921873.040795:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921873.040795:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0F:1506921873.041754:0:3914:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:11.0:1506921873.041758:0:3914:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921873.041761:0:3914:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:11.0:1506921873.041761:0:3914:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921873.041763:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:11.0:1506921873.041766:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921873.041767:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:11.0:1506921873.041768:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:5.0F:1506921873.041769:0:3919:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0F:1506921873.041771:0:3920:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:9.0F:1506921873.041771:0:3915:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:5.0:1506921873.041773:0:3919:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921873.041775:0:3920:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:9.0:1506921873.041775:0:3915:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:5.0:1506921873.041776:0:3919:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:5.0:1506921873.041777:0:3919:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921873.041778:0:3920:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:9.0:1506921873.041778:0:3915:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921873.041779:0:3920:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921873.041779:0:3915:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:5.0:1506921873.041779:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921873.041780:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:9.0:1506921873.041780:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:5.0:1506921873.041782:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:5.0:1506921873.041783:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921873.041784:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921873.041784:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921873.041785:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:5.0:1506921873.041785:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921873.041786:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921873.041787:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921873.041787:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921873.042769:0:3916:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921873.042771:0:3916:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921873.042771:0:3921:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:15.0:1506921873.042773:0:3921:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921873.042774:0:3916:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921873.042775:0:3916:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921873.042776:0:3921:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921873.042777:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921873.042777:0:3921:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921873.042778:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921873.042779:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921873.042781:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921873.042781:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921873.042782:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921873.042783:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921873.042785:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1506921873.584779:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.31@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921873.584784:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.32@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921873.584788:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.41@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921873.584790:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.42@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000100:00000001:16.0:1506921873.611770:0:3902:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921873.611772:0:3902:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921873.611775:0:3902:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921873.611776:0:3902:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921873.611776:0:3903:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921873.611777:0:3905:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921873.611777:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921873.611777:0:3911:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:4.0:1506921873.611778:0:3903:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921873.611779:0:3905:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:6.0:1506921873.611779:0:3911:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921873.611781:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921873.611781:0:3903:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921873.611782:0:3905:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921873.611782:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921873.611782:0:3911:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:6.0:1506921873.611782:0:3911:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921873.611783:0:3905:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921873.611783:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921873.611783:0:3903:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921873.611784:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921873.611784:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921873.611785:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921873.611789:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921873.611789:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921873.611789:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921873.611790:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921873.611791:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921873.611791:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921873.611793:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921873.611794:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921873.611794:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921873.611800:0:3910:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1506921873.611802:0:3910:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1506921873.611805:0:3910:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921873.611806:0:3910:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921873.611808:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921873.611811:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921873.611813:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921873.611814:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921873.612767:0:3907:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921873.612769:0:3909:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921873.612769:0:3907:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921873.612771:0:3909:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921873.612772:0:3907:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921873.612773:0:3907:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921873.612774:0:3909:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921873.612775:0:3909:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921873.612775:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921873.612777:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921873.612778:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921873.612778:0:3906:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921873.612779:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921873.612780:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921873.612780:0:3906:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921873.612781:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921873.612781:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921873.612782:0:3906:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921873.612783:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921873.612783:0:3906:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921873.612785:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921873.612788:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921873.612790:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921873.612791:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921873.613767:0:3904:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:14.0:1506921873.613769:0:3904:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:14.0:1506921873.613772:0:3904:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:14.0:1506921873.613773:0:3904:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921873.613774:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921873.613777:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921873.613778:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921873.613780:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:12.0:1506921873.613783:0:3908:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:12.0:1506921873.613785:0:3908:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:12.0:1506921873.613788:0:3908:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:12.0:1506921873.613789:0:3908:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:12.0:1506921873.613791:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:12.0:1506921873.613794:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:12.0:1506921873.613795:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:12.0:1506921873.613796:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921874.035769:0:3901:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921874.035772:0:3901:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921874.035775:0:3901:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921874.035776:0:3901:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921874.035778:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921874.035780:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921874.035782:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921874.035783:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921874.040766:0:3918:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1506921874.040768:0:3918:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921874.040771:0:3918:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921874.040772:0:3918:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921874.040774:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921874.040777:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921874.040778:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921874.040779:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921874.040780:0:3913:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:17.0:1506921874.040782:0:3912:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:7.0:1506921874.040782:0:3913:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:17.0:1506921874.040783:0:3912:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:7.0:1506921874.040785:0:3913:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:17.0:1506921874.040786:0:3912:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:7.0:1506921874.040786:0:3913:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921874.040787:0:3912:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921874.040787:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921874.040789:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921874.040790:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921874.040792:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921874.040792:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921874.040793:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921874.040793:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921874.040795:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921874.040803:0:3917:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921874.040804:0:3917:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921874.040807:0:3917:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921874.040808:0:3917:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921874.040810:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921874.040814:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921874.040816:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921874.040817:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921874.041779:0:3915:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:11.0:1506921874.041780:0:3914:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:11.0:1506921874.041781:0:3914:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:9.0:1506921874.041781:0:3915:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.041784:0:3914:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:9.0:1506921874.041784:0:3915:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:11.0:1506921874.041785:0:3914:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921874.041785:0:3915:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921874.041786:0:3919:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:11.0:1506921874.041787:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:9.0:1506921874.041787:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921874.041788:0:3919:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.041791:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921874.041791:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921874.041791:0:3919:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:11.0:1506921874.041792:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:9.0:1506921874.041792:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921874.041792:0:3919:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921874.041793:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:11.0:1506921874.041795:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921874.041795:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921874.041796:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921874.041798:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921874.041799:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921874.041802:0:3920:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1506921874.041804:0:3920:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921874.041807:0:3920:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921874.041807:0:3920:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921874.041809:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921874.041812:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921874.041813:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921874.041814:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921874.042769:0:3916:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921874.042771:0:3916:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.042771:0:3921:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:15.0:1506921874.042773:0:3921:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921874.042774:0:3916:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921874.042774:0:3916:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.042775:0:3921:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921874.042776:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921874.042777:0:3921:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.042778:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921874.042779:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921874.042780:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921874.042781:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921874.042782:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.042783:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921874.042785:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1506921874.584781:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.31@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921874.584786:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.32@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921874.584789:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.41@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921874.584791:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.42@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000100:00000001:16.0:1506921874.611760:0:3902:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921874.611762:0:3902:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:6.0:1506921874.611762:0:3911:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:6.0:1506921874.611764:0:3911:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921874.611765:0:3902:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921874.611766:0:3902:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921874.611767:0:3911:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:6.0:1506921874.611767:0:3911:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.611768:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921874.611769:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921874.611771:0:3910:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921874.611772:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921874.611772:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921874.611773:0:3905:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921874.611773:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921874.611773:0:3903:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1506921874.611773:0:3910:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:6.0:1506921874.611774:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921874.611775:0:3905:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:4.0:1506921874.611775:0:3903:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1506921874.611775:0:3910:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921874.611776:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921874.611776:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921874.611777:0:3910:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921874.611778:0:3905:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:4.0:1506921874.611778:0:3903:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921874.611778:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921874.611779:0:3905:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921874.611779:0:3903:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921874.611781:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921874.611781:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921874.611781:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921874.611783:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921874.611785:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921874.611785:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921874.611785:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921874.611787:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921874.611787:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921874.611790:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921874.611790:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921874.612765:0:3909:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921874.612766:0:3907:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921874.612767:0:3909:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921874.612769:0:3907:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921874.612770:0:3909:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921874.612771:0:3909:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.612772:0:3907:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921874.612773:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921874.612773:0:3907:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.612775:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921874.612776:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921874.612776:0:3906:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921874.612777:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921874.612778:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.612778:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921874.612778:0:3906:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921874.612779:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921874.612781:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921874.612781:0:3906:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:8.0:1506921874.612782:0:3906:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921874.612783:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921874.612786:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921874.612788:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921874.612789:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921874.613751:0:3904:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:12.0:1506921874.613751:0:3908:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:14.0:1506921874.613753:0:3904:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:12.0:1506921874.613753:0:3908:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:14.0:1506921874.613756:0:3904:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:12.0:1506921874.613756:0:3908:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:14.0:1506921874.613757:0:3904:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:12.0:1506921874.613757:0:3908:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921874.613759:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:12.0:1506921874.613759:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921874.613762:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:12.0:1506921874.613762:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921874.613764:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:12.0:1506921874.613764:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921874.613766:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:12.0:1506921874.613766:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:11.0:1506921874.689288:0:38877:0:(file.c:3755:ll_inode_revalidate()) Process entered 00000080:00000001:11.0:1506921874.689291:0:38877:0:(file.c:3643:__ll_inode_revalidate()) Process entered 00000080:00200000:11.0:1506921874.689292:0:38877:0:(file.c:3648:__ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff881021f03690),name=/ 00000080:00000010:11.0:1506921874.689297:0:38877:0:(llite_lib.c:2434:ll_prep_md_op_data()) kmalloced 'op_data': 320 at ffff882011e16200. 00000080:00000001:11.0:1506921874.689300:0:38877:0:(obd_class.h:1379:md_intent_lock()) Process entered 00800000:00000001:11.0:1506921874.689302:0:38877:0:(lmv_intent.c:474:lmv_intent_lock()) Process entered 00800000:00000002:11.0:1506921874.689304:0:38877:0:(lmv_intent.c:482:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:11.0:1506921874.689307:0:38877:0:(lmv_intent.c:361:lmv_intent_lookup()) Process entered 00800000:00000002:11.0:1506921874.689308:0:38877:0:(lmv_intent.c:390:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 lsm= (null) lsm_magic=ffffffff 00800000:00000001:11.0:1506921874.689312:0:38877:0:(obd_class.h:1379:md_intent_lock()) Process entered 00000002:00000001:11.0:1506921874.689314:0:38877:0:(mdc_locks.c:1118:mdc_intent_lock()) Process entered 00000002:00010000:11.0:1506921874.689315:0:38877:0:(mdc_locks.c:1125:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:11.0:1506921874.689318:0:38877:0:(mdc_locks.c:1024:mdc_revalidate_lock()) Process entered 00000002:00000001:11.0:1506921874.689319:0:38877:0:(mdc_locks.c:142:mdc_lock_match()) Process entered 00010000:00000001:11.0:1506921874.689321:0:38877:0:(ldlm_lock.c:1384:ldlm_lock_match()) Process entered 00010000:00000040:11.0:1506921874.689324:0:38877:0:(ldlm_resource.c:1480:ldlm_resource_getref()) getref res: ffff88203b7332c0 count: 2 00010000:00000001:11.0:1506921874.689326:0:38877:0:(ldlm_resource.c:1415:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:11.0:1506921874.689329:0:38877:0:(ldlm_lock.c:260:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:11.0:1506921874.689330:0:38877:0:(ldlm_lock.c:271:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000040:11.0:1506921874.689332:0:38877:0:(ldlm_resource.c:1480:ldlm_resource_getref()) getref res: ffff88203b7332c0 count: 3 00010000:00010000:11.0:1506921874.689334:0:38877:0:(ldlm_lock.c:742:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: oak-MDT0000-mdc-ffff88203bc47000 lock: ffff88101d368000/0x228de8b50142155 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13 rrc: 3 type: IBT flags: 0x0 nid: local remote: 0xc1d2b9af9ec6a943 expref: -99 pid: 4287 timeout: 0 lvb_type: 0 00010000:00000040:11.0:1506921874.689347:0:38877:0:(ldlm_resource.c:1519:ldlm_resource_putref()) putref res: ffff88203b7332c0 count: 2 00010000:00000001:11.0:1506921874.689349:0:38877:0:(ldlm_lock.c:1410:ldlm_lock_match()) Process leaving via out (rc=1 : 1 : 0x1) 00010000:00000040:11.0:1506921874.689350:0:38877:0:(ldlm_resource.c:1519:ldlm_resource_putref()) putref res: ffff88203b7332c0 count: 1 00010000:00000040:11.0:1506921874.689352:0:38877:0:(ldlm_resource.c:1480:ldlm_resource_getref()) getref res: ffff88203b7332c0 count: 2 00010000:00010000:11.0:1506921874.689353:0:38877:0:(ldlm_lock.c:1470:ldlm_lock_match()) ### matched (0 0) ns: oak-MDT0000-mdc-ffff88203bc47000 lock: ffff88101d368000/0x228de8b50142155 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13 rrc: 2 type: IBT flags: 0x0 nid: local remote: 0xc1d2b9af9ec6a943 expref: -99 pid: 4287 timeout: 0 lvb_type: 0 00010000:00000040:11.0:1506921874.689357:0:38877:0:(ldlm_resource.c:1519:ldlm_resource_putref()) putref res: ffff88203b7332c0 count: 1 02000000:00000001:11.0:1506921874.689359:0:38877:0:(sec.c:936:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:11.0:1506921874.689361:0:38877:0:(sec.c:950:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:11.0:1506921874.689362:0:38877:0:(mdc_locks.c:149:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:11.0:1506921874.689363:0:38877:0:(mdc_locks.c:1076:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:11.0:1506921874.689365:0:38877:0:(mdc_locks.c:1138:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:11.0:1506921874.689366:0:38877:0:(obd_class.h:1384:md_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:11.0:1506921874.689367:0:38877:0:(lmv_intent.c:409:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:11.0:1506921874.689368:0:38877:0:(lmv_intent.c:515:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:11.0:1506921874.689370:0:38877:0:(obd_class.h:1384:md_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00000010:11.0:1506921874.689371:0:38877:0:(llite_lib.c:2485:ll_finish_md_op_data()) kfreed 'op_data': 320 at ffff882011e16200. 00000080:00000001:11.0:1506921874.689374:0:38877:0:(dcache.c:267:ll_revalidate_it_finish()) Process entered 00000080:00000001:11.0:1506921874.689376:0:38877:0:(dcache.c:270:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:11.0:1506921874.689377:0:38877:0:(dcache.c:290:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff881021f03690) 00000080:00010000:11.0:1506921874.689379:0:38877:0:(llite_internal.h:1368:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff881021f03690) for lock 0x228de8b50142155 00000080:00000001:11.0:1506921874.689381:0:38877:0:(obd_class.h:1545:md_set_lock_data()) Process entered 00800000:00000001:11.0:1506921874.689382:0:38877:0:(lmv_obd.c:2845:lmv_set_lock_data()) Process entered 00800000:00000001:11.0:1506921874.689384:0:38877:0:(obd_class.h:1545:md_set_lock_data()) Process entered 00000002:00000001:11.0:1506921874.689385:0:38877:0:(mdc_locks.c:103:mdc_set_lock_data()) Process entered 00010000:00000001:11.0:1506921874.689386:0:38877:0:(ldlm_lock.c:584:__ldlm_handle2lock()) Process entered 00000020:00000001:11.0:1506921874.689387:0:38877:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000001:11.0:1506921874.689389:0:38877:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612201523806208 : -131872185745408 : ffff88101d368000) 00010000:00000001:11.0:1506921874.689392:0:38877:0:(ldlm_lock.c:603:__ldlm_handle2lock()) Process leaving (rc=18446612201523806208 : -131872185745408 : ffff88101d368000) 00010000:00000001:11.0:1506921874.689393:0:38877:0:(ldlm_lock.c:188:ldlm_lock_put()) Process entered 00010000:00000001:11.0:1506921874.689394:0:38877:0:(ldlm_lock.c:222:ldlm_lock_put()) Process leaving 00000002:00000001:11.0:1506921874.689395:0:38877:0:(mdc_locks.c:132:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:11.0:1506921874.689397:0:38877:0:(obd_class.h:1548:md_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:11.0:1506921874.689398:0:38877:0:(lmv_obd.c:2850:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:11.0:1506921874.689399:0:38877:0:(obd_class.h:1548:md_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:11.0:1506921874.689400:0:38877:0:(dcache.c:201:ll_intent_drop_lock()) releasing lock with cookie 0x228de8b50142155 from it ffff880168d1fd58 00010000:00000001:11.0:1506921874.689402:0:38877:0:(ldlm_lock.c:584:__ldlm_handle2lock()) Process entered 00000020:00000001:11.0:1506921874.689402:0:38877:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000001:11.0:1506921874.689403:0:38877:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612201523806208 : -131872185745408 : ffff88101d368000) 00010000:00000001:11.0:1506921874.689405:0:38877:0:(ldlm_lock.c:603:__ldlm_handle2lock()) Process leaving (rc=18446612201523806208 : -131872185745408 : ffff88101d368000) 00010000:00000001:11.0:1506921874.689406:0:38877:0:(ldlm_lock.c:823:ldlm_lock_decref_internal()) Process entered 00010000:00010000:11.0:1506921874.689408:0:38877:0:(ldlm_lock.c:796:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: ?? lock: ffff88101d368000/0x228de8b50142155 lrc: 3/1,0 mode: PR/PR res: ?? rrc=?? type: ??? flags: 0x10000000000000 nid: local remote: 0xc1d2b9af9ec6a943 expref: -99 pid: 4287 timeout: 0 lvb_type: 0 00010000:00000001:11.0:1506921874.689411:0:38877:0:(ldlm_lock.c:188:ldlm_lock_put()) Process entered 00010000:00000001:11.0:1506921874.689412:0:38877:0:(ldlm_lock.c:222:ldlm_lock_put()) Process leaving 00010000:00010000:11.0:1506921874.689413:0:38877:0:(ldlm_lock.c:869:ldlm_lock_decref_internal()) ### add lock into lru list ns: ?? lock: ffff88101d368000/0x228de8b50142155 lrc: 2/0,0 mode: PR/PR res: ?? rrc=?? type: ??? flags: 0x10000000000000 nid: local remote: 0xc1d2b9af9ec6a943 expref: -99 pid: 4287 timeout: 0 lvb_type: 0 00010000:00000001:11.0:1506921874.689416:0:38877:0:(ldlm_lock.c:298:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:11.0:1506921874.689417:0:38877:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process leaving 00010000:00000001:11.0:1506921874.689418:0:38877:0:(ldlm_lock.c:890:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:11.0:1506921874.689419:0:38877:0:(ldlm_lock.c:188:ldlm_lock_put()) Process entered 00010000:00000001:11.0:1506921874.689419:0:38877:0:(ldlm_lock.c:222:ldlm_lock_put()) Process leaving 00000100:00000001:11.0:1506921874.689420:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000001:11.0:1506921874.689421:0:38877:0:(client.c:2514:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:11.0:1506921874.689423:0:38877:0:(file.c:3784:ll_inode_revalidate()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:11.0:1506921874.689634:0:38877:0:(llite_lib.c:2525:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:11.0:1506921874.689650:0:38877:0:(llite_lib.c:2525:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:11.0:1506921874.689704:0:38877:0:(file.c:3941:ll_inode_permission()) Process entered 00000080:00000001:11.0:1506921874.689706:0:38877:0:(file.c:3643:__ll_inode_revalidate()) Process entered 00000080:00200000:11.0:1506921874.689706:0:38877:0:(file.c:3648:__ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff881021f03690),name=/ 00000080:00000010:11.0:1506921874.689709:0:38877:0:(llite_lib.c:2434:ll_prep_md_op_data()) kmalloced 'op_data': 320 at ffff882011e16200. 00000080:00000001:11.0:1506921874.689711:0:38877:0:(obd_class.h:1379:md_intent_lock()) Process entered 00800000:00000001:11.0:1506921874.689712:0:38877:0:(lmv_intent.c:474:lmv_intent_lock()) Process entered 00800000:00000002:11.0:1506921874.689713:0:38877:0:(lmv_intent.c:482:lmv_intent_lock()) INTENT LOCK 'lookup' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:11.0:1506921874.689715:0:38877:0:(lmv_intent.c:361:lmv_intent_lookup()) Process entered 00800000:00000002:11.0:1506921874.689739:0:38877:0:(lmv_intent.c:390:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 lsm= (null) lsm_magic=ffffffff 00800000:00000001:11.0:1506921874.689744:0:38877:0:(obd_class.h:1379:md_intent_lock()) Process entered 00000002:00000001:11.0:1506921874.689745:0:38877:0:(mdc_locks.c:1118:mdc_intent_lock()) Process entered 00000002:00010000:11.0:1506921874.689746:0:38877:0:(mdc_locks.c:1125:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: lookup flags 0 00000002:00000001:11.0:1506921874.689749:0:38877:0:(mdc_locks.c:1024:mdc_revalidate_lock()) Process entered 00000002:00000001:11.0:1506921874.689750:0:38877:0:(mdc_locks.c:142:mdc_lock_match()) Process entered 00010000:00000001:11.0:1506921874.689751:0:38877:0:(ldlm_lock.c:1384:ldlm_lock_match()) Process entered 00010000:00000040:11.0:1506921874.689752:0:38877:0:(ldlm_resource.c:1480:ldlm_resource_getref()) getref res: ffff88203b7332c0 count: 2 00010000:00000001:11.0:1506921874.689754:0:38877:0:(ldlm_resource.c:1415:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:11.0:1506921874.689756:0:38877:0:(ldlm_lock.c:260:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:11.0:1506921874.689757:0:38877:0:(ldlm_lock.c:271:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000040:11.0:1506921874.689758:0:38877:0:(ldlm_resource.c:1480:ldlm_resource_getref()) getref res: ffff88203b7332c0 count: 3 00010000:00010000:11.0:1506921874.689759:0:38877:0:(ldlm_lock.c:742:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: oak-MDT0000-mdc-ffff88203bc47000 lock: ffff88101d368000/0x228de8b50142155 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13 rrc: 3 type: IBT flags: 0x0 nid: local remote: 0xc1d2b9af9ec6a943 expref: -99 pid: 4287 timeout: 0 lvb_type: 0 00010000:00000040:11.0:1506921874.689764:0:38877:0:(ldlm_resource.c:1519:ldlm_resource_putref()) putref res: ffff88203b7332c0 count: 2 00010000:00000001:11.0:1506921874.689765:0:38877:0:(ldlm_lock.c:1410:ldlm_lock_match()) Process leaving via out (rc=1 : 1 : 0x1) 00010000:00000040:11.0:1506921874.689767:0:38877:0:(ldlm_resource.c:1519:ldlm_resource_putref()) putref res: ffff88203b7332c0 count: 1 00010000:00000040:11.0:1506921874.689768:0:38877:0:(ldlm_resource.c:1480:ldlm_resource_getref()) getref res: ffff88203b7332c0 count: 2 00010000:00010000:11.0:1506921874.689769:0:38877:0:(ldlm_lock.c:1470:ldlm_lock_match()) ### matched (0 0) ns: oak-MDT0000-mdc-ffff88203bc47000 lock: ffff88101d368000/0x228de8b50142155 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13 rrc: 2 type: IBT flags: 0x0 nid: local remote: 0xc1d2b9af9ec6a943 expref: -99 pid: 4287 timeout: 0 lvb_type: 0 00010000:00000040:11.0:1506921874.689773:0:38877:0:(ldlm_resource.c:1519:ldlm_resource_putref()) putref res: ffff88203b7332c0 count: 1 02000000:00000001:11.0:1506921874.689774:0:38877:0:(sec.c:936:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:11.0:1506921874.689775:0:38877:0:(sec.c:950:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:11.0:1506921874.689776:0:38877:0:(mdc_locks.c:149:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:11.0:1506921874.689777:0:38877:0:(mdc_locks.c:1076:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:11.0:1506921874.689778:0:38877:0:(mdc_locks.c:1138:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:11.0:1506921874.689779:0:38877:0:(obd_class.h:1384:md_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:11.0:1506921874.689780:0:38877:0:(lmv_intent.c:409:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:11.0:1506921874.689781:0:38877:0:(lmv_intent.c:515:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:11.0:1506921874.689782:0:38877:0:(obd_class.h:1384:md_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00000010:11.0:1506921874.689784:0:38877:0:(llite_lib.c:2485:ll_finish_md_op_data()) kfreed 'op_data': 320 at ffff882011e16200. 00000080:00000001:11.0:1506921874.689785:0:38877:0:(dcache.c:267:ll_revalidate_it_finish()) Process entered 00000080:00000001:11.0:1506921874.689786:0:38877:0:(dcache.c:270:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:11.0:1506921874.689788:0:38877:0:(dcache.c:290:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff881021f03690) 00000080:00010000:11.0:1506921874.689789:0:38877:0:(llite_internal.h:1368:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff881021f03690) for lock 0x228de8b50142155 00000080:00000001:11.0:1506921874.689791:0:38877:0:(obd_class.h:1545:md_set_lock_data()) Process entered 00800000:00000001:11.0:1506921874.689792:0:38877:0:(lmv_obd.c:2845:lmv_set_lock_data()) Process entered 00800000:00000001:11.0:1506921874.689793:0:38877:0:(obd_class.h:1545:md_set_lock_data()) Process entered 00000002:00000001:11.0:1506921874.689794:0:38877:0:(mdc_locks.c:103:mdc_set_lock_data()) Process entered 00010000:00000001:11.0:1506921874.689794:0:38877:0:(ldlm_lock.c:584:__ldlm_handle2lock()) Process entered 00000020:00000001:11.0:1506921874.689795:0:38877:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000001:11.0:1506921874.689796:0:38877:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612201523806208 : -131872185745408 : ffff88101d368000) 00010000:00000001:11.0:1506921874.689798:0:38877:0:(ldlm_lock.c:603:__ldlm_handle2lock()) Process leaving (rc=18446612201523806208 : -131872185745408 : ffff88101d368000) 00010000:00000001:11.0:1506921874.689799:0:38877:0:(ldlm_lock.c:188:ldlm_lock_put()) Process entered 00010000:00000001:11.0:1506921874.689800:0:38877:0:(ldlm_lock.c:222:ldlm_lock_put()) Process leaving 00000002:00000001:11.0:1506921874.689801:0:38877:0:(mdc_locks.c:132:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:11.0:1506921874.689802:0:38877:0:(obd_class.h:1548:md_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:11.0:1506921874.689803:0:38877:0:(lmv_obd.c:2850:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:11.0:1506921874.689804:0:38877:0:(obd_class.h:1548:md_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:11.0:1506921874.689805:0:38877:0:(dcache.c:201:ll_intent_drop_lock()) releasing lock with cookie 0x228de8b50142155 from it ffff880168d1fbe8 00010000:00000001:11.0:1506921874.689806:0:38877:0:(ldlm_lock.c:584:__ldlm_handle2lock()) Process entered 00000020:00000001:11.0:1506921874.689807:0:38877:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000001:11.0:1506921874.689807:0:38877:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612201523806208 : -131872185745408 : ffff88101d368000) 00010000:00000001:11.0:1506921874.689808:0:38877:0:(ldlm_lock.c:603:__ldlm_handle2lock()) Process leaving (rc=18446612201523806208 : -131872185745408 : ffff88101d368000) 00010000:00000001:11.0:1506921874.689810:0:38877:0:(ldlm_lock.c:823:ldlm_lock_decref_internal()) Process entered 00010000:00010000:11.0:1506921874.689811:0:38877:0:(ldlm_lock.c:796:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: ?? lock: ffff88101d368000/0x228de8b50142155 lrc: 3/1,0 mode: PR/PR res: ?? rrc=?? type: ??? flags: 0x10000000000000 nid: local remote: 0xc1d2b9af9ec6a943 expref: -99 pid: 4287 timeout: 0 lvb_type: 0 00010000:00000001:11.0:1506921874.689814:0:38877:0:(ldlm_lock.c:188:ldlm_lock_put()) Process entered 00010000:00000001:11.0:1506921874.689815:0:38877:0:(ldlm_lock.c:222:ldlm_lock_put()) Process leaving 00010000:00010000:11.0:1506921874.689815:0:38877:0:(ldlm_lock.c:869:ldlm_lock_decref_internal()) ### add lock into lru list ns: ?? lock: ffff88101d368000/0x228de8b50142155 lrc: 2/0,0 mode: PR/PR res: ?? rrc=?? type: ??? flags: 0x10000000000000 nid: local remote: 0xc1d2b9af9ec6a943 expref: -99 pid: 4287 timeout: 0 lvb_type: 0 00010000:00000001:11.0:1506921874.689818:0:38877:0:(ldlm_lock.c:298:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:11.0:1506921874.689819:0:38877:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process leaving 00010000:00000001:11.0:1506921874.689820:0:38877:0:(ldlm_lock.c:890:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:11.0:1506921874.689821:0:38877:0:(ldlm_lock.c:188:ldlm_lock_put()) Process entered 00010000:00000001:11.0:1506921874.689821:0:38877:0:(ldlm_lock.c:222:ldlm_lock_put()) Process leaving 00000100:00000001:11.0:1506921874.689822:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000001:11.0:1506921874.689823:0:38877:0:(client.c:2514:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:11.0:1506921874.689825:0:38877:0:(file.c:3962:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff881021f03690), inode mode 41ed mask 44 00000080:00000001:11.0:1506921874.689827:0:38877:0:(file.c:4000:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:11.0:1506921874.689829:0:38877:0:(dir.c:1825:ll_dir_open()) Process entered 00000080:00000001:11.0:1506921874.689831:0:38877:0:(file.c:491:ll_file_open()) Process entered 00000080:00200000:11.0:1506921874.689832:0:38877:0:(file.c:494:ll_file_open()) VFS Op:inode=[0x200000007:0x1:0x0](ffff881021f03690), flags 300000 00000080:00000010:11.0:1506921874.689834:0:38877:0:(file.c:71:ll_file_data_get()) slab-alloced 'fd': 216 at ffff882035b5cb00. 00000080:00000001:11.0:1506921874.689836:0:38877:0:(file.c:509:ll_file_open()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:11.0:1506921874.689837:0:38877:0:(dir.c:1826:ll_dir_open()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:11.0:1506921874.689842:0:38877:0:(dir.c:1160:ll_dir_ioctl()) Process entered 00000080:00200000:11.0:1506921874.689843:0:38877:0:(dir.c:1163:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff881021f03690), cmd=0xc0b066a2 00000080:00000010:11.0:1506921874.689845:0:38877:0:(dir.c:1529:ll_dir_ioctl()) kmalloced 'qctl': 176 at ffff8820382d7080. 00000080:00000001:11.0:1506921874.689847:0:38877:0:(dir.c:1010:quotactl_ioctl()) Process entered 00000080:00000010:11.0:1506921874.689849:0:38877:0:(dir.c:1067:quotactl_ioctl()) kmalloced 'oqctl': 112 at ffff881032734280. 00000080:00000001:11.0:1506921874.689850:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00800000:00000001:11.0:1506921874.689852:0:38877:0:(lmv_obd.c:3038:lmv_quotactl()) Process entered 00800000:00000001:11.0:1506921874.689853:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000002:00000001:11.0:1506921874.689854:0:38877:0:(mdc_request.c:1808:mdc_quotactl()) Process entered 00000100:00000010:11.0:1506921874.689857:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881e1b238600. 00000020:00000040:11.0:1506921874.689858:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203bc57800 refcount=4 obd=oak-MDT0000-mdc-ffff88203bc47000 00000100:00000001:11.0:1506921874.689861:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:11.0:1506921874.689862:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:11.0:1506921874.689863:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:11.0:1506921874.689865:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff882011e16200. 00000100:00000001:11.0:1506921874.689868:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.689870:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:11.0:1506921874.689871:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:11.0:1506921874.689873:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.689874:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612270695673600 : -131803013878016 : ffff8820382d6f00) 00000100:00000001:11.0:1506921874.689876:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.689876:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000020:00000001:11.0:1506921874.689878:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:11.0:1506921874.689880:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:11.0:1506921874.689889:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:11.0:1506921874.689891:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:11.0:1506921874.689892:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:11.0:1506921874.689895:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:11.0:1506921874.689895:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:11.0:1506921874.689899:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881e1b238600 x1580016666301360/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.689905:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:11.0:1506921874.689906:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.689908:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:11.0:1506921874.689909:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:11.0:1506921874.689911:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301360:10.0.2.52@o2ib5:48 00000100:00000001:11.0:1506921874.689914:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:11.0:1506921874.689915:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:11.0:1506921874.689916:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.689917:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:11.0:1506921874.689919:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.689920:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:11.0:1506921874.689922:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203bf0a840. 00000400:00000010:11.0:1506921874.689925:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032734500. 00000100:00000200:11.0:1506921874.689927:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301360, portal 10 00000100:00000001:11.0:1506921874.689929:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.689929:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000100:00000040:11.0:1506921874.689932:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881e1b238600 x1580016666301360/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921881 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.689938:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:11.0:1506921874.689940:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.52@o2ib5 00000400:00000010:11.0:1506921874.689941:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032734800. 00000100:00000200:11.0:1506921874.689942:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 12, xid 1580016666301360, offset 0 00000400:00000010:11.0:1506921874.689944:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e15e00 (tot 59423868). 00000400:00000200:11.0:1506921874.689947:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.52@o2ib5 00000400:00000200:11.0:1506921874.689957:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.52@o2ib5(10.0.2.52@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:11.0:1506921874.689961:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:11.0:1506921874.689966:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:11.0:1506921874.689968:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:11.0:1506921874.689970:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:11.0:1506921874.689975:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:11.0:1506921874.689976:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.689978:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.689979:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.689981:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:11.0:1506921874.689982:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=6 : 6 : 6) 00000100:00100000:11.0:1506921874.689983:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8820382d6f00 going to sleep for 6 seconds 00000100:00000001:11.0:1506921874.689984:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.689986:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.689987:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.689988:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2F:1506921874.690017:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.690038:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921874.690043:0:3861:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921874.690047:0:3861:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921874.690051:0:3861:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881e1b238600 x1580016666301360/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921881 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.690059:0:3861:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921874.690061:0:3861:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881e1b238600 x1580016666301360/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921881 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.690066:0:3861:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.690068:0:3861:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921874.690069:0:3861:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032734800 00000400:00000010:16.0:1506921874.690071:0:3861:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032734800. 00000400:00000010:16.0:1506921874.690074:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e15e00 (tot 59423468). 00000800:00000200:0.0:1506921874.690077:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921874.690081:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:16.2:1506921874.690159:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921874.690165:0:3858:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.690169:0:3858:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921874.690174:0:3858:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.52@o2ib5 : PUT 00000400:00000010:16.0:1506921874.690176:0:3858:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.690181:0:3858:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.52@o2ib5 of length 304 into portal 10 MB=0x59d044c2253b0 00000400:00000200:16.0:1506921874.690186:0:3858:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index a from 12345-10.0.2.52@o2ib5 of length 304/304 into md 0x7a0d95 [1] + 192 00000100:00000001:16.0:1506921874.690190:0:3858:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.690192:0:3858:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881e1b238600 x1580016666301360/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921881 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:16.0:1506921874.690198:0:3858:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881e1b238600 x1580016666301360/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921881 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921874.690201:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:16.0:1506921874.690205:0:3858:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.690207:0:3858:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.690210:0:3858:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.690211:0:3858:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921874.690213:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:11.0:1506921874.690224:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:11.0:1506921874.690228:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:11.0:1506921874.690229:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:11.0:1506921874.690230:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921874.690233:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881e1b238600 x1580016666301360/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921881 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:11.0:1506921874.690239:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881e1b238600 x1580016666301360/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921881 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.690244:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921874.690247:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203bf0a840. 00000400:00000200:11.0:1506921874.690248:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032734500 00000400:00000010:11.0:1506921874.690250:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032734500. 00000100:00000001:11.0:1506921874.690251:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.690252:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:11.0:1506921874.690254:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:11.0:1506921874.690255:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:11.0:1506921874.690256:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.690258:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:11.0:1506921874.690261:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203bc57c00 time=69 v=1 (1 1 1 1) 00000100:00001000:11.0:1506921874.690263:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203bc57bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:11.0:1506921874.690265:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:11.0:1506921874.690266:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:11.0:1506921874.690267:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:11.0:1506921874.690269:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.690270:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:11.0:1506921874.690271:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-MDT0000-mdc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:11.0:1506921874.690273:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:11.0:1506921874.690274:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:11.0:1506921874.690276:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881e1b238600 x1580016666301360/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/304 e 0 to 0 dl 1506921881 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.690281:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.690282:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:11.0:1506921874.690283:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:11.0:1506921874.690285:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881e1b238600 x1580016666301360/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/304 e 0 to 0 dl 1506921881 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:11.0:1506921874.690291:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301360:10.0.2.52@o2ib5:48 00000100:00000001:11.0:1506921874.690293:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.690295:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.690296:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:11.0:1506921874.690298:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.690299:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881e1b238600 x1580016666301360/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/304 e 0 to 0 dl 1506921881 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.690304:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:11.0:1506921874.690305:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.690307:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:11.0:1506921874.690308:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.690310:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.690311:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881e1b238600 x1580016666301360/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/304 e 0 to 0 dl 1506921881 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.690315:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:11.0:1506921874.690316:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:11.0:1506921874.690317:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.690319:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:11.0:1506921874.690320:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:11.0:1506921874.690321:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203bc57800 refcount=3 obd=oak-MDT0000-mdc-ffff88203bc47000 00000020:00000001:11.0:1506921874.690322:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:11.0:1506921874.690323:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff882011e16200. 02000000:00000001:11.0:1506921874.690325:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:11.0:1506921874.690325:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:11.0:1506921874.690326:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881e1b238600. 00000100:00000001:11.0:1506921874.690328:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:11.0:1506921874.690329:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:11.0:1506921874.690331:0:38877:0:(mdc_request.c:1836:mdc_quotactl()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:11.0:1506921874.690332:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:11.0:1506921874.690334:0:38877:0:(lmv_obd.c:3050:lmv_quotactl()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:11.0:1506921874.690335:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:11.0:1506921874.690337:0:38877:0:(dir.c:1084:quotactl_ioctl()) kmalloced 'oqctl_tmp': 112 at ffff881032734500. 00000080:00000001:11.0:1506921874.690338:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00020000:00000001:11.0:1506921874.690340:0:38877:0:(lov_obd.c:1348:lov_quotactl()) Process entered 00020000:00000001:11.0:1506921874.690341:0:38877:0:(obd_class.h:976:obd_getref()) Process entered 00020000:00000001:11.0:1506921874.690342:0:38877:0:(obd_class.h:981:obd_getref()) Process leaving 00020000:00000001:11.0:1506921874.690343:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:11.0:1506921874.690345:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:11.0:1506921874.690346:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881e1b238600. 00000020:00000040:11.0:1506921874.690348:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203c6ad000 refcount=5 obd=oak-OST0000-osc-ffff88203bc47000 00000100:00000001:11.0:1506921874.690349:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:11.0:1506921874.690350:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:11.0:1506921874.690351:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:11.0:1506921874.690353:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff882011e16200. 00000100:00000001:11.0:1506921874.690355:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.690356:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:11.0:1506921874.690357:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:11.0:1506921874.690358:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.690359:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612270695673600 : -131803013878016 : ffff8820382d6f00) 00000100:00000001:11.0:1506921874.690360:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.690361:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000020:00000001:11.0:1506921874.690363:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:11.0:1506921874.690364:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:11.0:1506921874.690371:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:11.0:1506921874.690373:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:11.0:1506921874.690374:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:11.0:1506921874.690376:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:11.0:1506921874.690376:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:11.0:1506921874.690378:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881e1b238600 x1580016666301376/t0(0) o19->oak-OST0000-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.690383:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:11.0:1506921874.690385:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.690386:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:11.0:1506921874.690387:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:11.0:1506921874.690389:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301376:10.0.2.101@o2ib5:19 00000100:00000001:11.0:1506921874.690392:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:11.0:1506921874.690393:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:11.0:1506921874.690394:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.690395:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:11.0:1506921874.690396:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.690397:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:11.0:1506921874.690398:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203bf0a840. 00000400:00000010:11.0:1506921874.690400:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735f00. 00000100:00000200:11.0:1506921874.690402:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301376, portal 4 00000100:00000001:11.0:1506921874.690403:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.690404:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000100:00000040:11.0:1506921874.690406:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881e1b238600 x1580016666301376/t0(0) o19->oak-OST0000-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.690411:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:11.0:1506921874.690412:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:11.0:1506921874.690414:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735100. 00000100:00000200:11.0:1506921874.690415:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301376, offset 0 00000400:00000010:11.0:1506921874.690417:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e13400 (tot 59423868). 00000400:00000200:11.0:1506921874.690419:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:11.0:1506921874.690427:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:11.0:1506921874.690430:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:11.0:1506921874.690434:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:11.0:1506921874.690436:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:11.0:1506921874.690438:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921874.690441:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:11.0:1506921874.690442:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.690443:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.690444:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.690445:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:11.0:1506921874.690446:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:11.0:1506921874.690448:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8820382d6f00 going to sleep for 43 seconds 00000100:00000001:11.0:1506921874.690450:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.690450:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.690452:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.690452:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2F:1506921874.690478:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921874.690493:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:11.0:1506921874.690498:0:3862:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:11.0:1506921874.690500:0:3862:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:11.0:1506921874.690502:0:3862:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881e1b238600 x1580016666301376/t0(0) o19->oak-OST0000-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.690506:0:3862:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.690508:0:3862:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881e1b238600 x1580016666301376/t0(0) o19->oak-OST0000-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.690512:0:3862:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.690513:0:3862:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:11.0:1506921874.690514:0:3862:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735100 00000400:00000010:11.0:1506921874.690515:0:3862:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735100. 00000400:00000010:11.0:1506921874.690517:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e13400 (tot 59423468). 00000800:00000200:11.0:1506921874.690518:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:11.0:1506921874.690520:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.2:1506921874.690614:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921874.690631:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.690637:0:3859:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921874.690643:0:3859:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921874.690645:0:3859:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.690652:0:3859:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c2253c0 00000400:00000200:16.0:1506921874.690657:0:3859:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a0da5 [1] + 192 00000100:00000001:16.0:1506921874.690661:0:3859:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.690664:0:3859:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881e1b238600 x1580016666301376/t0(0) o19->oak-OST0000-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921874.690668:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921874.690672:0:3859:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881e1b238600 x1580016666301376/t0(0) o19->oak-OST0000-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.690681:0:3859:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.690683:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.690685:0:3859:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:11.0:1506921874.690685:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:11.0:1506921874.690686:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000800:00000200:16.0:1506921874.690687:0:3859:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000400:00000001:11.0:1506921874.690687:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:11.0:1506921874.690688:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000800:00000200:16.0:1506921874.690689:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000200:11.0:1506921874.690691:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881e1b238600 x1580016666301376/t0(0) o19->oak-OST0000-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:11.0:1506921874.690696:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881e1b238600 x1580016666301376/t0(0) o19->oak-OST0000-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.690700:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921874.690702:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203bf0a840. 00000400:00000200:11.0:1506921874.690703:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735f00 00000400:00000010:11.0:1506921874.690703:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735f00. 00000100:00000001:11.0:1506921874.690704:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.690705:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:11.0:1506921874.690706:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:11.0:1506921874.690707:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:11.0:1506921874.690709:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.690710:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:11.0:1506921874.690713:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88203c6ad400 time=69 v=31 (31 31 1 31) 00000100:00001000:11.0:1506921874.690715:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203c6ad3c8 time=69 v=1 (1 1 1 1) 00000100:00000001:11.0:1506921874.690723:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:11.0:1506921874.690724:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:11.0:1506921874.690725:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:11.0:1506921874.690726:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.690728:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:11.0:1506921874.690728:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0000-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:11.0:1506921874.690730:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:11.0:1506921874.690730:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:11.0:1506921874.690732:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881e1b238600 x1580016666301376/t0(0) o19->oak-OST0000-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.690736:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.690738:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:11.0:1506921874.690739:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:11.0:1506921874.690741:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881e1b238600 x1580016666301376/t0(0) o19->oak-OST0000-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:11.0:1506921874.690746:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301376:10.0.2.101@o2ib5:19 00000100:00000001:11.0:1506921874.690749:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.690751:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.690752:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:11.0:1506921874.690753:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.690754:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881e1b238600 x1580016666301376/t0(0) o19->oak-OST0000-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.690758:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:11.0:1506921874.690759:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.690761:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:11.0:1506921874.690761:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.690763:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.690764:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881e1b238600 x1580016666301376/t0(0) o19->oak-OST0000-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.690768:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:11.0:1506921874.690770:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:11.0:1506921874.690771:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.690772:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:11.0:1506921874.690773:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:11.0:1506921874.690774:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203c6ad000 refcount=4 obd=oak-OST0000-osc-ffff88203bc47000 00000020:00000001:11.0:1506921874.690775:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:11.0:1506921874.690776:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff882011e16200. 02000000:00000001:11.0:1506921874.690777:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:11.0:1506921874.690778:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:11.0:1506921874.690779:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881e1b238600. 00000100:00000001:11.0:1506921874.690780:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:11.0:1506921874.690781:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:11.0:1506921874.690782:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921874.690783:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921874.690785:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:11.0:1506921874.690785:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:11.0:1506921874.690786:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881e1b238600. 00000020:00000040:11.0:1506921874.690787:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203bc55000 refcount=5 obd=oak-OST0001-osc-ffff88203bc47000 00000100:00000001:11.0:1506921874.690789:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:11.0:1506921874.690790:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:11.0:1506921874.690790:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:11.0:1506921874.690792:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff882011e16200. 00000100:00000001:11.0:1506921874.690794:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.690795:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:11.0:1506921874.690796:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:11.0:1506921874.690797:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.690798:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612270695673600 : -131803013878016 : ffff8820382d6f00) 00000100:00000001:11.0:1506921874.690799:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.690800:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000020:00000001:11.0:1506921874.690801:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:11.0:1506921874.690802:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:11.0:1506921874.690809:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:11.0:1506921874.690810:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:11.0:1506921874.690811:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:11.0:1506921874.690813:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:11.0:1506921874.690813:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:11.0:1506921874.690816:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881e1b238600 x1580016666301392/t0(0) o19->oak-OST0001-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.690820:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:11.0:1506921874.690821:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.690822:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:11.0:1506921874.690823:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:11.0:1506921874.690825:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301392:10.0.2.102@o2ib5:19 00000100:00000001:11.0:1506921874.690826:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:11.0:1506921874.690827:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:11.0:1506921874.690828:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.690829:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:11.0:1506921874.690830:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.690831:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:11.0:1506921874.690833:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203bf0a840. 00000400:00000010:11.0:1506921874.690834:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735f00. 00000100:00000200:11.0:1506921874.690836:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301392, portal 4 00000100:00000001:11.0:1506921874.690837:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.690838:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000100:00000040:11.0:1506921874.690840:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881e1b238600 x1580016666301392/t0(0) o19->oak-OST0001-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.690844:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:11.0:1506921874.690846:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:11.0:1506921874.690847:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735100. 00000100:00000200:11.0:1506921874.690848:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301392, offset 0 00000400:00000010:11.0:1506921874.690850:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e13400 (tot 59423868). 00000400:00000200:11.0:1506921874.690852:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:11.0:1506921874.690859:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:11.0:1506921874.690862:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:11.0:1506921874.690866:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:11.0:1506921874.690868:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:11.0:1506921874.690870:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:11.0:1506921874.690873:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:11.0:1506921874.690874:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.690876:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.690877:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921874.690878:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:11.0:1506921874.690879:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:11.0:1506921874.690879:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:11.0:1506921874.690881:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8820382d6f00 going to sleep for 43 seconds 00000100:00000001:11.0:1506921874.690882:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.690883:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.690884:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.690884:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.0:1506921874.690885:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921874.690889:0:3860:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921874.690892:0:3860:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921874.690895:0:3860:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881e1b238600 x1580016666301392/t0(0) o19->oak-OST0001-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.690901:0:3860:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921874.690902:0:3860:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881e1b238600 x1580016666301392/t0(0) o19->oak-OST0001-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.690907:0:3860:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.690908:0:3860:0:(events.c:81:request_out_callback()) Process leaving 00000800:00000200:18.0:1506921874.690909:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000200:16.0:1506921874.690910:0:3860:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735100 00000400:00000010:16.0:1506921874.690911:0:3860:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735100. 00000400:00000010:16.0:1506921874.690914:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e13400 (tot 59423468). 00000800:00000200:16.0:1506921874.690917:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921874.691141:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:1.0:1506921874.691190:0:3864:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921874.691200:0:3864:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.42@o2ib4 00000400:00000200:1.0:1506921874.691207:0:3864:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:1.0:1506921874.691209:0:3864:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881c4b6cc800 (tot 59423868). 00000400:00000200:1.0:1506921874.691215:0:3864:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c2253d0 00000400:00000200:1.0:1506921874.691220:0:3864:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1c49 [1] + 192 00000100:00000001:1.0:1506921874.691225:0:3864:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:1.0:1506921874.691228:0:3864:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881e1b238600 x1580016666301392/t0(0) o19->oak-OST0001-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:13.0F:1506921874.691230:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:1.0:1506921874.691236:0:3864:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881e1b238600 x1580016666301392/t0(0) o19->oak-OST0001-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921874.691242:0:3864:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:1.0:1506921874.691244:0:3864:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881c4b6cc800 (tot 59423468). 00000800:00000200:1.0:1506921874.691249:0:3864:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921874.691251:0:3864:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:1.0:1506921874.691252:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:11.0:1506921874.691260:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:11.0:1506921874.691262:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:11.0:1506921874.691263:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:11.0:1506921874.691264:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921874.691266:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881e1b238600 x1580016666301392/t0(0) o19->oak-OST0001-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:11.0:1506921874.691271:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881e1b238600 x1580016666301392/t0(0) o19->oak-OST0001-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.691276:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921874.691277:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203bf0a840. 00000400:00000200:11.0:1506921874.691278:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735f00 00000400:00000010:11.0:1506921874.691279:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735f00. 00000100:00000001:11.0:1506921874.691280:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.691282:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:11.0:1506921874.691283:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:11.0:1506921874.691284:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:11.0:1506921874.691285:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.691286:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:11.0:1506921874.691288:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88203bc55400 time=69 v=31 (31 31 31 31) 00000100:00001000:11.0:1506921874.691291:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203bc553c8 time=69 v=1 (1 1 1 1) 00000100:00000001:11.0:1506921874.691292:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:11.0:1506921874.691293:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:11.0:1506921874.691294:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:11.0:1506921874.691295:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.691297:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:11.0:1506921874.691298:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0001-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:11.0:1506921874.691299:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:11.0:1506921874.691299:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:11.0:1506921874.691303:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881e1b238600 x1580016666301392/t0(0) o19->oak-OST0001-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.691307:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.691308:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:11.0:1506921874.691309:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:11.0:1506921874.691311:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881e1b238600 x1580016666301392/t0(0) o19->oak-OST0001-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:11.0:1506921874.691316:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301392:10.0.2.102@o2ib5:19 00000100:00000001:11.0:1506921874.691318:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.691320:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.691321:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:11.0:1506921874.691322:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.691324:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881e1b238600 x1580016666301392/t0(0) o19->oak-OST0001-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.691328:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:11.0:1506921874.691329:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.691331:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:11.0:1506921874.691331:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.691333:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.691334:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881e1b238600 x1580016666301392/t0(0) o19->oak-OST0001-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.691338:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:11.0:1506921874.691339:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:11.0:1506921874.691340:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.691341:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:11.0:1506921874.691342:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:11.0:1506921874.691343:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203bc55000 refcount=4 obd=oak-OST0001-osc-ffff88203bc47000 00000020:00000001:11.0:1506921874.691344:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:11.0:1506921874.691345:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff882011e16200. 02000000:00000001:11.0:1506921874.691347:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:11.0:1506921874.691348:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:11.0:1506921874.691348:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881e1b238600. 00000100:00000001:11.0:1506921874.691350:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:11.0:1506921874.691351:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:11.0:1506921874.691352:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921874.691354:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921874.691355:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:11.0:1506921874.691356:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:11.0:1506921874.691357:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881e1b238600. 00000020:00000040:11.0:1506921874.691358:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203c9bf000 refcount=5 obd=oak-OST0002-osc-ffff88203bc47000 00000100:00000001:11.0:1506921874.691359:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:11.0:1506921874.691360:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:11.0:1506921874.691361:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:11.0:1506921874.691362:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff882011e16200. 00000100:00000001:11.0:1506921874.691364:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.691365:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:11.0:1506921874.691366:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:11.0:1506921874.691367:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.691368:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612270695673600 : -131803013878016 : ffff8820382d6f00) 00000100:00000001:11.0:1506921874.691370:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.691370:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000020:00000001:11.0:1506921874.691372:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:11.0:1506921874.691373:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:11.0:1506921874.691379:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:11.0:1506921874.691381:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:11.0:1506921874.691382:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:11.0:1506921874.691384:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:11.0:1506921874.691384:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:11.0:1506921874.691386:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881e1b238600 x1580016666301408/t0(0) o19->oak-OST0002-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.691390:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:11.0:1506921874.691391:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.691392:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:11.0:1506921874.691393:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:11.0:1506921874.691395:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301408:10.0.2.101@o2ib5:19 00000100:00000001:11.0:1506921874.691396:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:11.0:1506921874.691397:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:11.0:1506921874.691398:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.691399:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:11.0:1506921874.691400:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.691402:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:11.0:1506921874.691404:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203bf0a840. 00000400:00000010:11.0:1506921874.691405:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735f00. 00000100:00000200:11.0:1506921874.691407:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301408, portal 4 00000100:00000001:11.0:1506921874.691409:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.691409:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000100:00000040:11.0:1506921874.691412:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881e1b238600 x1580016666301408/t0(0) o19->oak-OST0002-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.691416:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:11.0:1506921874.691417:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:11.0:1506921874.691419:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032734200. 00000100:00000200:11.0:1506921874.691420:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301408, offset 0 00000400:00000010:11.0:1506921874.691422:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e11600 (tot 59423868). 00000400:00000200:11.0:1506921874.691424:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:11.0:1506921874.691431:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:11.0:1506921874.691435:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:11.0:1506921874.691438:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:11.0:1506921874.691440:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:11.0:1506921874.691442:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921874.691444:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:11.0:1506921874.691446:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.691447:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.691448:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.691449:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:11.0:1506921874.691450:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:11.0:1506921874.691452:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8820382d6f00 going to sleep for 43 seconds 00000100:00000001:11.0:1506921874.691453:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.691454:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.691455:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.691455:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921874.691487:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921874.691499:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:11.0:1506921874.691502:0:3862:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:11.0:1506921874.691503:0:3862:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:11.0:1506921874.691505:0:3862:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881e1b238600 x1580016666301408/t0(0) o19->oak-OST0002-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.691510:0:3862:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.691512:0:3862:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881e1b238600 x1580016666301408/t0(0) o19->oak-OST0002-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.691516:0:3862:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.691517:0:3862:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:11.0:1506921874.691517:0:3862:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032734200 00000400:00000010:11.0:1506921874.691518:0:3862:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032734200. 00000400:00000010:11.0:1506921874.691520:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e11600 (tot 59423468). 00000800:00000200:11.0:1506921874.691522:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:11.0:1506921874.691524:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:2.2:1506921874.691689:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:13.0:1506921874.691731:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921874.691739:0:3863:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.42@o2ib4 00000400:00000200:13.0:1506921874.691746:0:3863:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:13.0:1506921874.691748:0:3863:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88203bd1e000 (tot 59423868). 00000400:00000200:13.0:1506921874.691753:0:3863:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c2253e0 00000400:00000200:13.0:1506921874.691758:0:3863:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a0dbd [1] + 192 00000100:00000001:13.0:1506921874.691763:0:3863:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:13.0:1506921874.691766:0:3863:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881e1b238600 x1580016666301408/t0(0) o19->oak-OST0002-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:1.0:1506921874.691770:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:13.0:1506921874.691774:0:3863:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881e1b238600 x1580016666301408/t0(0) o19->oak-OST0002-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.691782:0:3863:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:13.0:1506921874.691783:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88203bd1e000 (tot 59423468). 00000100:00000001:11.0:1506921874.691785:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:13.0:1506921874.691786:0:3863:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000400:00000001:11.0:1506921874.691786:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:11.0:1506921874.691787:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000800:00000200:13.0:1506921874.691788:0:3863:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:11.0:1506921874.691788:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921874.691789:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881e1b238600 x1580016666301408/t0(0) o19->oak-OST0002-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000800:00000200:13.0:1506921874.691790:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000200:11.0:1506921874.691794:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881e1b238600 x1580016666301408/t0(0) o19->oak-OST0002-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.691798:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921874.691799:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203bf0a840. 00000400:00000200:11.0:1506921874.691800:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735f00 00000400:00000010:11.0:1506921874.691801:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735f00. 00000100:00000001:11.0:1506921874.691802:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.691803:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:11.0:1506921874.691804:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:11.0:1506921874.691805:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:11.0:1506921874.691805:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.691807:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:11.0:1506921874.691808:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88203c9bf400 time=69 v=31 (31 1 1 31) 00000100:00001000:11.0:1506921874.691810:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203c9bf3c8 time=69 v=1 (1 1 1 1) 00000100:00000001:11.0:1506921874.691812:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:11.0:1506921874.691813:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:11.0:1506921874.691814:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:11.0:1506921874.691815:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.691816:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:11.0:1506921874.691817:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0002-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:11.0:1506921874.691818:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:11.0:1506921874.691819:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:11.0:1506921874.691821:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881e1b238600 x1580016666301408/t0(0) o19->oak-OST0002-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.691826:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.691827:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:11.0:1506921874.691828:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:11.0:1506921874.691830:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881e1b238600 x1580016666301408/t0(0) o19->oak-OST0002-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:11.0:1506921874.691835:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301408:10.0.2.101@o2ib5:19 00000100:00000001:11.0:1506921874.691837:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.691838:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.691839:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:11.0:1506921874.691840:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.691841:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881e1b238600 x1580016666301408/t0(0) o19->oak-OST0002-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.691845:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:11.0:1506921874.691847:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.691848:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:11.0:1506921874.691848:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.691850:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.691851:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881e1b238600 x1580016666301408/t0(0) o19->oak-OST0002-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.691855:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:11.0:1506921874.691856:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:11.0:1506921874.691857:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.691858:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:11.0:1506921874.691859:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:11.0:1506921874.691859:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203c9bf000 refcount=4 obd=oak-OST0002-osc-ffff88203bc47000 00000020:00000001:11.0:1506921874.691861:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:11.0:1506921874.691861:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff882011e16200. 02000000:00000001:11.0:1506921874.691863:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:11.0:1506921874.691863:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:11.0:1506921874.691868:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881e1b238600. 00000100:00000001:11.0:1506921874.691869:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:11.0:1506921874.691870:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:11.0:1506921874.691871:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921874.691872:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921874.691873:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:11.0:1506921874.691874:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:11.0:1506921874.691875:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881e1b238600. 00000020:00000040:11.0:1506921874.691876:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203bc53800 refcount=5 obd=oak-OST0003-osc-ffff88203bc47000 00000100:00000001:11.0:1506921874.691877:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:11.0:1506921874.691878:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:11.0:1506921874.691879:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:11.0:1506921874.691880:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff882011e16200. 00000100:00000001:11.0:1506921874.691882:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.691883:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:11.0:1506921874.691884:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:11.0:1506921874.691885:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.691886:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612270695673600 : -131803013878016 : ffff8820382d6f00) 00000100:00000001:11.0:1506921874.691887:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.691888:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000020:00000001:11.0:1506921874.691889:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:11.0:1506921874.691890:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:11.0:1506921874.691896:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:11.0:1506921874.691898:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:11.0:1506921874.691899:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:11.0:1506921874.691900:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:11.0:1506921874.691901:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:11.0:1506921874.691903:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881e1b238600 x1580016666301424/t0(0) o19->oak-OST0003-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.691907:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:11.0:1506921874.691908:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.691909:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:11.0:1506921874.691909:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:11.0:1506921874.691911:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301424:10.0.2.102@o2ib5:19 00000100:00000001:11.0:1506921874.691913:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:11.0:1506921874.691914:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:11.0:1506921874.691915:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.691916:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:11.0:1506921874.691917:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.691918:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:11.0:1506921874.691920:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203bf0a840. 00000400:00000010:11.0:1506921874.691921:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735f00. 00000100:00000200:11.0:1506921874.691923:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301424, portal 4 00000100:00000001:11.0:1506921874.691924:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.691924:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000100:00000040:11.0:1506921874.691927:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881e1b238600 x1580016666301424/t0(0) o19->oak-OST0003-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.691931:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:11.0:1506921874.691932:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:11.0:1506921874.691934:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032734200. 00000100:00000200:11.0:1506921874.691935:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301424, offset 0 00000400:00000010:11.0:1506921874.691936:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e11600 (tot 59423868). 00000400:00000200:11.0:1506921874.691939:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:11.0:1506921874.691945:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:11.0:1506921874.691948:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:11.0:1506921874.691952:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:11.0:1506921874.691954:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:11.0:1506921874.691955:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:11.0:1506921874.691958:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:11.0:1506921874.691959:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.691960:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.691961:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.691962:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:11.0:1506921874.691963:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:11.0:1506921874.691964:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8820382d6f00 going to sleep for 43 seconds 00000100:00000001:11.0:1506921874.691966:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.691966:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.691968:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.691968:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921874.691990:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.692008:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921874.692012:0:3861:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921874.692016:0:3861:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921874.692020:0:3861:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881e1b238600 x1580016666301424/t0(0) o19->oak-OST0003-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.692028:0:3861:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921874.692030:0:3861:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881e1b238600 x1580016666301424/t0(0) o19->oak-OST0003-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.692035:0:3861:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.692037:0:3861:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921874.692038:0:3861:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032734200 00000400:00000010:16.0:1506921874.692040:0:3861:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032734200. 00000400:00000010:16.0:1506921874.692044:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e11600 (tot 59423468). 00000800:00000200:18.0:1506921874.692045:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921874.692049:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:16.2:1506921874.692134:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921874.692142:0:3858:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.692146:0:3858:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.41@o2ib4 00000800:00000200:18.0:1506921874.692149:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000200:16.0:1506921874.692151:0:3858:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:16.0:1506921874.692154:0:3858:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.692158:0:3858:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c2253f0 00000400:00000200:16.0:1506921874.692164:0:3858:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1c51 [1] + 192 00000100:00000001:16.0:1506921874.692168:0:3858:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.692170:0:3858:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881e1b238600 x1580016666301424/t0(0) o19->oak-OST0003-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:16.0:1506921874.692176:0:3858:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881e1b238600 x1580016666301424/t0(0) o19->oak-OST0003-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.692183:0:3858:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.692184:0:3858:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.692186:0:3858:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.692188:0:3858:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921874.692190:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:11.0:1506921874.692200:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:11.0:1506921874.692204:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:11.0:1506921874.692205:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:11.0:1506921874.692206:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921874.692209:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881e1b238600 x1580016666301424/t0(0) o19->oak-OST0003-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:11.0:1506921874.692214:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881e1b238600 x1580016666301424/t0(0) o19->oak-OST0003-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.692219:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921874.692220:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203bf0a840. 00000400:00000200:11.0:1506921874.692221:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735f00 00000400:00000010:11.0:1506921874.692223:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735f00. 00000100:00000001:11.0:1506921874.692224:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.692225:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:11.0:1506921874.692226:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:11.0:1506921874.692227:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:11.0:1506921874.692228:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.692230:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:11.0:1506921874.692231:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88203bc53c00 time=69 v=31 (31 31 31 31) 00000100:00001000:11.0:1506921874.692233:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203bc53bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:11.0:1506921874.692235:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:11.0:1506921874.692236:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:11.0:1506921874.692237:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:11.0:1506921874.692238:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.692239:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:11.0:1506921874.692240:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0003-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:11.0:1506921874.692241:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:11.0:1506921874.692242:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:11.0:1506921874.692244:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881e1b238600 x1580016666301424/t0(0) o19->oak-OST0003-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.692248:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.692249:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:11.0:1506921874.692250:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:11.0:1506921874.692252:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881e1b238600 x1580016666301424/t0(0) o19->oak-OST0003-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:11.0:1506921874.692257:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301424:10.0.2.102@o2ib5:19 00000100:00000001:11.0:1506921874.692261:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.692263:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.692264:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:11.0:1506921874.692265:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.692267:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881e1b238600 x1580016666301424/t0(0) o19->oak-OST0003-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.692271:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:11.0:1506921874.692272:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.692274:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:11.0:1506921874.692274:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.692276:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.692277:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881e1b238600 x1580016666301424/t0(0) o19->oak-OST0003-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.692281:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:11.0:1506921874.692282:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:11.0:1506921874.692283:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.692284:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:11.0:1506921874.692285:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:11.0:1506921874.692286:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203bc53800 refcount=4 obd=oak-OST0003-osc-ffff88203bc47000 00000020:00000001:11.0:1506921874.692287:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:11.0:1506921874.692288:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff882011e16200. 02000000:00000001:11.0:1506921874.692289:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:11.0:1506921874.692290:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:11.0:1506921874.692291:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881e1b238600. 00000100:00000001:11.0:1506921874.692292:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:11.0:1506921874.692293:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:11.0:1506921874.692294:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921874.692296:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921874.692297:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:11.0:1506921874.692298:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:11.0:1506921874.692298:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881e1b238600. 00000020:00000040:11.0:1506921874.692300:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203c9bf800 refcount=5 obd=oak-OST0004-osc-ffff88203bc47000 00000100:00000001:11.0:1506921874.692301:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:11.0:1506921874.692302:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:11.0:1506921874.692303:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:11.0:1506921874.692304:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff882011e16200. 00000100:00000001:11.0:1506921874.692306:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.692307:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:11.0:1506921874.692308:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:11.0:1506921874.692309:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.692310:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612270695673600 : -131803013878016 : ffff8820382d6f00) 00000100:00000001:11.0:1506921874.692311:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.692312:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000020:00000001:11.0:1506921874.692314:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:11.0:1506921874.692315:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:11.0:1506921874.692321:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:11.0:1506921874.692323:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:11.0:1506921874.692324:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:11.0:1506921874.692325:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:11.0:1506921874.692326:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:11.0:1506921874.692327:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881e1b238600 x1580016666301440/t0(0) o19->oak-OST0004-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.692332:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:11.0:1506921874.692332:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.692334:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:11.0:1506921874.692335:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:11.0:1506921874.692336:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301440:10.0.2.101@o2ib5:19 00000100:00000001:11.0:1506921874.692338:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:11.0:1506921874.692339:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:11.0:1506921874.692340:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.692341:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:11.0:1506921874.692342:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.692343:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:11.0:1506921874.692344:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203bf0a840. 00000400:00000010:11.0:1506921874.692346:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735f00. 00000100:00000200:11.0:1506921874.692348:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301440, portal 4 00000100:00000001:11.0:1506921874.692349:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.692350:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000100:00000040:11.0:1506921874.692352:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881e1b238600 x1580016666301440/t0(0) o19->oak-OST0004-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.692361:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:11.0:1506921874.692363:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:11.0:1506921874.692364:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735f80. 00000100:00000200:11.0:1506921874.692365:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301440, offset 0 00000400:00000010:11.0:1506921874.692367:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e12400 (tot 59423868). 00000400:00000200:11.0:1506921874.692369:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:11.0:1506921874.692376:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:11.0:1506921874.692379:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:11.0:1506921874.692382:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:11.0:1506921874.692384:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:11.0:1506921874.692385:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921874.692387:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:11.0:1506921874.692388:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.692389:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.692391:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.692392:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:11.0:1506921874.692392:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:11.0:1506921874.692394:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8820382d6f00 going to sleep for 43 seconds 00000100:00000001:11.0:1506921874.692395:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.692396:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.692397:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.692397:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921874.692443:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921874.692455:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:11.0:1506921874.692458:0:3862:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:11.0:1506921874.692459:0:3862:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:11.0:1506921874.692461:0:3862:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881e1b238600 x1580016666301440/t0(0) o19->oak-OST0004-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.692465:0:3862:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.692467:0:3862:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881e1b238600 x1580016666301440/t0(0) o19->oak-OST0004-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.692471:0:3862:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.692471:0:3862:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:11.0:1506921874.692472:0:3862:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735f80 00000400:00000010:11.0:1506921874.692473:0:3862:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735f80. 00000400:00000010:11.0:1506921874.692476:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e12400 (tot 59423468). 00000800:00000200:11.0:1506921874.692477:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:11.0:1506921874.692479:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.2:1506921874.692705:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921874.692734:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.692741:0:3859:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921874.692746:0:3859:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921874.692751:0:3859:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.692757:0:3859:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225400 00000400:00000200:16.0:1506921874.692762:0:3859:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a0dd5 [1] + 192 00000100:00000001:16.0:1506921874.692767:0:3859:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.692770:0:3859:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881e1b238600 x1580016666301440/t0(0) o19->oak-OST0004-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921874.692772:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921874.692777:0:3859:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881e1b238600 x1580016666301440/t0(0) o19->oak-OST0004-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.692786:0:3859:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.692788:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.692790:0:3859:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:11.0:1506921874.692791:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:16.0:1506921874.692793:0:3859:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000400:00000001:11.0:1506921874.692793:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000800:00000200:16.0:1506921874.692794:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000400:00000001:11.0:1506921874.692794:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:11.0:1506921874.692795:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921874.692797:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881e1b238600 x1580016666301440/t0(0) o19->oak-OST0004-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:11.0:1506921874.692802:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881e1b238600 x1580016666301440/t0(0) o19->oak-OST0004-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.692806:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921874.692808:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203bf0a840. 00000400:00000200:11.0:1506921874.692809:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735f00 00000400:00000010:11.0:1506921874.692810:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735f00. 00000100:00000001:11.0:1506921874.692811:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.692812:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:11.0:1506921874.692813:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:11.0:1506921874.692813:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:11.0:1506921874.692814:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.692816:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:11.0:1506921874.692818:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88203c9bfc00 time=69 v=31 (31 31 1 31) 00000100:00001000:11.0:1506921874.692820:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203c9bfbc8 time=69 v=1 (1 1 1 1) 00000100:00000001:11.0:1506921874.692821:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:11.0:1506921874.692822:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:11.0:1506921874.692823:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:11.0:1506921874.692824:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.692826:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:11.0:1506921874.692826:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0004-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:11.0:1506921874.692827:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:11.0:1506921874.692828:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:11.0:1506921874.692830:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881e1b238600 x1580016666301440/t0(0) o19->oak-OST0004-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.692834:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.692835:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:11.0:1506921874.692836:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:11.0:1506921874.692839:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881e1b238600 x1580016666301440/t0(0) o19->oak-OST0004-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:11.0:1506921874.692844:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301440:10.0.2.101@o2ib5:19 00000100:00000001:11.0:1506921874.692846:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.692848:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.692849:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:11.0:1506921874.692850:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.692851:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881e1b238600 x1580016666301440/t0(0) o19->oak-OST0004-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.692855:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:11.0:1506921874.692856:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.692857:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:11.0:1506921874.692858:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.692860:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.692861:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881e1b238600 x1580016666301440/t0(0) o19->oak-OST0004-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.692865:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:11.0:1506921874.692866:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:11.0:1506921874.692866:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.692868:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:11.0:1506921874.692869:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:11.0:1506921874.692869:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203c9bf800 refcount=4 obd=oak-OST0004-osc-ffff88203bc47000 00000020:00000001:11.0:1506921874.692871:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:11.0:1506921874.692872:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff882011e16200. 02000000:00000001:11.0:1506921874.692873:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:11.0:1506921874.692874:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:11.0:1506921874.692874:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881e1b238600. 00000100:00000001:11.0:1506921874.692876:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:11.0:1506921874.692876:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:11.0:1506921874.692877:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921874.692878:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921874.692880:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:11.0:1506921874.692880:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:11.0:1506921874.692881:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881e1b238600. 00000020:00000040:11.0:1506921874.692883:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203c9bd800 refcount=5 obd=oak-OST0005-osc-ffff88203bc47000 00000100:00000001:11.0:1506921874.692884:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:11.0:1506921874.692885:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:11.0:1506921874.692886:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:11.0:1506921874.692887:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff882011e16200. 00000100:00000001:11.0:1506921874.692889:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.692890:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:11.0:1506921874.692891:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:11.0:1506921874.692892:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.692893:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612270695673600 : -131803013878016 : ffff8820382d6f00) 00000100:00000001:11.0:1506921874.692894:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.692895:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000020:00000001:11.0:1506921874.692896:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:11.0:1506921874.692897:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:11.0:1506921874.692903:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:11.0:1506921874.692905:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:11.0:1506921874.692906:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:11.0:1506921874.692907:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:11.0:1506921874.692908:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:11.0:1506921874.692910:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881e1b238600 x1580016666301456/t0(0) o19->oak-OST0005-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.692914:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:11.0:1506921874.692914:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.692916:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:11.0:1506921874.692916:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:11.0:1506921874.692918:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301456:10.0.2.102@o2ib5:19 00000100:00000001:11.0:1506921874.692920:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:11.0:1506921874.692921:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:11.0:1506921874.692921:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.692923:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:11.0:1506921874.692923:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.692925:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:11.0:1506921874.692926:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203bf0a840. 00000400:00000010:11.0:1506921874.692927:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735f00. 00000100:00000200:11.0:1506921874.692929:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301456, portal 4 00000100:00000001:11.0:1506921874.692930:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.692931:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000100:00000040:11.0:1506921874.692934:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881e1b238600 x1580016666301456/t0(0) o19->oak-OST0005-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.692938:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:11.0:1506921874.692939:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:11.0:1506921874.692941:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735f80. 00000100:00000200:11.0:1506921874.692942:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301456, offset 0 00000400:00000010:11.0:1506921874.692944:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e12400 (tot 59423868). 00000400:00000200:11.0:1506921874.692946:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:11.0:1506921874.692953:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:11.0:1506921874.692956:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:11.0:1506921874.692960:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:11.0:1506921874.692962:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:11.0:1506921874.692963:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:11.0:1506921874.692966:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:11.0:1506921874.692967:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.692968:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.692969:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.692970:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:11.0:1506921874.692971:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000800:00000200:16.2:1506921874.692972:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000100:00100000:11.0:1506921874.692972:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8820382d6f00 going to sleep for 43 seconds 00000100:00000001:11.0:1506921874.692973:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.692974:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.692975:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.692976:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.0:1506921874.692979:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921874.692983:0:3860:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000800:00000200:18.0:1506921874.692986:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:16.0:1506921874.692986:0:3860:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921874.692989:0:3860:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881e1b238600 x1580016666301456/t0(0) o19->oak-OST0005-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.692994:0:3860:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921874.692996:0:3860:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881e1b238600 x1580016666301456/t0(0) o19->oak-OST0005-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.693000:0:3860:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.693002:0:3860:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921874.693003:0:3860:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735f80 00000400:00000010:16.0:1506921874.693005:0:3860:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735f80. 00000400:00000010:16.0:1506921874.693007:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e12400 (tot 59423468). 00000800:00000200:16.0:1506921874.693010:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921874.693243:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:1.0:1506921874.693289:0:3864:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921874.693299:0:3864:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.42@o2ib4 00000400:00000200:1.0:1506921874.693306:0:3864:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:1.0:1506921874.693308:0:3864:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881c4b6cc800 (tot 59423868). 00000400:00000200:1.0:1506921874.693314:0:3864:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225410 00000400:00000200:1.0:1506921874.693319:0:3864:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1c59 [1] + 192 00000100:00000001:1.0:1506921874.693324:0:3864:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:1.0:1506921874.693327:0:3864:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881e1b238600 x1580016666301456/t0(0) o19->oak-OST0005-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:13.0:1506921874.693328:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:1.0:1506921874.693335:0:3864:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881e1b238600 x1580016666301456/t0(0) o19->oak-OST0005-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921874.693341:0:3864:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:1.0:1506921874.693343:0:3864:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881c4b6cc800 (tot 59423468). 00000800:00000200:1.0:1506921874.693346:0:3864:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921874.693348:0:3864:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:1.0:1506921874.693349:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:11.0:1506921874.693359:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:11.0:1506921874.693361:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:11.0:1506921874.693362:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:11.0:1506921874.693363:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921874.693365:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881e1b238600 x1580016666301456/t0(0) o19->oak-OST0005-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:11.0:1506921874.693370:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881e1b238600 x1580016666301456/t0(0) o19->oak-OST0005-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.693374:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921874.693377:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203bf0a840. 00000400:00000200:11.0:1506921874.693378:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735f00 00000400:00000010:11.0:1506921874.693379:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735f00. 00000100:00000001:11.0:1506921874.693380:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.693381:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:11.0:1506921874.693382:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:11.0:1506921874.693383:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:11.0:1506921874.693384:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.693386:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:11.0:1506921874.693387:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88203c9bdc00 time=69 v=31 (31 31 31 31) 00000100:00001000:11.0:1506921874.693390:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203c9bdbc8 time=69 v=1 (1 1 1 1) 00000100:00000001:11.0:1506921874.693391:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:11.0:1506921874.693392:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:11.0:1506921874.693393:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:11.0:1506921874.693394:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.693396:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:11.0:1506921874.693396:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0005-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:11.0:1506921874.693398:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:11.0:1506921874.693398:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:11.0:1506921874.693400:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881e1b238600 x1580016666301456/t0(0) o19->oak-OST0005-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.693404:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.693406:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:11.0:1506921874.693406:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:11.0:1506921874.693408:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881e1b238600 x1580016666301456/t0(0) o19->oak-OST0005-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:11.0:1506921874.693413:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301456:10.0.2.102@o2ib5:19 00000100:00000001:11.0:1506921874.693415:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.693417:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.693418:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:11.0:1506921874.693419:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.693420:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881e1b238600 x1580016666301456/t0(0) o19->oak-OST0005-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.693424:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:11.0:1506921874.693426:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.693428:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:11.0:1506921874.693428:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.693430:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.693431:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881e1b238600 x1580016666301456/t0(0) o19->oak-OST0005-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.693435:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:11.0:1506921874.693436:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:11.0:1506921874.693436:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.693438:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:11.0:1506921874.693439:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:11.0:1506921874.693440:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203c9bd800 refcount=4 obd=oak-OST0005-osc-ffff88203bc47000 00000020:00000001:11.0:1506921874.693441:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:11.0:1506921874.693442:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff882011e16200. 02000000:00000001:11.0:1506921874.693443:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:11.0:1506921874.693444:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:11.0:1506921874.693445:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881e1b238600. 00000100:00000001:11.0:1506921874.693446:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:11.0:1506921874.693447:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:11.0:1506921874.693448:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921874.693449:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921874.693450:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:11.0:1506921874.693452:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:11.0:1506921874.693452:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881e1b238600. 00000020:00000040:11.0:1506921874.693454:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203c6af800 refcount=5 obd=oak-OST0006-osc-ffff88203bc47000 00000100:00000001:11.0:1506921874.693455:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:11.0:1506921874.693456:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:11.0:1506921874.693457:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:11.0:1506921874.693458:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff882011e16200. 00000100:00000001:11.0:1506921874.693460:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.693461:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:11.0:1506921874.693462:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:11.0:1506921874.693463:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.693464:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612270695673600 : -131803013878016 : ffff8820382d6f00) 00000100:00000001:11.0:1506921874.693465:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.693466:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000020:00000001:11.0:1506921874.693467:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:11.0:1506921874.693469:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:11.0:1506921874.693475:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:11.0:1506921874.693477:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:11.0:1506921874.693478:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:11.0:1506921874.693479:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:11.0:1506921874.693480:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:11.0:1506921874.693481:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881e1b238600 x1580016666301472/t0(0) o19->oak-OST0006-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.693485:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:11.0:1506921874.693486:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.693488:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:11.0:1506921874.693488:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:11.0:1506921874.693490:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301472:10.0.2.101@o2ib5:19 00000100:00000001:11.0:1506921874.693492:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:11.0:1506921874.693493:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:11.0:1506921874.693494:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.693495:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:11.0:1506921874.693496:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.693497:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:11.0:1506921874.693498:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203bf0a840. 00000400:00000010:11.0:1506921874.693500:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735f00. 00000100:00000200:11.0:1506921874.693502:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301472, portal 4 00000100:00000001:11.0:1506921874.693503:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.693503:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000100:00000040:11.0:1506921874.693506:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881e1b238600 x1580016666301472/t0(0) o19->oak-OST0006-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.693510:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:11.0:1506921874.693511:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:11.0:1506921874.693513:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735e80. 00000100:00000200:11.0:1506921874.693514:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301472, offset 0 00000400:00000010:11.0:1506921874.693515:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e14a00 (tot 59423868). 00000400:00000200:11.0:1506921874.693518:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:11.0:1506921874.693524:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:11.0:1506921874.693528:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:11.0:1506921874.693532:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:11.0:1506921874.693533:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:11.0:1506921874.693535:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921874.693537:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:11.0:1506921874.693538:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.693539:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.693540:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.693541:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:11.0:1506921874.693542:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:11.0:1506921874.693543:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8820382d6f00 going to sleep for 43 seconds 00000100:00000001:11.0:1506921874.693544:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.693545:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.693546:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.693547:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921874.693574:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921874.693586:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:11.0:1506921874.693588:0:3862:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:11.0:1506921874.693590:0:3862:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:11.0:1506921874.693592:0:3862:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881e1b238600 x1580016666301472/t0(0) o19->oak-OST0006-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.693596:0:3862:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.693597:0:3862:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881e1b238600 x1580016666301472/t0(0) o19->oak-OST0006-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.693601:0:3862:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.693602:0:3862:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:11.0:1506921874.693603:0:3862:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735e80 00000400:00000010:11.0:1506921874.693604:0:3862:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735e80. 00000400:00000010:11.0:1506921874.693605:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e14a00 (tot 59423468). 00000800:00000200:11.0:1506921874.693607:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:11.0:1506921874.693608:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:2.2:1506921874.693759:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:13.0:1506921874.693797:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921874.693806:0:3863:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.42@o2ib4 00000400:00000200:13.0:1506921874.693812:0:3863:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:13.0:1506921874.693814:0:3863:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88203bd1e000 (tot 59423868). 00000400:00000200:13.0:1506921874.693820:0:3863:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225420 00000400:00000200:13.0:1506921874.693825:0:3863:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a0ded [1] + 192 00000100:00000001:13.0:1506921874.693829:0:3863:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:13.0:1506921874.693832:0:3863:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881e1b238600 x1580016666301472/t0(0) o19->oak-OST0006-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:1.0:1506921874.693836:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:13.0:1506921874.693840:0:3863:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881e1b238600 x1580016666301472/t0(0) o19->oak-OST0006-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.693848:0:3863:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:13.0:1506921874.693849:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88203bd1e000 (tot 59423468). 00000100:00000001:11.0:1506921874.693850:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:13.0:1506921874.693852:0:3863:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000400:00000001:11.0:1506921874.693852:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:11.0:1506921874.693853:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:11.0:1506921874.693853:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921874.693855:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881e1b238600 x1580016666301472/t0(0) o19->oak-OST0006-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000800:00000200:13.0:1506921874.693856:0:3863:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:13.0:1506921874.693858:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000200:11.0:1506921874.693860:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881e1b238600 x1580016666301472/t0(0) o19->oak-OST0006-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.693864:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921874.693865:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203bf0a840. 00000400:00000200:11.0:1506921874.693866:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735f00 00000400:00000010:11.0:1506921874.693867:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735f00. 00000100:00000001:11.0:1506921874.693868:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.693869:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:11.0:1506921874.693870:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:11.0:1506921874.693870:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:11.0:1506921874.693871:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.693872:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:11.0:1506921874.693874:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88203c6afc00 time=69 v=31 (31 1 1 31) 00000100:00001000:11.0:1506921874.693877:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203c6afbc8 time=69 v=1 (1 1 1 1) 00000100:00000001:11.0:1506921874.693879:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:11.0:1506921874.693879:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:11.0:1506921874.693881:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:11.0:1506921874.693882:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.693884:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:11.0:1506921874.693884:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0006-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:11.0:1506921874.693886:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:11.0:1506921874.693886:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:11.0:1506921874.693889:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881e1b238600 x1580016666301472/t0(0) o19->oak-OST0006-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.693894:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.693896:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:11.0:1506921874.693897:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:11.0:1506921874.693899:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881e1b238600 x1580016666301472/t0(0) o19->oak-OST0006-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:11.0:1506921874.693906:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301472:10.0.2.101@o2ib5:19 00000100:00000001:11.0:1506921874.693908:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.693910:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.693911:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:11.0:1506921874.693912:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.693914:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881e1b238600 x1580016666301472/t0(0) o19->oak-OST0006-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.693924:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:11.0:1506921874.693926:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.693927:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:11.0:1506921874.693928:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.693929:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.693931:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881e1b238600 x1580016666301472/t0(0) o19->oak-OST0006-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.693936:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:11.0:1506921874.693937:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:11.0:1506921874.693938:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.693940:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:11.0:1506921874.693940:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:11.0:1506921874.693941:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203c6af800 refcount=4 obd=oak-OST0006-osc-ffff88203bc47000 00000020:00000001:11.0:1506921874.693943:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:11.0:1506921874.693944:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff882011e16200. 02000000:00000001:11.0:1506921874.693945:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:11.0:1506921874.693946:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:11.0:1506921874.693947:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881e1b238600. 00000100:00000001:11.0:1506921874.693948:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:11.0:1506921874.693949:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:11.0:1506921874.693950:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921874.693951:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921874.693953:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:11.0:1506921874.693954:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:11.0:1506921874.693955:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881e1b238600. 00000020:00000040:11.0:1506921874.693956:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203c6a9800 refcount=5 obd=oak-OST0007-osc-ffff88203bc47000 00000100:00000001:11.0:1506921874.693958:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:11.0:1506921874.693959:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:11.0:1506921874.693960:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:11.0:1506921874.693961:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff882011e16200. 00000100:00000001:11.0:1506921874.693963:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.693964:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:11.0:1506921874.693965:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:11.0:1506921874.693966:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.693968:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612270695673600 : -131803013878016 : ffff8820382d6f00) 00000100:00000001:11.0:1506921874.693969:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.693970:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000020:00000001:11.0:1506921874.693972:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:11.0:1506921874.693973:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:11.0:1506921874.693979:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:11.0:1506921874.693982:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:11.0:1506921874.693983:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:11.0:1506921874.693984:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:11.0:1506921874.693985:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:11.0:1506921874.693987:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881e1b238600 x1580016666301488/t0(0) o19->oak-OST0007-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.693993:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:11.0:1506921874.693994:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.693995:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:11.0:1506921874.693996:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:11.0:1506921874.693998:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301488:10.0.2.102@o2ib5:19 00000100:00000001:11.0:1506921874.694000:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:11.0:1506921874.694001:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:11.0:1506921874.694002:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.694004:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:11.0:1506921874.694005:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.694006:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:11.0:1506921874.694008:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203bf0a840. 00000400:00000010:11.0:1506921874.694009:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735f00. 00000100:00000200:11.0:1506921874.694011:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301488, portal 4 00000100:00000001:11.0:1506921874.694013:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.694014:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000100:00000040:11.0:1506921874.694017:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881e1b238600 x1580016666301488/t0(0) o19->oak-OST0007-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.694022:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:11.0:1506921874.694024:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:11.0:1506921874.694025:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735e80. 00000100:00000200:11.0:1506921874.694027:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301488, offset 0 00000400:00000010:11.0:1506921874.694029:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e14a00 (tot 59423868). 00000400:00000200:11.0:1506921874.694032:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:11.0:1506921874.694041:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:11.0:1506921874.694045:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:11.0:1506921874.694049:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:11.0:1506921874.694052:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:11.0:1506921874.694053:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:11.0:1506921874.694056:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:11.0:1506921874.694058:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.694059:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.694061:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.694062:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:11.0:1506921874.694063:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:11.0:1506921874.694064:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8820382d6f00 going to sleep for 43 seconds 00000100:00000001:11.0:1506921874.694065:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.694066:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.694068:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.694068:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921874.694094:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.694112:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921874.694116:0:3861:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921874.694120:0:3861:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921874.694124:0:3861:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881e1b238600 x1580016666301488/t0(0) o19->oak-OST0007-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.694132:0:3861:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921874.694136:0:3861:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881e1b238600 x1580016666301488/t0(0) o19->oak-OST0007-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.694142:0:3861:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.694143:0:3861:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921874.694145:0:3861:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735e80 00000400:00000010:16.0:1506921874.694146:0:3861:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735e80. 00000800:00000200:18.0:1506921874.694149:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000010:16.0:1506921874.694150:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e14a00 (tot 59423468). 00000800:00000200:16.0:1506921874.694153:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:16.2:1506921874.694315:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921874.694322:0:3858:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.694327:0:3858:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.41@o2ib4 00000800:00000200:18.0:1506921874.694329:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000200:16.0:1506921874.694332:0:3858:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:16.0:1506921874.694334:0:3858:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.694339:0:3858:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225430 00000400:00000200:16.0:1506921874.694343:0:3858:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1c61 [1] + 192 00000100:00000001:16.0:1506921874.694348:0:3858:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.694350:0:3858:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881e1b238600 x1580016666301488/t0(0) o19->oak-OST0007-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:16.0:1506921874.694356:0:3858:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881e1b238600 x1580016666301488/t0(0) o19->oak-OST0007-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.694363:0:3858:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.694364:0:3858:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.694366:0:3858:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.694368:0:3858:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921874.694369:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:11.0:1506921874.694385:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:11.0:1506921874.694388:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:11.0:1506921874.694389:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:11.0:1506921874.694391:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921874.694393:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881e1b238600 x1580016666301488/t0(0) o19->oak-OST0007-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:11.0:1506921874.694399:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881e1b238600 x1580016666301488/t0(0) o19->oak-OST0007-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.694403:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921874.694405:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203bf0a840. 00000400:00000200:11.0:1506921874.694406:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735f00 00000400:00000010:11.0:1506921874.694408:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735f00. 00000100:00000001:11.0:1506921874.694409:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.694410:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:11.0:1506921874.694411:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:11.0:1506921874.694412:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:11.0:1506921874.694413:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.694415:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:11.0:1506921874.694417:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88203c6a9c00 time=69 v=31 (31 31 31 31) 00000100:00001000:11.0:1506921874.694419:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203c6a9bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:11.0:1506921874.694421:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:11.0:1506921874.694422:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:11.0:1506921874.694423:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:11.0:1506921874.694424:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.694426:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:11.0:1506921874.694427:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0007-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:11.0:1506921874.694428:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:11.0:1506921874.694429:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:11.0:1506921874.694430:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881e1b238600 x1580016666301488/t0(0) o19->oak-OST0007-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.694435:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.694436:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:11.0:1506921874.694437:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:11.0:1506921874.694439:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881e1b238600 x1580016666301488/t0(0) o19->oak-OST0007-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:11.0:1506921874.694444:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301488:10.0.2.102@o2ib5:19 00000100:00000001:11.0:1506921874.694446:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.694448:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.694449:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:11.0:1506921874.694450:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.694452:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881e1b238600 x1580016666301488/t0(0) o19->oak-OST0007-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.694456:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:11.0:1506921874.694457:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.694459:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:11.0:1506921874.694460:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.694461:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.694463:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881e1b238600 x1580016666301488/t0(0) o19->oak-OST0007-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.694466:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:11.0:1506921874.694467:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:11.0:1506921874.694468:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.694470:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:11.0:1506921874.694471:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:11.0:1506921874.694472:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203c6a9800 refcount=4 obd=oak-OST0007-osc-ffff88203bc47000 00000020:00000001:11.0:1506921874.694473:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:11.0:1506921874.694474:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff882011e16200. 02000000:00000001:11.0:1506921874.694476:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:11.0:1506921874.694477:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:11.0:1506921874.694477:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881e1b238600. 00000100:00000001:11.0:1506921874.694479:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:11.0:1506921874.694480:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:11.0:1506921874.694481:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921874.694482:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921874.694483:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:11.0:1506921874.694484:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:11.0:1506921874.694485:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881e1b238600. 00000020:00000040:11.0:1506921874.694487:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203969d800 refcount=5 obd=oak-OST0008-osc-ffff88203bc47000 00000100:00000001:11.0:1506921874.694488:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:11.0:1506921874.694489:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:11.0:1506921874.694490:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:11.0:1506921874.694491:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff882011e16200. 00000100:00000001:11.0:1506921874.694493:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.694494:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:11.0:1506921874.694495:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:11.0:1506921874.694496:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.694497:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612270695673600 : -131803013878016 : ffff8820382d6f00) 00000100:00000001:11.0:1506921874.694498:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.694499:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000020:00000001:11.0:1506921874.694500:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:11.0:1506921874.694502:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:11.0:1506921874.694508:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:11.0:1506921874.694510:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:11.0:1506921874.694511:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:11.0:1506921874.694512:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:11.0:1506921874.694513:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:11.0:1506921874.694515:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881e1b238600 x1580016666301504/t0(0) o19->oak-OST0008-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.694519:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:11.0:1506921874.694520:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.694521:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:11.0:1506921874.694522:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:11.0:1506921874.694524:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301504:10.0.2.101@o2ib5:19 00000100:00000001:11.0:1506921874.694526:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:11.0:1506921874.694527:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:11.0:1506921874.694528:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.694529:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:11.0:1506921874.694530:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.694531:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:11.0:1506921874.694532:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203bf0a840. 00000400:00000010:11.0:1506921874.694534:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735f00. 00000100:00000200:11.0:1506921874.694536:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301504, portal 4 00000100:00000001:11.0:1506921874.694537:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.694538:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000100:00000040:11.0:1506921874.694540:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881e1b238600 x1580016666301504/t0(0) o19->oak-OST0008-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.694544:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:11.0:1506921874.694546:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:11.0:1506921874.694547:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032734b80. 00000100:00000200:11.0:1506921874.694548:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301504, offset 0 00000400:00000010:11.0:1506921874.694550:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e15800 (tot 59423868). 00000400:00000200:11.0:1506921874.694553:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:11.0:1506921874.694560:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:11.0:1506921874.694563:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:11.0:1506921874.694566:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:11.0:1506921874.694568:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:11.0:1506921874.694569:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921874.694572:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:11.0:1506921874.694573:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.694574:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.694575:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.694576:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:11.0:1506921874.694577:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:11.0:1506921874.694578:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8820382d6f00 going to sleep for 43 seconds 00000100:00000001:11.0:1506921874.694580:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.694581:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.694582:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.694582:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921874.694605:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921874.694617:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:11.0:1506921874.694620:0:3862:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:11.0:1506921874.694621:0:3862:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:11.0:1506921874.694623:0:3862:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881e1b238600 x1580016666301504/t0(0) o19->oak-OST0008-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.694627:0:3862:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.694629:0:3862:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881e1b238600 x1580016666301504/t0(0) o19->oak-OST0008-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.694633:0:3862:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.694634:0:3862:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:11.0:1506921874.694634:0:3862:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032734b80 00000400:00000010:11.0:1506921874.694635:0:3862:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032734b80. 00000400:00000010:11.0:1506921874.694637:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e15800 (tot 59423468). 00000800:00000200:11.0:1506921874.694639:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:11.0:1506921874.694640:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.2:1506921874.694879:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921874.694896:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.694902:0:3859:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921874.694908:0:3859:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921874.694910:0:3859:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.694916:0:3859:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225440 00000400:00000200:16.0:1506921874.694921:0:3859:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a0e05 [1] + 192 00000100:00000001:16.0:1506921874.694926:0:3859:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.694929:0:3859:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881e1b238600 x1580016666301504/t0(0) o19->oak-OST0008-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921874.694933:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921874.694939:0:3859:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881e1b238600 x1580016666301504/t0(0) o19->oak-OST0008-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.694947:0:3859:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.694948:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.694951:0:3859:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.694953:0:3859:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921874.694955:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:11.0:1506921874.694970:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:11.0:1506921874.694972:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:11.0:1506921874.694973:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:11.0:1506921874.694974:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921874.694977:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881e1b238600 x1580016666301504/t0(0) o19->oak-OST0008-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:11.0:1506921874.694982:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881e1b238600 x1580016666301504/t0(0) o19->oak-OST0008-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.694986:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921874.694988:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203bf0a840. 00000400:00000200:11.0:1506921874.694989:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735f00 00000400:00000010:11.0:1506921874.694990:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735f00. 00000100:00000001:11.0:1506921874.694992:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.694993:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:11.0:1506921874.694994:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:11.0:1506921874.694995:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:11.0:1506921874.694996:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.694998:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:11.0:1506921874.695000:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88203969dc00 time=69 v=31 (31 3 1 31) 00000100:00001000:11.0:1506921874.695002:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203969dbc8 time=69 v=1 (1 1 1 1) 00000100:00000001:11.0:1506921874.695004:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:11.0:1506921874.695005:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:11.0:1506921874.695006:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:11.0:1506921874.695007:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.695008:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:11.0:1506921874.695009:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0008-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:11.0:1506921874.695010:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:11.0:1506921874.695011:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:11.0:1506921874.695013:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881e1b238600 x1580016666301504/t0(0) o19->oak-OST0008-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.695017:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.695018:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:11.0:1506921874.695019:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:11.0:1506921874.695021:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881e1b238600 x1580016666301504/t0(0) o19->oak-OST0008-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:11.0:1506921874.695026:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301504:10.0.2.101@o2ib5:19 00000100:00000001:11.0:1506921874.695028:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.695030:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.695031:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:11.0:1506921874.695032:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.695034:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881e1b238600 x1580016666301504/t0(0) o19->oak-OST0008-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.695038:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:11.0:1506921874.695039:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.695040:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:11.0:1506921874.695041:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.695043:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.695044:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881e1b238600 x1580016666301504/t0(0) o19->oak-OST0008-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.695048:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:11.0:1506921874.695049:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:11.0:1506921874.695050:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.695052:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:11.0:1506921874.695053:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:11.0:1506921874.695053:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203969d800 refcount=4 obd=oak-OST0008-osc-ffff88203bc47000 00000020:00000001:11.0:1506921874.695055:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:11.0:1506921874.695056:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff882011e16200. 02000000:00000001:11.0:1506921874.695057:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:11.0:1506921874.695058:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:11.0:1506921874.695058:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881e1b238600. 00000100:00000001:11.0:1506921874.695060:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:11.0:1506921874.695061:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:11.0:1506921874.695062:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921874.695063:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921874.695064:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:11.0:1506921874.695065:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:11.0:1506921874.695066:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881e1b238600. 00000020:00000040:11.0:1506921874.695067:0:38877:0:(genops.c:1012:class_import_get()) import ffff882039699800 refcount=5 obd=oak-OST0009-osc-ffff88203bc47000 00000100:00000001:11.0:1506921874.695069:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:11.0:1506921874.695069:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:11.0:1506921874.695070:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:11.0:1506921874.695072:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff882011e16200. 00000100:00000001:11.0:1506921874.695073:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.695075:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:11.0:1506921874.695075:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:11.0:1506921874.695076:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.695077:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612270695673600 : -131803013878016 : ffff8820382d6f00) 00000100:00000001:11.0:1506921874.695079:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.695079:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000020:00000001:11.0:1506921874.695081:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:11.0:1506921874.695082:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:11.0:1506921874.695089:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:11.0:1506921874.695091:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:11.0:1506921874.695092:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:11.0:1506921874.695093:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:11.0:1506921874.695094:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:11.0:1506921874.695095:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881e1b238600 x1580016666301520/t0(0) o19->oak-OST0009-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.695100:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:11.0:1506921874.695101:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.695102:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:11.0:1506921874.695103:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:11.0:1506921874.695104:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301520:10.0.2.102@o2ib5:19 00000100:00000001:11.0:1506921874.695106:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:11.0:1506921874.695107:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:11.0:1506921874.695108:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.695109:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:11.0:1506921874.695110:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.695111:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:11.0:1506921874.695113:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203bf0a840. 00000400:00000010:11.0:1506921874.695114:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735f00. 00000100:00000200:11.0:1506921874.695116:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301520, portal 4 00000100:00000001:11.0:1506921874.695117:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.695118:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000100:00000040:11.0:1506921874.695120:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881e1b238600 x1580016666301520/t0(0) o19->oak-OST0009-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.695124:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:11.0:1506921874.695126:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:11.0:1506921874.695127:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032734b80. 00000100:00000200:11.0:1506921874.695128:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301520, offset 0 00000400:00000010:11.0:1506921874.695130:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e15800 (tot 59423868). 00000400:00000200:11.0:1506921874.695132:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:11.0:1506921874.695139:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:11.0:1506921874.695143:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:11.0:1506921874.695147:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:11.0:1506921874.695149:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:11.0:1506921874.695151:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:11.0:1506921874.695154:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:11.0:1506921874.695155:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.695156:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.695157:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.695158:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:16.2:1506921874.695159:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:11.0:1506921874.695159:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:11.0:1506921874.695160:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8820382d6f00 going to sleep for 43 seconds 00000100:00000001:11.0:1506921874.695162:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.695162:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.695164:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.695164:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.0:1506921874.695166:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921874.695169:0:3860:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921874.695172:0:3860:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921874.695175:0:3860:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881e1b238600 x1580016666301520/t0(0) o19->oak-OST0009-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.695180:0:3860:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921874.695182:0:3860:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881e1b238600 x1580016666301520/t0(0) o19->oak-OST0009-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.695186:0:3860:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.695188:0:3860:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921874.695189:0:3860:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032734b80 00000400:00000010:16.0:1506921874.695191:0:3860:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032734b80. 00000400:00000010:16.0:1506921874.695194:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e15800 (tot 59423468). 00000800:00000200:16.0:1506921874.695196:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921874.695198:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921874.695422:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:1.0:1506921874.695469:0:3864:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921874.695478:0:3864:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.42@o2ib4 00000400:00000200:1.0:1506921874.695485:0:3864:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:1.0:1506921874.695487:0:3864:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881c4b6cc800 (tot 59423868). 00000400:00000200:1.0:1506921874.695493:0:3864:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225450 00000400:00000200:1.0:1506921874.695498:0:3864:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1c69 [1] + 192 00000100:00000001:1.0:1506921874.695503:0:3864:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:1.0:1506921874.695506:0:3864:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881e1b238600 x1580016666301520/t0(0) o19->oak-OST0009-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:13.0:1506921874.695508:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:1.0:1506921874.695514:0:3864:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881e1b238600 x1580016666301520/t0(0) o19->oak-OST0009-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921874.695531:0:3864:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:1.0:1506921874.695533:0:3864:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881c4b6cc800 (tot 59423468). 00000800:00000200:1.0:1506921874.695535:0:3864:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921874.695537:0:3864:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:1.0:1506921874.695539:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:11.0:1506921874.695548:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:11.0:1506921874.695550:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:11.0:1506921874.695551:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:11.0:1506921874.695552:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921874.695554:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881e1b238600 x1580016666301520/t0(0) o19->oak-OST0009-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:11.0:1506921874.695560:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881e1b238600 x1580016666301520/t0(0) o19->oak-OST0009-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.695564:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921874.695565:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203bf0a840. 00000400:00000200:11.0:1506921874.695566:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735f00 00000400:00000010:11.0:1506921874.695568:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735f00. 00000100:00000001:11.0:1506921874.695569:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.695570:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:11.0:1506921874.695571:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:11.0:1506921874.695572:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:11.0:1506921874.695573:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.695574:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:11.0:1506921874.695576:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff882039699c00 time=69 v=31 (31 31 31 31) 00000100:00001000:11.0:1506921874.695578:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff882039699bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:11.0:1506921874.695580:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:11.0:1506921874.695581:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:11.0:1506921874.695582:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:11.0:1506921874.695583:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.695584:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:11.0:1506921874.695585:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0009-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:11.0:1506921874.695586:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:11.0:1506921874.695587:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:11.0:1506921874.695589:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881e1b238600 x1580016666301520/t0(0) o19->oak-OST0009-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.695594:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.695595:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:11.0:1506921874.695596:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:11.0:1506921874.695598:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881e1b238600 x1580016666301520/t0(0) o19->oak-OST0009-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:11.0:1506921874.695603:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301520:10.0.2.102@o2ib5:19 00000100:00000001:11.0:1506921874.695605:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.695607:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.695608:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:11.0:1506921874.695609:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.695610:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881e1b238600 x1580016666301520/t0(0) o19->oak-OST0009-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.695615:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:11.0:1506921874.695616:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.695617:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:11.0:1506921874.695618:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.695619:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.695621:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881e1b238600 x1580016666301520/t0(0) o19->oak-OST0009-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.695625:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:11.0:1506921874.695625:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:11.0:1506921874.695626:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.695628:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:11.0:1506921874.695629:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:11.0:1506921874.695629:0:38877:0:(genops.c:1026:class_import_put()) import ffff882039699800 refcount=4 obd=oak-OST0009-osc-ffff88203bc47000 00000020:00000001:11.0:1506921874.695631:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:11.0:1506921874.695632:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff882011e16200. 02000000:00000001:11.0:1506921874.695633:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:11.0:1506921874.695634:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:11.0:1506921874.695634:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881e1b238600. 00000100:00000001:11.0:1506921874.695636:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:11.0:1506921874.695637:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:11.0:1506921874.695638:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921874.695640:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921874.695641:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:11.0:1506921874.695642:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:11.0:1506921874.695642:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881e1b238600. 00000020:00000040:11.0:1506921874.695644:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203969a000 refcount=5 obd=oak-OST000a-osc-ffff88203bc47000 00000100:00000001:11.0:1506921874.695645:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:11.0:1506921874.695646:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:11.0:1506921874.695647:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:11.0:1506921874.695648:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff882011e16200. 00000100:00000001:11.0:1506921874.695650:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.695651:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:11.0:1506921874.695652:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:11.0:1506921874.695653:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.695654:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612270695673600 : -131803013878016 : ffff8820382d6f00) 00000100:00000001:11.0:1506921874.695655:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.695656:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000020:00000001:11.0:1506921874.695657:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:11.0:1506921874.695658:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:11.0:1506921874.695665:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:11.0:1506921874.695666:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:11.0:1506921874.695668:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:11.0:1506921874.695669:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:11.0:1506921874.695670:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:11.0:1506921874.695671:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881e1b238600 x1580016666301536/t0(0) o19->oak-OST000a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.695675:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:11.0:1506921874.695676:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.695677:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:11.0:1506921874.695678:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:11.0:1506921874.695680:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301536:10.0.2.101@o2ib5:19 00000100:00000001:11.0:1506921874.695681:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:11.0:1506921874.695682:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:11.0:1506921874.695683:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.695684:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:11.0:1506921874.695685:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.695687:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:11.0:1506921874.695688:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203bf0a840. 00000400:00000010:11.0:1506921874.695690:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735f00. 00000100:00000200:11.0:1506921874.695692:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301536, portal 4 00000100:00000001:11.0:1506921874.695693:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.695694:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000100:00000040:11.0:1506921874.695696:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881e1b238600 x1580016666301536/t0(0) o19->oak-OST000a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.695700:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:11.0:1506921874.695702:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:11.0:1506921874.695703:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032734d00. 00000100:00000200:11.0:1506921874.695704:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301536, offset 0 00000400:00000010:11.0:1506921874.695706:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e12600 (tot 59423868). 00000400:00000200:11.0:1506921874.695708:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:11.0:1506921874.695715:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:11.0:1506921874.695726:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:11.0:1506921874.695729:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:11.0:1506921874.695731:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:11.0:1506921874.695732:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921874.695734:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:11.0:1506921874.695736:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.695737:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.695738:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.695739:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:11.0:1506921874.695740:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:11.0:1506921874.695741:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8820382d6f00 going to sleep for 43 seconds 00000100:00000001:11.0:1506921874.695742:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.695743:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.695744:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.695744:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921874.695770:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921874.695782:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:11.0:1506921874.695785:0:3862:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:11.0:1506921874.695786:0:3862:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:11.0:1506921874.695788:0:3862:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881e1b238600 x1580016666301536/t0(0) o19->oak-OST000a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.695793:0:3862:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.695794:0:3862:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881e1b238600 x1580016666301536/t0(0) o19->oak-OST000a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.695798:0:3862:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.695799:0:3862:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:11.0:1506921874.695800:0:3862:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032734d00 00000400:00000010:11.0:1506921874.695801:0:3862:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032734d00. 00000400:00000010:11.0:1506921874.695803:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e12600 (tot 59423468). 00000800:00000200:11.0:1506921874.695804:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:11.0:1506921874.695806:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.2:1506921874.695994:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921874.696012:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.696018:0:3861:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921874.696024:0:3861:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921874.696027:0:3861:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.696033:0:3861:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225460 00000400:00000200:16.0:1506921874.696038:0:3861:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a0e1d [1] + 192 00000100:00000001:16.0:1506921874.696042:0:3861:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.696045:0:3861:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881e1b238600 x1580016666301536/t0(0) o19->oak-OST000a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921874.696051:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921874.696053:0:3861:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881e1b238600 x1580016666301536/t0(0) o19->oak-OST000a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.696067:0:3861:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.696069:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.696071:0:3861:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:11.0:1506921874.696071:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:11.0:1506921874.696072:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:11.0:1506921874.696073:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000800:00000200:16.0:1506921874.696074:0:3861:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:11.0:1506921874.696074:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000800:00000200:16.0:1506921874.696075:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000200:11.0:1506921874.696076:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881e1b238600 x1580016666301536/t0(0) o19->oak-OST000a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:11.0:1506921874.696081:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881e1b238600 x1580016666301536/t0(0) o19->oak-OST000a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.696085:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921874.696087:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203bf0a840. 00000400:00000200:11.0:1506921874.696088:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735f00 00000400:00000010:11.0:1506921874.696088:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735f00. 00000100:00000001:11.0:1506921874.696089:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.696090:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:11.0:1506921874.696091:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:11.0:1506921874.696092:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:11.0:1506921874.696093:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.696094:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:11.0:1506921874.696096:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88203969a400 time=69 v=31 (31 31 1 31) 00000100:00001000:11.0:1506921874.696098:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203969a3c8 time=69 v=1 (1 1 1 1) 00000100:00000001:11.0:1506921874.696101:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:11.0:1506921874.696101:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:11.0:1506921874.696103:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:11.0:1506921874.696104:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.696105:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:11.0:1506921874.696107:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST000a-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:11.0:1506921874.696108:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:11.0:1506921874.696109:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:11.0:1506921874.696111:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881e1b238600 x1580016666301536/t0(0) o19->oak-OST000a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.696117:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.696118:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:11.0:1506921874.696119:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:11.0:1506921874.696121:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881e1b238600 x1580016666301536/t0(0) o19->oak-OST000a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:11.0:1506921874.696128:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301536:10.0.2.101@o2ib5:19 00000100:00000001:11.0:1506921874.696131:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.696133:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.696134:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:11.0:1506921874.696135:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.696137:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881e1b238600 x1580016666301536/t0(0) o19->oak-OST000a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.696142:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:11.0:1506921874.696143:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.696145:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:11.0:1506921874.696146:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.696147:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.696149:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881e1b238600 x1580016666301536/t0(0) o19->oak-OST000a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.696154:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:11.0:1506921874.696155:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:11.0:1506921874.696156:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.696158:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:11.0:1506921874.696159:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:11.0:1506921874.696159:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203969a000 refcount=4 obd=oak-OST000a-osc-ffff88203bc47000 00000020:00000001:11.0:1506921874.696161:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:11.0:1506921874.696162:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff882011e16200. 02000000:00000001:11.0:1506921874.696164:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:11.0:1506921874.696165:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:11.0:1506921874.696169:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881e1b238600. 00000100:00000001:11.0:1506921874.696170:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:11.0:1506921874.696171:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:11.0:1506921874.696172:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921874.696174:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921874.696175:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:11.0:1506921874.696176:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:11.0:1506921874.696177:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881e1b238600. 00000020:00000040:11.0:1506921874.696178:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203969e000 refcount=5 obd=oak-OST000b-osc-ffff88203bc47000 00000100:00000001:11.0:1506921874.696180:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:11.0:1506921874.696181:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:11.0:1506921874.696182:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:11.0:1506921874.696184:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff882011e16200. 00000100:00000001:11.0:1506921874.696185:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.696187:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:11.0:1506921874.696187:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:11.0:1506921874.696189:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.696190:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612270695673600 : -131803013878016 : ffff8820382d6f00) 00000100:00000001:11.0:1506921874.696191:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.696192:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000020:00000001:11.0:1506921874.696194:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:11.0:1506921874.696195:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:11.0:1506921874.696201:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:11.0:1506921874.696203:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:11.0:1506921874.696205:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:11.0:1506921874.696206:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:11.0:1506921874.696207:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:11.0:1506921874.696209:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881e1b238600 x1580016666301552/t0(0) o19->oak-OST000b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.696215:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:11.0:1506921874.696216:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.696217:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:11.0:1506921874.696218:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:11.0:1506921874.696221:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301552:10.0.2.102@o2ib5:19 00000100:00000001:11.0:1506921874.696223:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:11.0:1506921874.696224:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:11.0:1506921874.696225:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.696226:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:11.0:1506921874.696227:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.696229:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:11.0:1506921874.696230:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203bf0a840. 00000400:00000010:11.0:1506921874.696232:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735f00. 00000100:00000200:11.0:1506921874.696234:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301552, portal 4 00000100:00000001:11.0:1506921874.696235:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.696236:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000100:00000040:11.0:1506921874.696239:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881e1b238600 x1580016666301552/t0(0) o19->oak-OST000b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.696244:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:11.0:1506921874.696246:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:11.0:1506921874.696247:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032734d00. 00000100:00000200:11.0:1506921874.696249:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301552, offset 0 00000400:00000010:11.0:1506921874.696251:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e12600 (tot 59423868). 00000400:00000200:11.0:1506921874.696255:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:11.0:1506921874.696262:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:11.0:1506921874.696266:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:11.0:1506921874.696271:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:11.0:1506921874.696273:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:11.0:1506921874.696275:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:11.0:1506921874.696278:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:11.0:1506921874.696280:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.696281:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.696282:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921874.696283:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:11.0:1506921874.696284:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:11.0:1506921874.696284:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=6 : 6 : 6) 00000100:00100000:11.0:1506921874.696286:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8820382d6f00 going to sleep for 6 seconds 00000100:00000001:11.0:1506921874.696287:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.696288:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.696289:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:16.0:1506921874.696290:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000100:00000001:11.0:1506921874.696290:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.0:1506921874.696294:0:3860:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921874.696297:0:3860:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921874.696300:0:3860:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881e1b238600 x1580016666301552/t0(0) o19->oak-OST000b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.696306:0:3860:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921874.696308:0:3860:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881e1b238600 x1580016666301552/t0(0) o19->oak-OST000b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.696312:0:3860:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.696314:0:3860:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921874.696315:0:3860:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032734d00 00000400:00000010:16.0:1506921874.696317:0:3860:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032734d00. 00000400:00000010:16.0:1506921874.696320:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e12600 (tot 59423468). 00000800:00000200:16.0:1506921874.696322:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921874.696324:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:16.2:1506921874.696504:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921874.696510:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.696514:0:3859:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921874.696518:0:3859:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000800:00000200:18.0:1506921874.696519:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000010:16.0:1506921874.696520:0:3859:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.696523:0:3859:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225470 00000400:00000200:16.0:1506921874.696527:0:3859:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1c71 [1] + 192 00000100:00000001:16.0:1506921874.696530:0:3859:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.696532:0:3859:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881e1b238600 x1580016666301552/t0(0) o19->oak-OST000b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:16.0:1506921874.696537:0:3859:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881e1b238600 x1580016666301552/t0(0) o19->oak-OST000b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.696544:0:3859:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.696546:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.696547:0:3859:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:11.0:1506921874.696548:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:16.0:1506921874.696549:0:3859:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921874.696550:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000400:00000001:11.0:1506921874.696550:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:11.0:1506921874.696551:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:11.0:1506921874.696552:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921874.696555:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881e1b238600 x1580016666301552/t0(0) o19->oak-OST000b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:11.0:1506921874.696560:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881e1b238600 x1580016666301552/t0(0) o19->oak-OST000b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.696565:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921874.696566:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203bf0a840. 00000400:00000200:11.0:1506921874.696567:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735f00 00000400:00000010:11.0:1506921874.696568:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735f00. 00000100:00000001:11.0:1506921874.696569:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.696570:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:11.0:1506921874.696571:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:11.0:1506921874.696572:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:11.0:1506921874.696573:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.696575:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:11.0:1506921874.696576:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88203969e400 time=69 v=1 (31 31 31 31) 00000100:00001000:11.0:1506921874.696578:0:38877:0:(import.c:1737:at_measured()) AT ffff88203969e400 change: old=1 new=31 delta=30 (val=31) hist 31 31 31 31 00000100:00001000:11.0:1506921874.696581:0:38877:0:(client.c:345:ptlrpc_at_adj_service()) The RPC service estimate for oak-OST000b-osc-ffff88203bc47000 ptl 28 has changed from 1 to 31 00000100:00001000:11.0:1506921874.696583:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203969e3c8 time=69 v=1 (1 1 1 1) 00000100:00000001:11.0:1506921874.696584:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:11.0:1506921874.696585:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:11.0:1506921874.696586:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:11.0:1506921874.696587:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.696589:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:11.0:1506921874.696590:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST000b-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:11.0:1506921874.696591:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:11.0:1506921874.696591:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:11.0:1506921874.696593:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881e1b238600 x1580016666301552/t0(0) o19->oak-OST000b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921881 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.696597:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.696599:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:11.0:1506921874.696599:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:11.0:1506921874.696601:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881e1b238600 x1580016666301552/t0(0) o19->oak-OST000b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921881 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:11.0:1506921874.696607:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301552:10.0.2.102@o2ib5:19 00000100:00000001:11.0:1506921874.696609:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921874.696611:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.696612:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:11.0:1506921874.696613:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.696615:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881e1b238600 x1580016666301552/t0(0) o19->oak-OST000b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921881 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.696619:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:11.0:1506921874.696620:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.696621:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:11.0:1506921874.696622:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.696624:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.696625:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881e1b238600 x1580016666301552/t0(0) o19->oak-OST000b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921881 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921874.696629:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:11.0:1506921874.696630:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:11.0:1506921874.696630:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.696632:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:11.0:1506921874.696632:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:11.0:1506921874.696633:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203969e000 refcount=4 obd=oak-OST000b-osc-ffff88203bc47000 00000020:00000001:11.0:1506921874.696635:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:11.0:1506921874.696635:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff882011e16200. 02000000:00000001:11.0:1506921874.696637:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:11.0:1506921874.696637:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:11.0:1506921874.696638:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881e1b238600. 00000100:00000001:11.0:1506921874.696640:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:11.0:1506921874.696640:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:11.0:1506921874.696642:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921874.696643:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921874.696644:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:11.0:1506921874.696645:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:11.0:1506921874.696645:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881e1b238600. 00000020:00000040:11.0:1506921874.696647:0:38877:0:(genops.c:1012:class_import_get()) import ffff882036e42000 refcount=5 obd=oak-OST000c-osc-ffff88203bc47000 00000100:00000001:11.0:1506921874.696648:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:11.0:1506921874.696649:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:11.0:1506921874.696650:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:11.0:1506921874.696651:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff882011e16200. 00000100:00000001:11.0:1506921874.696653:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.696654:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:11.0:1506921874.696655:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:11.0:1506921874.696656:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8820382d6f00. 00000100:00000001:11.0:1506921874.696657:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612270695673600 : -131803013878016 : ffff8820382d6f00) 00000100:00000001:11.0:1506921874.696658:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.696659:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000020:00000001:11.0:1506921874.696660:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:11.0:1506921874.696661:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:11.0:1506921874.696667:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:11.0:1506921874.696669:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:11.0:1506921874.696671:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:11.0:1506921874.696672:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:11.0:1506921874.696673:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:11.0:1506921874.696674:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881e1b238600 x1580016666301568/t0(0) o19->oak-OST000c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.696678:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:11.0:1506921874.696679:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.696680:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:11.0:1506921874.696681:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:11.0:1506921874.696683:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301568:10.0.2.101@o2ib5:19 00000100:00000001:11.0:1506921874.696685:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:11.0:1506921874.696686:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:11.0:1506921874.696686:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921874.696687:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:11.0:1506921874.696688:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:11.0:1506921874.696689:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:11.0:1506921874.696691:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203bf0a840. 00000400:00000010:11.0:1506921874.696692:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735f00. 00000100:00000200:11.0:1506921874.696694:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301568, portal 4 00000100:00000001:11.0:1506921874.696695:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921874.696696:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618550272 : -131812091001344 : ffff881e1b238600) 00000100:00000040:11.0:1506921874.696699:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881e1b238600 x1580016666301568/t0(0) o19->oak-OST000c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.696703:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:11.0:1506921874.696705:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:11.0:1506921874.696706:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032734d80. 00000100:00000200:11.0:1506921874.696707:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301568, offset 0 00000400:00000010:11.0:1506921874.696710:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e12600 (tot 59423868). 00000400:00000200:11.0:1506921874.696712:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:11.0:1506921874.696725:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:11.0:1506921874.696729:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:11.0:1506921874.696732:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:11.0:1506921874.696733:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:11.0:1506921874.696734:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921874.696737:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:11.0:1506921874.696738:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.696739:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921874.696740:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.696741:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:11.0:1506921874.696742:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:11.0:1506921874.696743:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8820382d6f00 going to sleep for 43 seconds 00000100:00000001:11.0:1506921874.696745:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.696745:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.696747:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921874.696747:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921874.696773:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921874.696816:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:13.0:1506921874.696822:0:3863:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:13.0:1506921874.696826:0:3863:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:13.0:1506921874.696829:0:3863:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881e1b238600 x1580016666301568/t0(0) o19->oak-OST000c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.696837:0:3863:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:13.0:1506921874.696840:0:3863:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881e1b238600 x1580016666301568/t0(0) o19->oak-OST000c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.696845:0:3863:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.696847:0:3863:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:13.0:1506921874.696848:0:3863:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032734d80 00000400:00000010:13.0:1506921874.696849:0:3863:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032734d80. 00000400:00000010:13.0:1506921874.696853:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e12600 (tot 59423468). 00000800:00000200:1.0:1506921874.696855:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:13.0:1506921874.696859:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:2.2:1506921874.697012:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:11.0:1506921874.697021:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921874.697025:0:3862:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.42@o2ib4 00000400:00000200:11.0:1506921874.697029:0:3862:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:11.0:1506921874.697032:0:3862:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e15800 (tot 59423868). 00000400:00000200:11.0:1506921874.697035:0:3862:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225480 00000400:00000200:11.0:1506921874.697039:0:3862:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a0e35 [1] + 192 00000100:00000001:11.0:1506921874.697043:0:3862:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921874.697044:0:3862:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881e1b238600 x1580016666301568/t0(0) o19->oak-OST000c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:11.0:1506921874.697054:0:3862:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881e1b238600 x1580016666301568/t0(0) o19->oak-OST000c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000800:00000200:5.0:1506921874.697057:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000001:11.0:1506921874.697061:0:3862:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921874.697062:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e15800 (tot 59423468). 00000800:00000200:11.0:1506921874.697064:0:3862:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000100:00000001:13.0:1506921874.697066:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:11.0:1506921874.697066:0:3862:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:11.0:1506921874.697067:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000400:00000001:13.0:1506921874.697069:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:13.0:1506921874.697070:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:13.0:1506921874.697071:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:13.0:1506921874.697073:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881e1b238600 x1580016666301568/t0(0) o19->oak-OST000c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:13.0:1506921874.697079:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881e1b238600 x1580016666301568/t0(0) o19->oak-OST000c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.697084:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:13.0:1506921874.697085:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203bf0a840. 00000400:00000200:13.0:1506921874.697087:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735f00 00000400:00000010:13.0:1506921874.697088:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735f00. 00000100:00000001:13.0:1506921874.697089:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:13.0:1506921874.697090:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:13.0:1506921874.697092:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:13.0:1506921874.697093:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:13.0:1506921874.697094:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:13.0:1506921874.697096:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:13.0:1506921874.697099:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff882036e42400 time=69 v=31 (31 31 1 31) 00000100:00001000:13.0:1506921874.697104:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff882036e423c8 time=69 v=1 (1 1 1 1) 00000100:00000001:13.0:1506921874.697105:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:13.0:1506921874.697106:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:13.0:1506921874.697108:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:13.0:1506921874.697109:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.697111:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:13.0:1506921874.697111:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST000c-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:13.0:1506921874.697113:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:13.0:1506921874.697113:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:13.0:1506921874.697116:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881e1b238600 x1580016666301568/t0(0) o19->oak-OST000c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:13.0:1506921874.697121:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:13.0:1506921874.697123:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:13.0:1506921874.697124:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:13.0:1506921874.697126:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881e1b238600 x1580016666301568/t0(0) o19->oak-OST000c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:13.0:1506921874.697131:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301568:10.0.2.101@o2ib5:19 00000100:00000001:13.0:1506921874.697134:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:13.0:1506921874.697136:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.697137:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:13.0:1506921874.697138:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:13.0:1506921874.697139:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881e1b238600 x1580016666301568/t0(0) o19->oak-OST000c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:13.0:1506921874.697143:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:13.0:1506921874.697145:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8820382d6f00. 00000100:00000001:13.0:1506921874.697147:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:13.0:1506921874.697147:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.697150:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:13.0:1506921874.697151:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881e1b238600 x1580016666301568/t0(0) o19->oak-OST000c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:13.0:1506921874.697155:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:13.0:1506921874.697156:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:13.0:1506921874.697157:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8810e90bc800. 02000000:00000001:13.0:1506921874.697159:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:13.0:1506921874.697161:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:13.0:1506921874.697162:0:38877:0:(genops.c:1026:class_import_put()) import ffff882036e42000 refcount=4 obd=oak-OST000c-osc-ffff88203bc47000 00000020:00000001:13.0:1506921874.697164:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:13.0:1506921874.697165:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff882011e16200. 02000000:00000001:13.0:1506921874.697166:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:13.0:1506921874.697167:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:13.0:1506921874.697168:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881e1b238600. 00000100:00000001:13.0:1506921874.697170:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:13.0:1506921874.697171:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:13.0:1506921874.697172:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:13.0:1506921874.697174:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:13.0:1506921874.697175:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:13.0:1506921874.697176:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:13.0:1506921874.697178:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881cd2f6ef00. 00000020:00000040:13.0:1506921874.697179:0:38877:0:(genops.c:1012:class_import_get()) import ffff882036e44000 refcount=5 obd=oak-OST000d-osc-ffff88203bc47000 00000100:00000001:13.0:1506921874.697181:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:13.0:1506921874.697182:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:13.0:1506921874.697183:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:13.0:1506921874.697185:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff88203bd1e000. 00000100:00000001:13.0:1506921874.697188:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.697190:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:13.0:1506921874.697191:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:13.0:1506921874.697193:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff88203830f2c0. 00000100:00000001:13.0:1506921874.697194:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612270695903936 : -131803013647680 : ffff88203830f2c0) 00000100:00000001:13.0:1506921874.697196:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:13.0:1506921874.697197:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612256112701184 : -131817596850432 : ffff881cd2f6ef00) 00000020:00000001:13.0:1506921874.697199:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:13.0:1506921874.697200:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:13.0:1506921874.697210:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:13.0:1506921874.697212:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:13.0:1506921874.697214:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:13.0:1506921874.697216:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:13.0:1506921874.697217:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:13.0:1506921874.697219:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881cd2f6ef00 x1580016666301584/t0(0) o19->oak-OST000d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.697223:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:13.0:1506921874.697225:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:13.0:1506921874.697227:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:13.0:1506921874.697228:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:13.0:1506921874.697230:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301584:10.0.2.102@o2ib5:19 00000100:00000001:13.0:1506921874.697233:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:13.0:1506921874.697234:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:13.0:1506921874.697235:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:13.0:1506921874.697237:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:13.0:1506921874.697238:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff882033d7c400. 02000000:00000001:13.0:1506921874.697240:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:13.0:1506921874.697242:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203b78ba20. 00000400:00000010:13.0:1506921874.697244:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881dc4adf100. 00000100:00000200:13.0:1506921874.697246:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301584, portal 4 00000100:00000001:13.0:1506921874.697248:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:13.0:1506921874.697249:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612256112701184 : -131817596850432 : ffff881cd2f6ef00) 00000100:00000040:13.0:1506921874.697251:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881cd2f6ef00 x1580016666301584/t0(0) o19->oak-OST000d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.697256:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:13.0:1506921874.697258:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:13.0:1506921874.697259:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881dc4adf280. 00000100:00000200:13.0:1506921874.697261:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301584, offset 0 00000400:00000010:13.0:1506921874.697263:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88203bd1ae00 (tot 59423868). 00000400:00000200:13.0:1506921874.697266:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:13.0:1506921874.697275:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:13.0:1506921874.697279:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:13.0:1506921874.697284:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:13.0:1506921874.697287:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:13.0:1506921874.697288:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:13.0:1506921874.697292:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:13.0:1506921874.697294:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.697295:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:13.0:1506921874.697296:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.697298:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:13.0:1506921874.697299:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:13.0:1506921874.697301:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff88203830f2c0 going to sleep for 43 seconds 00000100:00000001:13.0:1506921874.697302:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:13.0:1506921874.697304:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.697305:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:13.0:1506921874.697306:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921874.697329:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.697347:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921874.697352:0:3860:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921874.697355:0:3860:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921874.697359:0:3860:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881cd2f6ef00 x1580016666301584/t0(0) o19->oak-OST000d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.697367:0:3860:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921874.697369:0:3860:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881cd2f6ef00 x1580016666301584/t0(0) o19->oak-OST000d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.697375:0:3860:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.697377:0:3860:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921874.697378:0:3860:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881dc4adf280 00000400:00000010:16.0:1506921874.697379:0:3860:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881dc4adf280. 00000400:00000010:16.0:1506921874.697383:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88203bd1ae00 (tot 59423468). 00000800:00000200:18.0:1506921874.697384:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921874.697388:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921874.697494:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:1.0:1506921874.697537:0:3864:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921874.697546:0:3864:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.42@o2ib4 00000400:00000200:1.0:1506921874.697552:0:3864:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:1.0:1506921874.697555:0:3864:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881c4b6cc800 (tot 59423868). 00000400:00000200:1.0:1506921874.697560:0:3864:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225490 00000400:00000200:1.0:1506921874.697565:0:3864:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1c79 [1] + 192 00000100:00000001:1.0:1506921874.697570:0:3864:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:1.0:1506921874.697573:0:3864:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881cd2f6ef00 x1580016666301584/t0(0) o19->oak-OST000d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:13.0:1506921874.697577:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:1.0:1506921874.697581:0:3864:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881cd2f6ef00 x1580016666301584/t0(0) o19->oak-OST000d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921874.697589:0:3864:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:1.0:1506921874.697591:0:3864:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881c4b6cc800 (tot 59423468). 00000100:00000001:13.0:1506921874.697593:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:1.0:1506921874.697594:0:3864:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921874.697596:0:3864:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000400:00000001:13.0:1506921874.697597:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:13.0:1506921874.697598:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000800:00000200:1.0:1506921874.697598:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:13.0:1506921874.697599:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:13.0:1506921874.697603:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881cd2f6ef00 x1580016666301584/t0(0) o19->oak-OST000d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:13.0:1506921874.697611:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881cd2f6ef00 x1580016666301584/t0(0) o19->oak-OST000d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.697617:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:13.0:1506921874.697619:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203b78ba20. 00000400:00000200:13.0:1506921874.697620:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881dc4adf100 00000400:00000010:13.0:1506921874.697622:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881dc4adf100. 00000100:00000001:13.0:1506921874.697623:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:13.0:1506921874.697625:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:13.0:1506921874.697626:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:13.0:1506921874.697627:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:13.0:1506921874.697629:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:13.0:1506921874.697631:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:13.0:1506921874.697634:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff882036e44400 time=69 v=31 (31 31 31 31) 00000100:00001000:13.0:1506921874.697636:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff882036e443c8 time=69 v=1 (1 1 1 1) 00000100:00000001:13.0:1506921874.697638:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:13.0:1506921874.697639:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:13.0:1506921874.697641:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:13.0:1506921874.697642:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.697644:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:13.0:1506921874.697645:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST000d-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:13.0:1506921874.697646:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:13.0:1506921874.697647:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:13.0:1506921874.697650:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881cd2f6ef00 x1580016666301584/t0(0) o19->oak-OST000d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:13.0:1506921874.697655:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:13.0:1506921874.697659:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:13.0:1506921874.697660:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:13.0:1506921874.697662:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881cd2f6ef00 x1580016666301584/t0(0) o19->oak-OST000d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:13.0:1506921874.697668:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301584:10.0.2.102@o2ib5:19 00000100:00000001:13.0:1506921874.697671:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:13.0:1506921874.697673:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.697674:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:13.0:1506921874.697675:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:13.0:1506921874.697677:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881cd2f6ef00 x1580016666301584/t0(0) o19->oak-OST000d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:13.0:1506921874.697682:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:13.0:1506921874.697683:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff88203830f2c0. 00000100:00000001:13.0:1506921874.697685:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:13.0:1506921874.697685:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.697687:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:13.0:1506921874.697689:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881cd2f6ef00 x1580016666301584/t0(0) o19->oak-OST000d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:13.0:1506921874.697693:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:13.0:1506921874.697694:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:13.0:1506921874.697695:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff882033d7c400. 02000000:00000001:13.0:1506921874.697696:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:13.0:1506921874.697697:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:13.0:1506921874.697698:0:38877:0:(genops.c:1026:class_import_put()) import ffff882036e44000 refcount=4 obd=oak-OST000d-osc-ffff88203bc47000 00000020:00000001:13.0:1506921874.697700:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:13.0:1506921874.697701:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff88203bd1e000. 02000000:00000001:13.0:1506921874.697703:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:13.0:1506921874.697704:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:13.0:1506921874.697704:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881cd2f6ef00. 00000100:00000001:13.0:1506921874.697706:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:13.0:1506921874.697707:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:13.0:1506921874.697708:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:13.0:1506921874.697710:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:13.0:1506921874.697711:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:13.0:1506921874.697712:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:13.0:1506921874.697779:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881cd2f6ef00. 00000020:00000040:13.0:1506921874.697781:0:38877:0:(genops.c:1012:class_import_get()) import ffff882036e46000 refcount=5 obd=oak-OST000e-osc-ffff88203bc47000 00000100:00000001:13.0:1506921874.697783:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:13.0:1506921874.697784:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:13.0:1506921874.697785:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:13.0:1506921874.697787:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff88203bd1e000. 00000100:00000001:13.0:1506921874.697789:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.697791:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:13.0:1506921874.697792:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:13.0:1506921874.697793:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff88203830f2c0. 00000100:00000001:13.0:1506921874.697794:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612270695903936 : -131803013647680 : ffff88203830f2c0) 00000100:00000001:13.0:1506921874.697796:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:13.0:1506921874.697797:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612256112701184 : -131817596850432 : ffff881cd2f6ef00) 00000020:00000001:13.0:1506921874.697798:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:13.0:1506921874.697800:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:13.0:1506921874.697808:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:13.0:1506921874.697811:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:13.0:1506921874.697812:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:13.0:1506921874.697814:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:13.0:1506921874.697814:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:13.0:1506921874.697816:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881cd2f6ef00 x1580016666301600/t0(0) o19->oak-OST000e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.697821:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:13.0:1506921874.697822:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:13.0:1506921874.697824:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:13.0:1506921874.697825:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:13.0:1506921874.697827:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301600:10.0.2.101@o2ib5:19 00000100:00000001:13.0:1506921874.697829:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:13.0:1506921874.697830:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:13.0:1506921874.697831:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:13.0:1506921874.697832:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:13.0:1506921874.697833:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff882033d7c400. 02000000:00000001:13.0:1506921874.697835:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:13.0:1506921874.697837:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203b78ba20. 00000400:00000010:13.0:1506921874.697839:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881dc4adf100. 00000100:00000200:13.0:1506921874.697842:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301600, portal 4 00000100:00000001:13.0:1506921874.697843:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:13.0:1506921874.697844:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612256112701184 : -131817596850432 : ffff881cd2f6ef00) 00000100:00000040:13.0:1506921874.697847:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881cd2f6ef00 x1580016666301600/t0(0) o19->oak-OST000e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.697851:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:13.0:1506921874.697853:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:13.0:1506921874.697854:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881dc4adf080. 00000100:00000200:13.0:1506921874.697856:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301600, offset 0 00000400:00000010:13.0:1506921874.697858:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88203bd1b600 (tot 59423868). 00000400:00000200:13.0:1506921874.697861:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:13.0:1506921874.697870:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:13.0:1506921874.697874:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:13.0:1506921874.697878:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:13.0:1506921874.697880:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:13.0:1506921874.697882:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921874.697885:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:13.0:1506921874.697886:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.697887:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:13.0:1506921874.697889:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.697890:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:13.0:1506921874.697891:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000800:00000200:2.2:1506921874.697891:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000100:00100000:13.0:1506921874.697892:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff88203830f2c0 going to sleep for 43 seconds 00000100:00000001:13.0:1506921874.697894:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:13.0:1506921874.697895:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.697896:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:13.0:1506921874.697897:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:5.0:1506921874.697930:0:3865:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:5.0:1506921874.697938:0:3865:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:5.0:1506921874.697941:0:3865:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:5.0:1506921874.697944:0:3865:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881cd2f6ef00 x1580016666301600/t0(0) o19->oak-OST000e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:11.0:1506921874.697950:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000001:5.0:1506921874.697952:0:3865:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:5.0:1506921874.697954:0:3865:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881cd2f6ef00 x1580016666301600/t0(0) o19->oak-OST000e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921874.697960:0:3865:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:5.0:1506921874.697962:0:3865:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:5.0:1506921874.697963:0:3865:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881dc4adf080 00000400:00000010:5.0:1506921874.697964:0:3865:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881dc4adf080. 00000400:00000010:5.0:1506921874.697967:0:3865:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88203bd1b600 (tot 59423468). 00000800:00000200:5.0:1506921874.697970:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.2:1506921874.698023:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921874.698041:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.698047:0:3861:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921874.698053:0:3861:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921874.698056:0:3861:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.698062:0:3861:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c2254a0 00000400:00000200:16.0:1506921874.698067:0:3861:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a0e4d [1] + 192 00000100:00000001:16.0:1506921874.698072:0:3861:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.698075:0:3861:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881cd2f6ef00 x1580016666301600/t0(0) o19->oak-OST000e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921874.698078:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921874.698083:0:3861:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881cd2f6ef00 x1580016666301600/t0(0) o19->oak-OST000e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.698092:0:3861:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.698094:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.698096:0:3861:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.698098:0:3861:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:13.0:1506921874.698098:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:13.0:1506921874.698100:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:13.0:1506921874.698101:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000800:00000200:16.0:1506921874.698102:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:13.0:1506921874.698102:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:13.0:1506921874.698105:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881cd2f6ef00 x1580016666301600/t0(0) o19->oak-OST000e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:13.0:1506921874.698110:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881cd2f6ef00 x1580016666301600/t0(0) o19->oak-OST000e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.698120:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:13.0:1506921874.698122:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203b78ba20. 00000400:00000200:13.0:1506921874.698123:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881dc4adf100 00000400:00000010:13.0:1506921874.698124:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881dc4adf100. 00000100:00000001:13.0:1506921874.698125:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:13.0:1506921874.698127:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:13.0:1506921874.698127:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:13.0:1506921874.698128:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:13.0:1506921874.698129:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:13.0:1506921874.698131:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:13.0:1506921874.698133:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff882036e46400 time=69 v=31 (31 1 1 31) 00000100:00001000:13.0:1506921874.698135:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff882036e463c8 time=69 v=1 (1 1 1 1) 00000100:00000001:13.0:1506921874.698137:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:13.0:1506921874.698138:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:13.0:1506921874.698139:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:13.0:1506921874.698140:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.698141:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:13.0:1506921874.698142:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST000e-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:13.0:1506921874.698143:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:13.0:1506921874.698144:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:13.0:1506921874.698146:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881cd2f6ef00 x1580016666301600/t0(0) o19->oak-OST000e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:13.0:1506921874.698150:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:13.0:1506921874.698152:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:13.0:1506921874.698152:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:13.0:1506921874.698154:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881cd2f6ef00 x1580016666301600/t0(0) o19->oak-OST000e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:13.0:1506921874.698159:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301600:10.0.2.101@o2ib5:19 00000100:00000001:13.0:1506921874.698162:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:13.0:1506921874.698164:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.698165:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:13.0:1506921874.698166:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:13.0:1506921874.698168:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881cd2f6ef00 x1580016666301600/t0(0) o19->oak-OST000e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:13.0:1506921874.698176:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:13.0:1506921874.698177:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff88203830f2c0. 00000100:00000001:13.0:1506921874.698179:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:13.0:1506921874.698179:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.698181:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:13.0:1506921874.698182:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881cd2f6ef00 x1580016666301600/t0(0) o19->oak-OST000e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:13.0:1506921874.698186:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:13.0:1506921874.698187:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:13.0:1506921874.698188:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff882033d7c400. 02000000:00000001:13.0:1506921874.698190:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:13.0:1506921874.698190:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:13.0:1506921874.698191:0:38877:0:(genops.c:1026:class_import_put()) import ffff882036e46000 refcount=4 obd=oak-OST000e-osc-ffff88203bc47000 00000020:00000001:13.0:1506921874.698193:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:13.0:1506921874.698193:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff88203bd1e000. 02000000:00000001:13.0:1506921874.698195:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:13.0:1506921874.698195:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:13.0:1506921874.698196:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881cd2f6ef00. 00000100:00000001:13.0:1506921874.698198:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:13.0:1506921874.698198:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:13.0:1506921874.698200:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:13.0:1506921874.698201:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:13.0:1506921874.698202:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:13.0:1506921874.698203:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:13.0:1506921874.698204:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881cd2f6ef00. 00000020:00000040:13.0:1506921874.698205:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203b61d000 refcount=5 obd=oak-OST000f-osc-ffff88203bc47000 00000100:00000001:13.0:1506921874.698207:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:13.0:1506921874.698207:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:13.0:1506921874.698208:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:13.0:1506921874.698210:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff88203bd1e000. 00000100:00000001:13.0:1506921874.698211:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.698213:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:13.0:1506921874.698213:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:13.0:1506921874.698215:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff88203830f2c0. 00000100:00000001:13.0:1506921874.698216:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612270695903936 : -131803013647680 : ffff88203830f2c0) 00000100:00000001:13.0:1506921874.698217:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:13.0:1506921874.698218:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612256112701184 : -131817596850432 : ffff881cd2f6ef00) 00000020:00000001:13.0:1506921874.698219:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:13.0:1506921874.698220:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:13.0:1506921874.698227:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:13.0:1506921874.698229:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:13.0:1506921874.698230:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:13.0:1506921874.698232:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:13.0:1506921874.698232:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:13.0:1506921874.698234:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881cd2f6ef00 x1580016666301616/t0(0) o19->oak-OST000f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.698238:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:13.0:1506921874.698239:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:13.0:1506921874.698240:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:13.0:1506921874.698241:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:13.0:1506921874.698243:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301616:10.0.2.102@o2ib5:19 00000100:00000001:13.0:1506921874.698245:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:13.0:1506921874.698246:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:13.0:1506921874.698247:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:13.0:1506921874.698248:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:13.0:1506921874.698249:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff882033d7c400. 02000000:00000001:13.0:1506921874.698250:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:13.0:1506921874.698251:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203b78ba20. 00000400:00000010:13.0:1506921874.698252:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881dc4adf100. 00000100:00000200:13.0:1506921874.698254:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301616, portal 4 00000100:00000001:13.0:1506921874.698255:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:13.0:1506921874.698256:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612256112701184 : -131817596850432 : ffff881cd2f6ef00) 00000100:00000040:13.0:1506921874.698259:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881cd2f6ef00 x1580016666301616/t0(0) o19->oak-OST000f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.698263:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:13.0:1506921874.698264:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:13.0:1506921874.698265:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881dc4adeb00. 00000100:00000200:13.0:1506921874.698267:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301616, offset 0 00000400:00000010:13.0:1506921874.698269:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88203bd1d000 (tot 59423868). 00000400:00000200:13.0:1506921874.698271:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:13.0:1506921874.698279:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:13.0:1506921874.698282:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:13.0:1506921874.698287:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:13.0:1506921874.698289:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:13.0:1506921874.698290:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:13.0:1506921874.698293:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:13.0:1506921874.698294:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.698295:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:13.0:1506921874.698296:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.698297:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:16.2:1506921874.698298:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:13.0:1506921874.698298:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:13.0:1506921874.698299:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff88203830f2c0 going to sleep for 43 seconds 00000100:00000001:13.0:1506921874.698301:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:13.0:1506921874.698302:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.698303:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:13.0:1506921874.698303:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.0:1506921874.698305:0:3858:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921874.698309:0:3858:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921874.698312:0:3858:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921874.698314:0:3858:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881cd2f6ef00 x1580016666301616/t0(0) o19->oak-OST000f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921874.698317:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:16.0:1506921874.698320:0:3858:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921874.698322:0:3858:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881cd2f6ef00 x1580016666301616/t0(0) o19->oak-OST000f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.698326:0:3858:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.698328:0:3858:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921874.698329:0:3858:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881dc4adeb00 00000400:00000010:16.0:1506921874.698331:0:3858:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881dc4adeb00. 00000400:00000010:16.0:1506921874.698333:0:3858:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88203bd1d000 (tot 59423468). 00000800:00000200:16.0:1506921874.698335:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:16.2:1506921874.698440:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921874.698446:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.698449:0:3859:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000800:00000200:18.0:1506921874.698453:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000200:16.0:1506921874.698454:0:3859:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:16.0:1506921874.698456:0:3859:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.698459:0:3859:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c2254b0 00000400:00000200:16.0:1506921874.698462:0:3859:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1c81 [1] + 192 00000100:00000001:16.0:1506921874.698465:0:3859:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.698467:0:3859:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881cd2f6ef00 x1580016666301616/t0(0) o19->oak-OST000f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:16.0:1506921874.698472:0:3859:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881cd2f6ef00 x1580016666301616/t0(0) o19->oak-OST000f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.698479:0:3859:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.698480:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.698481:0:3859:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.698482:0:3859:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921874.698484:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:13.0:1506921874.698484:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:13.0:1506921874.698486:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:13.0:1506921874.698487:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:13.0:1506921874.698488:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:13.0:1506921874.698490:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881cd2f6ef00 x1580016666301616/t0(0) o19->oak-OST000f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:13.0:1506921874.698496:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881cd2f6ef00 x1580016666301616/t0(0) o19->oak-OST000f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.698500:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:13.0:1506921874.698502:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203b78ba20. 00000400:00000200:13.0:1506921874.698503:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881dc4adf100 00000400:00000010:13.0:1506921874.698504:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881dc4adf100. 00000100:00000001:13.0:1506921874.698505:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:13.0:1506921874.698506:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:13.0:1506921874.698506:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:13.0:1506921874.698507:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:13.0:1506921874.698508:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:13.0:1506921874.698510:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:13.0:1506921874.698512:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88203b61d400 time=69 v=31 (31 31 31 31) 00000100:00001000:13.0:1506921874.698514:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203b61d3c8 time=69 v=1 (1 1 1 1) 00000100:00000001:13.0:1506921874.698516:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:13.0:1506921874.698517:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:13.0:1506921874.698518:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:13.0:1506921874.698519:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.698520:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:13.0:1506921874.698521:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST000f-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:13.0:1506921874.698522:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:13.0:1506921874.698523:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:13.0:1506921874.698524:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881cd2f6ef00 x1580016666301616/t0(0) o19->oak-OST000f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:13.0:1506921874.698529:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:13.0:1506921874.698530:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:13.0:1506921874.698531:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:13.0:1506921874.698533:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881cd2f6ef00 x1580016666301616/t0(0) o19->oak-OST000f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:13.0:1506921874.698538:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301616:10.0.2.102@o2ib5:19 00000100:00000001:13.0:1506921874.698540:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:13.0:1506921874.698542:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.698543:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:13.0:1506921874.698544:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:13.0:1506921874.698546:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881cd2f6ef00 x1580016666301616/t0(0) o19->oak-OST000f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:13.0:1506921874.698550:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:13.0:1506921874.698551:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff88203830f2c0. 00000100:00000001:13.0:1506921874.698552:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:13.0:1506921874.698553:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.698554:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:13.0:1506921874.698556:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881cd2f6ef00 x1580016666301616/t0(0) o19->oak-OST000f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:13.0:1506921874.698560:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:13.0:1506921874.698560:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:13.0:1506921874.698561:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff882033d7c400. 02000000:00000001:13.0:1506921874.698563:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:13.0:1506921874.698563:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:13.0:1506921874.698564:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203b61d000 refcount=4 obd=oak-OST000f-osc-ffff88203bc47000 00000020:00000001:13.0:1506921874.698566:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:13.0:1506921874.698567:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff88203bd1e000. 02000000:00000001:13.0:1506921874.698568:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:13.0:1506921874.698569:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:13.0:1506921874.698569:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881cd2f6ef00. 00000100:00000001:13.0:1506921874.698571:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:13.0:1506921874.698572:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:13.0:1506921874.698573:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:13.0:1506921874.698574:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:13.0:1506921874.698575:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:13.0:1506921874.698576:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:13.0:1506921874.698577:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881cd2f6ef00. 00000020:00000040:13.0:1506921874.698578:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203b61c800 refcount=5 obd=oak-OST0010-osc-ffff88203bc47000 00000100:00000001:13.0:1506921874.698579:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:13.0:1506921874.698580:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:13.0:1506921874.698581:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:13.0:1506921874.698582:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff88203bd1e000. 00000100:00000001:13.0:1506921874.698583:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.698585:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:13.0:1506921874.698585:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:13.0:1506921874.698586:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff88203830f2c0. 00000100:00000001:13.0:1506921874.698587:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612270695903936 : -131803013647680 : ffff88203830f2c0) 00000100:00000001:13.0:1506921874.698589:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:13.0:1506921874.698589:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612256112701184 : -131817596850432 : ffff881cd2f6ef00) 00000020:00000001:13.0:1506921874.698591:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:13.0:1506921874.698592:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:13.0:1506921874.698598:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:13.0:1506921874.698600:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:13.0:1506921874.698601:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:13.0:1506921874.698602:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:13.0:1506921874.698602:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:13.0:1506921874.698604:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881cd2f6ef00 x1580016666301632/t0(0) o19->oak-OST0010-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.698608:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:13.0:1506921874.698609:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:13.0:1506921874.698611:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:13.0:1506921874.698611:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:13.0:1506921874.698613:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301632:10.0.2.101@o2ib5:19 00000100:00000001:13.0:1506921874.698615:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:13.0:1506921874.698616:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:13.0:1506921874.698617:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:13.0:1506921874.698618:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:13.0:1506921874.698619:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff882033d7c400. 02000000:00000001:13.0:1506921874.698620:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:13.0:1506921874.698621:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203b78ba20. 00000400:00000010:13.0:1506921874.698622:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881dc4adf100. 00000100:00000200:13.0:1506921874.698624:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301632, portal 4 00000100:00000001:13.0:1506921874.698625:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:13.0:1506921874.698626:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612256112701184 : -131817596850432 : ffff881cd2f6ef00) 00000100:00000040:13.0:1506921874.698629:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881cd2f6ef00 x1580016666301632/t0(0) o19->oak-OST0010-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.698632:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:13.0:1506921874.698634:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:13.0:1506921874.698635:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881dc4adea80. 00000100:00000200:13.0:1506921874.698636:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301632, offset 0 00000400:00000010:13.0:1506921874.698638:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88203bd1c400 (tot 59423868). 00000400:00000200:13.0:1506921874.698641:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:13.0:1506921874.698647:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:13.0:1506921874.698650:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:13.0:1506921874.698653:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:13.0:1506921874.698654:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:13.0:1506921874.698655:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921874.698658:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:13.0:1506921874.698658:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.698660:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:13.0:1506921874.698661:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.698662:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:13.0:1506921874.698662:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:13.0:1506921874.698663:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff88203830f2c0 going to sleep for 43 seconds 00000100:00000001:13.0:1506921874.698665:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:13.0:1506921874.698666:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.698667:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:13.0:1506921874.698668:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921874.698691:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921874.698705:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:13.0:1506921874.698710:0:3863:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:13.0:1506921874.698712:0:3863:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:13.0:1506921874.698713:0:3863:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881cd2f6ef00 x1580016666301632/t0(0) o19->oak-OST0010-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.698723:0:3863:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:13.0:1506921874.698725:0:3863:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881cd2f6ef00 x1580016666301632/t0(0) o19->oak-OST0010-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.698729:0:3863:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.698730:0:3863:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:13.0:1506921874.698731:0:3863:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881dc4adea80 00000400:00000010:13.0:1506921874.698732:0:3863:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881dc4adea80. 00000400:00000010:13.0:1506921874.698734:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88203bd1c400 (tot 59423468). 00000800:00000200:13.0:1506921874.698736:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:13.0:1506921874.698738:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:2.2:1506921874.698807:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:11.0:1506921874.698817:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921874.698821:0:3862:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.42@o2ib4 00000400:00000200:11.0:1506921874.698825:0:3862:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:11.0:1506921874.698827:0:3862:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e15800 (tot 59423868). 00000400:00000200:11.0:1506921874.698831:0:3862:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c2254c0 00000400:00000200:11.0:1506921874.698834:0:3862:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a0e65 [1] + 192 00000100:00000001:11.0:1506921874.698837:0:3862:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921874.698839:0:3862:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881cd2f6ef00 x1580016666301632/t0(0) o19->oak-OST0010-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:11.0:1506921874.698844:0:3862:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881cd2f6ef00 x1580016666301632/t0(0) o19->oak-OST0010-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.698850:0:3862:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921874.698852:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e15800 (tot 59423468). 00000800:00000200:5.0:1506921874.698853:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000001:13.0:1506921874.698854:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:11.0:1506921874.698854:0:3862:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (22)++ 00000400:00000001:13.0:1506921874.698855:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000800:00000200:11.0:1506921874.698855:0:3862:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (23)-- 00000400:00000001:13.0:1506921874.698856:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:13.0:1506921874.698857:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000800:00000200:11.0:1506921874.698858:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000200:13.0:1506921874.698859:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881cd2f6ef00 x1580016666301632/t0(0) o19->oak-OST0010-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:13.0:1506921874.698864:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881cd2f6ef00 x1580016666301632/t0(0) o19->oak-OST0010-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.698868:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:13.0:1506921874.698869:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203b78ba20. 00000400:00000200:13.0:1506921874.698870:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881dc4adf100 00000400:00000010:13.0:1506921874.698870:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881dc4adf100. 00000100:00000001:13.0:1506921874.698871:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:13.0:1506921874.698873:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:13.0:1506921874.698873:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:13.0:1506921874.698874:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:13.0:1506921874.698875:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:13.0:1506921874.698877:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:13.0:1506921874.698879:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88203b61cc00 time=69 v=31 (31 31 1 31) 00000100:00001000:13.0:1506921874.698881:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203b61cbc8 time=69 v=1 (1 1 1 1) 00000100:00000001:13.0:1506921874.698882:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:13.0:1506921874.698883:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:13.0:1506921874.698884:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:13.0:1506921874.698885:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.698886:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:13.0:1506921874.698887:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0010-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:13.0:1506921874.698888:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:13.0:1506921874.698889:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:13.0:1506921874.698891:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881cd2f6ef00 x1580016666301632/t0(0) o19->oak-OST0010-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:13.0:1506921874.698895:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:13.0:1506921874.698896:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:13.0:1506921874.698897:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:13.0:1506921874.698899:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881cd2f6ef00 x1580016666301632/t0(0) o19->oak-OST0010-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:13.0:1506921874.698903:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301632:10.0.2.101@o2ib5:19 00000100:00000001:13.0:1506921874.698905:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:13.0:1506921874.698907:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.698908:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:13.0:1506921874.698909:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:13.0:1506921874.698910:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881cd2f6ef00 x1580016666301632/t0(0) o19->oak-OST0010-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:13.0:1506921874.698914:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:13.0:1506921874.698916:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff88203830f2c0. 00000100:00000001:13.0:1506921874.698917:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:13.0:1506921874.698918:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.698919:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:13.0:1506921874.698921:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881cd2f6ef00 x1580016666301632/t0(0) o19->oak-OST0010-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:13.0:1506921874.698925:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:13.0:1506921874.698925:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:13.0:1506921874.698927:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff882033d7c400. 02000000:00000001:13.0:1506921874.698928:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:13.0:1506921874.698929:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:13.0:1506921874.698929:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203b61c800 refcount=4 obd=oak-OST0010-osc-ffff88203bc47000 00000020:00000001:13.0:1506921874.698931:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:13.0:1506921874.698931:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff88203bd1e000. 02000000:00000001:13.0:1506921874.698933:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:13.0:1506921874.698933:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:13.0:1506921874.698934:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881cd2f6ef00. 00000100:00000001:13.0:1506921874.698935:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:13.0:1506921874.698936:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:13.0:1506921874.698937:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:13.0:1506921874.698938:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:13.0:1506921874.698939:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:13.0:1506921874.698940:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:13.0:1506921874.698941:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881cd2f6ef00. 00000020:00000040:13.0:1506921874.698942:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203b619000 refcount=5 obd=oak-OST0011-osc-ffff88203bc47000 00000100:00000001:13.0:1506921874.698944:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:13.0:1506921874.698944:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:13.0:1506921874.698945:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:13.0:1506921874.698946:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff88203bd1e000. 00000100:00000001:13.0:1506921874.698948:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.698949:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:13.0:1506921874.698950:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:13.0:1506921874.698951:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff88203830f2c0. 00000100:00000001:13.0:1506921874.698952:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612270695903936 : -131803013647680 : ffff88203830f2c0) 00000100:00000001:13.0:1506921874.698953:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:13.0:1506921874.698954:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612256112701184 : -131817596850432 : ffff881cd2f6ef00) 00000020:00000001:13.0:1506921874.698955:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:13.0:1506921874.698957:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:13.0:1506921874.698963:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:13.0:1506921874.698965:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:13.0:1506921874.698966:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:13.0:1506921874.698967:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:13.0:1506921874.698968:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:13.0:1506921874.698969:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881cd2f6ef00 x1580016666301648/t0(0) o19->oak-OST0011-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.698974:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:13.0:1506921874.698975:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:13.0:1506921874.698976:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:13.0:1506921874.698977:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:13.0:1506921874.698978:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301648:10.0.2.102@o2ib5:19 00000100:00000001:13.0:1506921874.698980:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:13.0:1506921874.698981:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:13.0:1506921874.698982:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:13.0:1506921874.698983:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:13.0:1506921874.698984:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff882033d7c400. 02000000:00000001:13.0:1506921874.698985:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:13.0:1506921874.698987:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203b78ba20. 00000400:00000010:13.0:1506921874.698988:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881dc4adf100. 00000100:00000200:13.0:1506921874.698990:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301648, portal 4 00000100:00000001:13.0:1506921874.698991:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:13.0:1506921874.698991:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612256112701184 : -131817596850432 : ffff881cd2f6ef00) 00000100:00000040:13.0:1506921874.698994:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881cd2f6ef00 x1580016666301648/t0(0) o19->oak-OST0011-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.698998:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:13.0:1506921874.698999:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:13.0:1506921874.699000:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881dc4adea80. 00000100:00000200:13.0:1506921874.699001:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301648, offset 0 00000400:00000010:13.0:1506921874.699003:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88203bd1c400 (tot 59423868). 00000400:00000200:13.0:1506921874.699006:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:13.0:1506921874.699012:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:13.0:1506921874.699015:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:13.0:1506921874.699019:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:13.0:1506921874.699020:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:13.0:1506921874.699022:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:13.0:1506921874.699024:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:13.0:1506921874.699025:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.699026:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:13.0:1506921874.699028:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.699029:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:13.0:1506921874.699030:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:13.0:1506921874.699031:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff88203830f2c0 going to sleep for 43 seconds 00000100:00000001:13.0:1506921874.699033:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:13.0:1506921874.699033:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.699035:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:13.0:1506921874.699035:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921874.699058:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.699076:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921874.699081:0:3860:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921874.699086:0:3860:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921874.699091:0:3860:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881cd2f6ef00 x1580016666301648/t0(0) o19->oak-OST0011-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.699098:0:3860:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921874.699101:0:3860:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881cd2f6ef00 x1580016666301648/t0(0) o19->oak-OST0011-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.699106:0:3860:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.699108:0:3860:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921874.699110:0:3860:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881dc4adea80 00000400:00000010:16.0:1506921874.699111:0:3860:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881dc4adea80. 00000800:00000200:18.0:1506921874.699113:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000010:16.0:1506921874.699115:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88203bd1c400 (tot 59423468). 00000800:00000200:16.0:1506921874.699118:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921874.699231:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:13.0:1506921874.699245:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921874.699249:0:3863:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.42@o2ib4 00000400:00000200:13.0:1506921874.699254:0:3863:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:13.0:1506921874.699256:0:3863:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88203bd18400 (tot 59423868). 00000400:00000200:13.0:1506921874.699259:0:3863:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c2254d0 00000400:00000200:13.0:1506921874.699263:0:3863:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1c89 [1] + 192 00000100:00000001:13.0:1506921874.699267:0:3863:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:13.0:1506921874.699269:0:3863:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881cd2f6ef00 x1580016666301648/t0(0) o19->oak-OST0011-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:13.0:1506921874.699274:0:3863:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881cd2f6ef00 x1580016666301648/t0(0) o19->oak-OST0011-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.699281:0:3863:0:(events.c:171:reply_in_callback()) Process leaving 00000800:00000200:1.0:1506921874.699281:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000400:00000010:13.0:1506921874.699283:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88203bd18400 (tot 59423468). 00000800:00000200:13.0:1506921874.699285:0:3863:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921874.699286:0:3863:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:13.0:1506921874.699288:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921874.699315:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921874.699320:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921874.699321:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.699322:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.699327:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881cd2f6ef00 x1580016666301648/t0(0) o19->oak-OST0011-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.699335:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881cd2f6ef00 x1580016666301648/t0(0) o19->oak-OST0011-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.699341:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.699343:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203b78ba20. 00000400:00000200:15.0:1506921874.699345:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881dc4adf100 00000400:00000010:15.0:1506921874.699346:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881dc4adf100. 00000100:00000001:15.0:1506921874.699348:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.699350:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.699351:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.699352:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.699355:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.699357:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.699361:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88203b619400 time=69 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921874.699363:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203b6193c8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.699365:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.699366:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.699368:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.699369:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.699371:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.699372:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0011-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.699373:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.699374:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.699376:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881cd2f6ef00 x1580016666301648/t0(0) o19->oak-OST0011-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.699382:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.699384:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.699385:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.699387:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881cd2f6ef00 x1580016666301648/t0(0) o19->oak-OST0011-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.699392:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301648:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.699396:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.699397:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.699399:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.699400:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.699402:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881cd2f6ef00 x1580016666301648/t0(0) o19->oak-OST0011-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.699406:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.699408:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff88203830f2c0. 00000100:00000001:15.0:1506921874.699409:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.699410:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.699412:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.699414:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881cd2f6ef00 x1580016666301648/t0(0) o19->oak-OST0011-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.699418:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.699419:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.699421:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff882033d7c400. 02000000:00000001:15.0:1506921874.699423:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.699424:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.699425:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203b619000 refcount=4 obd=oak-OST0011-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.699427:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.699428:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff88203bd1e000. 02000000:00000001:15.0:1506921874.699429:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.699430:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.699431:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881cd2f6ef00. 00000100:00000001:15.0:1506921874.699433:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.699434:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.699435:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.699437:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.699438:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.699440:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.699441:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.699443:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203b61a000 refcount=5 obd=oak-OST0012-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.699445:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.699446:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.699447:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.699449:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.699451:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.699453:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.699454:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.699456:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.699457:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.699459:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.699460:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.699461:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.699463:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.699473:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.699476:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.699477:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.699478:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.699479:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.699482:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666301664/t0(0) o19->oak-OST0012-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.699487:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.699488:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.699490:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.699491:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.699493:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301664:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.699495:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.699496:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.699497:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.699499:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.699501:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.699502:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.699504:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.699506:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.699508:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301664, portal 4 00000100:00000001:15.0:1506921874.699510:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.699511:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.699514:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666301664/t0(0) o19->oak-OST0012-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.699518:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.699520:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921874.699521:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fac800. 00000100:00000200:15.0:1506921874.699522:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301664, offset 0 00000400:00000010:15.0:1506921874.699524:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451ee00 (tot 59423868). 00000400:00000200:15.0:1506921874.699527:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921874.699536:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921874.699540:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921874.699545:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.699547:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921874.699548:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921874.699552:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921874.699553:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.699555:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.699557:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.699559:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921874.699560:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.699561:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.699562:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.699564:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.699565:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.699566:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921874.699591:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921874.699634:0:3865:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:5.0:1506921874.699641:0:3865:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:5.0:1506921874.699644:0:3865:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:5.0:1506921874.699648:0:3865:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666301664/t0(0) o19->oak-OST0012-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921874.699655:0:3865:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:5.0:1506921874.699658:0:3865:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666301664/t0(0) o19->oak-OST0012-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:11.0:1506921874.699659:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:11.0:1506921874.699663:0:3862:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.42@o2ib4 00000100:00000001:5.0:1506921874.699663:0:3865:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:5.0:1506921874.699665:0:3865:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:5.0:1506921874.699666:0:3865:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fac800 00000400:00000200:11.0:1506921874.699667:0:3862:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:5.0:1506921874.699667:0:3865:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fac800. 00000400:00000010:11.0:1506921874.699668:0:3862:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e15800 (tot 59424268). 00000400:00000010:5.0:1506921874.699672:0:3865:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451ee00 (tot 59423868). 00000800:00000200:5.0:1506921874.699677:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (23)-- 00000400:00000200:11.0:1506921874.699678:0:3862:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c2254e0 00000800:00000200:5.0:1506921874.699679:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000400:00000200:11.0:1506921874.699680:0:3862:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a0e7d [1] + 192 00000100:00000001:11.0:1506921874.699683:0:3862:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921874.699685:0:3862:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666301664/t0(0) o19->oak-OST0012-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:11.0:1506921874.699690:0:3862:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666301664/t0(0) o19->oak-OST0012-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.699695:0:3862:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921874.699697:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e15800 (tot 59423468). 00000800:00000200:11.0:1506921874.699698:0:3862:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000100:00000001:15.0:1506921874.699700:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:11.0:1506921874.699700:0:3862:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000400:00000001:15.0:1506921874.699701:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000800:00000200:11.0:1506921874.699701:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000400:00000001:15.0:1506921874.699702:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.699703:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.699705:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666301664/t0(0) o19->oak-OST0012-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.699711:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666301664/t0(0) o19->oak-OST0012-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.699715:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.699737:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.699739:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.699741:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.699742:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.699744:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.699745:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.699746:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.699747:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.699749:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.699751:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88203b61a400 time=69 v=31 (31 1 1 31) 00000100:00001000:15.0:1506921874.699753:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203b61a3c8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.699755:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.699756:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.699757:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.699759:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.699760:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.699761:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0012-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.699762:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.699763:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.699765:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666301664/t0(0) o19->oak-OST0012-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.699770:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.699771:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.699772:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.699774:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666301664/t0(0) o19->oak-OST0012-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.699779:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301664:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.699781:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.699783:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.699784:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.699785:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.699786:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666301664/t0(0) o19->oak-OST0012-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.699790:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.699792:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.699793:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.699794:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.699796:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.699797:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666301664/t0(0) o19->oak-OST0012-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.699801:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.699802:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.699803:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.699805:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.699805:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.699806:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203b61a000 refcount=4 obd=oak-OST0012-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.699807:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.699808:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.699809:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.699810:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.699811:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.699813:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.699813:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.699814:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.699816:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.699817:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.699818:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.699819:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.699821:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203c7d0000 refcount=5 obd=oak-OST0013-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.699822:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.699823:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.699824:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.699826:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.699827:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.699829:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.699830:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.699831:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.699832:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.699833:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.699834:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.699836:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.699837:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.699844:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.699846:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.699847:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.699849:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.699849:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.699851:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666301680/t0(0) o19->oak-OST0013-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.699855:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.699856:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.699857:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.699858:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.699860:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301680:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.699862:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.699863:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.699864:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.699865:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.699866:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.699868:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.699869:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.699871:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.699872:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301680, portal 4 00000100:00000001:15.0:1506921874.699874:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.699874:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.699877:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666301680/t0(0) o19->oak-OST0013-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.699881:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.699882:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921874.699883:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fac700. 00000100:00000200:15.0:1506921874.699885:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301680, offset 0 00000400:00000010:15.0:1506921874.699886:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451a400 (tot 59423868). 00000400:00000200:15.0:1506921874.699889:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921874.699896:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921874.699899:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921874.699903:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.699905:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921874.699907:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921874.699909:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921874.699910:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.699911:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.699913:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.699914:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921874.699914:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.699916:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.699917:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.699918:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.699919:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.699920:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921874.699941:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.699959:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921874.699963:0:3861:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921874.699967:0:3861:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921874.699971:0:3861:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666301680/t0(0) o19->oak-OST0013-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.699978:0:3861:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921874.699981:0:3861:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666301680/t0(0) o19->oak-OST0013-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.699986:0:3861:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.699988:0:3861:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921874.699989:0:3861:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fac700 00000400:00000010:16.0:1506921874.699991:0:3861:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fac700. 00000400:00000010:16.0:1506921874.699994:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451a400 (tot 59423468). 00000800:00000200:18.0:1506921874.699997:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921874.700002:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:16.2:1506921874.700081:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921874.700088:0:3858:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.700093:0:3858:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.41@o2ib4 00000800:00000200:18.0:1506921874.700096:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000200:16.0:1506921874.700098:0:3858:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:16.0:1506921874.700100:0:3858:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.700105:0:3858:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c2254f0 00000400:00000200:16.0:1506921874.700110:0:3858:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1c91 [1] + 192 00000100:00000001:16.0:1506921874.700115:0:3858:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.700116:0:3858:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666301680/t0(0) o19->oak-OST0013-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:16.0:1506921874.700125:0:3858:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666301680/t0(0) o19->oak-OST0013-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.700132:0:3858:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.700133:0:3858:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.700136:0:3858:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.700138:0:3858:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921874.700139:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:15.0:1506921874.700141:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921874.700143:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921874.700144:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.700145:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.700147:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666301680/t0(0) o19->oak-OST0013-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.700153:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666301680/t0(0) o19->oak-OST0013-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.700158:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.700159:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.700161:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.700162:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.700163:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.700164:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.700165:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.700166:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.700167:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.700168:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.700170:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88203c7d0400 time=69 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921874.700172:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203c7d03c8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.700174:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.700174:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.700176:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.700177:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.700178:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.700179:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0013-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.700180:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.700180:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.700182:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666301680/t0(0) o19->oak-OST0013-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.700186:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.700188:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.700189:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.700191:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666301680/t0(0) o19->oak-OST0013-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.700195:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301680:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.700198:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.700200:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.700201:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.700202:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.700204:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666301680/t0(0) o19->oak-OST0013-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.700208:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.700209:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.700211:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.700211:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.700213:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.700214:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666301680/t0(0) o19->oak-OST0013-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.700218:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.700219:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.700220:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.700221:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.700222:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.700223:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203c7d0000 refcount=4 obd=oak-OST0013-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.700224:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.700225:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.700226:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.700227:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.700228:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.700229:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.700230:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.700231:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.700232:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.700233:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.700234:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.700235:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.700236:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203c7d7000 refcount=5 obd=oak-OST0014-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.700238:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.700239:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.700239:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.700241:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.700242:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.700244:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.700244:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.700246:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.700247:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.700249:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.700249:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.700251:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.700252:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.700259:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.700261:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.700262:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.700263:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.700264:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.700265:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666301696/t0(0) o19->oak-OST0014-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.700269:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.700270:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.700271:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.700272:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.700274:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301696:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.700276:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.700277:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.700278:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.700279:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.700280:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.700281:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.700283:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.700285:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.700286:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301696, portal 4 00000100:00000001:15.0:1506921874.700288:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.700288:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.700291:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666301696/t0(0) o19->oak-OST0014-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.700294:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.700296:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921874.700297:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fadd00. 00000100:00000200:15.0:1506921874.700298:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301696, offset 0 00000400:00000010:15.0:1506921874.700301:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451ae00 (tot 59423868). 00000400:00000200:15.0:1506921874.700303:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921874.700310:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921874.700313:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921874.700316:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.700318:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921874.700319:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921874.700322:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921874.700323:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.700324:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.700325:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.700326:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921874.700327:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.700328:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.700329:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.700330:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.700331:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.700332:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921874.700357:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921874.700400:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:13.0:1506921874.700406:0:3863:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:13.0:1506921874.700409:0:3863:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:13.0:1506921874.700413:0:3863:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666301696/t0(0) o19->oak-OST0014-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.700421:0:3863:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:13.0:1506921874.700423:0:3863:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666301696/t0(0) o19->oak-OST0014-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:16.2:1506921874.700426:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000100:00000001:13.0:1506921874.700428:0:3863:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.700430:0:3863:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:13.0:1506921874.700433:0:3863:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fadd00 00000400:00000010:13.0:1506921874.700434:0:3863:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fadd00. 00000400:00000010:13.0:1506921874.700437:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451ae00 (tot 59423468). 00000800:00000200:5.0:1506921874.700439:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:16.0:1506921874.700443:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:13.0:1506921874.700443:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.0:1506921874.700449:0:3859:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921874.700454:0:3859:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921874.700456:0:3859:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.700462:0:3859:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225500 00000400:00000200:16.0:1506921874.700467:0:3859:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a0e95 [1] + 192 00000100:00000001:16.0:1506921874.700472:0:3859:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.700475:0:3859:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666301696/t0(0) o19->oak-OST0014-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921874.700479:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921874.700482:0:3859:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666301696/t0(0) o19->oak-OST0014-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.700491:0:3859:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.700493:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.700496:0:3859:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:15.0:1506921874.700497:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:16.0:1506921874.700498:0:3859:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000400:00000001:15.0:1506921874.700498:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921874.700499:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000800:00000200:16.0:1506921874.700500:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:15.0:1506921874.700500:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.700503:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666301696/t0(0) o19->oak-OST0014-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.700508:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666301696/t0(0) o19->oak-OST0014-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.700512:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.700513:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.700515:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.700516:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.700517:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.700518:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.700518:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.700519:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.700520:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.700522:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.700524:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88203c7d7400 time=69 v=31 (31 3 1 31) 00000100:00001000:15.0:1506921874.700527:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203c7d73c8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.700528:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.700529:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.700531:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.700532:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.700533:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.700534:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0014-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.700535:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.700536:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.700538:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666301696/t0(0) o19->oak-OST0014-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.700542:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.700543:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.700544:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.700546:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666301696/t0(0) o19->oak-OST0014-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.700551:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301696:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.700553:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.700555:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.700556:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.700557:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.700558:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666301696/t0(0) o19->oak-OST0014-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.700562:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.700563:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.700565:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.700565:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.700567:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.700568:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666301696/t0(0) o19->oak-OST0014-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.700572:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.700573:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.700574:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.700576:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.700577:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.700577:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203c7d7000 refcount=4 obd=oak-OST0014-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.700579:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.700580:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.700582:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.700583:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.700583:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.700585:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.700585:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.700587:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.700588:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.700589:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.700590:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.700590:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.700592:0:38877:0:(genops.c:1012:class_import_get()) import ffff88207fdf9000 refcount=5 obd=oak-OST0015-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.700593:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.700594:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.700595:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.700596:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.700598:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.700599:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.700600:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.700601:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.700602:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.700603:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.700604:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.700605:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.700606:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.700612:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.700614:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.700615:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.700616:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.700617:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.700619:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666301712/t0(0) o19->oak-OST0015-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.700628:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.700629:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.700630:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.700631:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.700633:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301712:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.700635:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.700636:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.700637:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.700638:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.700638:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.700640:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.700641:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.700642:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.700644:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301712, portal 4 00000100:00000001:15.0:1506921874.700645:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.700646:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.700648:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666301712/t0(0) o19->oak-OST0015-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.700652:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.700653:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921874.700655:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad100. 00000100:00000200:15.0:1506921874.700656:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301712, offset 0 00000400:00000010:15.0:1506921874.700657:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b04518200 (tot 59423868). 00000400:00000200:15.0:1506921874.700660:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921874.700667:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921874.700670:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921874.700674:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.700676:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921874.700678:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921874.700681:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921874.700682:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.700683:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:16.2:1506921874.700686:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:15.0:1506921874.700688:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.700690:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921874.700690:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=6 : 6 : 6) 00000100:00100000:15.0:1506921874.700691:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 6 seconds 00000100:00000001:15.0:1506921874.700692:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:16.0:1506921874.700693:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000100:00000001:15.0:1506921874.700693:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.700694:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.700695:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.0:1506921874.700697:0:3860:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921874.700700:0:3860:0:(events.c:52:request_out_callback()) Process entered 00000800:00000200:18.0:1506921874.700703:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000200:16.0:1506921874.700703:0:3860:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666301712/t0(0) o19->oak-OST0015-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.700708:0:3860:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921874.700710:0:3860:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666301712/t0(0) o19->oak-OST0015-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.700714:0:3860:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.700726:0:3860:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921874.700728:0:3860:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad100 00000400:00000010:16.0:1506921874.700730:0:3860:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad100. 00000400:00000010:16.0:1506921874.700733:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b04518200 (tot 59423468). 00000800:00000200:16.0:1506921874.700735:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921874.700866:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:1.0:1506921874.700909:0:3864:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921874.700919:0:3864:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.42@o2ib4 00000800:00000200:11.0:1506921874.700922:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000400:00000200:1.0:1506921874.700926:0:3864:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:1.0:1506921874.700928:0:3864:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881c4b6cc800 (tot 59423868). 00000400:00000200:1.0:1506921874.700934:0:3864:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225510 00000400:00000200:1.0:1506921874.700939:0:3864:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1c99 [1] + 192 00000100:00000001:1.0:1506921874.700944:0:3864:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:1.0:1506921874.700947:0:3864:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666301712/t0(0) o19->oak-OST0015-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:1.0:1506921874.700956:0:3864:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666301712/t0(0) o19->oak-OST0015-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921874.700963:0:3864:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:1.0:1506921874.700965:0:3864:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881c4b6cc800 (tot 59423468). 00000800:00000200:1.0:1506921874.700968:0:3864:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921874.700970:0:3864:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:1.0:1506921874.700971:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921874.701001:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921874.701005:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921874.701006:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.701008:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.701012:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666301712/t0(0) o19->oak-OST0015-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.701021:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666301712/t0(0) o19->oak-OST0015-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.701026:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.701028:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.701029:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.701031:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.701032:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.701034:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.701035:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.701036:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.701037:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.701039:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.701042:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88207fdf9400 time=69 v=1 (31 31 31 31) 00000100:00001000:15.0:1506921874.701044:0:38877:0:(import.c:1737:at_measured()) AT ffff88207fdf9400 change: old=1 new=31 delta=30 (val=31) hist 31 31 31 31 00000100:00001000:15.0:1506921874.701047:0:38877:0:(client.c:345:ptlrpc_at_adj_service()) The RPC service estimate for oak-OST0015-osc-ffff88203bc47000 ptl 28 has changed from 1 to 31 00000100:00001000:15.0:1506921874.701049:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88207fdf93c8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.701051:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.701051:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.701053:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.701054:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.701056:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.701057:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0015-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.701060:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.701060:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.701063:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666301712/t0(0) o19->oak-OST0015-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921881 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.701068:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.701070:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.701071:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.701073:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666301712/t0(0) o19->oak-OST0015-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921881 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.701079:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301712:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.701081:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.701083:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.701084:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.701086:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.701087:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666301712/t0(0) o19->oak-OST0015-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921881 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.701092:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.701093:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.701095:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.701095:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.701097:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.701099:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666301712/t0(0) o19->oak-OST0015-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921881 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.701103:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.701104:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.701105:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.701107:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.701108:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.701108:0:38877:0:(genops.c:1026:class_import_put()) import ffff88207fdf9000 refcount=4 obd=oak-OST0015-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.701110:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.701111:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.701112:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.701113:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.701114:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.701116:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.701117:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.701118:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.701120:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.701121:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.701122:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.701123:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.701125:0:38877:0:(genops.c:1012:class_import_get()) import ffff88207fdf8800 refcount=5 obd=oak-OST0016-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.701127:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.701128:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.701129:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.701131:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.701133:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.701135:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.701135:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.701137:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.701138:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.701140:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.701140:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.701142:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.701144:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.701152:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.701155:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.701156:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.701157:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.701158:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.701160:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666301728/t0(0) o19->oak-OST0016-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.701165:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.701166:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.701167:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.701168:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.701170:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301728:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.701172:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.701174:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.701175:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.701176:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.701177:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.701179:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.701181:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.701182:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.701185:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301728, portal 4 00000100:00000001:15.0:1506921874.701186:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.701187:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.701190:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666301728/t0(0) o19->oak-OST0016-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.701194:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.701196:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921874.701197:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad500. 00000100:00000200:15.0:1506921874.701199:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301728, offset 0 00000400:00000010:15.0:1506921874.701201:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451dc00 (tot 59423868). 00000400:00000200:15.0:1506921874.701204:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921874.701213:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921874.701217:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921874.701221:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.701223:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921874.701225:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921874.701229:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921874.701230:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.701231:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.701232:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.701234:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921874.701235:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.701236:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.701237:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.701238:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.701240:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.701240:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921874.701268:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921874.701310:0:3865:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:5.0:1506921874.701317:0:3865:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:5.0:1506921874.701320:0:3865:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:5.0:1506921874.701324:0:3865:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666301728/t0(0) o19->oak-OST0016-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921874.701332:0:3865:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:5.0:1506921874.701334:0:3865:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666301728/t0(0) o19->oak-OST0016-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921874.701339:0:3865:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:5.0:1506921874.701341:0:3865:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:5.0:1506921874.701342:0:3865:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad500 00000400:00000010:5.0:1506921874.701344:0:3865:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad500. 00000800:00000200:16.2:1506921874.701347:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000400:00000010:5.0:1506921874.701347:0:3865:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451dc00 (tot 59423468). 00000800:00000200:13.0:1506921874.701349:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:5.0:1506921874.701353:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.0:1506921874.701365:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.701371:0:3861:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921874.701377:0:3861:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921874.701380:0:3861:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.701386:0:3861:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225520 00000400:00000200:16.0:1506921874.701391:0:3861:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a0ead [1] + 192 00000100:00000001:16.0:1506921874.701396:0:3861:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.701399:0:3861:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666301728/t0(0) o19->oak-OST0016-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921874.701403:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921874.701407:0:3861:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666301728/t0(0) o19->oak-OST0016-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.701416:0:3861:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.701417:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.701420:0:3861:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.701422:0:3861:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921874.701422:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:16.0:1506921874.701424:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000400:00000001:15.0:1506921874.701425:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921874.701426:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.701427:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.701429:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666301728/t0(0) o19->oak-OST0016-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.701435:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666301728/t0(0) o19->oak-OST0016-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.701439:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.701441:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.701442:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.701443:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.701444:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.701445:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.701446:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.701447:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.701448:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.701450:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.701452:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88207fdf8c00 time=69 v=31 (31 3 1 31) 00000100:00001000:15.0:1506921874.701454:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88207fdf8bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.701456:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.701457:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.701458:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.701459:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.701460:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.701461:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0016-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.701462:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.701463:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.701465:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666301728/t0(0) o19->oak-OST0016-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.701469:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.701470:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.701471:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.701473:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666301728/t0(0) o19->oak-OST0016-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.701478:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301728:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.701483:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.701485:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.701486:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.701487:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.701488:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666301728/t0(0) o19->oak-OST0016-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.701492:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.701494:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.701495:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.701496:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.701497:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.701499:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666301728/t0(0) o19->oak-OST0016-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.701503:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.701504:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.701504:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.701506:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.701507:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.701507:0:38877:0:(genops.c:1026:class_import_put()) import ffff88207fdf8800 refcount=4 obd=oak-OST0016-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.701509:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.701510:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.701511:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.701512:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.701513:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.701514:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.701515:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.701516:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.701517:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.701518:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.701519:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.701520:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.701522:0:38877:0:(genops.c:1012:class_import_get()) import ffff88103aad6000 refcount=5 obd=oak-OST0017-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.701523:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.701524:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.701525:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.701526:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.701528:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.701530:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.701530:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.701531:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.701532:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.701534:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.701534:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.701536:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.701537:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.701543:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.701545:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.701547:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.701548:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.701549:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.701550:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666301744/t0(0) o19->oak-OST0017-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.701555:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.701555:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.701557:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.701557:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.701559:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301744:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.701561:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.701562:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.701563:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.701564:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.701565:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.701566:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.701567:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.701569:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.701570:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301744, portal 4 00000100:00000001:15.0:1506921874.701572:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.701572:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.701575:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666301744/t0(0) o19->oak-OST0017-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.701583:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.701585:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921874.701586:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fac300. 00000100:00000200:15.0:1506921874.701587:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301744, offset 0 00000400:00000010:15.0:1506921874.701589:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451a600 (tot 59423868). 00000400:00000200:15.0:1506921874.701592:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921874.701599:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921874.701602:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921874.701607:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.701609:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921874.701610:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921874.701613:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921874.701614:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.701615:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.701616:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921874.701618:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:15.0:1506921874.701618:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921874.701618:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.701620:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.701621:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.701622:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.701623:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.701624:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.0:1506921874.701626:0:3858:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921874.701630:0:3858:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921874.701633:0:3858:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921874.701635:0:3858:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666301744/t0(0) o19->oak-OST0017-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.701641:0:3858:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000800:00000200:18.0:1506921874.701643:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921874.701643:0:3858:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666301744/t0(0) o19->oak-OST0017-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.701648:0:3858:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.701649:0:3858:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921874.701651:0:3858:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fac300 00000400:00000010:16.0:1506921874.701652:0:3858:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fac300. 00000400:00000010:16.0:1506921874.701655:0:3858:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451a600 (tot 59423468). 00000800:00000200:16.0:1506921874.701659:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:16.2:1506921874.701760:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921874.701766:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.701770:0:3859:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921874.701773:0:3859:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000800:00000200:18.0:1506921874.701774:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000010:16.0:1506921874.701775:0:3859:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.701778:0:3859:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225530 00000400:00000200:16.0:1506921874.701782:0:3859:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1ca1 [1] + 192 00000100:00000001:16.0:1506921874.701785:0:3859:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.701787:0:3859:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666301744/t0(0) o19->oak-OST0017-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:16.0:1506921874.701792:0:3859:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666301744/t0(0) o19->oak-OST0017-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.701799:0:3859:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.701800:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.701802:0:3859:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.701803:0:3859:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921874.701804:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:15.0:1506921874.701805:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921874.701807:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921874.701808:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.701809:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.701811:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666301744/t0(0) o19->oak-OST0017-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.701817:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666301744/t0(0) o19->oak-OST0017-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.701821:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.701823:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.701824:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.701825:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.701827:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.701828:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.701829:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.701830:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.701831:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.701832:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.701834:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88103aad6400 time=69 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921874.701836:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88103aad63c8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.701837:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.701838:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.701839:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.701840:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.701842:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.701842:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0017-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.701843:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.701844:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.701846:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666301744/t0(0) o19->oak-OST0017-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.701850:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.701851:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.701852:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.701854:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666301744/t0(0) o19->oak-OST0017-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.701859:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301744:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.701861:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.701863:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.701864:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.701865:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.701867:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666301744/t0(0) o19->oak-OST0017-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.701871:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.701872:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.701873:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.701874:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.701875:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.701877:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666301744/t0(0) o19->oak-OST0017-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.701881:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.701882:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.701883:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.701884:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.701885:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.701885:0:38877:0:(genops.c:1026:class_import_put()) import ffff88103aad6000 refcount=4 obd=oak-OST0017-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.701887:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.701888:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.701889:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.701890:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.701890:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.701892:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.701893:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.701894:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.701895:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.701896:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.701897:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.701898:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.701899:0:38877:0:(genops.c:1012:class_import_get()) import ffff88103aad4000 refcount=5 obd=oak-OST0018-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.701900:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.701901:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.701902:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.701903:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.701905:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.701906:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.701907:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.701908:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.701909:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.701910:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.701911:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.701912:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.701913:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.701919:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.701921:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.701923:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.701924:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.701925:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.701926:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666301760/t0(0) o19->oak-OST0018-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.701930:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.701931:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.701932:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.701933:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.701935:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301760:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.701937:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.701937:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.701938:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.701939:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.701940:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.701941:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.701943:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.701944:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.701946:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301760, portal 4 00000100:00000001:15.0:1506921874.701947:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.701948:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.701950:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666301760/t0(0) o19->oak-OST0018-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.701954:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.701956:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921874.701957:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad680. 00000100:00000200:15.0:1506921874.701958:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301760, offset 0 00000400:00000010:15.0:1506921874.701960:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b04519e00 (tot 59423868). 00000400:00000200:15.0:1506921874.701962:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921874.701968:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921874.701971:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921874.701974:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.701976:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921874.701978:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921874.701980:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921874.701981:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.701982:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.701983:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.701984:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921874.701985:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.701986:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.701987:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.701988:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.701989:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.701990:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921874.702015:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921874.702057:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:11.0:1506921874.702063:0:3862:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:11.0:1506921874.702066:0:3862:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:11.0:1506921874.702070:0:3862:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666301760/t0(0) o19->oak-OST0018-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.702077:0:3862:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.702079:0:3862:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666301760/t0(0) o19->oak-OST0018-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.702084:0:3862:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.702085:0:3862:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:11.0:1506921874.702086:0:3862:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad680 00000400:00000010:11.0:1506921874.702088:0:3862:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad680. 00000400:00000010:11.0:1506921874.702091:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b04519e00 (tot 59423468). 00000800:00000200:11.0:1506921874.702093:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:11.0:1506921874.702096:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.2:1506921874.702098:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921874.702115:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.702121:0:3860:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921874.702127:0:3860:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921874.702135:0:3860:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.702141:0:3860:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225540 00000400:00000200:16.0:1506921874.702146:0:3860:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a0ec5 [1] + 192 00000100:00000001:16.0:1506921874.702151:0:3860:0:(events.c:91:reply_in_callback()) Process entered 00000800:00000200:18.0:1506921874.702152:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000200:16.0:1506921874.702154:0:3860:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666301760/t0(0) o19->oak-OST0018-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:16.0:1506921874.702162:0:3860:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666301760/t0(0) o19->oak-OST0018-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.702171:0:3860:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.702173:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.702176:0:3860:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:15.0:1506921874.702176:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921874.702178:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921874.702179:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000800:00000200:16.0:1506921874.702180:0:3860:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921874.702180:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000800:00000200:16.0:1506921874.702182:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000200:15.0:1506921874.702182:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666301760/t0(0) o19->oak-OST0018-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.702187:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666301760/t0(0) o19->oak-OST0018-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.702192:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.702193:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.702194:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.702195:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.702196:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.702197:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.702198:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.702199:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.702200:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.702201:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.702203:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88103aad4400 time=69 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921874.702205:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88103aad43c8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.702206:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.702207:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.702208:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.702209:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.702211:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.702212:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0018-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.702213:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.702214:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.702216:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666301760/t0(0) o19->oak-OST0018-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.702220:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.702221:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.702222:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.702224:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666301760/t0(0) o19->oak-OST0018-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.702229:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301760:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.702231:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.702233:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.702234:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.702235:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.702236:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666301760/t0(0) o19->oak-OST0018-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.702240:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.702241:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.702243:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.702243:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.702245:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.702246:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666301760/t0(0) o19->oak-OST0018-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.702250:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.702251:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.702251:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.702253:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.702254:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.702254:0:38877:0:(genops.c:1026:class_import_put()) import ffff88103aad4000 refcount=4 obd=oak-OST0018-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.702256:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.702256:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.702258:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.702259:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.702260:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.702261:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.702262:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.702263:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.702264:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.702265:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.702266:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.702267:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.702268:0:38877:0:(genops.c:1012:class_import_get()) import ffff88103aad2000 refcount=5 obd=oak-OST0019-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.702269:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.702270:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.702271:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.702272:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.702274:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.702275:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.702276:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.702277:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.702278:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.702279:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.702280:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.702281:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.702282:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.702288:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.702290:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.702291:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.702293:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.702293:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.702295:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666301776/t0(0) o19->oak-OST0019-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.702299:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.702300:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.702301:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.702302:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.702303:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301776:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.702305:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.702306:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.702307:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.702308:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.702309:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.702310:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.702312:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.702313:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.702314:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301776, portal 4 00000100:00000001:15.0:1506921874.702315:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.702316:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.702318:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666301776/t0(0) o19->oak-OST0019-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.702322:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.702324:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921874.702325:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fadc00. 00000100:00000200:15.0:1506921874.702326:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301776, offset 0 00000400:00000010:15.0:1506921874.702328:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451d000 (tot 59423868). 00000400:00000200:15.0:1506921874.702331:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921874.702337:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921874.702340:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921874.702344:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.702346:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921874.702347:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921874.702350:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921874.702351:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.702352:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.702353:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.702354:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:16.2:1506921874.702355:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:15.0:1506921874.702355:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.702356:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.702357:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.702358:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.702359:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.702360:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.0:1506921874.702362:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921874.702366:0:3861:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000800:00000200:18.0:1506921874.702369:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:16.0:1506921874.702369:0:3861:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921874.702372:0:3861:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666301776/t0(0) o19->oak-OST0019-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.702378:0:3861:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921874.702380:0:3861:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666301776/t0(0) o19->oak-OST0019-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.702384:0:3861:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.702386:0:3861:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921874.702387:0:3861:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fadc00 00000400:00000010:16.0:1506921874.702389:0:3861:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fadc00. 00000400:00000010:16.0:1506921874.702392:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451d000 (tot 59423468). 00000800:00000200:16.0:1506921874.702394:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921874.702514:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:13.0:1506921874.702556:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921874.702565:0:3863:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.42@o2ib4 00000400:00000200:13.0:1506921874.702571:0:3863:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:13.0:1506921874.702574:0:3863:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88203bd18400 (tot 59423868). 00000400:00000200:13.0:1506921874.702579:0:3863:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225550 00000400:00000200:13.0:1506921874.702584:0:3863:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1ca9 [1] + 192 00000100:00000001:13.0:1506921874.702589:0:3863:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:13.0:1506921874.702592:0:3863:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666301776/t0(0) o19->oak-OST0019-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:5.0:1506921874.702597:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:13.0:1506921874.702600:0:3863:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666301776/t0(0) o19->oak-OST0019-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.702607:0:3863:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:13.0:1506921874.702608:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88203bd18400 (tot 59423468). 00000800:00000200:13.0:1506921874.702611:0:3863:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921874.702613:0:3863:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:13.0:1506921874.702615:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921874.702626:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921874.702629:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921874.702630:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.702631:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.702633:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666301776/t0(0) o19->oak-OST0019-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.702639:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666301776/t0(0) o19->oak-OST0019-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.702643:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.702644:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.702645:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.702646:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.702647:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.702649:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.702650:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.702650:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.702651:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.702653:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.702655:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88103aad2400 time=69 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921874.702657:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88103aad23c8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.702658:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.702659:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.702660:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.702661:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.702663:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.702664:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0019-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.702665:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.702665:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.702667:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666301776/t0(0) o19->oak-OST0019-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.702671:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.702673:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.702673:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.702675:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666301776/t0(0) o19->oak-OST0019-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.702680:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301776:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.702684:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.702686:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.702687:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.702688:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.702690:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666301776/t0(0) o19->oak-OST0019-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.702694:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.702695:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.702696:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.702697:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.702699:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.702700:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666301776/t0(0) o19->oak-OST0019-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.702704:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.702705:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.702706:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.702707:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.702708:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.702709:0:38877:0:(genops.c:1026:class_import_put()) import ffff88103aad2000 refcount=4 obd=oak-OST0019-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.702710:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.702711:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.702712:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.702713:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.702714:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.702715:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.702727:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.702729:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.702730:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.702731:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.702732:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.702733:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.702734:0:38877:0:(genops.c:1012:class_import_get()) import ffff8820362b0800 refcount=5 obd=oak-OST001a-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.702736:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.702737:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.702738:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.702739:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.702741:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.702742:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.702743:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.702744:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.702745:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.702747:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.702747:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.702749:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.702750:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.702757:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.702759:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.702760:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.702761:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.702762:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.702763:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666301792/t0(0) o19->oak-OST001a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.702768:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.702768:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.702770:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.702771:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.702772:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301792:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.702774:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.702775:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.702776:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.702777:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.702778:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.702779:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.702780:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.702782:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.702784:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301792, portal 4 00000100:00000001:15.0:1506921874.702785:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.702786:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.702788:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666301792/t0(0) o19->oak-OST001a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.702796:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.702798:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921874.702799:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad300. 00000100:00000200:15.0:1506921874.702801:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301792, offset 0 00000400:00000010:15.0:1506921874.702802:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b04518000 (tot 59423868). 00000400:00000200:15.0:1506921874.702805:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921874.702811:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921874.702815:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921874.702818:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.702819:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921874.702821:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921874.702824:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921874.702825:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.702826:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.702827:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.702828:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921874.702829:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.702830:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.702831:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.702832:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.702833:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.702834:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921874.702866:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921874.702911:0:3864:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:1.0:1506921874.702918:0:3864:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:1.0:1506921874.702921:0:3864:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:1.0:1506921874.702925:0:3864:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666301792/t0(0) o19->oak-OST001a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921874.702933:0:3864:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1506921874.702935:0:3864:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666301792/t0(0) o19->oak-OST001a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921874.702940:0:3864:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921874.702942:0:3864:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:1.0:1506921874.702943:0:3864:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad300 00000400:00000010:1.0:1506921874.702944:0:3864:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad300. 00000400:00000010:1.0:1506921874.702948:0:3864:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b04518000 (tot 59423468). 00000800:00000200:11.0:1506921874.702949:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:1.0:1506921874.702953:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.2:1506921874.702954:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921874.702971:0:3858:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.702977:0:3858:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921874.702983:0:3858:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921874.702986:0:3858:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.702992:0:3858:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225560 00000400:00000200:16.0:1506921874.702997:0:3858:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a0edd [1] + 192 00000100:00000001:16.0:1506921874.703002:0:3858:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.703005:0:3858:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666301792/t0(0) o19->oak-OST001a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921874.703009:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921874.703012:0:3858:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666301792/t0(0) o19->oak-OST001a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.703021:0:3858:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.703023:0:3858:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.703025:0:3858:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:15.0:1506921874.703026:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:16.0:1506921874.703028:0:3858:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000400:00000001:15.0:1506921874.703028:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000800:00000200:16.0:1506921874.703029:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000400:00000001:15.0:1506921874.703029:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.703030:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.703032:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666301792/t0(0) o19->oak-OST001a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.703037:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666301792/t0(0) o19->oak-OST001a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.703041:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.703043:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.703044:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.703045:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.703047:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.703048:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.703048:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.703049:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.703050:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.703052:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.703053:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff8820362b0c00 time=69 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921874.703055:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff8820362b0bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.703057:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.703058:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.703059:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.703060:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.703061:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.703062:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST001a-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.703063:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.703064:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.703066:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666301792/t0(0) o19->oak-OST001a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.703070:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.703071:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.703072:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.703074:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666301792/t0(0) o19->oak-OST001a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.703079:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301792:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.703081:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.703083:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.703084:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.703085:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.703086:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666301792/t0(0) o19->oak-OST001a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.703090:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.703091:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.703093:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.703093:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.703095:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.703097:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666301792/t0(0) o19->oak-OST001a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.703100:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.703101:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.703102:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.703103:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.703104:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.703105:0:38877:0:(genops.c:1026:class_import_put()) import ffff8820362b0800 refcount=4 obd=oak-OST001a-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.703106:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.703107:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.703108:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.703109:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.703110:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.703111:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.703112:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.703113:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.703114:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.703115:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.703116:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.703116:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.703118:0:38877:0:(genops.c:1012:class_import_get()) import ffff8820362b2800 refcount=5 obd=oak-OST001b-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.703119:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.703120:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.703121:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.703122:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.703123:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.703124:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.703125:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.703126:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.703127:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.703128:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.703129:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.703130:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.703131:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.703137:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.703139:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.703141:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.703142:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.703143:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.703144:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666301808/t0(0) o19->oak-OST001b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.703148:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.703149:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.703150:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.703151:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.703153:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301808:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.703155:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.703155:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.703156:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.703157:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.703158:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.703159:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.703161:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.703162:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.703163:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301808, portal 4 00000100:00000001:15.0:1506921874.703164:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.703165:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.703167:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666301808/t0(0) o19->oak-OST001b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.703171:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.703173:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921874.703174:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad980. 00000100:00000200:15.0:1506921874.703175:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301808, offset 0 00000400:00000010:15.0:1506921874.703177:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451e200 (tot 59423868). 00000400:00000200:15.0:1506921874.703179:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921874.703186:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921874.703189:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921874.703193:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.703195:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921874.703197:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921874.703200:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921874.703201:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.703202:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.703203:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.703204:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921874.703204:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000800:00000200:16.2:1506921874.703205:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000100:00100000:15.0:1506921874.703206:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.703207:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.703208:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.703209:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.703209:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.0:1506921874.703212:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921874.703218:0:3859:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000800:00000200:18.0:1506921874.703222:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:16.0:1506921874.703222:0:3859:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921874.703224:0:3859:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666301808/t0(0) o19->oak-OST001b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.703230:0:3859:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921874.703232:0:3859:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666301808/t0(0) o19->oak-OST001b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.703236:0:3859:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.703238:0:3859:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921874.703239:0:3859:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad980 00000400:00000010:16.0:1506921874.703241:0:3859:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad980. 00000400:00000010:16.0:1506921874.703244:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451e200 (tot 59423468). 00000800:00000200:16.0:1506921874.703246:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:16.2:1506921874.703351:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921874.703358:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.703362:0:3860:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000800:00000200:18.0:1506921874.703365:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000200:16.0:1506921874.703366:0:3860:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:16.0:1506921874.703368:0:3860:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.703371:0:3860:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225570 00000400:00000200:16.0:1506921874.703375:0:3860:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1cb1 [1] + 192 00000100:00000001:16.0:1506921874.703378:0:3860:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.703380:0:3860:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666301808/t0(0) o19->oak-OST001b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:16.0:1506921874.703385:0:3860:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666301808/t0(0) o19->oak-OST001b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.703392:0:3860:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.703393:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.703394:0:3860:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.703396:0:3860:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921874.703397:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:15.0:1506921874.703397:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921874.703398:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921874.703399:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.703400:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.703402:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666301808/t0(0) o19->oak-OST001b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.703407:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666301808/t0(0) o19->oak-OST001b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.703412:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.703413:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.703415:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.703416:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.703417:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.703418:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.703419:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.703419:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.703420:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.703422:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.703423:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff8820362b2c00 time=69 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921874.703425:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff8820362b2bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.703427:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.703428:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.703429:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.703430:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.703431:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.703432:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST001b-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.703433:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.703434:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.703436:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666301808/t0(0) o19->oak-OST001b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.703440:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.703441:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.703442:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.703443:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666301808/t0(0) o19->oak-OST001b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.703448:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301808:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.703450:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.703452:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.703453:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.703454:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.703456:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666301808/t0(0) o19->oak-OST001b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.703460:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.703461:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.703462:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.703463:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.703465:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.703466:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666301808/t0(0) o19->oak-OST001b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.703470:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.703471:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.703471:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.703473:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.703473:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.703474:0:38877:0:(genops.c:1026:class_import_put()) import ffff8820362b2800 refcount=4 obd=oak-OST001b-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.703475:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.703476:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.703478:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.703478:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.703479:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.703481:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.703482:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.703483:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.703484:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.703485:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.703485:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.703486:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.703487:0:38877:0:(genops.c:1012:class_import_get()) import ffff8820362b4800 refcount=5 obd=oak-OST001c-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.703489:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.703489:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.703490:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.703492:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.703493:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.703494:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.703495:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.703496:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.703497:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.703498:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.703499:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.703500:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.703501:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.703507:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.703509:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.703510:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.703511:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.703512:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.703513:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666301824/t0(0) o19->oak-OST001c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.703517:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.703518:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.703519:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.703520:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.703522:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301824:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.703524:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.703525:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.703526:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.703527:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.703528:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.703529:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.703530:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.703531:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.703533:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301824, portal 4 00000100:00000001:15.0:1506921874.703534:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.703535:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.703537:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666301824/t0(0) o19->oak-OST001c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.703541:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.703543:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921874.703544:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fadf00. 00000100:00000200:15.0:1506921874.703545:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301824, offset 0 00000400:00000010:15.0:1506921874.703547:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451d600 (tot 59423868). 00000400:00000200:15.0:1506921874.703549:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921874.703555:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921874.703558:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921874.703561:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.703563:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921874.703564:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921874.703566:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921874.703567:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.703568:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.703569:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.703570:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921874.703570:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.703571:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.703573:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.703573:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.703575:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.703575:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921874.703598:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921874.703641:0:3865:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:5.0:1506921874.703648:0:3865:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:5.0:1506921874.703651:0:3865:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:5.0:1506921874.703654:0:3865:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666301824/t0(0) o19->oak-OST001c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921874.703662:0:3865:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:5.0:1506921874.703665:0:3865:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666301824/t0(0) o19->oak-OST001c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921874.703670:0:3865:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:5.0:1506921874.703672:0:3865:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:5.0:1506921874.703673:0:3865:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fadf00 00000400:00000010:5.0:1506921874.703674:0:3865:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fadf00. 00000800:00000200:16.2:1506921874.703675:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000400:00000010:5.0:1506921874.703677:0:3865:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451d600 (tot 59423468). 00000800:00000200:13.0:1506921874.703679:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:5.0:1506921874.703684:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.0:1506921874.703692:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.703698:0:3861:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921874.703703:0:3861:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921874.703706:0:3861:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.703714:0:3861:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225580 00000800:00000200:18.0:1506921874.703730:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000200:16.0:1506921874.703734:0:3861:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a0ef5 [1] + 192 00000100:00000001:16.0:1506921874.703739:0:3861:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.703742:0:3861:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666301824/t0(0) o19->oak-OST001c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:16.0:1506921874.703750:0:3861:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666301824/t0(0) o19->oak-OST001c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.703759:0:3861:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.703761:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.703764:0:3861:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:15.0:1506921874.703765:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:16.0:1506921874.703766:0:3861:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000400:00000001:15.0:1506921874.703767:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000800:00000200:16.0:1506921874.703768:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000400:00000001:15.0:1506921874.703768:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.703769:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.703771:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666301824/t0(0) o19->oak-OST001c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.703776:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666301824/t0(0) o19->oak-OST001c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.703781:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.703782:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.703783:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.703785:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.703786:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.703787:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.703787:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.703788:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.703789:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.703790:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.703792:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff8820362b4c00 time=69 v=31 (31 3 1 31) 00000100:00001000:15.0:1506921874.703794:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff8820362b4bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.703796:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.703797:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.703798:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.703799:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.703800:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.703801:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST001c-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.703802:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.703802:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.703804:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666301824/t0(0) o19->oak-OST001c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.703808:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.703809:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.703810:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.703812:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666301824/t0(0) o19->oak-OST001c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.703817:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301824:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.703821:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.703822:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.703823:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.703824:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.703826:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666301824/t0(0) o19->oak-OST001c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.703830:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.703831:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.703832:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.703833:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.703834:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.703836:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666301824/t0(0) o19->oak-OST001c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.703840:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.703840:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.703841:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.703842:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.703843:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.703844:0:38877:0:(genops.c:1026:class_import_put()) import ffff8820362b4800 refcount=4 obd=oak-OST001c-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.703845:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.703846:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.703847:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.703848:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.703849:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.703850:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.703851:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.703852:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.703853:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.703854:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.703855:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.703855:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.703857:0:38877:0:(genops.c:1012:class_import_get()) import ffff8820362b6800 refcount=5 obd=oak-OST001d-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.703858:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.703859:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.703860:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.703861:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.703863:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.703864:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.703865:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.703866:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.703867:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.703868:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.703869:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.703870:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.703871:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.703877:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.703878:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.703879:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.703881:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.703881:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.703883:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666301840/t0(0) o19->oak-OST001d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.703887:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.703888:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.703889:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.703890:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.703891:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301840:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.703893:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.703894:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.703895:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.703896:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.703897:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.703898:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.703899:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.703900:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.703902:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301840, portal 4 00000100:00000001:15.0:1506921874.703903:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.703904:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.703906:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666301840/t0(0) o19->oak-OST001d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.703914:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.703915:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921874.703917:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fadc80. 00000100:00000200:15.0:1506921874.703918:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301840, offset 0 00000400:00000010:15.0:1506921874.703919:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451ac00 (tot 59423868). 00000400:00000200:15.0:1506921874.703922:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921874.703928:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921874.703931:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921874.703935:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.703937:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921874.703938:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921874.703941:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921874.703942:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.703943:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.703944:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.703945:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:16.2:1506921874.703946:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:15.0:1506921874.703946:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.703947:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.703948:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.703949:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.703950:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.703951:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.0:1506921874.703953:0:3858:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921874.703957:0:3858:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921874.703960:0:3858:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921874.703963:0:3858:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666301840/t0(0) o19->oak-OST001d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.703968:0:3858:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921874.703970:0:3858:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666301840/t0(0) o19->oak-OST001d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921874.703971:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:16.0:1506921874.703975:0:3858:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.703976:0:3858:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921874.703977:0:3858:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fadc80 00000400:00000010:16.0:1506921874.703979:0:3858:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fadc80. 00000400:00000010:16.0:1506921874.703982:0:3858:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451ac00 (tot 59423468). 00000800:00000200:16.0:1506921874.703984:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921874.704114:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:11.0:1506921874.704156:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921874.704164:0:3862:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.42@o2ib4 00000400:00000200:11.0:1506921874.704171:0:3862:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:11.0:1506921874.704173:0:3862:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e15800 (tot 59423868). 00000400:00000200:11.0:1506921874.704178:0:3862:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225590 00000400:00000200:11.0:1506921874.704182:0:3862:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1cb9 [1] + 192 00000100:00000001:11.0:1506921874.704187:0:3862:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921874.704191:0:3862:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666301840/t0(0) o19->oak-OST001d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:1.0:1506921874.704197:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:11.0:1506921874.704198:0:3862:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666301840/t0(0) o19->oak-OST001d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.704205:0:3862:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921874.704207:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e15800 (tot 59423468). 00000800:00000200:11.0:1506921874.704209:0:3862:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921874.704211:0:3862:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:11.0:1506921874.704212:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921874.704224:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921874.704226:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921874.704227:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.704228:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.704230:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666301840/t0(0) o19->oak-OST001d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.704235:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666301840/t0(0) o19->oak-OST001d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.704239:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.704240:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.704242:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.704243:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.704245:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.704246:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.704247:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.704248:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.704249:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.704251:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.704252:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff8820362b6c00 time=69 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921874.704254:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff8820362b6bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.704256:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.704257:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.704258:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.704259:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.704261:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.704261:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST001d-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.704262:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.704263:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.704265:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666301840/t0(0) o19->oak-OST001d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.704269:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.704270:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.704271:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.704273:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666301840/t0(0) o19->oak-OST001d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.704278:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301840:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.704280:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.704281:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.704283:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.704284:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.704285:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666301840/t0(0) o19->oak-OST001d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.704289:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.704291:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.704292:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.704293:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.704294:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.704299:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666301840/t0(0) o19->oak-OST001d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.704303:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.704304:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.704305:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.704306:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.704307:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.704308:0:38877:0:(genops.c:1026:class_import_put()) import ffff8820362b6800 refcount=4 obd=oak-OST001d-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.704309:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.704310:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.704311:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.704312:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.704313:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.704315:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.704315:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.704317:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.704318:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.704319:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.704320:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.704321:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.704322:0:38877:0:(genops.c:1012:class_import_get()) import ffff882036f50800 refcount=5 obd=oak-OST001e-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.704324:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.704324:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.704325:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.704327:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.704328:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.704330:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.704330:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.704332:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.704333:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.704334:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.704335:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.704336:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.704337:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.704344:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.704345:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.704347:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.704348:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.704349:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.704350:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666301856/t0(0) o19->oak-OST001e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.704355:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.704355:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.704357:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.704357:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.704359:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301856:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.704361:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.704362:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.704363:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.704364:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.704365:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.704366:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.704367:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.704369:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.704371:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301856, portal 4 00000100:00000001:15.0:1506921874.704372:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.704373:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.704375:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666301856/t0(0) o19->oak-OST001e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.704379:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.704381:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921874.704382:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad780. 00000100:00000200:15.0:1506921874.704383:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301856, offset 0 00000400:00000010:15.0:1506921874.704385:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451fe00 (tot 59423868). 00000400:00000200:15.0:1506921874.704387:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921874.704394:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921874.704397:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921874.704400:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.704402:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921874.704404:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921874.704407:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921874.704408:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.704409:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.704410:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.704411:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921874.704412:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.704413:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.704414:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.704415:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.704416:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.704417:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921874.704443:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921874.704488:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:13.0:1506921874.704494:0:3863:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:13.0:1506921874.704497:0:3863:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:13.0:1506921874.704501:0:3863:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666301856/t0(0) o19->oak-OST001e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.704509:0:3863:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:13.0:1506921874.704511:0:3863:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666301856/t0(0) o19->oak-OST001e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.704516:0:3863:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.704518:0:3863:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:13.0:1506921874.704519:0:3863:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad780 00000400:00000010:13.0:1506921874.704520:0:3863:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad780. 00000400:00000010:13.0:1506921874.704525:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451fe00 (tot 59423468). 00000800:00000200:5.0:1506921874.704527:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:16.2:1506921874.704530:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:13.0:1506921874.704532:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.0:1506921874.704547:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.704554:0:3859:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921874.704560:0:3859:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921874.704562:0:3859:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.704568:0:3859:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c2255a0 00000400:00000200:16.0:1506921874.704574:0:3859:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a0f0d [1] + 192 00000100:00000001:16.0:1506921874.704578:0:3859:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.704583:0:3859:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666301856/t0(0) o19->oak-OST001e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921874.704585:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921874.704591:0:3859:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666301856/t0(0) o19->oak-OST001e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.704600:0:3859:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.704602:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.704604:0:3859:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:15.0:1506921874.704605:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:16.0:1506921874.704606:0:3859:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000400:00000001:15.0:1506921874.704607:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000800:00000200:16.0:1506921874.704608:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000400:00000001:15.0:1506921874.704608:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.704609:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.704611:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666301856/t0(0) o19->oak-OST001e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.704616:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666301856/t0(0) o19->oak-OST001e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.704620:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.704622:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.704623:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.704624:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.704625:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.704626:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.704627:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.704628:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.704629:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.704630:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.704632:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff882036f50c00 time=69 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921874.704634:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff882036f50bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.704635:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.704636:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.704637:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.704638:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.704639:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.704641:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST001e-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.704642:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.704642:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.704644:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666301856/t0(0) o19->oak-OST001e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.704648:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.704650:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.704650:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.704652:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666301856/t0(0) o19->oak-OST001e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.704657:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301856:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.704659:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.704661:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.704662:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.704663:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.704665:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666301856/t0(0) o19->oak-OST001e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.704669:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.704670:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.704671:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.704672:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.704673:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.704675:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666301856/t0(0) o19->oak-OST001e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.704678:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.704679:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.704680:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.704681:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.704682:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.704683:0:38877:0:(genops.c:1026:class_import_put()) import ffff882036f50800 refcount=4 obd=oak-OST001e-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.704684:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.704685:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.704686:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.704687:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.704688:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.704689:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.704690:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.704691:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.704692:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.704693:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.704694:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.704695:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.704696:0:38877:0:(genops.c:1012:class_import_get()) import ffff882036f52800 refcount=5 obd=oak-OST001f-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.704698:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.704698:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.704699:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.704700:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.704702:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.704703:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.704704:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.704705:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.704706:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.704707:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.704708:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.704709:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.704710:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.704716:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.704726:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.704727:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.704728:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.704729:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.704730:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666301872/t0(0) o19->oak-OST001f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.704735:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.704735:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.704737:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.704737:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.704739:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301872:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.704741:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.704742:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.704743:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.704744:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.704745:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.704746:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.704748:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.704749:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.704751:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301872, portal 4 00000100:00000001:15.0:1506921874.704752:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.704752:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.704755:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666301872/t0(0) o19->oak-OST001f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.704759:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.704760:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921874.704761:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fadd80. 00000100:00000200:15.0:1506921874.704763:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301872, offset 0 00000400:00000010:15.0:1506921874.704764:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451a800 (tot 59423868). 00000400:00000200:15.0:1506921874.704767:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921874.704773:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921874.704777:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921874.704780:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.704782:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921874.704784:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921874.704787:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921874.704788:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.704789:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.704790:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921874.704792:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:15.0:1506921874.704792:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921874.704792:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=6 : 6 : 6) 00000100:00100000:15.0:1506921874.704793:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 6 seconds 00000100:00000001:15.0:1506921874.704794:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.704795:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.704796:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.704797:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.0:1506921874.704800:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921874.704805:0:3860:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921874.704808:0:3860:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921874.704811:0:3860:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666301872/t0(0) o19->oak-OST001f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.704816:0:3860:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921874.704818:0:3860:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666301872/t0(0) o19->oak-OST001f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.704823:0:3860:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.704824:0:3860:0:(events.c:81:request_out_callback()) Process leaving 00000800:00000200:18.0:1506921874.704825:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000200:16.0:1506921874.704826:0:3860:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fadd80 00000400:00000010:16.0:1506921874.704827:0:3860:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fadd80. 00000400:00000010:16.0:1506921874.704830:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451a800 (tot 59423468). 00000800:00000200:16.0:1506921874.704832:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:16.2:1506921874.704895:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921874.704902:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.704906:0:3861:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921874.704909:0:3861:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000800:00000200:18.0:1506921874.704910:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000010:16.0:1506921874.704911:0:3861:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.704915:0:3861:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c2255b0 00000400:00000200:16.0:1506921874.704918:0:3861:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1cc1 [1] + 192 00000100:00000001:16.0:1506921874.704922:0:3861:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.704923:0:3861:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666301872/t0(0) o19->oak-OST001f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:16.0:1506921874.704941:0:3861:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666301872/t0(0) o19->oak-OST001f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.704948:0:3861:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.704949:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.704951:0:3861:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.704952:0:3861:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921874.704953:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:15.0:1506921874.704953:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921874.704956:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921874.704957:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.704958:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.704960:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666301872/t0(0) o19->oak-OST001f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.704965:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666301872/t0(0) o19->oak-OST001f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.704969:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.704971:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.704972:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.704973:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.704974:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.704975:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.704976:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.704976:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.704977:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.704979:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.704981:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff882036f52c00 time=69 v=1 (31 31 31 31) 00000100:00001000:15.0:1506921874.704983:0:38877:0:(import.c:1737:at_measured()) AT ffff882036f52c00 change: old=1 new=31 delta=30 (val=31) hist 31 31 31 31 00000100:00001000:15.0:1506921874.704985:0:38877:0:(client.c:345:ptlrpc_at_adj_service()) The RPC service estimate for oak-OST001f-osc-ffff88203bc47000 ptl 28 has changed from 1 to 31 00000100:00001000:15.0:1506921874.704987:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff882036f52bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.704988:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.704989:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.704990:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.704991:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.704992:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.704993:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST001f-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.704994:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.704995:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.704997:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666301872/t0(0) o19->oak-OST001f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921881 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.705001:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.705002:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.705003:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.705005:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666301872/t0(0) o19->oak-OST001f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921881 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.705010:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301872:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.705012:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.705014:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.705015:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.705016:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.705018:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666301872/t0(0) o19->oak-OST001f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921881 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.705022:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.705023:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.705024:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.705025:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.705026:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.705028:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666301872/t0(0) o19->oak-OST001f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921881 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.705032:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.705032:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.705033:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.705035:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.705035:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.705036:0:38877:0:(genops.c:1026:class_import_put()) import ffff882036f52800 refcount=4 obd=oak-OST001f-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.705037:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.705038:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.705040:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.705040:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.705041:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.705042:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.705043:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.705044:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.705045:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.705046:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.705047:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.705048:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.705049:0:38877:0:(genops.c:1012:class_import_get()) import ffff882036f54800 refcount=5 obd=oak-OST0020-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.705050:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.705051:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.705052:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.705054:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.705055:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.705056:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.705057:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.705058:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.705059:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.705060:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.705061:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.705062:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.705063:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.705069:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.705071:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.705072:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.705073:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.705074:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.705075:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666301888/t0(0) o19->oak-OST0020-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.705080:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.705080:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.705082:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.705082:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.705084:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301888:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.705086:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.705087:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.705087:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.705088:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.705089:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.705091:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.705092:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.705093:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.705095:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301888, portal 4 00000100:00000001:15.0:1506921874.705096:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.705097:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.705100:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666301888/t0(0) o19->oak-OST0020-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.705104:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.705105:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921874.705106:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fadb80. 00000100:00000200:15.0:1506921874.705108:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301888, offset 0 00000400:00000010:15.0:1506921874.705109:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b04519800 (tot 59423868). 00000400:00000200:15.0:1506921874.705112:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921874.705118:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921874.705121:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921874.705123:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.705125:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921874.705126:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921874.705128:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921874.705129:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.705130:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.705131:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.705132:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921874.705133:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.705134:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.705135:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.705136:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.705137:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.705138:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921874.705162:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921874.705207:0:3864:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:1.0:1506921874.705214:0:3864:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:1.0:1506921874.705217:0:3864:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:1.0:1506921874.705221:0:3864:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666301888/t0(0) o19->oak-OST0020-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:16.2:1506921874.705223:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000100:00000001:1.0:1506921874.705229:0:3864:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1506921874.705231:0:3864:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666301888/t0(0) o19->oak-OST0020-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921874.705236:0:3864:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921874.705238:0:3864:0:(events.c:81:request_out_callback()) Process leaving 00000800:00000200:16.0:1506921874.705239:0:3858:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000400:00000200:1.0:1506921874.705239:0:3864:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fadb80 00000400:00000010:1.0:1506921874.705242:0:3864:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fadb80. 00000800:00000200:16.0:1506921874.705245:0:3858:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.41@o2ib4 00000400:00000010:1.0:1506921874.705246:0:3864:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b04519800 (tot 59423468). 00000800:00000200:11.0:1506921874.705247:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000400:00000200:16.0:1506921874.705250:0:3858:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000800:00000200:1.0:1506921874.705251:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000400:00000010:16.0:1506921874.705252:0:3858:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.705258:0:3858:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c2255c0 00000400:00000200:16.0:1506921874.705262:0:3858:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a0f25 [1] + 192 00000100:00000001:16.0:1506921874.705267:0:3858:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.705270:0:3858:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666301888/t0(0) o19->oak-OST0020-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921874.705275:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921874.705277:0:3858:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666301888/t0(0) o19->oak-OST0020-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.705285:0:3858:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.705287:0:3858:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.705290:0:3858:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:15.0:1506921874.705290:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:16.0:1506921874.705292:0:3858:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000400:00000001:15.0:1506921874.705292:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921874.705293:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000800:00000200:16.0:1506921874.705294:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:15.0:1506921874.705294:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.705296:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666301888/t0(0) o19->oak-OST0020-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.705301:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666301888/t0(0) o19->oak-OST0020-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.705305:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.705307:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.705308:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.705310:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.705311:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.705312:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.705312:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.705313:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.705314:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.705316:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.705317:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff882036f54c00 time=69 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921874.705319:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff882036f54bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.705321:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.705322:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.705323:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.705323:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.705325:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.705325:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0020-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.705327:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.705327:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.705329:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666301888/t0(0) o19->oak-OST0020-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.705333:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.705334:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.705335:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.705337:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666301888/t0(0) o19->oak-OST0020-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.705341:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301888:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.705344:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.705345:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.705346:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.705347:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.705349:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666301888/t0(0) o19->oak-OST0020-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.705353:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.705354:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.705355:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.705356:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.705358:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.705359:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666301888/t0(0) o19->oak-OST0020-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.705363:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.705364:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.705364:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.705366:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.705367:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.705367:0:38877:0:(genops.c:1026:class_import_put()) import ffff882036f54800 refcount=4 obd=oak-OST0020-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.705369:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.705369:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.705371:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.705371:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.705372:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.705374:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.705374:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.705375:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.705376:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.705377:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.705378:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.705379:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.705380:0:38877:0:(genops.c:1012:class_import_get()) import ffff882036f56800 refcount=5 obd=oak-OST0021-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.705381:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.705382:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.705383:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.705384:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.705386:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.705387:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.705388:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.705388:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.705389:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.705391:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.705391:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.705393:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.705394:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.705400:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.705401:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.705403:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.705404:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.705405:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.705406:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666301904/t0(0) o19->oak-OST0021-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.705410:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.705411:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.705412:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.705413:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.705415:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301904:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.705416:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.705417:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.705418:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.705419:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.705420:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.705421:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.705422:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.705424:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.705425:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301904, portal 4 00000100:00000001:15.0:1506921874.705426:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.705427:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.705429:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666301904/t0(0) o19->oak-OST0021-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.705433:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.705435:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921874.705436:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032facd80. 00000100:00000200:15.0:1506921874.705437:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301904, offset 0 00000400:00000010:15.0:1506921874.705439:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451f800 (tot 59423868). 00000400:00000200:15.0:1506921874.705441:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921874.705448:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921874.705451:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921874.705461:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.705463:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921874.705465:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921874.705468:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921874.705469:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.705470:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.705471:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.705472:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:16.2:1506921874.705473:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:15.0:1506921874.705473:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.705474:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.705475:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.705476:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.705477:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.705478:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.0:1506921874.705480:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921874.705484:0:3859:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000800:00000200:18.0:1506921874.705488:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:16.0:1506921874.705488:0:3859:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921874.705491:0:3859:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666301904/t0(0) o19->oak-OST0021-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.705496:0:3859:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921874.705498:0:3859:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666301904/t0(0) o19->oak-OST0021-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.705502:0:3859:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.705504:0:3859:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921874.705506:0:3859:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032facd80 00000400:00000010:16.0:1506921874.705509:0:3859:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032facd80. 00000400:00000010:16.0:1506921874.705512:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451f800 (tot 59423468). 00000800:00000200:16.0:1506921874.705514:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921874.705615:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:5.0:1506921874.705659:0:3865:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921874.705667:0:3865:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.42@o2ib4 00000400:00000200:5.0:1506921874.705675:0:3865:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:5.0:1506921874.705678:0:3865:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881e48bc2800 (tot 59423868). 00000400:00000200:5.0:1506921874.705684:0:3865:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c2255d0 00000400:00000200:5.0:1506921874.705688:0:3865:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1cc9 [1] + 192 00000100:00000001:5.0:1506921874.705693:0:3865:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:5.0:1506921874.705696:0:3865:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666301904/t0(0) o19->oak-OST0021-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:13.0:1506921874.705697:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:5.0:1506921874.705704:0:3865:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666301904/t0(0) o19->oak-OST0021-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921874.705712:0:3865:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:5.0:1506921874.705714:0:3865:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881e48bc2800 (tot 59423468). 00000100:00000001:15.0:1506921874.705715:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921874.705725:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921874.705726:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.705727:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.705729:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666301904/t0(0) o19->oak-OST0021-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.705734:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666301904/t0(0) o19->oak-OST0021-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.705738:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.705739:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000800:00000200:5.0:1506921874.705739:0:3865:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000400:00000200:15.0:1506921874.705740:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000800:00000200:5.0:1506921874.705741:0:3865:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000400:00000010:15.0:1506921874.705742:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.705743:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:5.0:1506921874.705743:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921874.705744:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.705745:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.705746:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.705747:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.705748:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.705750:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff882036f56c00 time=69 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921874.705752:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff882036f56bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.705754:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.705755:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.705756:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.705757:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.705758:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.705760:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0021-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.705761:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.705762:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.705764:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666301904/t0(0) o19->oak-OST0021-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.705768:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.705769:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.705770:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.705772:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666301904/t0(0) o19->oak-OST0021-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.705777:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301904:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.705779:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.705780:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.705781:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.705782:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.705784:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666301904/t0(0) o19->oak-OST0021-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.705788:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.705790:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.705791:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.705792:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.705793:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.705794:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666301904/t0(0) o19->oak-OST0021-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.705798:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.705799:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.705800:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.705801:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.705802:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.705803:0:38877:0:(genops.c:1026:class_import_put()) import ffff882036f56800 refcount=4 obd=oak-OST0021-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.705804:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.705805:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.705807:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.705807:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.705809:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.705810:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.705811:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.705812:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.705813:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.705814:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.705815:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.705816:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.705817:0:38877:0:(genops.c:1012:class_import_get()) import ffff882036a00800 refcount=5 obd=oak-OST0022-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.705819:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.705820:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.705821:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.705822:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.705824:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.705825:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.705826:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.705827:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.705828:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.705830:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.705830:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.705832:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.705833:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.705839:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.705841:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.705842:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.705844:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.705844:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.705846:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666301920/t0(0) o19->oak-OST0022-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.705850:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.705851:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.705852:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.705853:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.705855:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301920:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.705857:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.705858:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.705859:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.705860:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.705861:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.705862:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.705863:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.705865:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.705867:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301920, portal 4 00000100:00000001:15.0:1506921874.705868:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.705869:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.705871:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666301920/t0(0) o19->oak-OST0022-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.705875:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.705877:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921874.705878:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fada00. 00000100:00000200:15.0:1506921874.705879:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301920, offset 0 00000400:00000010:15.0:1506921874.705881:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451aa00 (tot 59423868). 00000400:00000200:15.0:1506921874.705884:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921874.705891:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921874.705894:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921874.705897:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.705899:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921874.705900:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921874.705903:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921874.705904:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.705905:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.705906:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.705907:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921874.705908:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.705909:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.705910:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.705911:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.705912:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.705913:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921874.705943:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921874.705985:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:11.0:1506921874.705992:0:3862:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:11.0:1506921874.705994:0:3862:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:11.0:1506921874.705998:0:3862:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666301920/t0(0) o19->oak-OST0022-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.706005:0:3862:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.706007:0:3862:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666301920/t0(0) o19->oak-OST0022-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.706012:0:3862:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.706013:0:3862:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:11.0:1506921874.706014:0:3862:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fada00 00000400:00000010:11.0:1506921874.706016:0:3862:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fada00. 00000400:00000010:11.0:1506921874.706019:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451aa00 (tot 59423468). 00000800:00000200:11.0:1506921874.706021:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:1.0:1506921874.706024:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.2:1506921874.706032:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921874.706049:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.706055:0:3860:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921874.706061:0:3860:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921874.706063:0:3860:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.706069:0:3860:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c2255e0 00000400:00000200:16.0:1506921874.706074:0:3860:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a0f3d [1] + 192 00000100:00000001:16.0:1506921874.706078:0:3860:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.706081:0:3860:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666301920/t0(0) o19->oak-OST0022-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921874.706086:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921874.706088:0:3860:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666301920/t0(0) o19->oak-OST0022-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.706097:0:3860:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.706099:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.706101:0:3860:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:15.0:1506921874.706102:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:16.0:1506921874.706103:0:3860:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000400:00000001:15.0:1506921874.706104:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000800:00000200:16.0:1506921874.706105:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000400:00000001:15.0:1506921874.706105:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.706106:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.706109:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666301920/t0(0) o19->oak-OST0022-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.706113:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666301920/t0(0) o19->oak-OST0022-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.706118:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.706119:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.706120:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.706122:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.706122:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.706123:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.706124:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.706125:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.706126:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.706128:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.706129:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff882036a00c00 time=69 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921874.706131:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff882036a00bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.706133:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.706134:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.706135:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.706136:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.706137:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.706138:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0022-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.706139:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.706139:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.706141:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666301920/t0(0) o19->oak-OST0022-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.706145:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.706147:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.706147:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.706149:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666301920/t0(0) o19->oak-OST0022-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.706154:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301920:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.706158:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.706160:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.706161:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.706162:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.706163:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666301920/t0(0) o19->oak-OST0022-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.706168:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.706169:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.706170:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.706171:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.706172:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.706173:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666301920/t0(0) o19->oak-OST0022-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.706177:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.706178:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.706179:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.706180:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.706181:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.706181:0:38877:0:(genops.c:1026:class_import_put()) import ffff882036a00800 refcount=4 obd=oak-OST0022-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.706183:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.706184:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.706185:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.706186:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.706187:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.706188:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.706189:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.706190:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.706191:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.706192:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.706193:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.706194:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.706195:0:38877:0:(genops.c:1012:class_import_get()) import ffff882036a02800 refcount=5 obd=oak-OST0023-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.706196:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.706197:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.706198:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.706199:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.706201:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.706202:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.706203:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.706204:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.706205:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.706206:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.706207:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.706208:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.706209:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.706215:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.706217:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.706218:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.706219:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.706220:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.706221:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666301936/t0(0) o19->oak-OST0023-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.706226:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.706226:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.706228:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.706228:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.706230:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301936:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.706232:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.706233:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.706233:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.706234:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.706235:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.706236:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.706238:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.706239:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.706241:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301936, portal 4 00000100:00000001:15.0:1506921874.706242:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.706243:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.706245:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666301936/t0(0) o19->oak-OST0023-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.706253:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.706255:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921874.706256:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fac900. 00000100:00000200:15.0:1506921874.706257:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301936, offset 0 00000400:00000010:15.0:1506921874.706259:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451f600 (tot 59423868). 00000400:00000200:15.0:1506921874.706261:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921874.706268:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921874.706271:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921874.706275:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.706276:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921874.706278:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921874.706280:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921874.706281:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.706282:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.706283:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.706284:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:16.2:1506921874.706285:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:15.0:1506921874.706285:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.706286:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.706287:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.706288:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.706289:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.706290:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.0:1506921874.706292:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921874.706296:0:3861:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000800:00000200:18.0:1506921874.706299:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:16.0:1506921874.706299:0:3861:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921874.706302:0:3861:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666301936/t0(0) o19->oak-OST0023-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.706307:0:3861:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921874.706309:0:3861:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666301936/t0(0) o19->oak-OST0023-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.706313:0:3861:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.706315:0:3861:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921874.706316:0:3861:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fac900 00000400:00000010:16.0:1506921874.706317:0:3861:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fac900. 00000400:00000010:16.0:1506921874.706320:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451f600 (tot 59423468). 00000800:00000200:16.0:1506921874.706322:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:16.2:1506921874.706399:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921874.706405:0:3858:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.706409:0:3858:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921874.706412:0:3858:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000800:00000200:18.0:1506921874.706413:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000010:16.0:1506921874.706414:0:3858:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.706417:0:3858:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c2255f0 00000400:00000200:16.0:1506921874.706421:0:3858:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1cd1 [1] + 192 00000100:00000001:16.0:1506921874.706424:0:3858:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.706425:0:3858:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666301936/t0(0) o19->oak-OST0023-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:16.0:1506921874.706431:0:3858:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666301936/t0(0) o19->oak-OST0023-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.706437:0:3858:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.706438:0:3858:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.706439:0:3858:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.706441:0:3858:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921874.706441:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:16.0:1506921874.706442:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000400:00000001:15.0:1506921874.706443:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921874.706444:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.706445:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.706447:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666301936/t0(0) o19->oak-OST0023-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.706452:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666301936/t0(0) o19->oak-OST0023-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.706456:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.706458:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.706459:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.706460:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.706462:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.706463:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.706464:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.706464:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.706465:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.706467:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.706468:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff882036a02c00 time=69 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921874.706470:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff882036a02bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.706472:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.706473:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.706474:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.706475:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.706476:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.706477:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0023-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.706478:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.706478:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.706480:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666301936/t0(0) o19->oak-OST0023-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.706484:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.706486:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.706486:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.706488:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666301936/t0(0) o19->oak-OST0023-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.706493:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301936:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.706495:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.706497:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.706498:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.706499:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.706501:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666301936/t0(0) o19->oak-OST0023-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.706505:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.706506:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.706507:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.706508:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.706509:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.706511:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666301936/t0(0) o19->oak-OST0023-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.706515:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.706516:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.706516:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.706518:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.706518:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.706519:0:38877:0:(genops.c:1026:class_import_put()) import ffff882036a02800 refcount=4 obd=oak-OST0023-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.706521:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.706521:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.706523:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.706523:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.706524:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.706526:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.706526:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.706527:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.706528:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.706529:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.706530:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.706531:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.706532:0:38877:0:(genops.c:1012:class_import_get()) import ffff882036a04800 refcount=5 obd=oak-OST0024-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.706534:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.706534:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.706535:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.706536:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.706538:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.706539:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.706540:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.706541:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.706542:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.706543:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.706544:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.706545:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.706546:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.706552:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.706553:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.706555:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.706556:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.706557:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.706558:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666301952/t0(0) o19->oak-OST0024-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.706562:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.706563:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.706564:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.706565:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.706567:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301952:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.706568:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.706569:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.706570:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.706571:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.706572:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.706573:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.706574:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.706575:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.706577:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301952, portal 4 00000100:00000001:15.0:1506921874.706578:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.706579:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.706581:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666301952/t0(0) o19->oak-OST0024-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.706585:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.706587:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921874.706589:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fac900. 00000100:00000200:15.0:1506921874.706590:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301952, offset 0 00000400:00000010:15.0:1506921874.706592:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b04519a00 (tot 59423868). 00000400:00000200:15.0:1506921874.706594:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921874.706600:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921874.706603:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921874.706606:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.706607:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921874.706609:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921874.706619:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921874.706620:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.706621:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.706622:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.706623:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921874.706623:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.706624:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.706626:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.706627:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.706628:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.706628:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921874.706651:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921874.706694:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:13.0:1506921874.706700:0:3863:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:13.0:1506921874.706703:0:3863:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:13.0:1506921874.706707:0:3863:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666301952/t0(0) o19->oak-OST0024-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.706715:0:3863:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000800:00000200:16.2:1506921874.706720:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000100:00000040:13.0:1506921874.706726:0:3863:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666301952/t0(0) o19->oak-OST0024-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:5.0:1506921874.706726:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000001:13.0:1506921874.706732:0:3863:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.706734:0:3863:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:13.0:1506921874.706735:0:3863:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fac900 00000400:00000010:13.0:1506921874.706736:0:3863:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fac900. 00000800:00000200:16.0:1506921874.706737:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000400:00000010:13.0:1506921874.706740:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b04519a00 (tot 59423468). 00000800:00000200:16.0:1506921874.706742:0:3859:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.41@o2ib4 00000800:00000200:13.0:1506921874.706742:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000400:00000200:16.0:1506921874.706747:0:3859:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921874.706749:0:3859:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.706755:0:3859:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225600 00000400:00000200:16.0:1506921874.706760:0:3859:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a0f55 [1] + 192 00000100:00000001:16.0:1506921874.706764:0:3859:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.706767:0:3859:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666301952/t0(0) o19->oak-OST0024-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921874.706773:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921874.706774:0:3859:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666301952/t0(0) o19->oak-OST0024-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.706783:0:3859:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.706784:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.706787:0:3859:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:15.0:1506921874.706787:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:16.0:1506921874.706789:0:3859:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000400:00000001:15.0:1506921874.706789:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921874.706790:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.706791:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000800:00000200:16.0:1506921874.706793:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000200:15.0:1506921874.706793:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666301952/t0(0) o19->oak-OST0024-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.706798:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666301952/t0(0) o19->oak-OST0024-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.706802:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.706804:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.706805:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.706806:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.706807:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.706808:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.706809:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.706810:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.706811:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.706812:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.706814:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff882036a04c00 time=69 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921874.706816:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff882036a04bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.706817:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.706818:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.706819:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.706820:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.706822:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.706823:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0024-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.706824:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.706825:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.706826:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666301952/t0(0) o19->oak-OST0024-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.706831:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.706832:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.706833:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.706834:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666301952/t0(0) o19->oak-OST0024-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.706839:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301952:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.706841:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.706843:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.706844:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.706845:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.706846:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666301952/t0(0) o19->oak-OST0024-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.706850:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.706852:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.706853:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.706853:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.706855:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.706856:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666301952/t0(0) o19->oak-OST0024-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.706860:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.706861:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.706862:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.706863:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.706864:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.706865:0:38877:0:(genops.c:1026:class_import_put()) import ffff882036a04800 refcount=4 obd=oak-OST0024-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.706866:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.706867:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.706868:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.706869:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.706870:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.706871:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.706872:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.706873:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.706874:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.706875:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.706876:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.706877:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.706878:0:38877:0:(genops.c:1012:class_import_get()) import ffff882036a06800 refcount=5 obd=oak-OST0025-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.706879:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.706880:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.706881:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.706882:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.706884:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.706885:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.706886:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.706887:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.706888:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.706889:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.706890:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.706891:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.706892:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.706898:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.706900:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.706901:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.706902:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.706903:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.706904:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666301968/t0(0) o19->oak-OST0025-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.706908:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.706909:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.706910:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.706911:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.706913:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301968:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.706914:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.706916:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.706916:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.706917:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.706918:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.706919:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.706921:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.706922:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.706924:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301968, portal 4 00000100:00000001:15.0:1506921874.706925:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.706926:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.706928:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666301968/t0(0) o19->oak-OST0025-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.706932:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.706934:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921874.706935:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fada00. 00000100:00000200:15.0:1506921874.706936:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301968, offset 0 00000400:00000010:15.0:1506921874.706938:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451b800 (tot 59423868). 00000400:00000200:15.0:1506921874.706940:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921874.706946:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921874.706950:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921874.706953:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.706955:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921874.706957:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921874.706959:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921874.706960:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.706961:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.706962:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.706963:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:16.2:1506921874.706964:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:15.0:1506921874.706964:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.706965:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.706966:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.706967:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.706968:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.706969:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.0:1506921874.706971:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921874.706975:0:3860:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000800:00000200:18.0:1506921874.706978:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:16.0:1506921874.706978:0:3860:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921874.706980:0:3860:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666301968/t0(0) o19->oak-OST0025-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.706985:0:3860:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921874.706987:0:3860:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666301968/t0(0) o19->oak-OST0025-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.706992:0:3860:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.706993:0:3860:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921874.706994:0:3860:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fada00 00000400:00000010:16.0:1506921874.706996:0:3860:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fada00. 00000400:00000010:16.0:1506921874.706999:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451b800 (tot 59423468). 00000800:00000200:16.0:1506921874.707001:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921874.707107:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:11.0:1506921874.707151:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921874.707160:0:3862:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.42@o2ib4 00000400:00000200:11.0:1506921874.707166:0:3862:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:11.0:1506921874.707176:0:3862:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e15800 (tot 59423868). 00000400:00000200:11.0:1506921874.707182:0:3862:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225610 00000400:00000200:11.0:1506921874.707187:0:3862:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1cd9 [1] + 192 00000800:00000200:1.0:1506921874.707188:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000001:11.0:1506921874.707191:0:3862:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921874.707202:0:3862:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666301968/t0(0) o19->oak-OST0025-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:11.0:1506921874.707216:0:3862:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666301968/t0(0) o19->oak-OST0025-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.707222:0:3862:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921874.707224:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e15800 (tot 59423468). 00000800:00000200:11.0:1506921874.707226:0:3862:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921874.707228:0:3862:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:11.0:1506921874.707230:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921874.707256:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921874.707262:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921874.707263:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.707264:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.707268:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666301968/t0(0) o19->oak-OST0025-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.707277:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666301968/t0(0) o19->oak-OST0025-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.707283:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.707284:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.707286:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.707288:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.707289:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.707291:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.707292:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.707293:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.707294:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.707296:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.707299:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff882036a06c00 time=69 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921874.707301:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff882036a06bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.707303:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.707304:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.707305:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.707307:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.707308:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.707309:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0025-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.707311:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.707311:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.707314:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666301968/t0(0) o19->oak-OST0025-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.707319:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.707321:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.707322:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.707324:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666301968/t0(0) o19->oak-OST0025-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.707329:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301968:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.707334:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.707336:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.707337:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.707338:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.707340:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666301968/t0(0) o19->oak-OST0025-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.707344:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.707346:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.707347:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.707348:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.707350:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.707352:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666301968/t0(0) o19->oak-OST0025-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.707356:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.707356:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.707357:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.707359:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.707360:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.707361:0:38877:0:(genops.c:1026:class_import_put()) import ffff882036a06800 refcount=4 obd=oak-OST0025-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.707363:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.707364:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.707365:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.707366:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.707367:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.707368:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.707369:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.707370:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.707372:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.707373:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.707374:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.707376:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.707377:0:38877:0:(genops.c:1012:class_import_get()) import ffff882036aa0800 refcount=5 obd=oak-OST0026-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.707379:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.707380:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.707381:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.707383:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.707385:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.707387:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.707388:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.707389:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.707390:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.707392:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.707393:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.707395:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.707396:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.707404:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.707407:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.707408:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.707410:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.707410:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.707412:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666301984/t0(0) o19->oak-OST0026-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.707417:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.707418:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.707420:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.707420:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.707423:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301984:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.707425:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.707426:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.707427:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.707428:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.707429:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.707431:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.707433:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.707435:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.707437:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666301984, portal 4 00000100:00000001:15.0:1506921874.707438:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.707439:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.707442:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666301984/t0(0) o19->oak-OST0026-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.707454:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.707456:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921874.707458:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032facd80. 00000100:00000200:15.0:1506921874.707459:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666301984, offset 0 00000400:00000010:15.0:1506921874.707461:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451b600 (tot 59423868). 00000400:00000200:15.0:1506921874.707464:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921874.707473:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921874.707477:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921874.707482:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.707484:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921874.707486:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921874.707489:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921874.707490:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.707491:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.707493:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.707494:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921874.707495:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.707496:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.707498:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.707499:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.707500:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.707501:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921874.707525:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921874.707554:0:3865:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:5.0:1506921874.707557:0:3865:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:5.0:1506921874.707560:0:3865:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:5.0:1506921874.707563:0:3865:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666301984/t0(0) o19->oak-OST0026-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921874.707568:0:3865:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:5.0:1506921874.707570:0:3865:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666301984/t0(0) o19->oak-OST0026-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921874.707574:0:3865:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:5.0:1506921874.707576:0:3865:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:5.0:1506921874.707577:0:3865:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032facd80 00000400:00000010:5.0:1506921874.707578:0:3865:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032facd80. 00000400:00000010:5.0:1506921874.707581:0:3865:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451b600 (tot 59423468). 00000800:00000200:5.0:1506921874.707583:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:5.0:1506921874.707585:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:2.2:1506921874.707593:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:1.0:1506921874.707618:0:3864:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921874.707625:0:3864:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.42@o2ib4 00000400:00000200:1.0:1506921874.707632:0:3864:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:1.0:1506921874.707634:0:3864:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881c4b6cae00 (tot 59423868). 00000400:00000200:1.0:1506921874.707640:0:3864:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225620 00000400:00000200:1.0:1506921874.707644:0:3864:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a0f6d [1] + 192 00000100:00000001:1.0:1506921874.707649:0:3864:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:1.0:1506921874.707652:0:3864:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666301984/t0(0) o19->oak-OST0026-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:11.0:1506921874.707656:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:1.0:1506921874.707659:0:3864:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666301984/t0(0) o19->oak-OST0026-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921874.707667:0:3864:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:1.0:1506921874.707668:0:3864:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881c4b6cae00 (tot 59423468). 00000100:00000001:15.0:1506921874.707670:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921874.707671:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000800:00000200:1.0:1506921874.707671:0:3864:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000400:00000001:15.0:1506921874.707672:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.707673:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000800:00000200:1.0:1506921874.707673:0:3864:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:1.0:1506921874.707674:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000200:15.0:1506921874.707675:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666301984/t0(0) o19->oak-OST0026-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.707680:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666301984/t0(0) o19->oak-OST0026-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.707685:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.707686:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.707687:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.707688:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.707690:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.707691:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.707692:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.707693:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.707694:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.707695:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.707697:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff882036aa0c00 time=69 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921874.707699:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff882036aa0bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.707701:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.707702:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.707703:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.707704:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.707705:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.707706:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0026-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.707707:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.707708:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.707709:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666301984/t0(0) o19->oak-OST0026-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.707714:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.707715:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.707716:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.707729:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666301984/t0(0) o19->oak-OST0026-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.707734:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666301984:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.707737:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.707738:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.707739:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.707740:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.707741:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666301984/t0(0) o19->oak-OST0026-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.707746:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.707747:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.707748:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.707749:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.707750:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.707752:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666301984/t0(0) o19->oak-OST0026-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.707756:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.707757:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.707758:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.707759:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.707760:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.707760:0:38877:0:(genops.c:1026:class_import_put()) import ffff882036aa0800 refcount=4 obd=oak-OST0026-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.707762:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.707763:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.707764:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.707765:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.707765:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.707767:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.707768:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.707769:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.707770:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.707771:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.707772:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.707773:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.707774:0:38877:0:(genops.c:1012:class_import_get()) import ffff882036aa2800 refcount=5 obd=oak-OST0027-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.707775:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.707776:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.707777:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.707778:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.707780:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.707782:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.707782:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.707783:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.707784:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.707786:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.707786:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.707788:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.707789:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.707796:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.707798:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.707799:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.707801:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.707801:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.707803:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302000/t0(0) o19->oak-OST0027-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.707807:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.707808:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.707809:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.707810:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.707812:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666302000:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.707814:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.707815:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.707816:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.707817:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.707818:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.707819:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.707820:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.707822:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.707823:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302000, portal 4 00000100:00000001:15.0:1506921874.707825:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.707825:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.707828:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302000/t0(0) o19->oak-OST0027-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.707832:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.707834:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921874.707835:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fadb80. 00000100:00000200:15.0:1506921874.707836:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302000, offset 0 00000400:00000010:15.0:1506921874.707838:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b04518400 (tot 59423868). 00000400:00000200:15.0:1506921874.707840:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921874.707847:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921874.707851:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921874.707855:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.707857:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921874.707859:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921874.707862:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921874.707863:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.707864:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.707865:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.707866:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921874.707867:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.707868:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.707870:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.707870:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.707872:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.707872:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921874.707905:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.707922:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921874.707927:0:3861:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921874.707930:0:3861:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921874.707935:0:3861:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302000/t0(0) o19->oak-OST0027-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.707942:0:3861:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921874.707957:0:3861:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302000/t0(0) o19->oak-OST0027-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.707963:0:3861:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.707964:0:3861:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921874.707965:0:3861:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fadb80 00000800:00000200:18.0:1506921874.707966:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000010:16.0:1506921874.707967:0:3861:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fadb80. 00000400:00000010:16.0:1506921874.707971:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b04518400 (tot 59423468). 00000800:00000200:16.0:1506921874.707973:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:16.2:1506921874.707986:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921874.707993:0:3858:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.707997:0:3858:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.41@o2ib4 00000800:00000200:18.0:1506921874.708000:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000200:16.0:1506921874.708002:0:3858:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:16.0:1506921874.708004:0:3858:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.708009:0:3858:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225630 00000400:00000200:16.0:1506921874.708013:0:3858:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1ce1 [1] + 192 00000100:00000001:16.0:1506921874.708020:0:3858:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.708022:0:3858:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302000/t0(0) o19->oak-OST0027-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:16.0:1506921874.708027:0:3858:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302000/t0(0) o19->oak-OST0027-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.708035:0:3858:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.708036:0:3858:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.708038:0:3858:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.708040:0:3858:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921874.708040:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:16.0:1506921874.708041:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000400:00000001:15.0:1506921874.708042:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921874.708043:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.708044:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.708046:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302000/t0(0) o19->oak-OST0027-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.708051:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302000/t0(0) o19->oak-OST0027-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.708056:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.708057:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.708058:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.708059:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.708060:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.708061:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.708062:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.708063:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.708064:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.708066:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.708067:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff882036aa2c00 time=69 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921874.708069:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff882036aa2bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.708071:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.708072:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.708073:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.708074:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.708075:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.708077:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0027-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.708078:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.708079:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.708080:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302000/t0(0) o19->oak-OST0027-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.708085:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.708086:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.708087:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.708089:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302000/t0(0) o19->oak-OST0027-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.708093:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666302000:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.708096:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.708098:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.708099:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.708100:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.708101:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302000/t0(0) o19->oak-OST0027-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.708106:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.708107:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.708108:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.708109:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.708110:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.708112:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302000/t0(0) o19->oak-OST0027-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.708116:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.708116:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.708117:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.708118:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.708119:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.708120:0:38877:0:(genops.c:1026:class_import_put()) import ffff882036aa2800 refcount=4 obd=oak-OST0027-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.708121:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.708122:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.708124:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.708124:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.708125:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.708127:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.708128:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.708129:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.708130:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.708131:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.708132:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.708132:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.708134:0:38877:0:(genops.c:1012:class_import_get()) import ffff882036aa4800 refcount=5 obd=oak-OST0028-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.708135:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.708136:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.708136:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.708138:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.708139:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.708141:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.708141:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.708142:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.708144:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.708145:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.708145:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.708147:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.708148:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.708154:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.708156:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.708157:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.708158:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.708159:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.708160:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302016/t0(0) o19->oak-OST0028-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.708165:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.708165:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.708167:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.708167:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.708169:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666302016:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.708171:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.708172:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.708173:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.708174:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.708175:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.708176:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.708178:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.708179:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.708181:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302016, portal 4 00000100:00000001:15.0:1506921874.708182:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.708183:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.708185:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302016/t0(0) o19->oak-OST0028-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.708189:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.708191:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921874.708192:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fadd80. 00000100:00000200:15.0:1506921874.708193:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302016, offset 0 00000400:00000010:15.0:1506921874.708195:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451e800 (tot 59423868). 00000400:00000200:15.0:1506921874.708197:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921874.708203:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921874.708206:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921874.708209:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.708210:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921874.708212:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921874.708214:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921874.708215:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.708216:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.708217:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.708218:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921874.708219:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=6 : 6 : 6) 00000100:00100000:15.0:1506921874.708220:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 6 seconds 00000100:00000001:15.0:1506921874.708221:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.708222:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.708223:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.708224:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921874.708247:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921874.708275:0:3865:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:5.0:1506921874.708279:0:3865:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:5.0:1506921874.708281:0:3865:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:5.0:1506921874.708285:0:3865:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302016/t0(0) o19->oak-OST0028-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921874.708290:0:3865:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:5.0:1506921874.708292:0:3865:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302016/t0(0) o19->oak-OST0028-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921874.708296:0:3865:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:5.0:1506921874.708297:0:3865:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:5.0:1506921874.708298:0:3865:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fadd80 00000800:00000200:13.0:1506921874.708299:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000400:00000010:5.0:1506921874.708300:0:3865:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fadd80. 00000400:00000010:5.0:1506921874.708302:0:3865:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451e800 (tot 59423468). 00000800:00000200:5.0:1506921874.708304:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.2:1506921874.708352:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921874.708367:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.708373:0:3859:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921874.708378:0:3859:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921874.708381:0:3859:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.708387:0:3859:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225640 00000400:00000200:16.0:1506921874.708391:0:3859:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a0f85 [1] + 192 00000100:00000001:16.0:1506921874.708396:0:3859:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.708399:0:3859:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302016/t0(0) o19->oak-OST0028-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921874.708404:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921874.708406:0:3859:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302016/t0(0) o19->oak-OST0028-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.708415:0:3859:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.708417:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.708419:0:3859:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:15.0:1506921874.708419:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:16.0:1506921874.708421:0:3859:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000400:00000001:15.0:1506921874.708422:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000800:00000200:16.0:1506921874.708423:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000400:00000001:15.0:1506921874.708423:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.708424:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.708426:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302016/t0(0) o19->oak-OST0028-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.708431:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302016/t0(0) o19->oak-OST0028-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921881 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.708435:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.708437:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.708438:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.708439:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.708440:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.708441:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.708442:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.708442:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.708443:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.708445:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.708446:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff882036aa4c00 time=69 v=1 (1 31 1 31) 00000100:00001000:15.0:1506921874.708448:0:38877:0:(import.c:1737:at_measured()) AT ffff882036aa4c00 change: old=1 new=31 delta=30 (val=31) hist 31 31 1 31 00000100:00001000:15.0:1506921874.708451:0:38877:0:(client.c:345:ptlrpc_at_adj_service()) The RPC service estimate for oak-OST0028-osc-ffff88203bc47000 ptl 28 has changed from 1 to 31 00000100:00001000:15.0:1506921874.708453:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff882036aa4bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.708454:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.708455:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.708456:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.708457:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.708459:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.708459:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0028-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.708460:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.708461:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.708463:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302016/t0(0) o19->oak-OST0028-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921881 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.708467:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.708468:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.708469:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.708471:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302016/t0(0) o19->oak-OST0028-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921881 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.708476:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666302016:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.708478:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.708480:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.708481:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.708489:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.708491:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302016/t0(0) o19->oak-OST0028-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921881 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.708495:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.708496:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.708497:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.708498:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.708499:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.708501:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302016/t0(0) o19->oak-OST0028-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921881 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.708505:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.708505:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.708506:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.708507:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.708508:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.708509:0:38877:0:(genops.c:1026:class_import_put()) import ffff882036aa4800 refcount=4 obd=oak-OST0028-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.708510:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.708511:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.708512:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.708513:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.708514:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.708515:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.708516:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.708517:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.708518:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.708519:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.708520:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.708521:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.708522:0:38877:0:(genops.c:1012:class_import_get()) import ffff882036aa6800 refcount=5 obd=oak-OST0029-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.708523:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.708525:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.708525:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.708527:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.708528:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.708530:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.708530:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.708531:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.708532:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.708533:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.708534:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.708535:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.708536:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.708542:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.708544:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.708545:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.708547:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.708547:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.708549:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302032/t0(0) o19->oak-OST0029-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.708553:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.708554:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.708555:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.708556:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.708557:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666302032:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.708559:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.708560:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.708561:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.708562:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.708563:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.708564:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.708565:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.708567:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.708569:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302032, portal 4 00000100:00000001:15.0:1506921874.708570:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.708570:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.708573:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302032/t0(0) o19->oak-OST0029-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.708577:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.708579:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921874.708580:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad780. 00000100:00000200:15.0:1506921874.708581:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302032, offset 0 00000400:00000010:15.0:1506921874.708583:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451da00 (tot 59423868). 00000400:00000200:15.0:1506921874.708586:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921874.708592:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921874.708595:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921874.708598:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.708600:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921874.708601:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921874.708604:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921874.708605:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.708606:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.708607:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.708608:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:16.2:1506921874.708609:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:15.0:1506921874.708609:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.708610:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.708611:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.708612:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.708613:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.708614:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.0:1506921874.708616:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921874.708620:0:3860:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921874.708623:0:3860:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921874.708626:0:3860:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302032/t0(0) o19->oak-OST0029-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921874.708628:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:16.0:1506921874.708632:0:3860:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921874.708636:0:3860:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302032/t0(0) o19->oak-OST0029-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.708641:0:3860:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.708642:0:3860:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921874.708644:0:3860:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad780 00000400:00000010:16.0:1506921874.708646:0:3860:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad780. 00000400:00000010:16.0:1506921874.708649:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451da00 (tot 59423468). 00000800:00000200:16.0:1506921874.708650:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921874.708757:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:11.0:1506921874.708802:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921874.708811:0:3862:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.42@o2ib4 00000800:00000200:1.0:1506921874.708815:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000400:00000200:11.0:1506921874.708817:0:3862:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:11.0:1506921874.708819:0:3862:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e15800 (tot 59423868). 00000400:00000200:11.0:1506921874.708824:0:3862:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225650 00000400:00000200:11.0:1506921874.708829:0:3862:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1ce9 [1] + 192 00000100:00000001:11.0:1506921874.708834:0:3862:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921874.708836:0:3862:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302032/t0(0) o19->oak-OST0029-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:11.0:1506921874.708843:0:3862:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302032/t0(0) o19->oak-OST0029-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.708851:0:3862:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921874.708852:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e15800 (tot 59423468). 00000100:00000001:15.0:1506921874.708854:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921874.708855:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000800:00000200:11.0:1506921874.708855:0:3862:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000400:00000001:15.0:1506921874.708856:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.708857:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000800:00000200:11.0:1506921874.708857:0:3862:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:11.0:1506921874.708858:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000200:15.0:1506921874.708859:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302032/t0(0) o19->oak-OST0029-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.708864:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302032/t0(0) o19->oak-OST0029-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.708868:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.708869:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.708870:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.708871:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.708872:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.708873:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.708874:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.708875:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.708876:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.708877:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.708879:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff882036aa6c00 time=69 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921874.708881:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff882036aa6bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.708883:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.708884:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.708885:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.708886:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.708887:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.708888:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0029-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.708889:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.708889:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.708891:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302032/t0(0) o19->oak-OST0029-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.708895:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.708897:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.708898:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.708899:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302032/t0(0) o19->oak-OST0029-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.708904:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666302032:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.708906:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.708908:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.708909:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.708910:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.708911:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302032/t0(0) o19->oak-OST0029-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.708916:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.708917:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.708918:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.708919:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.708920:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.708922:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302032/t0(0) o19->oak-OST0029-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.708926:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.708926:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.708927:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.708929:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.708929:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.708930:0:38877:0:(genops.c:1026:class_import_put()) import ffff882036aa6800 refcount=4 obd=oak-OST0029-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.708931:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.708932:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.708933:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.708934:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.708935:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.708936:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.708937:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.708938:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.708939:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.708940:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.708941:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.708942:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.708943:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203dd40800 refcount=5 obd=oak-OST002a-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.708944:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.708945:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.708946:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.708947:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.708949:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.708950:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.708951:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.708952:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.708953:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.708954:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.708955:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.708956:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.708957:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.708963:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.708965:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.708967:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.708968:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.708968:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.708970:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302048/t0(0) o19->oak-OST002a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.708974:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.708975:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.708976:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.708977:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.708979:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666302048:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.708980:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.708981:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.708982:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.708983:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.708984:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.708985:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.708987:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.708988:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.708990:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302048, portal 4 00000100:00000001:15.0:1506921874.708991:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.708992:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.708994:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302048/t0(0) o19->oak-OST002a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.708998:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.708999:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921874.709001:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fadc80. 00000100:00000200:15.0:1506921874.709002:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302048, offset 0 00000400:00000010:15.0:1506921874.709004:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451fc00 (tot 59423868). 00000400:00000200:15.0:1506921874.709006:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921874.709012:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921874.709015:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921874.709018:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.709019:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921874.709021:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921874.709024:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921874.709025:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.709026:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.709027:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.709028:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921874.709029:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.709030:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.709031:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.709032:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.709033:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.709033:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921874.709060:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921874.709103:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:13.0:1506921874.709110:0:3863:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:13.0:1506921874.709113:0:3863:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:13.0:1506921874.709116:0:3863:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302048/t0(0) o19->oak-OST002a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.709124:0:3863:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:13.0:1506921874.709126:0:3863:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302048/t0(0) o19->oak-OST002a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.709131:0:3863:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.709133:0:3863:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:13.0:1506921874.709134:0:3863:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fadc80 00000400:00000010:13.0:1506921874.709136:0:3863:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fadc80. 00000400:00000010:13.0:1506921874.709139:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451fc00 (tot 59423468). 00000800:00000200:13.0:1506921874.709141:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:5.0:1506921874.709144:0:3865:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921874.709154:0:3865:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.42@o2ib4 00000400:00000200:5.0:1506921874.709160:0:3865:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:5.0:1506921874.709162:0:3865:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881e48bc2800 (tot 59423868). 00000400:00000200:5.0:1506921874.709168:0:3865:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225660 00000400:00000200:5.0:1506921874.709172:0:3865:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a0f9d [1] + 192 00000100:00000001:5.0:1506921874.709177:0:3865:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:5.0:1506921874.709180:0:3865:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302048/t0(0) o19->oak-OST002a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:1.0:1506921874.709184:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:5.0:1506921874.709188:0:3865:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302048/t0(0) o19->oak-OST002a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921874.709195:0:3865:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:5.0:1506921874.709197:0:3865:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881e48bc2800 (tot 59423468). 00000100:00000001:15.0:1506921874.709198:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921874.709199:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921874.709200:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000800:00000200:5.0:1506921874.709200:0:3865:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000100:00000001:15.0:1506921874.709201:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000800:00000200:5.0:1506921874.709202:0:3865:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000100:00000200:15.0:1506921874.709203:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302048/t0(0) o19->oak-OST002a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000800:00000200:5.0:1506921874.709203:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000200:15.0:1506921874.709208:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302048/t0(0) o19->oak-OST002a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.709212:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.709212:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.709214:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.709215:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.709216:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.709217:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.709217:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.709218:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.709219:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.709220:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.709222:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88203dd40c00 time=69 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921874.709224:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203dd40bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.709226:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.709227:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.709228:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.709229:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.709230:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.709231:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST002a-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.709232:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.709233:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.709234:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302048/t0(0) o19->oak-OST002a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.709239:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.709240:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.709241:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.709242:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302048/t0(0) o19->oak-OST002a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.709247:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666302048:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.709249:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.709251:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.709252:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.709253:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.709254:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302048/t0(0) o19->oak-OST002a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.709258:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.709259:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.709261:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.709261:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.709262:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.709264:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302048/t0(0) o19->oak-OST002a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.709268:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.709268:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.709269:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.709270:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.709271:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.709272:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203dd40800 refcount=4 obd=oak-OST002a-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.709273:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.709274:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.709275:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.709275:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.709277:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.709278:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.709279:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.709280:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.709281:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.709282:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.709282:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.709283:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.709285:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203dd42800 refcount=5 obd=oak-OST002b-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.709286:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.709287:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.709287:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.709289:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.709290:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.709291:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.709292:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.709293:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.709294:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.709295:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.709296:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.709297:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.709298:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.709304:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.709306:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.709307:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.709308:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.709309:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.709310:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302064/t0(0) o19->oak-OST002b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.709315:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.709315:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.709316:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.709317:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.709319:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666302064:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.709321:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.709322:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.709323:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.709324:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.709324:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.709326:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.709327:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.709328:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.709330:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302064, portal 4 00000100:00000001:15.0:1506921874.709331:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.709331:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.709334:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302064/t0(0) o19->oak-OST002b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.709338:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.709339:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921874.709340:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fadf00. 00000100:00000200:15.0:1506921874.709342:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302064, offset 0 00000400:00000010:15.0:1506921874.709343:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b04519400 (tot 59423868). 00000400:00000200:15.0:1506921874.709346:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921874.709352:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921874.709355:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921874.709358:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.709360:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921874.709361:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921874.709363:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921874.709364:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.709365:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.709366:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.709367:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921874.709368:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.709369:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.709370:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.709371:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.709372:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.709373:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921874.709401:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.709418:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921874.709423:0:3861:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921874.709427:0:3861:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921874.709431:0:3861:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302064/t0(0) o19->oak-OST002b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.709438:0:3861:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921874.709441:0:3861:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302064/t0(0) o19->oak-OST002b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.709446:0:3861:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.709448:0:3861:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921874.709449:0:3861:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fadf00 00000400:00000010:16.0:1506921874.709451:0:3861:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fadf00. 00000400:00000010:16.0:1506921874.709455:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b04519400 (tot 59423468). 00000800:00000200:18.0:1506921874.709456:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921874.709460:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:16.2:1506921874.709490:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921874.709498:0:3858:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.709502:0:3858:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.41@o2ib4 00000800:00000200:18.0:1506921874.709505:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000200:16.0:1506921874.709507:0:3858:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:16.0:1506921874.709510:0:3858:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.709515:0:3858:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225670 00000400:00000200:16.0:1506921874.709520:0:3858:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1cf1 [1] + 192 00000100:00000001:16.0:1506921874.709524:0:3858:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.709526:0:3858:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302064/t0(0) o19->oak-OST002b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:16.0:1506921874.709532:0:3858:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302064/t0(0) o19->oak-OST002b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.709547:0:3858:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.709548:0:3858:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.709550:0:3858:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:15.0:1506921874.709551:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:16.0:1506921874.709553:0:3858:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921874.709554:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000400:00000001:15.0:1506921874.709554:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921874.709555:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.709556:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.709558:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302064/t0(0) o19->oak-OST002b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.709563:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302064/t0(0) o19->oak-OST002b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.709567:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.709569:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.709570:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.709571:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.709572:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.709573:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.709574:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.709574:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.709575:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.709577:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.709578:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88203dd42c00 time=69 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921874.709580:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203dd42bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.709582:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.709583:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.709584:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.709585:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.709586:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.709587:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST002b-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.709588:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.709588:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.709590:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302064/t0(0) o19->oak-OST002b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.709594:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.709596:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.709596:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.709598:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302064/t0(0) o19->oak-OST002b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.709603:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666302064:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.709607:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.709609:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.709610:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.709611:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.709613:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302064/t0(0) o19->oak-OST002b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.709617:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.709618:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.709619:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.709620:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.709621:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.709622:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302064/t0(0) o19->oak-OST002b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.709626:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.709627:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.709628:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.709629:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.709630:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.709630:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203dd42800 refcount=4 obd=oak-OST002b-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.709632:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.709633:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.709634:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.709635:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.709635:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.709637:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.709638:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.709639:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.709640:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.709641:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.709642:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.709642:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.709644:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203dd44800 refcount=5 obd=oak-OST002c-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.709645:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.709646:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.709646:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.709648:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.709650:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.709651:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.709652:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.709653:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.709654:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.709655:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.709655:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.709657:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.709658:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.709664:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.709666:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.709667:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.709668:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.709669:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.709670:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302080/t0(0) o19->oak-OST002c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.709674:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.709675:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.709676:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.709677:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.709679:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666302080:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.709680:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.709681:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.709682:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.709683:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.709684:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.709685:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.709686:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.709688:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.709689:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302080, portal 4 00000100:00000001:15.0:1506921874.709691:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.709691:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.709694:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302080/t0(0) o19->oak-OST002c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.709702:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.709703:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921874.709704:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad980. 00000100:00000200:15.0:1506921874.709706:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302080, offset 0 00000400:00000010:15.0:1506921874.709707:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451c600 (tot 59423868). 00000400:00000200:15.0:1506921874.709710:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921874.709716:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921874.709729:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921874.709732:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.709734:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921874.709735:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921874.709737:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921874.709738:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.709739:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.709740:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.709741:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921874.709742:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.709743:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.709744:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.709745:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.709746:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.709747:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921874.709772:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921874.709814:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:11.0:1506921874.709820:0:3862:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:11.0:1506921874.709823:0:3862:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:11.0:1506921874.709827:0:3862:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302080/t0(0) o19->oak-OST002c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.709833:0:3862:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921874.709835:0:3862:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302080/t0(0) o19->oak-OST002c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.709847:0:3862:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921874.709848:0:3862:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:11.0:1506921874.709849:0:3862:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad980 00000400:00000010:11.0:1506921874.709850:0:3862:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad980. 00000800:00000200:13.0:1506921874.709853:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000400:00000010:11.0:1506921874.709854:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451c600 (tot 59423468). 00000800:00000200:11.0:1506921874.709857:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.2:1506921874.709900:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921874.709917:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.709923:0:3859:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921874.709929:0:3859:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921874.709932:0:3859:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921874.709937:0:3859:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225680 00000400:00000200:16.0:1506921874.709942:0:3859:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a0fb5 [1] + 192 00000100:00000001:16.0:1506921874.709947:0:3859:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921874.709950:0:3859:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302080/t0(0) o19->oak-OST002c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921874.709955:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921874.709957:0:3859:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302080/t0(0) o19->oak-OST002c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.709966:0:3859:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921874.709968:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921874.709971:0:3859:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:15.0:1506921874.709971:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:16.0:1506921874.709973:0:3859:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000400:00000001:15.0:1506921874.709973:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921874.709973:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.709974:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000800:00000200:16.0:1506921874.709975:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000200:15.0:1506921874.709977:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302080/t0(0) o19->oak-OST002c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.709981:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302080/t0(0) o19->oak-OST002c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.709986:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.709987:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.709988:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.709989:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.709991:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.709992:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.709993:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.709994:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.709995:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.709996:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.709998:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88203dd44c00 time=69 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921874.710000:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203dd44bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.710002:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.710002:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.710003:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.710004:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.710006:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.710006:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST002c-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.710008:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.710008:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.710010:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302080/t0(0) o19->oak-OST002c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.710014:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.710015:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.710016:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.710018:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302080/t0(0) o19->oak-OST002c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.710023:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666302080:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.710025:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.710027:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.710028:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.710029:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.710030:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302080/t0(0) o19->oak-OST002c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.710034:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.710035:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.710037:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.710037:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.710039:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.710041:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302080/t0(0) o19->oak-OST002c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.710045:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.710046:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.710046:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.710048:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.710048:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.710049:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203dd44800 refcount=4 obd=oak-OST002c-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.710050:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.710051:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.710053:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.710053:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.710054:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.710055:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.710056:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.710057:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.710058:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.710059:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.710060:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.710061:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.710062:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203dd46800 refcount=5 obd=oak-OST002d-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.710064:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.710064:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.710065:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.710066:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.710068:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.710069:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.710070:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.710071:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.710072:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.710073:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.710074:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.710075:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.710076:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.710082:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.710084:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.710086:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.710087:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.710087:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.710089:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302096/t0(0) o19->oak-OST002d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.710093:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.710094:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.710095:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.710096:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.710098:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666302096:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.710099:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.710100:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.710101:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.710102:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.710103:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.710104:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.710105:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.710106:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.710108:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302096, portal 4 00000100:00000001:15.0:1506921874.710109:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.710110:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.710112:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302096/t0(0) o19->oak-OST002d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.710116:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.710130:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921874.710131:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad300. 00000100:00000200:15.0:1506921874.710132:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302096, offset 0 00000400:00000010:15.0:1506921874.710134:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b04519200 (tot 59423868). 00000400:00000200:15.0:1506921874.710136:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921874.710142:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921874.710145:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921874.710148:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.710150:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921874.710152:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921874.710155:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921874.710156:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.710156:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.710158:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.710158:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:16.2:1506921874.710159:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:15.0:1506921874.710159:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.710160:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.710161:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.710162:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.710163:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.710164:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.0:1506921874.710166:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921874.710171:0:3860:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921874.710174:0:3860:0:(events.c:52:request_out_callback()) Process entered 00000800:00000200:18.0:1506921874.710176:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000200:16.0:1506921874.710177:0:3860:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302096/t0(0) o19->oak-OST002d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.710182:0:3860:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921874.710184:0:3860:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302096/t0(0) o19->oak-OST002d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.710188:0:3860:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921874.710190:0:3860:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921874.710192:0:3860:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad300 00000400:00000010:16.0:1506921874.710195:0:3860:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad300. 00000400:00000010:16.0:1506921874.710198:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b04519200 (tot 59423468). 00000800:00000200:16.0:1506921874.710200:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921874.710261:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:1.0:1506921874.710301:0:3864:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921874.710310:0:3864:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.42@o2ib4 00000400:00000200:1.0:1506921874.710317:0:3864:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:1.0:1506921874.710319:0:3864:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881c4b6cae00 (tot 59423868). 00000400:00000200:1.0:1506921874.710325:0:3864:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225690 00000400:00000200:1.0:1506921874.710330:0:3864:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1cf9 [1] + 192 00000100:00000001:1.0:1506921874.710335:0:3864:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:1.0:1506921874.710338:0:3864:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302096/t0(0) o19->oak-OST002d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:5.0:1506921874.710342:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:1.0:1506921874.710346:0:3864:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302096/t0(0) o19->oak-OST002d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921874.710356:0:3864:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:1.0:1506921874.710358:0:3864:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881c4b6cae00 (tot 59423468). 00000100:00000001:15.0:1506921874.710359:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921874.710360:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000800:00000200:1.0:1506921874.710360:0:3864:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000400:00000001:15.0:1506921874.710361:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921874.710361:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.710363:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302096/t0(0) o19->oak-OST002d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000800:00000200:1.0:1506921874.710363:0:3864:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:1.0:1506921874.710364:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000200:15.0:1506921874.710367:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302096/t0(0) o19->oak-OST002d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.710371:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.710372:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.710373:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.710374:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.710375:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.710376:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.710376:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.710377:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.710378:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.710379:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.710381:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88203dd46c00 time=69 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921874.710383:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203dd46bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.710384:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.710385:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.710386:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.710387:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.710388:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.710389:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST002d-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.710390:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.710391:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.710392:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302096/t0(0) o19->oak-OST002d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.710396:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.710397:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.710398:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.710399:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302096/t0(0) o19->oak-OST002d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.710404:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666302096:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.710406:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.710408:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.710408:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.710410:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.710411:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302096/t0(0) o19->oak-OST002d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.710415:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.710416:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.710417:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.710418:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.710419:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.710420:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302096/t0(0) o19->oak-OST002d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.710424:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.710424:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.710425:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.710426:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.710427:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.710427:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203dd46800 refcount=4 obd=oak-OST002d-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.710429:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.710429:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.710430:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.710431:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.710432:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.710433:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.710434:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.710435:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.710436:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.710437:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.710437:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.710438:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.710439:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203ddf0800 refcount=5 obd=oak-OST002e-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.710440:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.710441:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.710442:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.710443:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.710445:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.710446:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.710446:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.710447:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.710448:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.710449:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.710450:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.710451:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.710452:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.710457:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.710459:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.710460:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.710461:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.710462:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.710463:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302112/t0(0) o19->oak-OST002e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.710467:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.710467:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.710468:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.710469:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.710471:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666302112:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.710472:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.710474:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.710474:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.710475:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.710476:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.710477:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.710478:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.710480:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.710481:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302112, portal 4 00000100:00000001:15.0:1506921874.710482:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.710483:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.710485:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302112/t0(0) o19->oak-OST002e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.710489:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.710490:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921874.710491:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fadc00. 00000100:00000200:15.0:1506921874.710492:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302112, offset 0 00000400:00000010:15.0:1506921874.710494:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b04518c00 (tot 59423868). 00000400:00000200:15.0:1506921874.710496:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921874.710502:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921874.710505:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921874.710507:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.710508:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921874.710510:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921874.710512:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921874.710513:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.710514:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.710515:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.710516:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921874.710516:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.710517:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.710518:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.710519:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.710520:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.710521:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921874.710551:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921874.710593:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:13.0:1506921874.710600:0:3863:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:13.0:1506921874.710604:0:3863:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:13.0:1506921874.710608:0:3863:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302112/t0(0) o19->oak-OST002e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.710616:0:3863:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:13.0:1506921874.710618:0:3863:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302112/t0(0) o19->oak-OST002e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921874.710623:0:3863:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921874.710625:0:3863:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:13.0:1506921874.710626:0:3863:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fadc00 00000400:00000010:13.0:1506921874.710627:0:3863:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fadc00. 00000400:00000010:13.0:1506921874.710631:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b04518c00 (tot 59423468). 00000800:00000200:11.0:1506921874.710631:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:13.0:1506921874.710638:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (23)-- 00000800:00000200:13.0:1506921874.710641:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:11.0:1506921874.710641:0:3862:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.42@o2ib4 00000400:00000200:11.0:1506921874.710647:0:3862:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:11.0:1506921874.710649:0:3862:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e15800 (tot 59423868). 00000400:00000200:11.0:1506921874.710654:0:3862:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c2256a0 00000400:00000200:11.0:1506921874.710658:0:3862:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a0fcd [1] + 192 00000100:00000001:11.0:1506921874.710663:0:3862:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921874.710665:0:3862:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302112/t0(0) o19->oak-OST002e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:11.0:1506921874.710672:0:3862:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302112/t0(0) o19->oak-OST002e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921874.710680:0:3862:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921874.710681:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e15800 (tot 59423468). 00000100:00000001:15.0:1506921874.710682:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:11.0:1506921874.710683:0:3862:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000400:00000001:15.0:1506921874.710684:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921874.710685:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000800:00000200:11.0:1506921874.710685:0:3862:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921874.710686:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.710687:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302112/t0(0) o19->oak-OST002e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000800:00000200:11.0:1506921874.710687:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000200:15.0:1506921874.710692:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302112/t0(0) o19->oak-OST002e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.710695:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.710696:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.710697:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.710698:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.710699:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.710700:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.710701:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.710702:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.710702:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.710703:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.710705:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88203ddf0c00 time=69 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921874.710707:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203ddf0bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.710708:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.710709:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.710710:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.710711:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.710712:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.710712:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST002e-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.710713:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.710714:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.710715:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302112/t0(0) o19->oak-OST002e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.710726:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.710727:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.710728:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.710729:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302112/t0(0) o19->oak-OST002e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.710734:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666302112:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921874.710740:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.710742:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.710743:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.710744:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.710745:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302112/t0(0) o19->oak-OST002e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.710749:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.710750:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.710751:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.710752:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.710753:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.710754:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302112/t0(0) o19->oak-OST002e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.710758:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.710758:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.710759:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.710760:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.710761:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.710762:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203ddf0800 refcount=4 obd=oak-OST002e-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.710763:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.710763:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.710764:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.710765:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.710766:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.710767:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.710767:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.710768:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.710769:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.710770:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921874.710771:0:38877:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.710772:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.710773:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203ddf2800 refcount=5 obd=oak-OST002f-osc-ffff88203bc47000 00000100:00000001:15.0:1506921874.710774:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.710775:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.710776:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.710777:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.710779:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.710780:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.710780:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.710781:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.710782:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.710783:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.710784:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.710785:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.710786:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.710792:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.710794:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.710795:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.710796:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.710796:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.710798:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302128/t0(0) o19->oak-OST002f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.710801:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.710802:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.710803:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.710804:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.710805:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666302128:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.710807:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.710808:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.710809:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.710810:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.710810:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.710811:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.710813:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.710814:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.710815:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302128, portal 4 00000100:00000001:15.0:1506921874.710817:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.710817:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.710820:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302128/t0(0) o19->oak-OST002f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.710827:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.710828:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921874.710829:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad680. 00000100:00000200:15.0:1506921874.710830:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302128, offset 0 00000400:00000010:15.0:1506921874.710832:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b04519600 (tot 59423868). 00000400:00000200:15.0:1506921874.710834:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921874.710840:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921874.710842:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921874.710846:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.710847:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921874.710848:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921874.710851:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921874.710852:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.710853:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.710854:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.710855:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921874.710856:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921874.710856:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921874.710858:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.710858:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.710859:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.710860:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921874.710894:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921874.710922:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921874.710927:0:3861:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921874.710930:0:3861:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921874.710934:0:3861:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302128/t0(0) o19->oak-OST002f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921874.710941:0:3861:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000800:00000200:18.0:1506921874.710944:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921874.710944:0:3861:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302128/t0(0) o19->oak-OST002f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:16.2:1506921874.710950:0:3861:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:16.0:1506921874.710956:0:3861:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:18.0:1506921874.710965:0:3858:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000100:00000001:16.0:1506921874.710965:0:3861:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921874.710966:0:3861:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad680 00000400:00000010:16.0:1506921874.710968:0:3861:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad680. 00000400:00000010:16.0:1506921874.710971:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b04519600 (tot 59423468). 00000800:00000200:18.0:1506921874.710974:0:3858:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.41@o2ib4 00000800:00000200:16.0:1506921874.710974:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (23)-- 00000800:00000200:16.0:1506921874.710976:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000200:18.0:1506921874.710980:0:3858:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:18.0:1506921874.710984:0:3858:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103a767000 (tot 59423868). 00000400:00000200:18.0:1506921874.710990:0:3858:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c2256b0 00000400:00000200:18.0:1506921874.710995:0:3858:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1d01 [1] + 192 00000100:00000001:18.0:1506921874.711000:0:3858:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:18.0:1506921874.711003:0:3858:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302128/t0(0) o19->oak-OST002f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:18.0:1506921874.711011:0:3858:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302128/t0(0) o19->oak-OST002f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:18.0:1506921874.711020:0:3858:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:18.0:1506921874.711022:0:3858:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103a767000 (tot 59423468). 00000800:00000200:18.0:1506921874.711024:0:3858:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:15.0:1506921874.711025:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921874.711026:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000800:00000200:18.0:1506921874.711027:0:3858:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000400:00000001:15.0:1506921874.711027:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000800:00000200:18.0:1506921874.711028:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:15.0:1506921874.711028:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921874.711030:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302128/t0(0) o19->oak-OST002f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921874.711035:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302128/t0(0) o19->oak-OST002f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.711039:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.711040:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.711041:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.711042:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.711044:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.711045:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.711046:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.711047:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.711048:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.711049:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.711051:0:38877:0:(import.c:1678:at_measured()) add 31 to ffff88203ddf2c00 time=69 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921874.711052:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203ddf2bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.711054:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.711054:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.711055:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.711056:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.711057:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.711058:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-OST002f-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.711059:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.711060:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.711061:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302128/t0(0) o19->oak-OST002f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.711065:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.711066:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.711067:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.711069:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302128/t0(0) o19->oak-OST002f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.711073:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666302128:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921874.711075:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.711077:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.711078:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.711079:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.711080:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302128/t0(0) o19->oak-OST002f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.711084:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.711085:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.711086:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.711087:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.711088:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.711090:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302128/t0(0) o19->oak-OST002f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921918 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.711093:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.711094:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.711095:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.711096:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.711097:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.711097:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203ddf2800 refcount=4 obd=oak-OST002f-osc-ffff88203bc47000 00000020:00000001:15.0:1506921874.711099:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.711099:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.711100:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.711101:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.711102:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.711103:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.711104:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921874.711105:0:38877:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.711106:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921874.711107:0:38877:0:(obd_class.h:986:obd_putref()) Process entered 00020000:00000001:15.0:1506921874.711109:0:38877:0:(obd_class.h:991:obd_putref()) Process leaving 00020000:00000001:15.0:1506921874.711110:0:38877:0:(lov_obd.c:1396:lov_quotactl()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:15.0:1506921874.711111:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:15.0:1506921874.711112:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00800000:00000001:15.0:1506921874.711114:0:38877:0:(lmv_obd.c:3038:lmv_quotactl()) Process entered 00800000:00000001:15.0:1506921874.711114:0:38877:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000002:00000001:15.0:1506921874.711115:0:38877:0:(mdc_request.c:1808:mdc_quotactl()) Process entered 00000100:00000010:15.0:1506921874.711117:0:38877:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921874.711118:0:38877:0:(genops.c:1012:class_import_get()) import ffff88203bc57800 refcount=4 obd=oak-MDT0000-mdc-ffff88203bc47000 00000100:00000001:15.0:1506921874.711119:0:38877:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921874.711120:0:38877:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921874.711121:0:38877:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921874.711122:0:38877:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921874.711123:0:38877:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.711125:0:38877:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921874.711125:0:38877:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921874.711126:0:38877:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.711127:0:38877:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921874.711128:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.711129:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921874.711131:0:38877:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921874.711132:0:38877:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921874.711137:0:38877:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921874.711139:0:38877:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921874.711140:0:38877:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921874.711141:0:38877:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921874.711141:0:38877:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921874.711143:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302144/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.711147:0:38877:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921874.711147:0:38877:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.711149:0:38877:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921874.711149:0:38877:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921874.711151:0:38877:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666302144:10.0.2.52@o2ib5:48 00000100:00000001:15.0:1506921874.711153:0:38877:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921874.711153:0:38877:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921874.711154:0:38877:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.711155:0:38877:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921874.711156:0:38877:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.711157:0:38877:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921874.711158:0:38877:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921874.711160:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921874.711162:0:38877:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302144, portal 10 00000100:00000001:15.0:1506921874.711163:0:38877:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921874.711163:0:38877:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921874.711166:0:38877:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302144/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921881 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.711169:0:38877:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921874.711171:0:38877:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.52@o2ib5 00000400:00000010:15.0:1506921874.711172:0:38877:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fac300. 00000100:00000200:15.0:1506921874.711173:0:38877:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 12, xid 1580016666302144, offset 0 00000400:00000010:15.0:1506921874.711175:0:38877:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451c200 (tot 59423868). 00000400:00000200:15.0:1506921874.711177:0:38877:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.52@o2ib5 00000400:00000200:15.0:1506921874.711184:0:38877:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.52@o2ib5(10.0.2.52@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921874.711187:0:38877:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921874.711189:0:38877:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921874.711190:0:38877:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921874.711192:0:38877:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921874.711194:0:38877:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921874.711195:0:38877:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.711196:0:38877:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.711197:0:38877:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.711198:0:38877:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921874.711198:0:38877:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=6 : 6 : 6) 00000100:00100000:15.0:1506921874.711199:0:38877:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 6 seconds 00000100:00000001:15.0:1506921874.711200:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.711201:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.711202:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921874.711203:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921874.711227:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921874.711272:0:3864:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:1.0:1506921874.711279:0:3864:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:1.0:1506921874.711282:0:3864:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:1.0:1506921874.711286:0:3864:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302144/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921881 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921874.711294:0:3864:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1506921874.711296:0:3864:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302144/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921881 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921874.711301:0:3864:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921874.711303:0:3864:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:1.0:1506921874.711304:0:3864:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fac300 00000400:00000010:1.0:1506921874.711305:0:3864:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fac300. 00000400:00000010:1.0:1506921874.711309:0:3864:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451c200 (tot 59423468). 00000800:00000200:13.0:1506921874.711311:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:1.0:1506921874.711315:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:2.2:1506921874.711527:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:5.0:1506921874.711570:0:3865:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921874.711579:0:3865:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.42@o2ib4 00000400:00000200:5.0:1506921874.711585:0:3865:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.52@o2ib5 : PUT 00000400:00000010:5.0:1506921874.711587:0:3865:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881e48bc2800 (tot 59423868). 00000400:00000200:5.0:1506921874.711592:0:3865:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.52@o2ib5 of length 304 into portal 10 MB=0x59d044c2256c0 00000400:00000200:5.0:1506921874.711599:0:3865:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index a from 12345-10.0.2.52@o2ib5 of length 304/304 into md 0x7a0fe5 [1] + 192 00000100:00000001:5.0:1506921874.711604:0:3865:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:5.0:1506921874.711607:0:3865:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302144/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921881 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:11.0:1506921874.711610:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:5.0:1506921874.711615:0:3865:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302144/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921881 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921874.711623:0:3865:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:5.0:1506921874.711624:0:3865:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881e48bc2800 (tot 59423468). 00000100:00000001:15.0:1506921874.711625:0:38877:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921874.711627:0:38877:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921874.711627:0:38877:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000800:00000200:5.0:1506921874.711627:0:3865:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000100:00000001:15.0:1506921874.711628:0:38877:0:(events.c:91:reply_in_callback()) Process entered 00000800:00000200:5.0:1506921874.711629:0:3865:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000100:00000200:15.0:1506921874.711630:0:38877:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302144/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921881 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000800:00000200:5.0:1506921874.711631:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000200:15.0:1506921874.711635:0:38877:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302144/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921881 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921874.711639:0:38877:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921874.711640:0:38877:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921874.711641:0:38877:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921874.711642:0:38877:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921874.711643:0:38877:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.711644:0:38877:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921874.711644:0:38877:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921874.711645:0:38877:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921874.711646:0:38877:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921874.711647:0:38877:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921874.711649:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203bc57c00 time=69 v=1 (1 1 1 1) 00000100:00001000:15.0:1506921874.711651:0:38877:0:(import.c:1678:at_measured()) add 1 to ffff88203bc57bc8 time=69 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921874.711653:0:38877:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921874.711654:0:38877:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921874.711655:0:38877:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921874.711656:0:38877:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.711657:0:38877:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921874.711657:0:38877:0:(client.c:2683:ptlrpc_free_committed()) oak-MDT0000-mdc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921874.711658:0:38877:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921874.711659:0:38877:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921874.711661:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302144/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/304 e 0 to 0 dl 1506921881 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.711665:0:38877:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921874.711666:0:38877:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921874.711667:0:38877:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921874.711669:0:38877:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302144/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/304 e 0 to 0 dl 1506921881 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921874.711673:0:38877:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38877:1580016666302144:10.0.2.52@o2ib5:48 00000100:00000001:15.0:1506921874.711675:0:38877:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921874.711677:0:38877:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.711678:0:38877:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921874.711678:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.711680:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302144/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/304 e 0 to 0 dl 1506921881 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.711684:0:38877:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921874.711685:0:38877:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921874.711686:0:38877:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921874.711687:0:38877:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921874.711688:0:38877:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921874.711690:0:38877:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302144/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/304 e 0 to 0 dl 1506921881 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921874.711694:0:38877:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921874.711694:0:38877:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921874.711695:0:38877:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921874.711696:0:38877:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921874.711697:0:38877:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921874.711698:0:38877:0:(genops.c:1026:class_import_put()) import ffff88203bc57800 refcount=3 obd=oak-MDT0000-mdc-ffff88203bc47000 00000020:00000001:15.0:1506921874.711699:0:38877:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921874.711700:0:38877:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921874.711701:0:38877:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921874.711702:0:38877:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921874.711703:0:38877:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921874.711704:0:38877:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921874.711705:0:38877:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:15.0:1506921874.711706:0:38877:0:(mdc_request.c:1836:mdc_quotactl()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:15.0:1506921874.711707:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:15.0:1506921874.711708:0:38877:0:(lmv_obd.c:3074:lmv_quotactl()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:15.0:1506921874.711710:0:38877:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:15.0:1506921874.711711:0:38877:0:(dir.c:1115:quotactl_ioctl()) kfreed 'oqctl_tmp': 112 at ffff881032734500. 00000080:00000010:15.0:1506921874.711713:0:38877:0:(dir.c:1119:quotactl_ioctl()) kfreed 'oqctl': 112 at ffff881032734280. 00000080:00000001:15.0:1506921874.711714:0:38877:0:(dir.c:1122:quotactl_ioctl()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:15.0:1506921874.711716:0:38877:0:(dir.c:1543:ll_dir_ioctl()) kfreed 'qctl': 176 at ffff8820382d7080. 00000080:00000001:15.0:1506921874.711724:0:38877:0:(dir.c:1544:ll_dir_ioctl()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:15.0:1506921874.711730:0:38877:0:(dir.c:1831:ll_dir_release()) Process entered 00000080:00000001:15.0:1506921874.711731:0:38877:0:(file.c:316:ll_file_release()) Process entered 00000080:00200000:15.0:1506921874.711732:0:38877:0:(file.c:319:ll_file_release()) VFS Op:inode=[0x200000007:0x1:0x0](ffff881021f03690) 00000080:00400000:15.0:1506921874.711735:0:38877:0:(statahead.c:1237:ll_deauthorize_statahead()) deauthorize statahead for [0x200000007:0x1:0x0] 00000080:00000010:15.0:1506921874.711737:0:38877:0:(file.c:83:ll_file_data_put()) slab-freed '(fd)': 216 at ffff882035b5cb00. 00000080:00000001:15.0:1506921874.711739:0:38877:0:(file.c:334:ll_file_release()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:15.0:1506921874.711740:0:38877:0:(dir.c:1832:ll_dir_release()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921875.035774:0:3901:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921875.035776:0:3901:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921875.035779:0:3901:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921875.035780:0:3901:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921875.035781:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921875.035784:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921875.035785:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921875.035787:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921875.040767:0:3913:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:7.0:1506921875.040769:0:3913:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:7.0:1506921875.040772:0:3913:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:7.0:1506921875.040773:0:3913:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921875.040775:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921875.040778:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921875.040780:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921875.040780:0:3918:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:7.0:1506921875.040781:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921875.040782:0:3918:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921875.040785:0:3918:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921875.040786:0:3912:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1506921875.040786:0:3918:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921875.040787:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921875.040788:0:3912:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921875.040790:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921875.040791:0:3912:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921875.040792:0:3912:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921875.040792:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921875.040793:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921875.040793:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921875.040796:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921875.040798:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921875.040798:0:3917:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921875.040799:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921875.040800:0:3917:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:17.0:1506921875.040803:0:3917:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:17.0:1506921875.040804:0:3917:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921875.040806:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921875.040809:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921875.040810:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921875.040811:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921875.041772:0:3919:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:7.0:1506921875.041774:0:3919:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921875.041774:0:3920:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1506921875.041776:0:3920:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:7.0:1506921875.041777:0:3919:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:7.0:1506921875.041778:0:3919:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921875.041779:0:3920:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:7.0:1506921875.041780:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921875.041780:0:3920:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921875.041782:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921875.041783:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921875.041784:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921875.041785:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921875.041785:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921875.041786:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921875.041788:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921875.041802:0:3915:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:13.0:1506921875.041804:0:3915:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:13.0:1506921875.041807:0:3915:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921875.041808:0:3914:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:13.0:1506921875.041808:0:3915:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921875.041809:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921875.041810:0:3914:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:13.0:1506921875.041812:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921875.041813:0:3914:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921875.041814:0:3914:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921875.041814:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:13.0:1506921875.041815:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921875.041816:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921875.041819:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921875.041820:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921875.041822:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921875.042770:0:3916:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921875.042774:0:3916:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921875.042777:0:3916:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921875.042778:0:3916:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921875.042779:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921875.042781:0:3921:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921875.042782:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921875.042783:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921875.042783:0:3921:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921875.042785:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921875.042786:0:3921:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921875.042787:0:3921:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921875.042788:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921875.042791:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921875.042795:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921875.042796:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1506921875.584780:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.31@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921875.584785:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.32@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921875.584791:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.41@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921875.584794:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.42@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000100:00000001:18.0:1506921875.611772:0:3905:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921875.611772:0:3902:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1506921875.611773:0:3910:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921875.611774:0:3905:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921875.611774:0:3902:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:6.0:1506921875.611774:0:3911:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:6.0:1506921875.611775:0:3911:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:4.0:1506921875.611775:0:3903:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1506921875.611775:0:3910:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921875.611777:0:3905:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921875.611777:0:3902:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:4.0:1506921875.611777:0:3903:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921875.611778:0:3905:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921875.611778:0:3902:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921875.611778:0:3911:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921875.611778:0:3910:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:6.0:1506921875.611779:0:3911:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921875.611779:0:3903:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921875.611779:0:3910:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921875.611780:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921875.611780:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921875.611780:0:3903:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921875.611781:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921875.611781:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00010000:00000001:12.0:1506921875.611782:0:3937:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:4.0:1506921875.611782:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00010000:00000001:12.0:1506921875.611784:0:3937:0:(ldlm_pool.c:504:ldlm_cli_pool_recalc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:18.0:1506921875.611785:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921875.611785:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921875.611786:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921875.611786:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921875.611787:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921875.611787:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00010000:00000001:12.0:1506921875.611787:0:3937:0:(ldlm_pool.c:525:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921875.611787:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921875.611787:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921875.611788:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921875.611788:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00010000:00000001:12.0:1506921875.611789:0:3937:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:18.0:1506921875.611790:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:12.0:1506921875.611790:0:3937:0:(ldlm_pool.c:504:ldlm_cli_pool_recalc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:12.0:1506921875.611791:0:3937:0:(ldlm_pool.c:525:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921875.611791:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921875.611792:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921875.611792:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921875.611792:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:12.0:1506921875.611793:0:3937:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:12.0:1506921875.611794:0:3937:0:(ldlm_request.c:1846:ldlm_cancel_lru()) Process entered 00010000:00000001:12.0:1506921875.611795:0:3937:0:(ldlm_request.c:1687:ldlm_prepare_lru_list()) Process entered 00010000:00000001:12.0:1506921875.611797:0:3937:0:(ldlm_lock.c:188:ldlm_lock_put()) Process entered 00010000:00000001:12.0:1506921875.611798:0:3937:0:(ldlm_lock.c:222:ldlm_lock_put()) Process leaving 00010000:00000001:12.0:1506921875.611798:0:3937:0:(ldlm_request.c:1815:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:12.0:1506921875.611800:0:3937:0:(ldlm_lockd.c:2046:ldlm_bl_to_thread()) Process entered 00010000:00000001:12.0:1506921875.611801:0:3937:0:(ldlm_lockd.c:2049:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:12.0:1506921875.611802:0:3937:0:(ldlm_request.c:1853:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:12.0:1506921875.611803:0:3937:0:(ldlm_pool.c:525:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:8.0:1506921875.611812:0:3935:0:(genops.c:1774:obd_stale_export_get()) Process entered 00000020:00000001:8.0:1506921875.611814:0:3935:0:(genops.c:1788:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:8.0:1506921875.611817:0:3935:0:(genops.c:1774:obd_stale_export_get()) Process entered 00000020:00000001:8.0:1506921875.611817:0:3935:0:(genops.c:1788:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1506921875.611847:0:3936:0:(genops.c:1774:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1506921875.611851:0:3936:0:(genops.c:1788:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1506921875.611854:0:3936:0:(genops.c:1774:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1506921875.611854:0:3936:0:(genops.c:1788:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921875.612769:0:3909:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921875.612770:0:3907:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921875.612771:0:3909:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921875.612772:0:3907:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921875.612775:0:3909:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921875.612775:0:3907:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921875.612776:0:3909:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921875.612776:0:3907:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921875.612777:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921875.612777:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921875.612779:0:3906:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921875.612780:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921875.612781:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921875.612781:0:3906:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921875.612782:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921875.612782:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921875.612783:0:3906:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921875.612784:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921875.612784:0:3906:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921875.612786:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921875.612786:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921875.612789:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921875.612790:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921875.612792:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921875.613759:0:3904:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:14.0:1506921875.613761:0:3904:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:12.0:1506921875.613761:0:3908:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:12.0:1506921875.613763:0:3908:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:14.0:1506921875.613764:0:3904:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:14.0:1506921875.613765:0:3904:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921875.613766:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:12.0:1506921875.613766:0:3908:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:12.0:1506921875.613767:0:3908:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921875.613769:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:12.0:1506921875.613769:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921875.613771:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921875.613772:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:12.0:1506921875.613772:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:12.0:1506921875.613773:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:12.0:1506921875.613775:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921876.035788:0:3901:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1506921876.035790:0:3901:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1506921876.035793:0:3901:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921876.035793:0:3901:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921876.035795:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921876.035798:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921876.035799:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921876.035801:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921876.040754:0:3917:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:7.0:1506921876.040754:0:3913:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:17.0:1506921876.040756:0:3917:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:7.0:1506921876.040757:0:3913:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:17.0:1506921876.040759:0:3917:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:17.0:1506921876.040760:0:3917:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921876.040760:0:3913:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:17.0:1506921876.040761:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921876.040761:0:3913:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921876.040763:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921876.040765:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921876.040766:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921876.040766:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921876.040768:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921876.040768:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921876.040769:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921876.040780:0:3918:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921876.040781:0:3912:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1506921876.040781:0:3918:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921876.040783:0:3912:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921876.040784:0:3918:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921876.040785:0:3918:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921876.040786:0:3912:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921876.040787:0:3912:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921876.040787:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921876.040789:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921876.040790:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921876.040792:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921876.040792:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921876.040793:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921876.040793:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921876.040795:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921876.041758:0:3919:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1506921876.041758:0:3920:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:13.0:1506921876.041760:0:3915:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:7.0:1506921876.041760:0:3919:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921876.041760:0:3920:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:13.0:1506921876.041762:0:3915:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:7.0:1506921876.041763:0:3919:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921876.041763:0:3920:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:7.0:1506921876.041764:0:3919:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921876.041764:0:3920:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921876.041765:0:3915:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:13.0:1506921876.041765:0:3915:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921876.041765:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921876.041766:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:13.0:1506921876.041767:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921876.041769:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921876.041769:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921876.041770:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921876.041771:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921876.041771:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:13.0:1506921876.041772:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921876.041773:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921876.041773:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921876.041774:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921876.041780:0:3914:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:17.0:1506921876.041782:0:3914:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:17.0:1506921876.041785:0:3914:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:17.0:1506921876.041786:0:3914:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921876.041788:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921876.041791:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921876.041792:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921876.041793:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921876.042769:0:3921:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921876.042770:0:3916:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:15.0:1506921876.042771:0:3921:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921876.042772:0:3916:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921876.042774:0:3916:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921876.042774:0:3921:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921876.042775:0:3921:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921876.042776:0:3916:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921876.042777:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921876.042777:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921876.042780:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921876.042780:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921876.042781:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921876.042782:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921876.042783:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921876.042783:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1506921876.584781:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.31@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921876.584786:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.32@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921876.584789:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.41@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921876.584792:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.42@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000100:00000001:6.0:1506921876.611763:0:3911:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921876.611764:0:3902:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:4.0:1506921876.611764:0:3903:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:6.0:1506921876.611765:0:3911:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:4.0:1506921876.611765:0:3903:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921876.611766:0:3902:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921876.611769:0:3902:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:6.0:1506921876.611769:0:3911:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:4.0:1506921876.611769:0:3903:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:6.0:1506921876.611770:0:3911:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921876.611770:0:3903:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921876.611771:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921876.611772:0:3902:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921876.611772:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921876.611774:0:3905:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921876.611774:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921876.611776:0:3905:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:6.0:1506921876.611776:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921876.611776:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921876.611776:0:3910:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921876.611777:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921876.611777:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921876.611777:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921876.611778:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921876.611778:0:3910:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921876.611779:0:3905:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:6.0:1506921876.611779:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921876.611780:0:3905:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921876.611780:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921876.611781:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921876.611781:0:3910:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921876.611782:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921876.611782:0:3910:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921876.611783:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921876.611785:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921876.611786:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921876.611786:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921876.611788:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921876.611788:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921876.611789:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921876.612773:0:3909:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921876.612773:0:3907:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921876.612774:0:3909:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921876.612775:0:3907:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921876.612778:0:3909:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921876.612778:0:3909:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921876.612778:0:3907:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921876.612779:0:3907:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921876.612780:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921876.612781:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921876.612783:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921876.612783:0:3906:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921876.612784:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921876.612785:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921876.612785:0:3906:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921876.612786:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921876.612786:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921876.612787:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921876.612788:0:3906:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:8.0:1506921876.612788:0:3906:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921876.612790:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921876.612795:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921876.612797:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921876.612798:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921876.613752:0:3904:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:14.0:1506921876.613755:0:3904:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:14.0:1506921876.613758:0:3904:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:14.0:1506921876.613759:0:3904:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921876.613761:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921876.613763:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921876.613765:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921876.613766:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921876.613774:0:3908:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921876.613776:0:3908:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921876.613779:0:3908:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921876.613780:0:3908:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921876.613782:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921876.613785:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921876.613787:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921876.613788:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921877.035770:0:3901:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1506921877.035772:0:3901:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1506921877.035775:0:3901:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921877.035775:0:3901:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921877.035779:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921877.035782:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921877.035783:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921877.035785:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921877.040753:0:3912:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:17.0:1506921877.040753:0:3917:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921877.040755:0:3912:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:17.0:1506921877.040757:0:3917:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921877.040758:0:3912:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921877.040759:0:3912:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921877.040760:0:3917:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921877.040761:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921877.040761:0:3917:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921877.040762:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921877.040764:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921877.040765:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921877.040765:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921877.040766:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921877.040767:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921877.040768:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921877.040778:0:3918:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:7.0:1506921877.040779:0:3913:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1506921877.040780:0:3918:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:7.0:1506921877.040781:0:3913:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921877.040783:0:3918:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:7.0:1506921877.040784:0:3913:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921877.040784:0:3918:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921877.040785:0:3913:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921877.040785:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921877.040787:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921877.040788:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921877.040790:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921877.040790:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921877.040791:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921877.040792:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921877.040793:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921877.041752:0:3915:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:13.0:1506921877.041754:0:3915:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:13.0:1506921877.041757:0:3915:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:13.0:1506921877.041758:0:3915:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921877.041759:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921877.041761:0:3914:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:7.0:1506921877.041761:0:3919:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:13.0:1506921877.041762:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921877.041762:0:3920:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:17.0:1506921877.041763:0:3914:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:13.0:1506921877.041763:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921877.041763:0:3919:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921877.041763:0:3920:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:13.0:1506921877.041765:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921877.041766:0:3914:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:7.0:1506921877.041766:0:3919:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921877.041766:0:3920:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:17.0:1506921877.041767:0:3914:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921877.041767:0:3919:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921877.041767:0:3920:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921877.041769:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921877.041769:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921877.041769:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921877.041773:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921877.041773:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921877.041773:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921877.041775:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921877.041775:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921877.041775:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921877.041777:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921877.041778:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921877.041778:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921877.042751:0:3921:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:15.0:1506921877.042753:0:3921:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921877.042756:0:3921:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921877.042757:0:3921:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921877.042758:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921877.042761:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921877.042763:0:3916:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:15.0:1506921877.042763:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921877.042764:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921877.042765:0:3916:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921877.042768:0:3916:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921877.042769:0:3916:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921877.042771:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921877.042774:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921877.042775:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921877.042777:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1506921877.584776:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.31@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921877.584781:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.32@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921877.584784:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.41@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921877.584786:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.42@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000100:00000001:16.0:1506921877.611778:0:3902:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921877.611780:0:3902:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921877.611783:0:3902:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921877.611784:0:3902:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921877.611784:0:3910:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921877.611786:0:3905:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921877.611786:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921877.611786:0:3911:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:4.0:1506921877.611786:0:3903:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1506921877.611786:0:3910:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921877.611788:0:3905:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921877.611789:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921877.611789:0:3911:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1506921877.611789:0:3910:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:4.0:1506921877.611790:0:3903:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1506921877.611790:0:3910:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921877.611791:0:3905:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921877.611791:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921877.611792:0:3905:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921877.611792:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921877.611792:0:3911:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921877.611792:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921877.611793:0:3911:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921877.611793:0:3903:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921877.611794:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921877.611794:0:3903:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921877.611795:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921877.611795:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921877.611795:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921877.611796:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921877.611797:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921877.611798:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921877.611800:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921877.611800:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921877.611800:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921877.611801:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921877.611801:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921877.611802:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921877.611803:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921877.611804:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921877.612766:0:3909:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921877.612766:0:3907:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921877.612768:0:3909:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921877.612768:0:3907:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921877.612771:0:3909:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921877.612771:0:3907:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921877.612772:0:3909:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921877.612772:0:3907:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921877.612773:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921877.612774:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921877.612776:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921877.612776:0:3906:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921877.612777:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921877.612778:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921877.612778:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921877.612778:0:3906:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921877.612780:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921877.612780:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921877.612781:0:3906:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:8.0:1506921877.612782:0:3906:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921877.612783:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921877.612786:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921877.612788:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921877.612789:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921877.613773:0:3908:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921877.613775:0:3908:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921877.613778:0:3908:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921877.613781:0:3908:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921877.613783:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921877.613785:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921877.613786:0:3904:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921877.613787:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921877.613788:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921877.613788:0:3904:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:14.0:1506921877.613791:0:3904:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:14.0:1506921877.613792:0:3904:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921877.613794:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921877.613797:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921877.613798:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921877.613799:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921878.035780:0:3901:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1506921878.035782:0:3901:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1506921878.035784:0:3901:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921878.035785:0:3901:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921878.035787:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921878.035790:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921878.035791:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921878.035792:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921878.040768:0:3913:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:7.0:1506921878.040770:0:3913:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:7.0:1506921878.040773:0:3913:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:7.0:1506921878.040774:0:3913:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921878.040776:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921878.040779:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921878.040781:0:3912:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:7.0:1506921878.040781:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921878.040782:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921878.040783:0:3917:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1506921878.040783:0:3918:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1506921878.040784:0:3918:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921878.040785:0:3912:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:17.0:1506921878.040785:0:3917:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921878.040787:0:3918:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921878.040788:0:3912:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:17.0:1506921878.040788:0:3917:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921878.040788:0:3918:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921878.040789:0:3912:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921878.040789:0:3917:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921878.040790:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921878.040790:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921878.040790:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921878.040794:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921878.040794:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921878.040795:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921878.040796:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921878.040796:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921878.040796:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921878.040798:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921878.040799:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921878.040799:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921878.041763:0:3919:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:17.0:1506921878.041764:0:3914:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:17.0:1506921878.041766:0:3914:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921878.041766:0:3920:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:7.0:1506921878.041767:0:3919:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921878.041768:0:3920:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:17.0:1506921878.041769:0:3914:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:17.0:1506921878.041770:0:3914:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921878.041770:0:3919:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:7.0:1506921878.041771:0:3919:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921878.041771:0:3920:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:17.0:1506921878.041772:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921878.041772:0:3920:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921878.041773:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921878.041773:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921878.041775:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921878.041776:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921878.041776:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921878.041777:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921878.041777:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921878.041778:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921878.041778:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921878.041781:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921878.041781:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921878.041782:0:3915:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:13.0:1506921878.041784:0:3915:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:13.0:1506921878.041787:0:3915:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:13.0:1506921878.041788:0:3915:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921878.041790:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:13.0:1506921878.041793:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921878.041794:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:13.0:1506921878.041795:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921878.042767:0:3916:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921878.042768:0:3916:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921878.042771:0:3916:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921878.042772:0:3916:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921878.042774:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921878.042776:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921878.042778:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921878.042779:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.042781:0:3921:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:15.0:1506921878.042783:0:3921:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.042786:0:3921:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.042787:0:3921:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.042788:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921878.042791:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.042793:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921878.042794:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:11.0:1506921878.240971:0:38882:0:(file.c:3755:ll_inode_revalidate()) Process entered 00000080:00000001:11.0:1506921878.240973:0:38882:0:(file.c:3643:__ll_inode_revalidate()) Process entered 00000080:00200000:11.0:1506921878.240974:0:38882:0:(file.c:3648:__ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff881021f03690),name=/ 00000080:00000010:11.0:1506921878.240978:0:38882:0:(llite_lib.c:2434:ll_prep_md_op_data()) kmalloced 'op_data': 320 at ffff882011e15800. 00000080:00000001:11.0:1506921878.240981:0:38882:0:(obd_class.h:1379:md_intent_lock()) Process entered 00800000:00000001:11.0:1506921878.240983:0:38882:0:(lmv_intent.c:474:lmv_intent_lock()) Process entered 00800000:00000002:11.0:1506921878.240984:0:38882:0:(lmv_intent.c:482:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:11.0:1506921878.240987:0:38882:0:(lmv_intent.c:361:lmv_intent_lookup()) Process entered 00800000:00000002:11.0:1506921878.240988:0:38882:0:(lmv_intent.c:390:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 lsm= (null) lsm_magic=ffffffff 00800000:00000001:11.0:1506921878.240992:0:38882:0:(obd_class.h:1379:md_intent_lock()) Process entered 00000002:00000001:11.0:1506921878.240993:0:38882:0:(mdc_locks.c:1118:mdc_intent_lock()) Process entered 00000002:00010000:11.0:1506921878.240994:0:38882:0:(mdc_locks.c:1125:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:11.0:1506921878.240997:0:38882:0:(mdc_locks.c:1024:mdc_revalidate_lock()) Process entered 00000002:00000001:11.0:1506921878.240998:0:38882:0:(mdc_locks.c:142:mdc_lock_match()) Process entered 00010000:00000001:11.0:1506921878.240999:0:38882:0:(ldlm_lock.c:1384:ldlm_lock_match()) Process entered 00010000:00000040:11.0:1506921878.241001:0:38882:0:(ldlm_resource.c:1480:ldlm_resource_getref()) getref res: ffff88203b7332c0 count: 2 00010000:00000001:11.0:1506921878.241003:0:38882:0:(ldlm_resource.c:1415:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:11.0:1506921878.241006:0:38882:0:(ldlm_lock.c:260:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:11.0:1506921878.241007:0:38882:0:(ldlm_lock.c:271:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000040:11.0:1506921878.241009:0:38882:0:(ldlm_resource.c:1480:ldlm_resource_getref()) getref res: ffff88203b7332c0 count: 3 00010000:00010000:11.0:1506921878.241010:0:38882:0:(ldlm_lock.c:742:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: oak-MDT0000-mdc-ffff88203bc47000 lock: ffff88101d368000/0x228de8b50142155 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13 rrc: 3 type: IBT flags: 0x0 nid: local remote: 0xc1d2b9af9ec6a943 expref: -99 pid: 4287 timeout: 0 lvb_type: 0 00010000:00000040:11.0:1506921878.241016:0:38882:0:(ldlm_resource.c:1519:ldlm_resource_putref()) putref res: ffff88203b7332c0 count: 2 00010000:00000001:11.0:1506921878.241018:0:38882:0:(ldlm_lock.c:1410:ldlm_lock_match()) Process leaving via out (rc=1 : 1 : 0x1) 00010000:00000040:11.0:1506921878.241019:0:38882:0:(ldlm_resource.c:1519:ldlm_resource_putref()) putref res: ffff88203b7332c0 count: 1 00010000:00000040:11.0:1506921878.241020:0:38882:0:(ldlm_resource.c:1480:ldlm_resource_getref()) getref res: ffff88203b7332c0 count: 2 00010000:00010000:11.0:1506921878.241021:0:38882:0:(ldlm_lock.c:1470:ldlm_lock_match()) ### matched (0 0) ns: oak-MDT0000-mdc-ffff88203bc47000 lock: ffff88101d368000/0x228de8b50142155 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13 rrc: 2 type: IBT flags: 0x0 nid: local remote: 0xc1d2b9af9ec6a943 expref: -99 pid: 4287 timeout: 0 lvb_type: 0 00010000:00000040:11.0:1506921878.241027:0:38882:0:(ldlm_resource.c:1519:ldlm_resource_putref()) putref res: ffff88203b7332c0 count: 1 02000000:00000001:11.0:1506921878.241028:0:38882:0:(sec.c:936:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:11.0:1506921878.241030:0:38882:0:(sec.c:950:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:11.0:1506921878.241031:0:38882:0:(mdc_locks.c:149:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:11.0:1506921878.241032:0:38882:0:(mdc_locks.c:1076:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:11.0:1506921878.241033:0:38882:0:(mdc_locks.c:1138:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:11.0:1506921878.241034:0:38882:0:(obd_class.h:1384:md_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:11.0:1506921878.241035:0:38882:0:(lmv_intent.c:409:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:11.0:1506921878.241036:0:38882:0:(lmv_intent.c:515:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:11.0:1506921878.241037:0:38882:0:(obd_class.h:1384:md_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00000010:11.0:1506921878.241038:0:38882:0:(llite_lib.c:2485:ll_finish_md_op_data()) kfreed 'op_data': 320 at ffff882011e15800. 00000080:00000001:11.0:1506921878.241040:0:38882:0:(dcache.c:267:ll_revalidate_it_finish()) Process entered 00000080:00000001:11.0:1506921878.241041:0:38882:0:(dcache.c:270:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:11.0:1506921878.241043:0:38882:0:(dcache.c:290:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff881021f03690) 00000080:00010000:11.0:1506921878.241045:0:38882:0:(llite_internal.h:1368:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff881021f03690) for lock 0x228de8b50142155 00000080:00000001:11.0:1506921878.241047:0:38882:0:(obd_class.h:1545:md_set_lock_data()) Process entered 00800000:00000001:11.0:1506921878.241048:0:38882:0:(lmv_obd.c:2845:lmv_set_lock_data()) Process entered 00800000:00000001:11.0:1506921878.241048:0:38882:0:(obd_class.h:1545:md_set_lock_data()) Process entered 00000002:00000001:11.0:1506921878.241049:0:38882:0:(mdc_locks.c:103:mdc_set_lock_data()) Process entered 00010000:00000001:11.0:1506921878.241050:0:38882:0:(ldlm_lock.c:584:__ldlm_handle2lock()) Process entered 00000020:00000001:11.0:1506921878.241051:0:38882:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000001:11.0:1506921878.241052:0:38882:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612201523806208 : -131872185745408 : ffff88101d368000) 00010000:00000001:11.0:1506921878.241054:0:38882:0:(ldlm_lock.c:603:__ldlm_handle2lock()) Process leaving (rc=18446612201523806208 : -131872185745408 : ffff88101d368000) 00010000:00000001:11.0:1506921878.241056:0:38882:0:(ldlm_lock.c:188:ldlm_lock_put()) Process entered 00010000:00000001:11.0:1506921878.241057:0:38882:0:(ldlm_lock.c:222:ldlm_lock_put()) Process leaving 00000002:00000001:11.0:1506921878.241058:0:38882:0:(mdc_locks.c:132:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:11.0:1506921878.241059:0:38882:0:(obd_class.h:1548:md_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:11.0:1506921878.241060:0:38882:0:(lmv_obd.c:2850:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:11.0:1506921878.241061:0:38882:0:(obd_class.h:1548:md_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:11.0:1506921878.241062:0:38882:0:(dcache.c:201:ll_intent_drop_lock()) releasing lock with cookie 0x228de8b50142155 from it ffff88103bea3d58 00010000:00000001:11.0:1506921878.241063:0:38882:0:(ldlm_lock.c:584:__ldlm_handle2lock()) Process entered 00000020:00000001:11.0:1506921878.241064:0:38882:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000001:11.0:1506921878.241065:0:38882:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612201523806208 : -131872185745408 : ffff88101d368000) 00010000:00000001:11.0:1506921878.241066:0:38882:0:(ldlm_lock.c:603:__ldlm_handle2lock()) Process leaving (rc=18446612201523806208 : -131872185745408 : ffff88101d368000) 00010000:00000001:11.0:1506921878.241068:0:38882:0:(ldlm_lock.c:823:ldlm_lock_decref_internal()) Process entered 00010000:00010000:11.0:1506921878.241069:0:38882:0:(ldlm_lock.c:796:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: ?? lock: ffff88101d368000/0x228de8b50142155 lrc: 3/1,0 mode: PR/PR res: ?? rrc=?? type: ??? flags: 0x10000000000000 nid: local remote: 0xc1d2b9af9ec6a943 expref: -99 pid: 4287 timeout: 0 lvb_type: 0 00010000:00000001:11.0:1506921878.241073:0:38882:0:(ldlm_lock.c:188:ldlm_lock_put()) Process entered 00010000:00000001:11.0:1506921878.241073:0:38882:0:(ldlm_lock.c:222:ldlm_lock_put()) Process leaving 00010000:00010000:11.0:1506921878.241074:0:38882:0:(ldlm_lock.c:869:ldlm_lock_decref_internal()) ### add lock into lru list ns: ?? lock: ffff88101d368000/0x228de8b50142155 lrc: 2/0,0 mode: PR/PR res: ?? rrc=?? type: ??? flags: 0x10000000000000 nid: local remote: 0xc1d2b9af9ec6a943 expref: -99 pid: 4287 timeout: 0 lvb_type: 0 00010000:00000001:11.0:1506921878.241077:0:38882:0:(ldlm_lock.c:298:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:11.0:1506921878.241078:0:38882:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process leaving 00010000:00000001:11.0:1506921878.241079:0:38882:0:(ldlm_lock.c:890:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:11.0:1506921878.241079:0:38882:0:(ldlm_lock.c:188:ldlm_lock_put()) Process entered 00010000:00000001:11.0:1506921878.241080:0:38882:0:(ldlm_lock.c:222:ldlm_lock_put()) Process leaving 00000100:00000001:11.0:1506921878.241081:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000001:11.0:1506921878.241082:0:38882:0:(client.c:2514:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:11.0:1506921878.241083:0:38882:0:(file.c:3784:ll_inode_revalidate()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:11.0:1506921878.241279:0:38882:0:(llite_lib.c:2525:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:11.0:1506921878.241296:0:38882:0:(llite_lib.c:2525:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:11.0:1506921878.241348:0:38882:0:(file.c:3941:ll_inode_permission()) Process entered 00000080:00000001:11.0:1506921878.241349:0:38882:0:(file.c:3643:__ll_inode_revalidate()) Process entered 00000080:00200000:11.0:1506921878.241350:0:38882:0:(file.c:3648:__ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff881021f03690),name=/ 00000080:00000010:11.0:1506921878.241353:0:38882:0:(llite_lib.c:2434:ll_prep_md_op_data()) kmalloced 'op_data': 320 at ffff882011e15800. 00000080:00000001:11.0:1506921878.241354:0:38882:0:(obd_class.h:1379:md_intent_lock()) Process entered 00800000:00000001:11.0:1506921878.241355:0:38882:0:(lmv_intent.c:474:lmv_intent_lock()) Process entered 00800000:00000002:11.0:1506921878.241356:0:38882:0:(lmv_intent.c:482:lmv_intent_lock()) INTENT LOCK 'lookup' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:11.0:1506921878.241358:0:38882:0:(lmv_intent.c:361:lmv_intent_lookup()) Process entered 00800000:00000002:11.0:1506921878.241359:0:38882:0:(lmv_intent.c:390:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 lsm= (null) lsm_magic=ffffffff 00800000:00000001:11.0:1506921878.241362:0:38882:0:(obd_class.h:1379:md_intent_lock()) Process entered 00000002:00000001:11.0:1506921878.241363:0:38882:0:(mdc_locks.c:1118:mdc_intent_lock()) Process entered 00000002:00010000:11.0:1506921878.241364:0:38882:0:(mdc_locks.c:1125:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: lookup flags 0 00000002:00000001:11.0:1506921878.241366:0:38882:0:(mdc_locks.c:1024:mdc_revalidate_lock()) Process entered 00000002:00000001:11.0:1506921878.241367:0:38882:0:(mdc_locks.c:142:mdc_lock_match()) Process entered 00010000:00000001:11.0:1506921878.241368:0:38882:0:(ldlm_lock.c:1384:ldlm_lock_match()) Process entered 00010000:00000040:11.0:1506921878.241369:0:38882:0:(ldlm_resource.c:1480:ldlm_resource_getref()) getref res: ffff88203b7332c0 count: 2 00010000:00000001:11.0:1506921878.241371:0:38882:0:(ldlm_resource.c:1415:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:11.0:1506921878.241373:0:38882:0:(ldlm_lock.c:260:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:11.0:1506921878.241374:0:38882:0:(ldlm_lock.c:271:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000040:11.0:1506921878.241375:0:38882:0:(ldlm_resource.c:1480:ldlm_resource_getref()) getref res: ffff88203b7332c0 count: 3 00010000:00010000:11.0:1506921878.241376:0:38882:0:(ldlm_lock.c:742:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: oak-MDT0000-mdc-ffff88203bc47000 lock: ffff88101d368000/0x228de8b50142155 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13 rrc: 3 type: IBT flags: 0x0 nid: local remote: 0xc1d2b9af9ec6a943 expref: -99 pid: 4287 timeout: 0 lvb_type: 0 00010000:00000040:11.0:1506921878.241381:0:38882:0:(ldlm_resource.c:1519:ldlm_resource_putref()) putref res: ffff88203b7332c0 count: 2 00010000:00000001:11.0:1506921878.241382:0:38882:0:(ldlm_lock.c:1410:ldlm_lock_match()) Process leaving via out (rc=1 : 1 : 0x1) 00010000:00000040:11.0:1506921878.241384:0:38882:0:(ldlm_resource.c:1519:ldlm_resource_putref()) putref res: ffff88203b7332c0 count: 1 00010000:00000040:11.0:1506921878.241385:0:38882:0:(ldlm_resource.c:1480:ldlm_resource_getref()) getref res: ffff88203b7332c0 count: 2 00010000:00010000:11.0:1506921878.241386:0:38882:0:(ldlm_lock.c:1470:ldlm_lock_match()) ### matched (0 0) ns: oak-MDT0000-mdc-ffff88203bc47000 lock: ffff88101d368000/0x228de8b50142155 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13 rrc: 2 type: IBT flags: 0x0 nid: local remote: 0xc1d2b9af9ec6a943 expref: -99 pid: 4287 timeout: 0 lvb_type: 0 00010000:00000040:11.0:1506921878.241390:0:38882:0:(ldlm_resource.c:1519:ldlm_resource_putref()) putref res: ffff88203b7332c0 count: 1 02000000:00000001:11.0:1506921878.241391:0:38882:0:(sec.c:936:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:11.0:1506921878.241393:0:38882:0:(sec.c:950:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:11.0:1506921878.241394:0:38882:0:(mdc_locks.c:149:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:11.0:1506921878.241395:0:38882:0:(mdc_locks.c:1076:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:11.0:1506921878.241396:0:38882:0:(mdc_locks.c:1138:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:11.0:1506921878.241397:0:38882:0:(obd_class.h:1384:md_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:11.0:1506921878.241398:0:38882:0:(lmv_intent.c:409:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:11.0:1506921878.241399:0:38882:0:(lmv_intent.c:515:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:11.0:1506921878.241400:0:38882:0:(obd_class.h:1384:md_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00000010:11.0:1506921878.241402:0:38882:0:(llite_lib.c:2485:ll_finish_md_op_data()) kfreed 'op_data': 320 at ffff882011e15800. 00000080:00000001:11.0:1506921878.241403:0:38882:0:(dcache.c:267:ll_revalidate_it_finish()) Process entered 00000080:00000001:11.0:1506921878.241404:0:38882:0:(dcache.c:270:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:11.0:1506921878.241405:0:38882:0:(dcache.c:290:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff881021f03690) 00000080:00010000:11.0:1506921878.241407:0:38882:0:(llite_internal.h:1368:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff881021f03690) for lock 0x228de8b50142155 00000080:00000001:11.0:1506921878.241408:0:38882:0:(obd_class.h:1545:md_set_lock_data()) Process entered 00800000:00000001:11.0:1506921878.241409:0:38882:0:(lmv_obd.c:2845:lmv_set_lock_data()) Process entered 00800000:00000001:11.0:1506921878.241410:0:38882:0:(obd_class.h:1545:md_set_lock_data()) Process entered 00000002:00000001:11.0:1506921878.241411:0:38882:0:(mdc_locks.c:103:mdc_set_lock_data()) Process entered 00010000:00000001:11.0:1506921878.241412:0:38882:0:(ldlm_lock.c:584:__ldlm_handle2lock()) Process entered 00000020:00000001:11.0:1506921878.241412:0:38882:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000001:11.0:1506921878.241414:0:38882:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612201523806208 : -131872185745408 : ffff88101d368000) 00010000:00000001:11.0:1506921878.241415:0:38882:0:(ldlm_lock.c:603:__ldlm_handle2lock()) Process leaving (rc=18446612201523806208 : -131872185745408 : ffff88101d368000) 00010000:00000001:11.0:1506921878.241417:0:38882:0:(ldlm_lock.c:188:ldlm_lock_put()) Process entered 00010000:00000001:11.0:1506921878.241417:0:38882:0:(ldlm_lock.c:222:ldlm_lock_put()) Process leaving 00000002:00000001:11.0:1506921878.241418:0:38882:0:(mdc_locks.c:132:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:11.0:1506921878.241419:0:38882:0:(obd_class.h:1548:md_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:11.0:1506921878.241420:0:38882:0:(lmv_obd.c:2850:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:11.0:1506921878.241421:0:38882:0:(obd_class.h:1548:md_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:11.0:1506921878.241422:0:38882:0:(dcache.c:201:ll_intent_drop_lock()) releasing lock with cookie 0x228de8b50142155 from it ffff88103bea3be8 00010000:00000001:11.0:1506921878.241423:0:38882:0:(ldlm_lock.c:584:__ldlm_handle2lock()) Process entered 00000020:00000001:11.0:1506921878.241424:0:38882:0:(lustre_handles.c:156:class_handle2object()) Process entered 00000020:00000001:11.0:1506921878.241425:0:38882:0:(lustre_handles.c:179:class_handle2object()) Process leaving (rc=18446612201523806208 : -131872185745408 : ffff88101d368000) 00010000:00000001:11.0:1506921878.241426:0:38882:0:(ldlm_lock.c:603:__ldlm_handle2lock()) Process leaving (rc=18446612201523806208 : -131872185745408 : ffff88101d368000) 00010000:00000001:11.0:1506921878.241427:0:38882:0:(ldlm_lock.c:823:ldlm_lock_decref_internal()) Process entered 00010000:00010000:11.0:1506921878.241428:0:38882:0:(ldlm_lock.c:796:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: ?? lock: ffff88101d368000/0x228de8b50142155 lrc: 3/1,0 mode: PR/PR res: ?? rrc=?? type: ??? flags: 0x10000000000000 nid: local remote: 0xc1d2b9af9ec6a943 expref: -99 pid: 4287 timeout: 0 lvb_type: 0 00010000:00000001:11.0:1506921878.241432:0:38882:0:(ldlm_lock.c:188:ldlm_lock_put()) Process entered 00010000:00000001:11.0:1506921878.241433:0:38882:0:(ldlm_lock.c:222:ldlm_lock_put()) Process leaving 00010000:00010000:11.0:1506921878.241433:0:38882:0:(ldlm_lock.c:869:ldlm_lock_decref_internal()) ### add lock into lru list ns: ?? lock: ffff88101d368000/0x228de8b50142155 lrc: 2/0,0 mode: PR/PR res: ?? rrc=?? type: ??? flags: 0x10000000000000 nid: local remote: 0xc1d2b9af9ec6a943 expref: -99 pid: 4287 timeout: 0 lvb_type: 0 00010000:00000001:11.0:1506921878.241436:0:38882:0:(ldlm_lock.c:298:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:11.0:1506921878.241437:0:38882:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process leaving 00010000:00000001:11.0:1506921878.241438:0:38882:0:(ldlm_lock.c:890:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:11.0:1506921878.241438:0:38882:0:(ldlm_lock.c:188:ldlm_lock_put()) Process entered 00010000:00000001:11.0:1506921878.241439:0:38882:0:(ldlm_lock.c:222:ldlm_lock_put()) Process leaving 00000100:00000001:11.0:1506921878.241440:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000001:11.0:1506921878.241441:0:38882:0:(client.c:2514:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:11.0:1506921878.241442:0:38882:0:(file.c:3962:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff881021f03690), inode mode 41ed mask 44 00000080:00000001:11.0:1506921878.241444:0:38882:0:(file.c:4000:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:11.0:1506921878.241446:0:38882:0:(dir.c:1825:ll_dir_open()) Process entered 00000080:00000001:11.0:1506921878.241447:0:38882:0:(file.c:491:ll_file_open()) Process entered 00000080:00200000:11.0:1506921878.241448:0:38882:0:(file.c:494:ll_file_open()) VFS Op:inode=[0x200000007:0x1:0x0](ffff881021f03690), flags 300000 00000080:00000010:11.0:1506921878.241450:0:38882:0:(file.c:71:ll_file_data_get()) slab-alloced 'fd': 216 at ffff882035b5d800. 00000080:00000001:11.0:1506921878.241451:0:38882:0:(file.c:509:ll_file_open()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:11.0:1506921878.241452:0:38882:0:(dir.c:1826:ll_dir_open()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:11.0:1506921878.241457:0:38882:0:(dir.c:1160:ll_dir_ioctl()) Process entered 00000080:00200000:11.0:1506921878.241457:0:38882:0:(dir.c:1163:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff881021f03690), cmd=0xc0b066a2 00000080:00000010:11.0:1506921878.241459:0:38882:0:(dir.c:1529:ll_dir_ioctl()) kmalloced 'qctl': 176 at ffff8820382d7380. 00000080:00000001:11.0:1506921878.241461:0:38882:0:(dir.c:1010:quotactl_ioctl()) Process entered 00000080:00000010:11.0:1506921878.241463:0:38882:0:(dir.c:1067:quotactl_ioctl()) kmalloced 'oqctl': 112 at ffff881032734100. 00000080:00000001:11.0:1506921878.241464:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00800000:00000001:11.0:1506921878.241465:0:38882:0:(lmv_obd.c:3038:lmv_quotactl()) Process entered 00800000:00000001:11.0:1506921878.241466:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000002:00000001:11.0:1506921878.241467:0:38882:0:(mdc_request.c:1808:mdc_quotactl()) Process entered 00000100:00000010:11.0:1506921878.241469:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881e1b23c800. 00000020:00000040:11.0:1506921878.241471:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203bc57800 refcount=4 obd=oak-MDT0000-mdc-ffff88203bc47000 00000100:00000001:11.0:1506921878.241473:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:11.0:1506921878.241474:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:11.0:1506921878.241475:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:11.0:1506921878.241477:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff882011e15800. 00000100:00000001:11.0:1506921878.241480:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.241482:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:11.0:1506921878.241483:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:11.0:1506921878.241485:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8820382d75c0. 00000100:00000001:11.0:1506921878.241486:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612270695675328 : -131803013876288 : ffff8820382d75c0) 00000100:00000001:11.0:1506921878.241488:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921878.241488:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618567168 : -131812090984448 : ffff881e1b23c800) 00000020:00000001:11.0:1506921878.241490:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:11.0:1506921878.241491:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:11.0:1506921878.241500:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:11.0:1506921878.241502:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:11.0:1506921878.241503:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:11.0:1506921878.241505:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:11.0:1506921878.241506:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:11.0:1506921878.241509:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881e1b23c800 x1580016666302160/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.241516:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:11.0:1506921878.241517:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921878.241519:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:11.0:1506921878.241519:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:11.0:1506921878.241522:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302160:10.0.2.52@o2ib5:48 00000100:00000001:11.0:1506921878.241524:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:11.0:1506921878.241526:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:11.0:1506921878.241527:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921878.241528:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:11.0:1506921878.241530:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8810e90bc000. 02000000:00000001:11.0:1506921878.241531:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:11.0:1506921878.241533:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203bf0af00. 00000400:00000010:11.0:1506921878.241536:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735b00. 00000100:00000200:11.0:1506921878.241538:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302160, portal 10 00000100:00000001:11.0:1506921878.241540:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921878.241540:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618567168 : -131812090984448 : ffff881e1b23c800) 00000100:00000040:11.0:1506921878.241544:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881e1b23c800 x1580016666302160/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921885 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.241549:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:11.0:1506921878.241552:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.52@o2ib5 00000400:00000010:11.0:1506921878.241553:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032734400. 00000100:00000200:11.0:1506921878.241555:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 12, xid 1580016666302160, offset 0 00000400:00000010:11.0:1506921878.241557:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e14a00 (tot 59423868). 00000400:00000200:11.0:1506921878.241560:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.52@o2ib5 00000400:00000200:11.0:1506921878.241569:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.52@o2ib5(10.0.2.52@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:11.0:1506921878.241573:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:11.0:1506921878.241578:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:11.0:1506921878.241580:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:11.0:1506921878.241582:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:11.0:1506921878.241586:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:11.0:1506921878.241587:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.241588:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921878.241590:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.241591:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:11.0:1506921878.241592:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=6 : 6 : 6) 00000100:00100000:11.0:1506921878.241593:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8820382d75c0 going to sleep for 6 seconds 00000100:00000001:11.0:1506921878.241594:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921878.241595:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.241597:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921878.241598:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921878.241622:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.241641:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.241646:0:3859:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.241650:0:3859:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.241655:0:3859:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881e1b23c800 x1580016666302160/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921885 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.241662:0:3859:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921878.241665:0:3859:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881e1b23c800 x1580016666302160/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921885 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.241671:0:3859:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.241672:0:3859:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921878.241674:0:3859:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032734400 00000400:00000010:16.0:1506921878.241675:0:3859:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032734400. 00000800:00000200:18.0:1506921878.241679:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000010:16.0:1506921878.241679:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e14a00 (tot 59423468). 00000800:00000200:16.0:1506921878.241684:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921878.241800:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:13.0:1506921878.241844:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921878.241853:0:3863:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.42@o2ib4 00000400:00000200:13.0:1506921878.241858:0:3863:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.52@o2ib5 : PUT 00000400:00000010:13.0:1506921878.241861:0:3863:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88203bd18400 (tot 59423868). 00000400:00000200:13.0:1506921878.241866:0:3863:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.52@o2ib5 of length 304 into portal 10 MB=0x59d044c2256d0 00000400:00000200:13.0:1506921878.241871:0:3863:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index a from 12345-10.0.2.52@o2ib5 of length 304/304 into md 0x7a0ff5 [1] + 192 00000100:00000001:13.0:1506921878.241876:0:3863:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:13.0:1506921878.241879:0:3863:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881e1b23c800 x1580016666302160/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921885 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:1.0:1506921878.241884:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:13.0:1506921878.241886:0:3863:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881e1b23c800 x1580016666302160/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921885 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921878.241894:0:3863:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:13.0:1506921878.241895:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88203bd18400 (tot 59423468). 00000100:00000001:11.0:1506921878.241897:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:13.0:1506921878.241898:0:3863:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000400:00000001:11.0:1506921878.241899:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000800:00000200:13.0:1506921878.241900:0:3863:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000400:00000001:11.0:1506921878.241900:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:11.0:1506921878.241901:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000800:00000200:13.0:1506921878.241902:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000200:11.0:1506921878.241903:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881e1b23c800 x1580016666302160/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921885 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:11.0:1506921878.241909:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881e1b23c800 x1580016666302160/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921885 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.241914:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921878.241915:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203bf0af00. 00000400:00000200:11.0:1506921878.241916:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735b00 00000400:00000010:11.0:1506921878.241917:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735b00. 00000100:00000001:11.0:1506921878.241918:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921878.241920:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:11.0:1506921878.241921:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:11.0:1506921878.241922:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:11.0:1506921878.241924:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921878.241925:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:11.0:1506921878.241928:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203bc57c00 time=73 v=1 (1 1 1 1) 00000100:00001000:11.0:1506921878.241931:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203bc57bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:11.0:1506921878.241932:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:11.0:1506921878.241933:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:11.0:1506921878.241934:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:11.0:1506921878.241936:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.241938:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:11.0:1506921878.241938:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-MDT0000-mdc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:11.0:1506921878.241940:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:11.0:1506921878.241940:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:11.0:1506921878.241942:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881e1b23c800 x1580016666302160/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/304 e 0 to 0 dl 1506921885 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921878.241947:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921878.241948:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:11.0:1506921878.241949:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:11.0:1506921878.241951:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881e1b23c800 x1580016666302160/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/304 e 0 to 0 dl 1506921885 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:11.0:1506921878.241956:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302160:10.0.2.52@o2ib5:48 00000100:00000001:11.0:1506921878.241959:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921878.241961:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.241962:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:11.0:1506921878.241963:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921878.241964:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881e1b23c800 x1580016666302160/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/304 e 0 to 0 dl 1506921885 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921878.241969:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:11.0:1506921878.241970:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8820382d75c0. 00000100:00000001:11.0:1506921878.241972:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:11.0:1506921878.241973:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.241975:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921878.241976:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881e1b23c800 x1580016666302160/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/304 e 0 to 0 dl 1506921885 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921878.241980:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:11.0:1506921878.241982:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:11.0:1506921878.241983:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8810e90bc000. 02000000:00000001:11.0:1506921878.241984:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:11.0:1506921878.241985:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:11.0:1506921878.241986:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203bc57800 refcount=3 obd=oak-MDT0000-mdc-ffff88203bc47000 00000020:00000001:11.0:1506921878.241988:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:11.0:1506921878.241988:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff882011e15800. 02000000:00000001:11.0:1506921878.241990:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:11.0:1506921878.241991:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:11.0:1506921878.241991:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881e1b23c800. 00000100:00000001:11.0:1506921878.241993:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:11.0:1506921878.241994:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:11.0:1506921878.241995:0:38882:0:(mdc_request.c:1836:mdc_quotactl()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:11.0:1506921878.241996:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:11.0:1506921878.241997:0:38882:0:(lmv_obd.c:3050:lmv_quotactl()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:11.0:1506921878.241999:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:11.0:1506921878.242000:0:38882:0:(dir.c:1084:quotactl_ioctl()) kmalloced 'oqctl_tmp': 112 at ffff881032735b00. 00000080:00000001:11.0:1506921878.242001:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00020000:00000001:11.0:1506921878.242002:0:38882:0:(lov_obd.c:1348:lov_quotactl()) Process entered 00020000:00000001:11.0:1506921878.242003:0:38882:0:(obd_class.h:976:obd_getref()) Process entered 00020000:00000001:11.0:1506921878.242004:0:38882:0:(obd_class.h:981:obd_getref()) Process leaving 00020000:00000001:11.0:1506921878.242005:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:11.0:1506921878.242006:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:11.0:1506921878.242007:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881e1b23c800. 00000020:00000040:11.0:1506921878.242009:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203c6ad000 refcount=5 obd=oak-OST0000-osc-ffff88203bc47000 00000100:00000001:11.0:1506921878.242010:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:11.0:1506921878.242011:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:11.0:1506921878.242012:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:11.0:1506921878.242014:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff882011e15800. 00000100:00000001:11.0:1506921878.242015:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.242017:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:11.0:1506921878.242018:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:11.0:1506921878.242019:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8820382d75c0. 00000100:00000001:11.0:1506921878.242020:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612270695675328 : -131803013876288 : ffff8820382d75c0) 00000100:00000001:11.0:1506921878.242021:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921878.242022:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618567168 : -131812090984448 : ffff881e1b23c800) 00000020:00000001:11.0:1506921878.242024:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:11.0:1506921878.242025:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:11.0:1506921878.242032:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:11.0:1506921878.242034:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:11.0:1506921878.242035:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:11.0:1506921878.242036:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:11.0:1506921878.242037:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:11.0:1506921878.242039:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881e1b23c800 x1580016666302176/t0(0) o19->oak-OST0000-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.242044:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:11.0:1506921878.242044:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921878.242046:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:11.0:1506921878.242046:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:11.0:1506921878.242048:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302176:10.0.2.101@o2ib5:19 00000100:00000001:11.0:1506921878.242051:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:11.0:1506921878.242051:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:11.0:1506921878.242052:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921878.242054:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:11.0:1506921878.242055:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8810e90bc000. 02000000:00000001:11.0:1506921878.242056:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:11.0:1506921878.242057:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203bf0af00. 00000400:00000010:11.0:1506921878.242059:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735280. 00000100:00000200:11.0:1506921878.242061:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302176, portal 4 00000100:00000001:11.0:1506921878.242062:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921878.242063:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618567168 : -131812090984448 : ffff881e1b23c800) 00000100:00000040:11.0:1506921878.242065:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881e1b23c800 x1580016666302176/t0(0) o19->oak-OST0000-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.242069:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:11.0:1506921878.242071:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:11.0:1506921878.242073:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735380. 00000100:00000200:11.0:1506921878.242074:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302176, offset 0 00000400:00000010:11.0:1506921878.242076:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e12400 (tot 59423868). 00000400:00000200:11.0:1506921878.242078:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:11.0:1506921878.242085:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:11.0:1506921878.242088:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:11.0:1506921878.242092:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:11.0:1506921878.242094:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:11.0:1506921878.242096:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921878.242099:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:11.0:1506921878.242100:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.242101:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921878.242102:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.242103:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:11.0:1506921878.242104:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:11.0:1506921878.242105:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8820382d75c0 going to sleep for 43 seconds 00000100:00000001:11.0:1506921878.242107:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921878.242107:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.242109:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921878.242109:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.242143:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921878.242157:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:11.0:1506921878.242161:0:3862:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:11.0:1506921878.242163:0:3862:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:11.0:1506921878.242165:0:3862:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881e1b23c800 x1580016666302176/t0(0) o19->oak-OST0000-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.242169:0:3862:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921878.242171:0:3862:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881e1b23c800 x1580016666302176/t0(0) o19->oak-OST0000-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.242175:0:3862:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.242176:0:3862:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:11.0:1506921878.242177:0:3862:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735380 00000400:00000010:11.0:1506921878.242177:0:3862:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735380. 00000400:00000010:11.0:1506921878.242180:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e12400 (tot 59423468). 00000800:00000200:11.0:1506921878.242181:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:11.0:1506921878.242183:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.2:1506921878.242393:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921878.242411:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.242417:0:3860:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921878.242423:0:3860:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921878.242426:0:3860:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921878.242432:0:3860:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c2256e0 00000400:00000200:16.0:1506921878.242437:0:3860:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a1005 [1] + 192 00000100:00000001:16.0:1506921878.242442:0:3860:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921878.242445:0:3860:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881e1b23c800 x1580016666302176/t0(0) o19->oak-OST0000-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921878.242448:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921878.242452:0:3860:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881e1b23c800 x1580016666302176/t0(0) o19->oak-OST0000-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.242460:0:3860:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921878.242462:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921878.242465:0:3860:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.242467:0:3860:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921878.242471:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:11.0:1506921878.242484:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:11.0:1506921878.242487:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:11.0:1506921878.242488:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:11.0:1506921878.242489:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921878.242492:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881e1b23c800 x1580016666302176/t0(0) o19->oak-OST0000-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:11.0:1506921878.242497:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881e1b23c800 x1580016666302176/t0(0) o19->oak-OST0000-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.242502:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921878.242504:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203bf0af00. 00000400:00000200:11.0:1506921878.242505:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735280 00000400:00000010:11.0:1506921878.242506:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735280. 00000100:00000001:11.0:1506921878.242507:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921878.242508:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:11.0:1506921878.242509:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:11.0:1506921878.242510:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:11.0:1506921878.242511:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921878.242513:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:11.0:1506921878.242515:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88203c6ad400 time=73 v=31 (31 31 1 31) 00000100:00001000:11.0:1506921878.242518:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203c6ad3c8 time=73 v=1 (1 1 1 1) 00000100:00000001:11.0:1506921878.242519:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:11.0:1506921878.242520:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:11.0:1506921878.242521:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:11.0:1506921878.242523:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.242524:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:11.0:1506921878.242525:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0000-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:11.0:1506921878.242526:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:11.0:1506921878.242527:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:11.0:1506921878.242529:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881e1b23c800 x1580016666302176/t0(0) o19->oak-OST0000-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921878.242533:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921878.242534:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:11.0:1506921878.242535:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:11.0:1506921878.242537:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881e1b23c800 x1580016666302176/t0(0) o19->oak-OST0000-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:11.0:1506921878.242542:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302176:10.0.2.101@o2ib5:19 00000100:00000001:11.0:1506921878.242544:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921878.242546:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.242547:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:11.0:1506921878.242548:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921878.242550:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881e1b23c800 x1580016666302176/t0(0) o19->oak-OST0000-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921878.242559:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:11.0:1506921878.242560:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8820382d75c0. 00000100:00000001:11.0:1506921878.242562:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:11.0:1506921878.242562:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.242564:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921878.242566:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881e1b23c800 x1580016666302176/t0(0) o19->oak-OST0000-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921878.242570:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:11.0:1506921878.242570:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:11.0:1506921878.242571:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8810e90bc000. 02000000:00000001:11.0:1506921878.242573:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:11.0:1506921878.242573:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:11.0:1506921878.242574:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203c6ad000 refcount=4 obd=oak-OST0000-osc-ffff88203bc47000 00000020:00000001:11.0:1506921878.242576:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:11.0:1506921878.242577:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff882011e15800. 02000000:00000001:11.0:1506921878.242578:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:11.0:1506921878.242579:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:11.0:1506921878.242579:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881e1b23c800. 00000100:00000001:11.0:1506921878.242581:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:11.0:1506921878.242582:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:11.0:1506921878.242583:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921878.242584:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921878.242585:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:11.0:1506921878.242586:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:11.0:1506921878.242587:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881e1b23c800. 00000020:00000040:11.0:1506921878.242589:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203bc55000 refcount=5 obd=oak-OST0001-osc-ffff88203bc47000 00000100:00000001:11.0:1506921878.242590:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:11.0:1506921878.242591:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:11.0:1506921878.242592:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:11.0:1506921878.242593:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff882011e15800. 00000100:00000001:11.0:1506921878.242595:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.242596:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:11.0:1506921878.242597:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:11.0:1506921878.242598:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8820382d75c0. 00000100:00000001:11.0:1506921878.242599:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612270695675328 : -131803013876288 : ffff8820382d75c0) 00000100:00000001:11.0:1506921878.242601:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921878.242602:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618567168 : -131812090984448 : ffff881e1b23c800) 00000020:00000001:11.0:1506921878.242603:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:11.0:1506921878.242604:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:11.0:1506921878.242611:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:11.0:1506921878.242613:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:11.0:1506921878.242614:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:11.0:1506921878.242616:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:11.0:1506921878.242616:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:11.0:1506921878.242618:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881e1b23c800 x1580016666302192/t0(0) o19->oak-OST0001-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.242622:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:11.0:1506921878.242623:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921878.242624:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:11.0:1506921878.242625:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:11.0:1506921878.242627:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302192:10.0.2.102@o2ib5:19 00000100:00000001:11.0:1506921878.242629:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:11.0:1506921878.242630:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:11.0:1506921878.242631:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921878.242632:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:11.0:1506921878.242633:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8810e90bc000. 02000000:00000001:11.0:1506921878.242634:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:11.0:1506921878.242635:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203bf0af00. 00000400:00000010:11.0:1506921878.242637:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735280. 00000100:00000200:11.0:1506921878.242639:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302192, portal 4 00000100:00000001:11.0:1506921878.242640:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921878.242641:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618567168 : -131812090984448 : ffff881e1b23c800) 00000100:00000040:11.0:1506921878.242643:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881e1b23c800 x1580016666302192/t0(0) o19->oak-OST0001-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.242647:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:11.0:1506921878.242649:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:11.0:1506921878.242650:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735380. 00000100:00000200:11.0:1506921878.242651:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302192, offset 0 00000400:00000010:11.0:1506921878.242653:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e12400 (tot 59423868). 00000400:00000200:11.0:1506921878.242655:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:11.0:1506921878.242663:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:11.0:1506921878.242667:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:11.0:1506921878.242671:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:11.0:1506921878.242673:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:11.0:1506921878.242675:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:11.0:1506921878.242678:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:11.0:1506921878.242679:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.242680:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921878.242681:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.242682:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:16.2:1506921878.242683:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:11.0:1506921878.242683:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:11.0:1506921878.242684:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8820382d75c0 going to sleep for 43 seconds 00000100:00000001:11.0:1506921878.242686:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921878.242686:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.242688:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921878.242688:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.0:1506921878.242690:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.242694:0:3861:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.242697:0:3861:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.242699:0:3861:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881e1b23c800 x1580016666302192/t0(0) o19->oak-OST0001-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.242705:0:3861:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921878.242707:0:3861:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881e1b23c800 x1580016666302192/t0(0) o19->oak-OST0001-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.242711:0:3861:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.242713:0:3861:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921878.242714:0:3861:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735380 00000400:00000010:16.0:1506921878.242716:0:3861:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735380. 00000800:00000200:18.0:1506921878.242729:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000010:16.0:1506921878.242738:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e12400 (tot 59423468). 00000800:00000200:16.0:1506921878.242740:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:16.2:1506921878.242856:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921878.242863:0:3858:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.242866:0:3858:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000800:00000200:18.0:1506921878.242870:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000200:16.0:1506921878.242870:0:3858:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:16.0:1506921878.242872:0:3858:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921878.242875:0:3858:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c2256f0 00000400:00000200:16.0:1506921878.242879:0:3858:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1d09 [1] + 192 00000100:00000001:16.0:1506921878.242882:0:3858:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921878.242884:0:3858:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881e1b23c800 x1580016666302192/t0(0) o19->oak-OST0001-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:16.0:1506921878.242889:0:3858:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881e1b23c800 x1580016666302192/t0(0) o19->oak-OST0001-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.242896:0:3858:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921878.242897:0:3858:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921878.242898:0:3858:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.242900:0:3858:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:11.0:1506921878.242900:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:16.0:1506921878.242901:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000400:00000001:11.0:1506921878.242902:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:11.0:1506921878.242903:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:11.0:1506921878.242904:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921878.242906:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881e1b23c800 x1580016666302192/t0(0) o19->oak-OST0001-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:11.0:1506921878.242911:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881e1b23c800 x1580016666302192/t0(0) o19->oak-OST0001-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.242915:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921878.242917:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203bf0af00. 00000400:00000200:11.0:1506921878.242918:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735280 00000400:00000010:11.0:1506921878.242919:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735280. 00000100:00000001:11.0:1506921878.242920:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921878.242921:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:11.0:1506921878.242922:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:11.0:1506921878.242923:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:11.0:1506921878.242924:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921878.242925:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:11.0:1506921878.242927:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88203bc55400 time=73 v=31 (31 31 31 31) 00000100:00001000:11.0:1506921878.242929:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203bc553c8 time=73 v=1 (1 1 1 1) 00000100:00000001:11.0:1506921878.242931:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:11.0:1506921878.242932:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:11.0:1506921878.242933:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:11.0:1506921878.242934:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.242936:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:11.0:1506921878.242936:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0001-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:11.0:1506921878.242937:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:11.0:1506921878.242938:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:11.0:1506921878.242940:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881e1b23c800 x1580016666302192/t0(0) o19->oak-OST0001-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921878.242944:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921878.242945:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:11.0:1506921878.242946:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:11.0:1506921878.242948:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881e1b23c800 x1580016666302192/t0(0) o19->oak-OST0001-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:11.0:1506921878.242953:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302192:10.0.2.102@o2ib5:19 00000100:00000001:11.0:1506921878.242955:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:11.0:1506921878.242957:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.242958:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:11.0:1506921878.242960:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921878.242961:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881e1b23c800 x1580016666302192/t0(0) o19->oak-OST0001-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921878.242965:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:11.0:1506921878.242966:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8820382d75c0. 00000100:00000001:11.0:1506921878.242968:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:11.0:1506921878.242969:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.242970:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921878.242972:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881e1b23c800 x1580016666302192/t0(0) o19->oak-OST0001-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:11.0:1506921878.242975:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:11.0:1506921878.242976:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:11.0:1506921878.242977:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8810e90bc000. 02000000:00000001:11.0:1506921878.242978:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:11.0:1506921878.242979:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:11.0:1506921878.242980:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203bc55000 refcount=4 obd=oak-OST0001-osc-ffff88203bc47000 00000020:00000001:11.0:1506921878.242982:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:11.0:1506921878.242983:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff882011e15800. 02000000:00000001:11.0:1506921878.242984:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:11.0:1506921878.242985:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:11.0:1506921878.242986:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881e1b23c800. 00000100:00000001:11.0:1506921878.242987:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:11.0:1506921878.242988:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:11.0:1506921878.242989:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921878.242990:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:11.0:1506921878.242991:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:11.0:1506921878.242992:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:11.0:1506921878.242993:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881e1b23c800. 00000020:00000040:11.0:1506921878.242994:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203c9bf000 refcount=5 obd=oak-OST0002-osc-ffff88203bc47000 00000100:00000001:11.0:1506921878.242996:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:11.0:1506921878.242997:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:11.0:1506921878.242997:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:11.0:1506921878.242999:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff882011e15800. 00000100:00000001:11.0:1506921878.243000:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.243001:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:11.0:1506921878.243002:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:11.0:1506921878.243003:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8820382d75c0. 00000100:00000001:11.0:1506921878.243004:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612270695675328 : -131803013876288 : ffff8820382d75c0) 00000100:00000001:11.0:1506921878.243005:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921878.243006:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618567168 : -131812090984448 : ffff881e1b23c800) 00000020:00000001:11.0:1506921878.243007:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:11.0:1506921878.243008:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:11.0:1506921878.243015:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:11.0:1506921878.243016:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:11.0:1506921878.243018:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:11.0:1506921878.243019:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:11.0:1506921878.243020:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:11.0:1506921878.243021:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881e1b23c800 x1580016666302208/t0(0) o19->oak-OST0002-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.243025:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:11.0:1506921878.243026:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921878.243028:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:11.0:1506921878.243028:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:11.0:1506921878.243030:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302208:10.0.2.101@o2ib5:19 00000100:00000001:11.0:1506921878.243032:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:11.0:1506921878.243033:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:11.0:1506921878.243034:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:11.0:1506921878.243035:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:11.0:1506921878.243036:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8810e90bc000. 02000000:00000001:11.0:1506921878.243037:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:11.0:1506921878.243038:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203bf0af00. 00000400:00000010:11.0:1506921878.243039:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735280. 00000100:00000200:11.0:1506921878.243041:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302208, portal 4 00000100:00000001:11.0:1506921878.243042:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:11.0:1506921878.243043:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612261618567168 : -131812090984448 : ffff881e1b23c800) 00000100:00000040:11.0:1506921878.243045:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881e1b23c800 x1580016666302208/t0(0) o19->oak-OST0002-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.243049:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:11.0:1506921878.243051:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:11.0:1506921878.243052:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032735380. 00000100:00000200:11.0:1506921878.243054:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302208, offset 0 00000400:00000010:11.0:1506921878.243056:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e11600 (tot 59423868). 00000400:00000200:11.0:1506921878.243058:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:11.0:1506921878.243064:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:11.0:1506921878.243067:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:11.0:1506921878.243070:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:11.0:1506921878.243072:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:11.0:1506921878.243073:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921878.243075:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:11.0:1506921878.243076:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.243077:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:11.0:1506921878.243079:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.243080:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:11.0:1506921878.243080:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:11.0:1506921878.243081:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8820382d75c0 going to sleep for 43 seconds 00000100:00000001:11.0:1506921878.243083:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921878.243084:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.243085:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:11.0:1506921878.243086:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.243135:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921878.243179:0:3864:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:1.0:1506921878.243186:0:3864:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:1.0:1506921878.243190:0:3864:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:1.0:1506921878.243194:0:3864:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881e1b23c800 x1580016666302208/t0(0) o19->oak-OST0002-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921878.243202:0:3864:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1506921878.243204:0:3864:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881e1b23c800 x1580016666302208/t0(0) o19->oak-OST0002-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921878.243209:0:3864:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921878.243211:0:3864:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:1.0:1506921878.243212:0:3864:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735380 00000400:00000010:1.0:1506921878.243213:0:3864:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735380. 00000400:00000010:1.0:1506921878.243217:0:3864:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e11600 (tot 59423468). 00000800:00000200:13.0:1506921878.243218:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:1.0:1506921878.243222:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:2.2:1506921878.243328:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:11.0:1506921878.243352:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921878.243358:0:3862:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.42@o2ib4 00000400:00000200:11.0:1506921878.243363:0:3862:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:11.0:1506921878.243365:0:3862:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e13400 (tot 59423868). 00000400:00000200:11.0:1506921878.243369:0:3862:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225700 00000400:00000200:11.0:1506921878.243373:0:3862:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a101d [1] + 192 00000100:00000001:11.0:1506921878.243377:0:3862:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921878.243379:0:3862:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881e1b23c800 x1580016666302208/t0(0) o19->oak-OST0002-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:11.0:1506921878.243384:0:3862:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881e1b23c800 x1580016666302208/t0(0) o19->oak-OST0002-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000800:00000200:5.0:1506921878.243389:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000001:11.0:1506921878.243391:0:3862:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921878.243393:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e13400 (tot 59423468). 00000800:00000200:11.0:1506921878.243395:0:3862:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000100:00000001:13.0:1506921878.243396:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:11.0:1506921878.243397:0:3862:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:11.0:1506921878.243398:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000400:00000001:13.0:1506921878.243400:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:13.0:1506921878.243401:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:13.0:1506921878.243402:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:13.0:1506921878.243406:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881e1b23c800 x1580016666302208/t0(0) o19->oak-OST0002-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:13.0:1506921878.243414:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881e1b23c800 x1580016666302208/t0(0) o19->oak-OST0002-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921878.243419:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:13.0:1506921878.243423:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203bf0af00. 00000400:00000200:13.0:1506921878.243425:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032735280 00000400:00000010:13.0:1506921878.243426:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032735280. 00000100:00000001:13.0:1506921878.243428:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:13.0:1506921878.243429:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:13.0:1506921878.243431:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:13.0:1506921878.243432:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:13.0:1506921878.243433:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:13.0:1506921878.243435:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:13.0:1506921878.243438:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88203c9bf400 time=73 v=31 (31 1 1 31) 00000100:00001000:13.0:1506921878.243441:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203c9bf3c8 time=73 v=1 (1 1 1 1) 00000100:00000001:13.0:1506921878.243442:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:13.0:1506921878.243443:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:13.0:1506921878.243445:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:13.0:1506921878.243446:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921878.243447:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:13.0:1506921878.243448:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0002-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:13.0:1506921878.243450:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:13.0:1506921878.243450:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:13.0:1506921878.243453:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881e1b23c800 x1580016666302208/t0(0) o19->oak-OST0002-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:13.0:1506921878.243458:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:13.0:1506921878.243460:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:13.0:1506921878.243461:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:13.0:1506921878.243463:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881e1b23c800 x1580016666302208/t0(0) o19->oak-OST0002-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:13.0:1506921878.243469:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302208:10.0.2.101@o2ib5:19 00000100:00000001:13.0:1506921878.243471:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:13.0:1506921878.243473:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921878.243474:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:13.0:1506921878.243475:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:13.0:1506921878.243477:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881e1b23c800 x1580016666302208/t0(0) o19->oak-OST0002-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:13.0:1506921878.243481:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:13.0:1506921878.243483:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8820382d75c0. 00000100:00000001:13.0:1506921878.243485:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:13.0:1506921878.243486:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921878.243488:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:13.0:1506921878.243489:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881e1b23c800 x1580016666302208/t0(0) o19->oak-OST0002-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:13.0:1506921878.243493:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:13.0:1506921878.243494:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:13.0:1506921878.243495:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8810e90bc000. 02000000:00000001:13.0:1506921878.243497:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:13.0:1506921878.243498:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:13.0:1506921878.243499:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203c9bf000 refcount=4 obd=oak-OST0002-osc-ffff88203bc47000 00000020:00000001:13.0:1506921878.243500:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:13.0:1506921878.243501:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff882011e15800. 02000000:00000001:13.0:1506921878.243503:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:13.0:1506921878.243504:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:13.0:1506921878.243505:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881e1b23c800. 00000100:00000001:13.0:1506921878.243506:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:13.0:1506921878.243507:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:13.0:1506921878.243508:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:13.0:1506921878.243510:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:13.0:1506921878.243511:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:13.0:1506921878.243512:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:13.0:1506921878.243514:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881cd2f6a100. 00000020:00000040:13.0:1506921878.243515:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203bc53800 refcount=5 obd=oak-OST0003-osc-ffff88203bc47000 00000100:00000001:13.0:1506921878.243517:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:13.0:1506921878.243518:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:13.0:1506921878.243519:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:13.0:1506921878.243521:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff88203bd18400. 00000100:00000001:13.0:1506921878.243523:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921878.243525:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:13.0:1506921878.243526:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:13.0:1506921878.243527:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff88203830f8c0. 00000100:00000001:13.0:1506921878.243528:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612270695905472 : -131803013646144 : ffff88203830f8c0) 00000100:00000001:13.0:1506921878.243530:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:13.0:1506921878.243531:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612256112681216 : -131817596870400 : ffff881cd2f6a100) 00000020:00000001:13.0:1506921878.243533:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:13.0:1506921878.243534:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:13.0:1506921878.243543:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:13.0:1506921878.243546:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:13.0:1506921878.243547:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:13.0:1506921878.243549:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:13.0:1506921878.243549:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:13.0:1506921878.243551:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881cd2f6a100 x1580016666302224/t0(0) o19->oak-OST0003-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921878.243556:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:13.0:1506921878.243557:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:13.0:1506921878.243558:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:13.0:1506921878.243559:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:13.0:1506921878.243561:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302224:10.0.2.102@o2ib5:19 00000100:00000001:13.0:1506921878.243563:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:13.0:1506921878.243565:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:13.0:1506921878.243566:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:13.0:1506921878.243567:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:13.0:1506921878.243569:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff882033d7d400. 02000000:00000001:13.0:1506921878.243570:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:13.0:1506921878.243572:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203b78bd20. 00000400:00000010:13.0:1506921878.243574:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881dc4ade400. 00000100:00000200:13.0:1506921878.243576:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302224, portal 4 00000100:00000001:13.0:1506921878.243577:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:13.0:1506921878.243578:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612256112681216 : -131817596870400 : ffff881cd2f6a100) 00000100:00000040:13.0:1506921878.243581:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881cd2f6a100 x1580016666302224/t0(0) o19->oak-OST0003-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921878.243585:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:13.0:1506921878.243587:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:13.0:1506921878.243588:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881dc4adf200. 00000100:00000200:13.0:1506921878.243590:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302224, offset 0 00000400:00000010:13.0:1506921878.243592:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88203bd18000 (tot 59423868). 00000400:00000200:13.0:1506921878.243595:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:13.0:1506921878.243604:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:13.0:1506921878.243608:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:13.0:1506921878.243614:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:13.0:1506921878.243616:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:13.0:1506921878.243618:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:13.0:1506921878.243621:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:13.0:1506921878.243622:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921878.243623:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:13.0:1506921878.243625:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921878.243626:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:13.0:1506921878.243627:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:13.0:1506921878.243628:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff88203830f8c0 going to sleep for 43 seconds 00000100:00000001:13.0:1506921878.243630:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:13.0:1506921878.243631:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921878.243632:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:13.0:1506921878.243633:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921878.243656:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.243674:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.243679:0:3859:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.243685:0:3859:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.243689:0:3859:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881cd2f6a100 x1580016666302224/t0(0) o19->oak-OST0003-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.243697:0:3859:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921878.243699:0:3859:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881cd2f6a100 x1580016666302224/t0(0) o19->oak-OST0003-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.243705:0:3859:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.243706:0:3859:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921878.243708:0:3859:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881dc4adf200 00000400:00000010:16.0:1506921878.243710:0:3859:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881dc4adf200. 00000800:00000200:18.0:1506921878.243711:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000010:16.0:1506921878.243713:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88203bd18000 (tot 59423468). 00000800:00000200:16.0:1506921878.243716:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921878.243824:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:13.0:1506921878.243836:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921878.243842:0:3863:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.42@o2ib4 00000400:00000200:13.0:1506921878.243846:0:3863:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:13.0:1506921878.243848:0:3863:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88203bd18600 (tot 59423868). 00000400:00000200:13.0:1506921878.243852:0:3863:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225710 00000400:00000200:13.0:1506921878.243856:0:3863:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1d11 [1] + 192 00000100:00000001:13.0:1506921878.243860:0:3863:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:13.0:1506921878.243862:0:3863:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881cd2f6a100 x1580016666302224/t0(0) o19->oak-OST0003-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:13.0:1506921878.243867:0:3863:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881cd2f6a100 x1580016666302224/t0(0) o19->oak-OST0003-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921878.243875:0:3863:0:(events.c:171:reply_in_callback()) Process leaving 00000800:00000200:1.0:1506921878.243875:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000400:00000010:13.0:1506921878.243876:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88203bd18600 (tot 59423468). 00000800:00000200:13.0:1506921878.243879:0:3863:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (22)++ 00000800:00000200:13.0:1506921878.243880:0:3863:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:13.0:1506921878.243882:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921878.243908:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.243913:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.243914:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.243915:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.243919:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881cd2f6a100 x1580016666302224/t0(0) o19->oak-OST0003-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.243928:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881cd2f6a100 x1580016666302224/t0(0) o19->oak-OST0003-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.243934:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.243935:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203b78bd20. 00000400:00000200:15.0:1506921878.243937:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881dc4ade400 00000400:00000010:15.0:1506921878.243939:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881dc4ade400. 00000100:00000001:15.0:1506921878.243940:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.243944:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.243945:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.243947:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.243948:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.243950:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.243953:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88203bc53c00 time=73 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921878.243955:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203bc53bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.243957:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.243958:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.243960:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.243961:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.243963:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.243964:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0003-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.243965:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.243966:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.243968:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881cd2f6a100 x1580016666302224/t0(0) o19->oak-OST0003-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.243973:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.243975:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.243976:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.243978:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881cd2f6a100 x1580016666302224/t0(0) o19->oak-OST0003-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.243983:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302224:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.243986:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.243988:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.243989:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.243991:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.243992:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881cd2f6a100 x1580016666302224/t0(0) o19->oak-OST0003-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.243997:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.243998:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff88203830f8c0. 00000100:00000001:15.0:1506921878.244000:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.244001:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.244003:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.244004:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881cd2f6a100 x1580016666302224/t0(0) o19->oak-OST0003-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.244009:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.244010:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.244011:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff882033d7d400. 02000000:00000001:15.0:1506921878.244013:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.244014:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.244015:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203bc53800 refcount=4 obd=oak-OST0003-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.244016:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.244017:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff88203bd18400. 02000000:00000001:15.0:1506921878.244019:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.244020:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.244021:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881cd2f6a100. 00000100:00000001:15.0:1506921878.244022:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.244023:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.244025:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.244026:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.244027:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.244029:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.244030:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.244032:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203c9bf800 refcount=5 obd=oak-OST0004-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.244033:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.244034:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.244035:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.244037:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.244040:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.244041:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.244042:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.244044:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.244045:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.244047:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.244047:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.244049:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.244051:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.244060:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.244062:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.244063:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.244066:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.244066:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.244068:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302240/t0(0) o19->oak-OST0004-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.244073:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.244074:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.244076:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.244076:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.244078:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302240:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.244081:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.244082:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.244083:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.244084:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.244085:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.244087:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.244089:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.244091:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.244093:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302240, portal 4 00000100:00000001:15.0:1506921878.244095:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.244095:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.244098:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302240/t0(0) o19->oak-OST0004-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.244103:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.244105:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921878.244106:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad500. 00000100:00000200:15.0:1506921878.244107:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302240, offset 0 00000400:00000010:15.0:1506921878.244109:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451c400 (tot 59423868). 00000400:00000200:15.0:1506921878.244112:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921878.244121:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921878.244125:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921878.244129:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.244132:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921878.244133:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921878.244138:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.244139:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.244140:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.244142:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.244143:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.244144:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.244145:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.244146:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.244147:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.244149:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.244149:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.244175:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921878.244217:0:3865:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:5.0:1506921878.244224:0:3865:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:5.0:1506921878.244227:0:3865:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:5.0:1506921878.244231:0:3865:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302240/t0(0) o19->oak-OST0004-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921878.244239:0:3865:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:5.0:1506921878.244241:0:3865:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302240/t0(0) o19->oak-OST0004-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921878.244246:0:3865:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:5.0:1506921878.244248:0:3865:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:5.0:1506921878.244249:0:3865:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad500 00000400:00000010:5.0:1506921878.244250:0:3865:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad500. 00000400:00000010:5.0:1506921878.244253:0:3865:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451c400 (tot 59423468). 00000800:00000200:11.0:1506921878.244254:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:5.0:1506921878.244259:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.2:1506921878.244453:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921878.244471:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.244477:0:3860:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921878.244483:0:3860:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921878.244485:0:3860:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921878.244491:0:3860:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225720 00000400:00000200:16.0:1506921878.244496:0:3860:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a1035 [1] + 192 00000100:00000001:16.0:1506921878.244501:0:3860:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921878.244503:0:3860:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302240/t0(0) o19->oak-OST0004-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921878.244507:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921878.244511:0:3860:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302240/t0(0) o19->oak-OST0004-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.244520:0:3860:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921878.244522:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921878.244524:0:3860:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.244527:0:3860:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921878.244528:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:15.0:1506921878.244559:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.244564:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.244565:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.244566:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.244570:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302240/t0(0) o19->oak-OST0004-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.244579:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302240/t0(0) o19->oak-OST0004-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.244585:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.244587:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.244588:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.244590:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.244591:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.244593:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.244594:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.244595:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.244596:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.244598:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.244601:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88203c9bfc00 time=73 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921878.244604:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203c9bfbc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.244606:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.244607:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.244608:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.244609:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.244611:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.244612:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0004-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.244615:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.244616:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.244618:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302240/t0(0) o19->oak-OST0004-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.244623:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.244625:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.244626:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.244628:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302240/t0(0) o19->oak-OST0004-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.244633:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302240:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.244636:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.244639:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.244640:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.244641:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.244642:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302240/t0(0) o19->oak-OST0004-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.244647:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.244648:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.244650:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.244650:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.244652:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.244654:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302240/t0(0) o19->oak-OST0004-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.244658:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.244659:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.244660:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.244662:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.244663:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.244664:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203c9bf800 refcount=4 obd=oak-OST0004-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.244665:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.244666:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.244667:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.244668:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.244669:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.244671:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.244672:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.244673:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.244675:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.244676:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.244677:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.244678:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.244680:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203c9bd800 refcount=5 obd=oak-OST0005-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.244682:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.244683:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.244684:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.244686:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.244688:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.244690:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.244690:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.244692:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.244693:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.244695:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.244695:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.244697:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.244698:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.244707:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.244710:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.244711:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.244712:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.244713:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.244715:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302256/t0(0) o19->oak-OST0005-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.244735:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.244736:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.244738:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.244739:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.244741:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302256:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.244743:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.244744:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.244747:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.244748:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.244750:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.244751:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.244753:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.244755:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.244757:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302256, portal 4 00000100:00000001:15.0:1506921878.244759:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.244759:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.244762:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302256/t0(0) o19->oak-OST0005-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.244766:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.244768:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921878.244770:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad100. 00000100:00000200:15.0:1506921878.244771:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302256, offset 0 00000400:00000010:15.0:1506921878.244773:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451b000 (tot 59423868). 00000400:00000200:15.0:1506921878.244776:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921878.244785:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921878.244790:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921878.244794:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.244797:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921878.244799:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921878.244802:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.244804:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.244805:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.244806:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.244807:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.244808:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.244810:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.244811:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.244812:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.244813:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.244814:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921878.244835:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.244852:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.244857:0:3861:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.244860:0:3861:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.244864:0:3861:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302256/t0(0) o19->oak-OST0005-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.244879:0:3861:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921878.244881:0:3861:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302256/t0(0) o19->oak-OST0005-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.244886:0:3861:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.244888:0:3861:0:(events.c:81:request_out_callback()) Process leaving 00000800:00000200:18.0:1506921878.244889:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000200:16.0:1506921878.244889:0:3861:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad100 00000400:00000010:16.0:1506921878.244891:0:3861:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad100. 00000400:00000010:16.0:1506921878.244894:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451b000 (tot 59423468). 00000800:00000200:16.0:1506921878.244897:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:16.2:1506921878.244965:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921878.244972:0:3858:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.244976:0:3858:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000800:00000200:18.0:1506921878.244979:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000200:16.0:1506921878.244981:0:3858:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:16.0:1506921878.244984:0:3858:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921878.244989:0:3858:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225730 00000400:00000200:16.0:1506921878.244993:0:3858:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1d19 [1] + 192 00000100:00000001:16.0:1506921878.244998:0:3858:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921878.245000:0:3858:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302256/t0(0) o19->oak-OST0005-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:16.0:1506921878.245006:0:3858:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302256/t0(0) o19->oak-OST0005-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.245013:0:3858:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921878.245014:0:3858:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921878.245016:0:3858:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.245018:0:3858:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.245019:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:16.0:1506921878.245020:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000400:00000001:15.0:1506921878.245022:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.245023:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.245024:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.245027:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302256/t0(0) o19->oak-OST0005-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.245032:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302256/t0(0) o19->oak-OST0005-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.245037:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.245038:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.245039:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.245041:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.245041:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.245043:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.245043:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.245044:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.245046:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.245047:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.245049:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88203c9bdc00 time=73 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921878.245051:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203c9bdbc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.245053:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.245054:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.245055:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.245056:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.245058:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.245058:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0005-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.245060:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.245060:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.245062:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302256/t0(0) o19->oak-OST0005-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.245067:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.245068:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.245069:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.245071:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302256/t0(0) o19->oak-OST0005-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.245076:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302256:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.245080:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.245082:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.245083:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.245085:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.245086:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302256/t0(0) o19->oak-OST0005-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.245091:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.245092:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.245093:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.245094:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.245095:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.245097:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302256/t0(0) o19->oak-OST0005-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.245101:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.245102:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.245103:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.245104:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.245105:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.245106:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203c9bd800 refcount=4 obd=oak-OST0005-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.245107:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.245108:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.245109:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.245110:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.245111:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.245113:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.245113:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.245114:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.245115:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.245117:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.245117:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.245118:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.245120:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203c6af800 refcount=5 obd=oak-OST0006-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.245121:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.245122:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.245123:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.245124:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.245126:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.245128:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.245128:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.245130:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.245131:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.245132:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.245133:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.245134:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.245135:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.245142:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.245144:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.245145:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.245147:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.245147:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.245149:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302272/t0(0) o19->oak-OST0006-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.245153:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.245154:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.245155:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.245156:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.245158:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302272:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.245159:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.245160:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.245161:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.245162:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.245163:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.245164:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.245166:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.245167:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.245169:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302272, portal 4 00000100:00000001:15.0:1506921878.245170:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.245171:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.245173:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302272/t0(0) o19->oak-OST0006-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.245186:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.245187:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921878.245189:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fadd00. 00000100:00000200:15.0:1506921878.245190:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302272, offset 0 00000400:00000010:15.0:1506921878.245192:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451e600 (tot 59423868). 00000400:00000200:15.0:1506921878.245195:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921878.245201:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921878.245205:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921878.245208:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.245210:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921878.245211:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921878.245213:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.245214:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.245215:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.245217:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.245218:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.245218:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.245220:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.245221:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.245222:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.245223:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.245224:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.245246:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921878.245290:0:3864:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:1.0:1506921878.245297:0:3864:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:1.0:1506921878.245300:0:3864:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:1.0:1506921878.245305:0:3864:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302272/t0(0) o19->oak-OST0006-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921878.245312:0:3864:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1506921878.245314:0:3864:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302272/t0(0) o19->oak-OST0006-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921878.245319:0:3864:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921878.245323:0:3864:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:1.0:1506921878.245324:0:3864:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fadd00 00000400:00000010:1.0:1506921878.245325:0:3864:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fadd00. 00000400:00000010:1.0:1506921878.245329:0:3864:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451e600 (tot 59423468). 00000800:00000200:13.0:1506921878.245330:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:1.0:1506921878.245334:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:2.2:1506921878.245443:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:11.0:1506921878.245484:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921878.245492:0:3862:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.42@o2ib4 00000400:00000200:11.0:1506921878.245498:0:3862:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:11.0:1506921878.245501:0:3862:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e13400 (tot 59423868). 00000400:00000200:11.0:1506921878.245506:0:3862:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225740 00000400:00000200:11.0:1506921878.245511:0:3862:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a104d [1] + 192 00000100:00000001:11.0:1506921878.245515:0:3862:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921878.245518:0:3862:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302272/t0(0) o19->oak-OST0006-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:5.0:1506921878.245523:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:11.0:1506921878.245526:0:3862:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302272/t0(0) o19->oak-OST0006-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.245540:0:3862:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921878.245541:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e13400 (tot 59423468). 00000800:00000200:11.0:1506921878.245544:0:3862:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921878.245546:0:3862:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:11.0:1506921878.245547:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921878.245573:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.245577:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.245578:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.245579:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.245583:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302272/t0(0) o19->oak-OST0006-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.245592:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302272/t0(0) o19->oak-OST0006-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.245598:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.245599:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.245600:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.245602:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.245605:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.245607:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.245608:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.245609:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.245610:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.245612:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.245615:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88203c6afc00 time=73 v=31 (31 1 1 31) 00000100:00001000:15.0:1506921878.245618:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203c6afbc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.245619:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.245620:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.245622:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.245623:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.245625:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.245626:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0006-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.245627:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.245628:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.245630:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302272/t0(0) o19->oak-OST0006-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.245635:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.245637:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.245638:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.245640:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302272/t0(0) o19->oak-OST0006-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.245645:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302272:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.245648:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.245650:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.245651:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.245652:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.245653:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302272/t0(0) o19->oak-OST0006-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.245658:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.245659:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.245661:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.245661:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.245663:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.245665:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302272/t0(0) o19->oak-OST0006-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.245669:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.245670:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.245671:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.245673:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.245674:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.245675:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203c6af800 refcount=4 obd=oak-OST0006-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.245676:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.245677:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.245679:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.245679:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.245680:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.245682:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.245682:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.245684:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.245685:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.245686:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.245688:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.245689:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.245690:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203c6a9800 refcount=5 obd=oak-OST0007-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.245692:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.245693:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.245694:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.245696:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.245698:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.245700:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.245701:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.245702:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.245703:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.245705:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.245705:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.245707:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.245709:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.245728:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.245731:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.245733:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.245735:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.245735:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.245737:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302288/t0(0) o19->oak-OST0007-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.245742:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.245743:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.245744:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.245745:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.245749:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302288:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.245751:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.245752:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.245753:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.245754:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.245756:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.245757:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.245759:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.245761:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.245763:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302288, portal 4 00000100:00000001:15.0:1506921878.245765:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.245766:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.245768:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302288/t0(0) o19->oak-OST0007-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.245773:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.245775:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921878.245776:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fac700. 00000100:00000200:15.0:1506921878.245777:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302288, offset 0 00000400:00000010:15.0:1506921878.245779:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451ba00 (tot 59423868). 00000400:00000200:15.0:1506921878.245782:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921878.245791:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921878.245795:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921878.245800:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.245803:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921878.245805:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921878.245809:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.245810:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.245811:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.245813:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.245814:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.245815:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.245816:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.245817:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.245818:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.245820:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.245820:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921878.245841:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.245859:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.245864:0:3859:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.245868:0:3859:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.245873:0:3859:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302288/t0(0) o19->oak-OST0007-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.245880:0:3859:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921878.245883:0:3859:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302288/t0(0) o19->oak-OST0007-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.245890:0:3859:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.245892:0:3859:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921878.245893:0:3859:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fac700 00000400:00000010:16.0:1506921878.245895:0:3859:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fac700. 00000800:00000200:18.0:1506921878.245896:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000010:16.0:1506921878.245898:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451ba00 (tot 59423468). 00000800:00000200:16.0:1506921878.245901:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921878.246037:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:13.0:1506921878.246081:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921878.246089:0:3863:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.42@o2ib4 00000400:00000200:13.0:1506921878.246095:0:3863:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:13.0:1506921878.246099:0:3863:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88203bd18600 (tot 59423868). 00000400:00000200:13.0:1506921878.246104:0:3863:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225750 00000400:00000200:13.0:1506921878.246109:0:3863:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1d21 [1] + 192 00000100:00000001:13.0:1506921878.246114:0:3863:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:13.0:1506921878.246116:0:3863:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302288/t0(0) o19->oak-OST0007-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:1.0:1506921878.246120:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:13.0:1506921878.246124:0:3863:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302288/t0(0) o19->oak-OST0007-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921878.246131:0:3863:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:13.0:1506921878.246133:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88203bd18600 (tot 59423468). 00000800:00000200:13.0:1506921878.246135:0:3863:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921878.246138:0:3863:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:13.0:1506921878.246139:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921878.246165:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.246169:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.246170:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.246171:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.246175:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302288/t0(0) o19->oak-OST0007-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.246184:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302288/t0(0) o19->oak-OST0007-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.246190:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.246191:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.246193:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.246194:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.246196:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.246197:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.246199:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.246200:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.246201:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.246203:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.246206:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88203c6a9c00 time=73 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921878.246208:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203c6a9bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.246210:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.246211:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.246212:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.246214:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.246215:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.246218:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0007-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.246220:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.246220:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.246223:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302288/t0(0) o19->oak-OST0007-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.246228:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.246230:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.246231:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.246233:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302288/t0(0) o19->oak-OST0007-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.246238:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302288:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.246241:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.246243:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.246244:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.246245:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.246247:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302288/t0(0) o19->oak-OST0007-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.246251:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.246252:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.246254:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.246255:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.246257:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.246258:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302288/t0(0) o19->oak-OST0007-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.246262:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.246263:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.246264:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.246266:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.246267:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.246268:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203c6a9800 refcount=4 obd=oak-OST0007-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.246269:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.246270:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.246272:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.246272:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.246274:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.246275:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.246276:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.246277:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.246279:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.246280:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.246281:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.246282:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.246284:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203969d800 refcount=5 obd=oak-OST0008-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.246285:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.246287:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.246288:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.246290:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.246292:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.246294:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.246294:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.246296:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.246297:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.246298:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.246299:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.246301:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.246302:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.246311:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.246313:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.246314:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.246316:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.246317:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.246318:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302304/t0(0) o19->oak-OST0008-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.246323:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.246324:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.246326:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.246326:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.246328:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302304:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.246331:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.246332:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.246333:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.246335:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.246336:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.246337:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.246339:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.246341:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.246343:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302304, portal 4 00000100:00000001:15.0:1506921878.246344:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.246345:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.246348:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302304/t0(0) o19->oak-OST0008-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.246352:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.246354:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921878.246356:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fac800. 00000100:00000200:15.0:1506921878.246357:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302304, offset 0 00000400:00000010:15.0:1506921878.246359:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451f200 (tot 59423868). 00000400:00000200:15.0:1506921878.246362:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921878.246371:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921878.246375:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921878.246379:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.246382:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921878.246383:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921878.246387:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.246388:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.246389:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.246391:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.246392:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.246393:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.246394:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.246395:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.246396:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.246398:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.246398:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.246413:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921878.246453:0:3865:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:5.0:1506921878.246459:0:3865:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:5.0:1506921878.246462:0:3865:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:5.0:1506921878.246466:0:3865:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302304/t0(0) o19->oak-OST0008-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921878.246474:0:3865:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:5.0:1506921878.246478:0:3865:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302304/t0(0) o19->oak-OST0008-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921878.246484:0:3865:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:5.0:1506921878.246485:0:3865:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:5.0:1506921878.246487:0:3865:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fac800 00000400:00000010:5.0:1506921878.246488:0:3865:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fac800. 00000800:00000200:11.0:1506921878.246490:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000400:00000010:5.0:1506921878.246491:0:3865:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451f200 (tot 59423468). 00000800:00000200:5.0:1506921878.246494:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.2:1506921878.246720:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921878.246740:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.246746:0:3860:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921878.246753:0:3860:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921878.246755:0:3860:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921878.246761:0:3860:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225760 00000400:00000200:16.0:1506921878.246766:0:3860:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a1065 [1] + 192 00000100:00000001:16.0:1506921878.246771:0:3860:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921878.246774:0:3860:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302304/t0(0) o19->oak-OST0008-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921878.246777:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921878.246781:0:3860:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302304/t0(0) o19->oak-OST0008-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.246790:0:3860:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921878.246791:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921878.246794:0:3860:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.246796:0:3860:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921878.246798:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:15.0:1506921878.246826:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.246832:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.246833:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.246834:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.246839:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302304/t0(0) o19->oak-OST0008-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.246847:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302304/t0(0) o19->oak-OST0008-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.246853:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.246855:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.246856:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.246858:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.246859:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.246861:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.246862:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.246863:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.246865:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.246866:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.246869:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88203969dc00 time=73 v=31 (31 3 1 31) 00000100:00001000:15.0:1506921878.246872:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203969dbc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.246874:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.246875:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.246876:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.246878:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.246879:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.246880:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0008-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.246881:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.246882:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.246884:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302304/t0(0) o19->oak-OST0008-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.246890:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.246892:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.246893:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.246895:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302304/t0(0) o19->oak-OST0008-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.246900:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302304:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.246905:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.246907:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.246908:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.246909:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.246911:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302304/t0(0) o19->oak-OST0008-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.246915:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.246916:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.246918:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.246919:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.246921:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.246922:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302304/t0(0) o19->oak-OST0008-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.246926:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.246927:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.246928:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.246930:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.246931:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.246932:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203969d800 refcount=4 obd=oak-OST0008-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.246933:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.246934:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.246936:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.246936:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.246937:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.246939:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.246939:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.246941:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.246942:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.246943:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.246944:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.246946:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.246947:0:38882:0:(genops.c:1012:class_import_get()) import ffff882039699800 refcount=5 obd=oak-OST0009-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.246949:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.246950:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.246951:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.246953:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.246962:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.246964:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.246965:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.246966:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.246968:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.246970:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.246971:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.246972:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.246974:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.246982:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.246985:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.246986:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.246988:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.246989:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.246990:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302320/t0(0) o19->oak-OST0009-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.246995:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.246996:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.246998:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.246999:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.247001:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302320:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.247003:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.247004:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.247005:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.247007:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.247008:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.247009:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.247011:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.247013:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.247015:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302320, portal 4 00000100:00000001:15.0:1506921878.247017:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.247017:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.247020:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302320/t0(0) o19->oak-OST0009-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.247029:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.247031:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921878.247033:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032facd00. 00000100:00000200:15.0:1506921878.247035:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302320, offset 0 00000400:00000010:15.0:1506921878.247037:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451f200 (tot 59423868). 00000400:00000200:15.0:1506921878.247040:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921878.247050:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921878.247054:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921878.247059:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.247061:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921878.247063:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921878.247067:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.247068:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.247070:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.247071:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.247072:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.247073:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.247075:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.247076:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.247077:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.247079:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.247079:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921878.247102:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.247119:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.247124:0:3861:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.247127:0:3861:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.247132:0:3861:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302320/t0(0) o19->oak-OST0009-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.247139:0:3861:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921878.247142:0:3861:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302320/t0(0) o19->oak-OST0009-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.247147:0:3861:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.247149:0:3861:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921878.247152:0:3861:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032facd00 00000400:00000010:16.0:1506921878.247154:0:3861:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032facd00. 00000800:00000200:18.0:1506921878.247156:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000010:16.0:1506921878.247158:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451f200 (tot 59423468). 00000800:00000200:16.0:1506921878.247161:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:16.2:1506921878.247219:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921878.247225:0:3858:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.247230:0:3858:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000800:00000200:18.0:1506921878.247233:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000200:16.0:1506921878.247235:0:3858:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:16.0:1506921878.247237:0:3858:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921878.247242:0:3858:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225770 00000400:00000200:16.0:1506921878.247247:0:3858:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1d29 [1] + 192 00000100:00000001:16.0:1506921878.247251:0:3858:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921878.247253:0:3858:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302320/t0(0) o19->oak-OST0009-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:16.0:1506921878.247259:0:3858:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302320/t0(0) o19->oak-OST0009-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.247267:0:3858:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921878.247268:0:3858:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921878.247270:0:3858:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.247272:0:3858:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.247273:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:16.0:1506921878.247274:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000400:00000001:15.0:1506921878.247275:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.247276:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.247277:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.247280:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302320/t0(0) o19->oak-OST0009-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.247286:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302320/t0(0) o19->oak-OST0009-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.247291:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.247292:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.247293:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.247295:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.247297:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.247298:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.247299:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.247300:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.247301:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.247303:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.247304:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff882039699c00 time=73 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921878.247307:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff882039699bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.247309:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.247309:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.247311:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.247312:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.247313:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.247314:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0009-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.247315:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.247316:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.247318:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302320/t0(0) o19->oak-OST0009-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.247322:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.247323:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.247324:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.247326:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302320/t0(0) o19->oak-OST0009-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.247332:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302320:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.247334:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.247336:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.247337:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.247338:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.247340:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302320/t0(0) o19->oak-OST0009-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.247344:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.247345:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.247346:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.247347:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.247349:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.247351:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302320/t0(0) o19->oak-OST0009-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.247355:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.247356:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.247357:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.247359:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.247359:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.247360:0:38882:0:(genops.c:1026:class_import_put()) import ffff882039699800 refcount=4 obd=oak-OST0009-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.247362:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.247362:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.247364:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.247365:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.247365:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.247367:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.247368:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.247369:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.247370:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.247371:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.247372:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.247373:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.247374:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203969a000 refcount=5 obd=oak-OST000a-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.247375:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.247376:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.247377:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.247378:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.247380:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.247382:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.247382:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.247383:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.247384:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.247386:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.247387:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.247388:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.247389:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.247396:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.247398:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.247400:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.247401:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.247402:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.247404:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302336/t0(0) o19->oak-OST000a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.247408:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.247409:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.247410:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.247411:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.247413:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302336:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.247415:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.247416:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.247416:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.247417:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.247419:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.247420:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.247421:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.247423:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.247424:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302336, portal 4 00000100:00000001:15.0:1506921878.247426:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.247426:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.247429:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302336/t0(0) o19->oak-OST000a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.247433:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.247434:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921878.247436:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fac200. 00000100:00000200:15.0:1506921878.247437:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302336, offset 0 00000400:00000010:15.0:1506921878.247439:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451ba00 (tot 59423868). 00000400:00000200:15.0:1506921878.247441:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921878.247448:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921878.247451:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921878.247454:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.247456:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921878.247458:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921878.247461:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.247462:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.247463:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.247465:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.247466:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.247467:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.247468:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.247469:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.247470:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.247471:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.247472:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.247495:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921878.247538:0:3864:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:1.0:1506921878.247545:0:3864:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:1.0:1506921878.247549:0:3864:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:1.0:1506921878.247553:0:3864:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302336/t0(0) o19->oak-OST000a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921878.247561:0:3864:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1506921878.247563:0:3864:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302336/t0(0) o19->oak-OST000a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921878.247568:0:3864:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921878.247570:0:3864:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:1.0:1506921878.247571:0:3864:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fac200 00000400:00000010:1.0:1506921878.247573:0:3864:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fac200. 00000400:00000010:1.0:1506921878.247576:0:3864:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451ba00 (tot 59423468). 00000800:00000200:13.0:1506921878.247578:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:1.0:1506921878.247582:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:2.2:1506921878.247639:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:11.0:1506921878.247677:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921878.247685:0:3862:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.42@o2ib4 00000400:00000200:11.0:1506921878.247691:0:3862:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:11.0:1506921878.247694:0:3862:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e13400 (tot 59423868). 00000400:00000200:11.0:1506921878.247699:0:3862:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225780 00000400:00000200:11.0:1506921878.247704:0:3862:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a107d [1] + 192 00000100:00000001:11.0:1506921878.247708:0:3862:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921878.247711:0:3862:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302336/t0(0) o19->oak-OST000a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:5.0:1506921878.247716:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:11.0:1506921878.247729:0:3862:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302336/t0(0) o19->oak-OST000a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.247736:0:3862:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921878.247738:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e13400 (tot 59423468). 00000800:00000200:11.0:1506921878.247740:0:3862:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921878.247742:0:3862:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:11.0:1506921878.247744:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921878.247771:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.247775:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.247776:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.247777:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.247781:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302336/t0(0) o19->oak-OST000a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.247790:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302336/t0(0) o19->oak-OST000a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.247795:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.247797:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.247798:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.247800:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.247801:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.247803:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.247804:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.247806:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.247807:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.247809:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.247812:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88203969a400 time=73 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921878.247815:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203969a3c8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.247816:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.247817:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.247819:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.247820:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.247822:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.247825:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST000a-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.247826:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.247827:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.247830:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302336/t0(0) o19->oak-OST000a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.247835:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.247837:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.247838:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.247840:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302336/t0(0) o19->oak-OST000a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.247845:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302336:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.247848:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.247850:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.247851:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.247852:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.247853:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302336/t0(0) o19->oak-OST000a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.247858:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.247859:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.247861:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.247861:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.247863:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.247864:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302336/t0(0) o19->oak-OST000a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.247868:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.247869:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.247870:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.247872:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.247873:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.247874:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203969a000 refcount=4 obd=oak-OST000a-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.247875:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.247876:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.247878:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.247879:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.247880:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.247882:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.247883:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.247884:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.247886:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.247887:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.247889:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.247890:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.247891:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203969e000 refcount=5 obd=oak-OST000b-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.247893:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.247895:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.247896:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.247898:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.247900:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.247902:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.247903:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.247904:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.247905:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.247907:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.247908:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.247910:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.247911:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.247920:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.247922:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.247923:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.247925:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.247926:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.247928:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302352/t0(0) o19->oak-OST000b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.247933:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.247934:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.247936:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.247936:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.247938:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302352:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.247941:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.247942:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.247944:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.247945:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.247946:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.247948:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.247950:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.247952:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.247954:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302352, portal 4 00000100:00000001:15.0:1506921878.247955:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.247956:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.247959:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302352/t0(0) o19->oak-OST000b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.247963:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.247965:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921878.247967:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fada80. 00000100:00000200:15.0:1506921878.247968:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302352, offset 0 00000400:00000010:15.0:1506921878.247970:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451e600 (tot 59423868). 00000400:00000200:15.0:1506921878.247973:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921878.247982:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921878.247986:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921878.247991:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.247994:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921878.247995:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921878.247999:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.248001:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.248002:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.248003:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.248005:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.248006:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.248007:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.248009:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.248010:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.248011:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.248012:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921878.248042:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.248060:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.248065:0:3859:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.248068:0:3859:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.248072:0:3859:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302352/t0(0) o19->oak-OST000b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.248080:0:3859:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921878.248083:0:3859:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302352/t0(0) o19->oak-OST000b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.248088:0:3859:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.248090:0:3859:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921878.248091:0:3859:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fada80 00000400:00000010:16.0:1506921878.248093:0:3859:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fada80. 00000400:00000010:16.0:1506921878.248096:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451e600 (tot 59423468). 00000800:00000200:18.0:1506921878.248097:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921878.248101:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921878.248228:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:13.0:1506921878.248273:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921878.248282:0:3863:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.42@o2ib4 00000400:00000200:13.0:1506921878.248287:0:3863:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:13.0:1506921878.248290:0:3863:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88203bd18600 (tot 59423868). 00000400:00000200:13.0:1506921878.248295:0:3863:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225790 00000400:00000200:13.0:1506921878.248300:0:3863:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1d31 [1] + 192 00000100:00000001:13.0:1506921878.248304:0:3863:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:13.0:1506921878.248307:0:3863:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302352/t0(0) o19->oak-OST000b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:1.0:1506921878.248313:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:13.0:1506921878.248315:0:3863:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302352/t0(0) o19->oak-OST000b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921878.248322:0:3863:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:13.0:1506921878.248324:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88203bd18600 (tot 59423468). 00000800:00000200:13.0:1506921878.248326:0:3863:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921878.248329:0:3863:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:13.0:1506921878.248330:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921878.248355:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.248361:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.248362:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.248364:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.248368:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302352/t0(0) o19->oak-OST000b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.248377:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302352/t0(0) o19->oak-OST000b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.248383:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.248384:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.248386:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.248388:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.248389:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.248391:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.248392:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.248393:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.248394:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.248396:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.248399:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88203969e400 time=73 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921878.248402:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203969e3c8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.248404:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.248405:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.248406:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.248408:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.248409:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.248410:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST000b-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.248412:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.248412:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.248415:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302352/t0(0) o19->oak-OST000b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.248420:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.248422:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.248423:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.248425:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302352/t0(0) o19->oak-OST000b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.248430:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302352:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.248435:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.248437:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.248438:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.248440:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.248442:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302352/t0(0) o19->oak-OST000b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.248446:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.248447:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.248449:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.248450:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.248452:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.248453:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302352/t0(0) o19->oak-OST000b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.248457:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.248458:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.248459:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.248461:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.248462:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.248463:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203969e000 refcount=4 obd=oak-OST000b-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.248464:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.248465:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.248467:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.248468:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.248469:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.248471:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.248472:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.248473:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.248474:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.248475:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.248477:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.248478:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.248479:0:38882:0:(genops.c:1012:class_import_get()) import ffff882036e42000 refcount=5 obd=oak-OST000c-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.248481:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.248482:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.248483:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.248485:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.248488:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.248490:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.248491:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.248492:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.248493:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.248495:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.248496:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.248498:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.248500:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.248508:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.248511:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.248512:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.248513:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.248514:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.248516:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302368/t0(0) o19->oak-OST000c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.248521:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.248522:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.248524:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.248525:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.248527:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302368:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.248528:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.248530:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.248531:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.248532:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.248533:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.248534:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.248536:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.248538:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.248541:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302368, portal 4 00000100:00000001:15.0:1506921878.248542:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.248543:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.248546:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302368/t0(0) o19->oak-OST000c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.248555:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.248557:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921878.248558:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad380. 00000100:00000200:15.0:1506921878.248560:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302368, offset 0 00000400:00000010:15.0:1506921878.248562:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451b000 (tot 59423868). 00000400:00000200:15.0:1506921878.248565:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921878.248574:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921878.248578:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921878.248582:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.248584:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921878.248586:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921878.248590:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.248591:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.248592:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.248594:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.248595:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.248596:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.248597:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.248599:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.248600:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.248601:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.248602:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.248626:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921878.248668:0:3865:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:5.0:1506921878.248675:0:3865:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:5.0:1506921878.248678:0:3865:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:5.0:1506921878.248682:0:3865:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302368/t0(0) o19->oak-OST000c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921878.248690:0:3865:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:5.0:1506921878.248692:0:3865:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302368/t0(0) o19->oak-OST000c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:11.0:1506921878.248693:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000001:5.0:1506921878.248697:0:3865:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:5.0:1506921878.248699:0:3865:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:5.0:1506921878.248700:0:3865:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad380 00000400:00000010:5.0:1506921878.248701:0:3865:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad380. 00000400:00000010:5.0:1506921878.248705:0:3865:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451b000 (tot 59423468). 00000800:00000200:5.0:1506921878.248707:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.2:1506921878.248877:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921878.248894:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.248901:0:3860:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921878.248907:0:3860:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921878.248910:0:3860:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921878.248915:0:3860:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c2257a0 00000400:00000200:16.0:1506921878.248920:0:3860:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a1095 [1] + 192 00000100:00000001:16.0:1506921878.248925:0:3860:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921878.248928:0:3860:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302368/t0(0) o19->oak-OST000c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921878.248931:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921878.248936:0:3860:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302368/t0(0) o19->oak-OST000c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.248945:0:3860:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921878.248947:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921878.248949:0:3860:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.248951:0:3860:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921878.248953:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:15.0:1506921878.248985:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.248989:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.248991:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.248992:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.248997:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302368/t0(0) o19->oak-OST000c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.249006:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302368/t0(0) o19->oak-OST000c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.249012:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.249014:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.249015:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.249017:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.249020:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.249022:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.249023:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.249024:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.249026:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.249027:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.249031:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff882036e42400 time=73 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921878.249033:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff882036e423c8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.249035:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.249036:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.249038:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.249039:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.249041:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.249042:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST000c-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.249043:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.249044:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.249046:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302368/t0(0) o19->oak-OST000c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.249051:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.249053:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.249054:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.249056:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302368/t0(0) o19->oak-OST000c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.249061:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302368:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.249064:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.249066:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.249067:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.249069:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.249070:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302368/t0(0) o19->oak-OST000c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.249074:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.249076:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.249077:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.249078:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.249080:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.249082:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302368/t0(0) o19->oak-OST000c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.249086:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.249087:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.249089:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.249090:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.249091:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.249092:0:38882:0:(genops.c:1026:class_import_put()) import ffff882036e42000 refcount=4 obd=oak-OST000c-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.249094:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.249095:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.249096:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.249097:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.249098:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.249100:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.249100:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.249101:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.249103:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.249104:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.249105:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.249106:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.249108:0:38882:0:(genops.c:1012:class_import_get()) import ffff882036e44000 refcount=5 obd=oak-OST000d-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.249109:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.249111:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.249112:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.249114:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.249117:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.249118:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.249119:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.249121:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.249122:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.249124:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.249125:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.249127:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.249129:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.249137:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.249139:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.249141:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.249143:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.249144:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.249146:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302384/t0(0) o19->oak-OST000d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.249151:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.249152:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.249153:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.249154:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.249156:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302384:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.249158:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.249159:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.249160:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.249162:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.249163:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.249164:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.249166:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.249168:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.249171:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302384, portal 4 00000100:00000001:15.0:1506921878.249172:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.249173:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.249176:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302384/t0(0) o19->oak-OST000d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.249180:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.249182:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921878.249183:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad880. 00000100:00000200:15.0:1506921878.249184:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302384, offset 0 00000400:00000010:15.0:1506921878.249187:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451c400 (tot 59423868). 00000400:00000200:15.0:1506921878.249190:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921878.249199:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921878.249203:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921878.249208:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.249211:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921878.249213:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921878.249217:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.249219:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.249220:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.249221:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.249223:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.249224:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.249225:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.249227:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.249228:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.249229:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.249230:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921878.249254:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.249271:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.249276:0:3861:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.249279:0:3861:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.249284:0:3861:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302384/t0(0) o19->oak-OST000d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.249291:0:3861:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921878.249293:0:3861:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302384/t0(0) o19->oak-OST000d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.249299:0:3861:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.249301:0:3861:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921878.249302:0:3861:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad880 00000400:00000010:16.0:1506921878.249303:0:3861:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad880. 00000400:00000010:16.0:1506921878.249307:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451c400 (tot 59423468). 00000800:00000200:18.0:1506921878.249308:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921878.249312:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:16.2:1506921878.249377:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921878.249384:0:3858:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.249389:0:3858:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000800:00000200:18.0:1506921878.249392:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000200:16.0:1506921878.249394:0:3858:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:16.0:1506921878.249398:0:3858:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921878.249403:0:3858:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c2257b0 00000400:00000200:16.0:1506921878.249408:0:3858:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1d39 [1] + 192 00000100:00000001:16.0:1506921878.249413:0:3858:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921878.249414:0:3858:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302384/t0(0) o19->oak-OST000d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:16.0:1506921878.249420:0:3858:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302384/t0(0) o19->oak-OST000d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.249428:0:3858:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921878.249429:0:3858:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921878.249431:0:3858:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.249433:0:3858:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.249434:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:16.0:1506921878.249435:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000400:00000001:15.0:1506921878.249436:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.249437:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.249438:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.249440:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302384/t0(0) o19->oak-OST000d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.249446:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302384/t0(0) o19->oak-OST000d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.249451:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.249452:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.249454:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.249455:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.249456:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.249457:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.249458:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.249459:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.249460:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.249461:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.249463:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff882036e44400 time=73 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921878.249465:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff882036e443c8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.249467:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.249468:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.249469:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.249470:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.249472:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.249474:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST000d-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.249475:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.249476:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.249478:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302384/t0(0) o19->oak-OST000d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.249482:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.249484:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.249485:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.249487:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302384/t0(0) o19->oak-OST000d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.249491:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302384:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.249494:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.249496:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.249497:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.249498:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.249499:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302384/t0(0) o19->oak-OST000d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.249504:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.249505:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.249506:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.249507:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.249508:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.249510:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302384/t0(0) o19->oak-OST000d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.249514:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.249514:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.249515:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.249517:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.249518:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.249518:0:38882:0:(genops.c:1026:class_import_put()) import ffff882036e44000 refcount=4 obd=oak-OST000d-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.249520:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.249521:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.249522:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.249523:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.249524:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.249526:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.249527:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.249528:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.249529:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.249530:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.249531:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.249532:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.249533:0:38882:0:(genops.c:1012:class_import_get()) import ffff882036e46000 refcount=5 obd=oak-OST000e-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.249535:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.249535:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.249536:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.249538:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.249539:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.249541:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.249541:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.249542:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.249544:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.249545:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.249546:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.249547:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.249548:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.249555:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.249557:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.249558:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.249560:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.249560:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.249562:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302400/t0(0) o19->oak-OST000e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.249566:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.249567:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.249568:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.249569:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.249571:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302400:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.249573:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.249574:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.249575:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.249576:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.249577:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.249578:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.249580:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.249582:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.249583:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302400, portal 4 00000100:00000001:15.0:1506921878.249585:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.249585:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.249588:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302400/t0(0) o19->oak-OST000e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.249592:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.249593:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921878.249594:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fac680. 00000100:00000200:15.0:1506921878.249596:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302400, offset 0 00000400:00000010:15.0:1506921878.249598:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451c200 (tot 59423868). 00000400:00000200:15.0:1506921878.249600:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921878.249607:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921878.249610:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921878.249613:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.249615:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921878.249617:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921878.249619:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.249620:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.249622:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.249623:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.249624:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.249625:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.249626:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.249627:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.249628:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.249629:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.249630:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.249654:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921878.249698:0:3864:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:1.0:1506921878.249705:0:3864:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:1.0:1506921878.249708:0:3864:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:1.0:1506921878.249713:0:3864:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302400/t0(0) o19->oak-OST000e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921878.249733:0:3864:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1506921878.249736:0:3864:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302400/t0(0) o19->oak-OST000e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921878.249741:0:3864:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921878.249743:0:3864:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:1.0:1506921878.249744:0:3864:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fac680 00000400:00000010:1.0:1506921878.249745:0:3864:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fac680. 00000800:00000200:13.0:1506921878.249749:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000400:00000010:1.0:1506921878.249749:0:3864:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451c200 (tot 59423468). 00000800:00000200:1.0:1506921878.249755:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:2.2:1506921878.249774:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:11.0:1506921878.249788:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921878.249796:0:3862:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.42@o2ib4 00000400:00000200:11.0:1506921878.249802:0:3862:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:11.0:1506921878.249805:0:3862:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e13400 (tot 59423868). 00000400:00000200:11.0:1506921878.249810:0:3862:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c2257c0 00000400:00000200:11.0:1506921878.249815:0:3862:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a10ad [1] + 192 00000100:00000001:11.0:1506921878.249820:0:3862:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921878.249822:0:3862:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302400/t0(0) o19->oak-OST000e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:5.0:1506921878.249828:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:11.0:1506921878.249837:0:3862:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302400/t0(0) o19->oak-OST000e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.249845:0:3862:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921878.249847:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e13400 (tot 59423468). 00000800:00000200:11.0:1506921878.249849:0:3862:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921878.249852:0:3862:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:11.0:1506921878.249853:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921878.249864:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.249868:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.249869:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.249870:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.249872:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302400/t0(0) o19->oak-OST000e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.249878:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302400/t0(0) o19->oak-OST000e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.249882:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.249883:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.249885:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.249886:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.249887:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.249888:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.249889:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.249891:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.249892:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.249893:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.249895:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff882036e46400 time=73 v=31 (31 1 1 31) 00000100:00001000:15.0:1506921878.249898:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff882036e463c8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.249899:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.249900:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.249901:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.249902:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.249903:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.249904:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST000e-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.249905:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.249906:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.249908:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302400/t0(0) o19->oak-OST000e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.249913:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.249914:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.249915:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.249917:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302400/t0(0) o19->oak-OST000e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.249922:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302400:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.249926:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.249928:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.249929:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.249930:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.249932:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302400/t0(0) o19->oak-OST000e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.249936:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.249937:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.249939:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.249940:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.249941:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.249942:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302400/t0(0) o19->oak-OST000e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.249946:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.249947:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.249948:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.249950:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.249951:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.249951:0:38882:0:(genops.c:1026:class_import_put()) import ffff882036e46000 refcount=4 obd=oak-OST000e-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.249953:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.249953:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.249955:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.249955:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.249956:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.249958:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.249959:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.249960:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.249961:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.249963:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.249964:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.249965:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.249966:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203b61d000 refcount=5 obd=oak-OST000f-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.249968:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.249969:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.249970:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.249971:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.249974:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.249975:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.249976:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.249977:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.249978:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.249980:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.249980:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.249982:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.249983:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.249990:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.249992:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.249993:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.249994:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.249995:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.249997:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302416/t0(0) o19->oak-OST000f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.250001:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.250002:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.250003:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.250004:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.250006:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302416:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.250008:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.250009:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.250010:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.250011:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.250012:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.250013:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.250014:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.250016:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.250018:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302416, portal 4 00000100:00000001:15.0:1506921878.250019:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.250020:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.250022:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302416/t0(0) o19->oak-OST000f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.250031:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.250033:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921878.250034:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032facb00. 00000100:00000200:15.0:1506921878.250036:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302416, offset 0 00000400:00000010:15.0:1506921878.250038:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b04519600 (tot 59423868). 00000400:00000200:15.0:1506921878.250041:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921878.250047:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921878.250050:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921878.250055:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.250056:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921878.250058:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921878.250061:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.250062:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.250063:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.250065:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.250066:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.250067:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.250068:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.250069:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.250070:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.250072:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.250072:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921878.250102:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.250120:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.250125:0:3859:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.250129:0:3859:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.250133:0:3859:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302416/t0(0) o19->oak-OST000f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.250141:0:3859:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921878.250143:0:3859:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302416/t0(0) o19->oak-OST000f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.250149:0:3859:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.250150:0:3859:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921878.250152:0:3859:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032facb00 00000400:00000010:16.0:1506921878.250153:0:3859:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032facb00. 00000800:00000200:18.0:1506921878.250157:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000010:16.0:1506921878.250157:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b04519600 (tot 59423468). 00000800:00000200:16.0:1506921878.250161:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921878.250271:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:13.0:1506921878.250314:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921878.250323:0:3863:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.42@o2ib4 00000400:00000200:13.0:1506921878.250329:0:3863:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:13.0:1506921878.250331:0:3863:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88203bd18600 (tot 59423868). 00000400:00000200:13.0:1506921878.250337:0:3863:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c2257d0 00000400:00000200:13.0:1506921878.250341:0:3863:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1d41 [1] + 192 00000100:00000001:13.0:1506921878.250346:0:3863:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:13.0:1506921878.250349:0:3863:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302416/t0(0) o19->oak-OST000f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:13.0:1506921878.250356:0:3863:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302416/t0(0) o19->oak-OST000f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000800:00000200:1.0:1506921878.250358:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000001:13.0:1506921878.250363:0:3863:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:13.0:1506921878.250365:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88203bd18600 (tot 59423468). 00000800:00000200:13.0:1506921878.250368:0:3863:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921878.250370:0:3863:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:13.0:1506921878.250371:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921878.250396:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.250400:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.250402:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.250403:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.250407:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302416/t0(0) o19->oak-OST000f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.250416:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302416/t0(0) o19->oak-OST000f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.250422:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.250423:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.250425:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.250426:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.250430:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.250432:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.250433:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.250434:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.250435:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.250437:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.250440:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88203b61d400 time=73 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921878.250443:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203b61d3c8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.250445:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.250446:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.250447:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.250448:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.250450:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.250451:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST000f-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.250452:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.250453:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.250455:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302416/t0(0) o19->oak-OST000f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.250460:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.250462:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.250463:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.250465:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302416/t0(0) o19->oak-OST000f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.250470:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302416:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.250473:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.250475:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.250476:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.250477:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.250479:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302416/t0(0) o19->oak-OST000f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.250484:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.250485:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.250486:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.250487:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.250489:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.250491:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302416/t0(0) o19->oak-OST000f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.250496:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.250497:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.250498:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.250500:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.250501:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.250502:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203b61d000 refcount=4 obd=oak-OST000f-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.250503:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.250504:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.250506:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.250506:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.250507:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.250509:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.250510:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.250511:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.250512:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.250513:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.250515:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.250516:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.250517:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203b61c800 refcount=5 obd=oak-OST0010-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.250519:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.250520:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.250521:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.250523:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.250525:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.250527:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.250528:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.250529:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.250530:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.250532:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.250533:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.250535:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.250537:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.250545:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.250547:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.250549:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.250551:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.250552:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.250554:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302432/t0(0) o19->oak-OST0010-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.250559:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.250560:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.250562:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.250562:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.250564:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302432:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.250567:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.250568:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.250569:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.250570:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.250571:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.250573:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.250574:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.250576:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.250579:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302432, portal 4 00000100:00000001:15.0:1506921878.250580:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.250581:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.250584:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302432/t0(0) o19->oak-OST0010-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.250588:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.250590:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921878.250591:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032facb80. 00000100:00000200:15.0:1506921878.250593:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302432, offset 0 00000400:00000010:15.0:1506921878.250595:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b04518c00 (tot 59423868). 00000400:00000200:15.0:1506921878.250598:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921878.250607:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921878.250611:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921878.250615:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.250617:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921878.250620:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921878.250624:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.250625:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.250626:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.250628:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.250629:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.250630:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.250632:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.250633:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.250634:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.250635:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.250636:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.250662:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921878.250704:0:3865:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:5.0:1506921878.250711:0:3865:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:5.0:1506921878.250714:0:3865:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:5.0:1506921878.250730:0:3865:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302432/t0(0) o19->oak-OST0010-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921878.250738:0:3865:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:5.0:1506921878.250740:0:3865:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302432/t0(0) o19->oak-OST0010-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:11.0:1506921878.250743:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000001:5.0:1506921878.250745:0:3865:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:5.0:1506921878.250747:0:3865:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:5.0:1506921878.250748:0:3865:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032facb80 00000400:00000010:5.0:1506921878.250752:0:3865:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032facb80. 00000400:00000010:5.0:1506921878.250755:0:3865:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b04518c00 (tot 59423468). 00000800:00000200:5.0:1506921878.250758:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.2:1506921878.250924:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921878.250941:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.250947:0:3860:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921878.250954:0:3860:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921878.250956:0:3860:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921878.250962:0:3860:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c2257e0 00000400:00000200:16.0:1506921878.250967:0:3860:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a10c5 [1] + 192 00000100:00000001:16.0:1506921878.250974:0:3860:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921878.250977:0:3860:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302432/t0(0) o19->oak-OST0010-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921878.250978:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921878.250985:0:3860:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302432/t0(0) o19->oak-OST0010-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.250993:0:3860:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921878.250995:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921878.250997:0:3860:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.250999:0:3860:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921878.251001:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:15.0:1506921878.251030:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.251034:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.251036:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.251037:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.251042:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302432/t0(0) o19->oak-OST0010-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.251050:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302432/t0(0) o19->oak-OST0010-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.251057:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.251059:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.251060:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.251062:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.251063:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.251065:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.251067:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.251068:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.251069:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.251071:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.251074:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88203b61cc00 time=73 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921878.251077:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203b61cbc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.251079:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.251080:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.251081:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.251083:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.251084:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.251087:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0010-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.251089:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.251090:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.251092:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302432/t0(0) o19->oak-OST0010-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.251098:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.251100:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.251101:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.251103:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302432/t0(0) o19->oak-OST0010-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.251108:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302432:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.251111:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.251113:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.251114:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.251115:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.251117:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302432/t0(0) o19->oak-OST0010-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.251121:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.251122:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.251124:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.251125:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.251126:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.251128:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302432/t0(0) o19->oak-OST0010-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.251132:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.251133:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.251134:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.251136:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.251136:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.251137:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203b61c800 refcount=4 obd=oak-OST0010-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.251139:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.251140:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.251142:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.251143:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.251144:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.251146:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.251147:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.251148:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.251150:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.251151:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.251152:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.251154:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.251155:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203b619000 refcount=5 obd=oak-OST0011-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.251157:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.251158:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.251159:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.251161:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.251164:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.251166:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.251166:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.251168:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.251169:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.251170:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.251171:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.251173:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.251174:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.251183:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.251186:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.251187:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.251189:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.251190:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.251191:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302448/t0(0) o19->oak-OST0011-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.251196:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.251197:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.251199:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.251200:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.251202:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302448:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.251204:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.251206:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.251207:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.251208:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.251209:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.251211:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.251213:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.251215:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.251217:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302448, portal 4 00000100:00000001:15.0:1506921878.251219:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.251219:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.251223:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302448/t0(0) o19->oak-OST0011-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.251227:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.251229:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921878.251230:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fac480. 00000100:00000200:15.0:1506921878.251232:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302448, offset 0 00000400:00000010:15.0:1506921878.251234:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b04519200 (tot 59423868). 00000400:00000200:15.0:1506921878.251237:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921878.251246:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921878.251250:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921878.251255:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.251257:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921878.251259:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921878.251263:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.251264:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.251266:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.251267:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.251268:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.251269:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.251271:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.251272:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.251273:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.251275:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.251276:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921878.251303:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.251321:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.251326:0:3861:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.251330:0:3861:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.251334:0:3861:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302448/t0(0) o19->oak-OST0011-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.251342:0:3861:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921878.251344:0:3861:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302448/t0(0) o19->oak-OST0011-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.251350:0:3861:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.251352:0:3861:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921878.251353:0:3861:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fac480 00000400:00000010:16.0:1506921878.251354:0:3861:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fac480. 00000800:00000200:18.0:1506921878.251358:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000010:16.0:1506921878.251358:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b04519200 (tot 59423468). 00000800:00000200:16.0:1506921878.251362:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:16.2:1506921878.251384:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921878.251391:0:3858:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.251396:0:3858:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000800:00000200:18.0:1506921878.251398:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000200:16.0:1506921878.251400:0:3858:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:16.0:1506921878.251403:0:3858:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921878.251407:0:3858:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c2257f0 00000400:00000200:16.0:1506921878.251412:0:3858:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1d49 [1] + 192 00000100:00000001:16.0:1506921878.251417:0:3858:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921878.251419:0:3858:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302448/t0(0) o19->oak-OST0011-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:16.0:1506921878.251436:0:3858:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302448/t0(0) o19->oak-OST0011-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.251444:0:3858:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921878.251446:0:3858:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921878.251448:0:3858:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.251450:0:3858:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.251450:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:16.0:1506921878.251451:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000400:00000001:15.0:1506921878.251457:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.251458:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.251459:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.251462:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302448/t0(0) o19->oak-OST0011-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.251468:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302448/t0(0) o19->oak-OST0011-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.251472:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.251473:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.251475:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.251476:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.251477:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.251478:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.251479:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.251480:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.251481:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.251482:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.251484:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88203b619400 time=73 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921878.251487:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203b6193c8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.251488:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.251489:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.251490:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.251491:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.251493:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.251493:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0011-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.251495:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.251495:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.251497:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302448/t0(0) o19->oak-OST0011-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.251501:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.251503:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.251504:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.251506:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302448/t0(0) o19->oak-OST0011-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.251511:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302448:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.251515:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.251517:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.251518:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.251519:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.251521:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302448/t0(0) o19->oak-OST0011-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.251525:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.251526:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.251528:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.251528:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.251530:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.251531:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302448/t0(0) o19->oak-OST0011-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.251535:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.251536:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.251537:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.251538:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.251539:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.251540:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203b619000 refcount=4 obd=oak-OST0011-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.251541:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.251542:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.251544:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.251544:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.251545:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.251547:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.251548:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.251549:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.251550:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.251551:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.251552:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.251553:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.251554:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203b61a000 refcount=5 obd=oak-OST0012-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.251556:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.251556:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.251557:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.251559:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.251561:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.251562:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.251563:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.251564:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.251565:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.251566:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.251567:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.251568:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.251569:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.251576:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.251578:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.251579:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.251581:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.251581:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.251583:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302464/t0(0) o19->oak-OST0012-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.251587:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.251588:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.251589:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.251590:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.251592:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302464:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.251594:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.251595:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.251596:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.251597:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.251597:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.251599:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.251600:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.251602:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.251603:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302464, portal 4 00000100:00000001:15.0:1506921878.251605:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.251605:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.251608:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302464/t0(0) o19->oak-OST0012-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.251616:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.251618:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921878.251619:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fade00. 00000100:00000200:15.0:1506921878.251621:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302464, offset 0 00000400:00000010:15.0:1506921878.251623:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451c600 (tot 59423868). 00000400:00000200:15.0:1506921878.251625:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921878.251632:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921878.251635:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921878.251639:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.251641:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921878.251642:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921878.251644:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.251645:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.251646:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.251648:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.251649:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.251649:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.251651:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.251652:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.251653:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.251654:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.251655:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.251677:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921878.251732:0:3864:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:1.0:1506921878.251739:0:3864:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:1.0:1506921878.251742:0:3864:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:1.0:1506921878.251747:0:3864:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302464/t0(0) o19->oak-OST0012-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921878.251754:0:3864:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1506921878.251756:0:3864:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302464/t0(0) o19->oak-OST0012-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921878.251762:0:3864:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921878.251763:0:3864:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:1.0:1506921878.251764:0:3864:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fade00 00000400:00000010:1.0:1506921878.251766:0:3864:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fade00. 00000400:00000010:1.0:1506921878.251769:0:3864:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451c600 (tot 59423468). 00000800:00000200:13.0:1506921878.251772:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:1.0:1506921878.251776:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:2.2:1506921878.251892:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:11.0:1506921878.251931:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921878.251940:0:3862:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.42@o2ib4 00000400:00000200:11.0:1506921878.251946:0:3862:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:11.0:1506921878.251948:0:3862:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e13400 (tot 59423868). 00000400:00000200:11.0:1506921878.251954:0:3862:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225800 00000400:00000200:11.0:1506921878.251958:0:3862:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a10dd [1] + 192 00000800:00000200:5.0:1506921878.251959:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000001:11.0:1506921878.251963:0:3862:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921878.251966:0:3862:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302464/t0(0) o19->oak-OST0012-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:11.0:1506921878.251974:0:3862:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302464/t0(0) o19->oak-OST0012-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.251981:0:3862:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921878.251983:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e13400 (tot 59423468). 00000800:00000200:11.0:1506921878.251985:0:3862:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921878.251987:0:3862:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:11.0:1506921878.251989:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921878.252016:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.252019:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.252021:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.252022:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.252026:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302464/t0(0) o19->oak-OST0012-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.252035:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302464/t0(0) o19->oak-OST0012-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.252041:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.252042:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.252044:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.252045:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.252048:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.252050:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.252051:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.252052:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.252053:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.252055:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.252058:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88203b61a400 time=73 v=31 (31 1 1 31) 00000100:00001000:15.0:1506921878.252061:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203b61a3c8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.252062:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.252063:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.252065:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.252066:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.252068:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.252069:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0012-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.252070:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.252071:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.252073:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302464/t0(0) o19->oak-OST0012-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.252078:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.252080:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.252081:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.252083:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302464/t0(0) o19->oak-OST0012-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.252088:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302464:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.252091:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.252092:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.252093:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.252094:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.252096:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302464/t0(0) o19->oak-OST0012-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.252100:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.252102:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.252103:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.252104:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.252106:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.252108:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302464/t0(0) o19->oak-OST0012-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.252112:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.252113:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.252114:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.252116:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.252117:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.252118:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203b61a000 refcount=4 obd=oak-OST0012-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.252119:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.252120:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.252122:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.252122:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.252123:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.252125:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.252125:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.252126:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.252128:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.252129:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.252130:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.252132:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.252133:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203c7d0000 refcount=5 obd=oak-OST0013-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.252135:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.252136:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.252137:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.252139:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.252141:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.252143:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.252143:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.252145:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.252146:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.252148:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.252148:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.252150:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.252152:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.252160:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.252163:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.252164:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.252166:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.252166:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.252168:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302480/t0(0) o19->oak-OST0013-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.252173:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.252174:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.252176:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.252176:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.252178:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302480:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.252181:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.252182:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.252183:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.252184:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.252185:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.252186:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.252188:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.252190:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.252192:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302480, portal 4 00000100:00000001:15.0:1506921878.252194:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.252194:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.252197:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302480/t0(0) o19->oak-OST0013-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.252201:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.252203:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921878.252205:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad700. 00000100:00000200:15.0:1506921878.252206:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302480, offset 0 00000400:00000010:15.0:1506921878.252208:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b04519400 (tot 59423868). 00000400:00000200:15.0:1506921878.252211:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921878.252220:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921878.252225:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921878.252230:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.252232:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921878.252235:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921878.252239:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.252240:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.252241:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.252242:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.252244:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.252244:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.252246:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.252247:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.252248:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.252250:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.252250:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921878.252280:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.252298:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.252303:0:3859:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.252306:0:3859:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.252311:0:3859:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302480/t0(0) o19->oak-OST0013-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.252318:0:3859:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921878.252321:0:3859:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302480/t0(0) o19->oak-OST0013-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.252326:0:3859:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.252328:0:3859:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921878.252329:0:3859:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad700 00000400:00000010:16.0:1506921878.252331:0:3859:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad700. 00000800:00000200:18.0:1506921878.252335:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000010:16.0:1506921878.252335:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b04519400 (tot 59423468). 00000800:00000200:16.0:1506921878.252339:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921878.252372:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:13.0:1506921878.252416:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921878.252424:0:3863:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.42@o2ib4 00000400:00000200:13.0:1506921878.252430:0:3863:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:13.0:1506921878.252432:0:3863:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88203bd18600 (tot 59423868). 00000400:00000200:13.0:1506921878.252438:0:3863:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225810 00000400:00000200:13.0:1506921878.252443:0:3863:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1d51 [1] + 192 00000100:00000001:13.0:1506921878.252447:0:3863:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:13.0:1506921878.252450:0:3863:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302480/t0(0) o19->oak-OST0013-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:1.0:1506921878.252455:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:13.0:1506921878.252458:0:3863:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302480/t0(0) o19->oak-OST0013-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921878.252466:0:3863:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:13.0:1506921878.252468:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88203bd18600 (tot 59423468). 00000800:00000200:13.0:1506921878.252471:0:3863:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000100:00000001:15.0:1506921878.252473:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.252475:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000800:00000200:13.0:1506921878.252475:0:3863:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000400:00000001:15.0:1506921878.252476:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.252476:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000800:00000200:13.0:1506921878.252477:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000200:15.0:1506921878.252478:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302480/t0(0) o19->oak-OST0013-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.252484:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302480/t0(0) o19->oak-OST0013-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.252488:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.252489:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.252490:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.252492:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.252493:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.252494:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.252495:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.252496:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.252497:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.252499:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.252500:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88203c7d0400 time=73 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921878.252503:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203c7d03c8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.252505:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.252505:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.252507:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.252508:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.252509:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.252510:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0013-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.252512:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.252512:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.252514:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302480/t0(0) o19->oak-OST0013-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.252518:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.252520:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.252521:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.252523:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302480/t0(0) o19->oak-OST0013-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.252527:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302480:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.252530:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.252531:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.252532:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.252533:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.252535:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302480/t0(0) o19->oak-OST0013-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.252539:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.252541:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.252542:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.252543:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.252544:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.252545:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302480/t0(0) o19->oak-OST0013-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.252549:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.252550:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.252551:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.252552:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.252553:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.252554:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203c7d0000 refcount=4 obd=oak-OST0013-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.252555:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.252556:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.252558:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.252558:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.252560:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.252561:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.252562:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.252563:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.252564:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.252566:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.252566:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.252567:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.252569:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203c7d7000 refcount=5 obd=oak-OST0014-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.252570:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.252571:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.252572:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.252573:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.252575:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.252576:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.252577:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.252578:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.252579:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.252580:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.252581:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.252583:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.252584:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.252590:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.252592:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.252594:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.252595:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.252596:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.252597:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302496/t0(0) o19->oak-OST0014-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.252602:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.252602:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.252604:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.252604:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.252606:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302496:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.252608:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.252609:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.252610:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.252611:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.252612:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.252613:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.252615:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.252616:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.252618:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302496, portal 4 00000100:00000001:15.0:1506921878.252619:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.252620:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.252623:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302496/t0(0) o19->oak-OST0014-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.252627:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.252628:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921878.252629:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fac080. 00000100:00000200:15.0:1506921878.252631:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302496, offset 0 00000400:00000010:15.0:1506921878.252632:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451fc00 (tot 59423868). 00000400:00000200:15.0:1506921878.252635:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921878.252641:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921878.252645:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921878.252648:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.252650:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921878.252652:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921878.252654:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.252655:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.252656:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.252657:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.252658:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.252659:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.252660:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.252662:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.252663:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.252664:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.252664:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.252691:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921878.252736:0:3865:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:5.0:1506921878.252743:0:3865:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:5.0:1506921878.252746:0:3865:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:5.0:1506921878.252750:0:3865:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302496/t0(0) o19->oak-OST0014-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921878.252758:0:3865:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:5.0:1506921878.252760:0:3865:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302496/t0(0) o19->oak-OST0014-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921878.252766:0:3865:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:5.0:1506921878.252767:0:3865:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:5.0:1506921878.252769:0:3865:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fac080 00000400:00000010:5.0:1506921878.252770:0:3865:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fac080. 00000400:00000010:5.0:1506921878.252773:0:3865:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451fc00 (tot 59423468). 00000800:00000200:11.0:1506921878.252776:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:5.0:1506921878.252780:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.2:1506921878.252941:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921878.252958:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.252965:0:3860:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921878.252971:0:3860:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921878.252974:0:3860:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921878.252979:0:3860:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225820 00000400:00000200:16.0:1506921878.252984:0:3860:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a10f5 [1] + 192 00000100:00000001:16.0:1506921878.252989:0:3860:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921878.252991:0:3860:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302496/t0(0) o19->oak-OST0014-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921878.252995:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921878.252999:0:3860:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302496/t0(0) o19->oak-OST0014-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.253015:0:3860:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921878.253017:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921878.253019:0:3860:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.253021:0:3860:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921878.253023:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:15.0:1506921878.253063:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.253069:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.253070:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.253071:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.253076:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302496/t0(0) o19->oak-OST0014-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.253084:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302496/t0(0) o19->oak-OST0014-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.253090:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.253092:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.253094:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.253095:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.253096:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.253098:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.253099:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.253100:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.253102:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.253103:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.253106:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88203c7d7400 time=73 v=31 (31 3 1 31) 00000100:00001000:15.0:1506921878.253109:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203c7d73c8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.253111:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.253112:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.253114:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.253115:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.253117:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.253118:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0014-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.253119:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.253119:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.253122:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302496/t0(0) o19->oak-OST0014-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.253127:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.253129:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.253130:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.253132:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302496/t0(0) o19->oak-OST0014-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.253137:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302496:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.253142:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.253144:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.253146:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.253147:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.253148:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302496/t0(0) o19->oak-OST0014-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.253153:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.253154:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.253156:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.253157:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.253159:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.253160:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302496/t0(0) o19->oak-OST0014-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.253164:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.253165:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.253166:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.253168:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.253169:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.253170:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203c7d7000 refcount=4 obd=oak-OST0014-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.253171:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.253172:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.253174:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.253174:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.253175:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.253177:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.253177:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.253179:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.253180:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.253181:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.253183:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.253184:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.253185:0:38882:0:(genops.c:1012:class_import_get()) import ffff88207fdf9000 refcount=5 obd=oak-OST0015-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.253187:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.253188:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.253189:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.253191:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.253194:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.253195:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.253196:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.253197:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.253199:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.253200:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.253201:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.253203:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.253204:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.253213:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.253215:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.253216:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.253218:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.253218:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.253220:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302512/t0(0) o19->oak-OST0015-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.253225:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.253226:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.253227:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.253228:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.253230:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302512:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.253232:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.253234:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.253235:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.253236:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.253237:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.253238:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.253240:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.253243:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.253245:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302512, portal 4 00000100:00000001:15.0:1506921878.253247:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.253247:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.253250:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302512/t0(0) o19->oak-OST0015-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.253259:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.253261:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921878.253262:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032facc80. 00000100:00000200:15.0:1506921878.253264:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302512, offset 0 00000400:00000010:15.0:1506921878.253266:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451da00 (tot 59423868). 00000400:00000200:15.0:1506921878.253269:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921878.253278:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921878.253283:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921878.253288:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.253290:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921878.253292:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921878.253296:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.253297:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.253298:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.253299:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.253301:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.253301:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.253303:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.253304:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.253305:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.253307:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.253307:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921878.253331:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.253348:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.253353:0:3861:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.253357:0:3861:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.253361:0:3861:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302512/t0(0) o19->oak-OST0015-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.253369:0:3861:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921878.253371:0:3861:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302512/t0(0) o19->oak-OST0015-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.253376:0:3861:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.253378:0:3861:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921878.253379:0:3861:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032facc80 00000400:00000010:16.0:1506921878.253381:0:3861:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032facc80. 00000400:00000010:16.0:1506921878.253384:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451da00 (tot 59423468). 00000800:00000200:18.0:1506921878.253385:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921878.253389:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:16.2:1506921878.253395:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921878.253401:0:3858:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.253405:0:3858:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000800:00000200:18.0:1506921878.253408:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000200:16.0:1506921878.253410:0:3858:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:16.0:1506921878.253412:0:3858:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921878.253417:0:3858:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225830 00000400:00000200:16.0:1506921878.253422:0:3858:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1d59 [1] + 192 00000100:00000001:16.0:1506921878.253426:0:3858:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921878.253428:0:3858:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302512/t0(0) o19->oak-OST0015-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:16.0:1506921878.253434:0:3858:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302512/t0(0) o19->oak-OST0015-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.253441:0:3858:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921878.253442:0:3858:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921878.253446:0:3858:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:15.0:1506921878.253447:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:16.0:1506921878.253448:0:3858:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000400:00000001:15.0:1506921878.253449:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000800:00000200:16.0:1506921878.253450:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000400:00000001:15.0:1506921878.253450:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.253451:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.253453:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302512/t0(0) o19->oak-OST0015-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.253459:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302512/t0(0) o19->oak-OST0015-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.253463:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.253465:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.253466:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.253468:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.253469:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.253470:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.253471:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.253472:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.253473:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.253475:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.253477:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88207fdf9400 time=73 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921878.253479:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88207fdf93c8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.253481:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.253481:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.253483:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.253484:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.253485:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.253486:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0015-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.253487:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.253488:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.253490:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302512/t0(0) o19->oak-OST0015-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.253494:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.253495:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.253496:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.253498:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302512/t0(0) o19->oak-OST0015-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.253503:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302512:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.253505:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.253507:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.253508:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.253509:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.253511:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302512/t0(0) o19->oak-OST0015-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.253515:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.253516:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.253518:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.253518:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.253520:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.253522:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302512/t0(0) o19->oak-OST0015-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.253526:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.253526:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.253527:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.253529:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.253529:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.253530:0:38882:0:(genops.c:1026:class_import_put()) import ffff88207fdf9000 refcount=4 obd=oak-OST0015-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.253532:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.253533:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.253534:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.253535:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.253535:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.253537:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.253538:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.253539:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.253540:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.253541:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.253542:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.253543:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.253544:0:38882:0:(genops.c:1012:class_import_get()) import ffff88207fdf8800 refcount=5 obd=oak-OST0016-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.253546:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.253547:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.253547:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.253549:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.253550:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.253552:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.253553:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.253554:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.253555:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.253556:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.253557:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.253558:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.253559:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.253566:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.253568:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.253569:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.253571:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.253572:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.253573:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302528/t0(0) o19->oak-OST0016-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.253578:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.253578:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.253580:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.253580:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.253582:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302528:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.253584:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.253585:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.253586:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.253587:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.253588:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.253589:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.253590:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.253592:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.253593:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302528, portal 4 00000100:00000001:15.0:1506921878.253595:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.253595:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.253598:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302528/t0(0) o19->oak-OST0016-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.253602:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.253604:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921878.253605:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032facc00. 00000100:00000200:15.0:1506921878.253606:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302528, offset 0 00000400:00000010:15.0:1506921878.253608:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451e800 (tot 59423868). 00000400:00000200:15.0:1506921878.253610:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921878.253617:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921878.253620:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921878.253623:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.253625:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921878.253627:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921878.253630:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.253631:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.253632:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.253633:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.253634:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.253635:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.253636:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.253637:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.253638:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.253639:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.253640:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.253668:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921878.253713:0:3864:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:1.0:1506921878.253730:0:3864:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:1.0:1506921878.253734:0:3864:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:1.0:1506921878.253738:0:3864:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302528/t0(0) o19->oak-OST0016-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921878.253746:0:3864:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1506921878.253748:0:3864:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302528/t0(0) o19->oak-OST0016-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:13.0:1506921878.253752:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000001:1.0:1506921878.253753:0:3864:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921878.253755:0:3864:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:1.0:1506921878.253756:0:3864:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032facc00 00000400:00000010:1.0:1506921878.253757:0:3864:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032facc00. 00000400:00000010:1.0:1506921878.253761:0:3864:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451e800 (tot 59423468). 00000800:00000200:1.0:1506921878.253763:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:2.2:1506921878.253823:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:11.0:1506921878.253863:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921878.253871:0:3862:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.42@o2ib4 00000400:00000200:11.0:1506921878.253877:0:3862:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:11.0:1506921878.253880:0:3862:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e13400 (tot 59423868). 00000400:00000200:11.0:1506921878.253885:0:3862:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225840 00000400:00000200:11.0:1506921878.253890:0:3862:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a110d [1] + 192 00000100:00000001:11.0:1506921878.253897:0:3862:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921878.253899:0:3862:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302528/t0(0) o19->oak-OST0016-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:11.0:1506921878.253907:0:3862:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302528/t0(0) o19->oak-OST0016-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000800:00000200:5.0:1506921878.253909:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000001:11.0:1506921878.253914:0:3862:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921878.253916:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e13400 (tot 59423468). 00000800:00000200:11.0:1506921878.253919:0:3862:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921878.253921:0:3862:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:11.0:1506921878.253922:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921878.253947:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.253952:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.253953:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.253954:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.253958:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302528/t0(0) o19->oak-OST0016-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.253967:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302528/t0(0) o19->oak-OST0016-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.253973:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.253974:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.253975:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.253977:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.253978:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.253980:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.253981:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.253983:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.253984:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.253985:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.253988:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88207fdf8c00 time=73 v=31 (31 3 1 31) 00000100:00001000:15.0:1506921878.253991:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88207fdf8bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.253993:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.253994:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.253995:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.253997:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.253998:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.254001:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0016-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.254002:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.254003:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.254006:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302528/t0(0) o19->oak-OST0016-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.254011:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.254013:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.254014:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.254016:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302528/t0(0) o19->oak-OST0016-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.254021:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302528:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.254024:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.254025:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.254026:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.254027:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.254029:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302528/t0(0) o19->oak-OST0016-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.254033:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.254034:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.254036:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.254037:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.254039:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.254040:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302528/t0(0) o19->oak-OST0016-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.254044:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.254045:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.254046:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.254048:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.254049:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.254049:0:38882:0:(genops.c:1026:class_import_put()) import ffff88207fdf8800 refcount=4 obd=oak-OST0016-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.254051:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.254052:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.254053:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.254054:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.254055:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.254057:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.254058:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.254059:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.254060:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.254062:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.254063:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.254064:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.254065:0:38882:0:(genops.c:1012:class_import_get()) import ffff88103aad6000 refcount=5 obd=oak-OST0017-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.254067:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.254068:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.254069:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.254071:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.254073:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.254075:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.254075:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.254077:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.254078:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.254080:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.254080:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.254082:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.254084:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.254092:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.254095:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.254096:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.254097:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.254098:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.254100:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302544/t0(0) o19->oak-OST0017-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.254105:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.254105:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.254107:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.254108:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.254110:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302544:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.254112:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.254114:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.254115:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.254116:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.254117:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.254118:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.254120:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.254122:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.254124:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302544, portal 4 00000100:00000001:15.0:1506921878.254126:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.254126:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.254129:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302544/t0(0) o19->oak-OST0017-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.254134:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.254136:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921878.254137:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad480. 00000100:00000200:15.0:1506921878.254138:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302544, offset 0 00000400:00000010:15.0:1506921878.254141:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b04518400 (tot 59423868). 00000400:00000200:15.0:1506921878.254143:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921878.254152:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921878.254157:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921878.254162:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.254165:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921878.254166:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921878.254170:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.254171:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.254172:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.254174:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.254175:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.254176:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.254177:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.254179:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.254180:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.254181:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.254181:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921878.254206:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.254224:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.254229:0:3859:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.254232:0:3859:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.254237:0:3859:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302544/t0(0) o19->oak-OST0017-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.254244:0:3859:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921878.254246:0:3859:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302544/t0(0) o19->oak-OST0017-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.254252:0:3859:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.254253:0:3859:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921878.254255:0:3859:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad480 00000400:00000010:16.0:1506921878.254256:0:3859:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad480. 00000400:00000010:16.0:1506921878.254260:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b04518400 (tot 59423468). 00000800:00000200:18.0:1506921878.254262:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921878.254266:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921878.254326:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:13.0:1506921878.254369:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921878.254378:0:3863:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.42@o2ib4 00000400:00000200:13.0:1506921878.254383:0:3863:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:13.0:1506921878.254386:0:3863:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88203bd18600 (tot 59423868). 00000400:00000200:13.0:1506921878.254391:0:3863:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225850 00000400:00000200:13.0:1506921878.254396:0:3863:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1d61 [1] + 192 00000100:00000001:13.0:1506921878.254401:0:3863:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:13.0:1506921878.254404:0:3863:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302544/t0(0) o19->oak-OST0017-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:1.0:1506921878.254410:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:13.0:1506921878.254411:0:3863:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302544/t0(0) o19->oak-OST0017-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921878.254418:0:3863:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:13.0:1506921878.254420:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88203bd18600 (tot 59423468). 00000800:00000200:13.0:1506921878.254423:0:3863:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921878.254425:0:3863:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:13.0:1506921878.254427:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921878.254458:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.254463:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.254465:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.254466:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.254470:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302544/t0(0) o19->oak-OST0017-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.254479:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302544/t0(0) o19->oak-OST0017-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.254484:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.254486:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.254487:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.254489:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.254490:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.254492:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.254493:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.254494:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.254495:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.254497:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.254500:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88103aad6400 time=73 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921878.254503:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88103aad63c8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.254504:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.254505:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.254507:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.254508:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.254509:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.254510:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0017-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.254512:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.254512:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.254515:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302544/t0(0) o19->oak-OST0017-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.254520:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.254522:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.254523:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.254525:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302544/t0(0) o19->oak-OST0017-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.254530:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302544:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.254535:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.254537:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.254538:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.254539:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.254541:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302544/t0(0) o19->oak-OST0017-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.254546:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.254547:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.254549:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.254549:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.254551:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.254553:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302544/t0(0) o19->oak-OST0017-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.254557:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.254557:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.254559:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.254560:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.254561:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.254562:0:38882:0:(genops.c:1026:class_import_put()) import ffff88103aad6000 refcount=4 obd=oak-OST0017-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.254564:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.254565:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.254566:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.254567:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.254568:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.254569:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.254570:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.254571:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.254573:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.254574:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.254575:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.254576:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.254578:0:38882:0:(genops.c:1012:class_import_get()) import ffff88103aad4000 refcount=5 obd=oak-OST0018-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.254580:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.254581:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.254582:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.254584:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.254586:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.254588:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.254589:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.254590:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.254591:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.254593:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.254593:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.254595:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.254597:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.254605:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.254607:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.254609:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.254610:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.254611:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.254613:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302560/t0(0) o19->oak-OST0018-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.254617:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.254618:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.254620:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.254621:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.254623:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302560:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.254625:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.254626:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.254627:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.254628:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.254629:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.254631:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.254633:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.254635:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.254637:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302560, portal 4 00000100:00000001:15.0:1506921878.254639:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.254639:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.254642:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302560/t0(0) o19->oak-OST0018-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.254651:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.254653:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921878.254654:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fac580. 00000100:00000200:15.0:1506921878.254655:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302560, offset 0 00000400:00000010:15.0:1506921878.254657:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451b600 (tot 59423868). 00000400:00000200:15.0:1506921878.254660:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921878.254669:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921878.254673:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921878.254677:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.254679:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921878.254681:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921878.254684:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.254685:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.254687:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.254688:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.254689:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.254690:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.254691:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.254693:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.254694:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.254695:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.254696:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.254723:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921878.254769:0:3865:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:5.0:1506921878.254776:0:3865:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:5.0:1506921878.254779:0:3865:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:5.0:1506921878.254782:0:3865:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302560/t0(0) o19->oak-OST0018-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921878.254790:0:3865:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:5.0:1506921878.254792:0:3865:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302560/t0(0) o19->oak-OST0018-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921878.254798:0:3865:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:5.0:1506921878.254799:0:3865:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:5.0:1506921878.254801:0:3865:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fac580 00000400:00000010:5.0:1506921878.254802:0:3865:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fac580. 00000400:00000010:5.0:1506921878.254805:0:3865:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451b600 (tot 59423468). 00000800:00000200:5.0:1506921878.254808:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:11.0:1506921878.254810:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.2:1506921878.254906:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921878.254926:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.254932:0:3860:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921878.254939:0:3860:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921878.254941:0:3860:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921878.254947:0:3860:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225860 00000400:00000200:16.0:1506921878.254952:0:3860:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a1125 [1] + 192 00000100:00000001:16.0:1506921878.254957:0:3860:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921878.254960:0:3860:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302560/t0(0) o19->oak-OST0018-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921878.254964:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921878.254967:0:3860:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302560/t0(0) o19->oak-OST0018-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.254976:0:3860:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921878.254977:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921878.254980:0:3860:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.254982:0:3860:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921878.254983:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:15.0:1506921878.255012:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.255016:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.255017:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.255018:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.255023:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302560/t0(0) o19->oak-OST0018-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.255032:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302560/t0(0) o19->oak-OST0018-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.255038:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.255039:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.255041:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.255042:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.255046:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.255048:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.255049:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.255050:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.255051:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.255053:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.255056:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88103aad4400 time=73 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921878.255059:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88103aad43c8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.255060:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.255061:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.255062:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.255064:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.255065:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.255066:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0018-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.255067:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.255068:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.255070:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302560/t0(0) o19->oak-OST0018-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.255076:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.255078:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.255079:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.255081:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302560/t0(0) o19->oak-OST0018-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.255086:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302560:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.255089:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.255091:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.255092:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.255093:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.255095:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302560/t0(0) o19->oak-OST0018-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.255099:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.255101:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.255102:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.255103:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.255105:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.255107:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302560/t0(0) o19->oak-OST0018-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.255111:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.255112:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.255113:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.255115:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.255116:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.255117:0:38882:0:(genops.c:1026:class_import_put()) import ffff88103aad4000 refcount=4 obd=oak-OST0018-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.255118:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.255119:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.255120:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.255121:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.255122:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.255124:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.255124:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.255126:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.255127:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.255128:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.255129:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.255130:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.255132:0:38882:0:(genops.c:1012:class_import_get()) import ffff88103aad2000 refcount=5 obd=oak-OST0019-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.255134:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.255135:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.255136:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.255138:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.255140:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.255142:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.255142:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.255144:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.255145:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.255147:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.255147:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.255149:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.255150:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.255159:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.255162:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.255163:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.255165:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.255166:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.255167:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302576/t0(0) o19->oak-OST0019-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.255172:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.255173:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.255175:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.255176:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.255178:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302576:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.255180:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.255181:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.255182:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.255183:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.255184:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.255185:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.255187:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.255189:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.255191:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302576, portal 4 00000100:00000001:15.0:1506921878.255193:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.255194:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.255197:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302576/t0(0) o19->oak-OST0019-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.255201:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.255203:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921878.255204:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fac000. 00000100:00000200:15.0:1506921878.255206:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302576, offset 0 00000400:00000010:15.0:1506921878.255208:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451b800 (tot 59423868). 00000400:00000200:15.0:1506921878.255211:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921878.255220:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921878.255224:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921878.255229:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.255231:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921878.255234:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921878.255238:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.255239:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.255240:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.255241:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.255243:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.255244:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.255245:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.255246:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.255247:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.255249:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.255249:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921878.255275:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.255292:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.255297:0:3861:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.255300:0:3861:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.255304:0:3861:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302576/t0(0) o19->oak-OST0019-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.255312:0:3861:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921878.255315:0:3861:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302576/t0(0) o19->oak-OST0019-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.255320:0:3861:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.255322:0:3861:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921878.255323:0:3861:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fac000 00000400:00000010:16.0:1506921878.255324:0:3861:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fac000. 00000400:00000010:16.0:1506921878.255328:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451b800 (tot 59423468). 00000800:00000200:18.0:1506921878.255329:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.255337:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (23)-- 00000800:00000200:18.0:1506921878.255339:0:3859:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000800:00000200:16.0:1506921878.255339:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000200:18.0:1506921878.255345:0:3859:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:18.0:1506921878.255348:0:3859:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103a767000 (tot 59423868). 00000400:00000200:18.0:1506921878.255354:0:3859:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225870 00000400:00000200:18.0:1506921878.255359:0:3859:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1d69 [1] + 192 00000100:00000001:18.0:1506921878.255364:0:3859:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:18.0:1506921878.255367:0:3859:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302576/t0(0) o19->oak-OST0019-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:18.0:1506921878.255374:0:3859:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302576/t0(0) o19->oak-OST0019-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:18.0:1506921878.255383:0:3859:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:18.0:1506921878.255385:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103a767000 (tot 59423468). 00000800:00000200:18.0:1506921878.255388:0:3859:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:18.0:1506921878.255390:0:3859:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.255390:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.255391:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.255392:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.255393:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000800:00000200:18.0:1506921878.255394:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000200:15.0:1506921878.255396:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302576/t0(0) o19->oak-OST0019-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.255401:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302576/t0(0) o19->oak-OST0019-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.255406:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.255407:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.255408:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.255409:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.255410:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.255412:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.255412:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.255413:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.255415:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.255416:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.255418:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88103aad2400 time=73 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921878.255420:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88103aad23c8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.255422:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.255422:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.255424:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.255425:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.255426:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.255427:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0019-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.255429:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.255429:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.255431:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302576/t0(0) o19->oak-OST0019-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.255435:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.255437:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.255438:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.255440:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302576/t0(0) o19->oak-OST0019-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.255445:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302576:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.255447:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.255449:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.255450:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.255451:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.255453:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302576/t0(0) o19->oak-OST0019-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.255457:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.255458:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.255460:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.255460:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.255462:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.255463:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302576/t0(0) o19->oak-OST0019-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.255467:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.255468:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.255469:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.255470:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.255471:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.255472:0:38882:0:(genops.c:1026:class_import_put()) import ffff88103aad2000 refcount=4 obd=oak-OST0019-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.255473:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.255474:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.255476:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.255476:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.255483:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.255485:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.255486:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.255487:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.255488:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.255489:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.255490:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.255491:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.255492:0:38882:0:(genops.c:1012:class_import_get()) import ffff8820362b0800 refcount=5 obd=oak-OST001a-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.255494:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.255494:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.255495:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.255497:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.255498:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.255500:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.255500:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.255501:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.255502:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.255504:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.255504:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.255506:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.255507:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.255514:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.255516:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.255517:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.255518:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.255519:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.255520:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302592/t0(0) o19->oak-OST001a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.255525:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.255525:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.255527:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.255528:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.255529:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302592:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.255531:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.255532:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.255533:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.255534:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.255535:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.255537:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.255538:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.255540:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.255542:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302592, portal 4 00000100:00000001:15.0:1506921878.255543:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.255543:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.255546:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302592/t0(0) o19->oak-OST001a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.255550:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.255551:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921878.255553:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fac280. 00000100:00000200:15.0:1506921878.255554:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302592, offset 0 00000400:00000010:15.0:1506921878.255556:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b04519a00 (tot 59423868). 00000400:00000200:15.0:1506921878.255558:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921878.255565:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921878.255568:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921878.255571:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.255573:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921878.255574:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921878.255577:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.255578:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.255579:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.255580:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.255581:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.255582:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.255583:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.255584:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.255585:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.255586:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.255587:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.255610:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921878.255653:0:3864:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:1.0:1506921878.255660:0:3864:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:1.0:1506921878.255664:0:3864:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:1.0:1506921878.255668:0:3864:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302592/t0(0) o19->oak-OST001a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921878.255677:0:3864:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1506921878.255680:0:3864:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302592/t0(0) o19->oak-OST001a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921878.255685:0:3864:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921878.255687:0:3864:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:1.0:1506921878.255688:0:3864:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fac280 00000400:00000010:1.0:1506921878.255689:0:3864:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fac280. 00000800:00000200:13.0:1506921878.255693:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000400:00000010:1.0:1506921878.255693:0:3864:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b04519a00 (tot 59423468). 00000800:00000200:1.0:1506921878.255697:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:2.2:1506921878.255791:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:5.0:1506921878.255831:0:3865:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921878.255840:0:3865:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.42@o2ib4 00000400:00000200:5.0:1506921878.255848:0:3865:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:5.0:1506921878.255850:0:3865:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881e48bc2800 (tot 59423868). 00000400:00000200:5.0:1506921878.255856:0:3865:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225880 00000400:00000200:5.0:1506921878.255860:0:3865:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a113d [1] + 192 00000100:00000001:5.0:1506921878.255865:0:3865:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:5.0:1506921878.255868:0:3865:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302592/t0(0) o19->oak-OST001a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:11.0:1506921878.255871:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:5.0:1506921878.255876:0:3865:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302592/t0(0) o19->oak-OST001a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921878.255883:0:3865:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:5.0:1506921878.255884:0:3865:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881e48bc2800 (tot 59423468). 00000800:00000200:5.0:1506921878.255887:0:3865:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921878.255889:0:3865:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:5.0:1506921878.255890:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921878.255916:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.255922:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.255923:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.255924:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.255928:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302592/t0(0) o19->oak-OST001a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.255937:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302592/t0(0) o19->oak-OST001a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.255943:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.255944:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.255946:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.255947:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.255949:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.255950:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.255952:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.255953:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.255954:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.255955:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.255958:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff8820362b0c00 time=73 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921878.255961:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff8820362b0bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.255963:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.255964:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.255965:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.255967:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.255968:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.255969:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST001a-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.255970:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.255971:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.255973:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302592/t0(0) o19->oak-OST001a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.255979:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.255980:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.255981:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.255983:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302592/t0(0) o19->oak-OST001a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.255989:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302592:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.255993:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.255995:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.255996:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.255997:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.255999:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302592/t0(0) o19->oak-OST001a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.256004:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.256005:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.256007:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.256007:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.256009:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.256011:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302592/t0(0) o19->oak-OST001a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.256015:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.256016:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.256017:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.256018:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.256019:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.256020:0:38882:0:(genops.c:1026:class_import_put()) import ffff8820362b0800 refcount=4 obd=oak-OST001a-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.256022:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.256023:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.256024:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.256025:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.256025:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.256027:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.256028:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.256029:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.256030:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.256032:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.256033:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.256034:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.256035:0:38882:0:(genops.c:1012:class_import_get()) import ffff8820362b2800 refcount=5 obd=oak-OST001b-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.256037:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.256038:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.256039:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.256041:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.256044:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.256045:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.256046:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.256047:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.256049:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.256050:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.256051:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.256053:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.256054:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.256063:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.256065:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.256067:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.256068:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.256069:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.256071:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302608/t0(0) o19->oak-OST001b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.256075:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.256076:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.256078:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.256079:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.256081:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302608:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.256083:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.256084:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.256085:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.256086:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.256087:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.256089:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.256091:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.256093:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.256095:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302608, portal 4 00000100:00000001:15.0:1506921878.256096:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.256097:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.256100:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302608/t0(0) o19->oak-OST001b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.256109:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.256111:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921878.256112:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fac100. 00000100:00000200:15.0:1506921878.256113:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302608, offset 0 00000400:00000010:15.0:1506921878.256115:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451f600 (tot 59423868). 00000400:00000200:15.0:1506921878.256118:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921878.256127:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921878.256132:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921878.256137:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.256139:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921878.256141:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921878.256145:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.256146:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.256147:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.256148:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.256150:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.256151:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.256152:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.256153:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.256154:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.256156:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.256156:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921878.256181:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.256198:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.256203:0:3860:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.256207:0:3860:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.256211:0:3860:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302608/t0(0) o19->oak-OST001b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.256219:0:3860:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921878.256221:0:3860:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302608/t0(0) o19->oak-OST001b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.256227:0:3860:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.256228:0:3860:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921878.256229:0:3860:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fac100 00000400:00000010:16.0:1506921878.256231:0:3860:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fac100. 00000400:00000010:16.0:1506921878.256235:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451f600 (tot 59423468). 00000800:00000200:16.0:1506921878.256237:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921878.256239:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921878.256246:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:13.0:1506921878.256289:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921878.256297:0:3863:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.42@o2ib4 00000400:00000200:13.0:1506921878.256303:0:3863:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:13.0:1506921878.256306:0:3863:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88203bd18600 (tot 59423868). 00000400:00000200:13.0:1506921878.256311:0:3863:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225890 00000400:00000200:13.0:1506921878.256315:0:3863:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1d71 [1] + 192 00000100:00000001:13.0:1506921878.256320:0:3863:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:13.0:1506921878.256323:0:3863:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302608/t0(0) o19->oak-OST001b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:1.0:1506921878.256329:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:13.0:1506921878.256331:0:3863:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302608/t0(0) o19->oak-OST001b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921878.256339:0:3863:0:(events.c:171:reply_in_callback()) Process leaving 00000100:00000001:15.0:1506921878.256343:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000010:13.0:1506921878.256343:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88203bd18600 (tot 59423468). 00000400:00000001:15.0:1506921878.256344:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.256345:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000800:00000200:13.0:1506921878.256345:0:3863:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000100:00000001:15.0:1506921878.256346:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.256348:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302608/t0(0) o19->oak-OST001b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000800:00000200:13.0:1506921878.256348:0:3863:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:13.0:1506921878.256349:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000200:15.0:1506921878.256354:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302608/t0(0) o19->oak-OST001b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.256358:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.256359:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.256360:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.256361:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.256363:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.256364:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.256365:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.256366:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.256367:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.256368:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.256370:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff8820362b2c00 time=73 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921878.256373:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff8820362b2bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.256374:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.256375:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.256376:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.256377:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.256379:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.256379:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST001b-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.256381:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.256381:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.256383:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302608/t0(0) o19->oak-OST001b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.256387:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.256389:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.256390:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.256391:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302608/t0(0) o19->oak-OST001b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.256396:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302608:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.256399:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.256400:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.256401:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.256403:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.256404:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302608/t0(0) o19->oak-OST001b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.256408:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.256410:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.256411:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.256412:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.256413:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.256415:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302608/t0(0) o19->oak-OST001b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.256419:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.256420:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.256421:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.256422:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.256423:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.256424:0:38882:0:(genops.c:1026:class_import_put()) import ffff8820362b2800 refcount=4 obd=oak-OST001b-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.256425:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.256426:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.256428:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.256428:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.256429:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.256431:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.256431:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.256432:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.256434:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.256435:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.256435:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.256436:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.256438:0:38882:0:(genops.c:1012:class_import_get()) import ffff8820362b4800 refcount=5 obd=oak-OST001c-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.256439:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.256440:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.256441:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.256442:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.256444:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.256445:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.256446:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.256447:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.256448:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.256449:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.256450:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.256452:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.256453:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.256459:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.256461:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.256463:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.256464:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.256465:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.256467:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302624/t0(0) o19->oak-OST001c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.256471:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.256472:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.256473:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.256474:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.256475:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302624:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.256477:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.256478:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.256479:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.256480:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.256481:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.256482:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.256483:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.256485:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.256486:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302624, portal 4 00000100:00000001:15.0:1506921878.256488:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.256488:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.256491:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302624/t0(0) o19->oak-OST001c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.256495:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.256496:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921878.256498:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fac400. 00000100:00000200:15.0:1506921878.256499:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302624, offset 0 00000400:00000010:15.0:1506921878.256500:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451aa00 (tot 59423868). 00000400:00000200:15.0:1506921878.256503:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921878.256509:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921878.256513:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921878.256516:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.256518:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921878.256520:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921878.256523:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.256524:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.256525:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.256526:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.256527:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.256528:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.256529:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.256530:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.256531:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.256532:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.256533:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.256562:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921878.256609:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:11.0:1506921878.256615:0:3862:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:11.0:1506921878.256618:0:3862:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:11.0:1506921878.256622:0:3862:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302624/t0(0) o19->oak-OST001c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.256630:0:3862:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921878.256632:0:3862:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302624/t0(0) o19->oak-OST001c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.256638:0:3862:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.256640:0:3862:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:11.0:1506921878.256641:0:3862:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fac400 00000400:00000010:11.0:1506921878.256642:0:3862:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fac400. 00000400:00000010:11.0:1506921878.256645:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451aa00 (tot 59423468). 00000800:00000200:11.0:1506921878.256648:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:5.0:1506921878.256650:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.2:1506921878.256775:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921878.256792:0:3858:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.256798:0:3858:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921878.256804:0:3858:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921878.256807:0:3858:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921878.256812:0:3858:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c2258a0 00000400:00000200:16.0:1506921878.256819:0:3858:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a1155 [1] + 192 00000100:00000001:16.0:1506921878.256824:0:3858:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921878.256827:0:3858:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302624/t0(0) o19->oak-OST001c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921878.256832:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921878.256835:0:3858:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302624/t0(0) o19->oak-OST001c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.256843:0:3858:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921878.256845:0:3858:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921878.256847:0:3858:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.256850:0:3858:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921878.256851:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:15.0:1506921878.256880:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.256884:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.256885:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.256886:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.256891:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302624/t0(0) o19->oak-OST001c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.256900:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302624/t0(0) o19->oak-OST001c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.256906:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.256907:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.256909:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.256910:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.256912:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.256913:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.256915:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.256916:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.256917:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.256919:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.256922:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff8820362b4c00 time=73 v=31 (31 3 1 31) 00000100:00001000:15.0:1506921878.256925:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff8820362b4bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.256926:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.256927:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.256928:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.256930:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.256931:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.256934:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST001c-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.256936:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.256936:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.256939:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302624/t0(0) o19->oak-OST001c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.256944:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.256946:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.256947:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.256949:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302624/t0(0) o19->oak-OST001c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.256954:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302624:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.256957:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.256960:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.256961:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.256962:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.256963:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302624/t0(0) o19->oak-OST001c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.256968:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.256969:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.256971:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.256972:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.256973:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.256975:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302624/t0(0) o19->oak-OST001c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.256979:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.256980:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.256981:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.256983:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.256984:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.256985:0:38882:0:(genops.c:1026:class_import_put()) import ffff8820362b4800 refcount=4 obd=oak-OST001c-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.256986:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.256987:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.256989:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.256989:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.256991:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.256992:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.256993:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.256994:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.256995:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.256997:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.256998:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.256999:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.257001:0:38882:0:(genops.c:1012:class_import_get()) import ffff8820362b6800 refcount=5 obd=oak-OST001d-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.257002:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.257003:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.257004:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.257006:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.257009:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.257010:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.257011:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.257012:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.257013:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.257015:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.257016:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.257018:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.257019:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.257028:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.257030:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.257031:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.257033:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.257034:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.257035:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302640/t0(0) o19->oak-OST001d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.257040:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.257041:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.257043:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.257044:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.257046:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302640:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.257048:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.257050:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.257051:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.257052:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.257053:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.257054:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.257056:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.257058:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.257061:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302640, portal 4 00000100:00000001:15.0:1506921878.257062:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.257063:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.257066:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302640/t0(0) o19->oak-OST001d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.257070:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.257072:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921878.257073:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fade80. 00000100:00000200:15.0:1506921878.257075:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302640, offset 0 00000400:00000010:15.0:1506921878.257077:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451f000 (tot 59423868). 00000400:00000200:15.0:1506921878.257080:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921878.257089:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921878.257093:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921878.257098:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.257101:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921878.257102:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921878.257106:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.257107:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.257108:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.257110:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.257111:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.257112:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.257113:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.257115:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.257116:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.257117:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.257118:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921878.257144:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.257161:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.257165:0:3860:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.257169:0:3860:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.257173:0:3860:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302640/t0(0) o19->oak-OST001d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.257181:0:3860:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921878.257183:0:3860:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302640/t0(0) o19->oak-OST001d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.257189:0:3860:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.257191:0:3860:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921878.257192:0:3860:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fade80 00000400:00000010:16.0:1506921878.257193:0:3860:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fade80. 00000800:00000200:18.0:1506921878.257197:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000010:16.0:1506921878.257197:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451f000 (tot 59423468). 00000800:00000200:16.0:1506921878.257201:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921878.257281:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:1.0:1506921878.257325:0:3864:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921878.257334:0:3864:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.42@o2ib4 00000400:00000200:1.0:1506921878.257341:0:3864:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:1.0:1506921878.257343:0:3864:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881c4b6cd200 (tot 59423868). 00000400:00000200:1.0:1506921878.257349:0:3864:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c2258b0 00000400:00000200:1.0:1506921878.257354:0:3864:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1d79 [1] + 192 00000100:00000001:1.0:1506921878.257359:0:3864:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:1.0:1506921878.257362:0:3864:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302640/t0(0) o19->oak-OST001d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:13.0:1506921878.257364:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:1.0:1506921878.257370:0:3864:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302640/t0(0) o19->oak-OST001d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921878.257377:0:3864:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:1.0:1506921878.257379:0:3864:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881c4b6cd200 (tot 59423468). 00000800:00000200:1.0:1506921878.257381:0:3864:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921878.257383:0:3864:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:1.0:1506921878.257385:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921878.257410:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.257416:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.257417:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.257419:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.257423:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302640/t0(0) o19->oak-OST001d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.257431:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302640/t0(0) o19->oak-OST001d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.257437:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.257439:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.257440:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.257442:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.257443:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.257445:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.257446:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.257447:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.257449:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.257450:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.257453:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff8820362b6c00 time=73 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921878.257456:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff8820362b6bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.257458:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.257459:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.257460:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.257461:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.257463:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.257464:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST001d-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.257465:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.257466:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.257468:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302640/t0(0) o19->oak-OST001d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.257474:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.257476:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.257477:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.257479:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302640/t0(0) o19->oak-OST001d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.257484:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302640:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.257489:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.257491:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.257492:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.257493:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.257495:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302640/t0(0) o19->oak-OST001d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.257500:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.257501:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.257503:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.257503:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.257505:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.257507:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302640/t0(0) o19->oak-OST001d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.257511:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.257511:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.257513:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.257514:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.257515:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.257516:0:38882:0:(genops.c:1026:class_import_put()) import ffff8820362b6800 refcount=4 obd=oak-OST001d-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.257518:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.257519:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.257520:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.257521:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.257522:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.257524:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.257524:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.257526:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.257527:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.257528:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.257530:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.257531:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.257532:0:38882:0:(genops.c:1012:class_import_get()) import ffff882036f50800 refcount=5 obd=oak-OST001e-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.257534:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.257535:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.257536:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.257538:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.257541:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.257542:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.257543:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.257544:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.257545:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.257547:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.257548:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.257550:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.257551:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.257560:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.257562:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.257563:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.257565:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.257566:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.257567:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302656/t0(0) o19->oak-OST001e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.257572:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.257573:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.257574:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.257575:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.257577:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302656:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.257579:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.257580:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.257582:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.257583:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.257584:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.257585:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.257587:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.257589:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.257592:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302656, portal 4 00000100:00000001:15.0:1506921878.257593:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.257594:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.257597:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302656/t0(0) o19->oak-OST001e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.257606:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.257608:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921878.257609:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fac180. 00000100:00000200:15.0:1506921878.257611:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302656, offset 0 00000400:00000010:15.0:1506921878.257613:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b04518e00 (tot 59423868). 00000400:00000200:15.0:1506921878.257616:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921878.257625:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921878.257630:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921878.257634:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.257636:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921878.257638:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921878.257641:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.257643:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.257644:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.257645:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.257647:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.257648:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.257649:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.257650:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.257651:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.257653:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.257653:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.257678:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921878.257734:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:11.0:1506921878.257741:0:3862:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:11.0:1506921878.257744:0:3862:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:11.0:1506921878.257748:0:3862:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302656/t0(0) o19->oak-OST001e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.257755:0:3862:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921878.257757:0:3862:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302656/t0(0) o19->oak-OST001e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.257770:0:3862:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.257772:0:3862:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:11.0:1506921878.257773:0:3862:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fac180 00000800:00000200:5.0:1506921878.257774:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000400:00000010:11.0:1506921878.257775:0:3862:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fac180. 00000400:00000010:11.0:1506921878.257778:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b04518e00 (tot 59423468). 00000800:00000200:11.0:1506921878.257781:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:2.2:1506921878.257846:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:13.0:1506921878.257886:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921878.257894:0:3863:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.42@o2ib4 00000400:00000200:13.0:1506921878.257900:0:3863:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:13.0:1506921878.257902:0:3863:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88203bd18600 (tot 59423868). 00000400:00000200:13.0:1506921878.257908:0:3863:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c2258c0 00000400:00000200:13.0:1506921878.257912:0:3863:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a116d [1] + 192 00000100:00000001:13.0:1506921878.257917:0:3863:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:13.0:1506921878.257920:0:3863:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302656/t0(0) o19->oak-OST001e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:13.0:1506921878.257927:0:3863:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302656/t0(0) o19->oak-OST001e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000800:00000200:1.0:1506921878.257927:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000001:13.0:1506921878.257934:0:3863:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:13.0:1506921878.257936:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88203bd18600 (tot 59423468). 00000800:00000200:13.0:1506921878.257938:0:3863:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921878.257940:0:3863:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:13.0:1506921878.257942:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921878.257967:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.257971:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.257973:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.257974:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.257978:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302656/t0(0) o19->oak-OST001e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.257987:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302656/t0(0) o19->oak-OST001e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.257993:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.257994:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.257996:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.257998:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.258001:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.258003:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.258004:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.258006:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.258007:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.258008:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.258012:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff882036f50c00 time=73 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921878.258015:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff882036f50bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.258016:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.258017:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.258019:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.258020:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.258022:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.258023:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST001e-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.258025:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.258025:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.258028:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302656/t0(0) o19->oak-OST001e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.258033:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.258035:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.258036:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.258037:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302656/t0(0) o19->oak-OST001e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.258043:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302656:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.258045:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.258047:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.258048:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.258049:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.258051:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302656/t0(0) o19->oak-OST001e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.258055:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.258056:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.258058:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.258059:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.258061:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.258063:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302656/t0(0) o19->oak-OST001e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.258067:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.258068:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.258069:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.258071:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.258072:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.258072:0:38882:0:(genops.c:1026:class_import_put()) import ffff882036f50800 refcount=4 obd=oak-OST001e-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.258074:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.258075:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.258076:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.258077:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.258078:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.258080:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.258080:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.258081:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.258083:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.258084:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.258085:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.258086:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.258088:0:38882:0:(genops.c:1012:class_import_get()) import ffff882036f52800 refcount=5 obd=oak-OST001f-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.258090:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.258091:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.258092:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.258094:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.258096:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.258098:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.258098:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.258100:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.258101:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.258102:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.258103:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.258105:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.258106:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.258115:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.258117:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.258119:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.258120:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.258121:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.258123:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302672/t0(0) o19->oak-OST001f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.258127:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.258128:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.258130:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.258131:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.258133:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302672:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.258135:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.258136:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.258137:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.258138:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.258139:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.258141:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.258142:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.258144:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.258146:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302672, portal 4 00000100:00000001:15.0:1506921878.258148:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.258149:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.258151:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302672/t0(0) o19->oak-OST001f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.258156:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.258158:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921878.258159:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad000. 00000100:00000200:15.0:1506921878.258160:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302672, offset 0 00000400:00000010:15.0:1506921878.258162:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451f800 (tot 59423868). 00000400:00000200:15.0:1506921878.258165:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921878.258174:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921878.258178:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921878.258183:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.258185:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921878.258188:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921878.258192:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.258193:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.258194:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.258196:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.258197:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.258198:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.258200:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.258201:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.258202:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.258203:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.258204:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921878.258230:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.258248:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.258253:0:3859:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.258257:0:3859:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.258261:0:3859:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302672/t0(0) o19->oak-OST001f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.258269:0:3859:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921878.258271:0:3859:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302672/t0(0) o19->oak-OST001f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.258277:0:3859:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.258281:0:3859:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921878.258282:0:3859:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad000 00000400:00000010:16.0:1506921878.258284:0:3859:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad000. 00000800:00000200:18.0:1506921878.258285:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000010:16.0:1506921878.258287:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451f800 (tot 59423468). 00000800:00000200:16.0:1506921878.258290:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921878.258347:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:5.0:1506921878.258390:0:3865:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921878.258398:0:3865:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.42@o2ib4 00000400:00000200:5.0:1506921878.258404:0:3865:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:5.0:1506921878.258407:0:3865:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881e48bc2800 (tot 59423868). 00000400:00000200:5.0:1506921878.258412:0:3865:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c2258d0 00000800:00000200:11.0:1506921878.258416:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000400:00000200:5.0:1506921878.258417:0:3865:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1d81 [1] + 192 00000100:00000001:5.0:1506921878.258422:0:3865:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:5.0:1506921878.258425:0:3865:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302672/t0(0) o19->oak-OST001f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:5.0:1506921878.258432:0:3865:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302672/t0(0) o19->oak-OST001f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921878.258440:0:3865:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:5.0:1506921878.258442:0:3865:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881e48bc2800 (tot 59423468). 00000800:00000200:5.0:1506921878.258445:0:3865:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921878.258447:0:3865:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:5.0:1506921878.258451:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921878.258480:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.258484:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.258485:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.258486:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.258490:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302672/t0(0) o19->oak-OST001f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.258499:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302672/t0(0) o19->oak-OST001f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.258505:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.258506:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.258508:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.258510:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.258511:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.258513:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.258514:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.258515:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.258516:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.258518:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.258521:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff882036f52c00 time=73 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921878.258524:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff882036f52bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.258526:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.258527:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.258529:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.258530:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.258532:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.258535:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST001f-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.258536:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.258537:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.258539:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302672/t0(0) o19->oak-OST001f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.258544:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.258546:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.258547:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.258549:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302672/t0(0) o19->oak-OST001f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.258554:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302672:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.258557:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.258559:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.258560:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.258561:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.258563:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302672/t0(0) o19->oak-OST001f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.258567:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.258568:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.258570:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.258571:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.258573:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.258574:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302672/t0(0) o19->oak-OST001f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.258578:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.258579:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.258580:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.258582:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.258583:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.258584:0:38882:0:(genops.c:1026:class_import_put()) import ffff882036f52800 refcount=4 obd=oak-OST001f-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.258585:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.258586:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.258588:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.258589:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.258590:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.258592:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.258592:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.258594:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.258595:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.258596:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.258597:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.258599:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.258600:0:38882:0:(genops.c:1012:class_import_get()) import ffff882036f54800 refcount=5 obd=oak-OST0020-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.258602:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.258603:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.258604:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.258606:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.258608:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.258610:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.258610:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.258612:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.258613:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.258615:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.258615:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.258617:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.258619:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.258627:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.258630:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.258631:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.258632:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.258633:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.258635:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302688/t0(0) o19->oak-OST0020-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.258640:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.258641:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.258642:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.258643:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.258645:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302688:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.258647:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.258649:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.258650:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.258652:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.258653:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.258654:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.258656:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.258658:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.258660:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302688, portal 4 00000100:00000001:15.0:1506921878.258662:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.258662:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.258665:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302688/t0(0) o19->oak-OST0020-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.258669:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.258671:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921878.258673:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fac780. 00000100:00000200:15.0:1506921878.258674:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302688, offset 0 00000400:00000010:15.0:1506921878.258676:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b04519800 (tot 59423868). 00000400:00000200:15.0:1506921878.258679:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921878.258687:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921878.258692:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921878.258696:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.258698:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921878.258700:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921878.258704:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.258705:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.258706:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.258707:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.258709:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.258709:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.258711:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.258712:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.258713:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.258715:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.258715:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.258741:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921878.258784:0:3864:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:1.0:1506921878.258795:0:3864:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:1.0:1506921878.258799:0:3864:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:1.0:1506921878.258803:0:3864:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302688/t0(0) o19->oak-OST0020-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921878.258817:0:3864:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1506921878.258820:0:3864:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302688/t0(0) o19->oak-OST0020-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921878.258824:0:3864:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921878.258826:0:3864:0:(events.c:81:request_out_callback()) Process leaving 00000800:00000200:13.0:1506921878.258827:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000400:00000200:1.0:1506921878.258827:0:3864:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fac780 00000400:00000010:1.0:1506921878.258828:0:3864:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fac780. 00000400:00000010:1.0:1506921878.258832:0:3864:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b04519800 (tot 59423468). 00000800:00000200:1.0:1506921878.258834:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.2:1506921878.258912:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921878.258930:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.258935:0:3861:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921878.258942:0:3861:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921878.258944:0:3861:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921878.258950:0:3861:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c2258e0 00000400:00000200:16.0:1506921878.258955:0:3861:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a1185 [1] + 192 00000100:00000001:16.0:1506921878.258960:0:3861:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921878.258963:0:3861:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302688/t0(0) o19->oak-OST0020-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921878.258967:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921878.258971:0:3861:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302688/t0(0) o19->oak-OST0020-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.258979:0:3861:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921878.258980:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921878.258983:0:3861:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.258985:0:3861:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921878.258987:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:15.0:1506921878.259014:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.259020:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.259021:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.259023:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.259027:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302688/t0(0) o19->oak-OST0020-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.259035:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302688/t0(0) o19->oak-OST0020-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.259041:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.259043:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.259045:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.259046:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.259047:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.259049:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.259050:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.259051:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.259052:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.259054:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.259057:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff882036f54c00 time=73 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921878.259059:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff882036f54bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.259061:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.259062:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.259063:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.259064:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.259066:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.259067:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0020-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.259068:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.259069:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.259071:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302688/t0(0) o19->oak-OST0020-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.259076:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.259078:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.259079:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.259081:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302688/t0(0) o19->oak-OST0020-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.259086:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302688:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.259091:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.259093:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.259094:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.259095:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.259096:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302688/t0(0) o19->oak-OST0020-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.259101:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.259102:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.259103:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.259104:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.259106:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.259107:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302688/t0(0) o19->oak-OST0020-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.259111:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.259112:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.259113:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.259115:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.259115:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.259116:0:38882:0:(genops.c:1026:class_import_put()) import ffff882036f54800 refcount=4 obd=oak-OST0020-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.259118:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.259119:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.259120:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.259121:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.259122:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.259123:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.259124:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.259125:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.259126:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.259127:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.259129:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.259130:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.259131:0:38882:0:(genops.c:1012:class_import_get()) import ffff882036f56800 refcount=5 obd=oak-OST0021-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.259132:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.259134:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.259135:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.259137:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.259139:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.259141:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.259141:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.259143:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.259144:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.259146:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.259147:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.259149:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.259150:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.259158:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.259160:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.259161:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.259163:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.259163:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.259165:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302704/t0(0) o19->oak-OST0021-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.259170:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.259171:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.259173:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.259174:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.259176:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302704:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.259178:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.259179:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.259180:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.259181:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.259182:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.259183:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.259185:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.259187:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.259189:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302704, portal 4 00000100:00000001:15.0:1506921878.259191:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.259191:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.259194:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302704/t0(0) o19->oak-OST0021-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.259202:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.259204:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921878.259205:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fac380. 00000100:00000200:15.0:1506921878.259206:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302704, offset 0 00000400:00000010:15.0:1506921878.259209:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451a800 (tot 59423868). 00000400:00000200:15.0:1506921878.259212:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921878.259220:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921878.259224:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921878.259229:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.259231:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921878.259233:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921878.259237:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.259238:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.259239:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.259240:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.259241:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.259242:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.259243:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.259244:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.259245:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.259247:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.259247:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921878.259274:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.259292:0:3858:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.259297:0:3858:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.259300:0:3858:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.259304:0:3858:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302704/t0(0) o19->oak-OST0021-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.259312:0:3858:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921878.259314:0:3858:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302704/t0(0) o19->oak-OST0021-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.259320:0:3858:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.259321:0:3858:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921878.259322:0:3858:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fac380 00000400:00000010:16.0:1506921878.259324:0:3858:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fac380. 00000400:00000010:16.0:1506921878.259328:0:3858:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451a800 (tot 59423468). 00000800:00000200:18.0:1506921878.259329:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921878.259334:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921878.259364:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:11.0:1506921878.259408:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921878.259416:0:3862:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.42@o2ib4 00000400:00000200:11.0:1506921878.259422:0:3862:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:11.0:1506921878.259425:0:3862:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff882011e13400 (tot 59423868). 00000400:00000200:11.0:1506921878.259430:0:3862:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c2258f0 00000400:00000200:11.0:1506921878.259435:0:3862:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1d89 [1] + 192 00000100:00000001:11.0:1506921878.259440:0:3862:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:11.0:1506921878.259442:0:3862:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302704/t0(0) o19->oak-OST0021-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:5.0:1506921878.259447:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:11.0:1506921878.259450:0:3862:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302704/t0(0) o19->oak-OST0021-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.259457:0:3862:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:11.0:1506921878.259459:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff882011e13400 (tot 59423468). 00000100:00000001:15.0:1506921878.259462:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:11.0:1506921878.259462:0:3862:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000400:00000001:15.0:1506921878.259463:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.259464:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000800:00000200:11.0:1506921878.259464:0:3862:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.259465:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000800:00000200:11.0:1506921878.259465:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000200:15.0:1506921878.259466:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302704/t0(0) o19->oak-OST0021-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.259471:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302704/t0(0) o19->oak-OST0021-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.259475:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.259476:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.259477:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.259479:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.259480:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.259481:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.259482:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.259483:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.259484:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.259486:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.259487:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff882036f56c00 time=73 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921878.259489:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff882036f56bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.259491:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.259492:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.259493:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.259494:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.259495:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.259496:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0021-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.259497:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.259497:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.259499:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302704/t0(0) o19->oak-OST0021-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.259503:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.259504:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.259505:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.259507:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302704/t0(0) o19->oak-OST0021-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.259511:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302704:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.259513:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.259515:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.259516:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.259517:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.259518:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302704/t0(0) o19->oak-OST0021-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.259522:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.259524:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.259525:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.259525:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.259527:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.259528:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302704/t0(0) o19->oak-OST0021-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.259532:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.259533:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.259534:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.259535:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.259535:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.259536:0:38882:0:(genops.c:1026:class_import_put()) import ffff882036f56800 refcount=4 obd=oak-OST0021-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.259538:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.259538:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.259540:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.259540:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.259541:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.259542:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.259543:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.259544:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.259545:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.259546:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.259547:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.259548:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.259549:0:38882:0:(genops.c:1012:class_import_get()) import ffff882036a00800 refcount=5 obd=oak-OST0022-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.259550:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.259551:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.259552:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.259553:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.259555:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.259556:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.259557:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.259558:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.259559:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.259560:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.259560:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.259562:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.259563:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.259569:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.259571:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.259572:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.259573:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.259574:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.259576:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302720/t0(0) o19->oak-OST0022-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.259580:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.259580:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.259581:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.259582:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.259584:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302720:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.259586:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.259586:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.259587:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.259588:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.259589:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.259590:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.259591:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.259593:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.259594:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302720, portal 4 00000100:00000001:15.0:1506921878.259595:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.259596:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.259598:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302720/t0(0) o19->oak-OST0022-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.259602:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.259603:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921878.259605:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032faca80. 00000100:00000200:15.0:1506921878.259606:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302720, offset 0 00000400:00000010:15.0:1506921878.259607:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451fe00 (tot 59423868). 00000400:00000200:15.0:1506921878.259609:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921878.259615:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921878.259619:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921878.259622:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.259623:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921878.259632:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921878.259634:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.259636:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.259637:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.259638:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.259639:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.259639:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.259640:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.259642:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.259642:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.259643:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.259644:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.259671:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921878.259733:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:13.0:1506921878.259740:0:3863:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:13.0:1506921878.259743:0:3863:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:13.0:1506921878.259747:0:3863:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302720/t0(0) o19->oak-OST0022-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921878.259754:0:3863:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:13.0:1506921878.259756:0:3863:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302720/t0(0) o19->oak-OST0022-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921878.259762:0:3863:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921878.259763:0:3863:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:13.0:1506921878.259764:0:3863:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032faca80 00000400:00000010:13.0:1506921878.259766:0:3863:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032faca80. 00000400:00000010:13.0:1506921878.259771:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451fe00 (tot 59423468). 00000800:00000200:1.0:1506921878.259773:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:13.0:1506921878.259777:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:2.2:1506921878.259823:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:5.0:1506921878.259862:0:3865:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921878.259871:0:3865:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.42@o2ib4 00000400:00000200:5.0:1506921878.259877:0:3865:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:5.0:1506921878.259879:0:3865:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881e48bc2800 (tot 59423868). 00000400:00000200:5.0:1506921878.259885:0:3865:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225900 00000400:00000200:5.0:1506921878.259889:0:3865:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a119d [1] + 192 00000100:00000001:5.0:1506921878.259894:0:3865:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:5.0:1506921878.259896:0:3865:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302720/t0(0) o19->oak-OST0022-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:11.0:1506921878.259904:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:5.0:1506921878.259904:0:3865:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302720/t0(0) o19->oak-OST0022-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921878.259911:0:3865:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:5.0:1506921878.259913:0:3865:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881e48bc2800 (tot 59423468). 00000800:00000200:5.0:1506921878.259915:0:3865:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921878.259917:0:3865:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:5.0:1506921878.259919:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921878.259943:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.259947:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.259948:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.259949:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.259953:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302720/t0(0) o19->oak-OST0022-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.259961:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302720/t0(0) o19->oak-OST0022-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.259966:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.259967:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.259969:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.259970:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.259971:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.259973:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.259974:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.259975:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.259976:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.259978:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.259980:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff882036a00c00 time=73 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921878.259983:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff882036a00bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.259985:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.259985:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.259987:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.259988:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.259989:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.259992:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0022-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.259993:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.259994:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.259996:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302720/t0(0) o19->oak-OST0022-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.260001:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.260003:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.260004:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.260005:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302720/t0(0) o19->oak-OST0022-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.260010:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302720:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.260013:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.260014:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.260015:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.260016:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.260018:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302720/t0(0) o19->oak-OST0022-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.260022:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.260023:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.260025:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.260025:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.260027:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.260028:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302720/t0(0) o19->oak-OST0022-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.260032:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.260033:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.260034:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.260035:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.260036:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.260037:0:38882:0:(genops.c:1026:class_import_put()) import ffff882036a00800 refcount=4 obd=oak-OST0022-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.260039:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.260039:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.260041:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.260041:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.260042:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.260044:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.260045:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.260046:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.260047:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.260048:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.260049:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.260050:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.260052:0:38882:0:(genops.c:1012:class_import_get()) import ffff882036a02800 refcount=5 obd=oak-OST0023-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.260053:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.260054:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.260055:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.260057:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.260059:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.260061:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.260062:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.260063:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.260064:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.260065:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.260066:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.260068:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.260069:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.260077:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.260079:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.260080:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.260082:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.260082:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.260084:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302736/t0(0) o19->oak-OST0023-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.260088:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.260089:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.260091:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.260092:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.260093:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302736:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.260095:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.260097:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.260098:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.260099:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.260100:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.260101:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.260103:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.260105:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.260107:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302736, portal 4 00000100:00000001:15.0:1506921878.260108:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.260109:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.260112:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302736/t0(0) o19->oak-OST0023-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.260116:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.260118:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921878.260119:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad900. 00000100:00000200:15.0:1506921878.260120:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302736, offset 0 00000400:00000010:15.0:1506921878.260122:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451ac00 (tot 59423868). 00000400:00000200:15.0:1506921878.260125:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921878.260133:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921878.260137:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921878.260142:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.260144:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921878.260146:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921878.260149:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.260150:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.260151:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.260153:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.260154:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.260155:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.260156:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.260157:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.260158:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.260159:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.260160:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921878.260187:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.260204:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.260209:0:3860:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.260213:0:3860:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.260217:0:3860:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302736/t0(0) o19->oak-OST0023-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.260225:0:3860:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921878.260227:0:3860:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302736/t0(0) o19->oak-OST0023-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.260233:0:3860:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.260234:0:3860:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921878.260236:0:3860:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad900 00000400:00000010:16.0:1506921878.260237:0:3860:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad900. 00000400:00000010:16.0:1506921878.260241:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451ac00 (tot 59423468). 00000800:00000200:18.0:1506921878.260242:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921878.260246:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921878.260255:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:1.0:1506921878.260299:0:3864:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921878.260308:0:3864:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.42@o2ib4 00000400:00000200:1.0:1506921878.260314:0:3864:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:1.0:1506921878.260317:0:3864:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881c4b6cd200 (tot 59423868). 00000400:00000200:1.0:1506921878.260322:0:3864:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225910 00000400:00000200:1.0:1506921878.260327:0:3864:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1d91 [1] + 192 00000100:00000001:1.0:1506921878.260332:0:3864:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:1.0:1506921878.260335:0:3864:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302736/t0(0) o19->oak-OST0023-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:13.0:1506921878.260338:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:1.0:1506921878.260343:0:3864:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302736/t0(0) o19->oak-OST0023-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921878.260351:0:3864:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:1.0:1506921878.260353:0:3864:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881c4b6cd200 (tot 59423468). 00000100:00000001:15.0:1506921878.260355:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:1.0:1506921878.260355:0:3864:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000400:00000001:15.0:1506921878.260357:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.260358:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.260358:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000800:00000200:1.0:1506921878.260358:0:3864:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:1.0:1506921878.260359:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000200:15.0:1506921878.260360:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302736/t0(0) o19->oak-OST0023-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.260365:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302736/t0(0) o19->oak-OST0023-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.260369:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.260370:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.260371:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.260372:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.260373:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.260374:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.260375:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.260376:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.260377:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.260378:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.260380:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff882036a02c00 time=73 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921878.260382:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff882036a02bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.260383:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.260384:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.260385:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.260386:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.260388:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.260388:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0023-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.260389:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.260390:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.260392:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302736/t0(0) o19->oak-OST0023-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.260395:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.260397:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.260397:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.260399:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302736/t0(0) o19->oak-OST0023-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.260404:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302736:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.260408:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.260409:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.260410:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.260411:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.260413:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302736/t0(0) o19->oak-OST0023-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.260416:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.260418:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.260419:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.260419:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.260421:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.260422:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302736/t0(0) o19->oak-OST0023-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.260426:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.260426:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.260427:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.260428:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.260429:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.260430:0:38882:0:(genops.c:1026:class_import_put()) import ffff882036a02800 refcount=4 obd=oak-OST0023-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.260431:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.260432:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.260433:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.260434:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.260434:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.260436:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.260437:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.260438:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.260439:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.260440:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.260440:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.260441:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.260442:0:38882:0:(genops.c:1012:class_import_get()) import ffff882036a04800 refcount=5 obd=oak-OST0024-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.260444:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.260444:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.260445:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.260447:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.260448:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.260450:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.260450:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.260451:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.260452:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.260454:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.260454:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.260456:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.260457:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.260463:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.260464:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.260466:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.260467:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.260468:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.260469:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302752/t0(0) o19->oak-OST0024-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.260473:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.260474:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.260475:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.260476:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.260477:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302752:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.260479:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.260480:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.260481:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.260481:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.260482:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.260483:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.260485:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.260486:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.260488:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302752, portal 4 00000100:00000001:15.0:1506921878.260489:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.260490:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.260492:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302752/t0(0) o19->oak-OST0024-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.260500:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.260501:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921878.260503:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fac980. 00000100:00000200:15.0:1506921878.260504:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302752, offset 0 00000400:00000010:15.0:1506921878.260505:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451d600 (tot 59423868). 00000400:00000200:15.0:1506921878.260508:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921878.260514:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921878.260517:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921878.260520:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.260522:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921878.260524:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921878.260526:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.260527:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.260528:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.260529:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.260530:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.260531:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.260532:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.260533:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.260534:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.260535:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.260535:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.260562:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921878.260607:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:11.0:1506921878.260613:0:3862:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:11.0:1506921878.260616:0:3862:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:11.0:1506921878.260619:0:3862:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302752/t0(0) o19->oak-OST0024-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.260626:0:3862:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921878.260629:0:3862:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302752/t0(0) o19->oak-OST0024-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.260640:0:3862:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.260642:0:3862:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:11.0:1506921878.260643:0:3862:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fac980 00000400:00000010:11.0:1506921878.260644:0:3862:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fac980. 00000800:00000200:5.0:1506921878.260645:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000400:00000010:11.0:1506921878.260647:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451d600 (tot 59423468). 00000800:00000200:11.0:1506921878.260650:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.2:1506921878.260766:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921878.260784:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.260790:0:3859:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921878.260796:0:3859:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921878.260799:0:3859:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921878.260805:0:3859:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225920 00000400:00000200:16.0:1506921878.260810:0:3859:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a11b5 [1] + 192 00000100:00000001:16.0:1506921878.260815:0:3859:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921878.260817:0:3859:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302752/t0(0) o19->oak-OST0024-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921878.260821:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921878.260825:0:3859:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302752/t0(0) o19->oak-OST0024-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.260834:0:3859:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921878.260835:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921878.260838:0:3859:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.260840:0:3859:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921878.260842:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:15.0:1506921878.260856:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.260858:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.260859:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.260860:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.260863:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302752/t0(0) o19->oak-OST0024-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.260868:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302752/t0(0) o19->oak-OST0024-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.260872:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.260873:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.260875:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.260876:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.260878:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.260879:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.260880:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.260881:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.260882:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.260883:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.260885:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff882036a04c00 time=73 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921878.260887:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff882036a04bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.260889:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.260889:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.260890:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.260891:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.260893:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.260893:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0024-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.260894:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.260895:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.260897:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302752/t0(0) o19->oak-OST0024-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.260901:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.260902:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.260903:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.260905:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302752/t0(0) o19->oak-OST0024-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.260909:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302752:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.260911:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.260913:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.260914:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.260915:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.260916:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302752/t0(0) o19->oak-OST0024-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.260920:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.260921:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.260923:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.260923:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.260925:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.260927:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302752/t0(0) o19->oak-OST0024-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.260931:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.260931:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.260932:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.260934:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.260935:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.260935:0:38882:0:(genops.c:1026:class_import_put()) import ffff882036a04800 refcount=4 obd=oak-OST0024-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.260937:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.260938:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.260939:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.260940:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.260940:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.260942:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.260942:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.260943:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.260944:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.260946:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.260946:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.260947:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.260948:0:38882:0:(genops.c:1012:class_import_get()) import ffff882036a06800 refcount=5 obd=oak-OST0025-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.260950:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.260950:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.260951:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.260953:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.260954:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.260955:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.260956:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.260957:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.260958:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.260959:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.260960:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.260961:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.260962:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.260969:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.260970:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.260972:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.260973:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.260974:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.260975:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302768/t0(0) o19->oak-OST0025-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.260979:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.260980:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.260981:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.260982:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.260983:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302768:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.260985:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.260986:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.260987:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.260988:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.260988:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.260989:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.260991:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.260992:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.260994:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302768, portal 4 00000100:00000001:15.0:1506921878.260995:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.260996:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.260998:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302768/t0(0) o19->oak-OST0025-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.261001:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.261003:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921878.261004:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fadf80. 00000100:00000200:15.0:1506921878.261005:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302768, offset 0 00000400:00000010:15.0:1506921878.261007:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451e200 (tot 59423868). 00000400:00000200:15.0:1506921878.261009:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921878.261015:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921878.261019:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921878.261022:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.261024:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921878.261026:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921878.261030:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.261031:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.261031:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.261033:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.261034:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.261034:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000800:00000200:16.2:1506921878.261035:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000100:00100000:15.0:1506921878.261035:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.261037:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.261037:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.261038:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.261039:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.0:1506921878.261042:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.261046:0:3861:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.261051:0:3861:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.261054:0:3861:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302768/t0(0) o19->oak-OST0025-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.261059:0:3861:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000800:00000200:18.0:1506921878.261061:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921878.261061:0:3861:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302768/t0(0) o19->oak-OST0025-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.261065:0:3861:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.261067:0:3861:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921878.261068:0:3861:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fadf80 00000400:00000010:16.0:1506921878.261070:0:3861:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fadf80. 00000400:00000010:16.0:1506921878.261073:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451e200 (tot 59423468). 00000800:00000200:16.0:1506921878.261075:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921878.261127:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:13.0:1506921878.261170:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921878.261179:0:3863:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.42@o2ib4 00000400:00000200:13.0:1506921878.261186:0:3863:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:13.0:1506921878.261188:0:3863:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88203bd18600 (tot 59423868). 00000400:00000200:13.0:1506921878.261194:0:3863:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225930 00000400:00000200:13.0:1506921878.261199:0:3863:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1d99 [1] + 192 00000100:00000001:13.0:1506921878.261204:0:3863:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:13.0:1506921878.261207:0:3863:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302768/t0(0) o19->oak-OST0025-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:13.0:1506921878.261214:0:3863:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302768/t0(0) o19->oak-OST0025-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000800:00000200:1.0:1506921878.261216:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000001:13.0:1506921878.261222:0:3863:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:13.0:1506921878.261224:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88203bd18600 (tot 59423468). 00000100:00000001:15.0:1506921878.261226:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:13.0:1506921878.261226:0:3863:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000400:00000001:15.0:1506921878.261227:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.261228:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.261229:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000800:00000200:13.0:1506921878.261229:0:3863:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000100:00000200:15.0:1506921878.261230:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302768/t0(0) o19->oak-OST0025-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000800:00000200:13.0:1506921878.261230:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000200:15.0:1506921878.261235:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302768/t0(0) o19->oak-OST0025-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.261239:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.261240:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.261241:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.261242:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.261242:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.261243:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.261244:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.261245:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.261246:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.261247:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.261249:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff882036a06c00 time=73 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921878.261250:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff882036a06bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.261252:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.261252:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.261253:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.261254:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.261255:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.261257:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0025-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.261258:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.261259:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.261260:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302768/t0(0) o19->oak-OST0025-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.261264:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.261265:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.261266:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.261268:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302768/t0(0) o19->oak-OST0025-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.261272:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302768:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.261274:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.261276:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.261277:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.261278:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.261279:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302768/t0(0) o19->oak-OST0025-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.261283:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.261284:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.261285:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.261286:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.261287:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.261288:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302768/t0(0) o19->oak-OST0025-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.261292:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.261293:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.261293:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.261295:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.261295:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.261296:0:38882:0:(genops.c:1026:class_import_put()) import ffff882036a06800 refcount=4 obd=oak-OST0025-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.261297:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.261298:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.261299:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.261300:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.261301:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.261302:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.261303:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.261305:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.261306:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.261307:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.261308:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.261309:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.261310:0:38882:0:(genops.c:1012:class_import_get()) import ffff882036aa0800 refcount=5 obd=oak-OST0026-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.261311:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.261312:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.261312:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.261314:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.261315:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.261316:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.261317:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.261318:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.261319:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.261320:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.261320:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.261322:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.261323:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.261328:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.261330:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.261331:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.261332:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.261333:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.261334:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302784/t0(0) o19->oak-OST0026-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.261338:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.261338:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.261339:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.261340:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.261342:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302784:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.261344:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.261345:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.261346:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.261347:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.261347:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.261348:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.261350:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.261351:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.261353:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302784, portal 4 00000100:00000001:15.0:1506921878.261354:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.261354:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.261357:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302784/t0(0) o19->oak-OST0026-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.261360:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.261362:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921878.261363:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fac600. 00000100:00000200:15.0:1506921878.261364:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302784, offset 0 00000400:00000010:15.0:1506921878.261365:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b04518000 (tot 59423868). 00000400:00000200:15.0:1506921878.261367:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921878.261373:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921878.261376:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921878.261379:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.261380:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921878.261382:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921878.261384:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.261385:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.261386:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.261387:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.261388:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.261388:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.261389:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.261390:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.261391:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.261392:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.261393:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.261422:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921878.261467:0:3865:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:5.0:1506921878.261473:0:3865:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:5.0:1506921878.261476:0:3865:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:5.0:1506921878.261480:0:3865:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302784/t0(0) o19->oak-OST0026-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921878.261488:0:3865:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:5.0:1506921878.261490:0:3865:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302784/t0(0) o19->oak-OST0026-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921878.261496:0:3865:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:5.0:1506921878.261497:0:3865:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:5.0:1506921878.261499:0:3865:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fac600 00000400:00000010:5.0:1506921878.261500:0:3865:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fac600. 00000400:00000010:5.0:1506921878.261503:0:3865:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b04518000 (tot 59423468). 00000800:00000200:5.0:1506921878.261506:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:5.0:1506921878.261515:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:2.2:1506921878.261590:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:1.0:1506921878.261630:0:3864:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921878.261639:0:3864:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.42@o2ib4 00000400:00000200:1.0:1506921878.261645:0:3864:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:1.0:1506921878.261647:0:3864:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881c4b6cd200 (tot 59423868). 00000400:00000200:1.0:1506921878.261653:0:3864:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225940 00000400:00000200:1.0:1506921878.261660:0:3864:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a11cd [1] + 192 00000100:00000001:1.0:1506921878.261664:0:3864:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:1.0:1506921878.261667:0:3864:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302784/t0(0) o19->oak-OST0026-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:13.0:1506921878.261668:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:1.0:1506921878.261675:0:3864:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302784/t0(0) o19->oak-OST0026-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921878.261681:0:3864:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:1.0:1506921878.261683:0:3864:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881c4b6cd200 (tot 59423468). 00000800:00000200:1.0:1506921878.261686:0:3864:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921878.261688:0:3864:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:1.0:1506921878.261689:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921878.261714:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.261730:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.261731:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.261732:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.261736:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302784/t0(0) o19->oak-OST0026-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.261744:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302784/t0(0) o19->oak-OST0026-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.261749:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.261751:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.261752:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.261753:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.261755:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.261756:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.261757:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.261758:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.261759:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.261761:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.261764:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff882036aa0c00 time=73 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921878.261766:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff882036aa0bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.261768:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.261769:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.261770:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.261771:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.261773:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.261774:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0026-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.261775:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.261775:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.261778:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302784/t0(0) o19->oak-OST0026-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.261783:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.261784:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.261785:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.261787:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302784/t0(0) o19->oak-OST0026-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.261792:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302784:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.261797:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.261798:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.261799:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.261800:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.261802:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302784/t0(0) o19->oak-OST0026-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.261806:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.261807:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.261808:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.261809:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.261811:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.261812:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302784/t0(0) o19->oak-OST0026-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.261816:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.261817:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.261818:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.261819:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.261820:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.261821:0:38882:0:(genops.c:1026:class_import_put()) import ffff882036aa0800 refcount=4 obd=oak-OST0026-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.261822:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.261823:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.261824:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.261825:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.261826:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.261827:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.261828:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.261829:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.261831:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.261832:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.261833:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.261834:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.261835:0:38882:0:(genops.c:1012:class_import_get()) import ffff882036aa2800 refcount=5 obd=oak-OST0027-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.261837:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.261838:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.261839:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.261841:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.261844:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.261845:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.261846:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.261848:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.261849:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.261850:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.261851:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.261853:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.261854:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.261862:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.261864:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.261866:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.261867:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.261868:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.261869:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302800/t0(0) o19->oak-OST0027-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.261874:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.261875:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.261876:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.261877:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.261879:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302800:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.261881:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.261882:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.261883:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.261884:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.261885:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.261886:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.261888:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.261890:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.261892:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302800, portal 4 00000100:00000001:15.0:1506921878.261894:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.261894:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.261897:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302800/t0(0) o19->oak-OST0027-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.261906:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.261908:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921878.261909:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad080. 00000100:00000200:15.0:1506921878.261910:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302800, offset 0 00000400:00000010:15.0:1506921878.261913:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451d000 (tot 59423868). 00000400:00000200:15.0:1506921878.261915:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921878.261924:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921878.261927:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921878.261932:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.261934:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921878.261936:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921878.261940:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.261941:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.261942:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.261943:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.261945:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.261946:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.261947:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.261948:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.261949:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.261950:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.261951:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921878.261977:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.261995:0:3858:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.262000:0:3858:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.262003:0:3858:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.262008:0:3858:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302800/t0(0) o19->oak-OST0027-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.262015:0:3858:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921878.262018:0:3858:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302800/t0(0) o19->oak-OST0027-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.262023:0:3858:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.262025:0:3858:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921878.262026:0:3858:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad080 00000400:00000010:16.0:1506921878.262028:0:3858:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad080. 00000400:00000010:16.0:1506921878.262031:0:3858:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451d000 (tot 59423468). 00000800:00000200:18.0:1506921878.262032:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.262040:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (23)-- 00000800:00000200:18.0:1506921878.262042:0:3859:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.41@o2ib4 00000800:00000200:16.0:1506921878.262042:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000200:18.0:1506921878.262048:0:3859:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:18.0:1506921878.262051:0:3859:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103a767000 (tot 59423868). 00000400:00000200:18.0:1506921878.262057:0:3859:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225950 00000400:00000200:18.0:1506921878.262062:0:3859:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1da1 [1] + 192 00000100:00000001:18.0:1506921878.262067:0:3859:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:18.0:1506921878.262069:0:3859:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302800/t0(0) o19->oak-OST0027-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:18.0:1506921878.262077:0:3859:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302800/t0(0) o19->oak-OST0027-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:18.0:1506921878.262086:0:3859:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:18.0:1506921878.262088:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103a767000 (tot 59423468). 00000800:00000200:18.0:1506921878.262090:0:3859:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000100:00000001:15.0:1506921878.262092:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:18.0:1506921878.262093:0:3859:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:18.0:1506921878.262094:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000400:00000001:15.0:1506921878.262094:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.262095:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.262096:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.262099:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302800/t0(0) o19->oak-OST0027-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.262104:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302800/t0(0) o19->oak-OST0027-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.262109:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.262110:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.262112:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.262113:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.262114:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.262115:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.262116:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.262117:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.262118:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.262120:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.262121:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff882036aa2c00 time=73 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921878.262123:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff882036aa2bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.262125:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.262126:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.262127:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.262128:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.262129:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.262130:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0027-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.262131:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.262132:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.262134:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302800/t0(0) o19->oak-OST0027-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.262138:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.262139:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.262140:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.262142:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302800/t0(0) o19->oak-OST0027-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.262146:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302800:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.262149:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.262150:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.262151:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.262153:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.262154:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302800/t0(0) o19->oak-OST0027-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.262158:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.262159:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.262161:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.262161:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.262163:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.262164:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302800/t0(0) o19->oak-OST0027-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.262168:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.262169:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.262170:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.262171:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.262172:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.262172:0:38882:0:(genops.c:1026:class_import_put()) import ffff882036aa2800 refcount=4 obd=oak-OST0027-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.262174:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.262175:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.262176:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.262177:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.262177:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.262179:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.262179:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.262180:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.262181:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.262182:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.262183:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.262184:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.262185:0:38882:0:(genops.c:1012:class_import_get()) import ffff882036aa4800 refcount=5 obd=oak-OST0028-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.262186:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.262187:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.262188:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.262189:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.262191:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.262192:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.262193:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.262194:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.262195:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.262196:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.262197:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.262198:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.262199:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.262205:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.262207:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.262208:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.262210:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.262210:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.262212:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302816/t0(0) o19->oak-OST0028-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.262216:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.262217:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.262218:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.262219:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.262221:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302816:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.262222:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.262223:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.262224:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.262225:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.262226:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.262227:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.262228:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.262230:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.262231:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302816, portal 4 00000100:00000001:15.0:1506921878.262232:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.262233:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.262235:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302816/t0(0) o19->oak-OST0028-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.262239:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.262240:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921878.262242:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032faca00. 00000100:00000200:15.0:1506921878.262243:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302816, offset 0 00000400:00000010:15.0:1506921878.262244:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b04519e00 (tot 59423868). 00000400:00000200:15.0:1506921878.262247:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921878.262253:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921878.262256:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921878.262259:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.262261:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921878.262263:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921878.262265:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.262266:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.262267:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.262268:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.262270:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.262270:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.262272:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.262273:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.262273:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.262275:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.262275:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.262301:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921878.262342:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:11.0:1506921878.262349:0:3862:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:11.0:1506921878.262351:0:3862:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:11.0:1506921878.262355:0:3862:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302816/t0(0) o19->oak-OST0028-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.262362:0:3862:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921878.262364:0:3862:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302816/t0(0) o19->oak-OST0028-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.262370:0:3862:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.262371:0:3862:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:11.0:1506921878.262372:0:3862:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032faca00 00000400:00000010:11.0:1506921878.262373:0:3862:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032faca00. 00000400:00000010:11.0:1506921878.262377:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b04519e00 (tot 59423468). 00000800:00000200:11.0:1506921878.262379:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:5.0:1506921878.262382:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.2:1506921878.262518:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921878.262535:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.262541:0:3861:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921878.262547:0:3861:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921878.262549:0:3861:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921878.262555:0:3861:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225960 00000400:00000200:16.0:1506921878.262560:0:3861:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a11e5 [1] + 192 00000100:00000001:16.0:1506921878.262564:0:3861:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921878.262569:0:3861:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302816/t0(0) o19->oak-OST0028-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921878.262575:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921878.262577:0:3861:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302816/t0(0) o19->oak-OST0028-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.262585:0:3861:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921878.262587:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921878.262590:0:3861:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.262592:0:3861:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921878.262594:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:15.0:1506921878.262622:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.262626:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.262627:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.262628:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.262633:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302816/t0(0) o19->oak-OST0028-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.262641:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302816/t0(0) o19->oak-OST0028-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.262647:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.262649:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.262651:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.262652:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.262653:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.262655:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.262657:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.262658:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.262659:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.262661:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.262664:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff882036aa4c00 time=73 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921878.262667:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff882036aa4bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.262669:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.262669:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.262671:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.262672:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.262674:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.262677:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0028-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.262679:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.262679:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.262682:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302816/t0(0) o19->oak-OST0028-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.262687:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.262689:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.262690:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.262692:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302816/t0(0) o19->oak-OST0028-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.262697:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302816:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.262700:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.262702:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.262703:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.262705:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.262706:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302816/t0(0) o19->oak-OST0028-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.262710:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.262712:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.262713:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.262714:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.262716:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.262728:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302816/t0(0) o19->oak-OST0028-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.262733:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.262734:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.262735:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.262736:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.262737:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.262738:0:38882:0:(genops.c:1026:class_import_put()) import ffff882036aa4800 refcount=4 obd=oak-OST0028-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.262740:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.262741:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.262742:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.262743:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.262745:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.262747:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.262747:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.262749:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.262751:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.262752:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.262753:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.262754:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.262756:0:38882:0:(genops.c:1012:class_import_get()) import ffff882036aa6800 refcount=5 obd=oak-OST0029-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.262758:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.262759:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.262760:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.262763:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.262765:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.262767:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.262767:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.262769:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.262770:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.262772:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.262772:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.262774:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.262776:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.262785:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.262787:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.262789:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.262790:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.262791:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.262793:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302832/t0(0) o19->oak-OST0029-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.262798:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.262799:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.262800:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.262801:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.262803:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302832:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.262805:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.262811:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.262812:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.262814:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.262815:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.262816:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.262819:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.262821:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.262823:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302832, portal 4 00000100:00000001:15.0:1506921878.262824:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.262825:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.262828:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302832/t0(0) o19->oak-OST0029-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.262833:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.262835:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921878.262836:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad580. 00000100:00000200:15.0:1506921878.262837:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302832, offset 0 00000400:00000010:15.0:1506921878.262840:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451a600 (tot 59423868). 00000400:00000200:15.0:1506921878.262843:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921878.262853:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921878.262857:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921878.262862:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.262864:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921878.262866:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921878.262870:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.262872:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.262873:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.262875:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.262876:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.262877:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.262878:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.262880:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.262881:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.262883:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.262883:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921878.262908:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.262925:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.262929:0:3860:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.262933:0:3860:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.262937:0:3860:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302832/t0(0) o19->oak-OST0029-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.262945:0:3860:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921878.262947:0:3860:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302832/t0(0) o19->oak-OST0029-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.262953:0:3860:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.262955:0:3860:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921878.262956:0:3860:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad580 00000400:00000010:16.0:1506921878.262957:0:3860:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad580. 00000400:00000010:16.0:1506921878.262961:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451a600 (tot 59423468). 00000800:00000200:18.0:1506921878.262962:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921878.262966:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921878.262979:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:13.0:1506921878.263023:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921878.263031:0:3863:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.42@o2ib4 00000400:00000200:13.0:1506921878.263038:0:3863:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:13.0:1506921878.263040:0:3863:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88203bd18600 (tot 59423868). 00000400:00000200:13.0:1506921878.263046:0:3863:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225970 00000400:00000200:13.0:1506921878.263059:0:3863:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1da9 [1] + 192 00000100:00000001:13.0:1506921878.263064:0:3863:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:13.0:1506921878.263067:0:3863:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302832/t0(0) o19->oak-OST0029-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:1.0:1506921878.263068:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:13.0:1506921878.263074:0:3863:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302832/t0(0) o19->oak-OST0029-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921878.263082:0:3863:0:(events.c:171:reply_in_callback()) Process leaving 00000100:00000001:15.0:1506921878.263086:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000010:13.0:1506921878.263086:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88203bd18600 (tot 59423468). 00000400:00000001:15.0:1506921878.263089:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000800:00000200:13.0:1506921878.263089:0:3863:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000400:00000001:15.0:1506921878.263090:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.263091:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000800:00000200:13.0:1506921878.263091:0:3863:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:13.0:1506921878.263092:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000200:15.0:1506921878.263093:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302832/t0(0) o19->oak-OST0029-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.263098:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302832/t0(0) o19->oak-OST0029-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.263102:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.263102:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.263104:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.263105:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.263106:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.263107:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.263108:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.263108:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.263109:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.263111:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.263112:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff882036aa6c00 time=73 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921878.263115:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff882036aa6bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.263116:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.263117:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.263118:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.263119:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.263121:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.263121:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST0029-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.263123:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.263123:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.263125:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302832/t0(0) o19->oak-OST0029-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.263129:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.263130:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.263131:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.263133:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302832/t0(0) o19->oak-OST0029-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.263137:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302832:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.263142:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.263143:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.263145:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.263146:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.263148:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302832/t0(0) o19->oak-OST0029-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.263152:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.263153:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.263154:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.263155:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.263156:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.263157:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302832/t0(0) o19->oak-OST0029-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.263161:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.263162:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.263162:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.263164:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.263164:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.263165:0:38882:0:(genops.c:1026:class_import_put()) import ffff882036aa6800 refcount=4 obd=oak-OST0029-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.263167:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.263167:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.263169:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.263169:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.263170:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.263171:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.263172:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.263173:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.263174:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.263175:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.263176:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.263177:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.263178:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203dd40800 refcount=5 obd=oak-OST002a-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.263179:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.263180:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.263181:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.263182:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.263184:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.263186:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.263187:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.263188:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.263189:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.263190:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.263191:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.263192:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.263194:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.263200:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.263202:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.263203:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.263204:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.263205:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.263206:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302848/t0(0) o19->oak-OST002a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.263211:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.263212:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.263213:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.263214:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.263216:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302848:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.263218:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.263218:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.263219:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.263220:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.263221:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.263222:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.263223:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.263225:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.263226:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302848, portal 4 00000100:00000001:15.0:1506921878.263228:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.263228:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.263230:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302848/t0(0) o19->oak-OST002a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.263239:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.263240:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921878.263242:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032face00. 00000100:00000200:15.0:1506921878.263243:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302848, offset 0 00000400:00000010:15.0:1506921878.263244:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451dc00 (tot 59423868). 00000400:00000200:15.0:1506921878.263247:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921878.263253:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921878.263256:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921878.263259:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.263261:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921878.263262:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921878.263264:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.263265:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.263266:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.263268:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.263269:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.263270:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.263271:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.263272:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.263273:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.263274:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.263275:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.263301:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:11.0:1506921878.263346:0:3862:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:11.0:1506921878.263353:0:3862:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:11.0:1506921878.263356:0:3862:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:11.0:1506921878.263359:0:3862:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302848/t0(0) o19->oak-OST002a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.263366:0:3862:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:11.0:1506921878.263368:0:3862:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302848/t0(0) o19->oak-OST002a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:11.0:1506921878.263373:0:3862:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:11.0:1506921878.263375:0:3862:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:11.0:1506921878.263376:0:3862:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032face00 00000400:00000010:11.0:1506921878.263377:0:3862:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032face00. 00000400:00000010:11.0:1506921878.263380:0:3862:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451dc00 (tot 59423468). 00000800:00000200:11.0:1506921878.263383:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:11.0:1506921878.263387:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:2.2:1506921878.263470:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:1.0:1506921878.263510:0:3864:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921878.263519:0:3864:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.42@o2ib4 00000400:00000200:1.0:1506921878.263525:0:3864:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:1.0:1506921878.263527:0:3864:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881c4b6cd200 (tot 59423868). 00000400:00000200:1.0:1506921878.263533:0:3864:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c225980 00000400:00000200:1.0:1506921878.263537:0:3864:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a11fd [1] + 192 00000100:00000001:1.0:1506921878.263542:0:3864:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:1.0:1506921878.263545:0:3864:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302848/t0(0) o19->oak-OST002a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:13.0:1506921878.263549:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:1.0:1506921878.263552:0:3864:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302848/t0(0) o19->oak-OST002a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921878.263559:0:3864:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:1.0:1506921878.263561:0:3864:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881c4b6cd200 (tot 59423468). 00000800:00000200:1.0:1506921878.263563:0:3864:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921878.263565:0:3864:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:1.0:1506921878.263566:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921878.263591:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.263595:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.263596:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.263597:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.263601:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302848/t0(0) o19->oak-OST002a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.263609:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302848/t0(0) o19->oak-OST002a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.263614:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.263615:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.263617:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.263618:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.263626:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.263628:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.263629:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.263630:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.263631:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.263633:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.263636:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88203dd40c00 time=73 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921878.263638:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203dd40bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.263640:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.263641:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.263642:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.263643:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.263645:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.263646:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST002a-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.263647:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.263648:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.263650:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302848/t0(0) o19->oak-OST002a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.263655:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.263657:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.263658:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.263659:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302848/t0(0) o19->oak-OST002a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.263664:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302848:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.263667:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.263669:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.263670:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.263671:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.263672:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302848/t0(0) o19->oak-OST002a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.263676:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.263677:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.263679:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.263680:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.263681:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.263683:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302848/t0(0) o19->oak-OST002a-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.263687:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.263688:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.263690:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.263691:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.263692:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.263693:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203dd40800 refcount=4 obd=oak-OST002a-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.263694:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.263695:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.263697:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.263697:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.263698:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.263700:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.263700:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.263701:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.263703:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.263704:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.263705:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.263706:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.263707:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203dd42800 refcount=5 obd=oak-OST002b-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.263709:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.263710:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.263711:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.263713:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.263715:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.263728:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.263729:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.263730:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.263732:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.263733:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.263734:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.263736:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.263737:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.263745:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.263747:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.263749:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.263750:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.263751:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.263753:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302864/t0(0) o19->oak-OST002b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.263757:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.263758:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.263759:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.263760:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.263762:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302864:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.263764:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.263765:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.263766:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.263768:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.263769:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.263770:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.263772:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.263774:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.263776:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302864, portal 4 00000100:00000001:15.0:1506921878.263777:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.263778:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.263780:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302864/t0(0) o19->oak-OST002b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.263784:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.263786:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921878.263787:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fac500. 00000100:00000200:15.0:1506921878.263789:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302864, offset 0 00000400:00000010:15.0:1506921878.263791:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b04518200 (tot 59423868). 00000400:00000200:15.0:1506921878.263793:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921878.263802:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921878.263805:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921878.263810:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.263812:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921878.263815:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921878.263818:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.263820:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.263821:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.263822:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.263823:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.263824:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.263825:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.263826:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.263827:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.263829:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.263829:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921878.263855:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.263873:0:3858:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.263877:0:3858:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.263881:0:3858:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.263885:0:3858:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302864/t0(0) o19->oak-OST002b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.263893:0:3858:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921878.263895:0:3858:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302864/t0(0) o19->oak-OST002b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.263901:0:3858:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.263903:0:3858:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921878.263904:0:3858:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fac500 00000400:00000010:16.0:1506921878.263908:0:3858:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fac500. 00000800:00000200:18.0:1506921878.263910:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000400:00000010:16.0:1506921878.263911:0:3858:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b04518200 (tot 59423468). 00000800:00000200:16.0:1506921878.263918:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (23)-- 00000800:00000200:18.0:1506921878.263920:0:3861:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.41@o2ib4 00000800:00000200:16.0:1506921878.263920:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000400:00000200:18.0:1506921878.263926:0:3861:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:18.0:1506921878.263929:0:3861:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103a767000 (tot 59423868). 00000400:00000200:18.0:1506921878.263935:0:3861:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c225990 00000400:00000200:18.0:1506921878.263940:0:3861:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1db1 [1] + 192 00000100:00000001:18.0:1506921878.263946:0:3861:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:18.0:1506921878.263949:0:3861:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302864/t0(0) o19->oak-OST002b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000040:18.0:1506921878.263957:0:3861:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302864/t0(0) o19->oak-OST002b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:18.0:1506921878.263965:0:3861:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:18.0:1506921878.263967:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103a767000 (tot 59423468). 00000800:00000200:18.0:1506921878.263969:0:3861:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:18.0:1506921878.263972:0:3861:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.263972:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:18.0:1506921878.263973:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000400:00000001:15.0:1506921878.263973:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.263974:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.263975:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.263977:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302864/t0(0) o19->oak-OST002b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.263983:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302864/t0(0) o19->oak-OST002b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.263987:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.263988:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.263990:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.263991:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.263992:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.263993:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.263993:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.263994:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.263996:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.263997:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.263999:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88203dd42c00 time=73 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921878.264001:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203dd42bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.264002:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.264003:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.264005:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.264006:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.264007:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.264009:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST002b-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.264010:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.264011:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.264013:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302864/t0(0) o19->oak-OST002b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.264017:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.264018:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.264019:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.264021:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302864/t0(0) o19->oak-OST002b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.264025:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302864:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.264027:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.264029:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.264030:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.264031:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.264033:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302864/t0(0) o19->oak-OST002b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.264037:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.264038:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.264039:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.264040:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.264042:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.264043:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302864/t0(0) o19->oak-OST002b-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.264046:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.264047:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.264048:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.264049:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.264050:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.264051:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203dd42800 refcount=4 obd=oak-OST002b-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.264052:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.264053:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.264054:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.264055:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.264056:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.264058:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.264059:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.264060:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.264061:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.264062:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.264063:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.264064:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.264065:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203dd44800 refcount=5 obd=oak-OST002c-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.264066:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.264067:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.264068:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.264069:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.264071:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.264072:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.264073:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.264074:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.264075:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.264076:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.264077:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.264078:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.264080:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.264086:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.264088:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.264089:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.264090:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.264091:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.264092:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302880/t0(0) o19->oak-OST002c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.264096:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.264097:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.264098:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.264099:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.264101:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302880:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.264103:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.264104:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.264105:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.264106:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.264107:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.264108:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.264110:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.264111:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.264113:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302880, portal 4 00000100:00000001:15.0:1506921878.264114:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.264115:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.264117:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302880/t0(0) o19->oak-OST002c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.264121:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.264122:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921878.264124:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad800. 00000100:00000200:15.0:1506921878.264125:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302880, offset 0 00000400:00000010:15.0:1506921878.264126:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451ae00 (tot 59423868). 00000400:00000200:15.0:1506921878.264129:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921878.264135:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921878.264139:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921878.264142:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.264143:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921878.264145:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921878.264147:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.264149:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.264150:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.264151:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.264152:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.264153:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.264154:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.264155:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.264156:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.264157:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.264157:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.264186:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921878.264239:0:3865:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:5.0:1506921878.264246:0:3865:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:5.0:1506921878.264249:0:3865:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:5.0:1506921878.264253:0:3865:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302880/t0(0) o19->oak-OST002c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921878.264261:0:3865:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:5.0:1506921878.264263:0:3865:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302880/t0(0) o19->oak-OST002c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921878.264268:0:3865:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:5.0:1506921878.264269:0:3865:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:5.0:1506921878.264271:0:3865:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad800 00000400:00000010:5.0:1506921878.264272:0:3865:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad800. 00000400:00000010:5.0:1506921878.264275:0:3865:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451ae00 (tot 59423468). 00000800:00000200:5.0:1506921878.264278:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:11.0:1506921878.264280:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.2:1506921878.264347:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921878.264364:0:3860:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.264371:0:3860:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921878.264376:0:3860:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:16.0:1506921878.264378:0:3860:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921878.264384:0:3860:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c2259a0 00000400:00000200:16.0:1506921878.264389:0:3860:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a1215 [1] + 192 00000100:00000001:16.0:1506921878.264393:0:3860:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921878.264396:0:3860:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302880/t0(0) o19->oak-OST002c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921878.264401:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921878.264404:0:3860:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302880/t0(0) o19->oak-OST002c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.264413:0:3860:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921878.264415:0:3860:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921878.264417:0:3860:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.264420:0:3860:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921878.264422:0:3860:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:15.0:1506921878.264447:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.264453:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.264454:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.264455:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.264460:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302880/t0(0) o19->oak-OST002c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.264468:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302880/t0(0) o19->oak-OST002c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.264474:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.264476:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.264477:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.264479:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.264480:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.264482:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.264483:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.264484:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.264485:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.264487:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.264489:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88203dd44c00 time=73 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921878.264492:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203dd44bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.264493:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.264494:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.264495:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.264497:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.264498:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.264499:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST002c-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.264500:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.264501:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.264503:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302880/t0(0) o19->oak-OST002c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.264508:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.264510:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.264511:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.264512:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302880/t0(0) o19->oak-OST002c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.264517:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302880:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.264522:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.264524:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.264525:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.264526:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.264527:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302880/t0(0) o19->oak-OST002c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.264531:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.264532:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.264534:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.264535:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.264536:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.264538:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302880/t0(0) o19->oak-OST002c-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.264542:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.264542:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.264543:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.264545:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.264546:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.264547:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203dd44800 refcount=4 obd=oak-OST002c-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.264548:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.264549:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.264550:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.264551:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.264552:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.264553:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.264554:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.264555:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.264556:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.264557:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.264559:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.264560:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.264561:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203dd46800 refcount=5 obd=oak-OST002d-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.264563:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.264564:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.264565:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.264566:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.264569:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.264570:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.264571:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.264572:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.264573:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.264575:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.264576:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.264577:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.264578:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.264586:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.264589:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.264590:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.264591:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.264592:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.264594:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302896/t0(0) o19->oak-OST002d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.264598:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.264599:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.264600:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.264601:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.264603:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302896:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.264605:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.264606:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.264607:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.264608:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.264609:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.264610:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.264612:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.264614:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.264616:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302896, portal 4 00000100:00000001:15.0:1506921878.264617:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.264618:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.264621:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302896/t0(0) o19->oak-OST002d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.264629:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.264631:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921878.264632:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fadb00. 00000100:00000200:15.0:1506921878.264634:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302896, offset 0 00000400:00000010:15.0:1506921878.264636:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451a400 (tot 59423868). 00000400:00000200:15.0:1506921878.264638:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921878.264648:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921878.264652:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921878.264656:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.264659:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921878.264660:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921878.264664:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.264665:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.264666:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.264667:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.264669:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.264669:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.264671:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.264672:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.264673:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.264674:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.264675:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921878.264704:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.264740:0:3859:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.264745:0:3859:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.264749:0:3859:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.264753:0:3859:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302896/t0(0) o19->oak-OST002d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.264761:0:3859:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921878.264763:0:3859:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302896/t0(0) o19->oak-OST002d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.264769:0:3859:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.264771:0:3859:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921878.264772:0:3859:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fadb00 00000400:00000010:16.0:1506921878.264773:0:3859:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fadb00. 00000400:00000010:16.0:1506921878.264777:0:3859:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451a400 (tot 59423468). 00000800:00000200:18.0:1506921878.264778:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921878.264782:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921878.264792:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:13.0:1506921878.264835:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921878.264844:0:3863:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.42@o2ib4 00000400:00000200:13.0:1506921878.264850:0:3863:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:13.0:1506921878.264853:0:3863:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88203bd18600 (tot 59423868). 00000400:00000200:13.0:1506921878.264858:0:3863:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c2259b0 00000400:00000200:13.0:1506921878.264863:0:3863:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1db9 [1] + 192 00000100:00000001:13.0:1506921878.264868:0:3863:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:13.0:1506921878.264871:0:3863:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302896/t0(0) o19->oak-OST002d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:1.0:1506921878.264875:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:13.0:1506921878.264878:0:3863:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302896/t0(0) o19->oak-OST002d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921878.264886:0:3863:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:13.0:1506921878.264888:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88203bd18600 (tot 59423468). 00000100:00000001:15.0:1506921878.264891:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000800:00000200:13.0:1506921878.264891:0:3863:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000400:00000001:15.0:1506921878.264893:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.264893:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000800:00000200:13.0:1506921878.264893:0:3863:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.264894:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000800:00000200:13.0:1506921878.264894:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000200:15.0:1506921878.264896:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302896/t0(0) o19->oak-OST002d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.264901:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302896/t0(0) o19->oak-OST002d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.264905:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.264906:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.264907:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.264908:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.264910:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.264911:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.264912:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.264913:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.264914:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.264915:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.264917:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88203dd46c00 time=73 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921878.264919:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203dd46bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.264921:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.264921:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.264922:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.264923:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.264925:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.264925:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST002d-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.264927:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.264927:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.264929:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302896/t0(0) o19->oak-OST002d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.264933:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.264934:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.264935:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.264937:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302896/t0(0) o19->oak-OST002d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.264941:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302896:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.264943:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.264945:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.264946:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.264947:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.264948:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302896/t0(0) o19->oak-OST002d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.264952:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.264954:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.264955:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.264956:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.264957:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.264959:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302896/t0(0) o19->oak-OST002d-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.264962:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.264963:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.264964:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.264965:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.264966:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.264967:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203dd46800 refcount=4 obd=oak-OST002d-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.264968:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.264969:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.264970:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.264971:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.264971:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.264973:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.264974:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.264975:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.264976:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.264977:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.264978:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.264978:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.264980:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203ddf0800 refcount=5 obd=oak-OST002e-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.264981:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.264982:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.264982:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.264984:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.264985:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.264986:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.264987:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.264988:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.264989:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.264990:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.264991:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.264992:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.264993:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.264999:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.265001:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.265003:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.265004:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.265005:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.265006:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302912/t0(0) o19->oak-OST002e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.265010:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.265011:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.265012:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.265013:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.265014:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302912:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.265016:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.265017:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.265017:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.265018:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.265019:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.265020:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.265022:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.265023:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.265025:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302912, portal 4 00000100:00000001:15.0:1506921878.265026:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.265027:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.265029:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302912/t0(0) o19->oak-OST002e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.265032:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.265034:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.101@o2ib5 00000400:00000010:15.0:1506921878.265035:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad200. 00000100:00000200:15.0:1506921878.265036:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302912, offset 0 00000400:00000010:15.0:1506921878.265037:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451ee00 (tot 59423868). 00000400:00000200:15.0:1506921878.265040:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.101@o2ib5 00000400:00000200:15.0:1506921878.265046:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.101@o2ib5(10.0.2.101@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921878.265049:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921878.265052:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.265054:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921878.265056:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921878.265058:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.265059:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.265060:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.265061:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.265062:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.265063:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.265064:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.265066:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.265066:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.265067:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.265068:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.265097:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921878.265140:0:3865:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:5.0:1506921878.265146:0:3865:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:5.0:1506921878.265161:0:3865:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:5.0:1506921878.265165:0:3865:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302912/t0(0) o19->oak-OST002e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921878.265172:0:3865:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:5.0:1506921878.265175:0:3865:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302912/t0(0) o19->oak-OST002e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921878.265180:0:3865:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:5.0:1506921878.265181:0:3865:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:5.0:1506921878.265183:0:3865:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad200 00000400:00000010:5.0:1506921878.265184:0:3865:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad200. 00000400:00000010:5.0:1506921878.265187:0:3865:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451ee00 (tot 59423468). 00000800:00000200:5.0:1506921878.265190:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:11.0:1506921878.265193:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:2.2:1506921878.265267:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:1.0:1506921878.265307:0:3864:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921878.265316:0:3864:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[1] from 10.9.0.42@o2ib4 00000400:00000200:1.0:1506921878.265322:0:3864:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.101@o2ib5 : PUT 00000400:00000010:1.0:1506921878.265326:0:3864:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881c4b6cd200 (tot 59423868). 00000400:00000200:1.0:1506921878.265332:0:3864:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.101@o2ib5 of length 304 into portal 4 MB=0x59d044c2259c0 00000400:00000200:1.0:1506921878.265337:0:3864:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.101@o2ib5 of length 304/304 into md 0x7a122d [1] + 192 00000100:00000001:1.0:1506921878.265341:0:3864:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:1.0:1506921878.265344:0:3864:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302912/t0(0) o19->oak-OST002e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:13.0:1506921878.265348:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:1.0:1506921878.265352:0:3864:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302912/t0(0) o19->oak-OST002e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:1.0:1506921878.265359:0:3864:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:1.0:1506921878.265360:0:3864:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881c4b6cd200 (tot 59423468). 00000800:00000200:1.0:1506921878.265363:0:3864:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:1.0:1506921878.265365:0:3864:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:1.0:1506921878.265366:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921878.265391:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.265395:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.265396:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.265397:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.265401:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302912/t0(0) o19->oak-OST002e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.265409:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302912/t0(0) o19->oak-OST002e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.265414:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.265416:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.265417:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.265419:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.265420:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.265421:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.265422:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.265423:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.265425:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.265426:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.265429:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88203ddf0c00 time=73 v=31 (31 31 1 31) 00000100:00001000:15.0:1506921878.265431:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203ddf0bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.265433:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.265434:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.265435:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.265437:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.265438:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.265441:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST002e-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.265442:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.265443:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.265445:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302912/t0(0) o19->oak-OST002e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.265450:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.265451:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.265452:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.265454:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302912/t0(0) o19->oak-OST002e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.265459:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302912:10.0.2.101@o2ib5:19 00000100:00000001:15.0:1506921878.265462:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.265463:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.265464:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.265465:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.265467:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302912/t0(0) o19->oak-OST002e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.265470:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.265472:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.265473:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.265474:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.265476:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.265477:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302912/t0(0) o19->oak-OST002e-osc-ffff88203bc47000@10.0.2.101@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.265481:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.265481:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.265482:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.265484:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.265485:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.265486:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203ddf0800 refcount=4 obd=oak-OST002e-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.265487:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.265488:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.265489:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.265490:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.265491:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.265493:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.265493:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.265494:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.265496:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.265497:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000001:00000001:15.0:1506921878.265498:0:38882:0:(osc_quota.c:274:osc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.265499:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.265500:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203ddf2800 refcount=5 obd=oak-OST002f-osc-ffff88203bc47000 00000100:00000001:15.0:1506921878.265502:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.265503:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.265504:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.265506:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.265508:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.265509:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.265510:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.265511:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.265512:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.265514:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.265515:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.265516:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.265518:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.265525:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.265528:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.265529:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.265530:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.265531:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.265533:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302928/t0(0) o19->oak-OST002f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.265537:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.265538:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.265539:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.265540:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.265542:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302928:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.265544:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.265545:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.265546:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.265547:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.265549:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.265550:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.265551:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.265553:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.265555:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302928, portal 4 00000100:00000001:15.0:1506921878.265557:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.265557:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.265560:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302928/t0(0) o19->oak-OST002f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.265564:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.265566:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.102@o2ib5 00000400:00000010:15.0:1506921878.265567:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fac880. 00000100:00000200:15.0:1506921878.265568:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 28, xid 1580016666302928, offset 0 00000400:00000010:15.0:1506921878.265570:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b0451f400 (tot 59423868). 00000400:00000200:15.0:1506921878.265573:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.102@o2ib5 00000400:00000200:15.0:1506921878.265582:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.102@o2ib5(10.0.2.102@o2ib5:10.9.0.41@o2ib4) : PUT 00000800:00000200:15.0:1506921878.265585:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.41@o2ib4 00000800:00000200:15.0:1506921878.265590:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88101de4df80] -> 10.9.0.41@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.265592:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:15.0:1506921878.265594:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:15.0:1506921878.265597:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88103dc5e400] (22)-- 00000100:00000001:15.0:1506921878.265598:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.265599:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.265601:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.265602:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.265603:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=43 : 43 : 2b) 00000100:00100000:15.0:1506921878.265604:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 43 seconds 00000100:00000001:15.0:1506921878.265605:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.265606:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.265607:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.265608:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:16.2:1506921878.265622:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.265631:0:3858:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)++ 00000800:00000200:16.0:1506921878.265636:0:3858:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88103dc5e400] (23)-- 00000100:00000001:16.0:1506921878.265639:0:3858:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:16.0:1506921878.265642:0:3858:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302928/t0(0) o19->oak-OST002f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.265648:0:3858:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:16.0:1506921878.265650:0:3858:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302928/t0(0) o19->oak-OST002f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.265654:0:3858:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.265655:0:3858:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:16.0:1506921878.265656:0:3858:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fac880 00000400:00000010:16.0:1506921878.265658:0:3858:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fac880. 00000400:00000010:16.0:1506921878.265661:0:3858:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b0451f400 (tot 59423468). 00000800:00000200:16.0:1506921878.265663:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921878.265665:0:3858:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000800:00000200:2.2:1506921878.265754:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (20)++ 00000800:00000200:5.0:1506921878.265794:0:3865:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921878.265802:0:3865:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[0] from 10.9.0.42@o2ib4 00000400:00000200:5.0:1506921878.265808:0:3865:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.102@o2ib5 : PUT 00000400:00000010:5.0:1506921878.265810:0:3865:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881e48bc2800 (tot 59423868). 00000400:00000200:5.0:1506921878.265816:0:3865:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.102@o2ib5 of length 304 into portal 4 MB=0x59d044c2259d0 00000400:00000200:5.0:1506921878.265823:0:3865:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.0.2.102@o2ib5 of length 304/304 into md 0x2c1dc1 [1] + 192 00000100:00000001:5.0:1506921878.265828:0:3865:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:5.0:1506921878.265831:0:3865:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302928/t0(0) o19->oak-OST002f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:11.0:1506921878.265834:0:3862:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000100:00000040:5.0:1506921878.265839:0:3865:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302928/t0(0) o19->oak-OST002f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:5.0:1506921878.265846:0:3865:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:5.0:1506921878.265848:0:3865:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881e48bc2800 (tot 59423468). 00000800:00000200:5.0:1506921878.265851:0:3865:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88203af8e800] (21)++ 00000800:00000200:5.0:1506921878.265853:0:3865:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88203af8e800] (22)-- 00000800:00000200:5.0:1506921878.265855:0:3865:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000100:00000001:15.0:1506921878.265883:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.265889:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.265890:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.265892:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.265896:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302928/t0(0) o19->oak-OST002f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.265904:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302928/t0(0) o19->oak-OST002f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/336 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.265910:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.265911:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.265913:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.265915:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.265916:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.265918:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.265919:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.265920:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.265921:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.265923:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.265926:0:38882:0:(import.c:1678:at_measured()) add 31 to ffff88203ddf2c00 time=73 v=31 (31 31 31 31) 00000100:00001000:15.0:1506921878.265928:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203ddf2bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.265930:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.265931:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.265932:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.265934:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.265935:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.265936:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-OST002f-osc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.265937:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.265938:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.265940:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302928/t0(0) o19->oak-OST002f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.265946:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.265948:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.265949:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.265951:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302928/t0(0) o19->oak-OST002f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.265956:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302928:10.0.2.102@o2ib5:19 00000100:00000001:15.0:1506921878.265961:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.265963:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.265964:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.265966:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.265967:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302928/t0(0) o19->oak-OST002f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.265972:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.265973:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.265975:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.265975:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.265977:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.265979:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302928/t0(0) o19->oak-OST002f-osc-ffff88203bc47000@10.0.2.102@o2ib5:28/4 lens 336/304 e 0 to 0 dl 1506921922 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.265983:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.265984:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.265985:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.265987:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.265987:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.265988:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203ddf2800 refcount=4 obd=oak-OST002f-osc-ffff88203bc47000 00000020:00000001:15.0:1506921878.265990:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.265991:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.265992:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.265993:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.265994:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.265996:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.265996:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:15.0:1506921878.265998:0:38882:0:(osc_quota.c:302:osc_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.265999:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:15.0:1506921878.266000:0:38882:0:(obd_class.h:986:obd_putref()) Process entered 00020000:00000001:15.0:1506921878.266002:0:38882:0:(obd_class.h:991:obd_putref()) Process leaving 00020000:00000001:15.0:1506921878.266002:0:38882:0:(lov_obd.c:1396:lov_quotactl()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:15.0:1506921878.266004:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:15.0:1506921878.266005:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00800000:00000001:15.0:1506921878.266007:0:38882:0:(lmv_obd.c:3038:lmv_quotactl()) Process entered 00800000:00000001:15.0:1506921878.266007:0:38882:0:(obd_class.h:1230:obd_quotactl()) Process entered 00000002:00000001:15.0:1506921878.266009:0:38882:0:(mdc_request.c:1808:mdc_quotactl()) Process entered 00000100:00000010:15.0:1506921878.266010:0:38882:0:(client.c:493:ptlrpc_request_cache_alloc()) slab-alloced 'req': 760 at ffff881afbb89500. 00000020:00000040:15.0:1506921878.266012:0:38882:0:(genops.c:1012:class_import_get()) import ffff88203bc57800 refcount=4 obd=oak-MDT0000-mdc-ffff88203bc47000 00000100:00000001:15.0:1506921878.266014:0:38882:0:(client.c:700:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:15.0:1506921878.266015:0:38882:0:(sec.c:434:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:15.0:1506921878.266017:0:38882:0:(sec.c:452:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:15.0:1506921878.266019:0:38882:0:(sec_null.c:161:null_alloc_reqbuf()) kmalloced 'req->rq_reqbuf': 512 at ffff881b0451a000. 00000100:00000001:15.0:1506921878.266021:0:38882:0:(client.c:774:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.266023:0:38882:0:(client.c:2882:ptlrpc_queue_wait()) Process entered 00000100:00000001:15.0:1506921878.266024:0:38882:0:(client.c:947:ptlrpc_prep_set()) Process entered 00000100:00000010:15.0:1506921878.266025:0:38882:0:(client.c:949:ptlrpc_prep_set()) kmalloced 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.266026:0:38882:0:(client.c:965:ptlrpc_prep_set()) Process leaving (rc=18446612204945489024 : -131868764062592 : ffff8810e9293c80) 00000100:00000001:15.0:1506921878.266028:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.266029:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000020:00000001:15.0:1506921878.266031:0:38882:0:(class_obd.c:128:lustre_get_jobid()) Process entered 00000400:00000001:15.0:1506921878.266032:0:38882:0:(linux-curproc.c:197:cfs_get_environ()) Process entered 00000400:00000001:15.0:1506921878.266041:0:38882:0:(linux-curproc.c:274:cfs_get_environ()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00000020:00000040:15.0:1506921878.266043:0:38882:0:(class_obd.c:169:lustre_get_jobid()) Get jobid for (SLURM_JOB_ID) failed: rc = -2 00000020:00000001:15.0:1506921878.266044:0:38882:0:(class_obd.c:175:lustre_get_jobid()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:15.0:1506921878.266046:0:38882:0:(client.c:2313:ptlrpc_set_wait()) Process entered 00000100:00000001:15.0:1506921878.266047:0:38882:0:(client.c:1530:ptlrpc_send_new_req()) Process entered 00000100:00000040:15.0:1506921878.266049:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "New" -> "Rpc" req@ffff881afbb89500 x1580016666302944/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 0 ref 2 fl New:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.266055:0:38882:0:(client.c:1153:ptlrpc_import_delay_req()) Process entered 00000100:00000001:15.0:1506921878.266055:0:38882:0:(client.c:1204:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.266057:0:38882:0:(sec.c:685:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:15.0:1506921878.266058:0:38882:0:(sec.c:712:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:15.0:1506921878.266060:0:38882:0:(client.c:1620:ptlrpc_send_new_req()) Sending RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302944:10.0.2.52@o2ib5:48 00000100:00000001:15.0:1506921878.266062:0:38882:0:(niobuf.c:699:ptl_send_rpc()) Process entered 02000000:00000001:15.0:1506921878.266063:0:38882:0:(sec.c:986:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:15.0:1506921878.266064:0:38882:0:(sec.c:1022:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.266066:0:38882:0:(sec.c:1692:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:15.0:1506921878.266067:0:38882:0:(sec_null.c:204:null_alloc_repbuf()) kmalloced 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.266068:0:38882:0:(sec.c:1702:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:15.0:1506921878.266070:0:38882:0:(lib-lnet.h:268:lnet_me_alloc()) slab-alloced 'me' at ffff88203c6e4b40. 00000400:00000010:15.0:1506921878.266072:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad180. 00000100:00000200:15.0:1506921878.266074:0:38882:0:(niobuf.c:875:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1580016666302944, portal 10 00000100:00000001:15.0:1506921878.266076:0:38882:0:(client.c:2808:ptlrpc_request_addref()) Process entered 00000100:00000001:15.0:1506921878.266077:0:38882:0:(client.c:2810:ptlrpc_request_addref()) Process leaving (rc=18446612248206546176 : -131825503005440 : ffff881afbb89500) 00000100:00000040:15.0:1506921878.266080:0:38882:0:(niobuf.c:896:ptl_send_rpc()) @@@ send flg=0 req@ffff881afbb89500 x1580016666302944/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921885 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.266085:0:38882:0:(niobuf.c:54:ptl_send_buf()) Process entered 00000100:00000040:15.0:1506921878.266087:0:38882:0:(niobuf.c:57:ptl_send_buf()) peer_id 12345-10.0.2.52@o2ib5 00000400:00000010:15.0:1506921878.266088:0:38882:0:(lib-lnet.h:222:lnet_md_alloc()) slab-alloced 'md' of size 128 at ffff881032fad280. 00000100:00000200:15.0:1506921878.266089:0:38882:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 12, xid 1580016666302944, offset 0 00000400:00000010:15.0:1506921878.266092:0:38882:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff881b04519c00 (tot 59423868). 00000400:00000200:15.0:1506921878.266095:0:38882:0:(lib-move.c:2781:LNetPut()) LNetPut -> 12345-10.0.2.52@o2ib5 00000400:00000200:15.0:1506921878.266103:0:38882:0:(lib-move.c:1915:lnet_select_pathway()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4:) -> 10.0.2.52@o2ib5(10.0.2.52@o2ib5:10.9.0.42@o2ib4) : PUT 00000800:00000200:15.0:1506921878.266107:0:38882:0:(o2iblnd_cb.c:1526:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.9.0.42@o2ib4 00000800:00000200:15.0:1506921878.266111:0:38882:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff88203bf7df80] -> 10.9.0.42@o2ib4 (2) version: 12 00000800:00000200:15.0:1506921878.266113:0:38882:0:(o2iblnd_cb.c:1407:kiblnd_launch_tx()) conn[ffff88203af8e800] (20)++ 00000800:00000200:15.0:1506921878.266115:0:38882:0:(o2iblnd_cb.c:1177:kiblnd_queue_tx_locked()) conn[ffff88203af8e800] (21)++ 00000800:00000200:15.0:1506921878.266119:0:38882:0:(o2iblnd_cb.c:1413:kiblnd_launch_tx()) conn[ffff88203af8e800] (22)-- 00000100:00000001:15.0:1506921878.266120:0:38882:0:(niobuf.c:100:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.266121:0:38882:0:(niobuf.c:904:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.266122:0:38882:0:(client.c:1640:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.266124:0:38882:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921878.266124:0:38882:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=6 : 6 : 6) 00000100:00100000:15.0:1506921878.266125:0:38882:0:(client.c:2334:ptlrpc_set_wait()) set ffff8810e9293c80 going to sleep for 6 seconds 00000100:00000001:15.0:1506921878.266127:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.266128:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.266129:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000100:00000001:15.0:1506921878.266130:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.2:1506921878.266150:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88203af8e800] (21)++ 00000800:00000200:13.0:1506921878.266190:0:3863:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88203af8e800] (22)++ 00000800:00000200:13.0:1506921878.266197:0:3863:0:(o2iblnd_cb.c:74:kiblnd_tx_done()) conn[ffff88203af8e800] (23)-- 00000100:00000001:13.0:1506921878.266200:0:3863:0:(events.c:52:request_out_callback()) Process entered 00000100:00000200:13.0:1506921878.266203:0:3863:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff881afbb89500 x1580016666302944/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921885 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921878.266211:0:3863:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:13.0:1506921878.266213:0:3863:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 2 req@ffff881afbb89500 x1580016666302944/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921885 ref 3 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000001:13.0:1506921878.266218:0:3863:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:13.0:1506921878.266220:0:3863:0:(events.c:81:request_out_callback()) Process leaving 00000400:00000200:13.0:1506921878.266221:0:3863:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad280 00000400:00000010:13.0:1506921878.266222:0:3863:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad280. 00000400:00000010:13.0:1506921878.266236:0:3863:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff881b04519c00 (tot 59423468). 00000800:00000200:1.0:1506921878.266236:0:3864:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (22)-- 00000800:00000200:13.0:1506921878.266240:0:3863:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88203af8e800] (21)-- 00000800:00000200:16.2:1506921878.266548:0:0:0:(o2iblnd_cb.c:3531:kiblnd_cq_completion()) conn[ffff88103dc5e400] (20)++ 00000800:00000200:16.0:1506921878.266557:0:3861:0:(o2iblnd_cb.c:3653:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.266561:0:3861:0:(o2iblnd_cb.c:326:kiblnd_handle_rx()) Received d1[2] from 10.9.0.41@o2ib4 00000400:00000200:16.0:1506921878.266566:0:3861:0:(lib-move.c:2397:lnet_parse()) TRACE: 10.9.101.59@o2ib4(10.9.101.59@o2ib4) <- 10.0.2.52@o2ib5 : PUT 00000400:00000010:16.0:1506921878.266569:0:3861:0:(lib-lnet.h:444:lnet_msg_alloc()) alloc '(msg)': 400 at ffff88103c277600 (tot 59423868). 00000400:00000200:16.0:1506921878.266574:0:3861:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.2.52@o2ib5 of length 304 into portal 10 MB=0x59d044c2259e0 00000400:00000200:16.0:1506921878.266579:0:3861:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index a from 12345-10.0.2.52@o2ib5 of length 304/304 into md 0x7a1245 [1] + 192 00000100:00000001:16.0:1506921878.266584:0:3861:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:16.0:1506921878.266587:0:3861:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff881afbb89500 x1580016666302944/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921885 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1506921878.266592:0:3859:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (22)-- 00000100:00000040:16.0:1506921878.266592:0:3861:0:(events.c:161:reply_in_callback()) @@@ reply in flags=0 mlen=304 offset=192 replen=336 req@ffff881afbb89500 x1580016666302944/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921885 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:16.0:1506921878.266600:0:3861:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:16.0:1506921878.266601:0:3861:0:(lib-lnet.h:454:lnet_msg_free()) kfreed 'msg': 400 at ffff88103c277600 (tot 59423468). 00000800:00000200:16.0:1506921878.266604:0:3861:0:(o2iblnd_cb.c:194:kiblnd_post_rx()) conn[ffff88103dc5e400] (21)++ 00000800:00000200:16.0:1506921878.266606:0:3861:0:(o2iblnd_cb.c:223:kiblnd_post_rx()) conn[ffff88103dc5e400] (22)-- 00000800:00000200:16.0:1506921878.266607:0:3861:0:(o2iblnd_cb.c:3669:kiblnd_scheduler()) conn[ffff88103dc5e400] (21)-- 00000100:00000001:15.0:1506921878.266638:0:38882:0:(client.c:1680:ptlrpc_check_set()) Process entered 00000400:00000001:15.0:1506921878.266642:0:38882:0:(lib-msg.c:44:lnet_build_unlink_event()) Process entered 00000400:00000001:15.0:1506921878.266643:0:38882:0:(lib-msg.c:53:lnet_build_unlink_event()) Process leaving 00000100:00000001:15.0:1506921878.266645:0:38882:0:(events.c:91:reply_in_callback()) Process entered 00000100:00000200:15.0:1506921878.266649:0:38882:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff881afbb89500 x1580016666302944/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921885 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:15.0:1506921878.266657:0:38882:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff881afbb89500 x1580016666302944/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/336 e 0 to 0 dl 1506921885 ref 2 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000001:15.0:1506921878.266663:0:38882:0:(events.c:171:reply_in_callback()) Process leaving 00000400:00000010:15.0:1506921878.266667:0:38882:0:(lib-lnet.h:278:lnet_me_free()) slab-freed 'me' at ffff88203c6e4b40. 00000400:00000200:15.0:1506921878.266668:0:38882:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff881032fad180 00000400:00000010:15.0:1506921878.266669:0:38882:0:(lib-lnet.h:253:lnet_md_free()) slab-freed 'md' at ffff881032fad180. 00000100:00000001:15.0:1506921878.266670:0:38882:0:(client.c:2587:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.266672:0:38882:0:(client.c:1337:after_reply()) Process entered 02000000:00000001:15.0:1506921878.266673:0:38882:0:(sec.c:1029:do_cli_unwrap_reply()) Process entered 00000100:00000001:15.0:1506921878.266674:0:38882:0:(pack_generic.c:577:__lustre_unpack_msg()) Process entered 00000100:00000001:15.0:1506921878.266676:0:38882:0:(pack_generic.c:596:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:15.0:1506921878.266677:0:38882:0:(sec.c:1083:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:15.0:1506921878.266680:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203bc57c00 time=73 v=1 (1 1 1 1) 00000100:00001000:15.0:1506921878.266682:0:38882:0:(import.c:1678:at_measured()) add 1 to ffff88203bc57bc8 time=73 v=1 (1 1 1 1) 00000100:00000001:15.0:1506921878.266684:0:38882:0:(client.c:1251:ptlrpc_check_status()) Process entered 00000100:00000001:15.0:1506921878.266685:0:38882:0:(client.c:1277:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:15.0:1506921878.266686:0:38882:0:(ldlm_request.c:1308:ldlm_cli_update_pool()) Process entered 00010000:00000001:15.0:1506921878.266687:0:38882:0:(ldlm_request.c:1346:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.266689:0:38882:0:(client.c:2675:ptlrpc_free_committed()) Process entered 00000100:00000040:15.0:1506921878.266690:0:38882:0:(client.c:2683:ptlrpc_free_committed()) oak-MDT0000-mdc-ffff88203bc47000: skip recheck: last_committed 0 00000100:00000001:15.0:1506921878.266691:0:38882:0:(client.c:2684:ptlrpc_free_committed()) Process leaving 00000100:00000001:15.0:1506921878.266691:0:38882:0:(client.c:1517:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:15.0:1506921878.266694:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Rpc" -> "Interpret" req@ffff881afbb89500 x1580016666302944/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/304 e 0 to 0 dl 1506921885 ref 2 fl Rpc:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.266699:0:38882:0:(client.c:1990:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:15.0:1506921878.266701:0:38882:0:(niobuf.c:436:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:15.0:1506921878.266701:0:38882:0:(niobuf.c:446:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:15.0:1506921878.266703:0:38882:0:(lustre_net.h:2455:ptlrpc_rqphase_move()) @@@ move req "Interpret" -> "Complete" req@ffff881afbb89500 x1580016666302944/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/304 e 0 to 0 dl 1506921885 ref 2 fl Interpret:RN/0/0 rc 0/0 00000100:00100000:15.0:1506921878.266708:0:38882:0:(client.c:2045:ptlrpc_check_set()) Completed RPC pname:cluuid:pid:xid:nid:opc lfs:6b653517-b709-6c81-dbad-9c8fcf35bffa:38882:1580016666302944:10.0.2.52@o2ib5:48 00000100:00000001:15.0:1506921878.266711:0:38882:0:(client.c:2089:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921878.266712:0:38882:0:(client.c:2424:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.266714:0:38882:0:(client.c:1008:ptlrpc_set_destroy()) Process entered 00000100:00000001:15.0:1506921878.266715:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.266727:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 1 req@ffff881afbb89500 x1580016666302944/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/304 e 0 to 0 dl 1506921885 ref 2 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.266731:0:38882:0:(client.c:2531:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:15.0:1506921878.266732:0:38882:0:(ptlrpc_internal.h:352:ptlrpc_reqset_put()) kfreed 'set': 152 at ffff8810e9293c80. 00000100:00000001:15.0:1506921878.266735:0:38882:0:(client.c:1050:ptlrpc_set_destroy()) Process leaving 00000100:00000001:15.0:1506921878.266735:0:38882:0:(client.c:2902:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921878.266737:0:38882:0:(client.c:2512:__ptlrpc_req_finished()) Process entered 00000100:00000040:15.0:1506921878.266739:0:38882:0:(client.c:2524:__ptlrpc_req_finished()) @@@ refcount now 0 req@ffff881afbb89500 x1580016666302944/t0(0) o48->oak-MDT0000-mdc-ffff88203bc47000@10.0.2.52@o2ib5:12/10 lens 336/304 e 0 to 0 dl 1506921885 ref 1 fl Complete:RN/0/0 rc 0/0 00000100:00000001:15.0:1506921878.266742:0:38882:0:(client.c:2438:__ptlrpc_free_req()) Process entered 02000000:00000001:15.0:1506921878.266743:0:38882:0:(sec.c:1713:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:15.0:1506921878.266744:0:38882:0:(sec_null.c:218:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8820347cbc00. 02000000:00000001:15.0:1506921878.266746:0:38882:0:(sec.c:1727:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:15.0:1506921878.266747:0:38882:0:(genops.c:1019:class_import_put()) Process entered 00000020:00000040:15.0:1506921878.266748:0:38882:0:(genops.c:1026:class_import_put()) import ffff88203bc57800 refcount=3 obd=oak-MDT0000-mdc-ffff88203bc47000 00000020:00000001:15.0:1506921878.266749:0:38882:0:(genops.c:1035:class_import_put()) Process leaving 02000000:00000010:15.0:1506921878.266750:0:38882:0:(sec_null.c:188:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 512 at ffff881b0451a000. 02000000:00000001:15.0:1506921878.266751:0:38882:0:(sec.c:466:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:15.0:1506921878.266752:0:38882:0:(sec.c:482:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:15.0:1506921878.266753:0:38882:0:(client.c:499:ptlrpc_request_cache_free()) slab-freed '(req)': 760 at ffff881afbb89500. 00000100:00000001:15.0:1506921878.266754:0:38882:0:(client.c:2490:__ptlrpc_free_req()) Process leaving 00000100:00000001:15.0:1506921878.266755:0:38882:0:(client.c:2528:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:15.0:1506921878.266756:0:38882:0:(mdc_request.c:1836:mdc_quotactl()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:15.0:1506921878.266757:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:15.0:1506921878.266759:0:38882:0:(lmv_obd.c:3074:lmv_quotactl()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:15.0:1506921878.266760:0:38882:0:(obd_class.h:1236:obd_quotactl()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:15.0:1506921878.266761:0:38882:0:(dir.c:1115:quotactl_ioctl()) kfreed 'oqctl_tmp': 112 at ffff881032735b00. 00000080:00000010:15.0:1506921878.266763:0:38882:0:(dir.c:1119:quotactl_ioctl()) kfreed 'oqctl': 112 at ffff881032734100. 00000080:00000001:15.0:1506921878.266764:0:38882:0:(dir.c:1122:quotactl_ioctl()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:15.0:1506921878.266765:0:38882:0:(dir.c:1543:ll_dir_ioctl()) kfreed 'qctl': 176 at ffff8820382d7380. 00000080:00000001:15.0:1506921878.266766:0:38882:0:(dir.c:1544:ll_dir_ioctl()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:15.0:1506921878.266772:0:38882:0:(dir.c:1831:ll_dir_release()) Process entered 00000080:00000001:15.0:1506921878.266773:0:38882:0:(file.c:316:ll_file_release()) Process entered 00000080:00200000:15.0:1506921878.266774:0:38882:0:(file.c:319:ll_file_release()) VFS Op:inode=[0x200000007:0x1:0x0](ffff881021f03690) 00000080:00400000:15.0:1506921878.266776:0:38882:0:(statahead.c:1237:ll_deauthorize_statahead()) deauthorize statahead for [0x200000007:0x1:0x0] 00000080:00000010:15.0:1506921878.266778:0:38882:0:(file.c:83:ll_file_data_put()) slab-freed '(fd)': 216 at ffff882035b5d800. 00000080:00000001:15.0:1506921878.266779:0:38882:0:(file.c:334:ll_file_release()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:15.0:1506921878.266780:0:38882:0:(dir.c:1832:ll_dir_release()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1506921878.584774:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.31@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921878.584779:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.32@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921878.584782:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.41@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921878.584785:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.42@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000100:00000001:4.0:1506921878.611780:0:3903:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:6.0:1506921878.611781:0:3911:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921878.611782:0:3905:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:4.0:1506921878.611782:0:3903:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921878.611783:0:3902:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:6.0:1506921878.611783:0:3911:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921878.611784:0:3905:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1506921878.611784:0:3910:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:4.0:1506921878.611785:0:3903:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921878.611786:0:3902:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:6.0:1506921878.611786:0:3911:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:4.0:1506921878.611786:0:3903:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921878.611786:0:3910:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921878.611787:0:3905:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:6.0:1506921878.611787:0:3911:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921878.611788:0:3905:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921878.611788:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921878.611789:0:3902:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:6.0:1506921878.611789:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921878.611789:0:3910:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921878.611790:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921878.611790:0:3902:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921878.611790:0:3910:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921878.611791:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921878.611792:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921878.611792:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921878.611793:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921878.611794:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921878.611794:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921878.611795:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921878.611795:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921878.611796:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921878.611796:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921878.611797:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.611802:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.611804:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921878.611805:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921878.611807:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921878.611809:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921878.611810:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921878.612772:0:3909:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921878.612774:0:3909:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921878.612775:0:3907:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921878.612777:0:3909:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921878.612777:0:3907:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921878.612778:0:3909:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921878.612780:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921878.612780:0:3907:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921878.612781:0:3907:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921878.612782:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.612783:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921878.612783:0:3906:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921878.612784:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921878.612784:0:3906:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921878.612785:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.612786:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.612787:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921878.612787:0:3906:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:8.0:1506921878.612788:0:3906:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.612789:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921878.612790:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921878.612793:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921878.612794:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921878.612795:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921878.613754:0:3904:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:14.0:1506921878.613755:0:3904:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:14.0:1506921878.613758:0:3904:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:14.0:1506921878.613759:0:3904:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921878.613761:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921878.613762:0:3908:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:14.0:1506921878.613763:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.613764:0:3908:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:14.0:1506921878.613765:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921878.613766:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.613767:0:3908:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921878.613768:0:3908:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.613770:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921878.613773:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921878.613774:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921878.613776:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921879.035771:0:3901:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1506921879.035773:0:3901:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1506921879.035776:0:3901:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921879.035776:0:3901:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921879.035778:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921879.035780:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921879.035782:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921879.035783:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921879.040772:0:3918:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1506921879.040774:0:3918:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921879.040777:0:3918:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921879.040778:0:3918:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921879.040779:0:3912:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1506921879.040779:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921879.040781:0:3912:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921879.040783:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921879.040784:0:3912:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921879.040784:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921879.040785:0:3912:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921879.040785:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921879.040787:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921879.040789:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921879.040791:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921879.040792:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1506921879.040811:0:3917:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1506921879.040813:0:3917:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1506921879.040816:0:3917:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:9.0:1506921879.040817:0:3913:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1506921879.040817:0:3917:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921879.040819:0:3913:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1506921879.040819:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:9.0:1506921879.040822:0:3913:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1506921879.040822:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921879.040823:0:3913:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1506921879.040823:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:9.0:1506921879.040825:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1506921879.040825:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921879.040828:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921879.040829:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:9.0:1506921879.040831:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921879.041764:0:3914:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:7.0:1506921879.041764:0:3919:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1506921879.041765:0:3920:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:17.0:1506921879.041766:0:3914:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:7.0:1506921879.041766:0:3919:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921879.041766:0:3920:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:17.0:1506921879.041769:0:3914:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:7.0:1506921879.041769:0:3919:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921879.041769:0:3920:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:17.0:1506921879.041770:0:3914:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921879.041770:0:3919:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921879.041770:0:3920:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921879.041772:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921879.041772:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921879.041772:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921879.041776:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921879.041776:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921879.041776:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921879.041777:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921879.041778:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921879.041778:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921879.041780:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921879.041780:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921879.041780:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921879.041789:0:3915:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:15.0:1506921879.041791:0:3915:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921879.041794:0:3915:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921879.041797:0:3915:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921879.041799:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921879.041802:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921879.041803:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921879.041804:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921879.042767:0:3916:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921879.042769:0:3916:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921879.042772:0:3916:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921879.042772:0:3916:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921879.042774:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921879.042777:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921879.042778:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921879.042779:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921879.042782:0:3921:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:15.0:1506921879.042783:0:3921:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921879.042786:0:3921:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921879.042787:0:3921:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921879.042789:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921879.042792:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921879.042793:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921879.042794:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0F:1506921879.147379:0:38887:0:(llite_lib.c:2525:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1506921879.147397:0:38887:0:(llite_lib.c:2525:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1506921879.147626:0:38887:0:(llite_lib.c:2525:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1506921879.147641:0:38887:0:(llite_lib.c:2525:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1506921879.584781:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.31@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921879.584786:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.32@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921879.584789:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.41@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921879.584791:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.42@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000100:00000001:16.0:1506921879.611773:0:3902:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1506921879.611773:0:3910:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921879.611775:0:3902:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1506921879.611775:0:3910:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921879.611776:0:3905:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921879.611778:0:3905:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921879.611778:0:3902:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921879.611778:0:3910:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921879.611779:0:3902:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921879.611779:0:3910:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921879.611780:0:3905:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:4.0:1506921879.611780:0:3903:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921879.611781:0:3905:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921879.611781:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921879.611781:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921879.611782:0:3903:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921879.611783:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921879.611784:0:3911:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921879.611785:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921879.611785:0:3903:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921879.611785:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921879.611786:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921879.611786:0:3911:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:4.0:1506921879.611786:0:3903:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921879.611787:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921879.611787:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921879.611787:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921879.611788:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921879.611789:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921879.611789:0:3911:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921879.611790:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921879.611790:0:3911:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921879.611790:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921879.611791:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921879.611792:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921879.611793:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921879.611794:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921879.611795:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921879.611797:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921879.611798:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921879.612769:0:3909:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921879.612769:0:3907:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921879.612771:0:3909:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921879.612772:0:3907:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921879.612774:0:3909:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921879.612775:0:3909:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921879.612775:0:3907:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921879.612776:0:3907:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921879.612777:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921879.612777:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921879.612778:0:3906:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:8.0:1506921879.612780:0:3906:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921879.612781:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921879.612781:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921879.612782:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921879.612782:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921879.612783:0:3906:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921879.612784:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921879.612784:0:3906:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921879.612785:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921879.612785:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921879.612788:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921879.612790:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921879.612791:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921879.613753:0:3904:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:14.0:1506921879.613755:0:3904:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:14.0:1506921879.613758:0:3904:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921879.613759:0:3908:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:14.0:1506921879.613759:0:3904:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921879.613761:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921879.613762:0:3908:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:14.0:1506921879.613764:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921879.613765:0:3908:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:14.0:1506921879.613765:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921879.613766:0:3908:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921879.613766:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921879.613768:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921879.613771:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921879.613773:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921879.613774:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921880.035773:0:3901:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1506921880.035774:0:3901:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1506921880.035777:0:3901:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921880.035778:0:3901:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921880.035780:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921880.035783:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921880.035784:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921880.035785:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921880.040753:0:3912:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921880.040755:0:3912:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921880.040758:0:3912:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921880.040759:0:3912:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921880.040761:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1506921880.040761:0:3917:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1506921880.040763:0:3917:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921880.040764:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921880.040765:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1506921880.040766:0:3917:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921880.040767:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1506921880.040767:0:3917:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1506921880.040769:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:9.0:1506921880.040771:0:3913:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1506921880.040772:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921880.040773:0:3913:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1506921880.040773:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1506921880.040774:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921880.040776:0:3913:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921880.040776:0:3918:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:9.0:1506921880.040777:0:3913:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921880.040778:0:3918:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:9.0:1506921880.040779:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921880.040781:0:3918:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921880.040781:0:3918:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921880.040782:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921880.040783:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:9.0:1506921880.040784:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:9.0:1506921880.040786:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921880.040787:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921880.040790:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921880.040792:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921880.041758:0:3914:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:7.0:1506921880.041758:0:3919:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1506921880.041758:0:3920:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:7.0:1506921880.041760:0:3919:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921880.041760:0:3920:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:17.0:1506921880.041762:0:3914:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:7.0:1506921880.041763:0:3919:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921880.041763:0:3920:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:7.0:1506921880.041764:0:3919:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921880.041764:0:3920:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921880.041765:0:3914:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:7.0:1506921880.041765:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921880.041765:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921880.041766:0:3914:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921880.041768:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921880.041768:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921880.041769:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921880.041770:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921880.041771:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921880.041771:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921880.041771:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921880.041772:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921880.041772:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921880.041773:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921880.041784:0:3915:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921880.041788:0:3915:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921880.041791:0:3915:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921880.041792:0:3915:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921880.041793:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921880.041796:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921880.041798:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921880.041799:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921880.042769:0:3916:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:15.0:1506921880.042770:0:3921:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921880.042771:0:3916:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921880.042772:0:3921:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921880.042774:0:3916:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921880.042775:0:3916:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921880.042775:0:3921:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921880.042776:0:3921:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921880.042777:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921880.042777:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921880.042780:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921880.042781:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921880.042782:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921880.042782:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921880.042783:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921880.042784:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1506921880.584780:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.31@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921880.584785:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.32@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921880.584788:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.41@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921880.584791:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.42@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000100:00000001:0.0:1506921880.611768:0:3910:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:6.0:1506921880.611770:0:3911:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:4.0:1506921880.611770:0:3903:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1506921880.611770:0:3910:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:6.0:1506921880.611772:0:3911:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:4.0:1506921880.611772:0:3903:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1506921880.611773:0:3910:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921880.611774:0:3910:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921880.611775:0:3911:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:4.0:1506921880.611775:0:3903:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:6.0:1506921880.611776:0:3911:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921880.611776:0:3903:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921880.611776:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921880.611778:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921880.611778:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921880.611779:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921880.611780:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921880.611781:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921880.611781:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921880.611782:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921880.611783:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921880.611783:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921880.611785:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921880.611785:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921880.611788:0:3905:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921880.611790:0:3905:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921880.611793:0:3905:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921880.611793:0:3902:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921880.611794:0:3905:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921880.611795:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921880.611795:0:3902:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921880.611798:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921880.611798:0:3902:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921880.611799:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921880.611799:0:3902:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921880.611801:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921880.611801:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921880.611804:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921880.611805:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921880.611807:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921880.612765:0:3909:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921880.612765:0:3907:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921880.612767:0:3909:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921880.612767:0:3907:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921880.612770:0:3909:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921880.612770:0:3907:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921880.612771:0:3909:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921880.612771:0:3907:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921880.612773:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921880.612773:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921880.612775:0:3906:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921880.612776:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921880.612776:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921880.612777:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921880.612777:0:3906:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921880.612778:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921880.612779:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921880.612779:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921880.612780:0:3906:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:8.0:1506921880.612781:0:3906:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921880.612782:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921880.612786:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921880.612787:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921880.612788:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921880.613752:0:3904:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:14.0:1506921880.613753:0:3904:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921880.613756:0:3908:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:14.0:1506921880.613756:0:3904:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:14.0:1506921880.613757:0:3904:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921880.613758:0:3908:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:14.0:1506921880.613759:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921880.613761:0:3908:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921880.613762:0:3908:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921880.613762:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921880.613763:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921880.613764:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921880.613764:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921880.613767:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921880.613768:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921880.613770:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921881.035772:0:3901:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1506921881.035774:0:3901:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1506921881.035777:0:3901:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921881.035778:0:3901:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921881.035779:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921881.035782:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921881.035783:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921881.035785:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1506921881.040754:0:3917:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:9.0:1506921881.040755:0:3913:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1506921881.040756:0:3917:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:9.0:1506921881.040757:0:3913:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1506921881.040759:0:3917:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:9.0:1506921881.040760:0:3913:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1506921881.040760:0:3917:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921881.040761:0:3913:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1506921881.040762:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:9.0:1506921881.040763:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:9.0:1506921881.040766:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1506921881.040766:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1506921881.040767:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:9.0:1506921881.040768:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1506921881.040768:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921881.040769:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921881.040777:0:3918:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921881.040778:0:3912:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1506921881.040779:0:3918:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921881.040780:0:3912:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921881.040782:0:3918:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921881.040783:0:3912:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921881.040783:0:3918:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921881.040784:0:3912:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921881.040785:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921881.040786:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921881.040788:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921881.040789:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921881.040789:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921881.040790:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921881.040791:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921881.040792:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921881.041750:0:3919:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:7.0:1506921881.041752:0:3919:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:7.0:1506921881.041755:0:3919:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:7.0:1506921881.041756:0:3919:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921881.041757:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921881.041760:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921881.041762:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921881.041763:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921881.041767:0:3915:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1506921881.041767:0:3920:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921881.041769:0:3915:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921881.041769:0:3920:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921881.041772:0:3915:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921881.041772:0:3920:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921881.041773:0:3915:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921881.041773:0:3920:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921881.041774:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921881.041775:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921881.041778:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921881.041778:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921881.041779:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921881.041780:0:3914:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1506921881.041780:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921881.041781:0:3914:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921881.041782:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921881.041782:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921881.041784:0:3914:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:17.0:1506921881.041785:0:3914:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921881.041787:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921881.041790:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921881.041791:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921881.041793:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921881.042752:0:3921:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:15.0:1506921881.042754:0:3921:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921881.042757:0:3921:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921881.042758:0:3921:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921881.042759:0:3916:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:15.0:1506921881.042759:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921881.042761:0:3916:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921881.042762:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921881.042763:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921881.042764:0:3916:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921881.042764:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921881.042765:0:3916:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921881.042766:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921881.042769:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921881.042771:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921881.042772:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1506921881.584785:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.31@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921881.584790:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.32@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921881.584793:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.41@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921881.584795:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.42@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000100:00000001:16.0:1506921881.611784:0:3902:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1506921881.611784:0:3910:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1506921881.611785:0:3910:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921881.611786:0:3902:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921881.611789:0:3902:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921881.611789:0:3910:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921881.611790:0:3902:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921881.611790:0:3910:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921881.611791:0:3905:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:6.0:1506921881.611791:0:3911:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:4.0:1506921881.611791:0:3903:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921881.611792:0:3905:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921881.611792:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921881.611792:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921881.611793:0:3911:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:4.0:1506921881.611793:0:3903:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921881.611795:0:3905:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921881.611796:0:3905:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921881.611796:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921881.611796:0:3911:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:4.0:1506921881.611796:0:3903:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921881.611796:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921881.611797:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921881.611797:0:3911:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921881.611797:0:3903:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921881.611797:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921881.611798:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921881.611798:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921881.611799:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921881.611799:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921881.611800:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921881.611802:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921881.611803:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921881.611803:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921881.611804:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921881.611804:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921881.611804:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921881.611807:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921881.611807:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921881.611807:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921881.612764:0:3909:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921881.612764:0:3907:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921881.612766:0:3909:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921881.612768:0:3907:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921881.612769:0:3909:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921881.612770:0:3909:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921881.612771:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921881.612771:0:3907:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921881.612772:0:3907:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921881.612774:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921881.612774:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921881.612774:0:3906:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921881.612776:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921881.612776:0:3906:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921881.612777:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921881.612777:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921881.612778:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921881.612779:0:3906:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921881.612780:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921881.612780:0:3906:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921881.612782:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921881.612787:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921881.612788:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921881.612790:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921881.613773:0:3908:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921881.613774:0:3908:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921881.613777:0:3908:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921881.613778:0:3908:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921881.613780:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921881.613783:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921881.613784:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921881.613785:0:3904:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921881.613786:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921881.613789:0:3904:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:14.0:1506921881.613792:0:3904:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:14.0:1506921881.613793:0:3904:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921881.613795:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921881.613798:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921881.613799:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921881.613801:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921882.035770:0:3901:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1506921882.035772:0:3901:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1506921882.035775:0:3901:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921882.035776:0:3901:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921882.035777:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921882.035780:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921882.035781:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921882.035783:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1506921882.040760:0:3917:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921882.040762:0:3912:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1506921882.040762:0:3917:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1506921882.040765:0:3917:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921882.040766:0:3912:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1506921882.040766:0:3917:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921882.040767:0:3913:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1506921882.040768:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1506921882.040771:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921882.040778:0:3912:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1506921882.040778:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921882.040779:0:3912:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1506921882.040779:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921882.040779:0:3918:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921882.040780:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921882.040781:0:3918:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:9.0:1506921882.040782:0:3913:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921882.040783:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921882.040784:0:3918:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921882.040785:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:9.0:1506921882.040785:0:3913:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921882.040785:0:3918:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921882.040786:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921882.040786:0:3913:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921882.040786:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:9.0:1506921882.040788:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921882.040789:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921882.040791:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921882.040791:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:9.0:1506921882.040792:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921882.040792:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921882.040794:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921882.041761:0:3915:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1506921882.041762:0:3920:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921882.041763:0:3915:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921882.041764:0:3920:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921882.041766:0:3915:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921882.041767:0:3915:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921882.041767:0:3920:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921882.041767:0:3920:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921882.041769:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921882.041769:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921882.041772:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921882.041773:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921882.041774:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921882.041774:0:3914:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:7.0:1506921882.041774:0:3919:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921882.041775:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921882.041776:0:3914:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:7.0:1506921882.041776:0:3919:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921882.041777:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921882.041778:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921882.041779:0:3914:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:7.0:1506921882.041779:0:3919:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:17.0:1506921882.041780:0:3914:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921882.041780:0:3919:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921882.041782:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921882.041782:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921882.041785:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921882.041786:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921882.041787:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921882.041787:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921882.041790:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921882.041790:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921882.042773:0:3916:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921882.042775:0:3916:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921882.042778:0:3916:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921882.042779:0:3916:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921882.042780:0:3921:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921882.042781:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921882.042782:0:3921:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921882.042784:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921882.042785:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921882.042785:0:3921:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921882.042786:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921882.042786:0:3921:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921882.042787:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921882.042791:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921882.042792:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921882.042793:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1506921882.584776:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.31@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921882.584781:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.32@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921882.584784:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.41@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921882.584786:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.42@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000100:00000001:18.0:1506921882.611757:0:3905:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921882.611757:0:3902:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:6.0:1506921882.611757:0:3911:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:4.0:1506921882.611757:0:3903:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921882.611759:0:3905:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921882.611759:0:3902:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:6.0:1506921882.611761:0:3911:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:4.0:1506921882.611761:0:3903:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921882.611762:0:3905:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921882.611762:0:3902:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921882.611763:0:3905:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921882.611764:0:3902:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921882.611764:0:3911:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:4.0:1506921882.611764:0:3903:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921882.611765:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921882.611765:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921882.611765:0:3911:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921882.611765:0:3903:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921882.611766:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921882.611767:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921882.611769:0:3910:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921882.611770:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921882.611770:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921882.611771:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921882.611771:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921882.611771:0:3910:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921882.611772:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921882.611772:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921882.611772:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921882.611773:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921882.611774:0:3910:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921882.611775:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921882.611775:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921882.611775:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921882.611775:0:3910:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921882.611776:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921882.611777:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921882.611780:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921882.611781:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921882.611783:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921882.612768:0:3907:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921882.612770:0:3909:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921882.612770:0:3907:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921882.612772:0:3909:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921882.612773:0:3907:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921882.612774:0:3907:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921882.612775:0:3909:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921882.612775:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921882.612776:0:3909:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921882.612777:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921882.612778:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921882.612778:0:3906:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921882.612780:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921882.612780:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921882.612780:0:3906:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921882.612781:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921882.612782:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921882.612783:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921882.612783:0:3906:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:8.0:1506921882.612784:0:3906:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921882.612786:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921882.612789:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921882.612790:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921882.612791:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921882.613759:0:3908:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:14.0:1506921882.613760:0:3904:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921882.613761:0:3908:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:14.0:1506921882.613761:0:3904:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921882.613764:0:3908:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:14.0:1506921882.613764:0:3904:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921882.613765:0:3908:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921882.613765:0:3904:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921882.613766:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921882.613767:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921882.613769:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921882.613770:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921882.613771:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921882.613772:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921882.613773:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921882.613774:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921883.035777:0:3901:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1506921883.035779:0:3901:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1506921883.035781:0:3901:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921883.035782:0:3901:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921883.035784:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921883.035786:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921883.035788:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921883.035789:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1506921883.040752:0:3917:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1506921883.040754:0:3917:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1506921883.040757:0:3917:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1506921883.040758:0:3917:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1506921883.040759:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1506921883.040763:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1506921883.040764:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1506921883.040767:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921883.040771:0:3913:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:9.0:1506921883.040773:0:3913:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921883.040775:0:3912:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:9.0:1506921883.040776:0:3913:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921883.040777:0:3912:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:9.0:1506921883.040777:0:3913:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921883.040777:0:3918:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:9.0:1506921883.040779:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921883.040779:0:3918:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921883.040780:0:3912:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921883.040781:0:3912:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921883.040782:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921883.040782:0:3918:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921883.040783:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921883.040783:0:3918:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921883.040784:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921883.040785:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921883.040786:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921883.040787:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921883.040788:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921883.040788:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921883.040789:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921883.040789:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921883.040791:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921883.041758:0:3915:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:7.0:1506921883.041758:0:3919:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:17.0:1506921883.041759:0:3914:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1506921883.041759:0:3920:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921883.041760:0:3915:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:17.0:1506921883.041761:0:3914:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921883.041761:0:3920:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:7.0:1506921883.041762:0:3919:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921883.041763:0:3915:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921883.041764:0:3915:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921883.041764:0:3914:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921883.041764:0:3920:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:17.0:1506921883.041765:0:3914:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921883.041765:0:3919:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921883.041765:0:3920:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921883.041766:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921883.041766:0:3919:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921883.041767:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921883.041767:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921883.041767:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921883.041770:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921883.041771:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921883.041772:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921883.041772:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921883.041772:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921883.041773:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921883.041773:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921883.041773:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921883.041773:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921883.041775:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921883.041776:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921883.041776:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921883.042751:0:3921:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:15.0:1506921883.042753:0:3921:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921883.042756:0:3921:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921883.042758:0:3916:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:15.0:1506921883.042758:0:3921:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921883.042760:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921883.042762:0:3916:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921883.042763:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921883.042764:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921883.042765:0:3916:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921883.042766:0:3916:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921883.042766:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921883.042768:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921883.042770:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921883.042772:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921883.042773:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1506921883.584774:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.31@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921883.584780:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.32@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921883.584783:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.41@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921883.584785:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.42@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000100:00000001:18.0:1506921883.611755:0:3905:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:6.0:1506921883.611755:0:3911:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:4.0:1506921883.611755:0:3903:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921883.611757:0:3905:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:6.0:1506921883.611757:0:3911:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:4.0:1506921883.611757:0:3903:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921883.611760:0:3905:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:6.0:1506921883.611760:0:3911:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:4.0:1506921883.611760:0:3903:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921883.611761:0:3905:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921883.611762:0:3911:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921883.611762:0:3903:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921883.611763:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921883.611764:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921883.611764:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921883.611767:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921883.611768:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921883.611769:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921883.611769:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921883.611770:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921883.611770:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921883.611770:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921883.611772:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921883.611772:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921883.611774:0:3902:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921883.611778:0:3902:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1506921883.611780:0:3910:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921883.611781:0:3902:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921883.611782:0:3902:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921883.611782:0:3910:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921883.611784:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921883.611785:0:3910:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921883.611786:0:3910:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921883.611787:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921883.611788:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921883.611788:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921883.611790:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921883.611791:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921883.611792:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921883.611794:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921883.612764:0:3909:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921883.612764:0:3907:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921883.612766:0:3909:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921883.612766:0:3907:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921883.612769:0:3909:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921883.612769:0:3907:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921883.612770:0:3909:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921883.612770:0:3907:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921883.612772:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921883.612772:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921883.612775:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921883.612775:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921883.612775:0:3906:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921883.612777:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921883.612777:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921883.612777:0:3906:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921883.612779:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921883.612779:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921883.612779:0:3906:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:8.0:1506921883.612780:0:3906:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921883.612782:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921883.612785:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921883.612786:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921883.612787:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921883.613753:0:3904:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:14.0:1506921883.613755:0:3904:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921883.613757:0:3908:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:14.0:1506921883.613758:0:3904:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921883.613759:0:3908:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:14.0:1506921883.613759:0:3904:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921883.613760:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921883.613762:0:3908:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921883.613763:0:3908:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921883.613763:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921883.613765:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921883.613765:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921883.613766:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921883.613768:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921883.613770:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921883.613771:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921884.035769:0:3901:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1506921884.035773:0:3901:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1506921884.035775:0:3901:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921884.035776:0:3901:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921884.035778:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921884.035780:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921884.035782:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921884.035783:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921884.040775:0:3913:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:9.0:1506921884.040777:0:3913:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:9.0:1506921884.040780:0:3913:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:9.0:1506921884.040781:0:3913:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921884.040781:0:3912:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:9.0:1506921884.040783:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921884.040783:0:3912:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1506921884.040785:0:3918:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:9.0:1506921884.040786:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1506921884.040787:0:3918:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921884.040787:0:3912:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921884.040787:0:3912:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921884.040788:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:9.0:1506921884.040789:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921884.040789:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1506921884.040790:0:3918:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1506921884.040791:0:3918:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1506921884.040792:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921884.040792:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921884.040794:0:3917:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1506921884.040794:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921884.040795:0:3917:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1506921884.040796:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921884.040796:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921884.040798:0:3917:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1506921884.040798:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921884.040799:0:3917:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1506921884.040799:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921884.040801:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921884.040804:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921884.040805:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921884.040806:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921884.041765:0:3915:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1506921884.041765:0:3920:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921884.041767:0:3915:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921884.041767:0:3920:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921884.041769:0:3915:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921884.041770:0:3915:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921884.041770:0:3920:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921884.041770:0:3920:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921884.041772:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921884.041772:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921884.041775:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921884.041776:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921884.041777:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921884.041777:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921884.041779:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921884.041779:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921884.041784:0:3914:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:7.0:1506921884.041784:0:3919:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:17.0:1506921884.041786:0:3914:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:7.0:1506921884.041786:0:3919:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:17.0:1506921884.041789:0:3914:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:7.0:1506921884.041789:0:3919:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:17.0:1506921884.041790:0:3914:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921884.041790:0:3919:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921884.041791:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921884.041791:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921884.041795:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921884.041795:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921884.041796:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921884.041797:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921884.041798:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921884.041799:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921884.042761:0:3916:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921884.042763:0:3916:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921884.042766:0:3916:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921884.042766:0:3916:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921884.042768:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921884.042771:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921884.042772:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921884.042773:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921884.042780:0:3921:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:15.0:1506921884.042782:0:3921:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921884.042785:0:3921:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921884.042786:0:3921:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921884.042788:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921884.042791:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921884.042792:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921884.042793:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1506921884.584783:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.31@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921884.584788:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.32@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921884.584791:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.41@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921884.584793:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.42@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000100:00000001:16.0:1506921884.611754:0:3902:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921884.611755:0:3905:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:6.0:1506921884.611755:0:3911:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:4.0:1506921884.611755:0:3903:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921884.611756:0:3902:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921884.611757:0:3905:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:6.0:1506921884.611757:0:3911:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:4.0:1506921884.611757:0:3903:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921884.611759:0:3902:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921884.611760:0:3905:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921884.611760:0:3902:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921884.611760:0:3911:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:4.0:1506921884.611760:0:3903:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921884.611761:0:3905:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921884.611761:0:3911:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921884.611761:0:3903:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921884.611762:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921884.611763:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921884.611763:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921884.611763:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921884.611766:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921884.611767:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921884.611767:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921884.611768:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921884.611768:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921884.611768:0:3910:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921884.611769:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921884.611769:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921884.611769:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921884.611770:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921884.611770:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921884.611770:0:3910:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:6.0:1506921884.611771:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921884.611772:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921884.611773:0:3910:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921884.611774:0:3910:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921884.611776:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921884.611779:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921884.611780:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921884.611782:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921884.612771:0:3907:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921884.612773:0:3909:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921884.612773:0:3907:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921884.612775:0:3909:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921884.612776:0:3907:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921884.612777:0:3907:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921884.612778:0:3909:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921884.612778:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921884.612779:0:3909:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921884.612781:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921884.612781:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921884.612781:0:3906:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921884.612783:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921884.612783:0:3906:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921884.612784:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921884.612784:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921884.612785:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921884.612786:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921884.612786:0:3906:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:8.0:1506921884.612787:0:3906:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921884.612789:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921884.612792:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921884.612793:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921884.612794:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921884.613752:0:3904:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:14.0:1506921884.613753:0:3904:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:14.0:1506921884.613756:0:3904:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:14.0:1506921884.613757:0:3904:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921884.613759:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921884.613760:0:3908:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:14.0:1506921884.613762:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921884.613763:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921884.613764:0:3908:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:14.0:1506921884.613764:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921884.613767:0:3908:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921884.613768:0:3908:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921884.613769:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921884.613772:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921884.613774:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921884.613775:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921885.035771:0:3901:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1506921885.035773:0:3901:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1506921885.035776:0:3901:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921885.035777:0:3901:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921885.035778:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921885.035781:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921885.035782:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921885.035783:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921885.040753:0:3917:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1506921885.040753:0:3918:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921885.040755:0:3917:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1506921885.040755:0:3918:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921885.040758:0:3917:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1506921885.040758:0:3918:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921885.040759:0:3917:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1506921885.040759:0:3918:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921885.040761:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1506921885.040761:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1506921885.040764:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921885.040765:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921885.040766:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1506921885.040766:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921885.040768:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1506921885.040768:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921885.040771:0:3913:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:9.0:1506921885.040773:0:3913:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921885.040775:0:3912:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:9.0:1506921885.040776:0:3913:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:9.0:1506921885.040777:0:3913:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921885.040777:0:3912:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:9.0:1506921885.040779:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921885.040780:0:3912:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921885.040780:0:3912:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921885.040782:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921885.040782:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:9.0:1506921885.040783:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:9.0:1506921885.040786:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921885.040786:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921885.040790:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921885.040791:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921885.041756:0:3919:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:17.0:1506921885.041757:0:3914:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:7.0:1506921885.041758:0:3919:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:17.0:1506921885.041760:0:3914:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921885.041760:0:3920:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921885.041761:0:3915:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:7.0:1506921885.041761:0:3919:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:7.0:1506921885.041762:0:3919:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921885.041762:0:3920:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:17.0:1506921885.041763:0:3914:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:7.0:1506921885.041763:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921885.041764:0:3914:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921885.041765:0:3915:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921885.041765:0:3920:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:17.0:1506921885.041766:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921885.041766:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921885.041766:0:3920:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921885.041767:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921885.041768:0:3915:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921885.041768:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921885.041769:0:3915:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921885.041769:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921885.041769:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921885.041770:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921885.041771:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921885.041771:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921885.041772:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921885.041772:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921885.041774:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921885.041775:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921885.041776:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921885.041777:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921885.042754:0:3921:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:15.0:1506921885.042756:0:3921:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921885.042758:0:3916:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:15.0:1506921885.042759:0:3921:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921885.042760:0:3916:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921885.042760:0:3921:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921885.042762:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921885.042763:0:3916:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921885.042764:0:3916:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921885.042764:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921885.042765:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921885.042766:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921885.042767:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921885.042768:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921885.042770:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921885.042771:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1506921885.584775:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.31@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921885.584780:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.32@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921885.584783:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.41@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921885.584786:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.42@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000100:00000001:0.0:1506921885.611771:0:3910:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921885.611772:0:3902:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1506921885.611773:0:3910:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921885.611774:0:3902:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1506921885.611776:0:3910:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921885.611777:0:3902:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921885.611777:0:3902:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921885.611779:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921885.611780:0:3905:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:6.0:1506921885.611780:0:3911:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:4.0:1506921885.611780:0:3903:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1506921885.611780:0:3910:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921885.611781:0:3903:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1506921885.611781:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921885.611782:0:3905:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921885.611782:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921885.611782:0:3911:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921885.611784:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921885.611784:0:3903:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921885.611784:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921885.611785:0:3905:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921885.611785:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921885.611785:0:3911:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921885.611786:0:3905:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921885.611786:0:3911:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921885.611786:0:3903:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921885.611786:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921885.611787:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921885.611787:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921885.611788:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921885.611788:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921885.611792:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921885.611792:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921885.611792:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921885.611793:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921885.611793:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921885.611794:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921885.611796:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921885.611796:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921885.611797:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:12.0:1506921885.611848:0:3937:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:12.0:1506921885.611850:0:3937:0:(ldlm_pool.c:504:ldlm_cli_pool_recalc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:12.0:1506921885.611853:0:3937:0:(ldlm_pool.c:525:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:12.0:1506921885.611855:0:3937:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:12.0:1506921885.611856:0:3937:0:(ldlm_pool.c:504:ldlm_cli_pool_recalc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:12.0:1506921885.611857:0:3937:0:(ldlm_pool.c:525:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:12.0:1506921885.611859:0:3937:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:12.0:1506921885.611860:0:3937:0:(ldlm_request.c:1846:ldlm_cancel_lru()) Process entered 00010000:00000001:12.0:1506921885.611861:0:3937:0:(ldlm_request.c:1687:ldlm_prepare_lru_list()) Process entered 00010000:00000001:12.0:1506921885.611863:0:3937:0:(ldlm_lock.c:188:ldlm_lock_put()) Process entered 00010000:00000001:12.0:1506921885.611864:0:3937:0:(ldlm_lock.c:222:ldlm_lock_put()) Process leaving 00010000:00000001:12.0:1506921885.611864:0:3937:0:(ldlm_request.c:1815:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:12.0:1506921885.611866:0:3937:0:(ldlm_lockd.c:2046:ldlm_bl_to_thread()) Process entered 00010000:00000001:12.0:1506921885.611867:0:3937:0:(ldlm_lockd.c:2049:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:12.0:1506921885.611868:0:3937:0:(ldlm_request.c:1853:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:12.0:1506921885.611869:0:3937:0:(ldlm_pool.c:525:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:8.0:1506921885.611880:0:3935:0:(genops.c:1774:obd_stale_export_get()) Process entered 00000020:00000001:8.0:1506921885.611882:0:3935:0:(genops.c:1788:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:8.0:1506921885.611884:0:3935:0:(genops.c:1774:obd_stale_export_get()) Process entered 00000020:00000001:8.0:1506921885.611885:0:3935:0:(genops.c:1788:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1506921885.611914:0:3936:0:(genops.c:1774:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1506921885.611917:0:3936:0:(genops.c:1788:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1506921885.611919:0:3936:0:(genops.c:1774:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1506921885.611919:0:3936:0:(genops.c:1788:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921885.612767:0:3909:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921885.612767:0:3907:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921885.612769:0:3909:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921885.612769:0:3907:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921885.612772:0:3909:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921885.612772:0:3907:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921885.612773:0:3909:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921885.612773:0:3907:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921885.612774:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921885.612774:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921885.612775:0:3906:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:8.0:1506921885.612777:0:3906:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921885.612778:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921885.612778:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921885.612779:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921885.612779:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921885.612780:0:3906:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:8.0:1506921885.612781:0:3906:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921885.612782:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921885.612782:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921885.612783:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921885.612786:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921885.612787:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921885.612788:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921885.613774:0:3908:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921885.613775:0:3908:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921885.613778:0:3908:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921885.613779:0:3908:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921885.613781:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921885.613783:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921885.613785:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921885.613786:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921885.613787:0:3904:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:14.0:1506921885.613789:0:3904:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:14.0:1506921885.613792:0:3904:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:14.0:1506921885.613793:0:3904:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921885.613795:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921885.613798:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921885.613799:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921885.613800:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921886.035774:0:3901:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1506921886.035776:0:3901:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1506921886.035779:0:3901:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921886.035780:0:3901:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921886.035782:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921886.035784:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921886.035786:0:3901:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921886.035787:0:3901:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921886.040768:0:3912:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1506921886.040770:0:3912:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921886.040773:0:3912:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921886.040774:0:3912:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921886.040776:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:9.0:1506921886.040777:0:3913:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1506921886.040777:0:3918:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:9.0:1506921886.040779:0:3913:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1506921886.040779:0:3918:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921886.040780:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921886.040781:0:3917:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1506921886.040781:0:3912:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:9.0:1506921886.040782:0:3913:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1506921886.040782:0:3918:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921886.040782:0:3912:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921886.040783:0:3917:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:9.0:1506921886.040783:0:3913:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1506921886.040783:0:3918:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921886.040785:0:3917:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:9.0:1506921886.040785:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1506921886.040785:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921886.040786:0:3917:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921886.040788:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1506921886.040788:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921886.040789:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1506921886.040789:0:3918:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:9.0:1506921886.040790:0:3913:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921886.040792:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1506921886.040792:0:3918:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:9.0:1506921886.040793:0:3913:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921886.040794:0:3917:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921886.040795:0:3917:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921886.041769:0:3915:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1506921886.041769:0:3920:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921886.041771:0:3915:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1506921886.041771:0:3920:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921886.041774:0:3915:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921886.041774:0:3915:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921886.041774:0:3920:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1506921886.041775:0:3920:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921886.041776:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921886.041777:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921886.041779:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1506921886.041780:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921886.041781:0:3915:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1506921886.041781:0:3920:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921886.041782:0:3915:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921886.041782:0:3914:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:7.0:1506921886.041783:0:3919:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1506921886.041783:0:3920:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921886.041784:0:3914:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:7.0:1506921886.041784:0:3919:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:17.0:1506921886.041787:0:3914:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:7.0:1506921886.041787:0:3919:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:17.0:1506921886.041788:0:3914:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921886.041788:0:3919:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921886.041790:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921886.041790:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921886.041794:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921886.041794:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:17.0:1506921886.041795:0:3914:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:7.0:1506921886.041795:0:3919:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:17.0:1506921886.041798:0:3914:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:7.0:1506921886.041798:0:3919:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921886.042764:0:3916:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:19.0:1506921886.042766:0:3916:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:19.0:1506921886.042768:0:3916:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:19.0:1506921886.042769:0:3916:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921886.042771:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921886.042774:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:19.0:1506921886.042775:0:3916:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:19.0:1506921886.042776:0:3916:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921886.042777:0:3921:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:15.0:1506921886.042779:0:3921:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:15.0:1506921886.042782:0:3921:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:15.0:1506921886.042783:0:3921:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921886.042784:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921886.042787:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:15.0:1506921886.042788:0:3921:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:15.0:1506921886.042790:0:3921:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1506921886.584774:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.31@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921886.584779:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.32@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921886.584782:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.41@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000400:00000200:0.0:1506921886.584784:0:3866:0:(router.c:1061:lnet_ping_router_locked()) rtr 10.9.0.42@o2ib4 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 4395210798 00000100:00000001:16.0:1506921886.611753:0:3902:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:6.0:1506921886.611753:0:3911:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:4.0:1506921886.611753:0:3903:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921886.611754:0:3905:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921886.611755:0:3905:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921886.611755:0:3902:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:6.0:1506921886.611755:0:3911:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:4.0:1506921886.611755:0:3903:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921886.611758:0:3902:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:6.0:1506921886.611758:0:3911:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:4.0:1506921886.611758:0:3903:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921886.611759:0:3905:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921886.611759:0:3902:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921886.611759:0:3911:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921886.611759:0:3903:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921886.611760:0:3905:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921886.611761:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921886.611761:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921886.611761:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921886.611761:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921886.611765:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921886.611766:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921886.611766:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921886.611766:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921886.611767:0:3905:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921886.611767:0:3902:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:6.0:1506921886.611768:0:3911:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:4.0:1506921886.611768:0:3903:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:18.0:1506921886.611769:0:3905:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921886.611769:0:3910:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921886.611771:0:3902:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:6.0:1506921886.611771:0:3911:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:4.0:1506921886.611771:0:3903:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921886.611771:0:3910:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1506921886.611774:0:3910:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1506921886.611775:0:3910:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921886.611776:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921886.611779:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1506921886.611781:0:3910:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1506921886.611782:0:3910:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921886.612767:0:3909:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:16.0:1506921886.612767:0:3907:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921886.612769:0:3909:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921886.612771:0:3907:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921886.612772:0:3909:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:18.0:1506921886.612773:0:3909:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921886.612774:0:3907:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921886.612774:0:3907:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921886.612775:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921886.612776:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921886.612776:0:3906:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:18.0:1506921886.612778:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:18.0:1506921886.612779:0:3909:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921886.612779:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921886.612779:0:3906:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:18.0:1506921886.612780:0:3909:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921886.612780:0:3907:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921886.612782:0:3907:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921886.612782:0:3906:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:8.0:1506921886.612783:0:3906:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921886.612785:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921886.612788:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:8.0:1506921886.612790:0:3906:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:8.0:1506921886.612791:0:3906:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921886.613753:0:3904:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:14.0:1506921886.613756:0:3904:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:16.0:1506921886.613758:0:3908:0:(client.c:2174:ptlrpc_expired_set()) Process entered 00000100:00000001:14.0:1506921886.613759:0:3904:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921886.613760:0:3908:0:(client.c:2209:ptlrpc_expired_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:14.0:1506921886.613760:0:3904:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921886.613762:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921886.613763:0:3908:0:(client.c:2262:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:16.0:1506921886.613764:0:3908:0:(client.c:2298:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921886.613765:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:14.0:1506921886.613765:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921886.613767:0:3904:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921886.613769:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:14.0:1506921886.613769:0:3904:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:16.0:1506921886.613770:0:3908:0:(ptlrpcd.c:321:ptlrpcd_check()) Process entered 00000100:00000001:16.0:1506921886.613772:0:3908:0:(ptlrpcd.c:416:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)