diff --git a/lnet/include/lnet/lnetst.h b/lnet/include/lnet/lnetst.h index 6ca9020..0c33c75 100644 --- a/lnet/include/lnet/lnetst.h +++ b/lnet/include/lnet/lnetst.h @@ -89,6 +89,32 @@ typedef struct { int nde_state; /* state of node */ } lstcon_node_ent_t; /*** node entry, for list_group command */ +typedef enum { + XSTAT_TYPE_PING = 1 +} lst_xstat_type_t; + +typedef enum { + LST_BRW_READ = 1, + LST_BRW_WRITE = 2 +} lst_brw_type_t; + +typedef enum { + LST_BRW_CHECK_NONE = 1, + LST_BRW_CHECK_SIMPLE = 2, + LST_BRW_CHECK_FULL = 3 +} lst_brw_flags_t; + +typedef struct { + int blk_opc; /* bulk operation code */ + int blk_size; /* size (bytes) */ + int blk_flags; /* reserved flags */ +} lst_test_bulk_param_t; + +typedef struct { + int png_size; /* size of ping message */ + int png_flags; /* reserved flags */ +} lst_test_advping_param_t; + typedef struct { int nle_nnode; /* # of nodes */ int nle_nactive; /* # of active nodes */ @@ -101,6 +127,12 @@ typedef struct { int tse_type; /* test type */ int tse_loop; /* loop count */ int tse_concur; /* concurrency of test */ + int tse_dist; /* distribution */ + int tse_span; /* span */ + union { + lst_test_bulk_param_t prm_bulk; + lst_test_advping_param_t prm_advping; + } tse_params; } lstcon_test_ent_t; /*** test summary entry, for list_batch command */ typedef struct { @@ -394,13 +426,15 @@ typedef struct { int lstio_sta_nmlen; /* IN: group name length */ char *lstio_sta_namep; /* IN: group name */ int lstio_sta_count; /* IN: # of pid */ + int lstio_sta_type; /* IN: XSTAT id or -1 for STAT */ lnet_process_id_t *lstio_sta_idsp; /* IN: pid */ cfs_list_t *lstio_sta_resultp; /* OUT: list head of result buffer */ } lstio_stat_args_t; typedef enum { - LST_TEST_BULK = 1, - LST_TEST_PING = 2 + LST_TEST_BULK = 1, + LST_TEST_PING = 2, + LST_TEST_ADVPING = 3 } lst_test_type_t; /* create a test in a batch */ @@ -431,31 +465,6 @@ typedef struct { cfs_list_t *lstio_tes_resultp; /* OUT: list head of result buffer */ } lstio_test_args_t; -typedef enum { - LST_BRW_READ = 1, - LST_BRW_WRITE = 2 -} lst_brw_type_t; - -typedef enum { - LST_BRW_CHECK_NONE = 1, - LST_BRW_CHECK_SIMPLE = 2, - LST_BRW_CHECK_FULL = 3 -} lst_brw_flags_t; - -typedef struct { - int blk_opc; /* bulk operation code */ - int blk_size; /* size (bytes) */ - int blk_time; /* time of running the test*/ - int blk_flags; /* reserved flags */ -} lst_test_bulk_param_t; - -typedef struct { - int png_size; /* size of ping message */ - int png_time; /* time */ - int png_loop; /* loop */ - int png_flags; /* reserved flags */ -} lst_test_ping_param_t; - /* more tests */ #include typedef struct { @@ -464,8 +473,8 @@ typedef struct { __u32 rpcs_rcvd; __u32 rpcs_dropped; __u32 rpcs_expired; - __u64 bulk_get; - __u64 bulk_put; + __u64 bulk_in; + __u64 bulk_out; } WIRE_ATTR srpc_counters_t; typedef struct { @@ -475,6 +484,12 @@ typedef struct { __u32 brw_errors; __u32 ping_errors; } WIRE_ATTR sfw_counters_t; + +typedef struct { + __u32 ping_count; + __u32 ping_duration; + __u32 ping_errors; +} WIRE_ATTR png_counters_t; #include #endif diff --git a/lnet/selftest/Makefile.in b/lnet/selftest/Makefile.in index 7acc249..ed9e79c 100644 --- a/lnet/selftest/Makefile.in +++ b/lnet/selftest/Makefile.in @@ -1,6 +1,6 @@ MODULES := lnet_selftest -lnet_selftest-objs := console.o conrpc.o conctl.o framework.o timer.o rpc.o module.o ping_test.o brw_test.o +lnet_selftest-objs := console.o conrpc.o conctl.o framework.o timer.o rpc.o module.o ping_test.o brw_test.o advping_test.o default: all diff --git a/lnet/selftest/advping_test.c b/lnet/selftest/advping_test.c new file mode 100644 index 0000000..1fca6e3 --- /dev/null +++ b/lnet/selftest/advping_test.c @@ -0,0 +1,278 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * GPL HEADER START + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 only, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License version 2 for more details (a copy is included + * in the LICENSE file that accompanied this code). + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; If not, see + * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf + * + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, + * CA 95054 USA or visit www.sun.com if you need additional information or + * have any questions. + * + * GPL HEADER END + */ +/* + * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. + * Use is subject to license terms. + */ +/* + * This file is part of Lustre, http://www.lustre.org/ + * Lustre is a trademark of Sun Microsystems, Inc. + * + * lnet/selftest/conctl.c + * + * Test client & Server + * + * Author: Liang Zhen + */ + +#include "selftest.h" + +#define LST_PING_TEST_MAGIC 0xbabeface + +typedef struct { + cfs_spinlock_t pnd_lock; /* serialize */ + int pnd_counter; /* sequence counter */ +} lst_advping_data_t; + +static lst_advping_data_t lst_advping_data; + + +static inline __u64 +advping_get_timestamp(void) +{ + struct timeval tv; + cfs_gettimeofday(&tv); + + return tv.tv_sec * 1000000 + tv.tv_usec; +} + +static void +advping_client_fini (sfw_test_instance_t *tsi) +{ + sfw_session_t *sn = tsi->tsi_batch->bat_session; + sfw_test_unit_t *tsu; + int errors; + + LASSERT (sn != NULL); + LASSERT (tsi->tsi_is_client); + + errors = cfs_atomic_read(&sn->sn_ping_errors); + if (errors) + CWARN ("%d pings have failed.\n", errors); + else + CDEBUG (D_NET, "Ping test finished OK.\n"); + + cfs_list_for_each_entry_typed (tsu, &tsi->tsi_units, + sfw_test_unit_t, tsu_list) { + srpc_bulk_t *bulk = (srpc_bulk_t *)tsu->tsu_private; + if (bulk == NULL) + continue; + srpc_free_bulk(bulk); + } +} + +static int +advping_client_init(sfw_test_instance_t *tsi) +{ + sfw_test_unit_t *tsu; + srpc_bulk_t *bulk; + int npg = tsi->tsi_u.advping.png_size / CFS_PAGE_SIZE; + LASSERT (tsi->tsi_is_client); + + cfs_spin_lock_init(&lst_advping_data.pnd_lock); + lst_advping_data.pnd_counter = 0; + + if (npg > 0) { + cfs_list_for_each_entry_typed (tsu, &tsi->tsi_units, + sfw_test_unit_t, tsu_list) { + /* Pre-allocate an srpc_bulk_t for copying into requests */ + bulk = srpc_alloc_bulk(npg, 0); + if (bulk == NULL) { + advping_client_fini(tsi); + return -ENOMEM; + } + + tsu->tsu_private = bulk; + } + } + + return 0; +} + +static int +advping_client_prep_rpc(sfw_test_unit_t *tsu, + lnet_process_id_t dest, srpc_client_rpc_t **rpcpp) +{ + srpc_advping_reqst_t *req; + sfw_test_instance_t *tsi = tsu->tsu_instance; + srpc_client_rpc_t *rpc; + srpc_bulk_t *bulk = tsu->tsu_private; + test_advping_req_t *preq = &tsi->tsi_u.advping; + int const png_size = preq->png_size; + int const nblk = png_size / CFS_PAGE_SIZE; + int rc; + + rc = sfw_create_test_rpc(tsu, dest, nblk, png_size, &rpc); + if (rc != 0) + return rc; + + if (png_size > 0) { + memcpy(&rpc->crpc_bulk, bulk, offsetof(srpc_bulk_t, bk_iovs[nblk])); + } + + req = &rpc->crpc_reqstmsg.msg_body.advping_reqst; + + req->pnr_magic = LST_PING_TEST_MAGIC; + + cfs_spin_lock(&lst_advping_data.pnd_lock); + req->pnr_seq = lst_advping_data.pnd_counter ++; + cfs_spin_unlock(&lst_advping_data.pnd_lock); + + req->pnr_time_usec = advping_get_timestamp(); + req->pnr_size = png_size; + + *rpcpp = rpc; + + return rc; +} + +static void +advping_client_done_rpc (sfw_test_unit_t *tsu, srpc_client_rpc_t *rpc) +{ + sfw_test_instance_t *tsi = tsu->tsu_instance; + sfw_session_t *sn = tsi->tsi_batch->bat_session; + srpc_advping_reqst_t *reqst = &rpc->crpc_reqstmsg.msg_body.advping_reqst; + srpc_advping_reply_t *reply = &rpc->crpc_replymsg.msg_body.advping_reply; + __u64 now; + unsigned rtt_usec; + + LASSERT (sn != NULL); + + if (rpc->crpc_status != 0) { + if (!tsi->tsi_stopping) /* rpc could have been aborted */ + cfs_atomic_inc(&sn->sn_ping_errors); + CERROR ("Unable to ping %s (%d): %d\n", + libcfs_id2str(rpc->crpc_dest), + reqst->pnr_seq, rpc->crpc_status); + return; + } + + if (rpc->crpc_replymsg.msg_magic != SRPC_MSG_MAGIC) { + __swab32s(&reply->pnr_seq); + __swab32s(&reply->pnr_magic); + __swab32s(&reply->pnr_status); + } + + if (reply->pnr_magic != LST_PING_TEST_MAGIC) { + rpc->crpc_status = -EBADMSG; + cfs_atomic_inc(&sn->sn_ping_errors); + CERROR ("Bad magic %u from %s, %u expected.\n", + reply->pnr_magic, libcfs_id2str(rpc->crpc_dest), + LST_PING_TEST_MAGIC); + return; + } + + if (reply->pnr_seq != reqst->pnr_seq) { + rpc->crpc_status = -EBADMSG; + cfs_atomic_inc(&sn->sn_ping_errors); + CERROR ("Bad seq %u from %s, %u expected.\n", + reply->pnr_seq, libcfs_id2str(rpc->crpc_dest), + reqst->pnr_seq); + return; + } + + now = advping_get_timestamp(); + rtt_usec = now - reqst->pnr_time_usec; + + cfs_atomic_inc(&sn->sn_ping_count); + cfs_atomic_add(rtt_usec, &sn->sn_ping_duration); + + return; +} + +void +advping_server_rpc_done (srpc_server_rpc_t *rpc) +{ + sfw_free_pages(rpc); +} + +static int +advping_server_handle (srpc_server_rpc_t *rpc) +{ + srpc_service_t *sv = rpc->srpc_service; + srpc_msg_t *reqstmsg = &rpc->srpc_reqstbuf->buf_msg; + srpc_advping_reqst_t *req = &reqstmsg->msg_body.advping_reqst; + srpc_advping_reply_t *rep = &rpc->srpc_replymsg.msg_body.advping_reply; + + LASSERT (sv->sv_id == SRPC_SERVICE_ADVPING); + + if (reqstmsg->msg_magic != SRPC_MSG_MAGIC) { + LASSERT (reqstmsg->msg_magic == __swab32(SRPC_MSG_MAGIC)); + + __swab32s(&reqstmsg->msg_type); + __swab32s(&req->pnr_seq); + __swab32s(&req->pnr_magic); + __swab32s(&req->pnr_size); + __swab64s(&req->pnr_time_usec); + } + LASSERT (reqstmsg->msg_type == srpc_service2request(sv->sv_id)); + + if (req->pnr_magic != LST_PING_TEST_MAGIC) { + CERROR ("Unexpected magic %08x from %s\n", + req->pnr_magic, libcfs_id2str(rpc->srpc_peer)); + return -EINVAL; + } + + rep->pnr_seq = req->pnr_seq; + rep->pnr_magic = LST_PING_TEST_MAGIC; + + CDEBUG (D_NET, "Got ping %d from %s\n", + req->pnr_seq, libcfs_id2str(rpc->srpc_peer)); + + // Get ready to receive bulk part + if (req->pnr_size > 0) { + int rc; + rpc->srpc_done = advping_server_rpc_done; + rc = sfw_alloc_pages(rpc, req->pnr_size / CFS_PAGE_SIZE, 1); + if (rc != 0) { + CERROR("Failed to alloc recv pages"); + return -ENOMEM; + } + } + + return 0; +} + +sfw_test_client_ops_t advping_test_client; +void advping_init_test_client(void) +{ + advping_test_client.tso_init = advping_client_init; + advping_test_client.tso_fini = advping_client_fini; + advping_test_client.tso_prep_rpc = advping_client_prep_rpc; + advping_test_client.tso_done_rpc = advping_client_done_rpc; +} + +srpc_service_t advping_test_service; +void advping_init_test_service(void) +{ + advping_test_service.sv_id = SRPC_SERVICE_ADVPING; + advping_test_service.sv_name = "advping_test"; + advping_test_service.sv_handler = advping_server_handle; +} + + diff --git a/lnet/selftest/autoMakefile.am b/lnet/selftest/autoMakefile.am index 24d92bc..0a237b6 100644 --- a/lnet/selftest/autoMakefile.am +++ b/lnet/selftest/autoMakefile.am @@ -1,6 +1,6 @@ my_sources = console.c conrpc.c conctl.c console.h conrpc.h \ framework.c timer.c rpc.c module.c \ - ping_test.c brw_test.c + ping_test.c brw_test.c advping_test.c if LIBLUSTRE noinst_LIBRARIES= libselftest.a diff --git a/lnet/selftest/conctl.c b/lnet/selftest/conctl.c index 6602961..e18d48a 100644 --- a/lnet/selftest/conctl.c +++ b/lnet/selftest/conctl.c @@ -701,11 +701,13 @@ lst_stat_query_ioctl(lstio_stat_args_t *args) if (args->lstio_sta_idsp == NULL) { rc = lstcon_group_stat(name, args->lstio_sta_timeout, + args->lstio_sta_type, args->lstio_sta_resultp); } else { rc = lstcon_nodes_stat(args->lstio_sta_count, args->lstio_sta_idsp, args->lstio_sta_timeout, + args->lstio_sta_type, args->lstio_sta_resultp); } diff --git a/lnet/selftest/conrpc.c b/lnet/selftest/conrpc.c index e376ce3..08c20fc 100644 --- a/lnet/selftest/conrpc.c +++ b/lnet/selftest/conrpc.c @@ -698,7 +698,25 @@ lstcon_statrpc_prep(lstcon_node_t *nd, lstcon_rpc_t **crpc) srq = &(*crpc)->crp_rpc->crpc_reqstmsg.msg_body.stat_reqst; srq->str_sid = console_session.ses_id; - srq->str_type = 0; /* XXX remove it */ + srq->str_reserved0 = 0; + + return 0; +} + +int +lstcon_xstatrpc_prep(lstcon_node_t *nd, int type, lstcon_rpc_t **crpc) +{ + srpc_xstat_reqst_t *xsrq; + int rc; + + rc = lstcon_rpc_prep(nd, SRPC_SERVICE_QUERY_XSTAT, 0, crpc); + if (rc != 0) + return rc; + + xsrq = &(*crpc)->crp_rpc->crpc_reqstmsg.msg_body.xstat_reqst; + + xsrq->xst_sid = console_session.ses_id; + xsrq->xst_type = type; return 0; } @@ -775,13 +793,25 @@ lstcon_dstnodes_prep(lstcon_group_t *grp, int idx, } int -lstcon_pingrpc_prep(lst_test_ping_param_t *param, srpc_test_reqst_t *req) +lstcon_pingrpc_prep(srpc_test_reqst_t *req) { test_ping_req_t *prq = &req->tsr_u.ping; - + + /* Un-used parameters */ + prq->png_reserved0 = 0; + prq->png_reserved1 = 0; + + return 0; +} + +int +lstcon_advpingrpc_prep(lst_test_advping_param_t *param, srpc_test_reqst_t *req) +{ + test_advping_req_t *prq = &req->tsr_u.advping; + prq->png_size = param->png_size; prq->png_flags = param->png_flags; - /* TODO dest */ + return 0; } @@ -866,12 +896,16 @@ lstcon_testrpc_prep(lstcon_node_t *nd, int transop, switch (test->tes_type) { case LST_TEST_PING: trq->tsr_service = SRPC_SERVICE_PING; - rc = lstcon_pingrpc_prep((lst_test_ping_param_t *)&test->tes_param[0], trq); + rc = lstcon_pingrpc_prep(trq); break; case LST_TEST_BULK: trq->tsr_service = SRPC_SERVICE_BRW; rc = lstcon_bulkrpc_prep((lst_test_bulk_param_t *)&test->tes_param[0], trq); break; + case LST_TEST_ADVPING: + trq->tsr_service = SRPC_SERVICE_ADVPING; + rc = lstcon_advpingrpc_prep((lst_test_advping_param_t *)&test->tes_param[0], trq); + break; default: LBUG(); break; @@ -890,6 +924,7 @@ lstcon_rpc_stat_reply(int transop, srpc_msg_t *msg, srpc_batch_reply_t *bat_rep; srpc_test_reply_t *test_rep; srpc_stat_reply_t *stat_rep; + srpc_xstat_reply_t *xstat_rep; int rc = 0; switch (transop) { @@ -1003,6 +1038,19 @@ lstcon_rpc_stat_reply(int transop, srpc_msg_t *msg, rc = stat_rep->str_status; break; + case LST_TRANS_XSTATQRY: + xstat_rep = &msg->msg_body.xstat_reply; + + if (xstat_rep->xst_status == 0) { + /* Counting XSTAT in the 'stat' counter */ + lstcon_statqry_stat_success(stat, 1); + return; + } + + lstcon_statqry_stat_failure(stat, 1); + rc = xstat_rep->xst_status; + break; + default: LBUG(); } @@ -1072,6 +1120,9 @@ lstcon_rpc_trans_ndlist(cfs_list_t *ndlist, case LST_TRANS_STATQRY: rc = lstcon_statrpc_prep(nd, &rpc); break; + case LST_TRANS_XSTATQRY: + rc = lstcon_xstatrpc_prep(nd, *((int*)arg), &rpc); + break; default: rc = -EINVAL; break; diff --git a/lnet/selftest/conrpc.h b/lnet/selftest/conrpc.h index 281bc73..708444e 100644 --- a/lnet/selftest/conrpc.h +++ b/lnet/selftest/conrpc.h @@ -102,6 +102,7 @@ typedef struct lstcon_rpc_trans { #define LST_TRANS_TSBSRVQRY 0x16 #define LST_TRANS_STATQRY 0x21 +#define LST_TRANS_XSTATQRY 0x22 typedef int (* lstcon_rpc_cond_func_t)(int, struct lstcon_node *, void *); typedef int (* lstcon_rpc_readent_func_t)(int, srpc_msg_t *, lstcon_rpc_ent_t *); diff --git a/lnet/selftest/console.c b/lnet/selftest/console.c index 4eaf731..e41020f 100644 --- a/lnet/selftest/console.c +++ b/lnet/selftest/console.c @@ -986,10 +986,21 @@ lstcon_batch_info(char *name, lstcon_test_batch_ent_t *ent_up, int server, entp->u.tbe_batch.bae_state = bat->bat_state; } else { - entp->u.tbe_test.tse_type = test->tes_type; entp->u.tbe_test.tse_loop = test->tes_loop; entp->u.tbe_test.tse_concur = test->tes_concur; + entp->u.tbe_test.tse_dist = test->tes_dist; + entp->u.tbe_test.tse_span = test->tes_span; + + if (test->tes_type == LST_TEST_BULK) { + memcpy (&entp->u.tbe_test.tse_params.prm_bulk, + &test->tes_param[0], + sizeof(lst_test_bulk_param_t)); + } else if (test->tes_type == LST_TEST_ADVPING) { + memcpy (&entp->u.tbe_test.tse_params.prm_advping, + &test->tes_param[0], + sizeof(lst_test_advping_param_t)); + } } cfs_list_for_each_entry_typed(ndl, clilst, lstcon_ndlink_t, ndl_link) @@ -1444,7 +1455,7 @@ lstcon_statrpc_readent(int transop, srpc_msg_t *msg, sfw_counters_t *sfwk_stat; srpc_counters_t *srpc_stat; lnet_counters_t *lnet_stat; - + if (rep->str_status != 0) return 0; @@ -1461,8 +1472,32 @@ lstcon_statrpc_readent(int transop, srpc_msg_t *msg, } int +lstcon_xstatrpc_readent(int transop, srpc_msg_t *msg, + lstcon_rpc_ent_t *ent_up) +{ + srpc_xstat_reply_t *rep = &msg->msg_body.xstat_reply; + png_counters_t *ping_stat; + + if (rep->xst_status != 0) + return 0; + + if (rep->xst_type == XSTAT_TYPE_PING) { + ping_stat = (png_counters_t *)&ent_up->rpe_payload[0]; + + if (cfs_copy_to_user(ping_stat, &rep->xst_result.ping, sizeof(*ping_stat))) + return -EFAULT; + } else { + CERROR("Unknown stat type %d\n", rep->xst_type); + return -1; + } + + return 0; +} + +int lstcon_ndlist_stat(cfs_list_t *ndlist, - int timeout, cfs_list_t *result_up) + int timeout, int type, + cfs_list_t *result_up) { cfs_list_t head; lstcon_rpc_trans_t *trans; @@ -1470,24 +1505,40 @@ lstcon_ndlist_stat(cfs_list_t *ndlist, CFS_INIT_LIST_HEAD(&head); - rc = lstcon_rpc_trans_ndlist(ndlist, &head, - LST_TRANS_STATQRY, NULL, NULL, &trans); - if (rc != 0) { - CERROR("Can't create transaction: %d\n", rc); - return rc; - } + if (type < 0) { + rc = lstcon_rpc_trans_ndlist(ndlist, &head, + LST_TRANS_STATQRY, NULL, NULL, &trans); + if (rc != 0) { + CERROR("Can't create transaction: %d\n", rc); + return rc; + } - lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout)); + lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout)); - rc = lstcon_rpc_trans_interpreter(trans, result_up, - lstcon_statrpc_readent); - lstcon_rpc_trans_destroy(trans); + rc = lstcon_rpc_trans_interpreter(trans, result_up, + lstcon_statrpc_readent); + lstcon_rpc_trans_destroy(trans); + } else { + rc = lstcon_rpc_trans_ndlist(ndlist, &head, + LST_TRANS_XSTATQRY, + &type, NULL, &trans); + if (rc != 0) { + CERROR("Can't create transaction: %d\n", rc); + return rc; + } + + lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout)); + + rc = lstcon_rpc_trans_interpreter(trans, result_up, + lstcon_xstatrpc_readent); + lstcon_rpc_trans_destroy(trans); + } return rc; } int -lstcon_group_stat(char *grp_name, int timeout, cfs_list_t *result_up) +lstcon_group_stat(char *grp_name, int timeout, int type, cfs_list_t *result_up) { lstcon_group_t *grp; int rc; @@ -1498,7 +1549,7 @@ lstcon_group_stat(char *grp_name, int timeout, cfs_list_t *result_up) return rc; } - rc = lstcon_ndlist_stat(&grp->grp_ndl_list, timeout, result_up); + rc = lstcon_ndlist_stat(&grp->grp_ndl_list, timeout, type, result_up); lstcon_group_put(grp); @@ -1507,7 +1558,7 @@ lstcon_group_stat(char *grp_name, int timeout, cfs_list_t *result_up) int lstcon_nodes_stat(int count, lnet_process_id_t *ids_up, - int timeout, cfs_list_t *result_up) + int timeout, int type, cfs_list_t *result_up) { lstcon_ndlink_t *ndl; lstcon_group_t *tmp; @@ -1542,7 +1593,7 @@ lstcon_nodes_stat(int count, lnet_process_id_t *ids_up, return rc; } - rc = lstcon_ndlist_stat(&tmp->grp_ndl_list, timeout, result_up); + rc = lstcon_ndlist_stat(&tmp->grp_ndl_list, timeout, type, result_up); lstcon_group_put(tmp); diff --git a/lnet/selftest/console.h b/lnet/selftest/console.h index 48a4061..e013b49 100644 --- a/lnet/selftest/console.h +++ b/lnet/selftest/console.h @@ -212,10 +212,10 @@ extern int lstcon_batch_list(int idx, int namelen, char *name_up); extern int lstcon_batch_info(char *name, lstcon_test_batch_ent_t *ent_up, int server, int testidx, int *index_p, int *ndent_p, lstcon_node_ent_t *dents_up); -extern int lstcon_group_stat(char *grp_name, int timeout, +extern int lstcon_group_stat(char *grp_name, int timeout, int type, cfs_list_t *result_up); extern int lstcon_nodes_stat(int count, lnet_process_id_t *ids_up, - int timeout, cfs_list_t *result_up); + int timeout, int type, cfs_list_t *result_up); extern int lstcon_test_add(char *name, int type, int loop, int concur, int dist, int span, char *src_name, char * dst_name, void *param, int paramlen, int *retp, diff --git a/lnet/selftest/framework.c b/lnet/selftest/framework.c index 4bf02d9..6865ca3 100644 --- a/lnet/selftest/framework.c +++ b/lnet/selftest/framework.c @@ -90,8 +90,8 @@ do { \ __swab32s(&(rc).rpcs_rcvd); \ __swab32s(&(rc).rpcs_dropped); \ __swab32s(&(rc).rpcs_expired); \ - __swab64s(&(rc).bulk_get); \ - __swab64s(&(rc).bulk_put); \ + __swab64s(&(rc).bulk_in); \ + __swab64s(&(rc).bulk_out); \ } while (0) #define sfw_unpack_lnet_counters(lc) \ @@ -109,6 +109,15 @@ do { \ __swab64s(&(lc).route_length); \ } while (0) +#define sfw_unpack_png_counters(fc) \ +do { \ + __swab32s(&(fc).ping_count); \ + __swab32s(&(fc).ping_duration); \ + __swab32s(&(fc).ping_errors); \ +} while (0) + + + #define sfw_test_active(t) (cfs_atomic_read(&(t)->tsi_nactive) != 0) #define sfw_batch_active(b) (cfs_atomic_read(&(b)->bat_nactive) != 0) @@ -398,6 +407,44 @@ sfw_bid2batch (lst_bid_t bid) } int +sfw_get_xstats (srpc_xstat_reqst_t *request, srpc_xstat_reply_t *reply) +{ + sfw_session_t *sn = sfw_data.fw_session; + png_counters_t *png; + + reply->xst_sid = (sn == NULL) ? LST_INVALID_SID : sn->sn_id; + + if (request->xst_sid.ses_nid == LNET_NID_ANY) { + reply->xst_status = EINVAL; + return 0; + } + + if (sn == NULL || !sfw_sid_equal(request->xst_sid, sn->sn_id)) { + reply->xst_status = ESRCH; + return 0; + } + + switch (request->xst_type) { + case XSTAT_TYPE_PING: + png = &reply->xst_result.ping; + png->ping_duration = cfs_atomic_read(&sn->sn_ping_duration); + png->ping_count = cfs_atomic_read(&sn->sn_ping_count); + /* NB ping_errors is also available in sfwk_counters_t, + * it's duplicated here to be able to report ping results + * without querying both sets of counters */ + png->ping_errors = cfs_atomic_read(&sn->sn_ping_errors); + break; + default: + reply->xst_status = EINVAL; + return 0; + } + + reply->xst_type = request->xst_type; + reply->xst_status = 0; + return 0; +} + +int sfw_get_stats (srpc_stat_reqst_t *request, srpc_stat_reply_t *reply) { sfw_session_t *sn = sfw_data.fw_session; @@ -690,8 +737,8 @@ sfw_unpack_test_req (srpc_msg_t *msg) return; } - if (req->tsr_service == SRPC_SERVICE_PING) { - test_ping_req_t *ping = &req->tsr_u.ping; + if (req->tsr_service == SRPC_SERVICE_ADVPING) { + test_advping_req_t *ping = &req->tsr_u.advping; __swab32s(&ping->png_size); __swab32s(&ping->png_flags); @@ -908,10 +955,10 @@ sfw_create_test_rpc (sfw_test_unit_t *tsu, lnet_process_id_t peer, } cfs_spin_unlock(&tsi->tsi_lock); - + if (rpc == NULL) rpc = srpc_create_client_rpc(peer, tsi->tsi_service, nblk, - blklen, sfw_test_rpc_done, + blklen, sfw_test_rpc_done, sfw_test_rpc_fini, tsu); if (rpc == NULL) { CERROR ("Can't create rpc for test %d\n", tsi->tsi_service); @@ -1247,6 +1294,11 @@ sfw_handle_server_rpc (srpc_server_rpc_t *rpc) &reply->msg_body.stat_reply); break; + case SRPC_SERVICE_QUERY_XSTAT: + rc = sfw_get_xstats(&request->msg_body.xstat_reqst, + &reply->msg_body.xstat_reply); + break; + case SRPC_SERVICE_DEBUG: rc = sfw_debug_session(&request->msg_body.dbg_reqst, &reply->msg_body.dbg_reply); @@ -1376,7 +1428,6 @@ sfw_unpack_message (srpc_msg_t *msg) if (msg->msg_type == SRPC_MSG_STAT_REQST) { srpc_stat_reqst_t *req = &msg->msg_body.stat_reqst; - __swab32s(&req->str_type); __swab64s(&req->str_rpyid); sfw_unpack_sid(req->str_sid); return; @@ -1393,6 +1444,25 @@ sfw_unpack_message (srpc_msg_t *msg) return; } + if (msg->msg_type == SRPC_MSG_XSTAT_REQST) { + srpc_xstat_reqst_t *req = &msg->msg_body.xstat_reqst; + + __swab64s(&req->xst_rpyid); + sfw_unpack_sid(req->xst_sid); + __swab32s(&req->xst_type); + return; + } + + if (msg->msg_type == SRPC_MSG_XSTAT_REPLY) { + srpc_xstat_reply_t *rep = &msg->msg_body.xstat_reply; + + __swab32s(&rep->xst_status); + sfw_unpack_sid(rep->xst_sid); + __swab32s(&rep->xst_type); + sfw_unpack_png_counters(rep->xst_result.ping); + return; + } + if (msg->msg_type == SRPC_MSG_MKSN_REQST) { srpc_mksn_reqst_t *req = &msg->msg_body.mksn_reqst; @@ -1570,6 +1640,11 @@ static srpc_service_t sfw_services[] = 0 }, { + /* sv_id */ SRPC_SERVICE_QUERY_XSTAT, + /* sv_name */ "query extended stats", + 0 + }, + { /* sv_id */ 0, /* sv_name */ NULL, 0 @@ -1586,6 +1661,11 @@ extern srpc_service_t brw_test_service; extern void brw_init_test_client(void); extern void brw_init_test_service(void); +extern sfw_test_client_ops_t advping_test_client; +extern srpc_service_t advping_test_service; +extern void advping_init_test_client(void); +extern void advping_init_test_service(void); + int sfw_startup (void) @@ -1649,6 +1729,11 @@ sfw_startup (void) rc = sfw_register_test(&ping_test_service, &ping_test_client); LASSERT (rc == 0); + advping_init_test_client(); + advping_init_test_service(); + rc = sfw_register_test(&advping_test_service, &advping_test_client); + LASSERT (rc == 0); + error = 0; cfs_list_for_each_entry_typed (tsc, &sfw_data.fw_tests, sfw_test_case_t, tsc_list) { diff --git a/lnet/selftest/rpc.c b/lnet/selftest/rpc.c index 570e0e2..ec14e61 100644 --- a/lnet/selftest/rpc.c +++ b/lnet/selftest/rpc.c @@ -1255,7 +1255,9 @@ srpc_lnet_ev_handler (lnet_event_t *ev) srpc_buffer_t *buffer; srpc_service_t *sv; srpc_msg_t *msg; + __u64 *counter; srpc_msg_type_t type; + int mlen; LASSERT (!cfs_in_interrupt()); @@ -1298,10 +1300,29 @@ srpc_lnet_ev_handler (lnet_event_t *ev) LASSERT (rpcev->ev_fired == 0); rpcev->ev_fired = 1; rpcev->ev_status = (ev->type == LNET_EVENT_UNLINK) ? - -EINTR : ev->status; + -EINTR : ev->status; + if (rpcev->ev_status == 0 && + rpcev->ev_type == SRPC_BULK_REQ_RCVD) { + mlen = ev->mlength; + counter = crpc->crpc_bulk.bk_sink ? + &srpc_data.rpc_counters.bulk_in : + &srpc_data.rpc_counters.bulk_out; + } else { + mlen = 0; + counter = NULL; + } + + swi_schedule_workitem(&crpc->crpc_wi); cfs_spin_unlock(&crpc->crpc_lock); + + if (counter != NULL) { + cfs_spin_lock(&srpc_data.rpc_glock); + *counter += mlen; + cfs_spin_unlock(&srpc_data.rpc_glock); + } + break; case SRPC_REQUEST_RCVD: @@ -1386,9 +1407,9 @@ srpc_lnet_ev_handler (lnet_event_t *ev) cfs_spin_lock(&srpc_data.rpc_glock); if (rpcev->ev_type == SRPC_BULK_GET_RPLD) - srpc_data.rpc_counters.bulk_get += ev->mlength; + srpc_data.rpc_counters.bulk_in += ev->mlength; else - srpc_data.rpc_counters.bulk_put += ev->mlength; + srpc_data.rpc_counters.bulk_out += ev->mlength; cfs_spin_unlock(&srpc_data.rpc_glock); } diff --git a/lnet/selftest/rpc.h b/lnet/selftest/rpc.h index 575887d..164f797 100644 --- a/lnet/selftest/rpc.h +++ b/lnet/selftest/rpc.h @@ -63,6 +63,10 @@ typedef enum { SRPC_MSG_PING_REPLY = 15, SRPC_MSG_JOIN_REQST = 16, SRPC_MSG_JOIN_REPLY = 17, + SRPC_MSG_ADVPING_REQST = 18, + SRPC_MSG_ADVPING_REPLY = 19, + SRPC_MSG_XSTAT_REQST = 20, + SRPC_MSG_XSTAT_REPLY = 21 } srpc_msg_type_t; #include @@ -159,7 +163,7 @@ typedef struct { typedef struct { __u64 str_rpyid; /* reply buffer matchbits */ lst_sid_t str_sid; /* session id */ - __u32 str_type; /* type of stat */ + __u32 str_reserved0; /* unused */ } WIRE_ATTR srpc_stat_reqst_t; typedef struct { @@ -171,15 +175,43 @@ typedef struct { } WIRE_ATTR srpc_stat_reply_t; typedef struct { + __u64 xst_rpyid; /* reply buffer matchbits */ + lst_sid_t xst_sid; /* session id */ + __u32 xst_type; /* type of stat */ +} WIRE_ATTR srpc_xstat_reqst_t; + +/* 'Extended' stats to overcome 160 byte limit on size of + * srpc_msg_t imposed by lustre 1.x backard compatibility. + * This is used for transmitting things which we would like + * to put into srpc_stat_reply_t but have to transmit separately + * because of the size limit */ +typedef struct { + __u32 xst_status; + lst_sid_t xst_sid; /* session id */ + __u32 xst_type; /* type of stat */ + union { + png_counters_t ping; + } xst_result; +} WIRE_ATTR srpc_xstat_reply_t; + + + + +typedef struct { __u32 blk_opc; /* bulk operation code */ __u32 blk_npg; /* # of pages */ __u32 blk_flags; /* reserved flags */ } WIRE_ATTR test_bulk_req_t; typedef struct { + __u32 png_reserved0; /* unused */ + __u32 png_reserved1; /* unused */ +} WIRE_ATTR test_ping_req_t; + +typedef struct { __u32 png_size; /* size of ping message */ __u32 png_flags; /* reserved flags */ -} WIRE_ATTR test_ping_req_t; +} WIRE_ATTR test_advping_req_t; typedef struct { __u64 tsr_rpyid; /* reply buffer matchbits */ @@ -197,6 +229,7 @@ typedef struct { union { test_bulk_req_t bulk; test_ping_req_t ping; + test_advping_req_t advping; } tsr_u; } WIRE_ATTR srpc_test_reqst_t; @@ -215,12 +248,27 @@ typedef struct { } WIRE_ATTR srpc_ping_reqst_t; typedef struct { + __u64 pnr_rpyid; + __u64 pnr_bulkid; /* bulk buffer matchbits */ + __u32 pnr_magic; + __u32 pnr_seq; + __u32 pnr_size; + __u64 pnr_time_usec; +} WIRE_ATTR srpc_advping_reqst_t; + +typedef struct { __u32 pnr_status; __u32 pnr_magic; __u32 pnr_seq; } WIRE_ATTR srpc_ping_reply_t; typedef struct { + __u32 pnr_status; + __u32 pnr_magic; + __u32 pnr_seq; +} WIRE_ATTR srpc_advping_reply_t; + +typedef struct { __u64 brw_rpyid; /* reply buffer matchbits */ __u64 brw_bulkid; /* bulk buffer matchbits */ __u32 brw_rw; /* read or write */ @@ -255,6 +303,8 @@ typedef struct { srpc_batch_reply_t bat_reply; srpc_stat_reqst_t stat_reqst; srpc_stat_reply_t stat_reply; + srpc_xstat_reqst_t xstat_reqst; + srpc_xstat_reply_t xstat_reply; srpc_test_reqst_t tes_reqst; srpc_test_reply_t tes_reply; srpc_join_reqst_t join_reqst; @@ -262,6 +312,8 @@ typedef struct { srpc_ping_reqst_t ping_reqst; srpc_ping_reply_t ping_reply; + srpc_advping_reqst_t advping_reqst; + srpc_advping_reply_t advping_reply; srpc_brw_reqst_t brw_reqst; srpc_brw_reply_t brw_reply; } msg_body; diff --git a/lnet/selftest/selftest.h b/lnet/selftest/selftest.h index 9856472..691f5a2 100644 --- a/lnet/selftest/selftest.h +++ b/lnet/selftest/selftest.h @@ -89,11 +89,13 @@ struct sfw_test_instance; #define SRPC_SERVICE_TEST 4 #define SRPC_SERVICE_QUERY_STAT 5 #define SRPC_SERVICE_JOIN 6 +#define SRPC_SERVICE_QUERY_XSTAT 7 #define SRPC_FRAMEWORK_SERVICE_MAX_ID 10 /* other services start from SRPC_FRAMEWORK_SERVICE_MAX_ID+1 */ #define SRPC_SERVICE_BRW 11 #define SRPC_SERVICE_PING 12 -#define SRPC_SERVICE_MAX_ID 12 +#define SRPC_SERVICE_ADVPING 13 +#define SRPC_SERVICE_MAX_ID 13 #define SRPC_REQUEST_PORTAL 50 /* a lazy portal for framework RPC requests */ @@ -106,6 +108,7 @@ srpc_service2request (int service) { switch (service) { default: + CERROR("unknown service %d\n", service); LBUG (); case SRPC_SERVICE_DEBUG: return SRPC_MSG_DEBUG_REQST; @@ -125,12 +128,18 @@ srpc_service2request (int service) case SRPC_SERVICE_QUERY_STAT: return SRPC_MSG_STAT_REQST; + case SRPC_SERVICE_QUERY_XSTAT: + return SRPC_MSG_XSTAT_REQST; + case SRPC_SERVICE_BRW: return SRPC_MSG_BRW_REQST; case SRPC_SERVICE_PING: return SRPC_MSG_PING_REQST; + case SRPC_SERVICE_ADVPING: + return SRPC_MSG_ADVPING_REQST; + case SRPC_SERVICE_JOIN: return SRPC_MSG_JOIN_REQST; } @@ -306,6 +315,9 @@ typedef struct { cfs_atomic_t sn_refcount; cfs_atomic_t sn_brw_errors; cfs_atomic_t sn_ping_errors; + cfs_atomic_t sn_ping_count; /* Number of successful pings */ + cfs_atomic_t sn_ping_duration; /* Time spent in successful pings */ + } sfw_session_t; #define sfw_sid_equal(sid0, sid1) ((sid0).ses_nid == (sid1).ses_nid && \ @@ -351,8 +363,9 @@ typedef struct sfw_test_instance { cfs_list_t tsi_active_rpcs; /* active rpcs */ union { - test_bulk_req_t bulk; /* bulk parameter */ - test_ping_req_t ping; /* ping parameter */ + test_bulk_req_t bulk; /* bulk parameter */ + test_ping_req_t ping; /* ping parameter */ + test_advping_req_t advping; /* advanced ping parameter */ } tsi_u; } sfw_test_instance_t; diff --git a/lnet/utils/lst.c b/lnet/utils/lst.c index 94cf02a..e0ee4b4 100644 --- a/lnet/utils/lst.c +++ b/lnet/utils/lst.c @@ -326,12 +326,21 @@ lst_node_str2state(char *str) } char * -lst_test_type2name(int type) +lst_test_tse2name(lstcon_test_ent_t *test) { - if (type == LST_TEST_PING) + if (test->tse_type == LST_TEST_PING) return "ping"; - if (type == LST_TEST_BULK) - return "brw"; + + if (test->tse_type == LST_TEST_BULK) { + if (test->tse_params.prm_bulk.blk_opc == LST_BRW_READ) { + return "brw RD"; + } else if (test->tse_params.prm_bulk.blk_opc == LST_BRW_WRITE) { + return "brw WR"; + } else { + fprintf(stderr, "Selftest kernel module is old\n"); + exit (-1); + } + } return "unknown"; } @@ -341,6 +350,8 @@ lst_test_name2type(char *name) { if (strcasecmp(name, "ping") == 0) return LST_TEST_PING; + if (strcasecmp(name, "advping") == 0) + return LST_TEST_ADVPING; if (strcasecmp(name, "brw") == 0) return LST_TEST_BULK; @@ -358,6 +369,10 @@ lst_print_error(char *sub, const char *def_format, ...) { va_list ap; + va_start(ap, def_format); + vfprintf(stderr, def_format, ap); + va_end(ap); + /* local error returned from kernel */ switch (errno) { case ESRCH: @@ -382,10 +397,6 @@ lst_print_error(char *sub, const char *def_format, ...) fprintf(stderr, "%s already exists\n", sub); return; default: - va_start(ap, def_format); - vfprintf(stderr, def_format, ap); - va_end(ap); - return; } } @@ -602,8 +613,8 @@ jt_lst_new_session(int argc, char **argv) rc = lst_new_session_ioctl(name, timeout, force, &session_id); if (rc != 0) { - lst_print_error("session", "Failed to create session: %s\n", - strerror(errno)); + lst_print_error("session", "Failed to create session [%s]: %s\n", + name, strerror(errno)); return rc; } @@ -1413,7 +1424,7 @@ jt_lst_list_group(int argc, char **argv) int lst_stat_ioctl (char *name, int count, lnet_process_id_t *idsp, - int timeout, cfs_list_t *resultp) + int timeout, int type, cfs_list_t *resultp) { lstio_stat_args_t args = {0}; @@ -1424,16 +1435,26 @@ lst_stat_ioctl (char *name, int count, lnet_process_id_t *idsp, args.lstio_sta_count = count; args.lstio_sta_idsp = idsp; args.lstio_sta_resultp = resultp; + args.lstio_sta_type = type; return lst_ioctl (LSTIO_STAT_QUERY, &args, sizeof(args)); } typedef struct { + sfw_counters_t sfw; + srpc_counters_t srpc; + lnet_counters_t lnet; +} lst_stat_counters_t; + +typedef struct { cfs_list_t srp_link; int srp_count; char *srp_name; lnet_process_id_t *srp_ids; - cfs_list_t srp_result[2]; + /* List of lstcon_rpc_ent_t containing lst_stat_counters_t */ + cfs_list_t srp_stat_result[2]; + /* List of lstcon_rpc_ent_t containing png_counters_t */ + cfs_list_t srp_ping_result[2]; } lst_stat_req_param_t; static void @@ -1441,8 +1462,10 @@ lst_stat_req_param_free(lst_stat_req_param_t *srp) { int i; - for (i = 0; i < 2; i++) - lst_free_rpcent(&srp->srp_result[i]); + for (i = 0; i < 2; i++) { + lst_free_rpcent(&srp->srp_stat_result[i]); + lst_free_rpcent(&srp->srp_ping_result[i]); + } if (srp->srp_ids != NULL) free(srp->srp_ids); @@ -1451,7 +1474,7 @@ lst_stat_req_param_free(lst_stat_req_param_t *srp) } static int -lst_stat_req_param_alloc(char *name, lst_stat_req_param_t **srpp, int save_old) +lst_stat_req_param_alloc(char *name, lst_stat_req_param_t **srpp, int save_old, int stat, int ping) { lst_stat_req_param_t *srp = NULL; int count = save_old ? 2 : 1; @@ -1463,8 +1486,10 @@ lst_stat_req_param_alloc(char *name, lst_stat_req_param_t **srpp, int save_old) return -ENOMEM; memset(srp, 0, sizeof(*srp)); - CFS_INIT_LIST_HEAD(&srp->srp_result[0]); - CFS_INIT_LIST_HEAD(&srp->srp_result[1]); + for (i = 0; i < 2; ++i) { + CFS_INIT_LIST_HEAD(&srp->srp_stat_result[i]); + CFS_INIT_LIST_HEAD(&srp->srp_ping_result[i]); + } rc = lst_get_node_count(LST_OPC_GROUP, name, &srp->srp_count, NULL); @@ -1485,13 +1510,24 @@ lst_stat_req_param_alloc(char *name, lst_stat_req_param_t **srpp, int save_old) srp->srp_name = name; for (i = 0; i < count; i++) { - rc = lst_alloc_rpcent(&srp->srp_result[i], srp->srp_count, - sizeof(sfw_counters_t) + - sizeof(srpc_counters_t) + - sizeof(lnet_counters_t)); - if (rc != 0) { - fprintf(stderr, "Out of memory\n"); - break; + if (stat) { + rc = lst_alloc_rpcent(&srp->srp_stat_result[i], srp->srp_count, + sizeof(lst_stat_counters_t)); + + if (rc != 0) { + fprintf(stderr, "out of memory\n"); + break; + } + } + + if (ping) { + rc = lst_alloc_rpcent(&srp->srp_ping_result[i], srp->srp_count, + sizeof(png_counters_t)); + + if (rc != 0) { + fprintf(stderr, "out of memory\n"); + break; + } } } @@ -1505,55 +1541,80 @@ lst_stat_req_param_alloc(char *name, lst_stat_req_param_t **srpp, int save_old) return rc; } -typedef struct { - /* TODO */ - int foo; -} lst_srpc_stat_result; +#define LST_STAT_VALUE_AVG 1 +#define LST_STAT_VALUE_MIN 2 +#define LST_STAT_VALUE_MAX 4 -#define LST_LNET_AVG 0 -#define LST_LNET_MIN 1 -#define LST_LNET_MAX 2 +#define LST_STAT_BW 1 +#define LST_STAT_RATE 2 -typedef struct { - float lnet_avg_sndrate; - float lnet_min_sndrate; - float lnet_max_sndrate; - float lnet_total_sndrate; +#define LST_STAT_INPUT 1 +#define LST_STAT_OUTPUT 2 - float lnet_avg_rcvrate; - float lnet_min_rcvrate; - float lnet_max_rcvrate; - float lnet_total_rcvrate; +#define LST_MEGA_BYTES (1024 * 1024) - float lnet_avg_sndperf; - float lnet_min_sndperf; - float lnet_max_sndperf; - float lnet_total_sndperf; +typedef struct { + float avg; + float min; + float max; + float total; +} lst_stat_value_t; - float lnet_avg_rcvperf; - float lnet_min_rcvperf; - float lnet_max_rcvperf; - float lnet_total_rcvperf; +typedef struct { + lst_stat_value_t lst_snd_rate; + lst_stat_value_t lst_rcv_rate; + lst_stat_value_t lst_out_perf; + lst_stat_value_t lst_in_perf; + int lst_stat_count; +} lst_stat_result_t; - int lnet_stat_count; -} lst_lnet_stat_result_t; +typedef struct { + int lst_stat_count; /* Number of ping averages accumulated */ + int lst_ping_count; /* Number of pings over last stat period */ + int lst_ping_errors; /* Number of ping errors over last stat period */ + lst_stat_value_t lst_ping_time; /* Average ping rtt in us*/ +} lst_ping_result_t; -lst_lnet_stat_result_t lnet_stat_result; +lst_ping_result_t sfwk_ping_result; +lst_stat_result_t lnet_stat_result; +lst_stat_result_t srpc_stat_result; static float -lst_lnet_stat_value(int bw, int send, int off) +lst_stat_value(int lnet, int bw, int dir, int type) { - float *p; + lst_stat_value_t *p; - p = bw ? &lnet_stat_result.lnet_avg_sndperf : - &lnet_stat_result.lnet_avg_sndrate; + if (lnet) { /* LNet stats */ + if (bw == LST_STAT_BW) { /* bandwidth */ + p = (dir == 0) ? &lnet_stat_result.lst_out_perf : + &lnet_stat_result.lst_in_perf; + } else { /* Rates */ + p = (dir == 0) ? &lnet_stat_result.lst_snd_rate : + &lnet_stat_result.lst_rcv_rate; + } + } else { /* SRPC stats */ + if (bw == LST_STAT_BW) { /* bandwidth */ + p = (dir == 0) ? &srpc_stat_result.lst_out_perf : + &srpc_stat_result.lst_in_perf; + } else { /* Rates */ + p = (dir == 0) ? &srpc_stat_result.lst_snd_rate : + &srpc_stat_result.lst_rcv_rate; + } + } - if (!send) - p += 4; + switch (type) { + default: + return 0; - p += off; + case LST_STAT_VALUE_AVG: + return p->avg; - return *p; + case LST_STAT_VALUE_MIN: + return p->min; + + case LST_STAT_VALUE_MAX: + return p->max; + } } static void @@ -1572,117 +1633,155 @@ lst_timeval_diff(struct timeval *tv1, return; } -void -lst_cal_lnet_stat(float delta, lnet_counters_t *lnet_new, - lnet_counters_t *lnet_old) +static void +lst_cal_value(lst_stat_value_t *stat, int const count, float const val) { - float perf; - float rate; + stat->total += val; + stat->avg = stat->total / count; - perf = (float)(lnet_new->send_length - - lnet_old->send_length) / (1024 * 1024) / delta; - lnet_stat_result.lnet_total_sndperf += perf; + if (stat->min > val || + stat->min == 0) + stat->min = val; - if (lnet_stat_result.lnet_min_sndperf > perf || - lnet_stat_result.lnet_min_sndperf == 0) - lnet_stat_result.lnet_min_sndperf = perf; - - if (lnet_stat_result.lnet_max_sndperf < perf) - lnet_stat_result.lnet_max_sndperf = perf; + if (stat->max < val) + stat->max = val; +} - perf = (float)(lnet_new->recv_length - - lnet_old->recv_length) / (1024 * 1024) / delta; - lnet_stat_result.lnet_total_rcvperf += perf; +static void +lst_cal_stat(lst_stat_result_t *result, + float out_perf, float in_perf, + float send_rate, float recv_rate) +{ + result->lst_stat_count ++; + lst_cal_value(&result->lst_out_perf, result->lst_stat_count, out_perf); + lst_cal_value(&result->lst_in_perf, result->lst_stat_count, in_perf); + lst_cal_value(&result->lst_snd_rate, result->lst_stat_count, send_rate); + lst_cal_value(&result->lst_rcv_rate, result->lst_stat_count, recv_rate); +} - if (lnet_stat_result.lnet_min_rcvperf > perf || - lnet_stat_result.lnet_min_rcvperf == 0) - lnet_stat_result.lnet_min_rcvperf = perf; +void +lst_cal_png_stat(png_counters_t *png_new, png_counters_t *png_old) +{ + // NB we assume that no more than 2**31us worth of pings will have + // happened since we last polled, or this calculation comes out wrong. + __u32 period = png_new->ping_duration - png_old->ping_duration; + __u32 count = png_new->ping_count - png_old->ping_count; + __u32 errors = png_new->ping_errors - png_old->ping_errors; - if (lnet_stat_result.lnet_max_rcvperf < perf) - lnet_stat_result.lnet_max_rcvperf = perf; + if (count == 0) { + return; + } - rate = (lnet_new->send_count - lnet_old->send_count) / delta; - lnet_stat_result.lnet_total_sndrate += rate; + float latency = (float)period / (float)count; + sfwk_ping_result.lst_ping_count += count; + sfwk_ping_result.lst_ping_errors += errors; - if (lnet_stat_result.lnet_min_sndrate > rate || - lnet_stat_result.lnet_min_sndrate == 0) - lnet_stat_result.lnet_min_sndrate = rate; + sfwk_ping_result.lst_stat_count++; + lst_cal_value(&sfwk_ping_result.lst_ping_time, + sfwk_ping_result.lst_stat_count, latency); +} - if (lnet_stat_result.lnet_max_sndrate < rate) - lnet_stat_result.lnet_max_sndrate = rate; +void +lst_cal_lnet_stat(float delta, lnet_counters_t *lnet_new, + lnet_counters_t *lnet_old) +{ + float out_perf; + float in_perf; + float send_rate; + float recv_rate; - rate = (lnet_new->recv_count - lnet_old->recv_count) / delta; - lnet_stat_result.lnet_total_rcvrate += rate; + out_perf = (float)(lnet_new->send_length - + lnet_old->send_length) / LST_MEGA_BYTES / delta; + in_perf = (float)(lnet_new->recv_length - + lnet_old->recv_length) / LST_MEGA_BYTES / delta; - if (lnet_stat_result.lnet_min_rcvrate > rate || - lnet_stat_result.lnet_min_rcvrate == 0) - lnet_stat_result.lnet_min_rcvrate = rate; + send_rate = (lnet_new->send_count - lnet_old->send_count) / delta; + recv_rate = (lnet_new->recv_count - lnet_old->recv_count) / delta; - if (lnet_stat_result.lnet_max_rcvrate < rate) - lnet_stat_result.lnet_max_rcvrate = rate; + lst_cal_stat(&lnet_stat_result, out_perf, in_perf, send_rate, recv_rate); +} - lnet_stat_result.lnet_stat_count ++; +void +lst_cal_srpc_stat(float delta, srpc_counters_t *srpc_new, + srpc_counters_t *srpc_old) +{ + float out_perf; + float in_perf; + float send_rate; + float recv_rate; - lnet_stat_result.lnet_avg_sndrate = lnet_stat_result.lnet_total_sndrate / - lnet_stat_result.lnet_stat_count; - lnet_stat_result.lnet_avg_rcvrate = lnet_stat_result.lnet_total_rcvrate / - lnet_stat_result.lnet_stat_count; + out_perf = (float)(srpc_new->bulk_out - + srpc_old->bulk_out) / LST_MEGA_BYTES / delta; + in_perf = (float)(srpc_new->bulk_in - + srpc_old->bulk_in) / LST_MEGA_BYTES / delta; - lnet_stat_result.lnet_avg_sndperf = lnet_stat_result.lnet_total_sndperf / - lnet_stat_result.lnet_stat_count; - lnet_stat_result.lnet_avg_rcvperf = lnet_stat_result.lnet_total_rcvperf / - lnet_stat_result.lnet_stat_count; + recv_rate = (srpc_new->rpcs_rcvd - srpc_old->rpcs_rcvd) / delta; + send_rate = (srpc_new->rpcs_sent - srpc_old->rpcs_sent) / delta; + lst_cal_stat(&srpc_stat_result, out_perf, in_perf, send_rate, recv_rate); } void -lst_print_lnet_stat(char *name, int bwrt, int rdwr, int type) +lst_print_lst_stat(int lnet, char *name, int bwrt, int io, int type) { - int start1 = 0; - int end1 = 1; - int start2 = 0; - int end2 = 1; + int start; + int end; int i; - int j; - if (lnet_stat_result.lnet_stat_count == 0) + if ((lnet && lnet_stat_result.lst_stat_count == 0) || + (!lnet && srpc_stat_result.lst_stat_count == 0)) return; - if (bwrt == 1) /* bw only */ - start1 = 1; + if ((bwrt & LST_STAT_RATE) != 0) { + fprintf(stdout, "[%s Rates of %s]\n", lnet ? "LNet" : "SRPC", name); + + for (i = 0; i <= 1; i++) { + if (lnet) + fprintf(stdout, "[%s] ", i == 0 ? "Snd" : "Rcv"); + else + fprintf(stdout, "[%s] ", i == 0 ? "Req" : "Srv"); + + if ((type & LST_STAT_VALUE_AVG) != 0) { + fprintf(stdout, "Avg: %-8.1f RPC/s ", + lst_stat_value(lnet, LST_STAT_RATE, i, LST_STAT_VALUE_AVG)); + } + + if ((type & LST_STAT_VALUE_MIN) != 0) { + fprintf(stdout, "Min: %-8.1f RPC/s ", + lst_stat_value(lnet, LST_STAT_RATE, i, LST_STAT_VALUE_MIN)); + } - if (bwrt == 2) /* rates only */ - end1 = 0; + if ((type & LST_STAT_VALUE_MAX) != 0) { + fprintf(stdout, "Max: %-8.1f RPC/s", + lst_stat_value(lnet, LST_STAT_RATE, i, LST_STAT_VALUE_MAX)); + } - if (rdwr == 1) /* recv only */ - start2 = 1; + fprintf(stdout, "\n"); + } + } - if (rdwr == 2) /* send only */ - end2 = 0; + start = ((io & LST_STAT_OUTPUT) == 0) ? 1 : 0; + end = ((io & LST_STAT_INPUT) == 0) ? 0 : 1; - for (i = start1; i <= end1; i++) { - fprintf(stdout, "[LNet %s of %s]\n", - i == 0 ? "Rates" : "Bandwidth", name); + if ((bwrt & LST_STAT_BW) != 0) { + fprintf(stdout, "[%s Bandwidth of %s]\n", lnet ? "LNet" : "SRPC", name); - for (j = start2; j <= end2; j++) { - fprintf(stdout, "[%c] ", j == 0 ? 'R' : 'W'); + for (i = start; i <= end; i++) { + fprintf(stdout, "[%s] ", i == 0 ? "Out" : "In "); - if ((type & 1) != 0) { - fprintf(stdout, i == 0 ? "Avg: %-8.0f RPC/s " : - "Avg: %-8.2f MB/s ", - lst_lnet_stat_value(i, j, 0)); + if ((type & LST_STAT_VALUE_AVG) != 0) { + fprintf(stdout, "Avg: %-8.2f MB/s ", + lst_stat_value(lnet, LST_STAT_BW, i, LST_STAT_VALUE_AVG)); } - if ((type & 2) != 0) { - fprintf(stdout, i == 0 ? "Min: %-8.0f RPC/s " : - "Min: %-8.2f MB/s ", - lst_lnet_stat_value(i, j, 1)); + if ((type & LST_STAT_VALUE_MIN) != 0) { + fprintf(stdout, "Min: %-8.2f MB/s ", + lst_stat_value(lnet, LST_STAT_BW, i, LST_STAT_VALUE_MIN)); } - if ((type & 4) != 0) { - fprintf(stdout, i == 0 ? "Max: %-8.0f RPC/s" : - "Max: %-8.2f MB/s", - lst_lnet_stat_value(i, j, 2)); + if ((type & LST_STAT_VALUE_MAX) != 0) { + fprintf(stdout, "Max: %-8.2f MB/s", + lst_stat_value(lnet, LST_STAT_BW, i, LST_STAT_VALUE_MAX)); } fprintf(stdout, "\n"); @@ -1690,19 +1789,15 @@ lst_print_lnet_stat(char *name, int bwrt, int rdwr, int type) } } + void -lst_print_stat(char *name, cfs_list_t *resultp, - int idx, int lnet, int bwrt, int rdwr, int type) +lst_print_ping_stats(char *name, cfs_list_t *resultp, int idx, int type) { cfs_list_t tmp[2]; lstcon_rpc_ent_t *new; lstcon_rpc_ent_t *old; - sfw_counters_t *sfwk_new; - sfw_counters_t *sfwk_old; - srpc_counters_t *srpc_new; - srpc_counters_t *srpc_old; - lnet_counters_t *lnet_new; - lnet_counters_t *lnet_old; + png_counters_t *ping_new; + png_counters_t *ping_old; struct timeval tv; float delta; int errcount = 0; @@ -1710,7 +1805,7 @@ lst_print_stat(char *name, cfs_list_t *resultp, CFS_INIT_LIST_HEAD(&tmp[0]); CFS_INIT_LIST_HEAD(&tmp[1]); - memset(&lnet_stat_result, 0, sizeof(lnet_stat_result)); + memset(&sfwk_ping_result, 0, sizeof(sfwk_ping_result)); while (!cfs_list_empty(&resultp[idx])) { if (cfs_list_empty(&resultp[1 - idx])) { @@ -1724,11 +1819,13 @@ lst_print_stat(char *name, cfs_list_t *resultp, rpe_link); /* first time get stats result, can't calculate diff */ - if (new->rpe_peer.nid == LNET_NID_ANY) + if (old->rpe_peer.nid == LNET_NID_ANY) { break; + } if (new->rpe_peer.nid != old->rpe_peer.nid || new->rpe_peer.pid != old->rpe_peer.pid) { + fprintf (stderr, "Group changed, re-run stat\n"); /* Something wrong. i.e, somebody change the group */ break; } @@ -1741,27 +1838,108 @@ lst_print_stat(char *name, cfs_list_t *resultp, if (new->rpe_rpc_errno != 0 || new->rpe_fwk_errno != 0 || old->rpe_rpc_errno != 0 || old->rpe_fwk_errno != 0) { + printf ("errors\n"); errcount ++; continue; } - sfwk_new = (sfw_counters_t *)&new->rpe_payload[0]; - sfwk_old = (sfw_counters_t *)&old->rpe_payload[0]; + ping_new = (png_counters_t *)&new->rpe_payload[0]; + ping_old = (png_counters_t *)&old->rpe_payload[0]; - srpc_new = (srpc_counters_t *)((char *)sfwk_new + sizeof(*sfwk_new)); - srpc_old = (srpc_counters_t *)((char *)sfwk_old + sizeof(*sfwk_old)); + lst_cal_png_stat(ping_new, ping_old); + } - lnet_new = (lnet_counters_t *)((char *)srpc_new + sizeof(*srpc_new)); - lnet_old = (lnet_counters_t *)((char *)srpc_old + sizeof(*srpc_old)); + cfs_list_splice(&tmp[idx], &resultp[idx]); + cfs_list_splice(&tmp[1 - idx], &resultp[1 - idx]); - lst_timeval_diff(&new->rpe_stamp, &old->rpe_stamp, &tv); + if (errcount > 0) + fprintf(stdout, "Failed to get ping stats on %d nodes\n", errcount); + + fprintf(stdout, "[Ping time of %s over %d good pings (%d bad pings)]\n", name, + sfwk_ping_result.lst_ping_count, sfwk_ping_result.lst_ping_errors); + + if ((type & LST_STAT_VALUE_AVG) != 0) { + fprintf (stdout, "Avg: %-8.2f us ", + sfwk_ping_result.lst_ping_time.avg); + } + + if ((type & LST_STAT_VALUE_MIN) != 0) { + fprintf (stdout, "Min: %-8.2f us ", + sfwk_ping_result.lst_ping_time.min); + } + + if ((type & LST_STAT_VALUE_MAX) != 0) { + fprintf (stdout, "Max: %-8.2f us ", + sfwk_ping_result.lst_ping_time.max); + } + + fprintf(stdout, "\n"); +} + +void +lst_print_stat(char *name, cfs_list_t *resultp, + int idx, int lnet, int bwrt, int io, int type) +{ + cfs_list_t tmp[2]; + lstcon_rpc_ent_t *new; + lstcon_rpc_ent_t *old; + lst_stat_counters_t *new_counters; + lst_stat_counters_t *old_counters; + struct timeval tv; + float delta; + int errcount = 0; + int i; + + CFS_INIT_LIST_HEAD(&tmp[0]); + CFS_INIT_LIST_HEAD(&tmp[1]); + + memset(&lnet_stat_result, 0, sizeof(lnet_stat_result)); + memset(&srpc_stat_result, 0, sizeof(srpc_stat_result)); + + while (!cfs_list_empty(&resultp[idx])) { + if (cfs_list_empty(&resultp[1 - idx])) { + fprintf(stderr, "Group is changed, re-run stat\n"); + break; + } + + new = cfs_list_entry(resultp[idx].next, lstcon_rpc_ent_t, + rpe_link); + old = cfs_list_entry(resultp[1 - idx].next, lstcon_rpc_ent_t, + rpe_link); + + /* first time get stats result, can't calculate diff */ + if (old->rpe_peer.nid == LNET_NID_ANY) + break; + + if (new->rpe_peer.nid != old->rpe_peer.nid || + new->rpe_peer.pid != old->rpe_peer.pid) { + fprintf(stderr, "Group is changed, re-run stat\n"); + /* Something wrong. i.e, somebody change the group */ + break; + } - delta = tv.tv_sec + (float)tv.tv_usec/1000000; + cfs_list_del(&new->rpe_link); + cfs_list_add_tail(&new->rpe_link, &tmp[idx]); - if (!lnet) /* TODO */ + cfs_list_del(&old->rpe_link); + cfs_list_add_tail(&old->rpe_link, &tmp[1 - idx]); + + if (new->rpe_rpc_errno != 0 || new->rpe_fwk_errno != 0 || + old->rpe_rpc_errno != 0 || old->rpe_fwk_errno != 0) { + errcount ++; continue; + } + + new_counters = (lst_stat_counters_t *)&new->rpe_payload[0]; + + old_counters = (lst_stat_counters_t *)&old->rpe_payload[0]; + + lst_timeval_diff(&new->rpe_stamp, &old->rpe_stamp, &tv); - lst_cal_lnet_stat(delta, lnet_new, lnet_old); + delta = tv.tv_sec + (float)tv.tv_usec / 1000000; + + lst_cal_lnet_stat(delta, &new_counters->lnet, &old_counters->lnet); + lst_cal_srpc_stat(delta, &new_counters->srpc, &old_counters->srpc); } cfs_list_splice(&tmp[idx], &resultp[idx]); @@ -1770,10 +1948,7 @@ lst_print_stat(char *name, cfs_list_t *resultp, if (errcount > 0) fprintf(stdout, "Failed to stat on %d nodes\n", errcount); - if (!lnet) /* TODO */ - return; - - lst_print_lnet_stat(name, bwrt, rdwr, type); + lst_print_lst_stat(lnet, name, bwrt, io, type); } int @@ -1785,13 +1960,18 @@ jt_lst_stat(int argc, char **argv) int optidx = 0; int timeout = 5; /* default timeout, 5 sec */ int delay = 5; /* default delay, 5 sec */ - int lnet = 1; /* lnet stat by default */ - int bwrt = 0; - int rdwr = 0; + int bwrt = -1; + int io = -1; int type = -1; int idx = 0; int rc; int c; + enum { + MODE_DEFAULT = 0x00, + MODE_LNET = 0x01, + MODE_SRPC = 0x02, + MODE_PING = 0x04 + } mode = MODE_DEFAULT; static struct option stat_opts[] = { @@ -1799,10 +1979,11 @@ jt_lst_stat(int argc, char **argv) {"delay" , required_argument, 0, 'd' }, {"lnet" , no_argument, 0, 'l' }, {"rpc" , no_argument, 0, 'c' }, + {"ping" , no_argument, 0, 'p' }, {"bw" , no_argument, 0, 'b' }, {"rate" , no_argument, 0, 'a' }, - {"read" , no_argument, 0, 'r' }, - {"write" , no_argument, 0, 'w' }, + {"input" , no_argument, 0, 'i' }, + {"output" , no_argument, 0, 'o' }, {"avg" , no_argument, 0, 'g' }, {"min" , no_argument, 0, 'n' }, {"max" , no_argument, 0, 'x' }, @@ -1816,7 +1997,7 @@ jt_lst_stat(int argc, char **argv) } while (1) { - c = getopt_long(argc, argv, "t:d:lcbarwgnx", stat_opts, &optidx); + c = getopt_long(argc, argv, "t:d:lcpbarwgnx", stat_opts, &optidx); if (c == -1) break; @@ -1829,43 +2010,58 @@ jt_lst_stat(int argc, char **argv) delay = atoi(optarg); break; case 'l': - lnet = 1; + mode |= MODE_LNET; + break; case 'c': - lnet = 0; + mode |= MODE_SRPC; + + break; + case 'p': + mode |= MODE_PING; + break; case 'b': - bwrt |= 1; + if (bwrt == -1) + bwrt = LST_STAT_BW; + else + bwrt |= LST_STAT_BW; break; case 'a': - bwrt |= 2; + if (bwrt == -1) + bwrt = LST_STAT_RATE; + else + bwrt |= LST_STAT_RATE; break; - case 'r': - rdwr |= 1; + case 'i': + if (io == -1) + io = LST_STAT_INPUT; + else + io |= LST_STAT_INPUT; break; - case 'w': - rdwr |= 2; + case 'o': + if (io == -1) + io == LST_STAT_OUTPUT; + else + io |= LST_STAT_OUTPUT; break; case 'g': - if (type == -1) { - type = 1; - break; - } - type |= 1; + if (type == -1) + type = LST_STAT_VALUE_AVG; + else + type |= LST_STAT_VALUE_AVG; break; case 'n': - if (type == -1) { - type = 2; - break; - } - type |= 2; + if (type == -1) + type = LST_STAT_VALUE_MIN; + else + type |= LST_STAT_VALUE_MIN; break; case 'x': - if (type == -1) { - type = 4; - break; - } - type |= 4; + if (type == -1) + type = LST_STAT_VALUE_MAX; + else + type |= LST_STAT_VALUE_MAX;; break; default: lst_print_usage(argv[0]); @@ -1878,6 +2074,10 @@ jt_lst_stat(int argc, char **argv) return -1; } + if (mode == MODE_DEFAULT) { + mode = MODE_SRPC; + } + if (timeout <= 0 || delay <= 0) { fprintf(stderr, "Invalid timeout or delay value\n"); return -1; @@ -1886,7 +2086,10 @@ jt_lst_stat(int argc, char **argv) CFS_INIT_LIST_HEAD(&head); while (optind < argc) { - rc = lst_stat_req_param_alloc(argv[optind++], &srp, 1); + rc = lst_stat_req_param_alloc(argv[optind++], &srp, 1, + mode & (MODE_LNET | MODE_SRPC), /* Populate srp_stat_result ? */ + mode & MODE_PING); /* Populate srp_ping_result ? */ + if (rc != 0) goto out; @@ -1905,19 +2108,45 @@ jt_lst_stat(int argc, char **argv) cfs_list_for_each_entry_typed(srp, &head, lst_stat_req_param_t, srp_link) { - rc = lst_stat_ioctl(srp->srp_name, - srp->srp_count, srp->srp_ids, - timeout, &srp->srp_result[idx]); - if (rc == -1) { - lst_print_error("stat", "Failed to stat %s: %s\n", - srp->srp_name, strerror(errno)); - goto out; + if (mode & (MODE_LNET | MODE_SRPC)) { + rc = lst_stat_ioctl(srp->srp_name, + srp->srp_count, srp->srp_ids, + timeout, -1, &srp->srp_stat_result[idx]); + if (rc == -1) { + lst_print_error("stat", "Failed to stat %s: %s\n", + srp->srp_name, strerror(errno)); + goto out; + } + + if (mode & MODE_SRPC) { + lst_print_stat(srp->srp_name, srp->srp_stat_result, + idx, 0, bwrt, io, type); + } + + if (mode & MODE_LNET) { + lst_print_stat(srp->srp_name, srp->srp_stat_result, + idx, 1, bwrt, io, type); + } + + lst_reset_rpcent(&srp->srp_stat_result[1 - idx]); + } - lst_print_stat(srp->srp_name, srp->srp_result, - idx, lnet, bwrt, rdwr, type); + if (mode & MODE_PING) { + rc = lst_stat_ioctl(srp->srp_name, + srp->srp_count, srp->srp_ids, + timeout, XSTAT_TYPE_PING, &srp->srp_ping_result[idx]); + if (rc == -1) { + lst_print_error("stat", "Failed to stat %s: %s\n", + srp->srp_name, strerror(errno)); + goto out; + } + + lst_print_ping_stats(srp->srp_name, srp->srp_ping_result, + idx, type); - lst_reset_rpcent(&srp->srp_result[1 - idx]); + lst_reset_rpcent(&srp->srp_ping_result[1 - idx]); + } } idx = 1 - idx; @@ -1940,9 +2169,7 @@ jt_lst_show_error(int argc, char **argv) cfs_list_t head; lst_stat_req_param_t *srp; lstcon_rpc_ent_t *ent; - sfw_counters_t *sfwk; - srpc_counters_t *srpc; - lnet_counters_t *lnet; + lst_stat_counters_t *counters; int show_rpc = 1; int optidx = 0; int rc = 0; @@ -1986,7 +2213,7 @@ jt_lst_show_error(int argc, char **argv) CFS_INIT_LIST_HEAD(&head); while (optind < argc) { - rc = lst_stat_req_param_alloc(argv[optind++], &srp, 0); + rc = lst_stat_req_param_alloc(argv[optind++], &srp, 0, 1, 0); if (rc != 0) goto out; @@ -1996,7 +2223,7 @@ jt_lst_show_error(int argc, char **argv) cfs_list_for_each_entry_typed(srp, &head, lst_stat_req_param_t, srp_link) { rc = lst_stat_ioctl(srp->srp_name, srp->srp_count, - srp->srp_ids, 10, &srp->srp_result[0]); + srp->srp_ids, 10, -1, &srp->srp_stat_result[0]); if (rc == -1) { lst_print_error(srp->srp_name, "Failed to show errors of %s: %s\n", @@ -2008,7 +2235,7 @@ jt_lst_show_error(int argc, char **argv) ecount = 0; - cfs_list_for_each_entry_typed(ent, &srp->srp_result[0], + cfs_list_for_each_entry_typed(ent, &srp->srp_stat_result[0], lstcon_rpc_ent_t, rpe_link) { if (ent->rpe_rpc_errno != 0) { ecount ++; @@ -2024,30 +2251,29 @@ jt_lst_show_error(int argc, char **argv) continue; } - sfwk = (sfw_counters_t *)&ent->rpe_payload[0]; - srpc = (srpc_counters_t *)((char *)sfwk + sizeof(*sfwk)); - lnet = (lnet_counters_t *)((char *)srpc + sizeof(*srpc)); + counters = (lst_stat_counters_t *)&ent->rpe_payload[0]; - if (srpc->errors == 0 && - sfwk->brw_errors == 0 && sfwk->ping_errors == 0) + if (counters->srpc.errors == 0 && + counters->sfw.brw_errors == 0 && counters->sfw.ping_errors == 0) continue; if (!show_rpc && - sfwk->brw_errors == 0 && sfwk->ping_errors == 0) + counters->sfw.brw_errors == 0 && counters->sfw.ping_errors == 0) continue; ecount ++; fprintf(stderr, "%s: [Session %d brw errors, %d ping errors]%c", libcfs_id2str(ent->rpe_peer), - sfwk->brw_errors, sfwk->ping_errors, + counters->sfw.brw_errors, counters->sfw.ping_errors, show_rpc ? ' ' : '\n'); if (!show_rpc) continue; fprintf(stderr, "[RPC: %d errors, %d dropped, %d expired]\n", - srpc->errors, srpc->rpcs_dropped, srpc->rpcs_expired); + counters->srpc.errors, counters->srpc.rpcs_dropped, + counters->srpc.rpcs_expired); } fprintf(stdout, "Total %d error nodes in %s\n", ecount, srp->srp_name); @@ -2308,7 +2534,7 @@ jt_lst_stop_batch(int argc, char **argv) lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) break; - fprintf(stdout, "%d batch in stopping\n", + fprintf(stdout, "%d batch is stopping\n", lstcon_tsbqry_stat_run(&trans_stat, 0)); sleep(1); } @@ -2551,11 +2777,22 @@ loop: test = 1; /* starting from test 1 */ } else { + char params[64]; + + if (ent.u.tbe_test.tse_type == LST_TEST_BULK) + snprintf(params, 64, ", size = %d", + ent.u.tbe_test.tse_params.prm_bulk.blk_size); + else if (ent.u.tbe_test.tse_type == LST_TEST_ADVPING) + snprintf(params, 64, ", size = %d", + ent.u.tbe_test.tse_params.prm_advping.png_size); + else + params[0] = 0; + fprintf(stdout, - "\tTest %d(%s) (loop: %d, concurrency: %d)\n", - test, lst_test_type2name(ent.u.tbe_test.tse_type), - ent.u.tbe_test.tse_loop, - ent.u.tbe_test.tse_concur); + "\tTest %d(%s) (loop %d, concurrency %d, distribute %d:%d%s)\n", + test, lst_test_tse2name(&ent.u.tbe_test), + ent.u.tbe_test.tse_loop, ent.u.tbe_test.tse_concur, + ent.u.tbe_test.tse_dist, ent.u.tbe_test.tse_span, params); ntest --; test ++; } @@ -2823,6 +3060,81 @@ lst_parse_distribute(char *dstr, int *dist, int *span) return 0; } +/** + * @param token + * Null terminated string like "10", "10k", "10KB", "10m" + * representing a size in bytes. + * + * @return + * -1 on invalid string, else size in bytes. + */ +static int +parse_size(char const *tok, int multiple, int min, int max) +{ + int size; + char *end = NULL; + + size = strtol(tok, &end, 0); + if (size < 0 || end == tok) { + fprintf(stderr, "Invalid size '%s'\n", tok); + return -1; + } + + if (end != NULL) { + if (*end == 'k' || *end == 'K') { + size *= 1024; + } else if (*end == 'm' || *end == 'M') { + size *= LST_MEGA_BYTES; + } else if (*end != '\0') { + fprintf(stderr, "Invalid trailing character on size '%c'\n", *end); + return -1; + } + } + + if (size < min || size > max) { + fprintf(stderr, "Size %d bytes is out of bounds (%d-%d)\n", + size, min, max); + return -1; + } else if (size % multiple != 0) { + fprintf(stderr, "Size %d is not a multiple of %d\n", + size, multiple); + return -1; + } + + return size; +} + +int +lst_get_advping_param(int argc, char **argv, lst_test_advping_param_t *ping) +{ + char *tok = NULL; + char *end = NULL; + int rc = 0; + int i = 0; + + ping->png_size = 0; + ping->png_flags = 0; + + while (i < argc) { + if (strcasestr(argv[i], "size=") == argv[i] || + strcasestr(argv[i], "s=") == argv[i]) { + tok = strchr(argv[i], '=') + 1; + + ping->png_size = parse_size(tok, CFS_PAGE_SIZE, 0, CFS_PAGE_SIZE * LNET_MAX_IOV); + if (ping->png_size < 0) { + return -1; + } + } else { + fprintf(stderr, "Unknown parameter: %s\n", argv[i]); + return -1; + } + + i++; + } + + return rc; +} + int lst_get_bulk_param(int argc, char **argv, lst_test_bulk_param_t *bulk) { @@ -2844,8 +3156,10 @@ lst_get_bulk_param(int argc, char **argv, lst_test_bulk_param_t *bulk) bulk->blk_flags = LST_BRW_CHECK_FULL; } else if (strcasecmp(tok, "simple") == 0) { bulk->blk_flags = LST_BRW_CHECK_SIMPLE; + } else if (strcasecmp(tok, "none") == 0) { + bulk->blk_flags = LST_BRW_CHECK_NONE; } else { - fprintf(stderr, "Unknow flag %s\n", tok); + fprintf(stderr, "Unknown flag %s\n", tok); return -1; } @@ -2853,26 +3167,11 @@ lst_get_bulk_param(int argc, char **argv, lst_test_bulk_param_t *bulk) strcasestr(argv[i], "s=") == argv[i]) { tok = strchr(argv[i], '=') + 1; - bulk->blk_size = strtol(tok, &end, 0); - if (bulk->blk_size <= 0) { - fprintf(stderr, "Invalid size %s\n", tok); + bulk->blk_size = parse_size(tok, CFS_PAGE_SIZE, + 1, CFS_PAGE_SIZE * LNET_MAX_IOV); + if (bulk->blk_size < 0) { return -1; } - - if (end == NULL) - return 0; - - if (*end == 'k' || *end == 'K') - bulk->blk_size *= 1024; - else if (*end == 'm' || *end == 'M') - bulk->blk_size *= 1024 * 1024; - - if (bulk->blk_size > CFS_PAGE_SIZE * LNET_MAX_IOV) { - fprintf(stderr, "Size exceed limitation: %d bytes\n", - bulk->blk_size); - return -1; - } - } else if (strcasecmp(argv[i], "read") == 0 || strcasecmp(argv[i], "r") == 0) { bulk->blk_opc = LST_BRW_READ; @@ -2882,7 +3181,7 @@ lst_get_bulk_param(int argc, char **argv, lst_test_bulk_param_t *bulk) bulk->blk_opc = LST_BRW_WRITE; } else { - fprintf(stderr, "Unknow parameter: %s\n", argv[i]); + fprintf(stderr, "Unknown parameter: %s\n", argv[i]); return -1; } @@ -2896,16 +3195,33 @@ int lst_get_test_param(char *test, int argc, char **argv, void **param, int *plen) { lst_test_bulk_param_t *bulk = NULL; + lst_test_advping_param_t *ping = NULL; int type; type = lst_test_name2type(test); if (type < 0) { - fprintf(stderr, "Unknow test name %s\n", test); + fprintf(stderr, "Unknown test name %s\n", test); return -1; } switch (type) { - case LST_TEST_PING: + case LST_TEST_ADVPING: + ping = malloc(sizeof(*ping)); + if (ping == NULL) { + fprintf(stderr, "Out of memory\n"); + return -1; + } + + memset(ping, 0, sizeof(*ping)); + + if (lst_get_advping_param(argc, argv, ping) != 0) { + free(ping); + return -1; + } + + *param = ping; + *plen = sizeof(*ping); + break; case LST_TEST_BULK: @@ -2926,12 +3242,13 @@ lst_get_test_param(char *test, int argc, char **argv, void **param, int *plen) *plen = sizeof(*bulk); break; - + case LST_TEST_PING: + /* NB no params for PING */ + break; default: break; } - /* TODO: parse more parameter */ return type; } @@ -3128,30 +3445,30 @@ static command_t lst_cmdlist[] = { {"show_session", jt_lst_show_session, NULL, "Usage: lst show_session" }, {"ping", jt_lst_ping , NULL, - "Usage: lst ping [--group NAME] [--batch NAME] [--session] [--nodes IDS]" }, + "Usage: lst ping [--group NAME] [--batch NAME] [--session] [--nodes NIDs]" }, {"add_group", jt_lst_add_group, NULL, - "Usage: lst group NAME IDs [IDs]..." }, + "Usage: lst add_group NAME NIDs [NIDs]..." }, {"del_group", jt_lst_del_group, NULL, "Usage: lst del_group NAME" }, {"update_group", jt_lst_update_group, NULL, - "Usage: lst update_group NAME [--clean] [--refresh] [--remove IDs]" }, + "Usage: lst update_group NAME [--clean] [--refresh] [--remove NIDs]" }, {"list_group", jt_lst_list_group, NULL, - "Usage: lst list_group [--active] [--busy] [--down] [--unknown] GROUP ..." }, + "Usage: lst list_group [--active] [--busy] [--down] [--unknown] NAME ..." }, {"stat", jt_lst_stat, NULL, - "Usage: lst stat [--bw] [--rate] [--read] [--write] [--max] [--min] [--avg] " - " [--timeout #] [--delay #] GROUP [GROUP]" }, + "Usage: lst stat [--bw] [--rate] [--input] [--output] [--max] [--min] [--avg] " + "[--lnet] [--ping] [--timeout #] [--delay #] GROUP | NIDs [GROUP | NIDs]" }, {"show_error", jt_lst_show_error, NULL, - "Usage: lst show_error NAME | IDS ..." }, + "Usage: lst show_error GROUP | NIDs ..." }, {"add_batch", jt_lst_add_batch, NULL, "Usage: lst add_batch NAME" }, {"run", jt_lst_start_batch, NULL, - "Usage: lst run [--timeout TIME] [NAME]" }, + "Usage: lst run [--timeout TIME] [BATCH]" }, {"stop", jt_lst_stop_batch, NULL, - "Usage: lst stop [--force] BATCH_NAME" }, + "Usage: lst stop [--force] BATCH" }, {"list_batch", jt_lst_list_batch, NULL, - "Usage: lst list_batch NAME [--test ID] [--server]" }, + "Usage: lst list_batch BATCH [--test ID] [--server]" }, {"query", jt_lst_query_batch, NULL, - "Usage: lst query [--test ID] [--server] [--timeout TIME] NAME" }, + "Usage: lst query [--test ID] [--server] [--timeout TIME] GROUP | NIDs" }, {"add_test", jt_lst_add_test, NULL, "Usage: lst add_test [--batch BATCH] [--loop #] [--concurrency #] " " [--distribute #:#] [--from GROUP] [--to GROUP] TEST..." }, diff --git a/lustre/doc/lst.8 b/lustre/doc/lst.8 index 886ed4f..23ac6a7 100644 --- a/lustre/doc/lst.8 +++ b/lustre/doc/lst.8 @@ -57,6 +57,28 @@ LNDs and LNET. A user can use lstclient to join a self-test session: .nf lstclient -sesid CONSOLE_NID group NAME .fi +.LP +.SH TEST PARAMETERS +Tests are added using the following syntax: +.br +.B lctl add_test --from --to [--batch ] [--loop ] [--concurrency ] [--distribute ] +.LP +.SH TEST TYPES +There are three tests, each of which has its own set of +parameters. +.LP +.B bulk_rw [read|write] [check=] [size=] +.LP +Defaults: bulk_rw read check=none size=4096 +.LP +.B ping +.LP +Defaults: N/A +.LP +.B adv_ping [size=] +.LP +Defaults: adv_ping size=0 +.LP .SH EXAMPLE SCRIPT Below is a sample LNET self-test script which simulates the traffic pattern of a set of Lustre servers on a TCP network, accessed by Lustre