diff --git a/lnet/include/lnet/lnetst.h b/lnet/include/lnet/lnetst.h index 6ca9020..8ac2591 100644 --- a/lnet/include/lnet/lnetst.h +++ b/lnet/include/lnet/lnetst.h @@ -89,6 +89,29 @@ typedef struct { int nde_state; /* state of node */ } lstcon_node_ent_t; /*** node entry, for list_group command */ + +typedef enum { + LST_BRW_READ = 1, + LST_BRW_WRITE = 2 +} lst_brw_type_t; + +typedef enum { + LST_BRW_CHECK_NONE = 1, + LST_BRW_CHECK_SIMPLE = 2, + LST_BRW_CHECK_FULL = 3 +} lst_brw_flags_t; + +typedef struct { + int blk_opc; /* bulk operation code */ + int blk_size; /* size (bytes) */ + int blk_flags; /* reserved flags */ +} lst_test_bulk_param_t; + +typedef struct { + int png_size; /* size of ping message */ + int png_flags; /* reserved flags */ +} lst_test_ping_param_t; + typedef struct { int nle_nnode; /* # of nodes */ int nle_nactive; /* # of active nodes */ @@ -101,6 +124,12 @@ typedef struct { int tse_type; /* test type */ int tse_loop; /* loop count */ int tse_concur; /* concurrency of test */ + int tse_dist; /* distribution */ + int tse_span; /* span */ + union { + lst_test_bulk_param_t prm_bulk; + lst_test_ping_param_t prm_ping; + } tse_params; } lstcon_test_ent_t; /*** test summary entry, for list_batch command */ typedef struct { @@ -431,31 +460,6 @@ typedef struct { cfs_list_t *lstio_tes_resultp; /* OUT: list head of result buffer */ } lstio_test_args_t; -typedef enum { - LST_BRW_READ = 1, - LST_BRW_WRITE = 2 -} lst_brw_type_t; - -typedef enum { - LST_BRW_CHECK_NONE = 1, - LST_BRW_CHECK_SIMPLE = 2, - LST_BRW_CHECK_FULL = 3 -} lst_brw_flags_t; - -typedef struct { - int blk_opc; /* bulk operation code */ - int blk_size; /* size (bytes) */ - int blk_time; /* time of running the test*/ - int blk_flags; /* reserved flags */ -} lst_test_bulk_param_t; - -typedef struct { - int png_size; /* size of ping message */ - int png_time; /* time */ - int png_loop; /* loop */ - int png_flags; /* reserved flags */ -} lst_test_ping_param_t; - /* more tests */ #include typedef struct { @@ -464,8 +468,8 @@ typedef struct { __u32 rpcs_rcvd; __u32 rpcs_dropped; __u32 rpcs_expired; - __u64 bulk_get; - __u64 bulk_put; + __u64 bulk_in; + __u64 bulk_out; } WIRE_ATTR srpc_counters_t; typedef struct { @@ -474,6 +478,8 @@ typedef struct { __u32 zombie_sessions; __u32 brw_errors; __u32 ping_errors; + __u32 ping_count; + __u32 ping_duration; } WIRE_ATTR sfw_counters_t; #include diff --git a/lnet/selftest/conrpc.c b/lnet/selftest/conrpc.c index e376ce3..619f045 100644 --- a/lnet/selftest/conrpc.c +++ b/lnet/selftest/conrpc.c @@ -781,7 +781,7 @@ lstcon_pingrpc_prep(lst_test_ping_param_t *param, srpc_test_reqst_t *req) prq->png_size = param->png_size; prq->png_flags = param->png_flags; - /* TODO dest */ + return 0; } diff --git a/lnet/selftest/console.c b/lnet/selftest/console.c index 4eaf731..303df07 100644 --- a/lnet/selftest/console.c +++ b/lnet/selftest/console.c @@ -986,10 +986,21 @@ lstcon_batch_info(char *name, lstcon_test_batch_ent_t *ent_up, int server, entp->u.tbe_batch.bae_state = bat->bat_state; } else { - entp->u.tbe_test.tse_type = test->tes_type; entp->u.tbe_test.tse_loop = test->tes_loop; entp->u.tbe_test.tse_concur = test->tes_concur; + entp->u.tbe_test.tse_dist = test->tes_dist; + entp->u.tbe_test.tse_span = test->tes_span; + + if (test->tes_type == LST_TEST_BULK) { + memcpy (&entp->u.tbe_test.tse_params.prm_bulk, + &test->tes_param[0], + sizeof(lst_test_bulk_param_t)); + } else if (test->tes_type == LST_TEST_PING) { + memcpy (&entp->u.tbe_test.tse_params.prm_ping, + &test->tes_param[0], + sizeof(lst_test_ping_param_t)); + } } cfs_list_for_each_entry_typed(ndl, clilst, lstcon_ndlink_t, ndl_link) diff --git a/lnet/selftest/framework.c b/lnet/selftest/framework.c index 4bf02d9..3be77de 100644 --- a/lnet/selftest/framework.c +++ b/lnet/selftest/framework.c @@ -78,6 +78,8 @@ do { \ do { \ __swab32s(&(fc).brw_errors); \ __swab32s(&(fc).ping_errors); \ + __swab32s(&(fc).ping_count); \ + __swab32s(&(fc).ping_duration); \ __swab32s(&(fc).active_tests); \ __swab32s(&(fc).active_batches); \ __swab32s(&(fc).zombie_sessions); \ @@ -90,8 +92,8 @@ do { \ __swab32s(&(rc).rpcs_rcvd); \ __swab32s(&(rc).rpcs_dropped); \ __swab32s(&(rc).rpcs_expired); \ - __swab64s(&(rc).bulk_get); \ - __swab64s(&(rc).bulk_put); \ + __swab64s(&(rc).bulk_in); \ + __swab64s(&(rc).bulk_out); \ } while (0) #define sfw_unpack_lnet_counters(lc) \ @@ -424,6 +426,8 @@ sfw_get_stats (srpc_stat_reqst_t *request, srpc_stat_reply_t *reply) cnt->brw_errors = cfs_atomic_read(&sn->sn_brw_errors); cnt->ping_errors = cfs_atomic_read(&sn->sn_ping_errors); + cnt->ping_duration = cfs_atomic_read(&sn->sn_ping_duration); + cnt->ping_count = cfs_atomic_read(&sn->sn_ping_count); cnt->zombie_sessions = cfs_atomic_read(&sfw_data.fw_nzombies); cnt->active_tests = cnt->active_batches = 0; @@ -899,6 +903,10 @@ sfw_create_test_rpc (sfw_test_unit_t *tsu, lnet_process_id_t peer, /* pick request from buffer */ rpc = cfs_list_entry(tsi->tsi_free_rpcs.next, srpc_client_rpc_t, crpc_list); + /* XXX does this require that all RPCs for a given + * test instance use the same bk_niov? Shouldn't we + * just skip over the one in the freelist if it doesn't + * have the requisite number of blocks?*/ LASSERT (nblk == rpc->crpc_bulk.bk_niov); cfs_list_del_init(&rpc->crpc_list); diff --git a/lnet/selftest/module.c b/lnet/selftest/module.c index d957695..a10362f 100644 --- a/lnet/selftest/module.c +++ b/lnet/selftest/module.c @@ -73,11 +73,11 @@ lnet_selftest_fini (void) void lnet_selftest_structure_assertion(void) { - CLASSERT(sizeof(srpc_msg_t) == 160); + CLASSERT(sizeof(srpc_msg_t) == 168); CLASSERT(sizeof(srpc_test_reqst_t) == 70); CLASSERT(offsetof(srpc_msg_t, msg_body.tes_reqst.tsr_concur) == 72); CLASSERT(offsetof(srpc_msg_t, msg_body.tes_reqst.tsr_ndest) == 78); - CLASSERT(sizeof(srpc_stat_reply_t) == 136); + CLASSERT(sizeof(srpc_stat_reply_t) == 144); CLASSERT(sizeof(srpc_stat_reqst_t) == 28); } diff --git a/lnet/selftest/ping_test.c b/lnet/selftest/ping_test.c index f51416c..25a0d29 100644 --- a/lnet/selftest/ping_test.c +++ b/lnet/selftest/ping_test.c @@ -51,21 +51,12 @@ typedef struct { static lst_ping_data_t lst_ping_data; -static int -ping_client_init(sfw_test_instance_t *tsi) -{ - LASSERT (tsi->tsi_is_client); - - cfs_spin_lock_init(&lst_ping_data.pnd_lock); - lst_ping_data.pnd_counter = 0; - - return 0; -} - static void ping_client_fini (sfw_test_instance_t *tsi) { sfw_session_t *sn = tsi->tsi_batch->bat_session; + sfw_test_unit_t *tsu; + srpc_bulk_t *bulk; int errors; LASSERT (sn != NULL); @@ -76,21 +67,69 @@ ping_client_fini (sfw_test_instance_t *tsi) CWARN ("%d pings have failed.\n", errors); else CDEBUG (D_NET, "Ping test finished OK.\n"); + + cfs_list_for_each_entry_typed (tsu, &tsi->tsi_units, + sfw_test_unit_t, tsu_list) { + bulk = tsu->tsu_private; + if (bulk == NULL) continue; + + srpc_free_bulk(bulk); + tsu->tsu_private = NULL; + } +} + +static int +ping_client_init(sfw_test_instance_t *tsi) +{ + srpc_bulk_t *bulk; + int npg = tsi->tsi_u.ping.png_size / CFS_PAGE_SIZE; + sfw_test_unit_t *tsu; + + LASSERT (tsi->tsi_is_client); + + cfs_spin_lock_init(&lst_ping_data.pnd_lock); + lst_ping_data.pnd_counter = 0; + + if (npg > 0) { + cfs_list_for_each_entry_typed (tsu, &tsi->tsi_units, + sfw_test_unit_t, tsu_list) { + /* Pre-allocate an srpc_bulk_t for copying into requests */ + bulk = srpc_alloc_bulk(npg, 0); + if (bulk == NULL) { + ping_client_fini(tsi); + return -ENOMEM; + } + + tsu->tsu_private = bulk; + } + } + + return 0; } static int ping_client_prep_rpc(sfw_test_unit_t *tsu, - lnet_process_id_t dest, srpc_client_rpc_t **rpc) + lnet_process_id_t dest, srpc_client_rpc_t **rpcpp) { - srpc_ping_reqst_t *req; - struct timeval tv; - int rc; + srpc_ping_reqst_t *req; + struct timeval tv; + int rc; + sfw_test_instance_t *tsi = tsu->tsu_instance; + srpc_client_rpc_t *rpc; + srpc_bulk_t *bulk = tsu->tsu_private; + test_ping_req_t *preq = &tsi->tsi_u.ping; + int const png_size = preq->png_size; + int const nblk = png_size / CFS_PAGE_SIZE; - rc = sfw_create_test_rpc(tsu, dest, 0, 0, rpc); + rc = sfw_create_test_rpc(tsu, dest, nblk, png_size, &rpc); if (rc != 0) return rc; - req = &(*rpc)->crpc_reqstmsg.msg_body.ping_reqst; + if (png_size > 0) { + memcpy(&rpc->crpc_bulk, bulk, offsetof(srpc_bulk_t, bk_iovs[nblk])); + } + + req = &rpc->crpc_reqstmsg.msg_body.ping_reqst; req->pnr_magic = LST_PING_TEST_MAGIC; @@ -101,6 +140,9 @@ ping_client_prep_rpc(sfw_test_unit_t *tsu, cfs_fs_timeval(&tv); req->pnr_time_sec = tv.tv_sec; req->pnr_time_usec = tv.tv_usec; + req->pnr_size = png_size; + + *rpcpp = rpc; return rc; } @@ -113,6 +155,7 @@ ping_client_done_rpc (sfw_test_unit_t *tsu, srpc_client_rpc_t *rpc) srpc_ping_reqst_t *reqst = &rpc->crpc_reqstmsg.msg_body.ping_reqst; srpc_ping_reply_t *reply = &rpc->crpc_replymsg.msg_body.ping_reply; struct timeval tv; + unsigned rtt_usec; LASSERT (sn != NULL); @@ -150,12 +193,24 @@ ping_client_done_rpc (sfw_test_unit_t *tsu, srpc_client_rpc_t *rpc) } cfs_fs_timeval(&tv); - CDEBUG (D_NET, "%d reply in %u usec\n", reply->pnr_seq, - (unsigned)((tv.tv_sec - (unsigned)reqst->pnr_time_sec) * 1000000 - + (tv.tv_usec - reqst->pnr_time_usec))); + + rtt_usec = (tv.tv_sec - (unsigned)reqst->pnr_time_sec) * 1000000 + + (tv.tv_usec - reqst->pnr_time_usec); + + cfs_atomic_inc(&sn->sn_ping_count); + cfs_atomic_add(rtt_usec, &sn->sn_ping_duration); + + CDEBUG (D_NET, "%d reply in %u usec\n", reply->pnr_seq, rtt_usec); + return; } +void +ping_server_rpc_done (srpc_server_rpc_t *rpc) +{ + sfw_free_pages(rpc); +} + static int ping_server_handle (srpc_server_rpc_t *rpc) { @@ -172,13 +227,14 @@ ping_server_handle (srpc_server_rpc_t *rpc) __swab32s(&reqstmsg->msg_type); __swab32s(&req->pnr_seq); __swab32s(&req->pnr_magic); + __swab32s(&req->pnr_size); __swab64s(&req->pnr_time_sec); __swab64s(&req->pnr_time_usec); } LASSERT (reqstmsg->msg_type == srpc_service2request(sv->sv_id)); if (req->pnr_magic != LST_PING_TEST_MAGIC) { - CERROR ("Unexpect magic %08x from %s\n", + CERROR ("Unexpected magic %08x from %s\n", req->pnr_magic, libcfs_id2str(rpc->srpc_peer)); return -EINVAL; } @@ -186,8 +242,21 @@ ping_server_handle (srpc_server_rpc_t *rpc) rep->pnr_seq = req->pnr_seq; rep->pnr_magic = LST_PING_TEST_MAGIC; - CDEBUG (D_NET, "Get ping %d from %s\n", + CDEBUG (D_NET, "Got ping %d from %s\n", req->pnr_seq, libcfs_id2str(rpc->srpc_peer)); + + + // Get ready to receive bulk part + if (req->pnr_size > 0) { + int rc; + rpc->srpc_done = ping_server_rpc_done; + rc = sfw_alloc_pages(rpc, req->pnr_size / CFS_PAGE_SIZE, 1); + if (rc != 0) { + CERROR("Failed to alloc recv pages"); + return -ENOMEM; + } + } + return 0; } @@ -207,3 +276,5 @@ void ping_init_test_service(void) ping_test_service.sv_name = "ping_test"; ping_test_service.sv_handler = ping_server_handle; } + + diff --git a/lnet/selftest/rpc.c b/lnet/selftest/rpc.c index 570e0e2..ec14e61 100644 --- a/lnet/selftest/rpc.c +++ b/lnet/selftest/rpc.c @@ -1255,7 +1255,9 @@ srpc_lnet_ev_handler (lnet_event_t *ev) srpc_buffer_t *buffer; srpc_service_t *sv; srpc_msg_t *msg; + __u64 *counter; srpc_msg_type_t type; + int mlen; LASSERT (!cfs_in_interrupt()); @@ -1298,10 +1300,29 @@ srpc_lnet_ev_handler (lnet_event_t *ev) LASSERT (rpcev->ev_fired == 0); rpcev->ev_fired = 1; rpcev->ev_status = (ev->type == LNET_EVENT_UNLINK) ? - -EINTR : ev->status; + -EINTR : ev->status; + if (rpcev->ev_status == 0 && + rpcev->ev_type == SRPC_BULK_REQ_RCVD) { + mlen = ev->mlength; + counter = crpc->crpc_bulk.bk_sink ? + &srpc_data.rpc_counters.bulk_in : + &srpc_data.rpc_counters.bulk_out; + } else { + mlen = 0; + counter = NULL; + } + + swi_schedule_workitem(&crpc->crpc_wi); cfs_spin_unlock(&crpc->crpc_lock); + + if (counter != NULL) { + cfs_spin_lock(&srpc_data.rpc_glock); + *counter += mlen; + cfs_spin_unlock(&srpc_data.rpc_glock); + } + break; case SRPC_REQUEST_RCVD: @@ -1386,9 +1407,9 @@ srpc_lnet_ev_handler (lnet_event_t *ev) cfs_spin_lock(&srpc_data.rpc_glock); if (rpcev->ev_type == SRPC_BULK_GET_RPLD) - srpc_data.rpc_counters.bulk_get += ev->mlength; + srpc_data.rpc_counters.bulk_in += ev->mlength; else - srpc_data.rpc_counters.bulk_put += ev->mlength; + srpc_data.rpc_counters.bulk_out += ev->mlength; cfs_spin_unlock(&srpc_data.rpc_glock); } diff --git a/lnet/selftest/rpc.h b/lnet/selftest/rpc.h index 575887d..5e3508b 100644 --- a/lnet/selftest/rpc.h +++ b/lnet/selftest/rpc.h @@ -208,8 +208,10 @@ typedef struct { /* TEST RPCs */ typedef struct { __u64 pnr_rpyid; + __u64 pnr_bulkid; /* bulk buffer matchbits */ __u32 pnr_magic; __u32 pnr_seq; + __u32 pnr_size; __u64 pnr_time_sec; __u64 pnr_time_usec; } WIRE_ATTR srpc_ping_reqst_t; diff --git a/lnet/selftest/selftest.h b/lnet/selftest/selftest.h index 9856472..397beb1 100644 --- a/lnet/selftest/selftest.h +++ b/lnet/selftest/selftest.h @@ -306,6 +306,9 @@ typedef struct { cfs_atomic_t sn_refcount; cfs_atomic_t sn_brw_errors; cfs_atomic_t sn_ping_errors; + cfs_atomic_t sn_ping_count; /* Number of successful pings */ + cfs_atomic_t sn_ping_duration; /* Time spent in successful pings */ + } sfw_session_t; #define sfw_sid_equal(sid0, sid1) ((sid0).ses_nid == (sid1).ses_nid && \ diff --git a/lnet/utils/lst.c b/lnet/utils/lst.c index 94cf02a..22c83f4 100644 --- a/lnet/utils/lst.c +++ b/lnet/utils/lst.c @@ -326,12 +326,21 @@ lst_node_str2state(char *str) } char * -lst_test_type2name(int type) +lst_test_tse2name(lstcon_test_ent_t *test) { - if (type == LST_TEST_PING) + if (test->tse_type == LST_TEST_PING) return "ping"; - if (type == LST_TEST_BULK) - return "brw"; + + if (test->tse_type == LST_TEST_BULK) { + if (test->tse_params.prm_bulk.blk_opc == LST_BRW_READ) { + return "brw RD"; + } else if (test->tse_params.prm_bulk.blk_opc == LST_BRW_WRITE) { + return "brw WR"; + } else { + fprintf(stderr, "Selftest kernel module is old\n"); + exit (-1); + } + } return "unknown"; } @@ -358,6 +367,10 @@ lst_print_error(char *sub, const char *def_format, ...) { va_list ap; + va_start(ap, def_format); + vfprintf(stderr, def_format, ap); + va_end(ap); + /* local error returned from kernel */ switch (errno) { case ESRCH: @@ -382,10 +395,6 @@ lst_print_error(char *sub, const char *def_format, ...) fprintf(stderr, "%s already exists\n", sub); return; default: - va_start(ap, def_format); - vfprintf(stderr, def_format, ap); - va_end(ap); - return; } } @@ -602,8 +611,8 @@ jt_lst_new_session(int argc, char **argv) rc = lst_new_session_ioctl(name, timeout, force, &session_id); if (rc != 0) { - lst_print_error("session", "Failed to create session: %s\n", - strerror(errno)); + lst_print_error("session", "Failed to create session [%s]: %s\n", + name, strerror(errno)); return rc; } @@ -1505,55 +1514,78 @@ lst_stat_req_param_alloc(char *name, lst_stat_req_param_t **srpp, int save_old) return rc; } -typedef struct { - /* TODO */ - int foo; -} lst_srpc_stat_result; +#define LST_STAT_VALUE_AVG 1 +#define LST_STAT_VALUE_MIN 2 +#define LST_STAT_VALUE_MAX 4 -#define LST_LNET_AVG 0 -#define LST_LNET_MIN 1 -#define LST_LNET_MAX 2 +#define LST_STAT_BW 1 +#define LST_STAT_RATE 2 -typedef struct { - float lnet_avg_sndrate; - float lnet_min_sndrate; - float lnet_max_sndrate; - float lnet_total_sndrate; +#define LST_STAT_INPUT 1 +#define LST_STAT_OUTPUT 2 - float lnet_avg_rcvrate; - float lnet_min_rcvrate; - float lnet_max_rcvrate; - float lnet_total_rcvrate; +#define LST_MEGA_BYTES (1024 * 1024) - float lnet_avg_sndperf; - float lnet_min_sndperf; - float lnet_max_sndperf; - float lnet_total_sndperf; +typedef struct { + float avg; + float min; + float max; + float total; +} lst_stat_value_t; - float lnet_avg_rcvperf; - float lnet_min_rcvperf; - float lnet_max_rcvperf; - float lnet_total_rcvperf; +typedef struct { + lst_stat_value_t lst_snd_rate; + lst_stat_value_t lst_rcv_rate; + lst_stat_value_t lst_out_perf; + lst_stat_value_t lst_in_perf; + int lst_stat_count; +} lst_stat_result_t; - int lnet_stat_count; -} lst_lnet_stat_result_t; +typedef struct { + int lst_stat_count; + lst_stat_value_t lst_ping_time; +} lst_ping_result_t; -lst_lnet_stat_result_t lnet_stat_result; +lst_ping_result_t sfwk_ping_result; +lst_stat_result_t lnet_stat_result; +lst_stat_result_t srpc_stat_result; static float -lst_lnet_stat_value(int bw, int send, int off) +lst_stat_value(int lnet, int bw, int dir, int type) { - float *p; + lst_stat_value_t *p; - p = bw ? &lnet_stat_result.lnet_avg_sndperf : - &lnet_stat_result.lnet_avg_sndrate; + if (lnet) { /* LNet stats */ + if (bw == LST_STAT_BW) { /* bandwidth */ + p = (dir == 0) ? &lnet_stat_result.lst_out_perf : + &lnet_stat_result.lst_in_perf; + } else { /* Rates */ + p = (dir == 0) ? &lnet_stat_result.lst_snd_rate : + &lnet_stat_result.lst_rcv_rate; + } + } else { /* SRPC stats */ + if (bw == LST_STAT_BW) { /* bandwidth */ + p = (dir == 0) ? &srpc_stat_result.lst_out_perf : + &srpc_stat_result.lst_in_perf; + } else { /* Rates */ + p = (dir == 0) ? &srpc_stat_result.lst_snd_rate : + &srpc_stat_result.lst_rcv_rate; + } + } - if (!send) - p += 4; + switch (type) { + default: + return 0; - p += off; + case LST_STAT_VALUE_AVG: + return p->avg; - return *p; + case LST_STAT_VALUE_MIN: + return p->min; + + case LST_STAT_VALUE_MAX: + return p->max; + } } static void @@ -1572,127 +1604,182 @@ lst_timeval_diff(struct timeval *tv1, return; } -void -lst_cal_lnet_stat(float delta, lnet_counters_t *lnet_new, - lnet_counters_t *lnet_old) +static void +lst_cal_value(lst_stat_value_t *stat, int count, __u32 val) { - float perf; - float rate; + stat->total += val; + stat->avg = stat->total / count; - perf = (float)(lnet_new->send_length - - lnet_old->send_length) / (1024 * 1024) / delta; - lnet_stat_result.lnet_total_sndperf += perf; + if (stat->min > val || + stat->min == 0) + stat->min = val; - if (lnet_stat_result.lnet_min_sndperf > perf || - lnet_stat_result.lnet_min_sndperf == 0) - lnet_stat_result.lnet_min_sndperf = perf; - - if (lnet_stat_result.lnet_max_sndperf < perf) - lnet_stat_result.lnet_max_sndperf = perf; + if (stat->max < val) + stat->max = val; +} - perf = (float)(lnet_new->recv_length - - lnet_old->recv_length) / (1024 * 1024) / delta; - lnet_stat_result.lnet_total_rcvperf += perf; +static void +lst_cal_stat(lst_stat_result_t *result, + float out_perf, float in_perf, + float send_rate, float recv_rate) +{ + result->lst_stat_count ++; + lst_cal_value(&result->lst_out_perf, result->lst_stat_count, out_perf); + lst_cal_value(&result->lst_in_perf, result->lst_stat_count, in_perf); + lst_cal_value(&result->lst_snd_rate, result->lst_stat_count, send_rate); + lst_cal_value(&result->lst_rcv_rate, result->lst_stat_count, recv_rate); +} - if (lnet_stat_result.lnet_min_rcvperf > perf || - lnet_stat_result.lnet_min_rcvperf == 0) - lnet_stat_result.lnet_min_rcvperf = perf; +void +lst_cal_sfw_stat(sfw_counters_t *sfwk_new, sfw_counters_t *sfwk_old) +{ + // NB we assume that no more than 2**31us worth of pings will have + // happened since we last polled, or this calculation comes out wrong. + __u32 period = sfwk_new->ping_duration - sfwk_old->ping_duration; + __u32 count = sfwk_new->ping_count - sfwk_old->ping_count; - if (lnet_stat_result.lnet_max_rcvperf < perf) - lnet_stat_result.lnet_max_rcvperf = perf; + if (count == 0) { + return; + } - rate = (lnet_new->send_count - lnet_old->send_count) / delta; - lnet_stat_result.lnet_total_sndrate += rate; + float latency = (float)period / (float)count; - if (lnet_stat_result.lnet_min_sndrate > rate || - lnet_stat_result.lnet_min_sndrate == 0) - lnet_stat_result.lnet_min_sndrate = rate; + sfwk_ping_result.lst_stat_count++; + lst_cal_value(&sfwk_ping_result.lst_ping_time, + sfwk_ping_result.lst_stat_count, latency); +} - if (lnet_stat_result.lnet_max_sndrate < rate) - lnet_stat_result.lnet_max_sndrate = rate; +void +lst_cal_lnet_stat(float delta, lnet_counters_t *lnet_new, + lnet_counters_t *lnet_old) +{ + float out_perf; + float in_perf; + float send_rate; + float recv_rate; - rate = (lnet_new->recv_count - lnet_old->recv_count) / delta; - lnet_stat_result.lnet_total_rcvrate += rate; + out_perf = (float)(lnet_new->send_length - + lnet_old->send_length) / LST_MEGA_BYTES / delta; + in_perf = (float)(lnet_new->recv_length - + lnet_old->recv_length) / LST_MEGA_BYTES / delta; - if (lnet_stat_result.lnet_min_rcvrate > rate || - lnet_stat_result.lnet_min_rcvrate == 0) - lnet_stat_result.lnet_min_rcvrate = rate; + send_rate = (lnet_new->send_count - lnet_old->send_count) / delta; + recv_rate = (lnet_new->recv_count - lnet_old->recv_count) / delta; - if (lnet_stat_result.lnet_max_rcvrate < rate) - lnet_stat_result.lnet_max_rcvrate = rate; + lst_cal_stat(&lnet_stat_result, out_perf, in_perf, send_rate, recv_rate); +} - lnet_stat_result.lnet_stat_count ++; +void +lst_cal_srpc_stat(float delta, srpc_counters_t *srpc_new, + srpc_counters_t *srpc_old) +{ + float out_perf; + float in_perf; + float send_rate; + float recv_rate; - lnet_stat_result.lnet_avg_sndrate = lnet_stat_result.lnet_total_sndrate / - lnet_stat_result.lnet_stat_count; - lnet_stat_result.lnet_avg_rcvrate = lnet_stat_result.lnet_total_rcvrate / - lnet_stat_result.lnet_stat_count; + out_perf = (float)(srpc_new->bulk_out - + srpc_old->bulk_out) / LST_MEGA_BYTES / delta; + in_perf = (float)(srpc_new->bulk_in - + srpc_old->bulk_in) / LST_MEGA_BYTES / delta; - lnet_stat_result.lnet_avg_sndperf = lnet_stat_result.lnet_total_sndperf / - lnet_stat_result.lnet_stat_count; - lnet_stat_result.lnet_avg_rcvperf = lnet_stat_result.lnet_total_rcvperf / - lnet_stat_result.lnet_stat_count; + recv_rate = (srpc_new->rpcs_rcvd - srpc_old->rpcs_rcvd) / delta; + send_rate = (srpc_new->rpcs_sent - srpc_old->rpcs_sent) / delta; + lst_cal_stat(&srpc_stat_result, out_perf, in_perf, send_rate, recv_rate); } void -lst_print_lnet_stat(char *name, int bwrt, int rdwr, int type) +lst_print_lst_stat(int lnet, char *name, int bwrt, int io, int type, int ping) { - int start1 = 0; - int end1 = 1; - int start2 = 0; - int end2 = 1; + int start; + int end; int i; - int j; - if (lnet_stat_result.lnet_stat_count == 0) + if ((lnet && lnet_stat_result.lst_stat_count == 0) || + (!lnet && srpc_stat_result.lst_stat_count == 0)) return; - if (bwrt == 1) /* bw only */ - start1 = 1; + if ((bwrt & LST_STAT_RATE) != 0) { + fprintf(stdout, "[%s Rates of %s]\n", lnet ? "LNet" : "SRPC", name); + + for (i = 0; i <= 1; i++) { + if (lnet) + fprintf(stdout, "[%s] ", i == 0 ? "Snd" : "Rcv"); + else + fprintf(stdout, "[%s] ", i == 0 ? "Req" : "Srv"); - if (bwrt == 2) /* rates only */ - end1 = 0; + if ((type & LST_STAT_VALUE_AVG) != 0) { + fprintf(stdout, "Avg: %-8.1f RPC/s ", + lst_stat_value(lnet, LST_STAT_RATE, i, LST_STAT_VALUE_AVG)); + } - if (rdwr == 1) /* recv only */ - start2 = 1; + if ((type & LST_STAT_VALUE_MIN) != 0) { + fprintf(stdout, "Min: %-8.1f RPC/s ", + lst_stat_value(lnet, LST_STAT_RATE, i, LST_STAT_VALUE_MIN)); + } - if (rdwr == 2) /* send only */ - end2 = 0; + if ((type & LST_STAT_VALUE_MAX) != 0) { + fprintf(stdout, "Max: %-8.1f RPC/s", + lst_stat_value(lnet, LST_STAT_RATE, i, LST_STAT_VALUE_MAX)); + } - for (i = start1; i <= end1; i++) { - fprintf(stdout, "[LNet %s of %s]\n", - i == 0 ? "Rates" : "Bandwidth", name); + fprintf(stdout, "\n"); + } + } - for (j = start2; j <= end2; j++) { - fprintf(stdout, "[%c] ", j == 0 ? 'R' : 'W'); + start = ((io & LST_STAT_OUTPUT) == 0) ? 1 : 0; + end = ((io & LST_STAT_INPUT) == 0) ? 0 : 1; - if ((type & 1) != 0) { - fprintf(stdout, i == 0 ? "Avg: %-8.0f RPC/s " : - "Avg: %-8.2f MB/s ", - lst_lnet_stat_value(i, j, 0)); + if ((bwrt & LST_STAT_BW) != 0) { + fprintf(stdout, "[%s Bandwidth of %s]\n", lnet ? "LNet" : "SRPC", name); + + for (i = start; i <= end; i++) { + fprintf(stdout, "[%s] ", i == 0 ? "Out" : "In "); + + if ((type & LST_STAT_VALUE_AVG) != 0) { + fprintf(stdout, "Avg: %-8.2f MB/s ", + lst_stat_value(lnet, LST_STAT_BW, i, LST_STAT_VALUE_AVG)); } - if ((type & 2) != 0) { - fprintf(stdout, i == 0 ? "Min: %-8.0f RPC/s " : - "Min: %-8.2f MB/s ", - lst_lnet_stat_value(i, j, 1)); + if ((type & LST_STAT_VALUE_MIN) != 0) { + fprintf(stdout, "Min: %-8.2f MB/s ", + lst_stat_value(lnet, LST_STAT_BW, i, LST_STAT_VALUE_MIN)); } - if ((type & 4) != 0) { - fprintf(stdout, i == 0 ? "Max: %-8.0f RPC/s" : - "Max: %-8.2f MB/s", - lst_lnet_stat_value(i, j, 2)); + if ((type & LST_STAT_VALUE_MAX) != 0) { + fprintf(stdout, "Max: %-8.2f MB/s", + lst_stat_value(lnet, LST_STAT_BW, i, LST_STAT_VALUE_MAX)); } fprintf(stdout, "\n"); } } + + if (ping != 0) { + fprintf(stdout, "[Ping time of %s]\n", name, type); + if ((type & LST_STAT_VALUE_AVG) != 0) { + fprintf (stdout, "Avg: %-8.2f us ", + sfwk_ping_result.lst_ping_time.avg); + } + + if ((type & LST_STAT_VALUE_MIN) != 0) { + fprintf (stdout, "Min: %-8.2f us ", + sfwk_ping_result.lst_ping_time.min); + } + + if ((type & LST_STAT_VALUE_MAX) != 0) { + fprintf (stdout, "Max: %-8.2f us ", + sfwk_ping_result.lst_ping_time.max); + } + + fprintf(stdout, "\n"); + } } void lst_print_stat(char *name, cfs_list_t *resultp, - int idx, int lnet, int bwrt, int rdwr, int type) + int idx, int lnet, int bwrt, int io, int type, int ping) { cfs_list_t tmp[2]; lstcon_rpc_ent_t *new; @@ -1711,6 +1798,8 @@ lst_print_stat(char *name, cfs_list_t *resultp, CFS_INIT_LIST_HEAD(&tmp[1]); memset(&lnet_stat_result, 0, sizeof(lnet_stat_result)); + memset(&srpc_stat_result, 0, sizeof(srpc_stat_result)); + memset(&sfwk_ping_result, 0, sizeof(sfwk_ping_result)); while (!cfs_list_empty(&resultp[idx])) { if (cfs_list_empty(&resultp[1 - idx])) { @@ -1758,10 +1847,9 @@ lst_print_stat(char *name, cfs_list_t *resultp, delta = tv.tv_sec + (float)tv.tv_usec/1000000; - if (!lnet) /* TODO */ - continue; - + lst_cal_sfw_stat(sfwk_new, sfwk_old); lst_cal_lnet_stat(delta, lnet_new, lnet_old); + lst_cal_srpc_stat(delta, srpc_new, srpc_old); } cfs_list_splice(&tmp[idx], &resultp[idx]); @@ -1770,10 +1858,7 @@ lst_print_stat(char *name, cfs_list_t *resultp, if (errcount > 0) fprintf(stdout, "Failed to stat on %d nodes\n", errcount); - if (!lnet) /* TODO */ - return; - - lst_print_lnet_stat(name, bwrt, rdwr, type); + lst_print_lst_stat(lnet, name, bwrt, io, type, ping); } int @@ -1785,9 +1870,10 @@ jt_lst_stat(int argc, char **argv) int optidx = 0; int timeout = 5; /* default timeout, 5 sec */ int delay = 5; /* default delay, 5 sec */ - int lnet = 1; /* lnet stat by default */ - int bwrt = 0; - int rdwr = 0; + int lnet = 0; /* srpc stat by default */ + int ping = 0; /* ping stats, off by default */ + int bwrt = -1; + int io = -1; int type = -1; int idx = 0; int rc; @@ -1799,10 +1885,11 @@ jt_lst_stat(int argc, char **argv) {"delay" , required_argument, 0, 'd' }, {"lnet" , no_argument, 0, 'l' }, {"rpc" , no_argument, 0, 'c' }, + {"ping" , no_argument, 0, 'p' }, {"bw" , no_argument, 0, 'b' }, {"rate" , no_argument, 0, 'a' }, - {"read" , no_argument, 0, 'r' }, - {"write" , no_argument, 0, 'w' }, + {"input" , no_argument, 0, 'i' }, + {"output" , no_argument, 0, 'o' }, {"avg" , no_argument, 0, 'g' }, {"min" , no_argument, 0, 'n' }, {"max" , no_argument, 0, 'x' }, @@ -1816,7 +1903,7 @@ jt_lst_stat(int argc, char **argv) } while (1) { - c = getopt_long(argc, argv, "t:d:lcbarwgnx", stat_opts, &optidx); + c = getopt_long(argc, argv, "t:d:lcpbarwgnx", stat_opts, &optidx); if (c == -1) break; @@ -1834,38 +1921,50 @@ jt_lst_stat(int argc, char **argv) case 'c': lnet = 0; break; + case 'p': + ping = 1; + break; case 'b': - bwrt |= 1; + if (bwrt == -1) + bwrt = LST_STAT_BW; + else + bwrt |= LST_STAT_BW; break; case 'a': - bwrt |= 2; + if (bwrt == -1) + bwrt = LST_STAT_RATE; + else + bwrt |= LST_STAT_RATE; break; - case 'r': - rdwr |= 1; + case 'i': + if (io == -1) + io = LST_STAT_INPUT; + else + io |= LST_STAT_INPUT; break; - case 'w': - rdwr |= 2; + case 'o': + if (io == -1) + io == LST_STAT_OUTPUT; + else + io |= LST_STAT_OUTPUT; break; case 'g': - if (type == -1) { - type = 1; - break; - } - type |= 1; + if (type == -1) + type = LST_STAT_VALUE_AVG; + else + type |= LST_STAT_VALUE_AVG; break; case 'n': - if (type == -1) { - type = 2; - break; - } - type |= 2; + if (type == -1) + type = LST_STAT_VALUE_MIN; + else + type |= LST_STAT_VALUE_MIN; break; case 'x': - if (type == -1) { - type = 4; - break; - } - type |= 4; + if (type == -1) + type = LST_STAT_VALUE_MAX; + else + type |= LST_STAT_VALUE_MAX;; break; default: lst_print_usage(argv[0]); @@ -1915,7 +2014,7 @@ jt_lst_stat(int argc, char **argv) } lst_print_stat(srp->srp_name, srp->srp_result, - idx, lnet, bwrt, rdwr, type); + idx, lnet, bwrt, io, type, ping); lst_reset_rpcent(&srp->srp_result[1 - idx]); } @@ -2308,7 +2407,7 @@ jt_lst_stop_batch(int argc, char **argv) lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) break; - fprintf(stdout, "%d batch in stopping\n", + fprintf(stdout, "%d batch is stopping\n", lstcon_tsbqry_stat_run(&trans_stat, 0)); sleep(1); } @@ -2551,11 +2650,22 @@ loop: test = 1; /* starting from test 1 */ } else { + char params[64]; + + if (ent.u.tbe_test.tse_type == LST_TEST_BULK) + snprintf(params, 64, ", size = %d", + ent.u.tbe_test.tse_params.prm_bulk.blk_size); + else if (ent.u.tbe_test.tse_type == LST_TEST_PING) + snprintf(params, 64, ", size = %d", + ent.u.tbe_test.tse_params.prm_ping.png_size); + else + params[0] = 0; + fprintf(stdout, - "\tTest %d(%s) (loop: %d, concurrency: %d)\n", - test, lst_test_type2name(ent.u.tbe_test.tse_type), - ent.u.tbe_test.tse_loop, - ent.u.tbe_test.tse_concur); + "\tTest %d(%s) (loop %d, concurrency %d, distribute %d:%d%s)\n", + test, lst_test_tse2name(&ent.u.tbe_test), + ent.u.tbe_test.tse_loop, ent.u.tbe_test.tse_concur, + ent.u.tbe_test.tse_dist, ent.u.tbe_test.tse_span, params); ntest --; test ++; } @@ -2823,6 +2933,77 @@ lst_parse_distribute(char *dstr, int *dist, int *span) return 0; } +/** + * @param token + * Null terminated string like "10", "10k", "10KB", "10m" + * representing a size in bytes. + * + * @return + * -1 on invalid string, else size in bytes. + */ +static int +parse_size(char const *tok) +{ + int size; + char *end = NULL; + + size = strtol(tok, &end, 0); + if (size < 0 || end == tok) { + return -1; + } + + if (end == NULL) + return size; + + if (*end == 'k' || *end == 'K') { + size *= 1024; + } else if (*end == 'm' || *end == 'M') { + size *= LST_MEGA_BYTES; + } else if (*end != '\0') { + return -1; + } + + return size; +} + +int +lst_get_ping_param(int argc, char **argv, lst_test_ping_param_t *ping) +{ + char *tok = NULL; + char *end = NULL; + int rc = 0; + int i = 0; + + ping->png_size = 0; + ping->png_flags = 0; + + while (i < argc) { + if (strcasestr(argv[i], "size=") == argv[i] || + strcasestr(argv[i], "s=") == argv[i]) { + tok = strchr(argv[i], '=') + 1; + + ping->png_size = parse_size(tok); + if (ping->png_size < 0) { + fprintf(stderr, "Invalid size %s\n", tok); + return -1; + } else if (ping->png_size > CFS_PAGE_SIZE * LNET_MAX_IOV) { + fprintf(stderr, "Size %d exceeds maximum %d bytes\n", + ping->png_size, CFS_PAGE_SIZE * LNET_MAX_IOV); + return -1; + } + } else { + fprintf(stderr, "Unknown parameter: %s\n", argv[i]); + return -1; + } + + i++; + } + + return rc; +} + + + int lst_get_bulk_param(int argc, char **argv, lst_test_bulk_param_t *bulk) { @@ -2844,8 +3025,10 @@ lst_get_bulk_param(int argc, char **argv, lst_test_bulk_param_t *bulk) bulk->blk_flags = LST_BRW_CHECK_FULL; } else if (strcasecmp(tok, "simple") == 0) { bulk->blk_flags = LST_BRW_CHECK_SIMPLE; + } else if (strcasecmp(tok, "none") == 0) { + bulk->blk_flags = LST_BRW_CHECK_NONE; } else { - fprintf(stderr, "Unknow flag %s\n", tok); + fprintf(stderr, "Unknown flag %s\n", tok); return -1; } @@ -2853,26 +3036,15 @@ lst_get_bulk_param(int argc, char **argv, lst_test_bulk_param_t *bulk) strcasestr(argv[i], "s=") == argv[i]) { tok = strchr(argv[i], '=') + 1; - bulk->blk_size = strtol(tok, &end, 0); + bulk->blk_size = parse_size(tok); if (bulk->blk_size <= 0) { fprintf(stderr, "Invalid size %s\n", tok); return -1; - } - - if (end == NULL) - return 0; - - if (*end == 'k' || *end == 'K') - bulk->blk_size *= 1024; - else if (*end == 'm' || *end == 'M') - bulk->blk_size *= 1024 * 1024; - - if (bulk->blk_size > CFS_PAGE_SIZE * LNET_MAX_IOV) { - fprintf(stderr, "Size exceed limitation: %d bytes\n", - bulk->blk_size); + } else if (bulk->blk_size > CFS_PAGE_SIZE * LNET_MAX_IOV) { + fprintf(stderr, "Size %d exceeds maximum %d bytes\n", + bulk->blk_size, CFS_PAGE_SIZE * LNET_MAX_IOV); return -1; } - } else if (strcasecmp(argv[i], "read") == 0 || strcasecmp(argv[i], "r") == 0) { bulk->blk_opc = LST_BRW_READ; @@ -2882,7 +3054,7 @@ lst_get_bulk_param(int argc, char **argv, lst_test_bulk_param_t *bulk) bulk->blk_opc = LST_BRW_WRITE; } else { - fprintf(stderr, "Unknow parameter: %s\n", argv[i]); + fprintf(stderr, "Unknown parameter: %s\n", argv[i]); return -1; } @@ -2896,16 +3068,33 @@ int lst_get_test_param(char *test, int argc, char **argv, void **param, int *plen) { lst_test_bulk_param_t *bulk = NULL; + lst_test_ping_param_t *ping = NULL; int type; type = lst_test_name2type(test); if (type < 0) { - fprintf(stderr, "Unknow test name %s\n", test); + fprintf(stderr, "Unknown test name %s\n", test); return -1; } switch (type) { case LST_TEST_PING: + ping = malloc(sizeof(*ping)); + if (ping == NULL) { + fprintf(stderr, "Out of memory\n"); + return -1; + } + + memset(ping, 0, sizeof(*ping)); + + if (lst_get_ping_param(argc, argv, ping) != 0) { + free(ping); + return -1; + } + + *param = ping; + *plen = sizeof(*ping); + break; case LST_TEST_BULK: @@ -2931,7 +3120,6 @@ lst_get_test_param(char *test, int argc, char **argv, void **param, int *plen) break; } - /* TODO: parse more parameter */ return type; } @@ -3128,30 +3316,30 @@ static command_t lst_cmdlist[] = { {"show_session", jt_lst_show_session, NULL, "Usage: lst show_session" }, {"ping", jt_lst_ping , NULL, - "Usage: lst ping [--group NAME] [--batch NAME] [--session] [--nodes IDS]" }, + "Usage: lst ping [--group NAME] [--batch NAME] [--session] [--nodes NIDs]" }, {"add_group", jt_lst_add_group, NULL, - "Usage: lst group NAME IDs [IDs]..." }, + "Usage: lst add_group NAME NIDs [NIDs]..." }, {"del_group", jt_lst_del_group, NULL, "Usage: lst del_group NAME" }, {"update_group", jt_lst_update_group, NULL, - "Usage: lst update_group NAME [--clean] [--refresh] [--remove IDs]" }, + "Usage: lst update_group NAME [--clean] [--refresh] [--remove NIDs]" }, {"list_group", jt_lst_list_group, NULL, - "Usage: lst list_group [--active] [--busy] [--down] [--unknown] GROUP ..." }, + "Usage: lst list_group [--active] [--busy] [--down] [--unknown] NAME ..." }, {"stat", jt_lst_stat, NULL, - "Usage: lst stat [--bw] [--rate] [--read] [--write] [--max] [--min] [--avg] " - " [--timeout #] [--delay #] GROUP [GROUP]" }, + "Usage: lst stat [--bw] [--rate] [--input] [--output] [--max] [--min] [--avg] " + "[--lnet] [--ping] [--timeout #] [--delay #] GROUP | NIDs [GROUP | NIDs]" }, {"show_error", jt_lst_show_error, NULL, - "Usage: lst show_error NAME | IDS ..." }, + "Usage: lst show_error GROUP | NIDs ..." }, {"add_batch", jt_lst_add_batch, NULL, "Usage: lst add_batch NAME" }, {"run", jt_lst_start_batch, NULL, - "Usage: lst run [--timeout TIME] [NAME]" }, + "Usage: lst run [--timeout TIME] [BATCH]" }, {"stop", jt_lst_stop_batch, NULL, - "Usage: lst stop [--force] BATCH_NAME" }, + "Usage: lst stop [--force] BATCH" }, {"list_batch", jt_lst_list_batch, NULL, - "Usage: lst list_batch NAME [--test ID] [--server]" }, + "Usage: lst list_batch BATCH [--test ID] [--server]" }, {"query", jt_lst_query_batch, NULL, - "Usage: lst query [--test ID] [--server] [--timeout TIME] NAME" }, + "Usage: lst query [--test ID] [--server] [--timeout TIME] GROUP | NIDs" }, {"add_test", jt_lst_add_test, NULL, "Usage: lst add_test [--batch BATCH] [--loop #] [--concurrency #] " " [--distribute #:#] [--from GROUP] [--to GROUP] TEST..." },