diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index bf8f144..c507f5e 100755 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -722,6 +722,11 @@ lnet_prepare(lnet_pid_t requested_pid) struct lnet_res_container **recs; int rc = 0; + if (requested_pid == LNET_PID_ANY) { + /* Don't instantiate LNET just for me */ + return -ENETDOWN; + } + LASSERT (the_lnet.ln_refcount == 0); the_lnet.ln_routing = 0; @@ -1042,7 +1047,7 @@ lnet_ping_info_create(int num_ni) } static void -lnet_ping_info_to_ni(lnet_ping_info_t *ping_info) +lnet_ping_info_install(lnet_ping_info_t *ping_info) { int i; lnet_ni_t *ni; @@ -1113,7 +1118,7 @@ lnet_ping_info_destroy(void) static int lnet_ping_info_setup(lnet_ping_info_t **ppinfo, lnet_handle_md_t *md_handle, - int ni_count) + int ni_count, bool set_eq) { lnet_handle_me_t me_handle; lnet_process_id_t id; @@ -1121,9 +1126,22 @@ lnet_ping_info_setup(lnet_ping_info_t **ppinfo, lnet_handle_md_t *md_handle, int infosz; int rc, rc2; + if (set_eq) { + /* We can have a tiny EQ since we only need to see the unlink event + * on teardown, which by definition is the last one! */ + rc = LNetEQAlloc(0, lnet_ping_event_handler, + &the_lnet.ln_ping_target_eq); + if (rc != 0) { + CERROR("Can't allocate ping EQ: %d\n", rc); + return rc; + } + } + *ppinfo = lnet_ping_info_create(ni_count); - if (*ppinfo == NULL) - return -ENOMEM; + if (*ppinfo == NULL) { + rc = -ENOMEM; + goto failed_0; + } memset(&id, 0, sizeof(lnet_process_id_t)); id.nid = LNET_NID_ANY; @@ -1165,7 +1183,9 @@ failed_2: failed_1: lnet_ping_info_free(*ppinfo); *ppinfo = NULL; - +failed_0: + if (set_eq) + LNetEQFree(the_lnet.ln_ping_target_eq); return rc; } @@ -1192,7 +1212,7 @@ lnet_ping_target_update(lnet_ping_info_t *pinfo, lnet_handle_md_t md_handle) lnet_ping_info_t *old_pinfo; /* switch the NIs to point to the new ping info created */ - lnet_ping_info_to_ni(pinfo); + lnet_ping_info_install(pinfo); lnet_net_lock(LNET_LOCK_EX); if (!the_lnet.ln_routing) @@ -1231,37 +1251,6 @@ lnet_ping_event_handler(lnet_event_t *event) pinfo->pi_features = LNET_PING_FEAT_INVAL; } -static int -lnet_ping_target_init(void) -{ - lnet_ping_info_t *pinfo; - lnet_handle_md_t md_handle; - int ni_count; - int rc; - int rc2; - - /* We can have a tiny EQ since we only need to see the unlink event on - * teardown, which by definition is the last one! */ - rc = LNetEQAlloc(0, lnet_ping_event_handler, &the_lnet.ln_ping_target_eq); - if (rc != 0) { - CERROR("Can't allocate ping EQ: %d\n", rc); - return rc; - } - - ni_count = lnet_get_ni_count(); - - rc = lnet_ping_info_setup(&pinfo, &md_handle, ni_count); - if (rc != 0) { - rc2 = LNetEQFree(the_lnet.ln_ping_target_eq); - LASSERT(rc2 == 0); - return rc; - } - - lnet_ping_target_update(pinfo, md_handle); - - return 0; -} - static void lnet_ping_target_fini(void) { @@ -1426,7 +1415,7 @@ lnet_shutdown_lndni(__u32 net) ni_count = lnet_get_ni_count(); /* create and link a new ping info, before removing the old one */ - rc = lnet_ping_info_setup(&pinfo, &md_handle, ni_count - 1); + rc = lnet_ping_info_setup(&pinfo, &md_handle, ni_count - 1, false); if (rc != 0) return rc; @@ -1470,23 +1459,7 @@ lnet_shutdown_lndni(__u32 net) } static int -lnet_prepare_ni_config_init(lnet_pid_t requested_pid) -{ - int rc; - - lnet_get_tunables(); - - if (requested_pid == LNET_PID_ANY) { - /* Don't instantiate LNET just for me */ - return -ENETDOWN; - } - - rc = lnet_prepare(requested_pid); - return rc; -} - -static int -lnet_complete_ni_config_init(void) +lnet_misc_init(void) { int rc; @@ -1496,34 +1469,22 @@ lnet_complete_ni_config_init(void) the_lnet.ln_refcount = 1; /* Now I may use my own API functions... */ - /* NB router checker needs the_lnet.ln_ping_info in - * lnet_router_checker -> lnet_update_ni_status_locked */ - rc = lnet_ping_target_init(); - if (rc != 0) - goto failed0; - rc = lnet_router_checker_start(); if (rc != 0) - goto failed1; + goto failed; lnet_proc_init(); - return 0; - -failed1: - lnet_ping_target_fini(); -failed0: +failed: the_lnet.ln_refcount = 0; lnet_acceptor_stop(); - return rc; } static int -lnet_startup_lndnis(char *net, int do_full_config, __s32 peer_to, +lnet_startup_lndnis(struct list_head *nilist, int do_full_config, __s32 peer_to, __s32 peer_cr, __s32 peer_buf_cr, __s32 credits) { - cfs_list_t nilist; int rc = 0; struct lnet_ni *ni; int lnd_type; @@ -1531,31 +1492,8 @@ lnet_startup_lndnis(char *net, int do_full_config, __s32 peer_to, struct lnet_tx_queue *tq; int nicount = 0; int i; - int new_ni_count = 0; - lnet_ping_info_t *pinfo = NULL; - lnet_handle_md_t md_handle; - - if (net == NULL) - return -EINVAL; - INIT_LIST_HEAD(&nilist); - - /* Create a ni structure for the network string */ - rc = lnet_parse_networks(&nilist, net, &new_ni_count); - if (rc != 0) - goto failed; - - if (!do_full_config) { - /* We need to update the ping info. - * we are adding NIs so we need to prep for that by - * creating a new ping info, to replace the old one */ - rc = lnet_ping_info_setup(&pinfo, &md_handle, - new_ni_count + lnet_get_ni_count()); - if (rc != 0) - goto failed; - } - - while (!list_empty(&nilist)) { + while (!list_empty(nilist)) { ni = list_entry(nilist.next, lnet_ni_t, ni_list); lnd_type = LNET_NETTYP(LNET_NIDNET(ni->ni_nid)); @@ -1693,7 +1631,7 @@ lnet_startup_lndnis(char *net, int do_full_config, __s32 peer_to, LCONSOLE_ERROR_MSG(0x106, "LND %s not supported in a " "single-threaded runtime\n", libcfs_lnd2str(lnd_type)); - goto failed_ni_cleanup; + goto failed; # endif } #endif @@ -1703,7 +1641,7 @@ lnet_startup_lndnis(char *net, int do_full_config, __s32 peer_to, libcfs_lnd2str(lnd->lnd_type), ni->ni_peertxcredits == 0 ? "" : "per-peer "); - goto failed_ni_cleanup; + goto failed; } cfs_percpt_for_each(tq, i, ni->ni_tx_queues) { @@ -1720,21 +1658,8 @@ lnet_startup_lndnis(char *net, int do_full_config, __s32 peer_to, nicount++; } - if (pinfo != NULL) - lnet_ping_target_update(pinfo, md_handle); - return nicount; - -failed_ni_cleanup: - if (!do_full_config) - lnet_shutdown_lndni(LNET_NIDNET(ni->ni_nid)); - failed: - if (pinfo != NULL) { - lnet_ping_md_unlink(pinfo, &md_handle); - lnet_ping_info_free(pinfo); - } - while (!list_empty(&nilist)) { ni = list_entry(nilist.next, lnet_ni_t, ni_list); list_del(&ni->ni_list); @@ -1743,91 +1668,6 @@ failed: return -EINVAL; } -static int -lnet_init_ni_system(char *net, __s32 peer_to, __s32 peer_cr, - __s32 peer_buf_cr, __s32 credits) -{ - int rc; - - rc = lnet_prepare_ni_config_init(LUSTRE_SRV_LNET_PID); - if (rc < 0) - return rc; - - /* Indicate that we have initialized the NI system so - * that if the user calls "lctl network up", this call - * would be a noop, as ln_niinit_self has already been - * set. The goal of this code is to allow LNET to be - * configured in a completely dynamic way. The NI system - * would get initialized on the first added network, then - * other configurations can be done, such as configuring, - * routes, routing pools, etc. */ - the_lnet.ln_niinit_self = 1; - - rc = lnet_startup_lndnis(net, 1, peer_to, peer_cr, - peer_buf_cr, credits); - if (rc < 0) - goto failed; - - rc = lnet_complete_ni_config_init(); - if (rc != 0) - goto failed; - - return 0; - -failed: - lnet_shutdown_lndnis(); - lnet_unprepare(); - return rc; -} - -static int -lnet_startup_lndnis_from_ioctl(char *net, __s32 peer_to, __s32 peer_cr, - __s32 peer_buf_cr, __s32 credits) -{ - int rc; - - /* If we have never configured a NI yet, we need to do a full - * initialization of the NI system. */ - if (the_lnet.ln_refcount == 0 && - !the_lnet.ln_niinit_self) - return lnet_init_ni_system(net, peer_to, peer_cr, - peer_buf_cr, credits); - - - rc = lnet_startup_lndnis(net, 0, peer_to, peer_cr, - peer_buf_cr, credits); - if (rc < 0) - return rc; - - return rc; -} - -static int -lnet_startup_lndnis_from_params(void) -{ - int lnd_type; - int nicount = 0; - char *nets = lnet_get_networks(); - - nicount = lnet_startup_lndnis(nets, 1, -1, -1, -1, -1); - if (nicount < 0) - goto failed; - - if (the_lnet.ln_eq_waitni != NULL && nicount > 1) { - lnd_type = the_lnet.ln_eq_waitni->ni_lnd->lnd_type; - LCONSOLE_ERROR_MSG(0x109, "LND %s can only run single-network" - "\n", - libcfs_lnd2str(lnd_type)); - goto failed; - } - - return 0; - - failed: - lnet_shutdown_lndnis(); - return -ENETDOWN; -} - /** * Initialize LNet library. * @@ -1961,39 +1801,56 @@ LNetNIInit(lnet_pid_t requested_pid) goto out; } - rc = lnet_prepare_ni_config_init(requested_pid); + rc = lnet_prepare(requested_pid); if (rc != 0) goto failed0; - rc = lnet_startup_lndnis_from_params(); - if (rc != 0) + rc = lnet_startup_lndnis(nets, 1, -1, -1, -1, -1); + if (rc < 0) goto failed1; - rc = lnet_parse_routes(lnet_get_routes(), &im_a_router); + rc = lnet_ping_info_setup(&pinfo, &md_handle, rc, true); if (rc != 0) goto failed2; + if (the_lnet.ln_eq_waitni != NULL && nicount > 1) { + lnd_type = the_lnet.ln_eq_waitni->ni_lnd->lnd_type; + LCONSOLE_ERROR_MSG(0x109, "LND %s can only run single-network" + "\n", + libcfs_lnd2str(lnd_type)); + goto failed3; + } + + rc = lnet_parse_routes(lnet_get_routes(), &im_a_router); + if (rc != 0) + goto failed3; + rc = lnet_check_routes(); if (rc != 0) - goto failed2; + goto failed3; rc = lnet_rtrpools_alloc(im_a_router); if (rc != 0) - goto failed2; + goto failed3; - rc = lnet_complete_ni_config_init(); + rc = lnet_misc_init(); if (rc == 0) - goto out; + goto failed3; - failed2: + lnet_ping_target_update(pinfo, md_handle); + LNET_MUTEX_UNLOCK(&the_lnet.ln_api_mutex); + return 0; + failed3: lnet_destroy_routes(); lnet_shutdown_lndnis(); + failed2: + lnet_ping_md_unlink(pinfo, &mdh); + lnet_ping_info_free(pinfo); failed1: lnet_unprepare(); failed0: - LASSERT (rc < 0); - out: LNET_MUTEX_UNLOCK(&the_lnet.ln_api_mutex); + LASSERT (rc < 0); return rc; } EXPORT_SYMBOL(LNetNIInit); @@ -2038,6 +1895,77 @@ LNetNIFini() } EXPORT_SYMBOL(LNetNIFini); +int +LNetDynNIInit(lnet_pid_t requested_pid, bool prepare, char *nets, ...) +{ + lnet_handle_md_t mdh; + LIST_HEAD (head); + char *net; + int rc; + + /* Create a ni structure for the network string */ + rc = lnet_parse_networks(&head, nets); + if (rc < 0) + return rc; + + LNET_MUTEX_LOCK(&the_lnet.ln_api_mutex); + if (rc > 1) { + rc = -EINVAL; /* only add one interface per call */ + goto out_0; + } + + rc = lnet_ping_info_setup(&pinfo, &mdh, 1 + lnet_get_ni_count(), prepare); + if (rc != 0) + goto out_0; + + if (!prepare) { + rc = lnet_startup_lndnis(head, 0, peer_to, peer_cr, + peer_buf_cr, credits); + if (rc != 0) + goto out_1; + } else { + rc = lnet_prepare(requested_pid); + if (rc != 0) + goto out_1; + + rc = lnet_startup_lndnis(head, 0, peer_to, peer_cr, + peer_buf_cr, credits); + if (rc != 0) + goto out_2; + + rc = lnet_misc_init(); + if (rc != 0) + goto out_3; + } + + lnet_ping_target_update(pinfo, md_handle); + LNET_MUTEX_UNLOCK(&the_lnet.ln_api_mutex); + return 0; +out_3: + lnet_shutdown_lndnis(); +out_2: + lnet_unprepare(); +out_1: + lnet_ping_md_unlink(pinfo, &mdh); + lnet_ping_info_free(pinfo); +out_0: + LNET_MUTEX_UNLOCK(&the_lnet.ln_api_mutex); + while (!list_empty(&head)) { + ni = list_entry(head.next, struct lnet_ni, ni_list); + list_del_init(&ni->ni_list); + lnet_ni_free(ni); + } + return rc; +} + +void +LNetDynNIFini(__u32 net) +{ + LNET_MUTEX_LOCK(&the_lnet.ln_api_mutex); + rc = lnet_shutdown_lndni(net); + LNET_MUTEX_UNLOCK(&the_lnet.ln_api_mutex); +} + /** * This is an ugly hack to export IOC_LIBCFS_DEBUG_PEER and * IOC_LIBCFS_PORTALS_COMPATIBILITY commands to users, by tweaking the LNet @@ -2097,28 +2025,6 @@ LNetCtl(unsigned int cmd, void *arg) &config->ioc_config_u.route.flags, &config->ioc_config_u.route.priority); - case IOC_LIBCFS_ADD_NET: - config = arg; - LNET_MUTEX_LOCK(&the_lnet.ln_api_mutex); - rc = lnet_startup_lndnis_from_ioctl - (config->ioc_config_u.net.intf, - config->ioc_config_u.net.peer_timeout, - config->ioc_config_u.net. - peer_tx_credits, - config->ioc_config_u.net. - peer_rtr_credits, - config->ioc_config_u.net. - max_tx_credits); - LNET_MUTEX_UNLOCK(&the_lnet.ln_api_mutex); - return rc; - - case IOC_LIBCFS_DEL_NET: - config = arg; - LNET_MUTEX_LOCK(&the_lnet.ln_api_mutex); - rc = lnet_shutdown_lndni(config->ioc_net); - LNET_MUTEX_UNLOCK(&the_lnet.ln_api_mutex); - return rc; - case IOC_LIBCFS_GET_NET: { struct libcfs_ioctl_net_config *net_config; diff --git a/lnet/lnet/module.c b/lnet/lnet/module.c index ed89dea..2d4caeb 100755 --- a/lnet/lnet/module.c +++ b/lnet/lnet/module.c @@ -67,7 +67,7 @@ int lnet_unconfigure (void) { int refcount; - + LNET_MUTEX_LOCK(&lnet_config_mutex); if (the_lnet.ln_niinit_self) { @@ -84,6 +84,30 @@ lnet_unconfigure (void) } int +lnet_dyn_configure(struct libcfs_ioctl_config_data *conf) +{ + int rc; + + LNET_MUTEX_LOCK(&lnet_config_mutex); + rc = LNetDynNIInit(LUSTRE_SRV_LNET_PID, !the_lnet.ln_niinit_self, + conf->ioc_config_u.net.intf, + conf->ioc_config_u.net.peer_timeout, + conf->ioc_config_u.net.peer_tx_credits, + conf->ioc_config_u.net.peer_rtr_credits, + conf->ioc_config_u.net.max_tx_credits); + LNET_MUTEX_UNLOCK(&lnet_config_mutex); + return rc; +} + +int +lnet_dyn_unconfigure(...) +{ + LNET_MUTEX_LOCK(&lnet_config_mutex); + LNetDynNIFini(net); + LNET_MUTEX_UNLOCK(&lnet_config_mutex); +} + +int lnet_ioctl(unsigned int cmd, struct libcfs_ioctl_hdr *hdr) { int rc; @@ -96,7 +120,10 @@ lnet_ioctl(unsigned int cmd, struct libcfs_ioctl_hdr *hdr) return lnet_unconfigure(); case IOC_LIBCFS_ADD_NET: - return LNetCtl(cmd, hdr); + return lnet_dyn_configure(hdr); + + case IOC_LIBCFS_DEL_NET: + return lnet_dyn_configure(hdr); default: /* Passing LNET_PID_ANY only gives me a ref if the net is up