Lines Matching refs:state

429 static int ibd_part_attach(ibd_state_t *state, dev_info_t *dip);
430 static void ibd_part_unattach(ibd_state_t *state);
432 static int ibd_port_unattach(ibd_state_t *state, dev_info_t *dip);
598 ibd_print_warn(ibd_state_t *state, char *fmt, ...) in ibd_print_warn() argument
605 datalink_id_t linkid = state->id_plinkid; in ibd_print_warn()
607 hca_guid = ddi_prop_get_int64(DDI_DEV_T_ANY, state->id_dip, in ibd_print_warn()
612 ddi_driver_name(state->id_dip), ddi_get_instance(state->id_dip), in ibd_print_warn()
613 (u_longlong_t)hca_guid, state->id_port, state->id_pkey, in ibd_print_warn()
843 ibd_rc_chan_s::state in _NOTE()
1089 ibd_get_allroutergroup(ibd_state_t *state, ipoib_mac_t *mcmac, in ibd_get_allroutergroup() argument
1093 uint32_t adjscope = state->id_scope << 16; in ibd_get_allroutergroup()
1110 ((uint32_t)(state->id_pkey << 16))), in ibd_get_allroutergroup()
1148 #define IBD_MCACHE_INSERT_FULL(state, mce) \ argument
1149 list_insert_head(&state->id_mc_full, mce)
1150 #define IBD_MCACHE_INSERT_NON(state, mce) \ argument
1151 list_insert_head(&state->id_mc_non, mce)
1152 #define IBD_MCACHE_FIND_FULL(state, mgid) \ argument
1153 ibd_mcache_find(mgid, &state->id_mc_full)
1154 #define IBD_MCACHE_FIND_NON(state, mgid) \ argument
1155 ibd_mcache_find(mgid, &state->id_mc_non)
1156 #define IBD_MCACHE_PULLOUT_FULL(state, mce) \ argument
1157 list_remove(&state->id_mc_full, mce)
1158 #define IBD_MCACHE_PULLOUT_NON(state, mce) \ argument
1159 list_remove(&state->id_mc_non, mce)
1175 ibd_queue_work_slot(ibd_state_t *state, ibd_req_t *ptr, int op) in ibd_queue_work_slot() argument
1184 mutex_enter(&state->id_acache_req_lock); in ibd_queue_work_slot()
1185 list_insert_tail(&state->id_req_list, ptr); in ibd_queue_work_slot()
1188 cv_signal(&state->id_acache_req_cv); in ibd_queue_work_slot()
1189 mutex_exit(&state->id_acache_req_lock); in ibd_queue_work_slot()
1196 ibd_async_work(ibd_state_t *state) in ibd_async_work() argument
1201 mutex_enter(&state->id_acache_req_lock); in ibd_async_work()
1202 CALLB_CPR_INIT(&cprinfo, &state->id_acache_req_lock, in ibd_async_work()
1206 ptr = list_get_head(&state->id_req_list); in ibd_async_work()
1208 mutex_exit(&state->id_acache_req_lock); in ibd_async_work()
1216 if ((state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) && in ibd_async_work()
1231 ibd_async_acache(state, &ptr->rq_mac); in ibd_async_work()
1235 ibd_async_multicast(state, in ibd_async_work()
1239 ibd_async_setprom(state); in ibd_async_work()
1242 ibd_async_unsetprom(state); in ibd_async_work()
1245 ibd_async_reap_group(state, in ibd_async_work()
1256 ibd_async_trap(state, ptr); in ibd_async_work()
1259 ibd_async_txsched(state); in ibd_async_work()
1262 ibd_async_link(state, ptr); in ibd_async_work()
1265 mutex_enter(&state->id_acache_req_lock); in ibd_async_work()
1269 mutex_exit(&state->id_acache_req_lock); in ibd_async_work()
1273 ibd_async_rc_process_too_big(state, in ibd_async_work()
1277 ibd_async_rc_close_act_chan(state, ptr); in ibd_async_work()
1280 ibd_async_rc_recycle_ace(state, ptr); in ibd_async_work()
1289 kmem_cache_free(state->id_req_kmc, ptr); in ibd_async_work()
1291 mutex_enter(&state->id_acache_req_lock); in ibd_async_work()
1298 cv_wait(&state->id_acache_req_cv, in ibd_async_work()
1299 &state->id_acache_req_lock); in ibd_async_work()
1301 &state->id_acache_req_lock); in ibd_async_work()
1316 ibd_async_safe(ibd_state_t *state) in ibd_async_safe() argument
1318 mutex_enter(&state->id_trap_lock); in ibd_async_safe()
1319 if (state->id_trap_stop) { in ibd_async_safe()
1320 mutex_exit(&state->id_trap_lock); in ibd_async_safe()
1323 state->id_trap_inprog++; in ibd_async_safe()
1324 mutex_exit(&state->id_trap_lock); in ibd_async_safe()
1334 ibd_async_done(ibd_state_t *state) in ibd_async_done() argument
1336 mutex_enter(&state->id_trap_lock); in ibd_async_done()
1337 if (--state->id_trap_inprog == 0) in ibd_async_done()
1338 cv_signal(&state->id_trap_cv); in ibd_async_done()
1339 mutex_exit(&state->id_trap_lock); in ibd_async_done()
1383 ibd_acache_init(ibd_state_t *state) in ibd_acache_init() argument
1388 mutex_init(&state->id_ac_mutex, NULL, MUTEX_DRIVER, NULL); in ibd_acache_init()
1389 mutex_init(&state->id_mc_mutex, NULL, MUTEX_DRIVER, NULL); in ibd_acache_init()
1390 mutex_enter(&state->id_ac_mutex); in ibd_acache_init()
1391 list_create(&state->id_ah_free, sizeof (ibd_ace_t), in ibd_acache_init()
1393 list_create(&state->id_ah_active, sizeof (ibd_ace_t), in ibd_acache_init()
1395 state->id_ah_active_hash = mod_hash_create_extended("IBD AH hash", in ibd_acache_init()
1396 state->id_hash_size, mod_hash_null_keydtor, mod_hash_null_valdtor, in ibd_acache_init()
1398 list_create(&state->id_mc_full, sizeof (ibd_mce_t), in ibd_acache_init()
1400 list_create(&state->id_mc_non, sizeof (ibd_mce_t), in ibd_acache_init()
1402 state->id_ac_hot_ace = NULL; in ibd_acache_init()
1404 state->id_ac_list = ce = (ibd_ace_t *)kmem_zalloc(sizeof (ibd_ace_t) * in ibd_acache_init()
1405 state->id_num_ah, KM_SLEEP); in ibd_acache_init()
1406 for (i = 0; i < state->id_num_ah; i++, ce++) { in ibd_acache_init()
1407 if (ibt_alloc_ud_dest(state->id_hca_hdl, IBT_UD_DEST_NO_FLAGS, in ibd_acache_init()
1408 state->id_pd_hdl, &ce->ac_dest) != IBT_SUCCESS) { in ibd_acache_init()
1409 mutex_exit(&state->id_ac_mutex); in ibd_acache_init()
1410 ibd_acache_fini(state); in ibd_acache_init()
1417 IBD_ACACHE_INSERT_FREE(state, ce); in ibd_acache_init()
1420 mutex_exit(&state->id_ac_mutex); in ibd_acache_init()
1425 ibd_acache_fini(ibd_state_t *state) in ibd_acache_fini() argument
1429 mutex_enter(&state->id_ac_mutex); in ibd_acache_fini()
1431 while ((ptr = IBD_ACACHE_GET_ACTIVE(state)) != NULL) { in ibd_acache_fini()
1437 while ((ptr = IBD_ACACHE_GET_FREE(state)) != NULL) { in ibd_acache_fini()
1443 list_destroy(&state->id_ah_free); in ibd_acache_fini()
1444 list_destroy(&state->id_ah_active); in ibd_acache_fini()
1445 list_destroy(&state->id_mc_full); in ibd_acache_fini()
1446 list_destroy(&state->id_mc_non); in ibd_acache_fini()
1447 kmem_free(state->id_ac_list, sizeof (ibd_ace_t) * state->id_num_ah); in ibd_acache_fini()
1448 mutex_exit(&state->id_ac_mutex); in ibd_acache_fini()
1449 mutex_destroy(&state->id_ac_mutex); in ibd_acache_fini()
1450 mutex_destroy(&state->id_mc_mutex); in ibd_acache_fini()
1460 ibd_acache_find(ibd_state_t *state, ipoib_mac_t *mac, boolean_t hold, int num) in ibd_acache_find() argument
1464 ASSERT(mutex_owned(&state->id_ac_mutex)); in ibd_acache_find()
1469 if (mod_hash_find(state->id_ah_active_hash, in ibd_acache_find()
1484 ibd_acache_lookup(ibd_state_t *state, ipoib_mac_t *mac, int *err, int numwqe) in ibd_acache_lookup() argument
1501 mutex_enter(&state->id_ac_mutex); in ibd_acache_lookup()
1503 if (((ptr = state->id_ac_hot_ace) != NULL) && in ibd_acache_lookup()
1506 mutex_exit(&state->id_ac_mutex); in ibd_acache_lookup()
1509 if (((ptr = ibd_acache_find(state, mac, B_TRUE, numwqe)) != NULL)) { in ibd_acache_lookup()
1510 state->id_ac_hot_ace = ptr; in ibd_acache_lookup()
1511 mutex_exit(&state->id_ac_mutex); in ibd_acache_lookup()
1528 if (state->id_ah_op == IBD_OP_NOTSTARTED) { in ibd_acache_lookup()
1529 req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP); in ibd_acache_lookup()
1536 state->id_ah_op = IBD_OP_ONGOING; in ibd_acache_lookup()
1537 ibd_queue_work_slot(state, req, IBD_ASYNC_GETAH); in ibd_acache_lookup()
1538 bcopy(mac, &state->id_ah_addr, IPOIB_ADDRL); in ibd_acache_lookup()
1540 } else if ((state->id_ah_op != IBD_OP_ONGOING) && in ibd_acache_lookup()
1541 (bcmp(&state->id_ah_addr, mac, IPOIB_ADDRL) == 0)) { in ibd_acache_lookup()
1546 if (state->id_ah_op == IBD_OP_ERRORED) { in ibd_acache_lookup()
1548 state->id_ah_error++; in ibd_acache_lookup()
1559 (void) ibd_get_allroutergroup(state, mac, &routermac); in ibd_acache_lookup()
1560 ptr = ibd_acache_find(state, &routermac, B_TRUE, in ibd_acache_lookup()
1563 state->id_ah_op = IBD_OP_NOTSTARTED; in ibd_acache_lookup()
1564 } else if ((state->id_ah_op != IBD_OP_ONGOING) && in ibd_acache_lookup()
1565 (bcmp(&state->id_ah_addr, mac, IPOIB_ADDRL) != 0)) { in ibd_acache_lookup()
1571 state->id_ah_op = IBD_OP_NOTSTARTED; in ibd_acache_lookup()
1573 mutex_exit(&state->id_ac_mutex); in ibd_acache_lookup()
1584 ibd_acache_get_unref(ibd_state_t *state) in ibd_acache_get_unref() argument
1586 ibd_ace_t *ptr = list_tail(&state->id_ah_active); in ibd_acache_get_unref()
1589 ASSERT(mutex_owned(&state->id_ac_mutex)); in ibd_acache_get_unref()
1605 ASSERT(state->id_enable_rc == B_TRUE); in ibd_acache_get_unref()
1608 ibd_rc_signal_ace_recycle(state, ptr); in ibd_acache_get_unref()
1611 IBD_ACACHE_PULLOUT_ACTIVE(state, ptr); in ibd_acache_get_unref()
1615 ptr = list_prev(&state->id_ah_active, ptr); in ibd_acache_get_unref()
1635 ibd_acache_recycle(ibd_state_t *state, ipoib_mac_t *acmac, boolean_t force) in ibd_acache_recycle() argument
1640 ASSERT(mutex_owned(&state->id_ac_mutex)); in ibd_acache_recycle()
1642 if ((acactive = ibd_acache_find(state, acmac, B_FALSE, 0)) != NULL) { in ibd_acache_recycle()
1667 IBD_ACACHE_PULLOUT_ACTIVE(state, acactive); in ibd_acache_recycle()
1677 IBD_ACACHE_PULLOUT_ACTIVE(state, acactive); in ibd_acache_recycle()
1679 IBD_ACACHE_INSERT_FREE(state, acactive); in ibd_acache_recycle()
1694 ibd_async_mcache(ibd_state_t *state, ipoib_mac_t *mac, boolean_t *redirect) in ibd_async_mcache() argument
1708 mce = IBD_MCACHE_FIND_FULL(state, mgid); in ibd_async_mcache()
1718 if ((mce = ibd_join_group(state, mgid, IB_MC_JSTATE_SEND_ONLY_NON)) != in ibd_async_mcache()
1730 if (!ibd_get_allroutergroup(state, mac, &routermac)) in ibd_async_mcache()
1741 if ((mce = IBD_MCACHE_FIND_FULL(state, mgid)) != NULL) { in ibd_async_mcache()
1751 if ((mce = ibd_join_group(state, mgid, IB_MC_JSTATE_SEND_ONLY_NON)) != in ibd_async_mcache()
1764 ibd_async_acache(ibd_state_t *state, ipoib_mac_t *mac) in ibd_async_acache() argument
1790 if ((mce = ibd_async_mcache(state, mac, &redirected)) == in ibd_async_acache()
1792 state->id_ah_op = IBD_OP_ERRORED; in ibd_async_acache()
1814 mutex_enter(&state->id_ac_mutex); in ibd_async_acache()
1815 if (ibd_acache_find(state, mac, B_FALSE, 0) != NULL) { in ibd_async_acache()
1816 state->id_ah_op = IBD_OP_ROUTERED; in ibd_async_acache()
1817 mutex_exit(&state->id_ac_mutex); in ibd_async_acache()
1821 mutex_exit(&state->id_ac_mutex); in ibd_async_acache()
1828 mutex_enter(&state->id_ac_mutex); in ibd_async_acache()
1829 if ((ce = IBD_ACACHE_GET_FREE(state)) == NULL) { in ibd_async_acache()
1839 if ((ce = ibd_acache_get_unref(state)) == NULL) { in ibd_async_acache()
1843 state->id_ah_op = IBD_OP_NOTSTARTED; in ibd_async_acache()
1844 mutex_exit(&state->id_ac_mutex); in ibd_async_acache()
1856 mutex_exit(&state->id_ac_mutex); in ibd_async_acache()
1866 path_attr.pa_sgid = state->id_sgid; in ibd_async_acache()
1870 path_attr.pa_sl = state->id_mcinfo->mc_adds_vect.av_srvl; in ibd_async_acache()
1871 path_attr.pa_pkey = state->id_pkey; in ibd_async_acache()
1872 if (ibt_get_paths(state->id_ibt_hdl, IBT_PATH_PKEY, &path_attr, 1, in ibd_async_acache()
1877 if (ibt_modify_ud_dest(ce->ac_dest, state->id_mcinfo->mc_qkey, in ibd_async_acache()
1895 if (state->id_enable_rc && (mac->ipoib_qpn != htonl(IB_MC_QPN)) && in ibd_async_acache()
1900 ibd_rc_try_connect(state, ce, &path_info); in ibd_async_acache()
1904 state->rc_conn_fail++; in ibd_async_acache()
1909 mutex_enter(&state->id_ac_mutex); in ibd_async_acache()
1910 IBD_ACACHE_INSERT_ACTIVE(state, ce); in ibd_async_acache()
1911 state->id_ah_op = ret; in ibd_async_acache()
1912 mutex_exit(&state->id_ac_mutex); in ibd_async_acache()
1920 mutex_enter(&state->id_ac_mutex); in ibd_async_acache()
1921 state->id_ah_op = IBD_OP_ERRORED; in ibd_async_acache()
1922 IBD_ACACHE_INSERT_FREE(state, ce); in ibd_async_acache()
1923 mutex_exit(&state->id_ac_mutex); in ibd_async_acache()
1931 ibd_async_link(ibd_state_t *state, ibd_req_t *req) in ibd_async_link() argument
1949 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*state)) in ibd_async_link()
1950 state->id_link_speed = ibd_get_portspeed(state); in ibd_async_link()
1951 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*state)) in ibd_async_link()
1962 if (state->id_prom_op == IBD_OP_COMPLETED) { in ibd_async_link()
1966 ibd_async_unsetprom(state); in ibd_async_link()
1971 ibd_async_setprom(state); in ibd_async_link()
1979 mce = list_head(&state->id_mc_full); in ibd_async_link()
1981 mce = list_next(&state->id_mc_full, mce); in ibd_async_link()
1983 ibd_leave_group(state, in ibd_async_link()
1987 ibd_reacquire_group(state, pmce); in ibd_async_link()
1996 mutex_enter(&state->id_ac_mutex); in ibd_async_link()
1997 ace = list_head(&state->id_ah_active); in ibd_async_link()
2001 ace = list_next(&state->id_ah_active, ace); in ibd_async_link()
2005 ASSERT(state->id_enable_rc == B_TRUE); in ibd_async_link()
2009 IBD_ACACHE_PULLOUT_ACTIVE(state, pace); in ibd_async_link()
2012 ibd_rc_signal_act_close(state, pace); in ibd_async_link()
2014 state->rc_act_close_simultaneous++; in ibd_async_link()
2022 cycled = ibd_acache_recycle(state, in ibd_async_link()
2045 ibd_async_reap_group(state, mce, in ibd_async_link()
2049 mutex_exit(&state->id_ac_mutex); in ibd_async_link()
2058 mutex_enter(&state->id_link_mutex); in ibd_async_link()
2059 state->id_link_state = lstate; in ibd_async_link()
2060 mac_link_update(state->id_mh, lstate); in ibd_async_link()
2061 mutex_exit(&state->id_link_mutex); in ibd_async_link()
2063 ibd_async_done(state); in ibd_async_link()
2109 ibd_link_mod(ibd_state_t *state, ibt_async_code_t code) in ibd_link_mod() argument
2125 ibd_set_mac_progress(state, IBD_DRV_RESTART_IN_PROGRESS); in ibd_link_mod()
2127 mutex_enter(&state->id_link_mutex); in ibd_link_mod()
2133 if (state->id_link_state == LINK_STATE_UNKNOWN) { in ibd_link_mod()
2134 mutex_exit(&state->id_link_mutex); in ibd_link_mod()
2142 if ((state->id_link_state == LINK_STATE_DOWN) && in ibd_link_mod()
2143 ((state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) == 0) && in ibd_link_mod()
2144 ((state->id_mac_state & IBD_DRV_STARTED) == 0)) { in ibd_link_mod()
2145 mutex_exit(&state->id_link_mutex); in ibd_link_mod()
2165 ibt_status = ibt_query_hca_ports(state->id_hca_hdl, state->id_port, in ibd_link_mod()
2178 if (state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) { in ibd_link_mod()
2179 mutex_exit(&state->id_link_mutex); in ibd_link_mod()
2180 if ((ret = ibd_start(state)) != 0) { in ibd_link_mod()
2203 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_sgid)) in ibd_link_mod()
2205 &state->id_sgid, sizeof (ib_gid_t)) != 0) { in ibd_link_mod()
2209 } else if (port_infop->p_pkey_tbl[state->id_pkix] == in ibd_link_mod()
2210 state->id_pkey) { in ibd_link_mod()
2215 port_infop->p_pkey_tbl_sz, state->id_pkey, &pkix) == 0) { in ibd_link_mod()
2218 mutex_exit(&state->id_link_mutex); in ibd_link_mod()
2228 (void) ibd_undo_start(state, LINK_STATE_DOWN); in ibd_link_mod()
2229 if ((ret = ibd_start(state)) != 0) { in ibd_link_mod()
2238 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_sgid)) in ibd_link_mod()
2266 if ((state->id_link_state == new_link_state) && (opcode != in ibd_link_mod()
2268 mutex_exit(&state->id_link_mutex); in ibd_link_mod()
2276 if (!ibd_async_safe(state)) { in ibd_link_mod()
2277 state->id_link_state = new_link_state; in ibd_link_mod()
2278 mutex_exit(&state->id_link_mutex); in ibd_link_mod()
2282 mutex_exit(&state->id_link_mutex); in ibd_link_mod()
2288 req = kmem_cache_alloc(state->id_req_kmc, KM_SLEEP); in ibd_link_mod()
2290 ibd_queue_work_slot(state, req, IBD_ASYNC_LINK); in ibd_link_mod()
2293 ibd_clr_mac_progress(state, IBD_DRV_RESTART_IN_PROGRESS); in ibd_link_mod()
2306 ibd_state_t *state = (ibd_state_t *)clnt_private; in ibd_async_handler() local
2310 ibd_print_warn(state, "catastrophic channel error"); in ibd_async_handler()
2313 ibd_print_warn(state, "completion queue error"); in ibd_async_handler()
2324 ASSERT(state->id_hca_hdl == hca_hdl); in ibd_async_handler()
2325 if (state->id_port != event->ev_port) in ibd_async_handler()
2330 ibd_link_mod(state, code); in ibd_async_handler()
2344 ASSERT(state->id_hca_hdl == hca_hdl); in ibd_async_handler()
2345 if (state->id_port != event->ev_port) in ibd_async_handler()
2348 ibd_link_mod(state, code); in ibd_async_handler()
2372 ibd_register_mac(ibd_state_t *state, dev_info_t *dip) in ibd_register_mac() argument
2390 macp->m_driver = state; in ibd_register_mac()
2392 macp->m_src_addr = (uint8_t *)&state->id_macaddr; in ibd_register_mac()
2396 if (state->id_type == IBD_PORT_DRIVER) { in ibd_register_mac()
2398 } else if (state->id_enable_rc) { in ibd_register_mac()
2399 macp->m_max_sdu = state->rc_mtu - IPOIB_HDRSIZE; in ibd_register_mac()
2408 if ((ret = mac_register(macp, &state->id_mh)) != 0) { in ibd_register_mac()
2420 ibd_record_capab(ibd_state_t *state) in ibd_record_capab() argument
2425 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*state)) in ibd_record_capab()
2430 ibt_status = ibt_query_hca(state->id_hca_hdl, &hca_attrs); in ibd_record_capab()
2437 if (state->id_enable_rc) { in ibd_record_capab()
2438 state->id_hwcksum_capab = 0; in ibd_record_capab()
2442 state->id_hwcksum_capab = IBT_HCA_CKSUM_FULL; in ibd_record_capab()
2449 if (state->id_enable_rc) { in ibd_record_capab()
2450 state->id_lso_capable = B_FALSE; in ibd_record_capab()
2451 state->id_lso_maxlen = 0; in ibd_record_capab()
2454 state->id_lso_capable = B_TRUE; in ibd_record_capab()
2456 state->id_lso_maxlen = IBD_LSO_MAXLEN; in ibd_record_capab()
2458 state->id_lso_maxlen = in ibd_record_capab()
2461 state->id_lso_capable = B_FALSE; in ibd_record_capab()
2462 state->id_lso_maxlen = 0; in ibd_record_capab()
2470 state->id_hca_res_lkey_capab = 1; in ibd_record_capab()
2471 state->id_res_lkey = hca_attrs.hca_reserved_lkey; in ibd_record_capab()
2472 state->rc_enable_iov_map = B_TRUE; in ibd_record_capab()
2475 state->rc_enable_iov_map = B_FALSE; in ibd_record_capab()
2483 state->id_max_sqseg = hca_attrs.hca_ud_send_sgl_sz; in ibd_record_capab()
2484 state->rc_tx_max_sqseg = hca_attrs.hca_conn_send_sgl_sz; in ibd_record_capab()
2486 state->id_max_sqseg = hca_attrs.hca_max_sgl; in ibd_record_capab()
2487 state->rc_tx_max_sqseg = hca_attrs.hca_max_sgl; in ibd_record_capab()
2489 if (state->id_max_sqseg > IBD_MAX_SQSEG) { in ibd_record_capab()
2490 state->id_max_sqseg = IBD_MAX_SQSEG; in ibd_record_capab()
2491 } else if (state->id_max_sqseg < IBD_MAX_SQSEG) { in ibd_record_capab()
2492 ibd_print_warn(state, "Set #sgl = %d instead of default %d", in ibd_record_capab()
2493 state->id_max_sqseg, IBD_MAX_SQSEG); in ibd_record_capab()
2495 if (state->rc_tx_max_sqseg > IBD_MAX_SQSEG) { in ibd_record_capab()
2496 state->rc_tx_max_sqseg = IBD_MAX_SQSEG; in ibd_record_capab()
2497 } else if (state->rc_tx_max_sqseg < IBD_MAX_SQSEG) { in ibd_record_capab()
2498 ibd_print_warn(state, "RC mode: Set #sgl = %d instead of " in ibd_record_capab()
2499 "default %d", state->rc_tx_max_sqseg, IBD_MAX_SQSEG); in ibd_record_capab()
2508 state->id_max_sqseg_hiwm = (state->id_max_sqseg * 65) / 100; in ibd_record_capab()
2509 state->rc_max_sqseg_hiwm = (state->rc_tx_max_sqseg * 65) / 100; in ibd_record_capab()
2518 state->id_hca_max_chan_sz = hca_attrs.hca_max_chan_sz; in ibd_record_capab()
2520 if (hca_attrs.hca_max_chan_sz < state->id_ud_num_rwqe) in ibd_record_capab()
2521 state->id_ud_num_rwqe = hca_attrs.hca_max_chan_sz; in ibd_record_capab()
2523 state->id_rx_bufs_outstanding_limit = state->id_ud_num_rwqe - in ibd_record_capab()
2526 if (hca_attrs.hca_max_chan_sz < state->id_ud_num_swqe) in ibd_record_capab()
2527 state->id_ud_num_swqe = hca_attrs.hca_max_chan_sz; in ibd_record_capab()
2529 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*state)) in ibd_record_capab()
2535 ibd_part_busy(ibd_state_t *state) in ibd_part_busy() argument
2537 if (atomic_add_32_nv(&state->id_rx_list.dl_bufs_outstanding, 0) != 0) { in ibd_part_busy()
2542 if (state->rc_srq_rwqe_list.dl_bufs_outstanding != 0) { in ibd_part_busy()
2551 if (state->id_ah_op == IBD_OP_ONGOING) { in ibd_part_busy()
2561 ibd_part_unattach(ibd_state_t *state) in ibd_part_unattach() argument
2563 uint32_t progress = state->id_mac_state; in ibd_part_unattach()
2567 ibd_free_rx_rsrcs(state); in ibd_part_unattach()
2570 ASSERT(state->id_enable_rc); in ibd_part_unattach()
2571 ibd_rc_fini_srq_list(state); in ibd_part_unattach()
2572 state->id_mac_state &= (~IBD_DRV_RC_SRQ_ALLOCD); in ibd_part_unattach()
2576 (void) mac_unregister(state->id_mh); in ibd_part_unattach()
2577 state->id_mac_state &= (~IBD_DRV_MAC_REGISTERED); in ibd_part_unattach()
2594 ibd_queue_work_slot(state, &state->id_ah_req, IBD_ASYNC_EXIT); in ibd_part_unattach()
2595 thread_join(state->id_async_thrid); in ibd_part_unattach()
2597 state->id_mac_state &= (~IBD_DRV_ASYNC_THR_CREATED); in ibd_part_unattach()
2601 list_destroy(&state->id_req_list); in ibd_part_unattach()
2602 mutex_destroy(&state->id_acache_req_lock); in ibd_part_unattach()
2603 cv_destroy(&state->id_acache_req_cv); in ibd_part_unattach()
2604 state->id_mac_state &= ~IBD_DRV_REQ_LIST_INITED; in ibd_part_unattach()
2608 if ((ret = ibt_free_pd(state->id_hca_hdl, in ibd_part_unattach()
2609 state->id_pd_hdl)) != IBT_SUCCESS) { in ibd_part_unattach()
2610 ibd_print_warn(state, "failed to free " in ibd_part_unattach()
2613 state->id_pd_hdl = NULL; in ibd_part_unattach()
2614 state->id_mac_state &= (~IBD_DRV_PD_ALLOCD); in ibd_part_unattach()
2618 if ((ret = ibt_close_hca(state->id_hca_hdl)) != in ibd_part_unattach()
2620 ibd_print_warn(state, "failed to close " in ibd_part_unattach()
2623 state->id_hca_hdl = NULL; in ibd_part_unattach()
2624 state->id_mac_state &= (~IBD_DRV_HCA_OPENED); in ibd_part_unattach()
2629 if ((ret = ibt_detach(state->id_ibt_hdl)) != in ibd_part_unattach()
2631 ibd_print_warn(state, in ibd_part_unattach()
2634 state->id_ibt_hdl = NULL; in ibd_part_unattach()
2635 state->id_mac_state &= (~IBD_DRV_IBTL_ATTACH_DONE); in ibd_part_unattach()
2642 ibd_print_warn(state, "ibt_detach(): global " in ibd_part_unattach()
2650 ddi_remove_softintr(state->id_tx); in ibd_part_unattach()
2651 state->id_tx = NULL; in ibd_part_unattach()
2652 state->id_mac_state &= (~IBD_DRV_TXINTR_ADDED); in ibd_part_unattach()
2656 ddi_remove_softintr(state->id_rx); in ibd_part_unattach()
2657 state->id_rx = NULL; in ibd_part_unattach()
2658 state->id_mac_state &= (~IBD_DRV_RXINTR_ADDED); in ibd_part_unattach()
2663 kstat_delete(state->rc_ksp); in ibd_part_unattach()
2664 state->id_mac_state &= (~IBD_DRV_RC_PRIVATE_STATE); in ibd_part_unattach()
2669 ibd_state_fini(state); in ibd_part_unattach()
2670 state->id_mac_state &= (~IBD_DRV_STATE_INITIALIZED); in ibd_part_unattach()
2675 ibd_part_attach(ibd_state_t *state, dev_info_t *dip) in ibd_part_attach() argument
2684 if (ibd_state_init(state, dip) != DDI_SUCCESS) { in ibd_part_attach()
2688 state->id_mac_state |= IBD_DRV_STATE_INITIALIZED; in ibd_part_attach()
2694 if ((rv = ddi_add_softintr(dip, DDI_SOFTINT_LOW, &state->id_rx, in ibd_part_attach()
2695 NULL, NULL, ibd_intr, (caddr_t)state)) != DDI_SUCCESS) { in ibd_part_attach()
2700 state->id_mac_state |= IBD_DRV_RXINTR_ADDED; in ibd_part_attach()
2703 if ((rv = ddi_add_softintr(dip, DDI_SOFTINT_LOW, &state->id_tx, in ibd_part_attach()
2705 (caddr_t)state)) != DDI_SUCCESS) { in ibd_part_attach()
2710 state->id_mac_state |= IBD_DRV_TXINTR_ADDED; in ibd_part_attach()
2718 if ((ret = ibt_attach(&ibd_clnt_modinfo, dip, state, in ibd_part_attach()
2726 if ((ret = ibt_attach(&ibd_clnt_modinfo, dip, state, in ibd_part_attach()
2727 &state->id_ibt_hdl)) != IBT_SUCCESS) { in ibd_part_attach()
2735 state->id_mac_state |= IBD_DRV_IBTL_ATTACH_DONE; in ibd_part_attach()
2740 if ((ret = ibt_open_hca(state->id_ibt_hdl, state->id_hca_guid, in ibd_part_attach()
2741 &state->id_hca_hdl)) != IBT_SUCCESS) { in ibd_part_attach()
2746 state->id_mac_state |= IBD_DRV_HCA_OPENED; in ibd_part_attach()
2750 if (state->id_enable_rc) { in ibd_part_attach()
2751 if (ibd_rc_init_stats(state) != DDI_SUCCESS) { in ibd_part_attach()
2756 state->id_mac_state |= IBD_DRV_RC_PRIVATE_STATE; in ibd_part_attach()
2763 (void) ibd_record_capab(state); in ibd_part_attach()
2768 if ((ret = ibt_alloc_pd(state->id_hca_hdl, IBT_PD_NO_FLAGS, in ibd_part_attach()
2769 &state->id_pd_hdl)) != IBT_SUCCESS) { in ibd_part_attach()
2774 state->id_mac_state |= IBD_DRV_PD_ALLOCD; in ibd_part_attach()
2781 mutex_init(&state->id_acache_req_lock, NULL, MUTEX_DRIVER, NULL); in ibd_part_attach()
2782 cv_init(&state->id_acache_req_cv, NULL, CV_DEFAULT, NULL); in ibd_part_attach()
2783 list_create(&state->id_req_list, sizeof (ibd_req_t), in ibd_part_attach()
2785 state->id_mac_state |= IBD_DRV_REQ_LIST_INITED; in ibd_part_attach()
2790 kht = thread_create(NULL, 0, ibd_async_work, state, 0, &p0, in ibd_part_attach()
2792 state->id_async_thrid = kht->t_did; in ibd_part_attach()
2793 state->id_mac_state |= IBD_DRV_ASYNC_THR_CREATED; in ibd_part_attach()
2823 ibd_state_t *state; in ibd_detach() local
2836 state = ddi_get_soft_state(ibd_list, instance); in ibd_detach()
2843 return (ibd_port_unattach(state, dip)); in ibd_detach()
2850 ibd_state_init(ibd_state_t *state, dev_info_t *dip) in ibd_state_init() argument
2854 mutex_init(&state->id_link_mutex, NULL, MUTEX_DRIVER, NULL); in ibd_state_init()
2855 state->id_link_state = LINK_STATE_UNKNOWN; in ibd_state_init()
2857 mutex_init(&state->id_trap_lock, NULL, MUTEX_DRIVER, NULL); in ibd_state_init()
2858 cv_init(&state->id_trap_cv, NULL, CV_DEFAULT, NULL); in ibd_state_init()
2859 state->id_trap_stop = B_TRUE; in ibd_state_init()
2860 state->id_trap_inprog = 0; in ibd_state_init()
2862 mutex_init(&state->id_scq_poll_lock, NULL, MUTEX_DRIVER, NULL); in ibd_state_init()
2863 mutex_init(&state->id_rcq_poll_lock, NULL, MUTEX_DRIVER, NULL); in ibd_state_init()
2864 state->id_dip = dip; in ibd_state_init()
2866 mutex_init(&state->id_sched_lock, NULL, MUTEX_DRIVER, NULL); in ibd_state_init()
2868 mutex_init(&state->id_tx_list.dl_mutex, NULL, MUTEX_DRIVER, NULL); in ibd_state_init()
2869 mutex_init(&state->id_tx_rel_list.dl_mutex, NULL, MUTEX_DRIVER, NULL); in ibd_state_init()
2870 mutex_init(&state->id_txpost_lock, NULL, MUTEX_DRIVER, NULL); in ibd_state_init()
2871 state->id_tx_busy = 0; in ibd_state_init()
2872 mutex_init(&state->id_lso_lock, NULL, MUTEX_DRIVER, NULL); in ibd_state_init()
2874 state->id_rx_list.dl_bufs_outstanding = 0; in ibd_state_init()
2875 state->id_rx_list.dl_cnt = 0; in ibd_state_init()
2876 mutex_init(&state->id_rx_list.dl_mutex, NULL, MUTEX_DRIVER, NULL); in ibd_state_init()
2877 mutex_init(&state->id_rx_free_list.dl_mutex, NULL, MUTEX_DRIVER, NULL); in ibd_state_init()
2879 state->id_pkey, state->id_plinkid); in ibd_state_init()
2880 state->id_req_kmc = kmem_cache_create(buf, sizeof (ibd_req_t), in ibd_state_init()
2884 mutex_init(&state->rc_rx_lock, NULL, MUTEX_DRIVER, NULL); in ibd_state_init()
2885 mutex_init(&state->rc_tx_large_bufs_lock, NULL, MUTEX_DRIVER, NULL); in ibd_state_init()
2886 mutex_init(&state->rc_srq_rwqe_list.dl_mutex, NULL, MUTEX_DRIVER, NULL); in ibd_state_init()
2887 mutex_init(&state->rc_srq_free_list.dl_mutex, NULL, MUTEX_DRIVER, NULL); in ibd_state_init()
2888 mutex_init(&state->rc_pass_chan_list.chan_list_mutex, NULL, in ibd_state_init()
2890 mutex_init(&state->rc_timeout_lock, NULL, MUTEX_DRIVER, NULL); in ibd_state_init()
2897 state->id_enable_rc = IBD_DEF_LINK_MODE; in ibd_state_init()
2898 state->rc_mtu = IBD_DEF_RC_MAX_MTU; in ibd_state_init()
2899 state->id_mtu = IBD_DEF_MAX_MTU; in ibd_state_init()
2902 state->id_lso_policy = IBD_DEF_LSO_POLICY; in ibd_state_init()
2903 state->id_num_lso_bufs = IBD_DEF_NUM_LSO_BUFS; in ibd_state_init()
2904 state->id_num_ah = IBD_DEF_NUM_AH; in ibd_state_init()
2905 state->id_hash_size = IBD_DEF_HASH_SIZE; in ibd_state_init()
2906 state->id_create_broadcast_group = IBD_DEF_CREATE_BCAST_GROUP; in ibd_state_init()
2907 state->id_allow_coalesce_comp_tuning = IBD_DEF_COALESCE_COMPLETIONS; in ibd_state_init()
2908 state->id_ud_rx_comp_count = IBD_DEF_UD_RX_COMP_COUNT; in ibd_state_init()
2909 state->id_ud_rx_comp_usec = IBD_DEF_UD_RX_COMP_USEC; in ibd_state_init()
2910 state->id_ud_tx_comp_count = IBD_DEF_UD_TX_COMP_COUNT; in ibd_state_init()
2911 state->id_ud_tx_comp_usec = IBD_DEF_UD_TX_COMP_USEC; in ibd_state_init()
2912 state->id_rc_rx_comp_count = IBD_DEF_RC_RX_COMP_COUNT; in ibd_state_init()
2913 state->id_rc_rx_comp_usec = IBD_DEF_RC_RX_COMP_USEC; in ibd_state_init()
2914 state->id_rc_tx_comp_count = IBD_DEF_RC_TX_COMP_COUNT; in ibd_state_init()
2915 state->id_rc_tx_comp_usec = IBD_DEF_RC_TX_COMP_USEC; in ibd_state_init()
2916 state->id_ud_tx_copy_thresh = IBD_DEF_UD_TX_COPY_THRESH; in ibd_state_init()
2917 state->id_rc_rx_copy_thresh = IBD_DEF_RC_RX_COPY_THRESH; in ibd_state_init()
2918 state->id_rc_tx_copy_thresh = IBD_DEF_RC_TX_COPY_THRESH; in ibd_state_init()
2919 state->id_ud_num_rwqe = IBD_DEF_UD_NUM_RWQE; in ibd_state_init()
2920 state->id_ud_num_swqe = IBD_DEF_UD_NUM_SWQE; in ibd_state_init()
2921 state->id_rc_num_rwqe = IBD_DEF_RC_NUM_RWQE; in ibd_state_init()
2922 state->id_rc_num_swqe = IBD_DEF_RC_NUM_SWQE; in ibd_state_init()
2923 state->rc_enable_srq = IBD_DEF_RC_ENABLE_SRQ; in ibd_state_init()
2924 state->id_rc_num_srq = IBD_DEF_RC_NUM_SRQ; in ibd_state_init()
2925 state->id_rc_rx_rwqe_thresh = IBD_DEF_RC_RX_RWQE_THRESH; in ibd_state_init()
2934 ibd_state_fini(ibd_state_t *state) in ibd_state_fini() argument
2936 kmem_cache_destroy(state->id_req_kmc); in ibd_state_fini()
2938 mutex_destroy(&state->id_rx_list.dl_mutex); in ibd_state_fini()
2939 mutex_destroy(&state->id_rx_free_list.dl_mutex); in ibd_state_fini()
2941 mutex_destroy(&state->id_txpost_lock); in ibd_state_fini()
2942 mutex_destroy(&state->id_tx_list.dl_mutex); in ibd_state_fini()
2943 mutex_destroy(&state->id_tx_rel_list.dl_mutex); in ibd_state_fini()
2944 mutex_destroy(&state->id_lso_lock); in ibd_state_fini()
2946 mutex_destroy(&state->id_sched_lock); in ibd_state_fini()
2947 mutex_destroy(&state->id_scq_poll_lock); in ibd_state_fini()
2948 mutex_destroy(&state->id_rcq_poll_lock); in ibd_state_fini()
2950 cv_destroy(&state->id_trap_cv); in ibd_state_fini()
2951 mutex_destroy(&state->id_trap_lock); in ibd_state_fini()
2952 mutex_destroy(&state->id_link_mutex); in ibd_state_fini()
2955 mutex_destroy(&state->rc_timeout_lock); in ibd_state_fini()
2956 mutex_destroy(&state->rc_srq_free_list.dl_mutex); in ibd_state_fini()
2957 mutex_destroy(&state->rc_srq_rwqe_list.dl_mutex); in ibd_state_fini()
2958 mutex_destroy(&state->rc_pass_chan_list.chan_list_mutex); in ibd_state_fini()
2959 mutex_destroy(&state->rc_tx_large_bufs_lock); in ibd_state_fini()
2960 mutex_destroy(&state->rc_rx_lock); in ibd_state_fini()
2967 ibd_get_portspeed(ibd_state_t *state) in ibd_get_portspeed() argument
2987 path_attr.pa_dgids = &state->id_sgid; in ibd_get_portspeed()
2989 path_attr.pa_sgid = state->id_sgid; in ibd_get_portspeed()
2991 if (ibt_get_paths(state->id_ibt_hdl, IBT_PATH_NO_FLAGS, in ibd_get_portspeed()
3064 ibd_iba_join(ibd_state_t *state, ib_gid_t mgid, ibd_mce_t *mce) in ibd_iba_join() argument
3069 mcg_attr.mc_qkey = state->id_mcinfo->mc_qkey; in ibd_iba_join()
3072 mcg_attr.mc_scope = state->id_scope; in ibd_iba_join()
3073 mcg_attr.mc_pkey = state->id_pkey; in ibd_iba_join()
3074 mcg_attr.mc_flow = state->id_mcinfo->mc_adds_vect.av_flow; in ibd_iba_join()
3075 mcg_attr.mc_sl = state->id_mcinfo->mc_adds_vect.av_srvl; in ibd_iba_join()
3076 mcg_attr.mc_tclass = state->id_mcinfo->mc_adds_vect.av_tclass; in ibd_iba_join()
3077 return (ibt_join_mcg(state->id_sgid, &mcg_attr, &mce->mc_info, in ibd_iba_join()
3091 ibd_join_group(ibd_state_t *state, ib_gid_t mgid, uint8_t jstate) in ibd_join_group() argument
3116 IBD_MCACHE_FIND_FULL(state, mgid)) != NULL)) { in ibd_join_group()
3133 if ((ibt_status = ibd_iba_join(state, mgid, mce)) != IBT_SUCCESS) { in ibd_join_group()
3145 tmce = IBD_MCACHE_FIND_FULL(state, mgid); in ibd_join_group()
3149 if (IBD_MCACHE_FIND_NON(state, mgid) != NULL) in ibd_join_group()
3160 if ((ibt_status = ibt_attach_mcg(state->id_chnl_hdl, in ibd_join_group()
3169 (void) ibt_leave_mcg(state->id_sgid, mgid, in ibd_join_group()
3170 state->id_sgid, jstate); in ibd_join_group()
3180 IBD_MCACHE_INSERT_NON(state, mce); in ibd_join_group()
3200 (void) ibt_leave_mcg(state->id_sgid, mgid, in ibd_join_group()
3201 state->id_sgid, IB_MC_JSTATE_SEND_ONLY_NON); in ibd_join_group()
3208 mutex_enter(&state->id_mc_mutex); in ibd_join_group()
3209 IBD_MCACHE_INSERT_FULL(state, mce); in ibd_join_group()
3210 mutex_exit(&state->id_mc_mutex); in ibd_join_group()
3223 ibd_reacquire_group(ibd_state_t *state, ibd_mce_t *mce) in ibd_reacquire_group() argument
3241 (void) ibt_leave_mcg(state->id_sgid, mgid, state->id_sgid, in ibd_reacquire_group()
3243 if (ibd_iba_join(state, mgid, mce) != IBT_SUCCESS) in ibd_reacquire_group()
3244 ibd_print_warn(state, "Failure on port up to rejoin " in ibd_reacquire_group()
3260 ibd_async_reap_group(ibd_state_t *state, ibd_mce_t *mce, ib_gid_t mgid, in ibd_async_reap_group() argument
3279 mutex_enter(&state->id_mc_mutex); in ibd_async_reap_group()
3280 IBD_MCACHE_PULLOUT_FULL(state, mce); in ibd_async_reap_group()
3281 mutex_exit(&state->id_mc_mutex); in ibd_async_reap_group()
3282 if (IBD_MCACHE_FIND_NON(state, mgid) != NULL) in ibd_async_reap_group()
3285 IBD_MCACHE_PULLOUT_NON(state, mce); in ibd_async_reap_group()
3286 tmce = IBD_MCACHE_FIND_FULL(state, mgid); in ibd_async_reap_group()
3290 mutex_enter(&state->id_mc_mutex); in ibd_async_reap_group()
3291 IBD_MCACHE_PULLOUT_FULL(state, mce); in ibd_async_reap_group()
3292 mutex_exit(&state->id_mc_mutex); in ibd_async_reap_group()
3307 (void) ibt_detach_mcg(state->id_chnl_hdl, &mce->mc_info); in ibd_async_reap_group()
3310 (void) ibt_leave_mcg(state->id_sgid, mgid, state->id_sgid, jstate); in ibd_async_reap_group()
3322 ibd_leave_group(ibd_state_t *state, ib_gid_t mgid, uint8_t jstate) in ibd_leave_group() argument
3333 mce = IBD_MCACHE_FIND_NON(state, mgid); in ibd_leave_group()
3342 mce = IBD_MCACHE_FIND_FULL(state, mgid); in ibd_leave_group()
3382 mutex_enter(&state->id_ac_mutex); in ibd_leave_group()
3384 recycled = ibd_acache_recycle(state, &mcmac, (jstate == in ibd_leave_group()
3386 mutex_exit(&state->id_ac_mutex); in ibd_leave_group()
3392 ibd_async_reap_group(state, mce, mgid, jstate); in ibd_leave_group()
3402 ibd_find_bgroup(ibd_state_t *state) in ibd_find_bgroup() argument
3414 state->id_bgroup_created = B_FALSE; in ibd_find_bgroup()
3415 state->id_bgroup_present = B_FALSE; in ibd_find_bgroup()
3419 mcg_attr.mc_pkey = state->id_pkey; in ibd_find_bgroup()
3420 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_mgid)) in ibd_find_bgroup()
3421 state->id_mgid.gid_guid = IB_MGID_IPV4_LOWGRP_MASK; in ibd_find_bgroup()
3422 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_mgid)) in ibd_find_bgroup()
3425 state->id_scope = mcg_attr.mc_scope = scopes[i]; in ibd_find_bgroup()
3430 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_mgid)) in ibd_find_bgroup()
3431 state->id_mgid.gid_prefix = in ibd_find_bgroup()
3433 ((uint64_t)state->id_scope << 48) | in ibd_find_bgroup()
3434 ((uint32_t)(state->id_pkey << 16))); in ibd_find_bgroup()
3435 mcg_attr.mc_mgid = state->id_mgid; in ibd_find_bgroup()
3436 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_mgid)) in ibd_find_bgroup()
3437 if (ibt_query_mcg(state->id_sgid, &mcg_attr, 1, in ibd_find_bgroup()
3438 &state->id_mcinfo, &numg) == IBT_SUCCESS) { in ibd_find_bgroup()
3445 if (state->id_create_broadcast_group) { in ibd_find_bgroup()
3451 if (state->id_bgroup_created) { in ibd_find_bgroup()
3452 ibd_print_warn(state, "IPoIB broadcast group " in ibd_find_bgroup()
3464 mcg_attr.mc_pkey = state->id_pkey; in ibd_find_bgroup()
3468 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_mgid)) in ibd_find_bgroup()
3469 state->id_mgid.gid_prefix = in ibd_find_bgroup()
3472 ((uint32_t)(state->id_pkey << 16))); in ibd_find_bgroup()
3473 mcg_attr.mc_mgid = state->id_mgid; in ibd_find_bgroup()
3474 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_mgid)) in ibd_find_bgroup()
3476 if ((ret = ibt_join_mcg(state->id_sgid, &mcg_attr, in ibd_find_bgroup()
3478 ibd_print_warn(state, "IPoIB broadcast group " in ibd_find_bgroup()
3480 state->id_bgroup_created = B_FALSE; in ibd_find_bgroup()
3483 state->id_bgroup_created = B_TRUE; in ibd_find_bgroup()
3486 ibd_print_warn(state, "IPoIB broadcast group absent"); in ibd_find_bgroup()
3494 mcgmtu = (128 << state->id_mcinfo->mc_mtu); in ibd_find_bgroup()
3495 if (state->id_mtu < mcgmtu) { in ibd_find_bgroup()
3496 ibd_print_warn(state, "IPoIB broadcast group MTU %d " in ibd_find_bgroup()
3498 state->id_mtu); in ibd_find_bgroup()
3499 ibt_free_mcg_info(state->id_mcinfo, 1); in ibd_find_bgroup()
3502 state->id_mtu = mcgmtu; in ibd_find_bgroup()
3503 state->id_bgroup_present = B_TRUE; in ibd_find_bgroup()
3508 if (state->id_bgroup_created) { in ibd_find_bgroup()
3509 (void) ibt_leave_mcg(state->id_sgid, in ibd_find_bgroup()
3510 mcg_info.mc_adds_vect.av_dgid, state->id_sgid, in ibd_find_bgroup()
3518 ibd_alloc_tx_copybufs(ibd_state_t *state) in ibd_alloc_tx_copybufs() argument
3525 state->id_tx_buf_sz = state->id_mtu; in ibd_alloc_tx_copybufs()
3526 if (state->id_lso_policy && state->id_lso_capable && in ibd_alloc_tx_copybufs()
3527 (state->id_ud_tx_copy_thresh > state->id_mtu)) { in ibd_alloc_tx_copybufs()
3528 state->id_tx_buf_sz = state->id_ud_tx_copy_thresh; in ibd_alloc_tx_copybufs()
3531 state->id_tx_bufs = kmem_zalloc(state->id_ud_num_swqe * in ibd_alloc_tx_copybufs()
3532 state->id_tx_buf_sz, KM_SLEEP); in ibd_alloc_tx_copybufs()
3534 state->id_tx_wqes = kmem_zalloc(state->id_ud_num_swqe * in ibd_alloc_tx_copybufs()
3540 mem_attr.mr_vaddr = (uint64_t)(uintptr_t)state->id_tx_bufs; in ibd_alloc_tx_copybufs()
3541 mem_attr.mr_len = state->id_ud_num_swqe * state->id_tx_buf_sz; in ibd_alloc_tx_copybufs()
3544 if (ibt_register_mr(state->id_hca_hdl, state->id_pd_hdl, &mem_attr, in ibd_alloc_tx_copybufs()
3545 &state->id_tx_mr_hdl, &state->id_tx_mr_desc) != IBT_SUCCESS) { in ibd_alloc_tx_copybufs()
3547 kmem_free(state->id_tx_wqes, in ibd_alloc_tx_copybufs()
3548 state->id_ud_num_swqe * sizeof (ibd_swqe_t)); in ibd_alloc_tx_copybufs()
3549 kmem_free(state->id_tx_bufs, in ibd_alloc_tx_copybufs()
3550 state->id_ud_num_swqe * state->id_tx_buf_sz); in ibd_alloc_tx_copybufs()
3551 state->id_tx_bufs = NULL; in ibd_alloc_tx_copybufs()
3559 ibd_alloc_tx_lsobufs(ibd_state_t *state) in ibd_alloc_tx_lsobufs() argument
3579 memsz = state->id_num_lso_bufs * IBD_LSO_BUFSZ; in ibd_alloc_tx_lsobufs()
3586 if (ibt_register_mr(state->id_hca_hdl, state->id_pd_hdl, in ibd_alloc_tx_lsobufs()
3594 mutex_enter(&state->id_lso_lock); in ibd_alloc_tx_lsobufs()
3602 buflist = kmem_zalloc(state->id_num_lso_bufs * sizeof (ibd_lsobuf_t), in ibd_alloc_tx_lsobufs()
3610 for (i = 0; i < state->id_num_lso_bufs; i++) { in ibd_alloc_tx_lsobufs()
3628 bktp->bkt_nelem = state->id_num_lso_bufs; in ibd_alloc_tx_lsobufs()
3631 state->id_lso = bktp; in ibd_alloc_tx_lsobufs()
3632 mutex_exit(&state->id_lso_lock); in ibd_alloc_tx_lsobufs()
3641 ibd_init_txlist(ibd_state_t *state) in ibd_init_txlist() argument
3649 if (ibd_alloc_tx_copybufs(state) != DDI_SUCCESS) in ibd_init_txlist()
3652 if (state->id_lso_policy && state->id_lso_capable) { in ibd_init_txlist()
3653 if (ibd_alloc_tx_lsobufs(state) != DDI_SUCCESS) in ibd_init_txlist()
3654 state->id_lso_capable = B_FALSE; in ibd_init_txlist()
3657 mutex_enter(&state->id_tx_list.dl_mutex); in ibd_init_txlist()
3658 state->id_tx_list.dl_head = NULL; in ibd_init_txlist()
3659 state->id_tx_list.dl_pending_sends = B_FALSE; in ibd_init_txlist()
3660 state->id_tx_list.dl_cnt = 0; in ibd_init_txlist()
3661 mutex_exit(&state->id_tx_list.dl_mutex); in ibd_init_txlist()
3662 mutex_enter(&state->id_tx_rel_list.dl_mutex); in ibd_init_txlist()
3663 state->id_tx_rel_list.dl_head = NULL; in ibd_init_txlist()
3664 state->id_tx_rel_list.dl_pending_sends = B_FALSE; in ibd_init_txlist()
3665 state->id_tx_rel_list.dl_cnt = 0; in ibd_init_txlist()
3666 mutex_exit(&state->id_tx_rel_list.dl_mutex); in ibd_init_txlist()
3671 lkey = state->id_tx_mr_desc.md_lkey; in ibd_init_txlist()
3672 bufaddr = state->id_tx_bufs; in ibd_init_txlist()
3673 len = state->id_tx_buf_sz; in ibd_init_txlist()
3674 swqe = state->id_tx_wqes; in ibd_init_txlist()
3675 mutex_enter(&state->id_tx_list.dl_mutex); in ibd_init_txlist()
3676 for (i = 0; i < state->id_ud_num_swqe; i++, swqe++, bufaddr += len) { in ibd_init_txlist()
3695 state->id_tx_list.dl_cnt++; in ibd_init_txlist()
3696 swqe->swqe_next = state->id_tx_list.dl_head; in ibd_init_txlist()
3697 state->id_tx_list.dl_head = SWQE_TO_WQE(swqe); in ibd_init_txlist()
3699 mutex_exit(&state->id_tx_list.dl_mutex); in ibd_init_txlist()
3705 ibd_acquire_lsobufs(ibd_state_t *state, uint_t req_sz, ibt_wr_ds_t *sgl_p, in ibd_acquire_lsobufs() argument
3727 mutex_enter(&state->id_lso_lock); in ibd_acquire_lsobufs()
3732 ASSERT(state->id_lso != NULL); in ibd_acquire_lsobufs()
3733 bktp = state->id_lso; in ibd_acquire_lsobufs()
3735 mutex_exit(&state->id_lso_lock); in ibd_acquire_lsobufs()
3775 mutex_exit(&state->id_lso_lock); in ibd_acquire_lsobufs()
3783 ibd_release_lsobufs(ibd_state_t *state, ibt_wr_ds_t *sgl_p, uint32_t nds) in ibd_release_lsobufs() argument
3791 mutex_enter(&state->id_lso_lock); in ibd_release_lsobufs()
3793 bktp = state->id_lso; in ibd_release_lsobufs()
3819 mutex_exit(&state->id_lso_lock); in ibd_release_lsobufs()
3823 ibd_free_tx_copybufs(ibd_state_t *state) in ibd_free_tx_copybufs() argument
3828 if (ibt_deregister_mr(state->id_hca_hdl, in ibd_free_tx_copybufs()
3829 state->id_tx_mr_hdl) != IBT_SUCCESS) { in ibd_free_tx_copybufs()
3832 state->id_tx_mr_hdl = NULL; in ibd_free_tx_copybufs()
3837 kmem_free(state->id_tx_wqes, state->id_ud_num_swqe * in ibd_free_tx_copybufs()
3839 kmem_free(state->id_tx_bufs, state->id_ud_num_swqe * in ibd_free_tx_copybufs()
3840 state->id_tx_buf_sz); in ibd_free_tx_copybufs()
3841 state->id_tx_wqes = NULL; in ibd_free_tx_copybufs()
3842 state->id_tx_bufs = NULL; in ibd_free_tx_copybufs()
3846 ibd_free_tx_lsobufs(ibd_state_t *state) in ibd_free_tx_lsobufs() argument
3850 mutex_enter(&state->id_lso_lock); in ibd_free_tx_lsobufs()
3852 if ((bktp = state->id_lso) == NULL) { in ibd_free_tx_lsobufs()
3853 mutex_exit(&state->id_lso_lock); in ibd_free_tx_lsobufs()
3867 if (ibt_deregister_mr(state->id_hca_hdl, in ibd_free_tx_lsobufs()
3879 state->id_lso = NULL; in ibd_free_tx_lsobufs()
3881 mutex_exit(&state->id_lso_lock); in ibd_free_tx_lsobufs()
3888 ibd_fini_txlist(ibd_state_t *state) in ibd_fini_txlist() argument
3893 mutex_enter(&state->id_tx_list.dl_mutex); in ibd_fini_txlist()
3894 mutex_enter(&state->id_tx_rel_list.dl_mutex); in ibd_fini_txlist()
3895 state->id_tx_list.dl_head = NULL; in ibd_fini_txlist()
3896 state->id_tx_list.dl_pending_sends = B_FALSE; in ibd_fini_txlist()
3897 state->id_tx_list.dl_cnt = 0; in ibd_fini_txlist()
3898 state->id_tx_rel_list.dl_head = NULL; in ibd_fini_txlist()
3899 state->id_tx_rel_list.dl_pending_sends = B_FALSE; in ibd_fini_txlist()
3900 state->id_tx_rel_list.dl_cnt = 0; in ibd_fini_txlist()
3901 mutex_exit(&state->id_tx_rel_list.dl_mutex); in ibd_fini_txlist()
3902 mutex_exit(&state->id_tx_list.dl_mutex); in ibd_fini_txlist()
3904 ibd_free_tx_lsobufs(state); in ibd_fini_txlist()
3905 ibd_free_tx_copybufs(state); in ibd_fini_txlist()
3912 ibd_post_recv_list(ibd_state_t *state, ibd_rwqe_t *rwqe) in ibd_post_recv_list() argument
3939 atomic_add_32(&state->id_rx_list.dl_cnt, i); in ibd_post_recv_list()
3942 ibt_status = ibt_post_recv(state->id_chnl_hdl, wrs, i, in ibd_post_recv_list()
3946 ibd_print_warn(state, "ibd_post_recv: FATAL: " in ibd_post_recv_list()
3950 atomic_add_32(&state->id_rx_list.dl_cnt, in ibd_post_recv_list()
3960 ibd_post_recv_intr(ibd_state_t *state) in ibd_post_recv_intr() argument
3966 state->id_rx_post_queue_index = in ibd_post_recv_intr()
3967 (state->id_rx_post_queue_index + 1) & in ibd_post_recv_intr()
3968 (state->id_rx_nqueues - 1); in ibd_post_recv_intr()
3970 rxp = state->id_rx_queues + state->id_rx_post_queue_index; in ibd_post_recv_intr()
3976 ibd_post_recv_list(state, list); in ibd_post_recv_intr()
3981 (((uintptr_t)(rwqe) >> 8) & (state->id_rx_nqueues - 1))
3995 ibd_post_recv(ibd_state_t *state, ibd_rwqe_t *rwqe) in ibd_post_recv() argument
3999 rxp = state->id_rx_queues + RX_QUEUE_HASH(rwqe); in ibd_post_recv()
4003 rxp = state->id_rx_queues + RX_QUEUE_HASH(rwqe + 16); in ibd_post_recv()
4008 uint_t active = atomic_inc_32_nv(&state->id_rx_post_active); in ibd_post_recv()
4011 if ((active & (state->id_rx_nqueues - 1)) == 0) { in ibd_post_recv()
4015 ibd_post_recv_list(state, rwqe); in ibd_post_recv()
4024 ibd_alloc_rx_copybufs(ibd_state_t *state) in ibd_alloc_rx_copybufs() argument
4032 state->id_rx_buf_sz = state->id_mtu + IPOIB_GRH_SIZE; in ibd_alloc_rx_copybufs()
4034 state->id_rx_bufs = kmem_zalloc(state->id_ud_num_rwqe * in ibd_alloc_rx_copybufs()
4035 state->id_rx_buf_sz, KM_SLEEP); in ibd_alloc_rx_copybufs()
4037 state->id_rx_wqes = kmem_zalloc(state->id_ud_num_rwqe * in ibd_alloc_rx_copybufs()
4040 state->id_rx_nqueues = 1 << IBD_LOG_RX_POST; in ibd_alloc_rx_copybufs()
4041 state->id_rx_queues = kmem_zalloc(state->id_rx_nqueues * in ibd_alloc_rx_copybufs()
4043 for (i = 0; i < state->id_rx_nqueues; i++) { in ibd_alloc_rx_copybufs()
4044 ibd_rx_queue_t *rxp = state->id_rx_queues + i; in ibd_alloc_rx_copybufs()
4051 mem_attr.mr_vaddr = (uint64_t)(uintptr_t)state->id_rx_bufs; in ibd_alloc_rx_copybufs()
4052 mem_attr.mr_len = state->id_ud_num_rwqe * state->id_rx_buf_sz; in ibd_alloc_rx_copybufs()
4055 if (ibt_register_mr(state->id_hca_hdl, state->id_pd_hdl, &mem_attr, in ibd_alloc_rx_copybufs()
4056 &state->id_rx_mr_hdl, &state->id_rx_mr_desc) != IBT_SUCCESS) { in ibd_alloc_rx_copybufs()
4058 kmem_free(state->id_rx_wqes, in ibd_alloc_rx_copybufs()
4059 state->id_ud_num_rwqe * sizeof (ibd_rwqe_t)); in ibd_alloc_rx_copybufs()
4060 kmem_free(state->id_rx_bufs, in ibd_alloc_rx_copybufs()
4061 state->id_ud_num_rwqe * state->id_rx_buf_sz); in ibd_alloc_rx_copybufs()
4062 state->id_rx_bufs = NULL; in ibd_alloc_rx_copybufs()
4063 state->id_rx_wqes = NULL; in ibd_alloc_rx_copybufs()
4074 ibd_init_rxlist(ibd_state_t *state) in ibd_init_rxlist() argument
4083 mutex_enter(&state->id_rx_free_list.dl_mutex); in ibd_init_rxlist()
4084 if (state->id_rx_free_list.dl_head != NULL) { in ibd_init_rxlist()
4086 len = state->id_rx_buf_sz; in ibd_init_rxlist()
4087 list = state->id_rx_free_list.dl_head; in ibd_init_rxlist()
4088 state->id_rx_free_list.dl_head = NULL; in ibd_init_rxlist()
4089 state->id_rx_free_list.dl_cnt = 0; in ibd_init_rxlist()
4090 mutex_exit(&state->id_rx_free_list.dl_mutex); in ibd_init_rxlist()
4097 if (atomic_dec_32_nv(&state->id_running) != 0) { in ibd_init_rxlist()
4107 atomic_inc_32(&state-> in ibd_init_rxlist()
4112 ibd_free_rwqe(state, rwqe); in ibd_init_rxlist()
4114 atomic_inc_32(&state->id_running); in ibd_init_rxlist()
4118 ibd_post_recv_list(state, WQE_TO_RWQE(list)); in ibd_init_rxlist()
4121 mutex_exit(&state->id_rx_free_list.dl_mutex); in ibd_init_rxlist()
4123 if (ibd_alloc_rx_copybufs(state) != DDI_SUCCESS) in ibd_init_rxlist()
4129 len = state->id_rx_buf_sz; in ibd_init_rxlist()
4130 lkey = state->id_rx_mr_desc.md_lkey; in ibd_init_rxlist()
4131 rwqe = state->id_rx_wqes; in ibd_init_rxlist()
4132 bufaddr = state->id_rx_bufs; in ibd_init_rxlist()
4134 for (i = 0; i < state->id_ud_num_rwqe; i++, rwqe++, bufaddr += len) { in ibd_init_rxlist()
4135 rwqe->w_state = state; in ibd_init_rxlist()
4145 if (atomic_dec_32_nv(&state->id_running) != 0) { in ibd_init_rxlist()
4156 atomic_inc_32(&state->id_running); in ibd_init_rxlist()
4159 mutex_enter(&state->id_rx_free_list.dl_mutex); in ibd_init_rxlist()
4160 state->id_rx_free_list.dl_head = NULL; in ibd_init_rxlist()
4161 state->id_rx_free_list.dl_cnt = 0; in ibd_init_rxlist()
4162 mutex_exit(&state->id_rx_free_list.dl_mutex); in ibd_init_rxlist()
4164 ibd_fini_rxlist(state); in ibd_init_rxlist()
4179 ibd_post_recv_list(state, WQE_TO_RWQE(list)); in ibd_init_rxlist()
4185 ibd_free_rx_copybufs(ibd_state_t *state) in ibd_free_rx_copybufs() argument
4192 if (ibt_deregister_mr(state->id_hca_hdl, in ibd_free_rx_copybufs()
4193 state->id_rx_mr_hdl) != IBT_SUCCESS) { in ibd_free_rx_copybufs()
4196 state->id_rx_mr_hdl = NULL; in ibd_free_rx_copybufs()
4201 for (i = 0; i < state->id_rx_nqueues; i++) { in ibd_free_rx_copybufs()
4202 ibd_rx_queue_t *rxp = state->id_rx_queues + i; in ibd_free_rx_copybufs()
4205 kmem_free(state->id_rx_queues, state->id_rx_nqueues * in ibd_free_rx_copybufs()
4207 kmem_free(state->id_rx_wqes, state->id_ud_num_rwqe * in ibd_free_rx_copybufs()
4209 kmem_free(state->id_rx_bufs, state->id_ud_num_rwqe * in ibd_free_rx_copybufs()
4210 state->id_rx_buf_sz); in ibd_free_rx_copybufs()
4211 state->id_rx_queues = NULL; in ibd_free_rx_copybufs()
4212 state->id_rx_wqes = NULL; in ibd_free_rx_copybufs()
4213 state->id_rx_bufs = NULL; in ibd_free_rx_copybufs()
4217 ibd_free_rx_rsrcs(ibd_state_t *state) in ibd_free_rx_rsrcs() argument
4219 mutex_enter(&state->id_rx_free_list.dl_mutex); in ibd_free_rx_rsrcs()
4220 if (state->id_rx_free_list.dl_head == NULL) { in ibd_free_rx_rsrcs()
4222 mutex_exit(&state->id_rx_free_list.dl_mutex); in ibd_free_rx_rsrcs()
4225 ASSERT(state->id_rx_free_list.dl_cnt == state->id_ud_num_rwqe); in ibd_free_rx_rsrcs()
4226 ibd_free_rx_copybufs(state); in ibd_free_rx_rsrcs()
4227 state->id_rx_free_list.dl_cnt = 0; in ibd_free_rx_rsrcs()
4228 state->id_rx_free_list.dl_head = NULL; in ibd_free_rx_rsrcs()
4229 mutex_exit(&state->id_rx_free_list.dl_mutex); in ibd_free_rx_rsrcs()
4236 ibd_fini_rxlist(ibd_state_t *state) in ibd_fini_rxlist() argument
4242 for (i = 0; i < state->id_rx_nqueues; i++) { in ibd_fini_rxlist()
4243 ibd_rx_queue_t *rxp = state->id_rx_queues + i; in ibd_fini_rxlist()
4255 if (atomic_add_32_nv(&state->id_rx_list.dl_bufs_outstanding, 0) == 0) in ibd_fini_rxlist()
4256 ibd_free_rx_rsrcs(state); in ibd_fini_rxlist()
4264 ibd_free_rwqe(ibd_state_t *state, ibd_rwqe_t *rwqe) in ibd_free_rwqe() argument
4275 mutex_enter(&state->id_rx_free_list.dl_mutex); in ibd_free_rwqe()
4276 state->id_rx_free_list.dl_cnt++; in ibd_free_rwqe()
4277 rwqe->rwqe_next = state->id_rx_free_list.dl_head; in ibd_free_rwqe()
4278 state->id_rx_free_list.dl_head = RWQE_TO_WQE(rwqe); in ibd_free_rwqe()
4279 mutex_exit(&state->id_rx_free_list.dl_mutex); in ibd_free_rwqe()
4290 ibd_state_t *state = (ibd_state_t *)arg; in ibd_rcq_handler() local
4292 atomic_inc_64(&state->id_num_intrs); in ibd_rcq_handler()
4295 mutex_enter(&state->id_rcq_poll_lock); in ibd_rcq_handler()
4296 if (state->id_rcq_poll_busy & IBD_CQ_POLLING) { in ibd_rcq_handler()
4297 state->id_rcq_poll_busy |= IBD_REDO_CQ_POLLING; in ibd_rcq_handler()
4298 mutex_exit(&state->id_rcq_poll_lock); in ibd_rcq_handler()
4301 mutex_exit(&state->id_rcq_poll_lock); in ibd_rcq_handler()
4302 ddi_trigger_softintr(state->id_rx); in ibd_rcq_handler()
4305 (void) ibd_intr((caddr_t)state); in ibd_rcq_handler()
4316 ibd_state_t *state = (ibd_state_t *)arg; in ibd_scq_handler() local
4318 atomic_inc_64(&state->id_num_intrs); in ibd_scq_handler()
4321 mutex_enter(&state->id_scq_poll_lock); in ibd_scq_handler()
4322 if (state->id_scq_poll_busy & IBD_CQ_POLLING) { in ibd_scq_handler()
4323 state->id_scq_poll_busy |= IBD_REDO_CQ_POLLING; in ibd_scq_handler()
4324 mutex_exit(&state->id_scq_poll_lock); in ibd_scq_handler()
4327 mutex_exit(&state->id_scq_poll_lock); in ibd_scq_handler()
4328 ddi_trigger_softintr(state->id_tx); in ibd_scq_handler()
4331 (void) ibd_tx_recycle((caddr_t)state); in ibd_scq_handler()
4345 ibd_state_t *state = (ibd_state_t *)arg; in ibd_snet_notices_handler() local
4356 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_sgid)) in ibd_snet_notices_handler()
4357 if (bcmp(&gid, &state->id_sgid, sizeof (ib_gid_t)) != 0) in ibd_snet_notices_handler()
4359 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_sgid)) in ibd_snet_notices_handler()
4373 ibd_print_warn(state, "IBA multicast support " in ibd_snet_notices_handler()
4383 ibd_print_warn(state, "IBA multicast support " in ibd_snet_notices_handler()
4393 if (((state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) == in ibd_snet_notices_handler()
4407 if (!ibd_async_safe(state)) in ibd_snet_notices_handler()
4410 req = kmem_cache_alloc(state->id_req_kmc, KM_SLEEP); in ibd_snet_notices_handler()
4413 ibd_queue_work_slot(state, req, IBD_ASYNC_TRAP); in ibd_snet_notices_handler()
4419 ibd_async_trap(ibd_state_t *state, ibd_req_t *req) in ibd_async_trap() argument
4435 if (((state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) == in ibd_async_trap()
4436 IBD_DRV_IN_LATE_HCA_INIT) && (state->id_bgroup_present == 0) && in ibd_async_trap()
4444 state->id_pkey)) { in ibd_async_trap()
4445 ibd_async_done(state); in ibd_async_trap()
4448 ibd_set_mac_progress(state, IBD_DRV_RESTART_IN_PROGRESS); in ibd_async_trap()
4455 if (((state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) == in ibd_async_trap()
4457 ((ret = ibd_start(state)) != 0)) { in ibd_async_trap()
4461 ibd_clr_mac_progress(state, IBD_DRV_RESTART_IN_PROGRESS); in ibd_async_trap()
4462 ibd_async_done(state); in ibd_async_trap()
4470 ibd_leave_group(state, mgid, IB_MC_JSTATE_SEND_ONLY_NON); in ibd_async_trap()
4472 if (state->id_prom_op == IBD_OP_COMPLETED) { in ibd_async_trap()
4473 ibd_leave_group(state, mgid, IB_MC_JSTATE_NON); in ibd_async_trap()
4485 if ((ibd_join_group(state, mgid, IB_MC_JSTATE_NON) == in ibd_async_trap()
4487 ibd_print_warn(state, "IBA promiscuous mode missed " in ibd_async_trap()
4496 ibd_async_done(state); in ibd_async_trap()
4505 ibd_state_t *state = arg; in ibd_m_getcapab() local
4507 if (state->id_type == IBD_PORT_DRIVER) in ibd_m_getcapab()
4517 if (state->id_hwcksum_capab & IBT_HCA_CKSUM_FULL) in ibd_m_getcapab()
4535 if (!state->id_lso_policy || !state->id_lso_capable) in ibd_m_getcapab()
4538 if ((state->id_hwcksum_capab & IBT_HCA_CKSUM_FULL) == 0) in ibd_m_getcapab()
4541 if (state->id_hca_res_lkey_capab == 0) { in ibd_m_getcapab()
4542 ibd_print_warn(state, "no reserved-lkey capability, " in ibd_m_getcapab()
4548 cap_lso->lso_basic_tcp_ipv4.lso_max = state->id_lso_maxlen - 1; in ibd_m_getcapab()
4566 ibd_state_t *state = arg; in ibd_m_setprop() local
4571 if (state->id_type == IBD_PORT_DRIVER) { in ibd_m_setprop()
4577 if (state->id_mac_state & IBD_DRV_STARTED) { in ibd_m_setprop()
4591 if (state->id_enable_rc) { in ibd_m_setprop()
4594 state->id_enable_rc = 1; in ibd_m_setprop()
4596 err = mac_maxsdu_update2(state->id_mh, in ibd_m_setprop()
4597 state->rc_mtu - IPOIB_HDRSIZE, in ibd_m_setprop()
4598 state->id_mtu - IPOIB_HDRSIZE); in ibd_m_setprop()
4600 if (!state->id_enable_rc) { in ibd_m_setprop()
4603 state->id_enable_rc = 0; in ibd_m_setprop()
4604 err = mac_maxsdu_update2(state->id_mh, in ibd_m_setprop()
4605 state->id_mtu - IPOIB_HDRSIZE, in ibd_m_setprop()
4606 state->id_mtu - IPOIB_HDRSIZE); in ibd_m_setprop()
4608 (void) ibd_record_capab(state); in ibd_m_setprop()
4609 mac_capab_update(state->id_mh); in ibd_m_setprop()
4613 err = ibd_set_priv_prop(state, pr_name, in ibd_m_setprop()
4627 ibd_state_t *state = arg; in ibd_m_getprop() local
4634 if (state->id_type == IBD_PORT_DRIVER) { in ibd_m_getprop()
4642 *(uint_t *)pr_val = state->id_enable_rc; in ibd_m_getprop()
4645 err = ibd_get_priv_prop(state, pr_name, pr_valsize, in ibd_m_getprop()
4659 ibd_state_t *state = arg; in ibd_m_propinfo() local
4668 if (state->id_type == IBD_PORT_DRIVER) { in ibd_m_propinfo()
4671 } else if (state->id_enable_rc) { in ibd_m_propinfo()
4674 min = max = state->id_mtu - IPOIB_HDRSIZE; in ibd_m_propinfo()
4749 ibd_set_priv_prop(ibd_state_t *state, const char *pr_name, in ibd_set_priv_prop() argument
4763 state->id_allow_coalesce_comp_tuning = (result == 1) ? in ibd_set_priv_prop()
4769 if (state->id_mac_state & IBD_DRV_STARTED) { in ibd_set_priv_prop()
4779 state->id_create_broadcast_group = (result == 1) ? in ibd_set_priv_prop()
4785 if (state->id_mac_state & IBD_DRV_STARTED) { in ibd_set_priv_prop()
4795 state->id_hash_size = (uint32_t)result; in ibd_set_priv_prop()
4800 if (state->id_mac_state & IBD_DRV_STARTED) { in ibd_set_priv_prop()
4810 state->id_lso_policy = (result == 1) ? in ibd_set_priv_prop()
4813 mac_capab_update(state->id_mh); in ibd_set_priv_prop()
4817 if (state->id_mac_state & IBD_DRV_STARTED) { in ibd_set_priv_prop()
4827 state->id_num_ah = (uint32_t)result; in ibd_set_priv_prop()
4832 if (state->id_mac_state & IBD_DRV_STARTED) { in ibd_set_priv_prop()
4835 if (!state->id_lso_policy || !state->id_lso_capable) { in ibd_set_priv_prop()
4846 state->id_num_lso_bufs = (uint32_t)result; in ibd_set_priv_prop()
4851 if (state->id_mac_state & IBD_DRV_STARTED) { in ibd_set_priv_prop()
4861 state->rc_enable_srq = (result == 1) ? in ibd_set_priv_prop()
4864 if (!state->rc_enable_srq) { in ibd_set_priv_prop()
4865 state->id_rc_num_srq = 0; in ibd_set_priv_prop()
4870 if (state->id_mac_state & IBD_DRV_STARTED) { in ibd_set_priv_prop()
4881 state->id_rc_num_rwqe = (uint32_t)result; in ibd_set_priv_prop()
4882 if (state->id_allow_coalesce_comp_tuning && in ibd_set_priv_prop()
4883 state->id_rc_rx_comp_count > state->id_rc_num_rwqe) in ibd_set_priv_prop()
4884 state->id_rc_rx_comp_count = in ibd_set_priv_prop()
4885 state->id_rc_num_rwqe; in ibd_set_priv_prop()
4886 if (state->id_rc_num_srq > state->id_rc_num_rwqe) in ibd_set_priv_prop()
4887 state->id_rc_num_srq = in ibd_set_priv_prop()
4888 state->id_rc_num_rwqe - 1; in ibd_set_priv_prop()
4893 if (state->id_rc_rx_rwqe_thresh > state->id_rc_num_rwqe) in ibd_set_priv_prop()
4894 state->id_rc_rx_rwqe_thresh = in ibd_set_priv_prop()
4895 (state->id_rc_num_rwqe >> 2); in ibd_set_priv_prop()
4901 if (state->id_mac_state & IBD_DRV_STARTED) { in ibd_set_priv_prop()
4907 if (!state->rc_enable_srq) in ibd_set_priv_prop()
4912 result >= state->id_rc_num_rwqe) { in ibd_set_priv_prop()
4915 state->id_rc_num_srq = (uint32_t)result; in ibd_set_priv_prop()
4919 if (state->id_mac_state & IBD_DRV_STARTED) { in ibd_set_priv_prop()
4930 state->id_rc_num_swqe = (uint32_t)result; in ibd_set_priv_prop()
4931 if (state->id_allow_coalesce_comp_tuning && in ibd_set_priv_prop()
4932 state->id_rc_tx_comp_count > state->id_rc_num_swqe) in ibd_set_priv_prop()
4933 state->id_rc_tx_comp_count = in ibd_set_priv_prop()
4934 state->id_rc_num_swqe; in ibd_set_priv_prop()
4939 if (!state->id_allow_coalesce_comp_tuning) { in ibd_set_priv_prop()
4946 if (result < 1 || result > state->id_rc_num_rwqe) { in ibd_set_priv_prop()
4949 state->id_rc_rx_comp_count = (uint32_t)result; in ibd_set_priv_prop()
4954 if (!state->id_allow_coalesce_comp_tuning) { in ibd_set_priv_prop()
4964 state->id_rc_rx_comp_usec = (uint32_t)result; in ibd_set_priv_prop()
4969 if (state->id_mac_state & IBD_DRV_STARTED) { in ibd_set_priv_prop()
4977 result > state->rc_mtu) { in ibd_set_priv_prop()
4980 state->id_rc_rx_copy_thresh = (uint32_t)result; in ibd_set_priv_prop()
4985 if (state->id_mac_state & IBD_DRV_STARTED) { in ibd_set_priv_prop()
4993 result >= state->id_rc_num_rwqe) { in ibd_set_priv_prop()
4996 state->id_rc_rx_rwqe_thresh = (uint32_t)result; in ibd_set_priv_prop()
5001 if (!state->id_allow_coalesce_comp_tuning) { in ibd_set_priv_prop()
5008 if (result < 1 || result > state->id_rc_num_swqe) { in ibd_set_priv_prop()
5011 state->id_rc_tx_comp_count = (uint32_t)result; in ibd_set_priv_prop()
5016 if (!state->id_allow_coalesce_comp_tuning) { in ibd_set_priv_prop()
5026 state->id_rc_tx_comp_usec = (uint32_t)result; in ibd_set_priv_prop()
5031 if (state->id_mac_state & IBD_DRV_STARTED) { in ibd_set_priv_prop()
5039 result > state->rc_mtu) { in ibd_set_priv_prop()
5042 state->id_rc_tx_copy_thresh = (uint32_t)result; in ibd_set_priv_prop()
5047 if (state->id_mac_state & IBD_DRV_STARTED) { in ibd_set_priv_prop()
5058 if (result > state->id_hca_max_chan_sz) { in ibd_set_priv_prop()
5059 state->id_ud_num_rwqe = in ibd_set_priv_prop()
5060 state->id_hca_max_chan_sz; in ibd_set_priv_prop()
5062 state->id_ud_num_rwqe = (uint32_t)result; in ibd_set_priv_prop()
5064 if (state->id_allow_coalesce_comp_tuning && in ibd_set_priv_prop()
5065 state->id_ud_rx_comp_count > state->id_ud_num_rwqe) in ibd_set_priv_prop()
5066 state->id_ud_rx_comp_count = in ibd_set_priv_prop()
5067 state->id_ud_num_rwqe; in ibd_set_priv_prop()
5072 if (state->id_mac_state & IBD_DRV_STARTED) { in ibd_set_priv_prop()
5083 if (result > state->id_hca_max_chan_sz) { in ibd_set_priv_prop()
5084 state->id_ud_num_swqe = in ibd_set_priv_prop()
5085 state->id_hca_max_chan_sz; in ibd_set_priv_prop()
5087 state->id_ud_num_swqe = (uint32_t)result; in ibd_set_priv_prop()
5089 if (state->id_allow_coalesce_comp_tuning && in ibd_set_priv_prop()
5090 state->id_ud_tx_comp_count > state->id_ud_num_swqe) in ibd_set_priv_prop()
5091 state->id_ud_tx_comp_count = in ibd_set_priv_prop()
5092 state->id_ud_num_swqe; in ibd_set_priv_prop()
5097 if (!state->id_allow_coalesce_comp_tuning) { in ibd_set_priv_prop()
5104 if (result < 1 || result > state->id_ud_num_rwqe) { in ibd_set_priv_prop()
5107 state->id_ud_rx_comp_count = (uint32_t)result; in ibd_set_priv_prop()
5112 if (!state->id_allow_coalesce_comp_tuning) { in ibd_set_priv_prop()
5122 state->id_ud_rx_comp_usec = (uint32_t)result; in ibd_set_priv_prop()
5127 if (!state->id_allow_coalesce_comp_tuning) { in ibd_set_priv_prop()
5134 if (result < 1 || result > state->id_ud_num_swqe) { in ibd_set_priv_prop()
5137 state->id_ud_tx_comp_count = (uint32_t)result; in ibd_set_priv_prop()
5142 if (!state->id_allow_coalesce_comp_tuning) { in ibd_set_priv_prop()
5152 state->id_ud_tx_comp_usec = (uint32_t)result; in ibd_set_priv_prop()
5157 if (state->id_mac_state & IBD_DRV_STARTED) { in ibd_set_priv_prop()
5168 state->id_ud_tx_copy_thresh = (uint32_t)result; in ibd_set_priv_prop()
5176 ibd_get_priv_prop(ibd_state_t *state, const char *pr_name, uint_t pr_valsize, in ibd_get_priv_prop() argument
5183 value = state->id_bgroup_present; in ibd_get_priv_prop()
5188 value = state->id_allow_coalesce_comp_tuning; in ibd_get_priv_prop()
5193 value = state->id_create_broadcast_group; in ibd_get_priv_prop()
5198 value = state->id_hash_size; in ibd_get_priv_prop()
5203 value = state->id_lso_policy; in ibd_get_priv_prop()
5208 value = state->id_num_ah; in ibd_get_priv_prop()
5213 value = state->id_num_lso_bufs; in ibd_get_priv_prop()
5218 value = state->rc_enable_srq; in ibd_get_priv_prop()
5223 value = state->id_rc_num_rwqe; in ibd_get_priv_prop()
5228 value = state->id_rc_num_srq; in ibd_get_priv_prop()
5233 value = state->id_rc_num_swqe; in ibd_get_priv_prop()
5238 value = state->id_rc_rx_comp_count; in ibd_get_priv_prop()
5243 value = state->id_rc_rx_comp_usec; in ibd_get_priv_prop()
5248 value = state->id_rc_rx_copy_thresh; in ibd_get_priv_prop()
5253 value = state->id_rc_rx_rwqe_thresh; in ibd_get_priv_prop()
5258 value = state->id_rc_tx_comp_count; in ibd_get_priv_prop()
5263 value = state->id_rc_tx_comp_usec; in ibd_get_priv_prop()
5268 value = state->id_rc_tx_copy_thresh; in ibd_get_priv_prop()
5273 value = state->id_ud_num_rwqe; in ibd_get_priv_prop()
5278 value = state->id_ud_num_swqe; in ibd_get_priv_prop()
5283 value = state->id_ud_rx_comp_count; in ibd_get_priv_prop()
5288 value = state->id_ud_rx_comp_usec; in ibd_get_priv_prop()
5293 value = state->id_ud_tx_comp_count; in ibd_get_priv_prop()
5298 value = state->id_ud_tx_comp_usec; in ibd_get_priv_prop()
5303 value = state->id_ud_tx_copy_thresh; in ibd_get_priv_prop()
5315 ibd_get_port_details(ibd_state_t *state) in ibd_get_port_details() argument
5321 mutex_enter(&state->id_link_mutex); in ibd_get_port_details()
5326 ret = ibt_query_hca_ports(state->id_hca_hdl, state->id_port, in ibd_get_port_details()
5329 mutex_exit(&state->id_link_mutex); in ibd_get_port_details()
5339 if ((ret = ibt_pkey2index(state->id_hca_hdl, state->id_port, in ibd_get_port_details()
5340 state->id_pkey, &state->id_pkix)) != IBT_SUCCESS) { in ibd_get_port_details()
5341 state->id_link_state = LINK_STATE_DOWN; in ibd_get_port_details()
5343 state->id_link_state = LINK_STATE_UP; in ibd_get_port_details()
5345 state->id_mtu = (128 << port_infop->p_mtu); in ibd_get_port_details()
5346 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_sgid)) in ibd_get_port_details()
5347 state->id_sgid = *port_infop->p_sgid_tbl; in ibd_get_port_details()
5348 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_sgid)) in ibd_get_port_details()
5352 state->id_link_speed = ibd_get_portspeed(state); in ibd_get_port_details()
5355 state->id_mtu = 0; in ibd_get_port_details()
5356 state->id_link_state = LINK_STATE_DOWN; in ibd_get_port_details()
5357 state->id_link_speed = 0; in ibd_get_port_details()
5359 mutex_exit(&state->id_link_mutex); in ibd_get_port_details()
5366 ibd_alloc_cqs(ibd_state_t *state) in ibd_alloc_cqs() argument
5375 ret = ibt_query_hca(state->id_hca_hdl, &hca_attrs); in ibd_alloc_cqs()
5392 if (hca_attrs.hca_max_cq_sz >= (state->id_ud_num_rwqe + 1)) { in ibd_alloc_cqs()
5393 cq_attr.cq_size = state->id_ud_num_rwqe + 1; in ibd_alloc_cqs()
5396 num_rwqe_change = state->id_ud_num_rwqe; in ibd_alloc_cqs()
5397 state->id_ud_num_rwqe = cq_attr.cq_size - 1; in ibd_alloc_cqs()
5400 if ((ret = ibt_alloc_cq(state->id_hca_hdl, &cq_attr, in ibd_alloc_cqs()
5401 &state->id_rcq_hdl, &real_size)) != IBT_SUCCESS) { in ibd_alloc_cqs()
5407 if ((ret = ibt_modify_cq(state->id_rcq_hdl, state->id_ud_rx_comp_count, in ibd_alloc_cqs()
5408 state->id_ud_rx_comp_usec, 0)) != IBT_SUCCESS) { in ibd_alloc_cqs()
5414 state->id_rxwcs_size = IBD_MAX_RX_MP_LEN; in ibd_alloc_cqs()
5415 state->id_rxwcs = kmem_alloc(sizeof (ibt_wc_t) * in ibd_alloc_cqs()
5416 state->id_rxwcs_size, KM_SLEEP); in ibd_alloc_cqs()
5421 if (hca_attrs.hca_max_cq_sz >= (state->id_ud_num_swqe + 1)) { in ibd_alloc_cqs()
5422 cq_attr.cq_size = state->id_ud_num_swqe + 1; in ibd_alloc_cqs()
5425 num_swqe_change = state->id_ud_num_swqe; in ibd_alloc_cqs()
5426 state->id_ud_num_swqe = cq_attr.cq_size - 1; in ibd_alloc_cqs()
5429 if ((ret = ibt_alloc_cq(state->id_hca_hdl, &cq_attr, in ibd_alloc_cqs()
5430 &state->id_scq_hdl, &real_size)) != IBT_SUCCESS) { in ibd_alloc_cqs()
5433 kmem_free(state->id_rxwcs, sizeof (ibt_wc_t) * in ibd_alloc_cqs()
5434 state->id_rxwcs_size); in ibd_alloc_cqs()
5435 (void) ibt_free_cq(state->id_rcq_hdl); in ibd_alloc_cqs()
5438 if ((ret = ibt_modify_cq(state->id_scq_hdl, state->id_ud_tx_comp_count, in ibd_alloc_cqs()
5439 state->id_ud_tx_comp_usec, 0)) != IBT_SUCCESS) { in ibd_alloc_cqs()
5444 state->id_txwcs_size = IBD_TX_POLL_THRESH; in ibd_alloc_cqs()
5445 state->id_txwcs = kmem_alloc(sizeof (ibt_wc_t) * in ibd_alloc_cqs()
5446 state->id_txwcs_size, KM_SLEEP); in ibd_alloc_cqs()
5453 ibd_print_warn(state, "Setting #rwqe = %d instead of default " in ibd_alloc_cqs()
5454 "%d", state->id_ud_num_rwqe, num_rwqe_change); in ibd_alloc_cqs()
5457 ibd_print_warn(state, "Setting #swqe = %d instead of default " in ibd_alloc_cqs()
5458 "%d", state->id_ud_num_swqe, num_swqe_change); in ibd_alloc_cqs()
5465 ibd_setup_ud_channel(ibd_state_t *state) in ibd_setup_ud_channel() argument
5472 if (state->id_hca_res_lkey_capab) in ibd_setup_ud_channel()
5474 if (state->id_lso_policy && state->id_lso_capable) in ibd_setup_ud_channel()
5477 ud_alloc_attr.ud_hca_port_num = state->id_port; in ibd_setup_ud_channel()
5478 ud_alloc_attr.ud_sizes.cs_sq_sgl = state->id_max_sqseg; in ibd_setup_ud_channel()
5480 ud_alloc_attr.ud_sizes.cs_sq = state->id_ud_num_swqe; in ibd_setup_ud_channel()
5481 ud_alloc_attr.ud_sizes.cs_rq = state->id_ud_num_rwqe; in ibd_setup_ud_channel()
5482 ud_alloc_attr.ud_qkey = state->id_mcinfo->mc_qkey; in ibd_setup_ud_channel()
5483 ud_alloc_attr.ud_scq = state->id_scq_hdl; in ibd_setup_ud_channel()
5484 ud_alloc_attr.ud_rcq = state->id_rcq_hdl; in ibd_setup_ud_channel()
5485 ud_alloc_attr.ud_pd = state->id_pd_hdl; in ibd_setup_ud_channel()
5486 ud_alloc_attr.ud_pkey_ix = state->id_pkix; in ibd_setup_ud_channel()
5489 if ((ret = ibt_alloc_ud_channel(state->id_hca_hdl, IBT_ACHAN_NO_FLAGS, in ibd_setup_ud_channel()
5490 &ud_alloc_attr, &state->id_chnl_hdl, NULL)) != IBT_SUCCESS) { in ibd_setup_ud_channel()
5496 if ((ret = ibt_query_ud_channel(state->id_chnl_hdl, in ibd_setup_ud_channel()
5500 (void) ibt_free_channel(state->id_chnl_hdl); in ibd_setup_ud_channel()
5504 state->id_qpnum = ud_chan_attr.ud_qpn; in ibd_setup_ud_channel()
5510 ibd_undo_start(ibd_state_t *state, link_state_t cur_link_state) in ibd_undo_start() argument
5512 uint32_t progress = state->id_mac_state; in ibd_undo_start()
5520 if (atomic_dec_32_nv(&state->id_running) != 0) in ibd_undo_start()
5531 mutex_enter(&state->id_link_mutex); in ibd_undo_start()
5533 state->id_link_state = cur_link_state; in ibd_undo_start()
5535 state->id_link_state = LINK_STATE_UNKNOWN; in ibd_undo_start()
5537 mutex_exit(&state->id_link_mutex); in ibd_undo_start()
5538 bzero(&state->id_macaddr, sizeof (ipoib_mac_t)); in ibd_undo_start()
5539 mac_link_update(state->id_mh, state->id_link_state); in ibd_undo_start()
5541 state->id_mac_state &= (~IBD_DRV_PORT_DETAILS_OBTAINED); in ibd_undo_start()
5543 state->id_mac_state &= (~IBD_DRV_STARTED); in ibd_undo_start()
5547 state->id_mac_state &= (~IBD_DRV_IN_LATE_HCA_INIT); in ibd_undo_start()
5552 ASSERT(state->id_enable_rc); in ibd_undo_start()
5553 if (state->rc_listen_hdl != NULL) { in ibd_undo_start()
5554 ibd_rc_stop_listen(state); in ibd_undo_start()
5556 state->id_mac_state &= (~IBD_DRV_RC_LISTEN); in ibd_undo_start()
5561 ASSERT(state->id_enable_rc); in ibd_undo_start()
5562 mutex_enter(&state->rc_timeout_lock); in ibd_undo_start()
5563 state->rc_timeout_start = B_FALSE; in ibd_undo_start()
5564 tid = state->rc_timeout; in ibd_undo_start()
5565 state->rc_timeout = 0; in ibd_undo_start()
5566 mutex_exit(&state->rc_timeout_lock); in ibd_undo_start()
5569 state->id_mac_state &= (~IBD_DRV_RC_TIMEOUT); in ibd_undo_start()
5572 if ((state->id_enable_rc) && (progress & IBD_DRV_ACACHE_INITIALIZED)) { in ibd_undo_start()
5574 while (state->id_ah_op == IBD_OP_ONGOING) { in ibd_undo_start()
5582 state->rc_stop_connect++; in ibd_undo_start()
5587 mutex_enter(&state->id_sched_lock); in ibd_undo_start()
5588 state->id_sched_needed = 0; in ibd_undo_start()
5589 mutex_exit(&state->id_sched_lock); in ibd_undo_start()
5590 (void) ibd_rc_close_all_chan(state); in ibd_undo_start()
5600 while (atomic_add_32_nv(&state->id_rx_list.dl_bufs_outstanding, in ibd_undo_start()
5618 state->id_mac_state &= (~IBD_DRV_RCQ_NOTIFY_ENABLED); in ibd_undo_start()
5622 ibd_rc_fini_tx_largebuf_list(state); in ibd_undo_start()
5623 state->id_mac_state &= (~IBD_DRV_RC_LARGEBUF_ALLOCD); in ibd_undo_start()
5627 ASSERT(state->id_enable_rc); in ibd_undo_start()
5628 if (state->rc_srq_rwqe_list.dl_bufs_outstanding == 0) { in ibd_undo_start()
5629 if (state->id_ah_op == IBD_OP_ONGOING) { in ibd_undo_start()
5631 if (state->id_ah_op == IBD_OP_ONGOING) { in ibd_undo_start()
5638 state->rc_stop_connect++; in ibd_undo_start()
5642 ibd_rc_fini_srq_list(state); in ibd_undo_start()
5643 state->id_mac_state &= in ibd_undo_start()
5647 ibd_rc_fini_srq_list(state); in ibd_undo_start()
5648 state->id_mac_state &= (~IBD_DRV_RC_SRQ_ALLOCD); in ibd_undo_start()
5656 ibt_register_subnet_notices(state->id_ibt_hdl, NULL, NULL); in ibd_undo_start()
5658 mutex_enter(&state->id_trap_lock); in ibd_undo_start()
5659 state->id_trap_stop = B_TRUE; in ibd_undo_start()
5660 while (state->id_trap_inprog > 0) in ibd_undo_start()
5661 cv_wait(&state->id_trap_cv, &state->id_trap_lock); in ibd_undo_start()
5662 mutex_exit(&state->id_trap_lock); in ibd_undo_start()
5664 state->id_mac_state &= (~IBD_DRV_SM_NOTICES_REGISTERED); in ibd_undo_start()
5675 if ((ret = ibt_flush_channel(state->id_chnl_hdl)) != in ibd_undo_start()
5686 mutex_enter(&state->id_tx_list.dl_mutex); in ibd_undo_start()
5687 mutex_enter(&state->id_tx_rel_list.dl_mutex); in ibd_undo_start()
5688 while (state->id_tx_list.dl_cnt + state->id_tx_rel_list.dl_cnt in ibd_undo_start()
5689 != state->id_ud_num_swqe) { in ibd_undo_start()
5692 mutex_exit(&state->id_tx_rel_list.dl_mutex); in ibd_undo_start()
5693 mutex_exit(&state->id_tx_list.dl_mutex); in ibd_undo_start()
5695 mutex_enter(&state->id_tx_list.dl_mutex); in ibd_undo_start()
5696 mutex_enter(&state->id_tx_rel_list.dl_mutex); in ibd_undo_start()
5698 ibt_set_cq_handler(state->id_scq_hdl, 0, 0); in ibd_undo_start()
5699 if (state->id_tx_list.dl_cnt + state->id_tx_rel_list.dl_cnt != in ibd_undo_start()
5700 state->id_ud_num_swqe) { in ibd_undo_start()
5703 mutex_exit(&state->id_tx_rel_list.dl_mutex); in ibd_undo_start()
5704 mutex_exit(&state->id_tx_list.dl_mutex); in ibd_undo_start()
5707 while (atomic_add_32_nv(&state->id_rx_list.dl_cnt, 0) != 0) { in ibd_undo_start()
5712 ibt_set_cq_handler(state->id_rcq_hdl, 0, 0); in ibd_undo_start()
5713 if (atomic_add_32_nv(&state->id_rx_list.dl_cnt, 0) != 0) { in ibd_undo_start()
5717 state->id_mac_state &= (~IBD_DRV_SCQ_NOTIFY_ENABLED); in ibd_undo_start()
5732 mce = list_head(&state->id_mc_full); in ibd_undo_start()
5736 mce = list_next(&state->id_mc_full, mce); in ibd_undo_start()
5737 ibd_leave_group(state, mgid, jstate); in ibd_undo_start()
5739 state->id_mac_state &= (~IBD_DRV_BCAST_GROUP_JOINED); in ibd_undo_start()
5743 ibd_fini_rxlist(state); in ibd_undo_start()
5744 state->id_mac_state &= (~IBD_DRV_RXLIST_ALLOCD); in ibd_undo_start()
5748 ibd_fini_txlist(state); in ibd_undo_start()
5749 state->id_mac_state &= (~IBD_DRV_TXLIST_ALLOCD); in ibd_undo_start()
5753 if ((ret = ibt_free_channel(state->id_chnl_hdl)) != in ibd_undo_start()
5759 state->id_mac_state &= (~IBD_DRV_UD_CHANNEL_SETUP); in ibd_undo_start()
5763 kmem_free(state->id_txwcs, in ibd_undo_start()
5764 sizeof (ibt_wc_t) * state->id_txwcs_size); in ibd_undo_start()
5765 if ((ret = ibt_free_cq(state->id_scq_hdl)) != in ibd_undo_start()
5771 kmem_free(state->id_rxwcs, in ibd_undo_start()
5772 sizeof (ibt_wc_t) * state->id_rxwcs_size); in ibd_undo_start()
5773 if ((ret = ibt_free_cq(state->id_rcq_hdl)) != IBT_SUCCESS) { in ibd_undo_start()
5778 state->id_txwcs = NULL; in ibd_undo_start()
5779 state->id_rxwcs = NULL; in ibd_undo_start()
5780 state->id_scq_hdl = NULL; in ibd_undo_start()
5781 state->id_rcq_hdl = NULL; in ibd_undo_start()
5783 state->id_mac_state &= (~IBD_DRV_CQS_ALLOCD); in ibd_undo_start()
5787 mutex_enter(&state->id_ac_mutex); in ibd_undo_start()
5788 mod_hash_destroy_hash(state->id_ah_active_hash); in ibd_undo_start()
5789 mutex_exit(&state->id_ac_mutex); in ibd_undo_start()
5790 ibd_acache_fini(state); in ibd_undo_start()
5792 state->id_mac_state &= (~IBD_DRV_ACACHE_INITIALIZED); in ibd_undo_start()
5800 if (state->id_bgroup_created) { in ibd_undo_start()
5801 mgid = state->id_mcinfo->mc_adds_vect.av_dgid; in ibd_undo_start()
5803 (void) ibt_leave_mcg(state->id_sgid, mgid, in ibd_undo_start()
5804 state->id_sgid, jstate); in ibd_undo_start()
5806 ibt_free_mcg_info(state->id_mcinfo, 1); in ibd_undo_start()
5808 state->id_mac_state &= (~IBD_DRV_BCAST_GROUP_FOUND); in ibd_undo_start()
5823 ibd_set_mac_progress(ibd_state_t *state, uint_t flag) in ibd_set_mac_progress() argument
5825 mutex_enter(&state->id_macst_lock); in ibd_set_mac_progress()
5826 while (state->id_mac_state & IBD_DRV_RESTART_IN_PROGRESS) in ibd_set_mac_progress()
5827 cv_wait(&state->id_macst_cv, &state->id_macst_lock); in ibd_set_mac_progress()
5829 state->id_mac_state |= flag; in ibd_set_mac_progress()
5830 mutex_exit(&state->id_macst_lock); in ibd_set_mac_progress()
5834 ibd_clr_mac_progress(ibd_state_t *state, uint_t flag) in ibd_clr_mac_progress() argument
5836 mutex_enter(&state->id_macst_lock); in ibd_clr_mac_progress()
5837 state->id_mac_state &= (~flag); in ibd_clr_mac_progress()
5838 cv_signal(&state->id_macst_cv); in ibd_clr_mac_progress()
5839 mutex_exit(&state->id_macst_lock); in ibd_clr_mac_progress()
5849 ibd_state_t *state = arg; in ibd_m_start() local
5852 if (state->id_type == IBD_PORT_DRIVER) in ibd_m_start()
5855 ibd_set_mac_progress(state, IBD_DRV_START_IN_PROGRESS); in ibd_m_start()
5856 if (state->id_mac_state & IBD_DRV_IN_DELETION) { in ibd_m_start()
5857 ibd_clr_mac_progress(state, IBD_DRV_START_IN_PROGRESS); in ibd_m_start()
5861 ret = ibd_start(state); in ibd_m_start()
5862 ibd_clr_mac_progress(state, IBD_DRV_START_IN_PROGRESS); in ibd_m_start()
5867 ibd_start(ibd_state_t *state) in ibd_start() argument
5873 if (state->id_mac_state & IBD_DRV_STARTED) in ibd_start()
5881 if (!(state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) && in ibd_start()
5882 (atomic_inc_32_nv(&state->id_running) != 1)) { in ibd_start()
5885 atomic_dec_32(&state->id_running); in ibd_start()
5893 if ((err = ibd_get_port_details(state)) != 0) { in ibd_start()
5902 state->id_mac_state |= IBD_DRV_PORT_DETAILS_OBTAINED; in ibd_start()
5903 if (state->id_link_state == LINK_STATE_DOWN) { in ibd_start()
5911 if (ibd_find_bgroup(state) != IBT_SUCCESS) { in ibd_start()
5916 state->id_mac_state |= IBD_DRV_BCAST_GROUP_FOUND; in ibd_start()
5922 if (ibd_acache_init(state) != DDI_SUCCESS) { in ibd_start()
5927 state->id_mac_state |= IBD_DRV_ACACHE_INITIALIZED; in ibd_start()
5932 if (ibd_alloc_cqs(state) != DDI_SUCCESS) { in ibd_start()
5937 state->id_mac_state |= IBD_DRV_CQS_ALLOCD; in ibd_start()
5942 if (ibd_setup_ud_channel(state) != DDI_SUCCESS) { in ibd_start()
5947 state->id_mac_state |= IBD_DRV_UD_CHANNEL_SETUP; in ibd_start()
5952 if (ibd_init_txlist(state) != DDI_SUCCESS) { in ibd_start()
5957 state->id_mac_state |= IBD_DRV_TXLIST_ALLOCD; in ibd_start()
5962 ibt_set_cq_handler(state->id_scq_hdl, ibd_scq_handler, state); in ibd_start()
5963 if ((ret = ibt_enable_cq_notify(state->id_scq_hdl, in ibd_start()
5970 state->id_mac_state |= IBD_DRV_SCQ_NOTIFY_ENABLED; in ibd_start()
5975 if (ibd_init_rxlist(state) != DDI_SUCCESS) { in ibd_start()
5980 state->id_mac_state |= IBD_DRV_RXLIST_ALLOCD; in ibd_start()
5985 if (ibd_join_group(state, state->id_mgid, IB_MC_JSTATE_FULL) == NULL) { in ibd_start()
5990 state->id_mac_state |= IBD_DRV_BCAST_GROUP_JOINED; in ibd_start()
5999 if (state->id_enable_rc) { in ibd_start()
6000 ibd_h2n_mac(&state->id_macaddr, in ibd_start()
6001 IBD_MAC_ADDR_RC + state->id_qpnum, in ibd_start()
6002 state->id_sgid.gid_prefix, state->id_sgid.gid_guid); in ibd_start()
6003 ibd_h2n_mac(&state->rc_macaddr_loopback, state->id_qpnum, in ibd_start()
6004 state->id_sgid.gid_prefix, state->id_sgid.gid_guid); in ibd_start()
6006 ibd_h2n_mac(&state->id_macaddr, state->id_qpnum, in ibd_start()
6007 state->id_sgid.gid_prefix, state->id_sgid.gid_guid); in ibd_start()
6009 ibd_h2n_mac(&state->id_bcaddr, IB_QPN_MASK, in ibd_start()
6010 state->id_mgid.gid_prefix, state->id_mgid.gid_guid); in ibd_start()
6012 if (!state->id_enable_rc) { in ibd_start()
6013 (void) mac_maxsdu_update2(state->id_mh, in ibd_start()
6014 state->id_mtu - IPOIB_HDRSIZE, in ibd_start()
6015 state->id_mtu - IPOIB_HDRSIZE); in ibd_start()
6017 mac_unicst_update(state->id_mh, (uint8_t *)&state->id_macaddr); in ibd_start()
6022 ibt_set_cq_handler(state->id_rcq_hdl, ibd_rcq_handler, state); in ibd_start()
6023 if ((ret = ibt_enable_cq_notify(state->id_rcq_hdl, in ibd_start()
6030 state->id_mac_state |= IBD_DRV_RCQ_NOTIFY_ENABLED; in ibd_start()
6052 if ((state->id_mac_state & IBD_DRV_SM_NOTICES_REGISTERED) == 0) { in ibd_start()
6053 ibt_register_subnet_notices(state->id_ibt_hdl, in ibd_start()
6054 ibd_snet_notices_handler, state); in ibd_start()
6055 mutex_enter(&state->id_trap_lock); in ibd_start()
6056 state->id_trap_stop = B_FALSE; in ibd_start()
6057 mutex_exit(&state->id_trap_lock); in ibd_start()
6058 state->id_mac_state |= IBD_DRV_SM_NOTICES_REGISTERED; in ibd_start()
6063 state->id_mac_state |= IBD_DRV_IN_LATE_HCA_INIT; in ibd_start()
6069 state->id_link_state = LINK_STATE_DOWN; in ibd_start()
6070 mac_unicst_update(state->id_mh, (uint8_t *)&state->id_macaddr); in ibd_start()
6071 mac_link_update(state->id_mh, state->id_link_state); in ibd_start()
6075 if (state->id_enable_rc) { in ibd_start()
6076 if (state->rc_enable_srq) { in ibd_start()
6077 if (state->id_mac_state & IBD_DRV_RC_SRQ_ALLOCD) { in ibd_start()
6078 if (ibd_rc_repost_srq_free_list(state) != in ibd_start()
6085 if (ibd_rc_init_srq_list(state) != in ibd_start()
6090 state->id_mac_state |= IBD_DRV_RC_SRQ_ALLOCD; in ibd_start()
6094 if (ibd_rc_init_tx_largebuf_list(state) != IBT_SUCCESS) { in ibd_start()
6100 state->id_mac_state |= IBD_DRV_RC_LARGEBUF_ALLOCD; in ibd_start()
6103 if (ibd_rc_listen(state) != IBT_SUCCESS) { in ibd_start()
6108 state->id_mac_state |= IBD_DRV_RC_LISTEN; in ibd_start()
6121 mac_link_update(state->id_mh, state->id_link_state); in ibd_start()
6122 state->id_mac_state &= ~IBD_DRV_IN_LATE_HCA_INIT; in ibd_start()
6123 state->id_mac_state |= IBD_DRV_STARTED; in ibd_start()
6126 if (state->id_enable_rc) { in ibd_start()
6127 mutex_enter(&state->rc_timeout_lock); in ibd_start()
6128 state->rc_timeout_start = B_TRUE; in ibd_start()
6129 state->rc_timeout = timeout(ibd_rc_conn_timeout_call, state, in ibd_start()
6131 mutex_exit(&state->rc_timeout_lock); in ibd_start()
6132 state->id_mac_state |= IBD_DRV_RC_TIMEOUT; in ibd_start()
6144 (void) ibd_undo_start(state, LINK_STATE_DOWN); in ibd_start()
6155 ibd_state_t *state = (ibd_state_t *)arg; in ibd_m_stop() local
6157 if (state->id_type == IBD_PORT_DRIVER) in ibd_m_stop()
6160 ibd_set_mac_progress(state, IBD_DRV_STOP_IN_PROGRESS); in ibd_m_stop()
6162 (void) ibd_undo_start(state, state->id_link_state); in ibd_m_stop()
6164 ibd_clr_mac_progress(state, IBD_DRV_STOP_IN_PROGRESS); in ibd_m_stop()
6174 ibd_state_t *state = arg; in ibd_m_unicst() local
6176 if (state->id_type == IBD_PORT_DRIVER) in ibd_m_unicst()
6183 if ((state->id_mac_state & IBD_DRV_STARTED) == 0) in ibd_m_unicst()
6186 if (bcmp(macaddr, &state->id_macaddr, IPOIB_ADDRL) == 0) in ibd_m_unicst()
6197 ibd_async_multicast(ibd_state_t *state, ib_gid_t mgid, int op) in ibd_async_multicast() argument
6203 if (ibd_join_group(state, mgid, IB_MC_JSTATE_FULL) == NULL) { in ibd_async_multicast()
6204 ibd_print_warn(state, "Join multicast group failed :" in ibd_async_multicast()
6212 ibd_leave_group(state, mgid, IB_MC_JSTATE_FULL); in ibd_async_multicast()
6224 ibd_state_t *state = (ibd_state_t *)arg; in ibd_m_multicst() local
6229 if (state->id_type == IBD_PORT_DRIVER) in ibd_m_multicst()
6237 if ((state->id_mac_state & IBD_DRV_STARTED) == 0) in ibd_m_multicst()
6263 IBD_FILL_SCOPE_PKEY(mcast, state->id_scope, state->id_pkey); in ibd_m_multicst()
6273 if (bcmp(mcast, &state->id_bcaddr, IPOIB_ADDRL) == 0) in ibd_m_multicst()
6277 req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP); in ibd_m_multicst()
6286 ibd_queue_work_slot(state, req, IBD_ASYNC_JOIN); in ibd_m_multicst()
6290 ibd_queue_work_slot(state, req, IBD_ASYNC_LEAVE); in ibd_m_multicst()
6302 ibd_async_unsetprom(ibd_state_t *state) in ibd_async_unsetprom() argument
6304 ibd_mce_t *mce = list_head(&state->id_mc_non); in ibd_async_unsetprom()
6311 mce = list_next(&state->id_mc_non, mce); in ibd_async_unsetprom()
6312 ibd_leave_group(state, mgid, IB_MC_JSTATE_NON); in ibd_async_unsetprom()
6314 state->id_prom_op = IBD_OP_NOTSTARTED; in ibd_async_unsetprom()
6324 ibd_async_setprom(ibd_state_t *state) in ibd_async_setprom() argument
6340 mcg_attr.mc_pkey = state->id_pkey; in ibd_async_setprom()
6341 mcg_attr.mc_scope = state->id_scope; in ibd_async_setprom()
6342 mcg_attr.mc_qkey = state->id_mcinfo->mc_qkey; in ibd_async_setprom()
6343 mcg_attr.mc_mtu_req.r_mtu = state->id_mcinfo->mc_mtu; in ibd_async_setprom()
6345 if (ibt_query_mcg(state->id_sgid, &mcg_attr, 0, &mcg_info, &numg) != in ibd_async_setprom()
6347 ibd_print_warn(state, "Could not get list of IBA multicast " in ibd_async_setprom()
6362 if (ibd_join_group(state, mgid, IB_MC_JSTATE_NON) == NULL) in ibd_async_setprom()
6363 ibd_print_warn(state, "IBA promiscuous mode missed " in ibd_async_setprom()
6372 state->id_prom_op = ret; in ibd_async_setprom()
6384 ibd_state_t *state = (ibd_state_t *)arg; in ibd_m_promisc() local
6387 if (state->id_type == IBD_PORT_DRIVER) in ibd_m_promisc()
6394 if ((state->id_mac_state & IBD_DRV_STARTED) == 0) in ibd_m_promisc()
6397 req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP); in ibd_m_promisc()
6402 ibd_queue_work_slot(state, req, IBD_ASYNC_PROMON); in ibd_m_promisc()
6405 ibd_queue_work_slot(state, req, IBD_ASYNC_PROMOFF); in ibd_m_promisc()
6417 ibd_state_t *state = (ibd_state_t *)arg; in ibd_m_stat() local
6421 *val = state->id_link_speed; in ibd_m_stat()
6424 *val = state->id_multi_rcv; in ibd_m_stat()
6427 *val = state->id_brd_rcv; in ibd_m_stat()
6430 *val = state->id_multi_xmt; in ibd_m_stat()
6433 *val = state->id_brd_xmt; in ibd_m_stat()
6436 *val = state->id_rcv_bytes + state->rc_rcv_trans_byte in ibd_m_stat()
6437 + state->rc_rcv_copy_byte; in ibd_m_stat()
6440 *val = state->id_rcv_pkt + state->rc_rcv_trans_pkt in ibd_m_stat()
6441 + state->rc_rcv_copy_pkt; in ibd_m_stat()
6444 *val = state->id_xmt_bytes + state->rc_xmt_bytes; in ibd_m_stat()
6447 *val = state->id_xmt_pkt + state->rc_xmt_small_pkt + in ibd_m_stat()
6448 state->rc_xmt_fragmented_pkt + in ibd_m_stat()
6449 state->rc_xmt_map_fail_pkt + state->rc_xmt_map_succ_pkt; in ibd_m_stat()
6452 *val = state->id_ah_error; /* failed AH translation */ in ibd_m_stat()
6458 *val = state->id_tx_short + state->rc_swqe_short + in ibd_m_stat()
6459 state->rc_xmt_buf_short; in ibd_m_stat()
6470 ibd_async_txsched(ibd_state_t *state) in ibd_async_txsched() argument
6472 ibd_resume_transmission(state); in ibd_async_txsched()
6476 ibd_resume_transmission(ibd_state_t *state) in ibd_resume_transmission() argument
6483 mutex_enter(&state->id_sched_lock); in ibd_resume_transmission()
6484 if (state->id_sched_needed & IBD_RSRC_SWQE) { in ibd_resume_transmission()
6485 mutex_enter(&state->id_tx_list.dl_mutex); in ibd_resume_transmission()
6486 mutex_enter(&state->id_tx_rel_list.dl_mutex); in ibd_resume_transmission()
6487 met_thresh = state->id_tx_list.dl_cnt + in ibd_resume_transmission()
6488 state->id_tx_rel_list.dl_cnt; in ibd_resume_transmission()
6489 mutex_exit(&state->id_tx_rel_list.dl_mutex); in ibd_resume_transmission()
6490 mutex_exit(&state->id_tx_list.dl_mutex); in ibd_resume_transmission()
6493 } else if (state->id_sched_needed & IBD_RSRC_LSOBUF) { in ibd_resume_transmission()
6494 ASSERT(state->id_lso != NULL); in ibd_resume_transmission()
6495 mutex_enter(&state->id_lso_lock); in ibd_resume_transmission()
6496 met_thresh = state->id_lso->bkt_nfree; in ibd_resume_transmission()
6498 mutex_exit(&state->id_lso_lock); in ibd_resume_transmission()
6501 state->id_sched_lso_cnt++; in ibd_resume_transmission()
6504 state->id_sched_needed &= ~flag; in ibd_resume_transmission()
6505 state->id_sched_cnt++; in ibd_resume_transmission()
6508 mutex_exit(&state->id_sched_lock); in ibd_resume_transmission()
6511 mac_tx_update(state->id_mh); in ibd_resume_transmission()
6518 ibd_release_swqe(ibd_state_t *state, ibd_swqe_t *head, ibd_swqe_t *tail, int n) in ibd_release_swqe() argument
6524 mutex_enter(&state->id_tx_rel_list.dl_mutex); in ibd_release_swqe()
6525 state->id_tx_rel_list.dl_pending_sends = B_FALSE; in ibd_release_swqe()
6526 tail->swqe_next = state->id_tx_rel_list.dl_head; in ibd_release_swqe()
6527 state->id_tx_rel_list.dl_head = SWQE_TO_WQE(head); in ibd_release_swqe()
6528 state->id_tx_rel_list.dl_cnt += n; in ibd_release_swqe()
6529 mutex_exit(&state->id_tx_rel_list.dl_mutex); in ibd_release_swqe()
6537 ibd_acquire_swqe(ibd_state_t *state) in ibd_acquire_swqe() argument
6541 mutex_enter(&state->id_tx_rel_list.dl_mutex); in ibd_acquire_swqe()
6542 if (state->id_tx_rel_list.dl_head != NULL) { in ibd_acquire_swqe()
6544 state->id_tx_list.dl_head = in ibd_acquire_swqe()
6545 state->id_tx_rel_list.dl_head; in ibd_acquire_swqe()
6546 state->id_tx_list.dl_cnt = in ibd_acquire_swqe()
6547 state->id_tx_rel_list.dl_cnt; in ibd_acquire_swqe()
6548 state->id_tx_list.dl_pending_sends = B_FALSE; in ibd_acquire_swqe()
6551 state->id_tx_rel_list.dl_head = NULL; in ibd_acquire_swqe()
6552 state->id_tx_rel_list.dl_cnt = 0; in ibd_acquire_swqe()
6553 mutex_exit(&state->id_tx_rel_list.dl_mutex); in ibd_acquire_swqe()
6555 wqe = WQE_TO_SWQE(state->id_tx_list.dl_head); in ibd_acquire_swqe()
6556 state->id_tx_list.dl_cnt -= 1; in ibd_acquire_swqe()
6557 state->id_tx_list.dl_head = wqe->swqe_next; in ibd_acquire_swqe()
6559 mutex_exit(&state->id_tx_rel_list.dl_mutex); in ibd_acquire_swqe()
6560 state->id_tx_list.dl_pending_sends = B_TRUE; in ibd_acquire_swqe()
6562 state->id_tx_short++; in ibd_acquire_swqe()
6679 ibd_post_send(ibd_state_t *state, ibd_swqe_t *node) in ibd_post_send() argument
6690 ibt_status = ibt_post_send(state->id_chnl_hdl, in ibd_post_send()
6693 ibd_print_warn(state, "ibd_post_send: " in ibd_post_send()
6695 ibd_tx_cleanup(state, node); in ibd_post_send()
6701 mutex_enter(&state->id_txpost_lock); in ibd_post_send()
6702 tx_head = state->id_tx_head; in ibd_post_send()
6704 state->id_tx_busy = 0; in ibd_post_send()
6705 mutex_exit(&state->id_txpost_lock); in ibd_post_send()
6708 state->id_tx_head = NULL; in ibd_post_send()
6709 mutex_exit(&state->id_txpost_lock); in ibd_post_send()
6736 ibt_status = ibt_post_send(state->id_chnl_hdl, in ibd_post_send()
6739 ibd_print_warn(state, "ibd_post_send: " in ibd_post_send()
6745 ibd_tx_cleanup(state, nodes[i]); in ibd_post_send()
6751 ibd_prepare_sgl(ibd_state_t *state, mblk_t *mp, ibd_swqe_t *node, in ibd_prepare_sgl() argument
6800 if ((state->id_hca_res_lkey_capab) && in ibd_prepare_sgl()
6801 (pktsize > state->id_ud_tx_copy_thresh) && in ibd_prepare_sgl()
6802 (nmblks < state->id_max_sqseg_hiwm)) { in ibd_prepare_sgl()
6810 iov_attr.iov_wr_nds = state->id_max_sqseg; in ibd_prepare_sgl()
6826 ibt_status = ibt_map_mem_iov(state->id_hca_hdl, &iov_attr, in ibd_prepare_sgl()
6829 ibd_print_warn(state, "ibd_send: ibt_map_mem_iov " in ibd_prepare_sgl()
6838 if (pktsize <= state->id_tx_buf_sz) { in ibd_prepare_sgl()
6865 if (ibd_acquire_lsobufs(state, pktsize, in ibd_prepare_sgl()
6909 ibd_sched_poll(ibd_state_t *state, int resource_type, int q_flag) in ibd_sched_poll() argument
6913 mutex_enter(&state->id_sched_lock); in ibd_sched_poll()
6914 state->id_sched_needed |= resource_type; in ibd_sched_poll()
6915 mutex_exit(&state->id_sched_lock); in ibd_sched_poll()
6921 req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP); in ibd_sched_poll()
6925 ibd_queue_work_slot(state, req, IBD_ASYNC_SCHED); in ibd_sched_poll()
6936 ibd_send(ibd_state_t *state, mblk_t *mp) in ibd_send() argument
6960 if ((state->id_mac_state & IBD_DRV_STARTED) == 0) in ibd_send()
6969 IBD_FILL_SCOPE_PKEY(dest, state->id_scope, state->id_pkey); in ibd_send()
6972 ace = ibd_acache_lookup(state, dest, &ret, 1); in ibd_send()
6973 if (state->id_enable_rc && (ace != NULL) && in ibd_send()
6976 state->rc_null_conn++; in ibd_send()
6995 state->rc_swqe_short++; in ibd_send()
6996 mutex_enter(&state->id_sched_lock); in ibd_send()
6997 state->id_sched_needed |= in ibd_send()
6999 mutex_exit(&state->id_sched_lock); in ibd_send()
7000 ibd_dec_ref_ace(state, ace); in ibd_send()
7004 state->rc_no_estab_conn++; in ibd_send()
7010 mutex_enter(&state->id_tx_list.dl_mutex); in ibd_send()
7011 node = WQE_TO_SWQE(state->id_tx_list.dl_head); in ibd_send()
7013 state->id_tx_list.dl_cnt -= 1; in ibd_send()
7014 state->id_tx_list.dl_head = node->swqe_next; in ibd_send()
7016 node = ibd_acquire_swqe(state); in ibd_send()
7018 mutex_exit(&state->id_tx_list.dl_mutex); in ibd_send()
7025 if (ibd_sched_poll(state, IBD_RSRC_SWQE, 0) == 0) { in ibd_send()
7027 ibd_dec_ref_ace(state, ace); in ibd_send()
7036 ibd_print_warn(state, "ibd_send: no swqe, pkt drop"); in ibd_send()
7038 ibd_dec_ref_ace(state, ace); in ibd_send()
7063 if (bcmp(&ipibp->ib_dst, &state->id_bcaddr, IPOIB_ADDRL) == 0) in ibd_send()
7064 atomic_inc_64(&state->id_brd_xmt); in ibd_send()
7066 atomic_inc_64(&state->id_multi_xmt); in ibd_send()
7079 state->rc_ace_not_found++; in ibd_send()
7097 } else if (ibd_sched_poll(state, IBD_RSRC_SWQE, 1) != 0) { in ibd_send()
7162 atomic_add_64(&state->rc_xmt_bytes, pktsize); in ibd_send()
7175 if (pktsize <= state->id_rc_tx_copy_thresh) { in ibd_send()
7176 atomic_inc_64(&state->rc_xmt_small_pkt); in ibd_send()
7195 if ((state->rc_enable_iov_map) && in ibd_send()
7196 (nmblks < state->rc_max_sqseg_hiwm)) { in ibd_send()
7202 iov_attr.iov_wr_nds = state->rc_tx_max_sqseg; in ibd_send()
7218 ret = ibt_map_mem_iov(state->id_hca_hdl, in ibd_send()
7223 &state->rc_xmt_map_fail_pkt); in ibd_send()
7230 atomic_inc_64(&state->rc_xmt_map_succ_pkt); in ibd_send()
7234 atomic_inc_64(&state->rc_xmt_fragmented_pkt); in ibd_send()
7236 mutex_enter(&state->rc_tx_large_bufs_lock); in ibd_send()
7237 if (state->rc_tx_largebuf_nfree == 0) { in ibd_send()
7238 state->rc_xmt_buf_short++; in ibd_send()
7240 (&state->rc_tx_large_bufs_lock); in ibd_send()
7241 mutex_enter(&state->id_sched_lock); in ibd_send()
7242 state->id_sched_needed |= in ibd_send()
7244 mutex_exit(&state->id_sched_lock); in ibd_send()
7258 lbufp = state->rc_tx_largebuf_free_head; in ibd_send()
7260 state->rc_tx_largebuf_free_head = in ibd_send()
7264 state->rc_tx_largebuf_nfree --; in ibd_send()
7265 mutex_exit(&state->rc_tx_large_bufs_lock); in ibd_send()
7270 state->rc_tx_mr_desc.md_lkey; in ibd_send()
7310 if ((state->id_enable_rc) && (pktsize > state->id_mtu)) { in ibd_send()
7321 state->rc_xmt_reenter_too_long_pkt++; in ibd_send()
7326 state->rc_xmt_icmp_too_long_pkt++; in ibd_send()
7328 req = kmem_cache_alloc(state->id_req_kmc, in ibd_send()
7331 ibd_print_warn(state, "ibd_send: alloc " in ibd_send()
7338 ibd_queue_work_slot(state, req, in ibd_send()
7344 ibd_print_warn(state, "Reliable Connected mode is on. " in ibd_send()
7347 pktsize, state->id_mtu); in ibd_send()
7348 state->rc_xmt_drop_too_long_pkt++; in ibd_send()
7356 atomic_add_64(&state->id_xmt_bytes, pktsize); in ibd_send()
7357 atomic_inc_64(&state->id_xmt_pkt); in ibd_send()
7374 ibd_print_warn(state, in ibd_send()
7396 if (ibd_prepare_sgl(state, mp, node, lsohdr_sz) != 0) { in ibd_send()
7397 if (ibd_sched_poll(state, IBD_RSRC_LSOBUF, 1) != 0) { in ibd_send()
7415 mutex_enter(&state->id_txpost_lock); in ibd_send()
7416 if (state->id_tx_busy) { in ibd_send()
7417 if (state->id_tx_head) { in ibd_send()
7418 state->id_tx_tail->swqe_next = in ibd_send()
7421 state->id_tx_head = node; in ibd_send()
7423 state->id_tx_tail = node; in ibd_send()
7424 mutex_exit(&state->id_txpost_lock); in ibd_send()
7426 state->id_tx_busy = 1; in ibd_send()
7427 mutex_exit(&state->id_txpost_lock); in ibd_send()
7428 ibd_post_send(state, node); in ibd_send()
7444 ibd_tx_cleanup(state, node); in ibd_send()
7457 ibd_state_t *state = (ibd_state_t *)arg; in ibd_m_tx() local
7460 if (state->id_type == IBD_PORT_DRIVER) { in ibd_m_tx()
7465 if ((state->id_link_state != LINK_STATE_UP) || in ibd_m_tx()
7466 !(state->id_mac_state & IBD_DRV_STARTED)) { in ibd_m_tx()
7474 if (ibd_send(state, mp) == B_FALSE) { in ibd_m_tx()
7492 ibd_state_t *state = (ibd_state_t *)arg; in ibd_intr() local
7494 ibd_poll_rcq(state, state->id_rcq_hdl); in ibd_intr()
7503 ibd_drain_scq(ibd_state_t *state, ibt_cq_hdl_t cq_hdl) in ibd_drain_scq() argument
7505 ibt_wc_t *wcs = state->id_txwcs; in ibd_drain_scq()
7506 uint_t numwcs = state->id_txwcs_size; in ibd_drain_scq()
7545 ibd_tx_cleanup_list(state, head, tail); in ibd_drain_scq()
7550 ibd_resume_transmission(state); in ibd_drain_scq()
7558 ibd_drain_rcq(ibd_state_t *state, ibt_cq_hdl_t cq_hdl) in ibd_drain_rcq() argument
7560 ibt_wc_t *wcs = state->id_rxwcs; in ibd_drain_rcq()
7561 uint_t numwcs = state->id_rxwcs_size; in ibd_drain_rcq()
7585 &state->id_rx_list.dl_bufs_outstanding); in ibd_drain_rcq()
7589 mp = ibd_process_rx(state, rwqe, wc); in ibd_drain_rcq()
7603 mac_rx(state->id_mh, state->id_rh, head); in ibd_drain_rcq()
7609 if (atomic_add_32_nv(&state->id_rx_list.dl_cnt, -num_polled) < in ibd_drain_rcq()
7610 (state->id_ud_num_rwqe / 4)) in ibd_drain_rcq()
7611 ibd_post_recv_intr(state); in ibd_drain_rcq()
7620 ibd_poll_scq(ibd_state_t *state, ibt_cq_hdl_t cq_hdl) in ibd_poll_scq() argument
7628 mutex_enter(&state->id_scq_poll_lock); in ibd_poll_scq()
7629 if (state->id_scq_poll_busy & flag) { in ibd_poll_scq()
7630 ibd_print_warn(state, "ibd_poll_scq: multiple polling threads"); in ibd_poll_scq()
7631 state->id_scq_poll_busy |= redo_flag; in ibd_poll_scq()
7632 mutex_exit(&state->id_scq_poll_lock); in ibd_poll_scq()
7635 state->id_scq_poll_busy |= flag; in ibd_poll_scq()
7636 mutex_exit(&state->id_scq_poll_lock); in ibd_poll_scq()
7650 ibd_drain_scq(state, cq_hdl); in ibd_poll_scq()
7665 ibd_drain_scq(state, cq_hdl); in ibd_poll_scq()
7667 mutex_enter(&state->id_scq_poll_lock); in ibd_poll_scq()
7668 if (state->id_scq_poll_busy & redo_flag) in ibd_poll_scq()
7669 state->id_scq_poll_busy &= ~redo_flag; in ibd_poll_scq()
7671 state->id_scq_poll_busy &= ~flag; in ibd_poll_scq()
7674 mutex_exit(&state->id_scq_poll_lock); in ibd_poll_scq()
7684 ibd_poll_rcq(ibd_state_t *state, ibt_cq_hdl_t rcq) in ibd_poll_rcq() argument
7692 mutex_enter(&state->id_rcq_poll_lock); in ibd_poll_rcq()
7693 if (state->id_rcq_poll_busy & flag) { in ibd_poll_rcq()
7694 ibd_print_warn(state, "ibd_poll_rcq: multiple polling threads"); in ibd_poll_rcq()
7695 state->id_rcq_poll_busy |= redo_flag; in ibd_poll_rcq()
7696 mutex_exit(&state->id_rcq_poll_lock); in ibd_poll_rcq()
7699 state->id_rcq_poll_busy |= flag; in ibd_poll_rcq()
7700 mutex_exit(&state->id_rcq_poll_lock); in ibd_poll_rcq()
7705 ibd_drain_rcq(state, rcq); in ibd_poll_rcq()
7720 ibd_drain_rcq(state, rcq); in ibd_poll_rcq()
7722 mutex_enter(&state->id_rcq_poll_lock); in ibd_poll_rcq()
7723 if (state->id_rcq_poll_busy & redo_flag) in ibd_poll_rcq()
7724 state->id_rcq_poll_busy &= ~redo_flag; in ibd_poll_rcq()
7726 state->id_rcq_poll_busy &= ~flag; in ibd_poll_rcq()
7729 mutex_exit(&state->id_rcq_poll_lock); in ibd_poll_rcq()
7738 ibd_unmap_mem(ibd_state_t *state, ibd_swqe_t *swqe) in ibd_unmap_mem() argument
7745 if ((stat = ibt_unmap_mem_iov(state->id_hca_hdl, in ibd_unmap_mem()
7756 ibd_dec_ref_ace(ibd_state_t *state, ibd_ace_t *ace) in ibd_dec_ref_ace() argument
7787 mutex_enter(&state->id_ac_mutex); in ibd_dec_ref_ace()
7800 IBD_ACACHE_PULLOUT_ACTIVE(state, ace); in ibd_dec_ref_ace()
7805 ibd_queue_work_slot(state, in ibd_dec_ref_ace()
7808 IBD_ACACHE_INSERT_FREE(state, ace); in ibd_dec_ref_ace()
7810 mutex_exit(&state->id_ac_mutex); in ibd_dec_ref_ace()
7819 ibd_tx_cleanup(ibd_state_t *state, ibd_swqe_t *swqe) in ibd_tx_cleanup() argument
7834 ibd_unmap_mem(state, swqe); in ibd_tx_cleanup()
7836 ibd_release_lsobufs(state, in ibd_tx_cleanup()
7853 ibd_dec_ref_ace(state, ace); in ibd_tx_cleanup()
7860 ibd_release_swqe(state, swqe, swqe, 1); in ibd_tx_cleanup()
7864 ibd_tx_cleanup_list(ibd_state_t *state, ibd_swqe_t *head, ibd_swqe_t *tail) in ibd_tx_cleanup_list() argument
7884 ibd_unmap_mem(state, swqe); in ibd_tx_cleanup_list()
7886 ibd_release_lsobufs(state, in ibd_tx_cleanup_list()
7904 ibd_dec_ref_ace(state, ace); in ibd_tx_cleanup_list()
7912 ibd_release_swqe(state, head, tail, n); in ibd_tx_cleanup_list()
7921 ibd_process_rx(ibd_state_t *state, ibd_rwqe_t *rwqe, ibt_wc_t *wc) in ibd_process_rx() argument
7935 bufs = atomic_inc_32_nv(&state->id_rx_list.dl_bufs_outstanding); in ibd_process_rx()
7938 if (bufs >= state->id_rx_bufs_outstanding_limit) { in ibd_process_rx()
7939 atomic_dec_32(&state->id_rx_list.dl_bufs_outstanding); in ibd_process_rx()
7940 atomic_inc_32(&state->id_rx_allocb); in ibd_process_rx()
7944 ibd_post_recv(state, rwqe); in ibd_process_rx()
7946 atomic_inc_32(&state->id_rx_allocb_failed); in ibd_process_rx()
7947 ibd_post_recv(state, rwqe); in ibd_process_rx()
7964 ibd_print_warn(state, in ibd_process_rx()
7983 if (state->id_enable_rc) { in ibd_process_rx()
7985 &state->rc_macaddr_loopback, in ibd_process_rx()
7991 if (bcmp(&phdr->ib_grh.ipoib_sqpn, &state->id_macaddr, in ibd_process_rx()
8004 phdr->ib_dst.ipoib_qpn = state->id_macaddr.ipoib_qpn; in ibd_process_rx()
8012 ovbcopy(&state->id_macaddr, &phdr->ib_dst, in ibd_process_rx()
8034 atomic_add_64(&state->id_rcv_bytes, pkt_len); in ibd_process_rx()
8035 atomic_inc_64(&state->id_rcv_pkt); in ibd_process_rx()
8036 if (bcmp(&phdr->ib_dst, &state->id_bcaddr, IPOIB_ADDRL) == 0) in ibd_process_rx()
8037 atomic_inc_64(&state->id_brd_rcv); in ibd_process_rx()
8039 atomic_inc_64(&state->id_multi_rcv); in ibd_process_rx()
8068 ibd_state_t *state = rwqe->w_state; in ibd_freemsg_cb() local
8070 atomic_dec_32(&state->id_rx_list.dl_bufs_outstanding); in ibd_freemsg_cb()
8075 if (atomic_add_32_nv(&state->id_running, 0) == 0) { in ibd_freemsg_cb()
8078 ibd_free_rwqe(state, rwqe); in ibd_freemsg_cb()
8083 state->id_mtu + IPOIB_GRH_SIZE, 0, &rwqe->w_freemsg_cb); in ibd_freemsg_cb()
8085 ibd_free_rwqe(state, rwqe); in ibd_freemsg_cb()
8090 ibd_post_recv(state, rwqe); in ibd_freemsg_cb()
8096 ibd_state_t *state = (ibd_state_t *)arg; in ibd_tx_recycle() local
8101 ibd_poll_scq(state, state->id_scq_hdl); in ibd_tx_recycle()
8169 ibd_state_t *state, *port_state, *p; in ibd_create_partition() local
8243 state = kmem_zalloc(sizeof (ibd_state_t), KM_SLEEP); in ibd_create_partition()
8245 state->id_type = IBD_PARTITION_OBJ; in ibd_create_partition()
8247 state->id_plinkid = cmd->ioc_partid; in ibd_create_partition()
8248 state->id_dlinkid = cmd->ibdioc.ioc_linkid; in ibd_create_partition()
8249 state->id_port_inst = cmd->ibdioc.ioc_port_inst; in ibd_create_partition()
8251 state->id_dip = port_state->id_dip; in ibd_create_partition()
8252 state->id_port = port_state->id_port; in ibd_create_partition()
8253 state->id_pkey = cmd->ioc_pkey; in ibd_create_partition()
8254 state->id_hca_guid = port_state->id_hca_guid; in ibd_create_partition()
8255 state->id_port_guid = port_state->id_port_guid; in ibd_create_partition()
8256 state->id_force_create = force_create; in ibd_create_partition()
8258 mutex_init(&state->id_macst_lock, NULL, MUTEX_DRIVER, NULL); in ibd_create_partition()
8259 cv_init(&state->id_macst_cv, NULL, CV_DEFAULT, NULL); in ibd_create_partition()
8261 if (ibd_part_attach(state, state->id_dip) != DDI_SUCCESS) { in ibd_create_partition()
8275 macp->m_driver = state; in ibd_create_partition()
8276 macp->m_src_addr = (uint8_t *)&state->id_macaddr; in ibd_create_partition()
8280 if (state->id_enable_rc) { in ibd_create_partition()
8287 err = mac_register(macp, &state->id_mh); in ibd_create_partition()
8297 err = dls_devnet_create(state->id_mh, in ibd_create_partition()
8303 (void) mac_unregister(state->id_mh); in ibd_create_partition()
8312 state->id_next = ibd_objlist_head; in ibd_create_partition()
8314 ibd_objlist_head = state; in ibd_create_partition()
8327 ibd_part_unattach(state); in ibd_create_partition()
8328 kmem_free(state, sizeof (ibd_state_t)); in ibd_create_partition()
8422 ibd_state_t *state, *port_state; in ibd_get_partition_info() local
8443 for (state = ibd_objlist_head; state; state = state->id_next) { in ibd_get_partition_info()
8444 if (state->id_plinkid == cmd.ioc_linkid) { in ibd_get_partition_info()
8449 if (state == NULL) { in ibd_get_partition_info()
8454 partioc.ibdioc.ioc_linkid = state->id_dlinkid; in ibd_get_partition_info()
8455 partioc.ibdioc.ioc_port_inst = state->id_port_inst; in ibd_get_partition_info()
8456 partioc.ibdioc.ioc_portnum = state->id_port; in ibd_get_partition_info()
8457 partioc.ibdioc.ioc_hcaguid = state->id_hca_guid; in ibd_get_partition_info()
8458 partioc.ibdioc.ioc_portguid = state->id_port_guid; in ibd_get_partition_info()
8460 partioc.ioc_partid = state->id_plinkid; in ibd_get_partition_info()
8461 partioc.ioc_pkey = state->id_pkey; in ibd_get_partition_info()
8462 partioc.ioc_force_create = state->id_force_create; in ibd_get_partition_info()
8697 ibd_state_t *state = (ibd_state_t *)arg; in ibdpd_async_handler() local
8703 if (ibd_get_port_state(state, &lstate) != 0) in ibdpd_async_handler()
8706 if (state->id_link_state != lstate) { in ibdpd_async_handler()
8707 state->id_link_state = lstate; in ibdpd_async_handler()
8708 mac_link_update(state->id_mh, lstate); in ibdpd_async_handler()
8717 ibd_get_port_state(ibd_state_t *state, link_state_t *lstate) in ibd_get_port_state() argument
8723 ret = ibt_query_hca_ports(state->id_hca_hdl, state->id_port, in ibd_get_port_state()
8728 state->id_sgid = *port_infop->p_sgid_tbl; in ibd_get_port_state()
8729 state->id_link_speed = ibd_get_portspeed(state); in ibd_get_port_state()
8743 ibd_state_t *state; in ibd_port_attach() local
8757 state = ddi_get_soft_state(ibd_list, instance); in ibd_port_attach()
8759 state->id_dip = dip; in ibd_port_attach()
8760 state->id_type = IBD_PORT_DRIVER; in ibd_port_attach()
8762 if ((state->id_port = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0, in ibd_port_attach()
8765 state->id_port); in ibd_port_attach()
8768 if ((state->id_hca_guid = ddi_prop_get_int64(DDI_DEV_T_ANY, dip, 0, in ibd_port_attach()
8771 state->id_hca_guid); in ibd_port_attach()
8774 if ((state->id_port_guid = ddi_prop_get_int64(DDI_DEV_T_ANY, dip, 0, in ibd_port_attach()
8777 state->id_port_guid); in ibd_port_attach()
8784 if ((ret = ibt_attach(&ibdpd_clnt_modinfo, dip, state, in ibd_port_attach()
8785 &state->id_ibt_hdl)) != IBT_SUCCESS) { in ibd_port_attach()
8791 state->id_mac_state |= IBD_DRV_IBTL_ATTACH_DONE; in ibd_port_attach()
8793 if ((ret = ibt_open_hca(state->id_ibt_hdl, state->id_hca_guid, in ibd_port_attach()
8794 &state->id_hca_hdl)) != IBT_SUCCESS) { in ibd_port_attach()
8799 state->id_mac_state |= IBD_DRV_HCA_OPENED; in ibd_port_attach()
8803 if (ibd_get_port_state(state, &lstate) != 0) { in ibd_port_attach()
8808 state->id_link_state = lstate; in ibd_port_attach()
8812 if (ibd_register_mac(state, dip) != IBT_SUCCESS) { in ibd_port_attach()
8816 state->id_mac_state |= IBD_DRV_MAC_REGISTERED; in ibd_port_attach()
8818 mac_link_update(state->id_mh, lstate); in ibd_port_attach()
8822 (void) ibd_port_unattach(state, dip); in ibd_port_attach()
8827 ibd_port_unattach(ibd_state_t *state, dev_info_t *dip) in ibd_port_unattach() argument
8830 uint32_t progress = state->id_mac_state; in ibd_port_unattach()
8834 (void) mac_unregister(state->id_mh); in ibd_port_unattach()
8835 state->id_mac_state &= (~IBD_DRV_MAC_REGISTERED); in ibd_port_unattach()
8839 if ((ret = ibt_close_hca(state->id_hca_hdl)) != in ibd_port_unattach()
8841 ibd_print_warn(state, "failed to close " in ibd_port_unattach()
8844 state->id_hca_hdl = NULL; in ibd_port_unattach()
8845 state->id_mac_state &= (~IBD_DRV_HCA_OPENED); in ibd_port_unattach()
8849 if ((ret = ibt_detach(state->id_ibt_hdl)) != IBT_SUCCESS) { in ibd_port_unattach()
8850 ibd_print_warn(state, in ibd_port_unattach()
8853 state->id_ibt_hdl = NULL; in ibd_port_unattach()
8854 state->id_mac_state &= (~IBD_DRV_IBTL_ATTACH_DONE); in ibd_port_unattach()
8865 ibd_state_t *state; in ibd_get_part_attr() local
8870 for (state = ibd_objlist_head; state; state = state->id_next) { in ibd_get_part_attr()
8871 if (state->id_plinkid == linkid) { in ibd_get_part_attr()
8876 if (state == NULL) { in ibd_get_part_attr()
8881 attr->pa_dlinkid = state->id_dlinkid; in ibd_get_part_attr()
8882 attr->pa_plinkid = state->id_plinkid; in ibd_get_part_attr()
8883 attr->pa_port = state->id_port; in ibd_get_part_attr()
8884 attr->pa_hca_guid = state->id_hca_guid; in ibd_get_part_attr()
8885 attr->pa_port_guid = state->id_port_guid; in ibd_get_part_attr()
8886 attr->pa_pkey = state->id_pkey; in ibd_get_part_attr()
8896 ibd_state_t *state; in ibd_get_all_part_attr() local
8902 for (state = ibd_objlist_head; state; state = state->id_next) in ibd_get_all_part_attr()
8914 for (state = ibd_objlist_head; state; state = state->id_next) { in ibd_get_all_part_attr()
8919 attr->pa_dlinkid = state->id_dlinkid; in ibd_get_all_part_attr()
8920 attr->pa_plinkid = state->id_plinkid; in ibd_get_all_part_attr()
8921 attr->pa_port = state->id_port; in ibd_get_all_part_attr()
8922 attr->pa_hca_guid = state->id_hca_guid; in ibd_get_all_part_attr()
8923 attr->pa_port_guid = state->id_port_guid; in ibd_get_all_part_attr()
8924 attr->pa_pkey = state->id_pkey; in ibd_get_all_part_attr()