Lines Matching refs:state
123 ibd_async_rc_close_act_chan(ibd_state_t *state, ibd_req_t *req) in ibd_async_rc_close_act_chan() argument
133 mutex_enter(&state->id_ac_mutex); in ibd_async_rc_close_act_chan()
138 IBD_ACACHE_INSERT_FREE(state, ace); in ibd_async_rc_close_act_chan()
142 state->rc_delay_ace_recycle++; in ibd_async_rc_close_act_chan()
144 mutex_exit(&state->id_ac_mutex); in ibd_async_rc_close_act_chan()
146 &state->rc_obs_act_chan_list); in ibd_async_rc_close_act_chan()
151 ibd_async_rc_recycle_ace(ibd_state_t *state, ibd_req_t *req) in ibd_async_rc_recycle_ace() argument
161 mutex_enter(&state->id_ac_mutex); in ibd_async_rc_recycle_ace()
166 IBD_ACACHE_INSERT_FREE(state, ace); in ibd_async_rc_recycle_ace()
170 state->rc_delay_ace_recycle++; in ibd_async_rc_recycle_ace()
172 mutex_exit(&state->id_ac_mutex); in ibd_async_rc_recycle_ace()
173 mutex_enter(&state->rc_ace_recycle_lock); in ibd_async_rc_recycle_ace()
174 state->rc_ace_recycle = NULL; in ibd_async_rc_recycle_ace()
175 mutex_exit(&state->rc_ace_recycle_lock); in ibd_async_rc_recycle_ace()
185 ibd_async_rc_process_too_big(ibd_state_t *state, ibd_req_t *req) in ibd_async_rc_process_too_big() argument
189 uint16_t mtu = state->id_mtu - IPOIB_HDRSIZE; in ibd_async_rc_process_too_big()
240 bcopy(&state->id_macaddr, &ibha->ib_dst, IPOIB_ADDRL); in ibd_async_rc_process_too_big()
257 ibd_print_warn(state, "ibd_rc_process_too_big_pkt: msg_len(%d) " in ibd_async_rc_process_too_big()
283 mac_rx(state->id_mh, state->id_rh, pmtu_mp); in ibd_async_rc_process_too_big()
305 ibd_state_t *state = carg; in ibd_rc_conn_timeout_call() local
311 mutex_enter(&state->id_ac_mutex); in ibd_rc_conn_timeout_call()
312 ace = list_head(&state->id_ah_active); in ibd_rc_conn_timeout_call()
314 ace = list_next(&state->id_ah_active, ace); in ibd_rc_conn_timeout_call()
317 ASSERT(state->id_enable_rc == B_TRUE); in ibd_rc_conn_timeout_call()
320 state->rc_timeout_act++; in ibd_rc_conn_timeout_call()
322 IBD_ACACHE_PULLOUT_ACTIVE(state, in ibd_rc_conn_timeout_call()
326 ibd_rc_signal_act_close(state, pre_ace); in ibd_rc_conn_timeout_call()
333 mutex_exit(&state->id_ac_mutex); in ibd_rc_conn_timeout_call()
336 mutex_enter(&state->rc_pass_chan_list.chan_list_mutex); in ibd_rc_conn_timeout_call()
337 next_chan = state->rc_pass_chan_list.chan_list; in ibd_rc_conn_timeout_call()
342 req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP); in ibd_rc_conn_timeout_call()
345 state->rc_timeout_pas++; in ibd_rc_conn_timeout_call()
347 ibd_queue_work_slot(state, req, in ibd_rc_conn_timeout_call()
350 ibd_print_warn(state, "ibd_rc_conn_timeout: " in ibd_rc_conn_timeout_call()
353 state->rc_pass_chan_list.chan_list = in ibd_rc_conn_timeout_call()
362 state->rc_pass_chan_list.chan_list = chan; in ibd_rc_conn_timeout_call()
373 state->rc_pass_chan_list.chan_list = NULL; in ibd_rc_conn_timeout_call()
375 mutex_exit(&state->rc_pass_chan_list.chan_list_mutex); in ibd_rc_conn_timeout_call()
377 mutex_enter(&state->rc_timeout_lock); in ibd_rc_conn_timeout_call()
378 if (state->rc_timeout_start == B_TRUE) { in ibd_rc_conn_timeout_call()
379 state->rc_timeout = timeout(ibd_rc_conn_timeout_call, state, in ibd_rc_conn_timeout_call()
382 mutex_exit(&state->rc_timeout_lock); in ibd_rc_conn_timeout_call()
396 ibd_state_t *state; in ibd_rc_update_stats() local
402 state = (ibd_state_t *)ksp->ks_private; in ibd_rc_update_stats()
403 ASSERT(state != NULL); in ibd_rc_update_stats()
406 ibd_rc_ksp->rc_rcv_trans_byte.value.ul = state->rc_rcv_trans_byte; in ibd_rc_update_stats()
407 ibd_rc_ksp->rc_rcv_trans_pkt.value.ul = state->rc_rcv_trans_pkt; in ibd_rc_update_stats()
408 ibd_rc_ksp->rc_rcv_copy_byte.value.ul = state->rc_rcv_copy_byte; in ibd_rc_update_stats()
409 ibd_rc_ksp->rc_rcv_copy_pkt.value.ul = state->rc_rcv_copy_pkt; in ibd_rc_update_stats()
410 ibd_rc_ksp->rc_rcv_alloc_fail.value.ul = state->rc_rcv_alloc_fail; in ibd_rc_update_stats()
412 ibd_rc_ksp->rc_rcq_err.value.ul = state->rc_rcq_err; in ibd_rc_update_stats()
414 ibd_rc_ksp->rc_rwqe_short.value.ul = state->rc_rwqe_short; in ibd_rc_update_stats()
416 ibd_rc_ksp->rc_xmt_bytes.value.ul = state->rc_xmt_bytes; in ibd_rc_update_stats()
417 ibd_rc_ksp->rc_xmt_small_pkt.value.ul = state->rc_xmt_small_pkt; in ibd_rc_update_stats()
419 state->rc_xmt_fragmented_pkt; in ibd_rc_update_stats()
420 ibd_rc_ksp->rc_xmt_map_fail_pkt.value.ul = state->rc_xmt_map_fail_pkt; in ibd_rc_update_stats()
421 ibd_rc_ksp->rc_xmt_map_succ_pkt.value.ul = state->rc_xmt_map_succ_pkt; in ibd_rc_update_stats()
422 ibd_rc_ksp->rc_ace_not_found.value.ul = state->rc_ace_not_found; in ibd_rc_update_stats()
424 ibd_rc_ksp->rc_scq_no_swqe.value.ul = state->rc_scq_no_swqe; in ibd_rc_update_stats()
425 ibd_rc_ksp->rc_scq_no_largebuf.value.ul = state->rc_scq_no_largebuf; in ibd_rc_update_stats()
426 ibd_rc_ksp->rc_swqe_short.value.ul = state->rc_swqe_short; in ibd_rc_update_stats()
427 ibd_rc_ksp->rc_swqe_mac_update.value.ul = state->rc_swqe_mac_update; in ibd_rc_update_stats()
428 ibd_rc_ksp->rc_xmt_buf_short.value.ul = state->rc_xmt_buf_short; in ibd_rc_update_stats()
430 state->rc_xmt_buf_mac_update; in ibd_rc_update_stats()
432 ibd_rc_ksp->rc_conn_succ.value.ul = state->rc_conn_succ; in ibd_rc_update_stats()
433 ibd_rc_ksp->rc_conn_fail.value.ul = state->rc_conn_fail; in ibd_rc_update_stats()
434 ibd_rc_ksp->rc_null_conn.value.ul = state->rc_null_conn; in ibd_rc_update_stats()
435 ibd_rc_ksp->rc_no_estab_conn.value.ul = state->rc_no_estab_conn; in ibd_rc_update_stats()
437 ibd_rc_ksp->rc_act_close.value.ul = state->rc_act_close; in ibd_rc_update_stats()
438 ibd_rc_ksp->rc_pas_close.value.ul = state->rc_pas_close; in ibd_rc_update_stats()
439 ibd_rc_ksp->rc_delay_ace_recycle.value.ul = state->rc_delay_ace_recycle; in ibd_rc_update_stats()
441 state->rc_act_close_simultaneous; in ibd_rc_update_stats()
442 ibd_rc_ksp->rc_reset_cnt.value.ul = state->rc_reset_cnt; in ibd_rc_update_stats()
443 ibd_rc_ksp->rc_timeout_act.value.ul = state->rc_timeout_act; in ibd_rc_update_stats()
444 ibd_rc_ksp->rc_timeout_pas.value.ul = state->rc_timeout_pas; in ibd_rc_update_stats()
457 ibd_rc_init_stats(ibd_state_t *state) in ibd_rc_init_stats() argument
467 inst = ddi_get_instance(state->id_dip); in ibd_rc_init_stats()
469 state->id_pkey, state->id_plinkid); in ibd_rc_init_stats()
474 ibd_print_warn(state, "ibd_rc_init_stats: Could not create " in ibd_rc_init_stats()
479 state->rc_ksp = ksp; /* Fill in the ksp of ibd over RC mode */ in ibd_rc_init_stats()
561 ksp->ks_private = (void *)state; in ibd_rc_init_stats()
573 ibd_rc_alloc_chan(ibd_rc_chan_t **ret_chan, ibd_state_t *state, in ibd_rc_alloc_chan() argument
586 chan->state = state; in ibd_rc_alloc_chan()
596 chan->scq_size = state->id_rc_num_swqe; in ibd_rc_alloc_chan()
600 chan->rcq_size = state->id_rc_num_rwqe; in ibd_rc_alloc_chan()
605 result = ibt_alloc_cq(state->id_hca_hdl, &cq_atts, &chan->scq_hdl, in ibd_rc_alloc_chan()
614 if (ibt_modify_cq(chan->scq_hdl, state->id_rc_tx_comp_count, in ibd_rc_alloc_chan()
615 state->id_rc_tx_comp_usec, 0) != IBT_SUCCESS) { in ibd_rc_alloc_chan()
627 result = ibt_alloc_cq(state->id_hca_hdl, &cq_atts, &chan->rcq_hdl, in ibd_rc_alloc_chan()
630 ibd_print_warn(state, "ibd_rc_alloc_chan: error <%d> creating " in ibd_rc_alloc_chan()
635 if (ibt_modify_cq(chan->rcq_hdl, state->id_rc_rx_comp_count, in ibd_rc_alloc_chan()
636 state->id_rc_rx_comp_usec, 0) != IBT_SUCCESS) { in ibd_rc_alloc_chan()
648 ibd_print_warn(state, "ibd_rc_alloc_chan: " in ibd_rc_alloc_chan()
653 if ((rv = ddi_add_softintr(state->id_dip, in ibd_rc_alloc_chan()
672 ibd_print_warn(state, "ibd_rc_alloc_chan: ibt_enable_cq_notify" in ibd_rc_alloc_chan()
688 alloc_args.rc_pd = state->id_pd_hdl; in ibd_rc_alloc_chan()
690 alloc_args.rc_hca_port_num = state->id_port; in ibd_rc_alloc_chan()
694 alloc_args.rc_sizes.cs_sq_sgl = state->rc_tx_max_sqseg; in ibd_rc_alloc_chan()
707 if (state->id_hca_res_lkey_capab) { in ibd_rc_alloc_chan()
713 if (state->rc_enable_srq) { in ibd_rc_alloc_chan()
715 alloc_args.rc_srq = state->rc_srq_hdl; in ibd_rc_alloc_chan()
720 result = ibt_alloc_rc_channel(state->id_hca_hdl, in ibd_rc_alloc_chan()
723 ibd_print_warn(state, "ibd_rc_alloc_chan: ibd_rc_open_channel" in ibd_rc_alloc_chan()
729 atomic_inc_32(&state->rc_num_tx_chan); in ibd_rc_alloc_chan()
731 atomic_inc_32(&state->rc_num_rx_chan); in ibd_rc_alloc_chan()
807 atomic_dec_32(&chan->state->rc_num_tx_chan); in ibd_rc_free_chan()
809 if (!chan->state->rc_enable_srq) { in ibd_rc_free_chan()
812 atomic_dec_32(&chan->state->rc_num_rx_chan); in ibd_rc_free_chan()
847 ibd_state_t *state = chan->state; in ibd_rc_re_add_to_pas_chan_list() local
849 mutex_enter(&state->rc_pass_chan_list.chan_list_mutex); in ibd_rc_re_add_to_pas_chan_list()
850 if ((state->id_mac_state & IBD_DRV_STARTED) == 0) { in ibd_rc_re_add_to_pas_chan_list()
851 mutex_exit(&state->rc_pass_chan_list.chan_list_mutex); in ibd_rc_re_add_to_pas_chan_list()
854 if (state->rc_pass_chan_list.chan_list == NULL) { in ibd_rc_re_add_to_pas_chan_list()
855 state->rc_pass_chan_list.chan_list = chan; in ibd_rc_re_add_to_pas_chan_list()
858 chan->next = state->rc_pass_chan_list.chan_list; in ibd_rc_re_add_to_pas_chan_list()
859 state->rc_pass_chan_list.chan_list = chan; in ibd_rc_re_add_to_pas_chan_list()
861 mutex_exit(&state->rc_pass_chan_list.chan_list_mutex); in ibd_rc_re_add_to_pas_chan_list()
910 ibd_rc_alloc_srq_copybufs(ibd_state_t *state) in ibd_rc_alloc_srq_copybufs() argument
918 rc_rx_bufs_sz = (state->rc_mtu + IPOIB_GRH_SIZE) * state->rc_srq_size; in ibd_rc_alloc_srq_copybufs()
920 state->rc_srq_rx_bufs = kmem_zalloc(rc_rx_bufs_sz, KM_SLEEP); in ibd_rc_alloc_srq_copybufs()
922 state->rc_srq_rwqes = kmem_zalloc(state->rc_srq_size * in ibd_rc_alloc_srq_copybufs()
928 mem_attr.mr_vaddr = (uint64_t)(uintptr_t)state->rc_srq_rx_bufs; in ibd_rc_alloc_srq_copybufs()
932 if (ibt_register_mr(state->id_hca_hdl, state->id_pd_hdl, &mem_attr, in ibd_rc_alloc_srq_copybufs()
933 &state->rc_srq_rx_mr_hdl, &state->rc_srq_rx_mr_desc) in ibd_rc_alloc_srq_copybufs()
937 kmem_free(state->rc_srq_rwqes, in ibd_rc_alloc_srq_copybufs()
938 state->rc_srq_size * sizeof (ibd_rwqe_t)); in ibd_rc_alloc_srq_copybufs()
939 kmem_free(state->rc_srq_rx_bufs, rc_rx_bufs_sz); in ibd_rc_alloc_srq_copybufs()
940 state->rc_srq_rx_bufs = NULL; in ibd_rc_alloc_srq_copybufs()
941 state->rc_srq_rwqes = NULL; in ibd_rc_alloc_srq_copybufs()
949 ibd_rc_free_srq_copybufs(ibd_state_t *state) in ibd_rc_free_srq_copybufs() argument
957 rc_rx_buf_sz = state->rc_mtu + IPOIB_GRH_SIZE; in ibd_rc_free_srq_copybufs()
962 if (ibt_deregister_mr(state->id_hca_hdl, in ibd_rc_free_srq_copybufs()
963 state->rc_srq_rx_mr_hdl) != IBT_SUCCESS) { in ibd_rc_free_srq_copybufs()
967 state->rc_srq_rx_mr_hdl = NULL; in ibd_rc_free_srq_copybufs()
972 kmem_free(state->rc_srq_rwqes, in ibd_rc_free_srq_copybufs()
973 state->rc_srq_size * sizeof (ibd_rwqe_t)); in ibd_rc_free_srq_copybufs()
974 kmem_free(state->rc_srq_rx_bufs, state->rc_srq_size * rc_rx_buf_sz); in ibd_rc_free_srq_copybufs()
975 state->rc_srq_rwqes = NULL; in ibd_rc_free_srq_copybufs()
976 state->rc_srq_rx_bufs = NULL; in ibd_rc_free_srq_copybufs()
983 ibd_rc_init_srq_list(ibd_state_t *state) in ibd_rc_init_srq_list() argument
995 srq_sizes.srq_wr_sz = state->id_rc_num_srq; in ibd_rc_init_srq_list()
996 ret = ibt_alloc_srq(state->id_hca_hdl, IBT_SRQ_NO_FLAGS, in ibd_rc_init_srq_list()
997 state->id_pd_hdl, &srq_sizes, &state->rc_srq_hdl, &srq_real_sizes); in ibd_rc_init_srq_list()
1006 len = state->id_rc_num_srq; in ibd_rc_init_srq_list()
1012 ret = ibt_alloc_srq(state->id_hca_hdl, in ibd_rc_init_srq_list()
1013 IBT_SRQ_NO_FLAGS, state->id_pd_hdl, &srq_sizes, in ibd_rc_init_srq_list()
1014 &state->rc_srq_hdl, &srq_real_sizes); in ibd_rc_init_srq_list()
1023 state->id_rc_num_srq = len; in ibd_rc_init_srq_list()
1024 state->id_rc_num_rwqe = state->id_rc_num_srq + 1; in ibd_rc_init_srq_list()
1027 state->rc_srq_size = srq_real_sizes.srq_wr_sz; in ibd_rc_init_srq_list()
1028 if (ibd_rc_alloc_srq_copybufs(state) != DDI_SUCCESS) { in ibd_rc_init_srq_list()
1029 ret = ibt_free_srq(state->rc_srq_hdl); in ibd_rc_init_srq_list()
1031 ibd_print_warn(state, "ibd_rc_init_srq_list: " in ibd_rc_init_srq_list()
1040 lkey = state->rc_srq_rx_mr_desc.md_lkey; in ibd_rc_init_srq_list()
1041 rwqe = state->rc_srq_rwqes; in ibd_rc_init_srq_list()
1042 bufaddr = state->rc_srq_rx_bufs; in ibd_rc_init_srq_list()
1043 len = state->rc_mtu + IPOIB_GRH_SIZE; in ibd_rc_init_srq_list()
1044 state->rc_srq_rwqe_list.dl_cnt = 0; in ibd_rc_init_srq_list()
1045 state->rc_srq_rwqe_list.dl_bufs_outstanding = 0; in ibd_rc_init_srq_list()
1046 for (i = 0; i < state->rc_srq_size; i++, rwqe++, bufaddr += len) { in ibd_rc_init_srq_list()
1047 rwqe->w_state = state; in ibd_rc_init_srq_list()
1057 if (atomic_dec_32_nv(&state->id_running) != 0) { in ibd_rc_init_srq_list()
1061 ibd_rc_fini_srq_list(state); in ibd_rc_init_srq_list()
1062 atomic_inc_32(&state->id_running); in ibd_rc_init_srq_list()
1070 rwqe->rwqe_copybuf.ic_sgl.ds_len = state->rc_mtu; in ibd_rc_init_srq_list()
1074 (void) ibd_rc_post_srq(state, rwqe); in ibd_rc_init_srq_list()
1077 mutex_enter(&state->rc_srq_free_list.dl_mutex); in ibd_rc_init_srq_list()
1078 state->rc_srq_free_list.dl_head = NULL; in ibd_rc_init_srq_list()
1079 state->rc_srq_free_list.dl_cnt = 0; in ibd_rc_init_srq_list()
1080 mutex_exit(&state->rc_srq_free_list.dl_mutex); in ibd_rc_init_srq_list()
1089 ibd_rc_fini_srq_list(ibd_state_t *state) in ibd_rc_fini_srq_list() argument
1095 ASSERT(state->id_running == 0); in ibd_rc_fini_srq_list()
1096 ret = ibt_free_srq(state->rc_srq_hdl); in ibd_rc_fini_srq_list()
1098 ibd_print_warn(state, "ibd_rc_fini_srq_list: " in ibd_rc_fini_srq_list()
1102 mutex_enter(&state->rc_srq_rwqe_list.dl_mutex); in ibd_rc_fini_srq_list()
1103 rwqe = state->rc_srq_rwqes; in ibd_rc_fini_srq_list()
1104 for (i = 0; i < state->rc_srq_size; i++, rwqe++) { in ibd_rc_fini_srq_list()
1110 mutex_exit(&state->rc_srq_rwqe_list.dl_mutex); in ibd_rc_fini_srq_list()
1112 ibd_rc_free_srq_copybufs(state); in ibd_rc_fini_srq_list()
1117 ibd_rc_repost_srq_free_list(ibd_state_t *state) in ibd_rc_repost_srq_free_list() argument
1123 mutex_enter(&state->rc_srq_free_list.dl_mutex); in ibd_rc_repost_srq_free_list()
1124 if (state->rc_srq_free_list.dl_head != NULL) { in ibd_rc_repost_srq_free_list()
1126 len = state->rc_mtu + IPOIB_GRH_SIZE; in ibd_rc_repost_srq_free_list()
1127 list = state->rc_srq_free_list.dl_head; in ibd_rc_repost_srq_free_list()
1128 state->rc_srq_free_list.dl_head = NULL; in ibd_rc_repost_srq_free_list()
1129 state->rc_srq_free_list.dl_cnt = 0; in ibd_rc_repost_srq_free_list()
1130 mutex_exit(&state->rc_srq_free_list.dl_mutex); in ibd_rc_repost_srq_free_list()
1140 ibd_rc_srq_free_rwqe(state, rwqe); in ibd_rc_repost_srq_free_list()
1146 if (ibd_rc_post_srq(state, rwqe) == DDI_FAILURE) { in ibd_rc_repost_srq_free_list()
1147 ibd_rc_srq_free_rwqe(state, rwqe); in ibd_rc_repost_srq_free_list()
1153 mutex_exit(&state->rc_srq_free_list.dl_mutex); in ibd_rc_repost_srq_free_list()
1161 ibd_rc_srq_free_rwqe(ibd_state_t *state, ibd_rwqe_t *rwqe) in ibd_rc_srq_free_rwqe() argument
1172 mutex_enter(&state->rc_srq_free_list.dl_mutex); in ibd_rc_srq_free_rwqe()
1173 state->rc_srq_free_list.dl_cnt++; in ibd_rc_srq_free_rwqe()
1174 rwqe->rwqe_next = state->rc_srq_free_list.dl_head; in ibd_rc_srq_free_rwqe()
1175 state->rc_srq_free_list.dl_head = RWQE_TO_WQE(rwqe); in ibd_rc_srq_free_rwqe()
1176 mutex_exit(&state->rc_srq_free_list.dl_mutex); in ibd_rc_srq_free_rwqe()
1183 ibd_state_t *state = rwqe->w_state; in ibd_rc_srq_freemsg_cb() local
1185 ASSERT(state->rc_enable_srq); in ibd_rc_srq_freemsg_cb()
1190 if (atomic_add_32_nv(&state->id_running, 0) == 0) { in ibd_rc_srq_freemsg_cb()
1193 &state->rc_srq_rwqe_list.dl_bufs_outstanding); in ibd_rc_srq_freemsg_cb()
1196 ibd_rc_srq_free_rwqe(state, rwqe); in ibd_rc_srq_freemsg_cb()
1201 atomic_dec_32(&state->rc_srq_rwqe_list.dl_bufs_outstanding); in ibd_rc_srq_freemsg_cb()
1203 ASSERT(state->rc_srq_rwqe_list.dl_cnt < state->rc_srq_size); in ibd_rc_srq_freemsg_cb()
1212 state->rc_mtu + IPOIB_GRH_SIZE, 0, &rwqe->w_freemsg_cb); in ibd_rc_srq_freemsg_cb()
1215 ibd_rc_srq_free_rwqe(state, rwqe); in ibd_rc_srq_freemsg_cb()
1219 if (ibd_rc_post_srq(state, rwqe) == DDI_FAILURE) { in ibd_rc_srq_freemsg_cb()
1220 ibd_print_warn(state, "ibd_rc_srq_freemsg_cb: ibd_rc_post_srq" in ibd_rc_srq_freemsg_cb()
1222 ibd_rc_srq_free_rwqe(state, rwqe); in ibd_rc_srq_freemsg_cb()
1231 ibd_rc_post_srq(ibd_state_t *state, ibd_rwqe_t *rwqe) in ibd_rc_post_srq() argument
1238 ASSERT(state->rc_srq_rwqe_list.dl_cnt < state->rc_srq_size); in ibd_rc_post_srq()
1239 atomic_inc_32(&state->rc_srq_rwqe_list.dl_cnt); in ibd_rc_post_srq()
1240 if (ibt_post_srq(state->rc_srq_hdl, &rwqe->w_rwr, 1, NULL) != in ibd_rc_post_srq()
1242 atomic_dec_32(&state->rc_srq_rwqe_list.dl_cnt); in ibd_rc_post_srq()
1274 ibd_state_t *state = chan->state; in ibd_rc_alloc_rx_copybufs() local
1281 rc_rx_bufs_sz = (state->rc_mtu + IPOIB_GRH_SIZE) * chan->rcq_size; in ibd_rc_alloc_rx_copybufs()
1295 if (ibt_register_mr(state->id_hca_hdl, state->id_pd_hdl, &mem_attr, in ibd_rc_alloc_rx_copybufs()
1311 ibd_state_t *state = chan->state; in ibd_rc_free_rx_copybufs() local
1314 ASSERT(!state->rc_enable_srq); in ibd_rc_free_rx_copybufs()
1322 rc_rx_buf_sz = state->rc_mtu + IPOIB_GRH_SIZE; in ibd_rc_free_rx_copybufs()
1327 if (ibt_deregister_mr(state->id_hca_hdl, in ibd_rc_free_rx_copybufs()
1349 ibd_state_t *state = chan->state; in ibd_rc_init_rxlist() local
1356 ASSERT(!state->rc_enable_srq); in ibd_rc_init_rxlist()
1366 len = state->rc_mtu + IPOIB_GRH_SIZE; in ibd_rc_init_rxlist()
1368 rwqe->w_state = state; in ibd_rc_init_rxlist()
1386 rwqe->rwqe_copybuf.ic_sgl.ds_len = state->rc_mtu; in ibd_rc_init_rxlist()
1456 ibd_state_t *state = chan->state; in ibd_rc_process_rx() local
1469 if (state->rc_enable_srq) { in ibd_rc_process_rx()
1470 rxcnt = atomic_dec_32_nv(&state->rc_srq_rwqe_list.dl_cnt); in ibd_rc_process_rx()
1484 if (rxcnt < state->id_rc_rx_rwqe_thresh) { in ibd_rc_process_rx()
1485 state->rc_rwqe_short++; in ibd_rc_process_rx()
1492 if ((rxcnt >= state->id_rc_rx_rwqe_thresh) && in ibd_rc_process_rx()
1493 (wc->wc_bytes_xfer > state->id_rc_rx_copy_thresh)) { in ibd_rc_process_rx()
1494 atomic_add_64(&state->rc_rcv_trans_byte, wc->wc_bytes_xfer); in ibd_rc_process_rx()
1495 atomic_inc_64(&state->rc_rcv_trans_pkt); in ibd_rc_process_rx()
1501 if (state->rc_enable_srq) { in ibd_rc_process_rx()
1503 &state->rc_srq_rwqe_list.dl_bufs_outstanding); in ibd_rc_process_rx()
1509 atomic_add_64(&state->rc_rcv_copy_byte, wc->wc_bytes_xfer); in ibd_rc_process_rx()
1510 atomic_inc_64(&state->rc_rcv_copy_pkt); in ibd_rc_process_rx()
1515 state->rc_rcv_alloc_fail++; in ibd_rc_process_rx()
1516 if (state->rc_enable_srq) { in ibd_rc_process_rx()
1517 if (ibd_rc_post_srq(state, rwqe) == in ibd_rc_process_rx()
1519 ibd_rc_srq_free_rwqe(state, rwqe); in ibd_rc_process_rx()
1533 if (state->rc_enable_srq) { in ibd_rc_process_rx()
1534 if (ibd_rc_post_srq(state, rwqe) == DDI_FAILURE) { in ibd_rc_process_rx()
1535 ibd_rc_srq_free_rwqe(state, rwqe); in ibd_rc_process_rx()
1556 ovbcopy(&state->id_macaddr, &phdr->ib_dst, in ibd_rc_process_rx()
1570 ibd_print_warn(state, in ibd_rc_process_rx()
1579 if (state->rc_enable_srq) { in ibd_rc_process_rx()
1580 mutex_enter(&state->rc_rx_lock); in ibd_rc_process_rx()
1581 if (state->rc_rx_mp) { in ibd_rc_process_rx()
1582 ASSERT(state->rc_rx_mp_tail != NULL); in ibd_rc_process_rx()
1583 state->rc_rx_mp_tail->b_next = mp; in ibd_rc_process_rx()
1585 ASSERT(state->rc_rx_mp_tail == NULL); in ibd_rc_process_rx()
1586 state->rc_rx_mp = mp; in ibd_rc_process_rx()
1589 state->rc_rx_mp_tail = mp; in ibd_rc_process_rx()
1590 state->rc_rx_mp_len++; in ibd_rc_process_rx()
1592 if (state->rc_rx_mp_len >= IBD_MAX_RX_MP_LEN) { in ibd_rc_process_rx()
1593 mpc = state->rc_rx_mp; in ibd_rc_process_rx()
1595 state->rc_rx_mp = NULL; in ibd_rc_process_rx()
1596 state->rc_rx_mp_tail = NULL; in ibd_rc_process_rx()
1597 state->rc_rx_mp_len = 0; in ibd_rc_process_rx()
1598 mutex_exit(&state->rc_rx_lock); in ibd_rc_process_rx()
1599 mac_rx(state->id_mh, NULL, mpc); in ibd_rc_process_rx()
1601 mutex_exit(&state->rc_rx_lock); in ibd_rc_process_rx()
1623 mac_rx(state->id_mh, NULL, mpc); in ibd_rc_process_rx()
1639 ibd_state_t *state = rwqe->w_state; in ibd_rc_freemsg_cb() local
1648 ASSERT(!state->rc_enable_srq); in ibd_rc_freemsg_cb()
1652 state->rc_mtu + IPOIB_GRH_SIZE, 0, &rwqe->w_freemsg_cb); in ibd_rc_freemsg_cb()
1692 chan->state->rc_rcq_err++; in ibd_rc_poll_rcq()
1718 ibd_state_t *state = chan->state; in ibd_rc_rcq_handler() local
1748 if (state->rc_enable_srq) { in ibd_rc_rcq_handler()
1749 mutex_enter(&state->rc_rx_lock); in ibd_rc_rcq_handler()
1751 if (state->rc_rx_mp != NULL) { in ibd_rc_rcq_handler()
1753 mpc = state->rc_rx_mp; in ibd_rc_rcq_handler()
1755 state->rc_rx_mp = NULL; in ibd_rc_rcq_handler()
1756 state->rc_rx_mp_tail = NULL; in ibd_rc_rcq_handler()
1757 state->rc_rx_mp_len = 0; in ibd_rc_rcq_handler()
1759 mutex_exit(&state->rc_rx_lock); in ibd_rc_rcq_handler()
1760 mac_rx(state->id_mh, NULL, mpc); in ibd_rc_rcq_handler()
1762 mutex_exit(&state->rc_rx_lock); in ibd_rc_rcq_handler()
1776 mac_rx(state->id_mh, NULL, mpc); in ibd_rc_rcq_handler()
1788 ibd_rc_init_tx_largebuf_list(ibd_state_t *state) in ibd_rc_init_tx_largebuf_list() argument
1798 num_swqe = state->id_rc_num_swqe - 1; in ibd_rc_init_tx_largebuf_list()
1804 mem_size = num_swqe * state->rc_mtu; in ibd_rc_init_tx_largebuf_list()
1805 state->rc_tx_mr_bufs = kmem_zalloc(mem_size, KM_SLEEP); in ibd_rc_init_tx_largebuf_list()
1808 mem_attr.mr_vaddr = (uint64_t)(uintptr_t)state->rc_tx_mr_bufs; in ibd_rc_init_tx_largebuf_list()
1811 if (ibt_register_mr(state->id_hca_hdl, state->id_pd_hdl, &mem_attr, in ibd_rc_init_tx_largebuf_list()
1812 &state->rc_tx_mr_hdl, &state->rc_tx_mr_desc) != IBT_SUCCESS) { in ibd_rc_init_tx_largebuf_list()
1815 kmem_free(state->rc_tx_mr_bufs, mem_size); in ibd_rc_init_tx_largebuf_list()
1816 state->rc_tx_mr_bufs = NULL; in ibd_rc_init_tx_largebuf_list()
1820 state->rc_tx_largebuf_desc_base = kmem_zalloc(num_swqe * in ibd_rc_init_tx_largebuf_list()
1826 memp = state->rc_tx_mr_bufs; in ibd_rc_init_tx_largebuf_list()
1827 mutex_enter(&state->rc_tx_large_bufs_lock); in ibd_rc_init_tx_largebuf_list()
1828 lbufp = state->rc_tx_largebuf_desc_base; in ibd_rc_init_tx_largebuf_list()
1835 memp += state->rc_mtu; in ibd_rc_init_tx_largebuf_list()
1843 state->rc_tx_largebuf_free_head = state->rc_tx_largebuf_desc_base; in ibd_rc_init_tx_largebuf_list()
1844 state->rc_tx_largebuf_nfree = num_swqe; in ibd_rc_init_tx_largebuf_list()
1845 mutex_exit(&state->rc_tx_large_bufs_lock); in ibd_rc_init_tx_largebuf_list()
1850 ibd_rc_fini_tx_largebuf_list(ibd_state_t *state) in ibd_rc_fini_tx_largebuf_list() argument
1854 num_swqe = state->id_rc_num_swqe - 1; in ibd_rc_fini_tx_largebuf_list()
1856 if (ibt_deregister_mr(state->id_hca_hdl, in ibd_rc_fini_tx_largebuf_list()
1857 state->rc_tx_mr_hdl) != IBT_SUCCESS) { in ibd_rc_fini_tx_largebuf_list()
1861 state->rc_tx_mr_hdl = NULL; in ibd_rc_fini_tx_largebuf_list()
1863 kmem_free(state->rc_tx_mr_bufs, num_swqe * state->rc_mtu); in ibd_rc_fini_tx_largebuf_list()
1864 state->rc_tx_mr_bufs = NULL; in ibd_rc_fini_tx_largebuf_list()
1866 kmem_free(state->rc_tx_largebuf_desc_base, in ibd_rc_fini_tx_largebuf_list()
1868 state->rc_tx_largebuf_desc_base = NULL; in ibd_rc_fini_tx_largebuf_list()
1875 ibd_state_t *state; in ibd_rc_alloc_tx_copybufs() local
1877 state = chan->state; in ibd_rc_alloc_tx_copybufs()
1878 ASSERT(state != NULL); in ibd_rc_alloc_tx_copybufs()
1883 mem_attr.mr_len = chan->scq_size * state->id_rc_tx_copy_thresh; in ibd_rc_alloc_tx_copybufs()
1893 if (ibt_register_mr(state->id_hca_hdl, state->id_pd_hdl, &mem_attr, in ibd_rc_alloc_tx_copybufs()
1897 chan->scq_size * state->id_rc_tx_copy_thresh); in ibd_rc_alloc_tx_copybufs()
1915 ibd_state_t *state = chan->state; in ibd_rc_init_txlist() local
1937 (chan->tx_mr_bufs + i * state->id_rc_tx_copy_thresh); in ibd_rc_init_txlist()
1957 ibd_state_t *state = chan->state; in ibd_rc_fini_txlist() local
1959 if (ibt_deregister_mr(chan->state->id_hca_hdl, in ibd_rc_fini_txlist()
1969 state->id_rc_tx_copy_thresh); in ibd_rc_fini_txlist()
2047 ibd_print_warn(chan->state, "ibd_post_send: " in ibd_rc_post_send()
2093 ibd_print_warn(chan->state, "ibd_post_send: " in ibd_rc_post_send()
2112 ibd_state_t *state; in ibd_rc_tx_cleanup() local
2117 state = ace->ac_chan->state; in ibd_rc_tx_cleanup()
2126 ibd_unmap_mem(state, swqe); in ibd_rc_tx_cleanup()
2140 mutex_enter(&state->rc_tx_large_bufs_lock); in ibd_rc_tx_cleanup()
2141 lbufp->lb_next = state->rc_tx_largebuf_free_head; in ibd_rc_tx_cleanup()
2142 state->rc_tx_largebuf_free_head = lbufp; in ibd_rc_tx_cleanup()
2143 state->rc_tx_largebuf_nfree ++; in ibd_rc_tx_cleanup()
2144 mutex_exit(&state->rc_tx_large_bufs_lock); in ibd_rc_tx_cleanup()
2162 ibd_dec_ref_ace(state, ace); in ibd_rc_tx_cleanup()
2168 ibd_state_t *state = chan->state; in ibd_rc_drain_scq() local
2190 mutex_enter(&state->id_ac_mutex); in ibd_rc_drain_scq()
2193 (chan->state->id_link_state == in ibd_rc_drain_scq()
2195 ((ace = ibd_acache_find(state, in ibd_rc_drain_scq()
2201 state, ace); in ibd_rc_drain_scq()
2204 mutex_exit(&state->id_ac_mutex); in ibd_rc_drain_scq()
2205 state->rc_reset_cnt++; in ibd_rc_drain_scq()
2212 chan->ace, chan->state-> in ibd_rc_drain_scq()
2215 state, ace); in ibd_rc_drain_scq()
2217 mutex_exit(&state->id_ac_mutex); in ibd_rc_drain_scq()
2218 state-> in ibd_rc_drain_scq()
2227 chan->state->id_link_state); in ibd_rc_drain_scq()
2234 mutex_enter(&state->id_sched_lock); in ibd_rc_drain_scq()
2235 if (state->id_sched_needed == 0) { in ibd_rc_drain_scq()
2236 mutex_exit(&state->id_sched_lock); in ibd_rc_drain_scq()
2237 } else if (state->id_sched_needed & IBD_RSRC_RC_SWQE) { in ibd_rc_drain_scq()
2242 state->id_sched_needed &= ~IBD_RSRC_RC_SWQE; in ibd_rc_drain_scq()
2245 mutex_exit(&state->id_sched_lock); in ibd_rc_drain_scq()
2246 state->rc_swqe_mac_update++; in ibd_rc_drain_scq()
2247 mac_tx_update(state->id_mh); in ibd_rc_drain_scq()
2249 state->rc_scq_no_swqe++; in ibd_rc_drain_scq()
2252 mutex_exit(&state->id_sched_lock); in ibd_rc_drain_scq()
2254 } else if (state->id_sched_needed & IBD_RSRC_RC_TX_LARGEBUF) { in ibd_rc_drain_scq()
2255 mutex_enter(&state->rc_tx_large_bufs_lock); in ibd_rc_drain_scq()
2256 if (state->rc_tx_largebuf_nfree > in ibd_rc_drain_scq()
2258 ASSERT(state->rc_tx_largebuf_free_head != NULL); in ibd_rc_drain_scq()
2259 state->id_sched_needed &= in ibd_rc_drain_scq()
2261 mutex_exit(&state->rc_tx_large_bufs_lock); in ibd_rc_drain_scq()
2262 mutex_exit(&state->id_sched_lock); in ibd_rc_drain_scq()
2263 state->rc_xmt_buf_mac_update++; in ibd_rc_drain_scq()
2264 mac_tx_update(state->id_mh); in ibd_rc_drain_scq()
2266 state->rc_scq_no_largebuf++; in ibd_rc_drain_scq()
2267 mutex_exit(&state->rc_tx_large_bufs_lock); in ibd_rc_drain_scq()
2268 mutex_exit(&state->id_sched_lock); in ibd_rc_drain_scq()
2270 } else if (state->id_sched_needed & IBD_RSRC_SWQE) { in ibd_rc_drain_scq()
2271 mutex_enter(&state->id_tx_list.dl_mutex); in ibd_rc_drain_scq()
2272 mutex_enter(&state->id_tx_rel_list.dl_mutex); in ibd_rc_drain_scq()
2273 if ((state->id_tx_list.dl_cnt + in ibd_rc_drain_scq()
2274 state->id_tx_rel_list.dl_cnt) in ibd_rc_drain_scq()
2276 state->id_sched_needed &= ~IBD_RSRC_SWQE; in ibd_rc_drain_scq()
2277 state->id_sched_cnt++; in ibd_rc_drain_scq()
2278 mutex_exit(&state->id_tx_rel_list.dl_mutex); in ibd_rc_drain_scq()
2279 mutex_exit(&state->id_tx_list.dl_mutex); in ibd_rc_drain_scq()
2280 mutex_exit(&state->id_sched_lock); in ibd_rc_drain_scq()
2281 mac_tx_update(state->id_mh); in ibd_rc_drain_scq()
2283 mutex_exit(&state->id_tx_rel_list.dl_mutex); in ibd_rc_drain_scq()
2284 mutex_exit(&state->id_tx_list.dl_mutex); in ibd_rc_drain_scq()
2285 mutex_exit(&state->id_sched_lock); in ibd_rc_drain_scq()
2288 mutex_exit(&state->id_sched_lock); in ibd_rc_drain_scq()
2318 ibd_state_t *state = chan->state; in ibd_rc_tx_recycle() local
2327 ibd_print_warn(state, "ibd_rc_tx_recycle: multiple polling " in ibd_rc_tx_recycle()
2433 ibd_rc_listen(ibd_state_t *state) in ibd_rc_listen() argument
2440 if (state->rc_listen_hdl != NULL) { in ibd_rc_listen()
2454 IBD_RC_QPN_TO_SID(state->id_qpnum), in ibd_rc_listen()
2455 1, &state->rc_listen_hdl, &ret_sid); in ibd_rc_listen()
2462 gid = state->id_sgid; in ibd_rc_listen()
2465 status = ibt_bind_service(state->rc_listen_hdl, in ibd_rc_listen()
2466 gid, NULL, state, &state->rc_listen_bind); in ibd_rc_listen()
2470 (void) ibd_deregister_service(state->rc_listen_hdl); in ibd_rc_listen()
2479 ASSERT(state->rc_listen_hdl_OFED_interop == NULL); in ibd_rc_listen()
2490 IBD_RC_QPN_TO_SID_OFED_INTEROP(state->id_qpnum), in ibd_rc_listen()
2491 1, &state->rc_listen_hdl_OFED_interop, &ret_sid); in ibd_rc_listen()
2496 (void) ibt_unbind_service(state->rc_listen_hdl, in ibd_rc_listen()
2497 state->rc_listen_bind); in ibd_rc_listen()
2498 (void) ibd_deregister_service(state->rc_listen_hdl); in ibd_rc_listen()
2502 gid = state->id_sgid; in ibd_rc_listen()
2505 status = ibt_bind_service(state->rc_listen_hdl_OFED_interop, in ibd_rc_listen()
2506 gid, NULL, state, &state->rc_listen_bind_OFED_interop); in ibd_rc_listen()
2511 state->rc_listen_hdl_OFED_interop); in ibd_rc_listen()
2512 (void) ibt_unbind_service(state->rc_listen_hdl, in ibd_rc_listen()
2513 state->rc_listen_bind); in ibd_rc_listen()
2514 (void) ibd_deregister_service(state->rc_listen_hdl); in ibd_rc_listen()
2522 ibd_rc_stop_listen(ibd_state_t *state) in ibd_rc_stop_listen() argument
2527 if (state->rc_listen_hdl != NULL) { in ibd_rc_stop_listen()
2528 ret = ibt_unbind_all_services(state->rc_listen_hdl); in ibd_rc_stop_listen()
2533 ret = ibd_deregister_service(state->rc_listen_hdl); in ibd_rc_stop_listen()
2538 state->rc_listen_hdl = NULL; in ibd_rc_stop_listen()
2543 if (state->rc_listen_hdl_OFED_interop != NULL) { in ibd_rc_stop_listen()
2545 state->rc_listen_hdl_OFED_interop); in ibd_rc_stop_listen()
2550 ret = ibd_deregister_service(state->rc_listen_hdl_OFED_interop); in ibd_rc_stop_listen()
2555 state->rc_listen_hdl_OFED_interop = NULL; in ibd_rc_stop_listen()
2561 ibd_rc_close_all_chan(ibd_state_t *state) in ibd_rc_close_all_chan() argument
2568 mutex_enter(&state->rc_pass_chan_list.chan_list_mutex); in ibd_rc_close_all_chan()
2569 rc_chan = state->rc_pass_chan_list.chan_list; in ibd_rc_close_all_chan()
2574 mutex_exit(&state->rc_pass_chan_list.chan_list_mutex); in ibd_rc_close_all_chan()
2576 if (state->rc_enable_srq) { in ibd_rc_close_all_chan()
2578 while (state->rc_srq_rwqe_list.dl_bufs_outstanding > 0) { in ibd_rc_close_all_chan()
2596 rc_chan = ibd_rc_rm_header_chan_list(&state->rc_pass_chan_list); in ibd_rc_close_all_chan()
2599 rc_chan = ibd_rc_rm_header_chan_list(&state->rc_pass_chan_list); in ibd_rc_close_all_chan()
2603 mutex_enter(&state->id_ac_mutex); in ibd_rc_close_all_chan()
2604 state->id_ac_hot_ace = NULL; in ibd_rc_close_all_chan()
2605 ace = list_head(&state->id_ah_active); in ibd_rc_close_all_chan()
2607 ace = list_next(&state->id_ah_active, ace); in ibd_rc_close_all_chan()
2610 IBD_ACACHE_PULLOUT_ACTIVE(state, pre_ace); in ibd_rc_close_all_chan()
2612 ibd_rc_add_to_chan_list(&state->rc_obs_act_chan_list, in ibd_rc_close_all_chan()
2616 mutex_exit(&state->id_ac_mutex); in ibd_rc_close_all_chan()
2618 rc_chan = ibd_rc_rm_header_chan_list(&state->rc_obs_act_chan_list); in ibd_rc_close_all_chan()
2623 mutex_enter(&state->id_ac_mutex); in ibd_rc_close_all_chan()
2628 IBD_ACACHE_INSERT_FREE(state, ace); in ibd_rc_close_all_chan()
2632 state->rc_delay_ace_recycle++; in ibd_rc_close_all_chan()
2634 mutex_exit(&state->id_ac_mutex); in ibd_rc_close_all_chan()
2637 &state->rc_obs_act_chan_list); in ibd_rc_close_all_chan()
2641 while (((state->rc_num_tx_chan != 0) || in ibd_rc_close_all_chan()
2642 (state->rc_num_rx_chan != 0)) && (attempts > 0)) { in ibd_rc_close_all_chan()
2650 ibd_rc_try_connect(ibd_state_t *state, ibd_ace_t *ace, ibt_path_info_t *path) in ibd_rc_try_connect() argument
2654 if ((state->id_mac_state & IBD_DRV_STARTED) == 0) in ibd_rc_try_connect()
2657 status = ibd_rc_connect(state, ace, path, in ibd_rc_try_connect()
2663 if ((state->id_mac_state & IBD_DRV_STARTED) == 0) in ibd_rc_try_connect()
2665 status = ibd_rc_connect(state, ace, path, in ibd_rc_try_connect()
2672 if ((state->id_mac_state & IBD_DRV_STARTED) == 0) in ibd_rc_try_connect()
2674 (void) ibd_rc_connect(state, ace, path, in ibd_rc_try_connect()
2684 ibd_rc_connect(ibd_state_t *state, ibd_ace_t *ace, ibt_path_info_t *path, in ibd_rc_connect() argument
2699 if ((status = ibd_rc_alloc_chan(&chan, state, B_TRUE)) != IBT_SUCCESS) { in ibd_rc_connect()
2705 chan->state = state; in ibd_rc_connect()
2748 hello_req_msg.reserved_qpn = htonl(state->id_qpnum); in ibd_rc_connect()
2749 hello_req_msg.rx_mtu = htonl(state->rc_mtu); in ibd_rc_connect()
2763 state->rc_conn_succ++; in ibd_rc_connect()
2784 ibd_rc_signal_act_close(ibd_state_t *state, ibd_ace_t *ace) in ibd_rc_signal_act_close() argument
2788 req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP); in ibd_rc_signal_act_close()
2790 ibd_print_warn(state, "ibd_rc_signal_act_close: alloc " in ibd_rc_signal_act_close()
2792 mutex_enter(&state->rc_obs_act_chan_list.chan_list_mutex); in ibd_rc_signal_act_close()
2793 ace->ac_chan->next = state->rc_obs_act_chan_list.chan_list; in ibd_rc_signal_act_close()
2794 state->rc_obs_act_chan_list.chan_list = ace->ac_chan; in ibd_rc_signal_act_close()
2795 mutex_exit(&state->rc_obs_act_chan_list.chan_list_mutex); in ibd_rc_signal_act_close()
2798 ibd_queue_work_slot(state, req, IBD_ASYNC_RC_CLOSE_ACT_CHAN); in ibd_rc_signal_act_close()
2803 ibd_rc_signal_ace_recycle(ibd_state_t *state, ibd_ace_t *ace) in ibd_rc_signal_ace_recycle() argument
2807 mutex_enter(&state->rc_ace_recycle_lock); in ibd_rc_signal_ace_recycle()
2808 if (state->rc_ace_recycle != NULL) { in ibd_rc_signal_ace_recycle()
2809 mutex_exit(&state->rc_ace_recycle_lock); in ibd_rc_signal_ace_recycle()
2813 req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP); in ibd_rc_signal_ace_recycle()
2815 mutex_exit(&state->rc_ace_recycle_lock); in ibd_rc_signal_ace_recycle()
2819 state->rc_ace_recycle = ace; in ibd_rc_signal_ace_recycle()
2820 mutex_exit(&state->rc_ace_recycle_lock); in ibd_rc_signal_ace_recycle()
2823 IBD_ACACHE_PULLOUT_ACTIVE(state, ace); in ibd_rc_signal_ace_recycle()
2825 ibd_queue_work_slot(state, req, IBD_ASYNC_RC_RECYCLE_ACE); in ibd_rc_signal_ace_recycle()
2836 ibd_state_t *state; in ibd_rc_act_close() local
2843 chan->state->rc_act_close++; in ibd_rc_act_close()
2861 state = chan->state; in ibd_rc_act_close()
2862 ASSERT(state != NULL); in ibd_rc_act_close()
2863 mutex_enter(&state->id_ac_mutex); in ibd_rc_act_close()
2871 mutex_exit(&state->id_ac_mutex); in ibd_rc_act_close()
2874 state->rc_act_close_not_clean++; in ibd_rc_act_close()
2897 mutex_enter(&state->id_ac_mutex); in ibd_rc_act_close()
2904 mutex_exit(&state->id_ac_mutex); in ibd_rc_act_close()
2961 chan->state->rc_pas_close++; in ibd_rc_pas_close()
2967 ((!chan->state->rc_enable_srq) && in ibd_rc_pas_close()
2983 if (!chan->state->rc_enable_srq) { in ibd_rc_pas_close()
3004 chan->state->rc_pas_close_rcq_invoking++; in ibd_rc_pas_close()
3052 ibd_state_t *state = (ibd_state_t *)arg; in ibd_rc_handle_req() local
3055 if (ibd_rc_alloc_chan(&chan, state, B_FALSE) != IBT_SUCCESS) { in ibd_rc_handle_req()
3060 ibd_rc_add_to_chan_list(&state->rc_pass_chan_list, chan); in ibd_rc_handle_req()
3064 if (!state->rc_enable_srq) { in ibd_rc_handle_req()
3087 hello_msg->reserved_qpn = htonl(state->id_qpnum); in ibd_rc_handle_req()
3088 hello_msg->rx_mtu = htonl(state->rc_mtu); in ibd_rc_handle_req()
3164 ibd_state_t *state; in ibd_rc_dispatch_actv_mad() local
3193 state = rc_chan->state; in ibd_rc_dispatch_actv_mad()
3194 mutex_enter(&state->id_ac_mutex); in ibd_rc_dispatch_actv_mad()
3196 ((ace = ibd_acache_find(state, &ace->ac_mac, B_FALSE, 0)) in ibd_rc_dispatch_actv_mad()
3201 IBD_ACACHE_PULLOUT_ACTIVE(state, ace); in ibd_rc_dispatch_actv_mad()
3202 mutex_exit(&state->id_ac_mutex); in ibd_rc_dispatch_actv_mad()
3208 mutex_exit(&state->id_ac_mutex); in ibd_rc_dispatch_actv_mad()
3209 state->rc_act_close_simultaneous++; in ibd_rc_dispatch_actv_mad()
3216 mutex_enter(&state->id_ac_mutex); in ibd_rc_dispatch_actv_mad()
3221 IBD_ACACHE_INSERT_FREE(state, ace); in ibd_rc_dispatch_actv_mad()
3225 state->rc_delay_ace_recycle++; in ibd_rc_dispatch_actv_mad()
3227 mutex_exit(&state->id_ac_mutex); in ibd_rc_dispatch_actv_mad()
3301 chan = ibd_rc_rm_from_chan_list(&chan->state->rc_pass_chan_list, in ibd_rc_dispatch_pass_mad()
3312 chan = ibd_rc_rm_from_chan_list(&chan->state->rc_pass_chan_list, in ibd_rc_dispatch_pass_mad()