1*17a2b317SBill Taylor /*
2*17a2b317SBill Taylor  * CDDL HEADER START
3*17a2b317SBill Taylor  *
4*17a2b317SBill Taylor  * The contents of this file are subject to the terms of the
5*17a2b317SBill Taylor  * Common Development and Distribution License (the "License").
6*17a2b317SBill Taylor  * You may not use this file except in compliance with the License.
7*17a2b317SBill Taylor  *
8*17a2b317SBill Taylor  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9*17a2b317SBill Taylor  * or http://www.opensolaris.org/os/licensing.
10*17a2b317SBill Taylor  * See the License for the specific language governing permissions
11*17a2b317SBill Taylor  * and limitations under the License.
12*17a2b317SBill Taylor  *
13*17a2b317SBill Taylor  * When distributing Covered Code, include this CDDL HEADER in each
14*17a2b317SBill Taylor  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15*17a2b317SBill Taylor  * If applicable, add the following below this CDDL HEADER, with the
16*17a2b317SBill Taylor  * fields enclosed by brackets "[]" replaced with your own identifying
17*17a2b317SBill Taylor  * information: Portions Copyright [yyyy] [name of copyright owner]
18*17a2b317SBill Taylor  *
19*17a2b317SBill Taylor  * CDDL HEADER END
20*17a2b317SBill Taylor  */
21*17a2b317SBill Taylor 
22*17a2b317SBill Taylor /*
23*17a2b317SBill Taylor  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24*17a2b317SBill Taylor  */
25*17a2b317SBill Taylor 
26*17a2b317SBill Taylor /*
27*17a2b317SBill Taylor  * hermon_fcoib.c
28*17a2b317SBill Taylor  *    Hermon Fibre Channel over IB routines
29*17a2b317SBill Taylor  *
30*17a2b317SBill Taylor  *    Implements all the routines necessary for setting up, using, and
31*17a2b317SBill Taylor  *    (later) tearing down all the FCoIB state.
32*17a2b317SBill Taylor  */
33*17a2b317SBill Taylor 
34*17a2b317SBill Taylor #include <sys/ib/adapters/hermon/hermon.h>
35*17a2b317SBill Taylor 
36*17a2b317SBill Taylor /*
37*17a2b317SBill Taylor  * hermon_fcoib_enable()
38*17a2b317SBill Taylor  *    Context: user or kernel context
39*17a2b317SBill Taylor  */
40*17a2b317SBill Taylor static int
hermon_fcoib_enable(hermon_state_t * state,int port)41*17a2b317SBill Taylor hermon_fcoib_enable(hermon_state_t *state, int port)
42*17a2b317SBill Taylor {
43*17a2b317SBill Taylor 	hermon_fcoib_t	*fcoib;
44*17a2b317SBill Taylor 	hermon_hw_config_fc_basic_t config_fc_basic;
45*17a2b317SBill Taylor 	int		status;
46*17a2b317SBill Taylor 
47*17a2b317SBill Taylor 	port--;		/* passed in as 1 or 2, used as 0 or 1 */
48*17a2b317SBill Taylor 	ASSERT(port >= 0 && port < HERMON_MAX_PORTS);
49*17a2b317SBill Taylor 	fcoib = &state->hs_fcoib;
50*17a2b317SBill Taylor 
51*17a2b317SBill Taylor 	/* Configure FCoIB on the port */
52*17a2b317SBill Taylor 	bzero(&config_fc_basic, sizeof (config_fc_basic));
53*17a2b317SBill Taylor 	config_fc_basic.fexch_base_hi = fcoib->hfc_fexch_base[port] >> 16;
54*17a2b317SBill Taylor 	config_fc_basic.fx_base_mpt_hi = fcoib->hfc_mpt_base[port] >> 17;
55*17a2b317SBill Taylor 	config_fc_basic.fx_base_mpt_lo = 0;
56*17a2b317SBill Taylor 	config_fc_basic.log2_num_rfci =
57*17a2b317SBill Taylor 	    state->hs_ibtfinfo.hca_attr->hca_rfci_max_log2_qp;
58*17a2b317SBill Taylor 	config_fc_basic.rfci_base = fcoib->hfc_rfci_qps_per_port * port +
59*17a2b317SBill Taylor 	    fcoib->hfc_rfci_rsrc->hr_indx;
60*17a2b317SBill Taylor #if 1
61*17a2b317SBill Taylor 	status = hermon_config_fc_cmd_post(state, &config_fc_basic, 1,
62*17a2b317SBill Taylor 	    HERMON_HW_FC_CONF_BASIC, 0, port + 1, HERMON_CMD_NOSLEEP_SPIN);
63*17a2b317SBill Taylor #else
64*17a2b317SBill Taylor 	status = hermon_config_fc_cmd_post(state, &config_fc_basic, 1,
65*17a2b317SBill Taylor 	    HERMON_HW_FC_CONF_BASIC, 0, 0, HERMON_CMD_NOSLEEP_SPIN);
66*17a2b317SBill Taylor #endif
67*17a2b317SBill Taylor 	if (status != HERMON_CMD_SUCCESS) {
68*17a2b317SBill Taylor 		cmn_err(CE_CONT, "fcoib_enable failed: status 0x%x\n", status);
69*17a2b317SBill Taylor 		HERMON_WARNING(state, "fcoib_enable failed");
70*17a2b317SBill Taylor 		return (DDI_FAILURE);
71*17a2b317SBill Taylor 	}
72*17a2b317SBill Taylor 	fcoib->hfc_port_enabled[port] = 1;
73*17a2b317SBill Taylor 	state->hs_fcoib_may_be_running = B_TRUE;
74*17a2b317SBill Taylor 	return (DDI_SUCCESS);
75*17a2b317SBill Taylor }
76*17a2b317SBill Taylor 
77*17a2b317SBill Taylor /*
78*17a2b317SBill Taylor  * hermon_fcoib_set_id()
79*17a2b317SBill Taylor  *    Context: user or kernel context
80*17a2b317SBill Taylor  */
81*17a2b317SBill Taylor int
hermon_fcoib_set_id(hermon_state_t * state,int port,uint32_t rfci_qpn,uint32_t src_id)82*17a2b317SBill Taylor hermon_fcoib_set_id(hermon_state_t *state, int port, uint32_t rfci_qpn,
83*17a2b317SBill Taylor     uint32_t src_id)
84*17a2b317SBill Taylor {
85*17a2b317SBill Taylor 	hermon_fcoib_t	*fcoib;
86*17a2b317SBill Taylor 	int		status;
87*17a2b317SBill Taylor 	int		offset;
88*17a2b317SBill Taylor 	uint32_t	*n_port_ids;
89*17a2b317SBill Taylor 
90*17a2b317SBill Taylor 	port--;		/* passed in as 1 or 2, used as 0 or 1 */
91*17a2b317SBill Taylor 	ASSERT(port >= 0 && port < HERMON_MAX_PORTS);
92*17a2b317SBill Taylor 	fcoib = &state->hs_fcoib;
93*17a2b317SBill Taylor 	mutex_enter(&fcoib->hfc_lock);
94*17a2b317SBill Taylor 
95*17a2b317SBill Taylor 	if (fcoib->hfc_port_enabled[port] == 0) {
96*17a2b317SBill Taylor 		if (hermon_fcoib_enable(state, port + 1) != DDI_SUCCESS) {
97*17a2b317SBill Taylor 			mutex_exit(&fcoib->hfc_lock);
98*17a2b317SBill Taylor 			return (DDI_FAILURE);
99*17a2b317SBill Taylor 		}
100*17a2b317SBill Taylor 	}
101*17a2b317SBill Taylor 
102*17a2b317SBill Taylor 	n_port_ids = fcoib->hfc_n_port_ids[port];
103*17a2b317SBill Taylor 	offset = rfci_qpn - fcoib->hfc_rfci_base[port];
104*17a2b317SBill Taylor 	ASSERT(offset >= 0 && offset < fcoib->hfc_rfci_qps_per_port);
105*17a2b317SBill Taylor 	n_port_ids[offset] = src_id;
106*17a2b317SBill Taylor 
107*17a2b317SBill Taylor 	status = hermon_config_fc_cmd_post(state, n_port_ids, 1,
108*17a2b317SBill Taylor 	    HERMON_HW_FC_CONF_NPORT, fcoib->hfc_rfci_qps_per_port,
109*17a2b317SBill Taylor 	    port + 1, HERMON_CMD_NOSLEEP_SPIN);
110*17a2b317SBill Taylor 	if (status != HERMON_CMD_SUCCESS) {
111*17a2b317SBill Taylor 		HERMON_WARNING(state, "fcoib_set_id failed");
112*17a2b317SBill Taylor 		mutex_exit(&fcoib->hfc_lock);
113*17a2b317SBill Taylor 		return (DDI_FAILURE);
114*17a2b317SBill Taylor 	}
115*17a2b317SBill Taylor 	mutex_exit(&fcoib->hfc_lock);
116*17a2b317SBill Taylor 	return (DDI_SUCCESS);
117*17a2b317SBill Taylor }
118*17a2b317SBill Taylor 
119*17a2b317SBill Taylor /*
120*17a2b317SBill Taylor  * hermon_fcoib_get_id_idx()
121*17a2b317SBill Taylor  *    Context: user or kernel context
122*17a2b317SBill Taylor  */
123*17a2b317SBill Taylor int
hermon_fcoib_get_id_idx(hermon_state_t * state,int port,ibt_fc_attr_t * fcp)124*17a2b317SBill Taylor hermon_fcoib_get_id_idx(hermon_state_t *state, int port, ibt_fc_attr_t *fcp)
125*17a2b317SBill Taylor {
126*17a2b317SBill Taylor 	hermon_fcoib_t	*fcoib;
127*17a2b317SBill Taylor 	int		idx;
128*17a2b317SBill Taylor 
129*17a2b317SBill Taylor 	port--;		/* passed in as 1 or 2, used as 0 or 1 */
130*17a2b317SBill Taylor 	ASSERT(port >= 0 && port < HERMON_MAX_PORTS);
131*17a2b317SBill Taylor 	fcoib = &state->hs_fcoib;
132*17a2b317SBill Taylor 
133*17a2b317SBill Taylor 	idx = fcp->fc_rfci_qpn - fcoib->hfc_rfci_base[port];
134*17a2b317SBill Taylor 	if (idx < 0 || idx >= fcoib->hfc_rfci_qps_per_port)
135*17a2b317SBill Taylor 		idx = -1;
136*17a2b317SBill Taylor 
137*17a2b317SBill Taylor 	return (idx);
138*17a2b317SBill Taylor }
139*17a2b317SBill Taylor 
140*17a2b317SBill Taylor /*
141*17a2b317SBill Taylor  * hermon_fcoib_get_exch_base()
142*17a2b317SBill Taylor  *    Context: user or kernel context
143*17a2b317SBill Taylor  */
144*17a2b317SBill Taylor int
hermon_fcoib_check_exch_base_off(hermon_state_t * state,int port,ibt_fc_attr_t * fcp)145*17a2b317SBill Taylor hermon_fcoib_check_exch_base_off(hermon_state_t *state, int port,
146*17a2b317SBill Taylor     ibt_fc_attr_t *fcp)
147*17a2b317SBill Taylor {
148*17a2b317SBill Taylor 	hermon_fcoib_t	*fcoib;
149*17a2b317SBill Taylor 	int		exch_base_off;
150*17a2b317SBill Taylor 
151*17a2b317SBill Taylor 	port--;		/* passed in as 1 or 2, used as 0 or 1 */
152*17a2b317SBill Taylor 	ASSERT(port >= 0 && port < HERMON_MAX_PORTS);
153*17a2b317SBill Taylor 	fcoib = &state->hs_fcoib;
154*17a2b317SBill Taylor 
155*17a2b317SBill Taylor 	exch_base_off = fcp->fc_exch_base_off;
156*17a2b317SBill Taylor 	if (exch_base_off >= fcoib->hfc_fexch_qps_per_port)
157*17a2b317SBill Taylor 		exch_base_off = -1;
158*17a2b317SBill Taylor 
159*17a2b317SBill Taylor 	return (exch_base_off);
160*17a2b317SBill Taylor }
161*17a2b317SBill Taylor 
162*17a2b317SBill Taylor /*
163*17a2b317SBill Taylor  * hermon_fcoib_qpnum_from_fexch()
164*17a2b317SBill Taylor  *    Context: user, kernel, or interrupt context
165*17a2b317SBill Taylor  */
166*17a2b317SBill Taylor int
hermon_fcoib_is_fexch_qpn(hermon_state_t * state,uint_t qpnum)167*17a2b317SBill Taylor hermon_fcoib_is_fexch_qpn(hermon_state_t *state, uint_t qpnum)
168*17a2b317SBill Taylor {
169*17a2b317SBill Taylor 	hermon_fcoib_t	*fcoib;
170*17a2b317SBill Taylor 
171*17a2b317SBill Taylor 	fcoib = &state->hs_fcoib;
172*17a2b317SBill Taylor 	qpnum -= fcoib->hfc_fexch_rsrc->hr_indx;
173*17a2b317SBill Taylor 	return (qpnum < fcoib->hfc_nports * fcoib->hfc_fexch_qps_per_port);
174*17a2b317SBill Taylor }
175*17a2b317SBill Taylor 
176*17a2b317SBill Taylor /*
177*17a2b317SBill Taylor  * hermon_fcoib_qpnum_from_fexch()
178*17a2b317SBill Taylor  *    Context: user, kernel, or interrupt context
179*17a2b317SBill Taylor  */
180*17a2b317SBill Taylor uint_t
hermon_fcoib_qpnum_from_fexch(hermon_state_t * state,int port,uint16_t fexch)181*17a2b317SBill Taylor hermon_fcoib_qpnum_from_fexch(hermon_state_t *state, int port,
182*17a2b317SBill Taylor     uint16_t fexch)
183*17a2b317SBill Taylor {
184*17a2b317SBill Taylor 	hermon_fcoib_t	*fcoib;
185*17a2b317SBill Taylor 	uint_t		qpnum;
186*17a2b317SBill Taylor 
187*17a2b317SBill Taylor 	port--;		/* passed in as 1 or 2, used as 0 or 1 */
188*17a2b317SBill Taylor 	ASSERT(port >= 0 && port < HERMON_MAX_PORTS);
189*17a2b317SBill Taylor 	fcoib = &state->hs_fcoib;
190*17a2b317SBill Taylor 	qpnum = fexch + fcoib->hfc_fexch_base[port];
191*17a2b317SBill Taylor 	return (qpnum);
192*17a2b317SBill Taylor }
193*17a2b317SBill Taylor 
194*17a2b317SBill Taylor /*
195*17a2b317SBill Taylor  * hermon_fcoib_qpn_to_mkey
196*17a2b317SBill Taylor  *    Context: user or kernel context
197*17a2b317SBill Taylor  */
198*17a2b317SBill Taylor uint32_t
hermon_fcoib_qpn_to_mkey(hermon_state_t * state,uint_t qpnum)199*17a2b317SBill Taylor hermon_fcoib_qpn_to_mkey(hermon_state_t *state, uint_t qpnum)
200*17a2b317SBill Taylor {
201*17a2b317SBill Taylor 	int		i;
202*17a2b317SBill Taylor 	hermon_fcoib_t	*fcoib;
203*17a2b317SBill Taylor 	uint32_t	qp_indx;
204*17a2b317SBill Taylor 
205*17a2b317SBill Taylor 	fcoib = &state->hs_fcoib;
206*17a2b317SBill Taylor 	for (i = 0; i < fcoib->hfc_nports; i++) {
207*17a2b317SBill Taylor 		qp_indx = qpnum - fcoib->hfc_fexch_base[i];
208*17a2b317SBill Taylor 		if (qp_indx < fcoib->hfc_fexch_qps_per_port)
209*17a2b317SBill Taylor 			return ((qp_indx + fcoib->hfc_mpt_base[i]) << 8);
210*17a2b317SBill Taylor 	}
211*17a2b317SBill Taylor 	return ((uint32_t)-1);	/* cannot get here with valid qpnum argument */
212*17a2b317SBill Taylor }
213*17a2b317SBill Taylor 
214*17a2b317SBill Taylor /*
215*17a2b317SBill Taylor  * hermon_fcoib_fexch_relative_qpn()
216*17a2b317SBill Taylor  *    Context: user or kernel context
217*17a2b317SBill Taylor  */
218*17a2b317SBill Taylor uint32_t
hermon_fcoib_fexch_relative_qpn(hermon_state_t * state,uint8_t port,uint32_t qp_indx)219*17a2b317SBill Taylor hermon_fcoib_fexch_relative_qpn(hermon_state_t *state, uint8_t port,
220*17a2b317SBill Taylor     uint32_t qp_indx)
221*17a2b317SBill Taylor {
222*17a2b317SBill Taylor 	port--;
223*17a2b317SBill Taylor 	ASSERT(port < HERMON_MAX_PORTS);
224*17a2b317SBill Taylor 	qp_indx -= state->hs_fcoib.hfc_fexch_base[port];
225*17a2b317SBill Taylor 	return (qp_indx);
226*17a2b317SBill Taylor }
227*17a2b317SBill Taylor 
228*17a2b317SBill Taylor /*
229*17a2b317SBill Taylor  * hermon_fcoib_fexch_mkey_init()
230*17a2b317SBill Taylor  *    Context: user or kernel context
231*17a2b317SBill Taylor  */
232*17a2b317SBill Taylor int
hermon_fcoib_fexch_mkey_init(hermon_state_t * state,hermon_pdhdl_t pd,uint8_t port,uint32_t qp_indx,uint_t sleep)233*17a2b317SBill Taylor hermon_fcoib_fexch_mkey_init(hermon_state_t *state, hermon_pdhdl_t pd,
234*17a2b317SBill Taylor     uint8_t port, uint32_t qp_indx, uint_t sleep)
235*17a2b317SBill Taylor {
236*17a2b317SBill Taylor 	int		status;
237*17a2b317SBill Taylor 	uint32_t	mpt_indx;
238*17a2b317SBill Taylor 	uint_t		nummtt;
239*17a2b317SBill Taylor 	uint64_t	mtt_addr;
240*17a2b317SBill Taylor 	hermon_fcoib_t	*fcoib;
241*17a2b317SBill Taylor 
242*17a2b317SBill Taylor 	port--;
243*17a2b317SBill Taylor 	ASSERT(port < HERMON_MAX_PORTS);
244*17a2b317SBill Taylor 	fcoib = &state->hs_fcoib;
245*17a2b317SBill Taylor 	qp_indx -= fcoib->hfc_fexch_base[port];	/* relative to FEXCH base */
246*17a2b317SBill Taylor 	if (qp_indx > fcoib->hfc_fexch_qps_per_port)
247*17a2b317SBill Taylor 		return (IBT_INVALID_PARAM);
248*17a2b317SBill Taylor 	mpt_indx = qp_indx + fcoib->hfc_mpt_base[port];
249*17a2b317SBill Taylor 	nummtt = fcoib->hfc_mtts_per_mpt;
250*17a2b317SBill Taylor 	mtt_addr = ((uint64_t)qp_indx * nummtt + fcoib->hfc_mtt_base[port]) <<
251*17a2b317SBill Taylor 	    HERMON_MTT_SIZE_SHIFT;
252*17a2b317SBill Taylor 
253*17a2b317SBill Taylor 	status = hermon_mr_fexch_mpt_init(state, pd, mpt_indx,
254*17a2b317SBill Taylor 	    nummtt, mtt_addr, sleep);
255*17a2b317SBill Taylor 	return (status);
256*17a2b317SBill Taylor }
257*17a2b317SBill Taylor 
258*17a2b317SBill Taylor /*
259*17a2b317SBill Taylor  * hermon_fcoib_fexch_mkey_fini()
260*17a2b317SBill Taylor  *    Context: user or kernel context
261*17a2b317SBill Taylor  */
262*17a2b317SBill Taylor int
hermon_fcoib_fexch_mkey_fini(hermon_state_t * state,hermon_pdhdl_t pd,uint32_t qpnum,uint_t sleep)263*17a2b317SBill Taylor hermon_fcoib_fexch_mkey_fini(hermon_state_t *state, hermon_pdhdl_t pd,
264*17a2b317SBill Taylor     uint32_t qpnum, uint_t sleep)
265*17a2b317SBill Taylor {
266*17a2b317SBill Taylor 	int		status;
267*17a2b317SBill Taylor 	uint8_t		port;
268*17a2b317SBill Taylor 	uint32_t	qp_indx;
269*17a2b317SBill Taylor 	uint32_t	mpt_indx;
270*17a2b317SBill Taylor 	hermon_fcoib_t	*fcoib;
271*17a2b317SBill Taylor 
272*17a2b317SBill Taylor 	fcoib = &state->hs_fcoib;
273*17a2b317SBill Taylor 	for (port = 0; port < fcoib->hfc_nports; port++) {
274*17a2b317SBill Taylor 		qp_indx = qpnum - fcoib->hfc_fexch_base[port];
275*17a2b317SBill Taylor 		if (qp_indx < fcoib->hfc_fexch_qps_per_port)
276*17a2b317SBill Taylor 			goto found;
277*17a2b317SBill Taylor 	}
278*17a2b317SBill Taylor 	return (IBT_INVALID_PARAM);
279*17a2b317SBill Taylor found:
280*17a2b317SBill Taylor 	/* qp_indx relative to FEXCH base */
281*17a2b317SBill Taylor 	mpt_indx = qp_indx + fcoib->hfc_mpt_base[port];
282*17a2b317SBill Taylor 
283*17a2b317SBill Taylor 	status = hermon_mr_fexch_mpt_fini(state, pd, mpt_indx, sleep);
284*17a2b317SBill Taylor 	return (status);
285*17a2b317SBill Taylor }
286*17a2b317SBill Taylor 
287*17a2b317SBill Taylor /*
288*17a2b317SBill Taylor  * hermon_fcoib_query_fc()
289*17a2b317SBill Taylor  *    Context: user or kernel context
290*17a2b317SBill Taylor  */
291*17a2b317SBill Taylor void
hermon_fcoib_query_fc(hermon_state_t * state,hermon_fcoib_t * fcoib)292*17a2b317SBill Taylor hermon_fcoib_query_fc(hermon_state_t *state, hermon_fcoib_t *fcoib)
293*17a2b317SBill Taylor {
294*17a2b317SBill Taylor 	int status;
295*17a2b317SBill Taylor 	struct hermon_hw_query_fc_s query_fc;
296*17a2b317SBill Taylor 
297*17a2b317SBill Taylor 	status = hermon_cmn_query_cmd_post(state, QUERY_FC, 0, 0, &query_fc,
298*17a2b317SBill Taylor 	    sizeof (query_fc), HERMON_CMD_NOSLEEP_SPIN);
299*17a2b317SBill Taylor 	if (status == HERMON_CMD_SUCCESS) {
300*17a2b317SBill Taylor 		fcoib->hfc_log2_max_port_ids_queried = query_fc.log2_max_nports;
301*17a2b317SBill Taylor 		fcoib->hfc_log2_max_fexch_queried = query_fc.log2_max_fexch;
302*17a2b317SBill Taylor 		fcoib->hfc_log2_max_rfci_queried = query_fc.log2_max_rfci;
303*17a2b317SBill Taylor 	} else
304*17a2b317SBill Taylor 		cmn_err(CE_CONT, "!query_fc status 0x%x\n", status);
305*17a2b317SBill Taylor }
306*17a2b317SBill Taylor 
307*17a2b317SBill Taylor /*
308*17a2b317SBill Taylor  * hermon_fcoib_init()
309*17a2b317SBill Taylor  *    Context: Only called from attach() path context
310*17a2b317SBill Taylor  */
311*17a2b317SBill Taylor int
hermon_fcoib_init(hermon_state_t * state)312*17a2b317SBill Taylor hermon_fcoib_init(hermon_state_t *state)
313*17a2b317SBill Taylor {
314*17a2b317SBill Taylor 	hermon_fcoib_t	*fcoib;
315*17a2b317SBill Taylor 	uint_t		numports;
316*17a2b317SBill Taylor 	char		string[128];
317*17a2b317SBill Taylor 	int		i;
318*17a2b317SBill Taylor 	uintptr_t	vmemstart = (uintptr_t)0x10000000;
319*17a2b317SBill Taylor 
320*17a2b317SBill Taylor 	/* used for fast checking for FCoIB during cqe_consume */
321*17a2b317SBill Taylor 	state->hs_fcoib_may_be_running = B_FALSE;
322*17a2b317SBill Taylor 
323*17a2b317SBill Taylor 	if ((state->hs_ibtfinfo.hca_attr->hca_flags2 & IBT_HCA2_FC) == 0)
324*17a2b317SBill Taylor 		return (DDI_SUCCESS);
325*17a2b317SBill Taylor 
326*17a2b317SBill Taylor 	fcoib = &state->hs_fcoib;
327*17a2b317SBill Taylor 	bzero(fcoib, sizeof (*fcoib));
328*17a2b317SBill Taylor 
329*17a2b317SBill Taylor 	hermon_fcoib_query_fc(state, fcoib);
330*17a2b317SBill Taylor 
331*17a2b317SBill Taylor 	mutex_init(&fcoib->hfc_lock, NULL, MUTEX_DRIVER, NULL);
332*17a2b317SBill Taylor 	mutex_enter(&fcoib->hfc_lock);
333*17a2b317SBill Taylor 
334*17a2b317SBill Taylor 	/* use a ROUND value that works on both 32 and 64-bit kernels */
335*17a2b317SBill Taylor 	fcoib->hfc_vmemstart = vmemstart;
336*17a2b317SBill Taylor 
337*17a2b317SBill Taylor 	fcoib->hfc_nports = numports = state->hs_cfg_profile->cp_num_ports;
338*17a2b317SBill Taylor 	fcoib->hfc_fexch_qps_per_port =
339*17a2b317SBill Taylor 	    1 << state->hs_ibtfinfo.hca_attr->hca_fexch_max_log2_qp;
340*17a2b317SBill Taylor 	fcoib->hfc_mpts_per_port = fcoib->hfc_fexch_qps_per_port * 2;
341*17a2b317SBill Taylor 	fcoib->hfc_mtts_per_mpt =
342*17a2b317SBill Taylor 	    (1 << state->hs_ibtfinfo.hca_attr->hca_fexch_max_log2_mem) >>
343*17a2b317SBill Taylor 	    PAGESHIFT;
344*17a2b317SBill Taylor 	fcoib->hfc_rfci_qps_per_port =
345*17a2b317SBill Taylor 	    1 << state->hs_ibtfinfo.hca_attr->hca_rfci_max_log2_qp;
346*17a2b317SBill Taylor 
347*17a2b317SBill Taylor 	if (hermon_rsrc_reserve(state, HERMON_DMPT, numports *
348*17a2b317SBill Taylor 	    fcoib->hfc_mpts_per_port, HERMON_SLEEP,
349*17a2b317SBill Taylor 	    &fcoib->hfc_mpt_rsrc) != DDI_SUCCESS) {
350*17a2b317SBill Taylor 		mutex_exit(&fcoib->hfc_lock);
351*17a2b317SBill Taylor 		hermon_fcoib_fini(state);
352*17a2b317SBill Taylor 		return (DDI_FAILURE);
353*17a2b317SBill Taylor 	}
354*17a2b317SBill Taylor 
355*17a2b317SBill Taylor 	/*
356*17a2b317SBill Taylor 	 * Only reserve MTTs for the Primary MPTs (first half of the
357*17a2b317SBill Taylor 	 * range for each port).
358*17a2b317SBill Taylor 	 */
359*17a2b317SBill Taylor 	if (hermon_rsrc_reserve(state, HERMON_MTT, numports *
360*17a2b317SBill Taylor 	    fcoib->hfc_mpts_per_port * fcoib->hfc_mtts_per_mpt / 2,
361*17a2b317SBill Taylor 	    HERMON_SLEEP, &fcoib->hfc_mtt_rsrc) != DDI_SUCCESS) {
362*17a2b317SBill Taylor 		mutex_exit(&fcoib->hfc_lock);
363*17a2b317SBill Taylor 		hermon_fcoib_fini(state);
364*17a2b317SBill Taylor 		return (DDI_FAILURE);
365*17a2b317SBill Taylor 	}
366*17a2b317SBill Taylor 	if (hermon_rsrc_reserve(state, HERMON_QPC, numports *
367*17a2b317SBill Taylor 	    fcoib->hfc_fexch_qps_per_port, HERMON_SLEEP,
368*17a2b317SBill Taylor 	    &fcoib->hfc_fexch_rsrc) != DDI_SUCCESS) {
369*17a2b317SBill Taylor 		mutex_exit(&fcoib->hfc_lock);
370*17a2b317SBill Taylor 		hermon_fcoib_fini(state);
371*17a2b317SBill Taylor 		return (DDI_FAILURE);
372*17a2b317SBill Taylor 	}
373*17a2b317SBill Taylor 	if (hermon_rsrc_reserve(state, HERMON_QPC, numports *
374*17a2b317SBill Taylor 	    fcoib->hfc_rfci_qps_per_port, HERMON_SLEEP,
375*17a2b317SBill Taylor 	    &fcoib->hfc_rfci_rsrc) != DDI_SUCCESS) {
376*17a2b317SBill Taylor 		mutex_exit(&fcoib->hfc_lock);
377*17a2b317SBill Taylor 		hermon_fcoib_fini(state);
378*17a2b317SBill Taylor 		return (DDI_FAILURE);
379*17a2b317SBill Taylor 	}
380*17a2b317SBill Taylor 
381*17a2b317SBill Taylor 	for (i = 0; i < numports; i++) {
382*17a2b317SBill Taylor 		fcoib->hfc_port_enabled[i] = 0;
383*17a2b317SBill Taylor 		fcoib->hfc_n_port_ids[i] = kmem_zalloc(sizeof (uint32_t) *
384*17a2b317SBill Taylor 		    fcoib->hfc_rfci_qps_per_port, KM_SLEEP);
385*17a2b317SBill Taylor 
386*17a2b317SBill Taylor 		fcoib->hfc_mpt_base[i] = i * fcoib->hfc_mpts_per_port +
387*17a2b317SBill Taylor 		    fcoib->hfc_mpt_rsrc->hr_indx;
388*17a2b317SBill Taylor 		/* "/ 2" is for Secondary MKEYs never used on Client side */
389*17a2b317SBill Taylor 		fcoib->hfc_mtt_base[i] = (i * fcoib->hfc_mpts_per_port *
390*17a2b317SBill Taylor 		    fcoib->hfc_mtts_per_mpt / 2) + fcoib->hfc_mtt_rsrc->hr_indx;
391*17a2b317SBill Taylor 		fcoib->hfc_fexch_base[i] = i * fcoib->hfc_fexch_qps_per_port +
392*17a2b317SBill Taylor 		    fcoib->hfc_fexch_rsrc->hr_indx;
393*17a2b317SBill Taylor 		fcoib->hfc_rfci_base[i] = i * fcoib->hfc_rfci_qps_per_port +
394*17a2b317SBill Taylor 		    fcoib->hfc_rfci_rsrc->hr_indx;
395*17a2b317SBill Taylor 
396*17a2b317SBill Taylor 		/* init FEXCH QP rsrc pool */
397*17a2b317SBill Taylor 		(void) sprintf(string, "hermon%d_port%d_fexch_vmem",
398*17a2b317SBill Taylor 		    state->hs_instance, i + 1);
399*17a2b317SBill Taylor 		fcoib->hfc_fexch_vmemp[i] = vmem_create(string,
400*17a2b317SBill Taylor 		    (void *)vmemstart, fcoib->hfc_fexch_qps_per_port,
401*17a2b317SBill Taylor 		    1, NULL, NULL, NULL, 0, VM_SLEEP);
402*17a2b317SBill Taylor 
403*17a2b317SBill Taylor 		/* init RFCI QP rsrc pool */
404*17a2b317SBill Taylor 		(void) sprintf(string, "hermon%d_port%d_rfci_vmem",
405*17a2b317SBill Taylor 		    state->hs_instance, i + 1);
406*17a2b317SBill Taylor 		fcoib->hfc_rfci_vmemp[i] = vmem_create(string,
407*17a2b317SBill Taylor 		    (void *)vmemstart, fcoib->hfc_rfci_qps_per_port,
408*17a2b317SBill Taylor 		    1, NULL, NULL, NULL, 0, VM_SLEEP);
409*17a2b317SBill Taylor 	}
410*17a2b317SBill Taylor 
411*17a2b317SBill Taylor 	mutex_exit(&fcoib->hfc_lock);
412*17a2b317SBill Taylor 
413*17a2b317SBill Taylor 	return (DDI_SUCCESS);
414*17a2b317SBill Taylor }
415*17a2b317SBill Taylor 
416*17a2b317SBill Taylor 
417*17a2b317SBill Taylor /*
418*17a2b317SBill Taylor  * hermon_fcoib_fini()
419*17a2b317SBill Taylor  *    Context: Only called from attach() and/or detach() path contexts
420*17a2b317SBill Taylor  */
421*17a2b317SBill Taylor void
hermon_fcoib_fini(hermon_state_t * state)422*17a2b317SBill Taylor hermon_fcoib_fini(hermon_state_t *state)
423*17a2b317SBill Taylor {
424*17a2b317SBill Taylor 	hermon_fcoib_t	*fcoib;
425*17a2b317SBill Taylor 	uint_t		numports;
426*17a2b317SBill Taylor 	int		i;
427*17a2b317SBill Taylor 
428*17a2b317SBill Taylor 	if ((state->hs_ibtfinfo.hca_attr->hca_flags2 & IBT_HCA2_FC) == 0)
429*17a2b317SBill Taylor 		return;
430*17a2b317SBill Taylor 
431*17a2b317SBill Taylor 	fcoib = &state->hs_fcoib;
432*17a2b317SBill Taylor 
433*17a2b317SBill Taylor 	mutex_enter(&fcoib->hfc_lock);
434*17a2b317SBill Taylor 
435*17a2b317SBill Taylor 	numports = fcoib->hfc_nports;
436*17a2b317SBill Taylor 
437*17a2b317SBill Taylor 	for (i = 0; i < numports; i++) {
438*17a2b317SBill Taylor 		if (fcoib->hfc_rfci_vmemp[i])
439*17a2b317SBill Taylor 			vmem_destroy(fcoib->hfc_rfci_vmemp[i]);
440*17a2b317SBill Taylor 		if (fcoib->hfc_fexch_vmemp[i])
441*17a2b317SBill Taylor 			vmem_destroy(fcoib->hfc_fexch_vmemp[i]);
442*17a2b317SBill Taylor 		if (fcoib->hfc_n_port_ids[i])
443*17a2b317SBill Taylor 			kmem_free(fcoib->hfc_n_port_ids[i], sizeof (uint32_t) *
444*17a2b317SBill Taylor 			    fcoib->hfc_rfci_qps_per_port);
445*17a2b317SBill Taylor 
446*17a2b317SBill Taylor 		/* XXX --- should we issue HERMON_HW_FC_CONF_BASIC disable? */
447*17a2b317SBill Taylor 		fcoib->hfc_port_enabled[i] = 0;
448*17a2b317SBill Taylor 	}
449*17a2b317SBill Taylor 	if (fcoib->hfc_rfci_rsrc)
450*17a2b317SBill Taylor 		hermon_rsrc_free(state, &fcoib->hfc_rfci_rsrc);
451*17a2b317SBill Taylor 	if (fcoib->hfc_fexch_rsrc)
452*17a2b317SBill Taylor 		hermon_rsrc_free(state, &fcoib->hfc_fexch_rsrc);
453*17a2b317SBill Taylor 	if (fcoib->hfc_mpt_rsrc)
454*17a2b317SBill Taylor 		hermon_rsrc_free(state, &fcoib->hfc_mpt_rsrc);
455*17a2b317SBill Taylor 	if (fcoib->hfc_mtt_rsrc)
456*17a2b317SBill Taylor 		hermon_rsrc_free(state, &fcoib->hfc_mtt_rsrc);
457*17a2b317SBill Taylor 
458*17a2b317SBill Taylor 	mutex_exit(&fcoib->hfc_lock);
459*17a2b317SBill Taylor 	mutex_destroy(&fcoib->hfc_lock);
460*17a2b317SBill Taylor 
461*17a2b317SBill Taylor 	bzero(fcoib, sizeof (*fcoib));
462*17a2b317SBill Taylor }