1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*
28  * tavor_rsrc.c
29  *    Tavor Resource Management Routines
30  *
31  *    Implements all the routines necessary for setup, teardown, and
32  *    alloc/free of all Tavor resources, including those that are managed
33  *    by Tavor hardware or which live in Tavor's direct attached DDR memory.
34  */
35 
36 #include <sys/sysmacros.h>
37 #include <sys/types.h>
38 #include <sys/conf.h>
39 #include <sys/ddi.h>
40 #include <sys/sunddi.h>
41 #include <sys/modctl.h>
42 #include <sys/vmem.h>
43 #include <sys/bitmap.h>
44 
45 #include <sys/ib/adapters/tavor/tavor.h>
46 
47 /*
48  * The following routines are used for initializing and destroying
49  * the resource pools used by the Tavor resource allocation routines.
50  * They consist of four classes of object:
51  *
52  * Mailboxes:  The "In" and "Out" mailbox types are used by the Tavor
53  *    command interface routines.  Mailboxes are used to pass information
54  *    back and forth to the Tavor firmware.  Either type of mailbox may
55  *    be allocated from Tavor's direct attached DDR memory or from system
56  *    memory (although currently all "In" mailboxes are in DDR and all "out"
57  *    mailboxes come from system memory.
58  *
59  * HW entry objects:  These objects represent resources required by the Tavor
60  *    hardware.  These objects include things like Queue Pair contexts (QPC),
61  *    Completion Queue contexts (CQC), Event Queue contexts (EQC), RDB (for
62  *    supporting RDMA Read/Atomic), Multicast Group entries (MCG), Memory
63  *    Protection Table entries (MPT), Memory Translation Table entries (MTT).
64  *
65  *    What these objects all have in common is that they are each required
66  *    to come from DDR memory, they are always allocated from tables, and
67  *    they are not to be directly accessed (read or written) by driver
68  *    software.
69  *    One notable exceptions to this rule are the Extended QP contexts (EQPC),
70  *    and the UAR scratch area (UAR_SCR), both of which are not directly
71  *    accessible through the Tavor resource allocation routines, but both
72  *    of which are also required to reside in DDR memory and are not to be
73  *    manipulated by driver software (they are privately managed by Tavor
74  *    hardware).
75  *    The other notable exceptions are the UAR pages (UAR_PG) which are
76  *    allocated from the UAR address space rather than DDR, and the UD
77  *    address vectors (UDAV) which are similar to the common object types
78  *    with the major difference being that UDAVs _are_ directly read and
79  *    written by driver software.
80  *
81  * SW handle objects: These objects represent resources required by Tavor
82  *    driver software.  They are primarily software tracking structures,
83  *    which are allocated from system memory (using kmem_cache).  Several of
84  *    the objects have both a "constructor" and "destructor" method
85  *    associated with them (see below).
86  *
87  * Protection Domain (PD) handle objects:  These objects are very much like
88  *    a SW handle object with the notable difference that all PD handle
89  *    objects have an actual Protection Domain number (PD) associated with
90  *    them (and the PD number is allocated/managed through a separate
91  *    vmem_arena specifically set aside for this purpose.
92  */
93 
94 static int tavor_rsrc_mbox_init(tavor_state_t *state,
95     tavor_rsrc_mbox_info_t *info);
96 static void tavor_rsrc_mbox_fini(tavor_state_t *state,
97     tavor_rsrc_mbox_info_t *info);
98 
99 static int tavor_rsrc_hw_entries_init(tavor_state_t *state,
100     tavor_rsrc_hw_entry_info_t *info);
101 static void tavor_rsrc_hw_entries_fini(tavor_state_t *state,
102     tavor_rsrc_hw_entry_info_t *info);
103 
104 static int tavor_rsrc_sw_handles_init(tavor_state_t *state,
105     tavor_rsrc_sw_hdl_info_t *info);
106 static void tavor_rsrc_sw_handles_fini(tavor_state_t *state,
107     tavor_rsrc_sw_hdl_info_t *info);
108 
109 static int tavor_rsrc_pd_handles_init(tavor_state_t *state,
110     tavor_rsrc_sw_hdl_info_t *info);
111 static void tavor_rsrc_pd_handles_fini(tavor_state_t *state,
112     tavor_rsrc_sw_hdl_info_t *info);
113 
114 /*
115  * The following routines are used for allocating and freeing the specific
116  * types of objects described above from their associated resource pools.
117  */
118 static int tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t *pool_info,
119     uint_t num, tavor_rsrc_t *hdl);
120 static void tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t *pool_info,
121     tavor_rsrc_t *hdl);
122 
123 static int tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t *pool_info,
124     uint_t num, uint_t num_align, ddi_acc_handle_t acc_handle,
125     uint_t sleepflag, tavor_rsrc_t *hdl);
126 static void tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t *pool_info,
127     tavor_rsrc_t *hdl);
128 
129 static int tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t *pool_info,
130     uint_t sleepflag, tavor_rsrc_t *hdl);
131 static void tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t *pool_info,
132     tavor_rsrc_t *hdl);
133 
134 static int tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t *pool_info,
135     uint_t sleepflag, tavor_rsrc_t *hdl);
136 static void tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t *pool_info,
137     tavor_rsrc_t *hdl);
138 
139 /*
140  * The following routines are the constructors and destructors for several
141  * of the SW handle type objects.  For certain types of SW handles objects
142  * (all of which are implemented using kmem_cache), we need to do some
143  * special field initialization (specifically, mutex_init/destroy).  These
144  * routines enable that init and teardown.
145  */
146 static int tavor_rsrc_pdhdl_constructor(void *pd, void *priv, int flags);
147 static void tavor_rsrc_pdhdl_destructor(void *pd, void *state);
148 static int tavor_rsrc_cqhdl_constructor(void *cq, void *priv, int flags);
149 static void tavor_rsrc_cqhdl_destructor(void *cq, void *state);
150 static int tavor_rsrc_qphdl_constructor(void *cq, void *priv, int flags);
151 static void tavor_rsrc_qphdl_destructor(void *cq, void *state);
152 static int tavor_rsrc_srqhdl_constructor(void *srq, void *priv, int flags);
153 static void tavor_rsrc_srqhdl_destructor(void *srq, void *state);
154 static int tavor_rsrc_refcnt_constructor(void *rc, void *priv, int flags);
155 static void tavor_rsrc_refcnt_destructor(void *rc, void *state);
156 static int tavor_rsrc_ahhdl_constructor(void *ah, void *priv, int flags);
157 static void tavor_rsrc_ahhdl_destructor(void *ah, void *state);
158 static int tavor_rsrc_mrhdl_constructor(void *mr, void *priv, int flags);
159 static void tavor_rsrc_mrhdl_destructor(void *mr, void *state);
160 
161 /*
162  * Special routine to calculate and return the size of a MCG object based
163  * on current driver configuration (specifically, the number of QP per MCG
164  * that has been configured.
165  */
166 static int tavor_rsrc_mcg_entry_get_size(tavor_state_t *state,
167     uint_t *mcg_size_shift);
168 
169 
170 /*
171  * tavor_rsrc_alloc()
172  *
173  *    Context: Can be called from interrupt or base context.
174  *    The "sleepflag" parameter is used by all object allocators to
175  *    determine whether to SLEEP for resources or not.
176  */
177 int
tavor_rsrc_alloc(tavor_state_t * state,tavor_rsrc_type_t rsrc,uint_t num,uint_t sleepflag,tavor_rsrc_t ** hdl)178 tavor_rsrc_alloc(tavor_state_t *state, tavor_rsrc_type_t rsrc, uint_t num,
179     uint_t sleepflag, tavor_rsrc_t **hdl)
180 {
181 	tavor_rsrc_pool_info_t	*rsrc_pool;
182 	tavor_rsrc_t		*tmp_rsrc_hdl;
183 	int			flag, status = DDI_FAILURE;
184 
185 	ASSERT(state != NULL);
186 	ASSERT(hdl != NULL);
187 
188 	rsrc_pool = &state->ts_rsrc_hdl[rsrc];
189 	ASSERT(rsrc_pool != NULL);
190 
191 	/*
192 	 * Allocate space for the object used to track the resource handle
193 	 */
194 	flag = (sleepflag == TAVOR_SLEEP) ? KM_SLEEP : KM_NOSLEEP;
195 	tmp_rsrc_hdl = (tavor_rsrc_t *)kmem_cache_alloc(state->ts_rsrc_cache,
196 	    flag);
197 	if (tmp_rsrc_hdl == NULL) {
198 		return (DDI_FAILURE);
199 	}
200 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*tmp_rsrc_hdl))
201 
202 	/*
203 	 * Set rsrc_hdl type.  This is later used by the tavor_rsrc_free call
204 	 * to know what type of resource is being freed.
205 	 */
206 	tmp_rsrc_hdl->rsrc_type = rsrc;
207 
208 	/*
209 	 * Depending on resource type, call the appropriate alloc routine
210 	 */
211 	switch (rsrc_pool->rsrc_type) {
212 	case TAVOR_IN_MBOX:
213 	case TAVOR_OUT_MBOX:
214 	case TAVOR_INTR_IN_MBOX:
215 	case TAVOR_INTR_OUT_MBOX:
216 		status = tavor_rsrc_mbox_alloc(rsrc_pool, num, tmp_rsrc_hdl);
217 		break;
218 
219 	case TAVOR_QPC:
220 	case TAVOR_CQC:
221 	case TAVOR_SRQC:
222 	case TAVOR_EQC:
223 	case TAVOR_RDB:
224 		/*
225 		 * Because these objects are NOT accessed by Tavor driver
226 		 * software, we set the acc_handle parameter to zero.  But
227 		 * if they are allocated in multiples, we specify here that
228 		 * they must be aligned on a more restrictive boundary.
229 		 */
230 		status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, num, 0,
231 		    sleepflag, tmp_rsrc_hdl);
232 		break;
233 
234 	case TAVOR_MPT:
235 		/*
236 		 * Because these MPT objects are sometimes accessed by Tavor
237 		 * driver software (FMR), we set the acc_handle parameter.  But
238 		 * if they are allocated in multiples, we specify here that
239 		 * they must be aligned on a more restrictive boundary.
240 		 */
241 		status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, num,
242 		    state->ts_reg_ddrhdl, sleepflag, tmp_rsrc_hdl);
243 		break;
244 
245 	case TAVOR_MCG:
246 		/*
247 		 * Tavor MCG entries are also NOT accessed by Tavor driver
248 		 * software, but because MCG entries do not have the same
249 		 * alignnment restrictions we loosen the constraint here.
250 		 */
251 		status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1, 0,
252 		    sleepflag, tmp_rsrc_hdl);
253 		break;
254 
255 	case TAVOR_MTT:
256 	case TAVOR_UDAV:
257 		/*
258 		 * Because MTT segments are among the few HW resources that
259 		 * may be allocated in odd numbers, we specify a less
260 		 * restrictive alignment than for the above resources.
261 		 *
262 		 * Also because both UDAV and MTT segment objects are read
263 		 * and/or written by Tavor driver software, we set the
264 		 * acc_handle parameter to point to the ddi_acc_handle_t for
265 		 * the Tavor DDR memory.
266 		 */
267 		status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1,
268 		    state->ts_reg_ddrhdl, sleepflag, tmp_rsrc_hdl);
269 		break;
270 
271 	case TAVOR_UARPG:
272 		/*
273 		 * Because UAR pages are written by Tavor driver software (for
274 		 * doorbells), we set the acc_handle parameter to point to
275 		 * the ddi_acc_handle_t for the Tavor UAR memory.
276 		 */
277 		status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1,
278 		    state->ts_reg_uarhdl, sleepflag, tmp_rsrc_hdl);
279 		break;
280 
281 	case TAVOR_MRHDL:
282 	case TAVOR_EQHDL:
283 	case TAVOR_CQHDL:
284 	case TAVOR_SRQHDL:
285 	case TAVOR_AHHDL:
286 	case TAVOR_QPHDL:
287 	case TAVOR_REFCNT:
288 		status = tavor_rsrc_swhdl_alloc(rsrc_pool, sleepflag,
289 		    tmp_rsrc_hdl);
290 		break;
291 
292 	case TAVOR_PDHDL:
293 		status = tavor_rsrc_pdhdl_alloc(rsrc_pool, sleepflag,
294 		    tmp_rsrc_hdl);
295 		break;
296 
297 	default:
298 		TAVOR_WARNING(state, "unexpected resource type in alloc");
299 		break;
300 	}
301 
302 	/*
303 	 * If the resource allocation failed, then free the special resource
304 	 * tracking structure and return failure.  Otherwise return the
305 	 * handle for the resource tracking structure.
306 	 */
307 	if (status != DDI_SUCCESS) {
308 		kmem_cache_free(state->ts_rsrc_cache, tmp_rsrc_hdl);
309 		tmp_rsrc_hdl = NULL;
310 		return (DDI_FAILURE);
311 	} else {
312 		*hdl = tmp_rsrc_hdl;
313 		return (DDI_SUCCESS);
314 	}
315 }
316 
317 
318 /*
319  * tavor_rsrc_free()
320  *    Context: Can be called from interrupt or base context.
321  */
322 void
tavor_rsrc_free(tavor_state_t * state,tavor_rsrc_t ** hdl)323 tavor_rsrc_free(tavor_state_t *state, tavor_rsrc_t **hdl)
324 {
325 	tavor_rsrc_pool_info_t	*rsrc_pool;
326 
327 	ASSERT(state != NULL);
328 	ASSERT(hdl != NULL);
329 
330 	rsrc_pool = &state->ts_rsrc_hdl[(*hdl)->rsrc_type];
331 	ASSERT(rsrc_pool != NULL);
332 
333 	/*
334 	 * Depending on resource type, call the appropriate free routine
335 	 */
336 	switch (rsrc_pool->rsrc_type) {
337 	case TAVOR_IN_MBOX:
338 	case TAVOR_OUT_MBOX:
339 	case TAVOR_INTR_IN_MBOX:
340 	case TAVOR_INTR_OUT_MBOX:
341 		tavor_rsrc_mbox_free(rsrc_pool, *hdl);
342 		break;
343 
344 	case TAVOR_QPC:
345 	case TAVOR_CQC:
346 	case TAVOR_SRQC:
347 	case TAVOR_EQC:
348 	case TAVOR_RDB:
349 	case TAVOR_MCG:
350 	case TAVOR_MPT:
351 	case TAVOR_MTT:
352 	case TAVOR_UDAV:
353 	case TAVOR_UARPG:
354 		tavor_rsrc_hw_entry_free(rsrc_pool, *hdl);
355 		break;
356 
357 	case TAVOR_MRHDL:
358 	case TAVOR_EQHDL:
359 	case TAVOR_CQHDL:
360 	case TAVOR_SRQHDL:
361 	case TAVOR_AHHDL:
362 	case TAVOR_QPHDL:
363 	case TAVOR_REFCNT:
364 		tavor_rsrc_swhdl_free(rsrc_pool, *hdl);
365 		break;
366 
367 	case TAVOR_PDHDL:
368 		tavor_rsrc_pdhdl_free(rsrc_pool, *hdl);
369 		break;
370 
371 	default:
372 		TAVOR_WARNING(state, "unexpected resource type in free");
373 		break;
374 	}
375 
376 	/*
377 	 * Free the special resource tracking structure, set the handle to
378 	 * NULL, and return.
379 	 */
380 	kmem_cache_free(state->ts_rsrc_cache, *hdl);
381 	*hdl = NULL;
382 }
383 
384 
385 /*
386  * tavor_rsrc_init_phase1()
387  *
388  *    Completes the first phase of Tavor resource/configuration init.
389  *    This involves creating the kmem_cache for the "tavor_rsrc_t"
390  *    structs, allocating the space for the resource pool handles,
391  *    and setting up the "Out" mailboxes.
392  *
393  *    When this function completes, the Tavor driver is ready to
394  *    post the following commands which return information only in the
395  *    "Out" mailbox: QUERY_DDR, QUERY_FW, QUERY_DEV_LIM, and QUERY_ADAPTER
396  *    If any of these commands are to be posted at this time, they must be
397  *    done so only when "spinning" (as the outstanding command list and
398  *    EQ setup code has not yet run)
399  *
400  *    Context: Only called from attach() path context
401  */
402 int
tavor_rsrc_init_phase1(tavor_state_t * state)403 tavor_rsrc_init_phase1(tavor_state_t *state)
404 {
405 	tavor_rsrc_pool_info_t		*rsrc_pool;
406 	tavor_rsrc_mbox_info_t 		mbox_info;
407 	tavor_rsrc_cleanup_level_t	cleanup;
408 	tavor_cfg_profile_t		*cfgprof;
409 	uint64_t			num, size;
410 	int				status;
411 	char				*rsrc_name;
412 
413 	ASSERT(state != NULL);
414 
415 	/* This is where Phase 1 of resource initialization begins */
416 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL0;
417 
418 	/* Build kmem cache name from Tavor instance */
419 	rsrc_name = (char *)kmem_zalloc(TAVOR_RSRC_NAME_MAXLEN, KM_SLEEP);
420 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_RSRC_CACHE);
421 
422 	/*
423 	 * Create the kmem_cache for "tavor_rsrc_t" structures
424 	 * (kmem_cache_create will SLEEP until successful)
425 	 */
426 	state->ts_rsrc_cache = kmem_cache_create(rsrc_name,
427 	    sizeof (tavor_rsrc_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
428 
429 	/*
430 	 * Allocate an array of tavor_rsrc_pool_info_t's (used in all
431 	 * subsequent resource allocations)
432 	 */
433 	state->ts_rsrc_hdl = kmem_zalloc(TAVOR_NUM_RESOURCES *
434 	    sizeof (tavor_rsrc_pool_info_t), KM_SLEEP);
435 
436 	cfgprof = state->ts_cfg_profile;
437 
438 	/*
439 	 * Initialize the resource pool for "Out" mailboxes.  Notice that
440 	 * the number of "Out" mailboxes, their size, and their location
441 	 * (DDR or system memory) is configurable.  By default, however,
442 	 * all "Out" mailboxes are located in system memory only (because
443 	 * they are primarily read from and never written to)
444 	 */
445 	num  =  ((uint64_t)1 << cfgprof->cp_log_num_outmbox);
446 	size =  ((uint64_t)1 << cfgprof->cp_log_outmbox_size);
447 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_OUT_MBOX];
448 	rsrc_pool->rsrc_type	  = TAVOR_OUT_MBOX;
449 	rsrc_pool->rsrc_loc	  = TAVOR_IN_SYSMEM;
450 	rsrc_pool->rsrc_pool_size = (size * num);
451 	rsrc_pool->rsrc_shift	  = cfgprof->cp_log_outmbox_size;
452 	rsrc_pool->rsrc_quantum	  = size;
453 	rsrc_pool->rsrc_align	  = TAVOR_MBOX_ALIGN;
454 	rsrc_pool->rsrc_state	  = state;
455 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_OUTMBOX_VMEM);
456 	mbox_info.mbi_num	  = num;
457 	mbox_info.mbi_size	  = size;
458 	mbox_info.mbi_rsrcpool	  = rsrc_pool;
459 	mbox_info.mbi_rsrcname	  = rsrc_name;
460 	status = tavor_rsrc_mbox_init(state, &mbox_info);
461 	if (status != DDI_SUCCESS) {
462 		tavor_rsrc_fini(state, cleanup);
463 		goto rsrcinitp1_fail;
464 	}
465 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL1;
466 
467 	/*
468 	 * Initialize the Tavor "Out" mailbox list.  This step actually uses
469 	 * the tavor_rsrc_alloc() for TAVOR_OUT_MBOX to preallocate the
470 	 * "Out" mailboxes, bind them for DMA access, and arrange them into
471 	 * an easily accessed fast-allocation mechanism (see tavor_cmd.c
472 	 * for more details)
473 	 */
474 	status = tavor_outmbox_list_init(state);
475 	if (status != DDI_SUCCESS) {
476 		tavor_rsrc_fini(state, cleanup);
477 		goto rsrcinitp1_fail;
478 	}
479 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL2;
480 
481 	/*
482 	 * Initialize the resource pool for interrupt "Out" mailboxes.  Notice
483 	 * that the number of interrupt "Out" mailboxes, their size, and their
484 	 * location (DDR or system memory) is configurable.  By default,
485 	 * however, all interrupt "Out" mailboxes are located in system memory
486 	 * only (because they are primarily read from and never written to)
487 	 */
488 	num  =  ((uint64_t)1 << cfgprof->cp_log_num_intr_outmbox);
489 	size =  ((uint64_t)1 << cfgprof->cp_log_outmbox_size);
490 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_INTR_OUT_MBOX];
491 	rsrc_pool->rsrc_type	  = TAVOR_INTR_OUT_MBOX;
492 	rsrc_pool->rsrc_loc	  = TAVOR_IN_SYSMEM;
493 	rsrc_pool->rsrc_pool_size = (size * num);
494 	rsrc_pool->rsrc_shift	  = cfgprof->cp_log_outmbox_size;
495 	rsrc_pool->rsrc_quantum	  = size;
496 	rsrc_pool->rsrc_align	  = TAVOR_MBOX_ALIGN;
497 	rsrc_pool->rsrc_state	  = state;
498 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INTR_OUTMBOX_VMEM);
499 	mbox_info.mbi_num	  = num;
500 	mbox_info.mbi_size	  = size;
501 	mbox_info.mbi_rsrcpool	  = rsrc_pool;
502 	mbox_info.mbi_rsrcname	  = rsrc_name;
503 	status = tavor_rsrc_mbox_init(state, &mbox_info);
504 	if (status != DDI_SUCCESS) {
505 		tavor_rsrc_fini(state, cleanup);
506 		goto rsrcinitp1_fail;
507 	}
508 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL3;
509 
510 	/*
511 	 * Initialize the Tavor "Out" mailbox list.  This step actually uses
512 	 * the tavor_rsrc_alloc() for TAVOR_OUT_MBOX to preallocate the
513 	 * "Out" mailboxes, bind them for DMA access, and arrange them into
514 	 * an easily accessed fast-allocation mechanism (see tavor_cmd.c
515 	 * for more details)
516 	 */
517 	status = tavor_intr_outmbox_list_init(state);
518 	if (status != DDI_SUCCESS) {
519 		tavor_rsrc_fini(state, cleanup);
520 		goto rsrcinitp1_fail;
521 	}
522 	cleanup = TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE;
523 
524 	kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
525 	return (DDI_SUCCESS);
526 
527 rsrcinitp1_fail:
528 	kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
529 	return (status);
530 }
531 
532 
533 /*
534  * tavor_rsrc_init_phase2()
535  *    Context: Only called from attach() path context
536  */
537 int
tavor_rsrc_init_phase2(tavor_state_t * state)538 tavor_rsrc_init_phase2(tavor_state_t *state)
539 {
540 	tavor_rsrc_sw_hdl_info_t	hdl_info;
541 	tavor_rsrc_hw_entry_info_t	entry_info;
542 	tavor_rsrc_mbox_info_t		mbox_info;
543 	tavor_rsrc_pool_info_t		*rsrc_pool;
544 	tavor_rsrc_cleanup_level_t	cleanup;
545 	tavor_cfg_profile_t		*cfgprof;
546 	uint64_t			num, max, size, num_prealloc;
547 	uint64_t			ddr_size, fw_size;
548 	uint_t				mcg_size, mcg_size_shift;
549 	uint_t				uarscr_size, mttsegment_sz;
550 	int				status;
551 	char				*rsrc_name;
552 
553 	ASSERT(state != NULL);
554 
555 	/* Phase 2 initialization begins where Phase 1 left off */
556 	cleanup = TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE;
557 
558 	/*
559 	 * Calculate the extent of the DDR size and portion of which that
560 	 * is already reserved for Tavor firmware.  (Note: this information
561 	 * is available because the QUERY_DDR and QUERY_FW commands have
562 	 * been posted to Tavor firmware prior to calling this routine)
563 	 */
564 	ddr_size = state->ts_ddr.ddr_endaddr - state->ts_ddr.ddr_baseaddr + 1;
565 	fw_size  = state->ts_fw.fw_endaddr - state->ts_fw.fw_baseaddr + 1;
566 
567 	/* Build the DDR vmem arena name from Tavor instance */
568 	rsrc_name = (char *)kmem_zalloc(TAVOR_RSRC_NAME_MAXLEN, KM_SLEEP);
569 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_VMEM);
570 
571 	/*
572 	 * Do a vmem_create for the entire DDR range (not including the
573 	 * portion consumed by Tavor firmware).  This creates the vmem arena
574 	 * from which all other DDR objects (specifically, tables of HW
575 	 * entries) will be allocated.
576 	 */
577 	state->ts_ddrvmem = vmem_create(rsrc_name,
578 	    (void *)(uintptr_t)state->ts_ddr.ddr_baseaddr, (ddr_size - fw_size),
579 	    sizeof (uint64_t), NULL, NULL, NULL, 0, VM_SLEEP);
580 	if (state->ts_ddrvmem == NULL) {
581 		tavor_rsrc_fini(state, cleanup);
582 		goto rsrcinitp2_fail;
583 	}
584 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL5;
585 
586 	/*
587 	 * Initialize the resource pools for all objects that exist in
588 	 * Tavor DDR memory.  This includes ("In") mailboxes, context tables
589 	 * (QPC, CQC, EQC, etc...), and other miscellaneous HW objects.
590 	 */
591 	cfgprof = state->ts_cfg_profile;
592 
593 	/*
594 	 * Initialize the resource pool for the MPT table entries.  Notice
595 	 * that the number of MPTs is configurable.  The configured value must
596 	 * be less that the maximum value (obtained from the QUERY_DEV_LIM
597 	 * command) or the initialization will fail.  Note also that a certain
598 	 * number of MPTs must be set aside for Tavor firmware use.
599 	 */
600 	num = ((uint64_t)1 << cfgprof->cp_log_num_mpt);
601 	max = ((uint64_t)1 << state->ts_devlim.log_max_mpt);
602 	num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_mpt);
603 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MPT];
604 	rsrc_pool->rsrc_type	  = TAVOR_MPT;
605 	rsrc_pool->rsrc_loc	  = TAVOR_IN_DDR;
606 	rsrc_pool->rsrc_pool_size = (TAVOR_MPT_SIZE * num);
607 	rsrc_pool->rsrc_shift	  = TAVOR_MPT_SIZE_SHIFT;
608 	rsrc_pool->rsrc_quantum	  = TAVOR_MPT_SIZE;
609 	rsrc_pool->rsrc_align	  = (TAVOR_MPT_SIZE * num);
610 	rsrc_pool->rsrc_state	  = state;
611 	rsrc_pool->rsrc_start	  = NULL;
612 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MPT_VMEM);
613 	entry_info.hwi_num	  = num;
614 	entry_info.hwi_max	  = max;
615 	entry_info.hwi_prealloc	  = num_prealloc;
616 	entry_info.hwi_rsrcpool	  = rsrc_pool;
617 	entry_info.hwi_rsrcname	  = rsrc_name;
618 	status = tavor_rsrc_hw_entries_init(state, &entry_info);
619 	if (status != DDI_SUCCESS) {
620 		tavor_rsrc_fini(state, cleanup);
621 		goto rsrcinitp2_fail;
622 	}
623 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL6;
624 
625 	/*
626 	 * Initialize the resource pool for the MTT table entries.  Notice
627 	 * that the number of MTTs is configurable.  The configured value must
628 	 * be less that the maximum value (obtained from the QUERY_DEV_LIM
629 	 * command) or the initialization will fail.  Note also that a certain
630 	 * number of MTT segments must be set aside for Tavor firmware use.
631 	 */
632 	mttsegment_sz = (TAVOR_MTTSEG_SIZE << TAVOR_MTT_SIZE_SHIFT);
633 	num = ((uint64_t)1 << cfgprof->cp_log_num_mttseg);
634 	max = ((uint64_t)1 << state->ts_devlim.log_max_mttseg);
635 	num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_mttseg);
636 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MTT];
637 	rsrc_pool->rsrc_type	  = TAVOR_MTT;
638 	rsrc_pool->rsrc_loc	  = TAVOR_IN_DDR;
639 	rsrc_pool->rsrc_pool_size = (TAVOR_MTT_SIZE * num);
640 	rsrc_pool->rsrc_shift	  = TAVOR_MTT_SIZE_SHIFT;
641 	rsrc_pool->rsrc_quantum	  = mttsegment_sz;
642 	rsrc_pool->rsrc_align	  = (TAVOR_MTT_SIZE * num);
643 	rsrc_pool->rsrc_state	  = state;
644 	rsrc_pool->rsrc_start	  = NULL;
645 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MTT_VMEM);
646 	entry_info.hwi_num	  = num;
647 	entry_info.hwi_max	  = max;
648 	entry_info.hwi_prealloc	  = num_prealloc;
649 	entry_info.hwi_rsrcpool	  = rsrc_pool;
650 	entry_info.hwi_rsrcname	  = rsrc_name;
651 	status = tavor_rsrc_hw_entries_init(state, &entry_info);
652 	if (status != DDI_SUCCESS) {
653 		tavor_rsrc_fini(state, cleanup);
654 		goto rsrcinitp2_fail;
655 	}
656 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL7;
657 
658 	/*
659 	 * Initialize the resource pool for the QPC table entries.  Notice
660 	 * that the number of QPs is configurable.  The configured value must
661 	 * be less that the maximum value (obtained from the QUERY_DEV_LIM
662 	 * command) or the initialization will fail.  Note also that a certain
663 	 * number of QP contexts must be set aside for Tavor firmware use.
664 	 */
665 	num = ((uint64_t)1 << cfgprof->cp_log_num_qp);
666 	max = ((uint64_t)1 << state->ts_devlim.log_max_qp);
667 	num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_qp);
668 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPC];
669 	rsrc_pool->rsrc_type	  = TAVOR_QPC;
670 	rsrc_pool->rsrc_loc	  = TAVOR_IN_DDR;
671 	rsrc_pool->rsrc_pool_size = (TAVOR_QPC_SIZE * num);
672 	rsrc_pool->rsrc_shift	  = TAVOR_QPC_SIZE_SHIFT;
673 	rsrc_pool->rsrc_quantum	  = TAVOR_QPC_SIZE;
674 	rsrc_pool->rsrc_align	  = (TAVOR_QPC_SIZE * num);
675 	rsrc_pool->rsrc_state	  = state;
676 	rsrc_pool->rsrc_start	  = NULL;
677 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_QPC_VMEM);
678 	entry_info.hwi_num	  = num;
679 	entry_info.hwi_max	  = max;
680 	entry_info.hwi_prealloc	  = num_prealloc;
681 	entry_info.hwi_rsrcpool	  = rsrc_pool;
682 	entry_info.hwi_rsrcname	  = rsrc_name;
683 	status = tavor_rsrc_hw_entries_init(state, &entry_info);
684 	if (status != DDI_SUCCESS) {
685 		tavor_rsrc_fini(state, cleanup);
686 		goto rsrcinitp2_fail;
687 	}
688 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL8;
689 
690 	/*
691 	 * Initialize the resource pool for the RDB table entries.  Notice
692 	 * that the number of RDBs is configurable.  The configured value must
693 	 * be less that the maximum value (obtained from the QUERY_DEV_LIM
694 	 * command) or the initialization will fail.
695 	 */
696 	num = ((uint64_t)1 << cfgprof->cp_log_num_rdb);
697 	max = ((uint64_t)1 << state->ts_devlim.log_max_ra_glob);
698 	num_prealloc = 0;
699 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_RDB];
700 	rsrc_pool->rsrc_type	  = TAVOR_RDB;
701 	rsrc_pool->rsrc_loc	  = TAVOR_IN_DDR;
702 	rsrc_pool->rsrc_pool_size = (TAVOR_RDB_SIZE * num);
703 	rsrc_pool->rsrc_shift	  = TAVOR_RDB_SIZE_SHIFT;
704 	rsrc_pool->rsrc_quantum	  = TAVOR_RDB_SIZE;
705 	rsrc_pool->rsrc_align	  = (TAVOR_RDB_SIZE * num);
706 	rsrc_pool->rsrc_state	  = state;
707 	rsrc_pool->rsrc_start	  = NULL;
708 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_RDB_VMEM);
709 	entry_info.hwi_num	  = num;
710 	entry_info.hwi_max	  = max;
711 	entry_info.hwi_prealloc	  = num_prealloc;
712 	entry_info.hwi_rsrcpool	  = rsrc_pool;
713 	entry_info.hwi_rsrcname	  = rsrc_name;
714 	status = tavor_rsrc_hw_entries_init(state, &entry_info);
715 	if (status != DDI_SUCCESS) {
716 		tavor_rsrc_fini(state, cleanup);
717 		goto rsrcinitp2_fail;
718 	}
719 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL9;
720 
721 	/*
722 	 * Initialize the resource pool for the CQC table entries.  Notice
723 	 * that the number of CQs is configurable.  The configured value must
724 	 * be less that the maximum value (obtained from the QUERY_DEV_LIM
725 	 * command) or the initialization will fail.  Note also that a certain
726 	 * number of CQ contexts must be set aside for Tavor firmware use.
727 	 */
728 	num = ((uint64_t)1 << cfgprof->cp_log_num_cq);
729 	max = ((uint64_t)1 << state->ts_devlim.log_max_cq);
730 	num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_cq);
731 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQC];
732 	rsrc_pool->rsrc_type	  = TAVOR_CQC;
733 	rsrc_pool->rsrc_loc	  = TAVOR_IN_DDR;
734 	rsrc_pool->rsrc_pool_size = (TAVOR_CQC_SIZE * num);
735 	rsrc_pool->rsrc_shift	  = TAVOR_CQC_SIZE_SHIFT;
736 	rsrc_pool->rsrc_quantum	  = TAVOR_CQC_SIZE;
737 	rsrc_pool->rsrc_align	  = (TAVOR_CQC_SIZE * num);
738 	rsrc_pool->rsrc_state	  = state;
739 	rsrc_pool->rsrc_start	  = NULL;
740 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_CQC_VMEM);
741 	entry_info.hwi_num	  = num;
742 	entry_info.hwi_max	  = max;
743 	entry_info.hwi_prealloc	  = num_prealloc;
744 	entry_info.hwi_rsrcpool	  = rsrc_pool;
745 	entry_info.hwi_rsrcname	  = rsrc_name;
746 	status = tavor_rsrc_hw_entries_init(state, &entry_info);
747 	if (status != DDI_SUCCESS) {
748 		tavor_rsrc_fini(state, cleanup);
749 		goto rsrcinitp2_fail;
750 	}
751 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL10;
752 
753 	/*
754 	 * Initialize the resource pool for the Extended QPC table entries.
755 	 * Notice that the number of EQPCs must be the same as the number
756 	 * of QP contexts.  So the initialization is constructed in a
757 	 * similar way as above (for TAVOR_QPC).  One notable difference
758 	 * here, however, is that by setting the rsrc_quantum field to
759 	 * zero (indicating a zero-sized object) we indicate that the
760 	 * object is not allocatable.  The EQPC table is, in fact, managed
761 	 * internally by the hardware and it is, therefore, unnecessary to
762 	 * initialize an additional vmem_arena for this type of object.
763 	 */
764 	num = ((uint64_t)1 << cfgprof->cp_log_num_qp);
765 	max = ((uint64_t)1 << state->ts_devlim.log_max_qp);
766 	num_prealloc = 0;
767 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQPC];
768 	rsrc_pool->rsrc_type	  = TAVOR_EQPC;
769 	rsrc_pool->rsrc_loc	  = TAVOR_IN_DDR;
770 	rsrc_pool->rsrc_pool_size = (TAVOR_EQPC_SIZE * num);
771 	rsrc_pool->rsrc_shift	  = 0;
772 	rsrc_pool->rsrc_quantum	  = 0;
773 	rsrc_pool->rsrc_align	  = TAVOR_EQPC_SIZE;
774 	rsrc_pool->rsrc_state	  = state;
775 	rsrc_pool->rsrc_start	  = NULL;
776 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_EQPC_VMEM);
777 	entry_info.hwi_num	  = num;
778 	entry_info.hwi_max	  = max;
779 	entry_info.hwi_prealloc	  = num_prealloc;
780 	entry_info.hwi_rsrcpool	  = rsrc_pool;
781 	entry_info.hwi_rsrcname	  = rsrc_name;
782 	status = tavor_rsrc_hw_entries_init(state, &entry_info);
783 	if (status != DDI_SUCCESS) {
784 		tavor_rsrc_fini(state, cleanup);
785 		goto rsrcinitp2_fail;
786 	}
787 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL11;
788 
789 	/*
790 	 * Initialize the resource pool for the UD address vector table
791 	 * entries.  Notice that the number of UDAVs is configurable.  The
792 	 * configured value must be less that the maximum value (obtained
793 	 * from the QUERY_DEV_LIM command) or the initialization will fail.
794 	 */
795 	num = ((uint64_t)1 << cfgprof->cp_log_num_ah);
796 	max = ((uint64_t)1 << state->ts_devlim.log_max_av);
797 	num_prealloc = 0;
798 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UDAV];
799 	rsrc_pool->rsrc_type	  = TAVOR_UDAV;
800 	rsrc_pool->rsrc_loc	  = TAVOR_IN_DDR;
801 	rsrc_pool->rsrc_pool_size = (TAVOR_UDAV_SIZE * num);
802 	rsrc_pool->rsrc_shift	  = TAVOR_UDAV_SIZE_SHIFT;
803 	rsrc_pool->rsrc_quantum	  = TAVOR_UDAV_SIZE;
804 	rsrc_pool->rsrc_align	  = TAVOR_UDAV_SIZE;
805 	rsrc_pool->rsrc_state	  = state;
806 	rsrc_pool->rsrc_start	  = NULL;
807 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_UDAV_VMEM);
808 	entry_info.hwi_num	  = num;
809 	entry_info.hwi_max	  = max;
810 	entry_info.hwi_prealloc	  = num_prealloc;
811 	entry_info.hwi_rsrcpool	  = rsrc_pool;
812 	entry_info.hwi_rsrcname	  = rsrc_name;
813 	status = tavor_rsrc_hw_entries_init(state, &entry_info);
814 	if (status != DDI_SUCCESS) {
815 		tavor_rsrc_fini(state, cleanup);
816 		goto rsrcinitp2_fail;
817 	}
818 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL12;
819 
820 	/*
821 	 * Initialize the resource pool for the UAR scratch table entries.
822 	 * Notice that the number of UARSCRs is configurable.  The configured
823 	 * value must be less that the maximum value (obtained from the
824 	 * QUERY_DEV_LIM command) or the initialization will fail.
825 	 * Like the EQPCs above, UARSCR objects are not allocatable.  The
826 	 * UARSCR table is also managed internally by the hardware and it
827 	 * is, therefore, unnecessary to initialize an additional vmem_arena
828 	 * for this type of object.  We indicate this by setting the
829 	 * rsrc_quantum field to zero (indicating a zero-sized object).
830 	 */
831 	uarscr_size = state->ts_devlim.uarscr_entry_sz;
832 	num = ((uint64_t)1 << cfgprof->cp_log_num_uar);
833 	max = ((uint64_t)1 << (state->ts_devlim.log_max_uar_sz + 20 -
834 	    PAGESHIFT));
835 	num_prealloc = 0;
836 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR];
837 	rsrc_pool->rsrc_type	  = TAVOR_UAR_SCR;
838 	rsrc_pool->rsrc_loc	  = TAVOR_IN_DDR;
839 	rsrc_pool->rsrc_pool_size = (uarscr_size * num);
840 	rsrc_pool->rsrc_shift	  = 0;
841 	rsrc_pool->rsrc_quantum	  = 0;
842 	rsrc_pool->rsrc_align	  = uarscr_size;
843 	rsrc_pool->rsrc_state	  = state;
844 	rsrc_pool->rsrc_start	  = NULL;
845 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_UARSCR_VMEM);
846 	entry_info.hwi_num	  = num;
847 	entry_info.hwi_max	  = max;
848 	entry_info.hwi_prealloc	  = num_prealloc;
849 	entry_info.hwi_rsrcpool	  = rsrc_pool;
850 	entry_info.hwi_rsrcname	  = rsrc_name;
851 	status = tavor_rsrc_hw_entries_init(state, &entry_info);
852 	if (status != DDI_SUCCESS) {
853 		tavor_rsrc_fini(state, cleanup);
854 		goto rsrcinitp2_fail;
855 	}
856 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL13;
857 
858 	/*
859 	 * Initialize the resource pool for the SRQC table entries.  Notice
860 	 * that the number of SRQs is configurable.  The configured value must
861 	 * be less that the maximum value (obtained from the QUERY_DEV_LIM
862 	 * command) or the initialization will fail.  Note also that a certain
863 	 * number of SRQ contexts must be set aside for Tavor firmware use.
864 	 *
865 	 * Note: We only allocate these resources if SRQ is enabled in the
866 	 * config profile; see below.
867 	 */
868 	num = ((uint64_t)1 << cfgprof->cp_log_num_srq);
869 	max = ((uint64_t)1 << state->ts_devlim.log_max_srq);
870 	num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_srq);
871 
872 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQC];
873 	rsrc_pool->rsrc_type	  = TAVOR_SRQC;
874 	rsrc_pool->rsrc_loc	  = TAVOR_IN_DDR;
875 	rsrc_pool->rsrc_pool_size = (TAVOR_SRQC_SIZE * num);
876 	rsrc_pool->rsrc_shift	  = TAVOR_SRQC_SIZE_SHIFT;
877 	rsrc_pool->rsrc_quantum	  = TAVOR_SRQC_SIZE;
878 	rsrc_pool->rsrc_align	  = (TAVOR_SRQC_SIZE * num);
879 	rsrc_pool->rsrc_state	  = state;
880 	rsrc_pool->rsrc_start	  = NULL;
881 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_SRQC_VMEM);
882 	entry_info.hwi_num	  = num;
883 	entry_info.hwi_max	  = max;
884 	entry_info.hwi_prealloc	  = num_prealloc;
885 	entry_info.hwi_rsrcpool	  = rsrc_pool;
886 	entry_info.hwi_rsrcname	  = rsrc_name;
887 
888 	/*
889 	 * SRQ support is configurable.  Only if SRQ is enabled (the default)
890 	 * do we actually try to configure these resources.  Otherwise, we
891 	 * simply set the cleanup level and continue on to the next resource
892 	 */
893 	if (state->ts_cfg_profile->cp_srq_enable != 0) {
894 		status = tavor_rsrc_hw_entries_init(state, &entry_info);
895 		if (status != DDI_SUCCESS) {
896 			tavor_rsrc_fini(state, cleanup);
897 			goto rsrcinitp2_fail;
898 		}
899 	}
900 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL14;
901 
902 	/*
903 	 * Initialize the resource pool for "In" mailboxes.  Notice that
904 	 * the number of "In" mailboxes, their size, and their location
905 	 * (DDR or system memory) is configurable.  By default, however,
906 	 * all "In" mailboxes are located in system memory only (because
907 	 * they are primarily written to and rarely read from)
908 	 */
909 	num  =  ((uint64_t)1 << cfgprof->cp_log_num_inmbox);
910 	size =  ((uint64_t)1 << cfgprof->cp_log_inmbox_size);
911 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_IN_MBOX];
912 	rsrc_pool->rsrc_type	  = TAVOR_IN_MBOX;
913 	rsrc_pool->rsrc_loc	  = TAVOR_IN_DDR;
914 	rsrc_pool->rsrc_pool_size = (size * num);
915 	rsrc_pool->rsrc_shift	  = cfgprof->cp_log_inmbox_size;
916 	rsrc_pool->rsrc_quantum	  = size;
917 	rsrc_pool->rsrc_align	  = TAVOR_MBOX_ALIGN;
918 	rsrc_pool->rsrc_state	  = state;
919 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INMBOX_VMEM);
920 	mbox_info.mbi_num	  = num;
921 	mbox_info.mbi_size	  = size;
922 	mbox_info.mbi_rsrcpool	  = rsrc_pool;
923 	mbox_info.mbi_rsrcname	  = rsrc_name;
924 	status = tavor_rsrc_mbox_init(state, &mbox_info);
925 	if (status != DDI_SUCCESS) {
926 		tavor_rsrc_fini(state, cleanup);
927 		goto rsrcinitp2_fail;
928 	}
929 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL15;
930 
931 	/*
932 	 * Initialize the Tavor "In" mailbox list.  This step actually uses
933 	 * the tavor_rsrc_alloc() for TAVOR_IN_MBOX to preallocate the
934 	 * "In" mailboxes, bind them for DMA access, and arrange them into
935 	 * an easily accessed fast-allocation mechanism (see tavor_cmd.c
936 	 * for more details)
937 	 */
938 	status = tavor_inmbox_list_init(state);
939 	if (status != DDI_SUCCESS) {
940 		tavor_rsrc_fini(state, cleanup);
941 		goto rsrcinitp2_fail;
942 	}
943 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL16;
944 
945 	/*
946 	 * Initialize the resource pool for interrupt "In" mailboxes.  Notice
947 	 * that the number of interrupt "In" mailboxes, their size, and their
948 	 * location (DDR or system memory) is configurable.  By default,
949 	 * however, all interrupt "In" mailboxes are located in system memory
950 	 * only (because they are primarily written to and rarely read from)
951 	 */
952 	num  =  ((uint64_t)1 << cfgprof->cp_log_num_intr_inmbox);
953 	size =  ((uint64_t)1 << cfgprof->cp_log_inmbox_size);
954 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_INTR_IN_MBOX];
955 	rsrc_pool->rsrc_type	  = TAVOR_INTR_IN_MBOX;
956 	rsrc_pool->rsrc_loc	  = TAVOR_IN_DDR;
957 	rsrc_pool->rsrc_pool_size = (size * num);
958 	rsrc_pool->rsrc_shift	  = cfgprof->cp_log_inmbox_size;
959 	rsrc_pool->rsrc_quantum	  = size;
960 	rsrc_pool->rsrc_align	  = TAVOR_MBOX_ALIGN;
961 	rsrc_pool->rsrc_state	  = state;
962 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INTR_INMBOX_VMEM);
963 	mbox_info.mbi_num	  = num;
964 	mbox_info.mbi_size	  = size;
965 	mbox_info.mbi_rsrcpool	  = rsrc_pool;
966 	mbox_info.mbi_rsrcname	  = rsrc_name;
967 	status = tavor_rsrc_mbox_init(state, &mbox_info);
968 	if (status != DDI_SUCCESS) {
969 		tavor_rsrc_fini(state, cleanup);
970 		goto rsrcinitp2_fail;
971 	}
972 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL17;
973 
974 	/*
975 	 * Initialize the Tavor interrupt "In" mailbox list.  This step
976 	 * actually uses the tavor_rsrc_alloc() for TAVOR_IN_MBOX to
977 	 * preallocate the interrupt "In" mailboxes, bind them for DMA access,
978 	 * and arrange them into an easily accessed fast-allocation mechanism
979 	 * (see tavor_cmd.c for more details)
980 	 */
981 	status = tavor_intr_inmbox_list_init(state);
982 	if (status != DDI_SUCCESS) {
983 		tavor_rsrc_fini(state, cleanup);
984 		goto rsrcinitp2_fail;
985 	}
986 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL18;
987 
988 	/*
989 	 * Initialize the Tavor command handling interfaces.  This step
990 	 * sets up the outstanding command tracking mechanism for easy access
991 	 * and fast allocation (see tavor_cmd.c for more details).
992 	 */
993 	status = tavor_outstanding_cmdlist_init(state);
994 	if (status != DDI_SUCCESS) {
995 		tavor_rsrc_fini(state, cleanup);
996 		goto rsrcinitp2_fail;
997 	}
998 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL19;
999 
1000 	/*
1001 	 * Calculate (and validate) the size of Multicast Group (MCG) entries
1002 	 */
1003 	status = tavor_rsrc_mcg_entry_get_size(state, &mcg_size_shift);
1004 	if (status != DDI_SUCCESS) {
1005 		tavor_rsrc_fini(state, cleanup);
1006 		goto rsrcinitp2_fail;
1007 	}
1008 	mcg_size = TAVOR_MCGMEM_SZ(state);
1009 
1010 	/*
1011 	 * Initialize the resource pool for the MCG table entries.  Notice
1012 	 * that the number of MCGs is configurable.  The configured value must
1013 	 * be less that the maximum value (obtained from the QUERY_DEV_LIM
1014 	 * command) or the initialization will fail.  Note also that a certain
1015 	 * number of MCGs must be set aside for Tavor firmware use (they
1016 	 * correspond to the number of MCGs used by the internal hash
1017 	 * function.
1018 	 */
1019 	num = ((uint64_t)1 << cfgprof->cp_log_num_mcg);
1020 	max = ((uint64_t)1 << state->ts_devlim.log_max_mcg);
1021 	num_prealloc = ((uint64_t)1 << cfgprof->cp_log_num_mcg_hash);
1022 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MCG];
1023 	rsrc_pool->rsrc_type	  = TAVOR_MCG;
1024 	rsrc_pool->rsrc_loc	  = TAVOR_IN_DDR;
1025 	rsrc_pool->rsrc_pool_size = (mcg_size * num);
1026 	rsrc_pool->rsrc_shift	  = mcg_size_shift;
1027 	rsrc_pool->rsrc_quantum	  = mcg_size;
1028 	rsrc_pool->rsrc_align	  = mcg_size;
1029 	rsrc_pool->rsrc_state	  = state;
1030 	rsrc_pool->rsrc_start	  = NULL;
1031 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MCG_VMEM);
1032 	entry_info.hwi_num	  = num;
1033 	entry_info.hwi_max	  = max;
1034 	entry_info.hwi_prealloc	  = num_prealloc;
1035 	entry_info.hwi_rsrcpool	  = rsrc_pool;
1036 	entry_info.hwi_rsrcname	  = rsrc_name;
1037 	status = tavor_rsrc_hw_entries_init(state, &entry_info);
1038 	if (status != DDI_SUCCESS) {
1039 		tavor_rsrc_fini(state, cleanup);
1040 		goto rsrcinitp2_fail;
1041 	}
1042 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL20;
1043 
1044 	/*
1045 	 * Initialize the resource pool for the EQC table entries.  Notice
1046 	 * that the number of EQs is hardcoded.  The hardcoded value should
1047 	 * be less that the maximum value (obtained from the QUERY_DEV_LIM
1048 	 * command) or the initialization will fail.
1049 	 */
1050 	num = TAVOR_NUM_EQ;
1051 	max = ((uint64_t)1 << state->ts_devlim.log_max_eq);
1052 	num_prealloc = 0;
1053 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQC];
1054 	rsrc_pool->rsrc_type	  = TAVOR_EQC;
1055 	rsrc_pool->rsrc_loc	  = TAVOR_IN_DDR;
1056 	rsrc_pool->rsrc_pool_size = (TAVOR_EQC_SIZE * num);
1057 	rsrc_pool->rsrc_shift	  = TAVOR_EQC_SIZE_SHIFT;
1058 	rsrc_pool->rsrc_quantum	  = TAVOR_EQC_SIZE;
1059 	rsrc_pool->rsrc_align	  = (TAVOR_EQC_SIZE * num);
1060 	rsrc_pool->rsrc_state	  = state;
1061 	rsrc_pool->rsrc_start	  = NULL;
1062 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_EQC_VMEM);
1063 	entry_info.hwi_num	  = num;
1064 	entry_info.hwi_max	  = max;
1065 	entry_info.hwi_prealloc	  = num_prealloc;
1066 	entry_info.hwi_rsrcpool	  = rsrc_pool;
1067 	entry_info.hwi_rsrcname	  = rsrc_name;
1068 	status = tavor_rsrc_hw_entries_init(state, &entry_info);
1069 	if (status != DDI_SUCCESS) {
1070 		tavor_rsrc_fini(state, cleanup);
1071 		goto rsrcinitp2_fail;
1072 	}
1073 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL21;
1074 
1075 	/*
1076 	 * Initialize the resource pools for all objects that exist in
1077 	 * system memory.  This includes PD handles, MR handle, EQ handles,
1078 	 * QP handles, etc.  These objects are almost entirely managed using
1079 	 * kmem_cache routines.  (See comment above for more detail)
1080 	 */
1081 
1082 	/*
1083 	 * Initialize the resource pool for the PD handles.  Notice
1084 	 * that the number of PDHDLs is configurable.  The configured value
1085 	 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1086 	 * command) or the initialization will fail.  Note also that the PD
1087 	 * handle has constructor and destructor methods associated with it.
1088 	 */
1089 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_PDHDL];
1090 	rsrc_pool->rsrc_type	 = TAVOR_PDHDL;
1091 	rsrc_pool->rsrc_loc	 = TAVOR_IN_SYSMEM;
1092 	rsrc_pool->rsrc_quantum	 = sizeof (struct tavor_sw_pd_s);
1093 	rsrc_pool->rsrc_state	 = state;
1094 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_PDHDL_CACHE);
1095 	hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_pd);
1096 	hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_pd);
1097 	hdl_info.swi_rsrcpool	 = rsrc_pool;
1098 	hdl_info.swi_constructor = tavor_rsrc_pdhdl_constructor;
1099 	hdl_info.swi_destructor	 = tavor_rsrc_pdhdl_destructor;
1100 	hdl_info.swi_rsrcname	 = rsrc_name;
1101 	hdl_info.swi_flags	 = TAVOR_SWHDL_KMEMCACHE_INIT;
1102 	status = tavor_rsrc_pd_handles_init(state, &hdl_info);
1103 	if (status != DDI_SUCCESS) {
1104 		tavor_rsrc_fini(state, cleanup);
1105 		goto rsrcinitp2_fail;
1106 	}
1107 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL22;
1108 
1109 	/*
1110 	 * Initialize the resource pool for the MR handles.  Notice
1111 	 * that the number of MRHDLs is configurable.  The configured value
1112 	 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1113 	 * command) or the initialization will fail.
1114 	 */
1115 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MRHDL];
1116 	rsrc_pool->rsrc_type	 = TAVOR_MRHDL;
1117 	rsrc_pool->rsrc_loc	 = TAVOR_IN_SYSMEM;
1118 	rsrc_pool->rsrc_quantum	 = sizeof (struct tavor_sw_mr_s);
1119 	rsrc_pool->rsrc_state	 = state;
1120 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_MRHDL_CACHE);
1121 	hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mpt);
1122 	hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mpt);
1123 	hdl_info.swi_rsrcpool	 = rsrc_pool;
1124 	hdl_info.swi_constructor = tavor_rsrc_mrhdl_constructor;
1125 	hdl_info.swi_destructor	 = tavor_rsrc_mrhdl_destructor;
1126 	hdl_info.swi_rsrcname	 = rsrc_name;
1127 	hdl_info.swi_flags	 = TAVOR_SWHDL_KMEMCACHE_INIT;
1128 	status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1129 	if (status != DDI_SUCCESS) {
1130 		tavor_rsrc_fini(state, cleanup);
1131 		goto rsrcinitp2_fail;
1132 	}
1133 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL23;
1134 
1135 	/*
1136 	 * Initialize the resource pool for the EQ handles.  Notice
1137 	 * that the number of EQHDLs is hardcoded.  The hardcoded value
1138 	 * should be less that the maximum value (obtained from the
1139 	 * QUERY_DEV_LIM command) or the initialization will fail.
1140 	 */
1141 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQHDL];
1142 	rsrc_pool->rsrc_type	 = TAVOR_EQHDL;
1143 	rsrc_pool->rsrc_loc	 = TAVOR_IN_SYSMEM;
1144 	rsrc_pool->rsrc_quantum	 = sizeof (struct tavor_sw_eq_s);
1145 	rsrc_pool->rsrc_state	 = state;
1146 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_EQHDL_CACHE);
1147 	hdl_info.swi_num = TAVOR_NUM_EQ;
1148 	hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_eq);
1149 	hdl_info.swi_rsrcpool	 = rsrc_pool;
1150 	hdl_info.swi_constructor = NULL;
1151 	hdl_info.swi_destructor	 = NULL;
1152 	hdl_info.swi_rsrcname	 = rsrc_name;
1153 	hdl_info.swi_flags	 = TAVOR_SWHDL_KMEMCACHE_INIT;
1154 	status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1155 	if (status != DDI_SUCCESS) {
1156 		tavor_rsrc_fini(state, cleanup);
1157 		goto rsrcinitp2_fail;
1158 	}
1159 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL24;
1160 
1161 	/*
1162 	 * Initialize the resource pool for the CQ handles.  Notice
1163 	 * that the number of CQHDLs is configurable.  The configured value
1164 	 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1165 	 * command) or the initialization will fail.  Note also that the CQ
1166 	 * handle has constructor and destructor methods associated with it.
1167 	 */
1168 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQHDL];
1169 	rsrc_pool->rsrc_type	 = TAVOR_CQHDL;
1170 	rsrc_pool->rsrc_loc	 = TAVOR_IN_SYSMEM;
1171 	rsrc_pool->rsrc_quantum	 = sizeof (struct tavor_sw_cq_s);
1172 	rsrc_pool->rsrc_state	 = state;
1173 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_CQHDL_CACHE);
1174 	hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_cq);
1175 	hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_cq);
1176 	hdl_info.swi_rsrcpool	 = rsrc_pool;
1177 	hdl_info.swi_constructor = tavor_rsrc_cqhdl_constructor;
1178 	hdl_info.swi_destructor	 = tavor_rsrc_cqhdl_destructor;
1179 	hdl_info.swi_rsrcname	 = rsrc_name;
1180 	hdl_info.swi_flags	 = (TAVOR_SWHDL_KMEMCACHE_INIT |
1181 	    TAVOR_SWHDL_TABLE_INIT);
1182 	hdl_info.swi_prealloc_sz = sizeof (tavor_cqhdl_t);
1183 	status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1184 	if (status != DDI_SUCCESS) {
1185 		tavor_rsrc_fini(state, cleanup);
1186 		goto rsrcinitp2_fail;
1187 	}
1188 
1189 	/*
1190 	 * Save away the pointer to the central list of CQ handle pointers
1191 	 * This this is used as a mechanism to enable fast CQnumber-to-CQhandle
1192 	 * lookup during EQ event processing.  The table is a list of
1193 	 * tavor_cqhdl_t allocated by the above routine because of the
1194 	 * TAVOR_SWHDL_TABLE_INIT flag.  The table has as many tavor_cqhdl_t
1195 	 * as the number of CQs.
1196 	 */
1197 	state->ts_cqhdl = hdl_info.swi_table_ptr;
1198 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL25;
1199 
1200 	/*
1201 	 * Initialize the resource pool for the SRQ handles.  Notice
1202 	 * that the number of SRQHDLs is configurable.  The configured value
1203 	 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1204 	 * command) or the initialization will fail.  Note also that the SRQ
1205 	 * handle has constructor and destructor methods associated with it.
1206 	 *
1207 	 * Note: We only allocate these resources if SRQ is enabled in the
1208 	 * config profile; see below.
1209 	 */
1210 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQHDL];
1211 	rsrc_pool->rsrc_type	 = TAVOR_SRQHDL;
1212 	rsrc_pool->rsrc_loc	 = TAVOR_IN_SYSMEM;
1213 	rsrc_pool->rsrc_quantum	 = sizeof (struct tavor_sw_srq_s);
1214 	rsrc_pool->rsrc_state	 = state;
1215 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_SRQHDL_CACHE);
1216 	hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_srq);
1217 	hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_srq);
1218 	hdl_info.swi_rsrcpool	 = rsrc_pool;
1219 	hdl_info.swi_constructor = tavor_rsrc_srqhdl_constructor;
1220 	hdl_info.swi_destructor	 = tavor_rsrc_srqhdl_destructor;
1221 	hdl_info.swi_rsrcname	 = rsrc_name;
1222 	hdl_info.swi_flags	 = (TAVOR_SWHDL_KMEMCACHE_INIT |
1223 	    TAVOR_SWHDL_TABLE_INIT);
1224 	hdl_info.swi_prealloc_sz = sizeof (tavor_srqhdl_t);
1225 
1226 	/*
1227 	 * SRQ support is configurable.  Only if SRQ is enabled (the default)
1228 	 * do we actually try to configure these resources.  Otherwise, we
1229 	 * simply set the cleanup level and continue on to the next resource
1230 	 */
1231 	if (state->ts_cfg_profile->cp_srq_enable != 0) {
1232 		status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1233 		if (status != DDI_SUCCESS) {
1234 			tavor_rsrc_fini(state, cleanup);
1235 			goto rsrcinitp2_fail;
1236 		}
1237 
1238 		/*
1239 		 * Save away the pointer to the central list of SRQ handle
1240 		 * pointers This this is used as a mechanism to enable fast
1241 		 * SRQnumber-to-SRQhandle lookup.  The table is a list of
1242 		 * tavor_srqhdl_t allocated by the above routine because of the
1243 		 * TAVOR_SWHDL_TABLE_INIT flag.  The table has as many
1244 		 * tavor_srqhdl_t as the number of SRQs.
1245 		 */
1246 		state->ts_srqhdl = hdl_info.swi_table_ptr;
1247 	}
1248 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL26;
1249 
1250 	/*
1251 	 * Initialize the resource pool for the address handles.  Notice
1252 	 * that the number of AHHDLs is configurable.  The configured value
1253 	 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1254 	 * command) or the initialization will fail.
1255 	 */
1256 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_AHHDL];
1257 	rsrc_pool->rsrc_type	 = TAVOR_AHHDL;
1258 	rsrc_pool->rsrc_loc	 = TAVOR_IN_SYSMEM;
1259 	rsrc_pool->rsrc_quantum	 = sizeof (struct tavor_sw_ah_s);
1260 	rsrc_pool->rsrc_state	 = state;
1261 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_AHHDL_CACHE);
1262 	hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_ah);
1263 	hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_av);
1264 	hdl_info.swi_rsrcpool	 = rsrc_pool;
1265 	hdl_info.swi_constructor = tavor_rsrc_ahhdl_constructor;
1266 	hdl_info.swi_destructor	 = tavor_rsrc_ahhdl_destructor;
1267 	hdl_info.swi_rsrcname	 = rsrc_name;
1268 	hdl_info.swi_flags	 = TAVOR_SWHDL_KMEMCACHE_INIT;
1269 	status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1270 	if (status != DDI_SUCCESS) {
1271 		tavor_rsrc_fini(state, cleanup);
1272 		goto rsrcinitp2_fail;
1273 	}
1274 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL27;
1275 
1276 	/*
1277 	 * Initialize the resource pool for the QP handles.  Notice
1278 	 * that the number of QPHDLs is configurable.  The configured value
1279 	 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1280 	 * command) or the initialization will fail.  Note also that the QP
1281 	 * handle has constructor and destructor methods associated with it.
1282 	 */
1283 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPHDL];
1284 	rsrc_pool->rsrc_type	 = TAVOR_QPHDL;
1285 	rsrc_pool->rsrc_loc	 = TAVOR_IN_SYSMEM;
1286 	rsrc_pool->rsrc_quantum	 = sizeof (struct tavor_sw_qp_s);
1287 	rsrc_pool->rsrc_state	 = state;
1288 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_QPHDL_CACHE);
1289 	hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_qp);
1290 	hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_qp);
1291 	hdl_info.swi_rsrcpool	 = rsrc_pool;
1292 	hdl_info.swi_constructor = tavor_rsrc_qphdl_constructor;
1293 	hdl_info.swi_destructor	 = tavor_rsrc_qphdl_destructor;
1294 	hdl_info.swi_rsrcname	 = rsrc_name;
1295 	hdl_info.swi_flags	 = (TAVOR_SWHDL_KMEMCACHE_INIT |
1296 	    TAVOR_SWHDL_TABLE_INIT);
1297 	hdl_info.swi_prealloc_sz = sizeof (tavor_qphdl_t);
1298 	status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1299 	if (status != DDI_SUCCESS) {
1300 		tavor_rsrc_fini(state, cleanup);
1301 		goto rsrcinitp2_fail;
1302 	}
1303 
1304 	/*
1305 	 * Save away the pointer to the central list of QP handle pointers
1306 	 * This this is used as a mechanism to enable fast QPnumber-to-QPhandle
1307 	 * lookup during CQ event processing.  The table is a list of
1308 	 * tavor_qphdl_t allocated by the above routine because of the
1309 	 * TAVOR_SWHDL_TABLE_INIT flag.  The table has as many tavor_qphdl_t
1310 	 * as the number of QPs.
1311 	 */
1312 	state->ts_qphdl = hdl_info.swi_table_ptr;
1313 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL28;
1314 
1315 	/*
1316 	 * Initialize the resource pool for the reference count handles.
1317 	 * Notice that the number of REFCNTs is configurable, but it's value
1318 	 * is set to the number of MPTs.  Since REFCNTs are used to support
1319 	 * shared memory regions, it is possible that we might require as
1320 	 * one REFCNT for every MPT.
1321 	 */
1322 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_REFCNT];
1323 	rsrc_pool->rsrc_type	 = TAVOR_REFCNT;
1324 	rsrc_pool->rsrc_loc	 = TAVOR_IN_SYSMEM;
1325 	rsrc_pool->rsrc_quantum	 = sizeof (tavor_sw_refcnt_t);
1326 	rsrc_pool->rsrc_state	 = state;
1327 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_REFCNT_CACHE);
1328 	hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mpt);
1329 	hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mpt);
1330 	hdl_info.swi_rsrcpool	 = rsrc_pool;
1331 	hdl_info.swi_constructor = tavor_rsrc_refcnt_constructor;
1332 	hdl_info.swi_destructor	 = tavor_rsrc_refcnt_destructor;
1333 	hdl_info.swi_rsrcname	 = rsrc_name;
1334 	hdl_info.swi_flags	 = TAVOR_SWHDL_KMEMCACHE_INIT;
1335 	status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1336 	if (status != DDI_SUCCESS) {
1337 		tavor_rsrc_fini(state, cleanup);
1338 		goto rsrcinitp2_fail;
1339 	}
1340 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL29;
1341 
1342 	/*
1343 	 * Initialize the resource pool for the MCG handles.  Notice that for
1344 	 * these MCG handles, we are allocating a table of structures (used to
1345 	 * keep track of the MCG entries that are being written to hardware
1346 	 * and to speed up multicast attach/detach operations).
1347 	 */
1348 	hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mcg);
1349 	hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mcg);
1350 	hdl_info.swi_flags = TAVOR_SWHDL_TABLE_INIT;
1351 	hdl_info.swi_prealloc_sz = sizeof (struct tavor_sw_mcg_list_s);
1352 	status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1353 	if (status != DDI_SUCCESS) {
1354 		tavor_rsrc_fini(state, cleanup);
1355 		goto rsrcinitp2_fail;
1356 	}
1357 	state->ts_mcghdl = hdl_info.swi_table_ptr;
1358 	cleanup = TAVOR_RSRC_CLEANUP_LEVEL30;
1359 
1360 	/*
1361 	 * Initialize the resource pools for all objects that exist in
1362 	 * UAR memory.  The only objects that are allocated from UAR memory
1363 	 * are the UAR pages which are used for holding Tavor hardware's
1364 	 * doorbell registers.
1365 	 */
1366 
1367 	/*
1368 	 * Initialize the resource pool for the UAR pages.  Notice
1369 	 * that the number of UARPGs is configurable.  The configured value
1370 	 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1371 	 * command) or the initialization will fail.  Note also that by
1372 	 * specifying the rsrc_start parameter in advance, we direct the
1373 	 * initialization routine not to attempt to allocated space from the
1374 	 * Tavor DDR vmem_arena.
1375 	 */
1376 	num  = ((uint64_t)1 << cfgprof->cp_log_num_uar);
1377 	max  = ((uint64_t)1 << (state->ts_devlim.log_max_uar_sz + 20 -
1378 	    PAGESHIFT));
1379 	num_prealloc = 0;
1380 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UARPG];
1381 	rsrc_pool->rsrc_type	  = TAVOR_UARPG;
1382 	rsrc_pool->rsrc_loc	  = TAVOR_IN_UAR;
1383 	rsrc_pool->rsrc_pool_size = (num << PAGESHIFT);
1384 	rsrc_pool->rsrc_shift	  = PAGESHIFT;
1385 	rsrc_pool->rsrc_quantum	  = PAGESIZE;
1386 	rsrc_pool->rsrc_align	  = PAGESIZE;
1387 	rsrc_pool->rsrc_state	  = state;
1388 	rsrc_pool->rsrc_start	  = (void *)state->ts_reg_uar_baseaddr;
1389 	TAVOR_RSRC_NAME(rsrc_name, TAVOR_UAR_VMEM);
1390 	entry_info.hwi_num	  = num;
1391 	entry_info.hwi_max	  = max;
1392 	entry_info.hwi_prealloc	  = num_prealloc;
1393 	entry_info.hwi_rsrcpool	  = rsrc_pool;
1394 	entry_info.hwi_rsrcname	  = rsrc_name;
1395 	status = tavor_rsrc_hw_entries_init(state, &entry_info);
1396 	if (status != DDI_SUCCESS) {
1397 		tavor_rsrc_fini(state, cleanup);
1398 		goto rsrcinitp2_fail;
1399 	}
1400 	cleanup = TAVOR_RSRC_CLEANUP_ALL;
1401 
1402 	kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
1403 	return (DDI_SUCCESS);
1404 
1405 rsrcinitp2_fail:
1406 	kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
1407 	return (status);
1408 }
1409 
1410 
1411 /*
1412  * tavor_rsrc_fini()
1413  *    Context: Only called from attach() and/or detach() path contexts
1414  */
1415 void
tavor_rsrc_fini(tavor_state_t * state,tavor_rsrc_cleanup_level_t clean)1416 tavor_rsrc_fini(tavor_state_t *state, tavor_rsrc_cleanup_level_t clean)
1417 {
1418 	tavor_rsrc_sw_hdl_info_t	hdl_info;
1419 	tavor_rsrc_hw_entry_info_t	entry_info;
1420 	tavor_rsrc_mbox_info_t		mbox_info;
1421 	tavor_cfg_profile_t		*cfgprof;
1422 
1423 	ASSERT(state != NULL);
1424 
1425 	cfgprof = state->ts_cfg_profile;
1426 
1427 	switch (clean) {
1428 	/*
1429 	 * If we add more resources that need to be cleaned up here, we should
1430 	 * ensure that TAVOR_RSRC_CLEANUP_ALL is still the first entry (i.e.
1431 	 * corresponds to the last resource allocated).
1432 	 */
1433 	case TAVOR_RSRC_CLEANUP_ALL:
1434 		/* Cleanup the UAR page resource pool */
1435 		entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UARPG];
1436 		tavor_rsrc_hw_entries_fini(state, &entry_info);
1437 		/* FALLTHROUGH */
1438 
1439 	case TAVOR_RSRC_CLEANUP_LEVEL30:
1440 		/* Cleanup the central MCG handle pointers list */
1441 		hdl_info.swi_rsrcpool  = NULL;
1442 		hdl_info.swi_table_ptr = state->ts_mcghdl;
1443 		hdl_info.swi_num =
1444 		    ((uint64_t)1 << cfgprof->cp_log_num_mcg);
1445 		hdl_info.swi_prealloc_sz = sizeof (struct tavor_sw_mcg_list_s);
1446 		tavor_rsrc_sw_handles_fini(state, &hdl_info);
1447 		/* FALLTHROUGH */
1448 
1449 	case TAVOR_RSRC_CLEANUP_LEVEL29:
1450 		/* Cleanup the reference count resource pool */
1451 		hdl_info.swi_rsrcpool  = &state->ts_rsrc_hdl[TAVOR_REFCNT];
1452 		hdl_info.swi_table_ptr = NULL;
1453 		tavor_rsrc_sw_handles_fini(state, &hdl_info);
1454 		/* FALLTHROUGH */
1455 
1456 	case TAVOR_RSRC_CLEANUP_LEVEL28:
1457 		/* Cleanup the QP handle resource pool */
1458 		hdl_info.swi_rsrcpool  = &state->ts_rsrc_hdl[TAVOR_QPHDL];
1459 		hdl_info.swi_table_ptr = state->ts_qphdl;
1460 		hdl_info.swi_num =
1461 		    ((uint64_t)1 << cfgprof->cp_log_num_qp);
1462 		hdl_info.swi_prealloc_sz = sizeof (tavor_qphdl_t);
1463 		tavor_rsrc_sw_handles_fini(state, &hdl_info);
1464 		/* FALLTHROUGH */
1465 
1466 	case TAVOR_RSRC_CLEANUP_LEVEL27:
1467 		/* Cleanup the address handle resource pool */
1468 		hdl_info.swi_rsrcpool  = &state->ts_rsrc_hdl[TAVOR_AHHDL];
1469 		hdl_info.swi_table_ptr = NULL;
1470 		tavor_rsrc_sw_handles_fini(state, &hdl_info);
1471 		/* FALLTHROUGH */
1472 
1473 	case TAVOR_RSRC_CLEANUP_LEVEL26:
1474 		/*
1475 		 * Cleanup the SRQ handle resource pool.
1476 		 *
1477 		 * Note: We only clean up if SRQ is enabled.  Otherwise we
1478 		 * simply fallthrough to the next resource cleanup.
1479 		 */
1480 		if (state->ts_cfg_profile->cp_srq_enable != 0) {
1481 			hdl_info.swi_rsrcpool  =
1482 			    &state->ts_rsrc_hdl[TAVOR_SRQHDL];
1483 			hdl_info.swi_table_ptr = state->ts_srqhdl;
1484 			hdl_info.swi_num =
1485 			    ((uint64_t)1 << cfgprof->cp_log_num_srq);
1486 			hdl_info.swi_prealloc_sz = sizeof (tavor_srqhdl_t);
1487 			tavor_rsrc_sw_handles_fini(state, &hdl_info);
1488 		}
1489 		/* FALLTHROUGH */
1490 
1491 	case TAVOR_RSRC_CLEANUP_LEVEL25:
1492 		/* Cleanup the CQ handle resource pool */
1493 		hdl_info.swi_rsrcpool  = &state->ts_rsrc_hdl[TAVOR_CQHDL];
1494 		hdl_info.swi_table_ptr = state->ts_cqhdl;
1495 		hdl_info.swi_num =
1496 		    ((uint64_t)1 << cfgprof->cp_log_num_cq);
1497 		hdl_info.swi_prealloc_sz = sizeof (tavor_cqhdl_t);
1498 		tavor_rsrc_sw_handles_fini(state, &hdl_info);
1499 		/* FALLTHROUGH */
1500 
1501 	case TAVOR_RSRC_CLEANUP_LEVEL24:
1502 		/* Cleanup the EQ handle resource pool */
1503 		hdl_info.swi_rsrcpool  = &state->ts_rsrc_hdl[TAVOR_EQHDL];
1504 		hdl_info.swi_table_ptr = NULL;
1505 		tavor_rsrc_sw_handles_fini(state, &hdl_info);
1506 		/* FALLTHROUGH */
1507 
1508 	case TAVOR_RSRC_CLEANUP_LEVEL23:
1509 		/* Cleanup the MR handle resource pool */
1510 		hdl_info.swi_rsrcpool  = &state->ts_rsrc_hdl[TAVOR_MRHDL];
1511 		hdl_info.swi_table_ptr = NULL;
1512 		tavor_rsrc_sw_handles_fini(state, &hdl_info);
1513 		/* FALLTHROUGH */
1514 
1515 	case TAVOR_RSRC_CLEANUP_LEVEL22:
1516 		/* Cleanup the PD handle resource pool */
1517 		hdl_info.swi_rsrcpool  = &state->ts_rsrc_hdl[TAVOR_PDHDL];
1518 		hdl_info.swi_table_ptr = NULL;
1519 		tavor_rsrc_pd_handles_fini(state, &hdl_info);
1520 		/* FALLTHROUGH */
1521 
1522 	case TAVOR_RSRC_CLEANUP_LEVEL21:
1523 		/* Cleanup the EQC table resource pool */
1524 		entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQC];
1525 		tavor_rsrc_hw_entries_fini(state, &entry_info);
1526 		/* FALLTHROUGH */
1527 
1528 	case TAVOR_RSRC_CLEANUP_LEVEL20:
1529 		/* Cleanup the MCG table resource pool */
1530 		entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MCG];
1531 		tavor_rsrc_hw_entries_fini(state, &entry_info);
1532 		/* FALLTHROUGH */
1533 
1534 	case TAVOR_RSRC_CLEANUP_LEVEL19:
1535 		/* Cleanup the outstanding command list  */
1536 		tavor_outstanding_cmdlist_fini(state);
1537 		/* FALLTHROUGH */
1538 
1539 	case TAVOR_RSRC_CLEANUP_LEVEL18:
1540 		/* Cleanup the "In" mailbox list  */
1541 		tavor_intr_inmbox_list_fini(state);
1542 		/* FALLTHROUGH */
1543 
1544 	case TAVOR_RSRC_CLEANUP_LEVEL17:
1545 		/* Cleanup the interrupt "In" mailbox resource pool */
1546 		mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[
1547 		    TAVOR_INTR_IN_MBOX];
1548 		tavor_rsrc_mbox_fini(state, &mbox_info);
1549 		/* FALLTHROUGH */
1550 
1551 	case TAVOR_RSRC_CLEANUP_LEVEL16:
1552 		/* Cleanup the "In" mailbox list  */
1553 		tavor_inmbox_list_fini(state);
1554 		/* FALLTHROUGH */
1555 
1556 	case TAVOR_RSRC_CLEANUP_LEVEL15:
1557 		/* Cleanup the "In" mailbox resource pool */
1558 		mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_IN_MBOX];
1559 		tavor_rsrc_mbox_fini(state, &mbox_info);
1560 		/* FALLTHROUGH */
1561 
1562 	case TAVOR_RSRC_CLEANUP_LEVEL14:
1563 		/*
1564 		 * Cleanup the SRQC table resource pool.
1565 		 *
1566 		 * Note: We only clean up if SRQ is enabled.  Otherwise we
1567 		 * simply fallthrough to the next resource cleanup.
1568 		 */
1569 		if (state->ts_cfg_profile->cp_srq_enable != 0) {
1570 			entry_info.hwi_rsrcpool =
1571 			    &state->ts_rsrc_hdl[TAVOR_SRQC];
1572 			tavor_rsrc_hw_entries_fini(state, &entry_info);
1573 		}
1574 		/* FALLTHROUGH */
1575 
1576 	case TAVOR_RSRC_CLEANUP_LEVEL13:
1577 		/* Cleanup the UAR scratch table resource pool */
1578 		entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR];
1579 		tavor_rsrc_hw_entries_fini(state, &entry_info);
1580 		/* FALLTHROUGH */
1581 
1582 	case TAVOR_RSRC_CLEANUP_LEVEL12:
1583 		/* Cleanup the UDAV table resource pool */
1584 		entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UDAV];
1585 		tavor_rsrc_hw_entries_fini(state, &entry_info);
1586 		/* FALLTHROUGH */
1587 
1588 	case TAVOR_RSRC_CLEANUP_LEVEL11:
1589 		/* Cleanup the EQPC table resource pool */
1590 		entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQPC];
1591 		tavor_rsrc_hw_entries_fini(state, &entry_info);
1592 		/* FALLTHROUGH */
1593 
1594 	case TAVOR_RSRC_CLEANUP_LEVEL10:
1595 		/* Cleanup the CQC table resource pool */
1596 		entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_CQC];
1597 		tavor_rsrc_hw_entries_fini(state, &entry_info);
1598 		/* FALLTHROUGH */
1599 
1600 	case TAVOR_RSRC_CLEANUP_LEVEL9:
1601 		/* Cleanup the RDB table resource pool */
1602 		entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_RDB];
1603 		tavor_rsrc_hw_entries_fini(state, &entry_info);
1604 		/* FALLTHROUGH */
1605 
1606 	case TAVOR_RSRC_CLEANUP_LEVEL8:
1607 		/* Cleanup the QPC table resource pool */
1608 		entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_QPC];
1609 		tavor_rsrc_hw_entries_fini(state, &entry_info);
1610 		/* FALLTHROUGH */
1611 
1612 	case TAVOR_RSRC_CLEANUP_LEVEL7:
1613 		/* Cleanup the MTT table resource pool */
1614 		entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MTT];
1615 		tavor_rsrc_hw_entries_fini(state, &entry_info);
1616 		/* FALLTHROUGH */
1617 
1618 	case TAVOR_RSRC_CLEANUP_LEVEL6:
1619 		/* Cleanup the MPT table resource pool */
1620 		entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MPT];
1621 		tavor_rsrc_hw_entries_fini(state, &entry_info);
1622 		/* FALLTHROUGH */
1623 
1624 	case TAVOR_RSRC_CLEANUP_LEVEL5:
1625 		/* Destroy the vmem arena for DDR memory */
1626 		vmem_destroy(state->ts_ddrvmem);
1627 		break;
1628 
1629 	/*
1630 	 * The cleanup below comes from the "Phase 1" initialization step.
1631 	 * (see tavor_rsrc_init_phase1() above)
1632 	 */
1633 	case TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE:
1634 		/* Cleanup the interrupt "Out" mailbox list  */
1635 		tavor_intr_outmbox_list_fini(state);
1636 		/* FALLTHROUGH */
1637 
1638 	case TAVOR_RSRC_CLEANUP_LEVEL3:
1639 		/* Cleanup the "Out" mailbox resource pool */
1640 		mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[
1641 		    TAVOR_INTR_OUT_MBOX];
1642 		tavor_rsrc_mbox_fini(state, &mbox_info);
1643 		/* FALLTHROUGH */
1644 
1645 	case TAVOR_RSRC_CLEANUP_LEVEL2:
1646 		/* Cleanup the "Out" mailbox list  */
1647 		tavor_outmbox_list_fini(state);
1648 		/* FALLTHROUGH */
1649 
1650 	case TAVOR_RSRC_CLEANUP_LEVEL1:
1651 		/* Cleanup the "Out" mailbox resource pool */
1652 		mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_OUT_MBOX];
1653 		tavor_rsrc_mbox_fini(state, &mbox_info);
1654 		/* FALLTHROUGH */
1655 
1656 	case TAVOR_RSRC_CLEANUP_LEVEL0:
1657 		/* Free the array of tavor_rsrc_pool_info_t's */
1658 		kmem_free(state->ts_rsrc_hdl, TAVOR_NUM_RESOURCES *
1659 		    sizeof (tavor_rsrc_pool_info_t));
1660 		kmem_cache_destroy(state->ts_rsrc_cache);
1661 		break;
1662 
1663 	default:
1664 		TAVOR_WARNING(state, "unexpected resource cleanup level");
1665 	}
1666 }
1667 
1668 
1669 /*
1670  * tavor_rsrc_mbox_init()
1671  *    Context: Only called from attach() path context
1672  */
1673 static int
tavor_rsrc_mbox_init(tavor_state_t * state,tavor_rsrc_mbox_info_t * info)1674 tavor_rsrc_mbox_init(tavor_state_t *state, tavor_rsrc_mbox_info_t *info)
1675 {
1676 	tavor_rsrc_pool_info_t	*rsrc_pool;
1677 	tavor_rsrc_priv_mbox_t	*priv;
1678 	vmem_t			*vmp;
1679 	uint64_t		offset;
1680 	uint_t			dma_xfer_mode;
1681 
1682 	ASSERT(state != NULL);
1683 	ASSERT(info != NULL);
1684 
1685 	rsrc_pool = info->mbi_rsrcpool;
1686 	ASSERT(rsrc_pool != NULL);
1687 
1688 	dma_xfer_mode = state->ts_cfg_profile->cp_streaming_consistent;
1689 
1690 	/* Allocate and initialize mailbox private structure */
1691 	priv = kmem_zalloc(sizeof (tavor_rsrc_priv_mbox_t), KM_SLEEP);
1692 	priv->pmb_dip		= state->ts_dip;
1693 	priv->pmb_acchdl	= state->ts_reg_ddrhdl;
1694 	priv->pmb_devaccattr	= state->ts_reg_accattr;
1695 	priv->pmb_xfer_mode	= dma_xfer_mode;
1696 
1697 	/*
1698 	 * Initialize many of the default DMA attributes.  Then set alignment
1699 	 * and scatter-gather restrictions specific for mailbox memory.
1700 	 */
1701 	tavor_dma_attr_init(&priv->pmb_dmaattr);
1702 	priv->pmb_dmaattr.dma_attr_align  = TAVOR_MBOX_ALIGN;
1703 	priv->pmb_dmaattr.dma_attr_sgllen = 1;
1704 
1705 	rsrc_pool->rsrc_private = priv;
1706 
1707 	/* Is object in DDR memory or system memory? */
1708 	if (rsrc_pool->rsrc_loc == TAVOR_IN_DDR) {
1709 		rsrc_pool->rsrc_ddr_offset = vmem_xalloc(state->ts_ddrvmem,
1710 		    rsrc_pool->rsrc_pool_size, rsrc_pool->rsrc_align,
1711 		    0, 0, NULL, NULL, VM_SLEEP);
1712 		if (rsrc_pool->rsrc_ddr_offset == NULL) {
1713 			/* Unable to alloc space for mailboxes */
1714 			kmem_free(priv, sizeof (tavor_rsrc_priv_mbox_t));
1715 			return (DDI_FAILURE);
1716 		}
1717 
1718 		/* Calculate offset and starting point (in DDR) */
1719 		offset = ((uintptr_t)rsrc_pool->rsrc_ddr_offset -
1720 		    state->ts_ddr.ddr_baseaddr);
1721 		rsrc_pool->rsrc_start =
1722 		    (void *)(uintptr_t)((uintptr_t)state->ts_reg_ddr_baseaddr +
1723 		    offset);
1724 
1725 		/* Create new vmem arena for the mailboxes */
1726 		vmp = vmem_create(info->mbi_rsrcname,
1727 		    rsrc_pool->rsrc_ddr_offset, rsrc_pool->rsrc_pool_size,
1728 		    rsrc_pool->rsrc_quantum, NULL, NULL, NULL, 0, VM_SLEEP);
1729 		if (vmp == NULL) {
1730 			/* Unable to create vmem arena */
1731 			vmem_xfree(state->ts_ddrvmem,
1732 			    rsrc_pool->rsrc_ddr_offset,
1733 			    rsrc_pool->rsrc_pool_size);
1734 			kmem_free(priv, sizeof (tavor_rsrc_priv_mbox_t));
1735 			return (DDI_FAILURE);
1736 		}
1737 		rsrc_pool->rsrc_vmp = vmp;
1738 	} else {
1739 		rsrc_pool->rsrc_ddr_offset = NULL;
1740 		rsrc_pool->rsrc_start = NULL;
1741 		rsrc_pool->rsrc_vmp = NULL;
1742 	}
1743 
1744 	return (DDI_SUCCESS);
1745 }
1746 
1747 
1748 /*
1749  * tavor_rsrc_mbox_fini()
1750  *    Context: Only called from attach() and/or detach() path contexts
1751  */
1752 static void
tavor_rsrc_mbox_fini(tavor_state_t * state,tavor_rsrc_mbox_info_t * info)1753 tavor_rsrc_mbox_fini(tavor_state_t *state, tavor_rsrc_mbox_info_t *info)
1754 {
1755 	tavor_rsrc_pool_info_t	*rsrc_pool;
1756 
1757 	ASSERT(state != NULL);
1758 	ASSERT(info != NULL);
1759 
1760 	rsrc_pool = info->mbi_rsrcpool;
1761 	ASSERT(rsrc_pool != NULL);
1762 
1763 	/* If mailboxes are DDR memory, then destroy and free up vmem */
1764 	if (rsrc_pool->rsrc_loc == TAVOR_IN_DDR) {
1765 
1766 		/* Destroy the specially created mbox vmem arena */
1767 		vmem_destroy(rsrc_pool->rsrc_vmp);
1768 
1769 		/* Free up the region from the ddr_vmem arena */
1770 		vmem_xfree(state->ts_ddrvmem, rsrc_pool->rsrc_ddr_offset,
1771 		    rsrc_pool->rsrc_pool_size);
1772 	}
1773 
1774 	/* Free up the private struct */
1775 	kmem_free(rsrc_pool->rsrc_private, sizeof (tavor_rsrc_priv_mbox_t));
1776 }
1777 
1778 
1779 /*
1780  * tavor_rsrc_hw_entries_init()
1781  *    Context: Only called from attach() path context
1782  */
1783 static int
tavor_rsrc_hw_entries_init(tavor_state_t * state,tavor_rsrc_hw_entry_info_t * info)1784 tavor_rsrc_hw_entries_init(tavor_state_t *state,
1785     tavor_rsrc_hw_entry_info_t *info)
1786 {
1787 	tavor_rsrc_pool_info_t	*rsrc_pool;
1788 	tavor_rsrc_t		*rsvd_rsrc = NULL;
1789 	vmem_t			*vmp;
1790 	uint64_t		num_hwentry, max_hwentry, num_prealloc;
1791 	uint64_t		offset;
1792 	int			status;
1793 
1794 	ASSERT(state != NULL);
1795 	ASSERT(info != NULL);
1796 
1797 	rsrc_pool	= info->hwi_rsrcpool;
1798 	ASSERT(rsrc_pool != NULL);
1799 	num_hwentry	= info->hwi_num;
1800 	max_hwentry	= info->hwi_max;
1801 	num_prealloc	= info->hwi_prealloc;
1802 
1803 	/* Make sure number of HW entries makes sense */
1804 	if (num_hwentry > max_hwentry) {
1805 		return (DDI_FAILURE);
1806 	}
1807 
1808 	/*
1809 	 * Determine if we need to allocate DDR space to set up the
1810 	 * "rsrc_start" pointer.  Not necessary if "rsrc_start" has already
1811 	 * been initialized (as is the case for the UAR page init).
1812 	 */
1813 	if (rsrc_pool->rsrc_start == NULL) {
1814 		/* Make sure HW entries table is aligned as specified */
1815 		rsrc_pool->rsrc_ddr_offset = vmem_xalloc(state->ts_ddrvmem,
1816 		    rsrc_pool->rsrc_pool_size, rsrc_pool->rsrc_align,
1817 		    0, 0, NULL, NULL, VM_NOSLEEP | VM_FIRSTFIT);
1818 		if (rsrc_pool->rsrc_ddr_offset == NULL) {
1819 			/* Unable to alloc space for aligned HW table */
1820 			return (DDI_FAILURE);
1821 		}
1822 
1823 		/* Calculate offset and starting point (in DDR) */
1824 		offset = ((uintptr_t)rsrc_pool->rsrc_ddr_offset -
1825 		    state->ts_ddr.ddr_baseaddr);
1826 		rsrc_pool->rsrc_start =
1827 		    (void *)(uintptr_t)((uintptr_t)state->ts_reg_ddr_baseaddr +
1828 		    offset);
1829 	} else {
1830 		rsrc_pool->rsrc_ddr_offset = rsrc_pool->rsrc_start;
1831 	}
1832 
1833 	/*
1834 	 * Create new vmem arena for the HW entries table (if rsrc_quantum
1835 	 * is non-zero).  Otherwise if rsrc_quantum is zero, then these HW
1836 	 * entries are not going to be dynamically allocatable (i.e. they
1837 	 * won't be allocated/freed through tavor_rsrc_alloc/free).  This
1838 	 * latter option is used for EQPC and UARSCR resource which are, in
1839 	 * fact, managed by the Tavor hardware.
1840 	 */
1841 	if (rsrc_pool->rsrc_quantum != 0) {
1842 		vmp = vmem_create(info->hwi_rsrcname,
1843 		    rsrc_pool->rsrc_ddr_offset, rsrc_pool->rsrc_pool_size,
1844 		    rsrc_pool->rsrc_quantum, NULL, NULL, NULL, 0, VM_SLEEP);
1845 		if (vmp == NULL) {
1846 			/* Unable to create vmem arena */
1847 			if (rsrc_pool->rsrc_ddr_offset !=
1848 			    rsrc_pool->rsrc_start) {
1849 				vmem_xfree(state->ts_ddrvmem,
1850 				    rsrc_pool->rsrc_ddr_offset,
1851 				    rsrc_pool->rsrc_pool_size);
1852 			}
1853 			return (DDI_FAILURE);
1854 		}
1855 		rsrc_pool->rsrc_vmp = vmp;
1856 	} else {
1857 		rsrc_pool->rsrc_vmp = NULL;
1858 	}
1859 
1860 	/* The first HW entries may be reserved by Tavor firmware */
1861 	if (num_prealloc != 0) {
1862 		status = tavor_rsrc_alloc(state, rsrc_pool->rsrc_type,
1863 		    num_prealloc, TAVOR_SLEEP, &rsvd_rsrc);
1864 		if (status != DDI_SUCCESS) {
1865 			/* Unable to preallocate the reserved HW entries */
1866 			if (rsrc_pool->rsrc_vmp != NULL) {
1867 				vmem_destroy(rsrc_pool->rsrc_vmp);
1868 			}
1869 			if (rsrc_pool->rsrc_ddr_offset !=
1870 			    rsrc_pool->rsrc_start) {
1871 				vmem_xfree(state->ts_ddrvmem,
1872 				    rsrc_pool->rsrc_ddr_offset,
1873 				    rsrc_pool->rsrc_pool_size);
1874 			}
1875 			return (DDI_FAILURE);
1876 		}
1877 	}
1878 	rsrc_pool->rsrc_private = rsvd_rsrc;
1879 
1880 	return (DDI_SUCCESS);
1881 }
1882 
1883 
1884 /*
1885  * tavor_rsrc_hw_entries_fini()
1886  *    Context: Only called from attach() and/or detach() path contexts
1887  */
1888 static void
tavor_rsrc_hw_entries_fini(tavor_state_t * state,tavor_rsrc_hw_entry_info_t * info)1889 tavor_rsrc_hw_entries_fini(tavor_state_t *state,
1890     tavor_rsrc_hw_entry_info_t *info)
1891 {
1892 	tavor_rsrc_pool_info_t	*rsrc_pool;
1893 	tavor_rsrc_t		*rsvd_rsrc;
1894 
1895 	ASSERT(state != NULL);
1896 	ASSERT(info != NULL);
1897 
1898 	rsrc_pool = info->hwi_rsrcpool;
1899 	ASSERT(rsrc_pool != NULL);
1900 
1901 	/* Free up any "reserved" (i.e. preallocated) HW entries */
1902 	rsvd_rsrc = (tavor_rsrc_t *)rsrc_pool->rsrc_private;
1903 	if (rsvd_rsrc != NULL) {
1904 		tavor_rsrc_free(state, &rsvd_rsrc);
1905 	}
1906 
1907 	/*
1908 	 * If we've actually setup a vmem arena for the HW entries, then
1909 	 * destroy it now
1910 	 */
1911 	if (rsrc_pool->rsrc_vmp != NULL) {
1912 		vmem_destroy(rsrc_pool->rsrc_vmp);
1913 	}
1914 
1915 	/*
1916 	 * Determine if a region was allocated from the tavor_ddr_vmem
1917 	 * arena (and free it up if necessary)
1918 	 */
1919 	if (rsrc_pool->rsrc_ddr_offset != rsrc_pool->rsrc_start) {
1920 		vmem_xfree(state->ts_ddrvmem, rsrc_pool->rsrc_ddr_offset,
1921 		    rsrc_pool->rsrc_pool_size);
1922 	}
1923 }
1924 
1925 
1926 /*
1927  * tavor_rsrc_sw_handles_init()
1928  *    Context: Only called from attach() path context
1929  */
1930 /* ARGSUSED */
1931 static int
tavor_rsrc_sw_handles_init(tavor_state_t * state,tavor_rsrc_sw_hdl_info_t * info)1932 tavor_rsrc_sw_handles_init(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
1933 {
1934 	tavor_rsrc_pool_info_t	*rsrc_pool;
1935 	uint64_t		num_swhdl, max_swhdl, prealloc_sz;
1936 
1937 	ASSERT(state != NULL);
1938 	ASSERT(info != NULL);
1939 
1940 	rsrc_pool	= info->swi_rsrcpool;
1941 	ASSERT(rsrc_pool != NULL);
1942 	num_swhdl	= info->swi_num;
1943 	max_swhdl	= info->swi_max;
1944 	prealloc_sz	= info->swi_prealloc_sz;
1945 
1946 	/* Make sure number of SW handles makes sense */
1947 	if (num_swhdl > max_swhdl) {
1948 		return (DDI_FAILURE);
1949 	}
1950 
1951 	/*
1952 	 * Depending on the flags parameter, create a kmem_cache for some
1953 	 * number of software handle structures.  Note: kmem_cache_create()
1954 	 * will SLEEP until successful.
1955 	 */
1956 	if (info->swi_flags & TAVOR_SWHDL_KMEMCACHE_INIT) {
1957 		rsrc_pool->rsrc_private = kmem_cache_create(
1958 		    info->swi_rsrcname, rsrc_pool->rsrc_quantum, 0,
1959 		    info->swi_constructor, info->swi_destructor, NULL,
1960 		    rsrc_pool->rsrc_state, NULL, 0);
1961 	}
1962 
1963 	/* Allocate the central list of SW handle pointers */
1964 	if (info->swi_flags & TAVOR_SWHDL_TABLE_INIT) {
1965 		info->swi_table_ptr = kmem_zalloc(num_swhdl * prealloc_sz,
1966 		    KM_SLEEP);
1967 	}
1968 
1969 	return (DDI_SUCCESS);
1970 }
1971 
1972 
1973 /*
1974  * tavor_rsrc_sw_handles_fini()
1975  *    Context: Only called from attach() and/or detach() path contexts
1976  */
1977 /* ARGSUSED */
1978 static void
tavor_rsrc_sw_handles_fini(tavor_state_t * state,tavor_rsrc_sw_hdl_info_t * info)1979 tavor_rsrc_sw_handles_fini(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
1980 {
1981 	tavor_rsrc_pool_info_t	*rsrc_pool;
1982 	uint64_t		num_swhdl, prealloc_sz;
1983 
1984 	ASSERT(state != NULL);
1985 	ASSERT(info != NULL);
1986 
1987 	rsrc_pool	= info->swi_rsrcpool;
1988 	num_swhdl	= info->swi_num;
1989 	prealloc_sz	= info->swi_prealloc_sz;
1990 
1991 	/*
1992 	 * If a "software handle" kmem_cache exists for this resource, then
1993 	 * destroy it now
1994 	 */
1995 	if (rsrc_pool != NULL) {
1996 		kmem_cache_destroy(rsrc_pool->rsrc_private);
1997 	}
1998 
1999 	/* Free up this central list of SW handle pointers */
2000 	if (info->swi_table_ptr != NULL) {
2001 		kmem_free(info->swi_table_ptr, num_swhdl * prealloc_sz);
2002 	}
2003 }
2004 
2005 
2006 /*
2007  * tavor_rsrc_pd_handles_init()
2008  *    Context: Only called from attach() path context
2009  */
2010 static int
tavor_rsrc_pd_handles_init(tavor_state_t * state,tavor_rsrc_sw_hdl_info_t * info)2011 tavor_rsrc_pd_handles_init(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
2012 {
2013 	tavor_rsrc_pool_info_t	*rsrc_pool;
2014 	vmem_t			*vmp;
2015 	char			vmem_name[TAVOR_RSRC_NAME_MAXLEN];
2016 	int			status;
2017 
2018 	ASSERT(state != NULL);
2019 	ASSERT(info != NULL);
2020 
2021 	rsrc_pool = info->swi_rsrcpool;
2022 	ASSERT(rsrc_pool != NULL);
2023 
2024 	/* Initialize the resource pool for software handle table */
2025 	status = tavor_rsrc_sw_handles_init(state, info);
2026 	if (status != DDI_SUCCESS) {
2027 		return (DDI_FAILURE);
2028 	}
2029 
2030 	/* Build vmem arena name from Tavor instance */
2031 	TAVOR_RSRC_NAME(vmem_name, TAVOR_PDHDL_VMEM);
2032 
2033 	/* Create new vmem arena for PD numbers */
2034 	vmp = vmem_create(vmem_name, (caddr_t)1, info->swi_num, 1, NULL,
2035 	    NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
2036 	if (vmp == NULL) {
2037 		/* Unable to create vmem arena */
2038 		info->swi_table_ptr = NULL;
2039 		tavor_rsrc_sw_handles_fini(state, info);
2040 		return (DDI_FAILURE);
2041 	}
2042 	rsrc_pool->rsrc_vmp = vmp;
2043 
2044 	return (DDI_SUCCESS);
2045 }
2046 
2047 
2048 /*
2049  * tavor_rsrc_pd_handles_fini()
2050  *    Context: Only called from attach() and/or detach() path contexts
2051  */
2052 static void
tavor_rsrc_pd_handles_fini(tavor_state_t * state,tavor_rsrc_sw_hdl_info_t * info)2053 tavor_rsrc_pd_handles_fini(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
2054 {
2055 	tavor_rsrc_pool_info_t	*rsrc_pool;
2056 
2057 	ASSERT(state != NULL);
2058 	ASSERT(info != NULL);
2059 
2060 	rsrc_pool = info->swi_rsrcpool;
2061 
2062 	/* Destroy the specially created UAR scratch table vmem arena */
2063 	vmem_destroy(rsrc_pool->rsrc_vmp);
2064 
2065 	/* Destroy the "tavor_sw_pd_t" kmem_cache */
2066 	tavor_rsrc_sw_handles_fini(state, info);
2067 }
2068 
2069 
2070 /*
2071  * tavor_rsrc_mbox_alloc()
2072  *    Context: Only called from attach() path context
2073  */
2074 static int
tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t * pool_info,uint_t num,tavor_rsrc_t * hdl)2075 tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t num,
2076     tavor_rsrc_t *hdl)
2077 {
2078 	tavor_rsrc_priv_mbox_t	*priv;
2079 	void			*addr;
2080 	caddr_t			kaddr;
2081 	uint64_t		offset;
2082 	size_t			real_len, temp_len;
2083 	int			status;
2084 
2085 	ASSERT(pool_info != NULL);
2086 	ASSERT(hdl != NULL);
2087 
2088 	/* Get the private pointer for the mailboxes */
2089 	priv = pool_info->rsrc_private;
2090 	ASSERT(priv != NULL);
2091 
2092 	/*
2093 	 * Allocate a DMA handle for the mailbox.  This will be used for
2094 	 * two purposes (potentially).  First, it could be used below in
2095 	 * the call to ddi_dma_mem_alloc() - if the mailbox is to come from
2096 	 * system memory.  Second, it is definitely used later to bind
2097 	 * the mailbox for DMA access from/by the hardware.
2098 	 */
2099 	status = ddi_dma_alloc_handle(priv->pmb_dip, &priv->pmb_dmaattr,
2100 	    DDI_DMA_SLEEP, NULL, &hdl->tr_dmahdl);
2101 	if (status != DDI_SUCCESS) {
2102 		return (DDI_FAILURE);
2103 	}
2104 
2105 	/* Is mailbox in DDR memory or system memory? */
2106 	if (pool_info->rsrc_loc == TAVOR_IN_DDR) {
2107 		/* Use vmem_alloc() to get DDR address of mbox */
2108 		hdl->tr_len = (num * pool_info->rsrc_quantum);
2109 		addr = vmem_alloc(pool_info->rsrc_vmp, hdl->tr_len,
2110 		    VM_SLEEP);
2111 		if (addr == NULL) {
2112 			/* No more DDR available for mailbox entries */
2113 			ddi_dma_free_handle(&hdl->tr_dmahdl);
2114 			return (DDI_FAILURE);
2115 		}
2116 		hdl->tr_acchdl = priv->pmb_acchdl;
2117 
2118 		/* Calculate kernel virtual address (from the DDR offset) */
2119 		offset = ((uintptr_t)addr -
2120 		    (uintptr_t)pool_info->rsrc_ddr_offset);
2121 		hdl->tr_addr = (void *)(uintptr_t)(offset +
2122 		    (uintptr_t)pool_info->rsrc_start);
2123 
2124 	} else { /* TAVOR_IN_SYSMEM */
2125 
2126 		/* Use ddi_dma_mem_alloc() to get memory for mailbox */
2127 		temp_len = (num * pool_info->rsrc_quantum);
2128 		status = ddi_dma_mem_alloc(hdl->tr_dmahdl, temp_len,
2129 		    &priv->pmb_devaccattr, priv->pmb_xfer_mode, DDI_DMA_SLEEP,
2130 		    NULL, &kaddr, &real_len, &hdl->tr_acchdl);
2131 		if (status != DDI_SUCCESS) {
2132 			/* No more sys memory available for mailbox entries */
2133 			ddi_dma_free_handle(&hdl->tr_dmahdl);
2134 			return (DDI_FAILURE);
2135 		}
2136 		hdl->tr_addr = (void *)kaddr;
2137 		hdl->tr_len  = real_len;
2138 	}
2139 
2140 	return (DDI_SUCCESS);
2141 }
2142 
2143 
2144 /*
2145  * tavor_rsrc_mbox_free()
2146  *    Context: Can be called from interrupt or base context.
2147  */
2148 static void
tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t * pool_info,tavor_rsrc_t * hdl)2149 tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
2150 {
2151 	void		*addr;
2152 	uint64_t	offset;
2153 
2154 	ASSERT(pool_info != NULL);
2155 	ASSERT(hdl != NULL);
2156 
2157 	/* Is mailbox in DDR memory or system memory? */
2158 	if (pool_info->rsrc_loc == TAVOR_IN_DDR) {
2159 
2160 		/* Calculate the allocated address (the mbox's DDR offset) */
2161 		offset = ((uintptr_t)hdl->tr_addr -
2162 		    (uintptr_t)pool_info->rsrc_start);
2163 		addr = (void *)(uintptr_t)(offset +
2164 		    (uintptr_t)pool_info->rsrc_ddr_offset);
2165 
2166 		/* Use vmem_free() to free up DDR memory for mailbox */
2167 		vmem_free(pool_info->rsrc_vmp, addr, hdl->tr_len);
2168 
2169 	} else { /* TAVOR_IN_SYSMEM */
2170 
2171 		/* Use ddi_dma_mem_free() to free up sys memory for mailbox */
2172 		ddi_dma_mem_free(&hdl->tr_acchdl);
2173 	}
2174 
2175 	/* Free the DMA handle for the mailbox */
2176 	ddi_dma_free_handle(&hdl->tr_dmahdl);
2177 }
2178 
2179 
2180 /*
2181  * tavor_rsrc_hw_entry_alloc()
2182  *    Context: Can be called from interrupt or base context.
2183  */
2184 static int
tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t * pool_info,uint_t num,uint_t num_align,ddi_acc_handle_t acc_handle,uint_t sleepflag,tavor_rsrc_t * hdl)2185 tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t num,
2186     uint_t num_align, ddi_acc_handle_t acc_handle, uint_t sleepflag,
2187     tavor_rsrc_t *hdl)
2188 {
2189 	void		*addr;
2190 	uint64_t	offset;
2191 	uint32_t	align;
2192 	int		flag;
2193 
2194 	ASSERT(pool_info != NULL);
2195 	ASSERT(hdl != NULL);
2196 
2197 	/*
2198 	 * Tavor hardware entries (QPC, CQC, EQC, MPT, MTT, etc.) do not
2199 	 * use dma_handle (because they are in Tavor locally attached DDR
2200 	 * memory) and, generally, don't use the acc_handle (because the
2201 	 * entries are not directly accessed by software).  The exceptions
2202 	 * to this rule are the UARPG and UDAV entries.
2203 	 */
2204 
2205 	/*
2206 	 * Use vmem_xalloc() to get a properly aligned pointer (based on
2207 	 * the number requested) to the HW entry(ies).  This handles the
2208 	 * cases (for special QPCs and for RDB entries) where we need more
2209 	 * than one and need to ensure that they are properly aligned.
2210 	 */
2211 	flag = (sleepflag == TAVOR_SLEEP) ? VM_SLEEP : VM_NOSLEEP;
2212 	hdl->tr_len = (num * pool_info->rsrc_quantum);
2213 	align	    = (num_align * pool_info->rsrc_quantum);
2214 	addr = vmem_xalloc(pool_info->rsrc_vmp, hdl->tr_len,
2215 	    align, 0, 0, NULL, NULL, flag | VM_FIRSTFIT);
2216 	if (addr == NULL) {
2217 		/* No more HW entries available */
2218 		return (DDI_FAILURE);
2219 	}
2220 
2221 	/* If an access handle was provided, fill it in */
2222 	if (acc_handle != 0) {
2223 		hdl->tr_acchdl = acc_handle;
2224 	}
2225 
2226 	/* Calculate vaddr and HW table index (from the DDR offset) */
2227 	offset = ((uintptr_t)addr - (uintptr_t)pool_info->rsrc_ddr_offset);
2228 	hdl->tr_addr = (void *)(uintptr_t)(offset +
2229 	    (uintptr_t)pool_info->rsrc_start);
2230 	hdl->tr_indx = (offset >> pool_info->rsrc_shift);
2231 
2232 	return (DDI_SUCCESS);
2233 }
2234 
2235 
2236 /*
2237  * tavor_rsrc_hw_entry_free()
2238  *    Context: Can be called from interrupt or base context.
2239  */
2240 static void
tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t * pool_info,tavor_rsrc_t * hdl)2241 tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
2242 {
2243 	void		*addr;
2244 	uint64_t	offset;
2245 
2246 	ASSERT(pool_info != NULL);
2247 	ASSERT(hdl != NULL);
2248 
2249 	/* Calculate the allocated address (the entry's DDR offset) */
2250 	offset = ((uintptr_t)hdl->tr_addr - (uintptr_t)pool_info->rsrc_start);
2251 	addr   = (void *)(uintptr_t)(offset +
2252 	    (uintptr_t)pool_info->rsrc_ddr_offset);
2253 
2254 	/* Use vmem_xfree() to free up the HW table entry */
2255 	vmem_xfree(pool_info->rsrc_vmp, addr, hdl->tr_len);
2256 }
2257 
2258 
2259 /*
2260  * tavor_rsrc_swhdl_alloc()
2261  *    Context: Can be called from interrupt or base context.
2262  */
2263 static int
tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t * pool_info,uint_t sleepflag,tavor_rsrc_t * hdl)2264 tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t sleepflag,
2265     tavor_rsrc_t *hdl)
2266 {
2267 	void	*addr;
2268 	int	flag;
2269 
2270 	ASSERT(pool_info != NULL);
2271 	ASSERT(hdl != NULL);
2272 
2273 	/* Allocate the software handle structure */
2274 	flag = (sleepflag == TAVOR_SLEEP) ? KM_SLEEP : KM_NOSLEEP;
2275 	addr = kmem_cache_alloc(pool_info->rsrc_private, flag);
2276 	if (addr == NULL) {
2277 		return (DDI_FAILURE);
2278 	}
2279 	hdl->tr_len  = pool_info->rsrc_quantum;
2280 	hdl->tr_addr = addr;
2281 
2282 	return (DDI_SUCCESS);
2283 }
2284 
2285 
2286 /*
2287  * tavor_rsrc_swhdl_free()
2288  *    Context: Can be called from interrupt or base context.
2289  */
2290 static void
tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t * pool_info,tavor_rsrc_t * hdl)2291 tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
2292 {
2293 	ASSERT(pool_info != NULL);
2294 	ASSERT(hdl != NULL);
2295 
2296 	/* Free the software handle structure */
2297 	kmem_cache_free(pool_info->rsrc_private, hdl->tr_addr);
2298 }
2299 
2300 
2301 /*
2302  * tavor_rsrc_pdhdl_alloc()
2303  *    Context: Can be called from interrupt or base context.
2304  */
2305 static int
tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t * pool_info,uint_t sleepflag,tavor_rsrc_t * hdl)2306 tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t sleepflag,
2307     tavor_rsrc_t *hdl)
2308 {
2309 	tavor_pdhdl_t	addr;
2310 	void		*tmpaddr;
2311 	int		flag, status;
2312 
2313 	ASSERT(pool_info != NULL);
2314 	ASSERT(hdl != NULL);
2315 
2316 	/* Allocate the software handle */
2317 	status = tavor_rsrc_swhdl_alloc(pool_info, sleepflag, hdl);
2318 	if (status != DDI_SUCCESS) {
2319 		return (DDI_FAILURE);
2320 	}
2321 	addr = (tavor_pdhdl_t)hdl->tr_addr;
2322 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*addr))
2323 
2324 	/* Allocate a PD number for the handle */
2325 	flag = (sleepflag == TAVOR_SLEEP) ? VM_SLEEP : VM_NOSLEEP;
2326 	tmpaddr = vmem_alloc(pool_info->rsrc_vmp, 1, flag);
2327 	if (tmpaddr == NULL) {
2328 		/* No more PD number entries available */
2329 		tavor_rsrc_swhdl_free(pool_info, hdl);
2330 		return (DDI_FAILURE);
2331 	}
2332 	addr->pd_pdnum = (uint32_t)(uintptr_t)tmpaddr;
2333 	addr->pd_rsrcp = hdl;
2334 	hdl->tr_indx   = addr->pd_pdnum;
2335 
2336 	return (DDI_SUCCESS);
2337 }
2338 
2339 
2340 /*
2341  * tavor_rsrc_pdhdl_free()
2342  *    Context: Can be called from interrupt or base context.
2343  */
2344 static void
tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t * pool_info,tavor_rsrc_t * hdl)2345 tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
2346 {
2347 	ASSERT(pool_info != NULL);
2348 	ASSERT(hdl != NULL);
2349 
2350 	/* Use vmem_free() to free up the PD number */
2351 	vmem_free(pool_info->rsrc_vmp, (void *)(uintptr_t)hdl->tr_indx, 1);
2352 
2353 	/* Free the software handle structure */
2354 	tavor_rsrc_swhdl_free(pool_info, hdl);
2355 }
2356 
2357 
2358 /*
2359  * tavor_rsrc_pdhdl_constructor()
2360  *    Context: Can be called from interrupt or base context.
2361  */
2362 /* ARGSUSED */
2363 static int
tavor_rsrc_pdhdl_constructor(void * pd,void * priv,int flags)2364 tavor_rsrc_pdhdl_constructor(void *pd, void *priv, int flags)
2365 {
2366 	tavor_pdhdl_t	pdhdl;
2367 	tavor_state_t	*state;
2368 
2369 	pdhdl = (tavor_pdhdl_t)pd;
2370 	state = (tavor_state_t *)priv;
2371 
2372 	mutex_init(&pdhdl->pd_lock, NULL, MUTEX_DRIVER,
2373 	    DDI_INTR_PRI(state->ts_intrmsi_pri));
2374 
2375 	return (DDI_SUCCESS);
2376 }
2377 
2378 
2379 /*
2380  * tavor_rsrc_pdhdl_destructor()
2381  *    Context: Can be called from interrupt or base context.
2382  */
2383 /* ARGSUSED */
2384 static void
tavor_rsrc_pdhdl_destructor(void * pd,void * priv)2385 tavor_rsrc_pdhdl_destructor(void *pd, void *priv)
2386 {
2387 	tavor_pdhdl_t	pdhdl;
2388 
2389 	pdhdl = (tavor_pdhdl_t)pd;
2390 
2391 	mutex_destroy(&pdhdl->pd_lock);
2392 }
2393 
2394 
2395 /*
2396  * tavor_rsrc_cqhdl_constructor()
2397  *    Context: Can be called from interrupt or base context.
2398  */
2399 /* ARGSUSED */
2400 static int
tavor_rsrc_cqhdl_constructor(void * cq,void * priv,int flags)2401 tavor_rsrc_cqhdl_constructor(void *cq, void *priv, int flags)
2402 {
2403 	tavor_cqhdl_t	cqhdl;
2404 	tavor_state_t	*state;
2405 
2406 	cqhdl = (tavor_cqhdl_t)cq;
2407 	state = (tavor_state_t *)priv;
2408 
2409 	mutex_init(&cqhdl->cq_lock, NULL, MUTEX_DRIVER,
2410 	    DDI_INTR_PRI(state->ts_intrmsi_pri));
2411 	mutex_init(&cqhdl->cq_wrid_wqhdr_lock, NULL, MUTEX_DRIVER,
2412 	    DDI_INTR_PRI(state->ts_intrmsi_pri));
2413 
2414 	return (DDI_SUCCESS);
2415 }
2416 
2417 
2418 /*
2419  * tavor_rsrc_cqhdl_destructor()
2420  *    Context: Can be called from interrupt or base context.
2421  */
2422 /* ARGSUSED */
2423 static void
tavor_rsrc_cqhdl_destructor(void * cq,void * priv)2424 tavor_rsrc_cqhdl_destructor(void *cq, void *priv)
2425 {
2426 	tavor_cqhdl_t	cqhdl;
2427 
2428 	cqhdl = (tavor_cqhdl_t)cq;
2429 
2430 	mutex_destroy(&cqhdl->cq_wrid_wqhdr_lock);
2431 	mutex_destroy(&cqhdl->cq_lock);
2432 }
2433 
2434 
2435 /*
2436  * tavor_rsrc_qphdl_constructor()
2437  *    Context: Can be called from interrupt or base context.
2438  */
2439 /* ARGSUSED */
2440 static int
tavor_rsrc_qphdl_constructor(void * qp,void * priv,int flags)2441 tavor_rsrc_qphdl_constructor(void *qp, void *priv, int flags)
2442 {
2443 	tavor_qphdl_t	qphdl;
2444 	tavor_state_t	*state;
2445 
2446 	qphdl = (tavor_qphdl_t)qp;
2447 	state = (tavor_state_t *)priv;
2448 
2449 	mutex_init(&qphdl->qp_lock, NULL, MUTEX_DRIVER,
2450 	    DDI_INTR_PRI(state->ts_intrmsi_pri));
2451 
2452 	return (DDI_SUCCESS);
2453 }
2454 
2455 
2456 /*
2457  * tavor_rsrc_qphdl_destructor()
2458  *    Context: Can be called from interrupt or base context.
2459  */
2460 /* ARGSUSED */
2461 static void
tavor_rsrc_qphdl_destructor(void * qp,void * priv)2462 tavor_rsrc_qphdl_destructor(void *qp, void *priv)
2463 {
2464 	tavor_qphdl_t	qphdl;
2465 
2466 	qphdl = (tavor_qphdl_t)qp;
2467 
2468 	mutex_destroy(&qphdl->qp_lock);
2469 }
2470 
2471 
2472 /*
2473  * tavor_rsrc_srqhdl_constructor()
2474  *    Context: Can be called from interrupt or base context.
2475  */
2476 /* ARGSUSED */
2477 static int
tavor_rsrc_srqhdl_constructor(void * srq,void * priv,int flags)2478 tavor_rsrc_srqhdl_constructor(void *srq, void *priv, int flags)
2479 {
2480 	tavor_srqhdl_t	srqhdl;
2481 	tavor_state_t	*state;
2482 
2483 	srqhdl = (tavor_srqhdl_t)srq;
2484 	state = (tavor_state_t *)priv;
2485 
2486 	mutex_init(&srqhdl->srq_lock, NULL, MUTEX_DRIVER,
2487 	    DDI_INTR_PRI(state->ts_intrmsi_pri));
2488 
2489 	return (DDI_SUCCESS);
2490 }
2491 
2492 
2493 /*
2494  * tavor_rsrc_srqhdl_destructor()
2495  *    Context: Can be called from interrupt or base context.
2496  */
2497 /* ARGSUSED */
2498 static void
tavor_rsrc_srqhdl_destructor(void * srq,void * priv)2499 tavor_rsrc_srqhdl_destructor(void *srq, void *priv)
2500 {
2501 	tavor_srqhdl_t	srqhdl;
2502 
2503 	srqhdl = (tavor_srqhdl_t)srq;
2504 
2505 	mutex_destroy(&srqhdl->srq_lock);
2506 }
2507 
2508 
2509 /*
2510  * tavor_rsrc_refcnt_constructor()
2511  *    Context: Can be called from interrupt or base context.
2512  */
2513 /* ARGSUSED */
2514 static int
tavor_rsrc_refcnt_constructor(void * rc,void * priv,int flags)2515 tavor_rsrc_refcnt_constructor(void *rc, void *priv, int flags)
2516 {
2517 	tavor_sw_refcnt_t	*refcnt;
2518 	tavor_state_t		*state;
2519 
2520 	refcnt = (tavor_sw_refcnt_t *)rc;
2521 	state  = (tavor_state_t *)priv;
2522 
2523 	mutex_init(&refcnt->swrc_lock, NULL, MUTEX_DRIVER,
2524 	    DDI_INTR_PRI(state->ts_intrmsi_pri));
2525 
2526 	return (DDI_SUCCESS);
2527 }
2528 
2529 
2530 /*
2531  * tavor_rsrc_refcnt_destructor()
2532  *    Context: Can be called from interrupt or base context.
2533  */
2534 /* ARGSUSED */
2535 static void
tavor_rsrc_refcnt_destructor(void * rc,void * priv)2536 tavor_rsrc_refcnt_destructor(void *rc, void *priv)
2537 {
2538 	tavor_sw_refcnt_t	*refcnt;
2539 
2540 	refcnt = (tavor_sw_refcnt_t *)rc;
2541 
2542 	mutex_destroy(&refcnt->swrc_lock);
2543 }
2544 
2545 
2546 /*
2547  * tavor_rsrc_ahhdl_constructor()
2548  *    Context: Can be called from interrupt or base context.
2549  */
2550 /* ARGSUSED */
2551 static int
tavor_rsrc_ahhdl_constructor(void * ah,void * priv,int flags)2552 tavor_rsrc_ahhdl_constructor(void *ah, void *priv, int flags)
2553 {
2554 	tavor_ahhdl_t	ahhdl;
2555 	tavor_state_t	*state;
2556 
2557 	ahhdl = (tavor_ahhdl_t)ah;
2558 	state = (tavor_state_t *)priv;
2559 
2560 	mutex_init(&ahhdl->ah_lock, NULL, MUTEX_DRIVER,
2561 	    DDI_INTR_PRI(state->ts_intrmsi_pri));
2562 
2563 	return (DDI_SUCCESS);
2564 }
2565 
2566 
2567 /*
2568  * tavor_rsrc_ahhdl_destructor()
2569  *    Context: Can be called from interrupt or base context.
2570  */
2571 /* ARGSUSED */
2572 static void
tavor_rsrc_ahhdl_destructor(void * ah,void * priv)2573 tavor_rsrc_ahhdl_destructor(void *ah, void *priv)
2574 {
2575 	tavor_ahhdl_t	ahhdl;
2576 
2577 	ahhdl = (tavor_ahhdl_t)ah;
2578 
2579 	mutex_destroy(&ahhdl->ah_lock);
2580 }
2581 
2582 
2583 /*
2584  * tavor_rsrc_mrhdl_constructor()
2585  *    Context: Can be called from interrupt or base context.
2586  */
2587 /* ARGSUSED */
2588 static int
tavor_rsrc_mrhdl_constructor(void * mr,void * priv,int flags)2589 tavor_rsrc_mrhdl_constructor(void *mr, void *priv, int flags)
2590 {
2591 	tavor_mrhdl_t	mrhdl;
2592 	tavor_state_t	*state;
2593 
2594 	mrhdl = (tavor_mrhdl_t)mr;
2595 	state = (tavor_state_t *)priv;
2596 
2597 	mutex_init(&mrhdl->mr_lock, NULL, MUTEX_DRIVER,
2598 	    DDI_INTR_PRI(state->ts_intrmsi_pri));
2599 
2600 	return (DDI_SUCCESS);
2601 }
2602 
2603 
2604 /*
2605  * tavor_rsrc_mrhdl_destructor()
2606  *    Context: Can be called from interrupt or base context.
2607  */
2608 /* ARGSUSED */
2609 static void
tavor_rsrc_mrhdl_destructor(void * mr,void * priv)2610 tavor_rsrc_mrhdl_destructor(void *mr, void *priv)
2611 {
2612 	tavor_mrhdl_t	mrhdl;
2613 
2614 	mrhdl = (tavor_mrhdl_t)mr;
2615 
2616 	mutex_destroy(&mrhdl->mr_lock);
2617 }
2618 
2619 
2620 /*
2621  * tavor_rsrc_mcg_entry_get_size()
2622  */
2623 static int
tavor_rsrc_mcg_entry_get_size(tavor_state_t * state,uint_t * mcg_size_shift)2624 tavor_rsrc_mcg_entry_get_size(tavor_state_t *state, uint_t *mcg_size_shift)
2625 {
2626 	uint_t	num_qp_per_mcg, max_qp_per_mcg, log2;
2627 
2628 	/*
2629 	 * Round the configured number of QP per MCG to next larger
2630 	 * power-of-2 size and update.
2631 	 */
2632 	num_qp_per_mcg = state->ts_cfg_profile->cp_num_qp_per_mcg + 8;
2633 	log2 = highbit(num_qp_per_mcg);
2634 	if (ISP2(num_qp_per_mcg)) {
2635 		log2 = log2 - 1;
2636 	}
2637 	state->ts_cfg_profile->cp_num_qp_per_mcg = (1 << log2) - 8;
2638 
2639 	/* Now make sure number of QP per MCG makes sense */
2640 	num_qp_per_mcg = state->ts_cfg_profile->cp_num_qp_per_mcg;
2641 	max_qp_per_mcg = (1 << state->ts_devlim.log_max_qp_mcg);
2642 	if (num_qp_per_mcg > max_qp_per_mcg) {
2643 		return (DDI_FAILURE);
2644 	}
2645 
2646 	/* Return the (shift) size of an individual MCG HW entry */
2647 	*mcg_size_shift = log2 + 2;
2648 
2649 	return (DDI_SUCCESS);
2650 }
2651