1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * tavor_rsrc.c 29 * Tavor Resource Management Routines 30 * 31 * Implements all the routines necessary for setup, teardown, and 32 * alloc/free of all Tavor resources, including those that are managed 33 * by Tavor hardware or which live in Tavor's direct attached DDR memory. 34 */ 35 36 #include <sys/types.h> 37 #include <sys/conf.h> 38 #include <sys/ddi.h> 39 #include <sys/sunddi.h> 40 #include <sys/modctl.h> 41 #include <sys/vmem.h> 42 #include <sys/bitmap.h> 43 44 #include <sys/ib/adapters/tavor/tavor.h> 45 46 /* 47 * The following routines are used for initializing and destroying 48 * the resource pools used by the Tavor resource allocation routines. 49 * They consist of four classes of object: 50 * 51 * Mailboxes: The "In" and "Out" mailbox types are used by the Tavor 52 * command interface routines. Mailboxes are used to pass information 53 * back and forth to the Tavor firmware. Either type of mailbox may 54 * be allocated from Tavor's direct attached DDR memory or from system 55 * memory (although currently all "In" mailboxes are in DDR and all "out" 56 * mailboxes come from system memory. 57 * 58 * HW entry objects: These objects represent resources required by the Tavor 59 * hardware. These objects include things like Queue Pair contexts (QPC), 60 * Completion Queue contexts (CQC), Event Queue contexts (EQC), RDB (for 61 * supporting RDMA Read/Atomic), Multicast Group entries (MCG), Memory 62 * Protection Table entries (MPT), Memory Translation Table entries (MTT). 63 * 64 * What these objects all have in common is that they are each required 65 * to come from DDR memory, they are always allocated from tables, and 66 * they are not to be directly accessed (read or written) by driver 67 * software. 68 * One notable exceptions to this rule are the Extended QP contexts (EQPC), 69 * and the UAR scratch area (UAR_SCR), both of which are not directly 70 * accessible through the Tavor resource allocation routines, but both 71 * of which are also required to reside in DDR memory and are not to be 72 * manipulated by driver software (they are privately managed by Tavor 73 * hardware). 74 * The other notable exceptions are the UAR pages (UAR_PG) which are 75 * allocated from the UAR address space rather than DDR, and the UD 76 * address vectors (UDAV) which are similar to the common object types 77 * with the major difference being that UDAVs _are_ directly read and 78 * written by driver software. 79 * 80 * SW handle objects: These objects represent resources required by Tavor 81 * driver software. They are primarily software tracking structures, 82 * which are allocated from system memory (using kmem_cache). Several of 83 * the objects have both a "constructor" and "destructor" method 84 * associated with them (see below). 85 * 86 * Protection Domain (PD) handle objects: These objects are very much like 87 * a SW handle object with the notable difference that all PD handle 88 * objects have an actual Protection Domain number (PD) associated with 89 * them (and the PD number is allocated/managed through a separate 90 * vmem_arena specifically set aside for this purpose. 91 */ 92 93 static int tavor_rsrc_mbox_init(tavor_state_t *state, 94 tavor_rsrc_mbox_info_t *info); 95 static void tavor_rsrc_mbox_fini(tavor_state_t *state, 96 tavor_rsrc_mbox_info_t *info); 97 98 static int tavor_rsrc_hw_entries_init(tavor_state_t *state, 99 tavor_rsrc_hw_entry_info_t *info); 100 static void tavor_rsrc_hw_entries_fini(tavor_state_t *state, 101 tavor_rsrc_hw_entry_info_t *info); 102 103 static int tavor_rsrc_sw_handles_init(tavor_state_t *state, 104 tavor_rsrc_sw_hdl_info_t *info); 105 static void tavor_rsrc_sw_handles_fini(tavor_state_t *state, 106 tavor_rsrc_sw_hdl_info_t *info); 107 108 static int tavor_rsrc_pd_handles_init(tavor_state_t *state, 109 tavor_rsrc_sw_hdl_info_t *info); 110 static void tavor_rsrc_pd_handles_fini(tavor_state_t *state, 111 tavor_rsrc_sw_hdl_info_t *info); 112 113 /* 114 * The following routines are used for allocating and freeing the specific 115 * types of objects described above from their associated resource pools. 116 */ 117 static int tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t *pool_info, 118 uint_t num, tavor_rsrc_t *hdl); 119 static void tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t *pool_info, 120 tavor_rsrc_t *hdl); 121 122 static int tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t *pool_info, 123 uint_t num, uint_t num_align, ddi_acc_handle_t acc_handle, 124 uint_t sleepflag, tavor_rsrc_t *hdl); 125 static void tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t *pool_info, 126 tavor_rsrc_t *hdl); 127 128 static int tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t *pool_info, 129 uint_t sleepflag, tavor_rsrc_t *hdl); 130 static void tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t *pool_info, 131 tavor_rsrc_t *hdl); 132 133 static int tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t *pool_info, 134 uint_t sleepflag, tavor_rsrc_t *hdl); 135 static void tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t *pool_info, 136 tavor_rsrc_t *hdl); 137 138 /* 139 * The following routines are the constructors and destructors for several 140 * of the SW handle type objects. For certain types of SW handles objects 141 * (all of which are implemented using kmem_cache), we need to do some 142 * special field initialization (specifically, mutex_init/destroy). These 143 * routines enable that init and teardown. 144 */ 145 static int tavor_rsrc_pdhdl_constructor(void *pd, void *priv, int flags); 146 static void tavor_rsrc_pdhdl_destructor(void *pd, void *state); 147 static int tavor_rsrc_cqhdl_constructor(void *cq, void *priv, int flags); 148 static void tavor_rsrc_cqhdl_destructor(void *cq, void *state); 149 static int tavor_rsrc_qphdl_constructor(void *cq, void *priv, int flags); 150 static void tavor_rsrc_qphdl_destructor(void *cq, void *state); 151 static int tavor_rsrc_srqhdl_constructor(void *srq, void *priv, int flags); 152 static void tavor_rsrc_srqhdl_destructor(void *srq, void *state); 153 static int tavor_rsrc_refcnt_constructor(void *rc, void *priv, int flags); 154 static void tavor_rsrc_refcnt_destructor(void *rc, void *state); 155 static int tavor_rsrc_ahhdl_constructor(void *ah, void *priv, int flags); 156 static void tavor_rsrc_ahhdl_destructor(void *ah, void *state); 157 static int tavor_rsrc_mrhdl_constructor(void *mr, void *priv, int flags); 158 static void tavor_rsrc_mrhdl_destructor(void *mr, void *state); 159 160 /* 161 * Special routine to calculate and return the size of a MCG object based 162 * on current driver configuration (specifically, the number of QP per MCG 163 * that has been configured. 164 */ 165 static int tavor_rsrc_mcg_entry_get_size(tavor_state_t *state, 166 uint_t *mcg_size_shift); 167 168 169 /* 170 * tavor_rsrc_alloc() 171 * 172 * Context: Can be called from interrupt or base context. 173 * The "sleepflag" parameter is used by all object allocators to 174 * determine whether to SLEEP for resources or not. 175 */ 176 int 177 tavor_rsrc_alloc(tavor_state_t *state, tavor_rsrc_type_t rsrc, uint_t num, 178 uint_t sleepflag, tavor_rsrc_t **hdl) 179 { 180 tavor_rsrc_pool_info_t *rsrc_pool; 181 tavor_rsrc_t *tmp_rsrc_hdl; 182 int flag, status = DDI_FAILURE; 183 184 TAVOR_TNF_ENTER(tavor_rsrc_alloc); 185 186 ASSERT(state != NULL); 187 ASSERT(hdl != NULL); 188 189 rsrc_pool = &state->ts_rsrc_hdl[rsrc]; 190 ASSERT(rsrc_pool != NULL); 191 192 /* 193 * Allocate space for the object used to track the resource handle 194 */ 195 flag = (sleepflag == TAVOR_SLEEP) ? KM_SLEEP : KM_NOSLEEP; 196 tmp_rsrc_hdl = (tavor_rsrc_t *)kmem_cache_alloc(state->ts_rsrc_cache, 197 flag); 198 if (tmp_rsrc_hdl == NULL) { 199 TNF_PROBE_0(tavor_rsrc_alloc_kmca_fail, TAVOR_TNF_ERROR, ""); 200 TAVOR_TNF_EXIT(tavor_rsrc_alloc); 201 return (DDI_FAILURE); 202 } 203 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*tmp_rsrc_hdl)) 204 205 /* 206 * Set rsrc_hdl type. This is later used by the tavor_rsrc_free call 207 * to know what type of resource is being freed. 208 */ 209 tmp_rsrc_hdl->rsrc_type = rsrc; 210 211 /* 212 * Depending on resource type, call the appropriate alloc routine 213 */ 214 switch (rsrc_pool->rsrc_type) { 215 case TAVOR_IN_MBOX: 216 case TAVOR_OUT_MBOX: 217 case TAVOR_INTR_IN_MBOX: 218 case TAVOR_INTR_OUT_MBOX: 219 status = tavor_rsrc_mbox_alloc(rsrc_pool, num, tmp_rsrc_hdl); 220 break; 221 222 case TAVOR_QPC: 223 case TAVOR_CQC: 224 case TAVOR_SRQC: 225 case TAVOR_EQC: 226 case TAVOR_RDB: 227 /* 228 * Because these objects are NOT accessed by Tavor driver 229 * software, we set the acc_handle parameter to zero. But 230 * if they are allocated in multiples, we specify here that 231 * they must be aligned on a more restrictive boundary. 232 */ 233 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, num, 0, 234 sleepflag, tmp_rsrc_hdl); 235 break; 236 237 case TAVOR_MPT: 238 /* 239 * Because these MPT objects are sometimes accessed by Tavor 240 * driver software (FMR), we set the acc_handle parameter. But 241 * if they are allocated in multiples, we specify here that 242 * they must be aligned on a more restrictive boundary. 243 */ 244 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, num, 245 state->ts_reg_ddrhdl, sleepflag, tmp_rsrc_hdl); 246 break; 247 248 case TAVOR_MCG: 249 /* 250 * Tavor MCG entries are also NOT accessed by Tavor driver 251 * software, but because MCG entries do not have the same 252 * alignnment restrictions we loosen the constraint here. 253 */ 254 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1, 0, 255 sleepflag, tmp_rsrc_hdl); 256 break; 257 258 case TAVOR_MTT: 259 case TAVOR_UDAV: 260 /* 261 * Because MTT segments are among the few HW resources that 262 * may be allocated in odd numbers, we specify a less 263 * restrictive alignment than for the above resources. 264 * 265 * Also because both UDAV and MTT segment objects are read 266 * and/or written by Tavor driver software, we set the 267 * acc_handle parameter to point to the ddi_acc_handle_t for 268 * the Tavor DDR memory. 269 */ 270 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1, 271 state->ts_reg_ddrhdl, sleepflag, tmp_rsrc_hdl); 272 break; 273 274 case TAVOR_UARPG: 275 /* 276 * Because UAR pages are written by Tavor driver software (for 277 * doorbells), we set the acc_handle parameter to point to 278 * the ddi_acc_handle_t for the Tavor UAR memory. 279 */ 280 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1, 281 state->ts_reg_uarhdl, sleepflag, tmp_rsrc_hdl); 282 break; 283 284 case TAVOR_MRHDL: 285 case TAVOR_EQHDL: 286 case TAVOR_CQHDL: 287 case TAVOR_SRQHDL: 288 case TAVOR_AHHDL: 289 case TAVOR_QPHDL: 290 case TAVOR_REFCNT: 291 status = tavor_rsrc_swhdl_alloc(rsrc_pool, sleepflag, 292 tmp_rsrc_hdl); 293 break; 294 295 case TAVOR_PDHDL: 296 status = tavor_rsrc_pdhdl_alloc(rsrc_pool, sleepflag, 297 tmp_rsrc_hdl); 298 break; 299 300 default: 301 TAVOR_WARNING(state, "unexpected resource type in alloc"); 302 TNF_PROBE_0(tavor_rsrc_alloc_inv_rsrctype_fail, 303 TAVOR_TNF_ERROR, ""); 304 break; 305 } 306 307 /* 308 * If the resource allocation failed, then free the special resource 309 * tracking structure and return failure. Otherwise return the 310 * handle for the resource tracking structure. 311 */ 312 if (status != DDI_SUCCESS) { 313 kmem_cache_free(state->ts_rsrc_cache, tmp_rsrc_hdl); 314 tmp_rsrc_hdl = NULL; 315 TNF_PROBE_1(tavor_rsrc_alloc_fail, TAVOR_TNF_ERROR, "", 316 tnf_uint, rsrc_type, rsrc_pool->rsrc_type); 317 TAVOR_TNF_EXIT(tavor_rsrc_alloc); 318 return (DDI_FAILURE); 319 } else { 320 *hdl = tmp_rsrc_hdl; 321 TAVOR_TNF_EXIT(tavor_rsrc_alloc); 322 return (DDI_SUCCESS); 323 } 324 } 325 326 327 /* 328 * tavor_rsrc_free() 329 * Context: Can be called from interrupt or base context. 330 */ 331 void 332 tavor_rsrc_free(tavor_state_t *state, tavor_rsrc_t **hdl) 333 { 334 tavor_rsrc_pool_info_t *rsrc_pool; 335 336 TAVOR_TNF_ENTER(tavor_rsrc_free); 337 338 ASSERT(state != NULL); 339 ASSERT(hdl != NULL); 340 341 rsrc_pool = &state->ts_rsrc_hdl[(*hdl)->rsrc_type]; 342 ASSERT(rsrc_pool != NULL); 343 344 /* 345 * Depending on resource type, call the appropriate free routine 346 */ 347 switch (rsrc_pool->rsrc_type) { 348 case TAVOR_IN_MBOX: 349 case TAVOR_OUT_MBOX: 350 case TAVOR_INTR_IN_MBOX: 351 case TAVOR_INTR_OUT_MBOX: 352 tavor_rsrc_mbox_free(rsrc_pool, *hdl); 353 break; 354 355 case TAVOR_QPC: 356 case TAVOR_CQC: 357 case TAVOR_SRQC: 358 case TAVOR_EQC: 359 case TAVOR_RDB: 360 case TAVOR_MCG: 361 case TAVOR_MPT: 362 case TAVOR_MTT: 363 case TAVOR_UDAV: 364 case TAVOR_UARPG: 365 tavor_rsrc_hw_entry_free(rsrc_pool, *hdl); 366 break; 367 368 case TAVOR_MRHDL: 369 case TAVOR_EQHDL: 370 case TAVOR_CQHDL: 371 case TAVOR_SRQHDL: 372 case TAVOR_AHHDL: 373 case TAVOR_QPHDL: 374 case TAVOR_REFCNT: 375 tavor_rsrc_swhdl_free(rsrc_pool, *hdl); 376 break; 377 378 case TAVOR_PDHDL: 379 tavor_rsrc_pdhdl_free(rsrc_pool, *hdl); 380 break; 381 382 default: 383 TAVOR_WARNING(state, "unexpected resource type in free"); 384 TNF_PROBE_0(tavor_rsrc_free_inv_rsrctype_fail, 385 TAVOR_TNF_ERROR, ""); 386 break; 387 } 388 389 /* 390 * Free the special resource tracking structure, set the handle to 391 * NULL, and return. 392 */ 393 kmem_cache_free(state->ts_rsrc_cache, *hdl); 394 *hdl = NULL; 395 396 TAVOR_TNF_EXIT(tavor_rsrc_free); 397 } 398 399 400 /* 401 * tavor_rsrc_init_phase1() 402 * 403 * Completes the first phase of Tavor resource/configuration init. 404 * This involves creating the kmem_cache for the "tavor_rsrc_t" 405 * structs, allocating the space for the resource pool handles, 406 * and setting up the "Out" mailboxes. 407 * 408 * When this function completes, the Tavor driver is ready to 409 * post the following commands which return information only in the 410 * "Out" mailbox: QUERY_DDR, QUERY_FW, QUERY_DEV_LIM, and QUERY_ADAPTER 411 * If any of these commands are to be posted at this time, they must be 412 * done so only when "spinning" (as the outstanding command list and 413 * EQ setup code has not yet run) 414 * 415 * Context: Only called from attach() path context 416 */ 417 int 418 tavor_rsrc_init_phase1(tavor_state_t *state) 419 { 420 tavor_rsrc_pool_info_t *rsrc_pool; 421 tavor_rsrc_mbox_info_t mbox_info; 422 tavor_rsrc_cleanup_level_t cleanup; 423 tavor_cfg_profile_t *cfgprof; 424 uint64_t num, size; 425 int status; 426 char *errormsg, *rsrc_name; 427 428 TAVOR_TNF_ENTER(tavor_rsrc_init_phase1); 429 430 ASSERT(state != NULL); 431 432 /* This is where Phase 1 of resource initialization begins */ 433 cleanup = TAVOR_RSRC_CLEANUP_LEVEL0; 434 435 /* Build kmem cache name from Tavor instance */ 436 rsrc_name = (char *)kmem_zalloc(TAVOR_RSRC_NAME_MAXLEN, KM_SLEEP); 437 TAVOR_RSRC_NAME(rsrc_name, TAVOR_RSRC_CACHE); 438 439 /* 440 * Create the kmem_cache for "tavor_rsrc_t" structures 441 * (kmem_cache_create will SLEEP until successful) 442 */ 443 state->ts_rsrc_cache = kmem_cache_create(rsrc_name, 444 sizeof (tavor_rsrc_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 445 446 /* 447 * Allocate an array of tavor_rsrc_pool_info_t's (used in all 448 * subsequent resource allocations) 449 */ 450 state->ts_rsrc_hdl = kmem_zalloc(TAVOR_NUM_RESOURCES * 451 sizeof (tavor_rsrc_pool_info_t), KM_SLEEP); 452 453 cfgprof = state->ts_cfg_profile; 454 455 /* 456 * Initialize the resource pool for "Out" mailboxes. Notice that 457 * the number of "Out" mailboxes, their size, and their location 458 * (DDR or system memory) is configurable. By default, however, 459 * all "Out" mailboxes are located in system memory only (because 460 * they are primarily read from and never written to) 461 */ 462 num = ((uint64_t)1 << cfgprof->cp_log_num_outmbox); 463 size = ((uint64_t)1 << cfgprof->cp_log_outmbox_size); 464 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_OUT_MBOX]; 465 rsrc_pool->rsrc_type = TAVOR_OUT_MBOX; 466 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 467 rsrc_pool->rsrc_pool_size = (size * num); 468 rsrc_pool->rsrc_shift = cfgprof->cp_log_outmbox_size; 469 rsrc_pool->rsrc_quantum = size; 470 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN; 471 rsrc_pool->rsrc_state = state; 472 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_OUTMBOX_VMEM); 473 mbox_info.mbi_num = num; 474 mbox_info.mbi_size = size; 475 mbox_info.mbi_rsrcpool = rsrc_pool; 476 mbox_info.mbi_rsrcname = rsrc_name; 477 status = tavor_rsrc_mbox_init(state, &mbox_info); 478 if (status != DDI_SUCCESS) { 479 tavor_rsrc_fini(state, cleanup); 480 /* Set "status" and "errormsg" and goto failure */ 481 TAVOR_TNF_FAIL(DDI_FAILURE, "out mailboxes"); 482 goto rsrcinitp1_fail; 483 } 484 cleanup = TAVOR_RSRC_CLEANUP_LEVEL1; 485 486 /* 487 * Initialize the Tavor "Out" mailbox list. This step actually uses 488 * the tavor_rsrc_alloc() for TAVOR_OUT_MBOX to preallocate the 489 * "Out" mailboxes, bind them for DMA access, and arrange them into 490 * an easily accessed fast-allocation mechanism (see tavor_cmd.c 491 * for more details) 492 */ 493 status = tavor_outmbox_list_init(state); 494 if (status != DDI_SUCCESS) { 495 tavor_rsrc_fini(state, cleanup); 496 /* Set "status" and "errormsg" and goto failure */ 497 TAVOR_TNF_FAIL(DDI_FAILURE, "out mailbox list"); 498 goto rsrcinitp1_fail; 499 } 500 cleanup = TAVOR_RSRC_CLEANUP_LEVEL2; 501 502 /* 503 * Initialize the resource pool for interrupt "Out" mailboxes. Notice 504 * that the number of interrupt "Out" mailboxes, their size, and their 505 * location (DDR or system memory) is configurable. By default, 506 * however, all interrupt "Out" mailboxes are located in system memory 507 * only (because they are primarily read from and never written to) 508 */ 509 num = ((uint64_t)1 << cfgprof->cp_log_num_intr_outmbox); 510 size = ((uint64_t)1 << cfgprof->cp_log_outmbox_size); 511 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_INTR_OUT_MBOX]; 512 rsrc_pool->rsrc_type = TAVOR_INTR_OUT_MBOX; 513 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 514 rsrc_pool->rsrc_pool_size = (size * num); 515 rsrc_pool->rsrc_shift = cfgprof->cp_log_outmbox_size; 516 rsrc_pool->rsrc_quantum = size; 517 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN; 518 rsrc_pool->rsrc_state = state; 519 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INTR_OUTMBOX_VMEM); 520 mbox_info.mbi_num = num; 521 mbox_info.mbi_size = size; 522 mbox_info.mbi_rsrcpool = rsrc_pool; 523 mbox_info.mbi_rsrcname = rsrc_name; 524 status = tavor_rsrc_mbox_init(state, &mbox_info); 525 if (status != DDI_SUCCESS) { 526 tavor_rsrc_fini(state, cleanup); 527 /* Set "status" and "errormsg" and goto failure */ 528 TAVOR_TNF_FAIL(DDI_FAILURE, "out intr mailboxes"); 529 goto rsrcinitp1_fail; 530 } 531 cleanup = TAVOR_RSRC_CLEANUP_LEVEL3; 532 533 /* 534 * Initialize the Tavor "Out" mailbox list. This step actually uses 535 * the tavor_rsrc_alloc() for TAVOR_OUT_MBOX to preallocate the 536 * "Out" mailboxes, bind them for DMA access, and arrange them into 537 * an easily accessed fast-allocation mechanism (see tavor_cmd.c 538 * for more details) 539 */ 540 status = tavor_intr_outmbox_list_init(state); 541 if (status != DDI_SUCCESS) { 542 tavor_rsrc_fini(state, cleanup); 543 /* Set "status" and "errormsg" and goto failure */ 544 TAVOR_TNF_FAIL(DDI_FAILURE, "out intr mailbox list"); 545 goto rsrcinitp1_fail; 546 } 547 cleanup = TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE; 548 549 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN); 550 TAVOR_TNF_EXIT(tavor_rsrc_init_phase1); 551 return (DDI_SUCCESS); 552 553 rsrcinitp1_fail: 554 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN); 555 TNF_PROBE_1(tavor_rsrc_init_phase1_fail, TAVOR_TNF_ERROR, "", 556 tnf_string, msg, errormsg); 557 TAVOR_TNF_EXIT(tavor_rsrc_init_phase1); 558 return (status); 559 } 560 561 562 /* 563 * tavor_rsrc_init_phase2() 564 * Context: Only called from attach() path context 565 */ 566 int 567 tavor_rsrc_init_phase2(tavor_state_t *state) 568 { 569 tavor_rsrc_sw_hdl_info_t hdl_info; 570 tavor_rsrc_hw_entry_info_t entry_info; 571 tavor_rsrc_mbox_info_t mbox_info; 572 tavor_rsrc_pool_info_t *rsrc_pool; 573 tavor_rsrc_cleanup_level_t cleanup; 574 tavor_cfg_profile_t *cfgprof; 575 uint64_t num, max, size, num_prealloc; 576 uint64_t ddr_size, fw_size; 577 uint_t mcg_size, mcg_size_shift; 578 uint_t uarscr_size, mttsegment_sz; 579 int status; 580 char *errormsg, *rsrc_name; 581 582 TAVOR_TNF_ENTER(tavor_rsrc_init_phase2); 583 584 ASSERT(state != NULL); 585 586 /* Phase 2 initialization begins where Phase 1 left off */ 587 cleanup = TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE; 588 589 /* 590 * Calculate the extent of the DDR size and portion of which that 591 * is already reserved for Tavor firmware. (Note: this information 592 * is available because the QUERY_DDR and QUERY_FW commands have 593 * been posted to Tavor firmware prior to calling this routine) 594 */ 595 ddr_size = state->ts_ddr.ddr_endaddr - state->ts_ddr.ddr_baseaddr + 1; 596 fw_size = state->ts_fw.fw_endaddr - state->ts_fw.fw_baseaddr + 1; 597 598 /* Build the DDR vmem arena name from Tavor instance */ 599 rsrc_name = (char *)kmem_zalloc(TAVOR_RSRC_NAME_MAXLEN, KM_SLEEP); 600 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_VMEM); 601 602 /* 603 * Do a vmem_create for the entire DDR range (not including the 604 * portion consumed by Tavor firmware). This creates the vmem arena 605 * from which all other DDR objects (specifically, tables of HW 606 * entries) will be allocated. 607 */ 608 state->ts_ddrvmem = vmem_create(rsrc_name, 609 (void *)(uintptr_t)state->ts_ddr.ddr_baseaddr, (ddr_size - fw_size), 610 sizeof (uint64_t), NULL, NULL, NULL, 0, VM_SLEEP); 611 if (state->ts_ddrvmem == NULL) { 612 tavor_rsrc_fini(state, cleanup); 613 /* Set "status" and "errormsg" and goto failure */ 614 TAVOR_TNF_FAIL(DDI_FAILURE, "DDR vmem"); 615 goto rsrcinitp2_fail; 616 } 617 cleanup = TAVOR_RSRC_CLEANUP_LEVEL5; 618 619 /* 620 * Initialize the resource pools for all objects that exist in 621 * Tavor DDR memory. This includes ("In") mailboxes, context tables 622 * (QPC, CQC, EQC, etc...), and other miscellaneous HW objects. 623 */ 624 cfgprof = state->ts_cfg_profile; 625 626 /* 627 * Initialize the resource pool for the MPT table entries. Notice 628 * that the number of MPTs is configurable. The configured value must 629 * be less that the maximum value (obtained from the QUERY_DEV_LIM 630 * command) or the initialization will fail. Note also that a certain 631 * number of MPTs must be set aside for Tavor firmware use. 632 */ 633 num = ((uint64_t)1 << cfgprof->cp_log_num_mpt); 634 max = ((uint64_t)1 << state->ts_devlim.log_max_mpt); 635 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_mpt); 636 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MPT]; 637 rsrc_pool->rsrc_type = TAVOR_MPT; 638 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 639 rsrc_pool->rsrc_pool_size = (TAVOR_MPT_SIZE * num); 640 rsrc_pool->rsrc_shift = TAVOR_MPT_SIZE_SHIFT; 641 rsrc_pool->rsrc_quantum = TAVOR_MPT_SIZE; 642 rsrc_pool->rsrc_align = (TAVOR_MPT_SIZE * num); 643 rsrc_pool->rsrc_state = state; 644 rsrc_pool->rsrc_start = NULL; 645 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MPT_VMEM); 646 entry_info.hwi_num = num; 647 entry_info.hwi_max = max; 648 entry_info.hwi_prealloc = num_prealloc; 649 entry_info.hwi_rsrcpool = rsrc_pool; 650 entry_info.hwi_rsrcname = rsrc_name; 651 status = tavor_rsrc_hw_entries_init(state, &entry_info); 652 if (status != DDI_SUCCESS) { 653 tavor_rsrc_fini(state, cleanup); 654 /* Set "status" and "errormsg" and goto failure */ 655 TAVOR_TNF_FAIL(DDI_FAILURE, "MPT table"); 656 goto rsrcinitp2_fail; 657 } 658 cleanup = TAVOR_RSRC_CLEANUP_LEVEL6; 659 660 /* 661 * Initialize the resource pool for the MTT table entries. Notice 662 * that the number of MTTs is configurable. The configured value must 663 * be less that the maximum value (obtained from the QUERY_DEV_LIM 664 * command) or the initialization will fail. Note also that a certain 665 * number of MTT segments must be set aside for Tavor firmware use. 666 */ 667 mttsegment_sz = (TAVOR_MTTSEG_SIZE << TAVOR_MTT_SIZE_SHIFT); 668 num = ((uint64_t)1 << cfgprof->cp_log_num_mttseg); 669 max = ((uint64_t)1 << state->ts_devlim.log_max_mttseg); 670 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_mttseg); 671 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MTT]; 672 rsrc_pool->rsrc_type = TAVOR_MTT; 673 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 674 rsrc_pool->rsrc_pool_size = (TAVOR_MTT_SIZE * num); 675 rsrc_pool->rsrc_shift = TAVOR_MTT_SIZE_SHIFT; 676 rsrc_pool->rsrc_quantum = mttsegment_sz; 677 rsrc_pool->rsrc_align = (TAVOR_MTT_SIZE * num); 678 rsrc_pool->rsrc_state = state; 679 rsrc_pool->rsrc_start = NULL; 680 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MTT_VMEM); 681 entry_info.hwi_num = num; 682 entry_info.hwi_max = max; 683 entry_info.hwi_prealloc = num_prealloc; 684 entry_info.hwi_rsrcpool = rsrc_pool; 685 entry_info.hwi_rsrcname = rsrc_name; 686 status = tavor_rsrc_hw_entries_init(state, &entry_info); 687 if (status != DDI_SUCCESS) { 688 tavor_rsrc_fini(state, cleanup); 689 /* Set "status" and "errormsg" and goto failure */ 690 TAVOR_TNF_FAIL(DDI_FAILURE, "MTT table"); 691 goto rsrcinitp2_fail; 692 } 693 cleanup = TAVOR_RSRC_CLEANUP_LEVEL7; 694 695 /* 696 * Initialize the resource pool for the QPC table entries. Notice 697 * that the number of QPs is configurable. The configured value must 698 * be less that the maximum value (obtained from the QUERY_DEV_LIM 699 * command) or the initialization will fail. Note also that a certain 700 * number of QP contexts must be set aside for Tavor firmware use. 701 */ 702 num = ((uint64_t)1 << cfgprof->cp_log_num_qp); 703 max = ((uint64_t)1 << state->ts_devlim.log_max_qp); 704 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_qp); 705 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPC]; 706 rsrc_pool->rsrc_type = TAVOR_QPC; 707 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 708 rsrc_pool->rsrc_pool_size = (TAVOR_QPC_SIZE * num); 709 rsrc_pool->rsrc_shift = TAVOR_QPC_SIZE_SHIFT; 710 rsrc_pool->rsrc_quantum = TAVOR_QPC_SIZE; 711 rsrc_pool->rsrc_align = (TAVOR_QPC_SIZE * num); 712 rsrc_pool->rsrc_state = state; 713 rsrc_pool->rsrc_start = NULL; 714 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_QPC_VMEM); 715 entry_info.hwi_num = num; 716 entry_info.hwi_max = max; 717 entry_info.hwi_prealloc = num_prealloc; 718 entry_info.hwi_rsrcpool = rsrc_pool; 719 entry_info.hwi_rsrcname = rsrc_name; 720 status = tavor_rsrc_hw_entries_init(state, &entry_info); 721 if (status != DDI_SUCCESS) { 722 tavor_rsrc_fini(state, cleanup); 723 /* Set "status" and "errormsg" and goto failure */ 724 TAVOR_TNF_FAIL(DDI_FAILURE, "QPC table"); 725 goto rsrcinitp2_fail; 726 } 727 cleanup = TAVOR_RSRC_CLEANUP_LEVEL8; 728 729 /* 730 * Initialize the resource pool for the RDB table entries. Notice 731 * that the number of RDBs is configurable. The configured value must 732 * be less that the maximum value (obtained from the QUERY_DEV_LIM 733 * command) or the initialization will fail. 734 */ 735 num = ((uint64_t)1 << cfgprof->cp_log_num_rdb); 736 max = ((uint64_t)1 << state->ts_devlim.log_max_ra_glob); 737 num_prealloc = 0; 738 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_RDB]; 739 rsrc_pool->rsrc_type = TAVOR_RDB; 740 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 741 rsrc_pool->rsrc_pool_size = (TAVOR_RDB_SIZE * num); 742 rsrc_pool->rsrc_shift = TAVOR_RDB_SIZE_SHIFT; 743 rsrc_pool->rsrc_quantum = TAVOR_RDB_SIZE; 744 rsrc_pool->rsrc_align = (TAVOR_RDB_SIZE * num); 745 rsrc_pool->rsrc_state = state; 746 rsrc_pool->rsrc_start = NULL; 747 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_RDB_VMEM); 748 entry_info.hwi_num = num; 749 entry_info.hwi_max = max; 750 entry_info.hwi_prealloc = num_prealloc; 751 entry_info.hwi_rsrcpool = rsrc_pool; 752 entry_info.hwi_rsrcname = rsrc_name; 753 status = tavor_rsrc_hw_entries_init(state, &entry_info); 754 if (status != DDI_SUCCESS) { 755 tavor_rsrc_fini(state, cleanup); 756 /* Set "status" and "errormsg" and goto failure */ 757 TAVOR_TNF_FAIL(DDI_FAILURE, "RDB table"); 758 goto rsrcinitp2_fail; 759 } 760 cleanup = TAVOR_RSRC_CLEANUP_LEVEL9; 761 762 /* 763 * Initialize the resource pool for the CQC table entries. Notice 764 * that the number of CQs is configurable. The configured value must 765 * be less that the maximum value (obtained from the QUERY_DEV_LIM 766 * command) or the initialization will fail. Note also that a certain 767 * number of CQ contexts must be set aside for Tavor firmware use. 768 */ 769 num = ((uint64_t)1 << cfgprof->cp_log_num_cq); 770 max = ((uint64_t)1 << state->ts_devlim.log_max_cq); 771 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_cq); 772 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQC]; 773 rsrc_pool->rsrc_type = TAVOR_CQC; 774 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 775 rsrc_pool->rsrc_pool_size = (TAVOR_CQC_SIZE * num); 776 rsrc_pool->rsrc_shift = TAVOR_CQC_SIZE_SHIFT; 777 rsrc_pool->rsrc_quantum = TAVOR_CQC_SIZE; 778 rsrc_pool->rsrc_align = (TAVOR_CQC_SIZE * num); 779 rsrc_pool->rsrc_state = state; 780 rsrc_pool->rsrc_start = NULL; 781 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_CQC_VMEM); 782 entry_info.hwi_num = num; 783 entry_info.hwi_max = max; 784 entry_info.hwi_prealloc = num_prealloc; 785 entry_info.hwi_rsrcpool = rsrc_pool; 786 entry_info.hwi_rsrcname = rsrc_name; 787 status = tavor_rsrc_hw_entries_init(state, &entry_info); 788 if (status != DDI_SUCCESS) { 789 tavor_rsrc_fini(state, cleanup); 790 /* Set "status" and "errormsg" and goto failure */ 791 TAVOR_TNF_FAIL(DDI_FAILURE, "CQC table"); 792 goto rsrcinitp2_fail; 793 } 794 cleanup = TAVOR_RSRC_CLEANUP_LEVEL10; 795 796 /* 797 * Initialize the resource pool for the Extended QPC table entries. 798 * Notice that the number of EQPCs must be the same as the number 799 * of QP contexts. So the initialization is constructed in a 800 * similar way as above (for TAVOR_QPC). One notable difference 801 * here, however, is that by setting the rsrc_quantum field to 802 * zero (indicating a zero-sized object) we indicate that the 803 * object is not allocatable. The EQPC table is, in fact, managed 804 * internally by the hardware and it is, therefore, unnecessary to 805 * initialize an additional vmem_arena for this type of object. 806 */ 807 num = ((uint64_t)1 << cfgprof->cp_log_num_qp); 808 max = ((uint64_t)1 << state->ts_devlim.log_max_qp); 809 num_prealloc = 0; 810 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQPC]; 811 rsrc_pool->rsrc_type = TAVOR_EQPC; 812 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 813 rsrc_pool->rsrc_pool_size = (TAVOR_EQPC_SIZE * num); 814 rsrc_pool->rsrc_shift = 0; 815 rsrc_pool->rsrc_quantum = 0; 816 rsrc_pool->rsrc_align = TAVOR_EQPC_SIZE; 817 rsrc_pool->rsrc_state = state; 818 rsrc_pool->rsrc_start = NULL; 819 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_EQPC_VMEM); 820 entry_info.hwi_num = num; 821 entry_info.hwi_max = max; 822 entry_info.hwi_prealloc = num_prealloc; 823 entry_info.hwi_rsrcpool = rsrc_pool; 824 entry_info.hwi_rsrcname = rsrc_name; 825 status = tavor_rsrc_hw_entries_init(state, &entry_info); 826 if (status != DDI_SUCCESS) { 827 tavor_rsrc_fini(state, cleanup); 828 /* Set "status" and "errormsg" and goto failure */ 829 TAVOR_TNF_FAIL(DDI_FAILURE, "Extended QPC table"); 830 goto rsrcinitp2_fail; 831 } 832 cleanup = TAVOR_RSRC_CLEANUP_LEVEL11; 833 834 /* 835 * Initialize the resource pool for the UD address vector table 836 * entries. Notice that the number of UDAVs is configurable. The 837 * configured value must be less that the maximum value (obtained 838 * from the QUERY_DEV_LIM command) or the initialization will fail. 839 */ 840 num = ((uint64_t)1 << cfgprof->cp_log_num_ah); 841 max = ((uint64_t)1 << state->ts_devlim.log_max_av); 842 num_prealloc = 0; 843 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UDAV]; 844 rsrc_pool->rsrc_type = TAVOR_UDAV; 845 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 846 rsrc_pool->rsrc_pool_size = (TAVOR_UDAV_SIZE * num); 847 rsrc_pool->rsrc_shift = TAVOR_UDAV_SIZE_SHIFT; 848 rsrc_pool->rsrc_quantum = TAVOR_UDAV_SIZE; 849 rsrc_pool->rsrc_align = TAVOR_UDAV_SIZE; 850 rsrc_pool->rsrc_state = state; 851 rsrc_pool->rsrc_start = NULL; 852 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_UDAV_VMEM); 853 entry_info.hwi_num = num; 854 entry_info.hwi_max = max; 855 entry_info.hwi_prealloc = num_prealloc; 856 entry_info.hwi_rsrcpool = rsrc_pool; 857 entry_info.hwi_rsrcname = rsrc_name; 858 status = tavor_rsrc_hw_entries_init(state, &entry_info); 859 if (status != DDI_SUCCESS) { 860 tavor_rsrc_fini(state, cleanup); 861 /* Set "status" and "errormsg" and goto failure */ 862 TAVOR_TNF_FAIL(DDI_FAILURE, "UDAV table"); 863 goto rsrcinitp2_fail; 864 } 865 cleanup = TAVOR_RSRC_CLEANUP_LEVEL12; 866 867 /* 868 * Initialize the resource pool for the UAR scratch table entries. 869 * Notice that the number of UARSCRs is configurable. The configured 870 * value must be less that the maximum value (obtained from the 871 * QUERY_DEV_LIM command) or the initialization will fail. 872 * Like the EQPCs above, UARSCR objects are not allocatable. The 873 * UARSCR table is also managed internally by the hardware and it 874 * is, therefore, unnecessary to initialize an additional vmem_arena 875 * for this type of object. We indicate this by setting the 876 * rsrc_quantum field to zero (indicating a zero-sized object). 877 */ 878 uarscr_size = state->ts_devlim.uarscr_entry_sz; 879 num = ((uint64_t)1 << cfgprof->cp_log_num_uar); 880 max = ((uint64_t)1 << (state->ts_devlim.log_max_uar_sz + 20 - 881 PAGESHIFT)); 882 num_prealloc = 0; 883 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR]; 884 rsrc_pool->rsrc_type = TAVOR_UAR_SCR; 885 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 886 rsrc_pool->rsrc_pool_size = (uarscr_size * num); 887 rsrc_pool->rsrc_shift = 0; 888 rsrc_pool->rsrc_quantum = 0; 889 rsrc_pool->rsrc_align = uarscr_size; 890 rsrc_pool->rsrc_state = state; 891 rsrc_pool->rsrc_start = NULL; 892 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_UARSCR_VMEM); 893 entry_info.hwi_num = num; 894 entry_info.hwi_max = max; 895 entry_info.hwi_prealloc = num_prealloc; 896 entry_info.hwi_rsrcpool = rsrc_pool; 897 entry_info.hwi_rsrcname = rsrc_name; 898 status = tavor_rsrc_hw_entries_init(state, &entry_info); 899 if (status != DDI_SUCCESS) { 900 tavor_rsrc_fini(state, cleanup); 901 /* Set "status" and "errormsg" and goto failure */ 902 TAVOR_TNF_FAIL(DDI_FAILURE, "UAR scratch table"); 903 goto rsrcinitp2_fail; 904 } 905 cleanup = TAVOR_RSRC_CLEANUP_LEVEL13; 906 907 /* 908 * Initialize the resource pool for the SRQC table entries. Notice 909 * that the number of SRQs is configurable. The configured value must 910 * be less that the maximum value (obtained from the QUERY_DEV_LIM 911 * command) or the initialization will fail. Note also that a certain 912 * number of SRQ contexts must be set aside for Tavor firmware use. 913 * 914 * Note: We only allocate these resources if SRQ is enabled in the 915 * config profile; see below. 916 */ 917 num = ((uint64_t)1 << cfgprof->cp_log_num_srq); 918 max = ((uint64_t)1 << state->ts_devlim.log_max_srq); 919 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_srq); 920 921 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQC]; 922 rsrc_pool->rsrc_type = TAVOR_SRQC; 923 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 924 rsrc_pool->rsrc_pool_size = (TAVOR_SRQC_SIZE * num); 925 rsrc_pool->rsrc_shift = TAVOR_SRQC_SIZE_SHIFT; 926 rsrc_pool->rsrc_quantum = TAVOR_SRQC_SIZE; 927 rsrc_pool->rsrc_align = (TAVOR_SRQC_SIZE * num); 928 rsrc_pool->rsrc_state = state; 929 rsrc_pool->rsrc_start = NULL; 930 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_SRQC_VMEM); 931 entry_info.hwi_num = num; 932 entry_info.hwi_max = max; 933 entry_info.hwi_prealloc = num_prealloc; 934 entry_info.hwi_rsrcpool = rsrc_pool; 935 entry_info.hwi_rsrcname = rsrc_name; 936 937 /* 938 * SRQ support is configurable. Only if SRQ is enabled (the default) 939 * do we actually try to configure these resources. Otherwise, we 940 * simply set the cleanup level and continue on to the next resource 941 */ 942 if (state->ts_cfg_profile->cp_srq_enable != 0) { 943 status = tavor_rsrc_hw_entries_init(state, &entry_info); 944 if (status != DDI_SUCCESS) { 945 tavor_rsrc_fini(state, cleanup); 946 /* Set "status" and "errormsg" and goto failure */ 947 TAVOR_TNF_FAIL(DDI_FAILURE, "SRQC table"); 948 goto rsrcinitp2_fail; 949 } 950 } 951 cleanup = TAVOR_RSRC_CLEANUP_LEVEL14; 952 953 /* 954 * Initialize the resource pool for "In" mailboxes. Notice that 955 * the number of "In" mailboxes, their size, and their location 956 * (DDR or system memory) is configurable. By default, however, 957 * all "In" mailboxes are located in system memory only (because 958 * they are primarily written to and rarely read from) 959 */ 960 num = ((uint64_t)1 << cfgprof->cp_log_num_inmbox); 961 size = ((uint64_t)1 << cfgprof->cp_log_inmbox_size); 962 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_IN_MBOX]; 963 rsrc_pool->rsrc_type = TAVOR_IN_MBOX; 964 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 965 rsrc_pool->rsrc_pool_size = (size * num); 966 rsrc_pool->rsrc_shift = cfgprof->cp_log_inmbox_size; 967 rsrc_pool->rsrc_quantum = size; 968 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN; 969 rsrc_pool->rsrc_state = state; 970 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INMBOX_VMEM); 971 mbox_info.mbi_num = num; 972 mbox_info.mbi_size = size; 973 mbox_info.mbi_rsrcpool = rsrc_pool; 974 mbox_info.mbi_rsrcname = rsrc_name; 975 status = tavor_rsrc_mbox_init(state, &mbox_info); 976 if (status != DDI_SUCCESS) { 977 tavor_rsrc_fini(state, cleanup); 978 /* Set "status" and "errormsg" and goto failure */ 979 TAVOR_TNF_FAIL(DDI_FAILURE, "in mailboxes"); 980 goto rsrcinitp2_fail; 981 } 982 cleanup = TAVOR_RSRC_CLEANUP_LEVEL15; 983 984 /* 985 * Initialize the Tavor "In" mailbox list. This step actually uses 986 * the tavor_rsrc_alloc() for TAVOR_IN_MBOX to preallocate the 987 * "In" mailboxes, bind them for DMA access, and arrange them into 988 * an easily accessed fast-allocation mechanism (see tavor_cmd.c 989 * for more details) 990 */ 991 status = tavor_inmbox_list_init(state); 992 if (status != DDI_SUCCESS) { 993 tavor_rsrc_fini(state, cleanup); 994 /* Set "status" and "errormsg" and goto failure */ 995 TAVOR_TNF_FAIL(DDI_FAILURE, "in mailbox list"); 996 goto rsrcinitp2_fail; 997 } 998 cleanup = TAVOR_RSRC_CLEANUP_LEVEL16; 999 1000 /* 1001 * Initialize the resource pool for interrupt "In" mailboxes. Notice 1002 * that the number of interrupt "In" mailboxes, their size, and their 1003 * location (DDR or system memory) is configurable. By default, 1004 * however, all interrupt "In" mailboxes are located in system memory 1005 * only (because they are primarily written to and rarely read from) 1006 */ 1007 num = ((uint64_t)1 << cfgprof->cp_log_num_intr_inmbox); 1008 size = ((uint64_t)1 << cfgprof->cp_log_inmbox_size); 1009 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_INTR_IN_MBOX]; 1010 rsrc_pool->rsrc_type = TAVOR_INTR_IN_MBOX; 1011 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 1012 rsrc_pool->rsrc_pool_size = (size * num); 1013 rsrc_pool->rsrc_shift = cfgprof->cp_log_inmbox_size; 1014 rsrc_pool->rsrc_quantum = size; 1015 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN; 1016 rsrc_pool->rsrc_state = state; 1017 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INTR_INMBOX_VMEM); 1018 mbox_info.mbi_num = num; 1019 mbox_info.mbi_size = size; 1020 mbox_info.mbi_rsrcpool = rsrc_pool; 1021 mbox_info.mbi_rsrcname = rsrc_name; 1022 status = tavor_rsrc_mbox_init(state, &mbox_info); 1023 if (status != DDI_SUCCESS) { 1024 tavor_rsrc_fini(state, cleanup); 1025 /* Set "status" and "errormsg" and goto failure */ 1026 TAVOR_TNF_FAIL(DDI_FAILURE, "in intr mailboxes"); 1027 goto rsrcinitp2_fail; 1028 } 1029 cleanup = TAVOR_RSRC_CLEANUP_LEVEL17; 1030 1031 /* 1032 * Initialize the Tavor interrupt "In" mailbox list. This step 1033 * actually uses the tavor_rsrc_alloc() for TAVOR_IN_MBOX to 1034 * preallocate the interrupt "In" mailboxes, bind them for DMA access, 1035 * and arrange them into an easily accessed fast-allocation mechanism 1036 * (see tavor_cmd.c for more details) 1037 */ 1038 status = tavor_intr_inmbox_list_init(state); 1039 if (status != DDI_SUCCESS) { 1040 tavor_rsrc_fini(state, cleanup); 1041 /* Set "status" and "errormsg" and goto failure */ 1042 TAVOR_TNF_FAIL(DDI_FAILURE, "in intr mailbox list"); 1043 goto rsrcinitp2_fail; 1044 } 1045 cleanup = TAVOR_RSRC_CLEANUP_LEVEL18; 1046 1047 /* 1048 * Initialize the Tavor command handling interfaces. This step 1049 * sets up the outstanding command tracking mechanism for easy access 1050 * and fast allocation (see tavor_cmd.c for more details). 1051 */ 1052 status = tavor_outstanding_cmdlist_init(state); 1053 if (status != DDI_SUCCESS) { 1054 tavor_rsrc_fini(state, cleanup); 1055 /* Set "status" and "errormsg" and goto failure */ 1056 TAVOR_TNF_FAIL(DDI_FAILURE, "outstanding cmd list"); 1057 goto rsrcinitp2_fail; 1058 } 1059 cleanup = TAVOR_RSRC_CLEANUP_LEVEL19; 1060 1061 /* 1062 * Calculate (and validate) the size of Multicast Group (MCG) entries 1063 */ 1064 status = tavor_rsrc_mcg_entry_get_size(state, &mcg_size_shift); 1065 if (status != DDI_SUCCESS) { 1066 tavor_rsrc_fini(state, cleanup); 1067 /* Set "status" and "errormsg" and goto failure */ 1068 TAVOR_TNF_FAIL(DDI_FAILURE, "failed get MCG size"); 1069 goto rsrcinitp2_fail; 1070 } 1071 mcg_size = TAVOR_MCGMEM_SZ(state); 1072 1073 /* 1074 * Initialize the resource pool for the MCG table entries. Notice 1075 * that the number of MCGs is configurable. The configured value must 1076 * be less that the maximum value (obtained from the QUERY_DEV_LIM 1077 * command) or the initialization will fail. Note also that a certain 1078 * number of MCGs must be set aside for Tavor firmware use (they 1079 * correspond to the number of MCGs used by the internal hash 1080 * function. 1081 */ 1082 num = ((uint64_t)1 << cfgprof->cp_log_num_mcg); 1083 max = ((uint64_t)1 << state->ts_devlim.log_max_mcg); 1084 num_prealloc = ((uint64_t)1 << cfgprof->cp_log_num_mcg_hash); 1085 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MCG]; 1086 rsrc_pool->rsrc_type = TAVOR_MCG; 1087 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 1088 rsrc_pool->rsrc_pool_size = (mcg_size * num); 1089 rsrc_pool->rsrc_shift = mcg_size_shift; 1090 rsrc_pool->rsrc_quantum = mcg_size; 1091 rsrc_pool->rsrc_align = mcg_size; 1092 rsrc_pool->rsrc_state = state; 1093 rsrc_pool->rsrc_start = NULL; 1094 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MCG_VMEM); 1095 entry_info.hwi_num = num; 1096 entry_info.hwi_max = max; 1097 entry_info.hwi_prealloc = num_prealloc; 1098 entry_info.hwi_rsrcpool = rsrc_pool; 1099 entry_info.hwi_rsrcname = rsrc_name; 1100 status = tavor_rsrc_hw_entries_init(state, &entry_info); 1101 if (status != DDI_SUCCESS) { 1102 tavor_rsrc_fini(state, cleanup); 1103 /* Set "status" and "errormsg" and goto failure */ 1104 TAVOR_TNF_FAIL(DDI_FAILURE, "MCG table"); 1105 goto rsrcinitp2_fail; 1106 } 1107 cleanup = TAVOR_RSRC_CLEANUP_LEVEL20; 1108 1109 /* 1110 * Initialize the resource pool for the EQC table entries. Notice 1111 * that the number of EQs is hardcoded. The hardcoded value should 1112 * be less that the maximum value (obtained from the QUERY_DEV_LIM 1113 * command) or the initialization will fail. 1114 */ 1115 num = TAVOR_NUM_EQ; 1116 max = ((uint64_t)1 << state->ts_devlim.log_max_eq); 1117 num_prealloc = 0; 1118 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQC]; 1119 rsrc_pool->rsrc_type = TAVOR_EQC; 1120 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 1121 rsrc_pool->rsrc_pool_size = (TAVOR_EQC_SIZE * num); 1122 rsrc_pool->rsrc_shift = TAVOR_EQC_SIZE_SHIFT; 1123 rsrc_pool->rsrc_quantum = TAVOR_EQC_SIZE; 1124 rsrc_pool->rsrc_align = (TAVOR_EQC_SIZE * num); 1125 rsrc_pool->rsrc_state = state; 1126 rsrc_pool->rsrc_start = NULL; 1127 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_EQC_VMEM); 1128 entry_info.hwi_num = num; 1129 entry_info.hwi_max = max; 1130 entry_info.hwi_prealloc = num_prealloc; 1131 entry_info.hwi_rsrcpool = rsrc_pool; 1132 entry_info.hwi_rsrcname = rsrc_name; 1133 status = tavor_rsrc_hw_entries_init(state, &entry_info); 1134 if (status != DDI_SUCCESS) { 1135 tavor_rsrc_fini(state, cleanup); 1136 /* Set "status" and "errormsg" and goto failure */ 1137 TAVOR_TNF_FAIL(DDI_FAILURE, "EQC table"); 1138 goto rsrcinitp2_fail; 1139 } 1140 cleanup = TAVOR_RSRC_CLEANUP_LEVEL21; 1141 1142 /* 1143 * Initialize the resource pools for all objects that exist in 1144 * system memory. This includes PD handles, MR handle, EQ handles, 1145 * QP handles, etc. These objects are almost entirely managed using 1146 * kmem_cache routines. (See comment above for more detail) 1147 */ 1148 1149 /* 1150 * Initialize the resource pool for the PD handles. Notice 1151 * that the number of PDHDLs is configurable. The configured value 1152 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1153 * command) or the initialization will fail. Note also that the PD 1154 * handle has constructor and destructor methods associated with it. 1155 */ 1156 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_PDHDL]; 1157 rsrc_pool->rsrc_type = TAVOR_PDHDL; 1158 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1159 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_pd_s); 1160 rsrc_pool->rsrc_state = state; 1161 TAVOR_RSRC_NAME(rsrc_name, TAVOR_PDHDL_CACHE); 1162 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_pd); 1163 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_pd); 1164 hdl_info.swi_rsrcpool = rsrc_pool; 1165 hdl_info.swi_constructor = tavor_rsrc_pdhdl_constructor; 1166 hdl_info.swi_destructor = tavor_rsrc_pdhdl_destructor; 1167 hdl_info.swi_rsrcname = rsrc_name; 1168 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT; 1169 status = tavor_rsrc_pd_handles_init(state, &hdl_info); 1170 if (status != DDI_SUCCESS) { 1171 tavor_rsrc_fini(state, cleanup); 1172 /* Set "status" and "errormsg" and goto failure */ 1173 TAVOR_TNF_FAIL(DDI_FAILURE, "PD handle"); 1174 goto rsrcinitp2_fail; 1175 } 1176 cleanup = TAVOR_RSRC_CLEANUP_LEVEL22; 1177 1178 /* 1179 * Initialize the resource pool for the MR handles. Notice 1180 * that the number of MRHDLs is configurable. The configured value 1181 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1182 * command) or the initialization will fail. 1183 */ 1184 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MRHDL]; 1185 rsrc_pool->rsrc_type = TAVOR_MRHDL; 1186 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1187 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_mr_s); 1188 rsrc_pool->rsrc_state = state; 1189 TAVOR_RSRC_NAME(rsrc_name, TAVOR_MRHDL_CACHE); 1190 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mpt); 1191 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mpt); 1192 hdl_info.swi_rsrcpool = rsrc_pool; 1193 hdl_info.swi_constructor = tavor_rsrc_mrhdl_constructor; 1194 hdl_info.swi_destructor = tavor_rsrc_mrhdl_destructor; 1195 hdl_info.swi_rsrcname = rsrc_name; 1196 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT; 1197 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1198 if (status != DDI_SUCCESS) { 1199 tavor_rsrc_fini(state, cleanup); 1200 /* Set "status" and "errormsg" and goto failure */ 1201 TAVOR_TNF_FAIL(DDI_FAILURE, "MR handle"); 1202 goto rsrcinitp2_fail; 1203 } 1204 cleanup = TAVOR_RSRC_CLEANUP_LEVEL23; 1205 1206 /* 1207 * Initialize the resource pool for the EQ handles. Notice 1208 * that the number of EQHDLs is hardcoded. The hardcoded value 1209 * should be less that the maximum value (obtained from the 1210 * QUERY_DEV_LIM command) or the initialization will fail. 1211 */ 1212 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQHDL]; 1213 rsrc_pool->rsrc_type = TAVOR_EQHDL; 1214 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1215 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_eq_s); 1216 rsrc_pool->rsrc_state = state; 1217 TAVOR_RSRC_NAME(rsrc_name, TAVOR_EQHDL_CACHE); 1218 hdl_info.swi_num = TAVOR_NUM_EQ; 1219 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_eq); 1220 hdl_info.swi_rsrcpool = rsrc_pool; 1221 hdl_info.swi_constructor = NULL; 1222 hdl_info.swi_destructor = NULL; 1223 hdl_info.swi_rsrcname = rsrc_name; 1224 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT; 1225 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1226 if (status != DDI_SUCCESS) { 1227 tavor_rsrc_fini(state, cleanup); 1228 /* Set "status" and "errormsg" and goto failure */ 1229 TAVOR_TNF_FAIL(DDI_FAILURE, "EQ handle"); 1230 goto rsrcinitp2_fail; 1231 } 1232 cleanup = TAVOR_RSRC_CLEANUP_LEVEL24; 1233 1234 /* 1235 * Initialize the resource pool for the CQ handles. Notice 1236 * that the number of CQHDLs is configurable. The configured value 1237 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1238 * command) or the initialization will fail. Note also that the CQ 1239 * handle has constructor and destructor methods associated with it. 1240 */ 1241 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQHDL]; 1242 rsrc_pool->rsrc_type = TAVOR_CQHDL; 1243 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1244 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_cq_s); 1245 rsrc_pool->rsrc_state = state; 1246 TAVOR_RSRC_NAME(rsrc_name, TAVOR_CQHDL_CACHE); 1247 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_cq); 1248 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_cq); 1249 hdl_info.swi_rsrcpool = rsrc_pool; 1250 hdl_info.swi_constructor = tavor_rsrc_cqhdl_constructor; 1251 hdl_info.swi_destructor = tavor_rsrc_cqhdl_destructor; 1252 hdl_info.swi_rsrcname = rsrc_name; 1253 hdl_info.swi_flags = (TAVOR_SWHDL_KMEMCACHE_INIT | 1254 TAVOR_SWHDL_TABLE_INIT); 1255 hdl_info.swi_prealloc_sz = sizeof (tavor_cqhdl_t); 1256 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1257 if (status != DDI_SUCCESS) { 1258 tavor_rsrc_fini(state, cleanup); 1259 /* Set "status" and "errormsg" and goto failure */ 1260 TAVOR_TNF_FAIL(DDI_FAILURE, "CQ handle"); 1261 goto rsrcinitp2_fail; 1262 } 1263 1264 /* 1265 * Save away the pointer to the central list of CQ handle pointers 1266 * This this is used as a mechanism to enable fast CQnumber-to-CQhandle 1267 * lookup during EQ event processing. The table is a list of 1268 * tavor_cqhdl_t allocated by the above routine because of the 1269 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many tavor_cqhdl_t 1270 * as the number of CQs. 1271 */ 1272 state->ts_cqhdl = hdl_info.swi_table_ptr; 1273 cleanup = TAVOR_RSRC_CLEANUP_LEVEL25; 1274 1275 /* 1276 * Initialize the resource pool for the SRQ handles. Notice 1277 * that the number of SRQHDLs is configurable. The configured value 1278 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1279 * command) or the initialization will fail. Note also that the SRQ 1280 * handle has constructor and destructor methods associated with it. 1281 * 1282 * Note: We only allocate these resources if SRQ is enabled in the 1283 * config profile; see below. 1284 */ 1285 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQHDL]; 1286 rsrc_pool->rsrc_type = TAVOR_SRQHDL; 1287 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1288 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_srq_s); 1289 rsrc_pool->rsrc_state = state; 1290 TAVOR_RSRC_NAME(rsrc_name, TAVOR_SRQHDL_CACHE); 1291 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_srq); 1292 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_srq); 1293 hdl_info.swi_rsrcpool = rsrc_pool; 1294 hdl_info.swi_constructor = tavor_rsrc_srqhdl_constructor; 1295 hdl_info.swi_destructor = tavor_rsrc_srqhdl_destructor; 1296 hdl_info.swi_rsrcname = rsrc_name; 1297 hdl_info.swi_flags = (TAVOR_SWHDL_KMEMCACHE_INIT | 1298 TAVOR_SWHDL_TABLE_INIT); 1299 hdl_info.swi_prealloc_sz = sizeof (tavor_srqhdl_t); 1300 1301 /* 1302 * SRQ support is configurable. Only if SRQ is enabled (the default) 1303 * do we actually try to configure these resources. Otherwise, we 1304 * simply set the cleanup level and continue on to the next resource 1305 */ 1306 if (state->ts_cfg_profile->cp_srq_enable != 0) { 1307 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1308 if (status != DDI_SUCCESS) { 1309 tavor_rsrc_fini(state, cleanup); 1310 /* Set "status" and "errormsg" and goto failure */ 1311 TAVOR_TNF_FAIL(DDI_FAILURE, "SRQ handle"); 1312 goto rsrcinitp2_fail; 1313 } 1314 1315 /* 1316 * Save away the pointer to the central list of SRQ handle 1317 * pointers This this is used as a mechanism to enable fast 1318 * SRQnumber-to-SRQhandle lookup. The table is a list of 1319 * tavor_srqhdl_t allocated by the above routine because of the 1320 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many 1321 * tavor_srqhdl_t as the number of SRQs. 1322 */ 1323 state->ts_srqhdl = hdl_info.swi_table_ptr; 1324 } 1325 cleanup = TAVOR_RSRC_CLEANUP_LEVEL26; 1326 1327 /* 1328 * Initialize the resource pool for the address handles. Notice 1329 * that the number of AHHDLs is configurable. The configured value 1330 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1331 * command) or the initialization will fail. 1332 */ 1333 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_AHHDL]; 1334 rsrc_pool->rsrc_type = TAVOR_AHHDL; 1335 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1336 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_ah_s); 1337 rsrc_pool->rsrc_state = state; 1338 TAVOR_RSRC_NAME(rsrc_name, TAVOR_AHHDL_CACHE); 1339 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_ah); 1340 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_av); 1341 hdl_info.swi_rsrcpool = rsrc_pool; 1342 hdl_info.swi_constructor = tavor_rsrc_ahhdl_constructor; 1343 hdl_info.swi_destructor = tavor_rsrc_ahhdl_destructor; 1344 hdl_info.swi_rsrcname = rsrc_name; 1345 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT; 1346 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1347 if (status != DDI_SUCCESS) { 1348 tavor_rsrc_fini(state, cleanup); 1349 /* Set "status" and "errormsg" and goto failure */ 1350 TAVOR_TNF_FAIL(DDI_FAILURE, "AH handle"); 1351 goto rsrcinitp2_fail; 1352 } 1353 cleanup = TAVOR_RSRC_CLEANUP_LEVEL27; 1354 1355 /* 1356 * Initialize the resource pool for the QP handles. Notice 1357 * that the number of QPHDLs is configurable. The configured value 1358 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1359 * command) or the initialization will fail. Note also that the QP 1360 * handle has constructor and destructor methods associated with it. 1361 */ 1362 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPHDL]; 1363 rsrc_pool->rsrc_type = TAVOR_QPHDL; 1364 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1365 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_qp_s); 1366 rsrc_pool->rsrc_state = state; 1367 TAVOR_RSRC_NAME(rsrc_name, TAVOR_QPHDL_CACHE); 1368 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_qp); 1369 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_qp); 1370 hdl_info.swi_rsrcpool = rsrc_pool; 1371 hdl_info.swi_constructor = tavor_rsrc_qphdl_constructor; 1372 hdl_info.swi_destructor = tavor_rsrc_qphdl_destructor; 1373 hdl_info.swi_rsrcname = rsrc_name; 1374 hdl_info.swi_flags = (TAVOR_SWHDL_KMEMCACHE_INIT | 1375 TAVOR_SWHDL_TABLE_INIT); 1376 hdl_info.swi_prealloc_sz = sizeof (tavor_qphdl_t); 1377 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1378 if (status != DDI_SUCCESS) { 1379 tavor_rsrc_fini(state, cleanup); 1380 /* Set "status" and "errormsg" and goto failure */ 1381 TAVOR_TNF_FAIL(DDI_FAILURE, "QP handle"); 1382 goto rsrcinitp2_fail; 1383 } 1384 1385 /* 1386 * Save away the pointer to the central list of QP handle pointers 1387 * This this is used as a mechanism to enable fast QPnumber-to-QPhandle 1388 * lookup during CQ event processing. The table is a list of 1389 * tavor_qphdl_t allocated by the above routine because of the 1390 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many tavor_qphdl_t 1391 * as the number of QPs. 1392 */ 1393 state->ts_qphdl = hdl_info.swi_table_ptr; 1394 cleanup = TAVOR_RSRC_CLEANUP_LEVEL28; 1395 1396 /* 1397 * Initialize the resource pool for the reference count handles. 1398 * Notice that the number of REFCNTs is configurable, but it's value 1399 * is set to the number of MPTs. Since REFCNTs are used to support 1400 * shared memory regions, it is possible that we might require as 1401 * one REFCNT for every MPT. 1402 */ 1403 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_REFCNT]; 1404 rsrc_pool->rsrc_type = TAVOR_REFCNT; 1405 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1406 rsrc_pool->rsrc_quantum = sizeof (tavor_sw_refcnt_t); 1407 rsrc_pool->rsrc_state = state; 1408 TAVOR_RSRC_NAME(rsrc_name, TAVOR_REFCNT_CACHE); 1409 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mpt); 1410 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mpt); 1411 hdl_info.swi_rsrcpool = rsrc_pool; 1412 hdl_info.swi_constructor = tavor_rsrc_refcnt_constructor; 1413 hdl_info.swi_destructor = tavor_rsrc_refcnt_destructor; 1414 hdl_info.swi_rsrcname = rsrc_name; 1415 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT; 1416 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1417 if (status != DDI_SUCCESS) { 1418 tavor_rsrc_fini(state, cleanup); 1419 /* Set "status" and "errormsg" and goto failure */ 1420 TAVOR_TNF_FAIL(DDI_FAILURE, "reference count handle"); 1421 goto rsrcinitp2_fail; 1422 } 1423 cleanup = TAVOR_RSRC_CLEANUP_LEVEL29; 1424 1425 /* 1426 * Initialize the resource pool for the MCG handles. Notice that for 1427 * these MCG handles, we are allocating a table of structures (used to 1428 * keep track of the MCG entries that are being written to hardware 1429 * and to speed up multicast attach/detach operations). 1430 */ 1431 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mcg); 1432 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mcg); 1433 hdl_info.swi_flags = TAVOR_SWHDL_TABLE_INIT; 1434 hdl_info.swi_prealloc_sz = sizeof (struct tavor_sw_mcg_list_s); 1435 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1436 if (status != DDI_SUCCESS) { 1437 tavor_rsrc_fini(state, cleanup); 1438 /* Set "status" and "errormsg" and goto failure */ 1439 TAVOR_TNF_FAIL(DDI_FAILURE, "MCG handle"); 1440 goto rsrcinitp2_fail; 1441 } 1442 state->ts_mcghdl = hdl_info.swi_table_ptr; 1443 cleanup = TAVOR_RSRC_CLEANUP_LEVEL30; 1444 1445 /* 1446 * Initialize the resource pools for all objects that exist in 1447 * UAR memory. The only objects that are allocated from UAR memory 1448 * are the UAR pages which are used for holding Tavor hardware's 1449 * doorbell registers. 1450 */ 1451 1452 /* 1453 * Initialize the resource pool for the UAR pages. Notice 1454 * that the number of UARPGs is configurable. The configured value 1455 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1456 * command) or the initialization will fail. Note also that by 1457 * specifying the rsrc_start parameter in advance, we direct the 1458 * initialization routine not to attempt to allocated space from the 1459 * Tavor DDR vmem_arena. 1460 */ 1461 num = ((uint64_t)1 << cfgprof->cp_log_num_uar); 1462 max = ((uint64_t)1 << (state->ts_devlim.log_max_uar_sz + 20 - 1463 PAGESHIFT)); 1464 num_prealloc = 0; 1465 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UARPG]; 1466 rsrc_pool->rsrc_type = TAVOR_UARPG; 1467 rsrc_pool->rsrc_loc = TAVOR_IN_UAR; 1468 rsrc_pool->rsrc_pool_size = (num << PAGESHIFT); 1469 rsrc_pool->rsrc_shift = PAGESHIFT; 1470 rsrc_pool->rsrc_quantum = PAGESIZE; 1471 rsrc_pool->rsrc_align = PAGESIZE; 1472 rsrc_pool->rsrc_state = state; 1473 rsrc_pool->rsrc_start = (void *)state->ts_reg_uar_baseaddr; 1474 TAVOR_RSRC_NAME(rsrc_name, TAVOR_UAR_VMEM); 1475 entry_info.hwi_num = num; 1476 entry_info.hwi_max = max; 1477 entry_info.hwi_prealloc = num_prealloc; 1478 entry_info.hwi_rsrcpool = rsrc_pool; 1479 entry_info.hwi_rsrcname = rsrc_name; 1480 status = tavor_rsrc_hw_entries_init(state, &entry_info); 1481 if (status != DDI_SUCCESS) { 1482 tavor_rsrc_fini(state, cleanup); 1483 /* Set "status" and "errormsg" and goto failure */ 1484 TAVOR_TNF_FAIL(DDI_FAILURE, "UAR page table"); 1485 goto rsrcinitp2_fail; 1486 } 1487 cleanup = TAVOR_RSRC_CLEANUP_ALL; 1488 1489 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN); 1490 TAVOR_TNF_EXIT(tavor_rsrc_init_phase2); 1491 return (DDI_SUCCESS); 1492 1493 rsrcinitp2_fail: 1494 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN); 1495 TNF_PROBE_1(tavor_rsrc_init_phase2_fail, TAVOR_TNF_ERROR, "", 1496 tnf_string, msg, errormsg); 1497 TAVOR_TNF_EXIT(tavor_rsrc_init_phase2); 1498 return (status); 1499 } 1500 1501 1502 /* 1503 * tavor_rsrc_fini() 1504 * Context: Only called from attach() and/or detach() path contexts 1505 */ 1506 void 1507 tavor_rsrc_fini(tavor_state_t *state, tavor_rsrc_cleanup_level_t clean) 1508 { 1509 tavor_rsrc_sw_hdl_info_t hdl_info; 1510 tavor_rsrc_hw_entry_info_t entry_info; 1511 tavor_rsrc_mbox_info_t mbox_info; 1512 tavor_cfg_profile_t *cfgprof; 1513 1514 TAVOR_TNF_ENTER(tavor_rsrc_fini); 1515 1516 ASSERT(state != NULL); 1517 1518 cfgprof = state->ts_cfg_profile; 1519 1520 switch (clean) { 1521 /* 1522 * If we add more resources that need to be cleaned up here, we should 1523 * ensure that TAVOR_RSRC_CLEANUP_ALL is still the first entry (i.e. 1524 * corresponds to the last resource allocated). 1525 */ 1526 case TAVOR_RSRC_CLEANUP_ALL: 1527 /* Cleanup the UAR page resource pool */ 1528 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UARPG]; 1529 tavor_rsrc_hw_entries_fini(state, &entry_info); 1530 /* FALLTHROUGH */ 1531 1532 case TAVOR_RSRC_CLEANUP_LEVEL30: 1533 /* Cleanup the central MCG handle pointers list */ 1534 hdl_info.swi_rsrcpool = NULL; 1535 hdl_info.swi_table_ptr = state->ts_mcghdl; 1536 hdl_info.swi_num = 1537 ((uint64_t)1 << cfgprof->cp_log_num_mcg); 1538 hdl_info.swi_prealloc_sz = sizeof (struct tavor_sw_mcg_list_s); 1539 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1540 /* FALLTHROUGH */ 1541 1542 case TAVOR_RSRC_CLEANUP_LEVEL29: 1543 /* Cleanup the reference count resource pool */ 1544 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_REFCNT]; 1545 hdl_info.swi_table_ptr = NULL; 1546 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1547 /* FALLTHROUGH */ 1548 1549 case TAVOR_RSRC_CLEANUP_LEVEL28: 1550 /* Cleanup the QP handle resource pool */ 1551 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_QPHDL]; 1552 hdl_info.swi_table_ptr = state->ts_qphdl; 1553 hdl_info.swi_num = 1554 ((uint64_t)1 << cfgprof->cp_log_num_qp); 1555 hdl_info.swi_prealloc_sz = sizeof (tavor_qphdl_t); 1556 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1557 /* FALLTHROUGH */ 1558 1559 case TAVOR_RSRC_CLEANUP_LEVEL27: 1560 /* Cleanup the address handle resource pool */ 1561 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_AHHDL]; 1562 hdl_info.swi_table_ptr = NULL; 1563 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1564 /* FALLTHROUGH */ 1565 1566 case TAVOR_RSRC_CLEANUP_LEVEL26: 1567 /* 1568 * Cleanup the SRQ handle resource pool. 1569 * 1570 * Note: We only clean up if SRQ is enabled. Otherwise we 1571 * simply fallthrough to the next resource cleanup. 1572 */ 1573 if (state->ts_cfg_profile->cp_srq_enable != 0) { 1574 hdl_info.swi_rsrcpool = 1575 &state->ts_rsrc_hdl[TAVOR_SRQHDL]; 1576 hdl_info.swi_table_ptr = state->ts_srqhdl; 1577 hdl_info.swi_num = 1578 ((uint64_t)1 << cfgprof->cp_log_num_srq); 1579 hdl_info.swi_prealloc_sz = sizeof (tavor_srqhdl_t); 1580 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1581 } 1582 /* FALLTHROUGH */ 1583 1584 case TAVOR_RSRC_CLEANUP_LEVEL25: 1585 /* Cleanup the CQ handle resource pool */ 1586 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_CQHDL]; 1587 hdl_info.swi_table_ptr = state->ts_cqhdl; 1588 hdl_info.swi_num = 1589 ((uint64_t)1 << cfgprof->cp_log_num_cq); 1590 hdl_info.swi_prealloc_sz = sizeof (tavor_cqhdl_t); 1591 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1592 /* FALLTHROUGH */ 1593 1594 case TAVOR_RSRC_CLEANUP_LEVEL24: 1595 /* Cleanup the EQ handle resource pool */ 1596 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQHDL]; 1597 hdl_info.swi_table_ptr = NULL; 1598 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1599 /* FALLTHROUGH */ 1600 1601 case TAVOR_RSRC_CLEANUP_LEVEL23: 1602 /* Cleanup the MR handle resource pool */ 1603 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MRHDL]; 1604 hdl_info.swi_table_ptr = NULL; 1605 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1606 /* FALLTHROUGH */ 1607 1608 case TAVOR_RSRC_CLEANUP_LEVEL22: 1609 /* Cleanup the PD handle resource pool */ 1610 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_PDHDL]; 1611 hdl_info.swi_table_ptr = NULL; 1612 tavor_rsrc_pd_handles_fini(state, &hdl_info); 1613 /* FALLTHROUGH */ 1614 1615 case TAVOR_RSRC_CLEANUP_LEVEL21: 1616 /* Cleanup the EQC table resource pool */ 1617 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQC]; 1618 tavor_rsrc_hw_entries_fini(state, &entry_info); 1619 /* FALLTHROUGH */ 1620 1621 case TAVOR_RSRC_CLEANUP_LEVEL20: 1622 /* Cleanup the MCG table resource pool */ 1623 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MCG]; 1624 tavor_rsrc_hw_entries_fini(state, &entry_info); 1625 /* FALLTHROUGH */ 1626 1627 case TAVOR_RSRC_CLEANUP_LEVEL19: 1628 /* Cleanup the outstanding command list */ 1629 tavor_outstanding_cmdlist_fini(state); 1630 /* FALLTHROUGH */ 1631 1632 case TAVOR_RSRC_CLEANUP_LEVEL18: 1633 /* Cleanup the "In" mailbox list */ 1634 tavor_intr_inmbox_list_fini(state); 1635 /* FALLTHROUGH */ 1636 1637 case TAVOR_RSRC_CLEANUP_LEVEL17: 1638 /* Cleanup the interrupt "In" mailbox resource pool */ 1639 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[ 1640 TAVOR_INTR_IN_MBOX]; 1641 tavor_rsrc_mbox_fini(state, &mbox_info); 1642 /* FALLTHROUGH */ 1643 1644 case TAVOR_RSRC_CLEANUP_LEVEL16: 1645 /* Cleanup the "In" mailbox list */ 1646 tavor_inmbox_list_fini(state); 1647 /* FALLTHROUGH */ 1648 1649 case TAVOR_RSRC_CLEANUP_LEVEL15: 1650 /* Cleanup the "In" mailbox resource pool */ 1651 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_IN_MBOX]; 1652 tavor_rsrc_mbox_fini(state, &mbox_info); 1653 /* FALLTHROUGH */ 1654 1655 case TAVOR_RSRC_CLEANUP_LEVEL14: 1656 /* 1657 * Cleanup the SRQC table resource pool. 1658 * 1659 * Note: We only clean up if SRQ is enabled. Otherwise we 1660 * simply fallthrough to the next resource cleanup. 1661 */ 1662 if (state->ts_cfg_profile->cp_srq_enable != 0) { 1663 entry_info.hwi_rsrcpool = 1664 &state->ts_rsrc_hdl[TAVOR_SRQC]; 1665 tavor_rsrc_hw_entries_fini(state, &entry_info); 1666 } 1667 /* FALLTHROUGH */ 1668 1669 case TAVOR_RSRC_CLEANUP_LEVEL13: 1670 /* Cleanup the UAR scratch table resource pool */ 1671 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR]; 1672 tavor_rsrc_hw_entries_fini(state, &entry_info); 1673 /* FALLTHROUGH */ 1674 1675 case TAVOR_RSRC_CLEANUP_LEVEL12: 1676 /* Cleanup the UDAV table resource pool */ 1677 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UDAV]; 1678 tavor_rsrc_hw_entries_fini(state, &entry_info); 1679 /* FALLTHROUGH */ 1680 1681 case TAVOR_RSRC_CLEANUP_LEVEL11: 1682 /* Cleanup the EQPC table resource pool */ 1683 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQPC]; 1684 tavor_rsrc_hw_entries_fini(state, &entry_info); 1685 /* FALLTHROUGH */ 1686 1687 case TAVOR_RSRC_CLEANUP_LEVEL10: 1688 /* Cleanup the CQC table resource pool */ 1689 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_CQC]; 1690 tavor_rsrc_hw_entries_fini(state, &entry_info); 1691 /* FALLTHROUGH */ 1692 1693 case TAVOR_RSRC_CLEANUP_LEVEL9: 1694 /* Cleanup the RDB table resource pool */ 1695 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_RDB]; 1696 tavor_rsrc_hw_entries_fini(state, &entry_info); 1697 /* FALLTHROUGH */ 1698 1699 case TAVOR_RSRC_CLEANUP_LEVEL8: 1700 /* Cleanup the QPC table resource pool */ 1701 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_QPC]; 1702 tavor_rsrc_hw_entries_fini(state, &entry_info); 1703 /* FALLTHROUGH */ 1704 1705 case TAVOR_RSRC_CLEANUP_LEVEL7: 1706 /* Cleanup the MTT table resource pool */ 1707 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MTT]; 1708 tavor_rsrc_hw_entries_fini(state, &entry_info); 1709 /* FALLTHROUGH */ 1710 1711 case TAVOR_RSRC_CLEANUP_LEVEL6: 1712 /* Cleanup the MPT table resource pool */ 1713 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MPT]; 1714 tavor_rsrc_hw_entries_fini(state, &entry_info); 1715 /* FALLTHROUGH */ 1716 1717 case TAVOR_RSRC_CLEANUP_LEVEL5: 1718 /* Destroy the vmem arena for DDR memory */ 1719 vmem_destroy(state->ts_ddrvmem); 1720 break; 1721 1722 /* 1723 * The cleanup below comes from the "Phase 1" initialization step. 1724 * (see tavor_rsrc_init_phase1() above) 1725 */ 1726 case TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE: 1727 /* Cleanup the interrupt "Out" mailbox list */ 1728 tavor_intr_outmbox_list_fini(state); 1729 /* FALLTHROUGH */ 1730 1731 case TAVOR_RSRC_CLEANUP_LEVEL3: 1732 /* Cleanup the "Out" mailbox resource pool */ 1733 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[ 1734 TAVOR_INTR_OUT_MBOX]; 1735 tavor_rsrc_mbox_fini(state, &mbox_info); 1736 /* FALLTHROUGH */ 1737 1738 case TAVOR_RSRC_CLEANUP_LEVEL2: 1739 /* Cleanup the "Out" mailbox list */ 1740 tavor_outmbox_list_fini(state); 1741 /* FALLTHROUGH */ 1742 1743 case TAVOR_RSRC_CLEANUP_LEVEL1: 1744 /* Cleanup the "Out" mailbox resource pool */ 1745 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_OUT_MBOX]; 1746 tavor_rsrc_mbox_fini(state, &mbox_info); 1747 /* FALLTHROUGH */ 1748 1749 case TAVOR_RSRC_CLEANUP_LEVEL0: 1750 /* Free the array of tavor_rsrc_pool_info_t's */ 1751 kmem_free(state->ts_rsrc_hdl, TAVOR_NUM_RESOURCES * 1752 sizeof (tavor_rsrc_pool_info_t)); 1753 kmem_cache_destroy(state->ts_rsrc_cache); 1754 break; 1755 1756 default: 1757 TAVOR_WARNING(state, "unexpected resource cleanup level"); 1758 TNF_PROBE_0(tavor_rsrc_fini_default_fail, TAVOR_TNF_ERROR, ""); 1759 TAVOR_TNF_EXIT(tavor_rsrc_fini); 1760 return; 1761 } 1762 1763 TAVOR_TNF_EXIT(tavor_rsrc_fini); 1764 } 1765 1766 1767 /* 1768 * tavor_rsrc_mbox_init() 1769 * Context: Only called from attach() path context 1770 */ 1771 static int 1772 tavor_rsrc_mbox_init(tavor_state_t *state, tavor_rsrc_mbox_info_t *info) 1773 { 1774 tavor_rsrc_pool_info_t *rsrc_pool; 1775 tavor_rsrc_priv_mbox_t *priv; 1776 vmem_t *vmp; 1777 uint64_t offset; 1778 uint_t dma_xfer_mode; 1779 1780 TAVOR_TNF_ENTER(tavor_rsrc_mbox_init); 1781 1782 ASSERT(state != NULL); 1783 ASSERT(info != NULL); 1784 1785 rsrc_pool = info->mbi_rsrcpool; 1786 ASSERT(rsrc_pool != NULL); 1787 1788 dma_xfer_mode = state->ts_cfg_profile->cp_streaming_consistent; 1789 1790 /* Allocate and initialize mailbox private structure */ 1791 priv = kmem_zalloc(sizeof (tavor_rsrc_priv_mbox_t), KM_SLEEP); 1792 priv->pmb_dip = state->ts_dip; 1793 priv->pmb_acchdl = state->ts_reg_ddrhdl; 1794 priv->pmb_devaccattr = state->ts_reg_accattr; 1795 priv->pmb_xfer_mode = dma_xfer_mode; 1796 1797 /* 1798 * Initialize many of the default DMA attributes. Then set alignment 1799 * and scatter-gather restrictions specific for mailbox memory. 1800 */ 1801 tavor_dma_attr_init(&priv->pmb_dmaattr); 1802 priv->pmb_dmaattr.dma_attr_align = TAVOR_MBOX_ALIGN; 1803 priv->pmb_dmaattr.dma_attr_sgllen = 1; 1804 1805 rsrc_pool->rsrc_private = priv; 1806 1807 /* Is object in DDR memory or system memory? */ 1808 if (rsrc_pool->rsrc_loc == TAVOR_IN_DDR) { 1809 rsrc_pool->rsrc_ddr_offset = vmem_xalloc(state->ts_ddrvmem, 1810 rsrc_pool->rsrc_pool_size, rsrc_pool->rsrc_align, 1811 0, 0, NULL, NULL, VM_SLEEP); 1812 if (rsrc_pool->rsrc_ddr_offset == NULL) { 1813 /* Unable to alloc space for mailboxes */ 1814 kmem_free(priv, sizeof (tavor_rsrc_priv_mbox_t)); 1815 TNF_PROBE_0(tavor_rsrc_mbox_init_vma_fail, 1816 TAVOR_TNF_ERROR, ""); 1817 TAVOR_TNF_EXIT(tavor_rsrc_mbox_init); 1818 return (DDI_FAILURE); 1819 } 1820 1821 /* Calculate offset and starting point (in DDR) */ 1822 offset = ((uintptr_t)rsrc_pool->rsrc_ddr_offset - 1823 state->ts_ddr.ddr_baseaddr); 1824 rsrc_pool->rsrc_start = 1825 (void *)(uintptr_t)((uintptr_t)state->ts_reg_ddr_baseaddr + 1826 offset); 1827 1828 /* Create new vmem arena for the mailboxes */ 1829 vmp = vmem_create(info->mbi_rsrcname, 1830 rsrc_pool->rsrc_ddr_offset, rsrc_pool->rsrc_pool_size, 1831 rsrc_pool->rsrc_quantum, NULL, NULL, NULL, 0, VM_SLEEP); 1832 if (vmp == NULL) { 1833 /* Unable to create vmem arena */ 1834 vmem_xfree(state->ts_ddrvmem, 1835 rsrc_pool->rsrc_ddr_offset, 1836 rsrc_pool->rsrc_pool_size); 1837 kmem_free(priv, sizeof (tavor_rsrc_priv_mbox_t)); 1838 TNF_PROBE_0(tavor_rsrc_mbox_init_vmem_create_fail, 1839 TAVOR_TNF_ERROR, ""); 1840 TAVOR_TNF_EXIT(tavor_rsrc_mbox_init); 1841 return (DDI_FAILURE); 1842 } 1843 rsrc_pool->rsrc_vmp = vmp; 1844 } else { 1845 rsrc_pool->rsrc_ddr_offset = NULL; 1846 rsrc_pool->rsrc_start = NULL; 1847 rsrc_pool->rsrc_vmp = NULL; 1848 } 1849 1850 TAVOR_TNF_EXIT(tavor_rsrc_mbox_init); 1851 return (DDI_SUCCESS); 1852 } 1853 1854 1855 /* 1856 * tavor_rsrc_mbox_fini() 1857 * Context: Only called from attach() and/or detach() path contexts 1858 */ 1859 static void 1860 tavor_rsrc_mbox_fini(tavor_state_t *state, tavor_rsrc_mbox_info_t *info) 1861 { 1862 tavor_rsrc_pool_info_t *rsrc_pool; 1863 1864 TAVOR_TNF_ENTER(tavor_rsrc_mbox_fini); 1865 1866 ASSERT(state != NULL); 1867 ASSERT(info != NULL); 1868 1869 rsrc_pool = info->mbi_rsrcpool; 1870 ASSERT(rsrc_pool != NULL); 1871 1872 /* If mailboxes are DDR memory, then destroy and free up vmem */ 1873 if (rsrc_pool->rsrc_loc == TAVOR_IN_DDR) { 1874 1875 /* Destroy the specially created mbox vmem arena */ 1876 vmem_destroy(rsrc_pool->rsrc_vmp); 1877 1878 /* Free up the region from the ddr_vmem arena */ 1879 vmem_xfree(state->ts_ddrvmem, rsrc_pool->rsrc_ddr_offset, 1880 rsrc_pool->rsrc_pool_size); 1881 } 1882 1883 /* Free up the private struct */ 1884 kmem_free(rsrc_pool->rsrc_private, sizeof (tavor_rsrc_priv_mbox_t)); 1885 1886 TAVOR_TNF_EXIT(tavor_rsrc_mbox_fini); 1887 } 1888 1889 1890 /* 1891 * tavor_rsrc_hw_entries_init() 1892 * Context: Only called from attach() path context 1893 */ 1894 static int 1895 tavor_rsrc_hw_entries_init(tavor_state_t *state, 1896 tavor_rsrc_hw_entry_info_t *info) 1897 { 1898 tavor_rsrc_pool_info_t *rsrc_pool; 1899 tavor_rsrc_t *rsvd_rsrc = NULL; 1900 vmem_t *vmp; 1901 uint64_t num_hwentry, max_hwentry, num_prealloc; 1902 uint64_t offset; 1903 int status; 1904 1905 TAVOR_TNF_ENTER(tavor_rsrc_hw_entries_init); 1906 1907 ASSERT(state != NULL); 1908 ASSERT(info != NULL); 1909 1910 rsrc_pool = info->hwi_rsrcpool; 1911 ASSERT(rsrc_pool != NULL); 1912 num_hwentry = info->hwi_num; 1913 max_hwentry = info->hwi_max; 1914 num_prealloc = info->hwi_prealloc; 1915 1916 /* Make sure number of HW entries makes sense */ 1917 if (num_hwentry > max_hwentry) { 1918 TNF_PROBE_2(tavor_rsrc_hw_entries_init_toomany_fail, 1919 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of HW " 1920 "entries exceeds device maximum", tnf_uint, maxhw, 1921 max_hwentry); 1922 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init); 1923 return (DDI_FAILURE); 1924 } 1925 1926 /* 1927 * Determine if we need to allocate DDR space to set up the 1928 * "rsrc_start" pointer. Not necessary if "rsrc_start" has already 1929 * been initialized (as is the case for the UAR page init). 1930 */ 1931 if (rsrc_pool->rsrc_start == NULL) { 1932 /* Make sure HW entries table is aligned as specified */ 1933 rsrc_pool->rsrc_ddr_offset = vmem_xalloc(state->ts_ddrvmem, 1934 rsrc_pool->rsrc_pool_size, rsrc_pool->rsrc_align, 1935 0, 0, NULL, NULL, VM_NOSLEEP | VM_FIRSTFIT); 1936 if (rsrc_pool->rsrc_ddr_offset == NULL) { 1937 /* Unable to alloc space for aligned HW table */ 1938 TNF_PROBE_0(tavor_rsrc_hw_entry_table_vmxalloc_fail, 1939 TAVOR_TNF_ERROR, ""); 1940 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init); 1941 return (DDI_FAILURE); 1942 } 1943 1944 /* Calculate offset and starting point (in DDR) */ 1945 offset = ((uintptr_t)rsrc_pool->rsrc_ddr_offset - 1946 state->ts_ddr.ddr_baseaddr); 1947 rsrc_pool->rsrc_start = 1948 (void *)(uintptr_t)((uintptr_t)state->ts_reg_ddr_baseaddr + 1949 offset); 1950 } else { 1951 rsrc_pool->rsrc_ddr_offset = rsrc_pool->rsrc_start; 1952 } 1953 1954 /* 1955 * Create new vmem arena for the HW entries table (if rsrc_quantum 1956 * is non-zero). Otherwise if rsrc_quantum is zero, then these HW 1957 * entries are not going to be dynamically allocatable (i.e. they 1958 * won't be allocated/freed through tavor_rsrc_alloc/free). This 1959 * latter option is used for EQPC and UARSCR resource which are, in 1960 * fact, managed by the Tavor hardware. 1961 */ 1962 if (rsrc_pool->rsrc_quantum != 0) { 1963 vmp = vmem_create(info->hwi_rsrcname, 1964 rsrc_pool->rsrc_ddr_offset, rsrc_pool->rsrc_pool_size, 1965 rsrc_pool->rsrc_quantum, NULL, NULL, NULL, 0, VM_SLEEP); 1966 if (vmp == NULL) { 1967 /* Unable to create vmem arena */ 1968 if (rsrc_pool->rsrc_ddr_offset != 1969 rsrc_pool->rsrc_start) { 1970 vmem_xfree(state->ts_ddrvmem, 1971 rsrc_pool->rsrc_ddr_offset, 1972 rsrc_pool->rsrc_pool_size); 1973 } 1974 TNF_PROBE_0(tavor_rsrc_hw_entries_init_vmemcreate_fail, 1975 TAVOR_TNF_ERROR, ""); 1976 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init); 1977 return (DDI_FAILURE); 1978 } 1979 rsrc_pool->rsrc_vmp = vmp; 1980 } else { 1981 rsrc_pool->rsrc_vmp = NULL; 1982 } 1983 1984 /* The first HW entries may be reserved by Tavor firmware */ 1985 if (num_prealloc != 0) { 1986 status = tavor_rsrc_alloc(state, rsrc_pool->rsrc_type, 1987 num_prealloc, TAVOR_SLEEP, &rsvd_rsrc); 1988 if (status != DDI_SUCCESS) { 1989 /* Unable to preallocate the reserved HW entries */ 1990 if (rsrc_pool->rsrc_vmp != NULL) { 1991 vmem_destroy(rsrc_pool->rsrc_vmp); 1992 } 1993 if (rsrc_pool->rsrc_ddr_offset != 1994 rsrc_pool->rsrc_start) { 1995 vmem_xfree(state->ts_ddrvmem, 1996 rsrc_pool->rsrc_ddr_offset, 1997 rsrc_pool->rsrc_pool_size); 1998 } 1999 TNF_PROBE_0(tavor_rsrc_hw_entries_init_pre_fail, 2000 TAVOR_TNF_ERROR, ""); 2001 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init); 2002 return (DDI_FAILURE); 2003 } 2004 } 2005 rsrc_pool->rsrc_private = rsvd_rsrc; 2006 2007 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init); 2008 return (DDI_SUCCESS); 2009 } 2010 2011 2012 /* 2013 * tavor_rsrc_hw_entries_fini() 2014 * Context: Only called from attach() and/or detach() path contexts 2015 */ 2016 static void 2017 tavor_rsrc_hw_entries_fini(tavor_state_t *state, 2018 tavor_rsrc_hw_entry_info_t *info) 2019 { 2020 tavor_rsrc_pool_info_t *rsrc_pool; 2021 tavor_rsrc_t *rsvd_rsrc; 2022 2023 TAVOR_TNF_ENTER(tavor_rsrc_hw_entries_fini); 2024 2025 ASSERT(state != NULL); 2026 ASSERT(info != NULL); 2027 2028 rsrc_pool = info->hwi_rsrcpool; 2029 ASSERT(rsrc_pool != NULL); 2030 2031 /* Free up any "reserved" (i.e. preallocated) HW entries */ 2032 rsvd_rsrc = (tavor_rsrc_t *)rsrc_pool->rsrc_private; 2033 if (rsvd_rsrc != NULL) { 2034 tavor_rsrc_free(state, &rsvd_rsrc); 2035 } 2036 2037 /* 2038 * If we've actually setup a vmem arena for the HW entries, then 2039 * destroy it now 2040 */ 2041 if (rsrc_pool->rsrc_vmp != NULL) { 2042 vmem_destroy(rsrc_pool->rsrc_vmp); 2043 } 2044 2045 /* 2046 * Determine if a region was allocated from the tavor_ddr_vmem 2047 * arena (and free it up if necessary) 2048 */ 2049 if (rsrc_pool->rsrc_ddr_offset != rsrc_pool->rsrc_start) { 2050 vmem_xfree(state->ts_ddrvmem, rsrc_pool->rsrc_ddr_offset, 2051 rsrc_pool->rsrc_pool_size); 2052 } 2053 2054 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_fini); 2055 } 2056 2057 2058 /* 2059 * tavor_rsrc_sw_handles_init() 2060 * Context: Only called from attach() path context 2061 */ 2062 /* ARGSUSED */ 2063 static int 2064 tavor_rsrc_sw_handles_init(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info) 2065 { 2066 tavor_rsrc_pool_info_t *rsrc_pool; 2067 uint64_t num_swhdl, max_swhdl, prealloc_sz; 2068 2069 TAVOR_TNF_ENTER(tavor_rsrc_sw_handles_init); 2070 2071 ASSERT(state != NULL); 2072 ASSERT(info != NULL); 2073 2074 rsrc_pool = info->swi_rsrcpool; 2075 ASSERT(rsrc_pool != NULL); 2076 num_swhdl = info->swi_num; 2077 max_swhdl = info->swi_max; 2078 prealloc_sz = info->swi_prealloc_sz; 2079 2080 /* Make sure number of SW handles makes sense */ 2081 if (num_swhdl > max_swhdl) { 2082 TNF_PROBE_2(tavor_rsrc_sw_handles_init_toomany_fail, 2083 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of SW " 2084 "handles exceeds maximum", tnf_uint, maxsw, max_swhdl); 2085 TAVOR_TNF_EXIT(tavor_rsrc_sw_handles_init); 2086 return (DDI_FAILURE); 2087 } 2088 2089 /* 2090 * Depending on the flags parameter, create a kmem_cache for some 2091 * number of software handle structures. Note: kmem_cache_create() 2092 * will SLEEP until successful. 2093 */ 2094 if (info->swi_flags & TAVOR_SWHDL_KMEMCACHE_INIT) { 2095 rsrc_pool->rsrc_private = kmem_cache_create( 2096 info->swi_rsrcname, rsrc_pool->rsrc_quantum, 0, 2097 info->swi_constructor, info->swi_destructor, NULL, 2098 rsrc_pool->rsrc_state, NULL, 0); 2099 } 2100 2101 /* Allocate the central list of SW handle pointers */ 2102 if (info->swi_flags & TAVOR_SWHDL_TABLE_INIT) { 2103 info->swi_table_ptr = kmem_zalloc(num_swhdl * prealloc_sz, 2104 KM_SLEEP); 2105 } 2106 2107 TAVOR_TNF_EXIT(tavor_rsrc_sw_handles_init); 2108 return (DDI_SUCCESS); 2109 } 2110 2111 2112 /* 2113 * tavor_rsrc_sw_handles_fini() 2114 * Context: Only called from attach() and/or detach() path contexts 2115 */ 2116 /* ARGSUSED */ 2117 static void 2118 tavor_rsrc_sw_handles_fini(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info) 2119 { 2120 tavor_rsrc_pool_info_t *rsrc_pool; 2121 uint64_t num_swhdl, prealloc_sz; 2122 2123 TAVOR_TNF_ENTER(tavor_rsrc_sw_handles_fini); 2124 2125 ASSERT(state != NULL); 2126 ASSERT(info != NULL); 2127 2128 rsrc_pool = info->swi_rsrcpool; 2129 num_swhdl = info->swi_num; 2130 prealloc_sz = info->swi_prealloc_sz; 2131 2132 /* 2133 * If a "software handle" kmem_cache exists for this resource, then 2134 * destroy it now 2135 */ 2136 if (rsrc_pool != NULL) { 2137 kmem_cache_destroy(rsrc_pool->rsrc_private); 2138 } 2139 2140 /* Free up this central list of SW handle pointers */ 2141 if (info->swi_table_ptr != NULL) { 2142 kmem_free(info->swi_table_ptr, num_swhdl * prealloc_sz); 2143 } 2144 2145 TAVOR_TNF_EXIT(tavor_rsrc_sw_handles_fini); 2146 } 2147 2148 2149 /* 2150 * tavor_rsrc_pd_handles_init() 2151 * Context: Only called from attach() path context 2152 */ 2153 static int 2154 tavor_rsrc_pd_handles_init(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info) 2155 { 2156 tavor_rsrc_pool_info_t *rsrc_pool; 2157 vmem_t *vmp; 2158 char vmem_name[TAVOR_RSRC_NAME_MAXLEN]; 2159 int status; 2160 2161 TAVOR_TNF_ENTER(tavor_rsrc_pd_handles_init); 2162 2163 ASSERT(state != NULL); 2164 ASSERT(info != NULL); 2165 2166 rsrc_pool = info->swi_rsrcpool; 2167 ASSERT(rsrc_pool != NULL); 2168 2169 /* Initialize the resource pool for software handle table */ 2170 status = tavor_rsrc_sw_handles_init(state, info); 2171 if (status != DDI_SUCCESS) { 2172 TNF_PROBE_0(tavor_rsrc_pdhdl_alloc_fail, TAVOR_TNF_ERROR, ""); 2173 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc); 2174 return (DDI_FAILURE); 2175 } 2176 2177 /* Build vmem arena name from Tavor instance */ 2178 TAVOR_RSRC_NAME(vmem_name, TAVOR_PDHDL_VMEM); 2179 2180 /* Create new vmem arena for PD numbers */ 2181 vmp = vmem_create(vmem_name, (caddr_t)1, info->swi_num, 1, NULL, 2182 NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER); 2183 if (vmp == NULL) { 2184 /* Unable to create vmem arena */ 2185 info->swi_table_ptr = NULL; 2186 tavor_rsrc_sw_handles_fini(state, info); 2187 TNF_PROBE_0(tavor_rsrc_pd_handles_init_vmem_create_fail, 2188 TAVOR_TNF_ERROR, ""); 2189 TAVOR_TNF_EXIT(tavor_rsrc_pd_handles_init); 2190 return (DDI_FAILURE); 2191 } 2192 rsrc_pool->rsrc_vmp = vmp; 2193 2194 TAVOR_TNF_EXIT(tavor_rsrc_pd_handles_init); 2195 return (DDI_SUCCESS); 2196 } 2197 2198 2199 /* 2200 * tavor_rsrc_pd_handles_fini() 2201 * Context: Only called from attach() and/or detach() path contexts 2202 */ 2203 static void 2204 tavor_rsrc_pd_handles_fini(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info) 2205 { 2206 tavor_rsrc_pool_info_t *rsrc_pool; 2207 2208 TAVOR_TNF_ENTER(tavor_rsrc_pd_handles_fini); 2209 2210 ASSERT(state != NULL); 2211 ASSERT(info != NULL); 2212 2213 rsrc_pool = info->swi_rsrcpool; 2214 2215 /* Destroy the specially created UAR scratch table vmem arena */ 2216 vmem_destroy(rsrc_pool->rsrc_vmp); 2217 2218 /* Destroy the "tavor_sw_pd_t" kmem_cache */ 2219 tavor_rsrc_sw_handles_fini(state, info); 2220 2221 TAVOR_TNF_EXIT(tavor_rsrc_pd_handles_fini); 2222 } 2223 2224 2225 /* 2226 * tavor_rsrc_mbox_alloc() 2227 * Context: Only called from attach() path context 2228 */ 2229 static int 2230 tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t num, 2231 tavor_rsrc_t *hdl) 2232 { 2233 tavor_rsrc_priv_mbox_t *priv; 2234 void *addr; 2235 caddr_t kaddr; 2236 uint64_t offset; 2237 size_t real_len, temp_len; 2238 int status; 2239 2240 TAVOR_TNF_ENTER(tavor_rsrc_mbox_alloc); 2241 2242 ASSERT(pool_info != NULL); 2243 ASSERT(hdl != NULL); 2244 2245 /* Get the private pointer for the mailboxes */ 2246 priv = pool_info->rsrc_private; 2247 ASSERT(priv != NULL); 2248 2249 /* 2250 * Allocate a DMA handle for the mailbox. This will be used for 2251 * two purposes (potentially). First, it could be used below in 2252 * the call to ddi_dma_mem_alloc() - if the mailbox is to come from 2253 * system memory. Second, it is definitely used later to bind 2254 * the mailbox for DMA access from/by the hardware. 2255 */ 2256 status = ddi_dma_alloc_handle(priv->pmb_dip, &priv->pmb_dmaattr, 2257 DDI_DMA_SLEEP, NULL, &hdl->tr_dmahdl); 2258 if (status != DDI_SUCCESS) { 2259 TNF_PROBE_1(tavor_rsrc_mbox_alloc_dmahdl_fail, TAVOR_TNF_ERROR, 2260 "", tnf_uint, status, status); 2261 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc); 2262 return (DDI_FAILURE); 2263 } 2264 2265 /* Is mailbox in DDR memory or system memory? */ 2266 if (pool_info->rsrc_loc == TAVOR_IN_DDR) { 2267 /* Use vmem_alloc() to get DDR address of mbox */ 2268 hdl->tr_len = (num * pool_info->rsrc_quantum); 2269 addr = vmem_alloc(pool_info->rsrc_vmp, hdl->tr_len, 2270 VM_SLEEP); 2271 if (addr == NULL) { 2272 /* No more DDR available for mailbox entries */ 2273 ddi_dma_free_handle(&hdl->tr_dmahdl); 2274 TNF_PROBE_0(tavor_rsrc_mbox_alloc_vma_fail, 2275 TAVOR_TNF_ERROR, ""); 2276 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc); 2277 return (DDI_FAILURE); 2278 } 2279 hdl->tr_acchdl = priv->pmb_acchdl; 2280 2281 /* Calculate kernel virtual address (from the DDR offset) */ 2282 offset = ((uintptr_t)addr - 2283 (uintptr_t)pool_info->rsrc_ddr_offset); 2284 hdl->tr_addr = (void *)(uintptr_t)(offset + 2285 (uintptr_t)pool_info->rsrc_start); 2286 2287 } else { /* TAVOR_IN_SYSMEM */ 2288 2289 /* Use ddi_dma_mem_alloc() to get memory for mailbox */ 2290 temp_len = (num * pool_info->rsrc_quantum); 2291 status = ddi_dma_mem_alloc(hdl->tr_dmahdl, temp_len, 2292 &priv->pmb_devaccattr, priv->pmb_xfer_mode, DDI_DMA_SLEEP, 2293 NULL, &kaddr, &real_len, &hdl->tr_acchdl); 2294 if (status != DDI_SUCCESS) { 2295 /* No more sys memory available for mailbox entries */ 2296 ddi_dma_free_handle(&hdl->tr_dmahdl); 2297 TNF_PROBE_0(tavor_rsrc_mbox_alloc_dma_memalloc_fail, 2298 TAVOR_TNF_ERROR, ""); 2299 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc); 2300 return (DDI_FAILURE); 2301 } 2302 hdl->tr_addr = (void *)kaddr; 2303 hdl->tr_len = real_len; 2304 } 2305 2306 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc); 2307 return (DDI_SUCCESS); 2308 } 2309 2310 2311 /* 2312 * tavor_rsrc_mbox_free() 2313 * Context: Can be called from interrupt or base context. 2314 */ 2315 static void 2316 tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl) 2317 { 2318 void *addr; 2319 uint64_t offset; 2320 2321 TAVOR_TNF_ENTER(tavor_rsrc_mbox_free); 2322 2323 ASSERT(pool_info != NULL); 2324 ASSERT(hdl != NULL); 2325 2326 /* Is mailbox in DDR memory or system memory? */ 2327 if (pool_info->rsrc_loc == TAVOR_IN_DDR) { 2328 2329 /* Calculate the allocated address (the mbox's DDR offset) */ 2330 offset = ((uintptr_t)hdl->tr_addr - 2331 (uintptr_t)pool_info->rsrc_start); 2332 addr = (void *)(uintptr_t)(offset + 2333 (uintptr_t)pool_info->rsrc_ddr_offset); 2334 2335 /* Use vmem_free() to free up DDR memory for mailbox */ 2336 vmem_free(pool_info->rsrc_vmp, addr, hdl->tr_len); 2337 2338 } else { /* TAVOR_IN_SYSMEM */ 2339 2340 /* Use ddi_dma_mem_free() to free up sys memory for mailbox */ 2341 ddi_dma_mem_free(&hdl->tr_acchdl); 2342 } 2343 2344 /* Free the DMA handle for the mailbox */ 2345 ddi_dma_free_handle(&hdl->tr_dmahdl); 2346 2347 TAVOR_TNF_EXIT(tavor_rsrc_mbox_free); 2348 } 2349 2350 2351 /* 2352 * tavor_rsrc_hw_entry_alloc() 2353 * Context: Can be called from interrupt or base context. 2354 */ 2355 static int 2356 tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t num, 2357 uint_t num_align, ddi_acc_handle_t acc_handle, uint_t sleepflag, 2358 tavor_rsrc_t *hdl) 2359 { 2360 void *addr; 2361 uint64_t offset; 2362 uint32_t align; 2363 int flag; 2364 2365 TAVOR_TNF_ENTER(tavor_rsrc_hw_entry_alloc); 2366 2367 ASSERT(pool_info != NULL); 2368 ASSERT(hdl != NULL); 2369 2370 /* 2371 * Tavor hardware entries (QPC, CQC, EQC, MPT, MTT, etc.) do not 2372 * use dma_handle (because they are in Tavor locally attached DDR 2373 * memory) and, generally, don't use the acc_handle (because the 2374 * entries are not directly accessed by software). The exceptions 2375 * to this rule are the UARPG and UDAV entries. 2376 */ 2377 2378 /* 2379 * Use vmem_xalloc() to get a properly aligned pointer (based on 2380 * the number requested) to the HW entry(ies). This handles the 2381 * cases (for special QPCs and for RDB entries) where we need more 2382 * than one and need to ensure that they are properly aligned. 2383 */ 2384 flag = (sleepflag == TAVOR_SLEEP) ? VM_SLEEP : VM_NOSLEEP; 2385 hdl->tr_len = (num * pool_info->rsrc_quantum); 2386 align = (num_align * pool_info->rsrc_quantum); 2387 addr = vmem_xalloc(pool_info->rsrc_vmp, hdl->tr_len, 2388 align, 0, 0, NULL, NULL, flag | VM_FIRSTFIT); 2389 if (addr == NULL) { 2390 /* No more HW entries available */ 2391 TNF_PROBE_0(tavor_rsrc_hw_entry_alloc_vmxa_fail, 2392 TAVOR_TNF_ERROR, ""); 2393 TAVOR_TNF_EXIT(tavor_rsrc_hw_entry_alloc); 2394 return (DDI_FAILURE); 2395 } 2396 2397 /* If an access handle was provided, fill it in */ 2398 if (acc_handle != 0) { 2399 hdl->tr_acchdl = acc_handle; 2400 } 2401 2402 /* Calculate vaddr and HW table index (from the DDR offset) */ 2403 offset = ((uintptr_t)addr - (uintptr_t)pool_info->rsrc_ddr_offset); 2404 hdl->tr_addr = (void *)(uintptr_t)(offset + 2405 (uintptr_t)pool_info->rsrc_start); 2406 hdl->tr_indx = (offset >> pool_info->rsrc_shift); 2407 2408 TAVOR_TNF_EXIT(tavor_rsrc_hw_entry_alloc); 2409 return (DDI_SUCCESS); 2410 } 2411 2412 2413 /* 2414 * tavor_rsrc_hw_entry_free() 2415 * Context: Can be called from interrupt or base context. 2416 */ 2417 static void 2418 tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl) 2419 { 2420 void *addr; 2421 uint64_t offset; 2422 2423 TAVOR_TNF_ENTER(tavor_rsrc_hw_entry_free); 2424 2425 ASSERT(pool_info != NULL); 2426 ASSERT(hdl != NULL); 2427 2428 /* Calculate the allocated address (the entry's DDR offset) */ 2429 offset = ((uintptr_t)hdl->tr_addr - (uintptr_t)pool_info->rsrc_start); 2430 addr = (void *)(uintptr_t)(offset + 2431 (uintptr_t)pool_info->rsrc_ddr_offset); 2432 2433 /* Use vmem_xfree() to free up the HW table entry */ 2434 vmem_xfree(pool_info->rsrc_vmp, addr, hdl->tr_len); 2435 2436 TAVOR_TNF_EXIT(tavor_rsrc_hw_entry_free); 2437 } 2438 2439 2440 /* 2441 * tavor_rsrc_swhdl_alloc() 2442 * Context: Can be called from interrupt or base context. 2443 */ 2444 static int 2445 tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t sleepflag, 2446 tavor_rsrc_t *hdl) 2447 { 2448 void *addr; 2449 int flag; 2450 2451 TAVOR_TNF_ENTER(tavor_rsrc_swhdl_alloc); 2452 2453 ASSERT(pool_info != NULL); 2454 ASSERT(hdl != NULL); 2455 2456 /* Allocate the software handle structure */ 2457 flag = (sleepflag == TAVOR_SLEEP) ? KM_SLEEP : KM_NOSLEEP; 2458 addr = kmem_cache_alloc(pool_info->rsrc_private, flag); 2459 if (addr == NULL) { 2460 TNF_PROBE_0(tavor_rsrc_swhdl_alloc_kmca_fail, TAVOR_TNF_ERROR, 2461 ""); 2462 TAVOR_TNF_EXIT(tavor_rsrc_swhdl_alloc); 2463 return (DDI_FAILURE); 2464 } 2465 hdl->tr_len = pool_info->rsrc_quantum; 2466 hdl->tr_addr = addr; 2467 2468 TAVOR_TNF_EXIT(tavor_rsrc_swhdl_alloc); 2469 return (DDI_SUCCESS); 2470 } 2471 2472 2473 /* 2474 * tavor_rsrc_swhdl_free() 2475 * Context: Can be called from interrupt or base context. 2476 */ 2477 static void 2478 tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl) 2479 { 2480 TAVOR_TNF_ENTER(tavor_rsrc_swhdl_free); 2481 2482 ASSERT(pool_info != NULL); 2483 ASSERT(hdl != NULL); 2484 2485 /* Free the software handle structure */ 2486 kmem_cache_free(pool_info->rsrc_private, hdl->tr_addr); 2487 2488 TAVOR_TNF_EXIT(tavor_rsrc_swhdl_free); 2489 } 2490 2491 2492 /* 2493 * tavor_rsrc_pdhdl_alloc() 2494 * Context: Can be called from interrupt or base context. 2495 */ 2496 static int 2497 tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t sleepflag, 2498 tavor_rsrc_t *hdl) 2499 { 2500 tavor_pdhdl_t addr; 2501 void *tmpaddr; 2502 int flag, status; 2503 2504 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_alloc); 2505 2506 ASSERT(pool_info != NULL); 2507 ASSERT(hdl != NULL); 2508 2509 /* Allocate the software handle */ 2510 status = tavor_rsrc_swhdl_alloc(pool_info, sleepflag, hdl); 2511 if (status != DDI_SUCCESS) { 2512 TNF_PROBE_0(tavor_rsrc_pdhdl_alloc_fail, TAVOR_TNF_ERROR, ""); 2513 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc); 2514 return (DDI_FAILURE); 2515 } 2516 addr = (tavor_pdhdl_t)hdl->tr_addr; 2517 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*addr)) 2518 2519 /* Allocate a PD number for the handle */ 2520 flag = (sleepflag == TAVOR_SLEEP) ? VM_SLEEP : VM_NOSLEEP; 2521 tmpaddr = vmem_alloc(pool_info->rsrc_vmp, 1, flag); 2522 if (tmpaddr == NULL) { 2523 /* No more PD number entries available */ 2524 tavor_rsrc_swhdl_free(pool_info, hdl); 2525 TNF_PROBE_0(tavor_rsrc_pdhdl_alloc_vma_fail, 2526 TAVOR_TNF_ERROR, ""); 2527 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc); 2528 return (DDI_FAILURE); 2529 } 2530 addr->pd_pdnum = (uint32_t)(uintptr_t)tmpaddr; 2531 addr->pd_rsrcp = hdl; 2532 hdl->tr_indx = addr->pd_pdnum; 2533 2534 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc); 2535 return (DDI_SUCCESS); 2536 } 2537 2538 2539 /* 2540 * tavor_rsrc_pdhdl_free() 2541 * Context: Can be called from interrupt or base context. 2542 */ 2543 static void 2544 tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl) 2545 { 2546 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_free); 2547 2548 ASSERT(pool_info != NULL); 2549 ASSERT(hdl != NULL); 2550 2551 /* Use vmem_free() to free up the PD number */ 2552 vmem_free(pool_info->rsrc_vmp, (void *)(uintptr_t)hdl->tr_indx, 1); 2553 2554 /* Free the software handle structure */ 2555 tavor_rsrc_swhdl_free(pool_info, hdl); 2556 2557 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_free); 2558 } 2559 2560 2561 /* 2562 * tavor_rsrc_pdhdl_constructor() 2563 * Context: Can be called from interrupt or base context. 2564 */ 2565 /* ARGSUSED */ 2566 static int 2567 tavor_rsrc_pdhdl_constructor(void *pd, void *priv, int flags) 2568 { 2569 tavor_pdhdl_t pdhdl; 2570 tavor_state_t *state; 2571 2572 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_constructor); 2573 2574 pdhdl = (tavor_pdhdl_t)pd; 2575 state = (tavor_state_t *)priv; 2576 2577 mutex_init(&pdhdl->pd_lock, NULL, MUTEX_DRIVER, 2578 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2579 2580 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_constructor); 2581 return (DDI_SUCCESS); 2582 } 2583 2584 2585 /* 2586 * tavor_rsrc_pdhdl_destructor() 2587 * Context: Can be called from interrupt or base context. 2588 */ 2589 /* ARGSUSED */ 2590 static void 2591 tavor_rsrc_pdhdl_destructor(void *pd, void *priv) 2592 { 2593 tavor_pdhdl_t pdhdl; 2594 2595 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_destructor); 2596 2597 pdhdl = (tavor_pdhdl_t)pd; 2598 2599 mutex_destroy(&pdhdl->pd_lock); 2600 2601 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_destructor); 2602 } 2603 2604 2605 /* 2606 * tavor_rsrc_cqhdl_constructor() 2607 * Context: Can be called from interrupt or base context. 2608 */ 2609 /* ARGSUSED */ 2610 static int 2611 tavor_rsrc_cqhdl_constructor(void *cq, void *priv, int flags) 2612 { 2613 tavor_cqhdl_t cqhdl; 2614 tavor_state_t *state; 2615 2616 TAVOR_TNF_ENTER(tavor_rsrc_cqhdl_constructor); 2617 2618 cqhdl = (tavor_cqhdl_t)cq; 2619 state = (tavor_state_t *)priv; 2620 2621 mutex_init(&cqhdl->cq_lock, NULL, MUTEX_DRIVER, 2622 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2623 mutex_init(&cqhdl->cq_wrid_wqhdr_lock, NULL, MUTEX_DRIVER, 2624 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2625 2626 TAVOR_TNF_EXIT(tavor_rsrc_cqhdl_constructor); 2627 return (DDI_SUCCESS); 2628 } 2629 2630 2631 /* 2632 * tavor_rsrc_cqhdl_destructor() 2633 * Context: Can be called from interrupt or base context. 2634 */ 2635 /* ARGSUSED */ 2636 static void 2637 tavor_rsrc_cqhdl_destructor(void *cq, void *priv) 2638 { 2639 tavor_cqhdl_t cqhdl; 2640 2641 TAVOR_TNF_ENTER(tavor_rsrc_cqhdl_destructor); 2642 2643 cqhdl = (tavor_cqhdl_t)cq; 2644 2645 mutex_destroy(&cqhdl->cq_wrid_wqhdr_lock); 2646 mutex_destroy(&cqhdl->cq_lock); 2647 2648 TAVOR_TNF_EXIT(tavor_rsrc_cqhdl_destructor); 2649 } 2650 2651 2652 /* 2653 * tavor_rsrc_qphdl_constructor() 2654 * Context: Can be called from interrupt or base context. 2655 */ 2656 /* ARGSUSED */ 2657 static int 2658 tavor_rsrc_qphdl_constructor(void *qp, void *priv, int flags) 2659 { 2660 tavor_qphdl_t qphdl; 2661 tavor_state_t *state; 2662 2663 TAVOR_TNF_ENTER(tavor_rsrc_qphdl_constructor); 2664 2665 qphdl = (tavor_qphdl_t)qp; 2666 state = (tavor_state_t *)priv; 2667 2668 mutex_init(&qphdl->qp_lock, NULL, MUTEX_DRIVER, 2669 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2670 2671 TAVOR_TNF_EXIT(tavor_rsrc_qphdl_constructor); 2672 return (DDI_SUCCESS); 2673 } 2674 2675 2676 /* 2677 * tavor_rsrc_qphdl_destructor() 2678 * Context: Can be called from interrupt or base context. 2679 */ 2680 /* ARGSUSED */ 2681 static void 2682 tavor_rsrc_qphdl_destructor(void *qp, void *priv) 2683 { 2684 tavor_qphdl_t qphdl; 2685 2686 TAVOR_TNF_ENTER(tavor_rsrc_qphdl_destructor); 2687 2688 qphdl = (tavor_qphdl_t)qp; 2689 2690 mutex_destroy(&qphdl->qp_lock); 2691 2692 TAVOR_TNF_ENTER(tavor_rsrc_qphdl_destructor); 2693 } 2694 2695 2696 /* 2697 * tavor_rsrc_srqhdl_constructor() 2698 * Context: Can be called from interrupt or base context. 2699 */ 2700 /* ARGSUSED */ 2701 static int 2702 tavor_rsrc_srqhdl_constructor(void *srq, void *priv, int flags) 2703 { 2704 tavor_srqhdl_t srqhdl; 2705 tavor_state_t *state; 2706 2707 TAVOR_TNF_ENTER(tavor_rsrc_srqhdl_constructor); 2708 2709 srqhdl = (tavor_srqhdl_t)srq; 2710 state = (tavor_state_t *)priv; 2711 2712 mutex_init(&srqhdl->srq_lock, NULL, MUTEX_DRIVER, 2713 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2714 2715 TAVOR_TNF_EXIT(tavor_rsrc_srqhdl_constructor); 2716 return (DDI_SUCCESS); 2717 } 2718 2719 2720 /* 2721 * tavor_rsrc_srqhdl_destructor() 2722 * Context: Can be called from interrupt or base context. 2723 */ 2724 /* ARGSUSED */ 2725 static void 2726 tavor_rsrc_srqhdl_destructor(void *srq, void *priv) 2727 { 2728 tavor_srqhdl_t srqhdl; 2729 2730 TAVOR_TNF_ENTER(tavor_rsrc_srqhdl_destructor); 2731 2732 srqhdl = (tavor_srqhdl_t)srq; 2733 2734 mutex_destroy(&srqhdl->srq_lock); 2735 2736 TAVOR_TNF_EXIT(tavor_rsrc_srqhdl_destructor); 2737 } 2738 2739 2740 /* 2741 * tavor_rsrc_refcnt_constructor() 2742 * Context: Can be called from interrupt or base context. 2743 */ 2744 /* ARGSUSED */ 2745 static int 2746 tavor_rsrc_refcnt_constructor(void *rc, void *priv, int flags) 2747 { 2748 tavor_sw_refcnt_t *refcnt; 2749 tavor_state_t *state; 2750 2751 TAVOR_TNF_ENTER(tavor_rsrc_refcnt_constructor); 2752 2753 refcnt = (tavor_sw_refcnt_t *)rc; 2754 state = (tavor_state_t *)priv; 2755 2756 mutex_init(&refcnt->swrc_lock, NULL, MUTEX_DRIVER, 2757 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2758 2759 TAVOR_TNF_EXIT(tavor_rsrc_refcnt_constructor); 2760 return (DDI_SUCCESS); 2761 } 2762 2763 2764 /* 2765 * tavor_rsrc_refcnt_destructor() 2766 * Context: Can be called from interrupt or base context. 2767 */ 2768 /* ARGSUSED */ 2769 static void 2770 tavor_rsrc_refcnt_destructor(void *rc, void *priv) 2771 { 2772 tavor_sw_refcnt_t *refcnt; 2773 2774 TAVOR_TNF_ENTER(tavor_rsrc_refcnt_destructor); 2775 2776 refcnt = (tavor_sw_refcnt_t *)rc; 2777 2778 mutex_destroy(&refcnt->swrc_lock); 2779 2780 TAVOR_TNF_ENTER(tavor_rsrc_refcnt_destructor); 2781 } 2782 2783 2784 /* 2785 * tavor_rsrc_ahhdl_constructor() 2786 * Context: Can be called from interrupt or base context. 2787 */ 2788 /* ARGSUSED */ 2789 static int 2790 tavor_rsrc_ahhdl_constructor(void *ah, void *priv, int flags) 2791 { 2792 tavor_ahhdl_t ahhdl; 2793 tavor_state_t *state; 2794 2795 TAVOR_TNF_ENTER(tavor_rsrc_ahhdl_constructor); 2796 2797 ahhdl = (tavor_ahhdl_t)ah; 2798 state = (tavor_state_t *)priv; 2799 2800 mutex_init(&ahhdl->ah_lock, NULL, MUTEX_DRIVER, 2801 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2802 2803 TAVOR_TNF_EXIT(tavor_rsrc_ahhdl_constructor); 2804 return (DDI_SUCCESS); 2805 } 2806 2807 2808 /* 2809 * tavor_rsrc_ahhdl_destructor() 2810 * Context: Can be called from interrupt or base context. 2811 */ 2812 /* ARGSUSED */ 2813 static void 2814 tavor_rsrc_ahhdl_destructor(void *ah, void *priv) 2815 { 2816 tavor_ahhdl_t ahhdl; 2817 2818 TAVOR_TNF_ENTER(tavor_rsrc_ahhdl_destructor); 2819 2820 ahhdl = (tavor_ahhdl_t)ah; 2821 2822 mutex_destroy(&ahhdl->ah_lock); 2823 2824 TAVOR_TNF_ENTER(tavor_rsrc_ahhdl_destructor); 2825 } 2826 2827 2828 /* 2829 * tavor_rsrc_mrhdl_constructor() 2830 * Context: Can be called from interrupt or base context. 2831 */ 2832 /* ARGSUSED */ 2833 static int 2834 tavor_rsrc_mrhdl_constructor(void *mr, void *priv, int flags) 2835 { 2836 tavor_mrhdl_t mrhdl; 2837 tavor_state_t *state; 2838 2839 TAVOR_TNF_ENTER(tavor_rsrc_mrhdl_constructor); 2840 2841 mrhdl = (tavor_mrhdl_t)mr; 2842 state = (tavor_state_t *)priv; 2843 2844 mutex_init(&mrhdl->mr_lock, NULL, MUTEX_DRIVER, 2845 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2846 2847 TAVOR_TNF_EXIT(tavor_rsrc_mrhdl_constructor); 2848 return (DDI_SUCCESS); 2849 } 2850 2851 2852 /* 2853 * tavor_rsrc_mrhdl_destructor() 2854 * Context: Can be called from interrupt or base context. 2855 */ 2856 /* ARGSUSED */ 2857 static void 2858 tavor_rsrc_mrhdl_destructor(void *mr, void *priv) 2859 { 2860 tavor_mrhdl_t mrhdl; 2861 2862 TAVOR_TNF_ENTER(tavor_rsrc_mrhdl_destructor); 2863 2864 mrhdl = (tavor_mrhdl_t)mr; 2865 2866 mutex_destroy(&mrhdl->mr_lock); 2867 2868 TAVOR_TNF_ENTER(tavor_rsrc_mrhdl_destructor); 2869 } 2870 2871 2872 /* 2873 * tavor_rsrc_mcg_entry_get_size() 2874 */ 2875 static int 2876 tavor_rsrc_mcg_entry_get_size(tavor_state_t *state, uint_t *mcg_size_shift) 2877 { 2878 uint_t num_qp_per_mcg, max_qp_per_mcg, log2; 2879 2880 TAVOR_TNF_ENTER(tavor_rsrc_mcg_entry_get_size); 2881 2882 /* 2883 * Round the configured number of QP per MCG to next larger 2884 * power-of-2 size and update. 2885 */ 2886 num_qp_per_mcg = state->ts_cfg_profile->cp_num_qp_per_mcg + 8; 2887 log2 = highbit(num_qp_per_mcg); 2888 if ((num_qp_per_mcg & (num_qp_per_mcg - 1)) == 0) { 2889 log2 = log2 - 1; 2890 } 2891 state->ts_cfg_profile->cp_num_qp_per_mcg = (1 << log2) - 8; 2892 2893 /* Now make sure number of QP per MCG makes sense */ 2894 num_qp_per_mcg = state->ts_cfg_profile->cp_num_qp_per_mcg; 2895 max_qp_per_mcg = (1 << state->ts_devlim.log_max_qp_mcg); 2896 if (num_qp_per_mcg > max_qp_per_mcg) { 2897 TNF_PROBE_1(tavor_rsrc_mcg_getsz_toomany_qppermcg_fail, 2898 TAVOR_TNF_ERROR, "", tnf_uint, maxqpmcg, max_qp_per_mcg); 2899 TAVOR_TNF_EXIT(tavor_rsrc_mcg_entry_get_size); 2900 return (DDI_FAILURE); 2901 } 2902 2903 /* Return the (shift) size of an individual MCG HW entry */ 2904 *mcg_size_shift = log2 + 2; 2905 2906 TAVOR_TNF_EXIT(tavor_rsrc_mcg_entry_get_size); 2907 return (DDI_SUCCESS); 2908 } 2909