1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * tavor_rsrc.c 29 * Tavor Resource Management Routines 30 * 31 * Implements all the routines necessary for setup, teardown, and 32 * alloc/free of all Tavor resources, including those that are managed 33 * by Tavor hardware or which live in Tavor's direct attached DDR memory. 34 */ 35 36 #include <sys/sysmacros.h> 37 #include <sys/types.h> 38 #include <sys/conf.h> 39 #include <sys/ddi.h> 40 #include <sys/sunddi.h> 41 #include <sys/modctl.h> 42 #include <sys/vmem.h> 43 #include <sys/bitmap.h> 44 45 #include <sys/ib/adapters/tavor/tavor.h> 46 47 /* 48 * The following routines are used for initializing and destroying 49 * the resource pools used by the Tavor resource allocation routines. 50 * They consist of four classes of object: 51 * 52 * Mailboxes: The "In" and "Out" mailbox types are used by the Tavor 53 * command interface routines. Mailboxes are used to pass information 54 * back and forth to the Tavor firmware. Either type of mailbox may 55 * be allocated from Tavor's direct attached DDR memory or from system 56 * memory (although currently all "In" mailboxes are in DDR and all "out" 57 * mailboxes come from system memory. 58 * 59 * HW entry objects: These objects represent resources required by the Tavor 60 * hardware. These objects include things like Queue Pair contexts (QPC), 61 * Completion Queue contexts (CQC), Event Queue contexts (EQC), RDB (for 62 * supporting RDMA Read/Atomic), Multicast Group entries (MCG), Memory 63 * Protection Table entries (MPT), Memory Translation Table entries (MTT). 64 * 65 * What these objects all have in common is that they are each required 66 * to come from DDR memory, they are always allocated from tables, and 67 * they are not to be directly accessed (read or written) by driver 68 * software. 69 * One notable exceptions to this rule are the Extended QP contexts (EQPC), 70 * and the UAR scratch area (UAR_SCR), both of which are not directly 71 * accessible through the Tavor resource allocation routines, but both 72 * of which are also required to reside in DDR memory and are not to be 73 * manipulated by driver software (they are privately managed by Tavor 74 * hardware). 75 * The other notable exceptions are the UAR pages (UAR_PG) which are 76 * allocated from the UAR address space rather than DDR, and the UD 77 * address vectors (UDAV) which are similar to the common object types 78 * with the major difference being that UDAVs _are_ directly read and 79 * written by driver software. 80 * 81 * SW handle objects: These objects represent resources required by Tavor 82 * driver software. They are primarily software tracking structures, 83 * which are allocated from system memory (using kmem_cache). Several of 84 * the objects have both a "constructor" and "destructor" method 85 * associated with them (see below). 86 * 87 * Protection Domain (PD) handle objects: These objects are very much like 88 * a SW handle object with the notable difference that all PD handle 89 * objects have an actual Protection Domain number (PD) associated with 90 * them (and the PD number is allocated/managed through a separate 91 * vmem_arena specifically set aside for this purpose. 92 */ 93 94 static int tavor_rsrc_mbox_init(tavor_state_t *state, 95 tavor_rsrc_mbox_info_t *info); 96 static void tavor_rsrc_mbox_fini(tavor_state_t *state, 97 tavor_rsrc_mbox_info_t *info); 98 99 static int tavor_rsrc_hw_entries_init(tavor_state_t *state, 100 tavor_rsrc_hw_entry_info_t *info); 101 static void tavor_rsrc_hw_entries_fini(tavor_state_t *state, 102 tavor_rsrc_hw_entry_info_t *info); 103 104 static int tavor_rsrc_sw_handles_init(tavor_state_t *state, 105 tavor_rsrc_sw_hdl_info_t *info); 106 static void tavor_rsrc_sw_handles_fini(tavor_state_t *state, 107 tavor_rsrc_sw_hdl_info_t *info); 108 109 static int tavor_rsrc_pd_handles_init(tavor_state_t *state, 110 tavor_rsrc_sw_hdl_info_t *info); 111 static void tavor_rsrc_pd_handles_fini(tavor_state_t *state, 112 tavor_rsrc_sw_hdl_info_t *info); 113 114 /* 115 * The following routines are used for allocating and freeing the specific 116 * types of objects described above from their associated resource pools. 117 */ 118 static int tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t *pool_info, 119 uint_t num, tavor_rsrc_t *hdl); 120 static void tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t *pool_info, 121 tavor_rsrc_t *hdl); 122 123 static int tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t *pool_info, 124 uint_t num, uint_t num_align, ddi_acc_handle_t acc_handle, 125 uint_t sleepflag, tavor_rsrc_t *hdl); 126 static void tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t *pool_info, 127 tavor_rsrc_t *hdl); 128 129 static int tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t *pool_info, 130 uint_t sleepflag, tavor_rsrc_t *hdl); 131 static void tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t *pool_info, 132 tavor_rsrc_t *hdl); 133 134 static int tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t *pool_info, 135 uint_t sleepflag, tavor_rsrc_t *hdl); 136 static void tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t *pool_info, 137 tavor_rsrc_t *hdl); 138 139 /* 140 * The following routines are the constructors and destructors for several 141 * of the SW handle type objects. For certain types of SW handles objects 142 * (all of which are implemented using kmem_cache), we need to do some 143 * special field initialization (specifically, mutex_init/destroy). These 144 * routines enable that init and teardown. 145 */ 146 static int tavor_rsrc_pdhdl_constructor(void *pd, void *priv, int flags); 147 static void tavor_rsrc_pdhdl_destructor(void *pd, void *state); 148 static int tavor_rsrc_cqhdl_constructor(void *cq, void *priv, int flags); 149 static void tavor_rsrc_cqhdl_destructor(void *cq, void *state); 150 static int tavor_rsrc_qphdl_constructor(void *cq, void *priv, int flags); 151 static void tavor_rsrc_qphdl_destructor(void *cq, void *state); 152 static int tavor_rsrc_srqhdl_constructor(void *srq, void *priv, int flags); 153 static void tavor_rsrc_srqhdl_destructor(void *srq, void *state); 154 static int tavor_rsrc_refcnt_constructor(void *rc, void *priv, int flags); 155 static void tavor_rsrc_refcnt_destructor(void *rc, void *state); 156 static int tavor_rsrc_ahhdl_constructor(void *ah, void *priv, int flags); 157 static void tavor_rsrc_ahhdl_destructor(void *ah, void *state); 158 static int tavor_rsrc_mrhdl_constructor(void *mr, void *priv, int flags); 159 static void tavor_rsrc_mrhdl_destructor(void *mr, void *state); 160 161 /* 162 * Special routine to calculate and return the size of a MCG object based 163 * on current driver configuration (specifically, the number of QP per MCG 164 * that has been configured. 165 */ 166 static int tavor_rsrc_mcg_entry_get_size(tavor_state_t *state, 167 uint_t *mcg_size_shift); 168 169 170 /* 171 * tavor_rsrc_alloc() 172 * 173 * Context: Can be called from interrupt or base context. 174 * The "sleepflag" parameter is used by all object allocators to 175 * determine whether to SLEEP for resources or not. 176 */ 177 int 178 tavor_rsrc_alloc(tavor_state_t *state, tavor_rsrc_type_t rsrc, uint_t num, 179 uint_t sleepflag, tavor_rsrc_t **hdl) 180 { 181 tavor_rsrc_pool_info_t *rsrc_pool; 182 tavor_rsrc_t *tmp_rsrc_hdl; 183 int flag, status = DDI_FAILURE; 184 185 TAVOR_TNF_ENTER(tavor_rsrc_alloc); 186 187 ASSERT(state != NULL); 188 ASSERT(hdl != NULL); 189 190 rsrc_pool = &state->ts_rsrc_hdl[rsrc]; 191 ASSERT(rsrc_pool != NULL); 192 193 /* 194 * Allocate space for the object used to track the resource handle 195 */ 196 flag = (sleepflag == TAVOR_SLEEP) ? KM_SLEEP : KM_NOSLEEP; 197 tmp_rsrc_hdl = (tavor_rsrc_t *)kmem_cache_alloc(state->ts_rsrc_cache, 198 flag); 199 if (tmp_rsrc_hdl == NULL) { 200 TNF_PROBE_0(tavor_rsrc_alloc_kmca_fail, TAVOR_TNF_ERROR, ""); 201 TAVOR_TNF_EXIT(tavor_rsrc_alloc); 202 return (DDI_FAILURE); 203 } 204 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*tmp_rsrc_hdl)) 205 206 /* 207 * Set rsrc_hdl type. This is later used by the tavor_rsrc_free call 208 * to know what type of resource is being freed. 209 */ 210 tmp_rsrc_hdl->rsrc_type = rsrc; 211 212 /* 213 * Depending on resource type, call the appropriate alloc routine 214 */ 215 switch (rsrc_pool->rsrc_type) { 216 case TAVOR_IN_MBOX: 217 case TAVOR_OUT_MBOX: 218 case TAVOR_INTR_IN_MBOX: 219 case TAVOR_INTR_OUT_MBOX: 220 status = tavor_rsrc_mbox_alloc(rsrc_pool, num, tmp_rsrc_hdl); 221 break; 222 223 case TAVOR_QPC: 224 case TAVOR_CQC: 225 case TAVOR_SRQC: 226 case TAVOR_EQC: 227 case TAVOR_RDB: 228 /* 229 * Because these objects are NOT accessed by Tavor driver 230 * software, we set the acc_handle parameter to zero. But 231 * if they are allocated in multiples, we specify here that 232 * they must be aligned on a more restrictive boundary. 233 */ 234 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, num, 0, 235 sleepflag, tmp_rsrc_hdl); 236 break; 237 238 case TAVOR_MPT: 239 /* 240 * Because these MPT objects are sometimes accessed by Tavor 241 * driver software (FMR), we set the acc_handle parameter. But 242 * if they are allocated in multiples, we specify here that 243 * they must be aligned on a more restrictive boundary. 244 */ 245 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, num, 246 state->ts_reg_ddrhdl, sleepflag, tmp_rsrc_hdl); 247 break; 248 249 case TAVOR_MCG: 250 /* 251 * Tavor MCG entries are also NOT accessed by Tavor driver 252 * software, but because MCG entries do not have the same 253 * alignnment restrictions we loosen the constraint here. 254 */ 255 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1, 0, 256 sleepflag, tmp_rsrc_hdl); 257 break; 258 259 case TAVOR_MTT: 260 case TAVOR_UDAV: 261 /* 262 * Because MTT segments are among the few HW resources that 263 * may be allocated in odd numbers, we specify a less 264 * restrictive alignment than for the above resources. 265 * 266 * Also because both UDAV and MTT segment objects are read 267 * and/or written by Tavor driver software, we set the 268 * acc_handle parameter to point to the ddi_acc_handle_t for 269 * the Tavor DDR memory. 270 */ 271 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1, 272 state->ts_reg_ddrhdl, sleepflag, tmp_rsrc_hdl); 273 break; 274 275 case TAVOR_UARPG: 276 /* 277 * Because UAR pages are written by Tavor driver software (for 278 * doorbells), we set the acc_handle parameter to point to 279 * the ddi_acc_handle_t for the Tavor UAR memory. 280 */ 281 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1, 282 state->ts_reg_uarhdl, sleepflag, tmp_rsrc_hdl); 283 break; 284 285 case TAVOR_MRHDL: 286 case TAVOR_EQHDL: 287 case TAVOR_CQHDL: 288 case TAVOR_SRQHDL: 289 case TAVOR_AHHDL: 290 case TAVOR_QPHDL: 291 case TAVOR_REFCNT: 292 status = tavor_rsrc_swhdl_alloc(rsrc_pool, sleepflag, 293 tmp_rsrc_hdl); 294 break; 295 296 case TAVOR_PDHDL: 297 status = tavor_rsrc_pdhdl_alloc(rsrc_pool, sleepflag, 298 tmp_rsrc_hdl); 299 break; 300 301 default: 302 TAVOR_WARNING(state, "unexpected resource type in alloc"); 303 TNF_PROBE_0(tavor_rsrc_alloc_inv_rsrctype_fail, 304 TAVOR_TNF_ERROR, ""); 305 break; 306 } 307 308 /* 309 * If the resource allocation failed, then free the special resource 310 * tracking structure and return failure. Otherwise return the 311 * handle for the resource tracking structure. 312 */ 313 if (status != DDI_SUCCESS) { 314 kmem_cache_free(state->ts_rsrc_cache, tmp_rsrc_hdl); 315 tmp_rsrc_hdl = NULL; 316 TNF_PROBE_1(tavor_rsrc_alloc_fail, TAVOR_TNF_ERROR, "", 317 tnf_uint, rsrc_type, rsrc_pool->rsrc_type); 318 TAVOR_TNF_EXIT(tavor_rsrc_alloc); 319 return (DDI_FAILURE); 320 } else { 321 *hdl = tmp_rsrc_hdl; 322 TAVOR_TNF_EXIT(tavor_rsrc_alloc); 323 return (DDI_SUCCESS); 324 } 325 } 326 327 328 /* 329 * tavor_rsrc_free() 330 * Context: Can be called from interrupt or base context. 331 */ 332 void 333 tavor_rsrc_free(tavor_state_t *state, tavor_rsrc_t **hdl) 334 { 335 tavor_rsrc_pool_info_t *rsrc_pool; 336 337 TAVOR_TNF_ENTER(tavor_rsrc_free); 338 339 ASSERT(state != NULL); 340 ASSERT(hdl != NULL); 341 342 rsrc_pool = &state->ts_rsrc_hdl[(*hdl)->rsrc_type]; 343 ASSERT(rsrc_pool != NULL); 344 345 /* 346 * Depending on resource type, call the appropriate free routine 347 */ 348 switch (rsrc_pool->rsrc_type) { 349 case TAVOR_IN_MBOX: 350 case TAVOR_OUT_MBOX: 351 case TAVOR_INTR_IN_MBOX: 352 case TAVOR_INTR_OUT_MBOX: 353 tavor_rsrc_mbox_free(rsrc_pool, *hdl); 354 break; 355 356 case TAVOR_QPC: 357 case TAVOR_CQC: 358 case TAVOR_SRQC: 359 case TAVOR_EQC: 360 case TAVOR_RDB: 361 case TAVOR_MCG: 362 case TAVOR_MPT: 363 case TAVOR_MTT: 364 case TAVOR_UDAV: 365 case TAVOR_UARPG: 366 tavor_rsrc_hw_entry_free(rsrc_pool, *hdl); 367 break; 368 369 case TAVOR_MRHDL: 370 case TAVOR_EQHDL: 371 case TAVOR_CQHDL: 372 case TAVOR_SRQHDL: 373 case TAVOR_AHHDL: 374 case TAVOR_QPHDL: 375 case TAVOR_REFCNT: 376 tavor_rsrc_swhdl_free(rsrc_pool, *hdl); 377 break; 378 379 case TAVOR_PDHDL: 380 tavor_rsrc_pdhdl_free(rsrc_pool, *hdl); 381 break; 382 383 default: 384 TAVOR_WARNING(state, "unexpected resource type in free"); 385 TNF_PROBE_0(tavor_rsrc_free_inv_rsrctype_fail, 386 TAVOR_TNF_ERROR, ""); 387 break; 388 } 389 390 /* 391 * Free the special resource tracking structure, set the handle to 392 * NULL, and return. 393 */ 394 kmem_cache_free(state->ts_rsrc_cache, *hdl); 395 *hdl = NULL; 396 397 TAVOR_TNF_EXIT(tavor_rsrc_free); 398 } 399 400 401 /* 402 * tavor_rsrc_init_phase1() 403 * 404 * Completes the first phase of Tavor resource/configuration init. 405 * This involves creating the kmem_cache for the "tavor_rsrc_t" 406 * structs, allocating the space for the resource pool handles, 407 * and setting up the "Out" mailboxes. 408 * 409 * When this function completes, the Tavor driver is ready to 410 * post the following commands which return information only in the 411 * "Out" mailbox: QUERY_DDR, QUERY_FW, QUERY_DEV_LIM, and QUERY_ADAPTER 412 * If any of these commands are to be posted at this time, they must be 413 * done so only when "spinning" (as the outstanding command list and 414 * EQ setup code has not yet run) 415 * 416 * Context: Only called from attach() path context 417 */ 418 int 419 tavor_rsrc_init_phase1(tavor_state_t *state) 420 { 421 tavor_rsrc_pool_info_t *rsrc_pool; 422 tavor_rsrc_mbox_info_t mbox_info; 423 tavor_rsrc_cleanup_level_t cleanup; 424 tavor_cfg_profile_t *cfgprof; 425 uint64_t num, size; 426 int status; 427 char *errormsg, *rsrc_name; 428 429 TAVOR_TNF_ENTER(tavor_rsrc_init_phase1); 430 431 ASSERT(state != NULL); 432 433 /* This is where Phase 1 of resource initialization begins */ 434 cleanup = TAVOR_RSRC_CLEANUP_LEVEL0; 435 436 /* Build kmem cache name from Tavor instance */ 437 rsrc_name = (char *)kmem_zalloc(TAVOR_RSRC_NAME_MAXLEN, KM_SLEEP); 438 TAVOR_RSRC_NAME(rsrc_name, TAVOR_RSRC_CACHE); 439 440 /* 441 * Create the kmem_cache for "tavor_rsrc_t" structures 442 * (kmem_cache_create will SLEEP until successful) 443 */ 444 state->ts_rsrc_cache = kmem_cache_create(rsrc_name, 445 sizeof (tavor_rsrc_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 446 447 /* 448 * Allocate an array of tavor_rsrc_pool_info_t's (used in all 449 * subsequent resource allocations) 450 */ 451 state->ts_rsrc_hdl = kmem_zalloc(TAVOR_NUM_RESOURCES * 452 sizeof (tavor_rsrc_pool_info_t), KM_SLEEP); 453 454 cfgprof = state->ts_cfg_profile; 455 456 /* 457 * Initialize the resource pool for "Out" mailboxes. Notice that 458 * the number of "Out" mailboxes, their size, and their location 459 * (DDR or system memory) is configurable. By default, however, 460 * all "Out" mailboxes are located in system memory only (because 461 * they are primarily read from and never written to) 462 */ 463 num = ((uint64_t)1 << cfgprof->cp_log_num_outmbox); 464 size = ((uint64_t)1 << cfgprof->cp_log_outmbox_size); 465 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_OUT_MBOX]; 466 rsrc_pool->rsrc_type = TAVOR_OUT_MBOX; 467 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 468 rsrc_pool->rsrc_pool_size = (size * num); 469 rsrc_pool->rsrc_shift = cfgprof->cp_log_outmbox_size; 470 rsrc_pool->rsrc_quantum = size; 471 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN; 472 rsrc_pool->rsrc_state = state; 473 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_OUTMBOX_VMEM); 474 mbox_info.mbi_num = num; 475 mbox_info.mbi_size = size; 476 mbox_info.mbi_rsrcpool = rsrc_pool; 477 mbox_info.mbi_rsrcname = rsrc_name; 478 status = tavor_rsrc_mbox_init(state, &mbox_info); 479 if (status != DDI_SUCCESS) { 480 tavor_rsrc_fini(state, cleanup); 481 /* Set "status" and "errormsg" and goto failure */ 482 TAVOR_TNF_FAIL(DDI_FAILURE, "out mailboxes"); 483 goto rsrcinitp1_fail; 484 } 485 cleanup = TAVOR_RSRC_CLEANUP_LEVEL1; 486 487 /* 488 * Initialize the Tavor "Out" mailbox list. This step actually uses 489 * the tavor_rsrc_alloc() for TAVOR_OUT_MBOX to preallocate the 490 * "Out" mailboxes, bind them for DMA access, and arrange them into 491 * an easily accessed fast-allocation mechanism (see tavor_cmd.c 492 * for more details) 493 */ 494 status = tavor_outmbox_list_init(state); 495 if (status != DDI_SUCCESS) { 496 tavor_rsrc_fini(state, cleanup); 497 /* Set "status" and "errormsg" and goto failure */ 498 TAVOR_TNF_FAIL(DDI_FAILURE, "out mailbox list"); 499 goto rsrcinitp1_fail; 500 } 501 cleanup = TAVOR_RSRC_CLEANUP_LEVEL2; 502 503 /* 504 * Initialize the resource pool for interrupt "Out" mailboxes. Notice 505 * that the number of interrupt "Out" mailboxes, their size, and their 506 * location (DDR or system memory) is configurable. By default, 507 * however, all interrupt "Out" mailboxes are located in system memory 508 * only (because they are primarily read from and never written to) 509 */ 510 num = ((uint64_t)1 << cfgprof->cp_log_num_intr_outmbox); 511 size = ((uint64_t)1 << cfgprof->cp_log_outmbox_size); 512 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_INTR_OUT_MBOX]; 513 rsrc_pool->rsrc_type = TAVOR_INTR_OUT_MBOX; 514 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 515 rsrc_pool->rsrc_pool_size = (size * num); 516 rsrc_pool->rsrc_shift = cfgprof->cp_log_outmbox_size; 517 rsrc_pool->rsrc_quantum = size; 518 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN; 519 rsrc_pool->rsrc_state = state; 520 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INTR_OUTMBOX_VMEM); 521 mbox_info.mbi_num = num; 522 mbox_info.mbi_size = size; 523 mbox_info.mbi_rsrcpool = rsrc_pool; 524 mbox_info.mbi_rsrcname = rsrc_name; 525 status = tavor_rsrc_mbox_init(state, &mbox_info); 526 if (status != DDI_SUCCESS) { 527 tavor_rsrc_fini(state, cleanup); 528 /* Set "status" and "errormsg" and goto failure */ 529 TAVOR_TNF_FAIL(DDI_FAILURE, "out intr mailboxes"); 530 goto rsrcinitp1_fail; 531 } 532 cleanup = TAVOR_RSRC_CLEANUP_LEVEL3; 533 534 /* 535 * Initialize the Tavor "Out" mailbox list. This step actually uses 536 * the tavor_rsrc_alloc() for TAVOR_OUT_MBOX to preallocate the 537 * "Out" mailboxes, bind them for DMA access, and arrange them into 538 * an easily accessed fast-allocation mechanism (see tavor_cmd.c 539 * for more details) 540 */ 541 status = tavor_intr_outmbox_list_init(state); 542 if (status != DDI_SUCCESS) { 543 tavor_rsrc_fini(state, cleanup); 544 /* Set "status" and "errormsg" and goto failure */ 545 TAVOR_TNF_FAIL(DDI_FAILURE, "out intr mailbox list"); 546 goto rsrcinitp1_fail; 547 } 548 cleanup = TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE; 549 550 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN); 551 TAVOR_TNF_EXIT(tavor_rsrc_init_phase1); 552 return (DDI_SUCCESS); 553 554 rsrcinitp1_fail: 555 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN); 556 TNF_PROBE_1(tavor_rsrc_init_phase1_fail, TAVOR_TNF_ERROR, "", 557 tnf_string, msg, errormsg); 558 TAVOR_TNF_EXIT(tavor_rsrc_init_phase1); 559 return (status); 560 } 561 562 563 /* 564 * tavor_rsrc_init_phase2() 565 * Context: Only called from attach() path context 566 */ 567 int 568 tavor_rsrc_init_phase2(tavor_state_t *state) 569 { 570 tavor_rsrc_sw_hdl_info_t hdl_info; 571 tavor_rsrc_hw_entry_info_t entry_info; 572 tavor_rsrc_mbox_info_t mbox_info; 573 tavor_rsrc_pool_info_t *rsrc_pool; 574 tavor_rsrc_cleanup_level_t cleanup; 575 tavor_cfg_profile_t *cfgprof; 576 uint64_t num, max, size, num_prealloc; 577 uint64_t ddr_size, fw_size; 578 uint_t mcg_size, mcg_size_shift; 579 uint_t uarscr_size, mttsegment_sz; 580 int status; 581 char *errormsg, *rsrc_name; 582 583 TAVOR_TNF_ENTER(tavor_rsrc_init_phase2); 584 585 ASSERT(state != NULL); 586 587 /* Phase 2 initialization begins where Phase 1 left off */ 588 cleanup = TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE; 589 590 /* 591 * Calculate the extent of the DDR size and portion of which that 592 * is already reserved for Tavor firmware. (Note: this information 593 * is available because the QUERY_DDR and QUERY_FW commands have 594 * been posted to Tavor firmware prior to calling this routine) 595 */ 596 ddr_size = state->ts_ddr.ddr_endaddr - state->ts_ddr.ddr_baseaddr + 1; 597 fw_size = state->ts_fw.fw_endaddr - state->ts_fw.fw_baseaddr + 1; 598 599 /* Build the DDR vmem arena name from Tavor instance */ 600 rsrc_name = (char *)kmem_zalloc(TAVOR_RSRC_NAME_MAXLEN, KM_SLEEP); 601 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_VMEM); 602 603 /* 604 * Do a vmem_create for the entire DDR range (not including the 605 * portion consumed by Tavor firmware). This creates the vmem arena 606 * from which all other DDR objects (specifically, tables of HW 607 * entries) will be allocated. 608 */ 609 state->ts_ddrvmem = vmem_create(rsrc_name, 610 (void *)(uintptr_t)state->ts_ddr.ddr_baseaddr, (ddr_size - fw_size), 611 sizeof (uint64_t), NULL, NULL, NULL, 0, VM_SLEEP); 612 if (state->ts_ddrvmem == NULL) { 613 tavor_rsrc_fini(state, cleanup); 614 /* Set "status" and "errormsg" and goto failure */ 615 TAVOR_TNF_FAIL(DDI_FAILURE, "DDR vmem"); 616 goto rsrcinitp2_fail; 617 } 618 cleanup = TAVOR_RSRC_CLEANUP_LEVEL5; 619 620 /* 621 * Initialize the resource pools for all objects that exist in 622 * Tavor DDR memory. This includes ("In") mailboxes, context tables 623 * (QPC, CQC, EQC, etc...), and other miscellaneous HW objects. 624 */ 625 cfgprof = state->ts_cfg_profile; 626 627 /* 628 * Initialize the resource pool for the MPT table entries. Notice 629 * that the number of MPTs is configurable. The configured value must 630 * be less that the maximum value (obtained from the QUERY_DEV_LIM 631 * command) or the initialization will fail. Note also that a certain 632 * number of MPTs must be set aside for Tavor firmware use. 633 */ 634 num = ((uint64_t)1 << cfgprof->cp_log_num_mpt); 635 max = ((uint64_t)1 << state->ts_devlim.log_max_mpt); 636 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_mpt); 637 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MPT]; 638 rsrc_pool->rsrc_type = TAVOR_MPT; 639 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 640 rsrc_pool->rsrc_pool_size = (TAVOR_MPT_SIZE * num); 641 rsrc_pool->rsrc_shift = TAVOR_MPT_SIZE_SHIFT; 642 rsrc_pool->rsrc_quantum = TAVOR_MPT_SIZE; 643 rsrc_pool->rsrc_align = (TAVOR_MPT_SIZE * num); 644 rsrc_pool->rsrc_state = state; 645 rsrc_pool->rsrc_start = NULL; 646 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MPT_VMEM); 647 entry_info.hwi_num = num; 648 entry_info.hwi_max = max; 649 entry_info.hwi_prealloc = num_prealloc; 650 entry_info.hwi_rsrcpool = rsrc_pool; 651 entry_info.hwi_rsrcname = rsrc_name; 652 status = tavor_rsrc_hw_entries_init(state, &entry_info); 653 if (status != DDI_SUCCESS) { 654 tavor_rsrc_fini(state, cleanup); 655 /* Set "status" and "errormsg" and goto failure */ 656 TAVOR_TNF_FAIL(DDI_FAILURE, "MPT table"); 657 goto rsrcinitp2_fail; 658 } 659 cleanup = TAVOR_RSRC_CLEANUP_LEVEL6; 660 661 /* 662 * Initialize the resource pool for the MTT table entries. Notice 663 * that the number of MTTs is configurable. The configured value must 664 * be less that the maximum value (obtained from the QUERY_DEV_LIM 665 * command) or the initialization will fail. Note also that a certain 666 * number of MTT segments must be set aside for Tavor firmware use. 667 */ 668 mttsegment_sz = (TAVOR_MTTSEG_SIZE << TAVOR_MTT_SIZE_SHIFT); 669 num = ((uint64_t)1 << cfgprof->cp_log_num_mttseg); 670 max = ((uint64_t)1 << state->ts_devlim.log_max_mttseg); 671 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_mttseg); 672 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MTT]; 673 rsrc_pool->rsrc_type = TAVOR_MTT; 674 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 675 rsrc_pool->rsrc_pool_size = (TAVOR_MTT_SIZE * num); 676 rsrc_pool->rsrc_shift = TAVOR_MTT_SIZE_SHIFT; 677 rsrc_pool->rsrc_quantum = mttsegment_sz; 678 rsrc_pool->rsrc_align = (TAVOR_MTT_SIZE * num); 679 rsrc_pool->rsrc_state = state; 680 rsrc_pool->rsrc_start = NULL; 681 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MTT_VMEM); 682 entry_info.hwi_num = num; 683 entry_info.hwi_max = max; 684 entry_info.hwi_prealloc = num_prealloc; 685 entry_info.hwi_rsrcpool = rsrc_pool; 686 entry_info.hwi_rsrcname = rsrc_name; 687 status = tavor_rsrc_hw_entries_init(state, &entry_info); 688 if (status != DDI_SUCCESS) { 689 tavor_rsrc_fini(state, cleanup); 690 /* Set "status" and "errormsg" and goto failure */ 691 TAVOR_TNF_FAIL(DDI_FAILURE, "MTT table"); 692 goto rsrcinitp2_fail; 693 } 694 cleanup = TAVOR_RSRC_CLEANUP_LEVEL7; 695 696 /* 697 * Initialize the resource pool for the QPC table entries. Notice 698 * that the number of QPs is configurable. The configured value must 699 * be less that the maximum value (obtained from the QUERY_DEV_LIM 700 * command) or the initialization will fail. Note also that a certain 701 * number of QP contexts must be set aside for Tavor firmware use. 702 */ 703 num = ((uint64_t)1 << cfgprof->cp_log_num_qp); 704 max = ((uint64_t)1 << state->ts_devlim.log_max_qp); 705 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_qp); 706 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPC]; 707 rsrc_pool->rsrc_type = TAVOR_QPC; 708 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 709 rsrc_pool->rsrc_pool_size = (TAVOR_QPC_SIZE * num); 710 rsrc_pool->rsrc_shift = TAVOR_QPC_SIZE_SHIFT; 711 rsrc_pool->rsrc_quantum = TAVOR_QPC_SIZE; 712 rsrc_pool->rsrc_align = (TAVOR_QPC_SIZE * num); 713 rsrc_pool->rsrc_state = state; 714 rsrc_pool->rsrc_start = NULL; 715 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_QPC_VMEM); 716 entry_info.hwi_num = num; 717 entry_info.hwi_max = max; 718 entry_info.hwi_prealloc = num_prealloc; 719 entry_info.hwi_rsrcpool = rsrc_pool; 720 entry_info.hwi_rsrcname = rsrc_name; 721 status = tavor_rsrc_hw_entries_init(state, &entry_info); 722 if (status != DDI_SUCCESS) { 723 tavor_rsrc_fini(state, cleanup); 724 /* Set "status" and "errormsg" and goto failure */ 725 TAVOR_TNF_FAIL(DDI_FAILURE, "QPC table"); 726 goto rsrcinitp2_fail; 727 } 728 cleanup = TAVOR_RSRC_CLEANUP_LEVEL8; 729 730 /* 731 * Initialize the resource pool for the RDB table entries. Notice 732 * that the number of RDBs is configurable. The configured value must 733 * be less that the maximum value (obtained from the QUERY_DEV_LIM 734 * command) or the initialization will fail. 735 */ 736 num = ((uint64_t)1 << cfgprof->cp_log_num_rdb); 737 max = ((uint64_t)1 << state->ts_devlim.log_max_ra_glob); 738 num_prealloc = 0; 739 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_RDB]; 740 rsrc_pool->rsrc_type = TAVOR_RDB; 741 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 742 rsrc_pool->rsrc_pool_size = (TAVOR_RDB_SIZE * num); 743 rsrc_pool->rsrc_shift = TAVOR_RDB_SIZE_SHIFT; 744 rsrc_pool->rsrc_quantum = TAVOR_RDB_SIZE; 745 rsrc_pool->rsrc_align = (TAVOR_RDB_SIZE * num); 746 rsrc_pool->rsrc_state = state; 747 rsrc_pool->rsrc_start = NULL; 748 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_RDB_VMEM); 749 entry_info.hwi_num = num; 750 entry_info.hwi_max = max; 751 entry_info.hwi_prealloc = num_prealloc; 752 entry_info.hwi_rsrcpool = rsrc_pool; 753 entry_info.hwi_rsrcname = rsrc_name; 754 status = tavor_rsrc_hw_entries_init(state, &entry_info); 755 if (status != DDI_SUCCESS) { 756 tavor_rsrc_fini(state, cleanup); 757 /* Set "status" and "errormsg" and goto failure */ 758 TAVOR_TNF_FAIL(DDI_FAILURE, "RDB table"); 759 goto rsrcinitp2_fail; 760 } 761 cleanup = TAVOR_RSRC_CLEANUP_LEVEL9; 762 763 /* 764 * Initialize the resource pool for the CQC table entries. Notice 765 * that the number of CQs is configurable. The configured value must 766 * be less that the maximum value (obtained from the QUERY_DEV_LIM 767 * command) or the initialization will fail. Note also that a certain 768 * number of CQ contexts must be set aside for Tavor firmware use. 769 */ 770 num = ((uint64_t)1 << cfgprof->cp_log_num_cq); 771 max = ((uint64_t)1 << state->ts_devlim.log_max_cq); 772 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_cq); 773 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQC]; 774 rsrc_pool->rsrc_type = TAVOR_CQC; 775 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 776 rsrc_pool->rsrc_pool_size = (TAVOR_CQC_SIZE * num); 777 rsrc_pool->rsrc_shift = TAVOR_CQC_SIZE_SHIFT; 778 rsrc_pool->rsrc_quantum = TAVOR_CQC_SIZE; 779 rsrc_pool->rsrc_align = (TAVOR_CQC_SIZE * num); 780 rsrc_pool->rsrc_state = state; 781 rsrc_pool->rsrc_start = NULL; 782 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_CQC_VMEM); 783 entry_info.hwi_num = num; 784 entry_info.hwi_max = max; 785 entry_info.hwi_prealloc = num_prealloc; 786 entry_info.hwi_rsrcpool = rsrc_pool; 787 entry_info.hwi_rsrcname = rsrc_name; 788 status = tavor_rsrc_hw_entries_init(state, &entry_info); 789 if (status != DDI_SUCCESS) { 790 tavor_rsrc_fini(state, cleanup); 791 /* Set "status" and "errormsg" and goto failure */ 792 TAVOR_TNF_FAIL(DDI_FAILURE, "CQC table"); 793 goto rsrcinitp2_fail; 794 } 795 cleanup = TAVOR_RSRC_CLEANUP_LEVEL10; 796 797 /* 798 * Initialize the resource pool for the Extended QPC table entries. 799 * Notice that the number of EQPCs must be the same as the number 800 * of QP contexts. So the initialization is constructed in a 801 * similar way as above (for TAVOR_QPC). One notable difference 802 * here, however, is that by setting the rsrc_quantum field to 803 * zero (indicating a zero-sized object) we indicate that the 804 * object is not allocatable. The EQPC table is, in fact, managed 805 * internally by the hardware and it is, therefore, unnecessary to 806 * initialize an additional vmem_arena for this type of object. 807 */ 808 num = ((uint64_t)1 << cfgprof->cp_log_num_qp); 809 max = ((uint64_t)1 << state->ts_devlim.log_max_qp); 810 num_prealloc = 0; 811 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQPC]; 812 rsrc_pool->rsrc_type = TAVOR_EQPC; 813 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 814 rsrc_pool->rsrc_pool_size = (TAVOR_EQPC_SIZE * num); 815 rsrc_pool->rsrc_shift = 0; 816 rsrc_pool->rsrc_quantum = 0; 817 rsrc_pool->rsrc_align = TAVOR_EQPC_SIZE; 818 rsrc_pool->rsrc_state = state; 819 rsrc_pool->rsrc_start = NULL; 820 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_EQPC_VMEM); 821 entry_info.hwi_num = num; 822 entry_info.hwi_max = max; 823 entry_info.hwi_prealloc = num_prealloc; 824 entry_info.hwi_rsrcpool = rsrc_pool; 825 entry_info.hwi_rsrcname = rsrc_name; 826 status = tavor_rsrc_hw_entries_init(state, &entry_info); 827 if (status != DDI_SUCCESS) { 828 tavor_rsrc_fini(state, cleanup); 829 /* Set "status" and "errormsg" and goto failure */ 830 TAVOR_TNF_FAIL(DDI_FAILURE, "Extended QPC table"); 831 goto rsrcinitp2_fail; 832 } 833 cleanup = TAVOR_RSRC_CLEANUP_LEVEL11; 834 835 /* 836 * Initialize the resource pool for the UD address vector table 837 * entries. Notice that the number of UDAVs is configurable. The 838 * configured value must be less that the maximum value (obtained 839 * from the QUERY_DEV_LIM command) or the initialization will fail. 840 */ 841 num = ((uint64_t)1 << cfgprof->cp_log_num_ah); 842 max = ((uint64_t)1 << state->ts_devlim.log_max_av); 843 num_prealloc = 0; 844 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UDAV]; 845 rsrc_pool->rsrc_type = TAVOR_UDAV; 846 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 847 rsrc_pool->rsrc_pool_size = (TAVOR_UDAV_SIZE * num); 848 rsrc_pool->rsrc_shift = TAVOR_UDAV_SIZE_SHIFT; 849 rsrc_pool->rsrc_quantum = TAVOR_UDAV_SIZE; 850 rsrc_pool->rsrc_align = TAVOR_UDAV_SIZE; 851 rsrc_pool->rsrc_state = state; 852 rsrc_pool->rsrc_start = NULL; 853 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_UDAV_VMEM); 854 entry_info.hwi_num = num; 855 entry_info.hwi_max = max; 856 entry_info.hwi_prealloc = num_prealloc; 857 entry_info.hwi_rsrcpool = rsrc_pool; 858 entry_info.hwi_rsrcname = rsrc_name; 859 status = tavor_rsrc_hw_entries_init(state, &entry_info); 860 if (status != DDI_SUCCESS) { 861 tavor_rsrc_fini(state, cleanup); 862 /* Set "status" and "errormsg" and goto failure */ 863 TAVOR_TNF_FAIL(DDI_FAILURE, "UDAV table"); 864 goto rsrcinitp2_fail; 865 } 866 cleanup = TAVOR_RSRC_CLEANUP_LEVEL12; 867 868 /* 869 * Initialize the resource pool for the UAR scratch table entries. 870 * Notice that the number of UARSCRs is configurable. The configured 871 * value must be less that the maximum value (obtained from the 872 * QUERY_DEV_LIM command) or the initialization will fail. 873 * Like the EQPCs above, UARSCR objects are not allocatable. The 874 * UARSCR table is also managed internally by the hardware and it 875 * is, therefore, unnecessary to initialize an additional vmem_arena 876 * for this type of object. We indicate this by setting the 877 * rsrc_quantum field to zero (indicating a zero-sized object). 878 */ 879 uarscr_size = state->ts_devlim.uarscr_entry_sz; 880 num = ((uint64_t)1 << cfgprof->cp_log_num_uar); 881 max = ((uint64_t)1 << (state->ts_devlim.log_max_uar_sz + 20 - 882 PAGESHIFT)); 883 num_prealloc = 0; 884 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR]; 885 rsrc_pool->rsrc_type = TAVOR_UAR_SCR; 886 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 887 rsrc_pool->rsrc_pool_size = (uarscr_size * num); 888 rsrc_pool->rsrc_shift = 0; 889 rsrc_pool->rsrc_quantum = 0; 890 rsrc_pool->rsrc_align = uarscr_size; 891 rsrc_pool->rsrc_state = state; 892 rsrc_pool->rsrc_start = NULL; 893 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_UARSCR_VMEM); 894 entry_info.hwi_num = num; 895 entry_info.hwi_max = max; 896 entry_info.hwi_prealloc = num_prealloc; 897 entry_info.hwi_rsrcpool = rsrc_pool; 898 entry_info.hwi_rsrcname = rsrc_name; 899 status = tavor_rsrc_hw_entries_init(state, &entry_info); 900 if (status != DDI_SUCCESS) { 901 tavor_rsrc_fini(state, cleanup); 902 /* Set "status" and "errormsg" and goto failure */ 903 TAVOR_TNF_FAIL(DDI_FAILURE, "UAR scratch table"); 904 goto rsrcinitp2_fail; 905 } 906 cleanup = TAVOR_RSRC_CLEANUP_LEVEL13; 907 908 /* 909 * Initialize the resource pool for the SRQC table entries. Notice 910 * that the number of SRQs is configurable. The configured value must 911 * be less that the maximum value (obtained from the QUERY_DEV_LIM 912 * command) or the initialization will fail. Note also that a certain 913 * number of SRQ contexts must be set aside for Tavor firmware use. 914 * 915 * Note: We only allocate these resources if SRQ is enabled in the 916 * config profile; see below. 917 */ 918 num = ((uint64_t)1 << cfgprof->cp_log_num_srq); 919 max = ((uint64_t)1 << state->ts_devlim.log_max_srq); 920 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_srq); 921 922 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQC]; 923 rsrc_pool->rsrc_type = TAVOR_SRQC; 924 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 925 rsrc_pool->rsrc_pool_size = (TAVOR_SRQC_SIZE * num); 926 rsrc_pool->rsrc_shift = TAVOR_SRQC_SIZE_SHIFT; 927 rsrc_pool->rsrc_quantum = TAVOR_SRQC_SIZE; 928 rsrc_pool->rsrc_align = (TAVOR_SRQC_SIZE * num); 929 rsrc_pool->rsrc_state = state; 930 rsrc_pool->rsrc_start = NULL; 931 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_SRQC_VMEM); 932 entry_info.hwi_num = num; 933 entry_info.hwi_max = max; 934 entry_info.hwi_prealloc = num_prealloc; 935 entry_info.hwi_rsrcpool = rsrc_pool; 936 entry_info.hwi_rsrcname = rsrc_name; 937 938 /* 939 * SRQ support is configurable. Only if SRQ is enabled (the default) 940 * do we actually try to configure these resources. Otherwise, we 941 * simply set the cleanup level and continue on to the next resource 942 */ 943 if (state->ts_cfg_profile->cp_srq_enable != 0) { 944 status = tavor_rsrc_hw_entries_init(state, &entry_info); 945 if (status != DDI_SUCCESS) { 946 tavor_rsrc_fini(state, cleanup); 947 /* Set "status" and "errormsg" and goto failure */ 948 TAVOR_TNF_FAIL(DDI_FAILURE, "SRQC table"); 949 goto rsrcinitp2_fail; 950 } 951 } 952 cleanup = TAVOR_RSRC_CLEANUP_LEVEL14; 953 954 /* 955 * Initialize the resource pool for "In" mailboxes. Notice that 956 * the number of "In" mailboxes, their size, and their location 957 * (DDR or system memory) is configurable. By default, however, 958 * all "In" mailboxes are located in system memory only (because 959 * they are primarily written to and rarely read from) 960 */ 961 num = ((uint64_t)1 << cfgprof->cp_log_num_inmbox); 962 size = ((uint64_t)1 << cfgprof->cp_log_inmbox_size); 963 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_IN_MBOX]; 964 rsrc_pool->rsrc_type = TAVOR_IN_MBOX; 965 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 966 rsrc_pool->rsrc_pool_size = (size * num); 967 rsrc_pool->rsrc_shift = cfgprof->cp_log_inmbox_size; 968 rsrc_pool->rsrc_quantum = size; 969 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN; 970 rsrc_pool->rsrc_state = state; 971 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INMBOX_VMEM); 972 mbox_info.mbi_num = num; 973 mbox_info.mbi_size = size; 974 mbox_info.mbi_rsrcpool = rsrc_pool; 975 mbox_info.mbi_rsrcname = rsrc_name; 976 status = tavor_rsrc_mbox_init(state, &mbox_info); 977 if (status != DDI_SUCCESS) { 978 tavor_rsrc_fini(state, cleanup); 979 /* Set "status" and "errormsg" and goto failure */ 980 TAVOR_TNF_FAIL(DDI_FAILURE, "in mailboxes"); 981 goto rsrcinitp2_fail; 982 } 983 cleanup = TAVOR_RSRC_CLEANUP_LEVEL15; 984 985 /* 986 * Initialize the Tavor "In" mailbox list. This step actually uses 987 * the tavor_rsrc_alloc() for TAVOR_IN_MBOX to preallocate the 988 * "In" mailboxes, bind them for DMA access, and arrange them into 989 * an easily accessed fast-allocation mechanism (see tavor_cmd.c 990 * for more details) 991 */ 992 status = tavor_inmbox_list_init(state); 993 if (status != DDI_SUCCESS) { 994 tavor_rsrc_fini(state, cleanup); 995 /* Set "status" and "errormsg" and goto failure */ 996 TAVOR_TNF_FAIL(DDI_FAILURE, "in mailbox list"); 997 goto rsrcinitp2_fail; 998 } 999 cleanup = TAVOR_RSRC_CLEANUP_LEVEL16; 1000 1001 /* 1002 * Initialize the resource pool for interrupt "In" mailboxes. Notice 1003 * that the number of interrupt "In" mailboxes, their size, and their 1004 * location (DDR or system memory) is configurable. By default, 1005 * however, all interrupt "In" mailboxes are located in system memory 1006 * only (because they are primarily written to and rarely read from) 1007 */ 1008 num = ((uint64_t)1 << cfgprof->cp_log_num_intr_inmbox); 1009 size = ((uint64_t)1 << cfgprof->cp_log_inmbox_size); 1010 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_INTR_IN_MBOX]; 1011 rsrc_pool->rsrc_type = TAVOR_INTR_IN_MBOX; 1012 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 1013 rsrc_pool->rsrc_pool_size = (size * num); 1014 rsrc_pool->rsrc_shift = cfgprof->cp_log_inmbox_size; 1015 rsrc_pool->rsrc_quantum = size; 1016 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN; 1017 rsrc_pool->rsrc_state = state; 1018 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INTR_INMBOX_VMEM); 1019 mbox_info.mbi_num = num; 1020 mbox_info.mbi_size = size; 1021 mbox_info.mbi_rsrcpool = rsrc_pool; 1022 mbox_info.mbi_rsrcname = rsrc_name; 1023 status = tavor_rsrc_mbox_init(state, &mbox_info); 1024 if (status != DDI_SUCCESS) { 1025 tavor_rsrc_fini(state, cleanup); 1026 /* Set "status" and "errormsg" and goto failure */ 1027 TAVOR_TNF_FAIL(DDI_FAILURE, "in intr mailboxes"); 1028 goto rsrcinitp2_fail; 1029 } 1030 cleanup = TAVOR_RSRC_CLEANUP_LEVEL17; 1031 1032 /* 1033 * Initialize the Tavor interrupt "In" mailbox list. This step 1034 * actually uses the tavor_rsrc_alloc() for TAVOR_IN_MBOX to 1035 * preallocate the interrupt "In" mailboxes, bind them for DMA access, 1036 * and arrange them into an easily accessed fast-allocation mechanism 1037 * (see tavor_cmd.c for more details) 1038 */ 1039 status = tavor_intr_inmbox_list_init(state); 1040 if (status != DDI_SUCCESS) { 1041 tavor_rsrc_fini(state, cleanup); 1042 /* Set "status" and "errormsg" and goto failure */ 1043 TAVOR_TNF_FAIL(DDI_FAILURE, "in intr mailbox list"); 1044 goto rsrcinitp2_fail; 1045 } 1046 cleanup = TAVOR_RSRC_CLEANUP_LEVEL18; 1047 1048 /* 1049 * Initialize the Tavor command handling interfaces. This step 1050 * sets up the outstanding command tracking mechanism for easy access 1051 * and fast allocation (see tavor_cmd.c for more details). 1052 */ 1053 status = tavor_outstanding_cmdlist_init(state); 1054 if (status != DDI_SUCCESS) { 1055 tavor_rsrc_fini(state, cleanup); 1056 /* Set "status" and "errormsg" and goto failure */ 1057 TAVOR_TNF_FAIL(DDI_FAILURE, "outstanding cmd list"); 1058 goto rsrcinitp2_fail; 1059 } 1060 cleanup = TAVOR_RSRC_CLEANUP_LEVEL19; 1061 1062 /* 1063 * Calculate (and validate) the size of Multicast Group (MCG) entries 1064 */ 1065 status = tavor_rsrc_mcg_entry_get_size(state, &mcg_size_shift); 1066 if (status != DDI_SUCCESS) { 1067 tavor_rsrc_fini(state, cleanup); 1068 /* Set "status" and "errormsg" and goto failure */ 1069 TAVOR_TNF_FAIL(DDI_FAILURE, "failed get MCG size"); 1070 goto rsrcinitp2_fail; 1071 } 1072 mcg_size = TAVOR_MCGMEM_SZ(state); 1073 1074 /* 1075 * Initialize the resource pool for the MCG table entries. Notice 1076 * that the number of MCGs is configurable. The configured value must 1077 * be less that the maximum value (obtained from the QUERY_DEV_LIM 1078 * command) or the initialization will fail. Note also that a certain 1079 * number of MCGs must be set aside for Tavor firmware use (they 1080 * correspond to the number of MCGs used by the internal hash 1081 * function. 1082 */ 1083 num = ((uint64_t)1 << cfgprof->cp_log_num_mcg); 1084 max = ((uint64_t)1 << state->ts_devlim.log_max_mcg); 1085 num_prealloc = ((uint64_t)1 << cfgprof->cp_log_num_mcg_hash); 1086 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MCG]; 1087 rsrc_pool->rsrc_type = TAVOR_MCG; 1088 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 1089 rsrc_pool->rsrc_pool_size = (mcg_size * num); 1090 rsrc_pool->rsrc_shift = mcg_size_shift; 1091 rsrc_pool->rsrc_quantum = mcg_size; 1092 rsrc_pool->rsrc_align = mcg_size; 1093 rsrc_pool->rsrc_state = state; 1094 rsrc_pool->rsrc_start = NULL; 1095 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MCG_VMEM); 1096 entry_info.hwi_num = num; 1097 entry_info.hwi_max = max; 1098 entry_info.hwi_prealloc = num_prealloc; 1099 entry_info.hwi_rsrcpool = rsrc_pool; 1100 entry_info.hwi_rsrcname = rsrc_name; 1101 status = tavor_rsrc_hw_entries_init(state, &entry_info); 1102 if (status != DDI_SUCCESS) { 1103 tavor_rsrc_fini(state, cleanup); 1104 /* Set "status" and "errormsg" and goto failure */ 1105 TAVOR_TNF_FAIL(DDI_FAILURE, "MCG table"); 1106 goto rsrcinitp2_fail; 1107 } 1108 cleanup = TAVOR_RSRC_CLEANUP_LEVEL20; 1109 1110 /* 1111 * Initialize the resource pool for the EQC table entries. Notice 1112 * that the number of EQs is hardcoded. The hardcoded value should 1113 * be less that the maximum value (obtained from the QUERY_DEV_LIM 1114 * command) or the initialization will fail. 1115 */ 1116 num = TAVOR_NUM_EQ; 1117 max = ((uint64_t)1 << state->ts_devlim.log_max_eq); 1118 num_prealloc = 0; 1119 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQC]; 1120 rsrc_pool->rsrc_type = TAVOR_EQC; 1121 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 1122 rsrc_pool->rsrc_pool_size = (TAVOR_EQC_SIZE * num); 1123 rsrc_pool->rsrc_shift = TAVOR_EQC_SIZE_SHIFT; 1124 rsrc_pool->rsrc_quantum = TAVOR_EQC_SIZE; 1125 rsrc_pool->rsrc_align = (TAVOR_EQC_SIZE * num); 1126 rsrc_pool->rsrc_state = state; 1127 rsrc_pool->rsrc_start = NULL; 1128 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_EQC_VMEM); 1129 entry_info.hwi_num = num; 1130 entry_info.hwi_max = max; 1131 entry_info.hwi_prealloc = num_prealloc; 1132 entry_info.hwi_rsrcpool = rsrc_pool; 1133 entry_info.hwi_rsrcname = rsrc_name; 1134 status = tavor_rsrc_hw_entries_init(state, &entry_info); 1135 if (status != DDI_SUCCESS) { 1136 tavor_rsrc_fini(state, cleanup); 1137 /* Set "status" and "errormsg" and goto failure */ 1138 TAVOR_TNF_FAIL(DDI_FAILURE, "EQC table"); 1139 goto rsrcinitp2_fail; 1140 } 1141 cleanup = TAVOR_RSRC_CLEANUP_LEVEL21; 1142 1143 /* 1144 * Initialize the resource pools for all objects that exist in 1145 * system memory. This includes PD handles, MR handle, EQ handles, 1146 * QP handles, etc. These objects are almost entirely managed using 1147 * kmem_cache routines. (See comment above for more detail) 1148 */ 1149 1150 /* 1151 * Initialize the resource pool for the PD handles. Notice 1152 * that the number of PDHDLs is configurable. The configured value 1153 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1154 * command) or the initialization will fail. Note also that the PD 1155 * handle has constructor and destructor methods associated with it. 1156 */ 1157 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_PDHDL]; 1158 rsrc_pool->rsrc_type = TAVOR_PDHDL; 1159 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1160 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_pd_s); 1161 rsrc_pool->rsrc_state = state; 1162 TAVOR_RSRC_NAME(rsrc_name, TAVOR_PDHDL_CACHE); 1163 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_pd); 1164 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_pd); 1165 hdl_info.swi_rsrcpool = rsrc_pool; 1166 hdl_info.swi_constructor = tavor_rsrc_pdhdl_constructor; 1167 hdl_info.swi_destructor = tavor_rsrc_pdhdl_destructor; 1168 hdl_info.swi_rsrcname = rsrc_name; 1169 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT; 1170 status = tavor_rsrc_pd_handles_init(state, &hdl_info); 1171 if (status != DDI_SUCCESS) { 1172 tavor_rsrc_fini(state, cleanup); 1173 /* Set "status" and "errormsg" and goto failure */ 1174 TAVOR_TNF_FAIL(DDI_FAILURE, "PD handle"); 1175 goto rsrcinitp2_fail; 1176 } 1177 cleanup = TAVOR_RSRC_CLEANUP_LEVEL22; 1178 1179 /* 1180 * Initialize the resource pool for the MR handles. Notice 1181 * that the number of MRHDLs is configurable. The configured value 1182 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1183 * command) or the initialization will fail. 1184 */ 1185 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MRHDL]; 1186 rsrc_pool->rsrc_type = TAVOR_MRHDL; 1187 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1188 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_mr_s); 1189 rsrc_pool->rsrc_state = state; 1190 TAVOR_RSRC_NAME(rsrc_name, TAVOR_MRHDL_CACHE); 1191 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mpt); 1192 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mpt); 1193 hdl_info.swi_rsrcpool = rsrc_pool; 1194 hdl_info.swi_constructor = tavor_rsrc_mrhdl_constructor; 1195 hdl_info.swi_destructor = tavor_rsrc_mrhdl_destructor; 1196 hdl_info.swi_rsrcname = rsrc_name; 1197 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT; 1198 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1199 if (status != DDI_SUCCESS) { 1200 tavor_rsrc_fini(state, cleanup); 1201 /* Set "status" and "errormsg" and goto failure */ 1202 TAVOR_TNF_FAIL(DDI_FAILURE, "MR handle"); 1203 goto rsrcinitp2_fail; 1204 } 1205 cleanup = TAVOR_RSRC_CLEANUP_LEVEL23; 1206 1207 /* 1208 * Initialize the resource pool for the EQ handles. Notice 1209 * that the number of EQHDLs is hardcoded. The hardcoded value 1210 * should be less that the maximum value (obtained from the 1211 * QUERY_DEV_LIM command) or the initialization will fail. 1212 */ 1213 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQHDL]; 1214 rsrc_pool->rsrc_type = TAVOR_EQHDL; 1215 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1216 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_eq_s); 1217 rsrc_pool->rsrc_state = state; 1218 TAVOR_RSRC_NAME(rsrc_name, TAVOR_EQHDL_CACHE); 1219 hdl_info.swi_num = TAVOR_NUM_EQ; 1220 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_eq); 1221 hdl_info.swi_rsrcpool = rsrc_pool; 1222 hdl_info.swi_constructor = NULL; 1223 hdl_info.swi_destructor = NULL; 1224 hdl_info.swi_rsrcname = rsrc_name; 1225 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT; 1226 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1227 if (status != DDI_SUCCESS) { 1228 tavor_rsrc_fini(state, cleanup); 1229 /* Set "status" and "errormsg" and goto failure */ 1230 TAVOR_TNF_FAIL(DDI_FAILURE, "EQ handle"); 1231 goto rsrcinitp2_fail; 1232 } 1233 cleanup = TAVOR_RSRC_CLEANUP_LEVEL24; 1234 1235 /* 1236 * Initialize the resource pool for the CQ handles. Notice 1237 * that the number of CQHDLs is configurable. The configured value 1238 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1239 * command) or the initialization will fail. Note also that the CQ 1240 * handle has constructor and destructor methods associated with it. 1241 */ 1242 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQHDL]; 1243 rsrc_pool->rsrc_type = TAVOR_CQHDL; 1244 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1245 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_cq_s); 1246 rsrc_pool->rsrc_state = state; 1247 TAVOR_RSRC_NAME(rsrc_name, TAVOR_CQHDL_CACHE); 1248 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_cq); 1249 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_cq); 1250 hdl_info.swi_rsrcpool = rsrc_pool; 1251 hdl_info.swi_constructor = tavor_rsrc_cqhdl_constructor; 1252 hdl_info.swi_destructor = tavor_rsrc_cqhdl_destructor; 1253 hdl_info.swi_rsrcname = rsrc_name; 1254 hdl_info.swi_flags = (TAVOR_SWHDL_KMEMCACHE_INIT | 1255 TAVOR_SWHDL_TABLE_INIT); 1256 hdl_info.swi_prealloc_sz = sizeof (tavor_cqhdl_t); 1257 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1258 if (status != DDI_SUCCESS) { 1259 tavor_rsrc_fini(state, cleanup); 1260 /* Set "status" and "errormsg" and goto failure */ 1261 TAVOR_TNF_FAIL(DDI_FAILURE, "CQ handle"); 1262 goto rsrcinitp2_fail; 1263 } 1264 1265 /* 1266 * Save away the pointer to the central list of CQ handle pointers 1267 * This this is used as a mechanism to enable fast CQnumber-to-CQhandle 1268 * lookup during EQ event processing. The table is a list of 1269 * tavor_cqhdl_t allocated by the above routine because of the 1270 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many tavor_cqhdl_t 1271 * as the number of CQs. 1272 */ 1273 state->ts_cqhdl = hdl_info.swi_table_ptr; 1274 cleanup = TAVOR_RSRC_CLEANUP_LEVEL25; 1275 1276 /* 1277 * Initialize the resource pool for the SRQ handles. Notice 1278 * that the number of SRQHDLs is configurable. The configured value 1279 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1280 * command) or the initialization will fail. Note also that the SRQ 1281 * handle has constructor and destructor methods associated with it. 1282 * 1283 * Note: We only allocate these resources if SRQ is enabled in the 1284 * config profile; see below. 1285 */ 1286 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQHDL]; 1287 rsrc_pool->rsrc_type = TAVOR_SRQHDL; 1288 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1289 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_srq_s); 1290 rsrc_pool->rsrc_state = state; 1291 TAVOR_RSRC_NAME(rsrc_name, TAVOR_SRQHDL_CACHE); 1292 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_srq); 1293 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_srq); 1294 hdl_info.swi_rsrcpool = rsrc_pool; 1295 hdl_info.swi_constructor = tavor_rsrc_srqhdl_constructor; 1296 hdl_info.swi_destructor = tavor_rsrc_srqhdl_destructor; 1297 hdl_info.swi_rsrcname = rsrc_name; 1298 hdl_info.swi_flags = (TAVOR_SWHDL_KMEMCACHE_INIT | 1299 TAVOR_SWHDL_TABLE_INIT); 1300 hdl_info.swi_prealloc_sz = sizeof (tavor_srqhdl_t); 1301 1302 /* 1303 * SRQ support is configurable. Only if SRQ is enabled (the default) 1304 * do we actually try to configure these resources. Otherwise, we 1305 * simply set the cleanup level and continue on to the next resource 1306 */ 1307 if (state->ts_cfg_profile->cp_srq_enable != 0) { 1308 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1309 if (status != DDI_SUCCESS) { 1310 tavor_rsrc_fini(state, cleanup); 1311 /* Set "status" and "errormsg" and goto failure */ 1312 TAVOR_TNF_FAIL(DDI_FAILURE, "SRQ handle"); 1313 goto rsrcinitp2_fail; 1314 } 1315 1316 /* 1317 * Save away the pointer to the central list of SRQ handle 1318 * pointers This this is used as a mechanism to enable fast 1319 * SRQnumber-to-SRQhandle lookup. The table is a list of 1320 * tavor_srqhdl_t allocated by the above routine because of the 1321 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many 1322 * tavor_srqhdl_t as the number of SRQs. 1323 */ 1324 state->ts_srqhdl = hdl_info.swi_table_ptr; 1325 } 1326 cleanup = TAVOR_RSRC_CLEANUP_LEVEL26; 1327 1328 /* 1329 * Initialize the resource pool for the address handles. Notice 1330 * that the number of AHHDLs is configurable. The configured value 1331 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1332 * command) or the initialization will fail. 1333 */ 1334 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_AHHDL]; 1335 rsrc_pool->rsrc_type = TAVOR_AHHDL; 1336 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1337 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_ah_s); 1338 rsrc_pool->rsrc_state = state; 1339 TAVOR_RSRC_NAME(rsrc_name, TAVOR_AHHDL_CACHE); 1340 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_ah); 1341 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_av); 1342 hdl_info.swi_rsrcpool = rsrc_pool; 1343 hdl_info.swi_constructor = tavor_rsrc_ahhdl_constructor; 1344 hdl_info.swi_destructor = tavor_rsrc_ahhdl_destructor; 1345 hdl_info.swi_rsrcname = rsrc_name; 1346 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT; 1347 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1348 if (status != DDI_SUCCESS) { 1349 tavor_rsrc_fini(state, cleanup); 1350 /* Set "status" and "errormsg" and goto failure */ 1351 TAVOR_TNF_FAIL(DDI_FAILURE, "AH handle"); 1352 goto rsrcinitp2_fail; 1353 } 1354 cleanup = TAVOR_RSRC_CLEANUP_LEVEL27; 1355 1356 /* 1357 * Initialize the resource pool for the QP handles. Notice 1358 * that the number of QPHDLs is configurable. The configured value 1359 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1360 * command) or the initialization will fail. Note also that the QP 1361 * handle has constructor and destructor methods associated with it. 1362 */ 1363 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPHDL]; 1364 rsrc_pool->rsrc_type = TAVOR_QPHDL; 1365 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1366 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_qp_s); 1367 rsrc_pool->rsrc_state = state; 1368 TAVOR_RSRC_NAME(rsrc_name, TAVOR_QPHDL_CACHE); 1369 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_qp); 1370 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_qp); 1371 hdl_info.swi_rsrcpool = rsrc_pool; 1372 hdl_info.swi_constructor = tavor_rsrc_qphdl_constructor; 1373 hdl_info.swi_destructor = tavor_rsrc_qphdl_destructor; 1374 hdl_info.swi_rsrcname = rsrc_name; 1375 hdl_info.swi_flags = (TAVOR_SWHDL_KMEMCACHE_INIT | 1376 TAVOR_SWHDL_TABLE_INIT); 1377 hdl_info.swi_prealloc_sz = sizeof (tavor_qphdl_t); 1378 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1379 if (status != DDI_SUCCESS) { 1380 tavor_rsrc_fini(state, cleanup); 1381 /* Set "status" and "errormsg" and goto failure */ 1382 TAVOR_TNF_FAIL(DDI_FAILURE, "QP handle"); 1383 goto rsrcinitp2_fail; 1384 } 1385 1386 /* 1387 * Save away the pointer to the central list of QP handle pointers 1388 * This this is used as a mechanism to enable fast QPnumber-to-QPhandle 1389 * lookup during CQ event processing. The table is a list of 1390 * tavor_qphdl_t allocated by the above routine because of the 1391 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many tavor_qphdl_t 1392 * as the number of QPs. 1393 */ 1394 state->ts_qphdl = hdl_info.swi_table_ptr; 1395 cleanup = TAVOR_RSRC_CLEANUP_LEVEL28; 1396 1397 /* 1398 * Initialize the resource pool for the reference count handles. 1399 * Notice that the number of REFCNTs is configurable, but it's value 1400 * is set to the number of MPTs. Since REFCNTs are used to support 1401 * shared memory regions, it is possible that we might require as 1402 * one REFCNT for every MPT. 1403 */ 1404 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_REFCNT]; 1405 rsrc_pool->rsrc_type = TAVOR_REFCNT; 1406 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1407 rsrc_pool->rsrc_quantum = sizeof (tavor_sw_refcnt_t); 1408 rsrc_pool->rsrc_state = state; 1409 TAVOR_RSRC_NAME(rsrc_name, TAVOR_REFCNT_CACHE); 1410 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mpt); 1411 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mpt); 1412 hdl_info.swi_rsrcpool = rsrc_pool; 1413 hdl_info.swi_constructor = tavor_rsrc_refcnt_constructor; 1414 hdl_info.swi_destructor = tavor_rsrc_refcnt_destructor; 1415 hdl_info.swi_rsrcname = rsrc_name; 1416 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT; 1417 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1418 if (status != DDI_SUCCESS) { 1419 tavor_rsrc_fini(state, cleanup); 1420 /* Set "status" and "errormsg" and goto failure */ 1421 TAVOR_TNF_FAIL(DDI_FAILURE, "reference count handle"); 1422 goto rsrcinitp2_fail; 1423 } 1424 cleanup = TAVOR_RSRC_CLEANUP_LEVEL29; 1425 1426 /* 1427 * Initialize the resource pool for the MCG handles. Notice that for 1428 * these MCG handles, we are allocating a table of structures (used to 1429 * keep track of the MCG entries that are being written to hardware 1430 * and to speed up multicast attach/detach operations). 1431 */ 1432 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mcg); 1433 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mcg); 1434 hdl_info.swi_flags = TAVOR_SWHDL_TABLE_INIT; 1435 hdl_info.swi_prealloc_sz = sizeof (struct tavor_sw_mcg_list_s); 1436 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1437 if (status != DDI_SUCCESS) { 1438 tavor_rsrc_fini(state, cleanup); 1439 /* Set "status" and "errormsg" and goto failure */ 1440 TAVOR_TNF_FAIL(DDI_FAILURE, "MCG handle"); 1441 goto rsrcinitp2_fail; 1442 } 1443 state->ts_mcghdl = hdl_info.swi_table_ptr; 1444 cleanup = TAVOR_RSRC_CLEANUP_LEVEL30; 1445 1446 /* 1447 * Initialize the resource pools for all objects that exist in 1448 * UAR memory. The only objects that are allocated from UAR memory 1449 * are the UAR pages which are used for holding Tavor hardware's 1450 * doorbell registers. 1451 */ 1452 1453 /* 1454 * Initialize the resource pool for the UAR pages. Notice 1455 * that the number of UARPGs is configurable. The configured value 1456 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1457 * command) or the initialization will fail. Note also that by 1458 * specifying the rsrc_start parameter in advance, we direct the 1459 * initialization routine not to attempt to allocated space from the 1460 * Tavor DDR vmem_arena. 1461 */ 1462 num = ((uint64_t)1 << cfgprof->cp_log_num_uar); 1463 max = ((uint64_t)1 << (state->ts_devlim.log_max_uar_sz + 20 - 1464 PAGESHIFT)); 1465 num_prealloc = 0; 1466 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UARPG]; 1467 rsrc_pool->rsrc_type = TAVOR_UARPG; 1468 rsrc_pool->rsrc_loc = TAVOR_IN_UAR; 1469 rsrc_pool->rsrc_pool_size = (num << PAGESHIFT); 1470 rsrc_pool->rsrc_shift = PAGESHIFT; 1471 rsrc_pool->rsrc_quantum = PAGESIZE; 1472 rsrc_pool->rsrc_align = PAGESIZE; 1473 rsrc_pool->rsrc_state = state; 1474 rsrc_pool->rsrc_start = (void *)state->ts_reg_uar_baseaddr; 1475 TAVOR_RSRC_NAME(rsrc_name, TAVOR_UAR_VMEM); 1476 entry_info.hwi_num = num; 1477 entry_info.hwi_max = max; 1478 entry_info.hwi_prealloc = num_prealloc; 1479 entry_info.hwi_rsrcpool = rsrc_pool; 1480 entry_info.hwi_rsrcname = rsrc_name; 1481 status = tavor_rsrc_hw_entries_init(state, &entry_info); 1482 if (status != DDI_SUCCESS) { 1483 tavor_rsrc_fini(state, cleanup); 1484 /* Set "status" and "errormsg" and goto failure */ 1485 TAVOR_TNF_FAIL(DDI_FAILURE, "UAR page table"); 1486 goto rsrcinitp2_fail; 1487 } 1488 cleanup = TAVOR_RSRC_CLEANUP_ALL; 1489 1490 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN); 1491 TAVOR_TNF_EXIT(tavor_rsrc_init_phase2); 1492 return (DDI_SUCCESS); 1493 1494 rsrcinitp2_fail: 1495 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN); 1496 TNF_PROBE_1(tavor_rsrc_init_phase2_fail, TAVOR_TNF_ERROR, "", 1497 tnf_string, msg, errormsg); 1498 TAVOR_TNF_EXIT(tavor_rsrc_init_phase2); 1499 return (status); 1500 } 1501 1502 1503 /* 1504 * tavor_rsrc_fini() 1505 * Context: Only called from attach() and/or detach() path contexts 1506 */ 1507 void 1508 tavor_rsrc_fini(tavor_state_t *state, tavor_rsrc_cleanup_level_t clean) 1509 { 1510 tavor_rsrc_sw_hdl_info_t hdl_info; 1511 tavor_rsrc_hw_entry_info_t entry_info; 1512 tavor_rsrc_mbox_info_t mbox_info; 1513 tavor_cfg_profile_t *cfgprof; 1514 1515 TAVOR_TNF_ENTER(tavor_rsrc_fini); 1516 1517 ASSERT(state != NULL); 1518 1519 cfgprof = state->ts_cfg_profile; 1520 1521 switch (clean) { 1522 /* 1523 * If we add more resources that need to be cleaned up here, we should 1524 * ensure that TAVOR_RSRC_CLEANUP_ALL is still the first entry (i.e. 1525 * corresponds to the last resource allocated). 1526 */ 1527 case TAVOR_RSRC_CLEANUP_ALL: 1528 /* Cleanup the UAR page resource pool */ 1529 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UARPG]; 1530 tavor_rsrc_hw_entries_fini(state, &entry_info); 1531 /* FALLTHROUGH */ 1532 1533 case TAVOR_RSRC_CLEANUP_LEVEL30: 1534 /* Cleanup the central MCG handle pointers list */ 1535 hdl_info.swi_rsrcpool = NULL; 1536 hdl_info.swi_table_ptr = state->ts_mcghdl; 1537 hdl_info.swi_num = 1538 ((uint64_t)1 << cfgprof->cp_log_num_mcg); 1539 hdl_info.swi_prealloc_sz = sizeof (struct tavor_sw_mcg_list_s); 1540 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1541 /* FALLTHROUGH */ 1542 1543 case TAVOR_RSRC_CLEANUP_LEVEL29: 1544 /* Cleanup the reference count resource pool */ 1545 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_REFCNT]; 1546 hdl_info.swi_table_ptr = NULL; 1547 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1548 /* FALLTHROUGH */ 1549 1550 case TAVOR_RSRC_CLEANUP_LEVEL28: 1551 /* Cleanup the QP handle resource pool */ 1552 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_QPHDL]; 1553 hdl_info.swi_table_ptr = state->ts_qphdl; 1554 hdl_info.swi_num = 1555 ((uint64_t)1 << cfgprof->cp_log_num_qp); 1556 hdl_info.swi_prealloc_sz = sizeof (tavor_qphdl_t); 1557 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1558 /* FALLTHROUGH */ 1559 1560 case TAVOR_RSRC_CLEANUP_LEVEL27: 1561 /* Cleanup the address handle resource pool */ 1562 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_AHHDL]; 1563 hdl_info.swi_table_ptr = NULL; 1564 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1565 /* FALLTHROUGH */ 1566 1567 case TAVOR_RSRC_CLEANUP_LEVEL26: 1568 /* 1569 * Cleanup the SRQ handle resource pool. 1570 * 1571 * Note: We only clean up if SRQ is enabled. Otherwise we 1572 * simply fallthrough to the next resource cleanup. 1573 */ 1574 if (state->ts_cfg_profile->cp_srq_enable != 0) { 1575 hdl_info.swi_rsrcpool = 1576 &state->ts_rsrc_hdl[TAVOR_SRQHDL]; 1577 hdl_info.swi_table_ptr = state->ts_srqhdl; 1578 hdl_info.swi_num = 1579 ((uint64_t)1 << cfgprof->cp_log_num_srq); 1580 hdl_info.swi_prealloc_sz = sizeof (tavor_srqhdl_t); 1581 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1582 } 1583 /* FALLTHROUGH */ 1584 1585 case TAVOR_RSRC_CLEANUP_LEVEL25: 1586 /* Cleanup the CQ handle resource pool */ 1587 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_CQHDL]; 1588 hdl_info.swi_table_ptr = state->ts_cqhdl; 1589 hdl_info.swi_num = 1590 ((uint64_t)1 << cfgprof->cp_log_num_cq); 1591 hdl_info.swi_prealloc_sz = sizeof (tavor_cqhdl_t); 1592 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1593 /* FALLTHROUGH */ 1594 1595 case TAVOR_RSRC_CLEANUP_LEVEL24: 1596 /* Cleanup the EQ handle resource pool */ 1597 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQHDL]; 1598 hdl_info.swi_table_ptr = NULL; 1599 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1600 /* FALLTHROUGH */ 1601 1602 case TAVOR_RSRC_CLEANUP_LEVEL23: 1603 /* Cleanup the MR handle resource pool */ 1604 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MRHDL]; 1605 hdl_info.swi_table_ptr = NULL; 1606 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1607 /* FALLTHROUGH */ 1608 1609 case TAVOR_RSRC_CLEANUP_LEVEL22: 1610 /* Cleanup the PD handle resource pool */ 1611 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_PDHDL]; 1612 hdl_info.swi_table_ptr = NULL; 1613 tavor_rsrc_pd_handles_fini(state, &hdl_info); 1614 /* FALLTHROUGH */ 1615 1616 case TAVOR_RSRC_CLEANUP_LEVEL21: 1617 /* Cleanup the EQC table resource pool */ 1618 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQC]; 1619 tavor_rsrc_hw_entries_fini(state, &entry_info); 1620 /* FALLTHROUGH */ 1621 1622 case TAVOR_RSRC_CLEANUP_LEVEL20: 1623 /* Cleanup the MCG table resource pool */ 1624 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MCG]; 1625 tavor_rsrc_hw_entries_fini(state, &entry_info); 1626 /* FALLTHROUGH */ 1627 1628 case TAVOR_RSRC_CLEANUP_LEVEL19: 1629 /* Cleanup the outstanding command list */ 1630 tavor_outstanding_cmdlist_fini(state); 1631 /* FALLTHROUGH */ 1632 1633 case TAVOR_RSRC_CLEANUP_LEVEL18: 1634 /* Cleanup the "In" mailbox list */ 1635 tavor_intr_inmbox_list_fini(state); 1636 /* FALLTHROUGH */ 1637 1638 case TAVOR_RSRC_CLEANUP_LEVEL17: 1639 /* Cleanup the interrupt "In" mailbox resource pool */ 1640 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[ 1641 TAVOR_INTR_IN_MBOX]; 1642 tavor_rsrc_mbox_fini(state, &mbox_info); 1643 /* FALLTHROUGH */ 1644 1645 case TAVOR_RSRC_CLEANUP_LEVEL16: 1646 /* Cleanup the "In" mailbox list */ 1647 tavor_inmbox_list_fini(state); 1648 /* FALLTHROUGH */ 1649 1650 case TAVOR_RSRC_CLEANUP_LEVEL15: 1651 /* Cleanup the "In" mailbox resource pool */ 1652 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_IN_MBOX]; 1653 tavor_rsrc_mbox_fini(state, &mbox_info); 1654 /* FALLTHROUGH */ 1655 1656 case TAVOR_RSRC_CLEANUP_LEVEL14: 1657 /* 1658 * Cleanup the SRQC table resource pool. 1659 * 1660 * Note: We only clean up if SRQ is enabled. Otherwise we 1661 * simply fallthrough to the next resource cleanup. 1662 */ 1663 if (state->ts_cfg_profile->cp_srq_enable != 0) { 1664 entry_info.hwi_rsrcpool = 1665 &state->ts_rsrc_hdl[TAVOR_SRQC]; 1666 tavor_rsrc_hw_entries_fini(state, &entry_info); 1667 } 1668 /* FALLTHROUGH */ 1669 1670 case TAVOR_RSRC_CLEANUP_LEVEL13: 1671 /* Cleanup the UAR scratch table resource pool */ 1672 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR]; 1673 tavor_rsrc_hw_entries_fini(state, &entry_info); 1674 /* FALLTHROUGH */ 1675 1676 case TAVOR_RSRC_CLEANUP_LEVEL12: 1677 /* Cleanup the UDAV table resource pool */ 1678 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UDAV]; 1679 tavor_rsrc_hw_entries_fini(state, &entry_info); 1680 /* FALLTHROUGH */ 1681 1682 case TAVOR_RSRC_CLEANUP_LEVEL11: 1683 /* Cleanup the EQPC table resource pool */ 1684 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQPC]; 1685 tavor_rsrc_hw_entries_fini(state, &entry_info); 1686 /* FALLTHROUGH */ 1687 1688 case TAVOR_RSRC_CLEANUP_LEVEL10: 1689 /* Cleanup the CQC table resource pool */ 1690 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_CQC]; 1691 tavor_rsrc_hw_entries_fini(state, &entry_info); 1692 /* FALLTHROUGH */ 1693 1694 case TAVOR_RSRC_CLEANUP_LEVEL9: 1695 /* Cleanup the RDB table resource pool */ 1696 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_RDB]; 1697 tavor_rsrc_hw_entries_fini(state, &entry_info); 1698 /* FALLTHROUGH */ 1699 1700 case TAVOR_RSRC_CLEANUP_LEVEL8: 1701 /* Cleanup the QPC table resource pool */ 1702 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_QPC]; 1703 tavor_rsrc_hw_entries_fini(state, &entry_info); 1704 /* FALLTHROUGH */ 1705 1706 case TAVOR_RSRC_CLEANUP_LEVEL7: 1707 /* Cleanup the MTT table resource pool */ 1708 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MTT]; 1709 tavor_rsrc_hw_entries_fini(state, &entry_info); 1710 /* FALLTHROUGH */ 1711 1712 case TAVOR_RSRC_CLEANUP_LEVEL6: 1713 /* Cleanup the MPT table resource pool */ 1714 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MPT]; 1715 tavor_rsrc_hw_entries_fini(state, &entry_info); 1716 /* FALLTHROUGH */ 1717 1718 case TAVOR_RSRC_CLEANUP_LEVEL5: 1719 /* Destroy the vmem arena for DDR memory */ 1720 vmem_destroy(state->ts_ddrvmem); 1721 break; 1722 1723 /* 1724 * The cleanup below comes from the "Phase 1" initialization step. 1725 * (see tavor_rsrc_init_phase1() above) 1726 */ 1727 case TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE: 1728 /* Cleanup the interrupt "Out" mailbox list */ 1729 tavor_intr_outmbox_list_fini(state); 1730 /* FALLTHROUGH */ 1731 1732 case TAVOR_RSRC_CLEANUP_LEVEL3: 1733 /* Cleanup the "Out" mailbox resource pool */ 1734 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[ 1735 TAVOR_INTR_OUT_MBOX]; 1736 tavor_rsrc_mbox_fini(state, &mbox_info); 1737 /* FALLTHROUGH */ 1738 1739 case TAVOR_RSRC_CLEANUP_LEVEL2: 1740 /* Cleanup the "Out" mailbox list */ 1741 tavor_outmbox_list_fini(state); 1742 /* FALLTHROUGH */ 1743 1744 case TAVOR_RSRC_CLEANUP_LEVEL1: 1745 /* Cleanup the "Out" mailbox resource pool */ 1746 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_OUT_MBOX]; 1747 tavor_rsrc_mbox_fini(state, &mbox_info); 1748 /* FALLTHROUGH */ 1749 1750 case TAVOR_RSRC_CLEANUP_LEVEL0: 1751 /* Free the array of tavor_rsrc_pool_info_t's */ 1752 kmem_free(state->ts_rsrc_hdl, TAVOR_NUM_RESOURCES * 1753 sizeof (tavor_rsrc_pool_info_t)); 1754 kmem_cache_destroy(state->ts_rsrc_cache); 1755 break; 1756 1757 default: 1758 TAVOR_WARNING(state, "unexpected resource cleanup level"); 1759 TNF_PROBE_0(tavor_rsrc_fini_default_fail, TAVOR_TNF_ERROR, ""); 1760 TAVOR_TNF_EXIT(tavor_rsrc_fini); 1761 return; 1762 } 1763 1764 TAVOR_TNF_EXIT(tavor_rsrc_fini); 1765 } 1766 1767 1768 /* 1769 * tavor_rsrc_mbox_init() 1770 * Context: Only called from attach() path context 1771 */ 1772 static int 1773 tavor_rsrc_mbox_init(tavor_state_t *state, tavor_rsrc_mbox_info_t *info) 1774 { 1775 tavor_rsrc_pool_info_t *rsrc_pool; 1776 tavor_rsrc_priv_mbox_t *priv; 1777 vmem_t *vmp; 1778 uint64_t offset; 1779 uint_t dma_xfer_mode; 1780 1781 TAVOR_TNF_ENTER(tavor_rsrc_mbox_init); 1782 1783 ASSERT(state != NULL); 1784 ASSERT(info != NULL); 1785 1786 rsrc_pool = info->mbi_rsrcpool; 1787 ASSERT(rsrc_pool != NULL); 1788 1789 dma_xfer_mode = state->ts_cfg_profile->cp_streaming_consistent; 1790 1791 /* Allocate and initialize mailbox private structure */ 1792 priv = kmem_zalloc(sizeof (tavor_rsrc_priv_mbox_t), KM_SLEEP); 1793 priv->pmb_dip = state->ts_dip; 1794 priv->pmb_acchdl = state->ts_reg_ddrhdl; 1795 priv->pmb_devaccattr = state->ts_reg_accattr; 1796 priv->pmb_xfer_mode = dma_xfer_mode; 1797 1798 /* 1799 * Initialize many of the default DMA attributes. Then set alignment 1800 * and scatter-gather restrictions specific for mailbox memory. 1801 */ 1802 tavor_dma_attr_init(&priv->pmb_dmaattr); 1803 priv->pmb_dmaattr.dma_attr_align = TAVOR_MBOX_ALIGN; 1804 priv->pmb_dmaattr.dma_attr_sgllen = 1; 1805 1806 rsrc_pool->rsrc_private = priv; 1807 1808 /* Is object in DDR memory or system memory? */ 1809 if (rsrc_pool->rsrc_loc == TAVOR_IN_DDR) { 1810 rsrc_pool->rsrc_ddr_offset = vmem_xalloc(state->ts_ddrvmem, 1811 rsrc_pool->rsrc_pool_size, rsrc_pool->rsrc_align, 1812 0, 0, NULL, NULL, VM_SLEEP); 1813 if (rsrc_pool->rsrc_ddr_offset == NULL) { 1814 /* Unable to alloc space for mailboxes */ 1815 kmem_free(priv, sizeof (tavor_rsrc_priv_mbox_t)); 1816 TNF_PROBE_0(tavor_rsrc_mbox_init_vma_fail, 1817 TAVOR_TNF_ERROR, ""); 1818 TAVOR_TNF_EXIT(tavor_rsrc_mbox_init); 1819 return (DDI_FAILURE); 1820 } 1821 1822 /* Calculate offset and starting point (in DDR) */ 1823 offset = ((uintptr_t)rsrc_pool->rsrc_ddr_offset - 1824 state->ts_ddr.ddr_baseaddr); 1825 rsrc_pool->rsrc_start = 1826 (void *)(uintptr_t)((uintptr_t)state->ts_reg_ddr_baseaddr + 1827 offset); 1828 1829 /* Create new vmem arena for the mailboxes */ 1830 vmp = vmem_create(info->mbi_rsrcname, 1831 rsrc_pool->rsrc_ddr_offset, rsrc_pool->rsrc_pool_size, 1832 rsrc_pool->rsrc_quantum, NULL, NULL, NULL, 0, VM_SLEEP); 1833 if (vmp == NULL) { 1834 /* Unable to create vmem arena */ 1835 vmem_xfree(state->ts_ddrvmem, 1836 rsrc_pool->rsrc_ddr_offset, 1837 rsrc_pool->rsrc_pool_size); 1838 kmem_free(priv, sizeof (tavor_rsrc_priv_mbox_t)); 1839 TNF_PROBE_0(tavor_rsrc_mbox_init_vmem_create_fail, 1840 TAVOR_TNF_ERROR, ""); 1841 TAVOR_TNF_EXIT(tavor_rsrc_mbox_init); 1842 return (DDI_FAILURE); 1843 } 1844 rsrc_pool->rsrc_vmp = vmp; 1845 } else { 1846 rsrc_pool->rsrc_ddr_offset = NULL; 1847 rsrc_pool->rsrc_start = NULL; 1848 rsrc_pool->rsrc_vmp = NULL; 1849 } 1850 1851 TAVOR_TNF_EXIT(tavor_rsrc_mbox_init); 1852 return (DDI_SUCCESS); 1853 } 1854 1855 1856 /* 1857 * tavor_rsrc_mbox_fini() 1858 * Context: Only called from attach() and/or detach() path contexts 1859 */ 1860 static void 1861 tavor_rsrc_mbox_fini(tavor_state_t *state, tavor_rsrc_mbox_info_t *info) 1862 { 1863 tavor_rsrc_pool_info_t *rsrc_pool; 1864 1865 TAVOR_TNF_ENTER(tavor_rsrc_mbox_fini); 1866 1867 ASSERT(state != NULL); 1868 ASSERT(info != NULL); 1869 1870 rsrc_pool = info->mbi_rsrcpool; 1871 ASSERT(rsrc_pool != NULL); 1872 1873 /* If mailboxes are DDR memory, then destroy and free up vmem */ 1874 if (rsrc_pool->rsrc_loc == TAVOR_IN_DDR) { 1875 1876 /* Destroy the specially created mbox vmem arena */ 1877 vmem_destroy(rsrc_pool->rsrc_vmp); 1878 1879 /* Free up the region from the ddr_vmem arena */ 1880 vmem_xfree(state->ts_ddrvmem, rsrc_pool->rsrc_ddr_offset, 1881 rsrc_pool->rsrc_pool_size); 1882 } 1883 1884 /* Free up the private struct */ 1885 kmem_free(rsrc_pool->rsrc_private, sizeof (tavor_rsrc_priv_mbox_t)); 1886 1887 TAVOR_TNF_EXIT(tavor_rsrc_mbox_fini); 1888 } 1889 1890 1891 /* 1892 * tavor_rsrc_hw_entries_init() 1893 * Context: Only called from attach() path context 1894 */ 1895 static int 1896 tavor_rsrc_hw_entries_init(tavor_state_t *state, 1897 tavor_rsrc_hw_entry_info_t *info) 1898 { 1899 tavor_rsrc_pool_info_t *rsrc_pool; 1900 tavor_rsrc_t *rsvd_rsrc = NULL; 1901 vmem_t *vmp; 1902 uint64_t num_hwentry, max_hwentry, num_prealloc; 1903 uint64_t offset; 1904 int status; 1905 1906 TAVOR_TNF_ENTER(tavor_rsrc_hw_entries_init); 1907 1908 ASSERT(state != NULL); 1909 ASSERT(info != NULL); 1910 1911 rsrc_pool = info->hwi_rsrcpool; 1912 ASSERT(rsrc_pool != NULL); 1913 num_hwentry = info->hwi_num; 1914 max_hwentry = info->hwi_max; 1915 num_prealloc = info->hwi_prealloc; 1916 1917 /* Make sure number of HW entries makes sense */ 1918 if (num_hwentry > max_hwentry) { 1919 TNF_PROBE_2(tavor_rsrc_hw_entries_init_toomany_fail, 1920 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of HW " 1921 "entries exceeds device maximum", tnf_uint, maxhw, 1922 max_hwentry); 1923 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init); 1924 return (DDI_FAILURE); 1925 } 1926 1927 /* 1928 * Determine if we need to allocate DDR space to set up the 1929 * "rsrc_start" pointer. Not necessary if "rsrc_start" has already 1930 * been initialized (as is the case for the UAR page init). 1931 */ 1932 if (rsrc_pool->rsrc_start == NULL) { 1933 /* Make sure HW entries table is aligned as specified */ 1934 rsrc_pool->rsrc_ddr_offset = vmem_xalloc(state->ts_ddrvmem, 1935 rsrc_pool->rsrc_pool_size, rsrc_pool->rsrc_align, 1936 0, 0, NULL, NULL, VM_NOSLEEP | VM_FIRSTFIT); 1937 if (rsrc_pool->rsrc_ddr_offset == NULL) { 1938 /* Unable to alloc space for aligned HW table */ 1939 TNF_PROBE_0(tavor_rsrc_hw_entry_table_vmxalloc_fail, 1940 TAVOR_TNF_ERROR, ""); 1941 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init); 1942 return (DDI_FAILURE); 1943 } 1944 1945 /* Calculate offset and starting point (in DDR) */ 1946 offset = ((uintptr_t)rsrc_pool->rsrc_ddr_offset - 1947 state->ts_ddr.ddr_baseaddr); 1948 rsrc_pool->rsrc_start = 1949 (void *)(uintptr_t)((uintptr_t)state->ts_reg_ddr_baseaddr + 1950 offset); 1951 } else { 1952 rsrc_pool->rsrc_ddr_offset = rsrc_pool->rsrc_start; 1953 } 1954 1955 /* 1956 * Create new vmem arena for the HW entries table (if rsrc_quantum 1957 * is non-zero). Otherwise if rsrc_quantum is zero, then these HW 1958 * entries are not going to be dynamically allocatable (i.e. they 1959 * won't be allocated/freed through tavor_rsrc_alloc/free). This 1960 * latter option is used for EQPC and UARSCR resource which are, in 1961 * fact, managed by the Tavor hardware. 1962 */ 1963 if (rsrc_pool->rsrc_quantum != 0) { 1964 vmp = vmem_create(info->hwi_rsrcname, 1965 rsrc_pool->rsrc_ddr_offset, rsrc_pool->rsrc_pool_size, 1966 rsrc_pool->rsrc_quantum, NULL, NULL, NULL, 0, VM_SLEEP); 1967 if (vmp == NULL) { 1968 /* Unable to create vmem arena */ 1969 if (rsrc_pool->rsrc_ddr_offset != 1970 rsrc_pool->rsrc_start) { 1971 vmem_xfree(state->ts_ddrvmem, 1972 rsrc_pool->rsrc_ddr_offset, 1973 rsrc_pool->rsrc_pool_size); 1974 } 1975 TNF_PROBE_0(tavor_rsrc_hw_entries_init_vmemcreate_fail, 1976 TAVOR_TNF_ERROR, ""); 1977 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init); 1978 return (DDI_FAILURE); 1979 } 1980 rsrc_pool->rsrc_vmp = vmp; 1981 } else { 1982 rsrc_pool->rsrc_vmp = NULL; 1983 } 1984 1985 /* The first HW entries may be reserved by Tavor firmware */ 1986 if (num_prealloc != 0) { 1987 status = tavor_rsrc_alloc(state, rsrc_pool->rsrc_type, 1988 num_prealloc, TAVOR_SLEEP, &rsvd_rsrc); 1989 if (status != DDI_SUCCESS) { 1990 /* Unable to preallocate the reserved HW entries */ 1991 if (rsrc_pool->rsrc_vmp != NULL) { 1992 vmem_destroy(rsrc_pool->rsrc_vmp); 1993 } 1994 if (rsrc_pool->rsrc_ddr_offset != 1995 rsrc_pool->rsrc_start) { 1996 vmem_xfree(state->ts_ddrvmem, 1997 rsrc_pool->rsrc_ddr_offset, 1998 rsrc_pool->rsrc_pool_size); 1999 } 2000 TNF_PROBE_0(tavor_rsrc_hw_entries_init_pre_fail, 2001 TAVOR_TNF_ERROR, ""); 2002 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init); 2003 return (DDI_FAILURE); 2004 } 2005 } 2006 rsrc_pool->rsrc_private = rsvd_rsrc; 2007 2008 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init); 2009 return (DDI_SUCCESS); 2010 } 2011 2012 2013 /* 2014 * tavor_rsrc_hw_entries_fini() 2015 * Context: Only called from attach() and/or detach() path contexts 2016 */ 2017 static void 2018 tavor_rsrc_hw_entries_fini(tavor_state_t *state, 2019 tavor_rsrc_hw_entry_info_t *info) 2020 { 2021 tavor_rsrc_pool_info_t *rsrc_pool; 2022 tavor_rsrc_t *rsvd_rsrc; 2023 2024 TAVOR_TNF_ENTER(tavor_rsrc_hw_entries_fini); 2025 2026 ASSERT(state != NULL); 2027 ASSERT(info != NULL); 2028 2029 rsrc_pool = info->hwi_rsrcpool; 2030 ASSERT(rsrc_pool != NULL); 2031 2032 /* Free up any "reserved" (i.e. preallocated) HW entries */ 2033 rsvd_rsrc = (tavor_rsrc_t *)rsrc_pool->rsrc_private; 2034 if (rsvd_rsrc != NULL) { 2035 tavor_rsrc_free(state, &rsvd_rsrc); 2036 } 2037 2038 /* 2039 * If we've actually setup a vmem arena for the HW entries, then 2040 * destroy it now 2041 */ 2042 if (rsrc_pool->rsrc_vmp != NULL) { 2043 vmem_destroy(rsrc_pool->rsrc_vmp); 2044 } 2045 2046 /* 2047 * Determine if a region was allocated from the tavor_ddr_vmem 2048 * arena (and free it up if necessary) 2049 */ 2050 if (rsrc_pool->rsrc_ddr_offset != rsrc_pool->rsrc_start) { 2051 vmem_xfree(state->ts_ddrvmem, rsrc_pool->rsrc_ddr_offset, 2052 rsrc_pool->rsrc_pool_size); 2053 } 2054 2055 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_fini); 2056 } 2057 2058 2059 /* 2060 * tavor_rsrc_sw_handles_init() 2061 * Context: Only called from attach() path context 2062 */ 2063 /* ARGSUSED */ 2064 static int 2065 tavor_rsrc_sw_handles_init(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info) 2066 { 2067 tavor_rsrc_pool_info_t *rsrc_pool; 2068 uint64_t num_swhdl, max_swhdl, prealloc_sz; 2069 2070 TAVOR_TNF_ENTER(tavor_rsrc_sw_handles_init); 2071 2072 ASSERT(state != NULL); 2073 ASSERT(info != NULL); 2074 2075 rsrc_pool = info->swi_rsrcpool; 2076 ASSERT(rsrc_pool != NULL); 2077 num_swhdl = info->swi_num; 2078 max_swhdl = info->swi_max; 2079 prealloc_sz = info->swi_prealloc_sz; 2080 2081 /* Make sure number of SW handles makes sense */ 2082 if (num_swhdl > max_swhdl) { 2083 TNF_PROBE_2(tavor_rsrc_sw_handles_init_toomany_fail, 2084 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of SW " 2085 "handles exceeds maximum", tnf_uint, maxsw, max_swhdl); 2086 TAVOR_TNF_EXIT(tavor_rsrc_sw_handles_init); 2087 return (DDI_FAILURE); 2088 } 2089 2090 /* 2091 * Depending on the flags parameter, create a kmem_cache for some 2092 * number of software handle structures. Note: kmem_cache_create() 2093 * will SLEEP until successful. 2094 */ 2095 if (info->swi_flags & TAVOR_SWHDL_KMEMCACHE_INIT) { 2096 rsrc_pool->rsrc_private = kmem_cache_create( 2097 info->swi_rsrcname, rsrc_pool->rsrc_quantum, 0, 2098 info->swi_constructor, info->swi_destructor, NULL, 2099 rsrc_pool->rsrc_state, NULL, 0); 2100 } 2101 2102 /* Allocate the central list of SW handle pointers */ 2103 if (info->swi_flags & TAVOR_SWHDL_TABLE_INIT) { 2104 info->swi_table_ptr = kmem_zalloc(num_swhdl * prealloc_sz, 2105 KM_SLEEP); 2106 } 2107 2108 TAVOR_TNF_EXIT(tavor_rsrc_sw_handles_init); 2109 return (DDI_SUCCESS); 2110 } 2111 2112 2113 /* 2114 * tavor_rsrc_sw_handles_fini() 2115 * Context: Only called from attach() and/or detach() path contexts 2116 */ 2117 /* ARGSUSED */ 2118 static void 2119 tavor_rsrc_sw_handles_fini(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info) 2120 { 2121 tavor_rsrc_pool_info_t *rsrc_pool; 2122 uint64_t num_swhdl, prealloc_sz; 2123 2124 TAVOR_TNF_ENTER(tavor_rsrc_sw_handles_fini); 2125 2126 ASSERT(state != NULL); 2127 ASSERT(info != NULL); 2128 2129 rsrc_pool = info->swi_rsrcpool; 2130 num_swhdl = info->swi_num; 2131 prealloc_sz = info->swi_prealloc_sz; 2132 2133 /* 2134 * If a "software handle" kmem_cache exists for this resource, then 2135 * destroy it now 2136 */ 2137 if (rsrc_pool != NULL) { 2138 kmem_cache_destroy(rsrc_pool->rsrc_private); 2139 } 2140 2141 /* Free up this central list of SW handle pointers */ 2142 if (info->swi_table_ptr != NULL) { 2143 kmem_free(info->swi_table_ptr, num_swhdl * prealloc_sz); 2144 } 2145 2146 TAVOR_TNF_EXIT(tavor_rsrc_sw_handles_fini); 2147 } 2148 2149 2150 /* 2151 * tavor_rsrc_pd_handles_init() 2152 * Context: Only called from attach() path context 2153 */ 2154 static int 2155 tavor_rsrc_pd_handles_init(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info) 2156 { 2157 tavor_rsrc_pool_info_t *rsrc_pool; 2158 vmem_t *vmp; 2159 char vmem_name[TAVOR_RSRC_NAME_MAXLEN]; 2160 int status; 2161 2162 TAVOR_TNF_ENTER(tavor_rsrc_pd_handles_init); 2163 2164 ASSERT(state != NULL); 2165 ASSERT(info != NULL); 2166 2167 rsrc_pool = info->swi_rsrcpool; 2168 ASSERT(rsrc_pool != NULL); 2169 2170 /* Initialize the resource pool for software handle table */ 2171 status = tavor_rsrc_sw_handles_init(state, info); 2172 if (status != DDI_SUCCESS) { 2173 TNF_PROBE_0(tavor_rsrc_pdhdl_alloc_fail, TAVOR_TNF_ERROR, ""); 2174 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc); 2175 return (DDI_FAILURE); 2176 } 2177 2178 /* Build vmem arena name from Tavor instance */ 2179 TAVOR_RSRC_NAME(vmem_name, TAVOR_PDHDL_VMEM); 2180 2181 /* Create new vmem arena for PD numbers */ 2182 vmp = vmem_create(vmem_name, (caddr_t)1, info->swi_num, 1, NULL, 2183 NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER); 2184 if (vmp == NULL) { 2185 /* Unable to create vmem arena */ 2186 info->swi_table_ptr = NULL; 2187 tavor_rsrc_sw_handles_fini(state, info); 2188 TNF_PROBE_0(tavor_rsrc_pd_handles_init_vmem_create_fail, 2189 TAVOR_TNF_ERROR, ""); 2190 TAVOR_TNF_EXIT(tavor_rsrc_pd_handles_init); 2191 return (DDI_FAILURE); 2192 } 2193 rsrc_pool->rsrc_vmp = vmp; 2194 2195 TAVOR_TNF_EXIT(tavor_rsrc_pd_handles_init); 2196 return (DDI_SUCCESS); 2197 } 2198 2199 2200 /* 2201 * tavor_rsrc_pd_handles_fini() 2202 * Context: Only called from attach() and/or detach() path contexts 2203 */ 2204 static void 2205 tavor_rsrc_pd_handles_fini(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info) 2206 { 2207 tavor_rsrc_pool_info_t *rsrc_pool; 2208 2209 TAVOR_TNF_ENTER(tavor_rsrc_pd_handles_fini); 2210 2211 ASSERT(state != NULL); 2212 ASSERT(info != NULL); 2213 2214 rsrc_pool = info->swi_rsrcpool; 2215 2216 /* Destroy the specially created UAR scratch table vmem arena */ 2217 vmem_destroy(rsrc_pool->rsrc_vmp); 2218 2219 /* Destroy the "tavor_sw_pd_t" kmem_cache */ 2220 tavor_rsrc_sw_handles_fini(state, info); 2221 2222 TAVOR_TNF_EXIT(tavor_rsrc_pd_handles_fini); 2223 } 2224 2225 2226 /* 2227 * tavor_rsrc_mbox_alloc() 2228 * Context: Only called from attach() path context 2229 */ 2230 static int 2231 tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t num, 2232 tavor_rsrc_t *hdl) 2233 { 2234 tavor_rsrc_priv_mbox_t *priv; 2235 void *addr; 2236 caddr_t kaddr; 2237 uint64_t offset; 2238 size_t real_len, temp_len; 2239 int status; 2240 2241 TAVOR_TNF_ENTER(tavor_rsrc_mbox_alloc); 2242 2243 ASSERT(pool_info != NULL); 2244 ASSERT(hdl != NULL); 2245 2246 /* Get the private pointer for the mailboxes */ 2247 priv = pool_info->rsrc_private; 2248 ASSERT(priv != NULL); 2249 2250 /* 2251 * Allocate a DMA handle for the mailbox. This will be used for 2252 * two purposes (potentially). First, it could be used below in 2253 * the call to ddi_dma_mem_alloc() - if the mailbox is to come from 2254 * system memory. Second, it is definitely used later to bind 2255 * the mailbox for DMA access from/by the hardware. 2256 */ 2257 status = ddi_dma_alloc_handle(priv->pmb_dip, &priv->pmb_dmaattr, 2258 DDI_DMA_SLEEP, NULL, &hdl->tr_dmahdl); 2259 if (status != DDI_SUCCESS) { 2260 TNF_PROBE_1(tavor_rsrc_mbox_alloc_dmahdl_fail, TAVOR_TNF_ERROR, 2261 "", tnf_uint, status, status); 2262 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc); 2263 return (DDI_FAILURE); 2264 } 2265 2266 /* Is mailbox in DDR memory or system memory? */ 2267 if (pool_info->rsrc_loc == TAVOR_IN_DDR) { 2268 /* Use vmem_alloc() to get DDR address of mbox */ 2269 hdl->tr_len = (num * pool_info->rsrc_quantum); 2270 addr = vmem_alloc(pool_info->rsrc_vmp, hdl->tr_len, 2271 VM_SLEEP); 2272 if (addr == NULL) { 2273 /* No more DDR available for mailbox entries */ 2274 ddi_dma_free_handle(&hdl->tr_dmahdl); 2275 TNF_PROBE_0(tavor_rsrc_mbox_alloc_vma_fail, 2276 TAVOR_TNF_ERROR, ""); 2277 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc); 2278 return (DDI_FAILURE); 2279 } 2280 hdl->tr_acchdl = priv->pmb_acchdl; 2281 2282 /* Calculate kernel virtual address (from the DDR offset) */ 2283 offset = ((uintptr_t)addr - 2284 (uintptr_t)pool_info->rsrc_ddr_offset); 2285 hdl->tr_addr = (void *)(uintptr_t)(offset + 2286 (uintptr_t)pool_info->rsrc_start); 2287 2288 } else { /* TAVOR_IN_SYSMEM */ 2289 2290 /* Use ddi_dma_mem_alloc() to get memory for mailbox */ 2291 temp_len = (num * pool_info->rsrc_quantum); 2292 status = ddi_dma_mem_alloc(hdl->tr_dmahdl, temp_len, 2293 &priv->pmb_devaccattr, priv->pmb_xfer_mode, DDI_DMA_SLEEP, 2294 NULL, &kaddr, &real_len, &hdl->tr_acchdl); 2295 if (status != DDI_SUCCESS) { 2296 /* No more sys memory available for mailbox entries */ 2297 ddi_dma_free_handle(&hdl->tr_dmahdl); 2298 TNF_PROBE_0(tavor_rsrc_mbox_alloc_dma_memalloc_fail, 2299 TAVOR_TNF_ERROR, ""); 2300 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc); 2301 return (DDI_FAILURE); 2302 } 2303 hdl->tr_addr = (void *)kaddr; 2304 hdl->tr_len = real_len; 2305 } 2306 2307 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc); 2308 return (DDI_SUCCESS); 2309 } 2310 2311 2312 /* 2313 * tavor_rsrc_mbox_free() 2314 * Context: Can be called from interrupt or base context. 2315 */ 2316 static void 2317 tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl) 2318 { 2319 void *addr; 2320 uint64_t offset; 2321 2322 TAVOR_TNF_ENTER(tavor_rsrc_mbox_free); 2323 2324 ASSERT(pool_info != NULL); 2325 ASSERT(hdl != NULL); 2326 2327 /* Is mailbox in DDR memory or system memory? */ 2328 if (pool_info->rsrc_loc == TAVOR_IN_DDR) { 2329 2330 /* Calculate the allocated address (the mbox's DDR offset) */ 2331 offset = ((uintptr_t)hdl->tr_addr - 2332 (uintptr_t)pool_info->rsrc_start); 2333 addr = (void *)(uintptr_t)(offset + 2334 (uintptr_t)pool_info->rsrc_ddr_offset); 2335 2336 /* Use vmem_free() to free up DDR memory for mailbox */ 2337 vmem_free(pool_info->rsrc_vmp, addr, hdl->tr_len); 2338 2339 } else { /* TAVOR_IN_SYSMEM */ 2340 2341 /* Use ddi_dma_mem_free() to free up sys memory for mailbox */ 2342 ddi_dma_mem_free(&hdl->tr_acchdl); 2343 } 2344 2345 /* Free the DMA handle for the mailbox */ 2346 ddi_dma_free_handle(&hdl->tr_dmahdl); 2347 2348 TAVOR_TNF_EXIT(tavor_rsrc_mbox_free); 2349 } 2350 2351 2352 /* 2353 * tavor_rsrc_hw_entry_alloc() 2354 * Context: Can be called from interrupt or base context. 2355 */ 2356 static int 2357 tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t num, 2358 uint_t num_align, ddi_acc_handle_t acc_handle, uint_t sleepflag, 2359 tavor_rsrc_t *hdl) 2360 { 2361 void *addr; 2362 uint64_t offset; 2363 uint32_t align; 2364 int flag; 2365 2366 TAVOR_TNF_ENTER(tavor_rsrc_hw_entry_alloc); 2367 2368 ASSERT(pool_info != NULL); 2369 ASSERT(hdl != NULL); 2370 2371 /* 2372 * Tavor hardware entries (QPC, CQC, EQC, MPT, MTT, etc.) do not 2373 * use dma_handle (because they are in Tavor locally attached DDR 2374 * memory) and, generally, don't use the acc_handle (because the 2375 * entries are not directly accessed by software). The exceptions 2376 * to this rule are the UARPG and UDAV entries. 2377 */ 2378 2379 /* 2380 * Use vmem_xalloc() to get a properly aligned pointer (based on 2381 * the number requested) to the HW entry(ies). This handles the 2382 * cases (for special QPCs and for RDB entries) where we need more 2383 * than one and need to ensure that they are properly aligned. 2384 */ 2385 flag = (sleepflag == TAVOR_SLEEP) ? VM_SLEEP : VM_NOSLEEP; 2386 hdl->tr_len = (num * pool_info->rsrc_quantum); 2387 align = (num_align * pool_info->rsrc_quantum); 2388 addr = vmem_xalloc(pool_info->rsrc_vmp, hdl->tr_len, 2389 align, 0, 0, NULL, NULL, flag | VM_FIRSTFIT); 2390 if (addr == NULL) { 2391 /* No more HW entries available */ 2392 TNF_PROBE_0(tavor_rsrc_hw_entry_alloc_vmxa_fail, 2393 TAVOR_TNF_ERROR, ""); 2394 TAVOR_TNF_EXIT(tavor_rsrc_hw_entry_alloc); 2395 return (DDI_FAILURE); 2396 } 2397 2398 /* If an access handle was provided, fill it in */ 2399 if (acc_handle != 0) { 2400 hdl->tr_acchdl = acc_handle; 2401 } 2402 2403 /* Calculate vaddr and HW table index (from the DDR offset) */ 2404 offset = ((uintptr_t)addr - (uintptr_t)pool_info->rsrc_ddr_offset); 2405 hdl->tr_addr = (void *)(uintptr_t)(offset + 2406 (uintptr_t)pool_info->rsrc_start); 2407 hdl->tr_indx = (offset >> pool_info->rsrc_shift); 2408 2409 TAVOR_TNF_EXIT(tavor_rsrc_hw_entry_alloc); 2410 return (DDI_SUCCESS); 2411 } 2412 2413 2414 /* 2415 * tavor_rsrc_hw_entry_free() 2416 * Context: Can be called from interrupt or base context. 2417 */ 2418 static void 2419 tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl) 2420 { 2421 void *addr; 2422 uint64_t offset; 2423 2424 TAVOR_TNF_ENTER(tavor_rsrc_hw_entry_free); 2425 2426 ASSERT(pool_info != NULL); 2427 ASSERT(hdl != NULL); 2428 2429 /* Calculate the allocated address (the entry's DDR offset) */ 2430 offset = ((uintptr_t)hdl->tr_addr - (uintptr_t)pool_info->rsrc_start); 2431 addr = (void *)(uintptr_t)(offset + 2432 (uintptr_t)pool_info->rsrc_ddr_offset); 2433 2434 /* Use vmem_xfree() to free up the HW table entry */ 2435 vmem_xfree(pool_info->rsrc_vmp, addr, hdl->tr_len); 2436 2437 TAVOR_TNF_EXIT(tavor_rsrc_hw_entry_free); 2438 } 2439 2440 2441 /* 2442 * tavor_rsrc_swhdl_alloc() 2443 * Context: Can be called from interrupt or base context. 2444 */ 2445 static int 2446 tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t sleepflag, 2447 tavor_rsrc_t *hdl) 2448 { 2449 void *addr; 2450 int flag; 2451 2452 TAVOR_TNF_ENTER(tavor_rsrc_swhdl_alloc); 2453 2454 ASSERT(pool_info != NULL); 2455 ASSERT(hdl != NULL); 2456 2457 /* Allocate the software handle structure */ 2458 flag = (sleepflag == TAVOR_SLEEP) ? KM_SLEEP : KM_NOSLEEP; 2459 addr = kmem_cache_alloc(pool_info->rsrc_private, flag); 2460 if (addr == NULL) { 2461 TNF_PROBE_0(tavor_rsrc_swhdl_alloc_kmca_fail, TAVOR_TNF_ERROR, 2462 ""); 2463 TAVOR_TNF_EXIT(tavor_rsrc_swhdl_alloc); 2464 return (DDI_FAILURE); 2465 } 2466 hdl->tr_len = pool_info->rsrc_quantum; 2467 hdl->tr_addr = addr; 2468 2469 TAVOR_TNF_EXIT(tavor_rsrc_swhdl_alloc); 2470 return (DDI_SUCCESS); 2471 } 2472 2473 2474 /* 2475 * tavor_rsrc_swhdl_free() 2476 * Context: Can be called from interrupt or base context. 2477 */ 2478 static void 2479 tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl) 2480 { 2481 TAVOR_TNF_ENTER(tavor_rsrc_swhdl_free); 2482 2483 ASSERT(pool_info != NULL); 2484 ASSERT(hdl != NULL); 2485 2486 /* Free the software handle structure */ 2487 kmem_cache_free(pool_info->rsrc_private, hdl->tr_addr); 2488 2489 TAVOR_TNF_EXIT(tavor_rsrc_swhdl_free); 2490 } 2491 2492 2493 /* 2494 * tavor_rsrc_pdhdl_alloc() 2495 * Context: Can be called from interrupt or base context. 2496 */ 2497 static int 2498 tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t sleepflag, 2499 tavor_rsrc_t *hdl) 2500 { 2501 tavor_pdhdl_t addr; 2502 void *tmpaddr; 2503 int flag, status; 2504 2505 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_alloc); 2506 2507 ASSERT(pool_info != NULL); 2508 ASSERT(hdl != NULL); 2509 2510 /* Allocate the software handle */ 2511 status = tavor_rsrc_swhdl_alloc(pool_info, sleepflag, hdl); 2512 if (status != DDI_SUCCESS) { 2513 TNF_PROBE_0(tavor_rsrc_pdhdl_alloc_fail, TAVOR_TNF_ERROR, ""); 2514 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc); 2515 return (DDI_FAILURE); 2516 } 2517 addr = (tavor_pdhdl_t)hdl->tr_addr; 2518 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*addr)) 2519 2520 /* Allocate a PD number for the handle */ 2521 flag = (sleepflag == TAVOR_SLEEP) ? VM_SLEEP : VM_NOSLEEP; 2522 tmpaddr = vmem_alloc(pool_info->rsrc_vmp, 1, flag); 2523 if (tmpaddr == NULL) { 2524 /* No more PD number entries available */ 2525 tavor_rsrc_swhdl_free(pool_info, hdl); 2526 TNF_PROBE_0(tavor_rsrc_pdhdl_alloc_vma_fail, 2527 TAVOR_TNF_ERROR, ""); 2528 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc); 2529 return (DDI_FAILURE); 2530 } 2531 addr->pd_pdnum = (uint32_t)(uintptr_t)tmpaddr; 2532 addr->pd_rsrcp = hdl; 2533 hdl->tr_indx = addr->pd_pdnum; 2534 2535 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc); 2536 return (DDI_SUCCESS); 2537 } 2538 2539 2540 /* 2541 * tavor_rsrc_pdhdl_free() 2542 * Context: Can be called from interrupt or base context. 2543 */ 2544 static void 2545 tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl) 2546 { 2547 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_free); 2548 2549 ASSERT(pool_info != NULL); 2550 ASSERT(hdl != NULL); 2551 2552 /* Use vmem_free() to free up the PD number */ 2553 vmem_free(pool_info->rsrc_vmp, (void *)(uintptr_t)hdl->tr_indx, 1); 2554 2555 /* Free the software handle structure */ 2556 tavor_rsrc_swhdl_free(pool_info, hdl); 2557 2558 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_free); 2559 } 2560 2561 2562 /* 2563 * tavor_rsrc_pdhdl_constructor() 2564 * Context: Can be called from interrupt or base context. 2565 */ 2566 /* ARGSUSED */ 2567 static int 2568 tavor_rsrc_pdhdl_constructor(void *pd, void *priv, int flags) 2569 { 2570 tavor_pdhdl_t pdhdl; 2571 tavor_state_t *state; 2572 2573 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_constructor); 2574 2575 pdhdl = (tavor_pdhdl_t)pd; 2576 state = (tavor_state_t *)priv; 2577 2578 mutex_init(&pdhdl->pd_lock, NULL, MUTEX_DRIVER, 2579 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2580 2581 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_constructor); 2582 return (DDI_SUCCESS); 2583 } 2584 2585 2586 /* 2587 * tavor_rsrc_pdhdl_destructor() 2588 * Context: Can be called from interrupt or base context. 2589 */ 2590 /* ARGSUSED */ 2591 static void 2592 tavor_rsrc_pdhdl_destructor(void *pd, void *priv) 2593 { 2594 tavor_pdhdl_t pdhdl; 2595 2596 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_destructor); 2597 2598 pdhdl = (tavor_pdhdl_t)pd; 2599 2600 mutex_destroy(&pdhdl->pd_lock); 2601 2602 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_destructor); 2603 } 2604 2605 2606 /* 2607 * tavor_rsrc_cqhdl_constructor() 2608 * Context: Can be called from interrupt or base context. 2609 */ 2610 /* ARGSUSED */ 2611 static int 2612 tavor_rsrc_cqhdl_constructor(void *cq, void *priv, int flags) 2613 { 2614 tavor_cqhdl_t cqhdl; 2615 tavor_state_t *state; 2616 2617 TAVOR_TNF_ENTER(tavor_rsrc_cqhdl_constructor); 2618 2619 cqhdl = (tavor_cqhdl_t)cq; 2620 state = (tavor_state_t *)priv; 2621 2622 mutex_init(&cqhdl->cq_lock, NULL, MUTEX_DRIVER, 2623 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2624 mutex_init(&cqhdl->cq_wrid_wqhdr_lock, NULL, MUTEX_DRIVER, 2625 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2626 2627 TAVOR_TNF_EXIT(tavor_rsrc_cqhdl_constructor); 2628 return (DDI_SUCCESS); 2629 } 2630 2631 2632 /* 2633 * tavor_rsrc_cqhdl_destructor() 2634 * Context: Can be called from interrupt or base context. 2635 */ 2636 /* ARGSUSED */ 2637 static void 2638 tavor_rsrc_cqhdl_destructor(void *cq, void *priv) 2639 { 2640 tavor_cqhdl_t cqhdl; 2641 2642 TAVOR_TNF_ENTER(tavor_rsrc_cqhdl_destructor); 2643 2644 cqhdl = (tavor_cqhdl_t)cq; 2645 2646 mutex_destroy(&cqhdl->cq_wrid_wqhdr_lock); 2647 mutex_destroy(&cqhdl->cq_lock); 2648 2649 TAVOR_TNF_EXIT(tavor_rsrc_cqhdl_destructor); 2650 } 2651 2652 2653 /* 2654 * tavor_rsrc_qphdl_constructor() 2655 * Context: Can be called from interrupt or base context. 2656 */ 2657 /* ARGSUSED */ 2658 static int 2659 tavor_rsrc_qphdl_constructor(void *qp, void *priv, int flags) 2660 { 2661 tavor_qphdl_t qphdl; 2662 tavor_state_t *state; 2663 2664 TAVOR_TNF_ENTER(tavor_rsrc_qphdl_constructor); 2665 2666 qphdl = (tavor_qphdl_t)qp; 2667 state = (tavor_state_t *)priv; 2668 2669 mutex_init(&qphdl->qp_lock, NULL, MUTEX_DRIVER, 2670 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2671 2672 TAVOR_TNF_EXIT(tavor_rsrc_qphdl_constructor); 2673 return (DDI_SUCCESS); 2674 } 2675 2676 2677 /* 2678 * tavor_rsrc_qphdl_destructor() 2679 * Context: Can be called from interrupt or base context. 2680 */ 2681 /* ARGSUSED */ 2682 static void 2683 tavor_rsrc_qphdl_destructor(void *qp, void *priv) 2684 { 2685 tavor_qphdl_t qphdl; 2686 2687 TAVOR_TNF_ENTER(tavor_rsrc_qphdl_destructor); 2688 2689 qphdl = (tavor_qphdl_t)qp; 2690 2691 mutex_destroy(&qphdl->qp_lock); 2692 2693 TAVOR_TNF_ENTER(tavor_rsrc_qphdl_destructor); 2694 } 2695 2696 2697 /* 2698 * tavor_rsrc_srqhdl_constructor() 2699 * Context: Can be called from interrupt or base context. 2700 */ 2701 /* ARGSUSED */ 2702 static int 2703 tavor_rsrc_srqhdl_constructor(void *srq, void *priv, int flags) 2704 { 2705 tavor_srqhdl_t srqhdl; 2706 tavor_state_t *state; 2707 2708 TAVOR_TNF_ENTER(tavor_rsrc_srqhdl_constructor); 2709 2710 srqhdl = (tavor_srqhdl_t)srq; 2711 state = (tavor_state_t *)priv; 2712 2713 mutex_init(&srqhdl->srq_lock, NULL, MUTEX_DRIVER, 2714 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2715 2716 TAVOR_TNF_EXIT(tavor_rsrc_srqhdl_constructor); 2717 return (DDI_SUCCESS); 2718 } 2719 2720 2721 /* 2722 * tavor_rsrc_srqhdl_destructor() 2723 * Context: Can be called from interrupt or base context. 2724 */ 2725 /* ARGSUSED */ 2726 static void 2727 tavor_rsrc_srqhdl_destructor(void *srq, void *priv) 2728 { 2729 tavor_srqhdl_t srqhdl; 2730 2731 TAVOR_TNF_ENTER(tavor_rsrc_srqhdl_destructor); 2732 2733 srqhdl = (tavor_srqhdl_t)srq; 2734 2735 mutex_destroy(&srqhdl->srq_lock); 2736 2737 TAVOR_TNF_EXIT(tavor_rsrc_srqhdl_destructor); 2738 } 2739 2740 2741 /* 2742 * tavor_rsrc_refcnt_constructor() 2743 * Context: Can be called from interrupt or base context. 2744 */ 2745 /* ARGSUSED */ 2746 static int 2747 tavor_rsrc_refcnt_constructor(void *rc, void *priv, int flags) 2748 { 2749 tavor_sw_refcnt_t *refcnt; 2750 tavor_state_t *state; 2751 2752 TAVOR_TNF_ENTER(tavor_rsrc_refcnt_constructor); 2753 2754 refcnt = (tavor_sw_refcnt_t *)rc; 2755 state = (tavor_state_t *)priv; 2756 2757 mutex_init(&refcnt->swrc_lock, NULL, MUTEX_DRIVER, 2758 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2759 2760 TAVOR_TNF_EXIT(tavor_rsrc_refcnt_constructor); 2761 return (DDI_SUCCESS); 2762 } 2763 2764 2765 /* 2766 * tavor_rsrc_refcnt_destructor() 2767 * Context: Can be called from interrupt or base context. 2768 */ 2769 /* ARGSUSED */ 2770 static void 2771 tavor_rsrc_refcnt_destructor(void *rc, void *priv) 2772 { 2773 tavor_sw_refcnt_t *refcnt; 2774 2775 TAVOR_TNF_ENTER(tavor_rsrc_refcnt_destructor); 2776 2777 refcnt = (tavor_sw_refcnt_t *)rc; 2778 2779 mutex_destroy(&refcnt->swrc_lock); 2780 2781 TAVOR_TNF_ENTER(tavor_rsrc_refcnt_destructor); 2782 } 2783 2784 2785 /* 2786 * tavor_rsrc_ahhdl_constructor() 2787 * Context: Can be called from interrupt or base context. 2788 */ 2789 /* ARGSUSED */ 2790 static int 2791 tavor_rsrc_ahhdl_constructor(void *ah, void *priv, int flags) 2792 { 2793 tavor_ahhdl_t ahhdl; 2794 tavor_state_t *state; 2795 2796 TAVOR_TNF_ENTER(tavor_rsrc_ahhdl_constructor); 2797 2798 ahhdl = (tavor_ahhdl_t)ah; 2799 state = (tavor_state_t *)priv; 2800 2801 mutex_init(&ahhdl->ah_lock, NULL, MUTEX_DRIVER, 2802 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2803 2804 TAVOR_TNF_EXIT(tavor_rsrc_ahhdl_constructor); 2805 return (DDI_SUCCESS); 2806 } 2807 2808 2809 /* 2810 * tavor_rsrc_ahhdl_destructor() 2811 * Context: Can be called from interrupt or base context. 2812 */ 2813 /* ARGSUSED */ 2814 static void 2815 tavor_rsrc_ahhdl_destructor(void *ah, void *priv) 2816 { 2817 tavor_ahhdl_t ahhdl; 2818 2819 TAVOR_TNF_ENTER(tavor_rsrc_ahhdl_destructor); 2820 2821 ahhdl = (tavor_ahhdl_t)ah; 2822 2823 mutex_destroy(&ahhdl->ah_lock); 2824 2825 TAVOR_TNF_ENTER(tavor_rsrc_ahhdl_destructor); 2826 } 2827 2828 2829 /* 2830 * tavor_rsrc_mrhdl_constructor() 2831 * Context: Can be called from interrupt or base context. 2832 */ 2833 /* ARGSUSED */ 2834 static int 2835 tavor_rsrc_mrhdl_constructor(void *mr, void *priv, int flags) 2836 { 2837 tavor_mrhdl_t mrhdl; 2838 tavor_state_t *state; 2839 2840 TAVOR_TNF_ENTER(tavor_rsrc_mrhdl_constructor); 2841 2842 mrhdl = (tavor_mrhdl_t)mr; 2843 state = (tavor_state_t *)priv; 2844 2845 mutex_init(&mrhdl->mr_lock, NULL, MUTEX_DRIVER, 2846 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2847 2848 TAVOR_TNF_EXIT(tavor_rsrc_mrhdl_constructor); 2849 return (DDI_SUCCESS); 2850 } 2851 2852 2853 /* 2854 * tavor_rsrc_mrhdl_destructor() 2855 * Context: Can be called from interrupt or base context. 2856 */ 2857 /* ARGSUSED */ 2858 static void 2859 tavor_rsrc_mrhdl_destructor(void *mr, void *priv) 2860 { 2861 tavor_mrhdl_t mrhdl; 2862 2863 TAVOR_TNF_ENTER(tavor_rsrc_mrhdl_destructor); 2864 2865 mrhdl = (tavor_mrhdl_t)mr; 2866 2867 mutex_destroy(&mrhdl->mr_lock); 2868 2869 TAVOR_TNF_ENTER(tavor_rsrc_mrhdl_destructor); 2870 } 2871 2872 2873 /* 2874 * tavor_rsrc_mcg_entry_get_size() 2875 */ 2876 static int 2877 tavor_rsrc_mcg_entry_get_size(tavor_state_t *state, uint_t *mcg_size_shift) 2878 { 2879 uint_t num_qp_per_mcg, max_qp_per_mcg, log2; 2880 2881 TAVOR_TNF_ENTER(tavor_rsrc_mcg_entry_get_size); 2882 2883 /* 2884 * Round the configured number of QP per MCG to next larger 2885 * power-of-2 size and update. 2886 */ 2887 num_qp_per_mcg = state->ts_cfg_profile->cp_num_qp_per_mcg + 8; 2888 log2 = highbit(num_qp_per_mcg); 2889 if (ISP2(num_qp_per_mcg)) { 2890 log2 = log2 - 1; 2891 } 2892 state->ts_cfg_profile->cp_num_qp_per_mcg = (1 << log2) - 8; 2893 2894 /* Now make sure number of QP per MCG makes sense */ 2895 num_qp_per_mcg = state->ts_cfg_profile->cp_num_qp_per_mcg; 2896 max_qp_per_mcg = (1 << state->ts_devlim.log_max_qp_mcg); 2897 if (num_qp_per_mcg > max_qp_per_mcg) { 2898 TNF_PROBE_1(tavor_rsrc_mcg_getsz_toomany_qppermcg_fail, 2899 TAVOR_TNF_ERROR, "", tnf_uint, maxqpmcg, max_qp_per_mcg); 2900 TAVOR_TNF_EXIT(tavor_rsrc_mcg_entry_get_size); 2901 return (DDI_FAILURE); 2902 } 2903 2904 /* Return the (shift) size of an individual MCG HW entry */ 2905 *mcg_size_shift = log2 + 2; 2906 2907 TAVOR_TNF_EXIT(tavor_rsrc_mcg_entry_get_size); 2908 return (DDI_SUCCESS); 2909 } 2910