1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * tavor.c 29 * Tavor (InfiniBand) HCA Driver attach/detach Routines 30 * 31 * Implements all the routines necessary for the attach, setup, 32 * initialization (and subsequent possible teardown and detach) of the 33 * Tavor InfiniBand HCA driver. 34 */ 35 36 #include <sys/types.h> 37 #include <sys/file.h> 38 #include <sys/open.h> 39 #include <sys/conf.h> 40 #include <sys/ddi.h> 41 #include <sys/sunddi.h> 42 #include <sys/modctl.h> 43 #include <sys/stat.h> 44 #include <sys/pci.h> 45 #include <sys/pci_cap.h> 46 #include <sys/bitmap.h> 47 #include <sys/policy.h> 48 49 #include <sys/ib/adapters/tavor/tavor.h> 50 #include <sys/pci.h> 51 52 /* Tavor HCA State Pointer */ 53 void *tavor_statep; 54 55 /* 56 * The Tavor "userland resource database" is common to instances of the 57 * Tavor HCA driver. This structure "tavor_userland_rsrc_db" contains all 58 * the necessary information to maintain it. 59 */ 60 tavor_umap_db_t tavor_userland_rsrc_db; 61 62 static int tavor_attach(dev_info_t *, ddi_attach_cmd_t); 63 static int tavor_detach(dev_info_t *, ddi_detach_cmd_t); 64 static int tavor_open(dev_t *, int, int, cred_t *); 65 static int tavor_close(dev_t, int, int, cred_t *); 66 static int tavor_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **); 67 static int tavor_drv_init(tavor_state_t *state, dev_info_t *dip, int instance); 68 static void tavor_drv_fini(tavor_state_t *state); 69 static void tavor_drv_fini2(tavor_state_t *state); 70 static int tavor_isr_init(tavor_state_t *state); 71 static void tavor_isr_fini(tavor_state_t *state); 72 static int tavor_hw_init(tavor_state_t *state); 73 static void tavor_hw_fini(tavor_state_t *state, 74 tavor_drv_cleanup_level_t cleanup); 75 static int tavor_soft_state_init(tavor_state_t *state); 76 static void tavor_soft_state_fini(tavor_state_t *state); 77 static int tavor_hca_port_init(tavor_state_t *state); 78 static int tavor_hca_ports_shutdown(tavor_state_t *state, uint_t num_init); 79 static void tavor_hca_config_setup(tavor_state_t *state, 80 tavor_hw_initqueryhca_t *inithca); 81 static int tavor_internal_uarpgs_init(tavor_state_t *state); 82 static void tavor_internal_uarpgs_fini(tavor_state_t *state); 83 static int tavor_special_qp_contexts_reserve(tavor_state_t *state); 84 static void tavor_special_qp_contexts_unreserve(tavor_state_t *state); 85 static int tavor_sw_reset(tavor_state_t *state); 86 static int tavor_mcg_init(tavor_state_t *state); 87 static void tavor_mcg_fini(tavor_state_t *state); 88 static int tavor_fw_version_check(tavor_state_t *state); 89 static void tavor_device_info_report(tavor_state_t *state); 90 static void tavor_pci_capability_list(tavor_state_t *state, 91 ddi_acc_handle_t hdl); 92 static void tavor_pci_capability_vpd(tavor_state_t *state, 93 ddi_acc_handle_t hdl, uint_t offset); 94 static int tavor_pci_read_vpd(ddi_acc_handle_t hdl, uint_t offset, 95 uint32_t addr, uint32_t *data); 96 static void tavor_pci_capability_pcix(tavor_state_t *state, 97 ddi_acc_handle_t hdl, uint_t offset); 98 static int tavor_intr_or_msi_init(tavor_state_t *state); 99 static int tavor_add_intrs(tavor_state_t *state, int intr_type); 100 static int tavor_intr_or_msi_fini(tavor_state_t *state); 101 102 /* X86 fastreboot support */ 103 static int tavor_intr_disable(tavor_state_t *); 104 static int tavor_quiesce(dev_info_t *); 105 106 /* Character/Block Operations */ 107 static struct cb_ops tavor_cb_ops = { 108 tavor_open, /* open */ 109 tavor_close, /* close */ 110 nodev, /* strategy (block) */ 111 nodev, /* print (block) */ 112 nodev, /* dump (block) */ 113 nodev, /* read */ 114 nodev, /* write */ 115 tavor_ioctl, /* ioctl */ 116 tavor_devmap, /* devmap */ 117 NULL, /* mmap */ 118 nodev, /* segmap */ 119 nochpoll, /* chpoll */ 120 ddi_prop_op, /* prop_op */ 121 NULL, /* streams */ 122 D_NEW | D_MP | 123 D_64BIT | D_HOTPLUG | 124 D_DEVMAP, /* flags */ 125 CB_REV /* rev */ 126 }; 127 128 /* Driver Operations */ 129 static struct dev_ops tavor_ops = { 130 DEVO_REV, /* struct rev */ 131 0, /* refcnt */ 132 tavor_getinfo, /* getinfo */ 133 nulldev, /* identify */ 134 nulldev, /* probe */ 135 tavor_attach, /* attach */ 136 tavor_detach, /* detach */ 137 nodev, /* reset */ 138 &tavor_cb_ops, /* cb_ops */ 139 NULL, /* bus_ops */ 140 nodev, /* power */ 141 tavor_quiesce, /* devo_quiesce */ 142 }; 143 144 /* Module Driver Info */ 145 static struct modldrv tavor_modldrv = { 146 &mod_driverops, 147 "Tavor InfiniBand HCA Driver", 148 &tavor_ops 149 }; 150 151 /* Module Linkage */ 152 static struct modlinkage tavor_modlinkage = { 153 MODREV_1, 154 &tavor_modldrv, 155 NULL 156 }; 157 158 /* 159 * This extern refers to the ibc_operations_t function vector that is defined 160 * in the tavor_ci.c file. 161 */ 162 extern ibc_operations_t tavor_ibc_ops; 163 164 #ifndef NPROBE 165 extern int tnf_mod_load(void); 166 extern int tnf_mod_unload(struct modlinkage *mlp); 167 #endif 168 169 170 /* 171 * _init() 172 */ 173 int 174 _init() 175 { 176 int status; 177 178 #ifndef NPROBE 179 (void) tnf_mod_load(); 180 #endif 181 TAVOR_TNF_ENTER(tavor_init); 182 183 status = ddi_soft_state_init(&tavor_statep, sizeof (tavor_state_t), 184 (size_t)TAVOR_INITIAL_STATES); 185 if (status != 0) { 186 TNF_PROBE_0(tavor_init_ssi_fail, TAVOR_TNF_ERROR, ""); 187 TAVOR_TNF_EXIT(tavor_init); 188 #ifndef NPROBE 189 (void) tnf_mod_unload(&tavor_modlinkage); 190 #endif 191 return (status); 192 } 193 194 status = ibc_init(&tavor_modlinkage); 195 if (status != 0) { 196 TNF_PROBE_0(tavor_init_ibc_init_fail, TAVOR_TNF_ERROR, ""); 197 ddi_soft_state_fini(&tavor_statep); 198 TAVOR_TNF_EXIT(tavor_init); 199 #ifndef NPROBE 200 (void) tnf_mod_unload(&tavor_modlinkage); 201 #endif 202 return (status); 203 } 204 status = mod_install(&tavor_modlinkage); 205 if (status != 0) { 206 TNF_PROBE_0(tavor_init_modi_fail, TAVOR_TNF_ERROR, ""); 207 ibc_fini(&tavor_modlinkage); 208 ddi_soft_state_fini(&tavor_statep); 209 TAVOR_TNF_EXIT(tavor_init); 210 #ifndef NPROBE 211 (void) tnf_mod_unload(&tavor_modlinkage); 212 #endif 213 return (status); 214 } 215 216 /* Initialize the Tavor "userland resources database" */ 217 tavor_umap_db_init(); 218 219 TAVOR_TNF_EXIT(tavor_init); 220 return (status); 221 } 222 223 224 /* 225 * _info() 226 */ 227 int 228 _info(struct modinfo *modinfop) 229 { 230 int status; 231 232 TAVOR_TNF_ENTER(tavor_info); 233 status = mod_info(&tavor_modlinkage, modinfop); 234 TAVOR_TNF_EXIT(tavor_info); 235 return (status); 236 } 237 238 239 /* 240 * _fini() 241 */ 242 int 243 _fini() 244 { 245 int status; 246 247 TAVOR_TNF_ENTER(tavor_fini); 248 249 status = mod_remove(&tavor_modlinkage); 250 if (status != 0) { 251 TNF_PROBE_0(tavor_fini_modr_fail, TAVOR_TNF_ERROR, ""); 252 TAVOR_TNF_EXIT(tavor_fini); 253 return (status); 254 } 255 256 /* Destroy the Tavor "userland resources database" */ 257 tavor_umap_db_fini(); 258 259 ibc_fini(&tavor_modlinkage); 260 ddi_soft_state_fini(&tavor_statep); 261 #ifndef NPROBE 262 (void) tnf_mod_unload(&tavor_modlinkage); 263 #endif 264 TAVOR_TNF_EXIT(tavor_fini); 265 return (status); 266 } 267 268 269 /* 270 * tavor_getinfo() 271 */ 272 /* ARGSUSED */ 273 static int 274 tavor_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result) 275 { 276 dev_t dev; 277 tavor_state_t *state; 278 minor_t instance; 279 280 TAVOR_TNF_ENTER(tavor_getinfo); 281 282 switch (cmd) { 283 case DDI_INFO_DEVT2DEVINFO: 284 dev = (dev_t)arg; 285 instance = TAVOR_DEV_INSTANCE(dev); 286 state = ddi_get_soft_state(tavor_statep, instance); 287 if (state == NULL) { 288 TNF_PROBE_0(tavor_getinfo_gss_fail, 289 TAVOR_TNF_ERROR, ""); 290 TAVOR_TNF_EXIT(tavor_getinfo); 291 return (DDI_FAILURE); 292 } 293 *result = (void *)state->ts_dip; 294 return (DDI_SUCCESS); 295 296 case DDI_INFO_DEVT2INSTANCE: 297 dev = (dev_t)arg; 298 instance = TAVOR_DEV_INSTANCE(dev); 299 *result = (void *)(uintptr_t)instance; 300 return (DDI_SUCCESS); 301 302 default: 303 TNF_PROBE_0(tavor_getinfo_default_fail, TAVOR_TNF_ERROR, ""); 304 break; 305 } 306 307 TAVOR_TNF_EXIT(tavor_getinfo); 308 return (DDI_FAILURE); 309 } 310 311 312 /* 313 * tavor_open() 314 */ 315 /* ARGSUSED */ 316 static int 317 tavor_open(dev_t *devp, int flag, int otyp, cred_t *credp) 318 { 319 tavor_state_t *state; 320 tavor_rsrc_t *rsrcp; 321 tavor_umap_db_entry_t *umapdb, *umapdb2; 322 minor_t instance; 323 uint64_t key, value; 324 uint_t tr_indx; 325 dev_t dev; 326 int status; 327 328 TAVOR_TNF_ENTER(tavor_open); 329 330 instance = TAVOR_DEV_INSTANCE(*devp); 331 state = ddi_get_soft_state(tavor_statep, instance); 332 if (state == NULL) { 333 TNF_PROBE_0(tavor_open_gss_fail, TAVOR_TNF_ERROR, ""); 334 TAVOR_TNF_EXIT(tavor_open); 335 return (ENXIO); 336 } 337 338 /* 339 * Only allow driver to be opened for character access, and verify 340 * whether exclusive access is allowed. 341 */ 342 if ((otyp != OTYP_CHR) || ((flag & FEXCL) && 343 secpolicy_excl_open(credp) != 0)) { 344 TNF_PROBE_0(tavor_open_invflags_fail, TAVOR_TNF_ERROR, ""); 345 TAVOR_TNF_EXIT(tavor_open); 346 return (EINVAL); 347 } 348 349 /* 350 * Search for the current process PID in the "userland resources 351 * database". If it is not found, then attempt to allocate a UAR 352 * page and add the ("key", "value") pair to the database. 353 * Note: As a last step we always return a devp appropriate for 354 * the open. Either we return a new minor number (based on the 355 * instance and the UAR page index) or we return the current minor 356 * number for the given client process. 357 * 358 * We also add an entry to the database to allow for lookup from 359 * "dev_t" to the current process PID. This is necessary because, 360 * under certain circumstance, the process PID that calls the Tavor 361 * close() entry point may not be the same as the one who called 362 * open(). Specifically, this can happen if a child process calls 363 * the Tavor's open() entry point, gets a UAR page, maps it out (using 364 * mmap()), and then exits without calling munmap(). Because mmap() 365 * adds a reference to the file descriptor, at the exit of the child 366 * process the file descriptor is "inherited" by the parent (and will 367 * be close()'d by the parent's PID only when it exits). 368 * 369 * Note: We use the tavor_umap_db_find_nolock() and 370 * tavor_umap_db_add_nolock() database access routines below (with 371 * an explicit mutex_enter of the database lock - "tdl_umapdb_lock") 372 * to ensure that the multiple accesses (in this case searching for, 373 * and then adding _two_ database entries) can be done atomically. 374 */ 375 key = ddi_get_pid(); 376 mutex_enter(&tavor_userland_rsrc_db.tdl_umapdb_lock); 377 status = tavor_umap_db_find_nolock(instance, key, 378 MLNX_UMAP_UARPG_RSRC, &value, 0, NULL); 379 if (status != DDI_SUCCESS) { 380 /* 381 * If we are in 'maintenance mode', we cannot alloc a UAR page. 382 * But we still need some rsrcp value, and a mostly unique 383 * tr_indx value. So we set rsrcp to NULL for maintenance 384 * mode, and use a rolling count for tr_indx. The field 385 * 'ts_open_tr_indx' is used only in this maintenance mode 386 * condition. 387 * 388 * Otherwise, if we are in operational mode then we allocate 389 * the UAR page as normal, and use the rsrcp value and tr_indx 390 * value from that allocation. 391 */ 392 if (!TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { 393 rsrcp = NULL; 394 tr_indx = state->ts_open_tr_indx++; 395 } else { 396 /* Allocate a new UAR page for this process */ 397 status = tavor_rsrc_alloc(state, TAVOR_UARPG, 1, 398 TAVOR_NOSLEEP, &rsrcp); 399 if (status != DDI_SUCCESS) { 400 mutex_exit( 401 &tavor_userland_rsrc_db.tdl_umapdb_lock); 402 TNF_PROBE_0(tavor_open_rsrcalloc_uarpg_fail, 403 TAVOR_TNF_ERROR, ""); 404 TAVOR_TNF_EXIT(tavor_open); 405 return (EAGAIN); 406 } 407 408 tr_indx = rsrcp->tr_indx; 409 } 410 411 /* 412 * Allocate an entry to track the UAR page resource in the 413 * "userland resources database". 414 */ 415 umapdb = tavor_umap_db_alloc(instance, key, 416 MLNX_UMAP_UARPG_RSRC, (uint64_t)(uintptr_t)rsrcp); 417 if (umapdb == NULL) { 418 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock); 419 /* If in "maintenance mode", don't free the rsrc */ 420 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { 421 tavor_rsrc_free(state, &rsrcp); 422 } 423 TNF_PROBE_0(tavor_open_umap_db_alloc_fail, 424 TAVOR_TNF_ERROR, ""); 425 TAVOR_TNF_EXIT(tavor_open); 426 return (EAGAIN); 427 } 428 429 /* 430 * Create a new device number. Minor number is a function of 431 * the UAR page index (15 bits) and the device instance number 432 * (3 bits). 433 */ 434 dev = makedevice(getmajor(*devp), (tr_indx << 435 TAVOR_MINORNUM_SHIFT) | instance); 436 437 /* 438 * Allocate another entry in the "userland resources database" 439 * to track the association of the device number (above) to 440 * the current process ID (in "key"). 441 */ 442 umapdb2 = tavor_umap_db_alloc(instance, dev, 443 MLNX_UMAP_PID_RSRC, (uint64_t)key); 444 if (umapdb2 == NULL) { 445 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock); 446 tavor_umap_db_free(umapdb); 447 /* If in "maintenance mode", don't free the rsrc */ 448 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { 449 tavor_rsrc_free(state, &rsrcp); 450 } 451 TNF_PROBE_0(tavor_open_umap_db_alloc_fail, 452 TAVOR_TNF_ERROR, ""); 453 TAVOR_TNF_EXIT(tavor_open); 454 return (EAGAIN); 455 } 456 457 /* Add the entries to the database */ 458 tavor_umap_db_add_nolock(umapdb); 459 tavor_umap_db_add_nolock(umapdb2); 460 461 } else { 462 /* 463 * Return the same device number as on the original open() 464 * call. This was calculated as a function of the UAR page 465 * index (top 16 bits) and the device instance number 466 */ 467 rsrcp = (tavor_rsrc_t *)(uintptr_t)value; 468 dev = makedevice(getmajor(*devp), (rsrcp->tr_indx << 469 TAVOR_MINORNUM_SHIFT) | instance); 470 } 471 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock); 472 473 *devp = dev; 474 475 TAVOR_TNF_EXIT(tavor_open); 476 return (0); 477 } 478 479 480 /* 481 * tavor_close() 482 */ 483 /* ARGSUSED */ 484 static int 485 tavor_close(dev_t dev, int flag, int otyp, cred_t *credp) 486 { 487 tavor_state_t *state; 488 tavor_rsrc_t *rsrcp; 489 tavor_umap_db_entry_t *umapdb; 490 tavor_umap_db_priv_t *priv; 491 minor_t instance; 492 uint64_t key, value; 493 int status; 494 495 TAVOR_TNF_ENTER(tavor_close); 496 497 instance = TAVOR_DEV_INSTANCE(dev); 498 state = ddi_get_soft_state(tavor_statep, instance); 499 if (state == NULL) { 500 TNF_PROBE_0(tavor_close_gss_fail, TAVOR_TNF_ERROR, ""); 501 TAVOR_TNF_EXIT(tavor_close); 502 return (ENXIO); 503 } 504 505 /* 506 * Search for "dev_t" in the "userland resources database". As 507 * explained above in tavor_open(), we can't depend on using the 508 * current process ID here to do the lookup because the process 509 * that ultimately closes may not be the same one who opened 510 * (because of inheritance). 511 * So we lookup the "dev_t" (which points to the PID of the process 512 * that opened), and we remove the entry from the database (and free 513 * it up). Then we do another query based on the PID value. And when 514 * we find that database entry, we free it up too and then free the 515 * Tavor UAR page resource. 516 * 517 * Note: We use the tavor_umap_db_find_nolock() database access 518 * routine below (with an explicit mutex_enter of the database lock) 519 * to ensure that the multiple accesses (which attempt to remove the 520 * two database entries) can be done atomically. 521 * 522 * This works the same in both maintenance mode and HCA mode, except 523 * for the call to tavor_rsrc_free(). In the case of maintenance mode, 524 * this call is not needed, as it was not allocated in tavor_open() 525 * above. 526 */ 527 key = dev; 528 mutex_enter(&tavor_userland_rsrc_db.tdl_umapdb_lock); 529 status = tavor_umap_db_find_nolock(instance, key, MLNX_UMAP_PID_RSRC, 530 &value, TAVOR_UMAP_DB_REMOVE, &umapdb); 531 if (status == DDI_SUCCESS) { 532 /* 533 * If the "tdb_priv" field is non-NULL, it indicates that 534 * some "on close" handling is still necessary. Call 535 * tavor_umap_db_handle_onclose_cb() to do the handling (i.e. 536 * to invoke all the registered callbacks). Then free up 537 * the resources associated with "tdb_priv" and continue 538 * closing. 539 */ 540 priv = (tavor_umap_db_priv_t *)umapdb->tdbe_common.tdb_priv; 541 if (priv != NULL) { 542 tavor_umap_db_handle_onclose_cb(priv); 543 kmem_free(priv, sizeof (tavor_umap_db_priv_t)); 544 umapdb->tdbe_common.tdb_priv = (void *)NULL; 545 } 546 547 tavor_umap_db_free(umapdb); 548 549 /* 550 * Now do another lookup using PID as the key (copy it from 551 * "value"). When this lookup is complete, the "value" field 552 * will contain the tavor_rsrc_t pointer for the UAR page 553 * resource. 554 */ 555 key = value; 556 status = tavor_umap_db_find_nolock(instance, key, 557 MLNX_UMAP_UARPG_RSRC, &value, TAVOR_UMAP_DB_REMOVE, 558 &umapdb); 559 if (status == DDI_SUCCESS) { 560 tavor_umap_db_free(umapdb); 561 /* If in "maintenance mode", don't free the rsrc */ 562 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { 563 rsrcp = (tavor_rsrc_t *)(uintptr_t)value; 564 tavor_rsrc_free(state, &rsrcp); 565 } 566 } 567 } 568 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock); 569 570 TAVOR_TNF_EXIT(tavor_close); 571 return (0); 572 } 573 574 575 /* 576 * tavor_attach() 577 * Context: Only called from attach() path context 578 */ 579 static int 580 tavor_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 581 { 582 tavor_state_t *state; 583 ibc_clnt_hdl_t tmp_ibtfpriv; 584 ibc_status_t ibc_status; 585 int instance; 586 int status; 587 588 TAVOR_TNF_ENTER(tavor_attach); 589 590 #ifdef __lock_lint 591 (void) tavor_quiesce(dip); 592 #endif 593 594 switch (cmd) { 595 case DDI_ATTACH: 596 instance = ddi_get_instance(dip); 597 status = ddi_soft_state_zalloc(tavor_statep, instance); 598 if (status != DDI_SUCCESS) { 599 TNF_PROBE_0(tavor_attach_ssz_fail, TAVOR_TNF_ERROR, ""); 600 cmn_err(CE_NOTE, "tavor%d: driver failed to attach: " 601 "attach_ssz_fail", instance); 602 goto fail_attach_nomsg; 603 604 } 605 state = ddi_get_soft_state(tavor_statep, instance); 606 if (state == NULL) { 607 ddi_soft_state_free(tavor_statep, instance); 608 TNF_PROBE_0(tavor_attach_gss_fail, TAVOR_TNF_ERROR, ""); 609 cmn_err(CE_NOTE, "tavor%d: driver failed to attach: " 610 "attach_gss_fail", instance); 611 goto fail_attach_nomsg; 612 } 613 614 /* clear the attach error buffer */ 615 TAVOR_ATTACH_MSG_INIT(state->ts_attach_buf); 616 617 /* 618 * Initialize Tavor driver and hardware. 619 * 620 * Note: If this initialization fails we may still wish to 621 * create a device node and remain operational so that Tavor 622 * firmware can be updated/flashed (i.e. "maintenance mode"). 623 * If this is the case, then "ts_operational_mode" will be 624 * equal to TAVOR_MAINTENANCE_MODE. We will not attempt to 625 * attach to the IBTF or register with the IBMF (i.e. no 626 * InfiniBand interfaces will be enabled). 627 */ 628 status = tavor_drv_init(state, dip, instance); 629 if ((status != DDI_SUCCESS) && 630 (TAVOR_IS_OPERATIONAL(state->ts_operational_mode))) { 631 TNF_PROBE_0(tavor_attach_drvinit_fail, 632 TAVOR_TNF_ERROR, ""); 633 goto fail_attach; 634 } 635 636 /* Create the minor node for device */ 637 status = ddi_create_minor_node(dip, "devctl", S_IFCHR, instance, 638 DDI_PSEUDO, 0); 639 if (status != DDI_SUCCESS) { 640 tavor_drv_fini(state); 641 TAVOR_ATTACH_MSG(state->ts_attach_buf, 642 "attach_create_mn_fail"); 643 TNF_PROBE_0(tavor_attach_create_mn_fail, 644 TAVOR_TNF_ERROR, ""); 645 goto fail_attach; 646 } 647 648 /* 649 * If we are in "maintenance mode", then we don't want to 650 * register with the IBTF. All InfiniBand interfaces are 651 * uninitialized, and the device is only capable of handling 652 * requests to update/flash firmware (or test/debug requests). 653 */ 654 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { 655 656 /* Attach to InfiniBand Transport Framework (IBTF) */ 657 ibc_status = ibc_attach(&tmp_ibtfpriv, 658 &state->ts_ibtfinfo); 659 if (ibc_status != IBC_SUCCESS) { 660 ddi_remove_minor_node(dip, "devctl"); 661 tavor_drv_fini(state); 662 TNF_PROBE_0(tavor_attach_ibcattach_fail, 663 TAVOR_TNF_ERROR, ""); 664 TAVOR_ATTACH_MSG(state->ts_attach_buf, 665 "attach_ibcattach_fail"); 666 goto fail_attach; 667 } 668 669 /* 670 * Now that we've successfully attached to the IBTF, 671 * we enable all appropriate asynch and CQ events to 672 * be forwarded to the IBTF. 673 */ 674 TAVOR_ENABLE_IBTF_CALLB(state, tmp_ibtfpriv); 675 676 ibc_post_attach(state->ts_ibtfpriv); 677 678 /* Register agents with IB Mgmt Framework (IBMF) */ 679 status = tavor_agent_handlers_init(state); 680 if (status != DDI_SUCCESS) { 681 (void) ibc_pre_detach(tmp_ibtfpriv, DDI_DETACH); 682 TAVOR_QUIESCE_IBTF_CALLB(state); 683 if (state->ts_in_evcallb != 0) { 684 TAVOR_WARNING(state, "unable to " 685 "quiesce Tavor IBTF callbacks"); 686 } 687 ibc_detach(tmp_ibtfpriv); 688 ddi_remove_minor_node(dip, "devctl"); 689 tavor_drv_fini(state); 690 TNF_PROBE_0(tavor_attach_agentinit_fail, 691 TAVOR_TNF_ERROR, ""); 692 TAVOR_ATTACH_MSG(state->ts_attach_buf, 693 "attach_agentinit_fail"); 694 goto fail_attach; 695 } 696 } 697 698 /* Report that driver was loaded */ 699 ddi_report_dev(dip); 700 701 /* Send device information to log file */ 702 tavor_device_info_report(state); 703 704 /* Report attach in maintenance mode, if appropriate */ 705 if (!(TAVOR_IS_OPERATIONAL(state->ts_operational_mode))) { 706 cmn_err(CE_NOTE, "tavor%d: driver attached " 707 "(for maintenance mode only)", state->ts_instance); 708 } 709 710 TAVOR_TNF_EXIT(tavor_attach); 711 return (DDI_SUCCESS); 712 713 case DDI_RESUME: 714 /* Add code here for DDI_RESUME XXX */ 715 TAVOR_TNF_EXIT(tavor_attach); 716 return (DDI_FAILURE); 717 718 default: 719 TNF_PROBE_0(tavor_attach_default_fail, TAVOR_TNF_ERROR, ""); 720 break; 721 } 722 723 fail_attach: 724 cmn_err(CE_NOTE, "tavor%d: driver failed to attach: %s", instance, 725 state->ts_attach_buf); 726 tavor_drv_fini2(state); 727 ddi_soft_state_free(tavor_statep, instance); 728 fail_attach_nomsg: 729 TAVOR_TNF_EXIT(tavor_attach); 730 return (DDI_FAILURE); 731 } 732 733 734 /* 735 * tavor_detach() 736 * Context: Only called from detach() path context 737 */ 738 static int 739 tavor_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 740 { 741 tavor_state_t *state; 742 ibc_clnt_hdl_t tmp_ibtfpriv; 743 ibc_status_t ibc_status; 744 int instance, status; 745 746 TAVOR_TNF_ENTER(tavor_detach); 747 748 instance = ddi_get_instance(dip); 749 state = ddi_get_soft_state(tavor_statep, instance); 750 if (state == NULL) { 751 TNF_PROBE_0(tavor_detach_gss_fail, TAVOR_TNF_ERROR, ""); 752 TAVOR_TNF_EXIT(tavor_detach); 753 return (DDI_FAILURE); 754 } 755 756 switch (cmd) { 757 case DDI_DETACH: 758 /* 759 * If we are in "maintenance mode", then we do not want to 760 * do teardown for any of the InfiniBand interfaces. 761 * Specifically, this means not detaching from IBTF (we never 762 * attached to begin with) and not deregistering from IBMF. 763 */ 764 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { 765 /* Unregister agents from IB Mgmt Framework (IBMF) */ 766 status = tavor_agent_handlers_fini(state); 767 if (status != DDI_SUCCESS) { 768 TNF_PROBE_0(tavor_detach_agentfini_fail, 769 TAVOR_TNF_ERROR, ""); 770 TAVOR_TNF_EXIT(tavor_detach); 771 return (DDI_FAILURE); 772 } 773 774 /* 775 * Attempt the "pre-detach" from InfiniBand Transport 776 * Framework (IBTF). At this point the IBTF is still 777 * capable of handling incoming asynch and completion 778 * events. This "pre-detach" is primarily a mechanism 779 * to notify the appropriate IBTF clients that the 780 * HCA is being removed/offlined. 781 */ 782 ibc_status = ibc_pre_detach(state->ts_ibtfpriv, cmd); 783 if (ibc_status != IBC_SUCCESS) { 784 status = tavor_agent_handlers_init(state); 785 if (status != DDI_SUCCESS) { 786 TAVOR_WARNING(state, "failed to " 787 "restart Tavor agents"); 788 } 789 TNF_PROBE_0(tavor_detach_ibcpredetach_fail, 790 TAVOR_TNF_ERROR, ""); 791 TAVOR_TNF_EXIT(tavor_detach); 792 return (DDI_FAILURE); 793 } 794 795 /* 796 * Before we can fully detach from the IBTF we need to 797 * ensure that we have handled all outstanding event 798 * callbacks. This is accomplished by quiescing the 799 * event callback mechanism. Note: if we are unable 800 * to successfully quiesce the callbacks, then this is 801 * an indication that something has probably gone 802 * seriously wrong. We print out a warning, but 803 * continue. 804 */ 805 tmp_ibtfpriv = state->ts_ibtfpriv; 806 TAVOR_QUIESCE_IBTF_CALLB(state); 807 if (state->ts_in_evcallb != 0) { 808 TAVOR_WARNING(state, "unable to quiesce Tavor " 809 "IBTF callbacks"); 810 } 811 812 /* Complete the detach from the IBTF */ 813 ibc_detach(tmp_ibtfpriv); 814 } 815 816 /* Remove the minor node for device */ 817 ddi_remove_minor_node(dip, "devctl"); 818 819 /* 820 * Only call tavor_drv_fini() if we are in Tavor HCA mode. 821 * (Because if we are in "maintenance mode", then we never 822 * successfully finished init.) Only report successful 823 * detach for normal HCA mode. 824 */ 825 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { 826 /* Cleanup driver resources and shutdown hardware */ 827 tavor_drv_fini(state); 828 cmn_err(CE_CONT, "Tavor driver successfully " 829 "detached\n"); 830 } 831 832 tavor_drv_fini2(state); 833 ddi_soft_state_free(tavor_statep, instance); 834 835 TAVOR_TNF_EXIT(tavor_detach); 836 return (DDI_SUCCESS); 837 838 case DDI_SUSPEND: 839 /* Add code here for DDI_SUSPEND XXX */ 840 TAVOR_TNF_EXIT(tavor_detach); 841 return (DDI_FAILURE); 842 843 default: 844 TNF_PROBE_0(tavor_detach_default_fail, TAVOR_TNF_ERROR, ""); 845 break; 846 } 847 848 TAVOR_TNF_EXIT(tavor_detach); 849 return (DDI_FAILURE); 850 } 851 852 853 /* 854 * tavor_drv_init() 855 * Context: Only called from attach() path context 856 */ 857 static int 858 tavor_drv_init(tavor_state_t *state, dev_info_t *dip, int instance) 859 { 860 int status; 861 862 TAVOR_TNF_ENTER(tavor_drv_init); 863 864 /* Save away devinfo and instance */ 865 state->ts_dip = dip; 866 state->ts_instance = instance; 867 868 /* 869 * Check and set the operational mode of the device. If the driver is 870 * bound to the Tavor device in "maintenance mode", then this generally 871 * means that either the device has been specifically jumpered to 872 * start in this mode or the firmware boot process has failed to 873 * successfully load either the primary or the secondary firmware 874 * image. 875 */ 876 if (TAVOR_IS_HCA_MODE(state->ts_dip)) { 877 state->ts_operational_mode = TAVOR_HCA_MODE; 878 879 } else if (TAVOR_IS_COMPAT_MODE(state->ts_dip)) { 880 state->ts_operational_mode = TAVOR_COMPAT_MODE; 881 882 } else if (TAVOR_IS_MAINTENANCE_MODE(state->ts_dip)) { 883 state->ts_operational_mode = TAVOR_MAINTENANCE_MODE; 884 return (DDI_FAILURE); 885 886 } else { 887 state->ts_operational_mode = 0; /* invalid operational mode */ 888 TAVOR_WARNING(state, "unexpected device type detected"); 889 TNF_PROBE_0(tavor_hw_init_unexpected_dev_fail, 890 TAVOR_TNF_ERROR, ""); 891 TAVOR_TNF_EXIT(tavor_hw_init); 892 return (DDI_FAILURE); 893 } 894 895 /* 896 * Initialize the Tavor hardware. 897 * Note: If this routine returns an error, it is often an reasonably 898 * good indication that something Tavor firmware-related has caused 899 * the failure. In order to give the user an opportunity (if desired) 900 * to update or reflash the Tavor firmware image, we set 901 * "ts_operational_mode" flag (described above) to indicate that we 902 * wish to enter maintenance mode. 903 */ 904 status = tavor_hw_init(state); 905 if (status != DDI_SUCCESS) { 906 state->ts_operational_mode = TAVOR_MAINTENANCE_MODE; 907 cmn_err(CE_NOTE, "tavor%d: error during attach: %s", instance, 908 state->ts_attach_buf); 909 TNF_PROBE_0(tavor_drv_init_hwinit_fail, TAVOR_TNF_ERROR, ""); 910 TAVOR_TNF_EXIT(tavor_drv_init); 911 return (DDI_FAILURE); 912 } 913 914 /* Setup Tavor interrupt handler */ 915 status = tavor_isr_init(state); 916 if (status != DDI_SUCCESS) { 917 tavor_hw_fini(state, TAVOR_DRV_CLEANUP_ALL); 918 TNF_PROBE_0(tavor_drv_init_isrinit_fail, TAVOR_TNF_ERROR, ""); 919 TAVOR_TNF_EXIT(tavor_drv_init); 920 return (DDI_FAILURE); 921 } 922 923 /* Initialize Tavor softstate */ 924 status = tavor_soft_state_init(state); 925 if (status != DDI_SUCCESS) { 926 tavor_isr_fini(state); 927 tavor_hw_fini(state, TAVOR_DRV_CLEANUP_ALL); 928 TNF_PROBE_0(tavor_drv_init_ssiinit_fail, TAVOR_TNF_ERROR, ""); 929 TAVOR_TNF_EXIT(tavor_drv_init); 930 return (DDI_FAILURE); 931 } 932 933 TAVOR_TNF_EXIT(tavor_drv_init); 934 return (DDI_SUCCESS); 935 } 936 937 938 /* 939 * tavor_drv_fini() 940 * Context: Only called from attach() and/or detach() path contexts 941 */ 942 static void 943 tavor_drv_fini(tavor_state_t *state) 944 { 945 TAVOR_TNF_ENTER(tavor_drv_fini); 946 947 /* Cleanup Tavor softstate */ 948 tavor_soft_state_fini(state); 949 950 /* Teardown Tavor interrupts */ 951 tavor_isr_fini(state); 952 953 /* Cleanup Tavor resources and shutdown hardware */ 954 tavor_hw_fini(state, TAVOR_DRV_CLEANUP_ALL); 955 956 TAVOR_TNF_EXIT(tavor_drv_fini); 957 } 958 959 /* 960 * tavor_drv_fini2() 961 * Context: Only called from attach() and/or detach() path contexts 962 */ 963 static void 964 tavor_drv_fini2(tavor_state_t *state) 965 { 966 TAVOR_TNF_ENTER(tavor_drv_fini2); 967 968 /* TAVOR_DRV_CLEANUP_LEVEL1 */ 969 if (state->ts_reg_cmdhdl) { 970 ddi_regs_map_free(&state->ts_reg_cmdhdl); 971 state->ts_reg_cmdhdl = NULL; 972 } 973 974 /* TAVOR_DRV_CLEANUP_LEVEL0 */ 975 if (state->ts_pci_cfghdl) { 976 pci_config_teardown(&state->ts_pci_cfghdl); 977 state->ts_pci_cfghdl = NULL; 978 } 979 980 TAVOR_TNF_EXIT(tavor_drv_fini2); 981 } 982 983 /* 984 * tavor_isr_init() 985 * Context: Only called from attach() path context 986 */ 987 static int 988 tavor_isr_init(tavor_state_t *state) 989 { 990 int status; 991 992 TAVOR_TNF_ENTER(tavor_isr_init); 993 994 /* 995 * Add a handler for the interrupt or MSI 996 */ 997 status = ddi_intr_add_handler(state->ts_intrmsi_hdl, tavor_isr, 998 (caddr_t)state, NULL); 999 if (status != DDI_SUCCESS) { 1000 TNF_PROBE_0(tavor_isr_init_addhndlr_fail, TAVOR_TNF_ERROR, ""); 1001 TAVOR_TNF_EXIT(tavor_isr_init); 1002 return (DDI_FAILURE); 1003 } 1004 1005 /* 1006 * Enable the software interrupt. Note: Even though we are only 1007 * using one (1) interrupt/MSI, depending on the value returned in 1008 * the capability flag, we have to call either ddi_intr_block_enable() 1009 * or ddi_intr_enable(). 1010 */ 1011 if (state->ts_intrmsi_cap & DDI_INTR_FLAG_BLOCK) { 1012 status = ddi_intr_block_enable(&state->ts_intrmsi_hdl, 1); 1013 if (status != DDI_SUCCESS) { 1014 TNF_PROBE_0(tavor_isr_init_blockenable_fail, 1015 TAVOR_TNF_ERROR, ""); 1016 TAVOR_TNF_EXIT(tavor_isr_init); 1017 return (DDI_FAILURE); 1018 } 1019 } else { 1020 status = ddi_intr_enable(state->ts_intrmsi_hdl); 1021 if (status != DDI_SUCCESS) { 1022 TNF_PROBE_0(tavor_isr_init_intrenable_fail, 1023 TAVOR_TNF_ERROR, ""); 1024 TAVOR_TNF_EXIT(tavor_isr_init); 1025 return (DDI_FAILURE); 1026 } 1027 } 1028 1029 /* 1030 * Now that the ISR has been setup, arm all the EQs for event 1031 * generation. 1032 */ 1033 tavor_eq_arm_all(state); 1034 1035 TAVOR_TNF_EXIT(tavor_isr_init); 1036 return (DDI_SUCCESS); 1037 } 1038 1039 1040 /* 1041 * tavor_isr_fini() 1042 * Context: Only called from attach() and/or detach() path contexts 1043 */ 1044 static void 1045 tavor_isr_fini(tavor_state_t *state) 1046 { 1047 TAVOR_TNF_ENTER(tavor_isr_fini); 1048 1049 /* Disable the software interrupt */ 1050 if (state->ts_intrmsi_cap & DDI_INTR_FLAG_BLOCK) { 1051 (void) ddi_intr_block_disable(&state->ts_intrmsi_hdl, 1); 1052 } else { 1053 (void) ddi_intr_disable(state->ts_intrmsi_hdl); 1054 } 1055 1056 /* 1057 * Remove the software handler for the interrupt or MSI 1058 */ 1059 (void) ddi_intr_remove_handler(state->ts_intrmsi_hdl); 1060 1061 TAVOR_TNF_EXIT(tavor_isr_fini); 1062 } 1063 1064 1065 /* 1066 * tavor_fix_error_buf() 1067 * Context: Only called from attach(). 1068 * 1069 * The error_buf_addr returned from QUERY_FW is a PCI address. 1070 * We need to convert it to an offset from the base address, 1071 * which is stored in the assigned-addresses property. 1072 */ 1073 static int 1074 tavor_fix_error_buf(tavor_state_t *state) 1075 { 1076 int assigned_addr_len; 1077 pci_regspec_t *assigned_addr; 1078 1079 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, state->ts_dip, 1080 DDI_PROP_DONTPASS, "assigned-addresses", (int **)&assigned_addr, 1081 (uint_t *)&assigned_addr_len) != DDI_PROP_SUCCESS) 1082 return (DDI_FAILURE); 1083 1084 state->ts_fw.error_buf_addr -= assigned_addr[0].pci_phys_low + 1085 ((uint64_t)(assigned_addr[0].pci_phys_mid) << 32); 1086 ddi_prop_free(assigned_addr); 1087 return (DDI_SUCCESS); 1088 } 1089 1090 /* 1091 * tavor_hw_init() 1092 * Context: Only called from attach() path context 1093 */ 1094 static int 1095 tavor_hw_init(tavor_state_t *state) 1096 { 1097 tavor_drv_cleanup_level_t cleanup; 1098 sm_nodeinfo_t nodeinfo; 1099 uint64_t errorcode; 1100 off_t ddr_size; 1101 int status; 1102 int retries; 1103 1104 TAVOR_TNF_ENTER(tavor_hw_init); 1105 1106 /* This is where driver initialization begins */ 1107 cleanup = TAVOR_DRV_CLEANUP_LEVEL0; 1108 1109 /* Setup device access attributes */ 1110 state->ts_reg_accattr.devacc_attr_version = DDI_DEVICE_ATTR_V0; 1111 state->ts_reg_accattr.devacc_attr_endian_flags = DDI_STRUCTURE_BE_ACC; 1112 state->ts_reg_accattr.devacc_attr_dataorder = DDI_STRICTORDER_ACC; 1113 1114 /* Setup for PCI config read/write of HCA device */ 1115 status = pci_config_setup(state->ts_dip, &state->ts_pci_cfghdl); 1116 if (status != DDI_SUCCESS) { 1117 tavor_hw_fini(state, cleanup); 1118 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1119 "hw_init_PCI_config_space_regmap_fail"); 1120 /* This case is not the degraded one */ 1121 return (DDI_FAILURE); 1122 } 1123 1124 /* Map in Tavor registers (CMD, UAR, DDR) and setup offsets */ 1125 status = ddi_regs_map_setup(state->ts_dip, TAVOR_CMD_BAR, 1126 &state->ts_reg_cmd_baseaddr, 0, 0, &state->ts_reg_accattr, 1127 &state->ts_reg_cmdhdl); 1128 if (status != DDI_SUCCESS) { 1129 tavor_hw_fini(state, cleanup); 1130 TNF_PROBE_0(tavor_hw_init_CMD_ddirms_fail, TAVOR_TNF_ERROR, ""); 1131 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1132 "hw_init_CMD_ddirms_fail"); 1133 TAVOR_TNF_EXIT(tavor_hw_init); 1134 return (DDI_FAILURE); 1135 } 1136 cleanup = TAVOR_DRV_CLEANUP_LEVEL1; 1137 1138 status = ddi_regs_map_setup(state->ts_dip, TAVOR_UAR_BAR, 1139 &state->ts_reg_uar_baseaddr, 0, 0, &state->ts_reg_accattr, 1140 &state->ts_reg_uarhdl); 1141 if (status != DDI_SUCCESS) { 1142 tavor_hw_fini(state, cleanup); 1143 TNF_PROBE_0(tavor_hw_init_UAR_ddirms_fail, TAVOR_TNF_ERROR, ""); 1144 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1145 "hw_init_UAR_ddirms_fail"); 1146 TAVOR_TNF_EXIT(tavor_hw_init); 1147 return (DDI_FAILURE); 1148 } 1149 cleanup = TAVOR_DRV_CLEANUP_LEVEL2; 1150 1151 status = ddi_dev_regsize(state->ts_dip, TAVOR_DDR_BAR, &ddr_size); 1152 if (status != DDI_SUCCESS) { 1153 cmn_err(CE_CONT, "Tavor: ddi_dev_regsize() failed " 1154 "(check HCA-attached DIMM memory?)\n"); 1155 tavor_hw_fini(state, cleanup); 1156 TNF_PROBE_0(tavor_hw_init_DDR_ddi_regsize_fail, 1157 TAVOR_TNF_ERROR, ""); 1158 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1159 "hw_init_DDR_ddi_regsize_fail"); 1160 TAVOR_TNF_EXIT(tavor_hw_init); 1161 return (DDI_FAILURE); 1162 } 1163 1164 #if !defined(_ELF64) && !defined(__sparc) 1165 /* 1166 * For 32 bit x86/x64 kernels, where there is limited kernel virtual 1167 * memory available, define a minimal memory footprint. This is 1168 * specified in order to not take up too much resources, thus starving 1169 * out others. Only specified if the HCA DIMM is equal to or greater 1170 * than 256MB. 1171 * 1172 * Note: x86/x64 install and safemode boot are both 32bit. 1173 */ 1174 ddr_size = TAVOR_DDR_SIZE_MIN; 1175 #endif /* !(_ELF64) && !(__sparc) */ 1176 1177 state->ts_cfg_profile_setting = ddr_size; 1178 1179 status = ddi_regs_map_setup(state->ts_dip, TAVOR_DDR_BAR, 1180 &state->ts_reg_ddr_baseaddr, 0, ddr_size, &state->ts_reg_accattr, 1181 &state->ts_reg_ddrhdl); 1182 1183 /* 1184 * On 32-bit platform testing (primarily x86), it was seen that the 1185 * ddi_regs_map_setup() call would fail because there wasn't enough 1186 * kernel virtual address space available to map in the entire 256MB 1187 * DDR. So we add this check in here, so that if the 256 (or other 1188 * larger value of DDR) map in fails, that we fallback to try the lower 1189 * size of 128MB. 1190 * 1191 * Note: If we only have 128MB of DDR in the system in the first place, 1192 * we don't try another ddi_regs_map_setup(), and just skip over this 1193 * check and return failures. 1194 */ 1195 if (status == DDI_ME_NORESOURCES && ddr_size > TAVOR_DDR_SIZE_128) { 1196 /* Try falling back to 128MB DDR mapping */ 1197 status = ddi_regs_map_setup(state->ts_dip, TAVOR_DDR_BAR, 1198 &state->ts_reg_ddr_baseaddr, 0, TAVOR_DDR_SIZE_128, 1199 &state->ts_reg_accattr, &state->ts_reg_ddrhdl); 1200 1201 /* 1202 * 128MB DDR mapping worked. 1203 * Set the updated config profile setting here. 1204 */ 1205 if (status == DDI_SUCCESS) { 1206 TNF_PROBE_0(tavor_hw_init_DDR_128mb_fallback_success, 1207 TAVOR_TNF_TRACE, ""); 1208 state->ts_cfg_profile_setting = TAVOR_DDR_SIZE_128; 1209 } 1210 } 1211 1212 if (status != DDI_SUCCESS) { 1213 if (status == DDI_ME_RNUMBER_RANGE) { 1214 cmn_err(CE_CONT, "Tavor: ddi_regs_map_setup() failed " 1215 "(check HCA-attached DIMM memory?)\n"); 1216 } 1217 tavor_hw_fini(state, cleanup); 1218 TNF_PROBE_0(tavor_hw_init_DDR_ddirms_fail, TAVOR_TNF_ERROR, ""); 1219 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1220 "hw_init_DDR_ddirms_fail"); 1221 TAVOR_TNF_EXIT(tavor_hw_init); 1222 return (DDI_FAILURE); 1223 } 1224 cleanup = TAVOR_DRV_CLEANUP_LEVEL3; 1225 1226 /* Setup Tavor Host Command Register (HCR) */ 1227 state->ts_cmd_regs.hcr = (tavor_hw_hcr_t *) 1228 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_HCR_OFFSET); 1229 1230 /* Setup Tavor Event Cause Register (ecr and clr_ecr) */ 1231 state->ts_cmd_regs.ecr = (uint64_t *) 1232 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_ECR_OFFSET); 1233 state->ts_cmd_regs.clr_ecr = (uint64_t *) 1234 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_CLR_ECR_OFFSET); 1235 1236 /* Setup Tavor Software Reset register (sw_reset) */ 1237 state->ts_cmd_regs.sw_reset = (uint32_t *) 1238 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_SW_RESET_OFFSET); 1239 1240 /* Setup Tavor Clear Interrupt register (clr_int) */ 1241 state->ts_cmd_regs.clr_int = (uint64_t *) 1242 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_CLR_INT_OFFSET); 1243 1244 /* Initialize the Phase1 Tavor configuration profile */ 1245 status = tavor_cfg_profile_init_phase1(state); 1246 if (status != DDI_SUCCESS) { 1247 tavor_hw_fini(state, cleanup); 1248 TNF_PROBE_0(tavor_hw_init_cfginit_fail, TAVOR_TNF_ERROR, ""); 1249 TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_cfginit_fail"); 1250 TAVOR_TNF_EXIT(tavor_hw_init); 1251 return (DDI_FAILURE); 1252 } 1253 cleanup = TAVOR_DRV_CLEANUP_LEVEL4; 1254 1255 /* Do a software reset of the Tavor HW to ensure proper state */ 1256 status = tavor_sw_reset(state); 1257 if (status != TAVOR_CMD_SUCCESS) { 1258 tavor_hw_fini(state, cleanup); 1259 TNF_PROBE_0(tavor_hw_init_sw_reset_fail, TAVOR_TNF_ERROR, ""); 1260 TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_sw_reset_fail"); 1261 TAVOR_TNF_EXIT(tavor_hw_init); 1262 return (DDI_FAILURE); 1263 } 1264 1265 /* Post the SYS_EN command to start the hardware */ 1266 status = tavor_sys_en_cmd_post(state, TAVOR_CMD_SYS_EN_NORMAL, 1267 &errorcode, TAVOR_CMD_NOSLEEP_SPIN); 1268 if (status != TAVOR_CMD_SUCCESS) { 1269 if ((status == TAVOR_CMD_BAD_NVMEM) || 1270 (status == TAVOR_CMD_DDR_MEM_ERR)) { 1271 cmn_err(CE_CONT, "Tavor: SYS_EN command failed: 0x%x " 1272 "0x%" PRIx64 " (invalid firmware image?)\n", 1273 status, errorcode); 1274 } else { 1275 cmn_err(CE_CONT, "Tavor: SYS_EN command failed: 0x%x " 1276 "0x%" PRIx64 "\n", status, errorcode); 1277 } 1278 tavor_hw_fini(state, cleanup); 1279 TNF_PROBE_0(tavor_hw_init_sys_en_cmd_fail, 1280 TAVOR_TNF_ERROR, ""); 1281 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1282 "hw_init_sys_en_cmd_fail"); 1283 TAVOR_TNF_EXIT(tavor_hw_init); 1284 return (DDI_FAILURE); 1285 } 1286 cleanup = TAVOR_DRV_CLEANUP_LEVEL5; 1287 1288 /* First phase of init for Tavor configuration/resources */ 1289 status = tavor_rsrc_init_phase1(state); 1290 if (status != DDI_SUCCESS) { 1291 tavor_hw_fini(state, cleanup); 1292 TNF_PROBE_0(tavor_hw_init_rsrcinit1_fail, TAVOR_TNF_ERROR, ""); 1293 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1294 "hw_init_rsrcinit1_fail"); 1295 TAVOR_TNF_EXIT(tavor_hw_init); 1296 return (DDI_FAILURE); 1297 } 1298 cleanup = TAVOR_DRV_CLEANUP_LEVEL6; 1299 1300 /* Query the DDR properties (e.g. total DDR size) */ 1301 status = tavor_cmn_query_cmd_post(state, QUERY_DDR, 0, 1302 &state->ts_ddr, sizeof (tavor_hw_queryddr_t), 1303 TAVOR_CMD_NOSLEEP_SPIN); 1304 if (status != TAVOR_CMD_SUCCESS) { 1305 cmn_err(CE_CONT, "Tavor: QUERY_DDR command failed: %08x\n", 1306 status); 1307 tavor_hw_fini(state, cleanup); 1308 TNF_PROBE_0(tavor_hw_init_query_ddr_cmd_fail, 1309 TAVOR_TNF_ERROR, ""); 1310 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1311 "hw_init_query_ddr_cmd_fail"); 1312 TAVOR_TNF_EXIT(tavor_hw_init); 1313 return (DDI_FAILURE); 1314 } 1315 1316 /* Figure out how big the firmware image (in DDR) is */ 1317 status = tavor_cmn_query_cmd_post(state, QUERY_FW, 0, &state->ts_fw, 1318 sizeof (tavor_hw_queryfw_t), TAVOR_CMD_NOSLEEP_SPIN); 1319 if (status != TAVOR_CMD_SUCCESS) { 1320 cmn_err(CE_CONT, "Tavor: QUERY_FW command failed: %08x\n", 1321 status); 1322 tavor_hw_fini(state, cleanup); 1323 TNF_PROBE_0(tavor_hw_init_query_fw_cmd_fail, 1324 TAVOR_TNF_ERROR, ""); 1325 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1326 "hw_init_query_fw_cmd_fail"); 1327 TAVOR_TNF_EXIT(tavor_hw_init); 1328 return (DDI_FAILURE); 1329 } 1330 1331 if (tavor_fix_error_buf(state) != DDI_SUCCESS) { 1332 tavor_hw_fini(state, cleanup); 1333 TNF_PROBE_0(tavor_hw_init_fixerrorbuf_fail, 1334 TAVOR_TNF_ERROR, ""); 1335 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1336 "hw_init_fixerrorbuf_fail"); 1337 TAVOR_TNF_EXIT(tavor_hw_init); 1338 return (DDI_FAILURE); 1339 } 1340 1341 /* Validate that the FW version is appropriate */ 1342 status = tavor_fw_version_check(state); 1343 if (status != DDI_SUCCESS) { 1344 if (state->ts_operational_mode == TAVOR_HCA_MODE) { 1345 cmn_err(CE_CONT, "Unsupported Tavor FW version: " 1346 "expected: %04d.%04d.%04d, " 1347 "actual: %04d.%04d.%04d\n", 1348 TAVOR_FW_VER_MAJOR, 1349 TAVOR_FW_VER_MINOR, 1350 TAVOR_FW_VER_SUBMINOR, 1351 state->ts_fw.fw_rev_major, 1352 state->ts_fw.fw_rev_minor, 1353 state->ts_fw.fw_rev_subminor); 1354 } else if (state->ts_operational_mode == TAVOR_COMPAT_MODE) { 1355 cmn_err(CE_CONT, "Unsupported Tavor Compat FW version: " 1356 "expected: %04d.%04d.%04d, " 1357 "actual: %04d.%04d.%04d\n", 1358 TAVOR_COMPAT_FW_VER_MAJOR, 1359 TAVOR_COMPAT_FW_VER_MINOR, 1360 TAVOR_COMPAT_FW_VER_SUBMINOR, 1361 state->ts_fw.fw_rev_major, 1362 state->ts_fw.fw_rev_minor, 1363 state->ts_fw.fw_rev_subminor); 1364 } else { 1365 cmn_err(CE_CONT, "Unsupported FW version: " 1366 "%04d.%04d.%04d\n", 1367 state->ts_fw.fw_rev_major, 1368 state->ts_fw.fw_rev_minor, 1369 state->ts_fw.fw_rev_subminor); 1370 } 1371 tavor_hw_fini(state, cleanup); 1372 TNF_PROBE_0(tavor_hw_init_checkfwver_fail, 1373 TAVOR_TNF_ERROR, ""); 1374 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1375 "hw_init_checkfwver_fail"); 1376 TAVOR_TNF_EXIT(tavor_hw_init); 1377 return (DDI_FAILURE); 1378 } 1379 1380 drv_usecwait(10); 1381 retries = 1000; /* retry up to 1 second before giving up */ 1382 retry: 1383 /* Call MOD_STAT_CFG to setup SRQ support (or disable) */ 1384 status = tavor_mod_stat_cfg_cmd_post(state); 1385 if (status != DDI_SUCCESS) { 1386 if (retries > 0) { 1387 drv_usecwait(1000); 1388 retries--; 1389 goto retry; 1390 } 1391 cmn_err(CE_CONT, "Tavor: MOD_STAT_CFG command failed: %08x\n", 1392 status); 1393 tavor_hw_fini(state, cleanup); 1394 TNF_PROBE_0(tavor_hw_init_mod_stat_cfg_cmd_fail, 1395 TAVOR_TNF_ERROR, ""); 1396 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1397 "hw_init_mod_stat_cfg_cmd_fail"); 1398 TAVOR_TNF_EXIT(tavor_hw_init); 1399 return (DDI_FAILURE); 1400 } 1401 1402 /* Figure out Tavor device limits */ 1403 status = tavor_cmn_query_cmd_post(state, QUERY_DEV_LIM, 0, 1404 &state->ts_devlim, sizeof (tavor_hw_querydevlim_t), 1405 TAVOR_CMD_NOSLEEP_SPIN); 1406 if (status != TAVOR_CMD_SUCCESS) { 1407 cmn_err(CE_CONT, "Tavor: QUERY_DEV_LIM command failed: %08x\n", 1408 status); 1409 tavor_hw_fini(state, cleanup); 1410 TNF_PROBE_0(tavor_hw_init_query_devlim_cmd_fail, 1411 TAVOR_TNF_ERROR, ""); 1412 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1413 "hw_init_query_devlim_cmd_fail"); 1414 TAVOR_TNF_EXIT(tavor_hw_init); 1415 return (DDI_FAILURE); 1416 } 1417 1418 /* Initialize the Phase2 Tavor configuration profile */ 1419 status = tavor_cfg_profile_init_phase2(state); 1420 if (status != DDI_SUCCESS) { 1421 tavor_hw_fini(state, cleanup); 1422 TNF_PROBE_0(tavor_hw_init_cfginit2_fail, TAVOR_TNF_ERROR, ""); 1423 TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_cfginit2_fail"); 1424 TAVOR_TNF_EXIT(tavor_hw_init); 1425 return (DDI_FAILURE); 1426 } 1427 1428 /* Second phase of init for Tavor configuration/resources */ 1429 status = tavor_rsrc_init_phase2(state); 1430 if (status != DDI_SUCCESS) { 1431 tavor_hw_fini(state, cleanup); 1432 TNF_PROBE_0(tavor_hw_init_rsrcinit2_fail, TAVOR_TNF_ERROR, ""); 1433 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1434 "hw_init_rsrcinit2_fail"); 1435 TAVOR_TNF_EXIT(tavor_hw_init); 1436 return (DDI_FAILURE); 1437 } 1438 cleanup = TAVOR_DRV_CLEANUP_LEVEL7; 1439 1440 /* Miscellaneous query information */ 1441 status = tavor_cmn_query_cmd_post(state, QUERY_ADAPTER, 0, 1442 &state->ts_adapter, sizeof (tavor_hw_queryadapter_t), 1443 TAVOR_CMD_NOSLEEP_SPIN); 1444 if (status != TAVOR_CMD_SUCCESS) { 1445 cmn_err(CE_CONT, "Tavor: QUERY_ADAPTER command failed: %08x\n", 1446 status); 1447 tavor_hw_fini(state, cleanup); 1448 TNF_PROBE_0(tavor_hw_init_query_adapter_cmd_fail, 1449 TAVOR_TNF_ERROR, ""); 1450 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1451 "hw_init_query_adapter_cmd_fail"); 1452 TAVOR_TNF_EXIT(tavor_hw_init); 1453 return (DDI_FAILURE); 1454 } 1455 1456 /* Prepare configuration for Tavor INIT_HCA command */ 1457 tavor_hca_config_setup(state, &state->ts_hcaparams); 1458 1459 /* Post command to init Tavor HCA */ 1460 status = tavor_init_hca_cmd_post(state, &state->ts_hcaparams, 1461 TAVOR_CMD_NOSLEEP_SPIN); 1462 if (status != TAVOR_CMD_SUCCESS) { 1463 cmn_err(CE_CONT, "Tavor: INIT_HCA command failed: %08x\n", 1464 status); 1465 tavor_hw_fini(state, cleanup); 1466 TNF_PROBE_0(tavor_hw_init_init_hca_cmd_fail, 1467 TAVOR_TNF_ERROR, ""); 1468 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1469 "hw_init_init_hca_cmd_fail"); 1470 TAVOR_TNF_EXIT(tavor_hw_init); 1471 return (DDI_FAILURE); 1472 } 1473 cleanup = TAVOR_DRV_CLEANUP_LEVEL8; 1474 1475 /* Allocate protection domain (PD) for Tavor internal use */ 1476 status = tavor_pd_alloc(state, &state->ts_pdhdl_internal, TAVOR_SLEEP); 1477 if (status != DDI_SUCCESS) { 1478 tavor_hw_fini(state, cleanup); 1479 TNF_PROBE_0(tavor_hw_init_internal_pd_alloc_fail, 1480 TAVOR_TNF_ERROR, ""); 1481 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1482 "hw_init_internal_pd_alloc_fail"); 1483 TAVOR_TNF_EXIT(tavor_hw_init); 1484 return (DDI_FAILURE); 1485 } 1486 cleanup = TAVOR_DRV_CLEANUP_LEVEL9; 1487 1488 /* Setup Tavor internal UAR pages (0 and 1) */ 1489 status = tavor_internal_uarpgs_init(state); 1490 if (status != DDI_SUCCESS) { 1491 tavor_hw_fini(state, cleanup); 1492 TNF_PROBE_0(tavor_hw_init_internal_uarpgs_alloc_fail, 1493 TAVOR_TNF_ERROR, ""); 1494 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1495 "hw_init_internal_uarpgs_alloc_fail"); 1496 TAVOR_TNF_EXIT(tavor_hw_init); 1497 return (DDI_FAILURE); 1498 } 1499 cleanup = TAVOR_DRV_CLEANUP_LEVEL10; 1500 1501 /* Query and initialize the Tavor interrupt/MSI information */ 1502 status = tavor_intr_or_msi_init(state); 1503 if (status != DDI_SUCCESS) { 1504 tavor_hw_fini(state, cleanup); 1505 TNF_PROBE_0(tavor_intr_or_msi_init_fail, 1506 TAVOR_TNF_ERROR, ""); 1507 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1508 "intr_or_msi_init_fail"); 1509 TAVOR_TNF_EXIT(tavor_hw_init); 1510 return (DDI_FAILURE); 1511 } 1512 cleanup = TAVOR_DRV_CLEANUP_LEVEL11; 1513 1514 /* Setup all of the Tavor EQs */ 1515 status = tavor_eq_init_all(state); 1516 if (status != DDI_SUCCESS) { 1517 tavor_hw_fini(state, cleanup); 1518 TNF_PROBE_0(tavor_hw_init_eqinitall_fail, TAVOR_TNF_ERROR, ""); 1519 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1520 "hw_init_eqinitall_fail"); 1521 TAVOR_TNF_EXIT(tavor_hw_init); 1522 return (DDI_FAILURE); 1523 } 1524 cleanup = TAVOR_DRV_CLEANUP_LEVEL12; 1525 1526 /* Set aside contexts for QP0 and QP1 */ 1527 status = tavor_special_qp_contexts_reserve(state); 1528 if (status != DDI_SUCCESS) { 1529 tavor_hw_fini(state, cleanup); 1530 TNF_PROBE_0(tavor_hw_init_reserve_special_qp_fail, 1531 TAVOR_TNF_ERROR, ""); 1532 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1533 "hw_init_reserve_special_qp_fail"); 1534 TAVOR_TNF_EXIT(tavor_hw_init); 1535 return (DDI_FAILURE); 1536 } 1537 cleanup = TAVOR_DRV_CLEANUP_LEVEL13; 1538 1539 /* Initialize for multicast group handling */ 1540 status = tavor_mcg_init(state); 1541 if (status != DDI_SUCCESS) { 1542 tavor_hw_fini(state, cleanup); 1543 TNF_PROBE_0(tavor_hw_init_mcg_init_fail, TAVOR_TNF_ERROR, ""); 1544 TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_mcg_init_fail"); 1545 TAVOR_TNF_EXIT(tavor_hw_init); 1546 return (DDI_FAILURE); 1547 } 1548 cleanup = TAVOR_DRV_CLEANUP_LEVEL14; 1549 1550 /* Initialize the Tavor IB port(s) */ 1551 status = tavor_hca_port_init(state); 1552 if (status != DDI_SUCCESS) { 1553 tavor_hw_fini(state, cleanup); 1554 TNF_PROBE_0(tavor_hw_init_hca_port_init_fail, 1555 TAVOR_TNF_ERROR, ""); 1556 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1557 "hw_init_hca_port_init_fail"); 1558 TAVOR_TNF_EXIT(tavor_hw_init); 1559 return (DDI_FAILURE); 1560 } 1561 cleanup = TAVOR_DRV_CLEANUP_ALL; 1562 1563 /* Determine NodeGUID and SystemImageGUID */ 1564 status = tavor_getnodeinfo_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN, 1565 &nodeinfo); 1566 if (status != TAVOR_CMD_SUCCESS) { 1567 cmn_err(CE_CONT, "Tavor: GetNodeInfo command failed: %08x\n", 1568 status); 1569 tavor_hw_fini(state, cleanup); 1570 TNF_PROBE_0(tavor_hw_init_getnodeinfo_cmd_fail, 1571 TAVOR_TNF_ERROR, ""); 1572 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1573 "hw_init_getnodeinfo_cmd_fail"); 1574 TAVOR_TNF_EXIT(tavor_hw_init); 1575 return (DDI_FAILURE); 1576 } 1577 1578 /* 1579 * If the NodeGUID value was set in OBP properties, then we use that 1580 * value. But we still print a message if the value we queried from 1581 * firmware does not match this value. 1582 * 1583 * Otherwise if OBP value is not set then we use the value from 1584 * firmware unconditionally. 1585 */ 1586 if (state->ts_cfg_profile->cp_nodeguid) { 1587 state->ts_nodeguid = state->ts_cfg_profile->cp_nodeguid; 1588 } else { 1589 state->ts_nodeguid = nodeinfo.NodeGUID; 1590 } 1591 1592 if (state->ts_nodeguid != nodeinfo.NodeGUID) { 1593 cmn_err(CE_NOTE, "!NodeGUID value queried from firmware " 1594 "does not match value set by device property"); 1595 } 1596 1597 /* 1598 * If the SystemImageGUID value was set in OBP properties, then we use 1599 * that value. But we still print a message if the value we queried 1600 * from firmware does not match this value. 1601 * 1602 * Otherwise if OBP value is not set then we use the value from 1603 * firmware unconditionally. 1604 */ 1605 if (state->ts_cfg_profile->cp_sysimgguid) { 1606 state->ts_sysimgguid = state->ts_cfg_profile->cp_sysimgguid; 1607 } else { 1608 state->ts_sysimgguid = nodeinfo.SystemImageGUID; 1609 } 1610 1611 if (state->ts_sysimgguid != nodeinfo.SystemImageGUID) { 1612 cmn_err(CE_NOTE, "!SystemImageGUID value queried from firmware " 1613 "does not match value set by device property"); 1614 } 1615 1616 /* Get NodeDescription */ 1617 status = tavor_getnodedesc_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN, 1618 (sm_nodedesc_t *)&state->ts_nodedesc); 1619 if (status != TAVOR_CMD_SUCCESS) { 1620 cmn_err(CE_CONT, "Tavor: GetNodeDesc command failed: %08x\n", 1621 status); 1622 tavor_hw_fini(state, cleanup); 1623 TNF_PROBE_0(tavor_hw_init_getnodedesc_cmd_fail, 1624 TAVOR_TNF_ERROR, ""); 1625 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1626 "hw_init_getnodedesc_cmd_fail"); 1627 TAVOR_TNF_EXIT(tavor_hw_init); 1628 return (DDI_FAILURE); 1629 } 1630 1631 TAVOR_TNF_EXIT(tavor_hw_init); 1632 return (DDI_SUCCESS); 1633 } 1634 1635 1636 /* 1637 * tavor_hw_fini() 1638 * Context: Only called from attach() and/or detach() path contexts 1639 */ 1640 static void 1641 tavor_hw_fini(tavor_state_t *state, tavor_drv_cleanup_level_t cleanup) 1642 { 1643 uint_t num_ports; 1644 int status; 1645 1646 TAVOR_TNF_ENTER(tavor_hw_fini); 1647 1648 switch (cleanup) { 1649 /* 1650 * If we add more driver initialization steps that should be cleaned 1651 * up here, we need to ensure that TAVOR_DRV_CLEANUP_ALL is still the 1652 * first entry (i.e. corresponds to the last init step). 1653 */ 1654 case TAVOR_DRV_CLEANUP_ALL: 1655 /* Shutdown the Tavor IB port(s) */ 1656 num_ports = state->ts_cfg_profile->cp_num_ports; 1657 (void) tavor_hca_ports_shutdown(state, num_ports); 1658 /* FALLTHROUGH */ 1659 1660 case TAVOR_DRV_CLEANUP_LEVEL14: 1661 /* Teardown resources used for multicast group handling */ 1662 tavor_mcg_fini(state); 1663 /* FALLTHROUGH */ 1664 1665 case TAVOR_DRV_CLEANUP_LEVEL13: 1666 /* Unreserve the special QP contexts */ 1667 tavor_special_qp_contexts_unreserve(state); 1668 /* FALLTHROUGH */ 1669 1670 case TAVOR_DRV_CLEANUP_LEVEL12: 1671 /* 1672 * Attempt to teardown all event queues (EQ). If we fail 1673 * here then print a warning message and return. Something 1674 * (either in HW or SW) has gone seriously wrong. 1675 */ 1676 status = tavor_eq_fini_all(state); 1677 if (status != DDI_SUCCESS) { 1678 TAVOR_WARNING(state, "failed to teardown EQs"); 1679 TNF_PROBE_0(tavor_hw_fini_eqfiniall_fail, 1680 TAVOR_TNF_ERROR, ""); 1681 TAVOR_TNF_EXIT(tavor_hw_fini); 1682 return; 1683 } 1684 /* FALLTHROUGH */ 1685 1686 case TAVOR_DRV_CLEANUP_LEVEL11: 1687 status = tavor_intr_or_msi_fini(state); 1688 if (status != DDI_SUCCESS) { 1689 TAVOR_WARNING(state, "failed to free intr/MSI"); 1690 TNF_PROBE_0(tavor_hw_fini_intrmsifini_fail, 1691 TAVOR_TNF_ERROR, ""); 1692 TAVOR_TNF_EXIT(tavor_hw_fini); 1693 return; 1694 } 1695 /* FALLTHROUGH */ 1696 1697 case TAVOR_DRV_CLEANUP_LEVEL10: 1698 /* Free the resources for the Tavor internal UAR pages */ 1699 tavor_internal_uarpgs_fini(state); 1700 /* FALLTHROUGH */ 1701 1702 case TAVOR_DRV_CLEANUP_LEVEL9: 1703 /* 1704 * Free the PD that was used internally by Tavor software. If 1705 * we fail here then print a warning and return. Something 1706 * (probably software-related, but perhaps HW) has gone wrong. 1707 */ 1708 status = tavor_pd_free(state, &state->ts_pdhdl_internal); 1709 if (status != DDI_SUCCESS) { 1710 TAVOR_WARNING(state, "failed to free internal PD"); 1711 TNF_PROBE_0(tavor_hw_fini_internal_pd_free_fail, 1712 TAVOR_TNF_ERROR, ""); 1713 TAVOR_TNF_EXIT(tavor_hw_fini); 1714 return; 1715 } 1716 /* FALLTHROUGH */ 1717 1718 case TAVOR_DRV_CLEANUP_LEVEL8: 1719 /* 1720 * Post the CLOSE_HCA command to Tavor firmware. If we fail 1721 * here then print a warning and return. Something (either in 1722 * HW or SW) has gone seriously wrong. 1723 */ 1724 status = tavor_close_hca_cmd_post(state, 1725 TAVOR_CMD_NOSLEEP_SPIN); 1726 if (status != TAVOR_CMD_SUCCESS) { 1727 TAVOR_WARNING(state, "failed to shutdown HCA"); 1728 TNF_PROBE_0(tavor_hw_fini_closehcacmd_fail, 1729 TAVOR_TNF_ERROR, ""); 1730 TAVOR_TNF_EXIT(tavor_hw_fini); 1731 return; 1732 } 1733 /* FALLTHROUGH */ 1734 1735 case TAVOR_DRV_CLEANUP_LEVEL7: 1736 /* Cleanup all the phase2 resources first */ 1737 tavor_rsrc_fini(state, TAVOR_RSRC_CLEANUP_ALL); 1738 /* FALLTHROUGH */ 1739 1740 case TAVOR_DRV_CLEANUP_LEVEL6: 1741 /* Then cleanup the phase1 resources */ 1742 tavor_rsrc_fini(state, TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE); 1743 /* FALLTHROUGH */ 1744 1745 case TAVOR_DRV_CLEANUP_LEVEL5: 1746 /* 1747 * Post the SYS_DIS command to Tavor firmware to shut 1748 * everything down again. If we fail here then print a 1749 * warning and return. Something (probably in HW, but maybe 1750 * in SW) has gone seriously wrong. 1751 */ 1752 status = tavor_sys_dis_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN); 1753 if (status != TAVOR_CMD_SUCCESS) { 1754 TAVOR_WARNING(state, "failed to shutdown hardware"); 1755 TNF_PROBE_0(tavor_hw_fini_sys_dis_fail, 1756 TAVOR_TNF_ERROR, ""); 1757 TAVOR_TNF_EXIT(tavor_hw_fini); 1758 return; 1759 } 1760 /* FALLTHROUGH */ 1761 1762 case TAVOR_DRV_CLEANUP_LEVEL4: 1763 /* Teardown any resources allocated for the config profile */ 1764 tavor_cfg_profile_fini(state); 1765 /* FALLTHROUGH */ 1766 1767 case TAVOR_DRV_CLEANUP_LEVEL3: 1768 ddi_regs_map_free(&state->ts_reg_ddrhdl); 1769 /* FALLTHROUGH */ 1770 1771 case TAVOR_DRV_CLEANUP_LEVEL2: 1772 ddi_regs_map_free(&state->ts_reg_uarhdl); 1773 /* FALLTHROUGH */ 1774 1775 case TAVOR_DRV_CLEANUP_LEVEL1: 1776 case TAVOR_DRV_CLEANUP_LEVEL0: 1777 /* 1778 * LEVEL1 and LEVEL0 resources are freed in 1779 * tavor_drv_fini2(). 1780 */ 1781 break; 1782 1783 default: 1784 TAVOR_WARNING(state, "unexpected driver cleanup level"); 1785 TNF_PROBE_0(tavor_hw_fini_default_fail, TAVOR_TNF_ERROR, ""); 1786 TAVOR_TNF_EXIT(tavor_hw_fini); 1787 return; 1788 } 1789 1790 TAVOR_TNF_EXIT(tavor_hw_fini); 1791 } 1792 1793 1794 /* 1795 * tavor_soft_state_init() 1796 * Context: Only called from attach() path context 1797 */ 1798 static int 1799 tavor_soft_state_init(tavor_state_t *state) 1800 { 1801 ibt_hca_attr_t *hca_attr; 1802 uint64_t maxval, val; 1803 ibt_hca_flags_t caps = IBT_HCA_NO_FLAGS; 1804 int status; 1805 1806 TAVOR_TNF_ENTER(tavor_soft_state_init); 1807 1808 /* 1809 * The ibc_hca_info_t struct is passed to the IBTF. This is the 1810 * routine where we initialize it. Many of the init values come from 1811 * either configuration variables or successful queries of the Tavor 1812 * hardware abilities 1813 */ 1814 state->ts_ibtfinfo.hca_ci_vers = IBCI_V3; 1815 state->ts_ibtfinfo.hca_dip = state->ts_dip; 1816 state->ts_ibtfinfo.hca_handle = (ibc_hca_hdl_t)state; 1817 state->ts_ibtfinfo.hca_ops = &tavor_ibc_ops; 1818 1819 hca_attr = kmem_zalloc(sizeof (ibt_hca_attr_t), KM_SLEEP); 1820 state->ts_ibtfinfo.hca_attr = hca_attr; 1821 1822 hca_attr->hca_fw_major_version = state->ts_fw.fw_rev_major; 1823 hca_attr->hca_fw_minor_version = state->ts_fw.fw_rev_minor; 1824 hca_attr->hca_fw_micro_version = state->ts_fw.fw_rev_subminor; 1825 1826 /* 1827 * Determine HCA capabilities: 1828 * No default support for IBT_HCA_RD, IBT_HCA_RAW_MULTICAST, 1829 * IBT_HCA_ATOMICS_GLOBAL, IBT_HCA_RESIZE_CHAN, IBT_HCA_INIT_TYPE, 1830 * or IBT_HCA_SHUTDOWN_PORT 1831 * But IBT_HCA_AH_PORT_CHECK, IBT_HCA_SQD_RTS_PORT, IBT_HCA_SI_GUID, 1832 * IBT_HCA_RNR_NAK, and IBT_HCA_CURRENT_QP_STATE are always 1833 * supported 1834 * All other features are conditionally supported, depending on the 1835 * status return by the Tavor HCA (in QUERY_DEV_LIM) 1836 */ 1837 if (state->ts_devlim.ud_multi) { 1838 caps |= IBT_HCA_UD_MULTICAST; 1839 } 1840 if (state->ts_devlim.atomic) { 1841 caps |= IBT_HCA_ATOMICS_HCA; 1842 } 1843 if (state->ts_devlim.apm) { 1844 caps |= IBT_HCA_AUTO_PATH_MIG; 1845 } 1846 if (state->ts_devlim.pkey_v) { 1847 caps |= IBT_HCA_PKEY_CNTR; 1848 } 1849 if (state->ts_devlim.qkey_v) { 1850 caps |= IBT_HCA_QKEY_CNTR; 1851 } 1852 if (state->ts_cfg_profile->cp_srq_enable) { 1853 caps |= IBT_HCA_SRQ | IBT_HCA_RESIZE_SRQ; 1854 } 1855 caps |= (IBT_HCA_AH_PORT_CHECK | IBT_HCA_SQD_SQD_PORT | 1856 IBT_HCA_SI_GUID | IBT_HCA_RNR_NAK | IBT_HCA_CURRENT_QP_STATE | 1857 IBT_HCA_PORT_UP | IBT_HCA_SQD_STATE); 1858 hca_attr->hca_flags = caps; 1859 1860 /* Determine VendorID, DeviceID, and revision ID */ 1861 hca_attr->hca_vendor_id = state->ts_adapter.vendor_id; 1862 hca_attr->hca_device_id = state->ts_adapter.device_id; 1863 hca_attr->hca_version_id = state->ts_adapter.rev_id; 1864 1865 /* 1866 * Determine number of available QPs and max QP size. Number of 1867 * available QPs is determined by subtracting the number of 1868 * "reserved QPs" (i.e. reserved for firmware use) from the 1869 * total number configured. 1870 */ 1871 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_qp); 1872 hca_attr->hca_max_qp = val - ((uint64_t)1 << 1873 state->ts_devlim.log_rsvd_qp); 1874 maxval = ((uint64_t)1 << state->ts_devlim.log_max_qp_sz); 1875 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_qp_sz); 1876 if (val > maxval) { 1877 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 1878 TNF_PROBE_2(tavor_soft_state_init_maxqpsz_toobig_fail, 1879 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max QP size " 1880 "exceeds device maximum", tnf_uint, maxsz, maxval); 1881 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1882 "soft_state_init_maxqpsz_toobig_fail"); 1883 TAVOR_TNF_EXIT(tavor_soft_state_init); 1884 return (DDI_FAILURE); 1885 } 1886 hca_attr->hca_max_qp_sz = val; 1887 1888 /* Determine max scatter-gather size in WQEs */ 1889 maxval = state->ts_devlim.max_sg; 1890 val = state->ts_cfg_profile->cp_wqe_max_sgl; 1891 if (val > maxval) { 1892 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 1893 TNF_PROBE_2(tavor_soft_state_init_toomanysgl_fail, 1894 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of sgl " 1895 "exceeds device maximum", tnf_uint, maxsgl, maxval); 1896 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1897 "soft_state_init_toomanysgl_fail"); 1898 TAVOR_TNF_EXIT(tavor_soft_state_init); 1899 return (DDI_FAILURE); 1900 } 1901 /* If the rounded value for max SGL is too large, cap it */ 1902 if (state->ts_cfg_profile->cp_wqe_real_max_sgl > maxval) { 1903 state->ts_cfg_profile->cp_wqe_real_max_sgl = maxval; 1904 val = maxval; 1905 } else { 1906 val = state->ts_cfg_profile->cp_wqe_real_max_sgl; 1907 } 1908 1909 hca_attr->hca_max_sgl = val; 1910 hca_attr->hca_max_rd_sgl = 0; /* zero because RD is unsupported */ 1911 1912 /* 1913 * Determine number of available CQs and max CQ size. Number of 1914 * available CQs is determined by subtracting the number of 1915 * "reserved CQs" (i.e. reserved for firmware use) from the 1916 * total number configured. 1917 */ 1918 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_cq); 1919 hca_attr->hca_max_cq = val - ((uint64_t)1 << 1920 state->ts_devlim.log_rsvd_cq); 1921 maxval = ((uint64_t)1 << state->ts_devlim.log_max_cq_sz); 1922 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_cq_sz) - 1; 1923 if (val > maxval) { 1924 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 1925 TNF_PROBE_2(tavor_soft_state_init_maxcqsz_toobig_fail, 1926 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max CQ size " 1927 "exceeds device maximum", tnf_uint, maxsz, maxval); 1928 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1929 "soft_state_init_maxcqsz_toobig_fail"); 1930 TAVOR_TNF_EXIT(tavor_soft_state_init); 1931 return (DDI_FAILURE); 1932 } 1933 hca_attr->hca_max_cq_sz = val; 1934 1935 /* 1936 * Determine number of available SRQs and max SRQ size. Number of 1937 * available SRQs is determined by subtracting the number of 1938 * "reserved SRQs" (i.e. reserved for firmware use) from the 1939 * total number configured. 1940 */ 1941 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_srq); 1942 hca_attr->hca_max_srqs = val - ((uint64_t)1 << 1943 state->ts_devlim.log_rsvd_srq); 1944 maxval = ((uint64_t)1 << state->ts_devlim.log_max_srq_sz); 1945 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_srq_sz); 1946 1947 if (val > maxval) { 1948 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 1949 TNF_PROBE_2(tavor_soft_state_init_maxsrqsz_toobig_fail, 1950 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max SRQ size " 1951 "exceeds device maximum", tnf_uint, maxsz, maxval); 1952 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1953 "soft_state_init_maxsrqsz_toobig_fail"); 1954 TAVOR_TNF_EXIT(tavor_soft_state_init); 1955 return (DDI_FAILURE); 1956 } 1957 hca_attr->hca_max_srqs_sz = val; 1958 1959 val = state->ts_cfg_profile->cp_srq_max_sgl; 1960 maxval = state->ts_devlim.max_sg; 1961 if (val > maxval) { 1962 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 1963 TNF_PROBE_2(tavor_soft_state_init_toomanysrqsgl_fail, 1964 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of srq " 1965 "sgl exceeds device maximum", tnf_uint, maxsgl, maxval); 1966 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1967 "soft_state_init_toomanysrqsgl_fail"); 1968 TAVOR_TNF_EXIT(tavor_soft_state_init); 1969 return (DDI_FAILURE); 1970 } 1971 hca_attr->hca_max_srq_sgl = val; 1972 1973 /* 1974 * Determine supported HCA page sizes 1975 * XXX 1976 * For now we simply return the system pagesize as the only supported 1977 * pagesize 1978 */ 1979 hca_attr->hca_page_sz = ((PAGESIZE == (1 << 13)) ? IBT_PAGE_8K : 1980 IBT_PAGE_4K); 1981 1982 /* 1983 * Determine number of available MemReg, MemWin, and their max size. 1984 * Number of available MRs and MWs is determined by subtracting 1985 * the number of "reserved MPTs" (i.e. reserved for firmware use) 1986 * from the total number configured for each. 1987 */ 1988 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_mpt); 1989 hca_attr->hca_max_memr = val - ((uint64_t)1 << 1990 state->ts_devlim.log_rsvd_mpt); 1991 hca_attr->hca_max_mem_win = val - ((uint64_t)1 << 1992 state->ts_devlim.log_rsvd_mpt); 1993 maxval = state->ts_devlim.log_max_mrw_sz; 1994 val = state->ts_cfg_profile->cp_log_max_mrw_sz; 1995 if (val > maxval) { 1996 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 1997 TNF_PROBE_2(tavor_soft_state_init_maxmrwsz_toobig_fail, 1998 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max mrw size " 1999 "exceeds device maximum", tnf_uint, maxsz, maxval); 2000 TAVOR_ATTACH_MSG(state->ts_attach_buf, 2001 "soft_state_init_maxmrwsz_toobig_fail"); 2002 TAVOR_TNF_EXIT(tavor_soft_state_init); 2003 return (DDI_FAILURE); 2004 } 2005 hca_attr->hca_max_memr_len = ((uint64_t)1 << val); 2006 2007 /* Determine RDMA/Atomic properties */ 2008 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_rdb); 2009 hca_attr->hca_max_rsc = val; 2010 val = state->ts_cfg_profile->cp_hca_max_rdma_in_qp; 2011 hca_attr->hca_max_rdma_in_qp = val; 2012 val = state->ts_cfg_profile->cp_hca_max_rdma_out_qp; 2013 hca_attr->hca_max_rdma_out_qp = val; 2014 hca_attr->hca_max_rdma_in_ee = 0; 2015 hca_attr->hca_max_rdma_out_ee = 0; 2016 2017 /* 2018 * Determine maximum number of raw IPv6 and Ether QPs. Set to 0 2019 * because neither type of raw QP is supported 2020 */ 2021 hca_attr->hca_max_ipv6_qp = 0; 2022 hca_attr->hca_max_ether_qp = 0; 2023 2024 /* Determine max number of MCGs and max QP-per-MCG */ 2025 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_qp); 2026 hca_attr->hca_max_mcg_qps = val; 2027 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_mcg); 2028 hca_attr->hca_max_mcg = val; 2029 val = state->ts_cfg_profile->cp_num_qp_per_mcg; 2030 hca_attr->hca_max_qp_per_mcg = val; 2031 2032 /* Determine max number partitions (i.e. PKeys) */ 2033 maxval = ((uint64_t)1 << state->ts_devlim.log_max_pkey); 2034 val = ((uint64_t)state->ts_cfg_profile->cp_num_ports << 2035 state->ts_cfg_profile->cp_log_max_pkeytbl); 2036 2037 if (val > maxval) { 2038 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 2039 TNF_PROBE_2(tavor_soft_state_init_toomanypkey_fail, 2040 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of PKeys " 2041 "exceeds device maximum", tnf_uint, maxpkey, maxval); 2042 TAVOR_ATTACH_MSG(state->ts_attach_buf, 2043 "soft_state_init_toomanypkey_fail"); 2044 TAVOR_TNF_EXIT(tavor_soft_state_init); 2045 return (DDI_FAILURE); 2046 } 2047 hca_attr->hca_max_partitions = val; 2048 2049 /* Determine number of ports */ 2050 maxval = state->ts_devlim.num_ports; 2051 val = state->ts_cfg_profile->cp_num_ports; 2052 if ((val > maxval) || (val == 0)) { 2053 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 2054 TNF_PROBE_2(tavor_soft_state_init_toomanyports_fail, 2055 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of ports " 2056 "exceeds device maximum", tnf_uint, maxports, maxval); 2057 TAVOR_ATTACH_MSG(state->ts_attach_buf, 2058 "soft_state_init_toomanyports_fail"); 2059 TAVOR_TNF_EXIT(tavor_soft_state_init); 2060 return (DDI_FAILURE); 2061 } 2062 hca_attr->hca_nports = val; 2063 2064 /* Copy NodeGUID and SystemImageGUID from softstate */ 2065 hca_attr->hca_node_guid = state->ts_nodeguid; 2066 hca_attr->hca_si_guid = state->ts_sysimgguid; 2067 2068 /* 2069 * Determine local ACK delay. Use the value suggested by the Tavor 2070 * hardware (from the QUERY_DEV_LIM command) 2071 */ 2072 hca_attr->hca_local_ack_delay = state->ts_devlim.ca_ack_delay; 2073 2074 /* Determine max SGID table and PKey table sizes */ 2075 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_gidtbl); 2076 hca_attr->hca_max_port_sgid_tbl_sz = val; 2077 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_pkeytbl); 2078 hca_attr->hca_max_port_pkey_tbl_sz = val; 2079 2080 /* Determine max number of PDs */ 2081 maxval = ((uint64_t)1 << state->ts_devlim.log_max_pd); 2082 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_pd); 2083 if (val > maxval) { 2084 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 2085 TNF_PROBE_2(tavor_soft_state_init_toomanypd_fail, 2086 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of PD " 2087 "exceeds device maximum", tnf_uint, maxpd, maxval); 2088 TAVOR_ATTACH_MSG(state->ts_attach_buf, 2089 "soft_state_init_toomanypd_fail"); 2090 TAVOR_TNF_EXIT(tavor_soft_state_init); 2091 return (DDI_FAILURE); 2092 } 2093 hca_attr->hca_max_pd = val; 2094 2095 /* Determine max number of Address Handles */ 2096 maxval = ((uint64_t)1 << state->ts_devlim.log_max_av); 2097 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_ah); 2098 if (val > maxval) { 2099 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 2100 TNF_PROBE_2(tavor_soft_state_init_toomanyah_fail, 2101 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of AH " 2102 "exceeds device maximum", tnf_uint, maxah, maxval); 2103 TAVOR_ATTACH_MSG(state->ts_attach_buf, 2104 "soft_state_init_toomanyah_fail"); 2105 TAVOR_TNF_EXIT(tavor_soft_state_init); 2106 return (DDI_FAILURE); 2107 } 2108 hca_attr->hca_max_ah = val; 2109 2110 /* No RDDs or EECs (since Reliable Datagram is not supported) */ 2111 hca_attr->hca_max_rdd = 0; 2112 hca_attr->hca_max_eec = 0; 2113 2114 /* Initialize lock for reserved UAR page access */ 2115 mutex_init(&state->ts_uar_lock, NULL, MUTEX_DRIVER, 2116 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2117 2118 /* Initialize the flash fields */ 2119 state->ts_fw_flashstarted = 0; 2120 mutex_init(&state->ts_fw_flashlock, NULL, MUTEX_DRIVER, 2121 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2122 2123 /* Initialize the lock for the info ioctl */ 2124 mutex_init(&state->ts_info_lock, NULL, MUTEX_DRIVER, 2125 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2126 2127 /* Initialize the AVL tree for QP number support */ 2128 tavor_qpn_avl_init(state); 2129 2130 /* Initialize the kstat info structure */ 2131 status = tavor_kstat_init(state); 2132 if (status != DDI_SUCCESS) { 2133 tavor_qpn_avl_fini(state); 2134 mutex_destroy(&state->ts_info_lock); 2135 mutex_destroy(&state->ts_fw_flashlock); 2136 mutex_destroy(&state->ts_uar_lock); 2137 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 2138 TNF_PROBE_0(tavor_soft_state_init_kstatinit_fail, 2139 TAVOR_TNF_ERROR, ""); 2140 TAVOR_ATTACH_MSG(state->ts_attach_buf, 2141 "soft_state_init_kstatinit_fail"); 2142 TAVOR_TNF_EXIT(tavor_soft_state_init); 2143 return (DDI_FAILURE); 2144 } 2145 2146 TAVOR_TNF_EXIT(tavor_soft_state_init); 2147 return (DDI_SUCCESS); 2148 } 2149 2150 2151 /* 2152 * tavor_soft_state_fini() 2153 * Context: Called only from detach() path context 2154 */ 2155 static void 2156 tavor_soft_state_fini(tavor_state_t *state) 2157 { 2158 TAVOR_TNF_ENTER(tavor_soft_state_fini); 2159 2160 /* Teardown the kstat info */ 2161 tavor_kstat_fini(state); 2162 2163 /* Teardown the AVL tree for QP number support */ 2164 tavor_qpn_avl_fini(state); 2165 2166 /* Free up info ioctl mutex */ 2167 mutex_destroy(&state->ts_info_lock); 2168 2169 /* Free up flash mutex */ 2170 mutex_destroy(&state->ts_fw_flashlock); 2171 2172 /* Free up the UAR page access mutex */ 2173 mutex_destroy(&state->ts_uar_lock); 2174 2175 /* Free up the hca_attr struct */ 2176 kmem_free(state->ts_ibtfinfo.hca_attr, sizeof (ibt_hca_attr_t)); 2177 2178 TAVOR_TNF_EXIT(tavor_soft_state_fini); 2179 } 2180 2181 2182 /* 2183 * tavor_hca_config_setup() 2184 * Context: Only called from attach() path context 2185 */ 2186 static void 2187 tavor_hca_config_setup(tavor_state_t *state, 2188 tavor_hw_initqueryhca_t *inithca) 2189 { 2190 tavor_rsrc_pool_info_t *rsrc_pool; 2191 uint64_t ddr_baseaddr, ddr_base_map_addr; 2192 uint64_t offset, addr; 2193 uint_t mcg_size; 2194 2195 TAVOR_TNF_ENTER(tavor_hca_config_setup); 2196 2197 /* Set "host endianness". Default is big endian */ 2198 #ifdef _LITTLE_ENDIAN 2199 inithca->big_endian = 0; 2200 #else 2201 inithca->big_endian = 1; 2202 #endif 2203 /* No Address Vector Protection, but Port Checking on by default */ 2204 inithca->udav_chk = TAVOR_UDAV_PROTECT_DISABLED; 2205 inithca->udav_port_chk = TAVOR_UDAV_PORTCHK_ENABLED; 2206 2207 ddr_baseaddr = (uint64_t)(uintptr_t)state->ts_reg_ddr_baseaddr; 2208 ddr_base_map_addr = (uint64_t)state->ts_ddr.ddr_baseaddr; 2209 2210 /* Setup QPC table */ 2211 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPC]; 2212 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2213 addr = ddr_base_map_addr + offset; 2214 inithca->context.qpc_baseaddr_h = (addr >> 32); 2215 inithca->context.qpc_baseaddr_l = (addr & 0xFFFFFFFF) >> 7; 2216 inithca->context.log_num_qp = state->ts_cfg_profile->cp_log_num_qp; 2217 2218 /* Setup EEC table (initialize to zero - RD unsupported) */ 2219 inithca->context.eec_baseaddr_h = 0; 2220 inithca->context.eec_baseaddr_l = 0; 2221 inithca->context.log_num_ee = 0; 2222 2223 /* Setup CQC table */ 2224 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQC]; 2225 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2226 addr = ddr_base_map_addr + offset; 2227 inithca->context.cqc_baseaddr_h = (addr >> 32); 2228 inithca->context.cqc_baseaddr_l = (addr & 0xFFFFFFFF) >> 6; 2229 inithca->context.log_num_cq = state->ts_cfg_profile->cp_log_num_cq; 2230 2231 /* Setup SRQC table */ 2232 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQC]; 2233 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2234 addr = ddr_base_map_addr + offset; 2235 inithca->context.srqc_baseaddr_h = (addr >> 32); 2236 inithca->context.srqc_baseaddr_l = (addr & 0xFFFFFFFF) >> 6; 2237 inithca->context.log_num_srq = 2238 state->ts_cfg_profile->cp_log_num_srq; 2239 2240 /* Setup EQPC table */ 2241 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQPC]; 2242 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2243 addr = ddr_base_map_addr + offset; 2244 inithca->context.eqpc_baseaddr = addr; 2245 2246 /* Setup EEEC table (initialize to zero - RD unsupported) */ 2247 inithca->context.eeec_baseaddr = 0; 2248 2249 /* Setup EQC table */ 2250 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQC]; 2251 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2252 addr = ddr_base_map_addr + offset; 2253 inithca->context.eqc_baseaddr_h = (addr >> 32); 2254 inithca->context.eqc_baseaddr_l = (addr & 0xFFFFFFFF) >> 6; 2255 inithca->context.log_num_eq = TAVOR_NUM_EQ_SHIFT; 2256 2257 /* Setup RDB table */ 2258 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_RDB]; 2259 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2260 addr = ddr_base_map_addr + offset; 2261 inithca->context.rdb_baseaddr_h = (addr >> 32); 2262 inithca->context.rdb_baseaddr_l = 0; 2263 2264 /* Setup Multicast */ 2265 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MCG]; 2266 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2267 addr = ddr_base_map_addr + offset; 2268 inithca->multi.mc_baseaddr = addr; 2269 mcg_size = TAVOR_MCGMEM_SZ(state); 2270 inithca->multi.log_mc_tbl_ent = highbit(mcg_size) - 1; 2271 inithca->multi.mc_tbl_hash_sz = 2272 (1 << state->ts_cfg_profile->cp_log_num_mcg_hash); 2273 inithca->multi.mc_hash_fn = TAVOR_MCG_DEFAULT_HASH_FN; 2274 inithca->multi.log_mc_tbl_sz = state->ts_cfg_profile->cp_log_num_mcg; 2275 2276 2277 /* Setup TPT */ 2278 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MPT]; 2279 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2280 addr = ddr_base_map_addr + offset; 2281 inithca->tpt.mpt_baseaddr = addr; 2282 inithca->tpt.mttseg_sz = TAVOR_MTTSEG_SIZE_SHIFT; 2283 inithca->tpt.log_mpt_sz = state->ts_cfg_profile->cp_log_num_mpt; 2284 inithca->tpt.mtt_version = TAVOR_MTT_PG_WALK_VER; 2285 2286 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MTT]; 2287 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2288 addr = ddr_base_map_addr + offset; 2289 inithca->tpt.mtt_baseaddr = addr; 2290 2291 /* Setup UAR */ 2292 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR]; 2293 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2294 addr = ddr_base_map_addr + offset; 2295 inithca->uar.uarscr_baseaddr = addr; 2296 2297 inithca->uar.uar_pg_sz = PAGESHIFT - 0xC; 2298 2299 TAVOR_TNF_EXIT(tavor_hca_config_setup); 2300 } 2301 2302 2303 /* 2304 * tavor_hca_port_init() 2305 * Context: Only called from attach() path context 2306 */ 2307 static int 2308 tavor_hca_port_init(tavor_state_t *state) 2309 { 2310 tavor_hw_initib_t *portinits, *initib; 2311 tavor_cfg_profile_t *cfgprof; 2312 uint_t num_ports; 2313 int i, status; 2314 uint64_t maxval, val; 2315 uint64_t sysimgguid, nodeguid, portguid; 2316 2317 TAVOR_TNF_ENTER(tavor_hca_port_init); 2318 2319 cfgprof = state->ts_cfg_profile; 2320 2321 /* Get number of HCA ports */ 2322 num_ports = cfgprof->cp_num_ports; 2323 2324 /* Allocate space for Tavor port init struct(s) */ 2325 portinits = (tavor_hw_initib_t *)kmem_zalloc(num_ports * 2326 sizeof (tavor_hw_initib_t), KM_SLEEP); 2327 2328 /* Post command to initialize Tavor HCA port */ 2329 for (i = 0; i < num_ports; i++) { 2330 initib = &portinits[i]; 2331 2332 /* 2333 * Determine whether we need to override the firmware's 2334 * default SystemImageGUID setting. 2335 */ 2336 sysimgguid = cfgprof->cp_sysimgguid; 2337 if (sysimgguid != 0) { 2338 initib->set_sysimg_guid = 1; 2339 initib->sysimg_guid = sysimgguid; 2340 } 2341 2342 /* 2343 * Determine whether we need to override the firmware's 2344 * default NodeGUID setting. 2345 */ 2346 nodeguid = cfgprof->cp_nodeguid; 2347 if (nodeguid != 0) { 2348 initib->set_node_guid = 1; 2349 initib->node_guid = nodeguid; 2350 } 2351 2352 /* 2353 * Determine whether we need to override the firmware's 2354 * default PortGUID setting. 2355 */ 2356 portguid = cfgprof->cp_portguid[i]; 2357 if (portguid != 0) { 2358 initib->set_port_guid0 = 1; 2359 initib->guid0 = portguid; 2360 } 2361 2362 /* Validate max MTU size */ 2363 maxval = state->ts_devlim.max_mtu; 2364 val = cfgprof->cp_max_mtu; 2365 if (val > maxval) { 2366 TNF_PROBE_2(tavor_hca_port_init_maxmtu_fail, 2367 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max " 2368 "MTU size exceeds device maximum", tnf_uint, 2369 maxmtu, maxval); 2370 TAVOR_TNF_EXIT(tavor_hca_port_init); 2371 goto init_ports_fail; 2372 } 2373 initib->mtu_cap = val; 2374 2375 /* Validate the max port width */ 2376 maxval = state->ts_devlim.max_port_width; 2377 val = cfgprof->cp_max_port_width; 2378 if (val > maxval) { 2379 TNF_PROBE_2(tavor_hca_port_init_maxportwidth_fail, 2380 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max " 2381 "port width exceeds device maximum", tnf_uint, 2382 maxportwidth, maxval); 2383 TAVOR_TNF_EXIT(tavor_hca_port_init); 2384 goto init_ports_fail; 2385 } 2386 initib->port_width_cap = val; 2387 2388 /* Validate max VL cap size */ 2389 maxval = state->ts_devlim.max_vl; 2390 val = cfgprof->cp_max_vlcap; 2391 if (val > maxval) { 2392 TNF_PROBE_2(tavor_hca_port_init_maxvlcap_fail, 2393 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max " 2394 "VLcap size exceeds device maximum", tnf_uint, 2395 maxvlcap, maxval); 2396 TAVOR_TNF_EXIT(tavor_hca_port_init); 2397 goto init_ports_fail; 2398 } 2399 initib->vl_cap = val; 2400 2401 /* Validate max GID table size */ 2402 maxval = ((uint64_t)1 << state->ts_devlim.log_max_gid); 2403 val = ((uint64_t)1 << cfgprof->cp_log_max_gidtbl); 2404 if (val > maxval) { 2405 TNF_PROBE_2(tavor_hca_port_init_gidtable_fail, 2406 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max " 2407 "GID table size exceeds device maximum", tnf_uint, 2408 maxgidtbl, maxval); 2409 TAVOR_TNF_EXIT(tavor_hca_port_init); 2410 goto init_ports_fail; 2411 } 2412 initib->max_gid = val; 2413 2414 /* Validate max PKey table size */ 2415 maxval = ((uint64_t)1 << state->ts_devlim.log_max_pkey); 2416 val = ((uint64_t)1 << cfgprof->cp_log_max_pkeytbl); 2417 if (val > maxval) { 2418 TNF_PROBE_2(tavor_hca_port_init_pkeytable_fail, 2419 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max " 2420 "PKey table size exceeds device maximum", tnf_uint, 2421 maxpkeytbl, maxval); 2422 TAVOR_TNF_EXIT(tavor_hca_port_init); 2423 goto init_ports_fail; 2424 } 2425 initib->max_pkey = val; 2426 2427 /* 2428 * Post the INIT_IB command to Tavor firmware. When this 2429 * command completes, the corresponding Tavor port will be 2430 * physically "Up" and initialized. 2431 */ 2432 status = tavor_init_ib_cmd_post(state, initib, i + 1, 2433 TAVOR_CMD_NOSLEEP_SPIN); 2434 if (status != TAVOR_CMD_SUCCESS) { 2435 cmn_err(CE_CONT, "Tavor: INIT_IB (port %02d) command " 2436 "failed: %08x\n", i + 1, status); 2437 TNF_PROBE_2(tavor_hca_port_init_init_ib_cmd_fail, 2438 TAVOR_TNF_ERROR, "", tnf_uint, cmd_status, status, 2439 tnf_uint, port, i + 1); 2440 TAVOR_TNF_EXIT(tavor_hca_port_init); 2441 goto init_ports_fail; 2442 } 2443 } 2444 2445 /* Free up the memory for Tavor port init struct(s), return success */ 2446 kmem_free(portinits, num_ports * sizeof (tavor_hw_initib_t)); 2447 TAVOR_TNF_EXIT(tavor_hca_port_init); 2448 return (DDI_SUCCESS); 2449 2450 init_ports_fail: 2451 /* 2452 * Free up the memory for Tavor port init struct(s), shutdown any 2453 * successfully initialized ports, and return failure 2454 */ 2455 kmem_free(portinits, num_ports * sizeof (tavor_hw_initib_t)); 2456 (void) tavor_hca_ports_shutdown(state, i); 2457 2458 TAVOR_TNF_EXIT(tavor_hca_port_init); 2459 return (DDI_FAILURE); 2460 } 2461 2462 2463 /* 2464 * tavor_hca_ports_shutdown() 2465 * Context: Only called from attach() and/or detach() path contexts 2466 */ 2467 static int 2468 tavor_hca_ports_shutdown(tavor_state_t *state, uint_t num_init) 2469 { 2470 int i, status; 2471 2472 TAVOR_TNF_ENTER(tavor_hca_ports_shutdown); 2473 2474 /* 2475 * Post commands to shutdown all init'd Tavor HCA ports. Note: if 2476 * any of these commands fail for any reason, it would be entirely 2477 * unexpected and probably indicative a serious problem (HW or SW). 2478 * Although we do return void from this function, this type of failure 2479 * should not go unreported. That is why we have the warning message 2480 * and the detailed TNF information. 2481 */ 2482 for (i = 0; i < num_init; i++) { 2483 status = tavor_close_ib_cmd_post(state, i + 1, 2484 TAVOR_CMD_NOSLEEP_SPIN); 2485 if (status != TAVOR_CMD_SUCCESS) { 2486 TAVOR_WARNING(state, "failed to shutdown HCA port"); 2487 TNF_PROBE_2(tavor_hca_ports_shutdown_close_ib_cmd_fail, 2488 TAVOR_TNF_ERROR, "", tnf_uint, cmd_status, status, 2489 tnf_uint, port, i + 1); 2490 TAVOR_TNF_EXIT(tavor_hca_ports_shutdown); 2491 return (status); 2492 } 2493 } 2494 2495 TAVOR_TNF_EXIT(tavor_hca_ports_shutdown); 2496 2497 return (TAVOR_CMD_SUCCESS); 2498 } 2499 2500 2501 /* 2502 * tavor_internal_uarpgs_init 2503 * Context: Only called from attach() path context 2504 */ 2505 static int 2506 tavor_internal_uarpgs_init(tavor_state_t *state) 2507 { 2508 int status; 2509 2510 TAVOR_TNF_ENTER(tavor_internal_uarpgs_init); 2511 2512 /* 2513 * Save away reserved Tavor UAR page #0. This UAR page is not to 2514 * be used by software. 2515 */ 2516 status = tavor_rsrc_alloc(state, TAVOR_UARPG, 1, TAVOR_SLEEP, 2517 &state->ts_uarpg0_rsrc_rsrvd); 2518 if (status != DDI_SUCCESS) { 2519 TNF_PROBE_0(tavor_uarpg0_rsrcalloc_fail, TAVOR_TNF_ERROR, ""); 2520 TAVOR_TNF_EXIT(tavor_internal_uarpgs_init); 2521 return (DDI_FAILURE); 2522 } 2523 2524 /* 2525 * Save away Tavor UAR page #1 (for internal use). This UAR page is 2526 * the privileged UAR page through which all kernel generated 2527 * doorbells will be rung. 2528 */ 2529 status = tavor_rsrc_alloc(state, TAVOR_UARPG, 1, TAVOR_SLEEP, 2530 &state->ts_uarpg1_rsrc); 2531 if (status != DDI_SUCCESS) { 2532 tavor_rsrc_free(state, &state->ts_uarpg0_rsrc_rsrvd); 2533 TNF_PROBE_0(tavor_uarpg1_rsrcalloc_fail, TAVOR_TNF_ERROR, ""); 2534 TAVOR_TNF_EXIT(tavor_internal_uarpgs_init); 2535 return (DDI_FAILURE); 2536 } 2537 2538 /* Setup pointer to UAR page #1 doorbells */ 2539 state->ts_uar = (tavor_hw_uar_t *)state->ts_uarpg1_rsrc->tr_addr; 2540 2541 TAVOR_TNF_EXIT(tavor_internal_uarpgs_init); 2542 return (DDI_SUCCESS); 2543 } 2544 2545 2546 /* 2547 * tavor_internal_uarpgs_fini 2548 * Context: Only called from attach() and/or detach() path contexts 2549 */ 2550 static void 2551 tavor_internal_uarpgs_fini(tavor_state_t *state) 2552 { 2553 TAVOR_TNF_ENTER(tavor_internal_uarpgs_fini); 2554 2555 /* Free up Tavor UAR page #1 (kernel driver doorbells) */ 2556 tavor_rsrc_free(state, &state->ts_uarpg1_rsrc); 2557 2558 /* Free up Tavor UAR page #0 (reserved) */ 2559 tavor_rsrc_free(state, &state->ts_uarpg0_rsrc_rsrvd); 2560 2561 TAVOR_TNF_EXIT(tavor_internal_uarpgs_fini); 2562 } 2563 2564 2565 /* 2566 * tavor_special_qp_contexts_reserve() 2567 * Context: Only called from attach() path context 2568 */ 2569 static int 2570 tavor_special_qp_contexts_reserve(tavor_state_t *state) 2571 { 2572 tavor_rsrc_t *qp0_rsrc, *qp1_rsrc; 2573 int status; 2574 2575 TAVOR_TNF_ENTER(tavor_special_qp_contexts_reserve); 2576 2577 /* Initialize the lock used for special QP rsrc management */ 2578 mutex_init(&state->ts_spec_qplock, NULL, MUTEX_DRIVER, 2579 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2580 2581 /* 2582 * Reserve contexts for QP0. These QP contexts will be setup to 2583 * act as aliases for the real QP0. Note: We are required to grab 2584 * two QPs (one per port) even if we are operating in single-port 2585 * mode. 2586 */ 2587 status = tavor_rsrc_alloc(state, TAVOR_QPC, 2, TAVOR_SLEEP, &qp0_rsrc); 2588 if (status != DDI_SUCCESS) { 2589 mutex_destroy(&state->ts_spec_qplock); 2590 TNF_PROBE_0(tavor_special_qp_contexts_reserve_qp0_fail, 2591 TAVOR_TNF_ERROR, ""); 2592 TAVOR_TNF_EXIT(tavor_special_qp_contexts_reserve); 2593 return (DDI_FAILURE); 2594 } 2595 state->ts_spec_qp0 = qp0_rsrc; 2596 2597 /* 2598 * Reserve contexts for QP1. These QP contexts will be setup to 2599 * act as aliases for the real QP1. Note: We are required to grab 2600 * two QPs (one per port) even if we are operating in single-port 2601 * mode. 2602 */ 2603 status = tavor_rsrc_alloc(state, TAVOR_QPC, 2, TAVOR_SLEEP, &qp1_rsrc); 2604 if (status != DDI_SUCCESS) { 2605 tavor_rsrc_free(state, &qp0_rsrc); 2606 mutex_destroy(&state->ts_spec_qplock); 2607 TNF_PROBE_0(tavor_special_qp_contexts_reserve_qp1_fail, 2608 TAVOR_TNF_ERROR, ""); 2609 TAVOR_TNF_EXIT(tavor_special_qp_contexts_reserve); 2610 return (DDI_FAILURE); 2611 } 2612 state->ts_spec_qp1 = qp1_rsrc; 2613 2614 TAVOR_TNF_EXIT(tavor_special_qp_contexts_reserve); 2615 return (DDI_SUCCESS); 2616 } 2617 2618 2619 /* 2620 * tavor_special_qp_contexts_unreserve() 2621 * Context: Only called from attach() and/or detach() path contexts 2622 */ 2623 static void 2624 tavor_special_qp_contexts_unreserve(tavor_state_t *state) 2625 { 2626 TAVOR_TNF_ENTER(tavor_special_qp_contexts_unreserve); 2627 2628 /* Unreserve contexts for QP1 */ 2629 tavor_rsrc_free(state, &state->ts_spec_qp1); 2630 2631 /* Unreserve contexts for QP0 */ 2632 tavor_rsrc_free(state, &state->ts_spec_qp0); 2633 2634 /* Destroy the lock used for special QP rsrc management */ 2635 mutex_destroy(&state->ts_spec_qplock); 2636 2637 TAVOR_TNF_EXIT(tavor_special_qp_contexts_unreserve); 2638 } 2639 2640 2641 /* 2642 * tavor_sw_reset() 2643 * Context: Currently called only from attach() path context 2644 */ 2645 static int 2646 tavor_sw_reset(tavor_state_t *state) 2647 { 2648 dev_info_t *dip, *pdip; 2649 ddi_acc_handle_t hdl = state->ts_pci_cfghdl, phdl; 2650 uint32_t reset_delay; 2651 int status, i; 2652 2653 TAVOR_TNF_ENTER(tavor_sw_reset); 2654 2655 /* 2656 * If the configured software reset delay is set to zero, then we 2657 * will not attempt a software reset of the Tavor device. 2658 */ 2659 reset_delay = state->ts_cfg_profile->cp_sw_reset_delay; 2660 if (reset_delay == 0) { 2661 TAVOR_TNF_EXIT(tavor_sw_reset); 2662 return (DDI_SUCCESS); 2663 } 2664 2665 /* 2666 * Get dip for HCA device _and_ parent device as well. Parent access 2667 * is necessary here because software reset of the Tavor hardware 2668 * will reinitialize both the config registers of the PCI bridge 2669 * (parent, if it exists) and the IB HCA (self) 2670 */ 2671 dip = state->ts_dip; 2672 pdip = ddi_get_parent(dip); 2673 2674 /* Query the PCI capabilities of the HCA device */ 2675 tavor_pci_capability_list(state, hdl); 2676 2677 /* 2678 * Read all PCI config info (reg0...reg63). Note: According to the 2679 * Tavor software reset application note, we should not read or 2680 * restore the values in reg22 and reg23. 2681 */ 2682 for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) { 2683 if ((i != TAVOR_SW_RESET_REG22_RSVD) && 2684 (i != TAVOR_SW_RESET_REG23_RSVD)) { 2685 state->ts_cfg_data[i] = pci_config_get32(hdl, i << 2); 2686 } 2687 } 2688 2689 if (TAVOR_PARENT_IS_BRIDGE(pdip)) { 2690 /* 2691 * Setup for PCI config read/write of bridge device 2692 */ 2693 status = pci_config_setup(pdip, &phdl); 2694 if (status != DDI_SUCCESS) { 2695 TNF_PROBE_0(tavor_sw_reset_pcicfg_p_fail, 2696 TAVOR_TNF_ERROR, ""); 2697 TAVOR_TNF_EXIT(tavor_sw_reset); 2698 return (DDI_FAILURE); 2699 } 2700 2701 /* 2702 * Read all PCI config info (reg0...reg63). Note: According to 2703 * the Tavor software reset application note, we should not 2704 * read or restore the values in reg22 and reg23. 2705 */ 2706 for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) { 2707 if ((i != TAVOR_SW_RESET_REG22_RSVD) && 2708 (i != TAVOR_SW_RESET_REG23_RSVD)) { 2709 state->ts_cfg_pdata[i] = 2710 pci_config_get32(phdl, i << 2); 2711 } 2712 } 2713 } 2714 2715 /* 2716 * Perform the software reset (by writing 1 at offset 0xF0010) 2717 */ 2718 ddi_put32(state->ts_reg_cmdhdl, state->ts_cmd_regs.sw_reset, 2719 TAVOR_SW_RESET_START); 2720 2721 drv_usecwait(reset_delay); 2722 2723 if (TAVOR_PARENT_IS_BRIDGE(pdip)) { 2724 /* 2725 * Bridge exists, so wait for the bridge to become ready. 2726 * 2727 * The above delay is necessary to avoid system panic from 2728 * Master Abort. If the device is accessed before this delay, 2729 * device will not respond to config cycles and they will be 2730 * terminate with a Master Abort which will panic the system. 2731 * Below is the loop we use to poll status from the device to 2732 * determine if it is OK to proceed. 2733 */ 2734 i = 0; 2735 while (pci_config_get32(phdl, 0) == TAVOR_SW_RESET_NOTDONE) { 2736 drv_usecwait(TAVOR_SW_RESET_POLL_DELAY); 2737 } 2738 2739 /* 2740 * Write all the PCI config registers back into each device 2741 * (except for reg22 and reg23 - see above) 2742 */ 2743 for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) { 2744 if ((i != TAVOR_SW_RESET_REG22_RSVD) && 2745 (i != TAVOR_SW_RESET_REG23_RSVD)) { 2746 pci_config_put32(phdl, i << 2, 2747 state->ts_cfg_pdata[i]); 2748 } 2749 } 2750 2751 /* 2752 * Tear down the config setup (for bridge device) 2753 */ 2754 pci_config_teardown(&phdl); 2755 2756 /* No Bridge Device */ 2757 } else { 2758 /* 2759 * Bridge does not exist, so instead wait for the device itself 2760 * to become ready. 2761 * 2762 * The above delay is necessary to avoid system panic from 2763 * Master Abort. If the device is accessed before this delay, 2764 * device will not respond to config cycles and they will be 2765 * terminate with a Master Abort which will panic the system. 2766 * Below is the loop we use to poll status from the device to 2767 * determine if it is OK to proceed. 2768 */ 2769 i = 0; 2770 while (pci_config_get32(hdl, 0) == TAVOR_SW_RESET_NOTDONE) { 2771 drv_usecwait(TAVOR_SW_RESET_POLL_DELAY); 2772 } 2773 } 2774 2775 for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) { 2776 if ((i != TAVOR_SW_RESET_REG22_RSVD) && 2777 (i != TAVOR_SW_RESET_REG23_RSVD)) { 2778 pci_config_put32(hdl, i << 2, state->ts_cfg_data[i]); 2779 } 2780 } 2781 2782 TAVOR_TNF_EXIT(tavor_sw_reset); 2783 return (DDI_SUCCESS); 2784 } 2785 2786 2787 /* 2788 * tavor_mcg_init() 2789 * Context: Only called from attach() path context 2790 */ 2791 static int 2792 tavor_mcg_init(tavor_state_t *state) 2793 { 2794 uint_t mcg_tmp_sz; 2795 2796 TAVOR_TNF_ENTER(tavor_mcg_init); 2797 2798 /* 2799 * Allocate space for the MCG temporary copy buffer. This is 2800 * used by the Attach/Detach Multicast Group code 2801 */ 2802 mcg_tmp_sz = TAVOR_MCGMEM_SZ(state); 2803 state->ts_mcgtmp = kmem_zalloc(mcg_tmp_sz, KM_SLEEP); 2804 2805 /* 2806 * Initialize the multicast group mutex. This ensures atomic 2807 * access to add, modify, and remove entries in the multicast 2808 * group hash lists. 2809 */ 2810 mutex_init(&state->ts_mcglock, NULL, MUTEX_DRIVER, 2811 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2812 2813 TAVOR_TNF_EXIT(tavor_mcg_init); 2814 return (DDI_SUCCESS); 2815 } 2816 2817 2818 /* 2819 * tavor_mcg_fini() 2820 * Context: Only called from attach() and/or detach() path contexts 2821 */ 2822 static void 2823 tavor_mcg_fini(tavor_state_t *state) 2824 { 2825 uint_t mcg_tmp_sz; 2826 2827 TAVOR_TNF_ENTER(tavor_mcg_fini); 2828 2829 /* Free up the space used for the MCG temporary copy buffer */ 2830 mcg_tmp_sz = TAVOR_MCGMEM_SZ(state); 2831 kmem_free(state->ts_mcgtmp, mcg_tmp_sz); 2832 2833 /* Destroy the multicast group mutex */ 2834 mutex_destroy(&state->ts_mcglock); 2835 2836 TAVOR_TNF_EXIT(tavor_mcg_fini); 2837 } 2838 2839 2840 /* 2841 * tavor_fw_version_check() 2842 * Context: Only called from attach() path context 2843 */ 2844 static int 2845 tavor_fw_version_check(tavor_state_t *state) 2846 { 2847 uint_t tavor_fw_ver_major; 2848 uint_t tavor_fw_ver_minor; 2849 uint_t tavor_fw_ver_subminor; 2850 2851 /* 2852 * Depending on which version of driver we have attached, the firmware 2853 * version checks will be different. We set up the comparison values 2854 * for both HCA Mode (Tavor hardware) or COMPAT Mode (Arbel hardware 2855 * running in tavor mode). 2856 */ 2857 switch (state->ts_operational_mode) { 2858 case TAVOR_HCA_MODE: 2859 tavor_fw_ver_major = TAVOR_FW_VER_MAJOR; 2860 tavor_fw_ver_minor = TAVOR_FW_VER_MINOR; 2861 tavor_fw_ver_subminor = TAVOR_FW_VER_SUBMINOR; 2862 break; 2863 2864 case TAVOR_COMPAT_MODE: 2865 tavor_fw_ver_major = TAVOR_COMPAT_FW_VER_MAJOR; 2866 tavor_fw_ver_minor = TAVOR_COMPAT_FW_VER_MINOR; 2867 tavor_fw_ver_subminor = TAVOR_COMPAT_FW_VER_SUBMINOR; 2868 break; 2869 2870 default: 2871 return (DDI_FAILURE); 2872 } 2873 2874 /* 2875 * If FW revision major number is less than acceptable, 2876 * return failure, else if greater return success. If 2877 * the major numbers are equal than check the minor number 2878 */ 2879 if (state->ts_fw.fw_rev_major < tavor_fw_ver_major) { 2880 return (DDI_FAILURE); 2881 } else if (state->ts_fw.fw_rev_major > tavor_fw_ver_major) { 2882 return (DDI_SUCCESS); 2883 } 2884 /* 2885 * Do the same check as above, except for minor revision numbers 2886 * If the minor numbers are equal than check the subminor number 2887 */ 2888 if (state->ts_fw.fw_rev_minor < tavor_fw_ver_minor) { 2889 return (DDI_FAILURE); 2890 } else if (state->ts_fw.fw_rev_minor > tavor_fw_ver_minor) { 2891 return (DDI_SUCCESS); 2892 } 2893 2894 /* 2895 * Once again we do the same check as above, except for the subminor 2896 * revision number. If the subminor numbers are equal here, then 2897 * these are the same firmware version, return success 2898 */ 2899 if (state->ts_fw.fw_rev_subminor < tavor_fw_ver_subminor) { 2900 return (DDI_FAILURE); 2901 } else if (state->ts_fw.fw_rev_subminor > tavor_fw_ver_subminor) { 2902 return (DDI_SUCCESS); 2903 } 2904 2905 return (DDI_SUCCESS); 2906 } 2907 2908 2909 /* 2910 * tavor_device_info_report() 2911 * Context: Only called from attach() path context 2912 */ 2913 static void 2914 tavor_device_info_report(tavor_state_t *state) 2915 { 2916 cmn_err(CE_CONT, "?tavor%d: FW ver: %04d.%04d.%04d, " 2917 "HW rev: %02x\n", state->ts_instance, state->ts_fw.fw_rev_major, 2918 state->ts_fw.fw_rev_minor, state->ts_fw.fw_rev_subminor, 2919 state->ts_adapter.rev_id); 2920 cmn_err(CE_CONT, "?tavor%d: %64s (0x%016" PRIx64 ")\n", 2921 state->ts_instance, state->ts_nodedesc, state->ts_nodeguid); 2922 } 2923 2924 2925 /* 2926 * tavor_pci_capability_list() 2927 * Context: Only called from attach() path context 2928 */ 2929 static void 2930 tavor_pci_capability_list(tavor_state_t *state, ddi_acc_handle_t hdl) 2931 { 2932 uint_t offset, data; 2933 2934 TAVOR_TNF_ENTER(tavor_pci_capability_list); 2935 2936 /* 2937 * Check for the "PCI Capabilities" bit in the "Status Register". 2938 * Bit 4 in this register indicates the presence of a "PCI 2939 * Capabilities" list. 2940 */ 2941 data = pci_config_get16(hdl, 0x6); 2942 if ((data & 0x10) == 0) { 2943 TNF_PROBE_0(tavor_pci_capab_list_fail, TAVOR_TNF_ERROR, ""); 2944 TAVOR_TNF_EXIT(tavor_pci_capability_list); 2945 return; 2946 } 2947 2948 /* 2949 * Starting from offset 0x34 in PCI config space, find the 2950 * head of "PCI capabilities" list, and walk the list. If 2951 * capabilities of a known type are encountered (e.g. 2952 * "PCI-X Capability"), then call the appropriate handler 2953 * function. 2954 */ 2955 offset = pci_config_get8(hdl, 0x34); 2956 while (offset != 0x0) { 2957 data = pci_config_get8(hdl, offset); 2958 2959 /* 2960 * Check for known capability types. Tavor has the 2961 * following: 2962 * o VPD Capability (0x03) 2963 * o PCI-X Capability (0x07) 2964 * o MSI Capability (0x05) 2965 * o MSIX Capability (0x11) 2966 */ 2967 switch (data) { 2968 case 0x03: 2969 tavor_pci_capability_vpd(state, hdl, offset); 2970 break; 2971 case 0x07: 2972 tavor_pci_capability_pcix(state, hdl, offset); 2973 break; 2974 case 0x05: 2975 break; 2976 default: 2977 break; 2978 } 2979 2980 /* Get offset of next entry in list */ 2981 offset = pci_config_get8(hdl, offset + 1); 2982 } 2983 2984 TAVOR_TNF_EXIT(tavor_pci_capability_list); 2985 } 2986 2987 /* 2988 * tavor_pci_read_vpd() 2989 * Context: Only called from attach() path context 2990 * utility routine for tavor_pci_capability_vpd() 2991 */ 2992 static int 2993 tavor_pci_read_vpd(ddi_acc_handle_t hdl, uint_t offset, uint32_t addr, 2994 uint32_t *data) 2995 { 2996 int retry = 4; /* retry counter for EEPROM poll */ 2997 uint32_t val; 2998 int vpd_addr = offset + 2; 2999 int vpd_data = offset + 4; 3000 3001 TAVOR_TNF_ENTER(tavor_pci_read_vpd); 3002 3003 /* 3004 * In order to read a 32-bit value from VPD, we are to write down 3005 * the address (offset in the VPD itself) to the address register. 3006 * To signal the read, we also clear bit 31. We then poll on bit 31 3007 * and when it is set, we can then read our 4 bytes from the data 3008 * register. 3009 */ 3010 (void) pci_config_put32(hdl, offset, addr << 16); 3011 do { 3012 drv_usecwait(1000); 3013 val = pci_config_get16(hdl, vpd_addr); 3014 if ((val >> 15) & 0x01) { 3015 *data = pci_config_get32(hdl, vpd_data); 3016 TAVOR_TNF_EXIT(tavor_pci_read_vpd); 3017 return (DDI_SUCCESS); 3018 } 3019 } while (--retry); 3020 3021 TNF_PROBE_0(tavor_pci_read_vpd_fail, TAVOR_TNF_ERROR, ""); 3022 TAVOR_TNF_EXIT(tavor_pci_read_vpd); 3023 return (DDI_FAILURE); 3024 } 3025 3026 3027 /* 3028 * tavor_pci_capability_vpd() 3029 * Context: Only called from attach() path context 3030 */ 3031 static void 3032 tavor_pci_capability_vpd(tavor_state_t *state, ddi_acc_handle_t hdl, 3033 uint_t offset) 3034 { 3035 uint8_t name_length; 3036 uint8_t pn_length; 3037 int i, err = 0; 3038 int vpd_str_id = 0; 3039 int vpd_ro_desc; 3040 int vpd_ro_pn_desc; 3041 #ifndef _LITTLE_ENDIAN 3042 uint32_t data32; 3043 #endif /* _LITTLE_ENDIAN */ 3044 union { 3045 uint32_t vpd_int[TAVOR_VPD_HDR_DWSIZE]; 3046 uchar_t vpd_char[TAVOR_VPD_HDR_BSIZE]; 3047 } vpd; 3048 3049 TAVOR_TNF_ENTER(tavor_pci_capability_vpd); 3050 3051 /* 3052 * Read Vital Product Data (VPD) from PCI-X capability. 3053 */ 3054 for (i = 0; i < TAVOR_VPD_HDR_DWSIZE; i++) { 3055 err = tavor_pci_read_vpd(hdl, offset, i << 2, &vpd.vpd_int[i]); 3056 if (err != DDI_SUCCESS) { 3057 cmn_err(CE_NOTE, "!VPD read failed\n"); 3058 goto out; 3059 } 3060 } 3061 3062 #ifndef _LITTLE_ENDIAN 3063 /* 3064 * Need to swap bytes for big endian. 3065 */ 3066 for (i = 0; i < TAVOR_VPD_HDR_DWSIZE; i++) { 3067 data32 = vpd.vpd_int[i]; 3068 vpd.vpd_char[(i << 2) + 3] = 3069 (uchar_t)((data32 & 0xFF000000) >> 24); 3070 vpd.vpd_char[(i << 2) + 2] = 3071 (uchar_t)((data32 & 0x00FF0000) >> 16); 3072 vpd.vpd_char[(i << 2) + 1] = 3073 (uchar_t)((data32 & 0x0000FF00) >> 8); 3074 vpd.vpd_char[i << 2] = (uchar_t)(data32 & 0x000000FF); 3075 } 3076 #endif /* _LITTLE_ENDIAN */ 3077 3078 /* Check for VPD String ID Tag */ 3079 if (vpd.vpd_char[vpd_str_id] == 0x82) { 3080 /* get the product name */ 3081 name_length = (uint8_t)vpd.vpd_char[vpd_str_id + 1]; 3082 if (name_length > sizeof (state->ts_hca_name)) { 3083 cmn_err(CE_NOTE, "!VPD name too large (0x%x)\n", 3084 name_length); 3085 goto out; 3086 } 3087 (void) memcpy(state->ts_hca_name, &vpd.vpd_char[vpd_str_id + 3], 3088 name_length); 3089 state->ts_hca_name[name_length] = 0; 3090 3091 /* get the part number */ 3092 vpd_ro_desc = name_length + 3; /* read-only tag location */ 3093 vpd_ro_pn_desc = vpd_ro_desc + 3; /* P/N keyword location */ 3094 /* 3095 * Verify read-only tag and Part Number keyword. 3096 */ 3097 if (vpd.vpd_char[vpd_ro_desc] != 0x90 || 3098 (vpd.vpd_char[vpd_ro_pn_desc] != 'P' && 3099 vpd.vpd_char[vpd_ro_pn_desc + 1] != 'N')) { 3100 cmn_err(CE_NOTE, "!VPD Part Number not found\n"); 3101 goto out; 3102 } 3103 3104 pn_length = (uint8_t)vpd.vpd_char[vpd_ro_pn_desc + 2]; 3105 if (pn_length > sizeof (state->ts_hca_pn)) { 3106 cmn_err(CE_NOTE, "!VPD part number too large (0x%x)\n", 3107 name_length); 3108 goto out; 3109 } 3110 (void) memcpy(state->ts_hca_pn, 3111 &vpd.vpd_char[vpd_ro_pn_desc + 3], 3112 pn_length); 3113 state->ts_hca_pn[pn_length] = 0; 3114 state->ts_hca_pn_len = pn_length; 3115 } else { 3116 /* Wrong VPD String ID Tag */ 3117 cmn_err(CE_NOTE, "!VPD String ID Tag not found, tag: %02x\n", 3118 vpd.vpd_char[0]); 3119 goto out; 3120 } 3121 TAVOR_TNF_EXIT(tavor_pci_capability_vpd); 3122 return; 3123 out: 3124 state->ts_hca_pn_len = 0; 3125 TNF_PROBE_0(tavor_pci_capability_vpd_fail, TAVOR_TNF_ERROR, ""); 3126 TAVOR_TNF_EXIT(tavor_pci_capability_vpd); 3127 } 3128 3129 /* 3130 * tavor_pci_capability_pcix() 3131 * Context: Only called from attach() path context 3132 */ 3133 static void 3134 tavor_pci_capability_pcix(tavor_state_t *state, ddi_acc_handle_t hdl, 3135 uint_t offset) 3136 { 3137 uint_t command, status; 3138 int max_out_splt_trans, max_mem_rd_byte_cnt; 3139 int designed_max_out_splt_trans, designed_max_mem_rd_byte_cnt; 3140 3141 TAVOR_TNF_ENTER(tavor_pci_capability_pcix); 3142 3143 /* 3144 * Query the current values for the PCI-X Command Register and 3145 * the PCI-X Status Register. 3146 */ 3147 command = pci_config_get16(hdl, offset + 2); 3148 status = pci_config_get32(hdl, offset + 4); 3149 3150 /* 3151 * Check for config property specifying "maximum outstanding 3152 * split transactions". If the property is defined and valid 3153 * (i.e. no larger than the so-called "designed maximum"), 3154 * then use the specified value to update the PCI-X Command Register. 3155 * Otherwise, extract the value from the Tavor config profile. 3156 */ 3157 designed_max_out_splt_trans = ((status >> 23) & 7); 3158 max_out_splt_trans = ddi_prop_get_int(DDI_DEV_T_ANY, state->ts_dip, 3159 DDI_PROP_DONTPASS, "pcix-max-outstanding-split-trans", -1); 3160 if ((max_out_splt_trans != -1) && 3161 ((max_out_splt_trans < 0) || 3162 (max_out_splt_trans > designed_max_out_splt_trans))) { 3163 cmn_err(CE_NOTE, "!tavor%d: property \"pcix-max-outstanding-" 3164 "split-trans\" (%d) invalid or exceeds device maximum" 3165 " (%d), using default value (%d)\n", state->ts_instance, 3166 max_out_splt_trans, designed_max_out_splt_trans, 3167 state->ts_cfg_profile->cp_max_out_splt_trans); 3168 max_out_splt_trans = 3169 state->ts_cfg_profile->cp_max_out_splt_trans; 3170 } else if (max_out_splt_trans == -1) { 3171 max_out_splt_trans = 3172 state->ts_cfg_profile->cp_max_out_splt_trans; 3173 } 3174 3175 /* 3176 * The config profile setting for max_out_splt_trans is determined 3177 * based on arch. Check tavor_cfg.c for more information. A value of 3178 * '-1' in the patchable variable means "do not change". A value of 3179 * '0' means 1 outstanding splt trans and other values as defined by 3180 * PCI. So we do one more check here, that if 'max_out_splt_trans' is 3181 * -1 (ie: < 0) we do not set the PCI command and leave it at the 3182 * default. 3183 */ 3184 if (max_out_splt_trans >= 0) { 3185 command = ((command & 0xFF8F) | max_out_splt_trans << 4); 3186 } 3187 3188 /* 3189 * Check for config property specifying "maximum memory read 3190 * byte count. If the property is defined and valid 3191 * (i.e. no larger than the so-called "designed maximum"), 3192 * then use the specified value to update the PCI-X Command Register. 3193 * Otherwise, extract the value from the Tavor config profile. 3194 */ 3195 designed_max_mem_rd_byte_cnt = ((status >> 21) & 3); 3196 max_mem_rd_byte_cnt = ddi_prop_get_int(DDI_DEV_T_ANY, state->ts_dip, 3197 DDI_PROP_DONTPASS, "pcix-max-read-byte-count", -1); 3198 if ((max_mem_rd_byte_cnt != -1) && 3199 ((max_mem_rd_byte_cnt < 0) || 3200 (max_mem_rd_byte_cnt > designed_max_mem_rd_byte_cnt))) { 3201 cmn_err(CE_NOTE, "!tavor%d: property \"pcix-max-read-byte-" 3202 "count\" (%d) invalid or exceeds device maximum" 3203 " (%d), using default value (%d)\n", state->ts_instance, 3204 max_mem_rd_byte_cnt, designed_max_mem_rd_byte_cnt, 3205 state->ts_cfg_profile->cp_max_mem_rd_byte_cnt); 3206 max_mem_rd_byte_cnt = 3207 state->ts_cfg_profile->cp_max_mem_rd_byte_cnt; 3208 } else if (max_mem_rd_byte_cnt == -1) { 3209 max_mem_rd_byte_cnt = 3210 state->ts_cfg_profile->cp_max_mem_rd_byte_cnt; 3211 } 3212 3213 /* 3214 * The config profile setting for max_mem_rd_byte_cnt is determined 3215 * based on arch. Check tavor_cfg.c for more information. A value of 3216 * '-1' in the patchable variable means "do not change". A value of 3217 * '0' means minimum (512B) read, and other values as defined by 3218 * PCI. So we do one more check here, that if 'max_mem_rd_byte_cnt' is 3219 * -1 (ie: < 0) we do not set the PCI command and leave it at the 3220 * default. 3221 */ 3222 if (max_mem_rd_byte_cnt >= 0) { 3223 command = ((command & 0xFFF3) | max_mem_rd_byte_cnt << 2); 3224 } 3225 3226 /* 3227 * Update the PCI-X Command Register with the newly configured 3228 * values. 3229 */ 3230 pci_config_put16(hdl, offset + 2, command); 3231 3232 TAVOR_TNF_EXIT(tavor_pci_capability_pcix); 3233 } 3234 3235 3236 /* 3237 * tavor_intr_or_msi_init() 3238 * Context: Only called from attach() path context 3239 */ 3240 static int 3241 tavor_intr_or_msi_init(tavor_state_t *state) 3242 { 3243 int status; 3244 3245 TAVOR_TNF_ENTER(tavor_intr_or_msi_init); 3246 3247 /* Query for the list of supported interrupt event types */ 3248 status = ddi_intr_get_supported_types(state->ts_dip, 3249 &state->ts_intr_types_avail); 3250 if (status != DDI_SUCCESS) { 3251 TNF_PROBE_0(tavor_intr_or_msi_init_gettypes_fail, 3252 TAVOR_TNF_ERROR, ""); 3253 TAVOR_TNF_EXIT(tavor_intr_or_msi_init); 3254 return (DDI_FAILURE); 3255 } 3256 3257 /* 3258 * If Tavor/Arbel supports MSI in this system (and, if it 3259 * hasn't been overridden by a configuration variable), then 3260 * the default behavior is to use a single MSI. Otherwise, 3261 * fallback to using legacy interrupts. Also, if MSI allocatis chosen, 3262 * but fails for whatever reasons, then fallback to using legacy 3263 * interrupts. 3264 */ 3265 if ((state->ts_cfg_profile->cp_use_msi_if_avail != 0) && 3266 (state->ts_intr_types_avail & DDI_INTR_TYPE_MSI)) { 3267 status = tavor_add_intrs(state, DDI_INTR_TYPE_MSI); 3268 if (status == DDI_SUCCESS) { 3269 state->ts_intr_type_chosen = DDI_INTR_TYPE_MSI; 3270 TAVOR_TNF_EXIT(tavor_intr_or_msi_init); 3271 return (DDI_SUCCESS); 3272 } 3273 } 3274 3275 /* 3276 * MSI interrupt allocation failed, or was not available. Fallback to 3277 * legacy interrupt support. 3278 */ 3279 if (state->ts_intr_types_avail & DDI_INTR_TYPE_FIXED) { 3280 status = tavor_add_intrs(state, DDI_INTR_TYPE_FIXED); 3281 if (status == DDI_SUCCESS) { 3282 state->ts_intr_type_chosen = DDI_INTR_TYPE_FIXED; 3283 TAVOR_TNF_EXIT(tavor_intr_or_msi_init); 3284 return (DDI_SUCCESS); 3285 } 3286 } 3287 3288 /* 3289 * Neither MSI or legacy interrupts were successful. return failure. 3290 */ 3291 TAVOR_TNF_EXIT(tavor_intr_or_msi_setup); 3292 return (DDI_FAILURE); 3293 } 3294 3295 /* 3296 * tavor_add_intrs() 3297 * Context: Only called from attach() patch context 3298 */ 3299 static int 3300 tavor_add_intrs(tavor_state_t *state, int intr_type) 3301 { 3302 int status; 3303 3304 TAVOR_TNF_ENTER(tavor_add_intrs); 3305 3306 /* Get number of interrupts/MSI supported */ 3307 status = ddi_intr_get_nintrs(state->ts_dip, intr_type, 3308 &state->ts_intrmsi_count); 3309 if (status != DDI_SUCCESS) { 3310 TNF_PROBE_0(tavor_add_intrs_getnintrs_fail, 3311 TAVOR_TNF_ERROR, ""); 3312 TAVOR_TNF_EXIT(tavor_add_intrs); 3313 return (DDI_FAILURE); 3314 } 3315 3316 /* Get number of available interrupts/MSI */ 3317 status = ddi_intr_get_navail(state->ts_dip, intr_type, 3318 &state->ts_intrmsi_avail); 3319 if (status != DDI_SUCCESS) { 3320 TNF_PROBE_0(tavor_add_intrs_getnavail_fail, 3321 TAVOR_TNF_ERROR, ""); 3322 TAVOR_TNF_EXIT(tavor_add_intrs); 3323 return (DDI_FAILURE); 3324 } 3325 3326 /* Ensure that we have at least one (1) usable MSI or interrupt */ 3327 if ((state->ts_intrmsi_avail < 1) || (state->ts_intrmsi_count < 1)) { 3328 TNF_PROBE_0(tavor_add_intrs_notenoughts_intrmsi_fail, 3329 TAVOR_TNF_ERROR, ""); 3330 TAVOR_TNF_EXIT(tavor_add_intrs); 3331 return (DDI_FAILURE); 3332 } 3333 3334 /* Attempt to allocate a single interrupt/MSI handle */ 3335 status = ddi_intr_alloc(state->ts_dip, &state->ts_intrmsi_hdl, 3336 intr_type, 0, 1, &state->ts_intrmsi_allocd, 3337 DDI_INTR_ALLOC_STRICT); 3338 if (status != DDI_SUCCESS) { 3339 TNF_PROBE_0(tavor_add_intrs_intralloc_fail, 3340 TAVOR_TNF_ERROR, ""); 3341 TAVOR_TNF_EXIT(tavor_add_intrs); 3342 return (DDI_FAILURE); 3343 } 3344 3345 /* Ensure that we have allocated at least one (1) MSI or interrupt */ 3346 if (state->ts_intrmsi_allocd < 1) { 3347 TNF_PROBE_0(tavor_add_intrs_noallocts_intrmsi_fail, 3348 TAVOR_TNF_ERROR, ""); 3349 TAVOR_TNF_EXIT(tavor_add_intrs); 3350 return (DDI_FAILURE); 3351 } 3352 3353 /* 3354 * Extract the priority for the allocated interrupt/MSI. This 3355 * will be used later when initializing certain mutexes. 3356 */ 3357 status = ddi_intr_get_pri(state->ts_intrmsi_hdl, 3358 &state->ts_intrmsi_pri); 3359 if (status != DDI_SUCCESS) { 3360 /* Free the allocated interrupt/MSI handle */ 3361 (void) ddi_intr_free(state->ts_intrmsi_hdl); 3362 3363 TNF_PROBE_0(tavor_add_intrs_getpri_fail, 3364 TAVOR_TNF_ERROR, ""); 3365 TAVOR_TNF_EXIT(tavor_add_intrs); 3366 return (DDI_FAILURE); 3367 } 3368 3369 /* Make sure the interrupt/MSI priority is below 'high level' */ 3370 if (state->ts_intrmsi_pri >= ddi_intr_get_hilevel_pri()) { 3371 /* Free the allocated interrupt/MSI handle */ 3372 (void) ddi_intr_free(state->ts_intrmsi_hdl); 3373 3374 TNF_PROBE_0(tavor_add_intrs_hilevelpri_fail, 3375 TAVOR_TNF_ERROR, ""); 3376 TAVOR_TNF_EXIT(tavor_add_intrs); 3377 return (DDI_FAILURE); 3378 } 3379 3380 /* Get add'l capability information regarding interrupt/MSI */ 3381 status = ddi_intr_get_cap(state->ts_intrmsi_hdl, 3382 &state->ts_intrmsi_cap); 3383 if (status != DDI_SUCCESS) { 3384 /* Free the allocated interrupt/MSI handle */ 3385 (void) ddi_intr_free(state->ts_intrmsi_hdl); 3386 3387 TNF_PROBE_0(tavor_add_intrs_getcap_fail, 3388 TAVOR_TNF_ERROR, ""); 3389 TAVOR_TNF_EXIT(tavor_add_intrs); 3390 return (DDI_FAILURE); 3391 } 3392 3393 TAVOR_TNF_EXIT(tavor_add_intrs); 3394 return (DDI_SUCCESS); 3395 } 3396 3397 3398 /* 3399 * tavor_intr_or_msi_fini() 3400 * Context: Only called from attach() and/or detach() path contexts 3401 */ 3402 static int 3403 tavor_intr_or_msi_fini(tavor_state_t *state) 3404 { 3405 int status; 3406 3407 TAVOR_TNF_ENTER(tavor_intr_or_msi_fini); 3408 3409 /* Free the allocated interrupt/MSI handle */ 3410 status = ddi_intr_free(state->ts_intrmsi_hdl); 3411 if (status != DDI_SUCCESS) { 3412 TNF_PROBE_0(tavor_intr_or_msi_fini_freehdl_fail, 3413 TAVOR_TNF_ERROR, ""); 3414 TAVOR_TNF_EXIT(tavor_intr_or_msi_fini); 3415 return (DDI_FAILURE); 3416 } 3417 3418 TAVOR_TNF_EXIT(tavor_intr_or_msi_fini); 3419 return (DDI_SUCCESS); 3420 } 3421 3422 3423 /* Disable Tavor interrupts */ 3424 static int 3425 tavor_intr_disable(tavor_state_t *state) 3426 { 3427 ushort_t msi_ctrl = 0, caps_ctrl = 0; 3428 ddi_acc_handle_t pci_cfg_hdl = state->ts_pci_cfghdl; 3429 ASSERT(pci_cfg_hdl != NULL); 3430 ASSERT(state->ts_intr_types_avail & 3431 (DDI_INTR_TYPE_FIXED | DDI_INTR_TYPE_MSI)); 3432 3433 /* 3434 * Check if MSI interrupts are used. If so, disable MSI interupts. 3435 * If not, since Tavor doesn't support MSI-X interrupts, assuming the 3436 * legacy interrupt is used instead, disable the legacy interrupt. 3437 */ 3438 if ((state->ts_cfg_profile->cp_use_msi_if_avail != 0) && 3439 (state->ts_intr_types_avail & DDI_INTR_TYPE_MSI)) { 3440 3441 if ((PCI_CAP_LOCATE(pci_cfg_hdl, PCI_CAP_ID_MSI, 3442 &caps_ctrl) == DDI_SUCCESS)) { 3443 if ((msi_ctrl = PCI_CAP_GET16(pci_cfg_hdl, NULL, 3444 caps_ctrl, PCI_MSI_CTRL)) == PCI_CAP_EINVAL16) 3445 return (DDI_FAILURE); 3446 } 3447 ASSERT(msi_ctrl != 0); 3448 3449 if (!(msi_ctrl & PCI_MSI_ENABLE_BIT)) 3450 return (DDI_SUCCESS); 3451 3452 if (msi_ctrl & PCI_MSI_PVM_MASK) { 3453 int offset = (msi_ctrl & PCI_MSI_64BIT_MASK) ? 3454 PCI_MSI_64BIT_MASKBITS : PCI_MSI_32BIT_MASK; 3455 3456 /* Clear all inums in MSI */ 3457 PCI_CAP_PUT32(pci_cfg_hdl, NULL, caps_ctrl, 3458 offset, 0x0); 3459 } 3460 3461 /* Disable MSI interrupts */ 3462 msi_ctrl &= ~PCI_MSI_ENABLE_BIT; 3463 PCI_CAP_PUT16(pci_cfg_hdl, NULL, caps_ctrl, PCI_MSI_CTRL, 3464 msi_ctrl); 3465 3466 } else { 3467 uint16_t cmdreg = pci_config_get16(pci_cfg_hdl, PCI_CONF_COMM); 3468 ASSERT(state->ts_intr_types_avail & DDI_INTR_TYPE_FIXED); 3469 3470 /* Disable the legacy interrupts */ 3471 cmdreg |= PCI_COMM_INTX_DISABLE; 3472 pci_config_put16(pci_cfg_hdl, PCI_CONF_COMM, cmdreg); 3473 } 3474 3475 return (DDI_SUCCESS); 3476 } 3477 3478 /* Tavor quiesce(9F) entry */ 3479 static int 3480 tavor_quiesce(dev_info_t *dip) 3481 { 3482 tavor_state_t *state = ddi_get_soft_state(tavor_statep, 3483 DEVI(dip)->devi_instance); 3484 ASSERT(state != NULL); 3485 3486 /* start fastreboot */ 3487 state->ts_quiescing = B_TRUE; 3488 3489 /* If it's in maintenance mode, do nothing but return with SUCCESS */ 3490 if (!TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { 3491 return (DDI_SUCCESS); 3492 } 3493 3494 /* Shutdown HCA ports */ 3495 if (tavor_hca_ports_shutdown(state, 3496 state->ts_cfg_profile->cp_num_ports) != TAVOR_CMD_SUCCESS) { 3497 state->ts_quiescing = B_FALSE; 3498 return (DDI_FAILURE); 3499 } 3500 3501 /* Close HCA */ 3502 if (tavor_close_hca_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN) != 3503 TAVOR_CMD_SUCCESS) { 3504 state->ts_quiescing = B_FALSE; 3505 return (DDI_FAILURE); 3506 } 3507 3508 /* Shutdown FW */ 3509 if (tavor_sys_dis_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN) != 3510 TAVOR_CMD_SUCCESS) { 3511 state->ts_quiescing = B_FALSE; 3512 return (DDI_FAILURE); 3513 } 3514 3515 /* Disable interrupts */ 3516 if (tavor_intr_disable(state) != DDI_SUCCESS) { 3517 state->ts_quiescing = B_FALSE; 3518 return (DDI_FAILURE); 3519 } 3520 3521 /* SW-reset */ 3522 if (tavor_sw_reset(state) != DDI_SUCCESS) { 3523 state->ts_quiescing = B_FALSE; 3524 return (DDI_FAILURE); 3525 } 3526 3527 return (DDI_SUCCESS); 3528 } 3529