1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * tavor.c 29 * Tavor (InfiniBand) HCA Driver attach/detach Routines 30 * 31 * Implements all the routines necessary for the attach, setup, 32 * initialization (and subsequent possible teardown and detach) of the 33 * Tavor InfiniBand HCA driver. 34 */ 35 36 #include <sys/types.h> 37 #include <sys/file.h> 38 #include <sys/open.h> 39 #include <sys/conf.h> 40 #include <sys/ddi.h> 41 #include <sys/sunddi.h> 42 #include <sys/modctl.h> 43 #include <sys/stat.h> 44 #include <sys/pci.h> 45 #include <sys/pci_cap.h> 46 #include <sys/bitmap.h> 47 #include <sys/policy.h> 48 49 #include <sys/ib/adapters/tavor/tavor.h> 50 #include <sys/pci.h> 51 52 /* Tavor HCA State Pointer */ 53 void *tavor_statep; 54 55 /* 56 * The Tavor "userland resource database" is common to instances of the 57 * Tavor HCA driver. This structure "tavor_userland_rsrc_db" contains all 58 * the necessary information to maintain it. 59 */ 60 tavor_umap_db_t tavor_userland_rsrc_db; 61 62 static int tavor_attach(dev_info_t *, ddi_attach_cmd_t); 63 static int tavor_detach(dev_info_t *, ddi_detach_cmd_t); 64 static int tavor_open(dev_t *, int, int, cred_t *); 65 static int tavor_close(dev_t, int, int, cred_t *); 66 static int tavor_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **); 67 static int tavor_drv_init(tavor_state_t *state, dev_info_t *dip, int instance); 68 static void tavor_drv_fini(tavor_state_t *state); 69 static int tavor_isr_init(tavor_state_t *state); 70 static void tavor_isr_fini(tavor_state_t *state); 71 static int tavor_hw_init(tavor_state_t *state); 72 static void tavor_hw_fini(tavor_state_t *state, 73 tavor_drv_cleanup_level_t cleanup); 74 static int tavor_soft_state_init(tavor_state_t *state); 75 static void tavor_soft_state_fini(tavor_state_t *state); 76 static int tavor_hca_port_init(tavor_state_t *state); 77 static int tavor_hca_ports_shutdown(tavor_state_t *state, uint_t num_init); 78 static void tavor_hca_config_setup(tavor_state_t *state, 79 tavor_hw_initqueryhca_t *inithca); 80 static int tavor_internal_uarpgs_init(tavor_state_t *state); 81 static void tavor_internal_uarpgs_fini(tavor_state_t *state); 82 static int tavor_special_qp_contexts_reserve(tavor_state_t *state); 83 static void tavor_special_qp_contexts_unreserve(tavor_state_t *state); 84 static int tavor_sw_reset(tavor_state_t *state); 85 static int tavor_mcg_init(tavor_state_t *state); 86 static void tavor_mcg_fini(tavor_state_t *state); 87 static int tavor_fw_version_check(tavor_state_t *state); 88 static void tavor_device_info_report(tavor_state_t *state); 89 static void tavor_pci_capability_list(tavor_state_t *state, 90 ddi_acc_handle_t hdl); 91 static void tavor_pci_capability_vpd(tavor_state_t *state, 92 ddi_acc_handle_t hdl, uint_t offset); 93 static int tavor_pci_read_vpd(ddi_acc_handle_t hdl, uint_t offset, 94 uint32_t addr, uint32_t *data); 95 static void tavor_pci_capability_pcix(tavor_state_t *state, 96 ddi_acc_handle_t hdl, uint_t offset); 97 static int tavor_intr_or_msi_init(tavor_state_t *state); 98 static int tavor_add_intrs(tavor_state_t *state, int intr_type); 99 static int tavor_intr_or_msi_fini(tavor_state_t *state); 100 101 /* X86 fastreboot support */ 102 static int tavor_intr_disable(tavor_state_t *); 103 static int tavor_quiesce(dev_info_t *); 104 105 /* Character/Block Operations */ 106 static struct cb_ops tavor_cb_ops = { 107 tavor_open, /* open */ 108 tavor_close, /* close */ 109 nodev, /* strategy (block) */ 110 nodev, /* print (block) */ 111 nodev, /* dump (block) */ 112 nodev, /* read */ 113 nodev, /* write */ 114 tavor_ioctl, /* ioctl */ 115 tavor_devmap, /* devmap */ 116 NULL, /* mmap */ 117 nodev, /* segmap */ 118 nochpoll, /* chpoll */ 119 ddi_prop_op, /* prop_op */ 120 NULL, /* streams */ 121 D_NEW | D_MP | 122 D_64BIT | D_HOTPLUG | 123 D_DEVMAP, /* flags */ 124 CB_REV /* rev */ 125 }; 126 127 /* Driver Operations */ 128 static struct dev_ops tavor_ops = { 129 DEVO_REV, /* struct rev */ 130 0, /* refcnt */ 131 tavor_getinfo, /* getinfo */ 132 nulldev, /* identify */ 133 nulldev, /* probe */ 134 tavor_attach, /* attach */ 135 tavor_detach, /* detach */ 136 nodev, /* reset */ 137 &tavor_cb_ops, /* cb_ops */ 138 NULL, /* bus_ops */ 139 nodev, /* power */ 140 tavor_quiesce, /* devo_quiesce */ 141 }; 142 143 /* Module Driver Info */ 144 static struct modldrv tavor_modldrv = { 145 &mod_driverops, 146 "Tavor InfiniBand HCA Driver", 147 &tavor_ops 148 }; 149 150 /* Module Linkage */ 151 static struct modlinkage tavor_modlinkage = { 152 MODREV_1, 153 &tavor_modldrv, 154 NULL 155 }; 156 157 /* 158 * This extern refers to the ibc_operations_t function vector that is defined 159 * in the tavor_ci.c file. 160 */ 161 extern ibc_operations_t tavor_ibc_ops; 162 163 #ifndef NPROBE 164 extern int tnf_mod_load(void); 165 extern int tnf_mod_unload(struct modlinkage *mlp); 166 #endif 167 168 169 /* 170 * _init() 171 */ 172 int 173 _init() 174 { 175 int status; 176 177 #ifndef NPROBE 178 (void) tnf_mod_load(); 179 #endif 180 TAVOR_TNF_ENTER(tavor_init); 181 182 status = ddi_soft_state_init(&tavor_statep, sizeof (tavor_state_t), 183 (size_t)TAVOR_INITIAL_STATES); 184 if (status != 0) { 185 TNF_PROBE_0(tavor_init_ssi_fail, TAVOR_TNF_ERROR, ""); 186 TAVOR_TNF_EXIT(tavor_init); 187 #ifndef NPROBE 188 (void) tnf_mod_unload(&tavor_modlinkage); 189 #endif 190 return (status); 191 } 192 193 status = ibc_init(&tavor_modlinkage); 194 if (status != 0) { 195 TNF_PROBE_0(tavor_init_ibc_init_fail, TAVOR_TNF_ERROR, ""); 196 ddi_soft_state_fini(&tavor_statep); 197 TAVOR_TNF_EXIT(tavor_init); 198 #ifndef NPROBE 199 (void) tnf_mod_unload(&tavor_modlinkage); 200 #endif 201 return (status); 202 } 203 status = mod_install(&tavor_modlinkage); 204 if (status != 0) { 205 TNF_PROBE_0(tavor_init_modi_fail, TAVOR_TNF_ERROR, ""); 206 ibc_fini(&tavor_modlinkage); 207 ddi_soft_state_fini(&tavor_statep); 208 TAVOR_TNF_EXIT(tavor_init); 209 #ifndef NPROBE 210 (void) tnf_mod_unload(&tavor_modlinkage); 211 #endif 212 return (status); 213 } 214 215 /* Initialize the Tavor "userland resources database" */ 216 tavor_umap_db_init(); 217 218 TAVOR_TNF_EXIT(tavor_init); 219 return (status); 220 } 221 222 223 /* 224 * _info() 225 */ 226 int 227 _info(struct modinfo *modinfop) 228 { 229 int status; 230 231 TAVOR_TNF_ENTER(tavor_info); 232 status = mod_info(&tavor_modlinkage, modinfop); 233 TAVOR_TNF_EXIT(tavor_info); 234 return (status); 235 } 236 237 238 /* 239 * _fini() 240 */ 241 int 242 _fini() 243 { 244 int status; 245 246 TAVOR_TNF_ENTER(tavor_fini); 247 248 status = mod_remove(&tavor_modlinkage); 249 if (status != 0) { 250 TNF_PROBE_0(tavor_fini_modr_fail, TAVOR_TNF_ERROR, ""); 251 TAVOR_TNF_EXIT(tavor_fini); 252 return (status); 253 } 254 255 /* Destroy the Tavor "userland resources database" */ 256 tavor_umap_db_fini(); 257 258 ibc_fini(&tavor_modlinkage); 259 ddi_soft_state_fini(&tavor_statep); 260 #ifndef NPROBE 261 (void) tnf_mod_unload(&tavor_modlinkage); 262 #endif 263 TAVOR_TNF_EXIT(tavor_fini); 264 return (status); 265 } 266 267 268 /* 269 * tavor_getinfo() 270 */ 271 /* ARGSUSED */ 272 static int 273 tavor_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result) 274 { 275 dev_t dev; 276 tavor_state_t *state; 277 minor_t instance; 278 279 TAVOR_TNF_ENTER(tavor_getinfo); 280 281 switch (cmd) { 282 case DDI_INFO_DEVT2DEVINFO: 283 dev = (dev_t)arg; 284 instance = TAVOR_DEV_INSTANCE(dev); 285 state = ddi_get_soft_state(tavor_statep, instance); 286 if (state == NULL) { 287 TNF_PROBE_0(tavor_getinfo_gss_fail, 288 TAVOR_TNF_ERROR, ""); 289 TAVOR_TNF_EXIT(tavor_getinfo); 290 return (DDI_FAILURE); 291 } 292 *result = (void *)state->ts_dip; 293 return (DDI_SUCCESS); 294 295 case DDI_INFO_DEVT2INSTANCE: 296 dev = (dev_t)arg; 297 instance = TAVOR_DEV_INSTANCE(dev); 298 *result = (void *)(uintptr_t)instance; 299 return (DDI_SUCCESS); 300 301 default: 302 TNF_PROBE_0(tavor_getinfo_default_fail, TAVOR_TNF_ERROR, ""); 303 break; 304 } 305 306 TAVOR_TNF_EXIT(tavor_getinfo); 307 return (DDI_FAILURE); 308 } 309 310 311 /* 312 * tavor_open() 313 */ 314 /* ARGSUSED */ 315 static int 316 tavor_open(dev_t *devp, int flag, int otyp, cred_t *credp) 317 { 318 tavor_state_t *state; 319 tavor_rsrc_t *rsrcp; 320 tavor_umap_db_entry_t *umapdb, *umapdb2; 321 minor_t instance; 322 uint64_t key, value; 323 uint_t tr_indx; 324 dev_t dev; 325 int status; 326 327 TAVOR_TNF_ENTER(tavor_open); 328 329 instance = TAVOR_DEV_INSTANCE(*devp); 330 state = ddi_get_soft_state(tavor_statep, instance); 331 if (state == NULL) { 332 TNF_PROBE_0(tavor_open_gss_fail, TAVOR_TNF_ERROR, ""); 333 TAVOR_TNF_EXIT(tavor_open); 334 return (ENXIO); 335 } 336 337 /* 338 * Only allow driver to be opened for character access, and verify 339 * whether exclusive access is allowed. 340 */ 341 if ((otyp != OTYP_CHR) || ((flag & FEXCL) && 342 secpolicy_excl_open(credp) != 0)) { 343 TNF_PROBE_0(tavor_open_invflags_fail, TAVOR_TNF_ERROR, ""); 344 TAVOR_TNF_EXIT(tavor_open); 345 return (EINVAL); 346 } 347 348 /* 349 * Search for the current process PID in the "userland resources 350 * database". If it is not found, then attempt to allocate a UAR 351 * page and add the ("key", "value") pair to the database. 352 * Note: As a last step we always return a devp appropriate for 353 * the open. Either we return a new minor number (based on the 354 * instance and the UAR page index) or we return the current minor 355 * number for the given client process. 356 * 357 * We also add an entry to the database to allow for lookup from 358 * "dev_t" to the current process PID. This is necessary because, 359 * under certain circumstance, the process PID that calls the Tavor 360 * close() entry point may not be the same as the one who called 361 * open(). Specifically, this can happen if a child process calls 362 * the Tavor's open() entry point, gets a UAR page, maps it out (using 363 * mmap()), and then exits without calling munmap(). Because mmap() 364 * adds a reference to the file descriptor, at the exit of the child 365 * process the file descriptor is "inherited" by the parent (and will 366 * be close()'d by the parent's PID only when it exits). 367 * 368 * Note: We use the tavor_umap_db_find_nolock() and 369 * tavor_umap_db_add_nolock() database access routines below (with 370 * an explicit mutex_enter of the database lock - "tdl_umapdb_lock") 371 * to ensure that the multiple accesses (in this case searching for, 372 * and then adding _two_ database entries) can be done atomically. 373 */ 374 key = ddi_get_pid(); 375 mutex_enter(&tavor_userland_rsrc_db.tdl_umapdb_lock); 376 status = tavor_umap_db_find_nolock(instance, key, 377 MLNX_UMAP_UARPG_RSRC, &value, 0, NULL); 378 if (status != DDI_SUCCESS) { 379 /* 380 * If we are in 'maintenance mode', we cannot alloc a UAR page. 381 * But we still need some rsrcp value, and a mostly unique 382 * tr_indx value. So we set rsrcp to NULL for maintenance 383 * mode, and use a rolling count for tr_indx. The field 384 * 'ts_open_tr_indx' is used only in this maintenance mode 385 * condition. 386 * 387 * Otherwise, if we are in operational mode then we allocate 388 * the UAR page as normal, and use the rsrcp value and tr_indx 389 * value from that allocation. 390 */ 391 if (!TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { 392 rsrcp = NULL; 393 tr_indx = state->ts_open_tr_indx++; 394 } else { 395 /* Allocate a new UAR page for this process */ 396 status = tavor_rsrc_alloc(state, TAVOR_UARPG, 1, 397 TAVOR_NOSLEEP, &rsrcp); 398 if (status != DDI_SUCCESS) { 399 mutex_exit( 400 &tavor_userland_rsrc_db.tdl_umapdb_lock); 401 TNF_PROBE_0(tavor_open_rsrcalloc_uarpg_fail, 402 TAVOR_TNF_ERROR, ""); 403 TAVOR_TNF_EXIT(tavor_open); 404 return (EAGAIN); 405 } 406 407 tr_indx = rsrcp->tr_indx; 408 } 409 410 /* 411 * Allocate an entry to track the UAR page resource in the 412 * "userland resources database". 413 */ 414 umapdb = tavor_umap_db_alloc(instance, key, 415 MLNX_UMAP_UARPG_RSRC, (uint64_t)(uintptr_t)rsrcp); 416 if (umapdb == NULL) { 417 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock); 418 /* If in "maintenance mode", don't free the rsrc */ 419 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { 420 tavor_rsrc_free(state, &rsrcp); 421 } 422 TNF_PROBE_0(tavor_open_umap_db_alloc_fail, 423 TAVOR_TNF_ERROR, ""); 424 TAVOR_TNF_EXIT(tavor_open); 425 return (EAGAIN); 426 } 427 428 /* 429 * Create a new device number. Minor number is a function of 430 * the UAR page index (15 bits) and the device instance number 431 * (3 bits). 432 */ 433 dev = makedevice(getmajor(*devp), (tr_indx << 434 TAVOR_MINORNUM_SHIFT) | instance); 435 436 /* 437 * Allocate another entry in the "userland resources database" 438 * to track the association of the device number (above) to 439 * the current process ID (in "key"). 440 */ 441 umapdb2 = tavor_umap_db_alloc(instance, dev, 442 MLNX_UMAP_PID_RSRC, (uint64_t)key); 443 if (umapdb2 == NULL) { 444 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock); 445 tavor_umap_db_free(umapdb); 446 /* If in "maintenance mode", don't free the rsrc */ 447 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { 448 tavor_rsrc_free(state, &rsrcp); 449 } 450 TNF_PROBE_0(tavor_open_umap_db_alloc_fail, 451 TAVOR_TNF_ERROR, ""); 452 TAVOR_TNF_EXIT(tavor_open); 453 return (EAGAIN); 454 } 455 456 /* Add the entries to the database */ 457 tavor_umap_db_add_nolock(umapdb); 458 tavor_umap_db_add_nolock(umapdb2); 459 460 } else { 461 /* 462 * Return the same device number as on the original open() 463 * call. This was calculated as a function of the UAR page 464 * index (top 16 bits) and the device instance number 465 */ 466 rsrcp = (tavor_rsrc_t *)(uintptr_t)value; 467 dev = makedevice(getmajor(*devp), (rsrcp->tr_indx << 468 TAVOR_MINORNUM_SHIFT) | instance); 469 } 470 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock); 471 472 *devp = dev; 473 474 TAVOR_TNF_EXIT(tavor_open); 475 return (0); 476 } 477 478 479 /* 480 * tavor_close() 481 */ 482 /* ARGSUSED */ 483 static int 484 tavor_close(dev_t dev, int flag, int otyp, cred_t *credp) 485 { 486 tavor_state_t *state; 487 tavor_rsrc_t *rsrcp; 488 tavor_umap_db_entry_t *umapdb; 489 tavor_umap_db_priv_t *priv; 490 minor_t instance; 491 uint64_t key, value; 492 int status; 493 494 TAVOR_TNF_ENTER(tavor_close); 495 496 instance = TAVOR_DEV_INSTANCE(dev); 497 state = ddi_get_soft_state(tavor_statep, instance); 498 if (state == NULL) { 499 TNF_PROBE_0(tavor_close_gss_fail, TAVOR_TNF_ERROR, ""); 500 TAVOR_TNF_EXIT(tavor_close); 501 return (ENXIO); 502 } 503 504 /* 505 * Search for "dev_t" in the "userland resources database". As 506 * explained above in tavor_open(), we can't depend on using the 507 * current process ID here to do the lookup because the process 508 * that ultimately closes may not be the same one who opened 509 * (because of inheritance). 510 * So we lookup the "dev_t" (which points to the PID of the process 511 * that opened), and we remove the entry from the database (and free 512 * it up). Then we do another query based on the PID value. And when 513 * we find that database entry, we free it up too and then free the 514 * Tavor UAR page resource. 515 * 516 * Note: We use the tavor_umap_db_find_nolock() database access 517 * routine below (with an explicit mutex_enter of the database lock) 518 * to ensure that the multiple accesses (which attempt to remove the 519 * two database entries) can be done atomically. 520 * 521 * This works the same in both maintenance mode and HCA mode, except 522 * for the call to tavor_rsrc_free(). In the case of maintenance mode, 523 * this call is not needed, as it was not allocated in tavor_open() 524 * above. 525 */ 526 key = dev; 527 mutex_enter(&tavor_userland_rsrc_db.tdl_umapdb_lock); 528 status = tavor_umap_db_find_nolock(instance, key, MLNX_UMAP_PID_RSRC, 529 &value, TAVOR_UMAP_DB_REMOVE, &umapdb); 530 if (status == DDI_SUCCESS) { 531 /* 532 * If the "tdb_priv" field is non-NULL, it indicates that 533 * some "on close" handling is still necessary. Call 534 * tavor_umap_db_handle_onclose_cb() to do the handling (i.e. 535 * to invoke all the registered callbacks). Then free up 536 * the resources associated with "tdb_priv" and continue 537 * closing. 538 */ 539 priv = (tavor_umap_db_priv_t *)umapdb->tdbe_common.tdb_priv; 540 if (priv != NULL) { 541 tavor_umap_db_handle_onclose_cb(priv); 542 kmem_free(priv, sizeof (tavor_umap_db_priv_t)); 543 umapdb->tdbe_common.tdb_priv = (void *)NULL; 544 } 545 546 tavor_umap_db_free(umapdb); 547 548 /* 549 * Now do another lookup using PID as the key (copy it from 550 * "value"). When this lookup is complete, the "value" field 551 * will contain the tavor_rsrc_t pointer for the UAR page 552 * resource. 553 */ 554 key = value; 555 status = tavor_umap_db_find_nolock(instance, key, 556 MLNX_UMAP_UARPG_RSRC, &value, TAVOR_UMAP_DB_REMOVE, 557 &umapdb); 558 if (status == DDI_SUCCESS) { 559 tavor_umap_db_free(umapdb); 560 /* If in "maintenance mode", don't free the rsrc */ 561 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { 562 rsrcp = (tavor_rsrc_t *)(uintptr_t)value; 563 tavor_rsrc_free(state, &rsrcp); 564 } 565 } 566 } 567 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock); 568 569 TAVOR_TNF_EXIT(tavor_close); 570 return (0); 571 } 572 573 574 /* 575 * tavor_attach() 576 * Context: Only called from attach() path context 577 */ 578 static int 579 tavor_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 580 { 581 tavor_state_t *state; 582 ibc_clnt_hdl_t tmp_ibtfpriv; 583 ibc_status_t ibc_status; 584 int instance; 585 int status; 586 587 TAVOR_TNF_ENTER(tavor_attach); 588 589 #ifdef __lock_lint 590 (void) tavor_quiesce(dip); 591 #endif 592 593 switch (cmd) { 594 case DDI_ATTACH: 595 instance = ddi_get_instance(dip); 596 status = ddi_soft_state_zalloc(tavor_statep, instance); 597 if (status != DDI_SUCCESS) { 598 TNF_PROBE_0(tavor_attach_ssz_fail, TAVOR_TNF_ERROR, ""); 599 cmn_err(CE_NOTE, "tavor%d: driver failed to attach: " 600 "attach_ssz_fail", instance); 601 goto fail_attach_nomsg; 602 603 } 604 state = ddi_get_soft_state(tavor_statep, instance); 605 if (state == NULL) { 606 ddi_soft_state_free(tavor_statep, instance); 607 TNF_PROBE_0(tavor_attach_gss_fail, TAVOR_TNF_ERROR, ""); 608 cmn_err(CE_NOTE, "tavor%d: driver failed to attach: " 609 "attach_gss_fail", instance); 610 goto fail_attach_nomsg; 611 } 612 613 /* clear the attach error buffer */ 614 TAVOR_ATTACH_MSG_INIT(state->ts_attach_buf); 615 616 /* 617 * Initialize Tavor driver and hardware. 618 * 619 * Note: If this initialization fails we may still wish to 620 * create a device node and remain operational so that Tavor 621 * firmware can be updated/flashed (i.e. "maintenance mode"). 622 * If this is the case, then "ts_operational_mode" will be 623 * equal to TAVOR_MAINTENANCE_MODE. We will not attempt to 624 * attach to the IBTF or register with the IBMF (i.e. no 625 * InfiniBand interfaces will be enabled). 626 */ 627 status = tavor_drv_init(state, dip, instance); 628 if ((status != DDI_SUCCESS) && 629 (TAVOR_IS_OPERATIONAL(state->ts_operational_mode))) { 630 TNF_PROBE_0(tavor_attach_drvinit_fail, 631 TAVOR_TNF_ERROR, ""); 632 goto fail_attach; 633 } 634 635 /* Create the minor node for device */ 636 status = ddi_create_minor_node(dip, "devctl", S_IFCHR, instance, 637 DDI_PSEUDO, 0); 638 if (status != DDI_SUCCESS) { 639 tavor_drv_fini(state); 640 TAVOR_ATTACH_MSG(state->ts_attach_buf, 641 "attach_create_mn_fail"); 642 TNF_PROBE_0(tavor_attach_create_mn_fail, 643 TAVOR_TNF_ERROR, ""); 644 goto fail_attach; 645 } 646 647 /* 648 * If we are in "maintenance mode", then we don't want to 649 * register with the IBTF. All InfiniBand interfaces are 650 * uninitialized, and the device is only capable of handling 651 * requests to update/flash firmware (or test/debug requests). 652 */ 653 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { 654 655 /* Attach to InfiniBand Transport Framework (IBTF) */ 656 ibc_status = ibc_attach(&tmp_ibtfpriv, 657 &state->ts_ibtfinfo); 658 if (ibc_status != IBC_SUCCESS) { 659 ddi_remove_minor_node(dip, "devctl"); 660 tavor_drv_fini(state); 661 TNF_PROBE_0(tavor_attach_ibcattach_fail, 662 TAVOR_TNF_ERROR, ""); 663 TAVOR_ATTACH_MSG(state->ts_attach_buf, 664 "attach_ibcattach_fail"); 665 goto fail_attach; 666 } 667 668 /* 669 * Now that we've successfully attached to the IBTF, 670 * we enable all appropriate asynch and CQ events to 671 * be forwarded to the IBTF. 672 */ 673 TAVOR_ENABLE_IBTF_CALLB(state, tmp_ibtfpriv); 674 675 ibc_post_attach(state->ts_ibtfpriv); 676 677 /* Register agents with IB Mgmt Framework (IBMF) */ 678 status = tavor_agent_handlers_init(state); 679 if (status != DDI_SUCCESS) { 680 (void) ibc_pre_detach(tmp_ibtfpriv, DDI_DETACH); 681 TAVOR_QUIESCE_IBTF_CALLB(state); 682 if (state->ts_in_evcallb != 0) { 683 TAVOR_WARNING(state, "unable to " 684 "quiesce Tavor IBTF callbacks"); 685 } 686 ibc_detach(tmp_ibtfpriv); 687 ddi_remove_minor_node(dip, "devctl"); 688 tavor_drv_fini(state); 689 TNF_PROBE_0(tavor_attach_agentinit_fail, 690 TAVOR_TNF_ERROR, ""); 691 TAVOR_ATTACH_MSG(state->ts_attach_buf, 692 "attach_agentinit_fail"); 693 goto fail_attach; 694 } 695 } 696 697 /* Report that driver was loaded */ 698 ddi_report_dev(dip); 699 700 /* Send device information to log file */ 701 tavor_device_info_report(state); 702 703 /* Report attach in maintenance mode, if appropriate */ 704 if (!(TAVOR_IS_OPERATIONAL(state->ts_operational_mode))) { 705 cmn_err(CE_NOTE, "tavor%d: driver attached " 706 "(for maintenance mode only)", state->ts_instance); 707 } 708 709 TAVOR_TNF_EXIT(tavor_attach); 710 return (DDI_SUCCESS); 711 712 case DDI_RESUME: 713 /* Add code here for DDI_RESUME XXX */ 714 TAVOR_TNF_EXIT(tavor_attach); 715 return (DDI_FAILURE); 716 717 default: 718 TNF_PROBE_0(tavor_attach_default_fail, TAVOR_TNF_ERROR, ""); 719 break; 720 } 721 722 fail_attach: 723 cmn_err(CE_NOTE, "tavor%d: driver failed to attach: %s", instance, 724 state->ts_attach_buf); 725 ddi_soft_state_free(tavor_statep, instance); 726 fail_attach_nomsg: 727 TAVOR_TNF_EXIT(tavor_attach); 728 return (DDI_FAILURE); 729 } 730 731 732 /* 733 * tavor_detach() 734 * Context: Only called from detach() path context 735 */ 736 static int 737 tavor_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 738 { 739 tavor_state_t *state; 740 ibc_clnt_hdl_t tmp_ibtfpriv; 741 ibc_status_t ibc_status; 742 int instance, status; 743 744 TAVOR_TNF_ENTER(tavor_detach); 745 746 instance = ddi_get_instance(dip); 747 state = ddi_get_soft_state(tavor_statep, instance); 748 if (state == NULL) { 749 TNF_PROBE_0(tavor_detach_gss_fail, TAVOR_TNF_ERROR, ""); 750 TAVOR_TNF_EXIT(tavor_detach); 751 return (DDI_FAILURE); 752 } 753 754 switch (cmd) { 755 case DDI_DETACH: 756 /* 757 * If we are in "maintenance mode", then we do not want to 758 * do teardown for any of the InfiniBand interfaces. 759 * Specifically, this means not detaching from IBTF (we never 760 * attached to begin with) and not deregistering from IBMF. 761 */ 762 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { 763 /* Unregister agents from IB Mgmt Framework (IBMF) */ 764 status = tavor_agent_handlers_fini(state); 765 if (status != DDI_SUCCESS) { 766 TNF_PROBE_0(tavor_detach_agentfini_fail, 767 TAVOR_TNF_ERROR, ""); 768 TAVOR_TNF_EXIT(tavor_detach); 769 return (DDI_FAILURE); 770 } 771 772 /* 773 * Attempt the "pre-detach" from InfiniBand Transport 774 * Framework (IBTF). At this point the IBTF is still 775 * capable of handling incoming asynch and completion 776 * events. This "pre-detach" is primarily a mechanism 777 * to notify the appropriate IBTF clients that the 778 * HCA is being removed/offlined. 779 */ 780 ibc_status = ibc_pre_detach(state->ts_ibtfpriv, cmd); 781 if (ibc_status != IBC_SUCCESS) { 782 status = tavor_agent_handlers_init(state); 783 if (status != DDI_SUCCESS) { 784 TAVOR_WARNING(state, "failed to " 785 "restart Tavor agents"); 786 } 787 TNF_PROBE_0(tavor_detach_ibcpredetach_fail, 788 TAVOR_TNF_ERROR, ""); 789 TAVOR_TNF_EXIT(tavor_detach); 790 return (DDI_FAILURE); 791 } 792 793 /* 794 * Before we can fully detach from the IBTF we need to 795 * ensure that we have handled all outstanding event 796 * callbacks. This is accomplished by quiescing the 797 * event callback mechanism. Note: if we are unable 798 * to successfully quiesce the callbacks, then this is 799 * an indication that something has probably gone 800 * seriously wrong. We print out a warning, but 801 * continue. 802 */ 803 tmp_ibtfpriv = state->ts_ibtfpriv; 804 TAVOR_QUIESCE_IBTF_CALLB(state); 805 if (state->ts_in_evcallb != 0) { 806 TAVOR_WARNING(state, "unable to quiesce Tavor " 807 "IBTF callbacks"); 808 } 809 810 /* Complete the detach from the IBTF */ 811 ibc_detach(tmp_ibtfpriv); 812 } 813 814 /* Remove the minor node for device */ 815 ddi_remove_minor_node(dip, "devctl"); 816 817 /* 818 * Only call tavor_drv_fini() if we are in Tavor HCA mode. 819 * (Because if we are in "maintenance mode", then we never 820 * successfully finished init.) Only report successful 821 * detach for normal HCA mode. 822 */ 823 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { 824 /* Cleanup driver resources and shutdown hardware */ 825 tavor_drv_fini(state); 826 cmn_err(CE_CONT, "Tavor driver successfully " 827 "detached\n"); 828 } 829 830 ddi_soft_state_free(tavor_statep, instance); 831 832 TAVOR_TNF_EXIT(tavor_detach); 833 return (DDI_SUCCESS); 834 835 case DDI_SUSPEND: 836 /* Add code here for DDI_SUSPEND XXX */ 837 TAVOR_TNF_EXIT(tavor_detach); 838 return (DDI_FAILURE); 839 840 default: 841 TNF_PROBE_0(tavor_detach_default_fail, TAVOR_TNF_ERROR, ""); 842 break; 843 } 844 845 TAVOR_TNF_EXIT(tavor_detach); 846 return (DDI_FAILURE); 847 } 848 849 850 /* 851 * tavor_drv_init() 852 * Context: Only called from attach() path context 853 */ 854 static int 855 tavor_drv_init(tavor_state_t *state, dev_info_t *dip, int instance) 856 { 857 int status; 858 859 TAVOR_TNF_ENTER(tavor_drv_init); 860 861 /* Save away devinfo and instance */ 862 state->ts_dip = dip; 863 state->ts_instance = instance; 864 865 /* 866 * Check and set the operational mode of the device. If the driver is 867 * bound to the Tavor device in "maintenance mode", then this generally 868 * means that either the device has been specifically jumpered to 869 * start in this mode or the firmware boot process has failed to 870 * successfully load either the primary or the secondary firmware 871 * image. 872 */ 873 if (TAVOR_IS_HCA_MODE(state->ts_dip)) { 874 state->ts_operational_mode = TAVOR_HCA_MODE; 875 876 } else if (TAVOR_IS_COMPAT_MODE(state->ts_dip)) { 877 state->ts_operational_mode = TAVOR_COMPAT_MODE; 878 879 } else if (TAVOR_IS_MAINTENANCE_MODE(state->ts_dip)) { 880 state->ts_operational_mode = TAVOR_MAINTENANCE_MODE; 881 return (DDI_FAILURE); 882 883 } else { 884 state->ts_operational_mode = 0; /* invalid operational mode */ 885 TAVOR_WARNING(state, "unexpected device type detected"); 886 TNF_PROBE_0(tavor_hw_init_unexpected_dev_fail, 887 TAVOR_TNF_ERROR, ""); 888 TAVOR_TNF_EXIT(tavor_hw_init); 889 return (DDI_FAILURE); 890 } 891 892 /* 893 * Initialize the Tavor hardware. 894 * 895 * Note: If this routine returns an error, it is often an reasonably 896 * good indication that something Tavor firmware-related has caused 897 * the failure. In order to give the user an opportunity (if desired) 898 * to update or reflash the Tavor firmware image, we set 899 * "ts_operational_mode" flag (described above) to indicate that we 900 * wish to enter maintenance mode. 901 */ 902 status = tavor_hw_init(state); 903 if (status != DDI_SUCCESS) { 904 state->ts_operational_mode = TAVOR_MAINTENANCE_MODE; 905 cmn_err(CE_NOTE, "tavor%d: error during attach: %s", instance, 906 state->ts_attach_buf); 907 TNF_PROBE_0(tavor_drv_init_hwinit_fail, TAVOR_TNF_ERROR, ""); 908 TAVOR_TNF_EXIT(tavor_drv_init); 909 return (DDI_FAILURE); 910 } 911 912 /* Setup Tavor interrupt handler */ 913 status = tavor_isr_init(state); 914 if (status != DDI_SUCCESS) { 915 tavor_hw_fini(state, TAVOR_DRV_CLEANUP_ALL); 916 TNF_PROBE_0(tavor_drv_init_isrinit_fail, TAVOR_TNF_ERROR, ""); 917 TAVOR_TNF_EXIT(tavor_drv_init); 918 return (DDI_FAILURE); 919 } 920 921 /* Initialize Tavor softstate */ 922 status = tavor_soft_state_init(state); 923 if (status != DDI_SUCCESS) { 924 tavor_isr_fini(state); 925 tavor_hw_fini(state, TAVOR_DRV_CLEANUP_ALL); 926 TNF_PROBE_0(tavor_drv_init_ssiinit_fail, TAVOR_TNF_ERROR, ""); 927 TAVOR_TNF_EXIT(tavor_drv_init); 928 return (DDI_FAILURE); 929 } 930 931 TAVOR_TNF_EXIT(tavor_drv_init); 932 return (DDI_SUCCESS); 933 } 934 935 936 /* 937 * tavor_drv_fini() 938 * Context: Only called from attach() and/or detach() path contexts 939 */ 940 static void 941 tavor_drv_fini(tavor_state_t *state) 942 { 943 TAVOR_TNF_ENTER(tavor_drv_fini); 944 945 /* Cleanup Tavor softstate */ 946 tavor_soft_state_fini(state); 947 948 /* Teardown Tavor interrupts */ 949 tavor_isr_fini(state); 950 951 /* Cleanup Tavor resources and shutdown hardware */ 952 tavor_hw_fini(state, TAVOR_DRV_CLEANUP_ALL); 953 954 TAVOR_TNF_EXIT(tavor_drv_fini); 955 } 956 957 958 /* 959 * tavor_isr_init() 960 * Context: Only called from attach() path context 961 */ 962 static int 963 tavor_isr_init(tavor_state_t *state) 964 { 965 int status; 966 967 TAVOR_TNF_ENTER(tavor_isr_init); 968 969 /* 970 * Add a handler for the interrupt or MSI 971 */ 972 status = ddi_intr_add_handler(state->ts_intrmsi_hdl, tavor_isr, 973 (caddr_t)state, NULL); 974 if (status != DDI_SUCCESS) { 975 TNF_PROBE_0(tavor_isr_init_addhndlr_fail, TAVOR_TNF_ERROR, ""); 976 TAVOR_TNF_EXIT(tavor_isr_init); 977 return (DDI_FAILURE); 978 } 979 980 /* 981 * Enable the software interrupt. Note: Even though we are only 982 * using one (1) interrupt/MSI, depending on the value returned in 983 * the capability flag, we have to call either ddi_intr_block_enable() 984 * or ddi_intr_enable(). 985 */ 986 if (state->ts_intrmsi_cap & DDI_INTR_FLAG_BLOCK) { 987 status = ddi_intr_block_enable(&state->ts_intrmsi_hdl, 1); 988 if (status != DDI_SUCCESS) { 989 TNF_PROBE_0(tavor_isr_init_blockenable_fail, 990 TAVOR_TNF_ERROR, ""); 991 TAVOR_TNF_EXIT(tavor_isr_init); 992 return (DDI_FAILURE); 993 } 994 } else { 995 status = ddi_intr_enable(state->ts_intrmsi_hdl); 996 if (status != DDI_SUCCESS) { 997 TNF_PROBE_0(tavor_isr_init_intrenable_fail, 998 TAVOR_TNF_ERROR, ""); 999 TAVOR_TNF_EXIT(tavor_isr_init); 1000 return (DDI_FAILURE); 1001 } 1002 } 1003 1004 /* 1005 * Now that the ISR has been setup, arm all the EQs for event 1006 * generation. 1007 */ 1008 tavor_eq_arm_all(state); 1009 1010 TAVOR_TNF_EXIT(tavor_isr_init); 1011 return (DDI_SUCCESS); 1012 } 1013 1014 1015 /* 1016 * tavor_isr_fini() 1017 * Context: Only called from attach() and/or detach() path contexts 1018 */ 1019 static void 1020 tavor_isr_fini(tavor_state_t *state) 1021 { 1022 TAVOR_TNF_ENTER(tavor_isr_fini); 1023 1024 /* Disable the software interrupt */ 1025 if (state->ts_intrmsi_cap & DDI_INTR_FLAG_BLOCK) { 1026 (void) ddi_intr_block_disable(&state->ts_intrmsi_hdl, 1); 1027 } else { 1028 (void) ddi_intr_disable(state->ts_intrmsi_hdl); 1029 } 1030 1031 /* 1032 * Remove the software handler for the interrupt or MSI 1033 */ 1034 (void) ddi_intr_remove_handler(state->ts_intrmsi_hdl); 1035 1036 TAVOR_TNF_EXIT(tavor_isr_fini); 1037 } 1038 1039 1040 /* 1041 * tavor_fix_error_buf() 1042 * Context: Only called from attach(). 1043 * 1044 * The error_buf_addr returned from QUERY_FW is a PCI address. 1045 * We need to convert it to an offset from the base address, 1046 * which is stored in the assigned-addresses property. 1047 */ 1048 static int 1049 tavor_fix_error_buf(tavor_state_t *state) 1050 { 1051 int assigned_addr_len; 1052 pci_regspec_t *assigned_addr; 1053 1054 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, state->ts_dip, 1055 DDI_PROP_DONTPASS, "assigned-addresses", (int **)&assigned_addr, 1056 (uint_t *)&assigned_addr_len) != DDI_PROP_SUCCESS) 1057 return (DDI_FAILURE); 1058 1059 state->ts_fw.error_buf_addr -= assigned_addr[0].pci_phys_low + 1060 ((uint64_t)(assigned_addr[0].pci_phys_mid) << 32); 1061 ddi_prop_free(assigned_addr); 1062 return (DDI_SUCCESS); 1063 } 1064 1065 /* 1066 * tavor_hw_init() 1067 * Context: Only called from attach() path context 1068 */ 1069 static int 1070 tavor_hw_init(tavor_state_t *state) 1071 { 1072 tavor_drv_cleanup_level_t cleanup; 1073 sm_nodeinfo_t nodeinfo; 1074 uint64_t errorcode; 1075 off_t ddr_size; 1076 int status; 1077 int retries; 1078 1079 TAVOR_TNF_ENTER(tavor_hw_init); 1080 1081 /* This is where driver initialization begins */ 1082 cleanup = TAVOR_DRV_CLEANUP_LEVEL0; 1083 1084 /* Setup device access attributes */ 1085 state->ts_reg_accattr.devacc_attr_version = DDI_DEVICE_ATTR_V0; 1086 state->ts_reg_accattr.devacc_attr_endian_flags = DDI_STRUCTURE_BE_ACC; 1087 state->ts_reg_accattr.devacc_attr_dataorder = DDI_STRICTORDER_ACC; 1088 1089 /* Setup for PCI config read/write of HCA device */ 1090 status = pci_config_setup(state->ts_dip, &state->ts_pci_cfghdl); 1091 if (status != DDI_SUCCESS) { 1092 tavor_hw_fini(state, cleanup); 1093 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1094 "hw_init_PCI_config_space_regmap_fail"); 1095 /* This case is not the degraded one */ 1096 return (DDI_FAILURE); 1097 } 1098 1099 /* Map in Tavor registers (CMD, UAR, DDR) and setup offsets */ 1100 status = ddi_regs_map_setup(state->ts_dip, TAVOR_CMD_BAR, 1101 &state->ts_reg_cmd_baseaddr, 0, 0, &state->ts_reg_accattr, 1102 &state->ts_reg_cmdhdl); 1103 if (status != DDI_SUCCESS) { 1104 tavor_hw_fini(state, cleanup); 1105 TNF_PROBE_0(tavor_hw_init_CMD_ddirms_fail, TAVOR_TNF_ERROR, ""); 1106 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1107 "hw_init_CMD_ddirms_fail"); 1108 TAVOR_TNF_EXIT(tavor_hw_init); 1109 return (DDI_FAILURE); 1110 } 1111 cleanup = TAVOR_DRV_CLEANUP_LEVEL1; 1112 1113 status = ddi_regs_map_setup(state->ts_dip, TAVOR_UAR_BAR, 1114 &state->ts_reg_uar_baseaddr, 0, 0, &state->ts_reg_accattr, 1115 &state->ts_reg_uarhdl); 1116 if (status != DDI_SUCCESS) { 1117 tavor_hw_fini(state, cleanup); 1118 TNF_PROBE_0(tavor_hw_init_UAR_ddirms_fail, TAVOR_TNF_ERROR, ""); 1119 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1120 "hw_init_UAR_ddirms_fail"); 1121 TAVOR_TNF_EXIT(tavor_hw_init); 1122 return (DDI_FAILURE); 1123 } 1124 cleanup = TAVOR_DRV_CLEANUP_LEVEL2; 1125 1126 status = ddi_dev_regsize(state->ts_dip, TAVOR_DDR_BAR, &ddr_size); 1127 if (status != DDI_SUCCESS) { 1128 cmn_err(CE_CONT, "Tavor: ddi_dev_regsize() failed " 1129 "(check HCA-attached DIMM memory?)\n"); 1130 tavor_hw_fini(state, cleanup); 1131 TNF_PROBE_0(tavor_hw_init_DDR_ddi_regsize_fail, 1132 TAVOR_TNF_ERROR, ""); 1133 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1134 "hw_init_DDR_ddi_regsize_fail"); 1135 TAVOR_TNF_EXIT(tavor_hw_init); 1136 return (DDI_FAILURE); 1137 } 1138 1139 #if !defined(_ELF64) && !defined(__sparc) 1140 /* 1141 * For 32 bit x86/x64 kernels, where there is limited kernel virtual 1142 * memory available, define a minimal memory footprint. This is 1143 * specified in order to not take up too much resources, thus starving 1144 * out others. Only specified if the HCA DIMM is equal to or greater 1145 * than 256MB. 1146 * 1147 * Note: x86/x64 install and safemode boot are both 32bit. 1148 */ 1149 ddr_size = TAVOR_DDR_SIZE_MIN; 1150 #endif /* !(_ELF64) && !(__sparc) */ 1151 1152 state->ts_cfg_profile_setting = ddr_size; 1153 1154 status = ddi_regs_map_setup(state->ts_dip, TAVOR_DDR_BAR, 1155 &state->ts_reg_ddr_baseaddr, 0, ddr_size, &state->ts_reg_accattr, 1156 &state->ts_reg_ddrhdl); 1157 1158 /* 1159 * On 32-bit platform testing (primarily x86), it was seen that the 1160 * ddi_regs_map_setup() call would fail because there wasn't enough 1161 * kernel virtual address space available to map in the entire 256MB 1162 * DDR. So we add this check in here, so that if the 256 (or other 1163 * larger value of DDR) map in fails, that we fallback to try the lower 1164 * size of 128MB. 1165 * 1166 * Note: If we only have 128MB of DDR in the system in the first place, 1167 * we don't try another ddi_regs_map_setup(), and just skip over this 1168 * check and return failures. 1169 */ 1170 if (status == DDI_ME_NORESOURCES && ddr_size > TAVOR_DDR_SIZE_128) { 1171 /* Try falling back to 128MB DDR mapping */ 1172 status = ddi_regs_map_setup(state->ts_dip, TAVOR_DDR_BAR, 1173 &state->ts_reg_ddr_baseaddr, 0, TAVOR_DDR_SIZE_128, 1174 &state->ts_reg_accattr, &state->ts_reg_ddrhdl); 1175 1176 /* 1177 * 128MB DDR mapping worked. 1178 * Set the updated config profile setting here. 1179 */ 1180 if (status == DDI_SUCCESS) { 1181 TNF_PROBE_0(tavor_hw_init_DDR_128mb_fallback_success, 1182 TAVOR_TNF_TRACE, ""); 1183 state->ts_cfg_profile_setting = TAVOR_DDR_SIZE_128; 1184 } 1185 } 1186 1187 if (status != DDI_SUCCESS) { 1188 if (status == DDI_ME_RNUMBER_RANGE) { 1189 cmn_err(CE_CONT, "Tavor: ddi_regs_map_setup() failed " 1190 "(check HCA-attached DIMM memory?)\n"); 1191 } 1192 tavor_hw_fini(state, cleanup); 1193 TNF_PROBE_0(tavor_hw_init_DDR_ddirms_fail, TAVOR_TNF_ERROR, ""); 1194 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1195 "hw_init_DDR_ddirms_fail"); 1196 TAVOR_TNF_EXIT(tavor_hw_init); 1197 return (DDI_FAILURE); 1198 } 1199 cleanup = TAVOR_DRV_CLEANUP_LEVEL3; 1200 1201 /* Setup Tavor Host Command Register (HCR) */ 1202 state->ts_cmd_regs.hcr = (tavor_hw_hcr_t *) 1203 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_HCR_OFFSET); 1204 1205 /* Setup Tavor Event Cause Register (ecr and clr_ecr) */ 1206 state->ts_cmd_regs.ecr = (uint64_t *) 1207 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_ECR_OFFSET); 1208 state->ts_cmd_regs.clr_ecr = (uint64_t *) 1209 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_CLR_ECR_OFFSET); 1210 1211 /* Setup Tavor Software Reset register (sw_reset) */ 1212 state->ts_cmd_regs.sw_reset = (uint32_t *) 1213 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_SW_RESET_OFFSET); 1214 1215 /* Setup Tavor Clear Interrupt register (clr_int) */ 1216 state->ts_cmd_regs.clr_int = (uint64_t *) 1217 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_CLR_INT_OFFSET); 1218 1219 /* Initialize the Phase1 Tavor configuration profile */ 1220 status = tavor_cfg_profile_init_phase1(state); 1221 if (status != DDI_SUCCESS) { 1222 tavor_hw_fini(state, cleanup); 1223 TNF_PROBE_0(tavor_hw_init_cfginit_fail, TAVOR_TNF_ERROR, ""); 1224 TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_cfginit_fail"); 1225 TAVOR_TNF_EXIT(tavor_hw_init); 1226 return (DDI_FAILURE); 1227 } 1228 cleanup = TAVOR_DRV_CLEANUP_LEVEL4; 1229 1230 /* Do a software reset of the Tavor HW to ensure proper state */ 1231 status = tavor_sw_reset(state); 1232 if (status != TAVOR_CMD_SUCCESS) { 1233 tavor_hw_fini(state, cleanup); 1234 TNF_PROBE_0(tavor_hw_init_sw_reset_fail, TAVOR_TNF_ERROR, ""); 1235 TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_sw_reset_fail"); 1236 TAVOR_TNF_EXIT(tavor_hw_init); 1237 return (DDI_FAILURE); 1238 } 1239 1240 /* Post the SYS_EN command to start the hardware */ 1241 status = tavor_sys_en_cmd_post(state, TAVOR_CMD_SYS_EN_NORMAL, 1242 &errorcode, TAVOR_CMD_NOSLEEP_SPIN); 1243 if (status != TAVOR_CMD_SUCCESS) { 1244 if ((status == TAVOR_CMD_BAD_NVMEM) || 1245 (status == TAVOR_CMD_DDR_MEM_ERR)) { 1246 cmn_err(CE_CONT, "Tavor: SYS_EN command failed: 0x%x " 1247 "0x%" PRIx64 " (invalid firmware image?)\n", 1248 status, errorcode); 1249 } else { 1250 cmn_err(CE_CONT, "Tavor: SYS_EN command failed: 0x%x " 1251 "0x%" PRIx64 "\n", status, errorcode); 1252 } 1253 tavor_hw_fini(state, cleanup); 1254 TNF_PROBE_0(tavor_hw_init_sys_en_cmd_fail, 1255 TAVOR_TNF_ERROR, ""); 1256 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1257 "hw_init_sys_en_cmd_fail"); 1258 TAVOR_TNF_EXIT(tavor_hw_init); 1259 return (DDI_FAILURE); 1260 } 1261 cleanup = TAVOR_DRV_CLEANUP_LEVEL5; 1262 1263 /* First phase of init for Tavor configuration/resources */ 1264 status = tavor_rsrc_init_phase1(state); 1265 if (status != DDI_SUCCESS) { 1266 tavor_hw_fini(state, cleanup); 1267 TNF_PROBE_0(tavor_hw_init_rsrcinit1_fail, TAVOR_TNF_ERROR, ""); 1268 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1269 "hw_init_rsrcinit1_fail"); 1270 TAVOR_TNF_EXIT(tavor_hw_init); 1271 return (DDI_FAILURE); 1272 } 1273 cleanup = TAVOR_DRV_CLEANUP_LEVEL6; 1274 1275 /* Query the DDR properties (e.g. total DDR size) */ 1276 status = tavor_cmn_query_cmd_post(state, QUERY_DDR, 0, 1277 &state->ts_ddr, sizeof (tavor_hw_queryddr_t), 1278 TAVOR_CMD_NOSLEEP_SPIN); 1279 if (status != TAVOR_CMD_SUCCESS) { 1280 cmn_err(CE_CONT, "Tavor: QUERY_DDR command failed: %08x\n", 1281 status); 1282 tavor_hw_fini(state, cleanup); 1283 TNF_PROBE_0(tavor_hw_init_query_ddr_cmd_fail, 1284 TAVOR_TNF_ERROR, ""); 1285 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1286 "hw_init_query_ddr_cmd_fail"); 1287 TAVOR_TNF_EXIT(tavor_hw_init); 1288 return (DDI_FAILURE); 1289 } 1290 1291 /* Figure out how big the firmware image (in DDR) is */ 1292 status = tavor_cmn_query_cmd_post(state, QUERY_FW, 0, &state->ts_fw, 1293 sizeof (tavor_hw_queryfw_t), TAVOR_CMD_NOSLEEP_SPIN); 1294 if (status != TAVOR_CMD_SUCCESS) { 1295 cmn_err(CE_CONT, "Tavor: QUERY_FW command failed: %08x\n", 1296 status); 1297 tavor_hw_fini(state, cleanup); 1298 TNF_PROBE_0(tavor_hw_init_query_fw_cmd_fail, 1299 TAVOR_TNF_ERROR, ""); 1300 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1301 "hw_init_query_fw_cmd_fail"); 1302 TAVOR_TNF_EXIT(tavor_hw_init); 1303 return (DDI_FAILURE); 1304 } 1305 1306 if (tavor_fix_error_buf(state) != DDI_SUCCESS) { 1307 tavor_hw_fini(state, cleanup); 1308 TNF_PROBE_0(tavor_hw_init_fixerrorbuf_fail, 1309 TAVOR_TNF_ERROR, ""); 1310 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1311 "hw_init_fixerrorbuf_fail"); 1312 TAVOR_TNF_EXIT(tavor_hw_init); 1313 return (DDI_FAILURE); 1314 } 1315 1316 /* Validate that the FW version is appropriate */ 1317 status = tavor_fw_version_check(state); 1318 if (status != DDI_SUCCESS) { 1319 if (state->ts_operational_mode == TAVOR_HCA_MODE) { 1320 cmn_err(CE_CONT, "Unsupported Tavor FW version: " 1321 "expected: %04d.%04d.%04d, " 1322 "actual: %04d.%04d.%04d\n", 1323 TAVOR_FW_VER_MAJOR, 1324 TAVOR_FW_VER_MINOR, 1325 TAVOR_FW_VER_SUBMINOR, 1326 state->ts_fw.fw_rev_major, 1327 state->ts_fw.fw_rev_minor, 1328 state->ts_fw.fw_rev_subminor); 1329 } else if (state->ts_operational_mode == TAVOR_COMPAT_MODE) { 1330 cmn_err(CE_CONT, "Unsupported Tavor Compat FW version: " 1331 "expected: %04d.%04d.%04d, " 1332 "actual: %04d.%04d.%04d\n", 1333 TAVOR_COMPAT_FW_VER_MAJOR, 1334 TAVOR_COMPAT_FW_VER_MINOR, 1335 TAVOR_COMPAT_FW_VER_SUBMINOR, 1336 state->ts_fw.fw_rev_major, 1337 state->ts_fw.fw_rev_minor, 1338 state->ts_fw.fw_rev_subminor); 1339 } else { 1340 cmn_err(CE_CONT, "Unsupported FW version: " 1341 "%04d.%04d.%04d\n", 1342 state->ts_fw.fw_rev_major, 1343 state->ts_fw.fw_rev_minor, 1344 state->ts_fw.fw_rev_subminor); 1345 } 1346 1347 tavor_hw_fini(state, cleanup); 1348 TNF_PROBE_0(tavor_hw_init_checkfwver_fail, 1349 TAVOR_TNF_ERROR, ""); 1350 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1351 "hw_init_checkfwver_fail"); 1352 TAVOR_TNF_EXIT(tavor_hw_init); 1353 return (DDI_FAILURE); 1354 } 1355 1356 drv_usecwait(10); 1357 retries = 1000; /* retry up to 1 second before giving up */ 1358 retry: 1359 /* Call MOD_STAT_CFG to setup SRQ support (or disable) */ 1360 status = tavor_mod_stat_cfg_cmd_post(state); 1361 if (status != DDI_SUCCESS) { 1362 if (retries > 0) { 1363 drv_usecwait(1000); 1364 retries--; 1365 goto retry; 1366 } 1367 cmn_err(CE_CONT, "Tavor: MOD_STAT_CFG command failed: %08x\n", 1368 status); 1369 tavor_hw_fini(state, cleanup); 1370 TNF_PROBE_0(tavor_hw_init_mod_stat_cfg_cmd_fail, 1371 TAVOR_TNF_ERROR, ""); 1372 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1373 "hw_init_mod_stat_cfg_cmd_fail"); 1374 TAVOR_TNF_EXIT(tavor_hw_init); 1375 return (DDI_FAILURE); 1376 } 1377 1378 /* Figure out Tavor device limits */ 1379 status = tavor_cmn_query_cmd_post(state, QUERY_DEV_LIM, 0, 1380 &state->ts_devlim, sizeof (tavor_hw_querydevlim_t), 1381 TAVOR_CMD_NOSLEEP_SPIN); 1382 if (status != TAVOR_CMD_SUCCESS) { 1383 cmn_err(CE_CONT, "Tavor: QUERY_DEV_LIM command failed: %08x\n", 1384 status); 1385 tavor_hw_fini(state, cleanup); 1386 TNF_PROBE_0(tavor_hw_init_query_devlim_cmd_fail, 1387 TAVOR_TNF_ERROR, ""); 1388 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1389 "hw_init_query_devlim_cmd_fail"); 1390 TAVOR_TNF_EXIT(tavor_hw_init); 1391 return (DDI_FAILURE); 1392 } 1393 1394 /* Initialize the Phase2 Tavor configuration profile */ 1395 status = tavor_cfg_profile_init_phase2(state); 1396 if (status != DDI_SUCCESS) { 1397 tavor_hw_fini(state, cleanup); 1398 TNF_PROBE_0(tavor_hw_init_cfginit2_fail, TAVOR_TNF_ERROR, ""); 1399 TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_cfginit2_fail"); 1400 TAVOR_TNF_EXIT(tavor_hw_init); 1401 return (DDI_FAILURE); 1402 } 1403 1404 /* Second phase of init for Tavor configuration/resources */ 1405 status = tavor_rsrc_init_phase2(state); 1406 if (status != DDI_SUCCESS) { 1407 tavor_hw_fini(state, cleanup); 1408 TNF_PROBE_0(tavor_hw_init_rsrcinit2_fail, TAVOR_TNF_ERROR, ""); 1409 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1410 "hw_init_rsrcinit2_fail"); 1411 TAVOR_TNF_EXIT(tavor_hw_init); 1412 return (DDI_FAILURE); 1413 } 1414 cleanup = TAVOR_DRV_CLEANUP_LEVEL7; 1415 1416 /* Miscellaneous query information */ 1417 status = tavor_cmn_query_cmd_post(state, QUERY_ADAPTER, 0, 1418 &state->ts_adapter, sizeof (tavor_hw_queryadapter_t), 1419 TAVOR_CMD_NOSLEEP_SPIN); 1420 if (status != TAVOR_CMD_SUCCESS) { 1421 cmn_err(CE_CONT, "Tavor: QUERY_ADAPTER command failed: %08x\n", 1422 status); 1423 tavor_hw_fini(state, cleanup); 1424 TNF_PROBE_0(tavor_hw_init_query_adapter_cmd_fail, 1425 TAVOR_TNF_ERROR, ""); 1426 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1427 "hw_init_query_adapter_cmd_fail"); 1428 TAVOR_TNF_EXIT(tavor_hw_init); 1429 return (DDI_FAILURE); 1430 } 1431 1432 /* Prepare configuration for Tavor INIT_HCA command */ 1433 tavor_hca_config_setup(state, &state->ts_hcaparams); 1434 1435 /* Post command to init Tavor HCA */ 1436 status = tavor_init_hca_cmd_post(state, &state->ts_hcaparams, 1437 TAVOR_CMD_NOSLEEP_SPIN); 1438 if (status != TAVOR_CMD_SUCCESS) { 1439 cmn_err(CE_CONT, "Tavor: INIT_HCA command failed: %08x\n", 1440 status); 1441 tavor_hw_fini(state, cleanup); 1442 TNF_PROBE_0(tavor_hw_init_init_hca_cmd_fail, 1443 TAVOR_TNF_ERROR, ""); 1444 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1445 "hw_init_init_hca_cmd_fail"); 1446 TAVOR_TNF_EXIT(tavor_hw_init); 1447 return (DDI_FAILURE); 1448 } 1449 cleanup = TAVOR_DRV_CLEANUP_LEVEL8; 1450 1451 /* Allocate protection domain (PD) for Tavor internal use */ 1452 status = tavor_pd_alloc(state, &state->ts_pdhdl_internal, TAVOR_SLEEP); 1453 if (status != DDI_SUCCESS) { 1454 tavor_hw_fini(state, cleanup); 1455 TNF_PROBE_0(tavor_hw_init_internal_pd_alloc_fail, 1456 TAVOR_TNF_ERROR, ""); 1457 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1458 "hw_init_internal_pd_alloc_fail"); 1459 TAVOR_TNF_EXIT(tavor_hw_init); 1460 return (DDI_FAILURE); 1461 } 1462 cleanup = TAVOR_DRV_CLEANUP_LEVEL9; 1463 1464 /* Setup Tavor internal UAR pages (0 and 1) */ 1465 status = tavor_internal_uarpgs_init(state); 1466 if (status != DDI_SUCCESS) { 1467 tavor_hw_fini(state, cleanup); 1468 TNF_PROBE_0(tavor_hw_init_internal_uarpgs_alloc_fail, 1469 TAVOR_TNF_ERROR, ""); 1470 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1471 "hw_init_internal_uarpgs_alloc_fail"); 1472 TAVOR_TNF_EXIT(tavor_hw_init); 1473 return (DDI_FAILURE); 1474 } 1475 cleanup = TAVOR_DRV_CLEANUP_LEVEL10; 1476 1477 /* Query and initialize the Tavor interrupt/MSI information */ 1478 status = tavor_intr_or_msi_init(state); 1479 if (status != DDI_SUCCESS) { 1480 tavor_hw_fini(state, cleanup); 1481 TNF_PROBE_0(tavor_intr_or_msi_init_fail, 1482 TAVOR_TNF_ERROR, ""); 1483 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1484 "intr_or_msi_init_fail"); 1485 TAVOR_TNF_EXIT(tavor_hw_init); 1486 return (DDI_FAILURE); 1487 } 1488 cleanup = TAVOR_DRV_CLEANUP_LEVEL11; 1489 1490 /* Setup all of the Tavor EQs */ 1491 status = tavor_eq_init_all(state); 1492 if (status != DDI_SUCCESS) { 1493 tavor_hw_fini(state, cleanup); 1494 TNF_PROBE_0(tavor_hw_init_eqinitall_fail, TAVOR_TNF_ERROR, ""); 1495 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1496 "hw_init_eqinitall_fail"); 1497 TAVOR_TNF_EXIT(tavor_hw_init); 1498 return (DDI_FAILURE); 1499 } 1500 cleanup = TAVOR_DRV_CLEANUP_LEVEL12; 1501 1502 /* Set aside contexts for QP0 and QP1 */ 1503 status = tavor_special_qp_contexts_reserve(state); 1504 if (status != DDI_SUCCESS) { 1505 tavor_hw_fini(state, cleanup); 1506 TNF_PROBE_0(tavor_hw_init_reserve_special_qp_fail, 1507 TAVOR_TNF_ERROR, ""); 1508 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1509 "hw_init_reserve_special_qp_fail"); 1510 TAVOR_TNF_EXIT(tavor_hw_init); 1511 return (DDI_FAILURE); 1512 } 1513 cleanup = TAVOR_DRV_CLEANUP_LEVEL13; 1514 1515 /* Initialize for multicast group handling */ 1516 status = tavor_mcg_init(state); 1517 if (status != DDI_SUCCESS) { 1518 tavor_hw_fini(state, cleanup); 1519 TNF_PROBE_0(tavor_hw_init_mcg_init_fail, TAVOR_TNF_ERROR, ""); 1520 TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_mcg_init_fail"); 1521 TAVOR_TNF_EXIT(tavor_hw_init); 1522 return (DDI_FAILURE); 1523 } 1524 cleanup = TAVOR_DRV_CLEANUP_LEVEL14; 1525 1526 /* Initialize the Tavor IB port(s) */ 1527 status = tavor_hca_port_init(state); 1528 if (status != DDI_SUCCESS) { 1529 tavor_hw_fini(state, cleanup); 1530 TNF_PROBE_0(tavor_hw_init_hca_port_init_fail, 1531 TAVOR_TNF_ERROR, ""); 1532 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1533 "hw_init_hca_port_init_fail"); 1534 TAVOR_TNF_EXIT(tavor_hw_init); 1535 return (DDI_FAILURE); 1536 } 1537 cleanup = TAVOR_DRV_CLEANUP_ALL; 1538 1539 /* Determine NodeGUID and SystemImageGUID */ 1540 status = tavor_getnodeinfo_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN, 1541 &nodeinfo); 1542 if (status != TAVOR_CMD_SUCCESS) { 1543 cmn_err(CE_CONT, "Tavor: GetNodeInfo command failed: %08x\n", 1544 status); 1545 tavor_hw_fini(state, cleanup); 1546 TNF_PROBE_0(tavor_hw_init_getnodeinfo_cmd_fail, 1547 TAVOR_TNF_ERROR, ""); 1548 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1549 "hw_init_getnodeinfo_cmd_fail"); 1550 TAVOR_TNF_EXIT(tavor_hw_init); 1551 return (DDI_FAILURE); 1552 } 1553 1554 /* 1555 * If the NodeGUID value was set in OBP properties, then we use that 1556 * value. But we still print a message if the value we queried from 1557 * firmware does not match this value. 1558 * 1559 * Otherwise if OBP value is not set then we use the value from 1560 * firmware unconditionally. 1561 */ 1562 if (state->ts_cfg_profile->cp_nodeguid) { 1563 state->ts_nodeguid = state->ts_cfg_profile->cp_nodeguid; 1564 } else { 1565 state->ts_nodeguid = nodeinfo.NodeGUID; 1566 } 1567 1568 if (state->ts_nodeguid != nodeinfo.NodeGUID) { 1569 cmn_err(CE_NOTE, "!NodeGUID value queried from firmware " 1570 "does not match value set by device property"); 1571 } 1572 1573 /* 1574 * If the SystemImageGUID value was set in OBP properties, then we use 1575 * that value. But we still print a message if the value we queried 1576 * from firmware does not match this value. 1577 * 1578 * Otherwise if OBP value is not set then we use the value from 1579 * firmware unconditionally. 1580 */ 1581 if (state->ts_cfg_profile->cp_sysimgguid) { 1582 state->ts_sysimgguid = state->ts_cfg_profile->cp_sysimgguid; 1583 } else { 1584 state->ts_sysimgguid = nodeinfo.SystemImageGUID; 1585 } 1586 1587 if (state->ts_sysimgguid != nodeinfo.SystemImageGUID) { 1588 cmn_err(CE_NOTE, "!SystemImageGUID value queried from firmware " 1589 "does not match value set by device property"); 1590 } 1591 1592 /* Get NodeDescription */ 1593 status = tavor_getnodedesc_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN, 1594 (sm_nodedesc_t *)&state->ts_nodedesc); 1595 if (status != TAVOR_CMD_SUCCESS) { 1596 cmn_err(CE_CONT, "Tavor: GetNodeDesc command failed: %08x\n", 1597 status); 1598 tavor_hw_fini(state, cleanup); 1599 TNF_PROBE_0(tavor_hw_init_getnodedesc_cmd_fail, 1600 TAVOR_TNF_ERROR, ""); 1601 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1602 "hw_init_getnodedesc_cmd_fail"); 1603 TAVOR_TNF_EXIT(tavor_hw_init); 1604 return (DDI_FAILURE); 1605 } 1606 1607 TAVOR_TNF_EXIT(tavor_hw_init); 1608 return (DDI_SUCCESS); 1609 } 1610 1611 1612 /* 1613 * tavor_hw_fini() 1614 * Context: Only called from attach() and/or detach() path contexts 1615 */ 1616 static void 1617 tavor_hw_fini(tavor_state_t *state, tavor_drv_cleanup_level_t cleanup) 1618 { 1619 uint_t num_ports; 1620 int status; 1621 1622 TAVOR_TNF_ENTER(tavor_hw_fini); 1623 1624 switch (cleanup) { 1625 /* 1626 * If we add more driver initialization steps that should be cleaned 1627 * up here, we need to ensure that TAVOR_DRV_CLEANUP_ALL is still the 1628 * first entry (i.e. corresponds to the last init step). 1629 */ 1630 case TAVOR_DRV_CLEANUP_ALL: 1631 /* Shutdown the Tavor IB port(s) */ 1632 num_ports = state->ts_cfg_profile->cp_num_ports; 1633 (void) tavor_hca_ports_shutdown(state, num_ports); 1634 /* FALLTHROUGH */ 1635 1636 case TAVOR_DRV_CLEANUP_LEVEL14: 1637 /* Teardown resources used for multicast group handling */ 1638 tavor_mcg_fini(state); 1639 /* FALLTHROUGH */ 1640 1641 case TAVOR_DRV_CLEANUP_LEVEL13: 1642 /* Unreserve the special QP contexts */ 1643 tavor_special_qp_contexts_unreserve(state); 1644 /* FALLTHROUGH */ 1645 1646 case TAVOR_DRV_CLEANUP_LEVEL12: 1647 /* 1648 * Attempt to teardown all event queues (EQ). If we fail 1649 * here then print a warning message and return. Something 1650 * (either in HW or SW) has gone seriously wrong. 1651 */ 1652 status = tavor_eq_fini_all(state); 1653 if (status != DDI_SUCCESS) { 1654 TAVOR_WARNING(state, "failed to teardown EQs"); 1655 TNF_PROBE_0(tavor_hw_fini_eqfiniall_fail, 1656 TAVOR_TNF_ERROR, ""); 1657 TAVOR_TNF_EXIT(tavor_hw_fini); 1658 return; 1659 } 1660 /* FALLTHROUGH */ 1661 1662 case TAVOR_DRV_CLEANUP_LEVEL11: 1663 status = tavor_intr_or_msi_fini(state); 1664 if (status != DDI_SUCCESS) { 1665 TAVOR_WARNING(state, "failed to free intr/MSI"); 1666 TNF_PROBE_0(tavor_hw_fini_intrmsifini_fail, 1667 TAVOR_TNF_ERROR, ""); 1668 TAVOR_TNF_EXIT(tavor_hw_fini); 1669 return; 1670 } 1671 /* FALLTHROUGH */ 1672 1673 case TAVOR_DRV_CLEANUP_LEVEL10: 1674 /* Free the resources for the Tavor internal UAR pages */ 1675 tavor_internal_uarpgs_fini(state); 1676 /* FALLTHROUGH */ 1677 1678 case TAVOR_DRV_CLEANUP_LEVEL9: 1679 /* 1680 * Free the PD that was used internally by Tavor software. If 1681 * we fail here then print a warning and return. Something 1682 * (probably software-related, but perhaps HW) has gone wrong. 1683 */ 1684 status = tavor_pd_free(state, &state->ts_pdhdl_internal); 1685 if (status != DDI_SUCCESS) { 1686 TAVOR_WARNING(state, "failed to free internal PD"); 1687 TNF_PROBE_0(tavor_hw_fini_internal_pd_free_fail, 1688 TAVOR_TNF_ERROR, ""); 1689 TAVOR_TNF_EXIT(tavor_hw_fini); 1690 return; 1691 } 1692 /* FALLTHROUGH */ 1693 1694 case TAVOR_DRV_CLEANUP_LEVEL8: 1695 /* 1696 * Post the CLOSE_HCA command to Tavor firmware. If we fail 1697 * here then print a warning and return. Something (either in 1698 * HW or SW) has gone seriously wrong. 1699 */ 1700 status = tavor_close_hca_cmd_post(state, 1701 TAVOR_CMD_NOSLEEP_SPIN); 1702 if (status != TAVOR_CMD_SUCCESS) { 1703 TAVOR_WARNING(state, "failed to shutdown HCA"); 1704 TNF_PROBE_0(tavor_hw_fini_closehcacmd_fail, 1705 TAVOR_TNF_ERROR, ""); 1706 TAVOR_TNF_EXIT(tavor_hw_fini); 1707 return; 1708 } 1709 /* FALLTHROUGH */ 1710 1711 case TAVOR_DRV_CLEANUP_LEVEL7: 1712 /* Cleanup all the phase2 resources first */ 1713 tavor_rsrc_fini(state, TAVOR_RSRC_CLEANUP_ALL); 1714 /* FALLTHROUGH */ 1715 1716 case TAVOR_DRV_CLEANUP_LEVEL6: 1717 /* Then cleanup the phase1 resources */ 1718 tavor_rsrc_fini(state, TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE); 1719 /* FALLTHROUGH */ 1720 1721 case TAVOR_DRV_CLEANUP_LEVEL5: 1722 /* 1723 * Post the SYS_DIS command to Tavor firmware to shut 1724 * everything down again. If we fail here then print a 1725 * warning and return. Something (probably in HW, but maybe 1726 * in SW) has gone seriously wrong. 1727 */ 1728 status = tavor_sys_dis_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN); 1729 if (status != TAVOR_CMD_SUCCESS) { 1730 TAVOR_WARNING(state, "failed to shutdown hardware"); 1731 TNF_PROBE_0(tavor_hw_fini_sys_dis_fail, 1732 TAVOR_TNF_ERROR, ""); 1733 TAVOR_TNF_EXIT(tavor_hw_fini); 1734 return; 1735 } 1736 /* FALLTHROUGH */ 1737 1738 case TAVOR_DRV_CLEANUP_LEVEL4: 1739 /* Teardown any resources allocated for the config profile */ 1740 tavor_cfg_profile_fini(state); 1741 /* FALLTHROUGH */ 1742 1743 case TAVOR_DRV_CLEANUP_LEVEL3: 1744 ddi_regs_map_free(&state->ts_reg_ddrhdl); 1745 /* FALLTHROUGH */ 1746 1747 case TAVOR_DRV_CLEANUP_LEVEL2: 1748 ddi_regs_map_free(&state->ts_reg_uarhdl); 1749 /* FALLTHROUGH */ 1750 1751 case TAVOR_DRV_CLEANUP_LEVEL1: 1752 ddi_regs_map_free(&state->ts_reg_cmdhdl); 1753 /* FALLTHROUGH */ 1754 1755 case TAVOR_DRV_CLEANUP_LEVEL0: 1756 if (state->ts_pci_cfghdl) { 1757 ddi_regs_map_free(&state->ts_pci_cfghdl); 1758 state->ts_pci_cfghdl = NULL; 1759 } 1760 break; 1761 1762 default: 1763 TAVOR_WARNING(state, "unexpected driver cleanup level"); 1764 TNF_PROBE_0(tavor_hw_fini_default_fail, TAVOR_TNF_ERROR, ""); 1765 TAVOR_TNF_EXIT(tavor_hw_fini); 1766 return; 1767 } 1768 1769 TAVOR_TNF_EXIT(tavor_hw_fini); 1770 } 1771 1772 1773 /* 1774 * tavor_soft_state_init() 1775 * Context: Only called from attach() path context 1776 */ 1777 static int 1778 tavor_soft_state_init(tavor_state_t *state) 1779 { 1780 ibt_hca_attr_t *hca_attr; 1781 uint64_t maxval, val; 1782 ibt_hca_flags_t caps = IBT_HCA_NO_FLAGS; 1783 int status; 1784 1785 TAVOR_TNF_ENTER(tavor_soft_state_init); 1786 1787 /* 1788 * The ibc_hca_info_t struct is passed to the IBTF. This is the 1789 * routine where we initialize it. Many of the init values come from 1790 * either configuration variables or successful queries of the Tavor 1791 * hardware abilities 1792 */ 1793 state->ts_ibtfinfo.hca_ci_vers = IBCI_V3; 1794 state->ts_ibtfinfo.hca_dip = state->ts_dip; 1795 state->ts_ibtfinfo.hca_handle = (ibc_hca_hdl_t)state; 1796 state->ts_ibtfinfo.hca_ops = &tavor_ibc_ops; 1797 1798 hca_attr = kmem_zalloc(sizeof (ibt_hca_attr_t), KM_SLEEP); 1799 state->ts_ibtfinfo.hca_attr = hca_attr; 1800 1801 hca_attr->hca_fw_major_version = state->ts_fw.fw_rev_major; 1802 hca_attr->hca_fw_minor_version = state->ts_fw.fw_rev_minor; 1803 hca_attr->hca_fw_micro_version = state->ts_fw.fw_rev_subminor; 1804 1805 /* 1806 * Determine HCA capabilities: 1807 * No default support for IBT_HCA_RD, IBT_HCA_RAW_MULTICAST, 1808 * IBT_HCA_ATOMICS_GLOBAL, IBT_HCA_RESIZE_CHAN, IBT_HCA_INIT_TYPE, 1809 * or IBT_HCA_SHUTDOWN_PORT 1810 * But IBT_HCA_AH_PORT_CHECK, IBT_HCA_SQD_RTS_PORT, IBT_HCA_SI_GUID, 1811 * IBT_HCA_RNR_NAK, and IBT_HCA_CURRENT_QP_STATE are always 1812 * supported 1813 * All other features are conditionally supported, depending on the 1814 * status return by the Tavor HCA (in QUERY_DEV_LIM) 1815 */ 1816 if (state->ts_devlim.ud_multi) { 1817 caps |= IBT_HCA_UD_MULTICAST; 1818 } 1819 if (state->ts_devlim.atomic) { 1820 caps |= IBT_HCA_ATOMICS_HCA; 1821 } 1822 if (state->ts_devlim.apm) { 1823 caps |= IBT_HCA_AUTO_PATH_MIG; 1824 } 1825 if (state->ts_devlim.pkey_v) { 1826 caps |= IBT_HCA_PKEY_CNTR; 1827 } 1828 if (state->ts_devlim.qkey_v) { 1829 caps |= IBT_HCA_QKEY_CNTR; 1830 } 1831 if (state->ts_cfg_profile->cp_srq_enable) { 1832 caps |= IBT_HCA_SRQ | IBT_HCA_RESIZE_SRQ; 1833 } 1834 if (state->ts_cfg_profile->cp_fmr_enable) { 1835 caps |= IBT_HCA_FMR; 1836 } 1837 caps |= (IBT_HCA_AH_PORT_CHECK | IBT_HCA_SQD_SQD_PORT | 1838 IBT_HCA_SI_GUID | IBT_HCA_RNR_NAK | IBT_HCA_CURRENT_QP_STATE | 1839 IBT_HCA_PORT_UP | IBT_HCA_SQD_STATE); 1840 hca_attr->hca_flags = caps; 1841 1842 /* Determine VendorID, DeviceID, and revision ID */ 1843 hca_attr->hca_vendor_id = state->ts_adapter.vendor_id; 1844 hca_attr->hca_device_id = state->ts_adapter.device_id; 1845 hca_attr->hca_version_id = state->ts_adapter.rev_id; 1846 1847 /* 1848 * Determine number of available QPs and max QP size. Number of 1849 * available QPs is determined by subtracting the number of 1850 * "reserved QPs" (i.e. reserved for firmware use) from the 1851 * total number configured. 1852 */ 1853 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_qp); 1854 hca_attr->hca_max_qp = val - ((uint64_t)1 << 1855 state->ts_devlim.log_rsvd_qp); 1856 maxval = ((uint64_t)1 << state->ts_devlim.log_max_qp_sz); 1857 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_qp_sz); 1858 if (val > maxval) { 1859 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 1860 TNF_PROBE_2(tavor_soft_state_init_maxqpsz_toobig_fail, 1861 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max QP size " 1862 "exceeds device maximum", tnf_uint, maxsz, maxval); 1863 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1864 "soft_state_init_maxqpsz_toobig_fail"); 1865 TAVOR_TNF_EXIT(tavor_soft_state_init); 1866 return (DDI_FAILURE); 1867 } 1868 hca_attr->hca_max_qp_sz = val; 1869 1870 /* Determine max scatter-gather size in WQEs */ 1871 maxval = state->ts_devlim.max_sg; 1872 val = state->ts_cfg_profile->cp_wqe_max_sgl; 1873 if (val > maxval) { 1874 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 1875 TNF_PROBE_2(tavor_soft_state_init_toomanysgl_fail, 1876 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of sgl " 1877 "exceeds device maximum", tnf_uint, maxsgl, maxval); 1878 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1879 "soft_state_init_toomanysgl_fail"); 1880 TAVOR_TNF_EXIT(tavor_soft_state_init); 1881 return (DDI_FAILURE); 1882 } 1883 /* If the rounded value for max SGL is too large, cap it */ 1884 if (state->ts_cfg_profile->cp_wqe_real_max_sgl > maxval) { 1885 state->ts_cfg_profile->cp_wqe_real_max_sgl = maxval; 1886 val = maxval; 1887 } else { 1888 val = state->ts_cfg_profile->cp_wqe_real_max_sgl; 1889 } 1890 1891 hca_attr->hca_max_sgl = val; 1892 hca_attr->hca_max_rd_sgl = 0; /* zero because RD is unsupported */ 1893 1894 /* 1895 * Determine number of available CQs and max CQ size. Number of 1896 * available CQs is determined by subtracting the number of 1897 * "reserved CQs" (i.e. reserved for firmware use) from the 1898 * total number configured. 1899 */ 1900 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_cq); 1901 hca_attr->hca_max_cq = val - ((uint64_t)1 << 1902 state->ts_devlim.log_rsvd_cq); 1903 maxval = ((uint64_t)1 << state->ts_devlim.log_max_cq_sz); 1904 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_cq_sz) - 1; 1905 if (val > maxval) { 1906 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 1907 TNF_PROBE_2(tavor_soft_state_init_maxcqsz_toobig_fail, 1908 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max CQ size " 1909 "exceeds device maximum", tnf_uint, maxsz, maxval); 1910 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1911 "soft_state_init_maxcqsz_toobig_fail"); 1912 TAVOR_TNF_EXIT(tavor_soft_state_init); 1913 return (DDI_FAILURE); 1914 } 1915 hca_attr->hca_max_cq_sz = val; 1916 1917 /* 1918 * Determine number of available SRQs and max SRQ size. Number of 1919 * available SRQs is determined by subtracting the number of 1920 * "reserved SRQs" (i.e. reserved for firmware use) from the 1921 * total number configured. 1922 */ 1923 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_srq); 1924 hca_attr->hca_max_srqs = val - ((uint64_t)1 << 1925 state->ts_devlim.log_rsvd_srq); 1926 maxval = ((uint64_t)1 << state->ts_devlim.log_max_srq_sz); 1927 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_srq_sz); 1928 1929 if (val > maxval) { 1930 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 1931 TNF_PROBE_2(tavor_soft_state_init_maxsrqsz_toobig_fail, 1932 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max SRQ size " 1933 "exceeds device maximum", tnf_uint, maxsz, maxval); 1934 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1935 "soft_state_init_maxsrqsz_toobig_fail"); 1936 TAVOR_TNF_EXIT(tavor_soft_state_init); 1937 return (DDI_FAILURE); 1938 } 1939 hca_attr->hca_max_srqs_sz = val; 1940 1941 val = state->ts_cfg_profile->cp_srq_max_sgl; 1942 maxval = state->ts_devlim.max_sg; 1943 if (val > maxval) { 1944 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 1945 TNF_PROBE_2(tavor_soft_state_init_toomanysrqsgl_fail, 1946 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of srq " 1947 "sgl exceeds device maximum", tnf_uint, maxsgl, maxval); 1948 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1949 "soft_state_init_toomanysrqsgl_fail"); 1950 TAVOR_TNF_EXIT(tavor_soft_state_init); 1951 return (DDI_FAILURE); 1952 } 1953 hca_attr->hca_max_srq_sgl = val; 1954 1955 /* 1956 * Determine supported HCA page sizes 1957 * XXX 1958 * For now we simply return the system pagesize as the only supported 1959 * pagesize 1960 */ 1961 hca_attr->hca_page_sz = ((PAGESIZE == (1 << 13)) ? IBT_PAGE_8K : 1962 IBT_PAGE_4K); 1963 1964 /* 1965 * Determine number of available MemReg, MemWin, and their max size. 1966 * Number of available MRs and MWs is determined by subtracting 1967 * the number of "reserved MPTs" (i.e. reserved for firmware use) 1968 * from the total number configured for each. 1969 */ 1970 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_mpt); 1971 hca_attr->hca_max_memr = val - ((uint64_t)1 << 1972 state->ts_devlim.log_rsvd_mpt); 1973 hca_attr->hca_max_mem_win = val - ((uint64_t)1 << 1974 state->ts_devlim.log_rsvd_mpt); 1975 maxval = state->ts_devlim.log_max_mrw_sz; 1976 val = state->ts_cfg_profile->cp_log_max_mrw_sz; 1977 if (val > maxval) { 1978 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 1979 TNF_PROBE_2(tavor_soft_state_init_maxmrwsz_toobig_fail, 1980 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max mrw size " 1981 "exceeds device maximum", tnf_uint, maxsz, maxval); 1982 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1983 "soft_state_init_maxmrwsz_toobig_fail"); 1984 TAVOR_TNF_EXIT(tavor_soft_state_init); 1985 return (DDI_FAILURE); 1986 } 1987 hca_attr->hca_max_memr_len = ((uint64_t)1 << val); 1988 1989 /* Determine RDMA/Atomic properties */ 1990 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_rdb); 1991 hca_attr->hca_max_rsc = val; 1992 val = state->ts_cfg_profile->cp_hca_max_rdma_in_qp; 1993 hca_attr->hca_max_rdma_in_qp = val; 1994 val = state->ts_cfg_profile->cp_hca_max_rdma_out_qp; 1995 hca_attr->hca_max_rdma_out_qp = val; 1996 hca_attr->hca_max_rdma_in_ee = 0; 1997 hca_attr->hca_max_rdma_out_ee = 0; 1998 1999 /* 2000 * Determine maximum number of raw IPv6 and Ether QPs. Set to 0 2001 * because neither type of raw QP is supported 2002 */ 2003 hca_attr->hca_max_ipv6_qp = 0; 2004 hca_attr->hca_max_ether_qp = 0; 2005 2006 /* Determine max number of MCGs and max QP-per-MCG */ 2007 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_qp); 2008 hca_attr->hca_max_mcg_qps = val; 2009 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_mcg); 2010 hca_attr->hca_max_mcg = val; 2011 val = state->ts_cfg_profile->cp_num_qp_per_mcg; 2012 hca_attr->hca_max_qp_per_mcg = val; 2013 2014 /* Determine max number partitions (i.e. PKeys) */ 2015 maxval = ((uint64_t)1 << state->ts_devlim.log_max_pkey); 2016 val = ((uint64_t)state->ts_cfg_profile->cp_num_ports << 2017 state->ts_cfg_profile->cp_log_max_pkeytbl); 2018 2019 if (val > maxval) { 2020 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 2021 TNF_PROBE_2(tavor_soft_state_init_toomanypkey_fail, 2022 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of PKeys " 2023 "exceeds device maximum", tnf_uint, maxpkey, maxval); 2024 TAVOR_ATTACH_MSG(state->ts_attach_buf, 2025 "soft_state_init_toomanypkey_fail"); 2026 TAVOR_TNF_EXIT(tavor_soft_state_init); 2027 return (DDI_FAILURE); 2028 } 2029 hca_attr->hca_max_partitions = val; 2030 2031 /* Determine number of ports */ 2032 maxval = state->ts_devlim.num_ports; 2033 val = state->ts_cfg_profile->cp_num_ports; 2034 if ((val > maxval) || (val == 0)) { 2035 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 2036 TNF_PROBE_2(tavor_soft_state_init_toomanyports_fail, 2037 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of ports " 2038 "exceeds device maximum", tnf_uint, maxports, maxval); 2039 TAVOR_ATTACH_MSG(state->ts_attach_buf, 2040 "soft_state_init_toomanyports_fail"); 2041 TAVOR_TNF_EXIT(tavor_soft_state_init); 2042 return (DDI_FAILURE); 2043 } 2044 hca_attr->hca_nports = val; 2045 2046 /* Copy NodeGUID and SystemImageGUID from softstate */ 2047 hca_attr->hca_node_guid = state->ts_nodeguid; 2048 hca_attr->hca_si_guid = state->ts_sysimgguid; 2049 2050 /* 2051 * Determine local ACK delay. Use the value suggested by the Tavor 2052 * hardware (from the QUERY_DEV_LIM command) 2053 */ 2054 hca_attr->hca_local_ack_delay = state->ts_devlim.ca_ack_delay; 2055 2056 /* Determine max SGID table and PKey table sizes */ 2057 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_gidtbl); 2058 hca_attr->hca_max_port_sgid_tbl_sz = val; 2059 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_pkeytbl); 2060 hca_attr->hca_max_port_pkey_tbl_sz = val; 2061 2062 /* Determine max number of PDs */ 2063 maxval = ((uint64_t)1 << state->ts_devlim.log_max_pd); 2064 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_pd); 2065 if (val > maxval) { 2066 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 2067 TNF_PROBE_2(tavor_soft_state_init_toomanypd_fail, 2068 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of PD " 2069 "exceeds device maximum", tnf_uint, maxpd, maxval); 2070 TAVOR_ATTACH_MSG(state->ts_attach_buf, 2071 "soft_state_init_toomanypd_fail"); 2072 TAVOR_TNF_EXIT(tavor_soft_state_init); 2073 return (DDI_FAILURE); 2074 } 2075 hca_attr->hca_max_pd = val; 2076 2077 /* Determine max number of Address Handles */ 2078 maxval = ((uint64_t)1 << state->ts_devlim.log_max_av); 2079 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_ah); 2080 if (val > maxval) { 2081 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 2082 TNF_PROBE_2(tavor_soft_state_init_toomanyah_fail, 2083 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of AH " 2084 "exceeds device maximum", tnf_uint, maxah, maxval); 2085 TAVOR_ATTACH_MSG(state->ts_attach_buf, 2086 "soft_state_init_toomanyah_fail"); 2087 TAVOR_TNF_EXIT(tavor_soft_state_init); 2088 return (DDI_FAILURE); 2089 } 2090 hca_attr->hca_max_ah = val; 2091 2092 /* No RDDs or EECs (since Reliable Datagram is not supported) */ 2093 hca_attr->hca_max_rdd = 0; 2094 hca_attr->hca_max_eec = 0; 2095 2096 /* Initialize lock for reserved UAR page access */ 2097 mutex_init(&state->ts_uar_lock, NULL, MUTEX_DRIVER, 2098 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2099 2100 /* Initialize the flash fields */ 2101 state->ts_fw_flashstarted = 0; 2102 mutex_init(&state->ts_fw_flashlock, NULL, MUTEX_DRIVER, 2103 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2104 2105 /* Initialize the lock for the info ioctl */ 2106 mutex_init(&state->ts_info_lock, NULL, MUTEX_DRIVER, 2107 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2108 2109 /* Initialize the AVL tree for QP number support */ 2110 tavor_qpn_avl_init(state); 2111 2112 /* Initialize the kstat info structure */ 2113 status = tavor_kstat_init(state); 2114 if (status != DDI_SUCCESS) { 2115 tavor_qpn_avl_fini(state); 2116 mutex_destroy(&state->ts_info_lock); 2117 mutex_destroy(&state->ts_fw_flashlock); 2118 mutex_destroy(&state->ts_uar_lock); 2119 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 2120 TNF_PROBE_0(tavor_soft_state_init_kstatinit_fail, 2121 TAVOR_TNF_ERROR, ""); 2122 TAVOR_ATTACH_MSG(state->ts_attach_buf, 2123 "soft_state_init_kstatinit_fail"); 2124 TAVOR_TNF_EXIT(tavor_soft_state_init); 2125 return (DDI_FAILURE); 2126 } 2127 2128 TAVOR_TNF_EXIT(tavor_soft_state_init); 2129 return (DDI_SUCCESS); 2130 } 2131 2132 2133 /* 2134 * tavor_soft_state_fini() 2135 * Context: Called only from detach() path context 2136 */ 2137 static void 2138 tavor_soft_state_fini(tavor_state_t *state) 2139 { 2140 TAVOR_TNF_ENTER(tavor_soft_state_fini); 2141 2142 /* Teardown the kstat info */ 2143 tavor_kstat_fini(state); 2144 2145 /* Teardown the AVL tree for QP number support */ 2146 tavor_qpn_avl_fini(state); 2147 2148 /* Free up info ioctl mutex */ 2149 mutex_destroy(&state->ts_info_lock); 2150 2151 /* Free up flash mutex */ 2152 mutex_destroy(&state->ts_fw_flashlock); 2153 2154 /* Free up the UAR page access mutex */ 2155 mutex_destroy(&state->ts_uar_lock); 2156 2157 /* Free up the hca_attr struct */ 2158 kmem_free(state->ts_ibtfinfo.hca_attr, sizeof (ibt_hca_attr_t)); 2159 2160 TAVOR_TNF_EXIT(tavor_soft_state_fini); 2161 } 2162 2163 2164 /* 2165 * tavor_hca_config_setup() 2166 * Context: Only called from attach() path context 2167 */ 2168 static void 2169 tavor_hca_config_setup(tavor_state_t *state, 2170 tavor_hw_initqueryhca_t *inithca) 2171 { 2172 tavor_rsrc_pool_info_t *rsrc_pool; 2173 uint64_t ddr_baseaddr, ddr_base_map_addr; 2174 uint64_t offset, addr; 2175 uint_t mcg_size; 2176 2177 TAVOR_TNF_ENTER(tavor_hca_config_setup); 2178 2179 /* Set "host endianness". Default is big endian */ 2180 #ifdef _LITTLE_ENDIAN 2181 inithca->big_endian = 0; 2182 #else 2183 inithca->big_endian = 1; 2184 #endif 2185 /* No Address Vector Protection, but Port Checking on by default */ 2186 inithca->udav_chk = TAVOR_UDAV_PROTECT_DISABLED; 2187 inithca->udav_port_chk = TAVOR_UDAV_PORTCHK_ENABLED; 2188 2189 ddr_baseaddr = (uint64_t)(uintptr_t)state->ts_reg_ddr_baseaddr; 2190 ddr_base_map_addr = (uint64_t)state->ts_ddr.ddr_baseaddr; 2191 2192 /* Setup QPC table */ 2193 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPC]; 2194 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2195 addr = ddr_base_map_addr + offset; 2196 inithca->context.qpc_baseaddr_h = (addr >> 32); 2197 inithca->context.qpc_baseaddr_l = (addr & 0xFFFFFFFF) >> 7; 2198 inithca->context.log_num_qp = state->ts_cfg_profile->cp_log_num_qp; 2199 2200 /* Setup EEC table (initialize to zero - RD unsupported) */ 2201 inithca->context.eec_baseaddr_h = 0; 2202 inithca->context.eec_baseaddr_l = 0; 2203 inithca->context.log_num_ee = 0; 2204 2205 /* Setup CQC table */ 2206 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQC]; 2207 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2208 addr = ddr_base_map_addr + offset; 2209 inithca->context.cqc_baseaddr_h = (addr >> 32); 2210 inithca->context.cqc_baseaddr_l = (addr & 0xFFFFFFFF) >> 6; 2211 inithca->context.log_num_cq = state->ts_cfg_profile->cp_log_num_cq; 2212 2213 /* Setup SRQC table */ 2214 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQC]; 2215 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2216 addr = ddr_base_map_addr + offset; 2217 inithca->context.srqc_baseaddr_h = (addr >> 32); 2218 inithca->context.srqc_baseaddr_l = (addr & 0xFFFFFFFF) >> 6; 2219 inithca->context.log_num_srq = 2220 state->ts_cfg_profile->cp_log_num_srq; 2221 2222 /* Setup EQPC table */ 2223 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQPC]; 2224 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2225 addr = ddr_base_map_addr + offset; 2226 inithca->context.eqpc_baseaddr = addr; 2227 2228 /* Setup EEEC table (initialize to zero - RD unsupported) */ 2229 inithca->context.eeec_baseaddr = 0; 2230 2231 /* Setup EQC table */ 2232 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQC]; 2233 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2234 addr = ddr_base_map_addr + offset; 2235 inithca->context.eqc_baseaddr_h = (addr >> 32); 2236 inithca->context.eqc_baseaddr_l = (addr & 0xFFFFFFFF) >> 6; 2237 inithca->context.log_num_eq = TAVOR_NUM_EQ_SHIFT; 2238 2239 /* Setup RDB table */ 2240 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_RDB]; 2241 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2242 addr = ddr_base_map_addr + offset; 2243 inithca->context.rdb_baseaddr_h = (addr >> 32); 2244 inithca->context.rdb_baseaddr_l = 0; 2245 2246 /* Setup Multicast */ 2247 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MCG]; 2248 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2249 addr = ddr_base_map_addr + offset; 2250 inithca->multi.mc_baseaddr = addr; 2251 mcg_size = TAVOR_MCGMEM_SZ(state); 2252 inithca->multi.log_mc_tbl_ent = highbit(mcg_size) - 1; 2253 inithca->multi.mc_tbl_hash_sz = 2254 (1 << state->ts_cfg_profile->cp_log_num_mcg_hash); 2255 inithca->multi.mc_hash_fn = TAVOR_MCG_DEFAULT_HASH_FN; 2256 inithca->multi.log_mc_tbl_sz = state->ts_cfg_profile->cp_log_num_mcg; 2257 2258 2259 /* Setup TPT */ 2260 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MPT]; 2261 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2262 addr = ddr_base_map_addr + offset; 2263 inithca->tpt.mpt_baseaddr = addr; 2264 inithca->tpt.mttseg_sz = TAVOR_MTTSEG_SIZE_SHIFT; 2265 inithca->tpt.log_mpt_sz = state->ts_cfg_profile->cp_log_num_mpt; 2266 inithca->tpt.mtt_version = TAVOR_MTT_PG_WALK_VER; 2267 2268 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MTT]; 2269 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2270 addr = ddr_base_map_addr + offset; 2271 inithca->tpt.mtt_baseaddr = addr; 2272 2273 /* Setup UAR */ 2274 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR]; 2275 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2276 addr = ddr_base_map_addr + offset; 2277 inithca->uar.uarscr_baseaddr = addr; 2278 2279 inithca->uar.uar_pg_sz = PAGESHIFT - 0xC; 2280 2281 TAVOR_TNF_EXIT(tavor_hca_config_setup); 2282 } 2283 2284 2285 /* 2286 * tavor_hca_port_init() 2287 * Context: Only called from attach() path context 2288 */ 2289 static int 2290 tavor_hca_port_init(tavor_state_t *state) 2291 { 2292 tavor_hw_initib_t *portinits, *initib; 2293 tavor_cfg_profile_t *cfgprof; 2294 uint_t num_ports; 2295 int i, status; 2296 uint64_t maxval, val; 2297 uint64_t sysimgguid, nodeguid, portguid; 2298 2299 TAVOR_TNF_ENTER(tavor_hca_port_init); 2300 2301 cfgprof = state->ts_cfg_profile; 2302 2303 /* Get number of HCA ports */ 2304 num_ports = cfgprof->cp_num_ports; 2305 2306 /* Allocate space for Tavor port init struct(s) */ 2307 portinits = (tavor_hw_initib_t *)kmem_zalloc(num_ports * 2308 sizeof (tavor_hw_initib_t), KM_SLEEP); 2309 2310 /* Post command to initialize Tavor HCA port */ 2311 for (i = 0; i < num_ports; i++) { 2312 initib = &portinits[i]; 2313 2314 /* 2315 * Determine whether we need to override the firmware's 2316 * default SystemImageGUID setting. 2317 */ 2318 sysimgguid = cfgprof->cp_sysimgguid; 2319 if (sysimgguid != 0) { 2320 initib->set_sysimg_guid = 1; 2321 initib->sysimg_guid = sysimgguid; 2322 } 2323 2324 /* 2325 * Determine whether we need to override the firmware's 2326 * default NodeGUID setting. 2327 */ 2328 nodeguid = cfgprof->cp_nodeguid; 2329 if (nodeguid != 0) { 2330 initib->set_node_guid = 1; 2331 initib->node_guid = nodeguid; 2332 } 2333 2334 /* 2335 * Determine whether we need to override the firmware's 2336 * default PortGUID setting. 2337 */ 2338 portguid = cfgprof->cp_portguid[i]; 2339 if (portguid != 0) { 2340 initib->set_port_guid0 = 1; 2341 initib->guid0 = portguid; 2342 } 2343 2344 /* Validate max MTU size */ 2345 maxval = state->ts_devlim.max_mtu; 2346 val = cfgprof->cp_max_mtu; 2347 if (val > maxval) { 2348 TNF_PROBE_2(tavor_hca_port_init_maxmtu_fail, 2349 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max " 2350 "MTU size exceeds device maximum", tnf_uint, 2351 maxmtu, maxval); 2352 TAVOR_TNF_EXIT(tavor_hca_port_init); 2353 goto init_ports_fail; 2354 } 2355 initib->mtu_cap = val; 2356 2357 /* Validate the max port width */ 2358 maxval = state->ts_devlim.max_port_width; 2359 val = cfgprof->cp_max_port_width; 2360 if (val > maxval) { 2361 TNF_PROBE_2(tavor_hca_port_init_maxportwidth_fail, 2362 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max " 2363 "port width exceeds device maximum", tnf_uint, 2364 maxportwidth, maxval); 2365 TAVOR_TNF_EXIT(tavor_hca_port_init); 2366 goto init_ports_fail; 2367 } 2368 initib->port_width_cap = val; 2369 2370 /* Validate max VL cap size */ 2371 maxval = state->ts_devlim.max_vl; 2372 val = cfgprof->cp_max_vlcap; 2373 if (val > maxval) { 2374 TNF_PROBE_2(tavor_hca_port_init_maxvlcap_fail, 2375 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max " 2376 "VLcap size exceeds device maximum", tnf_uint, 2377 maxvlcap, maxval); 2378 TAVOR_TNF_EXIT(tavor_hca_port_init); 2379 goto init_ports_fail; 2380 } 2381 initib->vl_cap = val; 2382 2383 /* Validate max GID table size */ 2384 maxval = ((uint64_t)1 << state->ts_devlim.log_max_gid); 2385 val = ((uint64_t)1 << cfgprof->cp_log_max_gidtbl); 2386 if (val > maxval) { 2387 TNF_PROBE_2(tavor_hca_port_init_gidtable_fail, 2388 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max " 2389 "GID table size exceeds device maximum", tnf_uint, 2390 maxgidtbl, maxval); 2391 TAVOR_TNF_EXIT(tavor_hca_port_init); 2392 goto init_ports_fail; 2393 } 2394 initib->max_gid = val; 2395 2396 /* Validate max PKey table size */ 2397 maxval = ((uint64_t)1 << state->ts_devlim.log_max_pkey); 2398 val = ((uint64_t)1 << cfgprof->cp_log_max_pkeytbl); 2399 if (val > maxval) { 2400 TNF_PROBE_2(tavor_hca_port_init_pkeytable_fail, 2401 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max " 2402 "PKey table size exceeds device maximum", tnf_uint, 2403 maxpkeytbl, maxval); 2404 TAVOR_TNF_EXIT(tavor_hca_port_init); 2405 goto init_ports_fail; 2406 } 2407 initib->max_pkey = val; 2408 2409 /* 2410 * Post the INIT_IB command to Tavor firmware. When this 2411 * command completes, the corresponding Tavor port will be 2412 * physically "Up" and initialized. 2413 */ 2414 status = tavor_init_ib_cmd_post(state, initib, i + 1, 2415 TAVOR_CMD_NOSLEEP_SPIN); 2416 if (status != TAVOR_CMD_SUCCESS) { 2417 cmn_err(CE_CONT, "Tavor: INIT_IB (port %02d) command " 2418 "failed: %08x\n", i + 1, status); 2419 TNF_PROBE_2(tavor_hca_port_init_init_ib_cmd_fail, 2420 TAVOR_TNF_ERROR, "", tnf_uint, cmd_status, status, 2421 tnf_uint, port, i + 1); 2422 TAVOR_TNF_EXIT(tavor_hca_port_init); 2423 goto init_ports_fail; 2424 } 2425 } 2426 2427 /* Free up the memory for Tavor port init struct(s), return success */ 2428 kmem_free(portinits, num_ports * sizeof (tavor_hw_initib_t)); 2429 TAVOR_TNF_EXIT(tavor_hca_port_init); 2430 return (DDI_SUCCESS); 2431 2432 init_ports_fail: 2433 /* 2434 * Free up the memory for Tavor port init struct(s), shutdown any 2435 * successfully initialized ports, and return failure 2436 */ 2437 kmem_free(portinits, num_ports * sizeof (tavor_hw_initib_t)); 2438 (void) tavor_hca_ports_shutdown(state, i); 2439 2440 TAVOR_TNF_EXIT(tavor_hca_port_init); 2441 return (DDI_FAILURE); 2442 } 2443 2444 2445 /* 2446 * tavor_hca_ports_shutdown() 2447 * Context: Only called from attach() and/or detach() path contexts 2448 */ 2449 static int 2450 tavor_hca_ports_shutdown(tavor_state_t *state, uint_t num_init) 2451 { 2452 int i, status; 2453 2454 TAVOR_TNF_ENTER(tavor_hca_ports_shutdown); 2455 2456 /* 2457 * Post commands to shutdown all init'd Tavor HCA ports. Note: if 2458 * any of these commands fail for any reason, it would be entirely 2459 * unexpected and probably indicative a serious problem (HW or SW). 2460 * Although we do return void from this function, this type of failure 2461 * should not go unreported. That is why we have the warning message 2462 * and the detailed TNF information. 2463 */ 2464 for (i = 0; i < num_init; i++) { 2465 status = tavor_close_ib_cmd_post(state, i + 1, 2466 TAVOR_CMD_NOSLEEP_SPIN); 2467 if (status != TAVOR_CMD_SUCCESS) { 2468 TAVOR_WARNING(state, "failed to shutdown HCA port"); 2469 TNF_PROBE_2(tavor_hca_ports_shutdown_close_ib_cmd_fail, 2470 TAVOR_TNF_ERROR, "", tnf_uint, cmd_status, status, 2471 tnf_uint, port, i + 1); 2472 TAVOR_TNF_EXIT(tavor_hca_ports_shutdown); 2473 return (status); 2474 } 2475 } 2476 2477 TAVOR_TNF_EXIT(tavor_hca_ports_shutdown); 2478 2479 return (TAVOR_CMD_SUCCESS); 2480 } 2481 2482 2483 /* 2484 * tavor_internal_uarpgs_init 2485 * Context: Only called from attach() path context 2486 */ 2487 static int 2488 tavor_internal_uarpgs_init(tavor_state_t *state) 2489 { 2490 int status; 2491 2492 TAVOR_TNF_ENTER(tavor_internal_uarpgs_init); 2493 2494 /* 2495 * Save away reserved Tavor UAR page #0. This UAR page is not to 2496 * be used by software. 2497 */ 2498 status = tavor_rsrc_alloc(state, TAVOR_UARPG, 1, TAVOR_SLEEP, 2499 &state->ts_uarpg0_rsrc_rsrvd); 2500 if (status != DDI_SUCCESS) { 2501 TNF_PROBE_0(tavor_uarpg0_rsrcalloc_fail, TAVOR_TNF_ERROR, ""); 2502 TAVOR_TNF_EXIT(tavor_internal_uarpgs_init); 2503 return (DDI_FAILURE); 2504 } 2505 2506 /* 2507 * Save away Tavor UAR page #1 (for internal use). This UAR page is 2508 * the privileged UAR page through which all kernel generated 2509 * doorbells will be rung. 2510 */ 2511 status = tavor_rsrc_alloc(state, TAVOR_UARPG, 1, TAVOR_SLEEP, 2512 &state->ts_uarpg1_rsrc); 2513 if (status != DDI_SUCCESS) { 2514 tavor_rsrc_free(state, &state->ts_uarpg0_rsrc_rsrvd); 2515 TNF_PROBE_0(tavor_uarpg1_rsrcalloc_fail, TAVOR_TNF_ERROR, ""); 2516 TAVOR_TNF_EXIT(tavor_internal_uarpgs_init); 2517 return (DDI_FAILURE); 2518 } 2519 2520 /* Setup pointer to UAR page #1 doorbells */ 2521 state->ts_uar = (tavor_hw_uar_t *)state->ts_uarpg1_rsrc->tr_addr; 2522 2523 TAVOR_TNF_EXIT(tavor_internal_uarpgs_init); 2524 return (DDI_SUCCESS); 2525 } 2526 2527 2528 /* 2529 * tavor_internal_uarpgs_fini 2530 * Context: Only called from attach() and/or detach() path contexts 2531 */ 2532 static void 2533 tavor_internal_uarpgs_fini(tavor_state_t *state) 2534 { 2535 TAVOR_TNF_ENTER(tavor_internal_uarpgs_fini); 2536 2537 /* Free up Tavor UAR page #1 (kernel driver doorbells) */ 2538 tavor_rsrc_free(state, &state->ts_uarpg1_rsrc); 2539 2540 /* Free up Tavor UAR page #0 (reserved) */ 2541 tavor_rsrc_free(state, &state->ts_uarpg0_rsrc_rsrvd); 2542 2543 TAVOR_TNF_EXIT(tavor_internal_uarpgs_fini); 2544 } 2545 2546 2547 /* 2548 * tavor_special_qp_contexts_reserve() 2549 * Context: Only called from attach() path context 2550 */ 2551 static int 2552 tavor_special_qp_contexts_reserve(tavor_state_t *state) 2553 { 2554 tavor_rsrc_t *qp0_rsrc, *qp1_rsrc; 2555 int status; 2556 2557 TAVOR_TNF_ENTER(tavor_special_qp_contexts_reserve); 2558 2559 /* Initialize the lock used for special QP rsrc management */ 2560 mutex_init(&state->ts_spec_qplock, NULL, MUTEX_DRIVER, 2561 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2562 2563 /* 2564 * Reserve contexts for QP0. These QP contexts will be setup to 2565 * act as aliases for the real QP0. Note: We are required to grab 2566 * two QPs (one per port) even if we are operating in single-port 2567 * mode. 2568 */ 2569 status = tavor_rsrc_alloc(state, TAVOR_QPC, 2, TAVOR_SLEEP, &qp0_rsrc); 2570 if (status != DDI_SUCCESS) { 2571 mutex_destroy(&state->ts_spec_qplock); 2572 TNF_PROBE_0(tavor_special_qp_contexts_reserve_qp0_fail, 2573 TAVOR_TNF_ERROR, ""); 2574 TAVOR_TNF_EXIT(tavor_special_qp_contexts_reserve); 2575 return (DDI_FAILURE); 2576 } 2577 state->ts_spec_qp0 = qp0_rsrc; 2578 2579 /* 2580 * Reserve contexts for QP1. These QP contexts will be setup to 2581 * act as aliases for the real QP1. Note: We are required to grab 2582 * two QPs (one per port) even if we are operating in single-port 2583 * mode. 2584 */ 2585 status = tavor_rsrc_alloc(state, TAVOR_QPC, 2, TAVOR_SLEEP, &qp1_rsrc); 2586 if (status != DDI_SUCCESS) { 2587 tavor_rsrc_free(state, &qp0_rsrc); 2588 mutex_destroy(&state->ts_spec_qplock); 2589 TNF_PROBE_0(tavor_special_qp_contexts_reserve_qp1_fail, 2590 TAVOR_TNF_ERROR, ""); 2591 TAVOR_TNF_EXIT(tavor_special_qp_contexts_reserve); 2592 return (DDI_FAILURE); 2593 } 2594 state->ts_spec_qp1 = qp1_rsrc; 2595 2596 TAVOR_TNF_EXIT(tavor_special_qp_contexts_reserve); 2597 return (DDI_SUCCESS); 2598 } 2599 2600 2601 /* 2602 * tavor_special_qp_contexts_unreserve() 2603 * Context: Only called from attach() and/or detach() path contexts 2604 */ 2605 static void 2606 tavor_special_qp_contexts_unreserve(tavor_state_t *state) 2607 { 2608 TAVOR_TNF_ENTER(tavor_special_qp_contexts_unreserve); 2609 2610 /* Unreserve contexts for QP1 */ 2611 tavor_rsrc_free(state, &state->ts_spec_qp1); 2612 2613 /* Unreserve contexts for QP0 */ 2614 tavor_rsrc_free(state, &state->ts_spec_qp0); 2615 2616 /* Destroy the lock used for special QP rsrc management */ 2617 mutex_destroy(&state->ts_spec_qplock); 2618 2619 TAVOR_TNF_EXIT(tavor_special_qp_contexts_unreserve); 2620 } 2621 2622 2623 /* 2624 * tavor_sw_reset() 2625 * Context: Currently called only from attach() path context 2626 */ 2627 static int 2628 tavor_sw_reset(tavor_state_t *state) 2629 { 2630 dev_info_t *dip, *pdip; 2631 ddi_acc_handle_t hdl = state->ts_pci_cfghdl, phdl; 2632 uint32_t reset_delay; 2633 int status, i; 2634 2635 TAVOR_TNF_ENTER(tavor_sw_reset); 2636 2637 /* 2638 * If the configured software reset delay is set to zero, then we 2639 * will not attempt a software reset of the Tavor device. 2640 */ 2641 reset_delay = state->ts_cfg_profile->cp_sw_reset_delay; 2642 if (reset_delay == 0) { 2643 TAVOR_TNF_EXIT(tavor_sw_reset); 2644 return (DDI_SUCCESS); 2645 } 2646 2647 /* 2648 * Get dip for HCA device _and_ parent device as well. Parent access 2649 * is necessary here because software reset of the Tavor hardware 2650 * will reinitialize both the config registers of the PCI bridge 2651 * (parent, if it exists) and the IB HCA (self) 2652 */ 2653 dip = state->ts_dip; 2654 pdip = ddi_get_parent(dip); 2655 2656 /* Query the PCI capabilities of the HCA device */ 2657 tavor_pci_capability_list(state, hdl); 2658 2659 /* 2660 * Read all PCI config info (reg0...reg63). Note: According to the 2661 * Tavor software reset application note, we should not read or 2662 * restore the values in reg22 and reg23. 2663 */ 2664 for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) { 2665 if ((i != TAVOR_SW_RESET_REG22_RSVD) && 2666 (i != TAVOR_SW_RESET_REG23_RSVD)) { 2667 state->ts_cfg_data[i] = pci_config_get32(hdl, i << 2); 2668 } 2669 } 2670 2671 if (TAVOR_PARENT_IS_BRIDGE(pdip)) { 2672 /* 2673 * Setup for PCI config read/write of bridge device 2674 */ 2675 status = pci_config_setup(pdip, &phdl); 2676 if (status != DDI_SUCCESS) { 2677 TNF_PROBE_0(tavor_sw_reset_pcicfg_p_fail, 2678 TAVOR_TNF_ERROR, ""); 2679 TAVOR_TNF_EXIT(tavor_sw_reset); 2680 return (DDI_FAILURE); 2681 } 2682 2683 /* 2684 * Read all PCI config info (reg0...reg63). Note: According to 2685 * the Tavor software reset application note, we should not 2686 * read or restore the values in reg22 and reg23. 2687 */ 2688 for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) { 2689 if ((i != TAVOR_SW_RESET_REG22_RSVD) && 2690 (i != TAVOR_SW_RESET_REG23_RSVD)) { 2691 state->ts_cfg_pdata[i] = 2692 pci_config_get32(phdl, i << 2); 2693 } 2694 } 2695 } 2696 2697 /* 2698 * Perform the software reset (by writing 1 at offset 0xF0010) 2699 */ 2700 ddi_put32(state->ts_reg_cmdhdl, state->ts_cmd_regs.sw_reset, 2701 TAVOR_SW_RESET_START); 2702 2703 drv_usecwait(reset_delay); 2704 2705 if (TAVOR_PARENT_IS_BRIDGE(pdip)) { 2706 /* 2707 * Bridge exists, so wait for the bridge to become ready. 2708 * 2709 * The above delay is necessary to avoid system panic from 2710 * Master Abort. If the device is accessed before this delay, 2711 * device will not respond to config cycles and they will be 2712 * terminate with a Master Abort which will panic the system. 2713 * Below is the loop we use to poll status from the device to 2714 * determine if it is OK to proceed. 2715 */ 2716 i = 0; 2717 while (pci_config_get32(phdl, 0) == TAVOR_SW_RESET_NOTDONE) { 2718 drv_usecwait(TAVOR_SW_RESET_POLL_DELAY); 2719 } 2720 2721 /* 2722 * Write all the PCI config registers back into each device 2723 * (except for reg22 and reg23 - see above) 2724 */ 2725 for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) { 2726 if ((i != TAVOR_SW_RESET_REG22_RSVD) && 2727 (i != TAVOR_SW_RESET_REG23_RSVD)) { 2728 pci_config_put32(phdl, i << 2, 2729 state->ts_cfg_pdata[i]); 2730 } 2731 } 2732 2733 /* 2734 * Tear down the config setup (for bridge device) 2735 */ 2736 pci_config_teardown(&phdl); 2737 2738 /* No Bridge Device */ 2739 } else { 2740 /* 2741 * Bridge does not exist, so instead wait for the device itself 2742 * to become ready. 2743 * 2744 * The above delay is necessary to avoid system panic from 2745 * Master Abort. If the device is accessed before this delay, 2746 * device will not respond to config cycles and they will be 2747 * terminate with a Master Abort which will panic the system. 2748 * Below is the loop we use to poll status from the device to 2749 * determine if it is OK to proceed. 2750 */ 2751 i = 0; 2752 while (pci_config_get32(hdl, 0) == TAVOR_SW_RESET_NOTDONE) { 2753 drv_usecwait(TAVOR_SW_RESET_POLL_DELAY); 2754 } 2755 } 2756 2757 for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) { 2758 if ((i != TAVOR_SW_RESET_REG22_RSVD) && 2759 (i != TAVOR_SW_RESET_REG23_RSVD)) { 2760 pci_config_put32(hdl, i << 2, state->ts_cfg_data[i]); 2761 } 2762 } 2763 2764 TAVOR_TNF_EXIT(tavor_sw_reset); 2765 return (DDI_SUCCESS); 2766 } 2767 2768 2769 /* 2770 * tavor_mcg_init() 2771 * Context: Only called from attach() path context 2772 */ 2773 static int 2774 tavor_mcg_init(tavor_state_t *state) 2775 { 2776 uint_t mcg_tmp_sz; 2777 2778 TAVOR_TNF_ENTER(tavor_mcg_init); 2779 2780 /* 2781 * Allocate space for the MCG temporary copy buffer. This is 2782 * used by the Attach/Detach Multicast Group code 2783 */ 2784 mcg_tmp_sz = TAVOR_MCGMEM_SZ(state); 2785 state->ts_mcgtmp = kmem_zalloc(mcg_tmp_sz, KM_SLEEP); 2786 2787 /* 2788 * Initialize the multicast group mutex. This ensures atomic 2789 * access to add, modify, and remove entries in the multicast 2790 * group hash lists. 2791 */ 2792 mutex_init(&state->ts_mcglock, NULL, MUTEX_DRIVER, 2793 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2794 2795 TAVOR_TNF_EXIT(tavor_mcg_init); 2796 return (DDI_SUCCESS); 2797 } 2798 2799 2800 /* 2801 * tavor_mcg_fini() 2802 * Context: Only called from attach() and/or detach() path contexts 2803 */ 2804 static void 2805 tavor_mcg_fini(tavor_state_t *state) 2806 { 2807 uint_t mcg_tmp_sz; 2808 2809 TAVOR_TNF_ENTER(tavor_mcg_fini); 2810 2811 /* Free up the space used for the MCG temporary copy buffer */ 2812 mcg_tmp_sz = TAVOR_MCGMEM_SZ(state); 2813 kmem_free(state->ts_mcgtmp, mcg_tmp_sz); 2814 2815 /* Destroy the multicast group mutex */ 2816 mutex_destroy(&state->ts_mcglock); 2817 2818 TAVOR_TNF_EXIT(tavor_mcg_fini); 2819 } 2820 2821 2822 /* 2823 * tavor_fw_version_check() 2824 * Context: Only called from attach() path context 2825 */ 2826 static int 2827 tavor_fw_version_check(tavor_state_t *state) 2828 { 2829 uint_t tavor_fw_ver_major; 2830 uint_t tavor_fw_ver_minor; 2831 uint_t tavor_fw_ver_subminor; 2832 2833 /* 2834 * Depending on which version of driver we have attached, the firmware 2835 * version checks will be different. We set up the comparison values 2836 * for both HCA Mode (Tavor hardware) or COMPAT Mode (Arbel hardware 2837 * running in tavor mode). 2838 */ 2839 switch (state->ts_operational_mode) { 2840 case TAVOR_HCA_MODE: 2841 tavor_fw_ver_major = TAVOR_FW_VER_MAJOR; 2842 tavor_fw_ver_minor = TAVOR_FW_VER_MINOR; 2843 tavor_fw_ver_subminor = TAVOR_FW_VER_SUBMINOR; 2844 break; 2845 2846 case TAVOR_COMPAT_MODE: 2847 tavor_fw_ver_major = TAVOR_COMPAT_FW_VER_MAJOR; 2848 tavor_fw_ver_minor = TAVOR_COMPAT_FW_VER_MINOR; 2849 tavor_fw_ver_subminor = TAVOR_COMPAT_FW_VER_SUBMINOR; 2850 break; 2851 2852 default: 2853 return (DDI_FAILURE); 2854 } 2855 2856 /* 2857 * If FW revision major number is less than acceptable, 2858 * return failure, else if greater return success. If 2859 * the major numbers are equal than check the minor number 2860 */ 2861 if (state->ts_fw.fw_rev_major < tavor_fw_ver_major) { 2862 return (DDI_FAILURE); 2863 } else if (state->ts_fw.fw_rev_major > tavor_fw_ver_major) { 2864 return (DDI_SUCCESS); 2865 } 2866 /* 2867 * Do the same check as above, except for minor revision numbers 2868 * If the minor numbers are equal than check the subminor number 2869 */ 2870 if (state->ts_fw.fw_rev_minor < tavor_fw_ver_minor) { 2871 return (DDI_FAILURE); 2872 } else if (state->ts_fw.fw_rev_minor > tavor_fw_ver_minor) { 2873 return (DDI_SUCCESS); 2874 } 2875 2876 /* 2877 * Once again we do the same check as above, except for the subminor 2878 * revision number. If the subminor numbers are equal here, then 2879 * these are the same firmware version, return success 2880 */ 2881 if (state->ts_fw.fw_rev_subminor < tavor_fw_ver_subminor) { 2882 return (DDI_FAILURE); 2883 } else if (state->ts_fw.fw_rev_subminor > tavor_fw_ver_subminor) { 2884 return (DDI_SUCCESS); 2885 } 2886 2887 return (DDI_SUCCESS); 2888 } 2889 2890 2891 /* 2892 * tavor_device_info_report() 2893 * Context: Only called from attach() path context 2894 */ 2895 static void 2896 tavor_device_info_report(tavor_state_t *state) 2897 { 2898 cmn_err(CE_CONT, "?tavor%d: FW ver: %04d.%04d.%04d, " 2899 "HW rev: %02x\n", state->ts_instance, state->ts_fw.fw_rev_major, 2900 state->ts_fw.fw_rev_minor, state->ts_fw.fw_rev_subminor, 2901 state->ts_adapter.rev_id); 2902 cmn_err(CE_CONT, "?tavor%d: %64s (0x%016" PRIx64 ")\n", 2903 state->ts_instance, state->ts_nodedesc, state->ts_nodeguid); 2904 } 2905 2906 2907 /* 2908 * tavor_pci_capability_list() 2909 * Context: Only called from attach() path context 2910 */ 2911 static void 2912 tavor_pci_capability_list(tavor_state_t *state, ddi_acc_handle_t hdl) 2913 { 2914 uint_t offset, data; 2915 2916 TAVOR_TNF_ENTER(tavor_pci_capability_list); 2917 2918 /* 2919 * Check for the "PCI Capabilities" bit in the "Status Register". 2920 * Bit 4 in this register indicates the presence of a "PCI 2921 * Capabilities" list. 2922 */ 2923 data = pci_config_get16(hdl, 0x6); 2924 if ((data & 0x10) == 0) { 2925 TNF_PROBE_0(tavor_pci_capab_list_fail, TAVOR_TNF_ERROR, ""); 2926 TAVOR_TNF_EXIT(tavor_pci_capability_list); 2927 return; 2928 } 2929 2930 /* 2931 * Starting from offset 0x34 in PCI config space, find the 2932 * head of "PCI capabilities" list, and walk the list. If 2933 * capabilities of a known type are encountered (e.g. 2934 * "PCI-X Capability"), then call the appropriate handler 2935 * function. 2936 */ 2937 offset = pci_config_get8(hdl, 0x34); 2938 while (offset != 0x0) { 2939 data = pci_config_get8(hdl, offset); 2940 2941 /* 2942 * Check for known capability types. Tavor has the 2943 * following: 2944 * o VPD Capability (0x03) 2945 * o PCI-X Capability (0x07) 2946 * o MSI Capability (0x05) 2947 * o MSIX Capability (0x11) 2948 */ 2949 switch (data) { 2950 case 0x03: 2951 tavor_pci_capability_vpd(state, hdl, offset); 2952 break; 2953 case 0x07: 2954 tavor_pci_capability_pcix(state, hdl, offset); 2955 break; 2956 case 0x05: 2957 break; 2958 default: 2959 break; 2960 } 2961 2962 /* Get offset of next entry in list */ 2963 offset = pci_config_get8(hdl, offset + 1); 2964 } 2965 2966 TAVOR_TNF_EXIT(tavor_pci_capability_list); 2967 } 2968 2969 /* 2970 * tavor_pci_read_vpd() 2971 * Context: Only called from attach() path context 2972 * utility routine for tavor_pci_capability_vpd() 2973 */ 2974 static int 2975 tavor_pci_read_vpd(ddi_acc_handle_t hdl, uint_t offset, uint32_t addr, 2976 uint32_t *data) 2977 { 2978 int retry = 4; /* retry counter for EEPROM poll */ 2979 uint32_t val; 2980 int vpd_addr = offset + 2; 2981 int vpd_data = offset + 4; 2982 2983 TAVOR_TNF_ENTER(tavor_pci_read_vpd); 2984 2985 /* 2986 * In order to read a 32-bit value from VPD, we are to write down 2987 * the address (offset in the VPD itself) to the address register. 2988 * To signal the read, we also clear bit 31. We then poll on bit 31 2989 * and when it is set, we can then read our 4 bytes from the data 2990 * register. 2991 */ 2992 (void) pci_config_put32(hdl, offset, addr << 16); 2993 do { 2994 drv_usecwait(1000); 2995 val = pci_config_get16(hdl, vpd_addr); 2996 if ((val >> 15) & 0x01) { 2997 *data = pci_config_get32(hdl, vpd_data); 2998 TAVOR_TNF_EXIT(tavor_pci_read_vpd); 2999 return (DDI_SUCCESS); 3000 } 3001 } while (--retry); 3002 3003 TNF_PROBE_0(tavor_pci_read_vpd_fail, TAVOR_TNF_ERROR, ""); 3004 TAVOR_TNF_EXIT(tavor_pci_read_vpd); 3005 return (DDI_FAILURE); 3006 } 3007 3008 3009 /* 3010 * tavor_pci_capability_vpd() 3011 * Context: Only called from attach() path context 3012 */ 3013 static void 3014 tavor_pci_capability_vpd(tavor_state_t *state, ddi_acc_handle_t hdl, 3015 uint_t offset) 3016 { 3017 uint8_t name_length; 3018 uint8_t pn_length; 3019 int i, err = 0; 3020 int vpd_str_id = 0; 3021 int vpd_ro_desc; 3022 int vpd_ro_pn_desc; 3023 #ifndef _LITTLE_ENDIAN 3024 uint32_t data32; 3025 #endif /* _LITTLE_ENDIAN */ 3026 union { 3027 uint32_t vpd_int[TAVOR_VPD_HDR_DWSIZE]; 3028 uchar_t vpd_char[TAVOR_VPD_HDR_BSIZE]; 3029 } vpd; 3030 3031 TAVOR_TNF_ENTER(tavor_pci_capability_vpd); 3032 3033 /* 3034 * Read Vital Product Data (VPD) from PCI-X capability. 3035 */ 3036 for (i = 0; i < TAVOR_VPD_HDR_DWSIZE; i++) { 3037 err = tavor_pci_read_vpd(hdl, offset, i << 2, &vpd.vpd_int[i]); 3038 if (err != DDI_SUCCESS) { 3039 cmn_err(CE_NOTE, "!VPD read failed\n"); 3040 goto out; 3041 } 3042 } 3043 3044 #ifndef _LITTLE_ENDIAN 3045 /* 3046 * Need to swap bytes for big endian. 3047 */ 3048 for (i = 0; i < TAVOR_VPD_HDR_DWSIZE; i++) { 3049 data32 = vpd.vpd_int[i]; 3050 vpd.vpd_char[(i << 2) + 3] = 3051 (uchar_t)((data32 & 0xFF000000) >> 24); 3052 vpd.vpd_char[(i << 2) + 2] = 3053 (uchar_t)((data32 & 0x00FF0000) >> 16); 3054 vpd.vpd_char[(i << 2) + 1] = 3055 (uchar_t)((data32 & 0x0000FF00) >> 8); 3056 vpd.vpd_char[i << 2] = (uchar_t)(data32 & 0x000000FF); 3057 } 3058 #endif /* _LITTLE_ENDIAN */ 3059 3060 /* Check for VPD String ID Tag */ 3061 if (vpd.vpd_char[vpd_str_id] == 0x82) { 3062 /* get the product name */ 3063 name_length = (uint8_t)vpd.vpd_char[vpd_str_id + 1]; 3064 if (name_length > sizeof (state->ts_hca_name)) { 3065 cmn_err(CE_NOTE, "!VPD name too large (0x%x)\n", 3066 name_length); 3067 goto out; 3068 } 3069 (void) memcpy(state->ts_hca_name, &vpd.vpd_char[vpd_str_id + 3], 3070 name_length); 3071 state->ts_hca_name[name_length] = 0; 3072 3073 /* get the part number */ 3074 vpd_ro_desc = name_length + 3; /* read-only tag location */ 3075 vpd_ro_pn_desc = vpd_ro_desc + 3; /* P/N keyword location */ 3076 /* 3077 * Verify read-only tag and Part Number keyword. 3078 */ 3079 if (vpd.vpd_char[vpd_ro_desc] != 0x90 || 3080 (vpd.vpd_char[vpd_ro_pn_desc] != 'P' && 3081 vpd.vpd_char[vpd_ro_pn_desc + 1] != 'N')) { 3082 cmn_err(CE_NOTE, "!VPD Part Number not found\n"); 3083 goto out; 3084 } 3085 3086 pn_length = (uint8_t)vpd.vpd_char[vpd_ro_pn_desc + 2]; 3087 if (pn_length > sizeof (state->ts_hca_pn)) { 3088 cmn_err(CE_NOTE, "!VPD part number too large (0x%x)\n", 3089 name_length); 3090 goto out; 3091 } 3092 (void) memcpy(state->ts_hca_pn, 3093 &vpd.vpd_char[vpd_ro_pn_desc + 3], 3094 pn_length); 3095 state->ts_hca_pn[pn_length] = 0; 3096 state->ts_hca_pn_len = pn_length; 3097 } else { 3098 /* Wrong VPD String ID Tag */ 3099 cmn_err(CE_NOTE, "!VPD String ID Tag not found, tag: %02x\n", 3100 vpd.vpd_char[0]); 3101 goto out; 3102 } 3103 TAVOR_TNF_EXIT(tavor_pci_capability_vpd); 3104 return; 3105 out: 3106 state->ts_hca_pn_len = 0; 3107 TNF_PROBE_0(tavor_pci_capability_vpd_fail, TAVOR_TNF_ERROR, ""); 3108 TAVOR_TNF_EXIT(tavor_pci_capability_vpd); 3109 } 3110 3111 /* 3112 * tavor_pci_capability_pcix() 3113 * Context: Only called from attach() path context 3114 */ 3115 static void 3116 tavor_pci_capability_pcix(tavor_state_t *state, ddi_acc_handle_t hdl, 3117 uint_t offset) 3118 { 3119 uint_t command, status; 3120 int max_out_splt_trans, max_mem_rd_byte_cnt; 3121 int designed_max_out_splt_trans, designed_max_mem_rd_byte_cnt; 3122 3123 TAVOR_TNF_ENTER(tavor_pci_capability_pcix); 3124 3125 /* 3126 * Query the current values for the PCI-X Command Register and 3127 * the PCI-X Status Register. 3128 */ 3129 command = pci_config_get16(hdl, offset + 2); 3130 status = pci_config_get32(hdl, offset + 4); 3131 3132 /* 3133 * Check for config property specifying "maximum outstanding 3134 * split transactions". If the property is defined and valid 3135 * (i.e. no larger than the so-called "designed maximum"), 3136 * then use the specified value to update the PCI-X Command Register. 3137 * Otherwise, extract the value from the Tavor config profile. 3138 */ 3139 designed_max_out_splt_trans = ((status >> 23) & 7); 3140 max_out_splt_trans = ddi_prop_get_int(DDI_DEV_T_ANY, state->ts_dip, 3141 DDI_PROP_DONTPASS, "pcix-max-outstanding-split-trans", -1); 3142 if ((max_out_splt_trans != -1) && 3143 ((max_out_splt_trans < 0) || 3144 (max_out_splt_trans > designed_max_out_splt_trans))) { 3145 cmn_err(CE_NOTE, "!tavor%d: property \"pcix-max-outstanding-" 3146 "split-trans\" (%d) invalid or exceeds device maximum" 3147 " (%d), using default value (%d)\n", state->ts_instance, 3148 max_out_splt_trans, designed_max_out_splt_trans, 3149 state->ts_cfg_profile->cp_max_out_splt_trans); 3150 max_out_splt_trans = 3151 state->ts_cfg_profile->cp_max_out_splt_trans; 3152 } else if (max_out_splt_trans == -1) { 3153 max_out_splt_trans = 3154 state->ts_cfg_profile->cp_max_out_splt_trans; 3155 } 3156 3157 /* 3158 * The config profile setting for max_out_splt_trans is determined 3159 * based on arch. Check tavor_cfg.c for more information. A value of 3160 * '-1' in the patchable variable means "do not change". A value of 3161 * '0' means 1 outstanding splt trans and other values as defined by 3162 * PCI. So we do one more check here, that if 'max_out_splt_trans' is 3163 * -1 (ie: < 0) we do not set the PCI command and leave it at the 3164 * default. 3165 */ 3166 if (max_out_splt_trans >= 0) { 3167 command = ((command & 0xFF8F) | max_out_splt_trans << 4); 3168 } 3169 3170 /* 3171 * Check for config property specifying "maximum memory read 3172 * byte count. If the property is defined and valid 3173 * (i.e. no larger than the so-called "designed maximum"), 3174 * then use the specified value to update the PCI-X Command Register. 3175 * Otherwise, extract the value from the Tavor config profile. 3176 */ 3177 designed_max_mem_rd_byte_cnt = ((status >> 21) & 3); 3178 max_mem_rd_byte_cnt = ddi_prop_get_int(DDI_DEV_T_ANY, state->ts_dip, 3179 DDI_PROP_DONTPASS, "pcix-max-read-byte-count", -1); 3180 if ((max_mem_rd_byte_cnt != -1) && 3181 ((max_mem_rd_byte_cnt < 0) || 3182 (max_mem_rd_byte_cnt > designed_max_mem_rd_byte_cnt))) { 3183 cmn_err(CE_NOTE, "!tavor%d: property \"pcix-max-read-byte-" 3184 "count\" (%d) invalid or exceeds device maximum" 3185 " (%d), using default value (%d)\n", state->ts_instance, 3186 max_mem_rd_byte_cnt, designed_max_mem_rd_byte_cnt, 3187 state->ts_cfg_profile->cp_max_mem_rd_byte_cnt); 3188 max_mem_rd_byte_cnt = 3189 state->ts_cfg_profile->cp_max_mem_rd_byte_cnt; 3190 } else if (max_mem_rd_byte_cnt == -1) { 3191 max_mem_rd_byte_cnt = 3192 state->ts_cfg_profile->cp_max_mem_rd_byte_cnt; 3193 } 3194 3195 /* 3196 * The config profile setting for max_mem_rd_byte_cnt is determined 3197 * based on arch. Check tavor_cfg.c for more information. A value of 3198 * '-1' in the patchable variable means "do not change". A value of 3199 * '0' means minimum (512B) read, and other values as defined by 3200 * PCI. So we do one more check here, that if 'max_mem_rd_byte_cnt' is 3201 * -1 (ie: < 0) we do not set the PCI command and leave it at the 3202 * default. 3203 */ 3204 if (max_mem_rd_byte_cnt >= 0) { 3205 command = ((command & 0xFFF3) | max_mem_rd_byte_cnt << 2); 3206 } 3207 3208 /* 3209 * Update the PCI-X Command Register with the newly configured 3210 * values. 3211 */ 3212 pci_config_put16(hdl, offset + 2, command); 3213 3214 TAVOR_TNF_EXIT(tavor_pci_capability_pcix); 3215 } 3216 3217 3218 /* 3219 * tavor_intr_or_msi_init() 3220 * Context: Only called from attach() path context 3221 */ 3222 static int 3223 tavor_intr_or_msi_init(tavor_state_t *state) 3224 { 3225 int status; 3226 3227 TAVOR_TNF_ENTER(tavor_intr_or_msi_init); 3228 3229 /* Query for the list of supported interrupt event types */ 3230 status = ddi_intr_get_supported_types(state->ts_dip, 3231 &state->ts_intr_types_avail); 3232 if (status != DDI_SUCCESS) { 3233 TNF_PROBE_0(tavor_intr_or_msi_init_gettypes_fail, 3234 TAVOR_TNF_ERROR, ""); 3235 TAVOR_TNF_EXIT(tavor_intr_or_msi_init); 3236 return (DDI_FAILURE); 3237 } 3238 3239 /* 3240 * If Tavor/Arbel supports MSI in this system (and, if it 3241 * hasn't been overridden by a configuration variable), then 3242 * the default behavior is to use a single MSI. Otherwise, 3243 * fallback to using legacy interrupts. Also, if MSI allocatis chosen, 3244 * but fails for whatever reasons, then fallback to using legacy 3245 * interrupts. 3246 */ 3247 if ((state->ts_cfg_profile->cp_use_msi_if_avail != 0) && 3248 (state->ts_intr_types_avail & DDI_INTR_TYPE_MSI)) { 3249 status = tavor_add_intrs(state, DDI_INTR_TYPE_MSI); 3250 if (status == DDI_SUCCESS) { 3251 state->ts_intr_type_chosen = DDI_INTR_TYPE_MSI; 3252 TAVOR_TNF_EXIT(tavor_intr_or_msi_init); 3253 return (DDI_SUCCESS); 3254 } 3255 } 3256 3257 /* 3258 * MSI interrupt allocation failed, or was not available. Fallback to 3259 * legacy interrupt support. 3260 */ 3261 if (state->ts_intr_types_avail & DDI_INTR_TYPE_FIXED) { 3262 status = tavor_add_intrs(state, DDI_INTR_TYPE_FIXED); 3263 if (status == DDI_SUCCESS) { 3264 state->ts_intr_type_chosen = DDI_INTR_TYPE_FIXED; 3265 TAVOR_TNF_EXIT(tavor_intr_or_msi_init); 3266 return (DDI_SUCCESS); 3267 } 3268 } 3269 3270 /* 3271 * Neither MSI or legacy interrupts were successful. return failure. 3272 */ 3273 TAVOR_TNF_EXIT(tavor_intr_or_msi_setup); 3274 return (DDI_FAILURE); 3275 } 3276 3277 /* 3278 * tavor_add_intrs() 3279 * Context: Only called from attach() patch context 3280 */ 3281 static int 3282 tavor_add_intrs(tavor_state_t *state, int intr_type) 3283 { 3284 int status; 3285 3286 TAVOR_TNF_ENTER(tavor_add_intrs); 3287 3288 /* Get number of interrupts/MSI supported */ 3289 status = ddi_intr_get_nintrs(state->ts_dip, intr_type, 3290 &state->ts_intrmsi_count); 3291 if (status != DDI_SUCCESS) { 3292 TNF_PROBE_0(tavor_add_intrs_getnintrs_fail, 3293 TAVOR_TNF_ERROR, ""); 3294 TAVOR_TNF_EXIT(tavor_add_intrs); 3295 return (DDI_FAILURE); 3296 } 3297 3298 /* Get number of available interrupts/MSI */ 3299 status = ddi_intr_get_navail(state->ts_dip, intr_type, 3300 &state->ts_intrmsi_avail); 3301 if (status != DDI_SUCCESS) { 3302 TNF_PROBE_0(tavor_add_intrs_getnavail_fail, 3303 TAVOR_TNF_ERROR, ""); 3304 TAVOR_TNF_EXIT(tavor_add_intrs); 3305 return (DDI_FAILURE); 3306 } 3307 3308 /* Ensure that we have at least one (1) usable MSI or interrupt */ 3309 if ((state->ts_intrmsi_avail < 1) || (state->ts_intrmsi_count < 1)) { 3310 TNF_PROBE_0(tavor_add_intrs_notenoughts_intrmsi_fail, 3311 TAVOR_TNF_ERROR, ""); 3312 TAVOR_TNF_EXIT(tavor_add_intrs); 3313 return (DDI_FAILURE); 3314 } 3315 3316 /* Attempt to allocate a single interrupt/MSI handle */ 3317 status = ddi_intr_alloc(state->ts_dip, &state->ts_intrmsi_hdl, 3318 intr_type, 0, 1, &state->ts_intrmsi_allocd, 3319 DDI_INTR_ALLOC_STRICT); 3320 if (status != DDI_SUCCESS) { 3321 TNF_PROBE_0(tavor_add_intrs_intralloc_fail, 3322 TAVOR_TNF_ERROR, ""); 3323 TAVOR_TNF_EXIT(tavor_add_intrs); 3324 return (DDI_FAILURE); 3325 } 3326 3327 /* Ensure that we have allocated at least one (1) MSI or interrupt */ 3328 if (state->ts_intrmsi_allocd < 1) { 3329 TNF_PROBE_0(tavor_add_intrs_noallocts_intrmsi_fail, 3330 TAVOR_TNF_ERROR, ""); 3331 TAVOR_TNF_EXIT(tavor_add_intrs); 3332 return (DDI_FAILURE); 3333 } 3334 3335 /* 3336 * Extract the priority for the allocated interrupt/MSI. This 3337 * will be used later when initializing certain mutexes. 3338 */ 3339 status = ddi_intr_get_pri(state->ts_intrmsi_hdl, 3340 &state->ts_intrmsi_pri); 3341 if (status != DDI_SUCCESS) { 3342 /* Free the allocated interrupt/MSI handle */ 3343 (void) ddi_intr_free(state->ts_intrmsi_hdl); 3344 3345 TNF_PROBE_0(tavor_add_intrs_getpri_fail, 3346 TAVOR_TNF_ERROR, ""); 3347 TAVOR_TNF_EXIT(tavor_add_intrs); 3348 return (DDI_FAILURE); 3349 } 3350 3351 /* Make sure the interrupt/MSI priority is below 'high level' */ 3352 if (state->ts_intrmsi_pri >= ddi_intr_get_hilevel_pri()) { 3353 /* Free the allocated interrupt/MSI handle */ 3354 (void) ddi_intr_free(state->ts_intrmsi_hdl); 3355 3356 TNF_PROBE_0(tavor_add_intrs_hilevelpri_fail, 3357 TAVOR_TNF_ERROR, ""); 3358 TAVOR_TNF_EXIT(tavor_add_intrs); 3359 return (DDI_FAILURE); 3360 } 3361 3362 /* Get add'l capability information regarding interrupt/MSI */ 3363 status = ddi_intr_get_cap(state->ts_intrmsi_hdl, 3364 &state->ts_intrmsi_cap); 3365 if (status != DDI_SUCCESS) { 3366 /* Free the allocated interrupt/MSI handle */ 3367 (void) ddi_intr_free(state->ts_intrmsi_hdl); 3368 3369 TNF_PROBE_0(tavor_add_intrs_getcap_fail, 3370 TAVOR_TNF_ERROR, ""); 3371 TAVOR_TNF_EXIT(tavor_add_intrs); 3372 return (DDI_FAILURE); 3373 } 3374 3375 TAVOR_TNF_EXIT(tavor_add_intrs); 3376 return (DDI_SUCCESS); 3377 } 3378 3379 3380 /* 3381 * tavor_intr_or_msi_fini() 3382 * Context: Only called from attach() and/or detach() path contexts 3383 */ 3384 static int 3385 tavor_intr_or_msi_fini(tavor_state_t *state) 3386 { 3387 int status; 3388 3389 TAVOR_TNF_ENTER(tavor_intr_or_msi_fini); 3390 3391 /* Free the allocated interrupt/MSI handle */ 3392 status = ddi_intr_free(state->ts_intrmsi_hdl); 3393 if (status != DDI_SUCCESS) { 3394 TNF_PROBE_0(tavor_intr_or_msi_fini_freehdl_fail, 3395 TAVOR_TNF_ERROR, ""); 3396 TAVOR_TNF_EXIT(tavor_intr_or_msi_fini); 3397 return (DDI_FAILURE); 3398 } 3399 3400 TAVOR_TNF_EXIT(tavor_intr_or_msi_fini); 3401 return (DDI_SUCCESS); 3402 } 3403 3404 3405 /* Disable Tavor interrupts */ 3406 static int 3407 tavor_intr_disable(tavor_state_t *state) 3408 { 3409 ushort_t msi_ctrl = 0, caps_ctrl = 0; 3410 ddi_acc_handle_t pci_cfg_hdl = state->ts_pci_cfghdl; 3411 ASSERT(pci_cfg_hdl != NULL); 3412 ASSERT(state->ts_intr_types_avail & 3413 (DDI_INTR_TYPE_FIXED | DDI_INTR_TYPE_MSI)); 3414 3415 /* 3416 * Check if MSI interrupts are used. If so, disable MSI interupts. 3417 * If not, since Tavor doesn't support MSI-X interrupts, assuming the 3418 * legacy interrupt is used instead, disable the legacy interrupt. 3419 */ 3420 if ((state->ts_cfg_profile->cp_use_msi_if_avail != 0) && 3421 (state->ts_intr_types_avail & DDI_INTR_TYPE_MSI)) { 3422 3423 if ((PCI_CAP_LOCATE(pci_cfg_hdl, PCI_CAP_ID_MSI, 3424 &caps_ctrl) == DDI_SUCCESS)) { 3425 if ((msi_ctrl = PCI_CAP_GET16(pci_cfg_hdl, NULL, 3426 caps_ctrl, PCI_MSI_CTRL)) == PCI_CAP_EINVAL16) 3427 return (DDI_FAILURE); 3428 } 3429 ASSERT(msi_ctrl != 0); 3430 3431 if (!(msi_ctrl & PCI_MSI_ENABLE_BIT)) 3432 return (DDI_SUCCESS); 3433 3434 if (msi_ctrl & PCI_MSI_PVM_MASK) { 3435 int offset = (msi_ctrl & PCI_MSI_64BIT_MASK) ? 3436 PCI_MSI_64BIT_MASKBITS : PCI_MSI_32BIT_MASK; 3437 3438 /* Clear all inums in MSI */ 3439 PCI_CAP_PUT32(pci_cfg_hdl, NULL, caps_ctrl, 3440 offset, 0x0); 3441 } 3442 3443 /* Disable MSI interrupts */ 3444 msi_ctrl &= ~PCI_MSI_ENABLE_BIT; 3445 PCI_CAP_PUT16(pci_cfg_hdl, NULL, caps_ctrl, PCI_MSI_CTRL, 3446 msi_ctrl); 3447 3448 } else { 3449 uint16_t cmdreg = pci_config_get16(pci_cfg_hdl, PCI_CONF_COMM); 3450 ASSERT(state->ts_intr_types_avail & DDI_INTR_TYPE_FIXED); 3451 3452 /* Disable the legacy interrupts */ 3453 cmdreg |= PCI_COMM_INTX_DISABLE; 3454 pci_config_put16(pci_cfg_hdl, PCI_CONF_COMM, cmdreg); 3455 } 3456 3457 return (DDI_SUCCESS); 3458 } 3459 3460 /* Tavor quiesce(9F) entry */ 3461 static int 3462 tavor_quiesce(dev_info_t *dip) 3463 { 3464 tavor_state_t *state = ddi_get_soft_state(tavor_statep, 3465 DEVI(dip)->devi_instance); 3466 ASSERT(state != NULL); 3467 3468 /* start fastreboot */ 3469 state->ts_quiescing = B_TRUE; 3470 3471 /* Shutdown HCA ports */ 3472 if (tavor_hca_ports_shutdown(state, 3473 state->ts_cfg_profile->cp_num_ports) != TAVOR_CMD_SUCCESS) { 3474 state->ts_quiescing = B_FALSE; 3475 return (DDI_FAILURE); 3476 } 3477 3478 /* Close HCA */ 3479 if (tavor_close_hca_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN) != 3480 TAVOR_CMD_SUCCESS) { 3481 state->ts_quiescing = B_FALSE; 3482 return (DDI_FAILURE); 3483 } 3484 3485 /* Shutdown FW */ 3486 if (tavor_sys_dis_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN) != 3487 TAVOR_CMD_SUCCESS) { 3488 state->ts_quiescing = B_FALSE; 3489 return (DDI_FAILURE); 3490 } 3491 3492 /* Disable interrupts */ 3493 if (tavor_intr_disable(state) != DDI_SUCCESS) { 3494 state->ts_quiescing = B_FALSE; 3495 return (DDI_FAILURE); 3496 } 3497 3498 /* SW-reset */ 3499 if (tavor_sw_reset(state) != DDI_SUCCESS) { 3500 state->ts_quiescing = B_FALSE; 3501 return (DDI_FAILURE); 3502 } 3503 3504 return (DDI_SUCCESS); 3505 } 3506