1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * ibtl_impl.c 28 * 29 * This file contains the IBTF module's initialization and 30 * IBTF Clients/Modules registration routines. 31 */ 32 33 #include <sys/modctl.h> 34 #include <sys/sunndi.h> 35 #include <sys/sunmdi.h> 36 #include <sys/ib/ibtl/impl/ibtl.h> 37 #include <sys/ib/ibtl/impl/ibtl_ibnex.h> 38 39 /* 40 * Globals. 41 */ 42 static char ibtf[] = "ibtl_impl"; 43 44 extern ibtl_ibnex_callback_t ibtl_ibnex_callback_routine; 45 46 /* 47 * ibtl_clnt_list: 48 * 49 * Head of the list of IBT Client Instances. The IBT Client List 50 * is modified by IBTF on an IBT client's ibt_attach/ibt_detach call. 51 * 52 * ibtl_hca_list: 53 * 54 * Head of the list of HCA devices. The HCA List is modified by IBTF on 55 * a CI's ibc_attach/ibc_detach call. 56 * The datap of the list elements points to an ibtl_hca_devinfo_s 57 * structure. 58 * 59 * (ibc_attach) 60 * ibtl_hca_list -> ibtl_hca_devinfo_t--> ... -->ibtl_hca_devinfo_t 61 * [per-hca_dev] | ^ {nth HCA Dev} 62 * | | 63 * | ibtl_hca_t (ibt_open_hca) 64 * | ^ | 65 * | | | 66 * v | V 67 * ibtl_clnt_list -> ibtl_clnt_t--> ...--> {n'th Module} 68 * [per-client_instance] (ibt_attach) 69 * 70 */ 71 72 /* Global List of IBT Client Instances, and associated mutex. */ 73 struct ibtl_clnt_s *ibtl_clnt_list = NULL; 74 kmutex_t ibtl_clnt_list_mutex; 75 76 /* Lock for the race between the client and CM to free QPs. */ 77 kmutex_t ibtl_free_qp_mutex; 78 79 /* Lock for the race between the client closing the HCA and QPN being freed. */ 80 kcondvar_t ibtl_close_hca_cv; 81 82 /* Global List of HCA Devices, and associated mutex. */ 83 struct ibtl_hca_devinfo_s *ibtl_hca_list = NULL; 84 85 /* Well-known async handlers and associated client private. */ 86 ibt_async_handler_t ibtl_cm_async_handler; 87 ibt_async_handler_t ibtl_dm_async_handler; 88 ibt_async_handler_t ibtl_ibma_async_handler; 89 void *ibtl_cm_clnt_private; 90 void *ibtl_dm_clnt_private; 91 void *ibtl_ibma_clnt_private; 92 93 extern int ib_hw_status; 94 _NOTE(SCHEME_PROTECTS_DATA("Scheme protects data", ib_hw_status)) 95 96 /* 97 * Misc Module Declarations. 98 */ 99 extern struct mod_ops mod_miscops; 100 static struct modlmisc modlmisc = { 101 &mod_miscops, /* Type of module - misc. */ 102 "IB Transport Layer" /* Name of the Module. */ 103 }; 104 105 static struct modlinkage modlinkage = { 106 MODREV_1, (void *)&modlmisc, NULL 107 }; 108 109 110 /* 111 * IBTF Loadable Module Routines. 112 */ 113 114 int 115 _init(void) 116 { 117 int rval; 118 119 if ((rval = mod_install(&modlinkage)) != 0) 120 return (rval); 121 122 /* 123 * initialize IBTL ib2usec table 124 */ 125 ibtl_ib2usec_init(); 126 127 /* 128 * Initialize Logging 129 */ 130 ibtl_logging_initialization(); 131 132 /* 133 * Initialize the Alloc QP States. 134 */ 135 ibtl_init_cep_states(); 136 137 /* 138 * Initialize all Global Link Lists. 139 */ 140 mutex_init(&ibtl_clnt_list_mutex, NULL, MUTEX_DEFAULT, NULL); 141 mutex_init(&ibtl_free_qp_mutex, NULL, MUTEX_DEFAULT, NULL); 142 cv_init(&ibtl_close_hca_cv, NULL, CV_DEFAULT, NULL); 143 144 mutex_init(&ibtl_qp_mutex, NULL, MUTEX_DEFAULT, NULL); 145 cv_init(&ibtl_qp_cv, NULL, CV_DEFAULT, NULL); 146 147 ibtl_thread_init(); 148 149 return (rval); 150 } 151 152 153 /* 154 * The IBTF Module is never unloaded. Actually there is no need of this 155 * routine, but provided just in case. 156 */ 157 int 158 _fini(void) 159 { 160 int rval; 161 162 if ((rval = mod_remove(&modlinkage)) != 0) { 163 return (rval); 164 } 165 166 ibtl_thread_fini(); 167 168 mutex_destroy(&ibtl_clnt_list_mutex); 169 mutex_destroy(&ibtl_free_qp_mutex); 170 cv_destroy(&ibtl_close_hca_cv); 171 mutex_destroy(&ibtl_qp_mutex); 172 cv_destroy(&ibtl_qp_cv); 173 174 /* 175 * Stop Logging 176 */ 177 ibtl_logging_destroy(); 178 179 return (rval); 180 } 181 182 183 int 184 _info(struct modinfo *modinfop) 185 { 186 /* Return the Module Information. */ 187 return (mod_info(&modlinkage, modinfop)); 188 } 189 190 191 /* 192 * IBTF Client Registration Routines. 193 */ 194 195 /* 196 * Function: 197 * ibt_attach 198 * Input: 199 * modinfop - Client Module info structure. 200 * arg - usually client's dip 201 * clnt_private - client's private data pointer. 202 * Output: 203 * ibt_hdl_p - pointer to client's specific IBT handle, 204 * which is opaque to clients. 205 * Returns: 206 * IBT_SUCCESS 207 * IBT_INVALID_PARAM 208 * Called by: 209 * IBTF Client module during its attach() to register its instance 210 * to IBTF. 211 * Description: 212 * Registers the IBTF client module instance and returns an opaque 213 * handler to the client to be used for future calls to IBTF. 214 * Adds this client module instance to ibtl_clnt_list list. 215 * Records well-known async handlers. 216 */ 217 ibt_status_t 218 ibt_attach(ibt_clnt_modinfo_t *mod_infop, dev_info_t *arg, void *clnt_private, 219 ibt_clnt_hdl_t *ibt_hdl_p) 220 { 221 dev_info_t *pdip; 222 ibtl_clnt_t *clntp; 223 224 IBTF_DPRINTF_L3(ibtf, "ibt_attach(%p, %p, %p)", 225 mod_infop, arg, clnt_private); 226 227 if (mod_infop->mi_clnt_name == NULL) { 228 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 229 "IB client needs to specify its name"); 230 return (IBT_INVALID_PARAM); 231 } 232 233 /* 234 * Validate the Transport API version. 235 */ 236 if (mod_infop->mi_ibt_version != IBTI_V2) { 237 IBTF_DPRINTF_L1(ibtf, "ibt_attach: IB client '%s' has an " 238 "invalid IB TI Version '%d'", mod_infop->mi_clnt_name, 239 mod_infop->mi_ibt_version); 240 return (IBT_NOT_SUPPORTED); 241 } 242 243 if (mod_infop->mi_async_handler == NULL) { 244 IBTF_DPRINTF_L2(ibtf, "ibt_attach: Client '%s' has not\n" 245 " provided an Asynchronous Event Handler.\n" 246 " This will be required soon.", 247 mod_infop->mi_clnt_name); 248 } 249 250 /* 251 * Check out Client's Class information. If it is not of mgmt class, 252 * we expect 'arg' to be Not NULL and point to client driver's 253 * device info struct. 254 */ 255 if ((!IBT_CLNT_MGMT_CLASS(mod_infop->mi_clnt_class)) && 256 (arg == NULL)) { 257 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 258 "arg not set with driver's dip."); 259 return (IBT_INVALID_PARAM); 260 } 261 262 if (!IBT_CLNT_MGMT_CLASS(mod_infop->mi_clnt_class)) { 263 pdip = ddi_get_parent(arg); 264 if (pdip == NULL || 265 ibtl_ibnex_valid_hca_parent(pdip) != IBT_SUCCESS) { 266 IBTF_DPRINTF_L2(ibtf, "ibt_attach: " 267 "client %s is not a child of IB nexus driver.", 268 ddi_driver_name(arg)); 269 return (IBT_INVALID_PARAM); 270 } 271 } 272 273 mutex_enter(&ibtl_clnt_list_mutex); 274 if (mod_infop->mi_clnt_class == IBT_CM) { 275 if (ibtl_cm_async_handler != NULL) { 276 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 277 "CM is already attached."); 278 mutex_exit(&ibtl_clnt_list_mutex); 279 return (IBT_INVALID_PARAM); 280 } 281 ibtl_cm_async_handler = mod_infop->mi_async_handler; 282 ibtl_cm_clnt_private = clnt_private; 283 } else if (mod_infop->mi_clnt_class == IBT_DM) { 284 if (ibtl_dm_async_handler != NULL) { 285 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 286 "DM is already attached."); 287 mutex_exit(&ibtl_clnt_list_mutex); 288 return (IBT_INVALID_PARAM); 289 } 290 ibtl_dm_async_handler = mod_infop->mi_async_handler; 291 ibtl_dm_clnt_private = clnt_private; 292 } else if (mod_infop->mi_clnt_class == IBT_IBMA) { 293 if (ibtl_ibma_async_handler != NULL) { 294 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 295 "IBMF is already attached."); 296 mutex_exit(&ibtl_clnt_list_mutex); 297 return (IBT_INVALID_PARAM); 298 } 299 ibtl_ibma_async_handler = mod_infop->mi_async_handler; 300 ibtl_ibma_clnt_private = clnt_private; 301 } 302 303 /* Allocate the memory for per-client-device info structure */ 304 clntp = kmem_zalloc(sizeof (ibtl_clnt_t), KM_SLEEP); 305 306 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(clntp->clnt_modinfop, 307 clntp->clnt_dip, clntp->clnt_name, clntp->clnt_async_cnt, 308 clntp->clnt_private)) 309 /* Update the Client info structure */ 310 clntp->clnt_modinfop = mod_infop; /* IBT Client's Mod Info */ 311 clntp->clnt_private = clnt_private; /* IBT Client's private */ 312 clntp->clnt_dip = arg; /* IBT Client's dip */ 313 clntp->clnt_async_cnt = 0; 314 /* using a count of 7 below guarantees it is NULL terminated */ 315 (void) strncpy(clntp->clnt_name, mod_infop->mi_clnt_name, 7); 316 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(clntp->clnt_modinfop, 317 clntp->clnt_dip, clntp->clnt_name, clntp->clnt_async_cnt, 318 clntp->clnt_private)) 319 320 /* 321 * Update Client Device Instance List. 322 */ 323 clntp->clnt_list_link = ibtl_clnt_list; 324 ibtl_clnt_list = clntp; 325 mutex_exit(&ibtl_clnt_list_mutex); 326 327 /* 328 * The ibt_hdl_p is a opaque handle which is the address of 329 * ibt_clnt_t structure passed back to the clients. 330 * The client will pass on this handle in its future calls to IBTF. 331 */ 332 *ibt_hdl_p = clntp; 333 334 return (IBT_SUCCESS); 335 } 336 337 338 /* 339 * Function: 340 * ibt_detach 341 * Input: 342 * ibt_hdl - IBT Handle as returned during ibt_attach call. 343 * Output: 344 * none 345 * Returns: 346 * IBT_SUCCESS 347 * IBT_INVALID_PARAM. 348 * Called by: 349 * IBTF Client module during its detach() to de-register its instance 350 * from IBTF. 351 * Description: 352 * Deregisters the IBTF client module instance from the IBTF. 353 * All resources and any reference to this ibt_hdl will be removed. 354 */ 355 ibt_status_t 356 ibt_detach(ibt_clnt_hdl_t ibt_hdl) 357 { 358 ibtl_clnt_t **clntpp; 359 360 IBTF_DPRINTF_L3(ibtf, "ibt_detach(%p)", ibt_hdl); 361 362 mutex_enter(&ibtl_clnt_list_mutex); 363 clntpp = &ibtl_clnt_list; 364 for (; *clntpp != NULL; clntpp = &(*clntpp)->clnt_list_link) 365 if (*clntpp == ibt_hdl) 366 break; 367 if (*clntpp == NULL) { 368 IBTF_DPRINTF_L1(ibtf, "ibt_detach: Client @ %p Not Found", 369 ibt_hdl); 370 mutex_exit(&ibtl_clnt_list_mutex); 371 return (IBT_INVALID_PARAM); 372 } 373 374 /* 375 * Check out whether the client has freed all its resources. 376 * If not done, then fail the detach. 377 * 378 * viz. A client has to close all the HCA they have opened, 379 * i.e. the HCA List maintained for clients has to be empty. 380 * If this list is not empty, then the client has not performed 381 * complete clean-up, so fail the detach. 382 */ 383 if (ibt_hdl->clnt_hca_list != NULL) { 384 mutex_exit(&ibtl_clnt_list_mutex); 385 386 IBTF_DPRINTF_L2(ibtf, "ibt_detach: " 387 "ERROR: Client '%s' has not closed all of its HCAs", 388 ibt_hdl->clnt_modinfop->mi_clnt_name); 389 return (IBT_HCA_RESOURCES_NOT_FREED); 390 } 391 392 if (ibt_hdl->clnt_srv_cnt != 0) { 393 mutex_exit(&ibtl_clnt_list_mutex); 394 IBTF_DPRINTF_L2(ibtf, "ibt_detach: client '%s' still has " 395 "services or subnet_notices registered", 396 ibt_hdl->clnt_modinfop->mi_clnt_name); 397 return (IBT_HCA_RESOURCES_NOT_FREED); 398 } 399 400 /* 401 * Delete the entry of this module from the ibtl_clnt_list List. 402 */ 403 *clntpp = ibt_hdl->clnt_list_link; /* remove us */ 404 405 /* make sure asyncs complete before freeing */ 406 ibtl_free_clnt_async_check(ibt_hdl); 407 408 if (ibt_hdl->clnt_modinfop->mi_clnt_class == IBT_CM) { 409 ibtl_cm_async_handler = NULL; 410 ibtl_cm_clnt_private = NULL; 411 } else if (ibt_hdl->clnt_modinfop->mi_clnt_class == IBT_DM) { 412 ibtl_dm_async_handler = NULL; 413 ibtl_dm_clnt_private = NULL; 414 } else if (ibt_hdl->clnt_modinfop->mi_clnt_class == IBT_IBMA) { 415 ibtl_ibma_async_handler = NULL; 416 ibtl_ibma_clnt_private = NULL; 417 } 418 mutex_exit(&ibtl_clnt_list_mutex); 419 420 /* Free up the memory of per-client info struct. */ 421 kmem_free(ibt_hdl, sizeof (ibtl_clnt_t)); 422 423 return (IBT_SUCCESS); 424 } 425 426 static void 427 ibtl_set_ibhw_status() 428 { 429 ib_hw_status++; 430 } 431 432 static void 433 ibtl_clear_ibhw_status() 434 { 435 ib_hw_status--; 436 } 437 438 /* 439 * Function: 440 * ibc_init 441 * Input: 442 * modlp - Pointer to IBC client module linkage structure 443 * Output: 444 * None 445 * Returns: 446 * 0 always for now 447 * Called by: 448 * CI client calls IBTF during its _init() to register HCA with 449 * Solaris I/O framework. 450 * Description: 451 * Initializes the CI clients module linkage structure with 452 * default bus_ops structure 453 */ 454 int 455 ibc_init(struct modlinkage *modlp) 456 { 457 ibtl_ibnex_cb_args_t cb_args; 458 459 mutex_enter(&ibtl_clnt_list_mutex); 460 cb_args.cb_flag = IBTL_IBNEX_IBC_INIT; 461 cb_args.cb_modlp = modlp; 462 if (ibtl_ibnex_callback_routine) { 463 (void) ((*ibtl_ibnex_callback_routine)(&cb_args)); 464 } 465 mutex_exit(&ibtl_clnt_list_mutex); 466 return (0); 467 } 468 469 470 /* 471 * Function: 472 * ibc_fini 473 * Input: 474 * modlp - Pointer to IBC client module linkage structure 475 * Output: 476 * None 477 * Returns: 478 * None 479 * Called by: 480 * CI client calls IBTF during its _fini() to remove HCA with 481 * Solaris I/O framework. 482 * Description: 483 * Undo what is done during ibc_init 484 */ 485 void 486 ibc_fini(struct modlinkage *modlp) 487 { 488 ibtl_ibnex_cb_args_t cb_args; 489 490 mutex_enter(&ibtl_clnt_list_mutex); 491 cb_args.cb_flag = IBTL_IBNEX_IBC_FINI; 492 cb_args.cb_modlp = modlp; 493 if (ibtl_ibnex_callback_routine) { 494 (void) ((*ibtl_ibnex_callback_routine)(&cb_args)); 495 } 496 mutex_exit(&ibtl_clnt_list_mutex); 497 } 498 499 /* 500 * Function: 501 * ibc_attach 502 * Input: 503 * info_p - IBC HCA Info. 504 * Output: 505 * ibc_hdl_p - IBC Client's HCA Handle. 506 * Returns: 507 * IBC_SUCCESS 508 * IBC_FAILURE 509 * Called by: 510 * CI calls IBTF during its attach() to register HCA Device with IBTF. 511 * Description: 512 * Registers the presence of HCA device by providing the HCA device info 513 * structure and provides an opaque HCA handler for future calls to this 514 * HCA device. 515 */ 516 ibc_status_t 517 ibc_attach(ibc_clnt_hdl_t *ibc_hdl_p, ibc_hca_info_t *info_p) 518 { 519 ibtl_hca_devinfo_t *hca_devp; 520 uint_t nports; 521 ibt_status_t status; 522 523 IBTF_DPRINTF_L2(ibtf, "ibc_attach(%p, %p)", ibc_hdl_p, info_p); 524 525 /* Validate the Transport API version */ 526 if (info_p->hca_ci_vers != IBCI_V2) { 527 IBTF_DPRINTF_L1(ibtf, "ibc_attach: Invalid IB CI Version '%d'", 528 info_p->hca_ci_vers); 529 return (IBC_FAILURE); 530 } 531 532 if (info_p->hca_attr == NULL) { 533 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 534 "HCA Attributes must be specified."); 535 return (IBC_FAILURE); 536 } 537 538 nports = info_p->hca_attr->hca_nports; 539 if (nports == 0) { 540 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 541 "Number of ports must be valid"); 542 return (IBC_FAILURE); 543 } 544 545 if (info_p->hca_attr->hca_max_port_pkey_tbl_sz == 0) { 546 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 547 "Number of Partitions must be at least 1"); 548 return (IBC_FAILURE); 549 } 550 551 if ((info_p->hca_attr->hca_flags & IBT_HCA_CURRENT_QP_STATE) == 0) { 552 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 553 "HCA driver must support QP current state checking"); 554 return (IBC_FAILURE); 555 } 556 557 if ((info_p->hca_attr->hca_flags & IBT_HCA_PORT_UP) == 0) { 558 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 559 "HCA driver must support PORT_UP async events"); 560 return (IBC_FAILURE); 561 } 562 563 /* 564 * Install IB nexus driver (if not installed already) 565 */ 566 ibtl_set_ibhw_status(); 567 if (ndi_devi_config_vhci("ib", 0) == NULL) { 568 IBTF_DPRINTF_L2(ibtf, "ibc_attach: IB nexus attach failed"); 569 ibtl_clear_ibhw_status(); 570 return (IBC_FAILURE); 571 } 572 573 ibtl_thread_init2(); 574 575 /* Allocate the memory for per-client info structure */ 576 hca_devp = kmem_zalloc(sizeof (ibtl_hca_devinfo_t) + 577 (nports - 1) * sizeof (ibtl_async_port_status_t), KM_SLEEP); 578 579 mutex_enter(&ibtl_clnt_list_mutex); 580 581 /* Update HCA dev info structure */ 582 hca_devp->hd_ibc_hca_hdl = info_p->hca_handle; 583 hca_devp->hd_ibc_ops = info_p->hca_ops; 584 hca_devp->hd_hca_attr = info_p->hca_attr; 585 hca_devp->hd_hca_dip = info_p->hca_dip; 586 587 status = ibtl_init_hca_portinfo(hca_devp); 588 if (status != IBT_SUCCESS) { 589 mutex_exit(&ibtl_clnt_list_mutex); 590 IBTF_DPRINTF_L1(ibtf, "ibc_attach: call to ibc_query_hca_ports " 591 "failed: status = %d", status); 592 kmem_free(hca_devp, sizeof (ibtl_hca_devinfo_t) + 593 (nports - 1) * sizeof (ibtl_async_port_status_t)); 594 return (IBC_FAILURE); 595 } 596 597 /* Register the with MPxIO as PHCI */ 598 if (mdi_phci_register(MDI_HCI_CLASS_IB, info_p->hca_dip, 0) != 599 MDI_SUCCESS) { 600 mutex_exit(&ibtl_clnt_list_mutex); 601 IBTF_DPRINTF_L1(ibtf, "ibc_attach: MPxIO register failed"); 602 kmem_free(hca_devp, sizeof (ibtl_hca_devinfo_t) + 603 (nports - 1) * sizeof (ibtl_async_port_status_t)); 604 return (IBC_FAILURE); 605 } 606 607 /* Initialize the Client List for this HCA. */ 608 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; 609 610 /* lock out asyncs until after we announce the new HCA */ 611 hca_devp->hd_async_busy = 1; 612 613 cv_init(&hca_devp->hd_async_task_cv, NULL, CV_DEFAULT, NULL); 614 cv_init(&hca_devp->hd_async_busy_cv, NULL, CV_DEFAULT, NULL); 615 616 /* init portinfo locking variables */ 617 hca_devp->hd_portinfo_locked_port = 0; 618 cv_init(&hca_devp->hd_portinfo_cv, NULL, CV_DEFAULT, NULL); 619 620 mutex_exit(&ibtl_clnt_list_mutex); 621 622 /* 623 * The ibc_hdl_p points to an opaque handle which is the address 624 * of ibt_hca_devinfo_t structure passed back to the CI. 625 * The CI will pass on this handle in its future upcalls to IBTF. 626 */ 627 *ibc_hdl_p = hca_devp; 628 629 return (IBC_SUCCESS); 630 } 631 632 633 /* 634 * Function: 635 * ibc_post_attach 636 * Input: 637 * ibc_hdl - IBC Client's HCA Handle. 638 * Returns: 639 * none 640 * Called by: 641 * CI calls IBTF during its attach() after a successful ibc_attach(). 642 * Description: 643 * Announces to all known clients the existence of this HCA (by GUID). 644 */ 645 void 646 ibc_post_attach(ibc_clnt_hdl_t ibc_hdl) 647 { 648 IBTF_DPRINTF_L2(ibtf, "ibc_post_attach(%p)", ibc_hdl); 649 650 /* 651 * Update the HCA Device List. 652 */ 653 mutex_enter(&ibtl_clnt_list_mutex); 654 ibc_hdl->hd_hca_dev_link = ibtl_hca_list; 655 ibtl_hca_list = ibc_hdl; 656 mutex_exit(&ibtl_clnt_list_mutex); 657 658 /* notify all IBT Client Device Instances of the new HCA Device */ 659 ibtl_announce_new_hca(ibc_hdl); 660 } 661 662 663 /* 664 * Function: 665 * ibc_pre_detach 666 * Input: 667 * ibc_clnt_hdl - IBC HCA Handle as returned during ibc_attach call. 668 * cmd - DDI_DETACH/DDI_SUSPEND command. 669 * Output: 670 * none 671 * Returns: 672 * IBC_SUCCESS 673 * IBC_FAILURE. 674 * Called by: 675 * CI to try to get all IBTF clients to close the HCA device. 676 * Description: 677 * Attempts to deregister the HCA device entry from the IBTF. 678 * If all resources are freed by the IBTF clients and this HCA 679 * is closed, then IBC_SUCCESS is returned. 680 */ 681 ibc_status_t 682 ibc_pre_detach(ibc_clnt_hdl_t hca_devp, ddi_detach_cmd_t cmd) 683 { 684 ibtl_hca_devinfo_t **hcapp, *hcap; 685 686 IBTF_DPRINTF_L2(ibtf, "ibc_pre_detach(%p, 0x%x)", hca_devp, cmd); 687 688 /* 689 * Return failure, if command is not DDI_DETACH 690 */ 691 switch (cmd) { 692 case DDI_DETACH: 693 break; 694 default: 695 return (IBC_FAILURE); /* TBD: DDI_FAILURE */ 696 } 697 698 /* Make sure this HCA is on the HCA Device List. */ 699 mutex_enter(&ibtl_clnt_list_mutex); 700 hcap = ibtl_hca_list; 701 while (hcap != NULL) { 702 if (hcap == hca_devp) 703 break; 704 hcap = hcap->hd_hca_dev_link; 705 } 706 if (hcap == NULL) { 707 mutex_exit(&ibtl_clnt_list_mutex); 708 return (IBC_FAILURE); 709 } 710 711 /* 712 * Initially set the state to "Detaching". 713 */ 714 hca_devp->hd_state = IBTL_HCA_DEV_DETACHING; 715 716 /* 717 * Try to detach all IBTI clients, and continue only if all 718 * of the detaches succeed. 719 */ 720 if (ibtl_detach_all_clients(hca_devp)) { 721 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; /* fix hd_state */ 722 mutex_exit(&ibtl_clnt_list_mutex); 723 724 return (IBC_FAILURE); 725 } 726 727 /* 728 * Check to see if all clients closed this HCA, or not. 729 * We only succeed if all clients cooperated. 730 */ 731 if (hca_devp->hd_clnt_list != NULL) { 732 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; 733 mutex_exit(&ibtl_clnt_list_mutex); 734 IBTF_DPRINTF_L2(ibtf, "ibc_pre_detach: HCA still has attached " 735 "clients"); 736 return (IBC_FAILURE); 737 } 738 739 /* 740 * mark this device as detached 741 */ 742 hca_devp->hd_state = IBTL_HCA_DEV_DETACHED; 743 744 /* Delete the entry for this hca_devp from hca_head_list */ 745 hcapp = &ibtl_hca_list; 746 while (*hcapp != NULL) { 747 if (*hcapp == hca_devp) 748 break; 749 hcapp = &(*hcapp)->hd_hca_dev_link; 750 } 751 752 if (mdi_phci_unregister(hca_devp->hd_hca_dip, 0) != MDI_SUCCESS) { 753 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; /* fix hd_state */ 754 mutex_exit(&ibtl_clnt_list_mutex); 755 IBTF_DPRINTF_L1(ibtf, "ibc_pre_detach: PHCI unregister failed"); 756 return (IBC_FAILURE); 757 } 758 759 if (*hcapp == NULL) { 760 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; /* fix hd_state */ 761 mutex_exit(&ibtl_clnt_list_mutex); 762 IBTF_DPRINTF_L1(ibtf, "ibc_pre_detach: HCA not attached"); 763 return (IBC_FAILURE); 764 } 765 *hcapp = hca_devp->hd_hca_dev_link; 766 ibtl_fast_gid_cache_valid = B_FALSE; /* invalidate fast_gid_cache */ 767 mutex_exit(&ibtl_clnt_list_mutex); 768 769 return (IBC_SUCCESS); 770 } 771 772 /* 773 * Function: 774 * ibc_detach 775 * Input: 776 * ibc_clnt_hdl - IBC HCA Handle as returned during ibc_attach call. 777 * Output: 778 * none 779 * Returns: 780 * None 781 * Called by: 782 * CI to detach the HCA device from IBTF. 783 * Description: 784 * Do the second step of detaching the HCA, which is required 785 * after a successful ibc_pre_detach. 786 */ 787 void 788 ibc_detach(ibc_clnt_hdl_t hca_devp) 789 { 790 IBTF_DPRINTF_L2(ibtf, "ibc_detach(%p)", hca_devp); 791 792 mutex_enter(&ibtl_clnt_list_mutex); 793 if (hca_devp->hd_state != IBTL_HCA_DEV_DETACHED) { 794 mutex_exit(&ibtl_clnt_list_mutex); 795 IBTF_DPRINTF_L0(ibtf, "ibc_detach: HCA has not successfully " 796 "pre-detached"); 797 return; 798 } 799 800 cv_destroy(&hca_devp->hd_async_task_cv); 801 cv_destroy(&hca_devp->hd_async_busy_cv); 802 cv_destroy(&hca_devp->hd_portinfo_cv); 803 804 kmem_free(hca_devp->hd_portinfop, hca_devp->hd_portinfo_len); 805 mutex_exit(&ibtl_clnt_list_mutex); 806 807 /* Free up the memory of per-client info struct */ 808 kmem_free(hca_devp, sizeof (ibtl_hca_devinfo_t) + 809 (hca_devp->hd_hca_attr->hca_nports - 1) * 810 sizeof (ibtl_async_port_status_t)); 811 ibtl_clear_ibhw_status(); 812 } 813 814 /* 815 * Function: 816 * ibt_ci_data_in() 817 * 818 * Input: 819 * hca_hdl HCA Handle. 820 * flags IBT_COMPLETE_ALLOC - Finish a deferred alloc. 821 * object Identifies the type object pointed to by 822 * ibt_object_handle. 823 * 824 * ibt_object_handle The handle of the object to be associated with 825 * the data in/out 826 * 827 * data_p Pointer data passed in to the CI. The buffer 828 * should be allocated by the caller. 829 * 830 * data_sz The size of the buffer pointed to by 831 * data_p. 832 * Output: 833 * 834 * Returns: 835 * IBT_SUCCESS 836 * IBT_NOT_SUPPORTED Feature not supported. 837 * IBT_INVALID_PARAM Invalid object type specified. 838 * IBT_HCA_HDL_INVALID 839 * IBT_AH_HDL_INVALID/IBT_UD_DEST_HDL_INVALID 840 * IBT_CHAN_HDL_INVALID/IBT_QP_HDL_INVALID 841 * IBT_CQ_HDL_INVALID 842 * IBT_EEC_HDL_INVALID 843 * IBT_RDD_HDL_INVALID 844 * IBT_MW_HDL_INVALID 845 * IBT_PD_HDL_INVALID 846 * IBT_SRQ_HDL_INVALID 847 * 848 * Description: 849 * Exchange CI private data for the specified CI object. 850 */ 851 ibt_status_t 852 ibt_ci_data_in(ibt_hca_hdl_t hca, ibt_ci_data_flags_t flags, 853 ibt_object_type_t object, void *ibt_object_handle, void *data_p, 854 size_t data_sz) 855 { 856 ibt_status_t retval; 857 void *ci_obj_hdl; 858 859 IBTF_DPRINTF_L3(ibtf, "ibt_ci_data_in(%p, %x, %d, %p, %p, %d)", 860 hca, flags, object, ibt_object_handle, data_p, data_sz); 861 862 switch (object) { 863 case IBT_HDL_HCA: 864 ci_obj_hdl = (void *) 865 (IBTL_HCA2CIHCA(((ibt_hca_hdl_t)ibt_object_handle))); 866 break; 867 868 case IBT_HDL_CHANNEL: 869 ci_obj_hdl = (void *) 870 (IBTL_CHAN2CIQP(((ibt_channel_hdl_t)ibt_object_handle))); 871 break; 872 873 case IBT_HDL_CQ: 874 ci_obj_hdl = (void *) 875 (((ibt_cq_hdl_t)(ibt_object_handle))->cq_ibc_cq_hdl); 876 break; 877 878 case IBT_HDL_EEC: 879 ci_obj_hdl = (void *) 880 (((ibt_eec_hdl_t)(ibt_object_handle))->eec_ibc_eec_hdl); 881 break; 882 883 case IBT_HDL_UD_DEST: 884 ci_obj_hdl = (void *) 885 (((ibt_ud_dest_hdl_t)(ibt_object_handle))->ud_ah); 886 break; 887 888 case IBT_HDL_SRQ: 889 ci_obj_hdl = (void *) 890 (((ibt_srq_hdl_t)(ibt_object_handle))->srq_ibc_srq_hdl); 891 break; 892 893 default: 894 ci_obj_hdl = ibt_object_handle; 895 break; 896 } 897 898 retval = (IBTL_HCA2CIHCAOPS_P(hca)->ibc_ci_data_in)(IBTL_HCA2CIHCA(hca), 899 flags, object, ci_obj_hdl, data_p, data_sz); 900 901 if (retval != IBT_SUCCESS) { 902 IBTF_DPRINTF_L2(ibtf, "ibt_ci_data_in: Failed : %d", retval); 903 } 904 return (retval); 905 } 906 907 /* 908 * Function: 909 * ibt_ci_data_out() 910 * 911 * Input: 912 * hca_hdl HCA Handle. 913 * flags IBT_COMPLETE_ALLOC - Finish a deferred alloc. 914 * object Identifies the type object pointed to by 915 * ibt_object_handle. 916 * 917 * ibt_object_handle The handle of the object to be associated with 918 * the data in/out 919 * 920 * data_p Pointer to a buffer in which to return the CI 921 * private data. The buffer should be allocated 922 * by the caller. 923 * 924 * data_sz The size of the buffer pointed to by 925 * data_p. 926 * Output: 927 * 928 * Returns: 929 * IBT_SUCCESS 930 * IBT_NOT_SUPPORTED Feature not supported. 931 * IBT_INSUFF_RESOURCE The buffer pointed to by data_p was too 932 * small to hold the data. 933 * IBT_INVALID_PARAM Invalid object type specified. 934 * IBT_HCA_HDL_INVALID 935 * IBT_AH_HDL_INVALID/IBT_UD_DEST_HDL_INVALID 936 * IBT_CHAN_HDL_INVALID/IBT_QP_HDL_INVALID 937 * IBT_CQ_HDL_INVALID 938 * IBT_EEC_HDL_INVALID 939 * IBT_RDD_HDL_INVALID 940 * IBT_MW_HDL_INVALID 941 * IBT_PD_HDL_INVALID 942 * IBT_SRQ_HDL_INVALID 943 * 944 * Description: 945 * Exchange CI private data for the specified CI object. 946 */ 947 ibt_status_t 948 ibt_ci_data_out(ibt_hca_hdl_t hca, ibt_ci_data_flags_t flags, 949 ibt_object_type_t object, void *ibt_object_handle, void *data_p, 950 size_t data_sz) 951 { 952 ibt_status_t retval; 953 void *ci_obj_hdl; 954 955 IBTF_DPRINTF_L3(ibtf, "ibt_ci_data_out(%p, %x, %d, %p, %p, %d)", 956 hca, flags, object, ibt_object_handle, data_p, data_sz); 957 958 switch (object) { 959 case IBT_HDL_HCA: 960 ci_obj_hdl = (void *) 961 (IBTL_HCA2CIHCA(((ibt_hca_hdl_t)ibt_object_handle))); 962 break; 963 964 case IBT_HDL_CHANNEL: 965 ci_obj_hdl = (void *) 966 (IBTL_CHAN2CIQP(((ibt_channel_hdl_t)ibt_object_handle))); 967 break; 968 969 case IBT_HDL_CQ: 970 ci_obj_hdl = (void *) 971 (((ibt_cq_hdl_t)(ibt_object_handle))->cq_ibc_cq_hdl); 972 break; 973 974 case IBT_HDL_EEC: 975 ci_obj_hdl = (void *) 976 (((ibt_eec_hdl_t)(ibt_object_handle))->eec_ibc_eec_hdl); 977 break; 978 979 case IBT_HDL_UD_DEST: 980 ci_obj_hdl = (void *) 981 (((ibt_ud_dest_hdl_t)(ibt_object_handle))->ud_ah); 982 break; 983 984 case IBT_HDL_SRQ: 985 ci_obj_hdl = (void *) 986 (((ibt_srq_hdl_t)(ibt_object_handle))->srq_ibc_srq_hdl); 987 break; 988 989 default: 990 ci_obj_hdl = ibt_object_handle; 991 break; 992 } 993 994 retval = (IBTL_HCA2CIHCAOPS_P(hca)->ibc_ci_data_out) 995 (IBTL_HCA2CIHCA(hca), flags, object, ci_obj_hdl, data_p, data_sz); 996 997 if (retval != IBT_SUCCESS) { 998 IBTF_DPRINTF_L2(ibtf, "ibt_ci_data_out: Failed : %d", retval); 999 } 1000 return (retval); 1001 } 1002 1003 1004 /* 1005 * FMA Support functions. 1006 */ 1007 1008 #define IBTL_ENA_MASK 0xC0000000 1009 #define IBTL_ENA_POSSIBLE 0x80000000 1010 #define IBTL_TYPE_SHIFT 27 1011 1012 /* 1013 * Function: 1014 * ibt_get_module_failure() 1015 * 1016 * Input: 1017 * type Identifies the failing IB module. 1018 * ena '0' or the data for Fault Management 1019 * Architecture (ENA). 1020 * 1021 * Returns: 1022 * status Special IB failure status. 1023 * 1024 * Description: 1025 * XXX Just stubbed out to return failures with no data for Fault 1026 * Management Architecture (ENAs) at the moment XXX 1027 */ 1028 ibt_status_t 1029 ibt_get_module_failure(ibt_failure_type_t type, uint64_t ena) 1030 { 1031 ibt_status_t ret; 1032 1033 IBTF_DPRINTF_L3(ibtf, "ibt_get_module_failure(%d, 0x%llX)", type, ena); 1034 1035 switch (type) { 1036 case IBT_FAILURE_CI: 1037 case IBT_FAILURE_IBMF: 1038 case IBT_FAILURE_IBCM: 1039 case IBT_FAILURE_IBDM: 1040 case IBT_FAILURE_IBTL: 1041 case IBT_FAILURE_IBSM: 1042 ret = IBTL_ENA_POSSIBLE | (type << IBTL_TYPE_SHIFT); 1043 break; 1044 default: 1045 ret = IBT_FAILURE; 1046 } 1047 IBTF_DPRINTF_L3(ibtf, "ibt_get_module_failure: ret = 0x%lX", ret); 1048 return (ret); 1049 } 1050 1051 1052 /* 1053 * Function: 1054 * ibc_get_ci_failure() 1055 * 1056 * Input: 1057 * ena '0' or the data for Fault Management 1058 * Architecture (ENA). 1059 * 1060 * Returns: 1061 * status Special CI failure status. 1062 * 1063 * Description: 1064 * Just use the function above to do the job. 1065 */ 1066 ibt_status_t 1067 ibc_get_ci_failure(uint64_t ena) 1068 { 1069 return (ibt_get_module_failure(IBT_FAILURE_CI, ena)); 1070 } 1071 1072 1073 /* 1074 * ibt_check_failure() 1075 * Function to test for special case failures. 1076 * 1077 * status An ibt_status_t returned from an IBTF function call. 1078 * 1079 * reserved_p NULL, or a pointer to where we store the data for 1080 * Fault Management Architecture (ENA). 1081 * 1082 * Description: 1083 * XXX Still need to determine the data for Fault Management Architecture 1084 * (ENA), using 0 for now XXX 1085 */ 1086 ibt_failure_type_t 1087 ibt_check_failure(ibt_status_t status, uint64_t *reserved_p) 1088 { 1089 ibt_failure_type_t type; 1090 1091 IBTF_DPRINTF_L3(ibtf, "ibt_check_failure(%X)", status); 1092 1093 if ((status & IBTL_ENA_MASK) == IBTL_ENA_POSSIBLE) { 1094 type = status & ~IBTL_ENA_POSSIBLE >> IBTL_TYPE_SHIFT; 1095 1096 /* XXX Need more work here... */ 1097 if (reserved_p != NULL) 1098 *reserved_p = 0; 1099 } else { 1100 type = IBT_FAILURE_STANDARD; 1101 if (reserved_p != NULL) 1102 *reserved_p = 0; /* No FMA Data Available. */ 1103 } 1104 IBTF_DPRINTF_L3(ibtf, "ibt_check_failure: type = 0x%X", type); 1105 return (type); 1106 } 1107