1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 /* 26 * ibtl_impl.c 27 * 28 * This file contains the IBTF module's initialization and 29 * IBTF Clients/Modules registration routines. 30 */ 31 32 #include <sys/modctl.h> 33 #include <sys/sunndi.h> 34 #include <sys/ib/ibtl/impl/ibtl.h> 35 #include <sys/ib/ibtl/impl/ibtl_ibnex.h> 36 37 /* 38 * Globals. 39 */ 40 static char ibtf[] = "ibtl_impl"; 41 42 extern ibtl_ibnex_callback_t ibtl_ibnex_callback_routine; 43 44 /* 45 * ibtl_clnt_list: 46 * 47 * Head of the list of IBT Client Instances. The IBT Client List 48 * is modified by IBTF on an IBT client's ibt_attach/ibt_detach call. 49 * 50 * ibtl_hca_list: 51 * 52 * Head of the list of HCA devices. The HCA List is modified by IBTF on 53 * a CI's ibc_attach/ibc_detach call. 54 * The datap of the list elements points to an ibtl_hca_devinfo_s 55 * structure. 56 * 57 * (ibc_attach) 58 * ibtl_hca_list -> ibtl_hca_devinfo_t--> ... -->ibtl_hca_devinfo_t 59 * [per-hca_dev] | ^ {nth HCA Dev} 60 * | | 61 * | ibtl_hca_t (ibt_open_hca) 62 * | ^ | 63 * | | | 64 * v | V 65 * ibtl_clnt_list -> ibtl_clnt_t--> ...--> {n'th Module} 66 * [per-client_instance] (ibt_attach) 67 * 68 */ 69 70 /* Global List of IBT Client Instances, and associated mutex. */ 71 struct ibtl_clnt_s *ibtl_clnt_list = NULL; 72 kmutex_t ibtl_clnt_list_mutex; 73 74 /* Lock for the race between the client and CM to free QPs. */ 75 kmutex_t ibtl_free_qp_mutex; 76 77 /* Lock for the race between the client closing the HCA and QPN being freed. */ 78 kcondvar_t ibtl_close_hca_cv; 79 80 /* Global List of HCA Devices, and associated mutex. */ 81 struct ibtl_hca_devinfo_s *ibtl_hca_list = NULL; 82 83 /* Well-known async handlers and associated client private. */ 84 ibt_async_handler_t ibtl_cm_async_handler; 85 ibt_async_handler_t ibtl_dm_async_handler; 86 ibt_async_handler_t ibtl_ibma_async_handler; 87 void *ibtl_cm_clnt_private; 88 void *ibtl_dm_clnt_private; 89 void *ibtl_ibma_clnt_private; 90 91 extern int ib_hw_status; 92 _NOTE(SCHEME_PROTECTS_DATA("Scheme protects data", ib_hw_status)) 93 94 /* 95 * Misc Module Declarations. 96 */ 97 extern struct mod_ops mod_miscops; 98 static struct modlmisc modlmisc = { 99 &mod_miscops, /* Type of module - misc. */ 100 "IB Transport Layer" /* Name of the Module. */ 101 }; 102 103 static struct modlinkage modlinkage = { 104 MODREV_1, (void *)&modlmisc, NULL 105 }; 106 107 static void ibtl_kstat_init(ibtl_hca_devinfo_t *); 108 static void ibtl_kstat_fini(ibtl_hca_devinfo_t *); 109 static void ibtl_kstat_stats_create(ibtl_hca_devinfo_t *, uint_t); 110 static void ibtl_kstat_pkeys_create(ibtl_hca_devinfo_t *, uint_t); 111 112 extern kmutex_t ibtl_part_attr_mutex; 113 114 /* 115 * IBTF Loadable Module Routines. 116 */ 117 118 int 119 _init(void) 120 { 121 int rval; 122 123 if ((rval = mod_install(&modlinkage)) != 0) 124 return (rval); 125 126 /* 127 * initialize IBTL ib2usec table 128 */ 129 ibtl_ib2usec_init(); 130 131 /* 132 * Initialize Logging 133 */ 134 ibtl_logging_initialization(); 135 136 /* 137 * Initialize the Alloc QP States. 138 */ 139 ibtl_init_cep_states(); 140 141 /* 142 * Initialize all Global Link Lists. 143 */ 144 mutex_init(&ibtl_clnt_list_mutex, NULL, MUTEX_DEFAULT, NULL); 145 mutex_init(&ibtl_free_qp_mutex, NULL, MUTEX_DEFAULT, NULL); 146 cv_init(&ibtl_close_hca_cv, NULL, CV_DEFAULT, NULL); 147 148 mutex_init(&ibtl_qp_mutex, NULL, MUTEX_DEFAULT, NULL); 149 cv_init(&ibtl_qp_cv, NULL, CV_DEFAULT, NULL); 150 151 mutex_init(&ibtl_part_attr_mutex, NULL, MUTEX_DEFAULT, NULL); 152 153 ibtl_thread_init(); 154 155 return (rval); 156 } 157 158 159 /* 160 * The IBTF Module is never unloaded. Actually there is no need of this 161 * routine, but provided just in case. 162 */ 163 int 164 _fini(void) 165 { 166 int rval; 167 168 if ((rval = mod_remove(&modlinkage)) != 0) { 169 return (rval); 170 } 171 172 ibtl_thread_fini(); 173 174 mutex_destroy(&ibtl_clnt_list_mutex); 175 mutex_destroy(&ibtl_free_qp_mutex); 176 cv_destroy(&ibtl_close_hca_cv); 177 mutex_destroy(&ibtl_qp_mutex); 178 cv_destroy(&ibtl_qp_cv); 179 mutex_destroy(&ibtl_part_attr_mutex); 180 181 /* 182 * Stop Logging 183 */ 184 ibtl_logging_destroy(); 185 186 return (rval); 187 } 188 189 190 int 191 _info(struct modinfo *modinfop) 192 { 193 /* Return the Module Information. */ 194 return (mod_info(&modlinkage, modinfop)); 195 } 196 197 198 /* 199 * IBTF Client Registration Routines. 200 */ 201 202 /* 203 * Function: 204 * ibt_attach 205 * Input: 206 * modinfop - Client Module info structure. 207 * arg - usually client's dip 208 * clnt_private - client's private data pointer. 209 * Output: 210 * ibt_hdl_p - pointer to client's specific IBT handle, 211 * which is opaque to clients. 212 * Returns: 213 * IBT_SUCCESS 214 * IBT_INVALID_PARAM 215 * Called by: 216 * IBTF Client module during its attach() to register its instance 217 * to IBTF. 218 * Description: 219 * Registers the IBTF client module instance and returns an opaque 220 * handler to the client to be used for future calls to IBTF. 221 * Adds this client module instance to ibtl_clnt_list list. 222 * Records well-known async handlers. 223 */ 224 ibt_status_t 225 ibt_attach(ibt_clnt_modinfo_t *mod_infop, dev_info_t *arg, void *clnt_private, 226 ibt_clnt_hdl_t *ibt_hdl_p) 227 { 228 dev_info_t *pdip; 229 ibtl_clnt_t *clntp; 230 231 IBTF_DPRINTF_L3(ibtf, "ibt_attach(%p, %p, %p)", 232 mod_infop, arg, clnt_private); 233 234 if (mod_infop->mi_clnt_name == NULL) { 235 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 236 "IB client needs to specify its name"); 237 return (IBT_INVALID_PARAM); 238 } 239 240 /* 241 * Validate the Transport API version. 242 */ 243 if (mod_infop->mi_ibt_version != IBTI_V_CURR) { 244 IBTF_DPRINTF_L1(ibtf, "ibt_attach: IB client '%s' has an " 245 "invalid IB TI Version '%d'", mod_infop->mi_clnt_name, 246 mod_infop->mi_ibt_version); 247 return (IBT_NOT_SUPPORTED); 248 } 249 250 if (mod_infop->mi_async_handler == NULL) { 251 IBTF_DPRINTF_L2(ibtf, "ibt_attach: Client '%s' has not\n" 252 " provided an Asynchronous Event Handler.\n" 253 " This will be required soon.", 254 mod_infop->mi_clnt_name); 255 } 256 257 /* 258 * Check out Client's Class information. If it is not of mgmt class, 259 * we expect 'arg' to be Not NULL and point to client driver's 260 * device info struct. 261 */ 262 if ((!IBT_MISCMOD_CLIENTS(mod_infop->mi_clnt_class)) && 263 (arg == NULL)) { 264 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 265 "arg not set with driver's dip."); 266 return (IBT_INVALID_PARAM); 267 } 268 269 if (!IBT_MISCMOD_CLIENTS(mod_infop->mi_clnt_class)) { 270 pdip = ddi_get_parent(arg); 271 if (pdip == NULL || 272 ibtl_ibnex_valid_hca_parent(pdip) != IBT_SUCCESS) { 273 IBTF_DPRINTF_L2(ibtf, "ibt_attach: " 274 "client %s is not a child of IB nexus driver.", 275 ddi_driver_name(arg)); 276 return (IBT_INVALID_PARAM); 277 } 278 } 279 280 mutex_enter(&ibtl_clnt_list_mutex); 281 if (mod_infop->mi_clnt_class == IBT_CM) { 282 if (ibtl_cm_async_handler != NULL) { 283 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 284 "CM is already attached."); 285 mutex_exit(&ibtl_clnt_list_mutex); 286 return (IBT_INVALID_PARAM); 287 } 288 ibtl_cm_async_handler = mod_infop->mi_async_handler; 289 ibtl_cm_clnt_private = clnt_private; 290 } else if (mod_infop->mi_clnt_class == IBT_DM) { 291 if (ibtl_dm_async_handler != NULL) { 292 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 293 "DM is already attached."); 294 mutex_exit(&ibtl_clnt_list_mutex); 295 return (IBT_INVALID_PARAM); 296 } 297 ibtl_dm_async_handler = mod_infop->mi_async_handler; 298 ibtl_dm_clnt_private = clnt_private; 299 } else if (mod_infop->mi_clnt_class == IBT_IBMA) { 300 if (ibtl_ibma_async_handler != NULL) { 301 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 302 "IBMF is already attached."); 303 mutex_exit(&ibtl_clnt_list_mutex); 304 return (IBT_INVALID_PARAM); 305 } 306 ibtl_ibma_async_handler = mod_infop->mi_async_handler; 307 ibtl_ibma_clnt_private = clnt_private; 308 } 309 310 /* Allocate the memory for per-client-device info structure */ 311 clntp = kmem_zalloc(sizeof (ibtl_clnt_t), KM_SLEEP); 312 313 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(clntp->clnt_modinfop, 314 clntp->clnt_dip, clntp->clnt_name, clntp->clnt_async_cnt, 315 clntp->clnt_private)) 316 /* Update the Client info structure */ 317 clntp->clnt_modinfop = mod_infop; /* IBT Client's Mod Info */ 318 clntp->clnt_private = clnt_private; /* IBT Client's private */ 319 clntp->clnt_dip = arg; /* IBT Client's dip */ 320 clntp->clnt_async_cnt = 0; 321 /* using a count of 7 below guarantees it is NULL terminated */ 322 (void) strncpy(clntp->clnt_name, mod_infop->mi_clnt_name, 7); 323 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(clntp->clnt_modinfop, 324 clntp->clnt_dip, clntp->clnt_name, clntp->clnt_async_cnt, 325 clntp->clnt_private)) 326 327 /* 328 * Update Client Device Instance List. 329 */ 330 clntp->clnt_list_link = ibtl_clnt_list; 331 ibtl_clnt_list = clntp; 332 mutex_exit(&ibtl_clnt_list_mutex); 333 334 /* 335 * The ibt_hdl_p is a opaque handle which is the address of 336 * ibt_clnt_t structure passed back to the clients. 337 * The client will pass on this handle in its future calls to IBTF. 338 */ 339 *ibt_hdl_p = clntp; 340 341 return (IBT_SUCCESS); 342 } 343 344 345 /* 346 * Function: 347 * ibt_detach 348 * Input: 349 * ibt_hdl - IBT Handle as returned during ibt_attach call. 350 * Output: 351 * none 352 * Returns: 353 * IBT_SUCCESS 354 * IBT_INVALID_PARAM. 355 * Called by: 356 * IBTF Client module during its detach() to de-register its instance 357 * from IBTF. 358 * Description: 359 * Deregisters the IBTF client module instance from the IBTF. 360 * All resources and any reference to this ibt_hdl will be removed. 361 */ 362 ibt_status_t 363 ibt_detach(ibt_clnt_hdl_t ibt_hdl) 364 { 365 ibtl_clnt_t **clntpp; 366 367 IBTF_DPRINTF_L3(ibtf, "ibt_detach(%p)", ibt_hdl); 368 369 mutex_enter(&ibtl_clnt_list_mutex); 370 clntpp = &ibtl_clnt_list; 371 for (; *clntpp != NULL; clntpp = &(*clntpp)->clnt_list_link) 372 if (*clntpp == ibt_hdl) 373 break; 374 if (*clntpp == NULL) { 375 IBTF_DPRINTF_L1(ibtf, "ibt_detach: Client @ %p Not Found", 376 ibt_hdl); 377 mutex_exit(&ibtl_clnt_list_mutex); 378 return (IBT_INVALID_PARAM); 379 } 380 381 /* 382 * Check out whether the client has freed all its resources. 383 * If not done, then fail the detach. 384 * 385 * viz. A client has to close all the HCA they have opened, 386 * i.e. the HCA List maintained for clients has to be empty. 387 * If this list is not empty, then the client has not performed 388 * complete clean-up, so fail the detach. 389 */ 390 if (ibt_hdl->clnt_hca_list != NULL) { 391 mutex_exit(&ibtl_clnt_list_mutex); 392 393 IBTF_DPRINTF_L2(ibtf, "ibt_detach: " 394 "ERROR: Client '%s' has not closed all of its HCAs", 395 ibt_hdl->clnt_modinfop->mi_clnt_name); 396 cmn_err(CE_CONT, "IBT DETACH failed: resources not yet " 397 "freed by client '%s'\n", 398 ibt_hdl->clnt_modinfop->mi_clnt_name); 399 return (IBT_HCA_RESOURCES_NOT_FREED); 400 } 401 402 if (ibt_hdl->clnt_srv_cnt != 0) { 403 mutex_exit(&ibtl_clnt_list_mutex); 404 IBTF_DPRINTF_L2(ibtf, "ibt_detach: client '%s' still has " 405 "services or subnet_notices registered", 406 ibt_hdl->clnt_modinfop->mi_clnt_name); 407 cmn_err(CE_CONT, "IBT DETACH failed: resources not yet " 408 "freed by client '%s'\n", 409 ibt_hdl->clnt_modinfop->mi_clnt_name); 410 return (IBT_HCA_RESOURCES_NOT_FREED); 411 } 412 413 /* 414 * Delete the entry of this module from the ibtl_clnt_list List. 415 */ 416 *clntpp = ibt_hdl->clnt_list_link; /* remove us */ 417 418 /* make sure asyncs complete before freeing */ 419 ibtl_free_clnt_async_check(ibt_hdl); 420 421 if (ibt_hdl->clnt_modinfop->mi_clnt_class == IBT_CM) { 422 ibtl_cm_async_handler = NULL; 423 ibtl_cm_clnt_private = NULL; 424 } else if (ibt_hdl->clnt_modinfop->mi_clnt_class == IBT_DM) { 425 ibtl_dm_async_handler = NULL; 426 ibtl_dm_clnt_private = NULL; 427 } else if (ibt_hdl->clnt_modinfop->mi_clnt_class == IBT_IBMA) { 428 ibtl_ibma_async_handler = NULL; 429 ibtl_ibma_clnt_private = NULL; 430 } 431 mutex_exit(&ibtl_clnt_list_mutex); 432 433 /* Free up the memory of per-client info struct. */ 434 kmem_free(ibt_hdl, sizeof (ibtl_clnt_t)); 435 436 return (IBT_SUCCESS); 437 } 438 439 static void 440 ibtl_set_ibhw_status() 441 { 442 ib_hw_status++; 443 } 444 445 static void 446 ibtl_clear_ibhw_status() 447 { 448 ib_hw_status--; 449 } 450 451 /* 452 * Function: 453 * ibc_init 454 * Input: 455 * modlp - Pointer to IBC client module linkage structure 456 * Output: 457 * None 458 * Returns: 459 * 0 always for now 460 * Called by: 461 * CI client calls IBTF during its _init() to register HCA with 462 * Solaris I/O framework. 463 * Description: 464 * Initializes the CI clients module linkage structure with 465 * default bus_ops structure 466 */ 467 int 468 ibc_init(struct modlinkage *modlp) 469 { 470 ibtl_ibnex_cb_args_t cb_args; 471 472 mutex_enter(&ibtl_clnt_list_mutex); 473 cb_args.cb_flag = IBTL_IBNEX_IBC_INIT; 474 cb_args.cb_modlp = modlp; 475 if (ibtl_ibnex_callback_routine) { 476 (void) ((*ibtl_ibnex_callback_routine)(&cb_args)); 477 } 478 mutex_exit(&ibtl_clnt_list_mutex); 479 return (0); 480 } 481 482 483 /* 484 * Function: 485 * ibc_fini 486 * Input: 487 * modlp - Pointer to IBC client module linkage structure 488 * Output: 489 * None 490 * Returns: 491 * None 492 * Called by: 493 * CI client calls IBTF during its _fini() to remove HCA with 494 * Solaris I/O framework. 495 * Description: 496 * Undo what is done during ibc_init 497 */ 498 void 499 ibc_fini(struct modlinkage *modlp) 500 { 501 ibtl_ibnex_cb_args_t cb_args; 502 503 mutex_enter(&ibtl_clnt_list_mutex); 504 cb_args.cb_flag = IBTL_IBNEX_IBC_FINI; 505 cb_args.cb_modlp = modlp; 506 if (ibtl_ibnex_callback_routine) { 507 (void) ((*ibtl_ibnex_callback_routine)(&cb_args)); 508 } 509 mutex_exit(&ibtl_clnt_list_mutex); 510 } 511 512 /* 513 * Function: 514 * ibc_attach 515 * Input: 516 * info_p - IBC HCA Info. 517 * Output: 518 * ibc_hdl_p - IBC Client's HCA Handle. 519 * Returns: 520 * IBC_SUCCESS 521 * IBC_FAILURE 522 * Called by: 523 * CI calls IBTF during its attach() to register HCA Device with IBTF. 524 * Description: 525 * Registers the presence of HCA device by providing the HCA device info 526 * structure and provides an opaque HCA handler for future calls to this 527 * HCA device. 528 */ 529 ibc_status_t 530 ibc_attach(ibc_clnt_hdl_t *ibc_hdl_p, ibc_hca_info_t *info_p) 531 { 532 ibtl_hca_devinfo_t *hca_devp; 533 uint_t nports; 534 ibt_status_t status; 535 536 IBTF_DPRINTF_L2(ibtf, "ibc_attach(%p, %p)", ibc_hdl_p, info_p); 537 538 /* Validate the Transport API version */ 539 if (info_p->hca_ci_vers != IBCI_V3) { 540 IBTF_DPRINTF_L1(ibtf, "ibc_attach: Invalid IB CI Version '%d'", 541 info_p->hca_ci_vers); 542 return (IBC_FAILURE); 543 } 544 545 if (info_p->hca_attr == NULL) { 546 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 547 "HCA Attributes must be specified."); 548 return (IBC_FAILURE); 549 } 550 551 nports = info_p->hca_attr->hca_nports; 552 if (nports == 0) { 553 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 554 "Number of ports must be valid"); 555 return (IBC_FAILURE); 556 } 557 558 if (info_p->hca_attr->hca_max_port_pkey_tbl_sz == 0) { 559 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 560 "Number of Partitions must be at least 1"); 561 return (IBC_FAILURE); 562 } 563 564 if ((info_p->hca_attr->hca_flags & IBT_HCA_CURRENT_QP_STATE) == 0) { 565 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 566 "HCA driver must support QP current state checking"); 567 return (IBC_FAILURE); 568 } 569 570 if ((info_p->hca_attr->hca_flags & IBT_HCA_PORT_UP) == 0) { 571 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 572 "HCA driver must support PORT_UP async events"); 573 return (IBC_FAILURE); 574 } 575 576 /* 577 * Install IB nexus driver (if not installed already) 578 */ 579 ibtl_set_ibhw_status(); 580 if (ndi_devi_config_vhci("ib", 0) == NULL) { 581 IBTF_DPRINTF_L2(ibtf, "ibc_attach: IB nexus attach failed"); 582 ibtl_clear_ibhw_status(); 583 return (IBC_FAILURE); 584 } 585 586 ibtl_thread_init2(); 587 588 /* Allocate the memory for per-client info structure */ 589 hca_devp = kmem_zalloc(sizeof (ibtl_hca_devinfo_t) + 590 (nports - 1) * sizeof (ibtl_async_port_event_t), KM_SLEEP); 591 592 mutex_enter(&ibtl_clnt_list_mutex); 593 594 /* Update HCA dev info structure */ 595 hca_devp->hd_ibc_hca_hdl = info_p->hca_handle; 596 hca_devp->hd_ibc_ops = info_p->hca_ops; 597 hca_devp->hd_hca_attr = info_p->hca_attr; 598 hca_devp->hd_hca_dip = info_p->hca_dip; 599 600 status = ibtl_init_hca_portinfo(hca_devp); 601 if (status != IBT_SUCCESS) { 602 mutex_exit(&ibtl_clnt_list_mutex); 603 IBTF_DPRINTF_L1(ibtf, "ibc_attach: call to ibc_query_hca_ports " 604 "failed: status = %d", status); 605 kmem_free(hca_devp, sizeof (ibtl_hca_devinfo_t) + 606 (nports - 1) * sizeof (ibtl_async_port_event_t)); 607 return (IBC_FAILURE); 608 } 609 610 /* Register the with MPxIO as PHCI */ 611 if (ibtl_ibnex_phci_register(info_p->hca_dip) != IBT_SUCCESS) { 612 mutex_exit(&ibtl_clnt_list_mutex); 613 IBTF_DPRINTF_L1(ibtf, "ibc_attach: MPxIO register failed"); 614 kmem_free(hca_devp, sizeof (ibtl_hca_devinfo_t) + 615 (nports - 1) * sizeof (ibtl_async_port_event_t)); 616 return (IBC_FAILURE); 617 } 618 619 /* Initialize the Client List for this HCA. */ 620 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; 621 622 /* lock out asyncs until after we announce the new HCA */ 623 hca_devp->hd_async_busy = 1; 624 625 cv_init(&hca_devp->hd_async_task_cv, NULL, CV_DEFAULT, NULL); 626 cv_init(&hca_devp->hd_async_busy_cv, NULL, CV_DEFAULT, NULL); 627 628 /* init portinfo locking variables */ 629 hca_devp->hd_portinfo_locked_port = 0; 630 cv_init(&hca_devp->hd_portinfo_cv, NULL, CV_DEFAULT, NULL); 631 632 ibtl_kstat_init(hca_devp); 633 634 mutex_exit(&ibtl_clnt_list_mutex); 635 636 /* 637 * The ibc_hdl_p points to an opaque handle which is the address 638 * of ibt_hca_devinfo_t structure passed back to the CI. 639 * The CI will pass on this handle in its future upcalls to IBTF. 640 */ 641 *ibc_hdl_p = hca_devp; 642 643 return (IBC_SUCCESS); 644 } 645 646 647 /* 648 * Function: 649 * ibc_post_attach 650 * Input: 651 * ibc_hdl - IBC Client's HCA Handle. 652 * Returns: 653 * none 654 * Called by: 655 * CI calls IBTF during its attach() after a successful ibc_attach(). 656 * Description: 657 * Announces to all known clients the existence of this HCA (by GUID). 658 */ 659 void 660 ibc_post_attach(ibc_clnt_hdl_t ibc_hdl) 661 { 662 IBTF_DPRINTF_L2(ibtf, "ibc_post_attach(%p)", ibc_hdl); 663 664 /* 665 * Update the HCA Device List. 666 */ 667 mutex_enter(&ibtl_clnt_list_mutex); 668 ibc_hdl->hd_hca_dev_link = ibtl_hca_list; 669 ibtl_hca_list = ibc_hdl; 670 mutex_exit(&ibtl_clnt_list_mutex); 671 672 /* notify all IBT Client Device Instances of the new HCA Device */ 673 ibtl_announce_new_hca(ibc_hdl); 674 } 675 676 677 /* 678 * Function: 679 * ibc_pre_detach 680 * Input: 681 * ibc_clnt_hdl - IBC HCA Handle as returned during ibc_attach call. 682 * cmd - DDI_DETACH/DDI_SUSPEND command. 683 * Output: 684 * none 685 * Returns: 686 * IBC_SUCCESS 687 * IBC_FAILURE. 688 * Called by: 689 * CI to try to get all IBTF clients to close the HCA device. 690 * Description: 691 * Attempts to deregister the HCA device entry from the IBTF. 692 * If all resources are freed by the IBTF clients and this HCA 693 * is closed, then IBC_SUCCESS is returned. 694 */ 695 ibc_status_t 696 ibc_pre_detach(ibc_clnt_hdl_t hca_devp, ddi_detach_cmd_t cmd) 697 { 698 ibtl_hca_devinfo_t **hcapp, *hcap; 699 700 IBTF_DPRINTF_L2(ibtf, "ibc_pre_detach(%p, 0x%x)", hca_devp, cmd); 701 702 /* 703 * Return failure, if command is not DDI_DETACH 704 */ 705 switch (cmd) { 706 case DDI_DETACH: 707 break; 708 default: 709 return (IBC_FAILURE); /* TBD: DDI_FAILURE */ 710 } 711 712 /* Make sure this HCA is on the HCA Device List. */ 713 mutex_enter(&ibtl_clnt_list_mutex); 714 hcap = ibtl_hca_list; 715 while (hcap != NULL) { 716 if (hcap == hca_devp) 717 break; 718 hcap = hcap->hd_hca_dev_link; 719 } 720 if (hcap == NULL) { 721 mutex_exit(&ibtl_clnt_list_mutex); 722 return (IBC_FAILURE); 723 } 724 725 /* 726 * Initially set the state to "Detaching". 727 */ 728 hca_devp->hd_state = IBTL_HCA_DEV_DETACHING; 729 730 /* 731 * Try to detach all IBTI clients, and continue only if all 732 * of the detaches succeed. 733 */ 734 if (ibtl_detach_all_clients(hca_devp)) { 735 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; /* fix hd_state */ 736 mutex_exit(&ibtl_clnt_list_mutex); 737 738 return (IBC_FAILURE); 739 } 740 741 /* 742 * Check to see if all clients closed this HCA, or not. 743 * We only succeed if all clients cooperated. 744 */ 745 if (hca_devp->hd_clnt_list != NULL) { 746 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; 747 mutex_exit(&ibtl_clnt_list_mutex); 748 IBTF_DPRINTF_L2(ibtf, "ibc_pre_detach: HCA still has attached " 749 "clients"); 750 return (IBC_FAILURE); 751 } 752 753 /* 754 * mark this device as detached 755 */ 756 hca_devp->hd_state = IBTL_HCA_DEV_DETACHED; 757 758 /* Delete the entry for this hca_devp from hca_head_list */ 759 hcapp = &ibtl_hca_list; 760 while (*hcapp != NULL) { 761 if (*hcapp == hca_devp) 762 break; 763 hcapp = &(*hcapp)->hd_hca_dev_link; 764 } 765 766 if (ibtl_ibnex_phci_unregister(hca_devp->hd_hca_dip) != IBT_SUCCESS) { 767 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; /* fix hd_state */ 768 mutex_exit(&ibtl_clnt_list_mutex); 769 IBTF_DPRINTF_L1(ibtf, "ibc_pre_detach: PHCI unregister failed"); 770 return (IBC_FAILURE); 771 } 772 773 if (*hcapp == NULL) { 774 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; /* fix hd_state */ 775 mutex_exit(&ibtl_clnt_list_mutex); 776 IBTF_DPRINTF_L1(ibtf, "ibc_pre_detach: HCA not attached"); 777 return (IBC_FAILURE); 778 } 779 *hcapp = hca_devp->hd_hca_dev_link; 780 ibtl_fast_gid_cache_valid = B_FALSE; /* invalidate fast_gid_cache */ 781 mutex_exit(&ibtl_clnt_list_mutex); 782 783 return (IBC_SUCCESS); 784 } 785 786 /* 787 * Function: 788 * ibc_detach 789 * Input: 790 * ibc_clnt_hdl - IBC HCA Handle as returned during ibc_attach call. 791 * Output: 792 * none 793 * Returns: 794 * None 795 * Called by: 796 * CI to detach the HCA device from IBTF. 797 * Description: 798 * Do the second step of detaching the HCA, which is required 799 * after a successful ibc_pre_detach. 800 */ 801 void 802 ibc_detach(ibc_clnt_hdl_t hca_devp) 803 { 804 IBTF_DPRINTF_L2(ibtf, "ibc_detach(%p)", hca_devp); 805 806 mutex_enter(&ibtl_clnt_list_mutex); 807 if (hca_devp->hd_state != IBTL_HCA_DEV_DETACHED) { 808 mutex_exit(&ibtl_clnt_list_mutex); 809 IBTF_DPRINTF_L0(ibtf, "ibc_detach: HCA has not successfully " 810 "pre-detached"); 811 return; 812 } 813 814 cv_destroy(&hca_devp->hd_async_task_cv); 815 cv_destroy(&hca_devp->hd_async_busy_cv); 816 cv_destroy(&hca_devp->hd_portinfo_cv); 817 818 kmem_free(hca_devp->hd_portinfop, hca_devp->hd_portinfo_len); 819 mutex_exit(&ibtl_clnt_list_mutex); 820 821 ibtl_kstat_fini(hca_devp); 822 823 /* Free up the memory of per-client info struct */ 824 kmem_free(hca_devp, sizeof (ibtl_hca_devinfo_t) + 825 (hca_devp->hd_hca_attr->hca_nports - 1) * 826 sizeof (ibtl_async_port_event_t)); 827 ibtl_clear_ibhw_status(); 828 } 829 830 /* 831 * Function: 832 * ibt_ci_data_in() 833 * 834 * Input: 835 * hca_hdl HCA Handle. 836 * flags IBT_COMPLETE_ALLOC - Finish a deferred alloc. 837 * object Identifies the type object pointed to by 838 * ibt_object_handle. 839 * 840 * ibt_object_handle The handle of the object to be associated with 841 * the data in/out 842 * 843 * data_p Pointer data passed in to the CI. The buffer 844 * should be allocated by the caller. 845 * 846 * data_sz The size of the buffer pointed to by 847 * data_p. 848 * Output: 849 * 850 * Returns: 851 * IBT_SUCCESS 852 * IBT_NOT_SUPPORTED Feature not supported. 853 * IBT_INVALID_PARAM Invalid object type specified. 854 * IBT_HCA_HDL_INVALID 855 * IBT_AH_HDL_INVALID/IBT_UD_DEST_HDL_INVALID 856 * IBT_CHAN_HDL_INVALID/IBT_QP_HDL_INVALID 857 * IBT_CQ_HDL_INVALID 858 * IBT_EEC_HDL_INVALID 859 * IBT_RDD_HDL_INVALID 860 * IBT_MW_HDL_INVALID 861 * IBT_PD_HDL_INVALID 862 * IBT_SRQ_HDL_INVALID 863 * 864 * Description: 865 * Exchange CI private data for the specified CI object. 866 */ 867 ibt_status_t 868 ibt_ci_data_in(ibt_hca_hdl_t hca, ibt_ci_data_flags_t flags, 869 ibt_object_type_t object, void *ibt_object_handle, void *data_p, 870 size_t data_sz) 871 { 872 ibt_status_t retval; 873 void *ci_obj_hdl; 874 875 IBTF_DPRINTF_L3(ibtf, "ibt_ci_data_in(%p, %x, %d, %p, %p, %d)", 876 hca, flags, object, ibt_object_handle, data_p, data_sz); 877 878 switch (object) { 879 case IBT_HDL_HCA: 880 ci_obj_hdl = (void *) 881 (IBTL_HCA2CIHCA(((ibt_hca_hdl_t)ibt_object_handle))); 882 break; 883 884 case IBT_HDL_CHANNEL: 885 ci_obj_hdl = (void *) 886 (IBTL_CHAN2CIQP(((ibt_channel_hdl_t)ibt_object_handle))); 887 break; 888 889 case IBT_HDL_CQ: 890 ci_obj_hdl = (void *) 891 (((ibt_cq_hdl_t)(ibt_object_handle))->cq_ibc_cq_hdl); 892 break; 893 894 case IBT_HDL_EEC: 895 ci_obj_hdl = (void *) 896 (((ibt_eec_hdl_t)(ibt_object_handle))->eec_ibc_eec_hdl); 897 break; 898 899 case IBT_HDL_UD_DEST: 900 ci_obj_hdl = (void *) 901 (((ibt_ud_dest_hdl_t)(ibt_object_handle))->ud_ah); 902 break; 903 904 case IBT_HDL_SRQ: 905 ci_obj_hdl = (void *) 906 (((ibt_srq_hdl_t)(ibt_object_handle))->srq_ibc_srq_hdl); 907 break; 908 909 default: 910 ci_obj_hdl = ibt_object_handle; 911 break; 912 } 913 914 retval = (IBTL_HCA2CIHCAOPS_P(hca)->ibc_ci_data_in)(IBTL_HCA2CIHCA(hca), 915 flags, object, ci_obj_hdl, data_p, data_sz); 916 917 if (retval != IBT_SUCCESS) { 918 IBTF_DPRINTF_L2(ibtf, "ibt_ci_data_in: Failed : %d", retval); 919 } 920 return (retval); 921 } 922 923 /* 924 * Function: 925 * ibt_ci_data_out() 926 * 927 * Input: 928 * hca_hdl HCA Handle. 929 * flags IBT_COMPLETE_ALLOC - Finish a deferred alloc. 930 * object Identifies the type object pointed to by 931 * ibt_object_handle. 932 * 933 * ibt_object_handle The handle of the object to be associated with 934 * the data in/out 935 * 936 * data_p Pointer to a buffer in which to return the CI 937 * private data. The buffer should be allocated 938 * by the caller. 939 * 940 * data_sz The size of the buffer pointed to by 941 * data_p. 942 * Output: 943 * 944 * Returns: 945 * IBT_SUCCESS 946 * IBT_NOT_SUPPORTED Feature not supported. 947 * IBT_INSUFF_RESOURCE The buffer pointed to by data_p was too 948 * small to hold the data. 949 * IBT_INVALID_PARAM Invalid object type specified. 950 * IBT_HCA_HDL_INVALID 951 * IBT_AH_HDL_INVALID/IBT_UD_DEST_HDL_INVALID 952 * IBT_CHAN_HDL_INVALID/IBT_QP_HDL_INVALID 953 * IBT_CQ_HDL_INVALID 954 * IBT_EEC_HDL_INVALID 955 * IBT_RDD_HDL_INVALID 956 * IBT_MW_HDL_INVALID 957 * IBT_PD_HDL_INVALID 958 * IBT_SRQ_HDL_INVALID 959 * 960 * Description: 961 * Exchange CI private data for the specified CI object. 962 */ 963 ibt_status_t 964 ibt_ci_data_out(ibt_hca_hdl_t hca, ibt_ci_data_flags_t flags, 965 ibt_object_type_t object, void *ibt_object_handle, void *data_p, 966 size_t data_sz) 967 { 968 ibt_status_t retval; 969 void *ci_obj_hdl; 970 971 IBTF_DPRINTF_L3(ibtf, "ibt_ci_data_out(%p, %x, %d, %p, %p, %d)", 972 hca, flags, object, ibt_object_handle, data_p, data_sz); 973 974 switch (object) { 975 case IBT_HDL_HCA: 976 ci_obj_hdl = (void *) 977 (IBTL_HCA2CIHCA(((ibt_hca_hdl_t)ibt_object_handle))); 978 break; 979 980 case IBT_HDL_CHANNEL: 981 ci_obj_hdl = (void *) 982 (IBTL_CHAN2CIQP(((ibt_channel_hdl_t)ibt_object_handle))); 983 break; 984 985 case IBT_HDL_CQ: 986 ci_obj_hdl = (void *) 987 (((ibt_cq_hdl_t)(ibt_object_handle))->cq_ibc_cq_hdl); 988 break; 989 990 case IBT_HDL_EEC: 991 ci_obj_hdl = (void *) 992 (((ibt_eec_hdl_t)(ibt_object_handle))->eec_ibc_eec_hdl); 993 break; 994 995 case IBT_HDL_UD_DEST: 996 ci_obj_hdl = (void *) 997 (((ibt_ud_dest_hdl_t)(ibt_object_handle))->ud_ah); 998 break; 999 1000 case IBT_HDL_SRQ: 1001 ci_obj_hdl = (void *) 1002 (((ibt_srq_hdl_t)(ibt_object_handle))->srq_ibc_srq_hdl); 1003 break; 1004 1005 default: 1006 ci_obj_hdl = ibt_object_handle; 1007 break; 1008 } 1009 1010 retval = (IBTL_HCA2CIHCAOPS_P(hca)->ibc_ci_data_out) 1011 (IBTL_HCA2CIHCA(hca), flags, object, ci_obj_hdl, data_p, data_sz); 1012 1013 if (retval != IBT_SUCCESS) { 1014 IBTF_DPRINTF_L2(ibtf, "ibt_ci_data_out: Failed : %d", retval); 1015 } 1016 return (retval); 1017 } 1018 1019 1020 /* 1021 * FMA Support functions. 1022 */ 1023 1024 #define IBTL_ENA_MASK 0xC0000000 1025 #define IBTL_ENA_POSSIBLE 0x80000000 1026 #define IBTL_TYPE_SHIFT 27 1027 1028 /* 1029 * Function: 1030 * ibt_get_module_failure() 1031 * 1032 * Input: 1033 * type Identifies the failing IB module. 1034 * ena '0' or the data for Fault Management 1035 * Architecture (ENA). 1036 * 1037 * Returns: 1038 * status Special IB failure status. 1039 * 1040 * Description: 1041 * XXX Just stubbed out to return failures with no data for Fault 1042 * Management Architecture (ENAs) at the moment XXX 1043 */ 1044 ibt_status_t 1045 ibt_get_module_failure(ibt_failure_type_t type, uint64_t ena) 1046 { 1047 ibt_status_t ret; 1048 1049 IBTF_DPRINTF_L3(ibtf, "ibt_get_module_failure(%d, 0x%llX)", type, ena); 1050 1051 switch (type) { 1052 case IBT_FAILURE_CI: 1053 case IBT_FAILURE_IBMF: 1054 case IBT_FAILURE_IBCM: 1055 case IBT_FAILURE_IBDM: 1056 case IBT_FAILURE_IBTL: 1057 case IBT_FAILURE_IBSM: 1058 ret = IBTL_ENA_POSSIBLE | (type << IBTL_TYPE_SHIFT); 1059 break; 1060 default: 1061 ret = IBT_FAILURE; 1062 } 1063 IBTF_DPRINTF_L3(ibtf, "ibt_get_module_failure: ret = 0x%lX", ret); 1064 return (ret); 1065 } 1066 1067 1068 /* 1069 * Function: 1070 * ibc_get_ci_failure() 1071 * 1072 * Input: 1073 * ena '0' or the data for Fault Management 1074 * Architecture (ENA). 1075 * 1076 * Returns: 1077 * status Special CI failure status. 1078 * 1079 * Description: 1080 * Just use the function above to do the job. 1081 */ 1082 ibt_status_t 1083 ibc_get_ci_failure(uint64_t ena) 1084 { 1085 return (ibt_get_module_failure(IBT_FAILURE_CI, ena)); 1086 } 1087 1088 1089 /* 1090 * ibt_check_failure() 1091 * Function to test for special case failures. 1092 * 1093 * status An ibt_status_t returned from an IBTF function call. 1094 * 1095 * reserved_p NULL, or a pointer to where we store the data for 1096 * Fault Management Architecture (ENA). 1097 * 1098 * Description: 1099 * XXX Still need to determine the data for Fault Management Architecture 1100 * (ENA), using 0 for now XXX 1101 */ 1102 ibt_failure_type_t 1103 ibt_check_failure(ibt_status_t status, uint64_t *reserved_p) 1104 { 1105 ibt_failure_type_t type; 1106 1107 IBTF_DPRINTF_L3(ibtf, "ibt_check_failure(%X)", status); 1108 1109 if ((status & IBTL_ENA_MASK) == IBTL_ENA_POSSIBLE) { 1110 type = status & ~IBTL_ENA_POSSIBLE >> IBTL_TYPE_SHIFT; 1111 1112 /* XXX Need more work here... */ 1113 if (reserved_p != NULL) 1114 *reserved_p = 0; 1115 } else { 1116 type = IBT_FAILURE_STANDARD; 1117 if (reserved_p != NULL) 1118 *reserved_p = 0; /* No FMA Data Available. */ 1119 } 1120 IBTF_DPRINTF_L3(ibtf, "ibt_check_failure: type = 0x%X", type); 1121 return (type); 1122 } 1123 1124 /* 1125 * Initialize and create kstats. 1126 * 1127 * We create the following kstats on all ports of the HCA: 1128 * <hca_driver_name><instance_number>/port<port_num>/stats 1129 * <hca_driver_name><instance_number>/port<port_num>/pkeys 1130 */ 1131 static void 1132 ibtl_kstat_init(ibtl_hca_devinfo_t *hca_devp) 1133 { 1134 uint_t nports = hca_devp->hd_hca_attr->hca_nports; 1135 ibtl_hca_port_kstat_t *pks; 1136 int i; 1137 1138 IBTF_DPRINTF_L3(ibtf, "ibtl_kstat_init(hca_devp = 0x%p)", hca_devp); 1139 1140 hca_devp->hd_hca_port_ks_info_len = 1141 sizeof (ibtl_hca_port_kstat_t) * nports; 1142 pks = kmem_zalloc(hca_devp->hd_hca_port_ks_info_len, KM_SLEEP); 1143 hca_devp->hd_hca_port_ks_info = pks; 1144 1145 for (i = 0; i < nports; i++, pks++) { 1146 pks->pks_hca_devp = hca_devp; 1147 pks->pks_port_num = i + 1; 1148 ibtl_kstat_stats_create(hca_devp, i + 1); 1149 ibtl_kstat_pkeys_create(hca_devp, i + 1); 1150 } 1151 } 1152 1153 /* 1154 * Delete kstats on all ports of the HCA. 1155 */ 1156 static void 1157 ibtl_kstat_fini(ibtl_hca_devinfo_t *hca_devp) 1158 { 1159 ibtl_hca_port_kstat_t *pks; 1160 int i; 1161 1162 IBTF_DPRINTF_L3(ibtf, "ibtl_kstat_fini(hca_devp = 0x%p)", hca_devp); 1163 1164 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hca_devp)) 1165 1166 pks = hca_devp->hd_hca_port_ks_info; 1167 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pks)) 1168 1169 if (pks == NULL) 1170 return; 1171 1172 for (i = 0; i < hca_devp->hd_hca_attr->hca_nports; i++, pks++) { 1173 if (pks->pks_stats_ksp) 1174 kstat_delete(pks->pks_stats_ksp); 1175 1176 if (pks->pks_pkeys_ksp) { 1177 ASSERT(!MUTEX_HELD(&ibtl_clnt_list_mutex)); 1178 kstat_delete(pks->pks_pkeys_ksp); 1179 } 1180 } 1181 1182 kmem_free(hca_devp->hd_hca_port_ks_info, 1183 hca_devp->hd_hca_port_ks_info_len); 1184 } 1185 1186 /* 1187 * Update "stats" kstat. 1188 * Called by kstat framework. 1189 */ 1190 static int 1191 ibtl_kstat_stats_update(kstat_t *ksp, int rw) 1192 { 1193 ibtl_hca_port_kstat_t *pks; 1194 ibtl_hca_devinfo_t *hca_devp; 1195 ibt_hca_portinfo_t *p; 1196 struct kstat_named *data; 1197 1198 IBTF_DPRINTF_L4(ibtf, "ibtl_kstat_stats_update(ksp = 0x%p, rw = %d)", 1199 ksp, rw); 1200 1201 if (rw == KSTAT_WRITE) 1202 return (EACCES); 1203 1204 mutex_enter(&ibtl_clnt_list_mutex); 1205 1206 /* 1207 * Update the link_state kstat using the value from portinfo cache. 1208 */ 1209 pks = ksp->ks_private; 1210 hca_devp = pks->pks_hca_devp; 1211 data = (struct kstat_named *)(ksp->ks_data); 1212 p = hca_devp->hd_portinfop + pks->pks_port_num - 1; 1213 data[0].value.ui32 = (uint32_t)p->p_linkstate; 1214 1215 mutex_exit(&ibtl_clnt_list_mutex); 1216 1217 return (0); 1218 } 1219 1220 /* 1221 * Create "stats" kstat for the specified HCA port in the form: 1222 * <hca_driver_name><instance_number>/port<port_num>/stats 1223 * At preset it contains only one named data of "link_state" 1224 */ 1225 static void 1226 ibtl_kstat_stats_create(ibtl_hca_devinfo_t *hca_devp, uint_t port_num) 1227 { 1228 struct kstat *ksp; 1229 struct kstat_named *named_data; 1230 char *drv_name; 1231 int drv_instance; 1232 ibtl_hca_port_kstat_t *pks; 1233 char kname[40]; 1234 1235 IBTF_DPRINTF_L3(ibtf, "ibtl_kstat_stats_create(hca_devp = 0x%p, " 1236 "port_num = 0x%u)", hca_devp, port_num); 1237 1238 drv_name = (char *)ddi_driver_name(hca_devp->hd_hca_dip); 1239 drv_instance = ddi_get_instance(hca_devp->hd_hca_dip); 1240 (void) snprintf(kname, sizeof (kname), "%s%d/port%d/stats", 1241 drv_name, drv_instance, port_num); 1242 1243 ksp = kstat_create("ibtf", 0, kname, "ib", KSTAT_TYPE_NAMED, 1, 0); 1244 if (ksp == NULL) { 1245 IBTF_DPRINTF_L2(ibtf, 1246 "ibtl_kstat_stats_create: kstat_create() failed"); 1247 return; 1248 } 1249 1250 named_data = (struct kstat_named *)(ksp->ks_data); 1251 kstat_named_init(&named_data[0], "link_state", KSTAT_DATA_UINT32); 1252 1253 pks = hca_devp->hd_hca_port_ks_info + port_num - 1; 1254 pks->pks_stats_ksp = ksp; 1255 1256 ksp->ks_private = pks; 1257 ksp->ks_update = ibtl_kstat_stats_update; 1258 1259 /* Install the kstat */ 1260 kstat_install(ksp); 1261 } 1262 1263 /* 1264 * Update "pkeys" kstat. 1265 * 1266 * Called by kstat framework. Since ks_lock was set to ibtl_clnt_list_mutex 1267 * at the time of the kstat creation, kstat framework will hold this lock 1268 * while calling this function. 1269 */ 1270 static int 1271 ibtl_kstat_pkeys_update(kstat_t *ksp, int rw) 1272 { 1273 ibtl_hca_port_kstat_t *pks; 1274 ibtl_hca_devinfo_t *hca_devp; 1275 ibt_hca_portinfo_t *p; 1276 1277 IBTF_DPRINTF_L4(ibtf, "ibtl_kstat_pkeys_update(ksp = 0x%p, rw = %d)", 1278 ksp, rw); 1279 1280 #ifndef __lock_lint 1281 ASSERT(MUTEX_HELD(&ibtl_clnt_list_mutex)); 1282 #endif 1283 1284 if (rw == KSTAT_WRITE) 1285 return (EACCES); 1286 1287 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ksp)) 1288 1289 pks = ksp->ks_private; 1290 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pks)) 1291 1292 hca_devp = pks->pks_hca_devp; 1293 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hca_devp)) 1294 1295 /* 1296 * Point kstat data to the pkey table in the portinfo cache. 1297 */ 1298 1299 p = hca_devp->hd_portinfop + pks->pks_port_num - 1; 1300 1301 ksp->ks_data = p->p_pkey_tbl; 1302 ksp->ks_ndata = p->p_pkey_tbl_sz; 1303 ksp->ks_data_size = p->p_pkey_tbl_sz * sizeof (ib_pkey_t); 1304 1305 return (0); 1306 } 1307 1308 /* 1309 * Create "pkeys" kstat for the specified HCA port in the form: 1310 * <hca_driver_name><instance_number>/port<port_num>/pkeys 1311 * 1312 * Currently kstat framework allows only some fixed data types as named 1313 * data components under a named kstat. Due to this limitation it is not 1314 * possible to add "pkeys" as a named data under the "stats" kstat. 1315 */ 1316 static void 1317 ibtl_kstat_pkeys_create(ibtl_hca_devinfo_t *hca_devp, uint_t port_num) 1318 { 1319 struct kstat *ksp; 1320 char *drv_name; 1321 int drv_instance; 1322 char kname[40]; 1323 ibtl_hca_port_kstat_t *pks; 1324 1325 IBTF_DPRINTF_L3(ibtf, "ibtl_kstat_stats_create(hca_devp = 0x%p, " 1326 "port_num = 0x%u)", hca_devp, port_num); 1327 1328 drv_name = (char *)ddi_driver_name(hca_devp->hd_hca_dip); 1329 drv_instance = ddi_get_instance(hca_devp->hd_hca_dip); 1330 (void) snprintf(kname, sizeof (kname), "%s%d/port%d/pkeys", 1331 drv_name, drv_instance, port_num); 1332 1333 ksp = kstat_create("ibtf", 0, kname, "ib", KSTAT_TYPE_RAW, 0, 1334 KSTAT_FLAG_VAR_SIZE | KSTAT_FLAG_VIRTUAL); 1335 if (ksp == NULL) { 1336 IBTF_DPRINTF_L2(ibtf, 1337 "ibtl_kstat_pkeys_create: kstat_create() failed"); 1338 return; 1339 } 1340 1341 pks = hca_devp->hd_hca_port_ks_info + port_num - 1; 1342 pks->pks_pkeys_ksp = ksp; 1343 1344 ksp->ks_private = pks; 1345 ksp->ks_update = ibtl_kstat_pkeys_update; 1346 ksp->ks_lock = &ibtl_clnt_list_mutex; 1347 1348 /* 1349 * We just go with the default_kstat_snapshot(). 1350 * So there is no need to set ks_snapshot field. 1351 */ 1352 1353 /* Install the kstat */ 1354 kstat_install(ksp); 1355 } 1356