1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2004 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/ib/mgt/ibmf/ibmf_saa_impl.h> 30 #include <sys/ib/mgt/ibmf/ibmf_saa_utils.h> 31 32 /* Global sa_access State Pointer */ 33 saa_state_t *saa_statep; 34 _NOTE(READ_ONLY_DATA(saa_statep)) 35 36 extern int ibmf_trace_level; 37 38 extern int ibmf_taskq_max_tasks; 39 40 static int 41 ibmf_saa_impl_new_smlid_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp, 42 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, int transport_flags); 43 static int 44 ibmf_saa_check_sa_and_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp, 45 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, 46 hrtime_t trans_send_time, int transport_flags); 47 static int ibmf_saa_impl_init_msg(saa_impl_trans_info_t *trans_info, 48 boolean_t sleep_flag, ibmf_msg_t **msgp, uint32_t *transport_flagsp, 49 ibmf_retrans_t *ibmf_retransp); 50 static int ibmf_saa_must_purge(saa_port_t *saa_portp); 51 static void ibmf_saa_impl_invalidate_port(saa_port_t *saa_portp); 52 static void ibmf_saa_impl_destroy_port(saa_port_t *saa_portp); 53 static void ibmf_saa_impl_uninit_kstats(saa_port_t *saa_portp); 54 static void ibmf_saa_impl_get_cpi_cb(void *arg, size_t length, char *buffer, 55 int status); 56 static void ibmf_saa_impl_async_event_cb(ibmf_handle_t ibmf_handle, 57 void *clnt_private, ibmf_async_event_t event_type); 58 static void ibmf_saa_impl_port_up(ib_guid_t ci_guid, uint8_t port_num); 59 static void ibmf_saa_impl_port_down(ib_guid_t ci_guid, uint8_t port_num); 60 static void ibmf_saa_impl_hca_detach(saa_port_t *saa_removed); 61 static void ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle, 62 ibmf_msg_t *msgp, boolean_t ignore_data, int *status, void **result, 63 size_t *length, boolean_t sleep_flag); 64 static int ibmf_saa_impl_check_sa_support(uint16_t cap_mask, uint16_t attr_id); 65 static uint_t ibmf_saa_impl_get_attr_id_length(uint16_t attr_id); 66 static void ibmf_saa_impl_free_msg(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msgp); 67 static int ibmf_saa_impl_get_port_guid(ibt_hca_portinfo_t *ibt_portinfop, 68 ib_guid_t *guid_ret); 69 static void ibmf_saa_impl_set_transaction_params(saa_port_t *saa_portp, 70 ibt_hca_portinfo_t *portinfop); 71 static void ibmf_saa_impl_ibmf_unreg(saa_port_t *saa_portp); 72 73 int ibmf_saa_max_wait_time = IBMF_SAA_MAX_WAIT_TIME_IN_SECS; 74 int ibmf_saa_trans_wait_time = IBMF_SAA_TRANS_WAIT_TIME_IN_SECS; 75 76 /* 77 * ibmf_saa_impl_init: 78 * Allocates memory for the ibmf_saa state structure and initializes the taskq. 79 * Called from the modules init() routine. 80 * 81 * Input Arguments 82 * none 83 * 84 * Output Arguments 85 * none 86 * 87 * Returns 88 * IBMF_NO_RESOURCES if taskq could not be created. 89 * IBMF_SUCCESS on success 90 * 91 */ 92 int 93 ibmf_saa_impl_init() 94 { 95 int res; 96 97 /* CONSTCOND */ 98 ASSERT(NO_COMPETING_THREADS); 99 100 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_init_start, 101 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init() enter\n"); 102 103 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_statep)) 104 105 saa_statep = kmem_zalloc(sizeof (saa_state_t), KM_SLEEP); 106 107 /* create taskq for notifying event subscribers */ 108 saa_statep->saa_event_taskq = taskq_create( 109 "ibmf_saa_event_taskq", IBMF_TASKQ_NTHREADS, 110 MINCLSYSPRI, 1, ibmf_taskq_max_tasks, TASKQ_DYNAMIC | 111 TASKQ_PREPOPULATE); 112 if (saa_statep->saa_event_taskq == NULL) { 113 114 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L4, 115 ibmf_saa_impl_init_end_err, 116 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init(): %s\n", 117 tnf_string, msg, "event taskq create failed"); 118 119 kmem_free(saa_statep, sizeof (saa_state_t)); 120 121 res = IBMF_NO_RESOURCES; 122 123 goto bail; 124 } 125 126 mutex_init(&saa_statep->saa_port_list_mutex, NULL, MUTEX_DRIVER, 127 NULL); 128 129 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*saa_statep)) 130 131 res = IBMF_SUCCESS; 132 bail: 133 134 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_init_end, 135 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init() exit: status = %d\n", 136 tnf_int, res, res); 137 138 return (res); 139 } 140 141 /* 142 * ibmf_saa_impl_fini: 143 * If there are no registered clients, cleans up all memory associated with the 144 * state, including each of the port list entries. 145 * Called from the modules fini() routine. 146 * 147 * Input Arguments 148 * none 149 * 150 * Output Arguments 151 * none 152 * 153 * Returns 154 * EBUSY if there are outstanding transactions or registered clients 155 * 0 if cleanup was sucessfull 156 * 157 */ 158 int 159 ibmf_saa_impl_fini() 160 { 161 int ret = 0; 162 saa_port_t *saa_portp; 163 164 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_fini_start, 165 IBMF_TNF_TRACE, "", "ibmf_saa_impl_fini() enter\n"); 166 167 /* make sure there are no registered clients */ 168 mutex_enter(&saa_statep->saa_port_list_mutex); 169 170 saa_portp = saa_statep->saa_port_list; 171 while (saa_portp != NULL) { 172 173 mutex_enter(&saa_portp->saa_pt_mutex); 174 175 if (saa_portp->saa_pt_reference_count > 0) { 176 177 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 178 ibmf_saa_impl_fini_err, IBMF_TNF_ERROR, "", 179 "ibmf_saa_impl_fini: %s, port %016" PRIx64 "\n", 180 tnf_string, msg, 181 "cannot unload ibmf_saa. Client on port still" 182 " registered", tnf_opaque, port, 183 saa_portp->saa_pt_port_guid); 184 185 mutex_exit(&saa_portp->saa_pt_mutex); 186 187 mutex_exit(&saa_statep->saa_port_list_mutex); 188 189 ret = EBUSY; 190 goto bail; 191 } 192 193 /* make sure there are no outstanding transactions */ 194 195 if (saa_portp->saa_pt_num_outstanding_trans > 0) { 196 197 IBMF_TRACE_3(IBMF_TNF_NODEBUG, DPRINT_L1, 198 ibmf_saa_impl_fini_err, IBMF_TNF_ERROR, "", 199 "ibmf_saa_impl_fini: %s, port = %016" PRIx64 200 ", num transactions = %d\n", 201 tnf_string, msg, "Cannot unload ibmf_saa." 202 " Outstanding transactions on port.", 203 tnf_opaque, port, 204 saa_portp->saa_pt_port_guid, 205 tnf_uint, outstanding_transactions, 206 saa_portp->saa_pt_num_outstanding_trans); 207 208 mutex_exit(&saa_portp->saa_pt_mutex); 209 210 mutex_exit(&saa_statep->saa_port_list_mutex); 211 212 ret = EBUSY; 213 goto bail; 214 } 215 216 mutex_exit(&saa_portp->saa_pt_mutex); 217 218 saa_portp = saa_portp->next; 219 } 220 221 mutex_exit(&saa_statep->saa_port_list_mutex); 222 223 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(saa_statep->saa_port_list, 224 *saa_portp)) 225 226 /* 227 * no more clients nor pending transaction: 228 * unregister ibmf and destroy port entries 229 */ 230 saa_portp = saa_statep->saa_port_list; 231 while (saa_portp != NULL) { 232 233 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 234 ibmf_saa_impl_fini, IBMF_TNF_TRACE, "", 235 "ibmf_saa_impl_fini: %s, prefix = %016" PRIx64 "\n", 236 tnf_string, msg, "deinitializing port", 237 tnf_opaque, port_guid, saa_portp->saa_pt_port_guid); 238 239 saa_statep->saa_port_list = saa_portp->next; 240 241 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_portp)) 242 243 mutex_enter(&saa_portp->saa_pt_mutex); 244 245 /* unregister from ibmf */ 246 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_READY) { 247 248 mutex_exit(&saa_portp->saa_pt_mutex); 249 250 ibmf_saa_impl_ibmf_unreg(saa_portp); 251 } else 252 253 mutex_exit(&saa_portp->saa_pt_mutex); 254 255 ibmf_saa_impl_destroy_port(saa_portp); 256 saa_portp = saa_statep->saa_port_list; 257 } 258 259 taskq_destroy(saa_statep->saa_event_taskq); 260 261 mutex_destroy(&saa_statep->saa_port_list_mutex); 262 263 kmem_free(saa_statep, sizeof (saa_state_t)); 264 265 bail: 266 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_fini_end, 267 IBMF_TNF_TRACE, "", "ibmf_saa_impl_fini() exit\n"); 268 269 return (ret); 270 } 271 272 /* 273 * ibmf_saa_is_valid 274 * Returns true the entry is valid. 275 * 276 * Input Arguments 277 * saa_portp pointer to state structure 278 * add_ref if B_TRUE ref count is incremented on a valid portp 279 * 280 * Output Arguments 281 * none 282 * 283 * Returns 284 * B_TRUE if entry was in a valid state, B_FALSE otherwise 285 */ 286 boolean_t 287 ibmf_saa_is_valid(saa_port_t *saa_portp, int add_ref) 288 { 289 boolean_t is_valid = B_TRUE; 290 291 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_is_valid_start, 292 IBMF_TNF_TRACE, "", "ibmf_saa_is_valid() enter\n"); 293 294 mutex_enter(&saa_portp->saa_pt_mutex); 295 296 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_INVALID || 297 saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_PURGING) { 298 299 is_valid = B_FALSE; 300 301 } else if (add_ref == B_TRUE) { 302 /* 303 * increment reference count here to ensure that 304 * entry does not get purged behind our backs 305 */ 306 saa_portp->saa_pt_reference_count++; 307 } 308 mutex_exit(&saa_portp->saa_pt_mutex); 309 310 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_is_valid_end, 311 IBMF_TNF_TRACE, "", "ibmf_saa_is_valid() exit\n"); 312 313 return (is_valid); 314 } 315 316 /* 317 * ibmf_saa_must_purge 318 * Determines if we can purge a portp (remove it from the list) based on the 319 * state and number of clients 320 * 321 * Input Arguments 322 * saa_portp pointer to state structure 323 * 324 * Output Arguments 325 * none 326 * 327 * Returns 328 * B_TRUE if the entry can be removed, B_FALSE otherwise 329 */ 330 static int 331 ibmf_saa_must_purge(saa_port_t *saa_portp) 332 { 333 int must_purge = B_FALSE; 334 335 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_must_purge_start, 336 IBMF_TNF_TRACE, "", "ibmf_saa_must_purge() enter\n"); 337 338 mutex_enter(&saa_portp->saa_pt_mutex); 339 340 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_INVALID && 341 saa_portp->saa_pt_reference_count == 0) { 342 343 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_PURGING; 344 must_purge = B_TRUE; 345 } 346 347 mutex_exit(&saa_portp->saa_pt_mutex); 348 349 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_must_purge_end, 350 IBMF_TNF_TRACE, "", "ibmf_saa_must_purge() exit\n"); 351 352 return (must_purge); 353 } 354 355 356 /* 357 * ibmf_saa_impl_purge: 358 * Removes invalid port state entries from the list 359 * 360 * Input Arguments 361 * none 362 * 363 * Output Arguments 364 * none 365 * 366 * Returns 367 * void 368 */ 369 void 370 ibmf_saa_impl_purge() 371 { 372 saa_port_t *cur_portp = NULL; 373 saa_port_t *prev_portp = NULL; 374 saa_port_t *rem_portp = NULL; 375 376 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_purge_start, 377 IBMF_TNF_TRACE, "", "ibmf_saa_impl_purge() enter\n"); 378 379 mutex_enter(&saa_statep->saa_port_list_mutex); 380 381 cur_portp = saa_statep->saa_port_list; 382 prev_portp = cur_portp; 383 384 while (cur_portp != NULL) { 385 386 if (ibmf_saa_must_purge(cur_portp) == B_TRUE) { 387 388 rem_portp = cur_portp; 389 390 /* unlink entry */ 391 if (cur_portp == saa_statep->saa_port_list) { 392 393 saa_statep->saa_port_list = cur_portp->next; 394 cur_portp = saa_statep->saa_port_list; 395 prev_portp = cur_portp; 396 397 } else { 398 399 prev_portp->next = cur_portp->next; 400 cur_portp = cur_portp->next; 401 } 402 403 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*rem_portp)) 404 405 /* destroy entry */ 406 ASSERT(rem_portp != NULL); 407 ibmf_saa_impl_destroy_port(rem_portp); 408 409 } else { 410 411 prev_portp = cur_portp; 412 cur_portp = cur_portp->next; 413 } 414 } 415 416 mutex_exit(&saa_statep->saa_port_list_mutex); 417 418 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_purge_end, 419 IBMF_TNF_TRACE, "", "ibmf_saa_impl_purge() exit\n"); 420 } 421 422 /* 423 * saa_impl_add_client: 424 * Adds a client for a particular portp. Reference count has been incremented 425 * before this call. It is decremented by saa_impl_add_client() if the call 426 * fails. 427 * 428 * Input Arguments 429 * none 430 * 431 * Output Arguments 432 * none 433 * 434 * Returns 435 * IBMF_BUSY if there are already too many clients registered, 436 * IBMF_BAD_PORT_STATE if the port is invalid (generally because a previous 437 * client failed during registration for this port) 438 * IBMF_SUCCESS otherwise 439 */ 440 int 441 ibmf_saa_impl_add_client(saa_port_t *saa_portp) 442 { 443 int status = IBMF_SUCCESS; 444 445 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_add_client_start, 446 IBMF_TNF_TRACE, "", "ibmf_saa_impl_add_client() enter\n"); 447 448 mutex_enter(&saa_portp->saa_pt_mutex); 449 450 /* 451 * check that we don't exceed max clients 452 */ 453 if (saa_portp->saa_pt_reference_count > 454 SAA_MAX_CLIENTS_PER_PORT) { 455 456 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 457 ibmf_saa_impl_add_client_err, IBMF_TNF_ERROR, "", 458 "ibmf_saa_impl_add_client: %s, num_reg_clients %d\n", 459 tnf_string, msg, "too many clients registered for" 460 " port", tnf_uint, num_reg_clients, 461 saa_portp->saa_pt_reference_count); 462 463 status = IBMF_BUSY; 464 goto bail; 465 } 466 467 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 468 ibmf_saa_impl_add_client, IBMF_TNF_TRACE, "", 469 "ibmf_saa_impl_add_client: num_registered_clients %d\n", 470 tnf_uint, num_registered_clients, 471 saa_portp->saa_pt_reference_count); 472 473 /* 474 * wait until anyone who is currently registering 475 * this port with ibmf is done 476 */ 477 while (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_REGISTERING) { 478 479 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 480 ibmf_saa_impl_add_client, IBMF_TNF_TRACE, "", 481 "ibmf_saa_impl_add_client: %s\n", 482 tnf_string, msg, "someone is registering. waiting" 483 " for them to finish"); 484 485 cv_wait(&saa_portp->saa_pt_ibmf_reg_cv, 486 &saa_portp->saa_pt_mutex); 487 488 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 489 ibmf_saa_impl_add_client, 490 IBMF_TNF_TRACE, "", "ibmf_saa_impl_add_client: %s\n", 491 tnf_string, msg, "done waiting"); 492 } 493 494 /* 495 * if port isn't ready here, fail. 496 */ 497 if (saa_portp->saa_pt_state != IBMF_SAA_PORT_STATE_READY) { 498 499 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 500 ibmf_saa_impl_add_client_err, IBMF_TNF_ERROR, 501 "", "ibmf_saa_impl_add_client: %s\n", 502 tnf_string, msg, "port state not ready," 503 " removing client."); 504 505 status = IBMF_BAD_PORT_STATE; 506 goto bail; 507 } 508 509 bail: 510 mutex_exit(&saa_portp->saa_pt_mutex); 511 512 if (status != IBMF_SUCCESS) { 513 514 mutex_enter( 515 &saa_portp->saa_pt_kstat_mutex); 516 517 IBMF_SAA_ADD32_KSTATS(saa_portp, 518 clients_reg_failed, 1); 519 520 mutex_exit( 521 &saa_portp->saa_pt_kstat_mutex); 522 523 /* decrementing refcount is last thing we do on entry */ 524 525 mutex_enter(&saa_portp->saa_pt_mutex); 526 527 ASSERT(saa_portp->saa_pt_reference_count > 0); 528 saa_portp->saa_pt_reference_count--; 529 530 mutex_exit(&saa_portp->saa_pt_mutex); 531 } 532 533 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 534 ibmf_saa_impl_add_client_end, IBMF_TNF_TRACE, "", 535 "ibmf_saa_impl_add_client() exit\n"); 536 537 return (status); 538 } 539 540 /* 541 * ibmf_saa_impl_create_port() 542 * Create port entry with mimimal inits because 543 * we're holding the list mutex: NO BLOCKING CALLS HERE, please. 544 * 545 * Initialize port state to "registering", so that clients accessing 546 * same port concurrently will wait for the end of the ibmf registration. 547 * Note: this thread will access port members without locking mutex. 548 * 549 * Input Arguments 550 * pt_guid guid of port 551 * 552 * Output Arguments 553 * saa_portpp pointer to new saa_portp structure 554 * 555 * Returns 556 * IBMF_NO_MEMORY if memory could not be allocated 557 * IBMF_SUCCESS otherwise 558 */ 559 int 560 ibmf_saa_impl_create_port(ib_guid_t pt_guid, saa_port_t **saa_portpp) 561 { 562 int status = IBMF_SUCCESS; 563 saa_port_t *saa_portp = NULL; 564 565 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_create_port_start, 566 IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port:" 567 " guid %016" PRIx64 "\n", 568 tnf_opaque, port_guid, pt_guid); 569 570 ASSERT(MUTEX_HELD(&saa_statep->saa_port_list_mutex)); 571 572 /* create & initialize new port */ 573 saa_portp = kmem_zalloc(sizeof (saa_port_t), KM_NOSLEEP); 574 575 if (saa_portp == NULL) { 576 577 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 578 ibmf_sa_session_open_err, IBMF_TNF_ERROR, "", 579 "ibmf_saa_impl_create_port: %s\n", 580 tnf_string, msg, "could not allocate memory for " 581 "new port"); 582 583 status = IBMF_NO_MEMORY; 584 goto bail; 585 } 586 587 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_sa_session_open, 588 IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port: %s\n", 589 tnf_string, msg, "first client registering, initializing"); 590 591 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_portp)) 592 593 /* tell everyone that kstats are not initialized */ 594 saa_portp->saa_pt_kstatp = NULL; 595 596 /* 597 * set up mutexe and state variable to indicate to 598 * other clients that were currently in the process of 599 * setting up the port data. This will prevent a subsequent 600 * client from trying to to register with ibmf before the 601 * port data has been initialized. 602 */ 603 mutex_init(&saa_portp->saa_pt_mutex, NULL, MUTEX_DRIVER, NULL); 604 cv_init(&saa_portp->saa_pt_ibmf_reg_cv, NULL, CV_DRIVER, NULL); 605 606 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_REGISTERING; 607 608 /* create other mutexes */ 609 mutex_init(&saa_portp->saa_pt_kstat_mutex, NULL, MUTEX_DRIVER, NULL); 610 611 mutex_init(&saa_portp->saa_pt_event_sub_mutex, NULL, MUTEX_DRIVER, 612 NULL); 613 614 /* 615 * clients assume all arrive; set mask to this so we only notify 616 * if something failed 617 */ 618 saa_portp->saa_pt_event_sub_last_success_mask = 619 IBMF_SAA_PORT_EVENT_SUB_ALL_ARRIVE; 620 621 /* 622 * set port_guid now so any immediately subsequent clients 623 * registering on this port, guid will know we're already here 624 */ 625 saa_portp->saa_pt_port_guid = pt_guid; 626 saa_portp->saa_pt_reference_count = 1; 627 saa_portp->saa_pt_current_tid = pt_guid << 32; 628 629 /* set sa_uptime now in case we never receive anything from SA */ 630 saa_portp->saa_pt_sa_uptime = gethrtime(); 631 632 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*saa_portp)) 633 634 /* Set new pointer in caller's */ 635 *saa_portpp = saa_portp; 636 637 bail: 638 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_create_port_end, 639 IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port() exit\n"); 640 641 return (status); 642 } 643 644 /* 645 * ibmf_saa_impl_invalidate_port: 646 * invalidates port entry (assumes exist) and deletes kstat object 647 * kstat object is destroyed in order to allow creating port entry 648 * even if this entry is not purged 649 */ 650 static void 651 ibmf_saa_impl_invalidate_port(saa_port_t *saa_portp) 652 { 653 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 654 ibmf_saa_impl_invalidate_port_start, 655 IBMF_TNF_TRACE, "", "ibmf_saa_impl_invalidate_port() enter\n"); 656 657 ASSERT(saa_portp != NULL); 658 ASSERT(MUTEX_HELD(&saa_portp->saa_pt_mutex)); 659 660 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_INVALID; 661 ibmf_saa_impl_uninit_kstats(saa_portp); 662 663 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 664 ibmf_saa_impl_invalidate_port_end, 665 IBMF_TNF_TRACE, "", "ibmf_saa_impl_invalidate_port() exit\n"); 666 } 667 668 /* 669 * ibmf_saa_impl_destroy_port: 670 * Frees the resources associated with an saa_portp structure. Assumes the 671 * saa_portp exists 672 * 673 * Input Arguments 674 * saa_portp pointer to saa_portp structure 675 * 676 * Output Arguments 677 * none 678 * 679 * Returns 680 * void 681 */ 682 static void 683 ibmf_saa_impl_destroy_port(saa_port_t *saa_portp) 684 { 685 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_destroy_start, 686 IBMF_TNF_TRACE, "", "ibmf_saa_impl_destroy() enter\n"); 687 688 ASSERT(saa_portp != NULL); 689 690 _NOTE(ASSUMING_PROTECTED(*saa_portp)) 691 692 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 693 ibmf_saa_impl_destroy, IBMF_TNF_TRACE, "", 694 "ibmf_saa_impl_destroy(): destroying port_guid %016" PRIx64 "\n", 695 tnf_opaque, port_guid, saa_portp->saa_pt_port_guid); 696 697 ibmf_saa_impl_uninit_kstats(saa_portp); 698 699 /* uninit synchronization variables used for registration */ 700 mutex_destroy(&saa_portp->saa_pt_mutex); 701 cv_destroy(&saa_portp->saa_pt_ibmf_reg_cv); 702 703 mutex_destroy(&saa_portp->saa_pt_event_sub_mutex); 704 mutex_destroy(&saa_portp->saa_pt_kstat_mutex); 705 706 kmem_free(saa_portp, sizeof (saa_port_t)); 707 708 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_destroy_end, 709 IBMF_TNF_TRACE, "", "ibmf_saa_impl_destroy() exit\n"); 710 } 711 712 /* 713 * ibmf_saa_impl_init_kstats: 714 * Create kstats structure. Should be called when memory is alloced for a new 715 * port entry. 716 */ 717 int 718 ibmf_saa_impl_init_kstats(saa_port_t *saa_portp) 719 { 720 char buf[128]; 721 ibmf_saa_kstat_t *ksp; 722 723 _NOTE(ASSUMING_PROTECTED(saa_portp->saa_pt_kstatp)) 724 725 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 726 ibmf_saa_impl_init_kstats_start, 727 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_kstats() enter\n"); 728 729 /* set up kstats structure */ 730 (void) sprintf(buf, "ibmf_saa_%016" PRIx64 "_stat", 731 saa_portp->saa_pt_port_guid); 732 733 saa_portp->saa_pt_kstatp = kstat_create("ibmf_saa", 734 0, buf, "misc", KSTAT_TYPE_NAMED, 735 sizeof (ibmf_saa_kstat_t) / sizeof (kstat_named_t), 736 KSTAT_FLAG_WRITABLE); 737 738 if (saa_portp->saa_pt_kstatp == NULL) 739 return (IBMF_NO_RESOURCES); 740 741 ksp = (ibmf_saa_kstat_t *)saa_portp->saa_pt_kstatp->ks_data; 742 743 kstat_named_init(&ksp->clients_registered, 744 "clients_registered", KSTAT_DATA_UINT32); 745 746 kstat_named_init(&ksp->clients_reg_failed, 747 "clients_reg_failed", KSTAT_DATA_UINT32); 748 749 kstat_named_init(&ksp->outstanding_requests, 750 "outstanding_requests", KSTAT_DATA_UINT32); 751 752 kstat_named_init(&ksp->total_requests, 753 "total_requests", KSTAT_DATA_UINT32); 754 755 kstat_named_init(&ksp->failed_requests, 756 "failed_requests", KSTAT_DATA_UINT32); 757 758 kstat_named_init(&ksp->requests_timedout, 759 "requests_timedout", KSTAT_DATA_UINT32); 760 761 kstat_install(saa_portp->saa_pt_kstatp); 762 763 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 764 ibmf_saa_impl_init_kstats_end, 765 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_kstats() exit\n"); 766 767 return (IBMF_SUCCESS); 768 } 769 770 /* 771 * ibmf_saa_impl_uninit_kstats: 772 * Free kstats context. Should be called when port is either destroyed 773 * or invalidated. 774 */ 775 static void 776 ibmf_saa_impl_uninit_kstats(saa_port_t *saa_portp) 777 { 778 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 779 ibmf_saa_impl_uninit_kstats_start, 780 IBMF_TNF_TRACE, "", "ibmf_saa_impl_uninit_kstats() enter\n"); 781 782 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 783 784 if (saa_portp->saa_pt_kstatp != NULL) { 785 kstat_delete(saa_portp->saa_pt_kstatp); 786 } 787 saa_portp->saa_pt_kstatp = NULL; 788 789 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 790 791 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 792 ibmf_saa_impl_uninit_kstats_end, 793 IBMF_TNF_TRACE, "", "ibmf_saa_impl_uninit_kstats() exit\n"); 794 } 795 796 /* 797 * ibmf_saa_impl_register_failed: 798 * invalidate entry and kick waiters 799 */ 800 void 801 ibmf_saa_impl_register_failed(saa_port_t *saa_portp) 802 { 803 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 804 ibmf_saa_impl_register_failed_start, 805 IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_failed() enter\n"); 806 807 mutex_enter(&saa_portp->saa_pt_mutex); 808 809 ibmf_saa_impl_invalidate_port(saa_portp); 810 811 cv_broadcast(&saa_portp->saa_pt_ibmf_reg_cv); 812 813 /* decrementing refcount is last thing we do on entry */ 814 815 ASSERT(saa_portp->saa_pt_reference_count > 0); 816 saa_portp->saa_pt_reference_count--; 817 818 mutex_exit(&saa_portp->saa_pt_mutex); 819 820 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 821 ibmf_saa_impl_register_failed_end, 822 IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_failed() exit\n"); 823 } 824 825 /* 826 * ibmf_saa_impl_register_port: 827 */ 828 int 829 ibmf_saa_impl_register_port( 830 saa_port_t *saa_portp) 831 { 832 uint_t hca_count = 0; 833 ib_guid_t *hca_list = NULL; 834 int status = IBMF_SUCCESS; 835 int unreg_status = IBMF_SUCCESS; 836 int ibt_status = IBT_SUCCESS; 837 ibt_hca_portinfo_t *port_info_list = NULL; 838 uint_t port_count = 0; 839 uint_t port_size = 0; 840 ib_pkey_t p_key; 841 ib_qkey_t q_key; 842 uint8_t portnum; 843 int ihca, iport; 844 ib_guid_t port_guid; 845 boolean_t qp_alloced = B_FALSE; 846 boolean_t ibmf_reg = B_FALSE; 847 848 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 849 ibmf_saa_impl_register_port_start, IBMF_TNF_TRACE, "", 850 "ibmf_saa_impl_register_port() enter\n"); 851 852 ASSERT(saa_portp != NULL); 853 854 _NOTE(ASSUMING_PROTECTED(*saa_portp)) 855 856 /* get the HCA list */ 857 858 hca_count = ibt_get_hca_list(&hca_list); 859 860 if (hca_count == 0) { 861 862 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 863 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 864 "ibmf_saa_impl_register_port: %s\n", 865 tnf_string, msg, "cannot register port (no HCAs).\n"); 866 867 status = IBMF_BAD_PORT; 868 goto bail; 869 } 870 871 /* lookup requested port guid in hca list */ 872 for (ihca = 0; ihca != hca_count; ihca++) { 873 874 ibt_status = ibt_query_hca_ports_byguid(hca_list[ihca], 875 0 /* all ports */, &port_info_list, 876 &port_count, &port_size); 877 878 if (ibt_status != IBT_SUCCESS) { 879 880 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 881 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 882 "ibmf_saa_impl_register_port: %s, %016" PRIx64 "\n", 883 tnf_string, msg, "Could not query hca. Exiting.", 884 tnf_opaque, guid, hca_list[ihca]); 885 886 status = IBMF_TRANSPORT_FAILURE; 887 break; 888 } 889 890 for (iport = 0; iport < port_count; iport++) { 891 892 /* get port guid associated with hca guid, port num */ 893 if (ibmf_saa_impl_get_port_guid( 894 port_info_list + iport, &port_guid) != IBMF_SUCCESS) 895 continue; 896 897 if (saa_portp->saa_pt_port_guid != port_guid) 898 continue; 899 900 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, 901 ibmf_saa_impl_register_port, 902 IBMF_TNF_TRACE, "", 903 "ibmf_saa_impl_register_port: %s, hca_guid = %016" 904 PRIx64 ", port_guid = %016" PRIx64 905 ", number = %d\n", 906 tnf_string, msg, "found port", 907 tnf_opaque, hca_guid, hca_list[ihca], 908 tnf_opaque, port_guid, port_guid, 909 tnf_uint, port, iport + 1); 910 911 /* 912 * we're here? then we found our port: 913 * fill in ibmf registration info 914 * and address parameters from the portinfo 915 */ 916 917 saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid 918 = hca_list[ihca]; 919 saa_portp->saa_pt_ibmf_reginfo.ir_port_num = iport+1; 920 saa_portp->saa_pt_ibmf_reginfo.ir_client_class 921 = SUBN_ADM_MANAGER; 922 923 saa_portp->saa_pt_node_guid = hca_list[ihca]; 924 saa_portp->saa_pt_port_num = iport + 1; 925 926 ibmf_saa_impl_set_transaction_params( 927 saa_portp, port_info_list + iport); 928 break; 929 } 930 931 ibt_free_portinfo(port_info_list, port_size); 932 933 if (iport != port_count) 934 break; /* found our port */ 935 } 936 937 ibt_free_hca_list(hca_list, hca_count); 938 939 if (ihca == hca_count) { 940 941 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 942 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 943 "ibmf_saa_impl_register_port: %s, port_guid %016" 944 PRIx64 "\n", 945 tnf_string, msg, "Could not find port, exiting", 946 tnf_opaque, port_guid, saa_portp->saa_pt_port_guid); 947 948 status = IBMF_BAD_PORT; 949 } 950 951 if (status != IBMF_SUCCESS) { 952 953 goto bail; 954 } 955 956 /* 957 * Now we found the port we searched for, 958 * and open an ibmf session on that port. 959 */ 960 961 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 962 ibmf_saa_impl_register_port, IBMF_TNF_TRACE, "", 963 "ibmf_saa_impl_register_port: %s, port_guid = %016" PRIx64 964 ", port = %d\n", tnf_string, msg, "Registering with ibmf", 965 tnf_opaque, port_guid, saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid, 966 tnf_uint, port, saa_portp->saa_pt_ibmf_reginfo.ir_port_num); 967 968 status = ibmf_register(&saa_portp->saa_pt_ibmf_reginfo, 969 IBMF_VERSION, IBMF_REG_FLAG_RMPP, 970 ibmf_saa_impl_async_event_cb, saa_portp, 971 &saa_portp->saa_pt_ibmf_handle, 972 &saa_portp->saa_pt_ibmf_impl_features); 973 974 if (status != IBMF_SUCCESS) { 975 976 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 977 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 978 "ibmf_saa_impl_register_port: %s, ibmf_status = %d\n", 979 tnf_string, msg, "Could not register with ibmf", 980 tnf_int, status, status); 981 982 goto bail; 983 } 984 985 ibmf_reg = B_TRUE; 986 987 /* allocate a qp through ibmf */ 988 status = ibmf_alloc_qp(saa_portp->saa_pt_ibmf_handle, 989 IB_PKEY_DEFAULT_LIMITED, IB_GSI_QKEY, IBMF_ALT_QP_MAD_RMPP, 990 &saa_portp->saa_pt_qp_handle); 991 992 if (status != IBMF_SUCCESS) { 993 994 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 995 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 996 "ibmf_saa_impl_register_port: %s, ibmf_status = %d\n", 997 tnf_string, msg, "Cannot alloc qp with ibmf", 998 tnf_int, status, status); 999 1000 goto bail; 1001 } 1002 1003 qp_alloced = B_TRUE; 1004 1005 /* 1006 * query the queue pair number; we will need it to unsubscribe from 1007 * notice reports 1008 */ 1009 status = ibmf_query_qp(saa_portp->saa_pt_ibmf_handle, 1010 saa_portp->saa_pt_qp_handle, &saa_portp->saa_pt_qpn, &p_key, &q_key, 1011 &portnum, 0); 1012 if (status != IBMF_SUCCESS) { 1013 1014 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1015 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 1016 "ibmf_saa_impl_register_port: %s, ibmf_status = %d\n", 1017 tnf_string, msg, "Cannot query alt qp to get qp num", 1018 tnf_int, status, status); 1019 1020 goto bail; 1021 } 1022 1023 /* 1024 * core ibmf is taking advantage of the fact that saa_portp is our 1025 * callback arg. If this changes, the code in ibmf_recv would need to 1026 * change as well 1027 */ 1028 status = ibmf_setup_async_cb(saa_portp->saa_pt_ibmf_handle, 1029 saa_portp->saa_pt_qp_handle, ibmf_saa_report_cb, saa_portp, 0); 1030 if (status != IBMF_SUCCESS) { 1031 1032 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1033 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 1034 "ibmf_saa_impl_register_port: %s, ibmf_status = %d\n", 1035 tnf_string, msg, "Cannot register async cb with ibmf", 1036 tnf_int, status, status); 1037 1038 goto bail; 1039 } 1040 1041 return (IBMF_SUCCESS); 1042 1043 bail: 1044 if (qp_alloced == B_TRUE) { 1045 1046 /* free alternate qp */ 1047 unreg_status = ibmf_free_qp(saa_portp->saa_pt_ibmf_handle, 1048 &saa_portp->saa_pt_qp_handle, 0); 1049 if (unreg_status != IBMF_SUCCESS) { 1050 1051 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1052 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 1053 "ibmf_saa_impl_register_port: %s, ibmf_status =" 1054 " %d\n", tnf_string, msg, 1055 "Cannot free alternate queue pair with ibmf", 1056 tnf_int, unreg_status, unreg_status); 1057 } 1058 } 1059 1060 if (ibmf_reg == B_TRUE) { 1061 /* unregister from ibmf */ 1062 unreg_status = ibmf_unregister( 1063 &saa_portp->saa_pt_ibmf_handle, 0); 1064 1065 if (unreg_status != IBMF_SUCCESS) { 1066 1067 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1068 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 1069 "ibmf_saa_impl_register_port: %s, ibmf_status =" 1070 " %d\n", tnf_string, msg, 1071 "Cannot unregister from ibmf", 1072 tnf_int, unreg_status, unreg_status); 1073 } 1074 } 1075 1076 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_register_port_end, 1077 IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_port() exit\n"); 1078 1079 return (status); 1080 } 1081 1082 /* 1083 * ibmf_saa_impl_getclassportinfo: 1084 */ 1085 void 1086 ibmf_saa_impl_get_classportinfo(saa_port_t *saa_portp) 1087 { 1088 int res; 1089 saa_impl_trans_info_t *trans_info; 1090 1091 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1092 ibmf_saa_impl_get_classportinfo_start, 1093 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_classportinfo() enter\n"); 1094 1095 /* 1096 * allocate memory for trans_info; send_request's callback will free up 1097 * memory since request is asynchronous 1098 */ 1099 trans_info = kmem_zalloc(sizeof (saa_impl_trans_info_t), KM_NOSLEEP); 1100 if (trans_info == NULL) { 1101 1102 mutex_enter(&saa_portp->saa_pt_mutex); 1103 1104 /* cpi transaction is handled as a client, decrement refcount */ 1105 ASSERT(saa_portp->saa_pt_reference_count > 0); 1106 saa_portp->saa_pt_reference_count--; 1107 1108 mutex_exit(&saa_portp->saa_pt_mutex); 1109 1110 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1111 ibmf_saa_impl_get_classportinfo_err, IBMF_TNF_ERROR, "", 1112 "ibmf_saa_impl_get_classportinfo: %s\n", tnf_string, msg, 1113 "Could not allocate memory for classportinfo trans_info"); 1114 1115 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1116 ibmf_saa_impl_get_classportinfo_end, IBMF_TNF_TRACE, "", 1117 "ibmf_saa_impl_get_classportinfo() exiting\n"); 1118 1119 return; 1120 } 1121 1122 /* no specific client associated with this transaction */ 1123 trans_info->si_trans_client_data = NULL; 1124 trans_info->si_trans_port = saa_portp; 1125 trans_info->si_trans_method = SA_SUBN_ADM_GET; 1126 trans_info->si_trans_attr_id = MAD_ATTR_ID_CLASSPORTINFO; 1127 1128 trans_info->si_trans_callback = ibmf_saa_impl_get_cpi_cb; 1129 trans_info->si_trans_callback_arg = saa_portp; 1130 1131 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 1132 1133 IBMF_SAA_ADD32_KSTATS(saa_portp, outstanding_requests, 1); 1134 IBMF_SAA_ADD32_KSTATS(saa_portp, total_requests, 1); 1135 1136 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 1137 1138 res = ibmf_saa_impl_send_request(trans_info); 1139 1140 if (res != IBMF_SUCCESS) { 1141 1142 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 1143 ibmf_saa_impl_get_classportinfo_err, IBMF_TNF_TRACE, "", 1144 "ibmf_saa_impl_get_classportinfo: %s, res = 0x%x\n", 1145 tnf_string, msg, "ibmf_saa_impl_send_request failed", 1146 tnf_opaque, res, res); 1147 1148 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 1149 1150 IBMF_SAA_SUB32_KSTATS(saa_portp, outstanding_requests, 1); 1151 IBMF_SAA_ADD32_KSTATS(saa_portp, failed_requests, 1); 1152 1153 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 1154 1155 mutex_enter(&saa_portp->saa_pt_mutex); 1156 1157 /* cpi transaction is handled as a client, decrement refcount */ 1158 ASSERT(saa_portp->saa_pt_reference_count > 0); 1159 saa_portp->saa_pt_reference_count--; 1160 1161 mutex_exit(&saa_portp->saa_pt_mutex); 1162 1163 } 1164 1165 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1166 ibmf_saa_impl_get_classportinfo_end, 1167 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_classportinfo() exit\n"); 1168 } 1169 1170 /* 1171 * ibmf_saa_impl_get_cpi_cb: 1172 * 1173 * Called when the asynchronous getportinfo request receives its response. 1174 * Checks the status. If success, updates the times in the port's 1175 * ibmf_retrans structure that is used in ibmf_msg_transport calls. If failure, 1176 * just use default values. 1177 * 1178 * Input Arguments 1179 * arg user-specified pointer (points to the current port data) 1180 * length length of payload returned (should be size of classportinfo_t) 1181 * buffer pointer to classportinfo returned (should not be null) 1182 * status status of sa access request 1183 * 1184 * Output Arguments 1185 * none 1186 * 1187 * Returns void 1188 */ 1189 static void 1190 ibmf_saa_impl_get_cpi_cb(void *arg, size_t length, char *buffer, int status) 1191 { 1192 saa_port_t *saa_portp; 1193 uint64_t base_time, resp_timeout, rttv_timeout; 1194 ib_mad_classportinfo_t *classportinfo; 1195 int resp_time_value; 1196 uint16_t sa_cap_mask; 1197 1198 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_get_cpi_cb_start, 1199 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_cpi_cb() enter\n"); 1200 1201 /* 1202 * access port entry: note that it may have become invalid 1203 * but we hold a ref count for cpi and the interactions on 1204 * the entry are harmless 1205 */ 1206 saa_portp = (saa_port_t *)arg; 1207 1208 /* process response */ 1209 1210 if ((status != IBMF_SUCCESS) || (buffer == NULL)) { 1211 1212 IBMF_TRACE_4(IBMF_TNF_NODEBUG, DPRINT_L1, 1213 ibmf_saa_impl_get_cpi_cb, IBMF_TNF_ERROR, "", 1214 "ibmf_saa_impl_get_cpi_cb: %s, status = %d, buffer = " 1215 " 0x%p, length = %d\n", tnf_string, msg, 1216 "could not get classportinfo. Check node and path to sm" 1217 " lid", tnf_int, status, status, 1218 tnf_opaque, buffer, buffer, tnf_uint, length, length); 1219 1220 /* 1221 * IB spec (C13-13) indicates 20 can be used as default or 1222 * intial value for classportinfo->resptimeout value 1223 */ 1224 resp_time_value = 20; 1225 1226 sa_cap_mask = 0xFFFF; 1227 1228 } else if (buffer != NULL) { 1229 1230 classportinfo = (ib_mad_classportinfo_t *)buffer; 1231 1232 resp_time_value = b2h32(classportinfo->RespTimeValue) & 0x1f; 1233 1234 sa_cap_mask = b2h16(classportinfo->CapabilityMask); 1235 1236 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 1237 ibmf_saa_impl_get_cpi_cb, IBMF_TNF_TRACE, "", 1238 "ibmf_saa_impl_get_cpi_cb: %s, timeout = 0x%x," 1239 " cap_mask = 0x%x\n", 1240 tnf_string, msg, "got classportinfo", 1241 tnf_opaque, timeout, resp_time_value, 1242 tnf_opaque, cap_mask, sa_cap_mask); 1243 1244 kmem_free(buffer, length); 1245 } 1246 1247 /* 1248 * using IB spec calculation from 13.4.6.2 1249 * use bit shifting for 2^x. 1250 */ 1251 base_time = (1 << resp_time_value); 1252 1253 resp_timeout = (4 * base_time * 1000 + 96 * base_time) / 1000; 1254 1255 mutex_enter(&saa_portp->saa_pt_mutex); 1256 1257 base_time = 2 * (1 << saa_portp->saa_pt_timeout); 1258 1259 rttv_timeout = (4 * base_time * 1000 + 96 * base_time) / 1000; 1260 1261 saa_portp->saa_pt_ibmf_retrans.retrans_rtv = resp_timeout; 1262 saa_portp->saa_pt_ibmf_retrans.retrans_rttv = rttv_timeout; 1263 saa_portp->saa_pt_sa_cap_mask = sa_cap_mask; 1264 1265 /* 1266 * cpi transaction is handled as a client, 1267 * decrement refcount; make sure it's the last 1268 * thing we do on this entry 1269 */ 1270 ASSERT(saa_portp->saa_pt_reference_count > 0); 1271 saa_portp->saa_pt_reference_count--; 1272 1273 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 1274 ibmf_saa_impl_get_cpi_cb, IBMF_TNF_TRACE, "", 1275 "ibmf_saa_impl_get_cpi_cb: %s, subnet_timeout = 0x%x, " 1276 "resp_time_value = 0x%x\n", 1277 tnf_string, msg, "updated resp timeout", 1278 tnf_opaque, subnet_timeout, saa_portp->saa_pt_timeout, 1279 tnf_opaque, resp_time_value, resp_time_value); 1280 1281 mutex_exit(&saa_portp->saa_pt_mutex); 1282 1283 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_get_cpi_cb_end, 1284 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_cpi_cb() exit\n"); 1285 } 1286 1287 /* 1288 * ibmf_saa_impl_send_request: 1289 * Sends a request to the sa. Can be used for both classportinfo and record 1290 * requests. Will set up all data structures for using the multi-packet 1291 * protocol, create the mad, and send it. Returns SA_SUCCESS if msg transport 1292 * worked, meaning succesful send for the async case and a succesful send and 1293 * recv for the sync case. 1294 */ 1295 int 1296 ibmf_saa_impl_send_request(saa_impl_trans_info_t *trans_info) 1297 { 1298 uint16_t attr_id; 1299 saa_client_data_t *client_data; 1300 saa_port_t *saa_portp; 1301 uint32_t transport_flags; 1302 ibmf_msg_cb_t ibmf_callback; 1303 void *ibmf_callback_arg; 1304 ibmf_msg_t *msgp; 1305 ibmf_retrans_t ibmf_retrans; 1306 uint16_t sa_cap_mask; 1307 boolean_t sleep_flag; 1308 int ibmf_status = IBMF_SUCCESS; 1309 int retry_count; 1310 uint16_t mad_status; 1311 1312 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1313 ibmf_saa_impl_send_request_start, 1314 IBMF_TNF_TRACE, "", "ibmf_saa_impl_send_request() enter\n"); 1315 1316 attr_id = trans_info->si_trans_attr_id; 1317 client_data = trans_info->si_trans_client_data; 1318 saa_portp = trans_info->si_trans_port; 1319 1320 /* 1321 * don't send on invalid entry 1322 * Note that there is a window where it could become 1323 * invalid after this test is done, but we'd rely on ibmf errors... 1324 */ 1325 if (ibmf_saa_is_valid(saa_portp, B_FALSE) == B_FALSE) { 1326 1327 IBMF_TRACE_4(IBMF_TNF_NODEBUG, DPRINT_L1, 1328 ibmf_saa_impl_send_request, 1329 IBMF_TNF_ERROR, "", 1330 "ibmf_saa_impl_send_request: %s, hca_guid = %016" 1331 PRIx64 ", port_guid = %016" PRIx64 1332 ", number = %d\n", 1333 tnf_string, msg, "sending on invalid port", 1334 tnf_opaque, hca_guid, 1335 saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid, 1336 tnf_opaque, port_guid, 1337 saa_portp->saa_pt_port_guid, 1338 tnf_uint, port, 1339 saa_portp->saa_pt_ibmf_reginfo.ir_port_num); 1340 1341 ibmf_status = IBMF_REQ_INVALID; 1342 goto bail; 1343 } 1344 1345 /* check whether SA supports this attribute */ 1346 mutex_enter(&saa_portp->saa_pt_mutex); 1347 1348 sa_cap_mask = saa_portp->saa_pt_sa_cap_mask; 1349 1350 mutex_exit(&saa_portp->saa_pt_mutex); 1351 1352 ibmf_status = ibmf_saa_impl_check_sa_support(sa_cap_mask, attr_id); 1353 1354 if (ibmf_status != IBMF_SUCCESS) { 1355 1356 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1357 ibmf_saa_impl_send_request_err, IBMF_TNF_ERROR, "", 1358 "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n", 1359 tnf_string, msg, "SA does not support attribute", 1360 tnf_int, ibmf_status, ibmf_status); 1361 1362 goto bail; 1363 } 1364 1365 /* make only non-blocking calls if this is an async request */ 1366 if ((trans_info->si_trans_callback == NULL) && 1367 (trans_info->si_trans_sub_callback == NULL)) { 1368 ibmf_callback = NULL; 1369 ibmf_callback_arg = NULL; 1370 sleep_flag = B_TRUE; 1371 } else { 1372 ibmf_callback = ibmf_saa_async_cb; 1373 ibmf_callback_arg = (void *)trans_info; 1374 sleep_flag = B_FALSE; 1375 } 1376 1377 ibmf_status = ibmf_saa_impl_init_msg(trans_info, sleep_flag, &msgp, 1378 &transport_flags, &ibmf_retrans); 1379 if (ibmf_status != IBMF_SUCCESS) { 1380 1381 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1382 ibmf_saa_impl_send_request_err, IBMF_TNF_ERROR, "", 1383 "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n", 1384 tnf_string, msg, "init_msg() failed", 1385 tnf_int, ibmf_status, ibmf_status); 1386 1387 goto bail; 1388 } 1389 1390 mutex_enter(&saa_portp->saa_pt_mutex); 1391 1392 saa_portp->saa_pt_num_outstanding_trans++; 1393 1394 mutex_exit(&saa_portp->saa_pt_mutex); 1395 1396 /* 1397 * increment the number of outstanding transaction so 1398 * ibmf_close_sa_session() will wait. classportinfo requests 1399 * don't have associated clients so check for valid clientp 1400 */ 1401 if (client_data != NULL) { 1402 1403 mutex_enter(&client_data->saa_client_mutex); 1404 1405 client_data->saa_client_num_pending_trans++; 1406 1407 mutex_exit(&client_data->saa_client_mutex); 1408 } 1409 1410 /* 1411 * make the call to msg_transport. If synchronous and success, 1412 * check that the response mad isn't status busy. If so, repeat the 1413 * call 1414 */ 1415 retry_count = 0; 1416 1417 /* 1418 * set the send time here. We only set this once at the beginning of 1419 * the transaction. Retrying because of busys or mastersmlid changes 1420 * does not change the original send time. It is meant to be an 1421 * absolute time out value and will only be used if there are other 1422 * problems (i.e. a buggy SA) 1423 */ 1424 trans_info->si_trans_send_time = gethrtime(); 1425 1426 for (;;) { 1427 1428 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle, 1429 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 1430 ibmf_callback, ibmf_callback_arg, transport_flags); 1431 1432 if (ibmf_callback != NULL) 1433 break; 1434 1435 /* 1436 * stop here for non-sequenced transactions since they wouldn't 1437 * receive a timeout or busy response 1438 */ 1439 if (!(transport_flags & IBMF_MSG_TRANS_FLAG_SEQ)) 1440 break; 1441 1442 /* 1443 * if the transaction timed out and this was a synchronous 1444 * request there's a possiblity we were talking to the wrong 1445 * master smlid. Check this and retry if necessary. 1446 */ 1447 if ((ibmf_status == IBMF_TRANS_TIMEOUT) && 1448 (sleep_flag == B_TRUE)) { 1449 ibmf_status = ibmf_saa_impl_new_smlid_retry(saa_portp, 1450 msgp, ibmf_callback, ibmf_callback_arg, 1451 transport_flags); 1452 } 1453 1454 /* 1455 * if the transaction timed out (and retrying with a new SM LID 1456 * didn't help) check how long it's been since we received an SA 1457 * packet. If it hasn't been max_wait_time then retry the 1458 * request. 1459 */ 1460 if ((ibmf_status == IBMF_TRANS_TIMEOUT) && 1461 (sleep_flag == B_TRUE)) { 1462 1463 ibmf_status = ibmf_saa_check_sa_and_retry( 1464 saa_portp, msgp, ibmf_callback, ibmf_callback_arg, 1465 trans_info->si_trans_send_time, transport_flags); 1466 } 1467 1468 if (ibmf_status != IBMF_SUCCESS) 1469 break; 1470 1471 if (retry_count >= IBMF_SAA_MAX_BUSY_RETRY_COUNT) 1472 break; 1473 1474 /* sync transaction with status SUCCESS should have response */ 1475 ASSERT(msgp->im_msgbufs_recv.im_bufs_mad_hdr != NULL); 1476 1477 mad_status = b2h16(msgp->im_msgbufs_recv. 1478 im_bufs_mad_hdr->Status); 1479 1480 if (mad_status != MAD_STATUS_BUSY) 1481 break; 1482 1483 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 1484 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1485 "ibmf_saa_impl_send_request: %s, retry_count = %d\n", 1486 tnf_string, msg, "response returned busy status", 1487 tnf_int, retry_count, retry_count); 1488 1489 retry_count++; 1490 1491 /* 1492 * since this is a blocking call, sleep for some time 1493 * to allow SA to transition from busy state 1494 */ 1495 delay(drv_usectohz(IBMF_SAA_BUSY_RETRY_SLEEP_SECS * 1000000)); 1496 } 1497 1498 if (ibmf_status != IBMF_SUCCESS) { 1499 1500 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 1501 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1502 "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n", 1503 tnf_string, msg, "ibmf_msg_transport() failed", 1504 tnf_int, ibmf_status, ibmf_status); 1505 1506 ibmf_saa_impl_free_msg(saa_portp->saa_pt_ibmf_handle, msgp); 1507 1508 mutex_enter(&saa_portp->saa_pt_mutex); 1509 1510 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0); 1511 saa_portp->saa_pt_num_outstanding_trans--; 1512 1513 mutex_exit(&saa_portp->saa_pt_mutex); 1514 1515 if (client_data != NULL) { 1516 1517 mutex_enter(&client_data->saa_client_mutex); 1518 1519 ASSERT(client_data->saa_client_num_pending_trans > 0); 1520 client_data->saa_client_num_pending_trans--; 1521 1522 if ((client_data->saa_client_num_pending_trans == 0) && 1523 (client_data->saa_client_state == 1524 SAA_CLIENT_STATE_WAITING)) 1525 cv_signal(&client_data->saa_client_state_cv); 1526 1527 mutex_exit(&client_data->saa_client_mutex); 1528 } 1529 1530 } else if (sleep_flag == B_TRUE) { 1531 1532 mutex_enter(&saa_portp->saa_pt_mutex); 1533 1534 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0); 1535 saa_portp->saa_pt_num_outstanding_trans--; 1536 1537 mutex_exit(&saa_portp->saa_pt_mutex); 1538 1539 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 1540 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1541 "ibmf_saa_impl_send_request: %s\n", 1542 tnf_string, msg, "Message sent and received successfully"); 1543 1544 /* fill in response values and free the message */ 1545 ibmf_saa_impl_prepare_response(saa_portp->saa_pt_ibmf_handle, 1546 msgp, B_FALSE, &trans_info->si_trans_status, 1547 &trans_info->si_trans_result, 1548 &trans_info->si_trans_length, sleep_flag); 1549 1550 if (client_data != NULL) { 1551 mutex_enter(&client_data->saa_client_mutex); 1552 1553 ASSERT(client_data->saa_client_num_pending_trans > 0); 1554 client_data->saa_client_num_pending_trans--; 1555 1556 if ((client_data->saa_client_num_pending_trans == 0) && 1557 (client_data->saa_client_state == 1558 SAA_CLIENT_STATE_WAITING)) 1559 cv_signal(&client_data->saa_client_state_cv); 1560 1561 mutex_exit(&client_data->saa_client_mutex); 1562 } 1563 } else { 1564 1565 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 1566 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1567 "ibmf_saa_impl_send_request: %s\n", 1568 tnf_string, msg, "Message sent successfully"); 1569 } 1570 1571 bail: 1572 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 1573 ibmf_saa_impl_send_request_end, 1574 IBMF_TNF_TRACE, "", "ibmf_saa_impl_send_request() exiting" 1575 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 1576 1577 return (ibmf_status); 1578 } 1579 1580 /* 1581 * ibmf_saa_impl_init_msg: 1582 * Allocates an ibmf message and fills out the header fields and formatted data 1583 * fields. Also sets up the correct transport_flags and retrans argument for 1584 * the message transport call based on the request information. 1585 * 1586 * Input Arguments 1587 * trans_info saa_trans_info structure passed to send_request 1588 * sleep_flag B_TRUE if init_msg can sleep in function calls 1589 * 1590 * Output Arguments 1591 * msgp ibmf message that should be given to msg_transport 1592 * transport_flagsp transport flags that should be given to msg_transport 1593 * ibmf_retrans_t retrans parameter that should be given to msg_transport 1594 * 1595 * Returns 1596 * ibmf_status 1597 */ 1598 static int 1599 ibmf_saa_impl_init_msg(saa_impl_trans_info_t *trans_info, boolean_t sleep_flag, 1600 ibmf_msg_t **msgp, uint32_t *transport_flagsp, 1601 ibmf_retrans_t *ibmf_retransp) 1602 { 1603 int ibmf_status; 1604 ibmf_msg_bufs_t *req_mad; 1605 ib_mad_hdr_t *mad_hdr; 1606 int ibmf_sleep_flag, km_sleep_flag; 1607 int free_res; 1608 ib_sa_hdr_t sa_hdr; 1609 ibmf_msg_t *ibmf_msg; 1610 uint16_t attr_id, pack_attr_id; 1611 uint8_t method; 1612 saa_client_data_t *client_data; 1613 saa_port_t *saa_portp; 1614 sa_multipath_record_t *multipath_template; 1615 size_t payload_length; 1616 uint32_t transport_flags; 1617 1618 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1619 ibmf_saa_impl_init_msg_start, 1620 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_msg() entering\n"); 1621 1622 attr_id = trans_info->si_trans_attr_id; 1623 method = trans_info->si_trans_method; 1624 client_data = trans_info->si_trans_client_data; 1625 saa_portp = trans_info->si_trans_port; 1626 1627 if (sleep_flag == B_TRUE) { 1628 ibmf_sleep_flag = IBMF_ALLOC_SLEEP; 1629 km_sleep_flag = KM_SLEEP; 1630 } else { 1631 ibmf_sleep_flag = IBMF_ALLOC_NOSLEEP; 1632 km_sleep_flag = KM_NOSLEEP; 1633 } 1634 1635 ibmf_status = ibmf_alloc_msg(saa_portp->saa_pt_ibmf_handle, 1636 ibmf_sleep_flag, &ibmf_msg); 1637 if (ibmf_status != IBMF_SUCCESS) { 1638 1639 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1640 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "", 1641 "ibmf_saa_impl_init_msg: %s, ibmf_status = %d\n", 1642 tnf_string, msg, "Cannot allocate msg_buf.", 1643 tnf_int, ibmf_status, ibmf_status); 1644 1645 goto bail; 1646 } 1647 1648 req_mad = &ibmf_msg->im_msgbufs_send; 1649 1650 /* create a template (SA MAD) */ 1651 mad_hdr = kmem_zalloc(sizeof (ib_mad_hdr_t), km_sleep_flag); 1652 1653 if (mad_hdr == NULL) { 1654 1655 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1656 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "", 1657 "ibmf_saa_impl_init_msg: %s\n", 1658 tnf_string, msg, "Cannot allocate mad header."); 1659 1660 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle, 1661 &ibmf_msg); 1662 ASSERT(free_res == IBMF_SUCCESS); 1663 1664 ibmf_status = IBMF_NO_MEMORY; 1665 goto bail; 1666 } 1667 1668 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mad_hdr, 1669 *req_mad)) 1670 1671 bzero(mad_hdr, sizeof (ib_mad_hdr_t)); 1672 mad_hdr->BaseVersion = SAA_MAD_BASE_VERSION; 1673 mad_hdr->MgmtClass = MAD_MGMT_CLASS_SUBN_ADM; 1674 mad_hdr->ClassVersion = SAA_MAD_CLASS_VERSION; 1675 mad_hdr->R_Method = method; 1676 mad_hdr->AttributeID = h2b16(attr_id); 1677 1678 /* attribute modifier is all Fs since RIDs are no longer used */ 1679 mad_hdr->AttributeModifier = h2b32(0xffffffff); 1680 1681 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, 1682 ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "", 1683 "ibmf_saa_impl_init_msg: %s, class = 0x%x, method = 0x%x," 1684 " attr_id = 0x%x\n", tnf_string, msg, "Sending MAD", 1685 tnf_opaque, class, mad_hdr->MgmtClass, 1686 tnf_opaque, method, mad_hdr->R_Method, 1687 tnf_opaque, attr_id, attr_id); 1688 1689 bzero(&sa_hdr, sizeof (ib_sa_hdr_t)); 1690 sa_hdr.ComponentMask = trans_info->si_trans_component_mask; 1691 1692 if (client_data != NULL) 1693 sa_hdr.SM_KEY = client_data->saa_client_sm_key; 1694 1695 /* 1696 * pack data for IB wire format; req_mad will have different pointers to 1697 * sa header and payload, mad_hdr will be the same 1698 */ 1699 req_mad->im_bufs_mad_hdr = mad_hdr; 1700 1701 ibmf_status = ibmf_saa_utils_pack_sa_hdr(&sa_hdr, 1702 &req_mad->im_bufs_cl_hdr, &req_mad->im_bufs_cl_hdr_len, 1703 km_sleep_flag); 1704 1705 if (ibmf_status != IBMF_SUCCESS) { 1706 1707 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1708 ibmf_saa_impl_init_msg, IBMF_TNF_ERROR, "", 1709 "ibmf_saa_impl_init_msg: %s, ibmf_status = %d\n", 1710 tnf_string, msg, "ibmf_saa_utils_pack_sa_hdr() failed", 1711 tnf_int, ibmf_status, ibmf_status); 1712 1713 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t)); 1714 1715 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle, 1716 &ibmf_msg); 1717 ASSERT(free_res == IBMF_SUCCESS); 1718 1719 goto bail; 1720 } 1721 1722 if (attr_id == SA_MULTIPATHRECORD_ATTRID) { 1723 1724 multipath_template = 1725 (sa_multipath_record_t *)trans_info->si_trans_template; 1726 1727 payload_length = sizeof (sa_multipath_record_t) + 1728 ((multipath_template->SGIDCount + 1729 multipath_template->DGIDCount) * sizeof (ib_gid_t)); 1730 1731 pack_attr_id = attr_id; 1732 } else { 1733 1734 /* trace record template is a path record */ 1735 pack_attr_id = (attr_id == SA_TRACERECORD_ATTRID) ? 1736 SA_PATHRECORD_ATTRID : attr_id; 1737 1738 payload_length = ibmf_saa_impl_get_attr_id_length(pack_attr_id); 1739 1740 if (payload_length == 0) { 1741 payload_length = trans_info->si_trans_template_length; 1742 1743 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 1744 ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "", 1745 "ibmf_saa_impl_init_msg: %s, length = %d\n", 1746 tnf_string, msg, 1747 "Unknown attribute. Using user-defined length.", 1748 tnf_uint, length, payload_length) 1749 } 1750 } 1751 1752 /* transport type depends on method */ 1753 switch (method) { 1754 1755 case SA_SUBN_ADM_GET: 1756 case SA_SUBN_ADM_DELETE: 1757 case SA_SUBN_ADM_GET_TABLE: 1758 case SA_SUBN_ADM_GET_TRACE_TABLE: 1759 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ; 1760 break; 1761 case SA_SUBN_ADM_SET: 1762 /* unsubscribes can be sequenced or unsequenced */ 1763 if (trans_info->si_trans_unseq_unsubscribe == B_TRUE) { 1764 transport_flags = 0; 1765 } else { 1766 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ; 1767 } 1768 break; 1769 case SA_SUBN_ADM_GET_MULTI: 1770 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ | 1771 IBMF_MSG_TRANS_FLAG_RMPP; 1772 break; 1773 default : 1774 ibmf_status = IBMF_UNSUPP_METHOD; 1775 goto bail; 1776 } 1777 1778 trans_info->si_trans_transport_flags = transport_flags; 1779 1780 if (trans_info->si_trans_template != NULL) { 1781 1782 ibmf_status = ibmf_saa_utils_pack_payload( 1783 trans_info->si_trans_template, payload_length, pack_attr_id, 1784 &req_mad->im_bufs_cl_data, &req_mad->im_bufs_cl_data_len, 1785 km_sleep_flag); 1786 if (ibmf_status != IBMF_SUCCESS) { 1787 1788 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1789 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "", 1790 "ibmf_saa_impl_init_msg: %s, ibmf_status =" 1791 " %d\n", tnf_string, msg, 1792 "ibmf_saa_utils_pack_payload() failed", 1793 tnf_int, ibmf_status, ibmf_status); 1794 1795 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t)); 1796 1797 kmem_free(req_mad->im_bufs_cl_hdr, 1798 req_mad->im_bufs_cl_hdr_len); 1799 1800 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle, 1801 &ibmf_msg); 1802 ASSERT(free_res == IBMF_SUCCESS); 1803 1804 goto bail; 1805 } 1806 1807 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 1808 ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "", 1809 "ibmf_saa_impl_init_msg: %s, attr_id = 0x%x, length =" 1810 " %d\n", tnf_string, msg, "Packed payload successfully", 1811 tnf_opaque, attr_id, attr_id, 1812 tnf_uint, length, req_mad->im_bufs_cl_data_len); 1813 1814 /* non-RMPP transactions have template size limit */ 1815 if (((transport_flags & IBMF_MSG_TRANS_FLAG_RMPP) == 0) && 1816 ((req_mad->im_bufs_cl_data_len + req_mad->im_bufs_cl_hdr_len 1817 + sizeof (ib_mad_hdr_t)) > IBMF_MAD_SIZE)) { 1818 1819 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1820 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "", 1821 "ibmf_saa_impl_init_msg: %s\n", tnf_string, msg, 1822 "Template too large to fit in single packet"); 1823 1824 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t)); 1825 1826 kmem_free(req_mad->im_bufs_cl_hdr, 1827 req_mad->im_bufs_cl_hdr_len); 1828 1829 kmem_free(req_mad->im_bufs_cl_data, 1830 req_mad->im_bufs_cl_data_len); 1831 1832 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle, 1833 &ibmf_msg); 1834 ASSERT(free_res == IBMF_SUCCESS); 1835 1836 ibmf_status = IBMF_REQ_INVALID; 1837 goto bail; 1838 } 1839 } 1840 1841 mutex_enter(&saa_portp->saa_pt_mutex); 1842 1843 mad_hdr->TransactionID = h2b64(saa_portp->saa_pt_current_tid++); 1844 1845 bcopy(&saa_portp->saa_pt_ibmf_retrans, ibmf_retransp, 1846 sizeof (ibmf_retrans_t)); 1847 1848 /* copy local addressing information to message */ 1849 bcopy(&saa_portp->saa_pt_ibmf_addr_info, &ibmf_msg->im_local_addr, 1850 sizeof (ibmf_addr_info_t)); 1851 1852 mutex_exit(&saa_portp->saa_pt_mutex); 1853 1854 *msgp = ibmf_msg; 1855 *transport_flagsp = transport_flags; 1856 bail: 1857 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 1858 ibmf_saa_impl_init_msg_end, 1859 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_msg() exiting" 1860 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 1861 1862 return (ibmf_status); 1863 1864 } 1865 1866 /* 1867 * ibmf_saa_impl_new_smlid_retry: 1868 * 1869 * It's possible for the MasterSMLID to change while ibmf_saa is running. The 1870 * MasterSMLID is set when we first register with ibmf_saa. If a request 1871 * timesout, this function should be called to check whether the SM LID changed. 1872 * If so, it will call msg_transport again with the request. 1873 * 1874 * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the 1875 * same values passed to the original ibmf_msg_transport that timedout. The 1876 * ibmf_retrans parameter will be re-retrieved from the saa_portp structure. 1877 * 1878 * If the lid did not change then this function returns IBMF_TRANS_TIMEOUT. 1879 * That way, callers can simply return the result of this function. 1880 * 1881 * Input Arguments 1882 * saa_portp pointer to saa_port structure 1883 * msgp ibmf message that timedout 1884 * ibmf_callback callback that should be called by msg_transport 1885 * ibmf_callback_arg args for ibmf_callback 1886 * transport_flags flags for ibmf_msg_transport 1887 * 1888 * Output Arguments 1889 * none 1890 * 1891 * Returns 1892 * IBMF_SUCCESS if lid changed and request was resent successfully, 1893 * IBMF_TRANS_TIMEOUT if lid did not change, 1894 * same values as ibmf_msg_transport() if lid changed but request could not be 1895 * resent. 1896 */ 1897 static int 1898 ibmf_saa_impl_new_smlid_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp, 1899 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, int transport_flags) 1900 { 1901 ibt_hca_portinfo_t *ibt_portinfop; 1902 ib_lid_t master_sm_lid; 1903 int subnet_timeout; 1904 uint_t nports, size; 1905 ibmf_retrans_t ibmf_retrans; 1906 int ibmf_status; 1907 ibt_status_t ibt_status; 1908 1909 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1910 ibmf_saa_impl_new_smlid_retry_start, 1911 IBMF_TNF_TRACE, "", "ibmf_saa_impl_new_smlid_retry() enter\n"); 1912 1913 _NOTE(ASSUMING_PROTECTED(*msgp)) 1914 _NOTE(ASSUMING_PROTECTED(*msgp->im_msgbufs_send.im_bufs_mad_hdr)) 1915 1916 /* first query the portinfo to see if the lid changed */ 1917 ibt_status = ibt_query_hca_ports_byguid(saa_portp->saa_pt_node_guid, 1918 saa_portp->saa_pt_port_num, &ibt_portinfop, &nports, &size); 1919 1920 if (ibt_status != IBT_SUCCESS) { 1921 1922 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1923 ibmf_saa_impl_new_smlid_retry_err, IBMF_TNF_ERROR, "", 1924 "ibmf_saa_impl_new_smlid_retry: %s, ibmf_status =" 1925 " %d\n", tnf_string, msg, 1926 "ibt_query_hca_ports_byguid() failed", 1927 tnf_int, ibt_status, ibt_status); 1928 1929 ibmf_status = IBMF_TRANSPORT_FAILURE; 1930 1931 goto bail; 1932 } 1933 1934 master_sm_lid = ibt_portinfop->p_sm_lid; 1935 subnet_timeout = ibt_portinfop->p_subnet_timeout; 1936 1937 ibt_free_portinfo(ibt_portinfop, size); 1938 1939 /* if master smlid is different than the remote lid we sent to */ 1940 if (master_sm_lid != msgp->im_local_addr.ia_remote_lid) { 1941 1942 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L2, 1943 ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "", 1944 "ibmf_saa_impl_new_smlid_retry: %s, new_lid 0x%x," 1945 " old_lid 0x%x\n", tnf_string, msg, 1946 "master smlid has changed. retrying msg_transport", 1947 tnf_opaque, new_lid, master_sm_lid, 1948 tnf_opaque, old_lid, msgp->im_local_addr.ia_remote_lid); 1949 1950 mutex_enter(&saa_portp->saa_pt_mutex); 1951 1952 /* update the master sm lid value in ibmf_saa */ 1953 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = 1954 master_sm_lid; 1955 1956 /* new tid needed */ 1957 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID = 1958 h2b64(saa_portp->saa_pt_current_tid++); 1959 1960 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans, 1961 sizeof (ibmf_retrans_t)); 1962 1963 /* update the subnet timeout since this may be a new sm/sa */ 1964 saa_portp->saa_pt_timeout = subnet_timeout; 1965 1966 /* place upper bound on subnet timeout in case of faulty SM */ 1967 if (saa_portp->saa_pt_timeout > IBMF_SAA_MAX_SUBNET_TIMEOUT) 1968 saa_portp->saa_pt_timeout = IBMF_SAA_MAX_SUBNET_TIMEOUT; 1969 1970 /* increment the reference count to account for the cpi call */ 1971 saa_portp->saa_pt_reference_count++; 1972 1973 mutex_exit(&saa_portp->saa_pt_mutex); 1974 1975 /* update the remote lid for this particular message */ 1976 msgp->im_local_addr.ia_remote_lid = master_sm_lid; 1977 1978 /* get the classportinfo again since this may be a new sm/sa */ 1979 ibmf_saa_impl_get_classportinfo(saa_portp); 1980 1981 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle, 1982 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 1983 ibmf_callback, ibmf_callback_arg, transport_flags); 1984 1985 if (ibmf_status != IBMF_SUCCESS) { 1986 1987 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 1988 ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "", 1989 "ibmf_saa_impl_new_smlid_retry: %s, ibmf_status = " 1990 "%d\n", tnf_string, msg, 1991 "ibmf_msg_transport() failed", 1992 tnf_int, ibmf_status, ibmf_status); 1993 } 1994 1995 goto bail; 1996 } 1997 1998 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 1999 ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "", 2000 "ibmf_saa_impl_new_smlid_retry: %s, master_smlid = 0x%x\n", 2001 tnf_string, msg, 2002 "master smlid did not change. returning failure", 2003 tnf_opaque, master_smlid, master_sm_lid); 2004 2005 /* mark status as timeout since that was original failure */ 2006 ibmf_status = IBMF_TRANS_TIMEOUT; 2007 2008 bail: 2009 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2010 ibmf_saa_impl_new_smlid_retry_end, 2011 IBMF_TNF_TRACE, "", "ibmf_saa_impl_new_smlid_retry() exiting" 2012 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 2013 2014 return (ibmf_status); 2015 } 2016 2017 /* 2018 * ibmf_saa_impl_async_event_cb: 2019 * ibmf event callback, argument to ibmf_register 2020 * ibmf_handle is unused 2021 */ 2022 /* ARGSUSED */ 2023 static void 2024 ibmf_saa_impl_async_event_cb( 2025 ibmf_handle_t ibmf_handle, 2026 void *clnt_private, 2027 ibmf_async_event_t event_type) 2028 { 2029 saa_port_t *saa_portp; 2030 2031 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2032 ibmf_saa_impl_async_event_cb_start, IBMF_TNF_TRACE, "", 2033 "ibmf_saa_impl_async_event_cb: Handling event type 0x%x\n", 2034 tnf_opaque, event_type, event_type); 2035 2036 saa_portp = (saa_port_t *)clnt_private; 2037 ASSERT(saa_portp != NULL); 2038 2039 switch (event_type) { 2040 2041 case IBMF_CI_OFFLINE: 2042 ibmf_saa_impl_hca_detach(saa_portp); 2043 break; 2044 default: 2045 break; 2046 } 2047 2048 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3, 2049 ibmf_saa_impl_async_event_cb_end, 2050 IBMF_TNF_TRACE, "", "ibmf_saa_impl_async_event_cb() exit\n"); 2051 } 2052 2053 2054 /* 2055 * ibmf_saa_impl_ibt_async_handler: 2056 * MUST NOT BE STATIC (referred from within IBMF) 2057 */ 2058 void 2059 ibmf_saa_impl_ibt_async_handler(ibt_async_code_t code, ibt_async_event_t *event) 2060 { 2061 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2062 ibmf_saa_impl_ibt_async_handler_start, IBMF_TNF_TRACE, "", 2063 "ibmf_saa_impl_ibt_async_handler: Handling event code 0x%x\n", 2064 tnf_opaque, code, code); 2065 2066 switch (code) { 2067 2068 case IBT_EVENT_PORT_UP: 2069 ibmf_saa_impl_port_up(event->ev_hca_guid, event->ev_port); 2070 break; 2071 case IBT_ERROR_PORT_DOWN: 2072 ibmf_saa_impl_port_down(event->ev_hca_guid, event->ev_port); 2073 break; 2074 default: 2075 break; 2076 } 2077 2078 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_async_handler_end, 2079 IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibt_async_handler() exit\n"); 2080 } 2081 2082 /* 2083 * ibmf_saa_impl_port_up: 2084 */ 2085 static void 2086 ibmf_saa_impl_port_up(ib_guid_t ci_guid, uint8_t port_num) 2087 { 2088 saa_port_t *saa_portp = NULL; 2089 int is_ready; 2090 ibt_hca_portinfo_t *ibt_portinfop; 2091 ib_lid_t master_sm_lid; 2092 uint_t nports, size; 2093 ibt_status_t ibt_status; 2094 boolean_t event_subs = B_FALSE; 2095 2096 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_up_start, 2097 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_up: Handling port up" 2098 " guid %016" PRIx64 " port %d\n", 2099 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num); 2100 2101 /* Get classportinfo of corresponding entry */ 2102 mutex_enter(&saa_statep->saa_port_list_mutex); 2103 2104 saa_portp = saa_statep->saa_port_list; 2105 while (saa_portp != NULL) { 2106 2107 if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid && 2108 saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) { 2109 2110 mutex_enter(&saa_portp->saa_pt_mutex); 2111 2112 is_ready = (saa_portp->saa_pt_state 2113 == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE; 2114 2115 /* 2116 * increment reference count to account for cpi and 2117 * informinfos. All 4 informinfo's sent are treated as 2118 * one port client reference 2119 */ 2120 if (is_ready == B_TRUE) 2121 saa_portp->saa_pt_reference_count += 2; 2122 2123 mutex_exit(&saa_portp->saa_pt_mutex); 2124 2125 if (is_ready == B_TRUE) 2126 break; /* normally, only 1 port entry */ 2127 } 2128 saa_portp = saa_portp->next; 2129 } 2130 2131 mutex_exit(&saa_statep->saa_port_list_mutex); 2132 2133 if (saa_portp != NULL && is_ready == B_TRUE) { 2134 2135 /* verify whether master sm lid changed */ 2136 2137 /* first query the portinfo to see if the lid changed */ 2138 ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num, 2139 &ibt_portinfop, &nports, &size); 2140 2141 if (ibt_status != IBT_SUCCESS) { 2142 2143 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 2144 ibmf_saa_impl_port_up_err, IBMF_TNF_ERROR, "", 2145 "ibmf_saa_impl_port_up: %s, ibmf_status =" 2146 " %d\n", tnf_string, msg, 2147 "ibt_query_hca_ports_byguid() failed", 2148 tnf_int, ibt_status, ibt_status); 2149 2150 goto bail; 2151 } 2152 2153 master_sm_lid = ibt_portinfop->p_sm_lid; 2154 2155 ibt_free_portinfo(ibt_portinfop, size); 2156 2157 /* check whether we need to subscribe for events */ 2158 mutex_enter(&saa_portp->saa_pt_event_sub_mutex); 2159 2160 event_subs = (saa_portp->saa_pt_event_sub_client_list != NULL) ? 2161 B_TRUE : B_FALSE; 2162 2163 mutex_exit(&saa_portp->saa_pt_event_sub_mutex); 2164 2165 /* update the master smlid */ 2166 mutex_enter(&saa_portp->saa_pt_mutex); 2167 2168 /* update the master sm lid value in ibmf_saa */ 2169 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = 2170 master_sm_lid; 2171 2172 /* if we're not subscribed for events, dec reference count */ 2173 if (event_subs == B_FALSE) 2174 saa_portp->saa_pt_reference_count--; 2175 2176 mutex_exit(&saa_portp->saa_pt_mutex); 2177 2178 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 2179 ibmf_saa_impl_port_up, IBMF_TNF_TRACE, "", 2180 "ibmf_saa_impl_port_up: %s, master_sm_lid = 0x%x\n", 2181 tnf_string, msg, 2182 "port is up. Sending classportinfo request", 2183 tnf_opaque, master_sm_lid, master_sm_lid); 2184 2185 /* get the classportinfo again */ 2186 ibmf_saa_impl_get_classportinfo(saa_portp); 2187 2188 /* 2189 * resubscribe to events if there are subscribers since SA may 2190 * have removed our subscription records when the port went down 2191 */ 2192 if (event_subs == B_TRUE) 2193 ibmf_saa_subscribe_events(saa_portp, B_TRUE, B_FALSE); 2194 } 2195 2196 bail: 2197 2198 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_up_end, 2199 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_up() exit\n"); 2200 } 2201 2202 /* 2203 * ibmf_saa_impl_port_down: 2204 */ 2205 static void 2206 ibmf_saa_impl_port_down(ib_guid_t ci_guid, uint8_t port_num) 2207 { 2208 2209 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_down_start, 2210 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_down: Handling port down" 2211 " guid %016" PRIx64 " port %d\n", 2212 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num); 2213 2214 2215 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_down_end, 2216 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_down() exit\n"); 2217 } 2218 2219 /* 2220 * ibmf_saa_impl_hca_detach: 2221 * find entry, unregister if there are no clients 2222 * have to unregister since ibmf needs to close the hca and will only do this if 2223 * no clients are registered 2224 */ 2225 static void 2226 ibmf_saa_impl_hca_detach(saa_port_t *saa_removed) 2227 { 2228 saa_port_t *saa_portp; 2229 boolean_t must_unreg, must_unsub; 2230 2231 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_hca_detach_start, 2232 IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach: Detaching" 2233 " entry %016" PRIx64 "\n", tnf_opaque, entry, saa_removed); 2234 2235 /* find this entry */ 2236 mutex_enter(&saa_statep->saa_port_list_mutex); 2237 2238 saa_portp = saa_statep->saa_port_list; 2239 while (saa_portp != NULL) { 2240 2241 if (saa_portp == saa_removed) 2242 break; 2243 2244 saa_portp = saa_portp->next; 2245 } 2246 mutex_exit(&saa_statep->saa_port_list_mutex); 2247 2248 ASSERT(saa_portp != NULL); 2249 2250 if (saa_portp == NULL) { 2251 2252 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 2253 ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "", 2254 "ibmf_saa_impl_hca_detach: %s, entry %016" 2255 PRIx64 "\n", 2256 tnf_string, msg, 2257 "Port entry NOT found", 2258 tnf_opaque, entryp, saa_removed); 2259 2260 goto bail; 2261 } 2262 2263 /* if there are clients expecting Reports(), unsusbscribe */ 2264 mutex_enter(&saa_portp->saa_pt_event_sub_mutex); 2265 2266 must_unsub = (saa_portp->saa_pt_event_sub_client_list != NULL) ? 2267 B_TRUE : B_FALSE; 2268 2269 mutex_exit(&saa_portp->saa_pt_event_sub_mutex); 2270 2271 /* fail if outstanding transactions */ 2272 mutex_enter(&saa_portp->saa_pt_mutex); 2273 2274 if (saa_portp->saa_pt_num_outstanding_trans > 0) { 2275 2276 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L1, 2277 ibmf_saa_impl_fini_err, IBMF_TNF_TRACE, "", 2278 "ibmf_saa_impl_fini: %s, port = %016" PRIx64 2279 ", num transactions = %d\n", 2280 tnf_string, msg, "Detaching HCA." 2281 " Outstanding transactions on port.", 2282 tnf_opaque, port, 2283 saa_portp->saa_pt_port_guid, 2284 tnf_uint, outstanding_transactions, 2285 saa_portp->saa_pt_num_outstanding_trans); 2286 2287 mutex_exit(&saa_portp->saa_pt_mutex); 2288 2289 goto bail; 2290 } 2291 2292 2293 /* 2294 * increment reference count by one to account for unsubscribe requests 2295 * that are about to be sent. All four informinfo's are treated as one 2296 * port client reference. The count will be decremented by 2297 * subscribe_events() before the call returns. 2298 */ 2299 if (must_unsub == B_TRUE) 2300 saa_portp->saa_pt_reference_count++; 2301 2302 mutex_exit(&saa_portp->saa_pt_mutex); 2303 2304 /* 2305 * try and unsubscribe from SA. Generate synchronous, unsequenced 2306 * unsubscribe requests. 2307 */ 2308 if (must_unsub == B_TRUE) 2309 ibmf_saa_subscribe_events(saa_portp, B_FALSE, B_TRUE); 2310 2311 /* warning if registered clients */ 2312 mutex_enter(&saa_portp->saa_pt_mutex); 2313 2314 if (saa_portp->saa_pt_reference_count > 0) { 2315 2316 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 2317 ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "", 2318 "ibmf_saa_impl_hca_detach: %s, port %016" 2319 PRIx64 "\n", 2320 tnf_string, msg, 2321 "Detaching HCA for port with clients still" 2322 " registered", tnf_opaque, port, 2323 saa_portp->saa_pt_port_guid); 2324 } 2325 2326 /* synchronize on end of registration */ 2327 while (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_REGISTERING) { 2328 2329 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L1, 2330 ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "", 2331 "ibmf_saa_impl_hca_detach: %s\n", 2332 tnf_string, msg, "someone is registering. waiting" 2333 " for them to finish"); 2334 2335 cv_wait(&saa_portp->saa_pt_ibmf_reg_cv, 2336 &saa_portp->saa_pt_mutex); 2337 2338 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L1, 2339 ibmf_saa_impl_hca_detach, 2340 IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach: %s\n", 2341 tnf_string, msg, "done waiting"); 2342 } 2343 2344 /* unregister from ibmf */ 2345 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_READY) { 2346 must_unreg = B_TRUE; 2347 } else 2348 must_unreg = B_FALSE; 2349 2350 ibmf_saa_impl_invalidate_port(saa_portp); 2351 2352 mutex_exit(&saa_portp->saa_pt_mutex); 2353 2354 if (must_unreg == B_TRUE) 2355 ibmf_saa_impl_ibmf_unreg(saa_portp); 2356 bail: 2357 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_hca_detach_end, 2358 IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach() exit\n"); 2359 } 2360 2361 /* ARGSUSED */ 2362 void 2363 ibmf_saa_async_cb(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp, void *args) 2364 { 2365 saa_impl_trans_info_t *trans_info; 2366 int status; 2367 size_t length; 2368 void *result; 2369 saa_port_t *saa_portp; 2370 saa_client_data_t *client_data; 2371 int ibmf_status; 2372 boolean_t ignore_data; 2373 ibmf_retrans_t ibmf_retrans; 2374 2375 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_async_cb_start, 2376 IBMF_TNF_TRACE, "", "ibmf_saa_async_cb() enter\n"); 2377 2378 trans_info = (saa_impl_trans_info_t *)args; 2379 2380 client_data = trans_info->si_trans_client_data; 2381 saa_portp = trans_info->si_trans_port; 2382 2383 if (msgp->im_msg_status == IBMF_TRANS_TIMEOUT) { 2384 2385 ibmf_status = ibmf_saa_impl_new_smlid_retry(saa_portp, msgp, 2386 ibmf_saa_async_cb, args, 2387 trans_info->si_trans_transport_flags); 2388 2389 /* 2390 * if smlid_retry() returns success sm lid changed and msg 2391 * was resent. Otherwise, lid did not change or msg could not 2392 * be resent. Continue normally. 2393 */ 2394 if (ibmf_status == IBMF_SUCCESS) 2395 goto bail; 2396 2397 /* 2398 * check whether we've received anything from the SA in a while. 2399 * If we have, this function will retry and return success. If 2400 * we haven't continue normally so that we return a timeout to 2401 * the client 2402 */ 2403 ibmf_status = ibmf_saa_check_sa_and_retry( 2404 saa_portp, msgp, ibmf_saa_async_cb, args, 2405 trans_info->si_trans_send_time, 2406 trans_info->si_trans_transport_flags); 2407 2408 if (ibmf_status == IBMF_SUCCESS) 2409 goto bail; 2410 } 2411 2412 /* if SA returned success but mad status is busy, retry a few times */ 2413 if (msgp->im_msg_status == IBMF_SUCCESS) { 2414 2415 ASSERT(msgp->im_msgbufs_recv.im_bufs_mad_hdr != NULL); 2416 2417 if ((b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->Status) == 2418 MAD_STATUS_BUSY) && 2419 (trans_info->si_trans_retry_busy_count < 2420 IBMF_SAA_MAX_BUSY_RETRY_COUNT)) { 2421 2422 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 2423 ibmf_saa_async_cb, IBMF_TNF_TRACE, "", 2424 "ibmf_saa_async_cb: %s, retry_count = %d\n", 2425 tnf_string, msg, 2426 "async response returned busy status", 2427 tnf_int, retry_count, 2428 trans_info->si_trans_retry_busy_count); 2429 2430 trans_info->si_trans_retry_busy_count++; 2431 2432 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans, 2433 sizeof (ibmf_retrans_t)); 2434 2435 ibmf_status = ibmf_msg_transport( 2436 saa_portp->saa_pt_ibmf_handle, 2437 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 2438 ibmf_saa_async_cb, args, 2439 trans_info->si_trans_transport_flags); 2440 2441 /* 2442 * if retry is successful, quit here since async_cb will 2443 * get called again; otherwise, let this function call 2444 * handle the cleanup 2445 */ 2446 if (ibmf_status == IBMF_SUCCESS) 2447 goto bail; 2448 } 2449 } 2450 2451 mutex_enter(&saa_portp->saa_pt_mutex); 2452 2453 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0); 2454 saa_portp->saa_pt_num_outstanding_trans--; 2455 2456 mutex_exit(&saa_portp->saa_pt_mutex); 2457 2458 if ((trans_info->si_trans_callback == NULL) && 2459 (trans_info->si_trans_sub_callback == NULL)) 2460 ignore_data = B_TRUE; 2461 else 2462 ignore_data = B_FALSE; 2463 2464 ibmf_saa_impl_prepare_response(ibmf_handle, msgp, ignore_data, &status, 2465 &result, &length, B_FALSE); 2466 2467 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 2468 2469 IBMF_SAA_SUB32_KSTATS(saa_portp, outstanding_requests, 1); 2470 2471 if (status != IBMF_SUCCESS) 2472 IBMF_SAA_ADD32_KSTATS(saa_portp, failed_requests, 1); 2473 2474 if (status == IBMF_TRANS_TIMEOUT) 2475 IBMF_SAA_ADD32_KSTATS(saa_portp, requests_timedout, 1); 2476 2477 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 2478 2479 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2480 ibmf_saa_async_cb, IBMF_TNF_TRACE, "", 2481 "ibmf_saa_async_cb: %s\n", tnf_string, msg, 2482 "Calling ibmf_saa client's callback"); 2483 2484 /* 2485 * there are three classes or trans_info users: ibmf_saa clients and 2486 * classportinfo requests; informinfo subscribe requests, and report 2487 * responses. For the first two, call the correct callback. For report 2488 * responses there's no need to notify anyone. 2489 */ 2490 if (trans_info->si_trans_callback != NULL) { 2491 /* ibmf_saa client or classportinfo request */ 2492 trans_info->si_trans_callback(trans_info->si_trans_callback_arg, 2493 length, result, status); 2494 } else if (trans_info->si_trans_sub_callback != NULL) { 2495 /* informinfo subscribe request */ 2496 trans_info->si_trans_sub_callback( 2497 trans_info->si_trans_callback_arg, length, result, status, 2498 trans_info->si_trans_sub_producer_type); 2499 } 2500 2501 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2502 ibmf_saa_async_cb, IBMF_TNF_TRACE, "", 2503 "ibmf_saa_async_cb: %s\n", tnf_string, msg, 2504 "Returned from callback"); 2505 2506 if (client_data != NULL) { 2507 mutex_enter(&client_data->saa_client_mutex); 2508 2509 ASSERT(client_data->saa_client_num_pending_trans > 0); 2510 client_data->saa_client_num_pending_trans--; 2511 2512 if ((client_data->saa_client_num_pending_trans == 0) && 2513 (client_data->saa_client_state == SAA_CLIENT_STATE_WAITING)) 2514 cv_signal(&client_data->saa_client_state_cv); 2515 2516 mutex_exit(&client_data->saa_client_mutex); 2517 } 2518 2519 kmem_free(trans_info, sizeof (saa_impl_trans_info_t)); 2520 2521 bail: 2522 2523 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_async_cb_end, 2524 IBMF_TNF_TRACE, "", "ibmf_saa_async_cb() exit\n"); 2525 } 2526 2527 /* 2528 * ibmf_saa_check_sa_and_retry: 2529 * 2530 * If a particular transaction times out, we don't want to give up if we know 2531 * the SA is responding. Check the time since we last received a response. If 2532 * it's less than ibmf_saa_max_wait_time retry the request. 2533 * 2534 * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the 2535 * same values passed to the original ibmf_msg_transport that timed out. The 2536 * ibmf_retrans parameter will be re-retrieved from the saa_portp structure. 2537 * 2538 * If max_wait_time seconds have passed, this function returns IBMF_TIMEOUT. 2539 * That way, callers can simply return the result of this function. 2540 * 2541 * Input Arguments 2542 * saa_portp pointer to saa_port structure 2543 * msgp ibmf message that timedout 2544 * ibmf_callback callback that should be called by msg_transport 2545 * ibmf_callback_arg args for ibmf_callback 2546 * transport_flags flags for ibmf_msg_transport 2547 * 2548 * Output Arguments 2549 * none 2550 * 2551 * Returns 2552 * IBMF_SUCCESS if we've recently received data from the SA and request was 2553 * resent. 2554 * IBMF_TRANS_TIMEOUT if no data has been received from the SA in max_wait_time 2555 * same values as ibmf_msg_transport() if data has been received but request 2556 * could not be resent. 2557 */ 2558 static int 2559 ibmf_saa_check_sa_and_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp, 2560 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, 2561 hrtime_t trans_send_time, int transport_flags) 2562 { 2563 hrtime_t curr_time, sa_uptime; 2564 ibmf_retrans_t ibmf_retrans; 2565 int ibmf_status; 2566 2567 do { 2568 2569 mutex_enter(&saa_portp->saa_pt_mutex); 2570 2571 sa_uptime = saa_portp->saa_pt_sa_uptime; 2572 2573 /* if nothing received from SA since we sent */ 2574 curr_time = gethrtime(); 2575 2576 /* 2577 * check if it's been a very long time since this 2578 * particular transaction was sent 2579 */ 2580 if (((curr_time - trans_send_time) / 1000000000) > 2581 ibmf_saa_trans_wait_time) { 2582 2583 mutex_exit(&saa_portp->saa_pt_mutex); 2584 2585 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L1, 2586 ibmf_saa_check_sa_and_retry_err, IBMF_TNF_ERROR, "", 2587 "ibmf_saa_check_sa_and_retry: %s, msgp = " 2588 "%p sa_uptime = %" PRIu64 ", trans send time = %" 2589 PRIu64 ", curr_time = %" PRIu64 "\n", 2590 tnf_string, msg, 2591 "Nothing received for this transaction", 2592 tnf_opaque, msgp, msgp, 2593 tnf_long, sa_uptime, sa_uptime, 2594 tnf_long, trans_send_time, trans_send_time, 2595 tnf_long, curr_time, curr_time); 2596 2597 ibmf_status = IBMF_TRANS_TIMEOUT; 2598 2599 break; 2600 } 2601 2602 /* 2603 * check time since we received something, 2604 * and make sure that it hasn't been an extra long 2605 * time for this particular transaction 2606 */ 2607 if (((curr_time - sa_uptime) / 1000000000) < 2608 ibmf_saa_max_wait_time) { 2609 2610 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L2, 2611 ibmf_saa_check_sa_and_retry, IBMF_TNF_TRACE, "", 2612 "ibmf_saa_check_sa_and_retry: %s, msgp = " 2613 "%p sa_uptime = %" PRIu64 " trans_send_time = %" 2614 PRIu64 " curr_time = %" PRIu64 "\n", 2615 tnf_string, msg, "Something received. Retrying", 2616 tnf_opaque, msgp, msgp, 2617 tnf_long, sa_uptime, sa_uptime, 2618 tnf_long, trans_send_time, trans_send_time, 2619 tnf_long, curr_time, curr_time); 2620 2621 /* 2622 * something received in WAIT_TIME_IN_SECS; 2623 * resend request 2624 */ 2625 2626 /* new tid needed */ 2627 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID = 2628 h2b64(saa_portp->saa_pt_current_tid++); 2629 2630 bcopy(&saa_portp->saa_pt_ibmf_retrans, 2631 &ibmf_retrans, sizeof (ibmf_retrans_t)); 2632 2633 mutex_exit(&saa_portp->saa_pt_mutex); 2634 2635 ibmf_status = ibmf_msg_transport( 2636 saa_portp->saa_pt_ibmf_handle, 2637 saa_portp->saa_pt_qp_handle, msgp, 2638 &ibmf_retrans, ibmf_callback, ibmf_callback_arg, 2639 transport_flags); 2640 2641 if (ibmf_status == IBMF_SUCCESS) 2642 goto bail; 2643 2644 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 2645 ibmf_saa_check_sa_and_retry, IBMF_TNF_TRACE, "", 2646 "ibmf_saa_check_sa_and_retry: %s, ibmf_status = " 2647 "%d\n", tnf_string, msg, 2648 "ibmf_msg_transport() failed", 2649 tnf_int, ibmf_status, ibmf_status); 2650 } else { 2651 2652 mutex_exit(&saa_portp->saa_pt_mutex); 2653 2654 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L1, 2655 ibmf_saa_check_sa_and_retry_err, IBMF_TNF_ERROR, "", 2656 "ibmf_saa_check_sa_and_retry: %s, msgp = " 2657 "%p sa_uptime = %" PRIu64 " curr_time = %" 2658 PRIu64 "\n", tnf_string, msg, 2659 "Nothing received. Timing out", 2660 tnf_opaque, msgp, msgp, 2661 tnf_long, sa_uptime, sa_uptime, 2662 tnf_long, curr_time, curr_time); 2663 2664 ibmf_status = IBMF_TRANS_TIMEOUT; 2665 2666 break; 2667 } 2668 } while (ibmf_status == IBMF_TRANS_TIMEOUT); 2669 2670 bail: 2671 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2672 ibmf_saa_check_sa_and_retry_end, 2673 IBMF_TNF_TRACE, "", "ibmf_saa_check_sa_and_retry() exiting" 2674 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 2675 2676 return (ibmf_status); 2677 } 2678 2679 2680 /* 2681 * ibmf_saa_impl_prepare_response: 2682 */ 2683 static void 2684 ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle, 2685 ibmf_msg_t *msgp, boolean_t ignore_data, int *status, void **result, 2686 size_t *length, boolean_t sleep_flag) 2687 { 2688 ibmf_msg_bufs_t *resp_buf; 2689 uint16_t attr_id; 2690 uint8_t method; 2691 boolean_t is_get_resp; 2692 uint16_t mad_status; 2693 uint16_t attr_offset; 2694 ib_sa_hdr_t *sa_hdr; 2695 2696 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 2697 ibmf_saa_impl_prepare_response_start, 2698 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response() enter\n"); 2699 2700 _NOTE(ASSUMING_PROTECTED(*msgp)) 2701 2702 *result = NULL; 2703 *length = 0; 2704 sa_hdr = NULL; 2705 2706 resp_buf = &msgp->im_msgbufs_recv; 2707 2708 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*resp_buf)) 2709 2710 if (msgp->im_msg_status != IBMF_SUCCESS) { 2711 2712 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 2713 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 2714 "ibmf_saa_impl_prepare_response: %s, msg_status = %d\n", 2715 tnf_string, msg, "Bad ibmf status", 2716 tnf_int, msg_status, msgp->im_msg_status); 2717 2718 *status = msgp->im_msg_status; 2719 2720 goto exit; 2721 } 2722 2723 if (resp_buf->im_bufs_mad_hdr == NULL) { 2724 2725 /* 2726 * this was an unsequenced transaction (from an unsubscribe for 2727 * following a CI_OFFLINE event) 2728 */ 2729 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2730 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 2731 "ibmf_saa_impl_prepare_response: %s\n", 2732 tnf_string, msg, "Unsequenced transaction callback"); 2733 2734 goto exit; 2735 } 2736 2737 if ((mad_status = b2h16(resp_buf->im_bufs_mad_hdr->Status)) != 2738 MAD_STATUS_NO_INVALID_FIELDS) { 2739 2740 /* convert mad packet status to IBMF status */ 2741 switch (mad_status) { 2742 2743 case SA_STATUS_ERR_NO_RESOURCES: 2744 *status = IBMF_NO_RESOURCES; 2745 break; 2746 case SA_STATUS_ERR_REQ_INVALID: 2747 *status = IBMF_REQ_INVALID; 2748 break; 2749 case SA_STATUS_ERR_NO_RECORDS: 2750 *status = IBMF_NO_RECORDS; 2751 break; 2752 case SA_STATUS_ERR_TOO_MANY_RECORDS: 2753 *status = IBMF_TOO_MANY_RECORDS; 2754 break; 2755 case SA_STATUS_ERR_REQ_INVALID_GID: 2756 *status = IBMF_INVALID_GID; 2757 break; 2758 case SA_STATUS_ERR_REQ_INSUFFICIENT_COMPONENTS: 2759 *status = IBMF_INSUFF_COMPS; 2760 break; 2761 case MAD_STATUS_UNSUPP_METHOD: 2762 *status = IBMF_UNSUPP_METHOD; 2763 break; 2764 case MAD_STATUS_UNSUPP_METHOD_ATTR: 2765 *status = IBMF_UNSUPP_METHOD_ATTR; 2766 break; 2767 case MAD_STATUS_INVALID_FIELD: 2768 *status = IBMF_INVALID_FIELD; 2769 break; 2770 default: 2771 *status = IBMF_REQ_INVALID; 2772 break; 2773 } 2774 2775 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 2776 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 2777 "ibmf_saa_impl_prepare_response: %s, mad_status = %x\n", 2778 tnf_string, msg, "Bad MAD status", 2779 tnf_int, mad_status, mad_status); 2780 2781 goto exit; 2782 } 2783 2784 attr_id = b2h16(resp_buf->im_bufs_mad_hdr->AttributeID); 2785 method = resp_buf->im_bufs_mad_hdr->R_Method; 2786 2787 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 2788 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 2789 "ibmf_saa_impl_prepare_response: attr_id = 0x%x, method = " 2790 "0x%x\n", 2791 tnf_opaque, attr_id, attr_id, 2792 tnf_opaque, method, method); 2793 2794 /* 2795 * ignore any data from deleteresp since there's no way to know whether 2796 * real data was returned; also ignore data if this was a Report 2797 * response 2798 */ 2799 if (method == SA_SUBN_ADM_DELETE_RESP) { 2800 2801 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2802 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 2803 "impf_saa_impl_prepare_response: %s\n", 2804 tnf_string, msg, 2805 "DeleteResp or NoticeResp returned. " 2806 "Ignoring response data"); 2807 2808 *status = IBMF_SUCCESS; 2809 2810 *length = 0; 2811 *result = NULL; 2812 2813 goto exit; 2814 } 2815 2816 if (attr_id == SA_MULTIPATHRECORD_ATTRID) { 2817 2818 /* 2819 * getmulti is only for requests; attribute should not 2820 * be returned from SA 2821 */ 2822 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 2823 ibmf_saa_impl_prepare_response_err, IBMF_TNF_ERROR, 2824 "", "ibmf_saa_impl_prepare_response: %s\n", 2825 tnf_string, msg, "SA returned getmulti record"); 2826 2827 *status = IBMF_REQ_INVALID; 2828 2829 goto exit; 2830 } 2831 2832 /* if we are supposed to ignore data, stop here */ 2833 if (ignore_data == B_TRUE) { 2834 2835 *status = IBMF_SUCCESS; 2836 2837 goto exit; 2838 } 2839 2840 is_get_resp = resp_buf->im_bufs_mad_hdr->R_Method == 2841 SA_SUBN_ADM_GET_RESP ? B_TRUE: B_FALSE; 2842 2843 /* unpack the sa header to get the attribute offset */ 2844 *status = ibmf_saa_utils_unpack_sa_hdr(resp_buf->im_bufs_cl_hdr, 2845 resp_buf->im_bufs_cl_hdr_len, &sa_hdr, sleep_flag); 2846 if (*status != IBMF_SUCCESS) { 2847 2848 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 2849 ibmf_saa_impl_prepare_response_err, 2850 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response: %s," 2851 " ibmf_status = %d\n", tnf_string, msg, 2852 "Could not unpack sa hdr", tnf_int, ibmf_status, *status); 2853 2854 goto exit; 2855 } 2856 2857 attr_offset = sa_hdr->AttributeOffset; 2858 2859 /* 2860 * unpack data payload; if unpack function doesn't return success 2861 * (because it could not allocate memory) forward this status to waiting 2862 * client 2863 */ 2864 *status = ibmf_saa_utils_unpack_payload(resp_buf->im_bufs_cl_data, 2865 resp_buf->im_bufs_cl_data_len, attr_id, result, length, 2866 attr_offset, is_get_resp, sleep_flag); 2867 if (*status == IBMF_SUCCESS) { 2868 2869 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, 2870 ibmf_saa_impl_prepare_response, 2871 IBMF_TNF_TRACE, "", 2872 "ibmf_saa_impl_prepare_response: attr_id = " 2873 "0x%x, attr_offset = %d, packed_payload_len = %d, " 2874 "unpacked_payload_len = %d\n", 2875 tnf_opaque, attr_id, attr_id, 2876 tnf_opaque, attr_offset, attr_offset, 2877 tnf_opaque, packed_payload_len, 2878 resp_buf->im_bufs_cl_data_len, 2879 tnf_opaque, unpacked_payload_len, *length); 2880 } else { 2881 2882 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L1, 2883 ibmf_saa_impl_prepare_response_err, 2884 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response: %s," 2885 "attr_id = 0x%x, attr_offset = %d, packed_payload_len = %d," 2886 "status = %d\n", 2887 tnf_string, msg, "Could not unpack payload", 2888 tnf_opaque, attr_id, attr_id, 2889 tnf_int, attr_offset, attr_offset, 2890 tnf_int, packed_payload_len, 2891 resp_buf->im_bufs_cl_data_len, 2892 tnf_int, status, *status); 2893 } 2894 exit: 2895 if (sa_hdr != NULL) 2896 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t)); 2897 2898 ibmf_saa_impl_free_msg(ibmf_handle, msgp); 2899 2900 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2901 ibmf_saa_impl_prepare_response_end, 2902 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response() exit," 2903 " status = 0x%d\n", tnf_int, status, *status); 2904 } 2905 2906 2907 /* 2908 * ibmf_saa_impl_check_sa_support: 2909 * Checks the capability mask (returned from the SA classportinfo response) to 2910 * determine whether the sa supports the specified attribute ID. 2911 * 2912 * Input Arguments 2913 * cap_mask 16-bit capability mask returned in SA's classportinfo 2914 * attr_id attribute ID of current request 2915 * 2916 * Returns 2917 * IBMF_NOT_SUPPORTED if capability mask indicates SA does not support attribute 2918 * IBMF_SUCCESS otherwise 2919 */ 2920 static int 2921 ibmf_saa_impl_check_sa_support(uint16_t cap_mask, uint16_t attr_id) 2922 { 2923 boolean_t attr_supported = B_TRUE; 2924 2925 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 2926 ibmf_saa_impl_check_sa_support, IBMF_TNF_TRACE, "", 2927 "ibmf_saa_impl_check_sa_support: cap_mask = 0x%x, " 2928 "attr_id = 0x%x\n", tnf_opaque, cap_mask, cap_mask, 2929 tnf_opaque, attr_id, attr_id); 2930 2931 switch (attr_id) { 2932 2933 case SA_SWITCHINFORECORD_ATTRID: 2934 case SA_LINEARFDBRECORD_ATTRID: 2935 case SA_RANDOMFDBRECORD_ATTRID: 2936 case SA_MULTICASTFDBRECORD_ATTRID: 2937 case SA_SMINFORECORD_ATTRID: 2938 case SA_INFORMINFORECORD_ATTRID: 2939 case SA_LINKRECORD_ATTRID: 2940 case SA_GUIDINFORECORD_ATTRID: 2941 case SA_TRACERECORD_ATTRID: 2942 case SA_SERVICEASSNRECORD_ATTRID: 2943 2944 if ((cap_mask & 2945 SA_CAPMASK_OPT_RECORDS_SUPPORTED) == 0) { 2946 2947 IBMF_TRACE_3(IBMF_TNF_NODEBUG, DPRINT_L1, 2948 ibmf_saa_impl_check_sa_support, 2949 IBMF_TNF_ERROR, "", 2950 "ibmf_saa_impl_check_sa_support: %s, " 2951 "cap_mask = 0x%x\n", tnf_string, msg, 2952 "SA does not support optional records", 2953 tnf_opaque, cap_mask, cap_mask, 2954 tnf_opaque, attr_id, attr_id); 2955 2956 attr_supported = B_FALSE; 2957 } 2958 break; 2959 2960 case SA_MULTIPATHRECORD_ATTRID: 2961 2962 if ((cap_mask & SA_CAPMASK_MULTIPATH_SUPPORTED) == 0) { 2963 2964 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 2965 ibmf_saa_impl_check_sa_support, 2966 IBMF_TNF_ERROR, "", 2967 "ibmf_saa_impl_check_sa_support: %s, " 2968 "cap_mask = 0x%x\n", tnf_string, msg, 2969 "SA does not support multipath records", 2970 tnf_opaque, cap_mask, cap_mask); 2971 2972 attr_supported = B_FALSE; 2973 } 2974 break; 2975 2976 case SA_MCMEMBERRECORD_ATTRID: 2977 2978 if ((cap_mask & SA_CAPMASK_UD_MCAST_SUPPORTED) == 0) { 2979 2980 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 2981 ibmf_saa_impl_check_sa_support, 2982 IBMF_TNF_ERROR, "", 2983 "ibmf_saa_impl_check_sa_support: %s, " 2984 "cap_mask = 0x%x\n", tnf_string, msg, 2985 "SA does not support ud multicast", 2986 tnf_opaque, cap_mask, cap_mask); 2987 2988 attr_supported = B_FALSE; 2989 } 2990 break; 2991 2992 default: 2993 break; 2994 } /* switch */ 2995 2996 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2997 ibmf_saa_impl_check_sa_support_end, IBMF_TNF_TRACE, "", 2998 "ibmf_saa_impl_check_sa_support() exiting, attr_supported = %d\n", 2999 tnf_opaque, attr_supported, attr_supported); 3000 3001 if (attr_supported == B_FALSE) 3002 return (IBMF_UNSUPP_METHOD_ATTR); 3003 else 3004 return (IBMF_SUCCESS); 3005 } 3006 3007 /* 3008 * ibmf_saa_impl_get_attr_id_length: 3009 * 3010 * Returns the host size of the specified sa record. Returns 0 for unknown 3011 * attributes. multipath record size is a dynamic value given as a parameter 3012 * specified with the ibmf_sa_access() call. 3013 */ 3014 static uint_t 3015 ibmf_saa_impl_get_attr_id_length(uint16_t attr_id) 3016 { 3017 uint_t attr_length; 3018 3019 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3020 ibmf_saa_impl_get_attr_id_length_start, 3021 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_attr_id_length() enter\n"); 3022 3023 /* this function should not be used for multipath record */ 3024 ASSERT(attr_id != SA_MULTIPATHRECORD_ATTRID); 3025 3026 switch (attr_id) { 3027 case SA_CLASSPORTINFO_ATTRID: 3028 attr_length = sizeof (ib_mad_classportinfo_t); 3029 break; 3030 case SA_NOTICE_ATTRID: 3031 attr_length = sizeof (ib_mad_notice_t); 3032 break; 3033 case SA_INFORMINFO_ATTRID: 3034 attr_length = sizeof (ib_mad_informinfo_t); 3035 break; 3036 case SA_NODERECORD_ATTRID: 3037 attr_length = sizeof (sa_node_record_t); 3038 break; 3039 case SA_PORTINFORECORD_ATTRID: 3040 attr_length = sizeof (sa_portinfo_record_t); 3041 break; 3042 case SA_SLTOVLRECORD_ATTRID: 3043 attr_length = sizeof (sa_SLtoVLmapping_record_t); 3044 break; 3045 case SA_SWITCHINFORECORD_ATTRID: 3046 attr_length = sizeof (sa_switchinfo_record_t); 3047 break; 3048 case SA_LINEARFDBRECORD_ATTRID: 3049 attr_length = sizeof (sa_linearft_record_t); 3050 break; 3051 case SA_RANDOMFDBRECORD_ATTRID: 3052 attr_length = sizeof (sa_randomft_record_t); 3053 break; 3054 case SA_MULTICASTFDBRECORD_ATTRID: 3055 attr_length = sizeof (sa_multicastft_record_t); 3056 break; 3057 case SA_SMINFORECORD_ATTRID: 3058 attr_length = sizeof (sa_sminfo_record_t); 3059 break; 3060 case SA_INFORMINFORECORD_ATTRID: 3061 attr_length = sizeof (sa_informinfo_record_t); 3062 break; 3063 case SA_LINKRECORD_ATTRID: 3064 attr_length = sizeof (sa_link_record_t); 3065 break; 3066 case SA_GUIDINFORECORD_ATTRID: 3067 attr_length = sizeof (sa_guidinfo_record_t); 3068 break; 3069 case SA_SERVICERECORD_ATTRID: 3070 attr_length = sizeof (sa_service_record_t); 3071 break; 3072 case SA_PARTITIONRECORD_ATTRID: 3073 attr_length = sizeof (sa_pkey_table_record_t); 3074 break; 3075 case SA_PATHRECORD_ATTRID: 3076 attr_length = sizeof (sa_path_record_t); 3077 break; 3078 case SA_VLARBRECORD_ATTRID: 3079 attr_length = sizeof (sa_VLarb_table_record_t); 3080 break; 3081 case SA_MCMEMBERRECORD_ATTRID: 3082 attr_length = sizeof (sa_mcmember_record_t); 3083 break; 3084 case SA_TRACERECORD_ATTRID: 3085 attr_length = sizeof (sa_trace_record_t); 3086 break; 3087 case SA_SERVICEASSNRECORD_ATTRID: 3088 attr_length = sizeof (sa_service_assn_record_t); 3089 break; 3090 default: 3091 /* should only get the above type of packets */ 3092 attr_length = 0; 3093 break; 3094 } 3095 3096 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 3097 ibmf_saa_impl_get_attr_id_length_end, 3098 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_attr_id_length():" 3099 " attr_id: 0x%x size %d\n", 3100 tnf_opaque, attr_id, attr_id, tnf_uint, attr_length, attr_length); 3101 3102 return (attr_length); 3103 } 3104 3105 /* 3106 * ibmf_saa_impl_free_msg: 3107 * Takes a completed message and free memory associated with the message, 3108 * including the individual fields of the im_msgbufs_send. 3109 * ibmf_free_msg, called at the end of this function, takes a pointer to the 3110 * message pointer so that it can set the message pointer to NULL. This 3111 * function takes just the message pointer so the msgp will not be NULL after 3112 * this function returns. 3113 * 3114 * Input Arguments 3115 * ibmf_hdl ibmf handle used in ibmf_msg_alloc 3116 * msgp pointer to ibmf_msg_t to free 3117 * 3118 * Returns 3119 * void 3120 */ 3121 static void 3122 ibmf_saa_impl_free_msg(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msgp) 3123 { 3124 int res; 3125 3126 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3127 ibmf_saa_impl_free_msg_start, 3128 IBMF_TNF_TRACE, "", "ibmf_saa_impl_free_msg() enter: msg %p\n", 3129 tnf_opaque, msg, msgp); 3130 3131 ASSERT(msgp != NULL); 3132 3133 kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr, 3134 sizeof (ib_mad_hdr_t)); 3135 3136 kmem_free(msgp->im_msgbufs_send.im_bufs_cl_hdr, 3137 msgp->im_msgbufs_send.im_bufs_cl_hdr_len); 3138 3139 if (msgp->im_msgbufs_send.im_bufs_cl_data_len > 0) 3140 kmem_free(msgp->im_msgbufs_send.im_bufs_cl_data, 3141 msgp->im_msgbufs_send.im_bufs_cl_data_len); 3142 3143 res = ibmf_free_msg(ibmf_hdl, &msgp); 3144 ASSERT(res == IBMF_SUCCESS); 3145 3146 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3147 ibmf_saa_impl_free_msg_end, 3148 IBMF_TNF_TRACE, "", "ibmf_saa_impl_free_msg() exit\n"); 3149 } 3150 3151 /* 3152 * ibmf_saa_impl_get_port_guid: 3153 */ 3154 static int 3155 ibmf_saa_impl_get_port_guid(ibt_hca_portinfo_t *ibt_portinfop, 3156 ib_guid_t *guid_ret) 3157 { 3158 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3159 ibmf_saa_impl_get_port_guid_start, 3160 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_port_guid() enter\n"); 3161 3162 if (ibt_portinfop->p_linkstate != IBT_PORT_ACTIVE) { 3163 3164 return (IBMF_BAD_PORT_STATE); 3165 } 3166 3167 if (ibt_portinfop->p_sgid_tbl_sz == 0) { 3168 3169 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L2, 3170 ibmf_saa_impl_get_port_guid_end, IBMF_TNF_TRACE, "", 3171 "ibmf_saa_impl_get_port_guid: %s\n", tnf_string, msg, 3172 "portinfo sgid table size is 0. Exiting.\n"); 3173 3174 return (IBMF_TRANSPORT_FAILURE); 3175 } 3176 3177 *guid_ret = ibt_portinfop->p_sgid_tbl[0].gid_guid; 3178 3179 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3180 ibmf_saa_impl_get_port_guid_end, IBMF_TNF_TRACE, "", 3181 "ibmf_saa_impl_get_port_guid: Returning port_guid %016" PRIx64 "\n", 3182 tnf_opaque, port_guid, *guid_ret); 3183 3184 return (IBMF_SUCCESS); 3185 } 3186 3187 /* 3188 * ibmf_saa_impl_set_transaction_params: 3189 */ 3190 static void 3191 ibmf_saa_impl_set_transaction_params(saa_port_t *saa_portp, 3192 ibt_hca_portinfo_t *portinfop) 3193 { 3194 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3195 ibmf_saa_impl_set_transaction_params_start, 3196 IBMF_TNF_TRACE, "", 3197 "ibmf_saa_impl_set_transaction_params() enter\n"); 3198 3199 _NOTE(ASSUMING_PROTECTED(*saa_portp)) 3200 3201 saa_portp->saa_pt_ibmf_retrans.retrans_retries = 3202 IBMF_SAA_RETRANS_RETRIES; 3203 /* 3204 * For the first transaction (generally getting the 3205 * classportinfo) have ibmf pick our timeouts. It should be using the 3206 * default IB spec values. 3207 * Once we get the classportinfo we'll update the correct response time 3208 * value (rtv) and round-trip time (rttv). ibmf should always calculate 3209 * trans_to since it depends on the particular transaction's number of 3210 * packets. 3211 */ 3212 saa_portp->saa_pt_ibmf_retrans.retrans_rtv = 0; 3213 saa_portp->saa_pt_ibmf_retrans.retrans_rttv = 0; 3214 saa_portp->saa_pt_ibmf_retrans.retrans_trans_to = 0; 3215 3216 /* 3217 * Assume that the SA supports all optional records. If it 3218 * does not, the request will get returned with ERR_NOT_SUPP. When 3219 * the classportinfo response comes back we will update the cap mask 3220 * to prevent unnecessary unsupported requests. 3221 */ 3222 saa_portp->saa_pt_sa_cap_mask = 0xFFFF; 3223 3224 saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = 1; 3225 saa_portp->saa_pt_ibmf_addr_info.ia_p_key = 3226 IB_PKEY_DEFAULT_LIMITED; 3227 saa_portp->saa_pt_ibmf_addr_info.ia_q_key = IB_GSI_QKEY; 3228 3229 /* 3230 * fill out addr information for MADs that will be sent 3231 * to SA on this port 3232 */ 3233 saa_portp->saa_pt_ibmf_addr_info.ia_local_lid = portinfop->p_base_lid; 3234 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = portinfop->p_sm_lid; 3235 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = portinfop->p_sm_sl; 3236 3237 /* place upper bound on subnet timeout in case of faulty SM */ 3238 saa_portp->saa_pt_timeout = portinfop->p_subnet_timeout; 3239 3240 if (saa_portp->saa_pt_timeout > IBMF_SAA_MAX_SUBNET_TIMEOUT) { 3241 3242 saa_portp->saa_pt_timeout = IBMF_SAA_MAX_SUBNET_TIMEOUT; 3243 } 3244 3245 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, 3246 ibmf_saa_impl_set_transaction_params, 3247 IBMF_TNF_TRACE, "", 3248 "ibmf_saa_impl_set_transaction_params: local_lid = 0x%x, " 3249 "sm_lid = 0x%x, sm_sl = 0x%x, sn_timeout = 0x%x\n", 3250 tnf_opaque, local_lid, portinfop->p_base_lid, 3251 tnf_opaque, sm_lid, portinfop->p_sm_lid, 3252 tnf_opaque, sm_sl, portinfop->p_sm_sl, 3253 tnf_opaque, subnet_timeout, portinfop->p_subnet_timeout); 3254 3255 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3256 ibmf_saa_impl_set_transaction_params_end, 3257 IBMF_TNF_TRACE, "", 3258 "ibmf_saa_impl_set_transaction_params() exit\n"); 3259 } 3260 3261 /* 3262 * ibmf_saa_impl_ibmf_unreg: 3263 */ 3264 static void 3265 ibmf_saa_impl_ibmf_unreg(saa_port_t *saa_portp) 3266 { 3267 int ibmf_status; 3268 3269 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_ibmf_unreg_start, 3270 IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibmf_unreg() enter\n"); 3271 3272 /* teardown async cb */ 3273 ibmf_status = ibmf_tear_down_async_cb(saa_portp->saa_pt_ibmf_handle, 3274 saa_portp->saa_pt_qp_handle, 0); 3275 if (ibmf_status != IBMF_SUCCESS) { 3276 3277 /* continue anyway even though unreg will probably fail */ 3278 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3279 ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "", 3280 "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n", 3281 tnf_string, msg, "Could not tear down async cb", 3282 tnf_int, ibmf_status, ibmf_status); 3283 } 3284 3285 /* free qp */ 3286 ibmf_status = ibmf_free_qp(saa_portp->saa_pt_ibmf_handle, 3287 &saa_portp->saa_pt_qp_handle, 0); 3288 3289 if (ibmf_status != IBMF_SUCCESS) { 3290 3291 /* continue anyway even though unreg will probably fail */ 3292 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3293 ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "", 3294 "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n", 3295 tnf_string, msg, "Could not free queue pair", 3296 tnf_int, ibmf_status, ibmf_status); 3297 } 3298 3299 ibmf_status = ibmf_unregister(&saa_portp->saa_pt_ibmf_handle, 0); 3300 3301 if (ibmf_status != IBMF_SUCCESS) { 3302 3303 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3304 ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "", 3305 "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n", 3306 tnf_string, msg, "ibmf_unregister() failed", 3307 tnf_int, ibmf_status, ibmf_status); 3308 } 3309 3310 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_ibmf_unreg_end, 3311 IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibmf_unreg() exit\n"); 3312 } 3313