1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/ib/mgt/ibmf/ibmf_saa_impl.h> 29 #include <sys/ib/mgt/ibmf/ibmf_saa_utils.h> 30 31 /* Global sa_access State Pointer */ 32 saa_state_t *saa_statep; 33 _NOTE(READ_ONLY_DATA(saa_statep)) 34 35 extern int ibmf_trace_level; 36 37 extern int ibmf_taskq_max_tasks; 38 39 static int 40 ibmf_saa_impl_new_smlid_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp, 41 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, int transport_flags); 42 static int 43 ibmf_saa_impl_revert_to_qp1(saa_port_t *saa_portp, ibmf_msg_t *msgp, 44 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_args, int transport_flags); 45 static int 46 ibmf_saa_check_sa_and_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp, 47 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, 48 hrtime_t trans_send_time, int transport_flags); 49 static int ibmf_saa_impl_init_msg(saa_impl_trans_info_t *trans_info, 50 boolean_t sleep_flag, ibmf_msg_t **msgp, uint32_t *transport_flagsp, 51 ibmf_retrans_t *ibmf_retransp); 52 static int ibmf_saa_must_purge(saa_port_t *saa_portp); 53 static void ibmf_saa_impl_invalidate_port(saa_port_t *saa_portp); 54 static void ibmf_saa_impl_destroy_port(saa_port_t *saa_portp); 55 static void ibmf_saa_impl_uninit_kstats(saa_port_t *saa_portp); 56 static void ibmf_saa_impl_get_cpi_cb(void *arg, size_t length, char *buffer, 57 int status); 58 static void ibmf_saa_impl_async_event_cb(ibmf_handle_t ibmf_handle, 59 void *clnt_private, ibmf_async_event_t event_type); 60 static void ibmf_saa_impl_port_up(ib_guid_t ci_guid, uint8_t port_num); 61 static void ibmf_saa_impl_port_down(ib_guid_t ci_guid, uint8_t port_num); 62 static void ibmf_saa_impl_hca_detach(saa_port_t *saa_removed); 63 static void ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle, 64 ibmf_msg_t *msgp, boolean_t ignore_data, int *status, void **result, 65 size_t *length, boolean_t sleep_flag); 66 static int ibmf_saa_impl_check_sa_support(uint16_t cap_mask, uint16_t attr_id); 67 static uint_t ibmf_saa_impl_get_attr_id_length(uint16_t attr_id); 68 static void ibmf_saa_impl_free_msg(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msgp); 69 static int ibmf_saa_impl_get_port_guid(ibt_hca_portinfo_t *ibt_portinfop, 70 ib_guid_t *guid_ret); 71 static void ibmf_saa_impl_set_transaction_params(saa_port_t *saa_portp, 72 ibt_hca_portinfo_t *portinfop); 73 static void ibmf_saa_impl_update_sa_address_info(saa_port_t *saa_portp, 74 ibmf_msg_t *msgp); 75 static void ibmf_saa_impl_ibmf_unreg(saa_port_t *saa_portp); 76 77 int ibmf_saa_max_wait_time = IBMF_SAA_MAX_WAIT_TIME_IN_SECS; 78 int ibmf_saa_trans_wait_time = IBMF_SAA_TRANS_WAIT_TIME_IN_SECS; 79 80 /* 81 * ibmf_saa_impl_init: 82 * Allocates memory for the ibmf_saa state structure and initializes the taskq. 83 * Called from the modules init() routine. 84 * 85 * Input Arguments 86 * none 87 * 88 * Output Arguments 89 * none 90 * 91 * Returns 92 * IBMF_NO_RESOURCES if taskq could not be created. 93 * IBMF_SUCCESS on success 94 * 95 */ 96 int 97 ibmf_saa_impl_init() 98 { 99 int res; 100 101 /* CONSTCOND */ 102 ASSERT(NO_COMPETING_THREADS); 103 104 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_init_start, 105 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init() enter\n"); 106 107 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_statep)) 108 109 saa_statep = kmem_zalloc(sizeof (saa_state_t), KM_SLEEP); 110 111 /* create taskq for notifying event subscribers */ 112 saa_statep->saa_event_taskq = taskq_create( 113 "ibmf_saa_event_taskq", IBMF_TASKQ_NTHREADS, 114 MINCLSYSPRI, 1, ibmf_taskq_max_tasks, TASKQ_DYNAMIC | 115 TASKQ_PREPOPULATE); 116 if (saa_statep->saa_event_taskq == NULL) { 117 118 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L4, 119 ibmf_saa_impl_init_end_err, 120 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init(): %s\n", 121 tnf_string, msg, "event taskq create failed"); 122 123 kmem_free(saa_statep, sizeof (saa_state_t)); 124 125 res = IBMF_NO_RESOURCES; 126 127 goto bail; 128 } 129 130 mutex_init(&saa_statep->saa_port_list_mutex, NULL, MUTEX_DRIVER, 131 NULL); 132 133 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*saa_statep)) 134 135 res = IBMF_SUCCESS; 136 bail: 137 138 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_init_end, 139 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init() exit: status = %d\n", 140 tnf_int, res, res); 141 142 return (res); 143 } 144 145 /* 146 * ibmf_saa_impl_fini: 147 * If there are no registered clients, cleans up all memory associated with the 148 * state, including each of the port list entries. 149 * Called from the modules fini() routine. 150 * 151 * Input Arguments 152 * none 153 * 154 * Output Arguments 155 * none 156 * 157 * Returns 158 * EBUSY if there are outstanding transactions or registered clients 159 * 0 if cleanup was sucessfull 160 * 161 */ 162 int 163 ibmf_saa_impl_fini() 164 { 165 int ret = 0; 166 saa_port_t *saa_portp; 167 168 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_fini_start, 169 IBMF_TNF_TRACE, "", "ibmf_saa_impl_fini() enter\n"); 170 171 /* make sure there are no registered clients */ 172 mutex_enter(&saa_statep->saa_port_list_mutex); 173 174 saa_portp = saa_statep->saa_port_list; 175 while (saa_portp != NULL) { 176 177 mutex_enter(&saa_portp->saa_pt_mutex); 178 179 if (saa_portp->saa_pt_reference_count > 0) { 180 181 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 182 ibmf_saa_impl_fini_err, IBMF_TNF_ERROR, "", 183 "ibmf_saa_impl_fini: %s, port %016" PRIx64 "\n", 184 tnf_string, msg, 185 "cannot unload ibmf_saa. Client on port still" 186 " registered", tnf_opaque, port, 187 saa_portp->saa_pt_port_guid); 188 189 mutex_exit(&saa_portp->saa_pt_mutex); 190 191 mutex_exit(&saa_statep->saa_port_list_mutex); 192 193 ret = EBUSY; 194 goto bail; 195 } 196 197 /* make sure there are no outstanding transactions */ 198 199 if (saa_portp->saa_pt_num_outstanding_trans > 0) { 200 201 IBMF_TRACE_3(IBMF_TNF_NODEBUG, DPRINT_L1, 202 ibmf_saa_impl_fini_err, IBMF_TNF_ERROR, "", 203 "ibmf_saa_impl_fini: %s, port = %016" PRIx64 204 ", num transactions = %d\n", 205 tnf_string, msg, "Cannot unload ibmf_saa." 206 " Outstanding transactions on port.", 207 tnf_opaque, port, 208 saa_portp->saa_pt_port_guid, 209 tnf_uint, outstanding_transactions, 210 saa_portp->saa_pt_num_outstanding_trans); 211 212 mutex_exit(&saa_portp->saa_pt_mutex); 213 214 mutex_exit(&saa_statep->saa_port_list_mutex); 215 216 ret = EBUSY; 217 goto bail; 218 } 219 220 mutex_exit(&saa_portp->saa_pt_mutex); 221 222 saa_portp = saa_portp->next; 223 } 224 225 mutex_exit(&saa_statep->saa_port_list_mutex); 226 227 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(saa_statep->saa_port_list, 228 *saa_portp)) 229 230 /* 231 * no more clients nor pending transaction: 232 * unregister ibmf and destroy port entries 233 */ 234 saa_portp = saa_statep->saa_port_list; 235 while (saa_portp != NULL) { 236 237 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 238 ibmf_saa_impl_fini, IBMF_TNF_TRACE, "", 239 "ibmf_saa_impl_fini: %s, prefix = %016" PRIx64 "\n", 240 tnf_string, msg, "deinitializing port", 241 tnf_opaque, port_guid, saa_portp->saa_pt_port_guid); 242 243 saa_statep->saa_port_list = saa_portp->next; 244 245 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_portp)) 246 247 mutex_enter(&saa_portp->saa_pt_mutex); 248 249 /* unregister from ibmf */ 250 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_READY) { 251 252 mutex_exit(&saa_portp->saa_pt_mutex); 253 254 ibmf_saa_impl_ibmf_unreg(saa_portp); 255 } else 256 257 mutex_exit(&saa_portp->saa_pt_mutex); 258 259 ibmf_saa_impl_destroy_port(saa_portp); 260 saa_portp = saa_statep->saa_port_list; 261 } 262 263 taskq_destroy(saa_statep->saa_event_taskq); 264 265 mutex_destroy(&saa_statep->saa_port_list_mutex); 266 267 kmem_free(saa_statep, sizeof (saa_state_t)); 268 269 bail: 270 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_fini_end, 271 IBMF_TNF_TRACE, "", "ibmf_saa_impl_fini() exit\n"); 272 273 return (ret); 274 } 275 276 /* 277 * ibmf_saa_is_valid 278 * Returns true the entry is valid. 279 * 280 * Input Arguments 281 * saa_portp pointer to state structure 282 * add_ref if B_TRUE ref count is incremented on a valid portp 283 * 284 * Output Arguments 285 * none 286 * 287 * Returns 288 * B_TRUE if entry was in a valid state, B_FALSE otherwise 289 */ 290 boolean_t 291 ibmf_saa_is_valid(saa_port_t *saa_portp, int add_ref) 292 { 293 boolean_t is_valid = B_TRUE; 294 295 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_is_valid_start, 296 IBMF_TNF_TRACE, "", "ibmf_saa_is_valid() enter\n"); 297 298 mutex_enter(&saa_portp->saa_pt_mutex); 299 300 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_INVALID || 301 saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_PURGING) { 302 303 is_valid = B_FALSE; 304 305 } else if (add_ref == B_TRUE) { 306 /* 307 * increment reference count here to ensure that 308 * entry does not get purged behind our backs 309 */ 310 saa_portp->saa_pt_reference_count++; 311 } 312 mutex_exit(&saa_portp->saa_pt_mutex); 313 314 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_is_valid_end, 315 IBMF_TNF_TRACE, "", "ibmf_saa_is_valid() exit\n"); 316 317 return (is_valid); 318 } 319 320 /* 321 * ibmf_saa_must_purge 322 * Determines if we can purge a portp (remove it from the list) based on the 323 * state and number of clients 324 * 325 * Input Arguments 326 * saa_portp pointer to state structure 327 * 328 * Output Arguments 329 * none 330 * 331 * Returns 332 * B_TRUE if the entry can be removed, B_FALSE otherwise 333 */ 334 static int 335 ibmf_saa_must_purge(saa_port_t *saa_portp) 336 { 337 int must_purge = B_FALSE; 338 339 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_must_purge_start, 340 IBMF_TNF_TRACE, "", "ibmf_saa_must_purge() enter\n"); 341 342 mutex_enter(&saa_portp->saa_pt_mutex); 343 344 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_INVALID && 345 saa_portp->saa_pt_reference_count == 0) { 346 347 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_PURGING; 348 must_purge = B_TRUE; 349 } 350 351 mutex_exit(&saa_portp->saa_pt_mutex); 352 353 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_must_purge_end, 354 IBMF_TNF_TRACE, "", "ibmf_saa_must_purge() exit\n"); 355 356 return (must_purge); 357 } 358 359 360 /* 361 * ibmf_saa_impl_purge: 362 * Removes invalid port state entries from the list 363 * 364 * Input Arguments 365 * none 366 * 367 * Output Arguments 368 * none 369 * 370 * Returns 371 * void 372 */ 373 void 374 ibmf_saa_impl_purge() 375 { 376 saa_port_t *cur_portp = NULL; 377 saa_port_t *prev_portp = NULL; 378 saa_port_t *rem_portp = NULL; 379 380 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_purge_start, 381 IBMF_TNF_TRACE, "", "ibmf_saa_impl_purge() enter\n"); 382 383 mutex_enter(&saa_statep->saa_port_list_mutex); 384 385 cur_portp = saa_statep->saa_port_list; 386 prev_portp = cur_portp; 387 388 while (cur_portp != NULL) { 389 390 if (ibmf_saa_must_purge(cur_portp) == B_TRUE) { 391 392 rem_portp = cur_portp; 393 394 /* unlink entry */ 395 if (cur_portp == saa_statep->saa_port_list) { 396 397 saa_statep->saa_port_list = cur_portp->next; 398 cur_portp = saa_statep->saa_port_list; 399 prev_portp = cur_portp; 400 401 } else { 402 403 prev_portp->next = cur_portp->next; 404 cur_portp = cur_portp->next; 405 } 406 407 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*rem_portp)) 408 409 /* destroy entry */ 410 ASSERT(rem_portp != NULL); 411 ibmf_saa_impl_destroy_port(rem_portp); 412 413 } else { 414 415 prev_portp = cur_portp; 416 cur_portp = cur_portp->next; 417 } 418 } 419 420 mutex_exit(&saa_statep->saa_port_list_mutex); 421 422 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_purge_end, 423 IBMF_TNF_TRACE, "", "ibmf_saa_impl_purge() exit\n"); 424 } 425 426 /* 427 * saa_impl_add_client: 428 * Adds a client for a particular portp. Reference count has been incremented 429 * before this call. It is decremented by saa_impl_add_client() if the call 430 * fails. 431 * 432 * Input Arguments 433 * none 434 * 435 * Output Arguments 436 * none 437 * 438 * Returns 439 * IBMF_BUSY if there are already too many clients registered, 440 * IBMF_BAD_PORT_STATE if the port is invalid (generally because a previous 441 * client failed during registration for this port) 442 * IBMF_SUCCESS otherwise 443 */ 444 int 445 ibmf_saa_impl_add_client(saa_port_t *saa_portp) 446 { 447 int status = IBMF_SUCCESS; 448 449 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_add_client_start, 450 IBMF_TNF_TRACE, "", "ibmf_saa_impl_add_client() enter\n"); 451 452 mutex_enter(&saa_portp->saa_pt_mutex); 453 454 /* 455 * check that we don't exceed max clients 456 */ 457 if (saa_portp->saa_pt_reference_count > 458 SAA_MAX_CLIENTS_PER_PORT) { 459 460 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 461 ibmf_saa_impl_add_client_err, IBMF_TNF_ERROR, "", 462 "ibmf_saa_impl_add_client: %s, num_reg_clients %d\n", 463 tnf_string, msg, "too many clients registered for" 464 " port", tnf_uint, num_reg_clients, 465 saa_portp->saa_pt_reference_count); 466 467 status = IBMF_BUSY; 468 goto bail; 469 } 470 471 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 472 ibmf_saa_impl_add_client, IBMF_TNF_TRACE, "", 473 "ibmf_saa_impl_add_client: num_registered_clients %d\n", 474 tnf_uint, num_registered_clients, 475 saa_portp->saa_pt_reference_count); 476 477 /* 478 * wait until anyone who is currently registering 479 * this port with ibmf is done 480 */ 481 while (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_REGISTERING) { 482 483 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 484 ibmf_saa_impl_add_client, IBMF_TNF_TRACE, "", 485 "ibmf_saa_impl_add_client: %s\n", 486 tnf_string, msg, "someone is registering. waiting" 487 " for them to finish"); 488 489 cv_wait(&saa_portp->saa_pt_ibmf_reg_cv, 490 &saa_portp->saa_pt_mutex); 491 492 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 493 ibmf_saa_impl_add_client, 494 IBMF_TNF_TRACE, "", "ibmf_saa_impl_add_client: %s\n", 495 tnf_string, msg, "done waiting"); 496 } 497 498 /* 499 * if port isn't ready here, fail. 500 */ 501 if (saa_portp->saa_pt_state != IBMF_SAA_PORT_STATE_READY) { 502 503 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 504 ibmf_saa_impl_add_client_err, IBMF_TNF_ERROR, 505 "", "ibmf_saa_impl_add_client: %s\n", 506 tnf_string, msg, "port state not ready," 507 " removing client."); 508 509 status = IBMF_BAD_PORT_STATE; 510 goto bail; 511 } 512 513 bail: 514 mutex_exit(&saa_portp->saa_pt_mutex); 515 516 if (status != IBMF_SUCCESS) { 517 518 mutex_enter( 519 &saa_portp->saa_pt_kstat_mutex); 520 521 IBMF_SAA_ADD32_KSTATS(saa_portp, 522 clients_reg_failed, 1); 523 524 mutex_exit( 525 &saa_portp->saa_pt_kstat_mutex); 526 527 /* decrementing refcount is last thing we do on entry */ 528 529 mutex_enter(&saa_portp->saa_pt_mutex); 530 531 ASSERT(saa_portp->saa_pt_reference_count > 0); 532 saa_portp->saa_pt_reference_count--; 533 534 mutex_exit(&saa_portp->saa_pt_mutex); 535 } 536 537 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 538 ibmf_saa_impl_add_client_end, IBMF_TNF_TRACE, "", 539 "ibmf_saa_impl_add_client() exit\n"); 540 541 return (status); 542 } 543 544 /* 545 * ibmf_saa_impl_create_port() 546 * Create port entry with mimimal inits because 547 * we're holding the list mutex: NO BLOCKING CALLS HERE, please. 548 * 549 * Initialize port state to "registering", so that clients accessing 550 * same port concurrently will wait for the end of the ibmf registration. 551 * Note: this thread will access port members without locking mutex. 552 * 553 * Input Arguments 554 * pt_guid guid of port 555 * 556 * Output Arguments 557 * saa_portpp pointer to new saa_portp structure 558 * 559 * Returns 560 * IBMF_NO_MEMORY if memory could not be allocated 561 * IBMF_SUCCESS otherwise 562 */ 563 int 564 ibmf_saa_impl_create_port(ib_guid_t pt_guid, saa_port_t **saa_portpp) 565 { 566 int status = IBMF_SUCCESS; 567 saa_port_t *saa_portp = NULL; 568 569 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_create_port_start, 570 IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port:" 571 " guid %016" PRIx64 "\n", 572 tnf_opaque, port_guid, pt_guid); 573 574 ASSERT(MUTEX_HELD(&saa_statep->saa_port_list_mutex)); 575 576 /* create & initialize new port */ 577 saa_portp = kmem_zalloc(sizeof (saa_port_t), KM_NOSLEEP); 578 579 if (saa_portp == NULL) { 580 581 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 582 ibmf_sa_session_open_err, IBMF_TNF_ERROR, "", 583 "ibmf_saa_impl_create_port: %s\n", 584 tnf_string, msg, "could not allocate memory for " 585 "new port"); 586 587 status = IBMF_NO_MEMORY; 588 goto bail; 589 } 590 591 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_sa_session_open, 592 IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port: %s\n", 593 tnf_string, msg, "first client registering, initializing"); 594 595 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_portp)) 596 597 /* tell everyone that kstats are not initialized */ 598 saa_portp->saa_pt_kstatp = NULL; 599 600 /* 601 * set up mutexe and state variable to indicate to 602 * other clients that were currently in the process of 603 * setting up the port data. This will prevent a subsequent 604 * client from trying to to register with ibmf before the 605 * port data has been initialized. 606 */ 607 mutex_init(&saa_portp->saa_pt_mutex, NULL, MUTEX_DRIVER, NULL); 608 cv_init(&saa_portp->saa_pt_ibmf_reg_cv, NULL, CV_DRIVER, NULL); 609 610 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_REGISTERING; 611 612 /* create other mutexes */ 613 mutex_init(&saa_portp->saa_pt_kstat_mutex, NULL, MUTEX_DRIVER, NULL); 614 615 mutex_init(&saa_portp->saa_pt_event_sub_mutex, NULL, MUTEX_DRIVER, 616 NULL); 617 618 /* 619 * clients assume all arrive; set mask to this so we only notify 620 * if something failed 621 */ 622 saa_portp->saa_pt_event_sub_last_success_mask = 623 IBMF_SAA_PORT_EVENT_SUB_ALL_ARRIVE; 624 625 /* 626 * set port_guid now so any immediately subsequent clients 627 * registering on this port, guid will know we're already here 628 */ 629 saa_portp->saa_pt_port_guid = pt_guid; 630 saa_portp->saa_pt_reference_count = 1; 631 saa_portp->saa_pt_current_tid = pt_guid << 32; 632 633 saa_portp->saa_pt_redirect_active = B_FALSE; 634 635 /* set sa_uptime now in case we never receive anything from SA */ 636 saa_portp->saa_pt_sa_uptime = gethrtime(); 637 638 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*saa_portp)) 639 640 /* Set new pointer in caller's */ 641 *saa_portpp = saa_portp; 642 643 bail: 644 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_create_port_end, 645 IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port() exit\n"); 646 647 return (status); 648 } 649 650 /* 651 * ibmf_saa_impl_invalidate_port: 652 * invalidates port entry (assumes exist) and deletes kstat object 653 * kstat object is destroyed in order to allow creating port entry 654 * even if this entry is not purged 655 */ 656 static void 657 ibmf_saa_impl_invalidate_port(saa_port_t *saa_portp) 658 { 659 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 660 ibmf_saa_impl_invalidate_port_start, 661 IBMF_TNF_TRACE, "", "ibmf_saa_impl_invalidate_port() enter\n"); 662 663 ASSERT(saa_portp != NULL); 664 ASSERT(MUTEX_HELD(&saa_portp->saa_pt_mutex)); 665 666 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_INVALID; 667 ibmf_saa_impl_uninit_kstats(saa_portp); 668 669 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 670 ibmf_saa_impl_invalidate_port_end, 671 IBMF_TNF_TRACE, "", "ibmf_saa_impl_invalidate_port() exit\n"); 672 } 673 674 /* 675 * ibmf_saa_impl_destroy_port: 676 * Frees the resources associated with an saa_portp structure. Assumes the 677 * saa_portp exists 678 * 679 * Input Arguments 680 * saa_portp pointer to saa_portp structure 681 * 682 * Output Arguments 683 * none 684 * 685 * Returns 686 * void 687 */ 688 static void 689 ibmf_saa_impl_destroy_port(saa_port_t *saa_portp) 690 { 691 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_destroy_start, 692 IBMF_TNF_TRACE, "", "ibmf_saa_impl_destroy() enter\n"); 693 694 ASSERT(saa_portp != NULL); 695 696 _NOTE(ASSUMING_PROTECTED(*saa_portp)) 697 698 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 699 ibmf_saa_impl_destroy, IBMF_TNF_TRACE, "", 700 "ibmf_saa_impl_destroy(): destroying port_guid %016" PRIx64 "\n", 701 tnf_opaque, port_guid, saa_portp->saa_pt_port_guid); 702 703 ibmf_saa_impl_uninit_kstats(saa_portp); 704 705 /* uninit synchronization variables used for registration */ 706 mutex_destroy(&saa_portp->saa_pt_mutex); 707 cv_destroy(&saa_portp->saa_pt_ibmf_reg_cv); 708 709 mutex_destroy(&saa_portp->saa_pt_event_sub_mutex); 710 mutex_destroy(&saa_portp->saa_pt_kstat_mutex); 711 712 kmem_free(saa_portp, sizeof (saa_port_t)); 713 714 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_destroy_end, 715 IBMF_TNF_TRACE, "", "ibmf_saa_impl_destroy() exit\n"); 716 } 717 718 /* 719 * ibmf_saa_impl_init_kstats: 720 * Create kstats structure. Should be called when memory is alloced for a new 721 * port entry. 722 */ 723 int 724 ibmf_saa_impl_init_kstats(saa_port_t *saa_portp) 725 { 726 char buf[128]; 727 ibmf_saa_kstat_t *ksp; 728 729 _NOTE(ASSUMING_PROTECTED(saa_portp->saa_pt_kstatp)) 730 731 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 732 ibmf_saa_impl_init_kstats_start, 733 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_kstats() enter\n"); 734 735 /* set up kstats structure */ 736 (void) sprintf(buf, "ibmf_saa_%016" PRIx64 "_stat", 737 saa_portp->saa_pt_port_guid); 738 739 saa_portp->saa_pt_kstatp = kstat_create("ibmf_saa", 740 0, buf, "misc", KSTAT_TYPE_NAMED, 741 sizeof (ibmf_saa_kstat_t) / sizeof (kstat_named_t), 742 KSTAT_FLAG_WRITABLE); 743 744 if (saa_portp->saa_pt_kstatp == NULL) 745 return (IBMF_NO_RESOURCES); 746 747 ksp = (ibmf_saa_kstat_t *)saa_portp->saa_pt_kstatp->ks_data; 748 749 kstat_named_init(&ksp->clients_registered, 750 "clients_registered", KSTAT_DATA_UINT32); 751 752 kstat_named_init(&ksp->clients_reg_failed, 753 "clients_reg_failed", KSTAT_DATA_UINT32); 754 755 kstat_named_init(&ksp->outstanding_requests, 756 "outstanding_requests", KSTAT_DATA_UINT32); 757 758 kstat_named_init(&ksp->total_requests, 759 "total_requests", KSTAT_DATA_UINT32); 760 761 kstat_named_init(&ksp->failed_requests, 762 "failed_requests", KSTAT_DATA_UINT32); 763 764 kstat_named_init(&ksp->requests_timedout, 765 "requests_timedout", KSTAT_DATA_UINT32); 766 767 kstat_install(saa_portp->saa_pt_kstatp); 768 769 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 770 ibmf_saa_impl_init_kstats_end, 771 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_kstats() exit\n"); 772 773 return (IBMF_SUCCESS); 774 } 775 776 /* 777 * ibmf_saa_impl_uninit_kstats: 778 * Free kstats context. Should be called when port is either destroyed 779 * or invalidated. 780 */ 781 static void 782 ibmf_saa_impl_uninit_kstats(saa_port_t *saa_portp) 783 { 784 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 785 ibmf_saa_impl_uninit_kstats_start, 786 IBMF_TNF_TRACE, "", "ibmf_saa_impl_uninit_kstats() enter\n"); 787 788 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 789 790 if (saa_portp->saa_pt_kstatp != NULL) { 791 kstat_delete(saa_portp->saa_pt_kstatp); 792 } 793 saa_portp->saa_pt_kstatp = NULL; 794 795 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 796 797 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 798 ibmf_saa_impl_uninit_kstats_end, 799 IBMF_TNF_TRACE, "", "ibmf_saa_impl_uninit_kstats() exit\n"); 800 } 801 802 /* 803 * ibmf_saa_impl_register_failed: 804 * invalidate entry and kick waiters 805 */ 806 void 807 ibmf_saa_impl_register_failed(saa_port_t *saa_portp) 808 { 809 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 810 ibmf_saa_impl_register_failed_start, 811 IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_failed() enter\n"); 812 813 mutex_enter(&saa_portp->saa_pt_mutex); 814 815 ibmf_saa_impl_invalidate_port(saa_portp); 816 817 cv_broadcast(&saa_portp->saa_pt_ibmf_reg_cv); 818 819 /* decrementing refcount is last thing we do on entry */ 820 821 ASSERT(saa_portp->saa_pt_reference_count > 0); 822 saa_portp->saa_pt_reference_count--; 823 824 mutex_exit(&saa_portp->saa_pt_mutex); 825 826 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 827 ibmf_saa_impl_register_failed_end, 828 IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_failed() exit\n"); 829 } 830 831 /* 832 * ibmf_saa_impl_register_port: 833 */ 834 int 835 ibmf_saa_impl_register_port( 836 saa_port_t *saa_portp) 837 { 838 uint_t hca_count = 0; 839 ib_guid_t *hca_list = NULL; 840 int status = IBMF_SUCCESS; 841 int unreg_status = IBMF_SUCCESS; 842 int ibt_status = IBT_SUCCESS; 843 ibt_hca_portinfo_t *port_info_list = NULL; 844 uint_t port_count = 0; 845 uint_t port_size = 0; 846 ib_pkey_t p_key; 847 ib_qkey_t q_key; 848 uint8_t portnum; 849 int ihca, iport; 850 ib_guid_t port_guid; 851 boolean_t qp_alloced = B_FALSE; 852 boolean_t ibmf_reg = B_FALSE; 853 854 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 855 ibmf_saa_impl_register_port_start, IBMF_TNF_TRACE, "", 856 "ibmf_saa_impl_register_port() enter\n"); 857 858 ASSERT(saa_portp != NULL); 859 860 _NOTE(ASSUMING_PROTECTED(*saa_portp)) 861 862 /* get the HCA list */ 863 864 hca_count = ibt_get_hca_list(&hca_list); 865 866 if (hca_count == 0) { 867 868 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 869 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 870 "ibmf_saa_impl_register_port: %s\n", 871 tnf_string, msg, "cannot register port (no HCAs).\n"); 872 873 status = IBMF_BAD_PORT; 874 goto bail; 875 } 876 877 /* lookup requested port guid in hca list */ 878 for (ihca = 0; ihca != hca_count; ihca++) { 879 880 ibt_status = ibt_query_hca_ports_byguid(hca_list[ihca], 881 0 /* all ports */, &port_info_list, 882 &port_count, &port_size); 883 884 if (ibt_status != IBT_SUCCESS) { 885 886 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 887 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 888 "ibmf_saa_impl_register_port: %s, %016" PRIx64 "\n", 889 tnf_string, msg, "Could not query hca. Exiting.", 890 tnf_opaque, guid, hca_list[ihca]); 891 892 status = IBMF_TRANSPORT_FAILURE; 893 break; 894 } 895 896 for (iport = 0; iport < port_count; iport++) { 897 898 /* get port guid associated with hca guid, port num */ 899 if (ibmf_saa_impl_get_port_guid( 900 port_info_list + iport, &port_guid) != IBMF_SUCCESS) 901 continue; 902 903 if (saa_portp->saa_pt_port_guid != port_guid) 904 continue; 905 906 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, 907 ibmf_saa_impl_register_port, 908 IBMF_TNF_TRACE, "", 909 "ibmf_saa_impl_register_port: %s, hca_guid = %016" 910 PRIx64 ", port_guid = %016" PRIx64 911 ", number = %d\n", 912 tnf_string, msg, "found port", 913 tnf_opaque, hca_guid, hca_list[ihca], 914 tnf_opaque, port_guid, port_guid, 915 tnf_uint, port, iport + 1); 916 917 /* 918 * we're here? then we found our port: 919 * fill in ibmf registration info 920 * and address parameters from the portinfo 921 */ 922 923 saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid 924 = hca_list[ihca]; 925 saa_portp->saa_pt_ibmf_reginfo.ir_port_num = iport+1; 926 saa_portp->saa_pt_ibmf_reginfo.ir_client_class 927 = SUBN_ADM_MANAGER; 928 929 saa_portp->saa_pt_node_guid = hca_list[ihca]; 930 saa_portp->saa_pt_port_num = iport + 1; 931 932 ibmf_saa_impl_set_transaction_params( 933 saa_portp, port_info_list + iport); 934 break; 935 } 936 937 ibt_free_portinfo(port_info_list, port_size); 938 939 if (iport != port_count) 940 break; /* found our port */ 941 } 942 943 ibt_free_hca_list(hca_list, hca_count); 944 945 if (ihca == hca_count) { 946 947 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 948 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 949 "ibmf_saa_impl_register_port: %s, port_guid %016" 950 PRIx64 "\n", 951 tnf_string, msg, "Could not find port, exiting", 952 tnf_opaque, port_guid, saa_portp->saa_pt_port_guid); 953 954 status = IBMF_BAD_PORT; 955 } 956 957 if (status != IBMF_SUCCESS) { 958 959 goto bail; 960 } 961 962 /* 963 * Now we found the port we searched for, 964 * and open an ibmf session on that port. 965 */ 966 967 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 968 ibmf_saa_impl_register_port, IBMF_TNF_TRACE, "", 969 "ibmf_saa_impl_register_port: %s, port_guid = %016" PRIx64 970 ", port = %d\n", tnf_string, msg, "Registering with ibmf", 971 tnf_opaque, port_guid, saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid, 972 tnf_uint, port, saa_portp->saa_pt_ibmf_reginfo.ir_port_num); 973 974 status = ibmf_register(&saa_portp->saa_pt_ibmf_reginfo, 975 IBMF_VERSION, IBMF_REG_FLAG_RMPP, 976 ibmf_saa_impl_async_event_cb, saa_portp, 977 &saa_portp->saa_pt_ibmf_handle, 978 &saa_portp->saa_pt_ibmf_impl_features); 979 980 if (status != IBMF_SUCCESS) { 981 982 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 983 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 984 "ibmf_saa_impl_register_port: %s, ibmf_status = %d\n", 985 tnf_string, msg, "Could not register with ibmf", 986 tnf_int, status, status); 987 988 goto bail; 989 } 990 991 ibmf_reg = B_TRUE; 992 993 /* allocate a qp through ibmf */ 994 status = ibmf_alloc_qp(saa_portp->saa_pt_ibmf_handle, 995 IB_PKEY_DEFAULT_LIMITED, IB_GSI_QKEY, IBMF_ALT_QP_MAD_RMPP, 996 &saa_portp->saa_pt_qp_handle); 997 998 if (status != IBMF_SUCCESS) { 999 1000 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1001 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 1002 "ibmf_saa_impl_register_port: %s, ibmf_status = %d\n", 1003 tnf_string, msg, "Cannot alloc qp with ibmf", 1004 tnf_int, status, status); 1005 1006 goto bail; 1007 } 1008 1009 qp_alloced = B_TRUE; 1010 1011 /* 1012 * query the queue pair number; we will need it to unsubscribe from 1013 * notice reports 1014 */ 1015 status = ibmf_query_qp(saa_portp->saa_pt_ibmf_handle, 1016 saa_portp->saa_pt_qp_handle, &saa_portp->saa_pt_qpn, &p_key, &q_key, 1017 &portnum, 0); 1018 if (status != IBMF_SUCCESS) { 1019 1020 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1021 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 1022 "ibmf_saa_impl_register_port: %s, ibmf_status = %d\n", 1023 tnf_string, msg, "Cannot query alt qp to get qp num", 1024 tnf_int, status, status); 1025 1026 goto bail; 1027 } 1028 1029 /* 1030 * core ibmf is taking advantage of the fact that saa_portp is our 1031 * callback arg. If this changes, the code in ibmf_recv would need to 1032 * change as well 1033 */ 1034 status = ibmf_setup_async_cb(saa_portp->saa_pt_ibmf_handle, 1035 saa_portp->saa_pt_qp_handle, ibmf_saa_report_cb, saa_portp, 0); 1036 if (status != IBMF_SUCCESS) { 1037 1038 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1039 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 1040 "ibmf_saa_impl_register_port: %s, ibmf_status = %d\n", 1041 tnf_string, msg, "Cannot register async cb with ibmf", 1042 tnf_int, status, status); 1043 1044 goto bail; 1045 } 1046 1047 return (IBMF_SUCCESS); 1048 1049 bail: 1050 if (qp_alloced == B_TRUE) { 1051 1052 /* free alternate qp */ 1053 unreg_status = ibmf_free_qp(saa_portp->saa_pt_ibmf_handle, 1054 &saa_portp->saa_pt_qp_handle, 0); 1055 if (unreg_status != IBMF_SUCCESS) { 1056 1057 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1058 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 1059 "ibmf_saa_impl_register_port: %s, ibmf_status =" 1060 " %d\n", tnf_string, msg, 1061 "Cannot free alternate queue pair with ibmf", 1062 tnf_int, unreg_status, unreg_status); 1063 } 1064 } 1065 1066 if (ibmf_reg == B_TRUE) { 1067 /* unregister from ibmf */ 1068 unreg_status = ibmf_unregister( 1069 &saa_portp->saa_pt_ibmf_handle, 0); 1070 1071 if (unreg_status != IBMF_SUCCESS) { 1072 1073 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1074 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 1075 "ibmf_saa_impl_register_port: %s, ibmf_status =" 1076 " %d\n", tnf_string, msg, 1077 "Cannot unregister from ibmf", 1078 tnf_int, unreg_status, unreg_status); 1079 } 1080 } 1081 1082 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_register_port_end, 1083 IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_port() exit\n"); 1084 1085 return (status); 1086 } 1087 1088 /* 1089 * ibmf_saa_impl_getclassportinfo: 1090 */ 1091 void 1092 ibmf_saa_impl_get_classportinfo(saa_port_t *saa_portp) 1093 { 1094 int res; 1095 saa_impl_trans_info_t *trans_info; 1096 1097 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1098 ibmf_saa_impl_get_classportinfo_start, 1099 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_classportinfo() enter\n"); 1100 1101 /* 1102 * allocate memory for trans_info; send_request's callback will free up 1103 * memory since request is asynchronous 1104 */ 1105 trans_info = kmem_zalloc(sizeof (saa_impl_trans_info_t), KM_NOSLEEP); 1106 if (trans_info == NULL) { 1107 1108 mutex_enter(&saa_portp->saa_pt_mutex); 1109 1110 /* cpi transaction is handled as a client, decrement refcount */ 1111 ASSERT(saa_portp->saa_pt_reference_count > 0); 1112 saa_portp->saa_pt_reference_count--; 1113 1114 mutex_exit(&saa_portp->saa_pt_mutex); 1115 1116 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1117 ibmf_saa_impl_get_classportinfo_err, IBMF_TNF_ERROR, "", 1118 "ibmf_saa_impl_get_classportinfo: %s\n", tnf_string, msg, 1119 "Could not allocate memory for classportinfo trans_info"); 1120 1121 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1122 ibmf_saa_impl_get_classportinfo_end, IBMF_TNF_TRACE, "", 1123 "ibmf_saa_impl_get_classportinfo() exiting\n"); 1124 1125 return; 1126 } 1127 1128 /* no specific client associated with this transaction */ 1129 trans_info->si_trans_client_data = NULL; 1130 trans_info->si_trans_port = saa_portp; 1131 trans_info->si_trans_method = SA_SUBN_ADM_GET; 1132 trans_info->si_trans_attr_id = MAD_ATTR_ID_CLASSPORTINFO; 1133 1134 trans_info->si_trans_callback = ibmf_saa_impl_get_cpi_cb; 1135 trans_info->si_trans_callback_arg = saa_portp; 1136 1137 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 1138 1139 IBMF_SAA_ADD32_KSTATS(saa_portp, outstanding_requests, 1); 1140 IBMF_SAA_ADD32_KSTATS(saa_portp, total_requests, 1); 1141 1142 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 1143 1144 res = ibmf_saa_impl_send_request(trans_info); 1145 1146 if (res != IBMF_SUCCESS) { 1147 1148 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 1149 ibmf_saa_impl_get_classportinfo_err, IBMF_TNF_TRACE, "", 1150 "ibmf_saa_impl_get_classportinfo: %s, res = 0x%x\n", 1151 tnf_string, msg, "ibmf_saa_impl_send_request failed", 1152 tnf_opaque, res, res); 1153 1154 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 1155 1156 IBMF_SAA_SUB32_KSTATS(saa_portp, outstanding_requests, 1); 1157 IBMF_SAA_ADD32_KSTATS(saa_portp, failed_requests, 1); 1158 1159 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 1160 1161 mutex_enter(&saa_portp->saa_pt_mutex); 1162 1163 /* cpi transaction is handled as a client, decrement refcount */ 1164 ASSERT(saa_portp->saa_pt_reference_count > 0); 1165 saa_portp->saa_pt_reference_count--; 1166 1167 mutex_exit(&saa_portp->saa_pt_mutex); 1168 1169 } 1170 1171 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1172 ibmf_saa_impl_get_classportinfo_end, 1173 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_classportinfo() exit\n"); 1174 } 1175 1176 /* 1177 * ibmf_saa_impl_get_cpi_cb: 1178 * 1179 * Called when the asynchronous getportinfo request receives its response. 1180 * Checks the status. If success, updates the times in the port's 1181 * ibmf_retrans structure that is used in ibmf_msg_transport calls. If failure, 1182 * just use default values. 1183 * 1184 * Input Arguments 1185 * arg user-specified pointer (points to the current port data) 1186 * length length of payload returned (should be size of classportinfo_t) 1187 * buffer pointer to classportinfo returned (should not be null) 1188 * status status of sa access request 1189 * 1190 * Output Arguments 1191 * none 1192 * 1193 * Returns void 1194 */ 1195 static void 1196 ibmf_saa_impl_get_cpi_cb(void *arg, size_t length, char *buffer, int status) 1197 { 1198 saa_port_t *saa_portp; 1199 uint64_t base_time, resp_timeout, rttv_timeout; 1200 ib_mad_classportinfo_t *classportinfo; 1201 int resp_time_value; 1202 uint16_t sa_cap_mask; 1203 1204 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_get_cpi_cb_start, 1205 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_cpi_cb() enter\n"); 1206 1207 /* 1208 * access port entry: note that it may have become invalid 1209 * but we hold a ref count for cpi and the interactions on 1210 * the entry are harmless 1211 */ 1212 saa_portp = (saa_port_t *)arg; 1213 1214 /* process response */ 1215 1216 if ((status != IBMF_SUCCESS) || (buffer == NULL)) { 1217 1218 IBMF_TRACE_4(IBMF_TNF_NODEBUG, DPRINT_L1, 1219 ibmf_saa_impl_get_cpi_cb, IBMF_TNF_ERROR, "", 1220 "ibmf_saa_impl_get_cpi_cb: %s, status = %d, buffer = " 1221 " 0x%p, length = %d\n", tnf_string, msg, 1222 "could not get classportinfo. Check node and path to sm" 1223 " lid", tnf_int, status, status, 1224 tnf_opaque, buffer, buffer, tnf_uint, length, length); 1225 1226 /* 1227 * IB spec (C13-13) indicates 20 can be used as default or 1228 * intial value for classportinfo->resptimeout value 1229 */ 1230 resp_time_value = 20; 1231 1232 sa_cap_mask = 0xFFFF; 1233 1234 } else if (buffer != NULL) { 1235 1236 classportinfo = (ib_mad_classportinfo_t *)buffer; 1237 1238 resp_time_value = classportinfo->RespTimeValue & 0x1f; 1239 1240 sa_cap_mask = classportinfo->CapabilityMask; 1241 1242 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 1243 ibmf_saa_impl_get_cpi_cb, IBMF_TNF_TRACE, "", 1244 "ibmf_saa_impl_get_cpi_cb: %s, timeout = 0x%x," 1245 " cap_mask = 0x%x\n", 1246 tnf_string, msg, "got classportinfo", 1247 tnf_opaque, timeout, resp_time_value, 1248 tnf_opaque, cap_mask, sa_cap_mask); 1249 1250 kmem_free(buffer, length); 1251 } 1252 1253 /* 1254 * using IB spec calculation from 13.4.6.2 1255 * use bit shifting for 2^x. 1256 */ 1257 base_time = (1 << resp_time_value); 1258 1259 resp_timeout = (4 * base_time * 1000 + 96 * base_time) / 1000; 1260 1261 mutex_enter(&saa_portp->saa_pt_mutex); 1262 1263 base_time = 2 * (1 << saa_portp->saa_pt_timeout); 1264 1265 rttv_timeout = (4 * base_time * 1000 + 96 * base_time) / 1000; 1266 1267 saa_portp->saa_pt_ibmf_retrans.retrans_rtv = resp_timeout; 1268 saa_portp->saa_pt_ibmf_retrans.retrans_rttv = rttv_timeout; 1269 saa_portp->saa_pt_sa_cap_mask = sa_cap_mask; 1270 1271 /* 1272 * cpi transaction is handled as a client, 1273 * decrement refcount; make sure it's the last 1274 * thing we do on this entry 1275 */ 1276 ASSERT(saa_portp->saa_pt_reference_count > 0); 1277 saa_portp->saa_pt_reference_count--; 1278 1279 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 1280 ibmf_saa_impl_get_cpi_cb, IBMF_TNF_TRACE, "", 1281 "ibmf_saa_impl_get_cpi_cb: %s, subnet_timeout = 0x%x, " 1282 "resp_time_value = 0x%x\n", 1283 tnf_string, msg, "updated resp timeout", 1284 tnf_opaque, subnet_timeout, saa_portp->saa_pt_timeout, 1285 tnf_opaque, resp_time_value, resp_time_value); 1286 1287 mutex_exit(&saa_portp->saa_pt_mutex); 1288 1289 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_get_cpi_cb_end, 1290 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_cpi_cb() exit\n"); 1291 } 1292 1293 /* 1294 * ibmf_saa_impl_send_request: 1295 * Sends a request to the sa. Can be used for both classportinfo and record 1296 * requests. Will set up all data structures for using the multi-packet 1297 * protocol, create the mad, and send it. Returns SA_SUCCESS if msg transport 1298 * worked, meaning succesful send for the async case and a succesful send and 1299 * recv for the sync case. 1300 */ 1301 int 1302 ibmf_saa_impl_send_request(saa_impl_trans_info_t *trans_info) 1303 { 1304 uint16_t attr_id; 1305 saa_client_data_t *client_data; 1306 saa_port_t *saa_portp; 1307 uint32_t transport_flags; 1308 ibmf_msg_cb_t ibmf_callback; 1309 void *ibmf_callback_arg; 1310 ibmf_msg_t *msgp; 1311 ibmf_retrans_t ibmf_retrans; 1312 uint16_t sa_cap_mask; 1313 boolean_t sleep_flag; 1314 int ibmf_status = IBMF_SUCCESS; 1315 int retry_count; 1316 uint16_t mad_status; 1317 boolean_t sa_is_redirected = B_FALSE; 1318 1319 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1320 ibmf_saa_impl_send_request_start, 1321 IBMF_TNF_TRACE, "", "ibmf_saa_impl_send_request() enter\n"); 1322 1323 attr_id = trans_info->si_trans_attr_id; 1324 client_data = trans_info->si_trans_client_data; 1325 saa_portp = trans_info->si_trans_port; 1326 1327 /* 1328 * don't send on invalid entry 1329 * Note that there is a window where it could become 1330 * invalid after this test is done, but we'd rely on ibmf errors... 1331 */ 1332 if (ibmf_saa_is_valid(saa_portp, B_FALSE) == B_FALSE) { 1333 1334 IBMF_TRACE_4(IBMF_TNF_NODEBUG, DPRINT_L1, 1335 ibmf_saa_impl_send_request, 1336 IBMF_TNF_ERROR, "", 1337 "ibmf_saa_impl_send_request: %s, hca_guid = %016" 1338 PRIx64 ", port_guid = %016" PRIx64 1339 ", number = %d\n", 1340 tnf_string, msg, "sending on invalid port", 1341 tnf_opaque, hca_guid, 1342 saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid, 1343 tnf_opaque, port_guid, 1344 saa_portp->saa_pt_port_guid, 1345 tnf_uint, port, 1346 saa_portp->saa_pt_ibmf_reginfo.ir_port_num); 1347 1348 ibmf_status = IBMF_REQ_INVALID; 1349 goto bail; 1350 } 1351 1352 /* check whether SA supports this attribute */ 1353 mutex_enter(&saa_portp->saa_pt_mutex); 1354 1355 sa_cap_mask = saa_portp->saa_pt_sa_cap_mask; 1356 sa_is_redirected = saa_portp->saa_pt_redirect_active; 1357 1358 mutex_exit(&saa_portp->saa_pt_mutex); 1359 1360 ibmf_status = ibmf_saa_impl_check_sa_support(sa_cap_mask, attr_id); 1361 1362 if (ibmf_status != IBMF_SUCCESS) { 1363 1364 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1365 ibmf_saa_impl_send_request_err, IBMF_TNF_ERROR, "", 1366 "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n", 1367 tnf_string, msg, "SA does not support attribute", 1368 tnf_int, ibmf_status, ibmf_status); 1369 1370 goto bail; 1371 } 1372 1373 /* make only non-blocking calls if this is an async request */ 1374 if ((trans_info->si_trans_callback == NULL) && 1375 (trans_info->si_trans_sub_callback == NULL)) { 1376 ibmf_callback = NULL; 1377 ibmf_callback_arg = NULL; 1378 sleep_flag = B_TRUE; 1379 } else { 1380 ibmf_callback = ibmf_saa_async_cb; 1381 ibmf_callback_arg = (void *)trans_info; 1382 sleep_flag = B_FALSE; 1383 } 1384 1385 ibmf_status = ibmf_saa_impl_init_msg(trans_info, sleep_flag, &msgp, 1386 &transport_flags, &ibmf_retrans); 1387 if (ibmf_status != IBMF_SUCCESS) { 1388 1389 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1390 ibmf_saa_impl_send_request_err, IBMF_TNF_ERROR, "", 1391 "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n", 1392 tnf_string, msg, "init_msg() failed", 1393 tnf_int, ibmf_status, ibmf_status); 1394 1395 goto bail; 1396 } 1397 1398 mutex_enter(&saa_portp->saa_pt_mutex); 1399 1400 saa_portp->saa_pt_num_outstanding_trans++; 1401 1402 mutex_exit(&saa_portp->saa_pt_mutex); 1403 1404 /* 1405 * increment the number of outstanding transaction so 1406 * ibmf_close_sa_session() will wait. classportinfo requests 1407 * don't have associated clients so check for valid clientp 1408 */ 1409 if (client_data != NULL) { 1410 1411 mutex_enter(&client_data->saa_client_mutex); 1412 1413 client_data->saa_client_num_pending_trans++; 1414 1415 mutex_exit(&client_data->saa_client_mutex); 1416 } 1417 1418 /* 1419 * make the call to msg_transport. If synchronous and success, 1420 * check that the response mad isn't status busy. If so, repeat the 1421 * call 1422 */ 1423 retry_count = 0; 1424 1425 /* 1426 * set the send time here. We only set this once at the beginning of 1427 * the transaction. Retrying because of busys or mastersmlid changes 1428 * does not change the original send time. It is meant to be an 1429 * absolute time out value and will only be used if there are other 1430 * problems (i.e. a buggy SA) 1431 */ 1432 trans_info->si_trans_send_time = gethrtime(); 1433 1434 for (;;) { 1435 1436 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle, 1437 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 1438 ibmf_callback, ibmf_callback_arg, transport_flags); 1439 1440 if (ibmf_callback != NULL) 1441 break; 1442 1443 /* 1444 * stop here for non-sequenced transactions since they wouldn't 1445 * receive a timeout or busy response 1446 */ 1447 if (!(transport_flags & IBMF_MSG_TRANS_FLAG_SEQ)) 1448 break; 1449 1450 /* 1451 * if the transaction timed out and this was a synchronous 1452 * request there's a possiblity we were talking to the wrong 1453 * master smlid or that the SA has stopped responding on the 1454 * redirected desination (if redirect is active). 1455 * Check this and retry if necessary. 1456 */ 1457 if ((ibmf_status == IBMF_TRANS_TIMEOUT) && 1458 (sleep_flag == B_TRUE)) { 1459 if (sa_is_redirected == B_TRUE) { 1460 ibmf_status = ibmf_saa_impl_revert_to_qp1( 1461 saa_portp, msgp, ibmf_callback, 1462 ibmf_callback_arg, transport_flags); 1463 } else { 1464 ibmf_status = ibmf_saa_impl_new_smlid_retry( 1465 saa_portp, msgp, ibmf_callback, 1466 ibmf_callback_arg, transport_flags); 1467 } 1468 } 1469 1470 /* 1471 * if the transaction timed out (and retrying with a new SM LID 1472 * didn't help) check how long it's been since we received an SA 1473 * packet. If it hasn't been max_wait_time then retry the 1474 * request. 1475 */ 1476 if ((ibmf_status == IBMF_TRANS_TIMEOUT) && 1477 (sleep_flag == B_TRUE)) { 1478 1479 ibmf_status = ibmf_saa_check_sa_and_retry( 1480 saa_portp, msgp, ibmf_callback, ibmf_callback_arg, 1481 trans_info->si_trans_send_time, transport_flags); 1482 } 1483 1484 if (ibmf_status != IBMF_SUCCESS) 1485 break; 1486 1487 if (retry_count >= IBMF_SAA_MAX_BUSY_RETRY_COUNT) 1488 break; 1489 1490 /* sync transaction with status SUCCESS should have response */ 1491 ASSERT(msgp->im_msgbufs_recv.im_bufs_mad_hdr != NULL); 1492 1493 mad_status = b2h16(msgp->im_msgbufs_recv. 1494 im_bufs_mad_hdr->Status); 1495 1496 if ((mad_status != MAD_STATUS_BUSY) && 1497 (mad_status != MAD_STATUS_REDIRECT_REQUIRED)) 1498 break; 1499 1500 if (mad_status == MAD_STATUS_REDIRECT_REQUIRED) { 1501 1502 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 1503 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1504 "ibmf_saa_impl_send_request: %s, retry_count %d\n", 1505 tnf_string, msg, 1506 "response returned redirect status", 1507 tnf_int, retry_count, retry_count); 1508 1509 /* update address info and copy it into msgp */ 1510 ibmf_saa_impl_update_sa_address_info(saa_portp, msgp); 1511 } else { 1512 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 1513 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1514 "ibmf_saa_impl_send_request: %s, retry_count %d\n", 1515 tnf_string, msg, "response returned busy status", 1516 tnf_int, retry_count, retry_count); 1517 } 1518 1519 retry_count++; 1520 1521 /* 1522 * since this is a blocking call, sleep for some time 1523 * to allow SA to transition from busy state (if busy) 1524 */ 1525 if (mad_status == MAD_STATUS_BUSY) 1526 delay(drv_usectohz( 1527 IBMF_SAA_BUSY_RETRY_SLEEP_SECS * 1000000)); 1528 } 1529 1530 if (ibmf_status != IBMF_SUCCESS) { 1531 1532 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 1533 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1534 "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n", 1535 tnf_string, msg, "ibmf_msg_transport() failed", 1536 tnf_int, ibmf_status, ibmf_status); 1537 1538 ibmf_saa_impl_free_msg(saa_portp->saa_pt_ibmf_handle, msgp); 1539 1540 mutex_enter(&saa_portp->saa_pt_mutex); 1541 1542 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0); 1543 saa_portp->saa_pt_num_outstanding_trans--; 1544 1545 mutex_exit(&saa_portp->saa_pt_mutex); 1546 1547 if (client_data != NULL) { 1548 1549 mutex_enter(&client_data->saa_client_mutex); 1550 1551 ASSERT(client_data->saa_client_num_pending_trans > 0); 1552 client_data->saa_client_num_pending_trans--; 1553 1554 if ((client_data->saa_client_num_pending_trans == 0) && 1555 (client_data->saa_client_state == 1556 SAA_CLIENT_STATE_WAITING)) 1557 cv_signal(&client_data->saa_client_state_cv); 1558 1559 mutex_exit(&client_data->saa_client_mutex); 1560 } 1561 1562 } else if (sleep_flag == B_TRUE) { 1563 1564 mutex_enter(&saa_portp->saa_pt_mutex); 1565 1566 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0); 1567 saa_portp->saa_pt_num_outstanding_trans--; 1568 1569 mutex_exit(&saa_portp->saa_pt_mutex); 1570 1571 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 1572 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1573 "ibmf_saa_impl_send_request: %s\n", 1574 tnf_string, msg, "Message sent and received successfully"); 1575 1576 /* fill in response values and free the message */ 1577 ibmf_saa_impl_prepare_response(saa_portp->saa_pt_ibmf_handle, 1578 msgp, B_FALSE, &trans_info->si_trans_status, 1579 &trans_info->si_trans_result, 1580 &trans_info->si_trans_length, sleep_flag); 1581 1582 if (client_data != NULL) { 1583 mutex_enter(&client_data->saa_client_mutex); 1584 1585 ASSERT(client_data->saa_client_num_pending_trans > 0); 1586 client_data->saa_client_num_pending_trans--; 1587 1588 if ((client_data->saa_client_num_pending_trans == 0) && 1589 (client_data->saa_client_state == 1590 SAA_CLIENT_STATE_WAITING)) 1591 cv_signal(&client_data->saa_client_state_cv); 1592 1593 mutex_exit(&client_data->saa_client_mutex); 1594 } 1595 } else { 1596 1597 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 1598 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1599 "ibmf_saa_impl_send_request: %s\n", 1600 tnf_string, msg, "Message sent successfully"); 1601 } 1602 1603 bail: 1604 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 1605 ibmf_saa_impl_send_request_end, 1606 IBMF_TNF_TRACE, "", "ibmf_saa_impl_send_request() exiting" 1607 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 1608 1609 return (ibmf_status); 1610 } 1611 1612 /* 1613 * ibmf_saa_impl_init_msg: 1614 * Allocates an ibmf message and fills out the header fields and formatted data 1615 * fields. Also sets up the correct transport_flags and retrans argument for 1616 * the message transport call based on the request information. 1617 * 1618 * Input Arguments 1619 * trans_info saa_trans_info structure passed to send_request 1620 * sleep_flag B_TRUE if init_msg can sleep in function calls 1621 * 1622 * Output Arguments 1623 * msgp ibmf message that should be given to msg_transport 1624 * transport_flagsp transport flags that should be given to msg_transport 1625 * ibmf_retrans_t retrans parameter that should be given to msg_transport 1626 * 1627 * Returns 1628 * ibmf_status 1629 */ 1630 static int 1631 ibmf_saa_impl_init_msg(saa_impl_trans_info_t *trans_info, boolean_t sleep_flag, 1632 ibmf_msg_t **msgp, uint32_t *transport_flagsp, 1633 ibmf_retrans_t *ibmf_retransp) 1634 { 1635 int ibmf_status; 1636 ibmf_msg_bufs_t *req_mad; 1637 ib_mad_hdr_t *mad_hdr; 1638 int ibmf_sleep_flag, km_sleep_flag; 1639 int free_res; 1640 ib_sa_hdr_t sa_hdr; 1641 ibmf_msg_t *ibmf_msg; 1642 uint16_t attr_id, pack_attr_id; 1643 uint8_t method; 1644 saa_client_data_t *client_data; 1645 saa_port_t *saa_portp; 1646 sa_multipath_record_t *multipath_template; 1647 size_t payload_length; 1648 uint32_t transport_flags; 1649 1650 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1651 ibmf_saa_impl_init_msg_start, 1652 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_msg() entering\n"); 1653 1654 attr_id = trans_info->si_trans_attr_id; 1655 method = trans_info->si_trans_method; 1656 client_data = trans_info->si_trans_client_data; 1657 saa_portp = trans_info->si_trans_port; 1658 1659 if (sleep_flag == B_TRUE) { 1660 ibmf_sleep_flag = IBMF_ALLOC_SLEEP; 1661 km_sleep_flag = KM_SLEEP; 1662 } else { 1663 ibmf_sleep_flag = IBMF_ALLOC_NOSLEEP; 1664 km_sleep_flag = KM_NOSLEEP; 1665 } 1666 1667 ibmf_status = ibmf_alloc_msg(saa_portp->saa_pt_ibmf_handle, 1668 ibmf_sleep_flag, &ibmf_msg); 1669 if (ibmf_status != IBMF_SUCCESS) { 1670 1671 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1672 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "", 1673 "ibmf_saa_impl_init_msg: %s, ibmf_status = %d\n", 1674 tnf_string, msg, "Cannot allocate msg_buf.", 1675 tnf_int, ibmf_status, ibmf_status); 1676 1677 goto bail; 1678 } 1679 1680 req_mad = &ibmf_msg->im_msgbufs_send; 1681 1682 /* create a template (SA MAD) */ 1683 mad_hdr = kmem_zalloc(sizeof (ib_mad_hdr_t), km_sleep_flag); 1684 1685 if (mad_hdr == NULL) { 1686 1687 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1688 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "", 1689 "ibmf_saa_impl_init_msg: %s\n", 1690 tnf_string, msg, "Cannot allocate mad header."); 1691 1692 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle, 1693 &ibmf_msg); 1694 ASSERT(free_res == IBMF_SUCCESS); 1695 1696 ibmf_status = IBMF_NO_MEMORY; 1697 goto bail; 1698 } 1699 1700 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mad_hdr, 1701 *req_mad)) 1702 1703 bzero(mad_hdr, sizeof (ib_mad_hdr_t)); 1704 mad_hdr->BaseVersion = SAA_MAD_BASE_VERSION; 1705 mad_hdr->MgmtClass = MAD_MGMT_CLASS_SUBN_ADM; 1706 mad_hdr->ClassVersion = SAA_MAD_CLASS_VERSION; 1707 mad_hdr->R_Method = method; 1708 mad_hdr->AttributeID = h2b16(attr_id); 1709 1710 /* attribute modifier is all Fs since RIDs are no longer used */ 1711 mad_hdr->AttributeModifier = h2b32(0xffffffff); 1712 1713 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, 1714 ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "", 1715 "ibmf_saa_impl_init_msg: %s, class = 0x%x, method = 0x%x," 1716 " attr_id = 0x%x\n", tnf_string, msg, "Sending MAD", 1717 tnf_opaque, class, mad_hdr->MgmtClass, 1718 tnf_opaque, method, mad_hdr->R_Method, 1719 tnf_opaque, attr_id, attr_id); 1720 1721 bzero(&sa_hdr, sizeof (ib_sa_hdr_t)); 1722 sa_hdr.ComponentMask = trans_info->si_trans_component_mask; 1723 1724 if (client_data != NULL) 1725 sa_hdr.SM_KEY = client_data->saa_client_sm_key; 1726 1727 /* 1728 * pack data for IB wire format; req_mad will have different pointers to 1729 * sa header and payload, mad_hdr will be the same 1730 */ 1731 req_mad->im_bufs_mad_hdr = mad_hdr; 1732 1733 ibmf_status = ibmf_saa_utils_pack_sa_hdr(&sa_hdr, 1734 &req_mad->im_bufs_cl_hdr, &req_mad->im_bufs_cl_hdr_len, 1735 km_sleep_flag); 1736 1737 if (ibmf_status != IBMF_SUCCESS) { 1738 1739 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1740 ibmf_saa_impl_init_msg, IBMF_TNF_ERROR, "", 1741 "ibmf_saa_impl_init_msg: %s, ibmf_status = %d\n", 1742 tnf_string, msg, "ibmf_saa_utils_pack_sa_hdr() failed", 1743 tnf_int, ibmf_status, ibmf_status); 1744 1745 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t)); 1746 1747 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle, 1748 &ibmf_msg); 1749 ASSERT(free_res == IBMF_SUCCESS); 1750 1751 goto bail; 1752 } 1753 1754 if (attr_id == SA_MULTIPATHRECORD_ATTRID) { 1755 1756 multipath_template = 1757 (sa_multipath_record_t *)trans_info->si_trans_template; 1758 1759 payload_length = sizeof (sa_multipath_record_t) + 1760 ((multipath_template->SGIDCount + 1761 multipath_template->DGIDCount) * sizeof (ib_gid_t)); 1762 1763 pack_attr_id = attr_id; 1764 } else { 1765 1766 /* trace record template is a path record */ 1767 pack_attr_id = (attr_id == SA_TRACERECORD_ATTRID) ? 1768 SA_PATHRECORD_ATTRID : attr_id; 1769 1770 payload_length = ibmf_saa_impl_get_attr_id_length(pack_attr_id); 1771 1772 if (payload_length == 0) { 1773 payload_length = trans_info->si_trans_template_length; 1774 1775 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 1776 ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "", 1777 "ibmf_saa_impl_init_msg: %s, length = %d\n", 1778 tnf_string, msg, 1779 "Unknown attribute. Using user-defined length.", 1780 tnf_uint, length, payload_length) 1781 } 1782 } 1783 1784 /* transport type depends on method */ 1785 switch (method) { 1786 1787 case SA_SUBN_ADM_GET: 1788 case SA_SUBN_ADM_DELETE: 1789 case SA_SUBN_ADM_GET_TABLE: 1790 case SA_SUBN_ADM_GET_TRACE_TABLE: 1791 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ; 1792 break; 1793 case SA_SUBN_ADM_SET: 1794 /* unsubscribes can be sequenced or unsequenced */ 1795 if (trans_info->si_trans_unseq_unsubscribe == B_TRUE) { 1796 transport_flags = 0; 1797 } else { 1798 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ; 1799 } 1800 break; 1801 case SA_SUBN_ADM_GET_MULTI: 1802 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ | 1803 IBMF_MSG_TRANS_FLAG_RMPP; 1804 break; 1805 default : 1806 ibmf_status = IBMF_UNSUPP_METHOD; 1807 goto bail; 1808 } 1809 1810 trans_info->si_trans_transport_flags = transport_flags; 1811 1812 if (trans_info->si_trans_template != NULL) { 1813 1814 ibmf_status = ibmf_saa_utils_pack_payload( 1815 trans_info->si_trans_template, payload_length, pack_attr_id, 1816 &req_mad->im_bufs_cl_data, &req_mad->im_bufs_cl_data_len, 1817 km_sleep_flag); 1818 if (ibmf_status != IBMF_SUCCESS) { 1819 1820 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1821 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "", 1822 "ibmf_saa_impl_init_msg: %s, ibmf_status =" 1823 " %d\n", tnf_string, msg, 1824 "ibmf_saa_utils_pack_payload() failed", 1825 tnf_int, ibmf_status, ibmf_status); 1826 1827 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t)); 1828 1829 kmem_free(req_mad->im_bufs_cl_hdr, 1830 req_mad->im_bufs_cl_hdr_len); 1831 1832 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle, 1833 &ibmf_msg); 1834 ASSERT(free_res == IBMF_SUCCESS); 1835 1836 goto bail; 1837 } 1838 1839 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 1840 ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "", 1841 "ibmf_saa_impl_init_msg: %s, attr_id = 0x%x, length =" 1842 " %d\n", tnf_string, msg, "Packed payload successfully", 1843 tnf_opaque, attr_id, attr_id, 1844 tnf_uint, length, req_mad->im_bufs_cl_data_len); 1845 1846 /* non-RMPP transactions have template size limit */ 1847 if (((transport_flags & IBMF_MSG_TRANS_FLAG_RMPP) == 0) && 1848 ((req_mad->im_bufs_cl_data_len + req_mad->im_bufs_cl_hdr_len 1849 + sizeof (ib_mad_hdr_t)) > IBMF_MAD_SIZE)) { 1850 1851 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1852 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "", 1853 "ibmf_saa_impl_init_msg: %s\n", tnf_string, msg, 1854 "Template too large to fit in single packet"); 1855 1856 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t)); 1857 1858 kmem_free(req_mad->im_bufs_cl_hdr, 1859 req_mad->im_bufs_cl_hdr_len); 1860 1861 kmem_free(req_mad->im_bufs_cl_data, 1862 req_mad->im_bufs_cl_data_len); 1863 1864 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle, 1865 &ibmf_msg); 1866 ASSERT(free_res == IBMF_SUCCESS); 1867 1868 ibmf_status = IBMF_REQ_INVALID; 1869 goto bail; 1870 } 1871 } 1872 1873 mutex_enter(&saa_portp->saa_pt_mutex); 1874 1875 mad_hdr->TransactionID = h2b64(saa_portp->saa_pt_current_tid++); 1876 1877 bcopy(&saa_portp->saa_pt_ibmf_retrans, ibmf_retransp, 1878 sizeof (ibmf_retrans_t)); 1879 1880 /* copy local addressing information to message */ 1881 bcopy(&saa_portp->saa_pt_ibmf_addr_info, &ibmf_msg->im_local_addr, 1882 sizeof (ibmf_addr_info_t)); 1883 1884 /* copy global addressing information to message if in use */ 1885 if (saa_portp->saa_pt_ibmf_msg_flags & IBMF_MSG_FLAGS_GLOBAL_ADDRESS) { 1886 1887 ibmf_msg->im_msg_flags = IBMF_MSG_FLAGS_GLOBAL_ADDRESS; 1888 1889 bcopy(&saa_portp->saa_pt_ibmf_global_addr, 1890 &ibmf_msg->im_global_addr, 1891 sizeof (ibmf_global_addr_info_t)); 1892 } else { 1893 ibmf_msg->im_msg_flags = 0; 1894 } 1895 1896 mutex_exit(&saa_portp->saa_pt_mutex); 1897 1898 *msgp = ibmf_msg; 1899 *transport_flagsp = transport_flags; 1900 bail: 1901 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 1902 ibmf_saa_impl_init_msg_end, 1903 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_msg() exiting" 1904 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 1905 1906 return (ibmf_status); 1907 1908 } 1909 1910 /* 1911 * ibmf_saa_impl_new_smlid_retry: 1912 * 1913 * It's possible for the MasterSMLID to change while ibmf_saa is running. The 1914 * MasterSMLID is set when we first register with ibmf_saa. If a request 1915 * timesout, this function should be called to check whether the SM LID changed. 1916 * If so, it will call msg_transport again with the request. 1917 * 1918 * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the 1919 * same values passed to the original ibmf_msg_transport that timedout. The 1920 * ibmf_retrans parameter will be re-retrieved from the saa_portp structure. 1921 * 1922 * If the lid did not change then this function returns IBMF_TRANS_TIMEOUT. 1923 * That way, callers can simply return the result of this function. 1924 * 1925 * Input Arguments 1926 * saa_portp pointer to saa_port structure 1927 * msgp ibmf message that timedout 1928 * ibmf_callback callback that should be called by msg_transport 1929 * ibmf_callback_arg args for ibmf_callback 1930 * transport_flags flags for ibmf_msg_transport 1931 * 1932 * Output Arguments 1933 * none 1934 * 1935 * Returns 1936 * IBMF_SUCCESS if lid changed and request was resent successfully, 1937 * IBMF_TRANS_TIMEOUT if lid did not change, 1938 * same values as ibmf_msg_transport() if lid changed but request could not be 1939 * resent. 1940 */ 1941 static int 1942 ibmf_saa_impl_new_smlid_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp, 1943 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, int transport_flags) 1944 { 1945 ibt_hca_portinfo_t *ibt_portinfop; 1946 ib_lid_t master_sm_lid; 1947 int subnet_timeout; 1948 uint_t nports, size; 1949 ibmf_retrans_t ibmf_retrans; 1950 int ibmf_status; 1951 ibt_status_t ibt_status; 1952 1953 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1954 ibmf_saa_impl_new_smlid_retry_start, 1955 IBMF_TNF_TRACE, "", "ibmf_saa_impl_new_smlid_retry() enter\n"); 1956 1957 _NOTE(ASSUMING_PROTECTED(*msgp)) 1958 _NOTE(ASSUMING_PROTECTED(*msgp->im_msgbufs_send.im_bufs_mad_hdr)) 1959 1960 /* first query the portinfo to see if the lid changed */ 1961 ibt_status = ibt_query_hca_ports_byguid(saa_portp->saa_pt_node_guid, 1962 saa_portp->saa_pt_port_num, &ibt_portinfop, &nports, &size); 1963 1964 if (ibt_status != IBT_SUCCESS) { 1965 1966 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1967 ibmf_saa_impl_new_smlid_retry_err, IBMF_TNF_ERROR, "", 1968 "ibmf_saa_impl_new_smlid_retry: %s, ibmf_status =" 1969 " %d\n", tnf_string, msg, 1970 "ibt_query_hca_ports_byguid() failed", 1971 tnf_int, ibt_status, ibt_status); 1972 1973 ibmf_status = IBMF_TRANSPORT_FAILURE; 1974 1975 goto bail; 1976 } 1977 1978 master_sm_lid = ibt_portinfop->p_sm_lid; 1979 subnet_timeout = ibt_portinfop->p_subnet_timeout; 1980 1981 ibt_free_portinfo(ibt_portinfop, size); 1982 1983 /* if master smlid is different than the remote lid we sent to */ 1984 if (master_sm_lid != msgp->im_local_addr.ia_remote_lid) { 1985 1986 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L2, 1987 ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "", 1988 "ibmf_saa_impl_new_smlid_retry: %s, new_lid 0x%x," 1989 " old_lid 0x%x\n", tnf_string, msg, 1990 "master smlid has changed. retrying msg_transport", 1991 tnf_opaque, new_lid, master_sm_lid, 1992 tnf_opaque, old_lid, msgp->im_local_addr.ia_remote_lid); 1993 1994 mutex_enter(&saa_portp->saa_pt_mutex); 1995 1996 /* update the master sm lid value in ibmf_saa */ 1997 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = 1998 master_sm_lid; 1999 2000 /* new tid needed */ 2001 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID = 2002 h2b64(saa_portp->saa_pt_current_tid++); 2003 2004 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans, 2005 sizeof (ibmf_retrans_t)); 2006 2007 /* update the subnet timeout since this may be a new sm/sa */ 2008 saa_portp->saa_pt_timeout = subnet_timeout; 2009 2010 /* place upper bound on subnet timeout in case of faulty SM */ 2011 if (saa_portp->saa_pt_timeout > IBMF_SAA_MAX_SUBNET_TIMEOUT) 2012 saa_portp->saa_pt_timeout = IBMF_SAA_MAX_SUBNET_TIMEOUT; 2013 2014 /* increment the reference count to account for the cpi call */ 2015 saa_portp->saa_pt_reference_count++; 2016 2017 mutex_exit(&saa_portp->saa_pt_mutex); 2018 2019 /* update the remote lid for this particular message */ 2020 msgp->im_local_addr.ia_remote_lid = master_sm_lid; 2021 2022 /* get the classportinfo again since this may be a new sm/sa */ 2023 ibmf_saa_impl_get_classportinfo(saa_portp); 2024 2025 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle, 2026 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 2027 ibmf_callback, ibmf_callback_arg, transport_flags); 2028 2029 if (ibmf_status != IBMF_SUCCESS) { 2030 2031 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 2032 ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "", 2033 "ibmf_saa_impl_new_smlid_retry: %s, ibmf_status = " 2034 "%d\n", tnf_string, msg, 2035 "ibmf_msg_transport() failed", 2036 tnf_int, ibmf_status, ibmf_status); 2037 } 2038 2039 goto bail; 2040 } 2041 2042 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 2043 ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "", 2044 "ibmf_saa_impl_new_smlid_retry: %s, master_smlid = 0x%x\n", 2045 tnf_string, msg, 2046 "master smlid did not change. returning failure", 2047 tnf_opaque, master_smlid, master_sm_lid); 2048 2049 /* mark status as timeout since that was original failure */ 2050 ibmf_status = IBMF_TRANS_TIMEOUT; 2051 2052 bail: 2053 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2054 ibmf_saa_impl_new_smlid_retry_end, 2055 IBMF_TNF_TRACE, "", "ibmf_saa_impl_new_smlid_retry() exiting" 2056 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 2057 2058 return (ibmf_status); 2059 } 2060 2061 /* 2062 * ibmf_saa_impl_revert_to_qp1() 2063 * 2064 * The SA that we had contact with via redirect may fail to respond. If this 2065 * occurs SA should revert back to qp1 and the SMLID set in the port. 2066 * msg_transport for the message that timed out will be retried with 2067 * these new parameters. 2068 * 2069 * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the 2070 * same values passed to the original ibmf_msg_transport that timedout. The 2071 * ibmf_retrans parameter will be re-retrieved from the saa_portp structure. 2072 * 2073 * Input Arguments 2074 * saa_portp pointer to saa_port structure 2075 * msgp ibmf message that timedout 2076 * ibmf_callback callback that should be called by msg_transport 2077 * ibmf_callback_arg args for ibmf_callback 2078 * transport_flags flags for ibmf_msg_transport 2079 * 2080 * Output Arguments 2081 * none 2082 * 2083 * Returns 2084 * none 2085 */ 2086 static int 2087 ibmf_saa_impl_revert_to_qp1(saa_port_t *saa_portp, ibmf_msg_t *msgp, 2088 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_args, int transport_flags) 2089 { 2090 ibt_hca_portinfo_t *ibt_portinfop; 2091 ib_lid_t master_sm_lid, base_lid; 2092 uint8_t sm_sl; 2093 int subnet_timeout; 2094 uint_t nports, size; 2095 ibmf_retrans_t ibmf_retrans; 2096 int ibmf_status; 2097 ibt_status_t ibt_status; 2098 2099 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 2100 ibmf_saa_impl_revert_to_qp1_start, 2101 IBMF_TNF_TRACE, "", "ibmf_saa_impl_revert_to_qp1() enter\n"); 2102 2103 _NOTE(ASSUMING_PROTECTED(*msgp)) 2104 _NOTE(ASSUMING_PROTECTED(*msgp->im_msgbufs_send.im_bufs_mad_hdr)) 2105 2106 /* first query the portinfo to see if the lid changed */ 2107 ibt_status = ibt_query_hca_ports_byguid(saa_portp->saa_pt_node_guid, 2108 saa_portp->saa_pt_port_num, &ibt_portinfop, &nports, &size); 2109 2110 if (ibt_status != IBT_SUCCESS) { 2111 2112 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 2113 ibmf_saa_impl_revert_to_qp1_err, IBMF_TNF_ERROR, "", 2114 "ibmf_saa_impl_revert_to_qp1: %s, ibmf_status =" 2115 " %d\n", tnf_string, msg, 2116 "ibt_query_hca_ports_byguid() failed", 2117 tnf_int, ibt_status, ibt_status); 2118 2119 ibmf_status = IBMF_TRANSPORT_FAILURE; 2120 2121 goto bail; 2122 } 2123 2124 master_sm_lid = ibt_portinfop->p_sm_lid; 2125 base_lid = ibt_portinfop->p_base_lid; 2126 sm_sl = ibt_portinfop->p_sm_sl; 2127 subnet_timeout = ibt_portinfop->p_subnet_timeout; 2128 2129 ibt_free_portinfo(ibt_portinfop, size); 2130 2131 2132 mutex_enter(&saa_portp->saa_pt_mutex); 2133 2134 saa_portp->saa_pt_redirect_active = B_FALSE; 2135 2136 /* update the address info in ibmf_saa */ 2137 saa_portp->saa_pt_ibmf_addr_info.ia_local_lid = base_lid; 2138 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = master_sm_lid; 2139 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = sm_sl; 2140 saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = 1; 2141 saa_portp->saa_pt_ibmf_addr_info.ia_p_key = IB_PKEY_DEFAULT_LIMITED; 2142 saa_portp->saa_pt_ibmf_addr_info.ia_q_key = IB_GSI_QKEY; 2143 saa_portp->saa_pt_ibmf_msg_flags = 0; 2144 2145 /* new tid needed */ 2146 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID = 2147 h2b64(saa_portp->saa_pt_current_tid++); 2148 2149 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans, 2150 sizeof (ibmf_retrans_t)); 2151 2152 /* update the subnet timeout since this may be a new sm/sa */ 2153 saa_portp->saa_pt_timeout = subnet_timeout; 2154 2155 /* place upper bound on subnet timeout in case of faulty SM */ 2156 if (saa_portp->saa_pt_timeout > IBMF_SAA_MAX_SUBNET_TIMEOUT) 2157 saa_portp->saa_pt_timeout = IBMF_SAA_MAX_SUBNET_TIMEOUT; 2158 2159 /* increment the reference count to account for the cpi call */ 2160 saa_portp->saa_pt_reference_count++; 2161 2162 mutex_exit(&saa_portp->saa_pt_mutex); 2163 2164 /* update the address info for this particular message */ 2165 bcopy(&saa_portp->saa_pt_ibmf_addr_info, &msgp->im_local_addr, 2166 sizeof (ibmf_addr_info_t)); 2167 msgp->im_msg_flags = 0; /* No GRH */ 2168 2169 /* get the classportinfo again since this may be a new sm/sa */ 2170 ibmf_saa_impl_get_classportinfo(saa_portp); 2171 2172 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle, 2173 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 2174 ibmf_callback, ibmf_callback_args, transport_flags); 2175 2176 if (ibmf_status != IBMF_SUCCESS) { 2177 2178 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 2179 ibmf_saa_impl_revert_to_qp1, IBMF_TNF_TRACE, "", 2180 "ibmf_saa_impl_revert_to_qp1: %s, ibmf_status = " 2181 "%d\n", tnf_string, msg, 2182 "ibmf_msg_transport() failed", 2183 tnf_int, ibmf_status, ibmf_status); 2184 } 2185 2186 bail: 2187 2188 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2189 ibmf_saa_impl_revert_to_qp1_end, 2190 IBMF_TNF_TRACE, "", "ibmf_saa_impl_revert_to_qp1() exiting" 2191 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 2192 2193 return (ibmf_status); 2194 } 2195 2196 /* 2197 * ibmf_saa_impl_async_event_cb: 2198 * ibmf event callback, argument to ibmf_register 2199 * ibmf_handle is unused 2200 */ 2201 /* ARGSUSED */ 2202 static void 2203 ibmf_saa_impl_async_event_cb( 2204 ibmf_handle_t ibmf_handle, 2205 void *clnt_private, 2206 ibmf_async_event_t event_type) 2207 { 2208 saa_port_t *saa_portp; 2209 2210 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2211 ibmf_saa_impl_async_event_cb_start, IBMF_TNF_TRACE, "", 2212 "ibmf_saa_impl_async_event_cb: Handling event type 0x%x\n", 2213 tnf_opaque, event_type, event_type); 2214 2215 saa_portp = (saa_port_t *)clnt_private; 2216 ASSERT(saa_portp != NULL); 2217 2218 switch (event_type) { 2219 2220 case IBMF_CI_OFFLINE: 2221 ibmf_saa_impl_hca_detach(saa_portp); 2222 break; 2223 default: 2224 break; 2225 } 2226 2227 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3, 2228 ibmf_saa_impl_async_event_cb_end, 2229 IBMF_TNF_TRACE, "", "ibmf_saa_impl_async_event_cb() exit\n"); 2230 } 2231 2232 2233 /* 2234 * ibmf_saa_impl_ibt_async_handler: 2235 * MUST NOT BE STATIC (referred from within IBMF) 2236 */ 2237 void 2238 ibmf_saa_impl_ibt_async_handler(ibt_async_code_t code, ibt_async_event_t *event) 2239 { 2240 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2241 ibmf_saa_impl_ibt_async_handler_start, IBMF_TNF_TRACE, "", 2242 "ibmf_saa_impl_ibt_async_handler: Handling event code 0x%x\n", 2243 tnf_opaque, code, code); 2244 2245 switch (code) { 2246 2247 case IBT_EVENT_PORT_UP: 2248 ibmf_saa_impl_port_up(event->ev_hca_guid, event->ev_port); 2249 break; 2250 case IBT_ERROR_PORT_DOWN: 2251 ibmf_saa_impl_port_down(event->ev_hca_guid, event->ev_port); 2252 break; 2253 default: 2254 break; 2255 } 2256 2257 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_async_handler_end, 2258 IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibt_async_handler() exit\n"); 2259 } 2260 2261 /* 2262 * ibmf_saa_impl_port_up: 2263 */ 2264 static void 2265 ibmf_saa_impl_port_up(ib_guid_t ci_guid, uint8_t port_num) 2266 { 2267 saa_port_t *saa_portp = NULL; 2268 int is_ready; 2269 ibt_hca_portinfo_t *ibt_portinfop; 2270 ib_lid_t master_sm_lid; 2271 uint_t nports, size; 2272 ibt_status_t ibt_status; 2273 boolean_t event_subs = B_FALSE; 2274 2275 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_up_start, 2276 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_up: Handling port up" 2277 " guid %016" PRIx64 " port %d\n", 2278 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num); 2279 2280 /* Get classportinfo of corresponding entry */ 2281 mutex_enter(&saa_statep->saa_port_list_mutex); 2282 2283 saa_portp = saa_statep->saa_port_list; 2284 while (saa_portp != NULL) { 2285 2286 if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid && 2287 saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) { 2288 2289 mutex_enter(&saa_portp->saa_pt_mutex); 2290 2291 is_ready = (saa_portp->saa_pt_state 2292 == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE; 2293 2294 /* 2295 * increment reference count to account for cpi and 2296 * informinfos. All 4 informinfo's sent are treated as 2297 * one port client reference 2298 */ 2299 if (is_ready == B_TRUE) 2300 saa_portp->saa_pt_reference_count += 2; 2301 2302 mutex_exit(&saa_portp->saa_pt_mutex); 2303 2304 if (is_ready == B_TRUE) 2305 break; /* normally, only 1 port entry */ 2306 } 2307 saa_portp = saa_portp->next; 2308 } 2309 2310 mutex_exit(&saa_statep->saa_port_list_mutex); 2311 2312 if (saa_portp != NULL && is_ready == B_TRUE) { 2313 2314 /* verify whether master sm lid changed */ 2315 2316 /* first query the portinfo to see if the lid changed */ 2317 ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num, 2318 &ibt_portinfop, &nports, &size); 2319 2320 if (ibt_status != IBT_SUCCESS) { 2321 2322 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 2323 ibmf_saa_impl_port_up_err, IBMF_TNF_ERROR, "", 2324 "ibmf_saa_impl_port_up: %s, ibmf_status =" 2325 " %d\n", tnf_string, msg, 2326 "ibt_query_hca_ports_byguid() failed", 2327 tnf_int, ibt_status, ibt_status); 2328 2329 goto bail; 2330 } 2331 2332 master_sm_lid = ibt_portinfop->p_sm_lid; 2333 2334 ibt_free_portinfo(ibt_portinfop, size); 2335 2336 /* check whether we need to subscribe for events */ 2337 mutex_enter(&saa_portp->saa_pt_event_sub_mutex); 2338 2339 event_subs = (saa_portp->saa_pt_event_sub_client_list != NULL) ? 2340 B_TRUE : B_FALSE; 2341 2342 mutex_exit(&saa_portp->saa_pt_event_sub_mutex); 2343 2344 /* update the master smlid */ 2345 mutex_enter(&saa_portp->saa_pt_mutex); 2346 2347 /* update the master sm lid value in ibmf_saa */ 2348 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = 2349 master_sm_lid; 2350 2351 /* if we're not subscribed for events, dec reference count */ 2352 if (event_subs == B_FALSE) 2353 saa_portp->saa_pt_reference_count--; 2354 2355 mutex_exit(&saa_portp->saa_pt_mutex); 2356 2357 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 2358 ibmf_saa_impl_port_up, IBMF_TNF_TRACE, "", 2359 "ibmf_saa_impl_port_up: %s, master_sm_lid = 0x%x\n", 2360 tnf_string, msg, 2361 "port is up. Sending classportinfo request", 2362 tnf_opaque, master_sm_lid, master_sm_lid); 2363 2364 /* get the classportinfo again */ 2365 ibmf_saa_impl_get_classportinfo(saa_portp); 2366 2367 /* 2368 * resubscribe to events if there are subscribers since SA may 2369 * have removed our subscription records when the port went down 2370 */ 2371 if (event_subs == B_TRUE) 2372 ibmf_saa_subscribe_events(saa_portp, B_TRUE, B_FALSE); 2373 } 2374 2375 bail: 2376 2377 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_up_end, 2378 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_up() exit\n"); 2379 } 2380 2381 /* 2382 * ibmf_saa_impl_port_down: 2383 */ 2384 static void 2385 ibmf_saa_impl_port_down(ib_guid_t ci_guid, uint8_t port_num) 2386 { 2387 2388 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_down_start, 2389 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_down: Handling port down" 2390 " guid %016" PRIx64 " port %d\n", 2391 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num); 2392 2393 2394 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_down_end, 2395 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_down() exit\n"); 2396 } 2397 2398 /* 2399 * ibmf_saa_impl_hca_detach: 2400 * find entry, unregister if there are no clients 2401 * have to unregister since ibmf needs to close the hca and will only do this if 2402 * no clients are registered 2403 */ 2404 static void 2405 ibmf_saa_impl_hca_detach(saa_port_t *saa_removed) 2406 { 2407 saa_port_t *saa_portp; 2408 boolean_t must_unreg, must_unsub; 2409 2410 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_hca_detach_start, 2411 IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach: Detaching" 2412 " entry %016" PRIx64 "\n", tnf_opaque, entry, saa_removed); 2413 2414 /* find this entry */ 2415 mutex_enter(&saa_statep->saa_port_list_mutex); 2416 2417 saa_portp = saa_statep->saa_port_list; 2418 while (saa_portp != NULL) { 2419 2420 if (saa_portp == saa_removed) 2421 break; 2422 2423 saa_portp = saa_portp->next; 2424 } 2425 mutex_exit(&saa_statep->saa_port_list_mutex); 2426 2427 ASSERT(saa_portp != NULL); 2428 2429 if (saa_portp == NULL) { 2430 2431 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 2432 ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "", 2433 "ibmf_saa_impl_hca_detach: %s, entry %016" 2434 PRIx64 "\n", 2435 tnf_string, msg, 2436 "Port entry NOT found", 2437 tnf_opaque, entryp, saa_removed); 2438 2439 goto bail; 2440 } 2441 2442 /* if there are clients expecting Reports(), unsusbscribe */ 2443 mutex_enter(&saa_portp->saa_pt_event_sub_mutex); 2444 2445 must_unsub = (saa_portp->saa_pt_event_sub_client_list != NULL) ? 2446 B_TRUE : B_FALSE; 2447 2448 mutex_exit(&saa_portp->saa_pt_event_sub_mutex); 2449 2450 /* fail if outstanding transactions */ 2451 mutex_enter(&saa_portp->saa_pt_mutex); 2452 2453 if (saa_portp->saa_pt_num_outstanding_trans > 0) { 2454 2455 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L1, 2456 ibmf_saa_impl_fini_err, IBMF_TNF_TRACE, "", 2457 "ibmf_saa_impl_fini: %s, port = %016" PRIx64 2458 ", num transactions = %d\n", 2459 tnf_string, msg, "Detaching HCA." 2460 " Outstanding transactions on port.", 2461 tnf_opaque, port, 2462 saa_portp->saa_pt_port_guid, 2463 tnf_uint, outstanding_transactions, 2464 saa_portp->saa_pt_num_outstanding_trans); 2465 2466 mutex_exit(&saa_portp->saa_pt_mutex); 2467 2468 goto bail; 2469 } 2470 2471 2472 /* 2473 * increment reference count by one to account for unsubscribe requests 2474 * that are about to be sent. All four informinfo's are treated as one 2475 * port client reference. The count will be decremented by 2476 * subscribe_events() before the call returns. 2477 */ 2478 if (must_unsub == B_TRUE) 2479 saa_portp->saa_pt_reference_count++; 2480 2481 mutex_exit(&saa_portp->saa_pt_mutex); 2482 2483 /* 2484 * try and unsubscribe from SA. Generate synchronous, unsequenced 2485 * unsubscribe requests. 2486 */ 2487 if (must_unsub == B_TRUE) 2488 ibmf_saa_subscribe_events(saa_portp, B_FALSE, B_TRUE); 2489 2490 /* warning if registered clients */ 2491 mutex_enter(&saa_portp->saa_pt_mutex); 2492 2493 if (saa_portp->saa_pt_reference_count > 0) { 2494 2495 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 2496 ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "", 2497 "ibmf_saa_impl_hca_detach: %s, port %016" 2498 PRIx64 "\n", 2499 tnf_string, msg, 2500 "Detaching HCA for port with clients still" 2501 " registered", tnf_opaque, port, 2502 saa_portp->saa_pt_port_guid); 2503 } 2504 2505 /* synchronize on end of registration */ 2506 while (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_REGISTERING) { 2507 2508 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L1, 2509 ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "", 2510 "ibmf_saa_impl_hca_detach: %s\n", 2511 tnf_string, msg, "someone is registering. waiting" 2512 " for them to finish"); 2513 2514 cv_wait(&saa_portp->saa_pt_ibmf_reg_cv, 2515 &saa_portp->saa_pt_mutex); 2516 2517 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L1, 2518 ibmf_saa_impl_hca_detach, 2519 IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach: %s\n", 2520 tnf_string, msg, "done waiting"); 2521 } 2522 2523 /* unregister from ibmf */ 2524 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_READY) { 2525 must_unreg = B_TRUE; 2526 } else 2527 must_unreg = B_FALSE; 2528 2529 ibmf_saa_impl_invalidate_port(saa_portp); 2530 2531 mutex_exit(&saa_portp->saa_pt_mutex); 2532 2533 if (must_unreg == B_TRUE) 2534 ibmf_saa_impl_ibmf_unreg(saa_portp); 2535 bail: 2536 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_hca_detach_end, 2537 IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach() exit\n"); 2538 } 2539 2540 /* ARGSUSED */ 2541 void 2542 ibmf_saa_async_cb(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp, void *args) 2543 { 2544 saa_impl_trans_info_t *trans_info; 2545 int status; 2546 size_t length; 2547 void *result; 2548 saa_port_t *saa_portp; 2549 saa_client_data_t *client_data; 2550 int ibmf_status; 2551 boolean_t ignore_data; 2552 ibmf_retrans_t ibmf_retrans; 2553 boolean_t sa_is_redirected = B_FALSE; 2554 2555 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_async_cb_start, 2556 IBMF_TNF_TRACE, "", "ibmf_saa_async_cb() enter\n"); 2557 2558 trans_info = (saa_impl_trans_info_t *)args; 2559 2560 client_data = trans_info->si_trans_client_data; 2561 saa_portp = trans_info->si_trans_port; 2562 2563 mutex_enter(&saa_portp->saa_pt_mutex); 2564 sa_is_redirected = saa_portp->saa_pt_redirect_active; 2565 mutex_exit(&saa_portp->saa_pt_mutex); 2566 2567 if ((msgp->im_msg_status == IBMF_TRANS_TIMEOUT) && 2568 (sa_is_redirected == B_TRUE)) { 2569 2570 /* 2571 * We should retry the request using SM_LID and QP1 if we 2572 * have been using redirect up until now 2573 */ 2574 ibmf_status = ibmf_saa_impl_revert_to_qp1( 2575 saa_portp, msgp, ibmf_saa_async_cb, args, 2576 trans_info->si_trans_transport_flags); 2577 2578 /* 2579 * If revert_to_qp1 returns success msg was resent. 2580 * Otherwise msg could not be resent. Continue normally 2581 */ 2582 if (ibmf_status == IBMF_SUCCESS) 2583 goto bail; 2584 2585 } else if (msgp->im_msg_status == IBMF_TRANS_TIMEOUT) { 2586 2587 2588 ibmf_status = ibmf_saa_impl_new_smlid_retry(saa_portp, msgp, 2589 ibmf_saa_async_cb, args, 2590 trans_info->si_trans_transport_flags); 2591 2592 /* 2593 * if smlid_retry() returns success sm lid changed and msg 2594 * was resent. Otherwise, lid did not change or msg could not 2595 * be resent. Continue normally. 2596 */ 2597 if (ibmf_status == IBMF_SUCCESS) 2598 goto bail; 2599 2600 /* 2601 * check whether we've received anything from the SA in a while. 2602 * If we have, this function will retry and return success. If 2603 * we haven't continue normally so that we return a timeout to 2604 * the client 2605 */ 2606 ibmf_status = ibmf_saa_check_sa_and_retry( 2607 saa_portp, msgp, ibmf_saa_async_cb, args, 2608 trans_info->si_trans_send_time, 2609 trans_info->si_trans_transport_flags); 2610 2611 if (ibmf_status == IBMF_SUCCESS) 2612 goto bail; 2613 } 2614 2615 /* 2616 * If SA returned success but mad status is busy, retry a few times. 2617 * If SA returned success but mad status says redirect is required, 2618 * update the address info and retry the request to the new SA address 2619 */ 2620 if (msgp->im_msg_status == IBMF_SUCCESS) { 2621 2622 ASSERT(msgp->im_msgbufs_recv.im_bufs_mad_hdr != NULL); 2623 2624 if ((b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->Status) == 2625 MAD_STATUS_BUSY) && 2626 (trans_info->si_trans_retry_busy_count < 2627 IBMF_SAA_MAX_BUSY_RETRY_COUNT)) { 2628 2629 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 2630 ibmf_saa_async_cb, IBMF_TNF_TRACE, "", 2631 "ibmf_saa_async_cb: %s, retry_count = %d\n", 2632 tnf_string, msg, 2633 "async response returned busy status", 2634 tnf_int, retry_count, 2635 trans_info->si_trans_retry_busy_count); 2636 2637 trans_info->si_trans_retry_busy_count++; 2638 2639 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans, 2640 sizeof (ibmf_retrans_t)); 2641 2642 ibmf_status = ibmf_msg_transport( 2643 saa_portp->saa_pt_ibmf_handle, 2644 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 2645 ibmf_saa_async_cb, args, 2646 trans_info->si_trans_transport_flags); 2647 2648 /* 2649 * if retry is successful, quit here since async_cb will 2650 * get called again; otherwise, let this function call 2651 * handle the cleanup 2652 */ 2653 if (ibmf_status == IBMF_SUCCESS) 2654 goto bail; 2655 } else if (b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->Status) 2656 == MAD_STATUS_REDIRECT_REQUIRED) { 2657 2658 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L2, 2659 ibmf_saa_async_cb, IBMF_TNF_TRACE, "", 2660 "ibmf_saa_async_cb: " 2661 "async response returned redirect status\n"); 2662 2663 /* update address info and copy it into msgp */ 2664 ibmf_saa_impl_update_sa_address_info(saa_portp, msgp); 2665 2666 /* retry with new address info */ 2667 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans, 2668 sizeof (ibmf_retrans_t)); 2669 2670 ibmf_status = ibmf_msg_transport( 2671 saa_portp->saa_pt_ibmf_handle, 2672 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 2673 ibmf_saa_async_cb, args, 2674 trans_info->si_trans_transport_flags); 2675 2676 /* 2677 * if retry is successful, quit here since async_cb will 2678 * get called again; otherwise, let this function call 2679 * handle the cleanup 2680 */ 2681 if (ibmf_status == IBMF_SUCCESS) 2682 goto bail; 2683 } 2684 } 2685 2686 mutex_enter(&saa_portp->saa_pt_mutex); 2687 2688 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0); 2689 saa_portp->saa_pt_num_outstanding_trans--; 2690 2691 mutex_exit(&saa_portp->saa_pt_mutex); 2692 2693 if ((trans_info->si_trans_callback == NULL) && 2694 (trans_info->si_trans_sub_callback == NULL)) 2695 ignore_data = B_TRUE; 2696 else 2697 ignore_data = B_FALSE; 2698 2699 ibmf_saa_impl_prepare_response(ibmf_handle, msgp, ignore_data, &status, 2700 &result, &length, B_FALSE); 2701 2702 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 2703 2704 IBMF_SAA_SUB32_KSTATS(saa_portp, outstanding_requests, 1); 2705 2706 if (status != IBMF_SUCCESS) 2707 IBMF_SAA_ADD32_KSTATS(saa_portp, failed_requests, 1); 2708 2709 if (status == IBMF_TRANS_TIMEOUT) 2710 IBMF_SAA_ADD32_KSTATS(saa_portp, requests_timedout, 1); 2711 2712 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 2713 2714 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2715 ibmf_saa_async_cb, IBMF_TNF_TRACE, "", 2716 "ibmf_saa_async_cb: %s\n", tnf_string, msg, 2717 "Calling ibmf_saa client's callback"); 2718 2719 /* 2720 * there are three classes or trans_info users: ibmf_saa clients and 2721 * classportinfo requests; informinfo subscribe requests, and report 2722 * responses. For the first two, call the correct callback. For report 2723 * responses there's no need to notify anyone. 2724 */ 2725 if (trans_info->si_trans_callback != NULL) { 2726 /* ibmf_saa client or classportinfo request */ 2727 trans_info->si_trans_callback(trans_info->si_trans_callback_arg, 2728 length, result, status); 2729 } else if (trans_info->si_trans_sub_callback != NULL) { 2730 /* informinfo subscribe request */ 2731 trans_info->si_trans_sub_callback( 2732 trans_info->si_trans_callback_arg, length, result, status, 2733 trans_info->si_trans_sub_producer_type); 2734 } 2735 2736 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2737 ibmf_saa_async_cb, IBMF_TNF_TRACE, "", 2738 "ibmf_saa_async_cb: %s\n", tnf_string, msg, 2739 "Returned from callback"); 2740 2741 if (client_data != NULL) { 2742 mutex_enter(&client_data->saa_client_mutex); 2743 2744 ASSERT(client_data->saa_client_num_pending_trans > 0); 2745 client_data->saa_client_num_pending_trans--; 2746 2747 if ((client_data->saa_client_num_pending_trans == 0) && 2748 (client_data->saa_client_state == SAA_CLIENT_STATE_WAITING)) 2749 cv_signal(&client_data->saa_client_state_cv); 2750 2751 mutex_exit(&client_data->saa_client_mutex); 2752 } 2753 2754 kmem_free(trans_info, sizeof (saa_impl_trans_info_t)); 2755 2756 bail: 2757 2758 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_async_cb_end, 2759 IBMF_TNF_TRACE, "", "ibmf_saa_async_cb() exit\n"); 2760 } 2761 2762 /* 2763 * ibmf_saa_check_sa_and_retry: 2764 * 2765 * If a particular transaction times out, we don't want to give up if we know 2766 * the SA is responding. Check the time since we last received a response. If 2767 * it's less than ibmf_saa_max_wait_time retry the request. 2768 * 2769 * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the 2770 * same values passed to the original ibmf_msg_transport that timed out. The 2771 * ibmf_retrans parameter will be re-retrieved from the saa_portp structure. 2772 * 2773 * If max_wait_time seconds have passed, this function returns IBMF_TIMEOUT. 2774 * That way, callers can simply return the result of this function. 2775 * 2776 * Input Arguments 2777 * saa_portp pointer to saa_port structure 2778 * msgp ibmf message that timedout 2779 * ibmf_callback callback that should be called by msg_transport 2780 * ibmf_callback_arg args for ibmf_callback 2781 * transport_flags flags for ibmf_msg_transport 2782 * 2783 * Output Arguments 2784 * none 2785 * 2786 * Returns 2787 * IBMF_SUCCESS if we've recently received data from the SA and request was 2788 * resent. 2789 * IBMF_TRANS_TIMEOUT if no data has been received from the SA in max_wait_time 2790 * same values as ibmf_msg_transport() if data has been received but request 2791 * could not be resent. 2792 */ 2793 static int 2794 ibmf_saa_check_sa_and_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp, 2795 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, 2796 hrtime_t trans_send_time, int transport_flags) 2797 { 2798 hrtime_t curr_time, sa_uptime; 2799 ibmf_retrans_t ibmf_retrans; 2800 int ibmf_status; 2801 2802 do { 2803 2804 mutex_enter(&saa_portp->saa_pt_mutex); 2805 2806 sa_uptime = saa_portp->saa_pt_sa_uptime; 2807 2808 /* if nothing received from SA since we sent */ 2809 curr_time = gethrtime(); 2810 2811 /* 2812 * check if it's been a very long time since this 2813 * particular transaction was sent 2814 */ 2815 if (((curr_time - trans_send_time) / 1000000000) > 2816 ibmf_saa_trans_wait_time) { 2817 2818 mutex_exit(&saa_portp->saa_pt_mutex); 2819 2820 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L1, 2821 ibmf_saa_check_sa_and_retry_err, IBMF_TNF_ERROR, "", 2822 "ibmf_saa_check_sa_and_retry: %s, msgp = " 2823 "%p sa_uptime = %" PRIu64 ", trans send time = %" 2824 PRIu64 ", curr_time = %" PRIu64 "\n", 2825 tnf_string, msg, 2826 "Nothing received for this transaction", 2827 tnf_opaque, msgp, msgp, 2828 tnf_long, sa_uptime, sa_uptime, 2829 tnf_long, trans_send_time, trans_send_time, 2830 tnf_long, curr_time, curr_time); 2831 2832 ibmf_status = IBMF_TRANS_TIMEOUT; 2833 2834 break; 2835 } 2836 2837 /* 2838 * check time since we received something, 2839 * and make sure that it hasn't been an extra long 2840 * time for this particular transaction 2841 */ 2842 if (((curr_time - sa_uptime) / 1000000000) < 2843 ibmf_saa_max_wait_time) { 2844 2845 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L2, 2846 ibmf_saa_check_sa_and_retry, IBMF_TNF_TRACE, "", 2847 "ibmf_saa_check_sa_and_retry: %s, msgp = " 2848 "%p sa_uptime = %" PRIu64 " trans_send_time = %" 2849 PRIu64 " curr_time = %" PRIu64 "\n", 2850 tnf_string, msg, "Something received. Retrying", 2851 tnf_opaque, msgp, msgp, 2852 tnf_long, sa_uptime, sa_uptime, 2853 tnf_long, trans_send_time, trans_send_time, 2854 tnf_long, curr_time, curr_time); 2855 2856 /* 2857 * something received in WAIT_TIME_IN_SECS; 2858 * resend request 2859 */ 2860 2861 /* new tid needed */ 2862 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID = 2863 h2b64(saa_portp->saa_pt_current_tid++); 2864 2865 bcopy(&saa_portp->saa_pt_ibmf_retrans, 2866 &ibmf_retrans, sizeof (ibmf_retrans_t)); 2867 2868 mutex_exit(&saa_portp->saa_pt_mutex); 2869 2870 ibmf_status = ibmf_msg_transport( 2871 saa_portp->saa_pt_ibmf_handle, 2872 saa_portp->saa_pt_qp_handle, msgp, 2873 &ibmf_retrans, ibmf_callback, ibmf_callback_arg, 2874 transport_flags); 2875 2876 if (ibmf_status == IBMF_SUCCESS) 2877 goto bail; 2878 2879 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 2880 ibmf_saa_check_sa_and_retry, IBMF_TNF_TRACE, "", 2881 "ibmf_saa_check_sa_and_retry: %s, ibmf_status = " 2882 "%d\n", tnf_string, msg, 2883 "ibmf_msg_transport() failed", 2884 tnf_int, ibmf_status, ibmf_status); 2885 } else { 2886 2887 mutex_exit(&saa_portp->saa_pt_mutex); 2888 2889 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L1, 2890 ibmf_saa_check_sa_and_retry_err, IBMF_TNF_ERROR, "", 2891 "ibmf_saa_check_sa_and_retry: %s, msgp = " 2892 "%p sa_uptime = %" PRIu64 " curr_time = %" 2893 PRIu64 "\n", tnf_string, msg, 2894 "Nothing received. Timing out", 2895 tnf_opaque, msgp, msgp, 2896 tnf_long, sa_uptime, sa_uptime, 2897 tnf_long, curr_time, curr_time); 2898 2899 ibmf_status = IBMF_TRANS_TIMEOUT; 2900 2901 break; 2902 } 2903 } while (ibmf_status == IBMF_TRANS_TIMEOUT); 2904 2905 bail: 2906 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2907 ibmf_saa_check_sa_and_retry_end, 2908 IBMF_TNF_TRACE, "", "ibmf_saa_check_sa_and_retry() exiting" 2909 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 2910 2911 return (ibmf_status); 2912 } 2913 2914 2915 /* 2916 * ibmf_saa_impl_prepare_response: 2917 */ 2918 static void 2919 ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle, 2920 ibmf_msg_t *msgp, boolean_t ignore_data, int *status, void **result, 2921 size_t *length, boolean_t sleep_flag) 2922 { 2923 ibmf_msg_bufs_t *resp_buf; 2924 uint16_t attr_id; 2925 uint8_t method; 2926 boolean_t is_get_resp; 2927 uint16_t mad_status; 2928 uint16_t attr_offset; 2929 ib_sa_hdr_t *sa_hdr; 2930 2931 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 2932 ibmf_saa_impl_prepare_response_start, 2933 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response() enter\n"); 2934 2935 _NOTE(ASSUMING_PROTECTED(*msgp)) 2936 2937 *result = NULL; 2938 *length = 0; 2939 sa_hdr = NULL; 2940 2941 resp_buf = &msgp->im_msgbufs_recv; 2942 2943 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*resp_buf)) 2944 2945 if (msgp->im_msg_status != IBMF_SUCCESS) { 2946 2947 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 2948 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 2949 "ibmf_saa_impl_prepare_response: %s, msg_status = %d\n", 2950 tnf_string, msg, "Bad ibmf status", 2951 tnf_int, msg_status, msgp->im_msg_status); 2952 2953 *status = msgp->im_msg_status; 2954 2955 goto exit; 2956 } 2957 2958 if (resp_buf->im_bufs_mad_hdr == NULL) { 2959 2960 /* 2961 * this was an unsequenced transaction (from an unsubscribe for 2962 * following a CI_OFFLINE event) 2963 */ 2964 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2965 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 2966 "ibmf_saa_impl_prepare_response: %s\n", 2967 tnf_string, msg, "Unsequenced transaction callback"); 2968 2969 goto exit; 2970 } 2971 2972 if ((mad_status = b2h16(resp_buf->im_bufs_mad_hdr->Status)) != 2973 MAD_STATUS_NO_INVALID_FIELDS) { 2974 2975 /* convert mad packet status to IBMF status */ 2976 switch (mad_status) { 2977 2978 case SA_STATUS_ERR_NO_RESOURCES: 2979 *status = IBMF_NO_RESOURCES; 2980 break; 2981 case SA_STATUS_ERR_REQ_INVALID: 2982 *status = IBMF_REQ_INVALID; 2983 break; 2984 case SA_STATUS_ERR_NO_RECORDS: 2985 *status = IBMF_NO_RECORDS; 2986 break; 2987 case SA_STATUS_ERR_TOO_MANY_RECORDS: 2988 *status = IBMF_TOO_MANY_RECORDS; 2989 break; 2990 case SA_STATUS_ERR_REQ_INVALID_GID: 2991 *status = IBMF_INVALID_GID; 2992 break; 2993 case SA_STATUS_ERR_REQ_INSUFFICIENT_COMPONENTS: 2994 *status = IBMF_INSUFF_COMPS; 2995 break; 2996 case MAD_STATUS_UNSUPP_METHOD: 2997 *status = IBMF_UNSUPP_METHOD; 2998 break; 2999 case MAD_STATUS_UNSUPP_METHOD_ATTR: 3000 *status = IBMF_UNSUPP_METHOD_ATTR; 3001 break; 3002 case MAD_STATUS_INVALID_FIELD: 3003 *status = IBMF_INVALID_FIELD; 3004 break; 3005 default: 3006 *status = IBMF_REQ_INVALID; 3007 break; 3008 } 3009 3010 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 3011 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 3012 "ibmf_saa_impl_prepare_response: %s, mad_status = %x\n", 3013 tnf_string, msg, "Bad MAD status", 3014 tnf_int, mad_status, mad_status); 3015 3016 goto exit; 3017 } 3018 3019 attr_id = b2h16(resp_buf->im_bufs_mad_hdr->AttributeID); 3020 method = resp_buf->im_bufs_mad_hdr->R_Method; 3021 3022 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 3023 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 3024 "ibmf_saa_impl_prepare_response: attr_id = 0x%x, method = " 3025 "0x%x\n", 3026 tnf_opaque, attr_id, attr_id, 3027 tnf_opaque, method, method); 3028 3029 /* 3030 * ignore any data from deleteresp since there's no way to know whether 3031 * real data was returned; also ignore data if this was a Report 3032 * response 3033 */ 3034 if (method == SA_SUBN_ADM_DELETE_RESP) { 3035 3036 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3037 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 3038 "impf_saa_impl_prepare_response: %s\n", 3039 tnf_string, msg, 3040 "DeleteResp or NoticeResp returned. " 3041 "Ignoring response data"); 3042 3043 *status = IBMF_SUCCESS; 3044 3045 *length = 0; 3046 *result = NULL; 3047 3048 goto exit; 3049 } 3050 3051 if (attr_id == SA_MULTIPATHRECORD_ATTRID) { 3052 3053 /* 3054 * getmulti is only for requests; attribute should not 3055 * be returned from SA 3056 */ 3057 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 3058 ibmf_saa_impl_prepare_response_err, IBMF_TNF_ERROR, 3059 "", "ibmf_saa_impl_prepare_response: %s\n", 3060 tnf_string, msg, "SA returned getmulti record"); 3061 3062 *status = IBMF_REQ_INVALID; 3063 3064 goto exit; 3065 } 3066 3067 /* if we are supposed to ignore data, stop here */ 3068 if (ignore_data == B_TRUE) { 3069 3070 *status = IBMF_SUCCESS; 3071 3072 goto exit; 3073 } 3074 3075 is_get_resp = resp_buf->im_bufs_mad_hdr->R_Method == 3076 SA_SUBN_ADM_GET_RESP ? B_TRUE: B_FALSE; 3077 3078 /* unpack the sa header to get the attribute offset */ 3079 *status = ibmf_saa_utils_unpack_sa_hdr(resp_buf->im_bufs_cl_hdr, 3080 resp_buf->im_bufs_cl_hdr_len, &sa_hdr, sleep_flag); 3081 if (*status != IBMF_SUCCESS) { 3082 3083 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3084 ibmf_saa_impl_prepare_response_err, 3085 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response: %s," 3086 " ibmf_status = %d\n", tnf_string, msg, 3087 "Could not unpack sa hdr", tnf_int, ibmf_status, *status); 3088 3089 goto exit; 3090 } 3091 3092 attr_offset = sa_hdr->AttributeOffset; 3093 3094 /* 3095 * unpack data payload; if unpack function doesn't return success 3096 * (because it could not allocate memory) forward this status to waiting 3097 * client 3098 */ 3099 *status = ibmf_saa_utils_unpack_payload(resp_buf->im_bufs_cl_data, 3100 resp_buf->im_bufs_cl_data_len, attr_id, result, length, 3101 attr_offset, is_get_resp, sleep_flag); 3102 if (*status == IBMF_SUCCESS) { 3103 3104 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, 3105 ibmf_saa_impl_prepare_response, 3106 IBMF_TNF_TRACE, "", 3107 "ibmf_saa_impl_prepare_response: attr_id = " 3108 "0x%x, attr_offset = %d, packed_payload_len = %d, " 3109 "unpacked_payload_len = %d\n", 3110 tnf_opaque, attr_id, attr_id, 3111 tnf_opaque, attr_offset, attr_offset, 3112 tnf_opaque, packed_payload_len, 3113 resp_buf->im_bufs_cl_data_len, 3114 tnf_opaque, unpacked_payload_len, *length); 3115 } else { 3116 3117 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L1, 3118 ibmf_saa_impl_prepare_response_err, 3119 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response: %s," 3120 "attr_id = 0x%x, attr_offset = %d, packed_payload_len = %d," 3121 "status = %d\n", 3122 tnf_string, msg, "Could not unpack payload", 3123 tnf_opaque, attr_id, attr_id, 3124 tnf_int, attr_offset, attr_offset, 3125 tnf_int, packed_payload_len, 3126 resp_buf->im_bufs_cl_data_len, 3127 tnf_int, status, *status); 3128 } 3129 exit: 3130 if (sa_hdr != NULL) 3131 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t)); 3132 3133 ibmf_saa_impl_free_msg(ibmf_handle, msgp); 3134 3135 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3136 ibmf_saa_impl_prepare_response_end, 3137 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response() exit," 3138 " status = 0x%d\n", tnf_int, status, *status); 3139 } 3140 3141 3142 /* 3143 * ibmf_saa_impl_check_sa_support: 3144 * Checks the capability mask (returned from the SA classportinfo response) to 3145 * determine whether the sa supports the specified attribute ID. 3146 * 3147 * Input Arguments 3148 * cap_mask 16-bit capability mask returned in SA's classportinfo 3149 * attr_id attribute ID of current request 3150 * 3151 * Returns 3152 * IBMF_NOT_SUPPORTED if capability mask indicates SA does not support attribute 3153 * IBMF_SUCCESS otherwise 3154 */ 3155 static int 3156 ibmf_saa_impl_check_sa_support(uint16_t cap_mask, uint16_t attr_id) 3157 { 3158 boolean_t attr_supported = B_TRUE; 3159 3160 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 3161 ibmf_saa_impl_check_sa_support, IBMF_TNF_TRACE, "", 3162 "ibmf_saa_impl_check_sa_support: cap_mask = 0x%x, " 3163 "attr_id = 0x%x\n", tnf_opaque, cap_mask, cap_mask, 3164 tnf_opaque, attr_id, attr_id); 3165 3166 switch (attr_id) { 3167 3168 case SA_SWITCHINFORECORD_ATTRID: 3169 case SA_LINEARFDBRECORD_ATTRID: 3170 case SA_RANDOMFDBRECORD_ATTRID: 3171 case SA_MULTICASTFDBRECORD_ATTRID: 3172 case SA_SMINFORECORD_ATTRID: 3173 case SA_INFORMINFORECORD_ATTRID: 3174 case SA_LINKRECORD_ATTRID: 3175 case SA_GUIDINFORECORD_ATTRID: 3176 case SA_TRACERECORD_ATTRID: 3177 case SA_SERVICEASSNRECORD_ATTRID: 3178 3179 if ((cap_mask & 3180 SA_CAPMASK_OPT_RECORDS_SUPPORTED) == 0) { 3181 3182 IBMF_TRACE_3(IBMF_TNF_NODEBUG, DPRINT_L1, 3183 ibmf_saa_impl_check_sa_support, 3184 IBMF_TNF_ERROR, "", 3185 "ibmf_saa_impl_check_sa_support: %s, " 3186 "cap_mask = 0x%x\n", tnf_string, msg, 3187 "SA does not support optional records", 3188 tnf_opaque, cap_mask, cap_mask, 3189 tnf_opaque, attr_id, attr_id); 3190 3191 attr_supported = B_FALSE; 3192 } 3193 break; 3194 3195 case SA_MULTIPATHRECORD_ATTRID: 3196 3197 if ((cap_mask & SA_CAPMASK_MULTIPATH_SUPPORTED) == 0) { 3198 3199 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 3200 ibmf_saa_impl_check_sa_support, 3201 IBMF_TNF_ERROR, "", 3202 "ibmf_saa_impl_check_sa_support: %s, " 3203 "cap_mask = 0x%x\n", tnf_string, msg, 3204 "SA does not support multipath records", 3205 tnf_opaque, cap_mask, cap_mask); 3206 3207 attr_supported = B_FALSE; 3208 } 3209 break; 3210 3211 case SA_MCMEMBERRECORD_ATTRID: 3212 3213 if ((cap_mask & SA_CAPMASK_UD_MCAST_SUPPORTED) == 0) { 3214 3215 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 3216 ibmf_saa_impl_check_sa_support, 3217 IBMF_TNF_ERROR, "", 3218 "ibmf_saa_impl_check_sa_support: %s, " 3219 "cap_mask = 0x%x\n", tnf_string, msg, 3220 "SA does not support ud multicast", 3221 tnf_opaque, cap_mask, cap_mask); 3222 3223 attr_supported = B_FALSE; 3224 } 3225 break; 3226 3227 default: 3228 break; 3229 } /* switch */ 3230 3231 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3232 ibmf_saa_impl_check_sa_support_end, IBMF_TNF_TRACE, "", 3233 "ibmf_saa_impl_check_sa_support() exiting, attr_supported = %d\n", 3234 tnf_opaque, attr_supported, attr_supported); 3235 3236 if (attr_supported == B_FALSE) 3237 return (IBMF_UNSUPP_METHOD_ATTR); 3238 else 3239 return (IBMF_SUCCESS); 3240 } 3241 3242 /* 3243 * ibmf_saa_impl_get_attr_id_length: 3244 * 3245 * Returns the host size of the specified sa record. Returns 0 for unknown 3246 * attributes. multipath record size is a dynamic value given as a parameter 3247 * specified with the ibmf_sa_access() call. 3248 */ 3249 static uint_t 3250 ibmf_saa_impl_get_attr_id_length(uint16_t attr_id) 3251 { 3252 uint_t attr_length; 3253 3254 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3255 ibmf_saa_impl_get_attr_id_length_start, 3256 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_attr_id_length() enter\n"); 3257 3258 /* this function should not be used for multipath record */ 3259 ASSERT(attr_id != SA_MULTIPATHRECORD_ATTRID); 3260 3261 switch (attr_id) { 3262 case SA_CLASSPORTINFO_ATTRID: 3263 attr_length = sizeof (ib_mad_classportinfo_t); 3264 break; 3265 case SA_NOTICE_ATTRID: 3266 attr_length = sizeof (ib_mad_notice_t); 3267 break; 3268 case SA_INFORMINFO_ATTRID: 3269 attr_length = sizeof (ib_mad_informinfo_t); 3270 break; 3271 case SA_NODERECORD_ATTRID: 3272 attr_length = sizeof (sa_node_record_t); 3273 break; 3274 case SA_PORTINFORECORD_ATTRID: 3275 attr_length = sizeof (sa_portinfo_record_t); 3276 break; 3277 case SA_SLTOVLRECORD_ATTRID: 3278 attr_length = sizeof (sa_SLtoVLmapping_record_t); 3279 break; 3280 case SA_SWITCHINFORECORD_ATTRID: 3281 attr_length = sizeof (sa_switchinfo_record_t); 3282 break; 3283 case SA_LINEARFDBRECORD_ATTRID: 3284 attr_length = sizeof (sa_linearft_record_t); 3285 break; 3286 case SA_RANDOMFDBRECORD_ATTRID: 3287 attr_length = sizeof (sa_randomft_record_t); 3288 break; 3289 case SA_MULTICASTFDBRECORD_ATTRID: 3290 attr_length = sizeof (sa_multicastft_record_t); 3291 break; 3292 case SA_SMINFORECORD_ATTRID: 3293 attr_length = sizeof (sa_sminfo_record_t); 3294 break; 3295 case SA_INFORMINFORECORD_ATTRID: 3296 attr_length = sizeof (sa_informinfo_record_t); 3297 break; 3298 case SA_LINKRECORD_ATTRID: 3299 attr_length = sizeof (sa_link_record_t); 3300 break; 3301 case SA_GUIDINFORECORD_ATTRID: 3302 attr_length = sizeof (sa_guidinfo_record_t); 3303 break; 3304 case SA_SERVICERECORD_ATTRID: 3305 attr_length = sizeof (sa_service_record_t); 3306 break; 3307 case SA_PARTITIONRECORD_ATTRID: 3308 attr_length = sizeof (sa_pkey_table_record_t); 3309 break; 3310 case SA_PATHRECORD_ATTRID: 3311 attr_length = sizeof (sa_path_record_t); 3312 break; 3313 case SA_VLARBRECORD_ATTRID: 3314 attr_length = sizeof (sa_VLarb_table_record_t); 3315 break; 3316 case SA_MCMEMBERRECORD_ATTRID: 3317 attr_length = sizeof (sa_mcmember_record_t); 3318 break; 3319 case SA_TRACERECORD_ATTRID: 3320 attr_length = sizeof (sa_trace_record_t); 3321 break; 3322 case SA_SERVICEASSNRECORD_ATTRID: 3323 attr_length = sizeof (sa_service_assn_record_t); 3324 break; 3325 default: 3326 /* should only get the above type of packets */ 3327 attr_length = 0; 3328 break; 3329 } 3330 3331 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 3332 ibmf_saa_impl_get_attr_id_length_end, 3333 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_attr_id_length():" 3334 " attr_id: 0x%x size %d\n", 3335 tnf_opaque, attr_id, attr_id, tnf_uint, attr_length, attr_length); 3336 3337 return (attr_length); 3338 } 3339 3340 /* 3341 * ibmf_saa_impl_free_msg: 3342 * Takes a completed message and free memory associated with the message, 3343 * including the individual fields of the im_msgbufs_send. 3344 * ibmf_free_msg, called at the end of this function, takes a pointer to the 3345 * message pointer so that it can set the message pointer to NULL. This 3346 * function takes just the message pointer so the msgp will not be NULL after 3347 * this function returns. 3348 * 3349 * Input Arguments 3350 * ibmf_hdl ibmf handle used in ibmf_msg_alloc 3351 * msgp pointer to ibmf_msg_t to free 3352 * 3353 * Returns 3354 * void 3355 */ 3356 static void 3357 ibmf_saa_impl_free_msg(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msgp) 3358 { 3359 int res; 3360 3361 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3362 ibmf_saa_impl_free_msg_start, 3363 IBMF_TNF_TRACE, "", "ibmf_saa_impl_free_msg() enter: msg %p\n", 3364 tnf_opaque, msg, msgp); 3365 3366 ASSERT(msgp != NULL); 3367 3368 kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr, 3369 sizeof (ib_mad_hdr_t)); 3370 3371 kmem_free(msgp->im_msgbufs_send.im_bufs_cl_hdr, 3372 msgp->im_msgbufs_send.im_bufs_cl_hdr_len); 3373 3374 if (msgp->im_msgbufs_send.im_bufs_cl_data_len > 0) 3375 kmem_free(msgp->im_msgbufs_send.im_bufs_cl_data, 3376 msgp->im_msgbufs_send.im_bufs_cl_data_len); 3377 3378 res = ibmf_free_msg(ibmf_hdl, &msgp); 3379 ASSERT(res == IBMF_SUCCESS); 3380 3381 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3382 ibmf_saa_impl_free_msg_end, 3383 IBMF_TNF_TRACE, "", "ibmf_saa_impl_free_msg() exit\n"); 3384 } 3385 3386 /* 3387 * ibmf_saa_impl_get_port_guid: 3388 */ 3389 static int 3390 ibmf_saa_impl_get_port_guid(ibt_hca_portinfo_t *ibt_portinfop, 3391 ib_guid_t *guid_ret) 3392 { 3393 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3394 ibmf_saa_impl_get_port_guid_start, 3395 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_port_guid() enter\n"); 3396 3397 if (ibt_portinfop->p_linkstate != IBT_PORT_ACTIVE) { 3398 3399 return (IBMF_BAD_PORT_STATE); 3400 } 3401 3402 if (ibt_portinfop->p_sgid_tbl_sz == 0) { 3403 3404 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L2, 3405 ibmf_saa_impl_get_port_guid_end, IBMF_TNF_TRACE, "", 3406 "ibmf_saa_impl_get_port_guid: %s\n", tnf_string, msg, 3407 "portinfo sgid table size is 0. Exiting.\n"); 3408 3409 return (IBMF_TRANSPORT_FAILURE); 3410 } 3411 3412 *guid_ret = ibt_portinfop->p_sgid_tbl[0].gid_guid; 3413 3414 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3415 ibmf_saa_impl_get_port_guid_end, IBMF_TNF_TRACE, "", 3416 "ibmf_saa_impl_get_port_guid: Returning port_guid %016" PRIx64 "\n", 3417 tnf_opaque, port_guid, *guid_ret); 3418 3419 return (IBMF_SUCCESS); 3420 } 3421 3422 /* 3423 * ibmf_saa_impl_set_transaction_params: 3424 */ 3425 static void 3426 ibmf_saa_impl_set_transaction_params(saa_port_t *saa_portp, 3427 ibt_hca_portinfo_t *portinfop) 3428 { 3429 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3430 ibmf_saa_impl_set_transaction_params_start, 3431 IBMF_TNF_TRACE, "", 3432 "ibmf_saa_impl_set_transaction_params() enter\n"); 3433 3434 _NOTE(ASSUMING_PROTECTED(*saa_portp)) 3435 3436 saa_portp->saa_pt_ibmf_retrans.retrans_retries = 3437 IBMF_SAA_RETRANS_RETRIES; 3438 /* 3439 * For the first transaction (generally getting the 3440 * classportinfo) have ibmf pick our timeouts. It should be using the 3441 * default IB spec values. 3442 * Once we get the classportinfo we'll update the correct response time 3443 * value (rtv) and round-trip time (rttv). ibmf should always calculate 3444 * trans_to since it depends on the particular transaction's number of 3445 * packets. 3446 */ 3447 saa_portp->saa_pt_ibmf_retrans.retrans_rtv = 0; 3448 saa_portp->saa_pt_ibmf_retrans.retrans_rttv = 0; 3449 saa_portp->saa_pt_ibmf_retrans.retrans_trans_to = 0; 3450 3451 /* 3452 * Assume that the SA supports all optional records. If it 3453 * does not, the request will get returned with ERR_NOT_SUPP. When 3454 * the classportinfo response comes back we will update the cap mask 3455 * to prevent unnecessary unsupported requests. 3456 */ 3457 saa_portp->saa_pt_sa_cap_mask = 0xFFFF; 3458 3459 saa_portp->saa_pt_ibmf_msg_flags = 0; 3460 saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = 1; 3461 saa_portp->saa_pt_ibmf_addr_info.ia_p_key = 3462 IB_PKEY_DEFAULT_LIMITED; 3463 saa_portp->saa_pt_ibmf_addr_info.ia_q_key = IB_GSI_QKEY; 3464 3465 /* 3466 * fill out addr information for MADs that will be sent 3467 * to SA on this port 3468 */ 3469 saa_portp->saa_pt_ibmf_addr_info.ia_local_lid = portinfop->p_base_lid; 3470 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = portinfop->p_sm_lid; 3471 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = portinfop->p_sm_sl; 3472 3473 /* place upper bound on subnet timeout in case of faulty SM */ 3474 saa_portp->saa_pt_timeout = portinfop->p_subnet_timeout; 3475 3476 if (saa_portp->saa_pt_timeout > IBMF_SAA_MAX_SUBNET_TIMEOUT) { 3477 3478 saa_portp->saa_pt_timeout = IBMF_SAA_MAX_SUBNET_TIMEOUT; 3479 } 3480 3481 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, 3482 ibmf_saa_impl_set_transaction_params, 3483 IBMF_TNF_TRACE, "", 3484 "ibmf_saa_impl_set_transaction_params: local_lid = 0x%x, " 3485 "sm_lid = 0x%x, sm_sl = 0x%x, sn_timeout = 0x%x\n", 3486 tnf_opaque, local_lid, portinfop->p_base_lid, 3487 tnf_opaque, sm_lid, portinfop->p_sm_lid, 3488 tnf_opaque, sm_sl, portinfop->p_sm_sl, 3489 tnf_opaque, subnet_timeout, portinfop->p_subnet_timeout); 3490 3491 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3492 ibmf_saa_impl_set_transaction_params_end, 3493 IBMF_TNF_TRACE, "", 3494 "ibmf_saa_impl_set_transaction_params() exit\n"); 3495 } 3496 3497 3498 /* 3499 * ibmf_saa_impl_update_sa_address_info 3500 */ 3501 static void 3502 ibmf_saa_impl_update_sa_address_info(saa_port_t *saa_portp, ibmf_msg_t *msgp) 3503 { 3504 void *result; 3505 ib_sa_hdr_t *sa_hdr; 3506 int rv; 3507 size_t length; 3508 uint16_t attr_id; 3509 ib_mad_classportinfo_t *cpi; 3510 ibmf_global_addr_info_t *gaddrp = &saa_portp->saa_pt_ibmf_global_addr; 3511 ibt_hca_portinfo_t *ibt_pinfo; 3512 uint_t nports, size; 3513 ibt_status_t ibt_status; 3514 3515 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3516 ibmf_saa_impl_update_sa_address_info, 3517 IBMF_TNF_TRACE, "", 3518 "ibmf_saa_impl_update_sa_address_info() enter\n"); 3519 3520 /* 3521 * decode the respons of msgp as a classportinfo attribute 3522 */ 3523 rv = ibmf_saa_utils_unpack_sa_hdr(msgp->im_msgbufs_recv.im_bufs_cl_hdr, 3524 msgp->im_msgbufs_recv.im_bufs_cl_hdr_len, &sa_hdr, KM_NOSLEEP); 3525 if (rv != IBMF_SUCCESS) { 3526 3527 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3528 ibmf_saa_impl_update_sa_address_err, 3529 IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: " 3530 "%s, ibmf_status = %d\n", tnf_string, msg, 3531 "Could not unpack sa hdr", tnf_int, ibmf_status, rv); 3532 3533 return; 3534 } 3535 3536 attr_id = b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID); 3537 if (attr_id != MAD_ATTR_ID_CLASSPORTINFO) { 3538 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3539 ibmf_saa_impl_update_sa_address_info_err, 3540 IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: " 3541 "%s, attrID = %x\n", tnf_string, msg, 3542 "Wrong attribute ID", tnf_int, ibmf_status, attr_id); 3543 3544 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t)); 3545 return; 3546 } 3547 rv = ibmf_saa_utils_unpack_payload( 3548 msgp->im_msgbufs_recv.im_bufs_cl_data, 3549 msgp->im_msgbufs_recv.im_bufs_cl_data_len, attr_id, &result, 3550 &length, sa_hdr->AttributeOffset, B_TRUE, KM_NOSLEEP); 3551 if (rv != IBMF_SUCCESS) { 3552 3553 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3554 ibmf_saa_impl_update_sa_address_err, 3555 IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: " 3556 "%s, ibmf_status = %d\n", tnf_string, msg, 3557 "Could not unpack payload", tnf_int, ibmf_status, rv); 3558 3559 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t)); 3560 return; 3561 } 3562 3563 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t)); 3564 3565 /* 3566 * Use the classportinfo contents to update the SA address info 3567 */ 3568 cpi = (ib_mad_classportinfo_t *)result; 3569 mutex_enter(&saa_portp->saa_pt_mutex); 3570 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = cpi->RedirectLID; 3571 saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = cpi->RedirectQP; 3572 saa_portp->saa_pt_ibmf_addr_info.ia_p_key = cpi->RedirectP_Key; 3573 saa_portp->saa_pt_ibmf_addr_info.ia_q_key = cpi->RedirectQ_Key; 3574 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = cpi->RedirectSL; 3575 3576 saa_portp->saa_pt_redirect_active = B_TRUE; 3577 3578 if ((cpi->RedirectGID_hi != 0) || (cpi->RedirectGID_lo != 0)) { 3579 3580 mutex_exit(&saa_portp->saa_pt_mutex); 3581 ibt_status = ibt_query_hca_ports_byguid( 3582 saa_portp->saa_pt_node_guid, saa_portp->saa_pt_port_num, 3583 &ibt_pinfo, &nports, &size); 3584 if (ibt_status != IBT_SUCCESS) { 3585 3586 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3587 ibmf_saa_impl_update_sa_address_err, IBMF_TNF_TRACE, 3588 "", "ibmf_saa_impl_update_sa_address_info: " 3589 "%s, ibt_status = %d\n", tnf_string, msg, 3590 "Could not query hca port", 3591 tnf_int, ibt_status, ibt_status); 3592 3593 kmem_free(result, length); 3594 return; 3595 } 3596 3597 mutex_enter(&saa_portp->saa_pt_mutex); 3598 /* 3599 * Fill in global address info parameters 3600 * 3601 * NOTE: The HopLimit value is not specified through the 3602 * contents of ClassPortInfo. It may be possible to find 3603 * out the proper value to use even for SA beeing redirected 3604 * to another subnet. But we do only support redirect within 3605 * our local subnet 3606 */ 3607 gaddrp->ig_sender_gid.gid_prefix = 3608 ibt_pinfo->p_sgid_tbl[0].gid_prefix; 3609 gaddrp->ig_sender_gid.gid_guid = saa_portp->saa_pt_port_guid; 3610 gaddrp->ig_recver_gid.gid_prefix = cpi->RedirectGID_hi; 3611 gaddrp->ig_recver_gid.gid_guid = cpi->RedirectGID_lo; 3612 gaddrp->ig_flow_label = cpi->RedirectFL; 3613 gaddrp->ig_tclass = cpi->RedirectTC; 3614 gaddrp->ig_hop_limit = 0; 3615 3616 saa_portp->saa_pt_ibmf_msg_flags = 3617 IBMF_MSG_FLAGS_GLOBAL_ADDRESS; 3618 3619 mutex_exit(&saa_portp->saa_pt_mutex); 3620 ibt_free_portinfo(ibt_pinfo, size); 3621 } else { 3622 saa_portp->saa_pt_ibmf_msg_flags = 0; 3623 mutex_exit(&saa_portp->saa_pt_mutex); 3624 } 3625 kmem_free(result, length); 3626 3627 /* 3628 * Update the address info of msgp with the new address parameters 3629 */ 3630 mutex_enter(&saa_portp->saa_pt_mutex); 3631 bcopy(&saa_portp->saa_pt_ibmf_addr_info, &msgp->im_local_addr, 3632 sizeof (ibmf_addr_info_t)); 3633 if (saa_portp->saa_pt_ibmf_msg_flags & IBMF_MSG_FLAGS_GLOBAL_ADDRESS) { 3634 3635 msgp->im_msg_flags = IBMF_MSG_FLAGS_GLOBAL_ADDRESS; 3636 3637 bcopy(&saa_portp->saa_pt_ibmf_global_addr, 3638 &msgp->im_global_addr, sizeof (ibmf_global_addr_info_t)); 3639 } else { 3640 msgp->im_msg_flags = 0; 3641 } 3642 mutex_exit(&saa_portp->saa_pt_mutex); 3643 } 3644 3645 /* 3646 * ibmf_saa_impl_ibmf_unreg: 3647 */ 3648 static void 3649 ibmf_saa_impl_ibmf_unreg(saa_port_t *saa_portp) 3650 { 3651 int ibmf_status; 3652 3653 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_ibmf_unreg_start, 3654 IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibmf_unreg() enter\n"); 3655 3656 /* teardown async cb */ 3657 ibmf_status = ibmf_tear_down_async_cb(saa_portp->saa_pt_ibmf_handle, 3658 saa_portp->saa_pt_qp_handle, 0); 3659 if (ibmf_status != IBMF_SUCCESS) { 3660 3661 /* continue anyway even though unreg will probably fail */ 3662 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3663 ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "", 3664 "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n", 3665 tnf_string, msg, "Could not tear down async cb", 3666 tnf_int, ibmf_status, ibmf_status); 3667 } 3668 3669 /* free qp */ 3670 ibmf_status = ibmf_free_qp(saa_portp->saa_pt_ibmf_handle, 3671 &saa_portp->saa_pt_qp_handle, 0); 3672 3673 if (ibmf_status != IBMF_SUCCESS) { 3674 3675 /* continue anyway even though unreg will probably fail */ 3676 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3677 ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "", 3678 "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n", 3679 tnf_string, msg, "Could not free queue pair", 3680 tnf_int, ibmf_status, ibmf_status); 3681 } 3682 3683 ibmf_status = ibmf_unregister(&saa_portp->saa_pt_ibmf_handle, 0); 3684 3685 if (ibmf_status != IBMF_SUCCESS) { 3686 3687 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3688 ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "", 3689 "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n", 3690 tnf_string, msg, "ibmf_unregister() failed", 3691 tnf_int, ibmf_status, ibmf_status); 3692 } 3693 3694 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_ibmf_unreg_end, 3695 IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibmf_unreg() exit\n"); 3696 } 3697