1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * hermon_agents.c 29 * Hermon InfiniBand Management Agent (SMA, PMA, BMA) routines 30 * 31 * Implements all the routines necessary for initializing, handling, 32 * and (later) tearing down all the infrastructure necessary for Hermon 33 * MAD processing. 34 */ 35 36 #include <sys/types.h> 37 #include <sys/conf.h> 38 #include <sys/ddi.h> 39 #include <sys/sunddi.h> 40 #include <sys/modctl.h> 41 42 #include <sys/ib/adapters/hermon/hermon.h> 43 #include <sys/ib/mgt/ibmf/ibmf.h> 44 #include <sys/disp.h> 45 46 static void hermon_agent_request_cb(ibmf_handle_t ibmf_handle, 47 ibmf_msg_t *msgp, void *args); 48 static void hermon_agent_handle_req(void *cb_args); 49 static void hermon_agent_response_cb(ibmf_handle_t ibmf_handle, 50 ibmf_msg_t *msgp, void *args); 51 static int hermon_agent_list_init(hermon_state_t *state); 52 static void hermon_agent_list_fini(hermon_state_t *state); 53 static int hermon_agent_register_all(hermon_state_t *state); 54 static int hermon_agent_unregister_all(hermon_state_t *state, int num_reg); 55 static void hermon_agent_mad_resp_handling(hermon_state_t *state, 56 ibmf_msg_t *msgp, uint_t port); 57 58 /* 59 * hermon_agent_handlers_init() 60 * Context: Only called from attach() and/or detach() path contexts 61 */ 62 int 63 hermon_agent_handlers_init(hermon_state_t *state) 64 { 65 int status; 66 char *rsrc_name; 67 68 /* Determine if we need to register any agents with the IBMF */ 69 if ((state->hs_cfg_profile->cp_qp0_agents_in_fw) && 70 (state->hs_cfg_profile->cp_qp1_agents_in_fw)) { 71 return (DDI_SUCCESS); 72 } 73 74 /* 75 * Build a unique name for the Hermon task queue from the Hermon driver 76 * instance number and HERMON_TASKQ_NAME 77 */ 78 rsrc_name = (char *)kmem_zalloc(HERMON_RSRC_NAME_MAXLEN, KM_SLEEP); 79 HERMON_RSRC_NAME(rsrc_name, HERMON_TASKQ_NAME); 80 81 /* Initialize the Hermon IB management agent list */ 82 status = hermon_agent_list_init(state); 83 if (status != DDI_SUCCESS) { 84 goto agentsinit_fail; 85 } 86 87 /* 88 * Initialize the agent handling task queue. Note: We set the task 89 * queue priority to the minimum system priority. At this point this 90 * is considered acceptable because MADs are unreliable datagrams 91 * and could get lost (in general) anyway. 92 */ 93 state->hs_taskq_agents = ddi_taskq_create(state->hs_dip, 94 rsrc_name, HERMON_TASKQ_NTHREADS, TASKQ_DEFAULTPRI, 0); 95 if (state->hs_taskq_agents == NULL) { 96 hermon_agent_list_fini(state); 97 goto agentsinit_fail; 98 } 99 100 /* Now attempt to register all of the agents with the IBMF */ 101 status = hermon_agent_register_all(state); 102 if (status != DDI_SUCCESS) { 103 ddi_taskq_destroy(state->hs_taskq_agents); 104 hermon_agent_list_fini(state); 105 goto agentsinit_fail; 106 } 107 108 kmem_free(rsrc_name, HERMON_RSRC_NAME_MAXLEN); 109 return (DDI_SUCCESS); 110 111 agentsinit_fail: 112 kmem_free(rsrc_name, HERMON_RSRC_NAME_MAXLEN); 113 return (status); 114 } 115 116 117 /* 118 * hermon_agent_handlers_fini() 119 * Context: Only called from detach() path context 120 */ 121 int 122 hermon_agent_handlers_fini(hermon_state_t *state) 123 { 124 int status; 125 126 /* Determine if we need to unregister any agents from the IBMF */ 127 if ((state->hs_cfg_profile->cp_qp0_agents_in_fw) && 128 (state->hs_cfg_profile->cp_qp1_agents_in_fw)) { 129 return (DDI_SUCCESS); 130 } 131 132 /* Now attempt to unregister all of the agents from the IBMF */ 133 status = hermon_agent_unregister_all(state, state->hs_num_agents); 134 if (status != DDI_SUCCESS) { 135 return (DDI_FAILURE); 136 } 137 138 /* 139 * Destroy the task queue. The task queue destroy is guaranteed to 140 * wait until any scheduled tasks have completed. We are able to 141 * guarantee that no _new_ tasks will be added the task queue while 142 * we are in the ddi_taskq_destroy() call because we have 143 * (at this point) successfully unregistered from IBMF (in 144 * hermon_agent_unregister_all() above). 145 */ 146 ddi_taskq_destroy(state->hs_taskq_agents); 147 148 /* Teardown the Hermon IB management agent list */ 149 hermon_agent_list_fini(state); 150 151 return (DDI_SUCCESS); 152 } 153 154 155 /* 156 * hermon_agent_request_cb() 157 * Context: Called from the IBMF context 158 */ 159 static void 160 hermon_agent_request_cb(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp, 161 void *args) 162 { 163 hermon_agent_handler_arg_t *cb_args; 164 hermon_agent_list_t *curr; 165 hermon_state_t *state; 166 int status; 167 168 curr = (hermon_agent_list_t *)args; 169 state = curr->agl_state; 170 171 /* 172 * Allocate space to hold the callback args (for passing to the 173 * task queue). Note: If we are unable to allocate space for the 174 * the callback args here, then we just return. But we must ensure 175 * that we call ibmf_free_msg() to free up the message. 176 */ 177 cb_args = (hermon_agent_handler_arg_t *)kmem_zalloc( 178 sizeof (hermon_agent_handler_arg_t), KM_NOSLEEP); 179 if (cb_args == NULL) { 180 (void) ibmf_free_msg(ibmf_handle, &msgp); 181 return; 182 } 183 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cb_args)) 184 185 /* Fill in the callback args */ 186 cb_args->ahd_ibmfhdl = ibmf_handle; 187 cb_args->ahd_ibmfmsg = msgp; 188 cb_args->ahd_agentlist = args; 189 190 /* 191 * Dispatch the message to the task queue. Note: Just like above, 192 * if this request fails for any reason then make sure to free up 193 * the IBMF message and then return 194 */ 195 status = ddi_taskq_dispatch(state->hs_taskq_agents, 196 hermon_agent_handle_req, cb_args, DDI_NOSLEEP); 197 if (status == DDI_FAILURE) { 198 kmem_free(cb_args, sizeof (hermon_agent_handler_arg_t)); 199 (void) ibmf_free_msg(ibmf_handle, &msgp); 200 } 201 } 202 203 /* 204 * hermon_get_smlid() 205 * Simple helper function for hermon_agent_handle_req() below. 206 * Get the portinfo and extract the smlid. 207 */ 208 static ib_lid_t 209 hermon_get_smlid(hermon_state_t *state, uint_t port) 210 { 211 sm_portinfo_t portinfo; 212 int status; 213 214 status = hermon_getportinfo_cmd_post(state, port, 215 HERMON_SLEEPFLAG_FOR_CONTEXT(), &portinfo); 216 if (status != HERMON_CMD_SUCCESS) { 217 cmn_err(CE_CONT, "Hermon: GetPortInfo (port %02d) command " 218 "failed: %08x\n", port, status); 219 return (0); 220 } 221 return (portinfo.MasterSMLID); 222 } 223 224 /* 225 * hermon_agent_handle_req() 226 * Context: Called with priority of taskQ thread 227 */ 228 static void 229 hermon_agent_handle_req(void *cb_args) 230 { 231 hermon_agent_handler_arg_t *agent_args; 232 hermon_agent_list_t *curr; 233 hermon_state_t *state; 234 ibmf_handle_t ibmf_handle; 235 ibmf_msg_t *msgp; 236 ibmf_msg_bufs_t *recv_msgbufp; 237 ibmf_msg_bufs_t *send_msgbufp; 238 ibmf_retrans_t retrans; 239 uint_t port; 240 int status; 241 242 /* Extract the necessary info from the callback args parameter */ 243 agent_args = (hermon_agent_handler_arg_t *)cb_args; 244 ibmf_handle = agent_args->ahd_ibmfhdl; 245 msgp = agent_args->ahd_ibmfmsg; 246 curr = agent_args->ahd_agentlist; 247 state = curr->agl_state; 248 port = curr->agl_port; 249 250 /* 251 * Set the message send buffer pointers to the message receive buffer 252 * pointers to reuse the IBMF provided buffers for the sender 253 * information. 254 */ 255 recv_msgbufp = &msgp->im_msgbufs_recv; 256 send_msgbufp = &msgp->im_msgbufs_send; 257 bcopy(recv_msgbufp, send_msgbufp, sizeof (ibmf_msg_bufs_t)); 258 259 /* 260 * Check if the incoming packet is a special "Hermon Trap" MAD. If it 261 * is, then do the special handling. If it isn't, then simply pass it 262 * on to the firmware and forward the response back to the IBMF. 263 * 264 * Note: Hermon has a unique method for handling internally generated 265 * Traps. All internally detected/generated Trap messages are 266 * automatically received by the IBMF (as receive completions on QP0), 267 * which (because all Hermon Trap MADs have SLID == 0) detects it as a 268 * special "Hermon Trap" and forwards it here to the driver's SMA. 269 * It is then our responsibility here to fill in the Trap MAD's DLID 270 * for forwarding to the real Master SM (as programmed in the port's 271 * PortInfo.MasterSMLID field.) 272 */ 273 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(msgp->im_local_addr)) 274 if (HERMON_IS_SPECIAL_TRAP_MAD(msgp)) { 275 msgp->im_local_addr.ia_remote_lid = 276 hermon_get_smlid(state, port); 277 } else { 278 /* 279 * Post the command to the firmware (using the MAD_IFC 280 * command). Note: We also reuse the command that was passed 281 * in. We pass the pointer to the original MAD payload as if 282 * it were both the source of the incoming MAD as well as the 283 * destination for the response. This is acceptable and saves 284 * us the step of one additional copy. Note: If this command 285 * fails for any reason other than HERMON_CMD_BAD_PKT, it 286 * probably indicates a serious problem. 287 */ 288 status = hermon_mad_ifc_cmd_post(state, port, 289 HERMON_CMD_SLEEP_NOSPIN, 290 (uint32_t *)recv_msgbufp->im_bufs_mad_hdr, 291 (uint32_t *)send_msgbufp->im_bufs_mad_hdr); 292 if (status != HERMON_CMD_SUCCESS) { 293 if ((status != HERMON_CMD_BAD_PKT) && 294 (status != HERMON_CMD_INSUFF_RSRC)) { 295 cmn_err(CE_CONT, "Hermon: MAD_IFC (port %02d) " 296 "command failed: %08x\n", port, status); 297 } 298 299 /* finish cleanup */ 300 goto hermon_agent_handle_req_skip_response; 301 } 302 } 303 304 /* 305 * If incoming MAD was "TrapRepress", then no response is necessary. 306 * Free the IBMF message and return. 307 */ 308 if (HERMON_IS_TRAP_REPRESS_MAD(msgp)) { 309 goto hermon_agent_handle_req_skip_response; 310 } 311 312 /* 313 * Modify the response MAD as necessary (for any special cases). 314 * Specifically, if this MAD was a directed route MAD, then some 315 * additional packet manipulation may be necessary because the Hermon 316 * firmware does not do all the required steps to respond to the 317 * MAD. 318 */ 319 hermon_agent_mad_resp_handling(state, msgp, port); 320 321 /* 322 * Send response (or forwarded "Trap" MAD) back to IBMF. We use the 323 * "response callback" to indicate when it is appropriate (later) to 324 * free the IBMF msg. 325 */ 326 status = ibmf_msg_transport(ibmf_handle, IBMF_QP_HANDLE_DEFAULT, 327 msgp, &retrans, hermon_agent_response_cb, state, 0); 328 if (status != IBMF_SUCCESS) { 329 goto hermon_agent_handle_req_skip_response; 330 } 331 332 /* Free up the callback args parameter */ 333 kmem_free(agent_args, sizeof (hermon_agent_handler_arg_t)); 334 return; 335 336 hermon_agent_handle_req_skip_response: 337 /* Free up the ibmf message */ 338 (void) ibmf_free_msg(ibmf_handle, &msgp); 339 340 /* Free up the callback args parameter */ 341 kmem_free(agent_args, sizeof (hermon_agent_handler_arg_t)); 342 } 343 344 345 /* 346 * hermon_agent_response_cb() 347 * Context: Called from the IBMF context 348 */ 349 /* ARGSUSED */ 350 static void 351 hermon_agent_response_cb(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp, 352 void *args) 353 { 354 /* 355 * It is the responsibility of each IBMF callback recipient to free 356 * the packets that it has been given. Now that we are in the 357 * response callback, we can be assured that it is safe to do so. 358 */ 359 (void) ibmf_free_msg(ibmf_handle, &msgp); 360 } 361 362 363 /* 364 * hermon_agent_list_init() 365 * Context: Only called from attach() path context 366 */ 367 static int 368 hermon_agent_list_init(hermon_state_t *state) 369 { 370 hermon_agent_list_t *curr; 371 uint_t num_ports, num_agents, num_agents_per_port; 372 uint_t num_sma_agents = 0; 373 uint_t num_pma_agents = 0; 374 uint_t num_bma_agents = 0; 375 uint_t do_qp0, do_qp1; 376 int i, j, indx; 377 378 /* 379 * Calculate the number of registered agents for each port 380 * (SMA, PMA, and BMA) and determine whether or not to register 381 * a given agent with the IBMF (or whether to let the Hermon firmware 382 * handle it) 383 */ 384 num_ports = state->hs_cfg_profile->cp_num_ports; 385 num_agents = 0; 386 num_agents_per_port = 0; 387 do_qp0 = state->hs_cfg_profile->cp_qp0_agents_in_fw; 388 do_qp1 = state->hs_cfg_profile->cp_qp1_agents_in_fw; 389 if (do_qp0 == 0) { 390 num_agents += (num_ports * HERMON_NUM_QP0_AGENTS_PER_PORT); 391 num_agents_per_port += HERMON_NUM_QP0_AGENTS_PER_PORT; 392 num_sma_agents = num_ports; 393 } 394 if (do_qp1 == 0) { 395 num_agents += (num_ports * HERMON_NUM_QP1_AGENTS_PER_PORT); 396 num_agents_per_port += HERMON_NUM_QP1_AGENTS_PER_PORT; 397 num_pma_agents = num_ports; 398 /* 399 * The following line is commented out because the Hermon 400 * firmware does not currently support a BMA. If it did, 401 * then we would want to register the agent with the IBMF. 402 * (We would also need to have HERMON_NUM_QP1_AGENTS_PER_PORT 403 * set to 2, instead of 1.) 404 * 405 * num_bma_agents = num_ports; 406 */ 407 } 408 409 state->hs_num_agents = num_agents; 410 411 /* 412 * Allocate the memory for all of the agent list entries 413 */ 414 state->hs_agents = (hermon_agent_list_t *)kmem_zalloc(num_agents * 415 sizeof (hermon_agent_list_t), KM_SLEEP); 416 if (state->hs_agents == NULL) { 417 return (DDI_FAILURE); 418 } 419 420 /* 421 * Fill in each of the agent list entries with the agent's 422 * MgmtClass, port number, and Hermon softstate pointer 423 */ 424 indx = 0; 425 for (i = 0; i < num_agents_per_port; i++) { 426 for (j = 0; j < num_ports; j++) { 427 curr = &state->hs_agents[indx]; 428 curr->agl_state = state; 429 curr->agl_port = j + 1; 430 431 if ((do_qp0 == 0) && num_sma_agents) { 432 curr->agl_mgmtclass = SUBN_AGENT; 433 num_sma_agents--; 434 indx++; 435 } else if ((do_qp1 == 0) && (num_pma_agents)) { 436 curr->agl_mgmtclass = PERF_AGENT; 437 num_pma_agents--; 438 indx++; 439 } else if ((do_qp1 == 0) && (num_bma_agents)) { 440 curr->agl_mgmtclass = BM_AGENT; 441 num_bma_agents--; 442 indx++; 443 } 444 } 445 } 446 447 return (DDI_SUCCESS); 448 } 449 450 451 /* 452 * hermon_agent_list_fini() 453 * Context: Only called from attach() and/or detach() path contexts 454 */ 455 static void 456 hermon_agent_list_fini(hermon_state_t *state) 457 { 458 /* Free up the memory for the agent list entries */ 459 kmem_free(state->hs_agents, 460 state->hs_num_agents * sizeof (hermon_agent_list_t)); 461 } 462 463 464 /* 465 * hermon_agent_register_all() 466 * Context: Only called from attach() path context 467 */ 468 static int 469 hermon_agent_register_all(hermon_state_t *state) 470 { 471 hermon_agent_list_t *curr; 472 ibmf_register_info_t ibmf_reg; 473 ibmf_impl_caps_t impl_caps; 474 ib_guid_t nodeguid; 475 int i, status, num_registered; 476 477 /* Get the Hermon NodeGUID from the softstate */ 478 nodeguid = state->hs_ibtfinfo.hca_attr->hca_node_guid; 479 480 /* 481 * Register each of the agents with the IBMF (and add callbacks for 482 * each to the hermon_agent_request_cb() routine). Note: If we 483 * fail somewhere along the line here, we attempt to cleanup as much 484 * of the mess as we can and then jump to hermon_agent_unregister_all() 485 * to cleanup the rest. 486 */ 487 num_registered = 0; 488 489 if (state->hs_num_agents == 0) { 490 return (DDI_SUCCESS); 491 } 492 493 for (i = 0; i < state->hs_num_agents; i++) { 494 /* Register each agent with the IBMF */ 495 curr = &state->hs_agents[i]; 496 ibmf_reg.ir_ci_guid = nodeguid; 497 ibmf_reg.ir_port_num = curr->agl_port; 498 ibmf_reg.ir_client_class = curr->agl_mgmtclass; 499 500 status = ibmf_register(&ibmf_reg, IBMF_VERSION, 0, 501 NULL, NULL, &curr->agl_ibmfhdl, &impl_caps); 502 if (status != IBMF_SUCCESS) { 503 goto agents_reg_fail; 504 } 505 506 /* Setup callbacks with the IBMF */ 507 status = ibmf_setup_async_cb(curr->agl_ibmfhdl, 508 IBMF_QP_HANDLE_DEFAULT, hermon_agent_request_cb, curr, 0); 509 if (status != IBMF_SUCCESS) { 510 (void) ibmf_unregister(&curr->agl_ibmfhdl, 0); 511 goto agents_reg_fail; 512 } 513 num_registered++; 514 } 515 516 return (DDI_SUCCESS); 517 518 agents_reg_fail: 519 (void) hermon_agent_unregister_all(state, num_registered); 520 return (DDI_FAILURE); 521 } 522 523 524 /* 525 * hermon_agent_unregister_all() 526 * Context: Only called from detach() path context 527 */ 528 static int 529 hermon_agent_unregister_all(hermon_state_t *state, int num_reg) 530 { 531 hermon_agent_list_t *curr; 532 int i, status; 533 534 if (num_reg == 0) { 535 return (DDI_SUCCESS); 536 } 537 538 /* 539 * For each registered agent in the agent list, teardown the 540 * callbacks from the IBMF and unregister. 541 */ 542 for (i = 0; i < num_reg; i++) { 543 curr = &state->hs_agents[i]; 544 545 /* Teardown the IBMF callback */ 546 status = ibmf_tear_down_async_cb(curr->agl_ibmfhdl, 547 IBMF_QP_HANDLE_DEFAULT, 0); 548 if (status != IBMF_SUCCESS) { 549 return (DDI_FAILURE); 550 } 551 552 /* Unregister the agent from the IBMF */ 553 status = ibmf_unregister(&curr->agl_ibmfhdl, 0); 554 if (status != IBMF_SUCCESS) { 555 return (DDI_FAILURE); 556 } 557 } 558 559 return (DDI_SUCCESS); 560 } 561 562 563 /* 564 * hermon_agent_mad_resp_handling() 565 * Context: Called with priority of taskQ thread 566 */ 567 /* ARGSUSED */ 568 static void 569 hermon_agent_mad_resp_handling(hermon_state_t *state, ibmf_msg_t *msgp, 570 uint_t port) 571 { 572 ib_mad_hdr_t *rmadhdrp = msgp->im_msgbufs_recv.im_bufs_mad_hdr; 573 ib_mad_hdr_t *smadhdrp = msgp->im_msgbufs_send.im_bufs_mad_hdr; 574 uint_t hop_count, hop_point; 575 uchar_t *resp, *ret_path; 576 577 resp = (uchar_t *)msgp->im_msgbufs_send.im_bufs_cl_data; 578 579 /* 580 * Handle directed route MADs as a special case. Hermon firmware 581 * does not update the "direction" bit, "hop pointer", "Return 582 * Path" or, in fact, any of the "directed route" parameters. So 583 * the responsibility falls on Hermon driver software to inspect the 584 * MADs and update those fields as appropriate (see section 14.2.2 585 * of the IBA specification, rev 1.1) 586 */ 587 if (HERMON_MAD_IS_DR(rmadhdrp)) { 588 589 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*((sm_dr_mad_hdr_t *)rmadhdrp))) 590 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*((sm_dr_mad_hdr_t *)smadhdrp))) 591 592 /* 593 * Set the "Direction" bit to one. This indicates that this 594 * is now directed route response 595 */ 596 HERMON_DRMAD_SET_DIRECTION(rmadhdrp); 597 598 /* Extract the "hop pointer" and "hop count" from the MAD */ 599 hop_count = HERMON_DRMAD_GET_HOPCOUNT(rmadhdrp); 600 hop_point = HERMON_DRMAD_GET_HOPPOINTER(rmadhdrp); 601 602 /* Append the port we came in on to the "Return Path" */ 603 if ((hop_count != 0) && ((hop_point == hop_count) || 604 (hop_point == hop_count + 1))) { 605 ret_path = &resp[HERMON_DRMAD_RETURN_PATH_OFFSET]; 606 ret_path[hop_point] = (uchar_t)port; 607 } 608 609 /* Then increment the "hop pointer" in the MAD */ 610 hop_point++; 611 HERMON_DRMAD_SET_HOPPOINTER(smadhdrp, (uint8_t)hop_point); 612 } 613 } 614