1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * hermon_agents.c
29 * Hermon InfiniBand Management Agent (SMA, PMA, BMA) routines
30 *
31 * Implements all the routines necessary for initializing, handling,
32 * and (later) tearing down all the infrastructure necessary for Hermon
33 * MAD processing.
34 */
35
36 #include <sys/types.h>
37 #include <sys/conf.h>
38 #include <sys/ddi.h>
39 #include <sys/sunddi.h>
40 #include <sys/modctl.h>
41
42 #include <sys/ib/adapters/hermon/hermon.h>
43 #include <sys/ib/mgt/ibmf/ibmf.h>
44 #include <sys/disp.h>
45
46 static void hermon_agent_request_cb(ibmf_handle_t ibmf_handle,
47 ibmf_msg_t *msgp, void *args);
48 static void hermon_agent_handle_req(void *cb_args);
49 static void hermon_agent_response_cb(ibmf_handle_t ibmf_handle,
50 ibmf_msg_t *msgp, void *args);
51 static int hermon_agent_list_init(hermon_state_t *state);
52 static void hermon_agent_list_fini(hermon_state_t *state);
53 static int hermon_agent_register_all(hermon_state_t *state);
54 static int hermon_agent_unregister_all(hermon_state_t *state, int num_reg);
55 static void hermon_agent_mad_resp_handling(hermon_state_t *state,
56 ibmf_msg_t *msgp, uint_t port);
57
58 /*
59 * hermon_agent_handlers_init()
60 * Context: Only called from attach() and/or detach() path contexts
61 */
62 int
hermon_agent_handlers_init(hermon_state_t * state)63 hermon_agent_handlers_init(hermon_state_t *state)
64 {
65 int status;
66 char *rsrc_name;
67
68 /* Determine if we need to register any agents with the IBMF */
69 if ((state->hs_cfg_profile->cp_qp0_agents_in_fw) &&
70 (state->hs_cfg_profile->cp_qp1_agents_in_fw)) {
71 return (DDI_SUCCESS);
72 }
73
74 /*
75 * Build a unique name for the Hermon task queue from the Hermon driver
76 * instance number and HERMON_TASKQ_NAME
77 */
78 rsrc_name = (char *)kmem_zalloc(HERMON_RSRC_NAME_MAXLEN, KM_SLEEP);
79 HERMON_RSRC_NAME(rsrc_name, HERMON_TASKQ_NAME);
80
81 /* Initialize the Hermon IB management agent list */
82 status = hermon_agent_list_init(state);
83 if (status != DDI_SUCCESS) {
84 goto agentsinit_fail;
85 }
86
87 /*
88 * Initialize the agent handling task queue. Note: We set the task
89 * queue priority to the minimum system priority. At this point this
90 * is considered acceptable because MADs are unreliable datagrams
91 * and could get lost (in general) anyway.
92 */
93 state->hs_taskq_agents = ddi_taskq_create(state->hs_dip,
94 rsrc_name, HERMON_TASKQ_NTHREADS, TASKQ_DEFAULTPRI, 0);
95 if (state->hs_taskq_agents == NULL) {
96 hermon_agent_list_fini(state);
97 goto agentsinit_fail;
98 }
99
100 /* Now attempt to register all of the agents with the IBMF */
101 status = hermon_agent_register_all(state);
102 if (status != DDI_SUCCESS) {
103 ddi_taskq_destroy(state->hs_taskq_agents);
104 hermon_agent_list_fini(state);
105 goto agentsinit_fail;
106 }
107
108 kmem_free(rsrc_name, HERMON_RSRC_NAME_MAXLEN);
109 return (DDI_SUCCESS);
110
111 agentsinit_fail:
112 kmem_free(rsrc_name, HERMON_RSRC_NAME_MAXLEN);
113 return (status);
114 }
115
116
117 /*
118 * hermon_agent_handlers_fini()
119 * Context: Only called from detach() path context
120 */
121 int
hermon_agent_handlers_fini(hermon_state_t * state)122 hermon_agent_handlers_fini(hermon_state_t *state)
123 {
124 int status;
125
126 /* Determine if we need to unregister any agents from the IBMF */
127 if ((state->hs_cfg_profile->cp_qp0_agents_in_fw) &&
128 (state->hs_cfg_profile->cp_qp1_agents_in_fw)) {
129 return (DDI_SUCCESS);
130 }
131
132 /* Now attempt to unregister all of the agents from the IBMF */
133 status = hermon_agent_unregister_all(state, state->hs_num_agents);
134 if (status != DDI_SUCCESS) {
135 return (DDI_FAILURE);
136 }
137
138 /*
139 * Destroy the task queue. The task queue destroy is guaranteed to
140 * wait until any scheduled tasks have completed. We are able to
141 * guarantee that no _new_ tasks will be added the task queue while
142 * we are in the ddi_taskq_destroy() call because we have
143 * (at this point) successfully unregistered from IBMF (in
144 * hermon_agent_unregister_all() above).
145 */
146 ddi_taskq_destroy(state->hs_taskq_agents);
147
148 /* Teardown the Hermon IB management agent list */
149 hermon_agent_list_fini(state);
150
151 return (DDI_SUCCESS);
152 }
153
154
155 /*
156 * hermon_agent_request_cb()
157 * Context: Called from the IBMF context
158 */
159 static void
hermon_agent_request_cb(ibmf_handle_t ibmf_handle,ibmf_msg_t * msgp,void * args)160 hermon_agent_request_cb(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp,
161 void *args)
162 {
163 hermon_agent_handler_arg_t *cb_args;
164 hermon_agent_list_t *curr;
165 hermon_state_t *state;
166 int status;
167
168 curr = (hermon_agent_list_t *)args;
169 state = curr->agl_state;
170
171 /*
172 * Allocate space to hold the callback args (for passing to the
173 * task queue). Note: If we are unable to allocate space for the
174 * the callback args here, then we just return. But we must ensure
175 * that we call ibmf_free_msg() to free up the message.
176 */
177 cb_args = (hermon_agent_handler_arg_t *)kmem_zalloc(
178 sizeof (hermon_agent_handler_arg_t), KM_NOSLEEP);
179 if (cb_args == NULL) {
180 (void) ibmf_free_msg(ibmf_handle, &msgp);
181 return;
182 }
183 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cb_args))
184
185 /* Fill in the callback args */
186 cb_args->ahd_ibmfhdl = ibmf_handle;
187 cb_args->ahd_ibmfmsg = msgp;
188 cb_args->ahd_agentlist = args;
189
190 /*
191 * Dispatch the message to the task queue. Note: Just like above,
192 * if this request fails for any reason then make sure to free up
193 * the IBMF message and then return
194 */
195 status = ddi_taskq_dispatch(state->hs_taskq_agents,
196 hermon_agent_handle_req, cb_args, DDI_NOSLEEP);
197 if (status == DDI_FAILURE) {
198 kmem_free(cb_args, sizeof (hermon_agent_handler_arg_t));
199 (void) ibmf_free_msg(ibmf_handle, &msgp);
200 }
201 }
202
203 /*
204 * hermon_get_smlid()
205 * Simple helper function for hermon_agent_handle_req() below.
206 * Get the portinfo and extract the smlid.
207 */
208 static ib_lid_t
hermon_get_smlid(hermon_state_t * state,uint_t port)209 hermon_get_smlid(hermon_state_t *state, uint_t port)
210 {
211 sm_portinfo_t portinfo;
212 int status;
213
214 status = hermon_getportinfo_cmd_post(state, port,
215 HERMON_SLEEPFLAG_FOR_CONTEXT(), &portinfo);
216 if (status != HERMON_CMD_SUCCESS) {
217 cmn_err(CE_CONT, "Hermon: GetPortInfo (port %02d) command "
218 "failed: %08x\n", port, status);
219 return (0);
220 }
221 return (portinfo.MasterSMLID);
222 }
223
224 /*
225 * hermon_get_port_change_flags()
226 * Helper function to determine the changes in the incoming MAD's portinfo
227 * for the Port Change event.
228 */
229 static ibt_port_change_t
hermon_port_change_flags(sm_portinfo_t * curpinfo,sm_portinfo_t * madpinfo)230 hermon_port_change_flags(sm_portinfo_t *curpinfo, sm_portinfo_t *madpinfo)
231 {
232 int SMDisabled, ReregSuppd;
233 ibt_port_change_t flags = 0;
234
235 SMDisabled = curpinfo->CapabilityMask & SM_CAP_MASK_IS_SM_DISABLED;
236 ReregSuppd = curpinfo->CapabilityMask & SM_CAP_MASK_IS_CLNT_REREG_SUPPD;
237
238 if (curpinfo->MasterSMLID != madpinfo->MasterSMLID) {
239 flags |= IBT_PORT_CHANGE_SM_LID;
240 }
241 if (curpinfo->MasterSMSL != madpinfo->MasterSMSL) {
242 flags |= IBT_PORT_CHANGE_SM_SL;
243 }
244 if (curpinfo->SubnetTimeOut != madpinfo->SubnetTimeOut) {
245 flags |= IBT_PORT_CHANGE_SUB_TIMEOUT;
246 }
247 if ((madpinfo->CapabilityMask & SM_CAP_MASK_IS_SM_DISABLED)
248 ^ SMDisabled) {
249 flags |= IBT_PORT_CHANGE_SM_FLAG;
250 }
251 if ((madpinfo->CapabilityMask & SM_CAP_MASK_IS_CLNT_REREG_SUPPD)
252 ^ ReregSuppd) {
253 flags |= IBT_PORT_CHANGE_REREG;
254 }
255 return (flags);
256 }
257
258 int
hermon_set_port_capability(hermon_state_t * state,uint8_t port,sm_portinfo_t * portinfo,ibt_port_change_t flags)259 hermon_set_port_capability(hermon_state_t *state, uint8_t port,
260 sm_portinfo_t *portinfo, ibt_port_change_t flags)
261 {
262 uint32_t capmask;
263 int status;
264 hermon_hw_set_port_t set_port;
265
266 bzero(&set_port, sizeof (set_port));
267
268 /* Validate that specified port number is legal */
269 if (!hermon_portnum_is_valid(state, port)) {
270 return (IBT_HCA_PORT_INVALID);
271 }
272
273 /*
274 * Convert InfiniBand-defined port capability flags to the format
275 * specified by the IBTF. Specifically, we modify the capability
276 * mask based on the specified values.
277 */
278 capmask = portinfo->CapabilityMask;
279
280 if (flags & IBT_PORT_CHANGE_SM_FLAG)
281 capmask ^= SM_CAP_MASK_IS_SM;
282
283 if (flags & IBT_PORT_CHANGE_REREG)
284 capmask ^= SM_CAP_MASK_IS_CLNT_REREG_SUPPD;
285 set_port.cap_mask = capmask;
286
287 /*
288 * Use the Hermon SET_PORT command to update the capability mask and
289 * (possibly) reset the QKey violation counter for the specified port.
290 * Note: In general, this operation shouldn't fail. If it does, then
291 * it is an indication that something (probably in HW, but maybe in
292 * SW) has gone seriously wrong.
293 */
294 status = hermon_set_port_cmd_post(state, &set_port, port,
295 HERMON_SLEEPFLAG_FOR_CONTEXT());
296 if (status != HERMON_CMD_SUCCESS) {
297 HERMON_WARNING(state, "failed to modify port capabilities");
298 cmn_err(CE_CONT, "Hermon: SET_IB (port %02d) command failed: "
299 "%08x\n", port, status);
300 return (DDI_FAILURE);
301 }
302
303 return (DDI_SUCCESS);
304 }
305
306 /*
307 * hermon_agent_handle_req()
308 * Context: Called with priority of taskQ thread
309 */
310 static void
hermon_agent_handle_req(void * cb_args)311 hermon_agent_handle_req(void *cb_args)
312 {
313 hermon_agent_handler_arg_t *agent_args;
314 hermon_agent_list_t *curr;
315 ibc_async_event_t event;
316 ibt_async_code_t type, code;
317 sm_portinfo_t curpinfo, tmadpinfo;
318 sm_portinfo_t *madpinfop;
319 hermon_state_t *state;
320 ibmf_handle_t ibmf_handle;
321 ibmf_msg_t *msgp;
322 ibmf_msg_bufs_t *recv_msgbufp;
323 ibmf_msg_bufs_t *send_msgbufp;
324 ib_mad_hdr_t *madhdrp;
325 ibmf_retrans_t retrans;
326 uint_t port;
327 int status;
328
329 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*((sm_portinfo_t *)madpinfop)))
330 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(curpinfo))
331 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(tmadpinfo))
332 /* Extract the necessary info from the callback args parameter */
333 agent_args = (hermon_agent_handler_arg_t *)cb_args;
334 ibmf_handle = agent_args->ahd_ibmfhdl;
335 msgp = agent_args->ahd_ibmfmsg;
336 curr = agent_args->ahd_agentlist;
337 state = curr->agl_state;
338 port = curr->agl_port;
339
340 /*
341 * Set the message send buffer pointers to the message receive buffer
342 * pointers to reuse the IBMF provided buffers for the sender
343 * information.
344 */
345 recv_msgbufp = &msgp->im_msgbufs_recv;
346 send_msgbufp = &msgp->im_msgbufs_send;
347 bcopy(recv_msgbufp, send_msgbufp, sizeof (ibmf_msg_bufs_t));
348
349 /*
350 * Check if the incoming packet is a special "Hermon Trap" MAD. If it
351 * is, then do the special handling. If it isn't, then simply pass it
352 * on to the firmware and forward the response back to the IBMF.
353 *
354 * Note: Hermon has a unique method for handling internally generated
355 * Traps. All internally detected/generated Trap messages are
356 * automatically received by the IBMF (as receive completions on QP0),
357 * which (because all Hermon Trap MADs have SLID == 0) detects it as a
358 * special "Hermon Trap" and forwards it here to the driver's SMA.
359 * It is then our responsibility here to fill in the Trap MAD's DLID
360 * for forwarding to the real Master SM (as programmed in the port's
361 * PortInfo.MasterSMLID field.)
362 */
363 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(msgp->im_local_addr))
364 if (HERMON_IS_SPECIAL_TRAP_MAD(msgp)) {
365 msgp->im_local_addr.ia_remote_lid =
366 hermon_get_smlid(state, port);
367 } else {
368 int isSMSet, isReregSuppd;
369 uint_t attr_id, method, mgmt_class;
370
371 madhdrp = recv_msgbufp->im_bufs_mad_hdr;
372 method = madhdrp->R_Method;
373 attr_id = b2h16(madhdrp->AttributeID);
374 mgmt_class = madhdrp->MgmtClass;
375
376 /*
377 * Is this a Subnet Manager MAD with SET method ? If so
378 * we will have to get the current portinfo to generate
379 * events based on what has changed in portinfo.
380 */
381 isSMSet = (((mgmt_class == MAD_MGMT_CLASS_SUBN_LID_ROUTED)||
382 (mgmt_class == MAD_MGMT_CLASS_SUBN_DIRECT_ROUTE)) &&
383 (method == MAD_METHOD_SET));
384
385 /*
386 * Get the current portinfo to compare with the portinfo
387 * received in the MAD for PortChange event.
388 */
389 if (isSMSet && (attr_id == SM_PORTINFO_ATTRID) ||
390 (attr_id == SM_PKEY_TABLE_ATTRID) ||
391 (attr_id == SM_GUIDINFO_ATTRID)) {
392 madpinfop = recv_msgbufp->im_bufs_cl_data;
393 tmadpinfo = *madpinfop;
394 HERMON_GETPORTINFO_SWAP(&tmadpinfo);
395 status = hermon_getportinfo_cmd_post(state, port,
396 HERMON_SLEEPFLAG_FOR_CONTEXT(), &curpinfo);
397 if (status != HERMON_CMD_SUCCESS) {
398 cmn_err(CE_CONT, "Hermon: GetPortInfo "
399 "(port %02d) command failed: %08x\n", port,
400 status);
401 goto hermon_agent_handle_req_skip_response;
402 }
403 }
404
405 /*
406 * Post the command to the firmware (using the MAD_IFC
407 * command). Note: We also reuse the command that was passed
408 * in. We pass the pointer to the original MAD payload as if
409 * it were both the source of the incoming MAD as well as the
410 * destination for the response. This is acceptable and saves
411 * us the step of one additional copy. Note: If this command
412 * fails for any reason other than HERMON_CMD_BAD_PKT, it
413 * probably indicates a serious problem.
414 */
415 status = hermon_mad_ifc_cmd_post(state, port,
416 HERMON_CMD_SLEEP_NOSPIN,
417 (uint32_t *)recv_msgbufp->im_bufs_mad_hdr,
418 (uint32_t *)send_msgbufp->im_bufs_mad_hdr);
419 if (status != HERMON_CMD_SUCCESS) {
420 if ((status != HERMON_CMD_BAD_PKT) &&
421 (status != HERMON_CMD_INSUFF_RSRC)) {
422 cmn_err(CE_CONT, "Hermon: MAD_IFC (port %02d) "
423 "command failed: %08x\n", port, status);
424 }
425
426 /* finish cleanup */
427 goto hermon_agent_handle_req_skip_response;
428 }
429
430 if (isSMSet) {
431 event.ev_port_flags = 0;
432 type = 0;
433 event.ev_port = (uint8_t)port;
434
435 switch (attr_id) {
436 case SM_PORTINFO_ATTRID:
437 /*
438 * This is a SM SET method with portinfo
439 * attribute. If ClientRereg bit was set in
440 * the MADs portinfo this is a REREG event
441 * (see section 14.4.11 in IB Spec 1.2.1). Else
442 * compare the current (before MAD_IFC command)
443 * portinfo with the portinfo in the MAD and
444 * signal PORT_CHANGE event with the proper
445 * ev_port_flags.
446 *
447 */
448 isReregSuppd = curpinfo.CapabilityMask &
449 SM_CAP_MASK_IS_CLNT_REREG_SUPPD;
450
451 madpinfop = recv_msgbufp->im_bufs_cl_data;
452 if (tmadpinfo.ClientRereg && isReregSuppd) {
453 type |= IBT_CLNT_REREG_EVENT;
454 }
455
456 type |= IBT_PORT_CHANGE_EVENT;
457 event.ev_port_flags = hermon_port_change_flags(
458 &curpinfo, &tmadpinfo);
459 if (event.ev_port_flags &
460 (IBT_PORT_CHANGE_REREG |
461 IBT_PORT_CHANGE_SM_FLAG)) {
462 if (hermon_set_port_capability(state,
463 port, &curpinfo,
464 event.ev_port_flags)
465 != DDI_SUCCESS) {
466 cmn_err(CE_CONT, "HERMON: Port "
467 "%d capability reset "
468 "failed\n", port);
469 }
470 }
471
472 /*
473 * If we have a SMLID change event but
474 * capability mask doesn't have Rereg support
475 * bit set, we have to do the Rereg part too.
476 */
477 if ((event.ev_port_flags &
478 IBT_PORT_CHANGE_SM_LID) && !isReregSuppd)
479 type |= IBT_CLNT_REREG_EVENT;
480 break;
481 case SM_PKEY_TABLE_ATTRID:
482 type |= IBT_PORT_CHANGE_EVENT;
483 event.ev_port_flags = IBT_PORT_CHANGE_PKEY;
484 break;
485 case SM_GUIDINFO_ATTRID:
486 type |= IBT_PORT_CHANGE_EVENT;
487 event.ev_port_flags = IBT_PORT_CHANGE_SGID;
488 break;
489 default:
490 break;
491
492 }
493
494 /*
495 * NOTE: here we call ibc_async_handler directly without
496 * using the HERMON_DO_IBTF_ASYNC_CALLB, since hermon
497 * can not be unloaded till ibmf_unregiter is done and
498 * this thread (hs_taskq_agents) will be destroyed
499 * before ibmf_uregister is called.
500 *
501 * The hermon event queue based hs_in_evcallb flag
502 * assumes that we will pick one event after another
503 * and dispatch them sequentially. If we use
504 * HERMON_DO_IBTF_ASYNC_CALLB, we will break this
505 * assumption make hs_in_evcallb inconsistent.
506 */
507 while (type != 0) {
508 if (type & IBT_PORT_CHANGE_EVENT) {
509 code = IBT_PORT_CHANGE_EVENT;
510 type &= ~IBT_PORT_CHANGE_EVENT;
511 } else {
512 code = IBT_CLNT_REREG_EVENT;
513 type = 0;
514 }
515 ibc_async_handler(state->hs_ibtfpriv, code,
516 &event);
517 }
518 }
519 }
520
521 /*
522 * If incoming MAD was "TrapRepress", then no response is necessary.
523 * Free the IBMF message and return.
524 */
525 if (HERMON_IS_TRAP_REPRESS_MAD(msgp)) {
526 goto hermon_agent_handle_req_skip_response;
527 }
528
529 /*
530 * Modify the response MAD as necessary (for any special cases).
531 * Specifically, if this MAD was a directed route MAD, then some
532 * additional packet manipulation may be necessary because the Hermon
533 * firmware does not do all the required steps to respond to the
534 * MAD.
535 */
536 hermon_agent_mad_resp_handling(state, msgp, port);
537
538 /*
539 * Send response (or forwarded "Trap" MAD) back to IBMF. We use the
540 * "response callback" to indicate when it is appropriate (later) to
541 * free the IBMF msg.
542 */
543 status = ibmf_msg_transport(ibmf_handle, IBMF_QP_HANDLE_DEFAULT,
544 msgp, &retrans, hermon_agent_response_cb, state, 0);
545 if (status != IBMF_SUCCESS) {
546 goto hermon_agent_handle_req_skip_response;
547 }
548
549 /* Free up the callback args parameter */
550 kmem_free(agent_args, sizeof (hermon_agent_handler_arg_t));
551 return;
552
553 hermon_agent_handle_req_skip_response:
554 /* Free up the ibmf message */
555 (void) ibmf_free_msg(ibmf_handle, &msgp);
556
557 /* Free up the callback args parameter */
558 kmem_free(agent_args, sizeof (hermon_agent_handler_arg_t));
559 }
560
561
562 /*
563 * hermon_agent_response_cb()
564 * Context: Called from the IBMF context
565 */
566 /* ARGSUSED */
567 static void
hermon_agent_response_cb(ibmf_handle_t ibmf_handle,ibmf_msg_t * msgp,void * args)568 hermon_agent_response_cb(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp,
569 void *args)
570 {
571 /*
572 * It is the responsibility of each IBMF callback recipient to free
573 * the packets that it has been given. Now that we are in the
574 * response callback, we can be assured that it is safe to do so.
575 */
576 (void) ibmf_free_msg(ibmf_handle, &msgp);
577 }
578
579
580 /*
581 * hermon_agent_list_init()
582 * Context: Only called from attach() path context
583 */
584 static int
hermon_agent_list_init(hermon_state_t * state)585 hermon_agent_list_init(hermon_state_t *state)
586 {
587 hermon_agent_list_t *curr;
588 uint_t num_ports, num_agents, num_agents_per_port;
589 uint_t num_sma_agents = 0;
590 uint_t num_pma_agents = 0;
591 uint_t num_bma_agents = 0;
592 uint_t do_qp0, do_qp1;
593 int i, j, indx;
594
595 /*
596 * Calculate the number of registered agents for each port
597 * (SMA, PMA, and BMA) and determine whether or not to register
598 * a given agent with the IBMF (or whether to let the Hermon firmware
599 * handle it)
600 */
601 num_ports = state->hs_cfg_profile->cp_num_ports;
602 num_agents = 0;
603 num_agents_per_port = 0;
604 do_qp0 = state->hs_cfg_profile->cp_qp0_agents_in_fw;
605 do_qp1 = state->hs_cfg_profile->cp_qp1_agents_in_fw;
606 if (do_qp0 == 0) {
607 num_agents += (num_ports * HERMON_NUM_QP0_AGENTS_PER_PORT);
608 num_agents_per_port += HERMON_NUM_QP0_AGENTS_PER_PORT;
609 num_sma_agents = num_ports;
610 }
611 if (do_qp1 == 0) {
612 num_agents += (num_ports * HERMON_NUM_QP1_AGENTS_PER_PORT);
613 num_agents_per_port += HERMON_NUM_QP1_AGENTS_PER_PORT;
614 num_pma_agents = num_ports;
615 /*
616 * The following line is commented out because the Hermon
617 * firmware does not currently support a BMA. If it did,
618 * then we would want to register the agent with the IBMF.
619 * (We would also need to have HERMON_NUM_QP1_AGENTS_PER_PORT
620 * set to 2, instead of 1.)
621 *
622 * num_bma_agents = num_ports;
623 */
624 }
625
626 state->hs_num_agents = num_agents;
627
628 /*
629 * Allocate the memory for all of the agent list entries
630 */
631 state->hs_agents = (hermon_agent_list_t *)kmem_zalloc(num_agents *
632 sizeof (hermon_agent_list_t), KM_SLEEP);
633 if (state->hs_agents == NULL) {
634 return (DDI_FAILURE);
635 }
636
637 /*
638 * Fill in each of the agent list entries with the agent's
639 * MgmtClass, port number, and Hermon softstate pointer
640 */
641 indx = 0;
642 for (i = 0; i < num_agents_per_port; i++) {
643 for (j = 0; j < num_ports; j++) {
644 curr = &state->hs_agents[indx];
645 curr->agl_state = state;
646 curr->agl_port = j + 1;
647
648 if ((do_qp0 == 0) && num_sma_agents) {
649 curr->agl_mgmtclass = SUBN_AGENT;
650 num_sma_agents--;
651 indx++;
652 } else if ((do_qp1 == 0) && (num_pma_agents)) {
653 curr->agl_mgmtclass = PERF_AGENT;
654 num_pma_agents--;
655 indx++;
656 } else if ((do_qp1 == 0) && (num_bma_agents)) {
657 curr->agl_mgmtclass = BM_AGENT;
658 num_bma_agents--;
659 indx++;
660 }
661 }
662 }
663
664 return (DDI_SUCCESS);
665 }
666
667
668 /*
669 * hermon_agent_list_fini()
670 * Context: Only called from attach() and/or detach() path contexts
671 */
672 static void
hermon_agent_list_fini(hermon_state_t * state)673 hermon_agent_list_fini(hermon_state_t *state)
674 {
675 /* Free up the memory for the agent list entries */
676 kmem_free(state->hs_agents,
677 state->hs_num_agents * sizeof (hermon_agent_list_t));
678 }
679
680
681 /*
682 * hermon_agent_register_all()
683 * Context: Only called from attach() path context
684 */
685 static int
hermon_agent_register_all(hermon_state_t * state)686 hermon_agent_register_all(hermon_state_t *state)
687 {
688 hermon_agent_list_t *curr;
689 ibmf_register_info_t ibmf_reg;
690 ibmf_impl_caps_t impl_caps;
691 ib_guid_t nodeguid;
692 int i, status, num_registered;
693
694 /* Get the Hermon NodeGUID from the softstate */
695 nodeguid = state->hs_ibtfinfo.hca_attr->hca_node_guid;
696
697 /*
698 * Register each of the agents with the IBMF (and add callbacks for
699 * each to the hermon_agent_request_cb() routine). Note: If we
700 * fail somewhere along the line here, we attempt to cleanup as much
701 * of the mess as we can and then jump to hermon_agent_unregister_all()
702 * to cleanup the rest.
703 */
704 num_registered = 0;
705
706 if (state->hs_num_agents == 0) {
707 return (DDI_SUCCESS);
708 }
709
710 for (i = 0; i < state->hs_num_agents; i++) {
711 /* Register each agent with the IBMF */
712 curr = &state->hs_agents[i];
713 ibmf_reg.ir_ci_guid = nodeguid;
714 ibmf_reg.ir_port_num = curr->agl_port;
715 ibmf_reg.ir_client_class = curr->agl_mgmtclass;
716
717 status = ibmf_register(&ibmf_reg, IBMF_VERSION, 0,
718 NULL, NULL, &curr->agl_ibmfhdl, &impl_caps);
719 if (status != IBMF_SUCCESS) {
720 goto agents_reg_fail;
721 }
722
723 /* Setup callbacks with the IBMF */
724 status = ibmf_setup_async_cb(curr->agl_ibmfhdl,
725 IBMF_QP_HANDLE_DEFAULT, hermon_agent_request_cb, curr, 0);
726 if (status != IBMF_SUCCESS) {
727 (void) ibmf_unregister(&curr->agl_ibmfhdl, 0);
728 goto agents_reg_fail;
729 }
730 num_registered++;
731 }
732
733 return (DDI_SUCCESS);
734
735 agents_reg_fail:
736 (void) hermon_agent_unregister_all(state, num_registered);
737 return (DDI_FAILURE);
738 }
739
740
741 /*
742 * hermon_agent_unregister_all()
743 * Context: Only called from detach() path context
744 */
745 static int
hermon_agent_unregister_all(hermon_state_t * state,int num_reg)746 hermon_agent_unregister_all(hermon_state_t *state, int num_reg)
747 {
748 hermon_agent_list_t *curr;
749 int i, status;
750
751 if (num_reg == 0) {
752 return (DDI_SUCCESS);
753 }
754
755 /*
756 * For each registered agent in the agent list, teardown the
757 * callbacks from the IBMF and unregister.
758 */
759 for (i = 0; i < num_reg; i++) {
760 curr = &state->hs_agents[i];
761
762 /* Teardown the IBMF callback */
763 status = ibmf_tear_down_async_cb(curr->agl_ibmfhdl,
764 IBMF_QP_HANDLE_DEFAULT, 0);
765 if (status != IBMF_SUCCESS) {
766 return (DDI_FAILURE);
767 }
768
769 /* Unregister the agent from the IBMF */
770 status = ibmf_unregister(&curr->agl_ibmfhdl, 0);
771 if (status != IBMF_SUCCESS) {
772 return (DDI_FAILURE);
773 }
774 }
775
776 return (DDI_SUCCESS);
777 }
778
779
780 /*
781 * hermon_agent_mad_resp_handling()
782 * Context: Called with priority of taskQ thread
783 */
784 /* ARGSUSED */
785 static void
hermon_agent_mad_resp_handling(hermon_state_t * state,ibmf_msg_t * msgp,uint_t port)786 hermon_agent_mad_resp_handling(hermon_state_t *state, ibmf_msg_t *msgp,
787 uint_t port)
788 {
789 ib_mad_hdr_t *rmadhdrp = msgp->im_msgbufs_recv.im_bufs_mad_hdr;
790 ib_mad_hdr_t *smadhdrp = msgp->im_msgbufs_send.im_bufs_mad_hdr;
791 uint_t hop_count, hop_point;
792 uchar_t *resp, *ret_path;
793
794 resp = (uchar_t *)msgp->im_msgbufs_send.im_bufs_cl_data;
795
796 /*
797 * Handle directed route MADs as a special case. Hermon firmware
798 * does not update the "direction" bit, "hop pointer", "Return
799 * Path" or, in fact, any of the "directed route" parameters. So
800 * the responsibility falls on Hermon driver software to inspect the
801 * MADs and update those fields as appropriate (see section 14.2.2
802 * of the IBA specification, rev 1.1)
803 */
804 if (HERMON_MAD_IS_DR(rmadhdrp)) {
805
806 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*((sm_dr_mad_hdr_t *)rmadhdrp)))
807 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*((sm_dr_mad_hdr_t *)smadhdrp)))
808
809 /*
810 * Set the "Direction" bit to one. This indicates that this
811 * is now directed route response
812 */
813 HERMON_DRMAD_SET_DIRECTION(rmadhdrp);
814
815 /* Extract the "hop pointer" and "hop count" from the MAD */
816 hop_count = HERMON_DRMAD_GET_HOPCOUNT(rmadhdrp);
817 hop_point = HERMON_DRMAD_GET_HOPPOINTER(rmadhdrp);
818
819 /* Append the port we came in on to the "Return Path" */
820 if ((hop_count != 0) && ((hop_point == hop_count) ||
821 (hop_point == hop_count + 1))) {
822 ret_path = &resp[HERMON_DRMAD_RETURN_PATH_OFFSET];
823 ret_path[hop_point] = (uchar_t)port;
824 }
825
826 /* Then increment the "hop pointer" in the MAD */
827 hop_point++;
828 HERMON_DRMAD_SET_HOPPOINTER(smadhdrp, (uint8_t)hop_point);
829 }
830 }
831