1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <sys/ib/mgt/ibmf/ibmf_saa_impl.h>
27 #include <sys/ib/mgt/ibmf/ibmf_saa_utils.h>
28 
29 /* Global sa_access State Pointer */
30 saa_state_t *saa_statep;
31 _NOTE(READ_ONLY_DATA(saa_statep))
32 
33 extern	int	ibmf_trace_level;
34 
35 extern	int	ibmf_taskq_max_tasks;
36 
37 static int
38 ibmf_saa_impl_new_smlid_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp,
39     ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, int transport_flags);
40 static int
41 ibmf_saa_impl_revert_to_qp1(saa_port_t *saa_portp, ibmf_msg_t *msgp,
42     ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_args, int transport_flags);
43 static int
44 ibmf_saa_check_sa_and_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp,
45     ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg,
46     hrtime_t trans_send_time, int transport_flags);
47 static int ibmf_saa_impl_init_msg(saa_impl_trans_info_t *trans_info,
48     boolean_t sleep_flag, ibmf_msg_t **msgp, uint32_t *transport_flagsp,
49     ibmf_retrans_t *ibmf_retransp);
50 static int ibmf_saa_must_purge(saa_port_t *saa_portp);
51 static void ibmf_saa_impl_invalidate_port(saa_port_t *saa_portp);
52 static void ibmf_saa_impl_destroy_port(saa_port_t *saa_portp);
53 static void ibmf_saa_impl_uninit_kstats(saa_port_t *saa_portp);
54 static void ibmf_saa_impl_get_cpi_cb(void *arg, size_t length, char *buffer,
55     int status);
56 static void ibmf_saa_impl_async_event_cb(ibmf_handle_t ibmf_handle,
57     void *clnt_private, ibmf_async_event_t event_type);
58 static void ibmf_saa_impl_port_up(ib_guid_t ci_guid, uint8_t port_num);
59 static void ibmf_saa_impl_port_down(ib_guid_t ci_guid, uint8_t port_num);
60 static void ibmf_saa_impl_port_chg(ibt_async_event_t *event);
61 static void ibmf_saa_impl_client_rereg(ib_guid_t ci_guid, uint8_t port_num);
62 static void ibmf_saa_impl_hca_detach(saa_port_t *saa_removed);
63 static void ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle,
64     ibmf_msg_t *msgp, boolean_t ignore_data, int *status, void **result,
65     size_t *length, boolean_t sleep_flag);
66 static int ibmf_saa_impl_check_sa_support(uint16_t cap_mask, uint16_t attr_id);
67 static uint_t ibmf_saa_impl_get_attr_id_length(uint16_t attr_id);
68 static void ibmf_saa_impl_free_msg(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msgp);
69 static int ibmf_saa_impl_get_port_guid(ibt_hca_portinfo_t *ibt_portinfop,
70     ib_guid_t *guid_ret);
71 static void ibmf_saa_impl_set_transaction_params(saa_port_t *saa_portp,
72     ibt_hca_portinfo_t *portinfop);
73 static void ibmf_saa_impl_update_sa_address_info(saa_port_t *saa_portp,
74     ibmf_msg_t *msgp);
75 static int ibmf_saa_impl_ibmf_unreg(saa_port_t *saa_portp);
76 
77 int	ibmf_saa_max_wait_time = IBMF_SAA_MAX_WAIT_TIME_IN_SECS;
78 int	ibmf_saa_trans_wait_time = IBMF_SAA_TRANS_WAIT_TIME_IN_SECS;
79 int	ibmf_saa_max_resp_time = IBMF_SAA_MAX_RESP_TIME;
80 int	ibmf_saa_max_subnet_timeout = IBMF_SAA_MAX_SUBNET_TIMEOUT;
81 int	ibmf_saa_retrans_retries = IBMF_SAA_RETRANS_RETRIES;
82 
83 /*
84  * ibmf_saa_impl_init:
85  * Allocates memory for the ibmf_saa state structure and initializes the taskq.
86  * Called from the modules init() routine.
87  *
88  * Input Arguments
89  * none
90  *
91  * Output Arguments
92  * none
93  *
94  * Returns
95  * IBMF_NO_RESOURCES if taskq could not be created.
96  * IBMF_SUCCESS on success
97  *
98  */
99 int
ibmf_saa_impl_init()100 ibmf_saa_impl_init()
101 {
102 	int		res;
103 
104 	/* CONSTCOND */
105 	ASSERT(NO_COMPETING_THREADS);
106 
107 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_init_start,
108 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_init() enter\n");
109 
110 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_statep))
111 
112 	saa_statep = kmem_zalloc(sizeof (saa_state_t), KM_SLEEP);
113 
114 	/* create taskq for notifying event subscribers */
115 	saa_statep->saa_event_taskq = taskq_create(
116 	    "ibmf_saa_event_taskq", IBMF_TASKQ_NTHREADS,
117 	    MINCLSYSPRI, 1, ibmf_taskq_max_tasks, TASKQ_DYNAMIC |
118 	    TASKQ_PREPOPULATE);
119 	if (saa_statep->saa_event_taskq == NULL) {
120 
121 		IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L4,
122 		    ibmf_saa_impl_init_end_err,
123 		    IBMF_TNF_TRACE, "", "ibmf_saa_impl_init(): %s\n",
124 		    tnf_string, msg, "event taskq create failed");
125 
126 		kmem_free(saa_statep, sizeof (saa_state_t));
127 
128 		res = IBMF_NO_RESOURCES;
129 
130 		goto bail;
131 	}
132 
133 	mutex_init(&saa_statep->saa_port_list_mutex, NULL, MUTEX_DRIVER,
134 	    NULL);
135 
136 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*saa_statep))
137 
138 	res = IBMF_SUCCESS;
139 bail:
140 
141 	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_init_end,
142 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_init() exit: status = %d\n",
143 	    tnf_int, res, res);
144 
145 	return (res);
146 }
147 
148 /*
149  * ibmf_saa_impl_fini:
150  * If there are no registered clients, cleans up all memory associated with the
151  * state, including each of the port list entries.
152  * Called from the modules fini() routine.
153  *
154  * Input Arguments
155  * none
156  *
157  * Output Arguments
158  * none
159  *
160  * Returns
161  * EBUSY if there are outstanding transactions or registered clients
162  * 0 if cleanup was sucessfull
163  *
164  */
165 int
ibmf_saa_impl_fini()166 ibmf_saa_impl_fini()
167 {
168 	int		ret = 0;
169 	saa_port_t	*saa_portp;
170 	saa_port_t	*next;
171 
172 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_fini_start,
173 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_fini() enter\n");
174 
175 	/* make sure there are no registered clients */
176 	mutex_enter(&saa_statep->saa_port_list_mutex);
177 
178 	saa_portp = saa_statep->saa_port_list;
179 	while (saa_portp != NULL) {
180 
181 		mutex_enter(&saa_portp->saa_pt_mutex);
182 
183 		if (saa_portp->saa_pt_reference_count > 0) {
184 
185 			IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
186 			    ibmf_saa_impl_fini_err, IBMF_TNF_ERROR, "",
187 			    "ibmf_saa_impl_fini: %s, port %016" PRIx64 "\n",
188 			    tnf_string, msg,
189 			    "cannot unload ibmf_saa. Client on port still"
190 			    " registered", tnf_opaque, port,
191 			    saa_portp->saa_pt_port_guid);
192 
193 			mutex_exit(&saa_portp->saa_pt_mutex);
194 
195 			mutex_exit(&saa_statep->saa_port_list_mutex);
196 
197 			ret = EBUSY;
198 			goto bail;
199 		}
200 
201 		/* make sure there are no outstanding transactions */
202 
203 		if (saa_portp->saa_pt_num_outstanding_trans > 0) {
204 
205 			IBMF_TRACE_3(IBMF_TNF_NODEBUG, DPRINT_L1,
206 			    ibmf_saa_impl_fini_err, IBMF_TNF_ERROR, "",
207 			    "ibmf_saa_impl_fini: %s, port = %016" PRIx64
208 			    ", num transactions = %d\n",
209 			    tnf_string, msg, "Cannot unload ibmf_saa."
210 			    "  Outstanding transactions on port.",
211 			    tnf_opaque, port,
212 			    saa_portp->saa_pt_port_guid,
213 			    tnf_uint, outstanding_transactions,
214 			    saa_portp->saa_pt_num_outstanding_trans);
215 
216 			mutex_exit(&saa_portp->saa_pt_mutex);
217 
218 			mutex_exit(&saa_statep->saa_port_list_mutex);
219 
220 			ret = EBUSY;
221 			goto bail;
222 		}
223 
224 		mutex_exit(&saa_portp->saa_pt_mutex);
225 
226 		saa_portp = saa_portp->next;
227 	}
228 
229 	mutex_exit(&saa_statep->saa_port_list_mutex);
230 
231 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(saa_statep->saa_port_list,
232 	    *saa_portp))
233 
234 	/*
235 	 * no more clients nor pending transaction:
236 	 * unregister ibmf and destroy port entries
237 	 */
238 	while (saa_statep->saa_port_list != NULL) {
239 
240 		saa_portp = saa_statep->saa_port_list;
241 		next = saa_portp->next;
242 
243 		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
244 		    ibmf_saa_impl_fini, IBMF_TNF_TRACE, "",
245 		    "ibmf_saa_impl_fini: %s, prefix = %016" PRIx64 "\n",
246 		    tnf_string, msg, "deinitializing port",
247 		    tnf_opaque, port_guid, saa_portp->saa_pt_port_guid);
248 
249 		_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_portp))
250 
251 		mutex_enter(&saa_portp->saa_pt_mutex);
252 
253 		/* unregister from ibmf */
254 		if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_READY) {
255 
256 			mutex_exit(&saa_portp->saa_pt_mutex);
257 
258 			if (ibmf_saa_impl_ibmf_unreg(saa_portp)
259 			    != IBMF_SUCCESS) {
260 				ret = EBUSY;
261 				goto bail;
262 			}
263 		} else
264 			mutex_exit(&saa_portp->saa_pt_mutex);
265 
266 		ibmf_saa_impl_destroy_port(saa_portp);
267 
268 		saa_statep->saa_port_list = next;
269 	}
270 
271 	taskq_destroy(saa_statep->saa_event_taskq);
272 
273 	mutex_destroy(&saa_statep->saa_port_list_mutex);
274 
275 	kmem_free(saa_statep, sizeof (saa_state_t));
276 
277 bail:
278 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_fini_end,
279 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_fini() exit\n");
280 
281 	return (ret);
282 }
283 
284 /*
285  * ibmf_saa_is_valid
286  * Returns true the entry is valid.
287  *
288  * Input Arguments
289  * saa_portp		pointer to state structure
290  * add_ref 		if B_TRUE ref count is incremented on a valid portp
291  *
292  * Output Arguments
293  * none
294  *
295  * Returns
296  * B_TRUE if entry was in a valid state, B_FALSE otherwise
297  */
298 boolean_t
ibmf_saa_is_valid(saa_port_t * saa_portp,int add_ref)299 ibmf_saa_is_valid(saa_port_t *saa_portp, int add_ref)
300 {
301 	boolean_t is_valid = B_TRUE;
302 
303 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_is_valid_start,
304 	    IBMF_TNF_TRACE, "", "ibmf_saa_is_valid() enter\n");
305 
306 	mutex_enter(&saa_portp->saa_pt_mutex);
307 
308 	if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_INVALID ||
309 	    saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_PURGING) {
310 
311 		is_valid = B_FALSE;
312 
313 	} else if (add_ref == B_TRUE) {
314 		/*
315 		 * increment reference count here to ensure that
316 		 * entry does not get purged behind our backs
317 		 */
318 		saa_portp->saa_pt_reference_count++;
319 	}
320 	mutex_exit(&saa_portp->saa_pt_mutex);
321 
322 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_is_valid_end,
323 	    IBMF_TNF_TRACE, "", "ibmf_saa_is_valid() exit\n");
324 
325 	return (is_valid);
326 }
327 
328 /*
329  * ibmf_saa_must_purge
330  * Determines if we can purge a portp (remove it from the list) based on the
331  * state and number of clients
332  *
333  * Input Arguments
334  * saa_portp		pointer to state structure
335  *
336  * Output Arguments
337  * none
338  *
339  * Returns
340  * B_TRUE if the entry can be removed, B_FALSE otherwise
341  */
342 static int
ibmf_saa_must_purge(saa_port_t * saa_portp)343 ibmf_saa_must_purge(saa_port_t *saa_portp)
344 {
345 	int must_purge = B_FALSE;
346 
347 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_must_purge_start,
348 	    IBMF_TNF_TRACE, "", "ibmf_saa_must_purge() enter\n");
349 
350 	mutex_enter(&saa_portp->saa_pt_mutex);
351 
352 	if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_INVALID &&
353 	    saa_portp->saa_pt_reference_count == 0) {
354 
355 		saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_PURGING;
356 		must_purge = B_TRUE;
357 	}
358 
359 	mutex_exit(&saa_portp->saa_pt_mutex);
360 
361 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_must_purge_end,
362 	    IBMF_TNF_TRACE, "", "ibmf_saa_must_purge() exit\n");
363 
364 	return (must_purge);
365 }
366 
367 
368 /*
369  * ibmf_saa_impl_purge:
370  * Removes invalid port state entries from the list
371  *
372  * Input Arguments
373  * none
374  *
375  * Output Arguments
376  * none
377  *
378  * Returns
379  * void
380  */
381 void
ibmf_saa_impl_purge()382 ibmf_saa_impl_purge()
383 {
384 	saa_port_t *cur_portp  = NULL;
385 	saa_port_t *prev_portp = NULL;
386 	saa_port_t *rem_portp  = NULL;
387 
388 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_purge_start,
389 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_purge() enter\n");
390 
391 	mutex_enter(&saa_statep->saa_port_list_mutex);
392 
393 	cur_portp = saa_statep->saa_port_list;
394 	prev_portp = cur_portp;
395 
396 	while (cur_portp != NULL) {
397 
398 		if (ibmf_saa_must_purge(cur_portp) == B_TRUE) {
399 
400 			rem_portp = cur_portp;
401 
402 			/* unlink entry */
403 			if (cur_portp == saa_statep->saa_port_list) {
404 
405 				saa_statep->saa_port_list = cur_portp->next;
406 				cur_portp = saa_statep->saa_port_list;
407 				prev_portp = cur_portp;
408 
409 			} else {
410 
411 				prev_portp->next = cur_portp->next;
412 				cur_portp = cur_portp->next;
413 			}
414 
415 			_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*rem_portp))
416 
417 			/* destroy entry */
418 			ASSERT(rem_portp != NULL);
419 			ibmf_saa_impl_destroy_port(rem_portp);
420 
421 		} else {
422 
423 			prev_portp = cur_portp;
424 			cur_portp = cur_portp->next;
425 		}
426 	}
427 
428 	mutex_exit(&saa_statep->saa_port_list_mutex);
429 
430 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_purge_end,
431 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_purge() exit\n");
432 }
433 
434 /*
435  * saa_impl_add_client:
436  * Adds a client for a particular portp.  Reference count has been incremented
437  * before this call.  It is decremented by saa_impl_add_client() if the call
438  * fails.
439  *
440  * Input Arguments
441  * none
442  *
443  * Output Arguments
444  * none
445  *
446  * Returns
447  * IBMF_BUSY if there are already too many clients registered,
448  * IBMF_BAD_PORT_STATE if the port is invalid (generally because a previous
449  * client failed during registration for this port)
450  * IBMF_SUCCESS otherwise
451  */
452 int
ibmf_saa_impl_add_client(saa_port_t * saa_portp)453 ibmf_saa_impl_add_client(saa_port_t *saa_portp)
454 {
455 	int status = IBMF_SUCCESS;
456 
457 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_add_client_start,
458 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_add_client() enter\n");
459 
460 	mutex_enter(&saa_portp->saa_pt_mutex);
461 
462 	/*
463 	 * check that we don't exceed max clients
464 	 */
465 	if (saa_portp->saa_pt_reference_count >
466 	    SAA_MAX_CLIENTS_PER_PORT) {
467 
468 		IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
469 		    ibmf_saa_impl_add_client_err, IBMF_TNF_ERROR, "",
470 		    "ibmf_saa_impl_add_client: %s, num_reg_clients %d\n",
471 		    tnf_string, msg, "too many clients registered for"
472 		    " port", tnf_uint, num_reg_clients,
473 		    saa_portp->saa_pt_reference_count);
474 
475 		status = IBMF_BUSY;
476 		goto bail;
477 	}
478 
479 	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
480 	    ibmf_saa_impl_add_client, IBMF_TNF_TRACE, "",
481 	    "ibmf_saa_impl_add_client: num_registered_clients %d\n",
482 	    tnf_uint, num_registered_clients,
483 	    saa_portp->saa_pt_reference_count);
484 
485 	/*
486 	 * wait until anyone who is currently registering
487 	 * this port with ibmf is done
488 	 */
489 	while (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_REGISTERING) {
490 
491 		IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
492 		    ibmf_saa_impl_add_client, IBMF_TNF_TRACE, "",
493 		    "ibmf_saa_impl_add_client: %s\n",
494 		    tnf_string, msg, "someone is registering. waiting"
495 		    " for them to finish");
496 
497 		cv_wait(&saa_portp->saa_pt_ibmf_reg_cv,
498 		    &saa_portp->saa_pt_mutex);
499 
500 		IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
501 		    ibmf_saa_impl_add_client,
502 		    IBMF_TNF_TRACE, "", "ibmf_saa_impl_add_client: %s\n",
503 		    tnf_string, msg, "done waiting");
504 	}
505 
506 	/*
507 	 * if port isn't ready here, fail.
508 	 */
509 	if (saa_portp->saa_pt_state != IBMF_SAA_PORT_STATE_READY) {
510 
511 		IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
512 		    ibmf_saa_impl_add_client_err, IBMF_TNF_ERROR,
513 		    "", "ibmf_saa_impl_add_client: %s\n",
514 		    tnf_string, msg, "port state not ready,"
515 		    " removing client.");
516 
517 		status = IBMF_BAD_PORT_STATE;
518 		goto bail;
519 	}
520 
521 bail:
522 	mutex_exit(&saa_portp->saa_pt_mutex);
523 
524 	if (status != IBMF_SUCCESS) {
525 
526 		mutex_enter(&saa_portp->saa_pt_kstat_mutex);
527 
528 		IBMF_SAA_ADD32_KSTATS(saa_portp,
529 		    clients_reg_failed, 1);
530 
531 		mutex_exit(&saa_portp->saa_pt_kstat_mutex);
532 
533 		/* decrementing refcount is last thing we do on entry */
534 
535 		mutex_enter(&saa_portp->saa_pt_mutex);
536 
537 		ASSERT(saa_portp->saa_pt_reference_count > 0);
538 		saa_portp->saa_pt_reference_count--;
539 
540 		mutex_exit(&saa_portp->saa_pt_mutex);
541 	}
542 
543 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
544 	    ibmf_saa_impl_add_client_end, IBMF_TNF_TRACE, "",
545 	    "ibmf_saa_impl_add_client() exit\n");
546 
547 	return (status);
548 }
549 
550 /*
551  * ibmf_saa_impl_create_port()
552  * Create port entry with mimimal inits because
553  * we're holding the list mutex: NO BLOCKING CALLS HERE, please.
554  *
555  * Initialize port state to "registering", so that clients accessing
556  * same port concurrently will wait for the end of the ibmf registration.
557  * Note: this thread will access port members without locking mutex.
558  *
559  * Input Arguments
560  * pt_guid		guid of port
561  *
562  * Output Arguments
563  * saa_portpp		pointer to new saa_portp structure
564  *
565  * Returns
566  * IBMF_NO_MEMORY if memory could not be allocated
567  * IBMF_SUCCESS otherwise
568  */
569 int
ibmf_saa_impl_create_port(ib_guid_t pt_guid,saa_port_t ** saa_portpp)570 ibmf_saa_impl_create_port(ib_guid_t pt_guid, saa_port_t **saa_portpp)
571 {
572 	int		status		= IBMF_SUCCESS;
573 	saa_port_t	*saa_portp	= NULL;
574 
575 	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_create_port_start,
576 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port:"
577 	    " guid %016" PRIx64 "\n",
578 	    tnf_opaque, port_guid, pt_guid);
579 
580 	ASSERT(MUTEX_HELD(&saa_statep->saa_port_list_mutex));
581 
582 	/* create & initialize new port */
583 	saa_portp = kmem_zalloc(sizeof (saa_port_t), KM_NOSLEEP);
584 
585 	if (saa_portp == NULL) {
586 
587 		IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
588 		    ibmf_sa_session_open_err, IBMF_TNF_ERROR, "",
589 		    "ibmf_saa_impl_create_port: %s\n",
590 		    tnf_string, msg, "could not allocate memory for "
591 		    "new port");
592 
593 		status = IBMF_NO_MEMORY;
594 		goto bail;
595 	}
596 
597 	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_sa_session_open,
598 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port: %s\n",
599 	    tnf_string, msg, "first client registering, initializing");
600 
601 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_portp))
602 
603 	/* tell everyone that kstats are not initialized */
604 	saa_portp->saa_pt_kstatp = NULL;
605 
606 	/*
607 	 * set up mutexe and state variable to indicate to
608 	 * other clients that were currently in the process of
609 	 * setting up the port data.  This will prevent a subsequent
610 	 * client from trying to to register with ibmf before the
611 	 * port data has been initialized.
612 	 */
613 	mutex_init(&saa_portp->saa_pt_mutex, NULL, MUTEX_DRIVER, NULL);
614 	cv_init(&saa_portp->saa_pt_ibmf_reg_cv, NULL, CV_DRIVER, NULL);
615 
616 	saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_REGISTERING;
617 
618 	/* create other mutexes */
619 	mutex_init(&saa_portp->saa_pt_kstat_mutex, NULL, MUTEX_DRIVER, NULL);
620 
621 	mutex_init(&saa_portp->saa_pt_event_sub_mutex, NULL, MUTEX_DRIVER,
622 	    NULL);
623 
624 	/*
625 	 * clients assume all arrive; set mask to this so we only notify
626 	 * if something failed
627 	 */
628 	saa_portp->saa_pt_event_sub_last_success_mask =
629 	    IBMF_SAA_PORT_EVENT_SUB_ALL_ARRIVE;
630 
631 	/*
632 	 * set port_guid now so any immediately subsequent clients
633 	 * registering on this port, guid will know we're already here
634 	 */
635 	saa_portp->saa_pt_port_guid = pt_guid;
636 	saa_portp->saa_pt_reference_count = 1;
637 	saa_portp->saa_pt_current_tid = pt_guid << 32;
638 
639 	saa_portp->saa_pt_redirect_active = B_FALSE;
640 
641 	/* set sa_uptime now in case we never receive anything from SA */
642 	saa_portp->saa_pt_sa_uptime = gethrtime();
643 
644 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*saa_portp))
645 
646 	/* Set new pointer in caller's */
647 	*saa_portpp = saa_portp;
648 
649 bail:
650 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_create_port_end,
651 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port() exit\n");
652 
653 	return (status);
654 }
655 
656 /*
657  * ibmf_saa_impl_invalidate_port:
658  * invalidates port entry (assumes exist) and deletes kstat object
659  * kstat object is destroyed in order to allow creating port entry
660  * even if this entry is not purged
661  */
662 static void
ibmf_saa_impl_invalidate_port(saa_port_t * saa_portp)663 ibmf_saa_impl_invalidate_port(saa_port_t *saa_portp)
664 {
665 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
666 	    ibmf_saa_impl_invalidate_port_start,
667 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_invalidate_port() enter\n");
668 
669 	ASSERT(saa_portp != NULL);
670 	ASSERT(MUTEX_HELD(&saa_portp->saa_pt_mutex));
671 
672 	saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_INVALID;
673 	ibmf_saa_impl_uninit_kstats(saa_portp);
674 
675 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
676 	    ibmf_saa_impl_invalidate_port_end,
677 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_invalidate_port() exit\n");
678 }
679 
680 /*
681  * ibmf_saa_impl_destroy_port:
682  * Frees the resources associated with an saa_portp structure.  Assumes the
683  * saa_portp exists
684  *
685  * Input Arguments
686  * saa_portp		pointer to saa_portp structure
687  *
688  * Output Arguments
689  * none
690  *
691  * Returns
692  * void
693  */
694 static void
ibmf_saa_impl_destroy_port(saa_port_t * saa_portp)695 ibmf_saa_impl_destroy_port(saa_port_t *saa_portp)
696 {
697 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_destroy_start,
698 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_destroy() enter\n");
699 
700 	ASSERT(saa_portp != NULL);
701 
702 	_NOTE(ASSUMING_PROTECTED(*saa_portp))
703 
704 	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
705 	    ibmf_saa_impl_destroy, IBMF_TNF_TRACE, "",
706 	    "ibmf_saa_impl_destroy(): destroying port_guid %016" PRIx64 "\n",
707 	    tnf_opaque, port_guid, saa_portp->saa_pt_port_guid);
708 
709 	ibmf_saa_impl_uninit_kstats(saa_portp);
710 
711 	/* uninit synchronization variables used for registration */
712 	mutex_destroy(&saa_portp->saa_pt_mutex);
713 	cv_destroy(&saa_portp->saa_pt_ibmf_reg_cv);
714 
715 	mutex_destroy(&saa_portp->saa_pt_event_sub_mutex);
716 	mutex_destroy(&saa_portp->saa_pt_kstat_mutex);
717 
718 	kmem_free(saa_portp, sizeof (saa_port_t));
719 
720 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_destroy_end,
721 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_destroy() exit\n");
722 }
723 
724 /*
725  * ibmf_saa_impl_init_kstats:
726  * Create kstats structure.  Should be called when memory is alloced for a new
727  * port entry.
728  */
729 int
ibmf_saa_impl_init_kstats(saa_port_t * saa_portp)730 ibmf_saa_impl_init_kstats(saa_port_t *saa_portp)
731 {
732 	char			buf[128];
733 	ibmf_saa_kstat_t	*ksp;
734 
735 	_NOTE(ASSUMING_PROTECTED(saa_portp->saa_pt_kstatp))
736 
737 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
738 	    ibmf_saa_impl_init_kstats_start,
739 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_kstats() enter\n");
740 
741 	/* set up kstats structure */
742 	(void) sprintf(buf, "ibmf_saa_%016" PRIx64 "_stat",
743 	    saa_portp->saa_pt_port_guid);
744 
745 	saa_portp->saa_pt_kstatp = kstat_create("ibmf_saa",
746 	    0, buf, "misc", KSTAT_TYPE_NAMED,
747 	    sizeof (ibmf_saa_kstat_t) / sizeof (kstat_named_t),
748 	    KSTAT_FLAG_WRITABLE);
749 
750 	if (saa_portp->saa_pt_kstatp == NULL)
751 		return (IBMF_NO_RESOURCES);
752 
753 	ksp = (ibmf_saa_kstat_t *)saa_portp->saa_pt_kstatp->ks_data;
754 
755 	kstat_named_init(&ksp->clients_registered,
756 	    "clients_registered", KSTAT_DATA_UINT32);
757 
758 	kstat_named_init(&ksp->clients_reg_failed,
759 	    "clients_reg_failed", KSTAT_DATA_UINT32);
760 
761 	kstat_named_init(&ksp->outstanding_requests,
762 	    "outstanding_requests", KSTAT_DATA_UINT32);
763 
764 	kstat_named_init(&ksp->total_requests,
765 	    "total_requests", KSTAT_DATA_UINT32);
766 
767 	kstat_named_init(&ksp->failed_requests,
768 	    "failed_requests", KSTAT_DATA_UINT32);
769 
770 	kstat_named_init(&ksp->requests_timedout,
771 	    "requests_timedout", KSTAT_DATA_UINT32);
772 
773 	kstat_install(saa_portp->saa_pt_kstatp);
774 
775 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
776 	    ibmf_saa_impl_init_kstats_end,
777 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_kstats() exit\n");
778 
779 	return (IBMF_SUCCESS);
780 }
781 
782 /*
783  * ibmf_saa_impl_uninit_kstats:
784  * Free kstats context.  Should be called when port is either destroyed
785  * or invalidated.
786  */
787 static void
ibmf_saa_impl_uninit_kstats(saa_port_t * saa_portp)788 ibmf_saa_impl_uninit_kstats(saa_port_t *saa_portp)
789 {
790 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
791 	    ibmf_saa_impl_uninit_kstats_start,
792 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_uninit_kstats() enter\n");
793 
794 	mutex_enter(&saa_portp->saa_pt_kstat_mutex);
795 
796 	if (saa_portp->saa_pt_kstatp != NULL) {
797 		kstat_delete(saa_portp->saa_pt_kstatp);
798 	}
799 	saa_portp->saa_pt_kstatp = NULL;
800 
801 	mutex_exit(&saa_portp->saa_pt_kstat_mutex);
802 
803 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
804 	    ibmf_saa_impl_uninit_kstats_end,
805 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_uninit_kstats() exit\n");
806 }
807 
808 /*
809  * ibmf_saa_impl_register_failed:
810  * invalidate entry and kick waiters
811  */
812 void
ibmf_saa_impl_register_failed(saa_port_t * saa_portp)813 ibmf_saa_impl_register_failed(saa_port_t *saa_portp)
814 {
815 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
816 	    ibmf_saa_impl_register_failed_start,
817 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_failed() enter\n");
818 
819 	mutex_enter(&saa_portp->saa_pt_mutex);
820 
821 	ibmf_saa_impl_invalidate_port(saa_portp);
822 
823 	cv_broadcast(&saa_portp->saa_pt_ibmf_reg_cv);
824 
825 	/* decrementing refcount is last thing we do on entry */
826 
827 	ASSERT(saa_portp->saa_pt_reference_count > 0);
828 	saa_portp->saa_pt_reference_count--;
829 
830 	mutex_exit(&saa_portp->saa_pt_mutex);
831 
832 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
833 	    ibmf_saa_impl_register_failed_end,
834 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_failed() exit\n");
835 }
836 
837 static int
ibmf_saa_impl_setup_qp_async_cb(saa_port_t * saa_portp,int setup_async_cb_only)838 ibmf_saa_impl_setup_qp_async_cb(saa_port_t *saa_portp, int setup_async_cb_only)
839 {
840 	int		status;
841 	int		unreg_status;
842 	ib_pkey_t	p_key;
843 	ib_qkey_t	q_key;
844 	uint8_t		portnum;
845 	boolean_t	qp_alloced = B_FALSE;
846 
847 	if (setup_async_cb_only == 0) {
848 
849 		/* allocate a qp through ibmf */
850 		status = ibmf_alloc_qp(saa_portp->saa_pt_ibmf_handle,
851 		    IB_PKEY_DEFAULT_LIMITED, IB_GSI_QKEY,
852 		    IBMF_ALT_QP_MAD_RMPP, &saa_portp->saa_pt_qp_handle);
853 
854 		if (status != IBMF_SUCCESS) {
855 
856 			IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
857 			    ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "",
858 			    "ibmf_saa_impl_setup_qp_async_cb: %s, "
859 			    "ibmf_status = %d\n",
860 			    tnf_string, msg, "Cannot alloc qp with ibmf",
861 			    tnf_int, status, status);
862 
863 			return (status);
864 		}
865 
866 		qp_alloced = B_TRUE;
867 
868 		/*
869 		 * query the queue pair number; we will need it to unsubscribe
870 		 * from notice reports
871 		 */
872 		status = ibmf_query_qp(saa_portp->saa_pt_ibmf_handle,
873 		    saa_portp->saa_pt_qp_handle, &saa_portp->saa_pt_qpn,
874 		    &p_key, &q_key, &portnum, 0);
875 
876 		if (status != IBMF_SUCCESS) {
877 
878 			IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
879 			    ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "",
880 			    "ibmf_saa_impl_setup_qp_async_cb: %s, "
881 			    "ibmf_status = %d\n",
882 			    tnf_string, msg,
883 			    "Cannot query alt qp to get qp num",
884 			    tnf_int, status, status);
885 
886 			goto bail;
887 		}
888 	}
889 
890 	/*
891 	 * core ibmf is taking advantage of the fact that saa_portp is our
892 	 * callback arg. If this changes, the code in ibmf_recv would need to
893 	 * change as well
894 	 */
895 	status = ibmf_setup_async_cb(saa_portp->saa_pt_ibmf_handle,
896 	    saa_portp->saa_pt_qp_handle, ibmf_saa_report_cb, saa_portp, 0);
897 	if (status != IBMF_SUCCESS) {
898 
899 		IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
900 		    ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "",
901 		    "ibmf_saa_impl_setup_qp_async_cb: %s, ibmf_status = %d\n",
902 		    tnf_string, msg, "Cannot register async cb with ibmf",
903 		    tnf_int, status, status);
904 
905 		goto bail;
906 	}
907 
908 	return (IBMF_SUCCESS);
909 
910 bail:
911 	if (qp_alloced == B_TRUE) {
912 		/* free alternate qp */
913 		unreg_status = ibmf_free_qp(saa_portp->saa_pt_ibmf_handle,
914 		    &saa_portp->saa_pt_qp_handle, 0);
915 		if (unreg_status != IBMF_SUCCESS) {
916 
917 			IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
918 			    ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "",
919 			    "ibmf_saa_impl_setup_qp_async_cb: %s, ibmf_status ="
920 			    " %d\n", tnf_string, msg,
921 			    "Cannot free alternate queue pair with ibmf",
922 			    tnf_int, unreg_status, unreg_status);
923 		}
924 	}
925 
926 	return (status);
927 }
928 
929 /*
930  * ibmf_saa_impl_register_port:
931  */
932 int
ibmf_saa_impl_register_port(saa_port_t * saa_portp)933 ibmf_saa_impl_register_port(
934 	saa_port_t *saa_portp)
935 {
936 	uint_t		hca_count	= 0;
937 	ib_guid_t	*hca_list 	= NULL;
938 	int		status 		= IBMF_SUCCESS;
939 	int		unreg_status 	= IBMF_SUCCESS;
940 	int		ibt_status	= IBT_SUCCESS;
941 	ibt_hca_portinfo_t *port_info_list = NULL;
942 	uint_t		port_count	= 0;
943 	uint_t		port_size	= 0;
944 	int		ihca, iport;
945 	ib_guid_t	port_guid;
946 	boolean_t	ibmf_reg = B_FALSE;
947 
948 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
949 	    ibmf_saa_impl_register_port_start, IBMF_TNF_TRACE, "",
950 	    "ibmf_saa_impl_register_port() enter\n");
951 
952 	ASSERT(saa_portp != NULL);
953 
954 	_NOTE(ASSUMING_PROTECTED(*saa_portp))
955 
956 	/* get the HCA list */
957 
958 	hca_count = ibt_get_hca_list(&hca_list);
959 
960 	if (hca_count == 0) {
961 
962 		IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
963 		    ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
964 		    "ibmf_saa_impl_register_port: %s\n",
965 		    tnf_string, msg, "cannot register port (no HCAs).\n");
966 
967 		status = IBMF_BAD_PORT;
968 		goto bail;
969 	}
970 
971 	/* lookup requested port guid in hca list */
972 	for (ihca = 0; ihca != hca_count; ihca++) {
973 
974 		ibt_status = ibt_query_hca_ports_byguid(hca_list[ihca],
975 		    0 /* all ports */, &port_info_list,
976 		    &port_count, &port_size);
977 
978 		if (ibt_status != IBT_SUCCESS) {
979 
980 			IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
981 			    ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
982 			    "ibmf_saa_impl_register_port: %s, %016" PRIx64 "\n",
983 			    tnf_string, msg, "Could not query hca.  Exiting.",
984 			    tnf_opaque, guid, hca_list[ihca]);
985 
986 			status = IBMF_TRANSPORT_FAILURE;
987 			break;
988 		}
989 
990 		for (iport = 0; iport < port_count; iport++) {
991 
992 			/* get port guid associated with hca guid, port num */
993 			if (ibmf_saa_impl_get_port_guid(
994 			    port_info_list + iport, &port_guid) != IBMF_SUCCESS)
995 				continue;
996 
997 			if (saa_portp->saa_pt_port_guid != port_guid)
998 				continue;
999 
1000 			IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3,
1001 			    ibmf_saa_impl_register_port,
1002 			    IBMF_TNF_TRACE, "",
1003 			    "ibmf_saa_impl_register_port: %s, hca_guid = %016"
1004 			    PRIx64 ", port_guid = %016" PRIx64
1005 			    ", number = %d\n",
1006 			    tnf_string, msg, "found port",
1007 			    tnf_opaque, hca_guid, hca_list[ihca],
1008 			    tnf_opaque, port_guid, port_guid,
1009 			    tnf_uint,   port, iport + 1);
1010 
1011 			/*
1012 			 * we're here? then we found our port:
1013 			 * fill in ibmf registration info
1014 			 * and address parameters from the portinfo
1015 			 */
1016 
1017 			saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid
1018 			    = hca_list[ihca];
1019 			saa_portp->saa_pt_ibmf_reginfo.ir_port_num = iport+1;
1020 			saa_portp->saa_pt_ibmf_reginfo.ir_client_class
1021 			    = SUBN_ADM_MANAGER;
1022 
1023 			saa_portp->saa_pt_node_guid = hca_list[ihca];
1024 			saa_portp->saa_pt_port_num = iport + 1;
1025 
1026 			ibmf_saa_impl_set_transaction_params(
1027 			    saa_portp, port_info_list + iport);
1028 			break;
1029 		}
1030 
1031 		ibt_free_portinfo(port_info_list, port_size);
1032 
1033 		if (iport != port_count)
1034 			break;	/* found our port */
1035 	}
1036 
1037 	ibt_free_hca_list(hca_list, hca_count);
1038 
1039 	if (ihca == hca_count) {
1040 
1041 		IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1042 		    ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
1043 		    "ibmf_saa_impl_register_port: %s, port_guid %016"
1044 		    PRIx64 "\n",
1045 		    tnf_string, msg, "Could not find port,  exiting",
1046 		    tnf_opaque, port_guid, saa_portp->saa_pt_port_guid);
1047 
1048 		status = IBMF_BAD_PORT;
1049 	}
1050 
1051 	if (status != IBMF_SUCCESS) {
1052 
1053 		goto bail;
1054 	}
1055 
1056 	/*
1057 	 * Now we found the port we searched for,
1058 	 * and open an ibmf session on that port.
1059 	 */
1060 
1061 	IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3,
1062 	    ibmf_saa_impl_register_port, IBMF_TNF_TRACE, "",
1063 	    "ibmf_saa_impl_register_port: %s, port_guid = %016" PRIx64
1064 	    ", port = %d\n", tnf_string, msg, "Registering with ibmf",
1065 	    tnf_opaque, port_guid, saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid,
1066 	    tnf_uint, port, saa_portp->saa_pt_ibmf_reginfo.ir_port_num);
1067 
1068 	status = ibmf_register(&saa_portp->saa_pt_ibmf_reginfo,
1069 	    IBMF_VERSION, IBMF_REG_FLAG_RMPP,
1070 	    ibmf_saa_impl_async_event_cb, saa_portp,
1071 	    &saa_portp->saa_pt_ibmf_handle,
1072 	    &saa_portp->saa_pt_ibmf_impl_features);
1073 
1074 	if (status != IBMF_SUCCESS) {
1075 
1076 		IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1077 		    ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
1078 		    "ibmf_saa_impl_register_port: %s, ibmf_status = %d\n",
1079 		    tnf_string, msg, "Could not register with ibmf",
1080 		    tnf_int, status, status);
1081 
1082 		goto bail;
1083 	}
1084 
1085 	ibmf_reg = B_TRUE;
1086 
1087 	if (ibmf_saa_impl_setup_qp_async_cb(saa_portp, 0) == IBMF_SUCCESS)
1088 		return (IBMF_SUCCESS);
1089 
1090 bail:
1091 	if (ibmf_reg == B_TRUE) {
1092 		/* unregister from ibmf */
1093 		unreg_status = ibmf_unregister(
1094 		    &saa_portp->saa_pt_ibmf_handle, 0);
1095 
1096 		if (unreg_status != IBMF_SUCCESS) {
1097 
1098 			IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1099 			    ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
1100 			    "ibmf_saa_impl_register_port: %s, ibmf_status ="
1101 			    " %d\n", tnf_string, msg,
1102 			    "Cannot unregister from ibmf",
1103 			    tnf_int, unreg_status, unreg_status);
1104 		}
1105 	}
1106 
1107 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_register_port_end,
1108 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_port() exit\n");
1109 
1110 	return (status);
1111 }
1112 
1113 /*
1114  * ibmf_saa_impl_getclassportinfo:
1115  */
1116 void
ibmf_saa_impl_get_classportinfo(saa_port_t * saa_portp)1117 ibmf_saa_impl_get_classportinfo(saa_port_t *saa_portp)
1118 {
1119 	int			res;
1120 	saa_impl_trans_info_t	*trans_info;
1121 
1122 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1123 	    ibmf_saa_impl_get_classportinfo_start,
1124 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_classportinfo() enter\n");
1125 
1126 	/*
1127 	 * allocate memory for trans_info; send_request's callback will free up
1128 	 * memory since request is asynchronous
1129 	 */
1130 	trans_info = kmem_zalloc(sizeof (saa_impl_trans_info_t), KM_NOSLEEP);
1131 	if (trans_info == NULL) {
1132 
1133 		mutex_enter(&saa_portp->saa_pt_mutex);
1134 
1135 		/* cpi transaction is handled as a client, decrement refcount */
1136 		ASSERT(saa_portp->saa_pt_reference_count > 0);
1137 		saa_portp->saa_pt_reference_count--;
1138 
1139 		mutex_exit(&saa_portp->saa_pt_mutex);
1140 
1141 		IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
1142 		    ibmf_saa_impl_get_classportinfo_err, IBMF_TNF_ERROR, "",
1143 		    "ibmf_saa_impl_get_classportinfo: %s\n", tnf_string, msg,
1144 		    "Could not allocate memory for classportinfo trans_info");
1145 
1146 		IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1147 		    ibmf_saa_impl_get_classportinfo_end, IBMF_TNF_TRACE, "",
1148 		    "ibmf_saa_impl_get_classportinfo() exiting\n");
1149 
1150 		return;
1151 	}
1152 
1153 	/* no specific client associated with this transaction */
1154 	trans_info->si_trans_client_data = NULL;
1155 	trans_info->si_trans_port	 = saa_portp;
1156 	trans_info->si_trans_method	 = SA_SUBN_ADM_GET;
1157 	trans_info->si_trans_attr_id	 = MAD_ATTR_ID_CLASSPORTINFO;
1158 
1159 	trans_info->si_trans_callback = ibmf_saa_impl_get_cpi_cb;
1160 	trans_info->si_trans_callback_arg = saa_portp;
1161 
1162 	mutex_enter(&saa_portp->saa_pt_kstat_mutex);
1163 
1164 	IBMF_SAA_ADD32_KSTATS(saa_portp, outstanding_requests, 1);
1165 	IBMF_SAA_ADD32_KSTATS(saa_portp, total_requests, 1);
1166 
1167 	mutex_exit(&saa_portp->saa_pt_kstat_mutex);
1168 
1169 	res = ibmf_saa_impl_send_request(trans_info);
1170 
1171 	if (res != IBMF_SUCCESS) {
1172 
1173 		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
1174 		    ibmf_saa_impl_get_classportinfo_err, IBMF_TNF_TRACE, "",
1175 		    "ibmf_saa_impl_get_classportinfo: %s, res = 0x%x\n",
1176 		    tnf_string, msg, "ibmf_saa_impl_send_request failed",
1177 		    tnf_opaque, res, res);
1178 
1179 		mutex_enter(&saa_portp->saa_pt_kstat_mutex);
1180 
1181 		IBMF_SAA_SUB32_KSTATS(saa_portp, outstanding_requests, 1);
1182 		IBMF_SAA_ADD32_KSTATS(saa_portp, failed_requests, 1);
1183 
1184 		mutex_exit(&saa_portp->saa_pt_kstat_mutex);
1185 
1186 		mutex_enter(&saa_portp->saa_pt_mutex);
1187 
1188 		/* cpi transaction is handled as a client, decrement refcount */
1189 		ASSERT(saa_portp->saa_pt_reference_count > 0);
1190 		saa_portp->saa_pt_reference_count--;
1191 
1192 		mutex_exit(&saa_portp->saa_pt_mutex);
1193 
1194 	}
1195 
1196 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1197 	    ibmf_saa_impl_get_classportinfo_end,
1198 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_classportinfo() exit\n");
1199 }
1200 
1201 /*
1202  * ibmf_saa_impl_get_cpi_cb:
1203  *
1204  * Called when the asynchronous getportinfo request receives its response.
1205  * Checks the status.  If success, updates the times in the port's
1206  * ibmf_retrans structure that is used in ibmf_msg_transport calls.  If failure,
1207  * just use default values.
1208  *
1209  * Input Arguments
1210  * arg		user-specified pointer (points to the current port data)
1211  * length	length of payload returned (should be size of classportinfo_t)
1212  * buffer	pointer to classportinfo returned (should not be null)
1213  * status	status of sa access request
1214  *
1215  * Output Arguments
1216  * none
1217  *
1218  * Returns void
1219  */
1220 static void
ibmf_saa_impl_get_cpi_cb(void * arg,size_t length,char * buffer,int status)1221 ibmf_saa_impl_get_cpi_cb(void *arg, size_t length, char *buffer, int status)
1222 {
1223 	saa_port_t		*saa_portp;
1224 	uint64_t		base_time, resp_timeout, rttv_timeout;
1225 	ib_mad_classportinfo_t	*classportinfo;
1226 	int			resp_time_value;
1227 	uint16_t		sa_cap_mask;
1228 
1229 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_get_cpi_cb_start,
1230 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_cpi_cb() enter\n");
1231 
1232 	/*
1233 	 * access port entry: note that it may have become invalid
1234 	 * but we hold a ref count for cpi and the interactions on
1235 	 * the entry are harmless
1236 	 */
1237 	saa_portp = (saa_port_t *)arg;
1238 
1239 	/* process response */
1240 
1241 	if ((status != IBMF_SUCCESS) || (buffer == NULL)) {
1242 
1243 		IBMF_TRACE_4(IBMF_TNF_NODEBUG, DPRINT_L1,
1244 		    ibmf_saa_impl_get_cpi_cb, IBMF_TNF_ERROR, "",
1245 		    "ibmf_saa_impl_get_cpi_cb: %s, status = %d, buffer = "
1246 		    " 0x%p, length = %d\n", tnf_string, msg,
1247 		    "could not get classportinfo.  Check node and path to sm"
1248 		    " lid", tnf_int, status, status,
1249 		    tnf_opaque, buffer, buffer, tnf_uint, length, length);
1250 
1251 		/*
1252 		 * IB spec (C13-13) indicates 20 can be used as default or
1253 		 * intial value for classportinfo->resptimeout value
1254 		 */
1255 		resp_time_value = 20;
1256 
1257 		sa_cap_mask = 0xFFFF;
1258 
1259 	} else if (buffer != NULL) {
1260 
1261 		classportinfo = (ib_mad_classportinfo_t *)buffer;
1262 
1263 		resp_time_value = classportinfo->RespTimeValue & 0x1f;
1264 
1265 		/*
1266 		 * Because some subnet managers might not provide sane
1267 		 * value for "resp_time_value", we limit it here.  In
1268 		 * case this limit is too restrictive (very large fabric),
1269 		 * we allow the limit to be raised (/etc/system).
1270 		 */
1271 		if (resp_time_value > ibmf_saa_max_resp_time) {
1272 			cmn_err(CE_CONT, "!ibmf_saa_max_resp_time (%d) "
1273 			    "exceeded.", ibmf_saa_max_resp_time);
1274 			cmn_err(CE_CONT, "!Reducing subnet administrator "
1275 			    "resp_time value from %d to %d.",
1276 			    resp_time_value, ibmf_saa_max_resp_time);
1277 			resp_time_value = ibmf_saa_max_resp_time;
1278 		}
1279 
1280 		sa_cap_mask = classportinfo->CapabilityMask;
1281 
1282 		IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3,
1283 		    ibmf_saa_impl_get_cpi_cb, IBMF_TNF_TRACE, "",
1284 		    "ibmf_saa_impl_get_cpi_cb: %s, timeout = 0x%x,"
1285 		    " cap_mask = 0x%x\n",
1286 		    tnf_string, msg, "got classportinfo",
1287 		    tnf_opaque, timeout, resp_time_value,
1288 		    tnf_opaque, cap_mask, sa_cap_mask);
1289 
1290 		kmem_free(buffer, length);
1291 	}
1292 
1293 	/*
1294 	 * using IB spec calculation from 13.4.6.2
1295 	 * use bit shifting for 2^x.
1296 	 */
1297 	base_time = (1 << resp_time_value);
1298 
1299 	resp_timeout = (4 * base_time * 1000 + 96 * base_time) / 1000;
1300 
1301 	mutex_enter(&saa_portp->saa_pt_mutex);
1302 
1303 	base_time = 2 * (1 << saa_portp->saa_pt_timeout);
1304 
1305 	rttv_timeout = (4 * base_time * 1000 + 96 * base_time) / 1000;
1306 
1307 	saa_portp->saa_pt_ibmf_retrans.retrans_rtv = resp_timeout;
1308 	saa_portp->saa_pt_ibmf_retrans.retrans_rttv = rttv_timeout;
1309 	saa_portp->saa_pt_sa_cap_mask = sa_cap_mask;
1310 
1311 	/*
1312 	 * cpi transaction is handled as a client,
1313 	 * decrement refcount; make sure it's the last
1314 	 * thing we do on this entry
1315 	 */
1316 	ASSERT(saa_portp->saa_pt_reference_count > 0);
1317 	saa_portp->saa_pt_reference_count--;
1318 
1319 	IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3,
1320 	    ibmf_saa_impl_get_cpi_cb, IBMF_TNF_TRACE, "",
1321 	    "ibmf_saa_impl_get_cpi_cb: %s, subnet_timeout = 0x%x, "
1322 	    "resp_time_value = 0x%x\n",
1323 	    tnf_string, msg, "updated resp timeout",
1324 	    tnf_opaque, subnet_timeout, saa_portp->saa_pt_timeout,
1325 	    tnf_opaque, resp_time_value, resp_time_value);
1326 
1327 	mutex_exit(&saa_portp->saa_pt_mutex);
1328 
1329 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_get_cpi_cb_end,
1330 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_cpi_cb() exit\n");
1331 }
1332 
1333 /*
1334  * ibmf_saa_impl_send_request:
1335  * Sends a request to the sa.  Can be used for both classportinfo and record
1336  * requests.  Will set up all data structures for using the multi-packet
1337  * protocol, create the mad, and send it.  Returns SA_SUCCESS if msg transport
1338  * worked, meaning succesful send for the async case and a succesful send and
1339  * recv for the sync case.
1340  */
1341 int
ibmf_saa_impl_send_request(saa_impl_trans_info_t * trans_info)1342 ibmf_saa_impl_send_request(saa_impl_trans_info_t *trans_info)
1343 {
1344 	uint16_t 		attr_id;
1345 	saa_client_data_t	*client_data;
1346 	saa_port_t		*saa_portp;
1347 	uint32_t		transport_flags;
1348 	ibmf_msg_cb_t		ibmf_callback;
1349 	void			*ibmf_callback_arg;
1350 	ibmf_msg_t		*msgp;
1351 	ibmf_retrans_t		ibmf_retrans;
1352 	uint16_t		sa_cap_mask;
1353 	boolean_t		sleep_flag;
1354 	int			ibmf_status = IBMF_SUCCESS;
1355 	int			retry_count;
1356 	uint16_t		mad_status;
1357 	boolean_t		sa_is_redirected = B_FALSE;
1358 
1359 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1360 	    ibmf_saa_impl_send_request_start,
1361 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_send_request() enter\n");
1362 
1363 	attr_id = trans_info->si_trans_attr_id;
1364 	client_data = trans_info->si_trans_client_data;
1365 	saa_portp   = trans_info->si_trans_port;
1366 
1367 	/*
1368 	 * don't send on invalid entry
1369 	 * Note that there is a window where it could become
1370 	 * invalid after this test is done, but we'd rely on ibmf errors...
1371 	 */
1372 	if (ibmf_saa_is_valid(saa_portp, B_FALSE) == B_FALSE) {
1373 
1374 		IBMF_TRACE_4(IBMF_TNF_NODEBUG, DPRINT_L1,
1375 		    ibmf_saa_impl_send_request,
1376 		    IBMF_TNF_ERROR, "",
1377 		    "ibmf_saa_impl_send_request: %s, hca_guid = %016"
1378 		    PRIx64 ", port_guid = %016" PRIx64
1379 		    ", number = %d\n",
1380 		    tnf_string, msg, "sending on invalid port",
1381 		    tnf_opaque, hca_guid,
1382 		    saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid,
1383 		    tnf_opaque, port_guid,
1384 		    saa_portp->saa_pt_port_guid,
1385 		    tnf_uint,   port,
1386 		    saa_portp->saa_pt_ibmf_reginfo.ir_port_num);
1387 
1388 		ibmf_status = IBMF_REQ_INVALID;
1389 		goto bail;
1390 	}
1391 
1392 	/* check whether SA supports this attribute */
1393 	mutex_enter(&saa_portp->saa_pt_mutex);
1394 
1395 	sa_cap_mask = saa_portp->saa_pt_sa_cap_mask;
1396 	sa_is_redirected = saa_portp->saa_pt_redirect_active;
1397 
1398 	mutex_exit(&saa_portp->saa_pt_mutex);
1399 
1400 	ibmf_status = ibmf_saa_impl_check_sa_support(sa_cap_mask, attr_id);
1401 
1402 	if (ibmf_status != IBMF_SUCCESS) {
1403 
1404 		IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1405 		    ibmf_saa_impl_send_request_err, IBMF_TNF_ERROR, "",
1406 		    "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n",
1407 		    tnf_string, msg, "SA does not support attribute",
1408 		    tnf_int, ibmf_status, ibmf_status);
1409 
1410 		goto bail;
1411 	}
1412 
1413 	/* make only non-blocking calls if this is an async request */
1414 	if ((trans_info->si_trans_callback == NULL) &&
1415 	    (trans_info->si_trans_sub_callback == NULL)) {
1416 		ibmf_callback = NULL;
1417 		ibmf_callback_arg = NULL;
1418 		sleep_flag = B_TRUE;
1419 	} else {
1420 		ibmf_callback = ibmf_saa_async_cb;
1421 		ibmf_callback_arg = (void *)trans_info;
1422 		sleep_flag = B_FALSE;
1423 	}
1424 
1425 	ibmf_status = ibmf_saa_impl_init_msg(trans_info, sleep_flag, &msgp,
1426 	    &transport_flags, &ibmf_retrans);
1427 	if (ibmf_status != IBMF_SUCCESS) {
1428 
1429 		IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1430 		    ibmf_saa_impl_send_request_err, IBMF_TNF_ERROR, "",
1431 		    "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n",
1432 		    tnf_string, msg, "init_msg() failed",
1433 		    tnf_int, ibmf_status, ibmf_status);
1434 
1435 		goto bail;
1436 	}
1437 
1438 	mutex_enter(&saa_portp->saa_pt_mutex);
1439 
1440 	saa_portp->saa_pt_num_outstanding_trans++;
1441 
1442 	mutex_exit(&saa_portp->saa_pt_mutex);
1443 
1444 	/*
1445 	 * increment the number of outstanding transaction so
1446 	 * ibmf_close_sa_session() will wait.  classportinfo requests
1447 	 * don't have associated clients so check for valid clientp
1448 	 */
1449 	if (client_data != NULL) {
1450 
1451 		mutex_enter(&client_data->saa_client_mutex);
1452 
1453 		client_data->saa_client_num_pending_trans++;
1454 
1455 		mutex_exit(&client_data->saa_client_mutex);
1456 	}
1457 
1458 	/*
1459 	 * make the call to msg_transport.  If synchronous and success,
1460 	 * check that the response mad isn't status busy.  If so, repeat the
1461 	 * call
1462 	 */
1463 	retry_count = 0;
1464 
1465 	/*
1466 	 * set the send time here. We only set this once at the beginning of
1467 	 * the transaction.  Retrying because of busys or mastersmlid changes
1468 	 * does not change the original send time.  It is meant to be an
1469 	 * absolute time out value and will only be used if there are other
1470 	 * problems (i.e. a buggy SA)
1471 	 */
1472 	trans_info->si_trans_send_time = gethrtime();
1473 
1474 	for (;;) {
1475 
1476 		ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle,
1477 		    saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
1478 		    ibmf_callback, ibmf_callback_arg, transport_flags);
1479 
1480 		if (ibmf_callback != NULL)
1481 			break;
1482 
1483 		/*
1484 		 * stop here for non-sequenced transactions since they wouldn't
1485 		 * receive a timeout or busy response
1486 		 */
1487 		if (!(transport_flags & IBMF_MSG_TRANS_FLAG_SEQ))
1488 			break;
1489 
1490 		/*
1491 		 * if the transaction timed out and this was a synchronous
1492 		 * request there's a possiblity we were talking to the wrong
1493 		 * master smlid or that the SA has stopped responding on the
1494 		 * redirected desination (if redirect is active).
1495 		 * Check this and retry if necessary.
1496 		 */
1497 		if ((ibmf_status == IBMF_TRANS_TIMEOUT) &&
1498 		    (sleep_flag == B_TRUE)) {
1499 			if (sa_is_redirected == B_TRUE) {
1500 				ibmf_status = ibmf_saa_impl_revert_to_qp1(
1501 				    saa_portp, msgp, ibmf_callback,
1502 				    ibmf_callback_arg, transport_flags);
1503 			} else {
1504 				ibmf_status = ibmf_saa_impl_new_smlid_retry(
1505 				    saa_portp, msgp, ibmf_callback,
1506 				    ibmf_callback_arg, transport_flags);
1507 			}
1508 		}
1509 
1510 		/*
1511 		 * if the transaction timed out (and retrying with a new SM LID
1512 		 * didn't help) check how long it's been since we received an SA
1513 		 * packet.  If it hasn't been max_wait_time then retry the
1514 		 * request.
1515 		 */
1516 		if ((ibmf_status == IBMF_TRANS_TIMEOUT) &&
1517 		    (sleep_flag == B_TRUE)) {
1518 
1519 			ibmf_status = ibmf_saa_check_sa_and_retry(
1520 			    saa_portp, msgp, ibmf_callback, ibmf_callback_arg,
1521 			    trans_info->si_trans_send_time, transport_flags);
1522 		}
1523 
1524 		if (ibmf_status != IBMF_SUCCESS)
1525 			break;
1526 
1527 		if (retry_count >= IBMF_SAA_MAX_BUSY_RETRY_COUNT)
1528 			break;
1529 
1530 		/* sync transaction with status SUCCESS should have response */
1531 		ASSERT(msgp->im_msgbufs_recv.im_bufs_mad_hdr != NULL);
1532 
1533 		mad_status = b2h16(msgp->im_msgbufs_recv.
1534 		    im_bufs_mad_hdr->Status);
1535 
1536 		if ((mad_status != MAD_STATUS_BUSY) &&
1537 		    (mad_status != MAD_STATUS_REDIRECT_REQUIRED))
1538 			break;
1539 
1540 		if (mad_status == MAD_STATUS_REDIRECT_REQUIRED) {
1541 
1542 			IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
1543 			    ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1544 			    "ibmf_saa_impl_send_request: %s, retry_count %d\n",
1545 			    tnf_string, msg,
1546 			    "response returned redirect status",
1547 			    tnf_int, retry_count, retry_count);
1548 
1549 			/* update address info and copy it into msgp */
1550 			ibmf_saa_impl_update_sa_address_info(saa_portp, msgp);
1551 		} else {
1552 			IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
1553 			    ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1554 			    "ibmf_saa_impl_send_request: %s, retry_count %d\n",
1555 			    tnf_string, msg, "response returned busy status",
1556 			    tnf_int, retry_count, retry_count);
1557 		}
1558 
1559 		retry_count++;
1560 
1561 		/*
1562 		 * since this is a blocking call, sleep for some time
1563 		 * to allow SA to transition from busy state (if busy)
1564 		 */
1565 		if (mad_status == MAD_STATUS_BUSY)
1566 			delay(drv_usectohz(
1567 			    IBMF_SAA_BUSY_RETRY_SLEEP_SECS * 1000000));
1568 	}
1569 
1570 	if (ibmf_status != IBMF_SUCCESS) {
1571 
1572 		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
1573 		    ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1574 		    "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n",
1575 		    tnf_string, msg, "ibmf_msg_transport() failed",
1576 		    tnf_int, ibmf_status, ibmf_status);
1577 
1578 		ibmf_saa_impl_free_msg(saa_portp->saa_pt_ibmf_handle, msgp);
1579 
1580 		mutex_enter(&saa_portp->saa_pt_mutex);
1581 
1582 		ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0);
1583 		saa_portp->saa_pt_num_outstanding_trans--;
1584 
1585 		mutex_exit(&saa_portp->saa_pt_mutex);
1586 
1587 		if (client_data != NULL) {
1588 
1589 			mutex_enter(&client_data->saa_client_mutex);
1590 
1591 			ASSERT(client_data->saa_client_num_pending_trans > 0);
1592 			client_data->saa_client_num_pending_trans--;
1593 
1594 			if ((client_data->saa_client_num_pending_trans == 0) &&
1595 			    (client_data->saa_client_state ==
1596 			    SAA_CLIENT_STATE_WAITING))
1597 				cv_signal(&client_data->saa_client_state_cv);
1598 
1599 			mutex_exit(&client_data->saa_client_mutex);
1600 		}
1601 
1602 	} else if (sleep_flag == B_TRUE) {
1603 
1604 		mutex_enter(&saa_portp->saa_pt_mutex);
1605 
1606 		ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0);
1607 		saa_portp->saa_pt_num_outstanding_trans--;
1608 
1609 		mutex_exit(&saa_portp->saa_pt_mutex);
1610 
1611 		IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
1612 		    ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1613 		    "ibmf_saa_impl_send_request: %s\n",
1614 		    tnf_string, msg, "Message sent and received successfully");
1615 
1616 		/* fill in response values and free the message */
1617 		ibmf_saa_impl_prepare_response(saa_portp->saa_pt_ibmf_handle,
1618 		    msgp, B_FALSE, &trans_info->si_trans_status,
1619 		    &trans_info->si_trans_result,
1620 		    &trans_info->si_trans_length, sleep_flag);
1621 
1622 		if (client_data != NULL) {
1623 			mutex_enter(&client_data->saa_client_mutex);
1624 
1625 			ASSERT(client_data->saa_client_num_pending_trans > 0);
1626 			client_data->saa_client_num_pending_trans--;
1627 
1628 			if ((client_data->saa_client_num_pending_trans == 0) &&
1629 			    (client_data->saa_client_state ==
1630 			    SAA_CLIENT_STATE_WAITING))
1631 				cv_signal(&client_data->saa_client_state_cv);
1632 
1633 			mutex_exit(&client_data->saa_client_mutex);
1634 		}
1635 	} else {
1636 
1637 		IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
1638 		    ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1639 		    "ibmf_saa_impl_send_request: %s\n",
1640 		    tnf_string, msg, "Message sent successfully");
1641 	}
1642 
1643 bail:
1644 	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
1645 	    ibmf_saa_impl_send_request_end,
1646 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_send_request() exiting"
1647 	    " ibmf_status = %d\n", tnf_int, result, ibmf_status);
1648 
1649 	return (ibmf_status);
1650 }
1651 
1652 /*
1653  * ibmf_saa_impl_init_msg:
1654  * Allocates an ibmf message and fills out the header fields and formatted data
1655  * fields.  Also sets up the correct transport_flags and retrans argument for
1656  * the message transport call based on the request information.
1657  *
1658  * Input Arguments
1659  * trans_info		saa_trans_info structure passed to send_request
1660  * sleep_flag		B_TRUE if init_msg can sleep in function calls
1661  *
1662  * Output Arguments
1663  * msgp			ibmf message that should be given to msg_transport
1664  * transport_flagsp	transport flags that should be given to msg_transport
1665  * ibmf_retrans_t	retrans parameter that should be given to msg_transport
1666  *
1667  * Returns
1668  * ibmf_status
1669  */
1670 static int
ibmf_saa_impl_init_msg(saa_impl_trans_info_t * trans_info,boolean_t sleep_flag,ibmf_msg_t ** msgp,uint32_t * transport_flagsp,ibmf_retrans_t * ibmf_retransp)1671 ibmf_saa_impl_init_msg(saa_impl_trans_info_t *trans_info, boolean_t sleep_flag,
1672     ibmf_msg_t **msgp, uint32_t *transport_flagsp,
1673     ibmf_retrans_t *ibmf_retransp)
1674 {
1675 	int			ibmf_status;
1676 	ibmf_msg_bufs_t		*req_mad;
1677 	ib_mad_hdr_t		*mad_hdr;
1678 	int			ibmf_sleep_flag, km_sleep_flag;
1679 	int 			free_res;
1680 	ib_sa_hdr_t		sa_hdr;
1681 	ibmf_msg_t		*ibmf_msg;
1682 	uint16_t 		attr_id, pack_attr_id;
1683 	uint8_t			method;
1684 	saa_client_data_t	*client_data;
1685 	saa_port_t		*saa_portp;
1686 	sa_multipath_record_t	*multipath_template;
1687 	size_t			payload_length;
1688 	uint32_t		transport_flags;
1689 
1690 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1691 	    ibmf_saa_impl_init_msg_start,
1692 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_msg() entering\n");
1693 
1694 	attr_id = trans_info->si_trans_attr_id;
1695 	method = trans_info->si_trans_method;
1696 	client_data = trans_info->si_trans_client_data;
1697 	saa_portp   = trans_info->si_trans_port;
1698 
1699 	if (sleep_flag == B_TRUE) {
1700 		ibmf_sleep_flag = IBMF_ALLOC_SLEEP;
1701 		km_sleep_flag = KM_SLEEP;
1702 	} else {
1703 		ibmf_sleep_flag = IBMF_ALLOC_NOSLEEP;
1704 		km_sleep_flag = KM_NOSLEEP;
1705 	}
1706 
1707 	ibmf_status = ibmf_alloc_msg(saa_portp->saa_pt_ibmf_handle,
1708 	    ibmf_sleep_flag, &ibmf_msg);
1709 	if (ibmf_status != IBMF_SUCCESS) {
1710 
1711 		IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1712 		    ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "",
1713 		    "ibmf_saa_impl_init_msg: %s, ibmf_status = %d\n",
1714 		    tnf_string, msg, "Cannot allocate msg_buf.",
1715 		    tnf_int, ibmf_status, ibmf_status);
1716 
1717 		goto bail;
1718 	}
1719 
1720 	req_mad = &ibmf_msg->im_msgbufs_send;
1721 
1722 	/* create a template (SA MAD) */
1723 	mad_hdr = kmem_zalloc(sizeof (ib_mad_hdr_t), km_sleep_flag);
1724 
1725 	if (mad_hdr == NULL) {
1726 
1727 		IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
1728 		    ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "",
1729 		    "ibmf_saa_impl_init_msg: %s\n",
1730 		    tnf_string, msg, "Cannot allocate mad header.");
1731 
1732 		free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle,
1733 		    &ibmf_msg);
1734 		ASSERT(free_res == IBMF_SUCCESS);
1735 
1736 		ibmf_status = IBMF_NO_MEMORY;
1737 		goto bail;
1738 	}
1739 
1740 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mad_hdr,
1741 	    *req_mad))
1742 
1743 	bzero(mad_hdr, sizeof (ib_mad_hdr_t));
1744 	mad_hdr->BaseVersion = SAA_MAD_BASE_VERSION;
1745 	mad_hdr->MgmtClass = MAD_MGMT_CLASS_SUBN_ADM;
1746 	mad_hdr->ClassVersion = SAA_MAD_CLASS_VERSION;
1747 	mad_hdr->R_Method = method;
1748 	mad_hdr->AttributeID = h2b16(attr_id);
1749 
1750 	/* attribute modifier is all Fs since RIDs are no longer used */
1751 	mad_hdr->AttributeModifier = h2b32(0xffffffff);
1752 
1753 	IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3,
1754 	    ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "",
1755 	    "ibmf_saa_impl_init_msg: %s, class = 0x%x, method = 0x%x,"
1756 	    " attr_id = 0x%x\n", tnf_string, msg, "Sending MAD",
1757 	    tnf_opaque, class, mad_hdr->MgmtClass,
1758 	    tnf_opaque, method, mad_hdr->R_Method,
1759 	    tnf_opaque, attr_id, attr_id);
1760 
1761 	bzero(&sa_hdr, sizeof (ib_sa_hdr_t));
1762 	sa_hdr.ComponentMask = trans_info->si_trans_component_mask;
1763 
1764 	if (client_data != NULL)
1765 		sa_hdr.SM_KEY = client_data->saa_client_sm_key;
1766 
1767 	/*
1768 	 * pack data for IB wire format; req_mad will have different pointers to
1769 	 * sa header and payload, mad_hdr will be the same
1770 	 */
1771 	req_mad->im_bufs_mad_hdr = mad_hdr;
1772 
1773 	ibmf_status = ibmf_saa_utils_pack_sa_hdr(&sa_hdr,
1774 	    &req_mad->im_bufs_cl_hdr, &req_mad->im_bufs_cl_hdr_len,
1775 	    km_sleep_flag);
1776 
1777 	if (ibmf_status != IBMF_SUCCESS) {
1778 
1779 		IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1780 		    ibmf_saa_impl_init_msg, IBMF_TNF_ERROR, "",
1781 		    "ibmf_saa_impl_init_msg: %s, ibmf_status = %d\n",
1782 		    tnf_string, msg, "ibmf_saa_utils_pack_sa_hdr() failed",
1783 		    tnf_int, ibmf_status, ibmf_status);
1784 
1785 		kmem_free(mad_hdr, sizeof (ib_mad_hdr_t));
1786 
1787 		free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle,
1788 		    &ibmf_msg);
1789 		ASSERT(free_res == IBMF_SUCCESS);
1790 
1791 		goto bail;
1792 	}
1793 
1794 	if (attr_id == SA_MULTIPATHRECORD_ATTRID) {
1795 
1796 		multipath_template =
1797 		    (sa_multipath_record_t *)trans_info->si_trans_template;
1798 
1799 		payload_length = sizeof (sa_multipath_record_t) +
1800 		    ((multipath_template->SGIDCount +
1801 		    multipath_template->DGIDCount) * sizeof (ib_gid_t));
1802 
1803 		pack_attr_id = attr_id;
1804 	} else {
1805 
1806 		/* trace record template is a path record */
1807 		pack_attr_id = (attr_id == SA_TRACERECORD_ATTRID) ?
1808 		    SA_PATHRECORD_ATTRID : attr_id;
1809 
1810 		payload_length = ibmf_saa_impl_get_attr_id_length(pack_attr_id);
1811 
1812 		if (payload_length == 0) {
1813 			payload_length = trans_info->si_trans_template_length;
1814 
1815 			IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
1816 			    ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "",
1817 			    "ibmf_saa_impl_init_msg: %s, length = %d\n",
1818 			    tnf_string, msg,
1819 			    "Unknown attribute.  Using user-defined length.",
1820 			    tnf_uint, length, payload_length)
1821 		}
1822 	}
1823 
1824 	/* transport type depends on method */
1825 	switch (method) {
1826 
1827 		case SA_SUBN_ADM_GET:
1828 		case SA_SUBN_ADM_DELETE:
1829 		case SA_SUBN_ADM_GET_TABLE:
1830 		case SA_SUBN_ADM_GET_TRACE_TABLE:
1831 			transport_flags = IBMF_MSG_TRANS_FLAG_SEQ;
1832 			break;
1833 		case SA_SUBN_ADM_SET:
1834 			/* unsubscribes can be sequenced or unsequenced */
1835 			if (trans_info->si_trans_unseq_unsubscribe == B_TRUE) {
1836 				transport_flags = 0;
1837 			} else {
1838 				transport_flags = IBMF_MSG_TRANS_FLAG_SEQ;
1839 			}
1840 			break;
1841 		case SA_SUBN_ADM_GET_MULTI:
1842 			transport_flags = IBMF_MSG_TRANS_FLAG_SEQ |
1843 			    IBMF_MSG_TRANS_FLAG_RMPP;
1844 			break;
1845 		default :
1846 			ibmf_status = IBMF_UNSUPP_METHOD;
1847 			goto bail;
1848 	}
1849 
1850 	trans_info->si_trans_transport_flags = transport_flags;
1851 
1852 	if (trans_info->si_trans_template != NULL) {
1853 
1854 		ibmf_status = ibmf_saa_utils_pack_payload(
1855 		    trans_info->si_trans_template, payload_length, pack_attr_id,
1856 		    &req_mad->im_bufs_cl_data, &req_mad->im_bufs_cl_data_len,
1857 		    km_sleep_flag);
1858 		if (ibmf_status != IBMF_SUCCESS) {
1859 
1860 			IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1861 			    ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "",
1862 			    "ibmf_saa_impl_init_msg: %s, ibmf_status ="
1863 			    " %d\n", tnf_string, msg,
1864 			    "ibmf_saa_utils_pack_payload() failed",
1865 			    tnf_int, ibmf_status, ibmf_status);
1866 
1867 			kmem_free(mad_hdr, sizeof (ib_mad_hdr_t));
1868 
1869 			kmem_free(req_mad->im_bufs_cl_hdr,
1870 			    req_mad->im_bufs_cl_hdr_len);
1871 
1872 			free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle,
1873 			    &ibmf_msg);
1874 			ASSERT(free_res == IBMF_SUCCESS);
1875 
1876 			goto bail;
1877 		}
1878 
1879 		IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3,
1880 		    ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "",
1881 		    "ibmf_saa_impl_init_msg: %s, attr_id = 0x%x, length ="
1882 		    " %d\n", tnf_string, msg, "Packed payload successfully",
1883 		    tnf_opaque, attr_id, attr_id,
1884 		    tnf_uint, length, req_mad->im_bufs_cl_data_len);
1885 
1886 		/* non-RMPP transactions have template size limit */
1887 		if (((transport_flags & IBMF_MSG_TRANS_FLAG_RMPP) == 0) &&
1888 		    ((req_mad->im_bufs_cl_data_len + req_mad->im_bufs_cl_hdr_len
1889 		    + sizeof (ib_mad_hdr_t)) > IBMF_MAD_SIZE)) {
1890 
1891 			IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
1892 			    ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "",
1893 			    "ibmf_saa_impl_init_msg: %s\n", tnf_string, msg,
1894 			    "Template too large to fit in single packet");
1895 
1896 			kmem_free(mad_hdr, sizeof (ib_mad_hdr_t));
1897 
1898 			kmem_free(req_mad->im_bufs_cl_hdr,
1899 			    req_mad->im_bufs_cl_hdr_len);
1900 
1901 			kmem_free(req_mad->im_bufs_cl_data,
1902 			    req_mad->im_bufs_cl_data_len);
1903 
1904 			free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle,
1905 			    &ibmf_msg);
1906 			ASSERT(free_res == IBMF_SUCCESS);
1907 
1908 			ibmf_status = IBMF_REQ_INVALID;
1909 			goto bail;
1910 		}
1911 	}
1912 
1913 	mutex_enter(&saa_portp->saa_pt_mutex);
1914 
1915 	mad_hdr->TransactionID = h2b64(saa_portp->saa_pt_current_tid++);
1916 
1917 	bcopy(&saa_portp->saa_pt_ibmf_retrans, ibmf_retransp,
1918 	    sizeof (ibmf_retrans_t));
1919 
1920 	/* copy local addressing information to message */
1921 	bcopy(&saa_portp->saa_pt_ibmf_addr_info, &ibmf_msg->im_local_addr,
1922 	    sizeof (ibmf_addr_info_t));
1923 
1924 	/* copy global addressing information to message if in use */
1925 	if (saa_portp->saa_pt_ibmf_msg_flags & IBMF_MSG_FLAGS_GLOBAL_ADDRESS) {
1926 
1927 		ibmf_msg->im_msg_flags = IBMF_MSG_FLAGS_GLOBAL_ADDRESS;
1928 
1929 		bcopy(&saa_portp->saa_pt_ibmf_global_addr,
1930 		    &ibmf_msg->im_global_addr,
1931 		    sizeof (ibmf_global_addr_info_t));
1932 	} else {
1933 		ibmf_msg->im_msg_flags = 0;
1934 	}
1935 
1936 	mutex_exit(&saa_portp->saa_pt_mutex);
1937 
1938 	*msgp = ibmf_msg;
1939 	*transport_flagsp = transport_flags;
1940 bail:
1941 	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
1942 	    ibmf_saa_impl_init_msg_end,
1943 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_msg() exiting"
1944 	    " ibmf_status = %d\n", tnf_int, result, ibmf_status);
1945 
1946 	return (ibmf_status);
1947 
1948 }
1949 
1950 /*
1951  * ibmf_saa_impl_new_smlid_retry:
1952  *
1953  * It's possible for the MasterSMLID to change while ibmf_saa is running.  The
1954  * MasterSMLID is set when we first register with ibmf_saa.  If a request
1955  * timesout, this function should be called to check whether the SM LID changed.
1956  * If so, it will call msg_transport again with the request.
1957  *
1958  * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the
1959  * same values passed to the original ibmf_msg_transport that timedout.  The
1960  * ibmf_retrans parameter will be re-retrieved from the saa_portp structure.
1961  *
1962  * If the lid did not change then this function returns IBMF_TRANS_TIMEOUT.
1963  * That way, callers can simply return the result of this function.
1964  *
1965  * Input Arguments
1966  * saa_portp		pointer to saa_port structure
1967  * msgp			ibmf message that timedout
1968  * ibmf_callback	callback that should be called by msg_transport
1969  * ibmf_callback_arg	args for ibmf_callback
1970  * transport_flags	flags for ibmf_msg_transport
1971  *
1972  * Output Arguments
1973  * none
1974  *
1975  * Returns
1976  * IBMF_SUCCESS if lid changed and request was resent successfully,
1977  * IBMF_TRANS_TIMEOUT if lid did not change,
1978  * same values as ibmf_msg_transport() if lid changed but request could not be
1979  * resent.
1980  */
1981 static int
ibmf_saa_impl_new_smlid_retry(saa_port_t * saa_portp,ibmf_msg_t * msgp,ibmf_msg_cb_t ibmf_callback,void * ibmf_callback_arg,int transport_flags)1982 ibmf_saa_impl_new_smlid_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp,
1983     ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, int transport_flags)
1984 {
1985 	ibt_hca_portinfo_t	*ibt_portinfop;
1986 	ib_lid_t		master_sm_lid;
1987 	int			subnet_timeout;
1988 	uint_t			nports, size;
1989 	ibmf_retrans_t		ibmf_retrans;
1990 	int			ibmf_status;
1991 	ibt_status_t		ibt_status;
1992 
1993 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1994 	    ibmf_saa_impl_new_smlid_retry_start,
1995 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_new_smlid_retry() enter\n");
1996 
1997 	_NOTE(ASSUMING_PROTECTED(*msgp))
1998 	_NOTE(ASSUMING_PROTECTED(*msgp->im_msgbufs_send.im_bufs_mad_hdr))
1999 
2000 	/* first query the portinfo to see if the lid changed */
2001 	ibt_status = ibt_query_hca_ports_byguid(saa_portp->saa_pt_node_guid,
2002 	    saa_portp->saa_pt_port_num, &ibt_portinfop, &nports, &size);
2003 
2004 	if (ibt_status != IBT_SUCCESS)  {
2005 
2006 		IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2007 		    ibmf_saa_impl_new_smlid_retry_err, IBMF_TNF_ERROR, "",
2008 		    "ibmf_saa_impl_new_smlid_retry: %s, ibmf_status ="
2009 		    " %d\n", tnf_string, msg,
2010 		    "ibt_query_hca_ports_byguid() failed",
2011 		    tnf_int, ibt_status, ibt_status);
2012 
2013 		ibmf_status = IBMF_TRANSPORT_FAILURE;
2014 
2015 		goto bail;
2016 	}
2017 
2018 	master_sm_lid = ibt_portinfop->p_sm_lid;
2019 	subnet_timeout = ibt_portinfop->p_subnet_timeout;
2020 
2021 	ibt_free_portinfo(ibt_portinfop, size);
2022 
2023 	/* if master smlid is different than the remote lid we sent to */
2024 	if (master_sm_lid != msgp->im_local_addr.ia_remote_lid) {
2025 
2026 		IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L2,
2027 		    ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "",
2028 		    "ibmf_saa_impl_new_smlid_retry: %s, new_lid 0x%x,"
2029 		    " old_lid 0x%x\n", tnf_string, msg,
2030 		    "master smlid has changed.  retrying msg_transport",
2031 		    tnf_opaque, new_lid, master_sm_lid,
2032 		    tnf_opaque, old_lid, msgp->im_local_addr.ia_remote_lid);
2033 
2034 		mutex_enter(&saa_portp->saa_pt_mutex);
2035 
2036 		/* update the master sm lid value in ibmf_saa */
2037 		saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid =
2038 		    master_sm_lid;
2039 
2040 		/* new tid needed */
2041 		msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID =
2042 		    h2b64(saa_portp->saa_pt_current_tid++);
2043 
2044 		bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans,
2045 		    sizeof (ibmf_retrans_t));
2046 
2047 		/* update the subnet timeout since this may be a new sm/sa */
2048 		saa_portp->saa_pt_timeout = subnet_timeout;
2049 
2050 		/* place upper bound on subnet timeout in case of faulty SM */
2051 		if (saa_portp->saa_pt_timeout > ibmf_saa_max_subnet_timeout)
2052 			saa_portp->saa_pt_timeout = ibmf_saa_max_subnet_timeout;
2053 
2054 		/* increment the reference count to account for the cpi call */
2055 		saa_portp->saa_pt_reference_count++;
2056 
2057 		mutex_exit(&saa_portp->saa_pt_mutex);
2058 
2059 		/* update the remote lid for this particular message */
2060 		msgp->im_local_addr.ia_remote_lid = master_sm_lid;
2061 
2062 		/* get the classportinfo again since this may be a new sm/sa */
2063 		ibmf_saa_impl_get_classportinfo(saa_portp);
2064 
2065 		ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle,
2066 		    saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
2067 		    ibmf_callback, ibmf_callback_arg, transport_flags);
2068 
2069 		if (ibmf_status != IBMF_SUCCESS) {
2070 
2071 			IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
2072 			    ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "",
2073 			    "ibmf_saa_impl_new_smlid_retry: %s, ibmf_status = "
2074 			    "%d\n", tnf_string, msg,
2075 			    "ibmf_msg_transport() failed",
2076 			    tnf_int, ibmf_status, ibmf_status);
2077 		}
2078 
2079 		goto bail;
2080 	}
2081 
2082 	IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
2083 	    ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "",
2084 	    "ibmf_saa_impl_new_smlid_retry: %s, master_smlid = 0x%x\n",
2085 	    tnf_string, msg,
2086 	    "master smlid did not change.  returning failure",
2087 	    tnf_opaque, master_smlid, master_sm_lid);
2088 
2089 	/* mark status as timeout since that was original failure */
2090 	ibmf_status = IBMF_TRANS_TIMEOUT;
2091 
2092 bail:
2093 	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2094 	    ibmf_saa_impl_new_smlid_retry_end,
2095 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_new_smlid_retry() exiting"
2096 	    " ibmf_status = %d\n", tnf_int, result, ibmf_status);
2097 
2098 	return (ibmf_status);
2099 }
2100 
2101 /*
2102  * ibmf_saa_impl_revert_to_qp1()
2103  *
2104  * The SA that we had contact with via redirect may fail to respond. If this
2105  * occurs SA should revert back to qp1 and the SMLID set in the port.
2106  * msg_transport for the message that timed out will be retried with
2107  * these new parameters.
2108  *
2109  * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the
2110  * same values passed to the original ibmf_msg_transport that timedout.  The
2111  * ibmf_retrans parameter will be re-retrieved from the saa_portp structure.
2112  *
2113  * Input Arguments
2114  * saa_portp		pointer to saa_port structure
2115  * msgp			ibmf message that timedout
2116  * ibmf_callback	callback that should be called by msg_transport
2117  * ibmf_callback_arg	args for ibmf_callback
2118  * transport_flags	flags for ibmf_msg_transport
2119  *
2120  * Output Arguments
2121  * none
2122  *
2123  * Returns
2124  * none
2125  */
2126 static int
ibmf_saa_impl_revert_to_qp1(saa_port_t * saa_portp,ibmf_msg_t * msgp,ibmf_msg_cb_t ibmf_callback,void * ibmf_callback_args,int transport_flags)2127 ibmf_saa_impl_revert_to_qp1(saa_port_t *saa_portp, ibmf_msg_t *msgp,
2128     ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_args, int transport_flags)
2129 {
2130 	ibt_hca_portinfo_t	*ibt_portinfop;
2131 	ib_lid_t		master_sm_lid, base_lid;
2132 	uint8_t			sm_sl;
2133 	int			subnet_timeout;
2134 	uint_t			nports, size;
2135 	ibmf_retrans_t		ibmf_retrans;
2136 	int			ibmf_status;
2137 	ibt_status_t		ibt_status;
2138 
2139 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
2140 	    ibmf_saa_impl_revert_to_qp1_start,
2141 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_revert_to_qp1() enter\n");
2142 
2143 	_NOTE(ASSUMING_PROTECTED(*msgp))
2144 	_NOTE(ASSUMING_PROTECTED(*msgp->im_msgbufs_send.im_bufs_mad_hdr))
2145 
2146 	/* first query the portinfo to see if the lid changed */
2147 	ibt_status = ibt_query_hca_ports_byguid(saa_portp->saa_pt_node_guid,
2148 	    saa_portp->saa_pt_port_num, &ibt_portinfop, &nports, &size);
2149 
2150 	if (ibt_status != IBT_SUCCESS)  {
2151 
2152 		IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2153 		    ibmf_saa_impl_revert_to_qp1_err, IBMF_TNF_ERROR, "",
2154 		    "ibmf_saa_impl_revert_to_qp1: %s, ibmf_status ="
2155 		    " %d\n", tnf_string, msg,
2156 		    "ibt_query_hca_ports_byguid() failed",
2157 		    tnf_int, ibt_status, ibt_status);
2158 
2159 		ibmf_status = IBMF_TRANSPORT_FAILURE;
2160 
2161 		goto bail;
2162 	}
2163 
2164 	master_sm_lid = ibt_portinfop->p_sm_lid;
2165 	base_lid = ibt_portinfop->p_base_lid;
2166 	sm_sl = ibt_portinfop->p_sm_sl;
2167 	subnet_timeout = ibt_portinfop->p_subnet_timeout;
2168 
2169 	ibt_free_portinfo(ibt_portinfop, size);
2170 
2171 
2172 	mutex_enter(&saa_portp->saa_pt_mutex);
2173 
2174 	saa_portp->saa_pt_redirect_active = B_FALSE;
2175 
2176 	/* update the address info in ibmf_saa */
2177 	saa_portp->saa_pt_ibmf_addr_info.ia_local_lid = base_lid;
2178 	saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = master_sm_lid;
2179 	saa_portp->saa_pt_ibmf_addr_info.ia_service_level = sm_sl;
2180 	saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = 1;
2181 	saa_portp->saa_pt_ibmf_addr_info.ia_p_key = IB_PKEY_DEFAULT_LIMITED;
2182 	saa_portp->saa_pt_ibmf_addr_info.ia_q_key = IB_GSI_QKEY;
2183 	saa_portp->saa_pt_ibmf_msg_flags = 0;
2184 
2185 	/* new tid needed */
2186 	msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID =
2187 	    h2b64(saa_portp->saa_pt_current_tid++);
2188 
2189 	bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans,
2190 	    sizeof (ibmf_retrans_t));
2191 
2192 	/* update the subnet timeout since this may be a new sm/sa */
2193 	saa_portp->saa_pt_timeout = subnet_timeout;
2194 
2195 	/* place upper bound on subnet timeout in case of faulty SM */
2196 	if (saa_portp->saa_pt_timeout > ibmf_saa_max_subnet_timeout)
2197 		saa_portp->saa_pt_timeout = ibmf_saa_max_subnet_timeout;
2198 
2199 	/* increment the reference count to account for the cpi call */
2200 	saa_portp->saa_pt_reference_count++;
2201 
2202 	mutex_exit(&saa_portp->saa_pt_mutex);
2203 
2204 	/* update the address info for this particular message */
2205 	bcopy(&saa_portp->saa_pt_ibmf_addr_info, &msgp->im_local_addr,
2206 	    sizeof (ibmf_addr_info_t));
2207 	msgp->im_msg_flags = 0; /* No GRH */
2208 
2209 	/* get the classportinfo again since this may be a new sm/sa */
2210 	ibmf_saa_impl_get_classportinfo(saa_portp);
2211 
2212 	ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle,
2213 	    saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
2214 	    ibmf_callback, ibmf_callback_args, transport_flags);
2215 
2216 	if (ibmf_status != IBMF_SUCCESS) {
2217 
2218 		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
2219 		    ibmf_saa_impl_revert_to_qp1, IBMF_TNF_TRACE, "",
2220 		    "ibmf_saa_impl_revert_to_qp1: %s, ibmf_status = "
2221 		    "%d\n", tnf_string, msg,
2222 		    "ibmf_msg_transport() failed",
2223 		    tnf_int, ibmf_status, ibmf_status);
2224 	}
2225 
2226 bail:
2227 
2228 	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2229 	    ibmf_saa_impl_revert_to_qp1_end,
2230 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_revert_to_qp1() exiting"
2231 	    " ibmf_status = %d\n", tnf_int, result, ibmf_status);
2232 
2233 	return (ibmf_status);
2234 }
2235 
2236 /*
2237  * ibmf_saa_impl_async_event_cb:
2238  *	ibmf event callback, argument to ibmf_register
2239  *	ibmf_handle is unused
2240  */
2241 /*  ARGSUSED */
2242 static void
ibmf_saa_impl_async_event_cb(ibmf_handle_t ibmf_handle,void * clnt_private,ibmf_async_event_t event_type)2243 ibmf_saa_impl_async_event_cb(
2244 	ibmf_handle_t		ibmf_handle,
2245 	void			*clnt_private,
2246 	ibmf_async_event_t	event_type)
2247 {
2248 	saa_port_t		*saa_portp;
2249 
2250 	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2251 	    ibmf_saa_impl_async_event_cb_start, IBMF_TNF_TRACE, "",
2252 	    "ibmf_saa_impl_async_event_cb: Handling event type 0x%x\n",
2253 	    tnf_opaque, event_type, event_type);
2254 
2255 	saa_portp = (saa_port_t *)clnt_private;
2256 	ASSERT(saa_portp != NULL);
2257 
2258 	switch (event_type) {
2259 
2260 	case IBMF_CI_OFFLINE:
2261 		ibmf_saa_impl_hca_detach(saa_portp);
2262 		break;
2263 	default:
2264 		break;
2265 	}
2266 
2267 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3,
2268 	    ibmf_saa_impl_async_event_cb_end,
2269 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_async_event_cb() exit\n");
2270 }
2271 
2272 
2273 /*
2274  * ibmf_saa_impl_ibt_async_handler:
2275  * MUST NOT BE STATIC (referred from within IBMF)
2276  */
2277 void
ibmf_saa_impl_ibt_async_handler(ibt_async_code_t code,ibt_async_event_t * event)2278 ibmf_saa_impl_ibt_async_handler(ibt_async_code_t code, ibt_async_event_t *event)
2279 {
2280 	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2281 	    ibmf_saa_impl_ibt_async_handler_start, IBMF_TNF_TRACE, "",
2282 	    "ibmf_saa_impl_ibt_async_handler: Handling event code 0x%x\n",
2283 	    tnf_opaque, code, code);
2284 
2285 	switch (code) {
2286 
2287 	case IBT_EVENT_PORT_UP:
2288 		ibmf_saa_impl_port_up(event->ev_hca_guid, event->ev_port);
2289 		break;
2290 	case IBT_ERROR_PORT_DOWN:
2291 		ibmf_saa_impl_port_down(event->ev_hca_guid, event->ev_port);
2292 		break;
2293 	case IBT_PORT_CHANGE_EVENT:
2294 		ibmf_saa_impl_port_chg(event);
2295 		break;
2296 	case IBT_CLNT_REREG_EVENT:
2297 		ibmf_saa_impl_client_rereg(event->ev_hca_guid, event->ev_port);
2298 		break;
2299 	default:
2300 		break;
2301 	}
2302 
2303 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_async_handler_end,
2304 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibt_async_handler() exit\n");
2305 }
2306 
2307 /*
2308  * ibmf_saa_impl_port_chg:
2309  */
2310 static void
ibmf_saa_impl_port_chg(ibt_async_event_t * event)2311 ibmf_saa_impl_port_chg(ibt_async_event_t *event)
2312 {
2313 	saa_port_t		*saa_portp	= NULL;
2314 	boolean_t		is_ready = B_FALSE;
2315 	ibt_hca_portinfo_t	*ibt_portinfop;
2316 	uint_t			nports, size;
2317 	ibt_status_t		ibt_status;
2318 	ib_guid_t		ci_guid;
2319 	int			port_num;
2320 
2321 	ci_guid = event->ev_hca_guid;
2322 	port_num = event->ev_port;
2323 
2324 	IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_chg_start,
2325 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_chg: Handling port chg"
2326 	    " guid %016" PRIx64 " port %d\n",
2327 	    tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num);
2328 
2329 	/* Get classportinfo of corresponding entry */
2330 	mutex_enter(&saa_statep->saa_port_list_mutex);
2331 
2332 	saa_portp = saa_statep->saa_port_list;
2333 	while (saa_portp != NULL) {
2334 		if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid &&
2335 		    saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) {
2336 			mutex_enter(&saa_portp->saa_pt_mutex);
2337 
2338 			is_ready = (saa_portp->saa_pt_state
2339 			    == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE;
2340 
2341 			/*
2342 			 * increment reference count to account for cpi and
2343 			 * informinfos.  All 4 informinfo's sent are treated as
2344 			 * one port client reference
2345 			 */
2346 			if (is_ready)
2347 				saa_portp->saa_pt_reference_count ++;
2348 
2349 			mutex_exit(&saa_portp->saa_pt_mutex);
2350 
2351 			if (is_ready)
2352 				break; /* normally, only 1 port entry */
2353 		}
2354 		saa_portp = saa_portp->next;
2355 	}
2356 
2357 	mutex_exit(&saa_statep->saa_port_list_mutex);
2358 
2359 	if (saa_portp != NULL) {
2360 		/* first query the portinfo to see if the lid changed */
2361 		ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num,
2362 		    &ibt_portinfop, &nports, &size);
2363 
2364 		if (ibt_status != IBT_SUCCESS) {
2365 
2366 			IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2367 			    ibmf_saa_impl_port_chg_err, IBMF_TNF_ERROR, "",
2368 			    "ibmf_saa_impl_port_chg: %s, ibmf_status ="
2369 			    " %d\n", tnf_string, msg,
2370 			    "ibt_query_hca_ports_byguid() failed",
2371 			    tnf_int, ibt_status, ibt_status);
2372 
2373 			goto bail;
2374 		}
2375 
2376 		mutex_enter(&saa_portp->saa_pt_mutex);
2377 		if (event->ev_port_flags & IBT_PORT_CHANGE_SM_LID) {
2378 			/* update the Master SM Lid value in ibmf_saa */
2379 			saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid =
2380 			    ibt_portinfop->p_sm_lid;
2381 		}
2382 		if (event->ev_port_flags & IBT_PORT_CHANGE_SM_SL) {
2383 			/* update the Master SM SL value in ibmf_saa */
2384 			saa_portp->saa_pt_ibmf_addr_info.ia_service_level =
2385 			    ibt_portinfop->p_sm_sl;
2386 		}
2387 		if (event->ev_port_flags & IBT_PORT_CHANGE_SUB_TIMEOUT) {
2388 			/* update the Subnet timeout value in ibmf_saa */
2389 			saa_portp->saa_pt_timeout =
2390 			    ibt_portinfop->p_subnet_timeout;
2391 		}
2392 		mutex_exit(&saa_portp->saa_pt_mutex);
2393 
2394 		ibt_free_portinfo(ibt_portinfop, size);
2395 
2396 		/* get the classportinfo again */
2397 		ibmf_saa_impl_get_classportinfo(saa_portp);
2398 	}
2399 bail:
2400 
2401 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_chg_end,
2402 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_chg() exit\n");
2403 }
2404 /*
2405  * ibmf_saa_impl_client_rereg:
2406  */
2407 static void
ibmf_saa_impl_client_rereg(ib_guid_t ci_guid,uint8_t port_num)2408 ibmf_saa_impl_client_rereg(ib_guid_t ci_guid, uint8_t port_num)
2409 {
2410 	saa_port_t		*saa_portp	= NULL;
2411 	boolean_t		is_ready = B_FALSE;
2412 	ibt_hca_portinfo_t	*ibt_portinfop;
2413 	ib_lid_t		master_sm_lid;
2414 	uint_t			nports, size;
2415 	ibt_status_t		ibt_status;
2416 	boolean_t		event_subs = B_FALSE;
2417 
2418 	IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_rereg_start,
2419 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_client_rereg: Handling clnt "
2420 	    "rereg guid %016" PRIx64 " port %d\n",
2421 	    tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num);
2422 
2423 	/* Get classportinfo of corresponding entry */
2424 	mutex_enter(&saa_statep->saa_port_list_mutex);
2425 
2426 	saa_portp = saa_statep->saa_port_list;
2427 	while (saa_portp != NULL) {
2428 
2429 		if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid &&
2430 		    saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) {
2431 
2432 			mutex_enter(&saa_portp->saa_pt_mutex);
2433 
2434 			is_ready = (saa_portp->saa_pt_state
2435 			    == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE;
2436 
2437 			/*
2438 			 * increment reference count to account for cpi and
2439 			 * informinfos.  All 4 informinfo's sent are treated as
2440 			 * one port client reference
2441 			 */
2442 			if (is_ready)
2443 				saa_portp->saa_pt_reference_count += 2;
2444 
2445 			mutex_exit(&saa_portp->saa_pt_mutex);
2446 
2447 			if (is_ready)
2448 				break; /* normally, only 1 port entry */
2449 		}
2450 		saa_portp = saa_portp->next;
2451 	}
2452 
2453 	mutex_exit(&saa_statep->saa_port_list_mutex);
2454 
2455 	if (saa_portp != NULL && is_ready == B_TRUE) {
2456 
2457 		/* verify whether master sm lid changed */
2458 
2459 		/* first query the portinfo to see if the lid changed */
2460 		ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num,
2461 		    &ibt_portinfop, &nports, &size);
2462 
2463 		if (ibt_status != IBT_SUCCESS) {
2464 
2465 			IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2466 			    ibmf_saa_impl_port_rereg_err, IBMF_TNF_ERROR, "",
2467 			    "ibmf_saa_impl_client_rereg: %s, ibmf_status ="
2468 			    " %d\n", tnf_string, msg,
2469 			    "ibt_query_hca_ports_byguid() failed",
2470 			    tnf_int, ibt_status, ibt_status);
2471 
2472 			goto bail;
2473 		}
2474 
2475 		master_sm_lid = ibt_portinfop->p_sm_lid;
2476 
2477 		ibt_free_portinfo(ibt_portinfop, size);
2478 
2479 		/* check whether we need to subscribe for events */
2480 		mutex_enter(&saa_portp->saa_pt_event_sub_mutex);
2481 
2482 		event_subs = (saa_portp->saa_pt_event_sub_client_list != NULL) ?
2483 		    B_TRUE : B_FALSE;
2484 
2485 		mutex_exit(&saa_portp->saa_pt_event_sub_mutex);
2486 
2487 		/* update the master smlid */
2488 		mutex_enter(&saa_portp->saa_pt_mutex);
2489 
2490 		/* update the master sm lid value in ibmf_saa */
2491 		saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid =
2492 		    master_sm_lid;
2493 
2494 		/* if we're not subscribed for events, dec reference count */
2495 		if (event_subs == B_FALSE)
2496 			saa_portp->saa_pt_reference_count--;
2497 
2498 		mutex_exit(&saa_portp->saa_pt_mutex);
2499 
2500 		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
2501 		    ibmf_saa_impl_port_rereg, IBMF_TNF_TRACE, "",
2502 		    "ibmf_saa_impl_client_rereg: %s, master_sm_lid = 0x%x\n",
2503 		    tnf_string, msg,
2504 		    "port is up.  Sending classportinfo request",
2505 		    tnf_opaque, master_sm_lid, master_sm_lid);
2506 
2507 		/* get the classportinfo again */
2508 		ibmf_saa_impl_get_classportinfo(saa_portp);
2509 
2510 		/*
2511 		 * resubscribe to events if there are subscribers since SA may
2512 		 * have removed our subscription records when the port went down
2513 		 */
2514 		if (event_subs == B_TRUE)
2515 			ibmf_saa_subscribe_events(saa_portp, B_TRUE, B_FALSE);
2516 	}
2517 
2518 bail:
2519 
2520 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_rereg_end,
2521 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_client_rereg() exit\n");
2522 }
2523 /*
2524  * ibmf_saa_impl_port_up:
2525  */
2526 static void
ibmf_saa_impl_port_up(ib_guid_t ci_guid,uint8_t port_num)2527 ibmf_saa_impl_port_up(ib_guid_t ci_guid, uint8_t port_num)
2528 {
2529 	saa_port_t		*saa_portp	= NULL;
2530 	int			is_ready;
2531 	ibt_hca_portinfo_t	*ibt_portinfop;
2532 	ib_lid_t		master_sm_lid;
2533 	uint_t			nports, size;
2534 	ibt_status_t		ibt_status;
2535 	boolean_t		event_subs = B_FALSE;
2536 
2537 	IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_up_start,
2538 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_up: Handling port up"
2539 	    " guid %016" PRIx64 " port %d\n",
2540 	    tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num);
2541 
2542 	/* Get classportinfo of corresponding entry */
2543 	mutex_enter(&saa_statep->saa_port_list_mutex);
2544 
2545 	saa_portp = saa_statep->saa_port_list;
2546 	while (saa_portp != NULL) {
2547 
2548 		if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid &&
2549 		    saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) {
2550 
2551 			mutex_enter(&saa_portp->saa_pt_mutex);
2552 
2553 			is_ready = (saa_portp->saa_pt_state
2554 			    == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE;
2555 
2556 			/*
2557 			 * increment reference count to account for cpi and
2558 			 * informinfos.  All 4 informinfo's sent are treated as
2559 			 * one port client reference
2560 			 */
2561 			if (is_ready == B_TRUE)
2562 				saa_portp->saa_pt_reference_count += 2;
2563 
2564 			mutex_exit(&saa_portp->saa_pt_mutex);
2565 
2566 			if (is_ready == B_TRUE)
2567 				break; /* normally, only 1 port entry */
2568 		}
2569 		saa_portp = saa_portp->next;
2570 	}
2571 
2572 	mutex_exit(&saa_statep->saa_port_list_mutex);
2573 
2574 	if (saa_portp != NULL && is_ready == B_TRUE) {
2575 
2576 		/* verify whether master sm lid changed */
2577 
2578 		/* first query the portinfo to see if the lid changed */
2579 		ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num,
2580 		    &ibt_portinfop, &nports, &size);
2581 
2582 		if (ibt_status != IBT_SUCCESS) {
2583 
2584 			IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2585 			    ibmf_saa_impl_port_up_err, IBMF_TNF_ERROR, "",
2586 			    "ibmf_saa_impl_port_up: %s, ibmf_status ="
2587 			    " %d\n", tnf_string, msg,
2588 			    "ibt_query_hca_ports_byguid() failed",
2589 			    tnf_int, ibt_status, ibt_status);
2590 
2591 			goto bail;
2592 		}
2593 
2594 		master_sm_lid = ibt_portinfop->p_sm_lid;
2595 
2596 		ibt_free_portinfo(ibt_portinfop, size);
2597 
2598 		/* check whether we need to subscribe for events */
2599 		mutex_enter(&saa_portp->saa_pt_event_sub_mutex);
2600 
2601 		event_subs = (saa_portp->saa_pt_event_sub_client_list != NULL) ?
2602 		    B_TRUE : B_FALSE;
2603 
2604 		mutex_exit(&saa_portp->saa_pt_event_sub_mutex);
2605 
2606 		/* update the master smlid */
2607 		mutex_enter(&saa_portp->saa_pt_mutex);
2608 
2609 		/* update the master sm lid value in ibmf_saa */
2610 		saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid =
2611 		    master_sm_lid;
2612 
2613 		/* if we're not subscribed for events, dec reference count */
2614 		if (event_subs == B_FALSE)
2615 			saa_portp->saa_pt_reference_count--;
2616 
2617 		mutex_exit(&saa_portp->saa_pt_mutex);
2618 
2619 		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
2620 		    ibmf_saa_impl_port_up, IBMF_TNF_TRACE, "",
2621 		    "ibmf_saa_impl_port_up: %s, master_sm_lid = 0x%x\n",
2622 		    tnf_string, msg,
2623 		    "port is up.  Sending classportinfo request",
2624 		    tnf_opaque, master_sm_lid, master_sm_lid);
2625 
2626 		/* get the classportinfo again */
2627 		ibmf_saa_impl_get_classportinfo(saa_portp);
2628 
2629 		/*
2630 		 * resubscribe to events if there are subscribers since SA may
2631 		 * have removed our subscription records when the port went down
2632 		 */
2633 		if (event_subs == B_TRUE)
2634 			ibmf_saa_subscribe_events(saa_portp, B_TRUE, B_FALSE);
2635 	}
2636 
2637 bail:
2638 
2639 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_up_end,
2640 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_up() exit\n");
2641 }
2642 
2643 /*
2644  * ibmf_saa_impl_port_down:
2645  */
2646 static void
ibmf_saa_impl_port_down(ib_guid_t ci_guid,uint8_t port_num)2647 ibmf_saa_impl_port_down(ib_guid_t ci_guid, uint8_t port_num)
2648 {
2649 
2650 	IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_down_start,
2651 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_down: Handling port down"
2652 	    " guid %016" PRIx64 " port %d\n",
2653 	    tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num);
2654 
2655 
2656 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_down_end,
2657 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_down() exit\n");
2658 }
2659 
2660 /*
2661  * ibmf_saa_impl_hca_detach:
2662  * find entry, unregister if there are no clients
2663  * have to unregister since ibmf needs to close the hca and will only do this if
2664  * no clients are registered
2665  */
2666 static void
ibmf_saa_impl_hca_detach(saa_port_t * saa_removed)2667 ibmf_saa_impl_hca_detach(saa_port_t *saa_removed)
2668 {
2669 	saa_port_t 	*saa_portp;
2670 	boolean_t	must_unreg, must_unsub;
2671 
2672 	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_hca_detach_start,
2673 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach: Detaching"
2674 	    " entry %016" PRIx64 "\n", tnf_opaque, entry, saa_removed);
2675 
2676 	/* find this entry */
2677 	mutex_enter(&saa_statep->saa_port_list_mutex);
2678 
2679 	saa_portp = saa_statep->saa_port_list;
2680 	while (saa_portp != NULL) {
2681 
2682 		if (saa_portp == saa_removed)
2683 			break;
2684 
2685 		saa_portp = saa_portp->next;
2686 	}
2687 	mutex_exit(&saa_statep->saa_port_list_mutex);
2688 
2689 	ASSERT(saa_portp != NULL);
2690 
2691 	if (saa_portp == NULL) {
2692 
2693 		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
2694 		    ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "",
2695 		    "ibmf_saa_impl_hca_detach: %s, entry %016"
2696 		    PRIx64 "\n",
2697 		    tnf_string, msg,
2698 		    "Port entry NOT found",
2699 		    tnf_opaque, entryp, saa_removed);
2700 
2701 		goto bail;
2702 	}
2703 
2704 	/* if there are clients expecting Reports(), unsusbscribe */
2705 	mutex_enter(&saa_portp->saa_pt_event_sub_mutex);
2706 
2707 	must_unsub = (saa_portp->saa_pt_event_sub_client_list != NULL) ?
2708 	    B_TRUE : B_FALSE;
2709 
2710 	mutex_exit(&saa_portp->saa_pt_event_sub_mutex);
2711 
2712 	/* fail if outstanding transactions */
2713 	mutex_enter(&saa_portp->saa_pt_mutex);
2714 
2715 	if (saa_portp->saa_pt_num_outstanding_trans > 0) {
2716 
2717 		IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L1,
2718 		    ibmf_saa_impl_fini_err, IBMF_TNF_TRACE, "",
2719 		    "ibmf_saa_impl_fini: %s, port = %016" PRIx64
2720 		    ", num transactions = %d\n",
2721 		    tnf_string, msg, "Detaching HCA."
2722 		    "  Outstanding transactions on port.",
2723 		    tnf_opaque, port,
2724 		    saa_portp->saa_pt_port_guid,
2725 		    tnf_uint, outstanding_transactions,
2726 		    saa_portp->saa_pt_num_outstanding_trans);
2727 
2728 		mutex_exit(&saa_portp->saa_pt_mutex);
2729 
2730 		goto bail;
2731 	}
2732 
2733 
2734 	/*
2735 	 * increment reference count by one to account for unsubscribe requests
2736 	 * that are about to be sent.  All four informinfo's are treated as one
2737 	 * port client reference.  The count will be decremented by
2738 	 * subscribe_events() before the call returns.
2739 	 */
2740 	if (must_unsub == B_TRUE)
2741 		saa_portp->saa_pt_reference_count++;
2742 
2743 	mutex_exit(&saa_portp->saa_pt_mutex);
2744 
2745 	/*
2746 	 * try and unsubscribe from SA.  Generate synchronous, unsequenced
2747 	 * unsubscribe requests.
2748 	 */
2749 	if (must_unsub == B_TRUE)
2750 		ibmf_saa_subscribe_events(saa_portp, B_FALSE, B_TRUE);
2751 
2752 	/* warning if registered clients */
2753 	mutex_enter(&saa_portp->saa_pt_mutex);
2754 
2755 	if (saa_portp->saa_pt_reference_count > 0) {
2756 
2757 		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
2758 		    ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "",
2759 		    "ibmf_saa_impl_hca_detach: %s, port %016"
2760 		    PRIx64 "\n",
2761 		    tnf_string, msg,
2762 		    "Detaching HCA for port with clients still"
2763 		    " registered", tnf_opaque, port,
2764 		    saa_portp->saa_pt_port_guid);
2765 	}
2766 
2767 	/* synchronize on end of registration */
2768 	while (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_REGISTERING) {
2769 
2770 		IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L1,
2771 		    ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "",
2772 		    "ibmf_saa_impl_hca_detach: %s\n",
2773 		    tnf_string, msg, "someone is registering. waiting"
2774 		    " for them to finish");
2775 
2776 		cv_wait(&saa_portp->saa_pt_ibmf_reg_cv,
2777 		    &saa_portp->saa_pt_mutex);
2778 
2779 		IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L1,
2780 		    ibmf_saa_impl_hca_detach,
2781 		    IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach: %s\n",
2782 		    tnf_string, msg, "done waiting");
2783 	}
2784 
2785 	/* unregister from ibmf */
2786 	if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_READY) {
2787 		must_unreg = B_TRUE;
2788 	} else
2789 		must_unreg = B_FALSE;
2790 
2791 	ibmf_saa_impl_invalidate_port(saa_portp);
2792 
2793 	mutex_exit(&saa_portp->saa_pt_mutex);
2794 
2795 	if (must_unreg == B_TRUE) {
2796 		if (ibmf_saa_impl_ibmf_unreg(saa_portp) != IBMF_SUCCESS) {
2797 			mutex_enter(&saa_portp->saa_pt_mutex);
2798 			mutex_enter(&saa_portp->saa_pt_kstat_mutex);
2799 			(void) ibmf_saa_impl_init_kstats(saa_portp);
2800 			mutex_exit(&saa_portp->saa_pt_kstat_mutex);
2801 			saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_READY;
2802 			if (must_unsub == B_TRUE)
2803 				saa_portp->saa_pt_reference_count++;
2804 			mutex_exit(&saa_portp->saa_pt_mutex);
2805 
2806 			if (must_unsub == B_TRUE) {
2807 				ibmf_saa_subscribe_events(saa_portp, B_TRUE,
2808 				    B_FALSE);
2809 			}
2810 		}
2811 	}
2812 bail:
2813 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_hca_detach_end,
2814 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach() exit\n");
2815 }
2816 
2817 /* ARGSUSED */
2818 void
ibmf_saa_async_cb(ibmf_handle_t ibmf_handle,ibmf_msg_t * msgp,void * args)2819 ibmf_saa_async_cb(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp, void *args)
2820 {
2821 	saa_impl_trans_info_t	*trans_info;
2822 	int			status;
2823 	size_t			length;
2824 	void			*result;
2825 	saa_port_t		*saa_portp;
2826 	saa_client_data_t	*client_data;
2827 	int			ibmf_status;
2828 	boolean_t		ignore_data;
2829 	ibmf_retrans_t		ibmf_retrans;
2830 	boolean_t		sa_is_redirected = B_FALSE;
2831 
2832 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_async_cb_start,
2833 	    IBMF_TNF_TRACE, "", "ibmf_saa_async_cb() enter\n");
2834 
2835 	trans_info = (saa_impl_trans_info_t *)args;
2836 
2837 	client_data = trans_info->si_trans_client_data;
2838 	saa_portp   = trans_info->si_trans_port;
2839 
2840 	mutex_enter(&saa_portp->saa_pt_mutex);
2841 	sa_is_redirected = saa_portp->saa_pt_redirect_active;
2842 	mutex_exit(&saa_portp->saa_pt_mutex);
2843 
2844 	if ((msgp->im_msg_status == IBMF_TRANS_TIMEOUT) &&
2845 	    (sa_is_redirected == B_TRUE)) {
2846 
2847 		/*
2848 		 * We should retry the request using SM_LID and QP1 if we
2849 		 * have been using redirect up until now
2850 		 */
2851 		ibmf_status = ibmf_saa_impl_revert_to_qp1(
2852 		    saa_portp, msgp, ibmf_saa_async_cb, args,
2853 		    trans_info->si_trans_transport_flags);
2854 
2855 		/*
2856 		 * If revert_to_qp1 returns success msg was resent.
2857 		 * Otherwise msg could not be resent. Continue normally
2858 		 */
2859 		if (ibmf_status == IBMF_SUCCESS)
2860 			goto bail;
2861 
2862 	} else if (msgp->im_msg_status == IBMF_TRANS_TIMEOUT) {
2863 
2864 
2865 		ibmf_status = ibmf_saa_impl_new_smlid_retry(saa_portp, msgp,
2866 		    ibmf_saa_async_cb, args,
2867 		    trans_info->si_trans_transport_flags);
2868 
2869 		/*
2870 		 * if smlid_retry() returns success sm lid changed and msg
2871 		 * was resent.  Otherwise, lid did not change or msg could not
2872 		 * be resent.  Continue normally.
2873 		 */
2874 		if (ibmf_status == IBMF_SUCCESS)
2875 			goto bail;
2876 
2877 		/*
2878 		 * check whether we've received anything from the SA in a while.
2879 		 * If we have, this function will retry and return success.  If
2880 		 * we haven't continue normally so that we return a timeout to
2881 		 * the client
2882 		 */
2883 		ibmf_status = ibmf_saa_check_sa_and_retry(
2884 		    saa_portp, msgp, ibmf_saa_async_cb, args,
2885 		    trans_info->si_trans_send_time,
2886 		    trans_info->si_trans_transport_flags);
2887 
2888 		if (ibmf_status == IBMF_SUCCESS)
2889 			goto bail;
2890 	}
2891 
2892 	/*
2893 	 * If SA returned success but mad status is busy, retry a few times.
2894 	 * If SA returned success but mad status says redirect is required,
2895 	 * update the address info and retry the request to the new SA address
2896 	 */
2897 	if (msgp->im_msg_status == IBMF_SUCCESS) {
2898 
2899 		ASSERT(msgp->im_msgbufs_recv.im_bufs_mad_hdr != NULL);
2900 
2901 		if ((b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->Status) ==
2902 		    MAD_STATUS_BUSY) &&
2903 		    (trans_info->si_trans_retry_busy_count <
2904 		    IBMF_SAA_MAX_BUSY_RETRY_COUNT)) {
2905 
2906 			IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
2907 			    ibmf_saa_async_cb, IBMF_TNF_TRACE, "",
2908 			    "ibmf_saa_async_cb: %s, retry_count = %d\n",
2909 			    tnf_string, msg,
2910 			    "async response returned busy status",
2911 			    tnf_int, retry_count,
2912 			    trans_info->si_trans_retry_busy_count);
2913 
2914 			trans_info->si_trans_retry_busy_count++;
2915 
2916 			bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans,
2917 			    sizeof (ibmf_retrans_t));
2918 
2919 			ibmf_status = ibmf_msg_transport(
2920 			    saa_portp->saa_pt_ibmf_handle,
2921 			    saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
2922 			    ibmf_saa_async_cb, args,
2923 			    trans_info->si_trans_transport_flags);
2924 
2925 			/*
2926 			 * if retry is successful, quit here since async_cb will
2927 			 * get called again; otherwise, let this function call
2928 			 * handle the cleanup
2929 			 */
2930 			if (ibmf_status == IBMF_SUCCESS)
2931 				goto bail;
2932 		} else if (b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->Status)
2933 		    == MAD_STATUS_REDIRECT_REQUIRED) {
2934 
2935 			IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L2,
2936 			    ibmf_saa_async_cb, IBMF_TNF_TRACE, "",
2937 			    "ibmf_saa_async_cb: "
2938 			    "async response returned redirect status\n");
2939 
2940 			/* update address info and copy it into msgp */
2941 			ibmf_saa_impl_update_sa_address_info(saa_portp, msgp);
2942 
2943 			/* retry with new address info */
2944 			bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans,
2945 			    sizeof (ibmf_retrans_t));
2946 
2947 			ibmf_status = ibmf_msg_transport(
2948 			    saa_portp->saa_pt_ibmf_handle,
2949 			    saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
2950 			    ibmf_saa_async_cb, args,
2951 			    trans_info->si_trans_transport_flags);
2952 
2953 			/*
2954 			 * if retry is successful, quit here since async_cb will
2955 			 * get called again; otherwise, let this function call
2956 			 * handle the cleanup
2957 			 */
2958 			if (ibmf_status == IBMF_SUCCESS)
2959 				goto bail;
2960 		}
2961 	}
2962 
2963 	mutex_enter(&saa_portp->saa_pt_mutex);
2964 
2965 	ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0);
2966 	saa_portp->saa_pt_num_outstanding_trans--;
2967 
2968 	mutex_exit(&saa_portp->saa_pt_mutex);
2969 
2970 	if ((trans_info->si_trans_callback == NULL) &&
2971 	    (trans_info->si_trans_sub_callback == NULL))
2972 		ignore_data = B_TRUE;
2973 	else
2974 		ignore_data = B_FALSE;
2975 
2976 	ibmf_saa_impl_prepare_response(ibmf_handle, msgp, ignore_data, &status,
2977 	    &result, &length, B_FALSE);
2978 
2979 	mutex_enter(&saa_portp->saa_pt_kstat_mutex);
2980 
2981 	IBMF_SAA_SUB32_KSTATS(saa_portp, outstanding_requests, 1);
2982 
2983 	if (status != IBMF_SUCCESS)
2984 		IBMF_SAA_ADD32_KSTATS(saa_portp, failed_requests, 1);
2985 
2986 	if (status == IBMF_TRANS_TIMEOUT)
2987 		IBMF_SAA_ADD32_KSTATS(saa_portp, requests_timedout, 1);
2988 
2989 	mutex_exit(&saa_portp->saa_pt_kstat_mutex);
2990 
2991 	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2992 	    ibmf_saa_async_cb, IBMF_TNF_TRACE, "",
2993 	    "ibmf_saa_async_cb: %s\n", tnf_string, msg,
2994 	    "Calling ibmf_saa client's callback");
2995 
2996 	/*
2997 	 * there are three classes or trans_info users: ibmf_saa clients and
2998 	 * classportinfo requests; informinfo subscribe requests, and report
2999 	 * responses.  For the first two, call the correct callback.  For report
3000 	 * responses there's no need to notify anyone.
3001 	 */
3002 	if (trans_info->si_trans_callback != NULL) {
3003 		/* ibmf_saa client or classportinfo request */
3004 		trans_info->si_trans_callback(trans_info->si_trans_callback_arg,
3005 		    length, result, status);
3006 	} else if (trans_info->si_trans_sub_callback != NULL) {
3007 		/* informinfo subscribe request */
3008 		trans_info->si_trans_sub_callback(
3009 		    trans_info->si_trans_callback_arg, length, result, status,
3010 		    trans_info->si_trans_sub_producer_type);
3011 	}
3012 
3013 	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3014 	    ibmf_saa_async_cb, IBMF_TNF_TRACE, "",
3015 	    "ibmf_saa_async_cb: %s\n", tnf_string, msg,
3016 	    "Returned from callback");
3017 
3018 	if (client_data != NULL) {
3019 		mutex_enter(&client_data->saa_client_mutex);
3020 
3021 		ASSERT(client_data->saa_client_num_pending_trans > 0);
3022 		client_data->saa_client_num_pending_trans--;
3023 
3024 		if ((client_data->saa_client_num_pending_trans == 0) &&
3025 		    (client_data->saa_client_state == SAA_CLIENT_STATE_WAITING))
3026 			cv_signal(&client_data->saa_client_state_cv);
3027 
3028 		mutex_exit(&client_data->saa_client_mutex);
3029 	}
3030 
3031 	kmem_free(trans_info, sizeof (saa_impl_trans_info_t));
3032 
3033 bail:
3034 
3035 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_async_cb_end,
3036 	    IBMF_TNF_TRACE, "", "ibmf_saa_async_cb() exit\n");
3037 }
3038 
3039 /*
3040  * ibmf_saa_check_sa_and_retry:
3041  *
3042  * If a particular transaction times out, we don't want to give up if we know
3043  * the SA is responding.  Check the time since we last received a response. If
3044  * it's less than ibmf_saa_max_wait_time retry the request.
3045  *
3046  * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the
3047  * same values passed to the original ibmf_msg_transport that timed out.  The
3048  * ibmf_retrans parameter will be re-retrieved from the saa_portp structure.
3049  *
3050  * If max_wait_time seconds have passed, this function returns IBMF_TIMEOUT.
3051  * That way, callers can simply return the result of this function.
3052  *
3053  * Input Arguments
3054  * saa_portp		pointer to saa_port structure
3055  * msgp			ibmf message that timedout
3056  * ibmf_callback	callback that should be called by msg_transport
3057  * ibmf_callback_arg	args for ibmf_callback
3058  * transport_flags	flags for ibmf_msg_transport
3059  *
3060  * Output Arguments
3061  * none
3062  *
3063  * Returns
3064  * IBMF_SUCCESS if we've recently received data from the SA and request was
3065  * resent.
3066  * IBMF_TRANS_TIMEOUT if no data has been received from the SA in max_wait_time
3067  * same values as ibmf_msg_transport() if data has been received but request
3068  * could not be resent.
3069  */
3070 static int
ibmf_saa_check_sa_and_retry(saa_port_t * saa_portp,ibmf_msg_t * msgp,ibmf_msg_cb_t ibmf_callback,void * ibmf_callback_arg,hrtime_t trans_send_time,int transport_flags)3071 ibmf_saa_check_sa_and_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp,
3072     ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg,
3073     hrtime_t trans_send_time, int transport_flags)
3074 {
3075 	hrtime_t		curr_time, sa_uptime;
3076 	ibmf_retrans_t		ibmf_retrans;
3077 	int			ibmf_status;
3078 
3079 	do {
3080 
3081 		mutex_enter(&saa_portp->saa_pt_mutex);
3082 
3083 		sa_uptime = saa_portp->saa_pt_sa_uptime;
3084 
3085 		/* if nothing received from SA since we sent */
3086 		curr_time = gethrtime();
3087 
3088 		/*
3089 		 * check if it's been a very long time since this
3090 		 * particular transaction was sent
3091 		 */
3092 		if (((curr_time - trans_send_time) / 1000000000) >
3093 		    ibmf_saa_trans_wait_time) {
3094 
3095 			mutex_exit(&saa_portp->saa_pt_mutex);
3096 
3097 			IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L1,
3098 			    ibmf_saa_check_sa_and_retry_err, IBMF_TNF_ERROR, "",
3099 			    "ibmf_saa_check_sa_and_retry: %s, msgp = "
3100 			    "%p sa_uptime = %" PRIu64 ", trans send time = %"
3101 			    PRIu64 ", curr_time = %" PRIu64 "\n",
3102 			    tnf_string, msg,
3103 			    "Nothing received for this transaction",
3104 			    tnf_opaque, msgp, msgp,
3105 			    tnf_long, sa_uptime, sa_uptime,
3106 			    tnf_long, trans_send_time, trans_send_time,
3107 			    tnf_long, curr_time, curr_time);
3108 
3109 			ibmf_status = IBMF_TRANS_TIMEOUT;
3110 
3111 			break;
3112 		}
3113 
3114 		/*
3115 		 * check time since we received something,
3116 		 * and make sure that it hasn't been an extra long
3117 		 * time for this particular transaction
3118 		 */
3119 		if (((curr_time - sa_uptime) / 1000000000) <
3120 		    ibmf_saa_max_wait_time) {
3121 
3122 			IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L2,
3123 			    ibmf_saa_check_sa_and_retry, IBMF_TNF_TRACE, "",
3124 			    "ibmf_saa_check_sa_and_retry: %s, msgp = "
3125 			    "%p sa_uptime = %" PRIu64 " trans_send_time = %"
3126 			    PRIu64 " curr_time = %" PRIu64 "\n",
3127 			    tnf_string, msg, "Something received.  Retrying",
3128 			    tnf_opaque, msgp, msgp,
3129 			    tnf_long, sa_uptime, sa_uptime,
3130 			    tnf_long, trans_send_time, trans_send_time,
3131 			    tnf_long, curr_time, curr_time);
3132 
3133 			/*
3134 			 * something received in WAIT_TIME_IN_SECS;
3135 			 * resend request
3136 			 */
3137 
3138 			/* new tid needed */
3139 			msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID =
3140 			    h2b64(saa_portp->saa_pt_current_tid++);
3141 
3142 			/*
3143 			 * We are going to retry the access to the SM but
3144 			 * Master SMLID could have changed due to a port change
3145 			 * event. So update the remote_lid of the message with
3146 			 * the SMLID from saa_portp for this port before the
3147 			 * retry.
3148 			 */
3149 			msgp->im_local_addr.ia_remote_lid =
3150 			    saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid;
3151 
3152 			bcopy(&saa_portp->saa_pt_ibmf_retrans,
3153 			    &ibmf_retrans, sizeof (ibmf_retrans_t));
3154 
3155 			mutex_exit(&saa_portp->saa_pt_mutex);
3156 
3157 			ibmf_status = ibmf_msg_transport(
3158 			    saa_portp->saa_pt_ibmf_handle,
3159 			    saa_portp->saa_pt_qp_handle, msgp,
3160 			    &ibmf_retrans, ibmf_callback, ibmf_callback_arg,
3161 			    transport_flags);
3162 
3163 			if (ibmf_status == IBMF_SUCCESS)
3164 				goto bail;
3165 
3166 			IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
3167 			    ibmf_saa_check_sa_and_retry, IBMF_TNF_TRACE, "",
3168 			    "ibmf_saa_check_sa_and_retry: %s, ibmf_status = "
3169 			    "%d\n", tnf_string, msg,
3170 			    "ibmf_msg_transport() failed",
3171 			    tnf_int, ibmf_status, ibmf_status);
3172 		} else {
3173 
3174 			mutex_exit(&saa_portp->saa_pt_mutex);
3175 
3176 			IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L1,
3177 			    ibmf_saa_check_sa_and_retry_err, IBMF_TNF_ERROR, "",
3178 			    "ibmf_saa_check_sa_and_retry: %s, msgp = "
3179 			    "%p sa_uptime = %" PRIu64 " curr_time = %"
3180 			    PRIu64 "\n", tnf_string, msg,
3181 			    "Nothing received.  Timing out",
3182 			    tnf_opaque, msgp, msgp,
3183 			    tnf_long, sa_uptime, sa_uptime,
3184 			    tnf_long, curr_time, curr_time);
3185 
3186 			ibmf_status = IBMF_TRANS_TIMEOUT;
3187 
3188 			break;
3189 		}
3190 	} while (ibmf_status == IBMF_TRANS_TIMEOUT);
3191 
3192 bail:
3193 	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3194 	    ibmf_saa_check_sa_and_retry_end,
3195 	    IBMF_TNF_TRACE, "", "ibmf_saa_check_sa_and_retry() exiting"
3196 	    " ibmf_status = %d\n", tnf_int, result, ibmf_status);
3197 
3198 	return (ibmf_status);
3199 }
3200 
3201 
3202 /*
3203  * ibmf_saa_impl_prepare_response:
3204  */
3205 static void
ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle,ibmf_msg_t * msgp,boolean_t ignore_data,int * status,void ** result,size_t * length,boolean_t sleep_flag)3206 ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle,
3207     ibmf_msg_t *msgp, boolean_t ignore_data, int *status, void **result,
3208     size_t *length, boolean_t sleep_flag)
3209 {
3210 	ibmf_msg_bufs_t	*resp_buf;
3211 	uint16_t	attr_id;
3212 	uint8_t		method;
3213 	boolean_t	is_get_resp;
3214 	uint16_t	mad_status;
3215 	uint16_t	attr_offset;
3216 	ib_sa_hdr_t	*sa_hdr;
3217 
3218 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3219 	    ibmf_saa_impl_prepare_response_start,
3220 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response() enter\n");
3221 
3222 	_NOTE(ASSUMING_PROTECTED(*msgp))
3223 
3224 	*result = NULL;
3225 	*length = 0;
3226 	sa_hdr = NULL;
3227 
3228 	resp_buf = &msgp->im_msgbufs_recv;
3229 
3230 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*resp_buf))
3231 
3232 	if (msgp->im_msg_status != IBMF_SUCCESS) {
3233 
3234 		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
3235 		    ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3236 		    "ibmf_saa_impl_prepare_response: %s, msg_status = %d\n",
3237 		    tnf_string, msg, "Bad ibmf status",
3238 		    tnf_int, msg_status, msgp->im_msg_status);
3239 
3240 		*status = msgp->im_msg_status;
3241 
3242 		goto exit;
3243 	}
3244 
3245 	if (resp_buf->im_bufs_mad_hdr == NULL) {
3246 
3247 		/*
3248 		 * this was an unsequenced transaction (from an unsubscribe for
3249 		 * following a CI_OFFLINE event)
3250 		 */
3251 		IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3252 		    ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3253 		    "ibmf_saa_impl_prepare_response: %s\n",
3254 		    tnf_string, msg, "Unsequenced transaction callback");
3255 
3256 		goto exit;
3257 	}
3258 
3259 	if ((mad_status = b2h16(resp_buf->im_bufs_mad_hdr->Status)) !=
3260 	    MAD_STATUS_NO_INVALID_FIELDS) {
3261 
3262 		/* convert mad packet status to IBMF status */
3263 		switch (mad_status) {
3264 
3265 			case SA_STATUS_ERR_NO_RESOURCES:
3266 				*status = IBMF_NO_RESOURCES;
3267 				break;
3268 			case SA_STATUS_ERR_REQ_INVALID:
3269 				*status = IBMF_REQ_INVALID;
3270 				break;
3271 			case SA_STATUS_ERR_NO_RECORDS:
3272 				*status = IBMF_NO_RECORDS;
3273 				break;
3274 			case SA_STATUS_ERR_TOO_MANY_RECORDS:
3275 				*status = IBMF_TOO_MANY_RECORDS;
3276 				break;
3277 			case SA_STATUS_ERR_REQ_INVALID_GID:
3278 				*status = IBMF_INVALID_GID;
3279 				break;
3280 			case SA_STATUS_ERR_REQ_INSUFFICIENT_COMPONENTS:
3281 				*status = IBMF_INSUFF_COMPS;
3282 				break;
3283 			case MAD_STATUS_UNSUPP_METHOD:
3284 				*status = IBMF_UNSUPP_METHOD;
3285 				break;
3286 			case MAD_STATUS_UNSUPP_METHOD_ATTR:
3287 				*status = IBMF_UNSUPP_METHOD_ATTR;
3288 				break;
3289 			case MAD_STATUS_INVALID_FIELD:
3290 				*status = IBMF_INVALID_FIELD;
3291 				break;
3292 			default:
3293 				*status = IBMF_REQ_INVALID;
3294 				break;
3295 		}
3296 
3297 		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
3298 		    ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3299 		    "ibmf_saa_impl_prepare_response: %s, mad_status = %x\n",
3300 		    tnf_string, msg, "Bad MAD status",
3301 		    tnf_int, mad_status, mad_status);
3302 
3303 		goto exit;
3304 	}
3305 
3306 	attr_id = b2h16(resp_buf->im_bufs_mad_hdr->AttributeID);
3307 	method = resp_buf->im_bufs_mad_hdr->R_Method;
3308 
3309 	IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
3310 	    ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3311 	    "ibmf_saa_impl_prepare_response: attr_id = 0x%x, method = "
3312 	    "0x%x\n",
3313 	    tnf_opaque, attr_id, attr_id,
3314 	    tnf_opaque, method, method);
3315 
3316 	/*
3317 	 * ignore any data from deleteresp since there's no way to know whether
3318 	 * real data was returned; also ignore data if this was a Report
3319 	 * response
3320 	 */
3321 	if (method == SA_SUBN_ADM_DELETE_RESP) {
3322 
3323 		IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3324 		    ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3325 		    "impf_saa_impl_prepare_response: %s\n",
3326 		    tnf_string, msg,
3327 		    "DeleteResp or NoticeResp returned.  "
3328 		    "Ignoring response data");
3329 
3330 		*status = IBMF_SUCCESS;
3331 
3332 		*length = 0;
3333 		*result = NULL;
3334 
3335 		goto exit;
3336 	}
3337 
3338 	if (attr_id == SA_MULTIPATHRECORD_ATTRID) {
3339 
3340 		/*
3341 		 * getmulti is only for requests; attribute should not
3342 		 * be returned from SA
3343 		 */
3344 		IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
3345 		    ibmf_saa_impl_prepare_response_err, IBMF_TNF_ERROR,
3346 		    "", "ibmf_saa_impl_prepare_response: %s\n",
3347 		    tnf_string, msg, "SA returned getmulti record");
3348 
3349 		*status = IBMF_REQ_INVALID;
3350 
3351 		goto exit;
3352 	}
3353 
3354 	/* if we are supposed to ignore data, stop here */
3355 	if (ignore_data == B_TRUE) {
3356 
3357 		*status = IBMF_SUCCESS;
3358 
3359 		goto exit;
3360 	}
3361 
3362 	is_get_resp = resp_buf->im_bufs_mad_hdr->R_Method ==
3363 	    SA_SUBN_ADM_GET_RESP ? B_TRUE: B_FALSE;
3364 
3365 	/* unpack the sa header to get the attribute offset */
3366 	*status = ibmf_saa_utils_unpack_sa_hdr(resp_buf->im_bufs_cl_hdr,
3367 	    resp_buf->im_bufs_cl_hdr_len, &sa_hdr, sleep_flag);
3368 	if (*status != IBMF_SUCCESS) {
3369 
3370 		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3371 		    ibmf_saa_impl_prepare_response_err,
3372 		    IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response: %s,"
3373 		    " ibmf_status = %d\n", tnf_string, msg,
3374 		    "Could not unpack sa hdr", tnf_int, ibmf_status, *status);
3375 
3376 		goto exit;
3377 	}
3378 
3379 	attr_offset = sa_hdr->AttributeOffset;
3380 
3381 	/*
3382 	 * unpack data payload; if unpack function doesn't return success
3383 	 * (because it could not allocate memory) forward this status to waiting
3384 	 * client
3385 	 */
3386 	*status = ibmf_saa_utils_unpack_payload(resp_buf->im_bufs_cl_data,
3387 	    resp_buf->im_bufs_cl_data_len, attr_id, result, length,
3388 	    attr_offset, is_get_resp, sleep_flag);
3389 	if (*status == IBMF_SUCCESS) {
3390 
3391 		IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3,
3392 		    ibmf_saa_impl_prepare_response,
3393 		    IBMF_TNF_TRACE, "",
3394 		    "ibmf_saa_impl_prepare_response: attr_id = "
3395 		    "0x%x, attr_offset = %d, packed_payload_len = %d, "
3396 		    "unpacked_payload_len = %d\n",
3397 		    tnf_opaque, attr_id, attr_id,
3398 		    tnf_opaque, attr_offset, attr_offset,
3399 		    tnf_opaque, packed_payload_len,
3400 		    resp_buf->im_bufs_cl_data_len,
3401 		    tnf_opaque, unpacked_payload_len, *length);
3402 	} else {
3403 
3404 		IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L1,
3405 		    ibmf_saa_impl_prepare_response_err,
3406 		    IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response: %s,"
3407 		    "attr_id = 0x%x, attr_offset = %d, packed_payload_len = %d,"
3408 		    "status = %d\n",
3409 		    tnf_string, msg, "Could not unpack payload",
3410 		    tnf_opaque, attr_id, attr_id,
3411 		    tnf_int, attr_offset, attr_offset,
3412 		    tnf_int, packed_payload_len,
3413 		    resp_buf->im_bufs_cl_data_len,
3414 		    tnf_int, status, *status);
3415 	}
3416 exit:
3417 	if (sa_hdr != NULL)
3418 		kmem_free(sa_hdr, sizeof (ib_sa_hdr_t));
3419 
3420 	ibmf_saa_impl_free_msg(ibmf_handle, msgp);
3421 
3422 	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3423 	    ibmf_saa_impl_prepare_response_end,
3424 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response() exit,"
3425 	    " status = 0x%d\n", tnf_int, status, *status);
3426 }
3427 
3428 
3429 /*
3430  * ibmf_saa_impl_check_sa_support:
3431  * Checks the capability mask (returned from the SA classportinfo response) to
3432  * determine whether the sa supports the specified attribute ID.
3433  *
3434  * Input Arguments
3435  * cap_mask	16-bit capability mask returned in SA's classportinfo
3436  * attr_id	attribute ID of current request
3437  *
3438  * Returns
3439  * IBMF_NOT_SUPPORTED if capability mask indicates SA does not support attribute
3440  * IBMF_SUCCESS otherwise
3441  */
3442 static int
ibmf_saa_impl_check_sa_support(uint16_t cap_mask,uint16_t attr_id)3443 ibmf_saa_impl_check_sa_support(uint16_t cap_mask, uint16_t attr_id)
3444 {
3445 	boolean_t	attr_supported = B_TRUE;
3446 
3447 	IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
3448 	    ibmf_saa_impl_check_sa_support, IBMF_TNF_TRACE, "",
3449 	    "ibmf_saa_impl_check_sa_support: cap_mask = 0x%x, "
3450 	    "attr_id = 0x%x\n", tnf_opaque, cap_mask, cap_mask,
3451 	    tnf_opaque, attr_id, attr_id);
3452 
3453 	switch (attr_id) {
3454 
3455 		case SA_SWITCHINFORECORD_ATTRID:
3456 		case SA_LINEARFDBRECORD_ATTRID:
3457 		case SA_RANDOMFDBRECORD_ATTRID:
3458 		case SA_MULTICASTFDBRECORD_ATTRID:
3459 		case SA_SMINFORECORD_ATTRID:
3460 		case SA_INFORMINFORECORD_ATTRID:
3461 		case SA_LINKRECORD_ATTRID:
3462 		case SA_GUIDINFORECORD_ATTRID:
3463 		case SA_TRACERECORD_ATTRID:
3464 		case SA_SERVICEASSNRECORD_ATTRID:
3465 
3466 			if ((cap_mask &
3467 			    SA_CAPMASK_OPT_RECORDS_SUPPORTED) == 0) {
3468 
3469 				IBMF_TRACE_3(IBMF_TNF_NODEBUG, DPRINT_L1,
3470 				    ibmf_saa_impl_check_sa_support,
3471 				    IBMF_TNF_ERROR, "",
3472 				    "ibmf_saa_impl_check_sa_support: %s, "
3473 				    "cap_mask = 0x%x\n", tnf_string, msg,
3474 				    "SA does not support optional records",
3475 				    tnf_opaque, cap_mask, cap_mask,
3476 				    tnf_opaque, attr_id, attr_id);
3477 
3478 				attr_supported = B_FALSE;
3479 			}
3480 			break;
3481 
3482 		case SA_MULTIPATHRECORD_ATTRID:
3483 
3484 			if ((cap_mask & SA_CAPMASK_MULTIPATH_SUPPORTED) == 0) {
3485 
3486 				IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
3487 				    ibmf_saa_impl_check_sa_support,
3488 				    IBMF_TNF_ERROR, "",
3489 				    "ibmf_saa_impl_check_sa_support: %s, "
3490 				    "cap_mask = 0x%x\n", tnf_string, msg,
3491 				    "SA does not support multipath records",
3492 				    tnf_opaque, cap_mask, cap_mask);
3493 
3494 				attr_supported = B_FALSE;
3495 			}
3496 			break;
3497 
3498 		case SA_MCMEMBERRECORD_ATTRID:
3499 
3500 			if ((cap_mask & SA_CAPMASK_UD_MCAST_SUPPORTED) == 0) {
3501 
3502 				IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
3503 				    ibmf_saa_impl_check_sa_support,
3504 				    IBMF_TNF_ERROR, "",
3505 				    "ibmf_saa_impl_check_sa_support: %s, "
3506 				    "cap_mask = 0x%x\n", tnf_string, msg,
3507 				    "SA does not support ud multicast",
3508 				    tnf_opaque, cap_mask, cap_mask);
3509 
3510 				attr_supported = B_FALSE;
3511 			}
3512 			break;
3513 
3514 		default:
3515 			break;
3516 	} /* switch */
3517 
3518 	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3519 	    ibmf_saa_impl_check_sa_support_end, IBMF_TNF_TRACE, "",
3520 	    "ibmf_saa_impl_check_sa_support() exiting, attr_supported = %d\n",
3521 	    tnf_opaque, attr_supported, attr_supported);
3522 
3523 	if (attr_supported == B_FALSE)
3524 		return (IBMF_UNSUPP_METHOD_ATTR);
3525 	else
3526 		return (IBMF_SUCCESS);
3527 }
3528 
3529 /*
3530  * ibmf_saa_impl_get_attr_id_length:
3531  *
3532  * Returns the host size of the specified sa record.  Returns 0 for unknown
3533  * attributes.  multipath record size is a dynamic value given as a parameter
3534  * specified with the ibmf_sa_access() call.
3535  */
3536 static uint_t
ibmf_saa_impl_get_attr_id_length(uint16_t attr_id)3537 ibmf_saa_impl_get_attr_id_length(uint16_t attr_id)
3538 {
3539 	uint_t	attr_length;
3540 
3541 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3542 	    ibmf_saa_impl_get_attr_id_length_start,
3543 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_attr_id_length() enter\n");
3544 
3545 	/* this function should not be used for multipath record */
3546 	ASSERT(attr_id != SA_MULTIPATHRECORD_ATTRID);
3547 
3548 	switch (attr_id) {
3549 		case SA_CLASSPORTINFO_ATTRID:
3550 			attr_length = sizeof (ib_mad_classportinfo_t);
3551 			break;
3552 		case SA_NOTICE_ATTRID:
3553 			attr_length = sizeof (ib_mad_notice_t);
3554 			break;
3555 		case SA_INFORMINFO_ATTRID:
3556 			attr_length = sizeof (ib_mad_informinfo_t);
3557 			break;
3558 		case SA_NODERECORD_ATTRID:
3559 			attr_length = sizeof (sa_node_record_t);
3560 			break;
3561 		case SA_PORTINFORECORD_ATTRID:
3562 			attr_length = sizeof (sa_portinfo_record_t);
3563 			break;
3564 		case SA_SLTOVLRECORD_ATTRID:
3565 			attr_length = sizeof (sa_SLtoVLmapping_record_t);
3566 			break;
3567 		case SA_SWITCHINFORECORD_ATTRID:
3568 			attr_length = sizeof (sa_switchinfo_record_t);
3569 			break;
3570 		case SA_LINEARFDBRECORD_ATTRID:
3571 			attr_length = sizeof (sa_linearft_record_t);
3572 			break;
3573 		case SA_RANDOMFDBRECORD_ATTRID:
3574 			attr_length = sizeof (sa_randomft_record_t);
3575 			break;
3576 		case SA_MULTICASTFDBRECORD_ATTRID:
3577 			attr_length = sizeof (sa_multicastft_record_t);
3578 			break;
3579 		case SA_SMINFORECORD_ATTRID:
3580 			attr_length = sizeof (sa_sminfo_record_t);
3581 			break;
3582 		case SA_INFORMINFORECORD_ATTRID:
3583 			attr_length = sizeof (sa_informinfo_record_t);
3584 			break;
3585 		case SA_LINKRECORD_ATTRID:
3586 			attr_length = sizeof (sa_link_record_t);
3587 			break;
3588 		case SA_GUIDINFORECORD_ATTRID:
3589 			attr_length = sizeof (sa_guidinfo_record_t);
3590 			break;
3591 		case SA_SERVICERECORD_ATTRID:
3592 			attr_length = sizeof (sa_service_record_t);
3593 			break;
3594 		case SA_PARTITIONRECORD_ATTRID:
3595 			attr_length = sizeof (sa_pkey_table_record_t);
3596 			break;
3597 		case SA_PATHRECORD_ATTRID:
3598 			attr_length = sizeof (sa_path_record_t);
3599 			break;
3600 		case SA_VLARBRECORD_ATTRID:
3601 			attr_length = sizeof (sa_VLarb_table_record_t);
3602 			break;
3603 		case SA_MCMEMBERRECORD_ATTRID:
3604 			attr_length = sizeof (sa_mcmember_record_t);
3605 			break;
3606 		case SA_TRACERECORD_ATTRID:
3607 			attr_length = sizeof (sa_trace_record_t);
3608 			break;
3609 		case SA_SERVICEASSNRECORD_ATTRID:
3610 			attr_length = sizeof (sa_service_assn_record_t);
3611 			break;
3612 		default:
3613 			/* should only get the above type of packets */
3614 			attr_length = 0;
3615 			break;
3616 	}
3617 
3618 	IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
3619 	    ibmf_saa_impl_get_attr_id_length_end,
3620 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_attr_id_length():"
3621 	    " attr_id: 0x%x size %d\n",
3622 	    tnf_opaque, attr_id, attr_id, tnf_uint, attr_length, attr_length);
3623 
3624 	return (attr_length);
3625 }
3626 
3627 /*
3628  * ibmf_saa_impl_free_msg:
3629  * Takes a completed message and free memory associated with the message,
3630  * including the individual fields of the im_msgbufs_send.
3631  * ibmf_free_msg, called at the end of this function, takes a pointer to the
3632  * message pointer so that it can set the message pointer to NULL.  This
3633  * function takes just the message pointer so the msgp will not be NULL after
3634  * this function returns.
3635  *
3636  * Input Arguments
3637  * ibmf_hdl	ibmf handle used in ibmf_msg_alloc
3638  * msgp		pointer to ibmf_msg_t to free
3639  *
3640  * Returns
3641  * void
3642  */
3643 static void
ibmf_saa_impl_free_msg(ibmf_handle_t ibmf_hdl,ibmf_msg_t * msgp)3644 ibmf_saa_impl_free_msg(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msgp)
3645 {
3646 	int	res;
3647 
3648 	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3649 	    ibmf_saa_impl_free_msg_start,
3650 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_free_msg() enter: msg %p\n",
3651 	    tnf_opaque, msg, msgp);
3652 
3653 	ASSERT(msgp != NULL);
3654 
3655 	kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr,
3656 	    sizeof (ib_mad_hdr_t));
3657 
3658 	kmem_free(msgp->im_msgbufs_send.im_bufs_cl_hdr,
3659 	    msgp->im_msgbufs_send.im_bufs_cl_hdr_len);
3660 
3661 	if (msgp->im_msgbufs_send.im_bufs_cl_data_len > 0)
3662 		kmem_free(msgp->im_msgbufs_send.im_bufs_cl_data,
3663 		    msgp->im_msgbufs_send.im_bufs_cl_data_len);
3664 
3665 	res = ibmf_free_msg(ibmf_hdl, &msgp);
3666 	ASSERT(res == IBMF_SUCCESS);
3667 
3668 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3669 	    ibmf_saa_impl_free_msg_end,
3670 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_free_msg() exit\n");
3671 }
3672 
3673 /*
3674  * ibmf_saa_impl_get_port_guid:
3675  */
3676 static int
ibmf_saa_impl_get_port_guid(ibt_hca_portinfo_t * ibt_portinfop,ib_guid_t * guid_ret)3677 ibmf_saa_impl_get_port_guid(ibt_hca_portinfo_t *ibt_portinfop,
3678     ib_guid_t *guid_ret)
3679 {
3680 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3681 	    ibmf_saa_impl_get_port_guid_start,
3682 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_port_guid() enter\n");
3683 
3684 	if (ibt_portinfop->p_linkstate != IBT_PORT_ACTIVE) {
3685 
3686 		return (IBMF_BAD_PORT_STATE);
3687 	}
3688 
3689 	if (ibt_portinfop->p_sgid_tbl_sz == 0) {
3690 
3691 		IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L2,
3692 		    ibmf_saa_impl_get_port_guid_end, IBMF_TNF_TRACE, "",
3693 		    "ibmf_saa_impl_get_port_guid: %s\n", tnf_string, msg,
3694 		    "portinfo sgid table size is 0. Exiting.\n");
3695 
3696 		return (IBMF_TRANSPORT_FAILURE);
3697 	}
3698 
3699 	*guid_ret = ibt_portinfop->p_sgid_tbl[0].gid_guid;
3700 
3701 	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3702 	    ibmf_saa_impl_get_port_guid_end, IBMF_TNF_TRACE, "",
3703 	    "ibmf_saa_impl_get_port_guid: Returning port_guid %016" PRIx64 "\n",
3704 	    tnf_opaque, port_guid, *guid_ret);
3705 
3706 	return (IBMF_SUCCESS);
3707 }
3708 
3709 /*
3710  * ibmf_saa_impl_set_transaction_params:
3711  */
3712 static void
ibmf_saa_impl_set_transaction_params(saa_port_t * saa_portp,ibt_hca_portinfo_t * portinfop)3713 ibmf_saa_impl_set_transaction_params(saa_port_t *saa_portp,
3714     ibt_hca_portinfo_t *portinfop)
3715 {
3716 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3717 	    ibmf_saa_impl_set_transaction_params_start,
3718 	    IBMF_TNF_TRACE, "",
3719 	    "ibmf_saa_impl_set_transaction_params() enter\n");
3720 
3721 	_NOTE(ASSUMING_PROTECTED(*saa_portp))
3722 
3723 	saa_portp->saa_pt_ibmf_retrans.retrans_retries =
3724 	    ibmf_saa_retrans_retries;
3725 	/*
3726 	 * For the first transaction (generally getting the
3727 	 * classportinfo) have ibmf pick our timeouts.  It should be using the
3728 	 * default IB spec values.
3729 	 * Once we get the classportinfo we'll update the correct response time
3730 	 * value (rtv) and round-trip time (rttv).  ibmf should always calculate
3731 	 * trans_to since it depends on the particular transaction's number of
3732 	 * packets.
3733 	 */
3734 	saa_portp->saa_pt_ibmf_retrans.retrans_rtv = 0;
3735 	saa_portp->saa_pt_ibmf_retrans.retrans_rttv = 0;
3736 	saa_portp->saa_pt_ibmf_retrans.retrans_trans_to = 0;
3737 
3738 	/*
3739 	 * Assume that the SA supports all optional records. If it
3740 	 * does not, the request will get returned with ERR_NOT_SUPP.  When
3741 	 * the classportinfo response comes back we will update the cap mask
3742 	 * to prevent unnecessary unsupported requests.
3743 	 */
3744 	saa_portp->saa_pt_sa_cap_mask = 0xFFFF;
3745 
3746 	saa_portp->saa_pt_ibmf_msg_flags = 0;
3747 	saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno 	= 1;
3748 	saa_portp->saa_pt_ibmf_addr_info.ia_p_key 	=
3749 	    IB_PKEY_DEFAULT_LIMITED;
3750 	saa_portp->saa_pt_ibmf_addr_info.ia_q_key 	= IB_GSI_QKEY;
3751 
3752 	/*
3753 	 * fill out addr information for MADs that will be sent
3754 	 * to SA on this port
3755 	 */
3756 	saa_portp->saa_pt_ibmf_addr_info.ia_local_lid 	= portinfop->p_base_lid;
3757 	saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid 	= portinfop->p_sm_lid;
3758 	saa_portp->saa_pt_ibmf_addr_info.ia_service_level = portinfop->p_sm_sl;
3759 
3760 	/* place upper bound on subnet timeout in case of faulty SM */
3761 	saa_portp->saa_pt_timeout = portinfop->p_subnet_timeout;
3762 
3763 	if (saa_portp->saa_pt_timeout > ibmf_saa_max_subnet_timeout)
3764 		saa_portp->saa_pt_timeout = ibmf_saa_max_subnet_timeout;
3765 
3766 	IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3,
3767 	    ibmf_saa_impl_set_transaction_params,
3768 	    IBMF_TNF_TRACE, "",
3769 	    "ibmf_saa_impl_set_transaction_params: local_lid = 0x%x, "
3770 	    "sm_lid = 0x%x, sm_sl = 0x%x, sn_timeout = 0x%x\n",
3771 	    tnf_opaque, local_lid, portinfop->p_base_lid,
3772 	    tnf_opaque, sm_lid, portinfop->p_sm_lid,
3773 	    tnf_opaque, sm_sl, portinfop->p_sm_sl,
3774 	    tnf_opaque, subnet_timeout, portinfop->p_subnet_timeout);
3775 
3776 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3777 	    ibmf_saa_impl_set_transaction_params_end,
3778 	    IBMF_TNF_TRACE, "",
3779 	    "ibmf_saa_impl_set_transaction_params() exit\n");
3780 }
3781 
3782 
3783 /*
3784  * ibmf_saa_impl_update_sa_address_info
3785  */
3786 static void
ibmf_saa_impl_update_sa_address_info(saa_port_t * saa_portp,ibmf_msg_t * msgp)3787 ibmf_saa_impl_update_sa_address_info(saa_port_t *saa_portp, ibmf_msg_t *msgp)
3788 {
3789 	void			*result;
3790 	ib_sa_hdr_t		*sa_hdr;
3791 	int			rv;
3792 	size_t			length;
3793 	uint16_t		attr_id;
3794 	ib_mad_classportinfo_t	*cpi;
3795 	ibmf_global_addr_info_t	*gaddrp = &saa_portp->saa_pt_ibmf_global_addr;
3796 	ibt_hca_portinfo_t	*ibt_pinfo;
3797 	uint_t			nports, size;
3798 	ibt_status_t		ibt_status;
3799 
3800 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3801 	    ibmf_saa_impl_update_sa_address_info,
3802 	    IBMF_TNF_TRACE, "",
3803 	    "ibmf_saa_impl_update_sa_address_info() enter\n");
3804 
3805 	/*
3806 	 * decode the respons of msgp as a classportinfo attribute
3807 	 */
3808 	rv = ibmf_saa_utils_unpack_sa_hdr(msgp->im_msgbufs_recv.im_bufs_cl_hdr,
3809 	    msgp->im_msgbufs_recv.im_bufs_cl_hdr_len, &sa_hdr, KM_NOSLEEP);
3810 	if (rv != IBMF_SUCCESS) {
3811 
3812 		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3813 		    ibmf_saa_impl_update_sa_address_err,
3814 		    IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: "
3815 		    "%s, ibmf_status = %d\n", tnf_string, msg,
3816 		    "Could not unpack sa hdr", tnf_int, ibmf_status, rv);
3817 
3818 		return;
3819 	}
3820 
3821 	attr_id = b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID);
3822 	if (attr_id != MAD_ATTR_ID_CLASSPORTINFO) {
3823 		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3824 		    ibmf_saa_impl_update_sa_address_info_err,
3825 		    IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: "
3826 		    "%s, attrID = %x\n", tnf_string, msg,
3827 		    "Wrong attribute ID", tnf_int, ibmf_status, attr_id);
3828 
3829 		kmem_free(sa_hdr, sizeof (ib_sa_hdr_t));
3830 		return;
3831 	}
3832 	rv = ibmf_saa_utils_unpack_payload(
3833 	    msgp->im_msgbufs_recv.im_bufs_cl_data,
3834 	    msgp->im_msgbufs_recv.im_bufs_cl_data_len, attr_id, &result,
3835 	    &length, sa_hdr->AttributeOffset, B_TRUE, KM_NOSLEEP);
3836 	if (rv != IBMF_SUCCESS) {
3837 
3838 		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3839 		    ibmf_saa_impl_update_sa_address_err,
3840 		    IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: "
3841 		    "%s, ibmf_status = %d\n", tnf_string, msg,
3842 		    "Could not unpack payload", tnf_int, ibmf_status, rv);
3843 
3844 		kmem_free(sa_hdr, sizeof (ib_sa_hdr_t));
3845 		return;
3846 	}
3847 
3848 	kmem_free(sa_hdr, sizeof (ib_sa_hdr_t));
3849 
3850 	/*
3851 	 * Use the classportinfo contents to update the SA address info
3852 	 */
3853 	cpi = (ib_mad_classportinfo_t *)result;
3854 	mutex_enter(&saa_portp->saa_pt_mutex);
3855 	saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid	= cpi->RedirectLID;
3856 	saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno 	= cpi->RedirectQP;
3857 	saa_portp->saa_pt_ibmf_addr_info.ia_p_key 	= cpi->RedirectP_Key;
3858 	saa_portp->saa_pt_ibmf_addr_info.ia_q_key 	= cpi->RedirectQ_Key;
3859 	saa_portp->saa_pt_ibmf_addr_info.ia_service_level = cpi->RedirectSL;
3860 
3861 	saa_portp->saa_pt_redirect_active = B_TRUE;
3862 
3863 	if ((cpi->RedirectGID_hi != 0) || (cpi->RedirectGID_lo != 0)) {
3864 
3865 		mutex_exit(&saa_portp->saa_pt_mutex);
3866 		ibt_status = ibt_query_hca_ports_byguid(
3867 		    saa_portp->saa_pt_node_guid, saa_portp->saa_pt_port_num,
3868 		    &ibt_pinfo, &nports, &size);
3869 		if (ibt_status != IBT_SUCCESS) {
3870 
3871 			IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3872 			    ibmf_saa_impl_update_sa_address_err, IBMF_TNF_TRACE,
3873 			    "", "ibmf_saa_impl_update_sa_address_info: "
3874 			    "%s, ibt_status = %d\n", tnf_string, msg,
3875 			    "Could not query hca port",
3876 			    tnf_int, ibt_status, ibt_status);
3877 
3878 			kmem_free(result, length);
3879 			return;
3880 		}
3881 
3882 		mutex_enter(&saa_portp->saa_pt_mutex);
3883 		/*
3884 		 * Fill in global address info parameters
3885 		 *
3886 		 * NOTE: The HopLimit value is not specified through the
3887 		 * contents of ClassPortInfo. It may be possible to find
3888 		 * out the proper value to use even for SA beeing redirected
3889 		 * to another subnet. But we do only support redirect within
3890 		 * our local subnet
3891 		 */
3892 		gaddrp->ig_sender_gid.gid_prefix =
3893 		    ibt_pinfo->p_sgid_tbl[0].gid_prefix;
3894 		gaddrp->ig_sender_gid.gid_guid = saa_portp->saa_pt_port_guid;
3895 		gaddrp->ig_recver_gid.gid_prefix = cpi->RedirectGID_hi;
3896 		gaddrp->ig_recver_gid.gid_guid = cpi->RedirectGID_lo;
3897 		gaddrp->ig_flow_label = cpi->RedirectFL;
3898 		gaddrp->ig_tclass = cpi->RedirectTC;
3899 		gaddrp->ig_hop_limit = 0;
3900 
3901 		saa_portp->saa_pt_ibmf_msg_flags =
3902 		    IBMF_MSG_FLAGS_GLOBAL_ADDRESS;
3903 
3904 		mutex_exit(&saa_portp->saa_pt_mutex);
3905 		ibt_free_portinfo(ibt_pinfo, size);
3906 	} else {
3907 		saa_portp->saa_pt_ibmf_msg_flags = 0;
3908 		mutex_exit(&saa_portp->saa_pt_mutex);
3909 	}
3910 	kmem_free(result, length);
3911 
3912 	/*
3913 	 * Update the address info of msgp with the new address parameters
3914 	 */
3915 	mutex_enter(&saa_portp->saa_pt_mutex);
3916 	bcopy(&saa_portp->saa_pt_ibmf_addr_info, &msgp->im_local_addr,
3917 	    sizeof (ibmf_addr_info_t));
3918 	if (saa_portp->saa_pt_ibmf_msg_flags & IBMF_MSG_FLAGS_GLOBAL_ADDRESS) {
3919 
3920 		msgp->im_msg_flags = IBMF_MSG_FLAGS_GLOBAL_ADDRESS;
3921 
3922 		bcopy(&saa_portp->saa_pt_ibmf_global_addr,
3923 		    &msgp->im_global_addr, sizeof (ibmf_global_addr_info_t));
3924 	} else {
3925 		msgp->im_msg_flags = 0;
3926 	}
3927 	mutex_exit(&saa_portp->saa_pt_mutex);
3928 }
3929 
3930 /*
3931  * ibmf_saa_impl_ibmf_unreg:
3932  */
3933 static int
ibmf_saa_impl_ibmf_unreg(saa_port_t * saa_portp)3934 ibmf_saa_impl_ibmf_unreg(saa_port_t *saa_portp)
3935 {
3936 	int	ibmf_status;
3937 
3938 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_ibmf_unreg_start,
3939 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibmf_unreg() enter\n");
3940 
3941 	/* teardown async cb */
3942 	ibmf_status = ibmf_tear_down_async_cb(saa_portp->saa_pt_ibmf_handle,
3943 	    saa_portp->saa_pt_qp_handle, 0);
3944 	if (ibmf_status != IBMF_SUCCESS) {
3945 
3946 		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3947 		    ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "",
3948 		    "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n",
3949 		    tnf_string, msg, "Could not tear down async cb",
3950 		    tnf_int, ibmf_status, ibmf_status);
3951 
3952 		goto bail;
3953 	}
3954 
3955 	/* free qp */
3956 	ibmf_status = ibmf_free_qp(saa_portp->saa_pt_ibmf_handle,
3957 	    &saa_portp->saa_pt_qp_handle, 0);
3958 
3959 	if (ibmf_status != IBMF_SUCCESS) {
3960 
3961 		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3962 		    ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "",
3963 		    "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n",
3964 		    tnf_string, msg, "Could not free queue pair",
3965 		    tnf_int, ibmf_status, ibmf_status);
3966 
3967 		(void) ibmf_saa_impl_setup_qp_async_cb(saa_portp, 1);
3968 		goto bail;
3969 	}
3970 
3971 	ibmf_status = ibmf_unregister(&saa_portp->saa_pt_ibmf_handle, 0);
3972 
3973 	if (ibmf_status != IBMF_SUCCESS) {
3974 
3975 		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3976 		    ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "",
3977 		    "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n",
3978 		    tnf_string, msg, "ibmf_unregister() failed",
3979 		    tnf_int, ibmf_status, ibmf_status);
3980 
3981 		(void) ibmf_saa_impl_setup_qp_async_cb(saa_portp, 0);
3982 	}
3983 
3984 bail:
3985 	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_ibmf_unreg_end,
3986 	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibmf_unreg() exit\n");
3987 
3988 	return (ibmf_status);
3989 }
3990