xref: /illumos-gate/usr/src/uts/common/io/ib/ibtl/ibtl_cm.c (revision 9d3d2ed09c8e9ba0b2ba44fdd1dd300b2c3f9e8e)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 #include <sys/ib/ibtl/impl/ibtl.h>
30 #include <sys/ib/ibtl/impl/ibtl_cm.h>
31 
32 /*
33  * ibtl_cm.c
34  *    These routines tie the Communication Manager into IBTL.
35  */
36 
37 /*
38  * Globals.
39  */
40 static char 		ibtf_cm[] = "ibtl_cm";
41 boolean_t		ibtl_fast_gid_cache_valid = B_FALSE;
42 
43 /*
44  * Function:
45  *	ibtl_cm_set_chan_private
46  * Input:
47  *	chan		Channel Handle.
48  *	cm_private	CM private data.
49  * Output:
50  *	none.
51  * Returns:
52  *	none.
53  * Description:
54  *	A helper function to store CM's Private data in the specified channel.
55  */
56 void
57 ibtl_cm_set_chan_private(ibt_channel_hdl_t chan, void *cm_private)
58 {
59 	IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_set_chan_private(%p, %p)",
60 	    chan, cm_private);
61 
62 	mutex_enter(&chan->ch_cm_mutex);
63 	chan->ch_cm_private = cm_private;
64 	if (cm_private == NULL)
65 		cv_signal(&chan->ch_cm_cv);
66 	mutex_exit(&chan->ch_cm_mutex);
67 }
68 
69 
70 /*
71  * Function:
72  *	ibtl_cm_get_chan_private
73  * Input:
74  *	chan		Channel Handle.
75  * Output:
76  *	cm_private_p	The CM private data.
77  * Returns:
78  *	CM private data.
79  * Description:
80  *	A helper function to get CM's Private data for the specified channel.
81  */
82 void *
83 ibtl_cm_get_chan_private(ibt_channel_hdl_t chan)
84 {
85 	void *cm_private;
86 
87 	IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_get_chan_private(%p)", chan);
88 	mutex_enter(&chan->ch_cm_mutex);
89 	cm_private = chan->ch_cm_private;
90 #ifndef __lock_lint
91 	/* IBCM will call the release function if cm_private is non-NULL */
92 	if (cm_private == NULL)
93 #endif
94 		mutex_exit(&chan->ch_cm_mutex);
95 	return (cm_private);
96 }
97 
98 void
99 ibtl_cm_release_chan_private(ibt_channel_hdl_t chan)
100 {
101 #ifndef __lock_lint
102 	mutex_exit(&chan->ch_cm_mutex);
103 #endif
104 }
105 
106 void
107 ibtl_cm_wait_chan_private(ibt_channel_hdl_t chan)
108 {
109 	mutex_enter(&chan->ch_cm_mutex);
110 	if (chan->ch_cm_private != NULL)
111 		cv_wait(&chan->ch_cm_cv, &chan->ch_cm_mutex);
112 	mutex_exit(&chan->ch_cm_mutex);
113 	delay(drv_usectohz(50000));
114 }
115 
116 
117 /*
118  * Function:
119  *	ibtl_cm_get_chan_type
120  * Input:
121  *	chan		Channel Handle.
122  * Output:
123  *	none.
124  * Returns:
125  *	Channel transport type.
126  * Description:
127  *	A helper function to get channel transport type.
128  */
129 ibt_tran_srv_t
130 ibtl_cm_get_chan_type(ibt_channel_hdl_t chan)
131 {
132 	IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_get_chan_type(%p)", chan);
133 
134 	return (chan->ch_qp.qp_type);
135 }
136 
137 /*
138  * Function:
139  *	ibtl_cm_change_service_cnt
140  * Input:
141  *	ibt_hdl		Client's IBT Handle.
142  *	delta_num_sids	The change in the number of service ids
143  *			(positive for ibt_register_service() and
144  *			negative fo ibt_service_deregister()).
145  */
146 void
147 ibtl_cm_change_service_cnt(ibt_clnt_hdl_t ibt_hdl, int delta_num_sids)
148 {
149 	IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_change_service_cnt(%p. %d)",
150 	    ibt_hdl, delta_num_sids);
151 
152 	mutex_enter(&ibtl_clnt_list_mutex);
153 	if ((delta_num_sids < 0) && (-delta_num_sids > ibt_hdl->clnt_srv_cnt)) {
154 		IBTF_DPRINTF_L2(ibtf_cm, "ibtl_cm_change_service_cnt: "
155 		    "ERROR: service registration counter underflow\n"
156 		    "current count = %d, requested delta = %d",
157 		    ibt_hdl->clnt_srv_cnt, delta_num_sids);
158 	}
159 	ibt_hdl->clnt_srv_cnt += delta_num_sids;
160 	mutex_exit(&ibtl_clnt_list_mutex);
161 }
162 
163 
164 /*
165  * Function:
166  *	ibtl_cm_get_hca_port
167  * Input:
168  *	gid		Source GID.
169  *	hca_guid	Optional source HCA GUID on which SGID is available.
170  *			Ignored if zero.
171  * Output:
172  *	hca_port	Pointer to ibtl_cm_hca_port_t struct.
173  * Returns:
174  *	IBT_SUCCESS.
175  * Description:
176  *	A helper function to get HCA node GUID, Base LID, SGID Index,
177  *	port number, LMC and MTU for the specified SGID.
178  *	Also filling default SGID, to be used in ibmf_sa_session_open.
179  */
180 ibt_status_t
181 ibtl_cm_get_hca_port(ib_gid_t gid, ib_guid_t hca_guid,
182     ibtl_cm_hca_port_t *hca_port)
183 {
184 	ibtl_hca_devinfo_t	*hca_devp;	/* HCA Dev Info */
185 	ibt_hca_portinfo_t	*portinfop;
186 	uint_t			ports, port;
187 	uint_t			i;
188 	ib_gid_t		*sgid;
189 	static ib_gid_t		fast_gid;	/* fast_gid_cache data */
190 	static uint8_t		fast_sgid_ix;
191 	static ibt_hca_portinfo_t *fast_portinfop;
192 	static ib_guid_t	fast_node_guid;
193 	static ib_guid_t	fast_port_guid;
194 
195 	IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_get_hca_port(%llX:%llX, %llX)",
196 	    gid.gid_prefix, gid.gid_guid, hca_guid);
197 
198 	if ((gid.gid_prefix == 0) || (gid.gid_guid == 0)) {
199 		IBTF_DPRINTF_L2(ibtf_cm, "ibtl_cm_get_hca_port: "
200 		    "NULL SGID specified.");
201 		return (IBT_INVALID_PARAM);
202 	}
203 
204 	mutex_enter(&ibtl_clnt_list_mutex);
205 
206 	if ((ibtl_fast_gid_cache_valid == B_TRUE) &&
207 	    (gid.gid_guid == fast_gid.gid_guid) &&
208 	    (gid.gid_prefix == fast_gid.gid_prefix)) {
209 
210 		if ((hca_guid != 0) && (hca_guid != fast_node_guid)) {
211 			IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_get_hca_port: "
212 			    "Mis-match hca_guid v/s sgid combination.");
213 			mutex_exit(&ibtl_clnt_list_mutex);
214 			return (IBT_INVALID_PARAM);
215 		}
216 
217 		portinfop = fast_portinfop;
218 		hca_port->hp_base_lid = portinfop->p_base_lid;
219 		hca_port->hp_port = portinfop->p_port_num;
220 		hca_port->hp_sgid_ix = fast_sgid_ix;
221 		hca_port->hp_lmc = portinfop->p_lmc;
222 		hca_port->hp_mtu = portinfop->p_mtu;
223 		hca_port->hp_hca_guid = fast_node_guid;
224 		hca_port->hp_port_guid = fast_port_guid;
225 
226 		mutex_exit(&ibtl_clnt_list_mutex);
227 
228 		return (IBT_SUCCESS);
229 	}
230 
231 	/* If HCA GUID is specified, then lookup in that device only. */
232 	if (hca_guid) {
233 		hca_devp = ibtl_get_hcadevinfo(hca_guid);
234 	} else {
235 		hca_devp = ibtl_hca_list;
236 	}
237 
238 	while (hca_devp != NULL) {
239 
240 		ports = hca_devp->hd_hca_attr->hca_nports;
241 		portinfop = hca_devp->hd_portinfop;
242 
243 		for (port = 0; port < ports; port++, portinfop++) {
244 			if (portinfop->p_linkstate != IBT_PORT_ACTIVE)
245 				continue;
246 			sgid = &portinfop->p_sgid_tbl[0];
247 			for (i = 0; i < portinfop->p_sgid_tbl_sz; i++, sgid++) {
248 				if ((gid.gid_guid != sgid->gid_guid) ||
249 				    (gid.gid_prefix != sgid->gid_prefix))
250 					continue;
251 
252 				/*
253 				 * Found the matching GID.
254 				 */
255 				ibtl_fast_gid_cache_valid = B_TRUE;
256 				fast_gid = gid;
257 				fast_portinfop = portinfop;
258 				fast_node_guid = hca_port->hp_hca_guid =
259 				    hca_devp->hd_hca_attr->hca_node_guid;
260 				fast_sgid_ix = hca_port->hp_sgid_ix = i;
261 				fast_port_guid =
262 				    portinfop->p_sgid_tbl[0].gid_guid;
263 				hca_port->hp_port_guid = fast_port_guid;
264 				hca_port->hp_base_lid = portinfop->p_base_lid;
265 				hca_port->hp_port = portinfop->p_port_num;
266 				hca_port->hp_lmc = portinfop->p_lmc;
267 				hca_port->hp_mtu = portinfop->p_mtu;
268 
269 				mutex_exit(&ibtl_clnt_list_mutex);
270 
271 				return (IBT_SUCCESS);
272 			}
273 		}
274 
275 		/* Asked to look in the specified HCA device only?. */
276 		if (hca_guid)
277 			break;
278 
279 		/* Get next in the list */
280 		hca_devp = hca_devp->hd_hca_dev_link;
281 	}
282 
283 	mutex_exit(&ibtl_clnt_list_mutex);
284 
285 	/* If we are here, then we failed to get a match, so return error. */
286 	return (IBT_INVALID_PARAM);
287 }
288 
289 
290 static ibt_status_t
291 ibtl_cm_get_cnt(ibt_path_attr_t *attr, ibt_path_flags_t flags,
292     ibtl_cm_port_list_t *plistp, uint_t *count)
293 {
294 	ibtl_hca_devinfo_t	*hdevp;
295 	ibt_hca_portinfo_t	*pinfop;
296 	ib_guid_t		hca_guid, tmp_hca_guid = 0;
297 	ib_gid_t		gid;
298 	uint_t			pcount = 0, tmp_pcount = 0;
299 	uint_t			cnt = *count;
300 	ibt_status_t		retval = IBT_SUCCESS;
301 	uint_t			i, j;
302 
303 	*count = 0;
304 
305 	/* If HCA GUID is specified, then lookup in that device only. */
306 	if (attr->pa_hca_guid) {
307 		hdevp = ibtl_get_hcadevinfo(attr->pa_hca_guid);
308 	} else {
309 		hdevp = ibtl_hca_list;
310 	}
311 
312 	while (hdevp != NULL) {
313 		hca_guid = hdevp->hd_hca_attr->hca_node_guid;
314 
315 		if ((flags & IBT_PATH_APM) &&
316 		    (!(hdevp->hd_hca_attr->hca_flags &
317 		    IBT_HCA_AUTO_PATH_MIG))) {
318 
319 			IBTF_DPRINTF_L2(ibtf_cm, "ibtl_cm_get_cnt: "
320 			    "HCA (%llX) - APM NOT SUPPORTED ", hca_guid);
321 
322 			retval = IBT_APM_NOT_SUPPORTED;
323 
324 			if (attr->pa_hca_guid)
325 				break;
326 			hdevp = hdevp->hd_hca_dev_link;
327 			continue;
328 		}
329 
330 		for (i = 0; i < hdevp->hd_hca_attr->hca_nports; i++) {
331 
332 			if ((attr->pa_hca_port_num) &&
333 			    (attr->pa_hca_port_num != (i + 1))) {
334 				IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_get_cnt: "
335 				    "Asked only on Port# %d, so skip this "
336 				    "port(%d)", attr->pa_hca_port_num, (i + 1));
337 				continue;
338 			}
339 			pinfop = hdevp->hd_portinfop + i;
340 
341 			if (pinfop->p_linkstate != IBT_PORT_ACTIVE) {
342 				retval = IBT_HCA_PORT_NOT_ACTIVE;
343 				continue;
344 			}
345 			if (attr->pa_mtu.r_mtu) {
346 				if ((attr->pa_mtu.r_selector == IBT_GT) &&
347 				    (attr->pa_mtu.r_mtu >= pinfop->p_mtu))
348 					continue;
349 				else if ((attr->pa_mtu.r_selector == IBT_EQU) &&
350 				    (attr->pa_mtu.r_mtu > pinfop->p_mtu))
351 					continue;
352 			}
353 
354 			for (j = 0; j < pinfop->p_sgid_tbl_sz; j++) {
355 				gid = pinfop->p_sgid_tbl[j];
356 				if (gid.gid_prefix && gid.gid_guid) {
357 					pcount++;
358 					if (plistp) {
359 						plistp->p_hca_guid = hca_guid;
360 						plistp->p_mtu = pinfop->p_mtu;
361 						plistp->p_base_lid =
362 						    pinfop->p_base_lid;
363 						plistp->p_port_num =
364 						    pinfop->p_port_num;
365 						plistp->p_sgid_ix = j;
366 						plistp->p_sgid = gid;
367 						plistp->p_count = cnt;
368 						if (hdevp->hd_multism)
369 							plistp->p_multi |=
370 							    IBTL_CM_MULTI_SM;
371 
372 						IBTF_DPRINTF_L3(ibtf_cm,
373 						    "ibtl_cm_get_cnt: HCA"
374 						    "(%llX,%d) SGID(%llX:%llX)",
375 						    plistp->p_hca_guid,
376 						    plistp->p_port_num,
377 						    plistp->p_sgid.gid_prefix,
378 						    plistp->p_sgid.gid_guid);
379 
380 						plistp++;
381 					}
382 				}
383 			}
384 		}
385 		/* Asked to look in the specified HCA device only?. */
386 		if (attr->pa_hca_guid)
387 			break;
388 
389 		if (flags & IBT_PATH_APM) {
390 			if (pcount == 2) {
391 				attr->pa_hca_guid = hca_guid;
392 				break;
393 			} else if (pcount == 1) {
394 				if (hdevp->hd_hca_dev_link) {
395 					tmp_hca_guid = hca_guid;
396 					tmp_pcount = pcount;
397 					pcount = 0;
398 				} else if (tmp_hca_guid) {
399 					attr->pa_hca_guid = tmp_hca_guid;
400 				} else {
401 					attr->pa_hca_guid = hca_guid;
402 				}
403 			} else if ((pcount == 0) && (tmp_hca_guid)) {
404 				attr->pa_hca_guid = tmp_hca_guid;
405 				pcount = tmp_pcount;
406 			}
407 		}
408 		hdevp = hdevp->hd_hca_dev_link;
409 	}
410 
411 	*count = pcount;
412 
413 	if (pcount) {
414 		retval = IBT_SUCCESS;
415 	} else {
416 		IBTF_DPRINTF_L2(ibtf_cm, "ibtl_cm_get_cnt: "
417 		    "Appropriate Source Points NOT found");
418 		if (retval == IBT_SUCCESS)
419 			retval = IBT_NO_HCAS_AVAILABLE;
420 	}
421 
422 	return (retval);
423 }
424 
425 
426 ibt_status_t
427 ibtl_cm_get_active_plist(ibt_path_attr_t *attr, ibt_path_flags_t flags,
428     ibtl_cm_port_list_t **port_list_p)
429 {
430 	ibtl_cm_port_list_t	*p_listp, tmp;
431 	uint_t			i, j;
432 	uint_t			count, rcount;
433 	boolean_t		multi_hca = B_FALSE;
434 	ibt_status_t		retval = IBT_SUCCESS;
435 
436 	IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_get_active_plist(%p, %X)",
437 	    attr, flags);
438 
439 get_plist_start:
440 	*port_list_p = NULL;
441 
442 	/* Get "number of active src points" so that we can allocate memory. */
443 	mutex_enter(&ibtl_clnt_list_mutex);
444 	retval = ibtl_cm_get_cnt(attr, flags, NULL, &count);
445 	mutex_exit(&ibtl_clnt_list_mutex);
446 
447 	IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_get_active_plist: Found %d SrcPoint",
448 	    count);
449 	if (retval != IBT_SUCCESS)
450 		return (retval);
451 
452 	/* Allocate Memory to hold Src Point information. */
453 	p_listp = kmem_zalloc(count * sizeof (ibtl_cm_port_list_t), KM_SLEEP);
454 
455 	/*
456 	 * Verify that the count we got previously is still valid, as we had
457 	 * dropped mutex to allocate memory. If not, restart the process.
458 	 */
459 	mutex_enter(&ibtl_clnt_list_mutex);
460 	retval = ibtl_cm_get_cnt(attr, flags, NULL, &rcount);
461 	if (retval != IBT_SUCCESS) {
462 		mutex_exit(&ibtl_clnt_list_mutex);
463 		kmem_free(p_listp, count * sizeof (ibtl_cm_port_list_t));
464 		return (retval);
465 	} else if (rcount != count) {
466 		mutex_exit(&ibtl_clnt_list_mutex);
467 		kmem_free(p_listp, count * sizeof (ibtl_cm_port_list_t));
468 		goto get_plist_start;
469 	}
470 
471 	*port_list_p = p_listp;
472 	/*
473 	 * Src count hasn't changed, still holding the lock fill-in the
474 	 * required source point information.
475 	 */
476 	retval = ibtl_cm_get_cnt(attr, flags, p_listp, &rcount);
477 	mutex_exit(&ibtl_clnt_list_mutex);
478 	if (retval != IBT_SUCCESS) {
479 		kmem_free(p_listp, count * sizeof (ibtl_cm_port_list_t));
480 		*port_list_p = NULL;
481 		return (retval);
482 	}
483 
484 	p_listp = *port_list_p;
485 
486 	_NOTE(NO_COMPETING_THREADS_NOW)
487 
488 	for (i = 0; i < count - 1; i++) {
489 		for (j = 0; j < count - 1 - i; j++) {
490 			if (p_listp[j].p_hca_guid != p_listp[j+1].p_hca_guid) {
491 				multi_hca = B_TRUE;
492 				break;
493 			}
494 		}
495 		if (multi_hca == B_TRUE)
496 			break;
497 	}
498 
499 	if (multi_hca == B_TRUE)
500 		for (i = 0; i < count; i++)
501 			p_listp[i].p_multi |= IBTL_CM_MULTI_HCA;
502 
503 	/*
504 	 * Sort (bubble sort) the list based on MTU quality (higher on top).
505 	 * Sorting is only performed, if IBT_PATH_AVAIL is set.
506 	 */
507 	if (((attr->pa_mtu.r_selector == IBT_GT) || (flags & IBT_PATH_AVAIL)) &&
508 	    (!(flags & IBT_PATH_APM))) {
509 		for (i = 0; i < count - 1; i++) {
510 			for (j = 0; j < count - 1 - i; j++) {
511 				if (p_listp[j].p_mtu < p_listp[j+1].p_mtu) {
512 					tmp = p_listp[j];
513 					p_listp[j] = p_listp[j+1];
514 					p_listp[j+1] = tmp;
515 				}
516 			}
517 		}
518 	}
519 
520 	if ((p_listp->p_multi & IBTL_CM_MULTI_HCA) &&
521 	    (flags & IBT_PATH_AVAIL) && (!(flags & IBT_PATH_APM))) {
522 		/* Avoid having same HCA next to each other in the list. */
523 		for (i = 0; i < count - 1; i++) {
524 			for (j = 0; j < (count - 1 - i); j++) {
525 				if ((p_listp[j].p_hca_guid ==
526 				    p_listp[j+1].p_hca_guid) &&
527 				    (j+2 < count)) {
528 					tmp = p_listp[j+1];
529 					p_listp[j+1] = p_listp[j+2];
530 					p_listp[j+2] = tmp;
531 				}
532 			}
533 		}
534 	}
535 
536 	/*
537 	 * If SGID is specified, then make sure that SGID info is first
538 	 * in the array.
539 	 */
540 	if (attr->pa_sgid.gid_guid && (p_listp->p_count > 1) &&
541 	    (p_listp[0].p_sgid.gid_guid != attr->pa_sgid.gid_guid)) {
542 		for (i = 1; i < count; i++) {
543 			if (p_listp[i].p_sgid.gid_guid ==
544 			    attr->pa_sgid.gid_guid) {
545 				tmp = p_listp[i];
546 				p_listp[i] = p_listp[0];
547 				p_listp[0] = tmp;
548 			}
549 		}
550 	}
551 
552 	_NOTE(COMPETING_THREADS_NOW)
553 
554 	IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_get_active_plist: "
555 	    "Returned <%d> entries @0x%p", count, *port_list_p);
556 
557 	return (retval);
558 }
559 
560 
561 void
562 ibtl_cm_free_active_plist(ibtl_cm_port_list_t *plist)
563 {
564 	int count;
565 
566 	IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_free_active_plist(%p)", plist);
567 
568 	if (plist != NULL) {
569 		_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*plist))
570 		count = plist->p_count;
571 		_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*plist))
572 
573 		kmem_free(plist, count * sizeof (ibtl_cm_port_list_t));
574 	}
575 }
576 
577 /*
578  * Function:
579  *	ibtl_cm_get_1st_full_pkey_ix
580  * Input:
581  *	hca_guid	HCA GUID.
582  *	port		Port Number.
583  * Output:
584  *	None.
585  * Returns:
586  *	P_Key Index of the first full member available from the P_Key table
587  *	of the specified HCA<->Port.
588  * Description:
589  *	A helper function to get P_Key Index of the first full member P_Key
590  *	available on the specified HCA and Port combination.
591  */
592 uint16_t
593 ibtl_cm_get_1st_full_pkey_ix(ib_guid_t hca_guid, uint8_t port)
594 {
595 	ibtl_hca_devinfo_t	*hca_devp;	/* HCA Dev Info */
596 	uint16_t		pkey_ix = 0;
597 
598 	IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_get_1st_full_pkey_ix(%llX, %d)",
599 	    hca_guid, port);
600 
601 	mutex_enter(&ibtl_clnt_list_mutex);
602 	hca_devp = ibtl_get_hcadevinfo(hca_guid);
603 
604 	if ((hca_devp != NULL) && (port <= hca_devp->hd_hca_attr->hca_nports) &&
605 	    (port != 0)) {
606 		pkey_ix = hca_devp->hd_portinfop[port - 1].p_def_pkey_ix;
607 	} else {
608 		IBTF_DPRINTF_L2(ibtf_cm, "ibtl_cm_get_1st_full_pkey_ix: "
609 		    "Invalid HCA (%llX), Port (%d) specified.", hca_guid, port);
610 	}
611 	mutex_exit(&ibtl_clnt_list_mutex);
612 
613 	return (pkey_ix);
614 }
615 
616 
617 ibt_status_t
618 ibtl_cm_get_local_comp_gids(ib_guid_t hca_guid, ib_gid_t gid, ib_gid_t **gids_p,
619     uint_t *num_gids_p)
620 {
621 	ibtl_hca_devinfo_t	*hdevp;	/* HCA Dev Info */
622 	ibt_hca_portinfo_t	*pinfop;
623 	ib_gid_t		sgid;
624 	ib_gid_t		*gidp = NULL;
625 	int			i, j, k;
626 	int			count = 0;
627 	int			gid_specified;
628 
629 	IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_get_local_comp_gids(%llX, %llX:%llX)",
630 	    hca_guid, gid.gid_prefix, gid.gid_guid);
631 
632 	mutex_enter(&ibtl_clnt_list_mutex);
633 	hdevp = ibtl_get_hcadevinfo(hca_guid);
634 
635 	if (hdevp == NULL) {
636 		IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_get_local_comp_gids: ",
637 		    "NO HCA (%llX) availble", hca_guid);
638 		mutex_exit(&ibtl_clnt_list_mutex);
639 		return (IBT_NO_HCAS_AVAILABLE);
640 	}
641 
642 	if (gid.gid_prefix && gid.gid_guid)
643 		gid_specified = 1;
644 	else
645 		gid_specified = 0;
646 
647 	for (i = 0; i < hdevp->hd_hca_attr->hca_nports; i++) {
648 		pinfop = hdevp->hd_portinfop + i;
649 
650 		if (pinfop->p_linkstate != IBT_PORT_ACTIVE)
651 			continue;
652 
653 		for (j = 0; j < pinfop->p_sgid_tbl_sz; j++) {
654 			sgid = pinfop->p_sgid_tbl[j];
655 			if (sgid.gid_prefix && sgid.gid_guid) {
656 				if (gid_specified &&
657 				    ((gid.gid_prefix == sgid.gid_prefix) &&
658 				    (gid.gid_guid == sgid.gid_guid))) {
659 					/*
660 					 * Don't return the input specified
661 					 * GID
662 					 */
663 					continue;
664 				}
665 				count++;
666 			}
667 		}
668 	}
669 
670 	if (count == 0) {
671 		IBTF_DPRINTF_L2(ibtf_cm, "ibtl_cm_get_local_comp_gids: "
672 		    "Companion GIDs not available");
673 		mutex_exit(&ibtl_clnt_list_mutex);
674 		return (IBT_GIDS_NOT_FOUND);
675 	}
676 
677 	gidp = kmem_zalloc(count * sizeof (ib_gid_t), KM_SLEEP);
678 	*num_gids_p = count;
679 	*gids_p = gidp;
680 	k = 0;
681 
682 	for (i = 0; i < hdevp->hd_hca_attr->hca_nports; i++) {
683 		pinfop = hdevp->hd_portinfop + i;
684 
685 		if (pinfop->p_linkstate != IBT_PORT_ACTIVE)
686 			continue;
687 
688 		for (j = 0; j < pinfop->p_sgid_tbl_sz; j++) {
689 			sgid = pinfop->p_sgid_tbl[j];
690 			if (sgid.gid_prefix && sgid.gid_guid) {
691 				if (gid_specified &&
692 				    ((gid.gid_prefix == sgid.gid_prefix) &&
693 				    (gid.gid_guid == sgid.gid_guid)))
694 					continue;
695 
696 				gidp[k].gid_prefix = sgid.gid_prefix;
697 				gidp[k].gid_guid = sgid.gid_guid;
698 
699 				IBTF_DPRINTF_L3(ibtf_cm,
700 				    "ibtl_cm_get_local_comp_gids: GID[%d]="
701 				    "%llX:%llX", k, gidp[k].gid_prefix,
702 				    gidp[k].gid_guid);
703 				k++;
704 				if (k == count)
705 					break;
706 			}
707 		}
708 		if (k == count)
709 			break;
710 	}
711 	mutex_exit(&ibtl_clnt_list_mutex);
712 
713 	return (IBT_SUCCESS);
714 }
715 
716 
717 int
718 ibtl_cm_is_multi_sm(ib_guid_t hca_guid)
719 {
720 	ibtl_hca_devinfo_t	*hdevp;	/* HCA Dev Info */
721 	uint_t			multi_sm;
722 
723 	mutex_enter(&ibtl_clnt_list_mutex);
724 	hdevp = ibtl_get_hcadevinfo(hca_guid);
725 	if (hdevp == NULL) {
726 		IBTF_DPRINTF_L2(ibtf_cm, "ibtl_cm_is_multi_sm: NO HCA (%llX) "
727 		    "availble", hca_guid);
728 		mutex_exit(&ibtl_clnt_list_mutex);
729 		return (-1);
730 	}
731 	multi_sm = hdevp->hd_multism;
732 	mutex_exit(&ibtl_clnt_list_mutex);
733 
734 	IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_is_multi_sm(%llX): %d", hca_guid,
735 	    multi_sm);
736 
737 	return (multi_sm);
738 }
739