xref: /illumos-gate/usr/src/uts/common/io/ib/mgt/ibdm/ibdm.c (revision add2b70b10ae748c42011265ee2e7cfe4e5d6950)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 /*
29  * ibdm.c
30  *
31  * This file contains the InifiniBand Device Manager (IBDM) support functions.
32  * IB nexus driver will only be the client for the IBDM module.
33  *
34  * IBDM registers with IBTF for HCA arrival/removal notification.
35  * IBDM registers with SA access to send DM MADs to discover the IOC's behind
36  * the IOU's.
37  *
38  * IB nexus driver registers with IBDM to find the information about the
39  * HCA's and IOC's (behind the IOU) present on the IB fabric.
40  */
41 
42 #include <sys/systm.h>
43 #include <sys/taskq.h>
44 #include <sys/ib/mgt/ibdm/ibdm_impl.h>
45 #include <sys/modctl.h>
46 
47 /* Function Prototype declarations */
48 static int	ibdm_free_iou_info(ibdm_dp_gidinfo_t *, ibdm_iou_info_t **);
49 static int	ibdm_fini(void);
50 static int	ibdm_init(void);
51 static int	ibdm_get_reachable_ports(ibdm_port_attr_t *,
52 			ibdm_hca_list_t *);
53 static ibdm_dp_gidinfo_t *ibdm_check_dgid(ib_guid_t, ib_sn_prefix_t);
54 static ibdm_dp_gidinfo_t *ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *);
55 static int	ibdm_send_classportinfo(ibdm_dp_gidinfo_t *);
56 static int	ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *);
57 static int	ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *);
58 static int	ibdm_get_node_port_guids(ibmf_saa_handle_t, ib_lid_t,
59 		    ib_guid_t *, ib_guid_t *);
60 static int	ibdm_retry_command(ibdm_timeout_cb_args_t *);
61 static int	ibdm_get_diagcode(ibdm_dp_gidinfo_t *, int);
62 static int	ibdm_verify_mad_status(ib_mad_hdr_t *);
63 static int	ibdm_handle_redirection(ibmf_msg_t *,
64 		    ibdm_dp_gidinfo_t *, int *);
65 static void	ibdm_wait_probe_completion(void);
66 static void	ibdm_sweep_fabric(int);
67 static void	ibdm_probe_gid_thread(void *);
68 static void	ibdm_wakeup_probe_gid_cv(void);
69 static void	ibdm_port_attr_ibmf_init(ibdm_port_attr_t *, ib_pkey_t, int);
70 static int	ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *, int);
71 static void	ibdm_update_port_attr(ibdm_port_attr_t *);
72 static void	ibdm_handle_hca_attach(ib_guid_t);
73 static void	ibdm_handle_srventry_mad(ibmf_msg_t *,
74 		    ibdm_dp_gidinfo_t *, int *);
75 static void	ibdm_ibmf_recv_cb(ibmf_handle_t, ibmf_msg_t *, void *);
76 static void	ibdm_recv_incoming_mad(void *);
77 static void	ibdm_process_incoming_mad(ibmf_handle_t, ibmf_msg_t *, void *);
78 static void	ibdm_ibmf_send_cb(ibmf_handle_t, ibmf_msg_t *, void *);
79 static void	ibdm_pkt_timeout_hdlr(void *arg);
80 static void	ibdm_initialize_port(ibdm_port_attr_t *);
81 static void	ibdm_handle_diagcode(ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
82 static void	ibdm_probe_gid(ibdm_dp_gidinfo_t *);
83 static void	ibdm_alloc_send_buffers(ibmf_msg_t *);
84 static void	ibdm_free_send_buffers(ibmf_msg_t *);
85 static void	ibdm_handle_hca_detach(ib_guid_t);
86 static int	ibdm_fini_port(ibdm_port_attr_t *);
87 static int	ibdm_uninit_hca(ibdm_hca_list_t *);
88 static void	ibdm_handle_iounitinfo(ibmf_handle_t,
89 		    ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
90 static void	ibdm_handle_ioc_profile(ibmf_handle_t,
91 		    ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
92 static void	ibdm_event_hdlr(void *, ibt_hca_hdl_t,
93 		    ibt_async_code_t, ibt_async_event_t *);
94 static void	ibdm_handle_classportinfo(ibmf_handle_t,
95 		    ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
96 static void	ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *,
97 		    ibdm_dp_gidinfo_t *);
98 
99 static ibdm_hca_list_t		*ibdm_dup_hca_attr(ibdm_hca_list_t *);
100 static ibdm_ioc_info_t		*ibdm_dup_ioc_info(ibdm_ioc_info_t *,
101 				    ibdm_dp_gidinfo_t *gid_list);
102 static void			ibdm_probe_ioc(ib_guid_t, ib_guid_t, int);
103 static ibdm_ioc_info_t		*ibdm_is_ioc_present(ib_guid_t,
104 				    ibdm_dp_gidinfo_t *, int *);
105 static ibdm_port_attr_t		*ibdm_get_port_attr(ibt_async_event_t *,
106 				    ibdm_hca_list_t **);
107 static sa_node_record_t		*ibdm_get_node_records(ibmf_saa_handle_t,
108 				    size_t *, ib_guid_t);
109 static sa_portinfo_record_t	*ibdm_get_portinfo(ibmf_saa_handle_t, size_t *,
110 				    ib_lid_t);
111 static ibdm_dp_gidinfo_t	*ibdm_create_gid_info(ibdm_port_attr_t *,
112 				    ib_gid_t, ib_gid_t);
113 static ibdm_dp_gidinfo_t	*ibdm_find_gid(ib_guid_t, ib_guid_t);
114 static int	ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *, uint8_t);
115 static ibdm_ioc_info_t	*ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *, int);
116 static void	ibdm_saa_event_cb(ibmf_saa_handle_t, ibmf_saa_subnet_event_t,
117 		    ibmf_saa_event_details_t *, void *);
118 static void	ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *,
119     ibdm_dp_gidinfo_t *);
120 static ibdm_dp_gidinfo_t *ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *);
121 static void ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *,
122     ibdm_dp_gidinfo_t *);
123 static void ibdm_addto_gidlist(ibdm_gid_t **, ibdm_gid_t *);
124 static void ibdm_free_gid_list(ibdm_gid_t *);
125 static void ibdm_rescan_gidlist(ib_guid_t *ioc_guid);
126 static void ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *);
127 static void ibdm_saa_event_taskq(void *);
128 static void ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *);
129 static void ibdm_get_next_port(ibdm_hca_list_t **,
130     ibdm_port_attr_t **, int);
131 static void ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *,
132     ibdm_dp_gidinfo_t *);
133 static void ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *,
134     ibdm_hca_list_t *);
135 static void ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *);
136 static void ibdm_saa_handle_new_gid(void *);
137 static void ibdm_reset_all_dgids(ibmf_saa_handle_t);
138 static void ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *);
139 static void ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *);
140 static void ibdm_fill_srv_attr_mod(ib_mad_hdr_t *, ibdm_timeout_cb_args_t *);
141 static void ibdm_bump_transactionID(ibdm_dp_gidinfo_t *);
142 static ibdm_ioc_info_t	*ibdm_handle_prev_iou();
143 static int ibdm_serv_cmp(ibdm_srvents_info_t *, ibdm_srvents_info_t *,
144     int);
145 
146 int	ibdm_dft_timeout	= IBDM_DFT_TIMEOUT;
147 int	ibdm_dft_retry_cnt	= IBDM_DFT_NRETRIES;
148 #ifdef DEBUG
149 int	ibdm_ignore_saa_event = 0;
150 #endif
151 
152 /* Modload support */
153 static struct modlmisc ibdm_modlmisc	= {
154 	&mod_miscops,
155 	"InfiniBand Device Manager %I%",
156 };
157 
158 struct modlinkage ibdm_modlinkage = {
159 	MODREV_1,
160 	(void *)&ibdm_modlmisc,
161 	NULL
162 };
163 
164 static ibt_clnt_modinfo_t ibdm_ibt_modinfo = {
165 	IBTI_V2,
166 	IBT_DM,
167 	ibdm_event_hdlr,
168 	NULL,
169 	"ibdm"
170 };
171 
172 /* Global variables */
173 ibdm_t	ibdm;
174 int	ibdm_taskq_enable = IBDM_ENABLE_TASKQ_HANDLING;
175 char	*ibdm_string = "ibdm";
176 
177 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by cv",
178     ibdm.ibdm_dp_gidlist_head))
179 
180 /*
181  * _init
182  *	Loadable module init, called before any other module.
183  *	Initialize mutex
184  *	Register with IBTF
185  */
186 int
187 _init(void)
188 {
189 	int		err;
190 
191 	IBTF_DPRINTF_L4("ibdm", "\t_init: addr of ibdm %p", &ibdm);
192 
193 	if ((err = ibdm_init()) != IBDM_SUCCESS) {
194 		IBTF_DPRINTF_L2("ibdm", "_init: ibdm_init failed 0x%x", err);
195 		(void) ibdm_fini();
196 		return (DDI_FAILURE);
197 	}
198 
199 	if ((err = mod_install(&ibdm_modlinkage)) != 0) {
200 		IBTF_DPRINTF_L2("ibdm", "_init: mod_install failed 0x%x", err);
201 		(void) ibdm_fini();
202 	}
203 	return (err);
204 }
205 
206 
207 int
208 _fini(void)
209 {
210 	int err;
211 
212 	if ((err = ibdm_fini()) != IBDM_SUCCESS) {
213 		IBTF_DPRINTF_L2("ibdm", "_fini: ibdm_fini failed 0x%x", err);
214 		(void) ibdm_init();
215 		return (EBUSY);
216 	}
217 
218 	if ((err = mod_remove(&ibdm_modlinkage)) != 0) {
219 		IBTF_DPRINTF_L2("ibdm", "_fini: mod_remove failed 0x%x", err);
220 		(void) ibdm_init();
221 	}
222 	return (err);
223 }
224 
225 
226 int
227 _info(struct modinfo *modinfop)
228 {
229 	return (mod_info(&ibdm_modlinkage, modinfop));
230 }
231 
232 
233 /*
234  * ibdm_init():
235  * 	Register with IBTF
236  *	Allocate memory for the HCAs
237  *	Allocate minor-nodes for the HCAs
238  */
239 static int
240 ibdm_init(void)
241 {
242 	int			i, hca_count;
243 	ib_guid_t		*hca_guids;
244 	ibt_status_t		status;
245 
246 	IBTF_DPRINTF_L4("ibdm", "\tibdm_init:");
247 	if (!(ibdm.ibdm_state & IBDM_LOCKS_ALLOCED)) {
248 		mutex_init(&ibdm.ibdm_mutex, NULL, MUTEX_DEFAULT, NULL);
249 		mutex_init(&ibdm.ibdm_hl_mutex, NULL, MUTEX_DEFAULT, NULL);
250 		mutex_init(&ibdm.ibdm_ibnex_mutex, NULL, MUTEX_DEFAULT, NULL);
251 		mutex_enter(&ibdm.ibdm_mutex);
252 		ibdm.ibdm_state |= IBDM_LOCKS_ALLOCED;
253 	}
254 
255 	if (!(ibdm.ibdm_state & IBDM_IBT_ATTACHED)) {
256 		if ((status = ibt_attach(&ibdm_ibt_modinfo, NULL, NULL,
257 		    (void *)&ibdm.ibdm_ibt_clnt_hdl)) != IBT_SUCCESS) {
258 			IBTF_DPRINTF_L2("ibdm", "ibdm_init: ibt_attach "
259 			    "failed %x", status);
260 			mutex_exit(&ibdm.ibdm_mutex);
261 			return (IBDM_FAILURE);
262 		}
263 
264 		ibdm.ibdm_state |= IBDM_IBT_ATTACHED;
265 		mutex_exit(&ibdm.ibdm_mutex);
266 	}
267 
268 
269 	if (!(ibdm.ibdm_state & IBDM_HCA_ATTACHED)) {
270 		hca_count = ibt_get_hca_list(&hca_guids);
271 		IBTF_DPRINTF_L4("ibdm", "ibdm_init: num_hcas = %d", hca_count);
272 		for (i = 0; i < hca_count; i++)
273 			(void) ibdm_handle_hca_attach(hca_guids[i]);
274 		if (hca_count)
275 			ibt_free_hca_list(hca_guids, hca_count);
276 
277 		mutex_enter(&ibdm.ibdm_mutex);
278 		ibdm.ibdm_state |= IBDM_HCA_ATTACHED;
279 		mutex_exit(&ibdm.ibdm_mutex);
280 	}
281 
282 	if (!(ibdm.ibdm_state & IBDM_CVS_ALLOCED)) {
283 		cv_init(&ibdm.ibdm_probe_cv, NULL, CV_DRIVER, NULL);
284 		cv_init(&ibdm.ibdm_busy_cv, NULL, CV_DRIVER, NULL);
285 		mutex_enter(&ibdm.ibdm_mutex);
286 		ibdm.ibdm_state |= IBDM_CVS_ALLOCED;
287 		mutex_exit(&ibdm.ibdm_mutex);
288 	}
289 	return (IBDM_SUCCESS);
290 }
291 
292 
293 static int
294 ibdm_free_iou_info(ibdm_dp_gidinfo_t *gid_info, ibdm_iou_info_t **ioup)
295 {
296 	int			ii, k, niocs;
297 	size_t			size;
298 	ibdm_gid_t		*delete, *head;
299 	timeout_id_t		timeout_id;
300 	ibdm_ioc_info_t		*ioc;
301 	ibdm_iou_info_t		*gl_iou = *ioup;
302 
303 	ASSERT(mutex_owned(&gid_info->gl_mutex));
304 	if (gl_iou == NULL) {
305 		IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: No IOU");
306 		return (0);
307 	}
308 
309 	niocs = gl_iou->iou_info.iou_num_ctrl_slots;
310 	IBTF_DPRINTF_L4("ibdm", "\tfree_iou_info: gid_info = %p, niocs %d",
311 	    gid_info, niocs);
312 
313 	for (ii = 0; ii < niocs; ii++) {
314 		ioc = (ibdm_ioc_info_t *)&gl_iou->iou_ioc_info[ii];
315 
316 		/* handle the case where an ioc_timeout_id is scheduled */
317 		if (ioc->ioc_timeout_id) {
318 			timeout_id = ioc->ioc_timeout_id;
319 			mutex_exit(&gid_info->gl_mutex);
320 			IBTF_DPRINTF_L5("ibdm", "free_iou_info: "
321 			    "ioc_timeout_id = 0x%x", timeout_id);
322 			if (untimeout(timeout_id) == -1) {
323 				IBTF_DPRINTF_L2("ibdm", "free_iou_info: "
324 				    "untimeout ioc_timeout_id failed");
325 				mutex_enter(&gid_info->gl_mutex);
326 				return (-1);
327 			}
328 			mutex_enter(&gid_info->gl_mutex);
329 			ioc->ioc_timeout_id = 0;
330 		}
331 
332 		/* handle the case where an ioc_dc_timeout_id is scheduled */
333 		if (ioc->ioc_dc_timeout_id) {
334 			timeout_id = ioc->ioc_dc_timeout_id;
335 			mutex_exit(&gid_info->gl_mutex);
336 			IBTF_DPRINTF_L5("ibdm", "free_iou_info: "
337 			    "ioc_dc_timeout_id = 0x%x", timeout_id);
338 			if (untimeout(timeout_id) == -1) {
339 				IBTF_DPRINTF_L2("ibdm", "free_iou_info: "
340 				    "untimeout ioc_dc_timeout_id failed");
341 				mutex_enter(&gid_info->gl_mutex);
342 				return (-1);
343 			}
344 			mutex_enter(&gid_info->gl_mutex);
345 			ioc->ioc_dc_timeout_id = 0;
346 		}
347 
348 		/* handle the case where serv[k].se_timeout_id is scheduled */
349 		for (k = 0; k < ioc->ioc_profile.ioc_service_entries; k++) {
350 			if (ioc->ioc_serv[k].se_timeout_id) {
351 				timeout_id = ioc->ioc_serv[k].se_timeout_id;
352 				mutex_exit(&gid_info->gl_mutex);
353 				IBTF_DPRINTF_L5("ibdm", "free_iou_info: "
354 				    "ioc->ioc_serv[%d].se_timeout_id = 0x%x",
355 				    k, timeout_id);
356 				if (untimeout(timeout_id) == -1) {
357 					IBTF_DPRINTF_L2("ibdm", "free_iou_info:"
358 					    " untimeout se_timeout_id failed");
359 					mutex_enter(&gid_info->gl_mutex);
360 					return (-1);
361 				}
362 				mutex_enter(&gid_info->gl_mutex);
363 				ioc->ioc_serv[k].se_timeout_id = 0;
364 			}
365 		}
366 
367 		/* delete GID list in IOC */
368 		head = ioc->ioc_gid_list;
369 		while (head) {
370 			IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: "
371 			    "Deleting gid_list struct %p", head);
372 			delete = head;
373 			head = head->gid_next;
374 			kmem_free(delete, sizeof (ibdm_gid_t));
375 		}
376 		ioc->ioc_gid_list = NULL;
377 
378 		/* delete ioc_serv */
379 		size = ioc->ioc_profile.ioc_service_entries *
380 		    sizeof (ibdm_srvents_info_t);
381 		if (ioc->ioc_serv && size) {
382 			kmem_free(ioc->ioc_serv, size);
383 			ioc->ioc_serv = NULL;
384 		}
385 	}
386 
387 	IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: deleting IOU & IOC");
388 	size = sizeof (ibdm_iou_info_t) + niocs * sizeof (ibdm_ioc_info_t);
389 	kmem_free(gl_iou, size);
390 	*ioup = NULL;
391 	return (0);
392 }
393 
394 
395 /*
396  * ibdm_fini():
397  * 	Un-register with IBTF
398  *	De allocate memory for the GID info
399  */
400 static int
401 ibdm_fini()
402 {
403 	int			ii;
404 	ibdm_hca_list_t		*hca_list, *temp;
405 	ibdm_dp_gidinfo_t	*gid_info, *tmp;
406 	ibdm_gid_t		*head, *delete;
407 
408 	IBTF_DPRINTF_L4("ibdm", "\tibdm_fini");
409 
410 	mutex_enter(&ibdm.ibdm_hl_mutex);
411 	if (ibdm.ibdm_state & IBDM_IBT_ATTACHED) {
412 		if (ibt_detach(ibdm.ibdm_ibt_clnt_hdl) != IBT_SUCCESS) {
413 			IBTF_DPRINTF_L2("ibdm", "\t_fini: ibt_detach failed");
414 			mutex_exit(&ibdm.ibdm_hl_mutex);
415 			return (IBDM_FAILURE);
416 		}
417 		ibdm.ibdm_state &= ~IBDM_IBT_ATTACHED;
418 		ibdm.ibdm_ibt_clnt_hdl = NULL;
419 	}
420 
421 	hca_list = ibdm.ibdm_hca_list_head;
422 	IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: nhcas %d", ibdm.ibdm_hca_count);
423 	for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
424 		temp = hca_list;
425 		hca_list = hca_list->hl_next;
426 		IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: hca %p", temp);
427 		if (ibdm_uninit_hca(temp) != IBDM_SUCCESS) {
428 			IBTF_DPRINTF_L2("ibdm", "\tibdm_fini: "
429 			    "uninit_hca %p failed", temp);
430 			mutex_exit(&ibdm.ibdm_hl_mutex);
431 			return (IBDM_FAILURE);
432 		}
433 	}
434 	mutex_exit(&ibdm.ibdm_hl_mutex);
435 
436 	mutex_enter(&ibdm.ibdm_mutex);
437 	if (ibdm.ibdm_state & IBDM_HCA_ATTACHED)
438 		ibdm.ibdm_state &= ~IBDM_HCA_ATTACHED;
439 
440 	gid_info = ibdm.ibdm_dp_gidlist_head;
441 	while (gid_info) {
442 		mutex_enter(&gid_info->gl_mutex);
443 		(void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou);
444 		mutex_exit(&gid_info->gl_mutex);
445 		ibdm_delete_glhca_list(gid_info);
446 
447 		tmp = gid_info;
448 		gid_info = gid_info->gl_next;
449 		mutex_destroy(&tmp->gl_mutex);
450 		head = tmp->gl_gid;
451 		while (head) {
452 			IBTF_DPRINTF_L4("ibdm",
453 			    "\tibdm_fini: Deleting gid structs");
454 			delete = head;
455 			head = head->gid_next;
456 			kmem_free(delete, sizeof (ibdm_gid_t));
457 		}
458 		kmem_free(tmp, sizeof (ibdm_dp_gidinfo_t));
459 	}
460 	mutex_exit(&ibdm.ibdm_mutex);
461 
462 	if (ibdm.ibdm_state & IBDM_LOCKS_ALLOCED) {
463 		ibdm.ibdm_state &= ~IBDM_LOCKS_ALLOCED;
464 		mutex_destroy(&ibdm.ibdm_mutex);
465 		mutex_destroy(&ibdm.ibdm_hl_mutex);
466 		mutex_destroy(&ibdm.ibdm_ibnex_mutex);
467 	}
468 	if (ibdm.ibdm_state & IBDM_CVS_ALLOCED) {
469 		ibdm.ibdm_state &= ~IBDM_CVS_ALLOCED;
470 		cv_destroy(&ibdm.ibdm_probe_cv);
471 		cv_destroy(&ibdm.ibdm_busy_cv);
472 	}
473 	return (IBDM_SUCCESS);
474 }
475 
476 
477 /*
478  * ibdm_event_hdlr()
479  *
480  *	IBDM registers  this asynchronous event handler at the time of
481  *	ibt_attach. IBDM support the following async events. For other
482  *	event, simply returns success.
483  *	IBT_HCA_ATTACH_EVENT:
484  *		Retrieves the  information about all the port that are
485  *		present on this HCA,  allocates  the  port  attributes
486  *		structure  and calls IB  nexus  callback  routine with
487  *		the port attributes structure as an input argument.
488  *	IBT_HCA_DETACH_EVENT:
489  *		Retrieves the information about all the ports that are
490  *		present on  this HCA and  calls IB nexus callback with
491  *		port guid as an argument
492  *	IBT_EVENT_PORT_UP:
493  *		Register with IBMF and SA access
494  *		Setup IBMF receive callback routine
495  *	IBT_EVENT_PORT_DOWN:
496  *		Un-Register with IBMF and SA access
497  *		Teardown IBMF receive callback routine
498  */
499 /*ARGSUSED*/
500 static void
501 ibdm_event_hdlr(void *clnt_hdl,
502     ibt_hca_hdl_t hca_hdl, ibt_async_code_t code, ibt_async_event_t *event)
503 {
504 	ibdm_hca_list_t		*hca_list;
505 	ibdm_port_attr_t	*port;
506 	ibmf_saa_handle_t	port_sa_hdl;
507 
508 	IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: async code 0x%x", code);
509 
510 	switch (code) {
511 	case IBT_HCA_ATTACH_EVENT:	/* New HCA registered with IBTF */
512 		ibdm_handle_hca_attach(event->ev_hca_guid);
513 		break;
514 
515 	case IBT_HCA_DETACH_EVENT:	/* HCA unregistered with IBTF */
516 		ibdm_handle_hca_detach(event->ev_hca_guid);
517 		mutex_enter(&ibdm.ibdm_ibnex_mutex);
518 		if (ibdm.ibdm_ibnex_callback != NULL) {
519 			(*ibdm.ibdm_ibnex_callback)((void *)
520 			    &event->ev_hca_guid, IBDM_EVENT_HCA_REMOVED);
521 		}
522 		mutex_exit(&ibdm.ibdm_ibnex_mutex);
523 		break;
524 
525 	case IBT_EVENT_PORT_UP:
526 		IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_UP");
527 		mutex_enter(&ibdm.ibdm_hl_mutex);
528 		port = ibdm_get_port_attr(event, &hca_list);
529 		if (port == NULL) {
530 			IBTF_DPRINTF_L2("ibdm",
531 				"\tevent_hdlr: HCA not present");
532 			mutex_exit(&ibdm.ibdm_hl_mutex);
533 			break;
534 		}
535 		ibdm_initialize_port(port);
536 		hca_list->hl_nports_active++;
537 		mutex_exit(&ibdm.ibdm_hl_mutex);
538 		break;
539 
540 	case IBT_ERROR_PORT_DOWN:
541 		IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_DOWN");
542 		mutex_enter(&ibdm.ibdm_hl_mutex);
543 		port = ibdm_get_port_attr(event, &hca_list);
544 		if (port == NULL) {
545 			IBTF_DPRINTF_L2("ibdm",
546 				"\tevent_hdlr: HCA not present");
547 			mutex_exit(&ibdm.ibdm_hl_mutex);
548 			break;
549 		}
550 		hca_list->hl_nports_active--;
551 		port_sa_hdl = port->pa_sa_hdl;
552 		(void) ibdm_fini_port(port);
553 		port->pa_state = IBT_PORT_DOWN;
554 		mutex_exit(&ibdm.ibdm_hl_mutex);
555 		ibdm_reset_all_dgids(port_sa_hdl);
556 		break;
557 
558 	default:		/* Ignore all other events/errors */
559 		break;
560 	}
561 }
562 
563 
564 /*
565  * ibdm_initialize_port()
566  *	Register with IBMF
567  *	Register with SA access
568  *	Register a receive callback routine with IBMF. IBMF invokes
569  *	this routine whenever a MAD arrives at this port.
570  *	Update the port attributes
571  */
572 static void
573 ibdm_initialize_port(ibdm_port_attr_t *port)
574 {
575 	int				ii;
576 	uint_t				nports, size;
577 	uint_t				pkey_idx;
578 	ib_pkey_t			pkey;
579 	ibt_hca_portinfo_t		*pinfop;
580 	ibmf_register_info_t		ibmf_reg;
581 	ibmf_saa_subnet_event_args_t	event_args;
582 
583 	IBTF_DPRINTF_L4("ibdm", "\tinitialize_port:");
584 	ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
585 
586 	/* Check whether the port is active */
587 	if (ibt_get_port_state(port->pa_hca_hdl, port->pa_port_num, NULL,
588 	    NULL) != IBT_SUCCESS)
589 		return;
590 
591 	if (port->pa_sa_hdl != NULL)
592 		return;
593 
594 	if (ibt_query_hca_ports(port->pa_hca_hdl, port->pa_port_num,
595 	    &pinfop, &nports, &size) != IBT_SUCCESS) {
596 		/* This should not occur */
597 		port->pa_npkeys		= 0;
598 		port->pa_pkey_tbl	= NULL;
599 		return;
600 	}
601 	port->pa_sn_prefix = pinfop->p_sgid_tbl[0].gid_prefix;
602 
603 	port->pa_state		= pinfop->p_linkstate;
604 	port->pa_npkeys		= pinfop->p_pkey_tbl_sz;
605 	port->pa_pkey_tbl	= (ibdm_pkey_tbl_t *)kmem_zalloc(
606 	    port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP);
607 
608 	for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++)
609 		port->pa_pkey_tbl[pkey_idx].pt_pkey =
610 		    pinfop->p_pkey_tbl[pkey_idx];
611 
612 	ibt_free_portinfo(pinfop, size);
613 
614 	event_args.is_event_callback = ibdm_saa_event_cb;
615 	event_args.is_event_callback_arg = port;
616 	if (ibmf_sa_session_open(port->pa_port_guid, 0, &event_args,
617 	    IBMF_VERSION, 0, &port->pa_sa_hdl) != IBMF_SUCCESS) {
618 		IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: "
619 		    "sa access registration failed");
620 		return;
621 	}
622 	ibmf_reg.ir_ci_guid		= port->pa_hca_guid;
623 	ibmf_reg.ir_port_num		= port->pa_port_num;
624 	ibmf_reg.ir_client_class	= DEV_MGT_MANAGER;
625 
626 	if (ibmf_register(&ibmf_reg, IBMF_VERSION, 0, NULL, NULL,
627 	    &port->pa_ibmf_hdl, &port->pa_ibmf_caps) != IBMF_SUCCESS) {
628 		IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: "
629 		    "IBMF registration failed");
630 		(void) ibdm_fini_port(port);
631 		return;
632 	}
633 	if (ibmf_setup_async_cb(port->pa_ibmf_hdl, IBMF_QP_HANDLE_DEFAULT,
634 	    ibdm_ibmf_recv_cb, 0, 0) != IBMF_SUCCESS) {
635 		IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: "
636 		    "IBMF setup recv cb failed");
637 		(void) ibdm_fini_port(port);
638 		return;
639 	}
640 
641 	for (ii = 0; ii < port->pa_npkeys; ii++) {
642 		pkey = port->pa_pkey_tbl[ii].pt_pkey;
643 		if (IBDM_INVALID_PKEY(pkey)) {
644 			port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
645 			continue;
646 		}
647 		ibdm_port_attr_ibmf_init(port, pkey, ii);
648 	}
649 }
650 
651 
652 /*
653  * ibdm_port_attr_ibmf_init:
654  *	With IBMF - Alloc QP Handle and Setup Async callback
655  */
656 static void
657 ibdm_port_attr_ibmf_init(ibdm_port_attr_t *port, ib_pkey_t pkey, int ii)
658 {
659 	int ret;
660 
661 	if ((ret = ibmf_alloc_qp(port->pa_ibmf_hdl, pkey, IB_GSI_QKEY,
662 	    IBMF_ALT_QP_MAD_NO_RMPP, &port->pa_pkey_tbl[ii].pt_qp_hdl)) !=
663 	    IBMF_SUCCESS) {
664 		IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: "
665 		    "IBMF failed to alloc qp %d", ret);
666 		port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
667 		return;
668 	}
669 
670 	IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_init: QP handle is %p",
671 	    port->pa_ibmf_hdl);
672 
673 	if ((ret = ibmf_setup_async_cb(port->pa_ibmf_hdl,
674 	    port->pa_pkey_tbl[ii].pt_qp_hdl, ibdm_ibmf_recv_cb, 0, 0)) !=
675 	    IBMF_SUCCESS) {
676 		IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: "
677 		    "IBMF setup recv cb failed %d", ret);
678 		(void) ibmf_free_qp(port->pa_ibmf_hdl,
679 		    &port->pa_pkey_tbl[ii].pt_qp_hdl, 0);
680 		port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
681 	}
682 }
683 
684 
685 /*
686  * ibdm_get_port_attr()
687  *	Get port attributes from HCA guid and port number
688  *	Return pointer to ibdm_port_attr_t on Success
689  *	and NULL on failure
690  */
691 static ibdm_port_attr_t *
692 ibdm_get_port_attr(ibt_async_event_t *event, ibdm_hca_list_t **retval)
693 {
694 	ibdm_hca_list_t		*hca_list;
695 	ibdm_port_attr_t	*port_attr;
696 	int			ii;
697 
698 	IBTF_DPRINTF_L4("ibdm", "\tget_port_attr: port# %d", event->ev_port);
699 	ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
700 	hca_list = ibdm.ibdm_hca_list_head;
701 	while (hca_list) {
702 		if (hca_list->hl_hca_guid == event->ev_hca_guid) {
703 			for (ii = 0; ii < hca_list->hl_nports; ii++) {
704 				port_attr = &hca_list->hl_port_attr[ii];
705 				if (port_attr->pa_port_num == event->ev_port) {
706 					*retval = hca_list;
707 					return (port_attr);
708 				}
709 			}
710 		}
711 		hca_list = hca_list->hl_next;
712 	}
713 	return (NULL);
714 }
715 
716 
717 /*
718  * ibdm_update_port_attr()
719  *	Update the port attributes
720  */
721 static void
722 ibdm_update_port_attr(ibdm_port_attr_t *port)
723 {
724 	uint_t			nports, size;
725 	uint_t			pkey_idx;
726 	ibt_hca_portinfo_t	*portinfop;
727 
728 	IBTF_DPRINTF_L4("ibdm", "\tupdate_port_attr: Begin");
729 	if (ibt_query_hca_ports(port->pa_hca_hdl,
730 	    port->pa_port_num, &portinfop, &nports, &size) != IBT_SUCCESS) {
731 		/* This should not occur */
732 		port->pa_npkeys		= 0;
733 		port->pa_pkey_tbl	= NULL;
734 		return;
735 	}
736 	port->pa_sn_prefix = portinfop->p_sgid_tbl[0].gid_prefix;
737 
738 	port->pa_state		= portinfop->p_linkstate;
739 
740 	/*
741 	 * PKey information in portinfo valid only if port is
742 	 * ACTIVE. Bail out if not.
743 	 */
744 	if (port->pa_state != IBT_PORT_ACTIVE) {
745 		port->pa_npkeys		= 0;
746 		port->pa_pkey_tbl	= NULL;
747 		ibt_free_portinfo(portinfop, size);
748 		return;
749 	}
750 
751 	port->pa_npkeys		= portinfop->p_pkey_tbl_sz;
752 	port->pa_pkey_tbl	= (ibdm_pkey_tbl_t *)kmem_zalloc(
753 	    port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP);
754 
755 	for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++) {
756 		port->pa_pkey_tbl[pkey_idx].pt_pkey =
757 		    portinfop->p_pkey_tbl[pkey_idx];
758 	}
759 	ibt_free_portinfo(portinfop, size);
760 }
761 
762 
763 /*
764  * ibdm_handle_hca_attach()
765  */
766 static void
767 ibdm_handle_hca_attach(ib_guid_t hca_guid)
768 {
769 	uint_t			size;
770 	uint_t			ii, nports;
771 	ibt_status_t		status;
772 	ibt_hca_hdl_t		hca_hdl;
773 	ibt_hca_attr_t		*hca_attr;
774 	ibdm_hca_list_t		*hca_list, *temp;
775 	ibdm_port_attr_t	*port_attr;
776 	ibt_hca_portinfo_t	*portinfop;
777 
778 	IBTF_DPRINTF_L4("ibdm",
779 	    "\thandle_hca_attach: hca_guid = 0x%llX", hca_guid);
780 
781 	/* open the HCA first */
782 	if ((status = ibt_open_hca(ibdm.ibdm_ibt_clnt_hdl, hca_guid,
783 	    &hca_hdl)) != IBT_SUCCESS) {
784 		IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: "
785 		    "open_hca failed, status 0x%x", status);
786 		return;
787 	}
788 
789 	hca_attr = (ibt_hca_attr_t *)
790 	    kmem_alloc(sizeof (ibt_hca_attr_t), KM_SLEEP);
791 	/* ibt_query_hca always returns IBT_SUCCESS */
792 	(void) ibt_query_hca(hca_hdl, hca_attr);
793 
794 	IBTF_DPRINTF_L4("ibdm", "\tvid: 0x%x, pid: 0x%x, ver: 0x%x,"
795 	    " #ports: %d", hca_attr->hca_vendor_id, hca_attr->hca_device_id,
796 	    hca_attr->hca_version_id, hca_attr->hca_nports);
797 
798 	if ((status = ibt_query_hca_ports(hca_hdl, 0, &portinfop, &nports,
799 	    &size)) != IBT_SUCCESS) {
800 		IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: "
801 		    "ibt_query_hca_ports failed, status 0x%x", status);
802 		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
803 		(void) ibt_close_hca(hca_hdl);
804 		return;
805 	}
806 	hca_list = (ibdm_hca_list_t *)
807 	    kmem_zalloc((sizeof (ibdm_hca_list_t)), KM_SLEEP);
808 	hca_list->hl_port_attr = (ibdm_port_attr_t *)kmem_zalloc(
809 	    (sizeof (ibdm_port_attr_t) * hca_attr->hca_nports), KM_SLEEP);
810 	hca_list->hl_hca_guid = hca_attr->hca_node_guid;
811 	hca_list->hl_nports = hca_attr->hca_nports;
812 	hca_list->hl_attach_time = ddi_get_time();
813 	hca_list->hl_hca_hdl = hca_hdl;
814 
815 	/*
816 	 * Init a dummy port attribute for the HCA node
817 	 * This is for Per-HCA Node. Initialize port_attr :
818 	 * 	hca_guid & port_guid -> hca_guid
819 	 *	npkeys, pkey_tbl is NULL
820 	 *	port_num, sn_prefix is 0
821 	 *	vendorid, product_id, dev_version from HCA
822 	 *	pa_state is IBT_PORT_ACTIVE
823 	 */
824 	hca_list->hl_hca_port_attr = (ibdm_port_attr_t *)kmem_zalloc(
825 	    sizeof (ibdm_port_attr_t), KM_SLEEP);
826 	port_attr = hca_list->hl_hca_port_attr;
827 	port_attr->pa_vendorid  = hca_attr->hca_vendor_id;
828 	port_attr->pa_productid	= hca_attr->hca_device_id;
829 	port_attr->pa_dev_version = hca_attr->hca_version_id;
830 	port_attr->pa_hca_guid	= hca_attr->hca_node_guid;
831 	port_attr->pa_hca_hdl	= hca_list->hl_hca_hdl;
832 	port_attr->pa_port_guid	= hca_attr->hca_node_guid;
833 	port_attr->pa_state	= IBT_PORT_ACTIVE;
834 
835 
836 	for (ii = 0; ii < nports; ii++) {
837 		port_attr		= &hca_list->hl_port_attr[ii];
838 		port_attr->pa_vendorid	= hca_attr->hca_vendor_id;
839 		port_attr->pa_productid	= hca_attr->hca_device_id;
840 		port_attr->pa_dev_version = hca_attr->hca_version_id;
841 		port_attr->pa_hca_guid	= hca_attr->hca_node_guid;
842 		port_attr->pa_hca_hdl	= hca_list->hl_hca_hdl;
843 		port_attr->pa_port_guid	= portinfop[ii].p_sgid_tbl->gid_guid;
844 		port_attr->pa_sn_prefix	= portinfop[ii].p_sgid_tbl->gid_prefix;
845 		port_attr->pa_port_num	= portinfop[ii].p_port_num;
846 		port_attr->pa_state	= portinfop[ii].p_linkstate;
847 
848 		/*
849 		 * Register with IBMF, SA access when the port is in
850 		 * ACTIVE state. Also register a callback routine
851 		 * with IBMF to receive incoming DM MAD's.
852 		 * The IBDM event handler takes care of registration of
853 		 * port which are not active.
854 		 */
855 		IBTF_DPRINTF_L4("ibdm",
856 		    "\thandle_hca_attach: port guid %llx Port state 0x%x",
857 		    port_attr->pa_port_guid, portinfop[ii].p_linkstate);
858 
859 		if (portinfop[ii].p_linkstate == IBT_PORT_ACTIVE) {
860 			mutex_enter(&ibdm.ibdm_hl_mutex);
861 			hca_list->hl_nports_active++;
862 			ibdm_initialize_port(port_attr);
863 			mutex_exit(&ibdm.ibdm_hl_mutex);
864 		}
865 	}
866 	mutex_enter(&ibdm.ibdm_hl_mutex);
867 	for (temp = ibdm.ibdm_hca_list_head; temp; temp = temp->hl_next) {
868 		if (temp->hl_hca_guid == hca_guid) {
869 			IBTF_DPRINTF_L2("ibdm", "hca_attach: HCA %llX "
870 			    "already seen by IBDM", hca_guid);
871 			mutex_exit(&ibdm.ibdm_hl_mutex);
872 			(void) ibdm_uninit_hca(hca_list);
873 			return;
874 		}
875 	}
876 	ibdm.ibdm_hca_count++;
877 	if (ibdm.ibdm_hca_list_head == NULL) {
878 		ibdm.ibdm_hca_list_head = hca_list;
879 		ibdm.ibdm_hca_list_tail = hca_list;
880 	} else {
881 		ibdm.ibdm_hca_list_tail->hl_next = hca_list;
882 		ibdm.ibdm_hca_list_tail = hca_list;
883 	}
884 	mutex_exit(&ibdm.ibdm_hl_mutex);
885 	mutex_enter(&ibdm.ibdm_ibnex_mutex);
886 	if (ibdm.ibdm_ibnex_callback != NULL) {
887 		(*ibdm.ibdm_ibnex_callback)((void *)
888 		    &hca_guid, IBDM_EVENT_HCA_ADDED);
889 	}
890 	mutex_exit(&ibdm.ibdm_ibnex_mutex);
891 
892 	kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
893 	ibt_free_portinfo(portinfop, size);
894 }
895 
896 
897 /*
898  * ibdm_handle_hca_detach()
899  */
900 static void
901 ibdm_handle_hca_detach(ib_guid_t hca_guid)
902 {
903 	ibdm_hca_list_t		*head, *prev = NULL;
904 	size_t			len;
905 	ibdm_dp_gidinfo_t	*gidinfo;
906 
907 	IBTF_DPRINTF_L4("ibdm",
908 	    "\thandle_hca_detach: hca_guid = 0x%llx", hca_guid);
909 
910 	/* Make sure no probes are running */
911 	mutex_enter(&ibdm.ibdm_mutex);
912 	while (ibdm.ibdm_busy & IBDM_BUSY)
913 		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
914 	ibdm.ibdm_busy |= IBDM_BUSY;
915 	mutex_exit(&ibdm.ibdm_mutex);
916 
917 	mutex_enter(&ibdm.ibdm_hl_mutex);
918 	head = ibdm.ibdm_hca_list_head;
919 	while (head) {
920 		if (head->hl_hca_guid == hca_guid) {
921 			if (prev == NULL)
922 				ibdm.ibdm_hca_list_head = head->hl_next;
923 			else
924 				prev->hl_next = head->hl_next;
925 			ibdm.ibdm_hca_count--;
926 			break;
927 		}
928 		prev = head;
929 		head = head->hl_next;
930 	}
931 	mutex_exit(&ibdm.ibdm_hl_mutex);
932 	if (ibdm_uninit_hca(head) != IBDM_SUCCESS)
933 		(void) ibdm_handle_hca_attach(hca_guid);
934 
935 	/*
936 	 * Now clean up the HCA lists in the gidlist.
937 	 */
938 	for (gidinfo = ibdm.ibdm_dp_gidlist_head; gidinfo; gidinfo =
939 	    gidinfo->gl_next) {
940 		prev = NULL;
941 		head = gidinfo->gl_hca_list;
942 		while (head) {
943 			if (head->hl_hca_guid == hca_guid) {
944 				if (prev == NULL)
945 					gidinfo->gl_hca_list =
946 					    head->hl_next;
947 				else
948 					prev->hl_next = head->hl_next;
949 
950 				len = sizeof (ibdm_hca_list_t) +
951 				    (head->hl_nports *
952 				    sizeof (ibdm_port_attr_t));
953 				kmem_free(head, len);
954 
955 				break;
956 			}
957 			prev = head;
958 			head = head->hl_next;
959 		}
960 	}
961 
962 	mutex_enter(&ibdm.ibdm_mutex);
963 	ibdm.ibdm_busy &= ~IBDM_BUSY;
964 	cv_broadcast(&ibdm.ibdm_busy_cv);
965 	mutex_exit(&ibdm.ibdm_mutex);
966 }
967 
968 
969 static int
970 ibdm_uninit_hca(ibdm_hca_list_t *head)
971 {
972 	int			ii;
973 	ibdm_port_attr_t	*port_attr;
974 
975 	for (ii = 0; ii < head->hl_nports; ii++) {
976 		port_attr = &head->hl_port_attr[ii];
977 		if (ibdm_fini_port(port_attr) != IBDM_SUCCESS) {
978 			IBTF_DPRINTF_L2("ibdm", "uninit_hca: HCA %p port 0x%x "
979 			    "ibdm_fini_port() failed", head, ii);
980 			return (IBDM_FAILURE);
981 		}
982 	}
983 	if (head->hl_hca_hdl)
984 		if (ibt_close_hca(head->hl_hca_hdl) != IBT_SUCCESS)
985 			return (IBDM_FAILURE);
986 	kmem_free(head->hl_port_attr,
987 	    head->hl_nports * sizeof (ibdm_port_attr_t));
988 	kmem_free(head->hl_hca_port_attr, sizeof (ibdm_port_attr_t));
989 	kmem_free(head, sizeof (ibdm_hca_list_t));
990 	return (IBDM_SUCCESS);
991 }
992 
993 
994 /*
995  * For each port on the HCA,
996  *	1) Teardown IBMF receive callback function
997  *	2) Unregister with IBMF
998  *	3) Unregister with SA access
999  */
1000 static int
1001 ibdm_fini_port(ibdm_port_attr_t *port_attr)
1002 {
1003 	int	ii, ibmf_status;
1004 
1005 	for (ii = 0; ii < port_attr->pa_npkeys; ii++) {
1006 		if (port_attr->pa_pkey_tbl == NULL)
1007 			break;
1008 		if (!port_attr->pa_pkey_tbl[ii].pt_qp_hdl)
1009 			continue;
1010 		if (ibdm_port_attr_ibmf_fini(port_attr, ii) != IBDM_SUCCESS) {
1011 			IBTF_DPRINTF_L4("ibdm", "\tfini_port: "
1012 			    "ibdm_port_attr_ibmf_fini failed for "
1013 			    "port pkey 0x%x", ii);
1014 			return (IBDM_FAILURE);
1015 		}
1016 	}
1017 
1018 	if (port_attr->pa_ibmf_hdl) {
1019 		ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl,
1020 		    IBMF_QP_HANDLE_DEFAULT, 0);
1021 		if (ibmf_status != IBMF_SUCCESS) {
1022 			IBTF_DPRINTF_L4("ibdm", "\tfini_port: "
1023 			    "ibmf_tear_down_async_cb failed %d", ibmf_status);
1024 			return (IBDM_FAILURE);
1025 		}
1026 
1027 		ibmf_status = ibmf_unregister(&port_attr->pa_ibmf_hdl, 0);
1028 		if (ibmf_status != IBMF_SUCCESS) {
1029 			IBTF_DPRINTF_L4("ibdm", "\tfini_port: "
1030 			    "ibmf_unregister failed %d", ibmf_status);
1031 			return (IBDM_FAILURE);
1032 		}
1033 
1034 		port_attr->pa_ibmf_hdl = NULL;
1035 	}
1036 
1037 	if (port_attr->pa_sa_hdl) {
1038 		ibmf_status = ibmf_sa_session_close(&port_attr->pa_sa_hdl, 0);
1039 		if (ibmf_status != IBMF_SUCCESS) {
1040 			IBTF_DPRINTF_L4("ibdm", "\tfini_port: "
1041 			    "ibmf_sa_session_close failed %d", ibmf_status);
1042 			return (IBDM_FAILURE);
1043 		}
1044 		port_attr->pa_sa_hdl = NULL;
1045 	}
1046 
1047 	if (port_attr->pa_pkey_tbl != NULL) {
1048 		kmem_free(port_attr->pa_pkey_tbl,
1049 		    port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t));
1050 		port_attr->pa_pkey_tbl = NULL;
1051 		port_attr->pa_npkeys = 0;
1052 	}
1053 
1054 	return (IBDM_SUCCESS);
1055 }
1056 
1057 
1058 /*
1059  * ibdm_port_attr_ibmf_fini:
1060  *	With IBMF - Tear down Async callback and free QP Handle
1061  */
1062 static int
1063 ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *port_attr, int ii)
1064 {
1065 	int ibmf_status;
1066 
1067 	IBTF_DPRINTF_L5("ibdm", "\tport_attr_ibmf_fini:");
1068 
1069 	if (port_attr->pa_pkey_tbl[ii].pt_qp_hdl) {
1070 		ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl,
1071 		    port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0);
1072 		if (ibmf_status != IBMF_SUCCESS) {
1073 			IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: "
1074 			    "ibmf_tear_down_async_cb failed %d", ibmf_status);
1075 			return (IBDM_FAILURE);
1076 		}
1077 		ibmf_status = ibmf_free_qp(port_attr->pa_ibmf_hdl,
1078 		    &port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0);
1079 		if (ibmf_status != IBMF_SUCCESS) {
1080 			IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: "
1081 			    "ibmf_free_qp failed %d", ibmf_status);
1082 			return (IBDM_FAILURE);
1083 		}
1084 		port_attr->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
1085 	}
1086 	return (IBDM_SUCCESS);
1087 }
1088 
1089 
1090 /*
1091  * ibdm_gid_decr_pending:
1092  *	decrement gl_pending_cmds. If zero wakeup sleeping threads
1093  */
1094 static void
1095 ibdm_gid_decr_pending(ibdm_dp_gidinfo_t *gidinfo)
1096 {
1097 	mutex_enter(&ibdm.ibdm_mutex);
1098 	mutex_enter(&gidinfo->gl_mutex);
1099 	if (--gidinfo->gl_pending_cmds == 0) {
1100 		/*
1101 		 * Handle DGID getting removed.
1102 		 */
1103 		if (gidinfo->gl_disconnected) {
1104 			mutex_exit(&gidinfo->gl_mutex);
1105 			mutex_exit(&ibdm.ibdm_mutex);
1106 
1107 			IBTF_DPRINTF_L3(ibdm_string, "\tgid_decr_pending: "
1108 			    "gidinfo %p hot removal", gidinfo);
1109 			ibdm_delete_gidinfo(gidinfo);
1110 
1111 			mutex_enter(&ibdm.ibdm_mutex);
1112 			ibdm.ibdm_ngid_probes_in_progress--;
1113 			ibdm_wait_probe_completion();
1114 			mutex_exit(&ibdm.ibdm_mutex);
1115 			return;
1116 		}
1117 		mutex_exit(&gidinfo->gl_mutex);
1118 		mutex_exit(&ibdm.ibdm_mutex);
1119 		ibdm_notify_newgid_iocs(gidinfo);
1120 		mutex_enter(&ibdm.ibdm_mutex);
1121 		mutex_enter(&gidinfo->gl_mutex);
1122 
1123 		ibdm.ibdm_ngid_probes_in_progress--;
1124 		ibdm_wait_probe_completion();
1125 	}
1126 	mutex_exit(&gidinfo->gl_mutex);
1127 	mutex_exit(&ibdm.ibdm_mutex);
1128 }
1129 
1130 
1131 /*
1132  * ibdm_wait_probe_completion:
1133  *	wait for probing to complete
1134  */
1135 static void
1136 ibdm_wait_probe_completion(void)
1137 {
1138 	ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
1139 	if (ibdm.ibdm_ngid_probes_in_progress) {
1140 		IBTF_DPRINTF_L4("ibdm",	"\twait for probe complete");
1141 		ibdm.ibdm_busy |= IBDM_PROBE_IN_PROGRESS;
1142 		while (ibdm.ibdm_busy & IBDM_PROBE_IN_PROGRESS)
1143 			cv_wait(&ibdm.ibdm_probe_cv, &ibdm.ibdm_mutex);
1144 	}
1145 }
1146 
1147 
1148 /*
1149  * ibdm_wakeup_probe_gid_cv:
1150  *	wakeup waiting threads (based on ibdm_ngid_probes_in_progress)
1151  */
1152 static void
1153 ibdm_wakeup_probe_gid_cv(void)
1154 {
1155 	ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
1156 	if (!ibdm.ibdm_ngid_probes_in_progress) {
1157 		IBTF_DPRINTF_L4("ibdm", "wakeup_probe_gid_thread: Wakeup");
1158 		ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS;
1159 		cv_broadcast(&ibdm.ibdm_probe_cv);
1160 	}
1161 
1162 }
1163 
1164 
1165 /*
1166  * ibdm_sweep_fabric(reprobe_flag)
1167  *	Find all possible Managed IOU's and their IOC's that are visible
1168  *	to the host. The algorithm used is as follows
1169  *
1170  *	Send a "bus walk" request for each port on the host HCA to SA access
1171  *	SA returns complete set of GID's that are reachable from
1172  *	source port. This is done in parallel.
1173  *
1174  *	Initialize GID state to IBDM_GID_PROBE_NOT_DONE
1175  *
1176  *	Sort the GID list and eliminate duplicate GID's
1177  *		1) Use DGID for sorting
1178  *		2) use PortGuid for sorting
1179  *			Send SA query to retrieve NodeRecord and
1180  *			extract PortGuid from that.
1181  *
1182  *	Set GID state to IBDM_GID_PROBE_FAILED to all the ports that dont
1183  *	support DM MAD's
1184  *		Send a "Portinfo" query to get the port capabilities and
1185  *		then check for DM MAD's support
1186  *
1187  *	Send "ClassPortInfo" request for all the GID's in parallel,
1188  *	set the GID state to IBDM_GET_CLASSPORTINFO and wait on the
1189  *	cv_signal to complete.
1190  *
1191  *	When DM agent on the remote GID sends back the response, IBMF
1192  *	invokes DM callback routine.
1193  *
1194  *	If the response is proper, send "IOUnitInfo" request and set
1195  *	GID state to IBDM_GET_IOUNITINFO.
1196  *
1197  *	If the response is proper, send "IocProfileInfo" request to
1198  *	all the IOC simultaneously and set GID state to IBDM_GET_IOC_DETAILS.
1199  *
1200  *	Send request to get Service entries simultaneously
1201  *
1202  *	Signal the waiting thread when received response for all the commands.
1203  *
1204  *	Set the GID state to IBDM_GID_PROBE_FAILED when received a error
1205  *	response during the probing period.
1206  *
1207  *	Note:
1208  *	ibdm.ibdm_ngid_probes_in_progress and ibdm_gid_list_t:gl_pending_cmds
1209  *	keep track of number commands in progress at any point of time.
1210  *	MAD transaction ID is used to identify a particular GID
1211  *	TBD: Consider registering the IBMF receive callback on demand
1212  *
1213  *	Note: This routine must be called with ibdm.ibdm_mutex held
1214  *	TBD: Re probe the failure GID (for certain failures) when requested
1215  *	     for fabric sweep next time
1216  *
1217  *	Parameters : If reprobe_flag is set, All IOCs will be reprobed.
1218  */
1219 static void
1220 ibdm_sweep_fabric(int reprobe_flag)
1221 {
1222 	int			ii;
1223 	int			new_paths = 0;
1224 	uint8_t			niocs;
1225 	taskqid_t		tid;
1226 	ibdm_ioc_info_t		*ioc;
1227 	ibdm_hca_list_t		*hca_list = NULL;
1228 	ibdm_port_attr_t	*port = NULL;
1229 	ibdm_dp_gidinfo_t 	*gid_info;
1230 
1231 	IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: Enter");
1232 	ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
1233 
1234 	/*
1235 	 * Check whether a sweep already in progress. If so, just
1236 	 * wait for the fabric sweep to complete
1237 	 */
1238 	while (ibdm.ibdm_busy & IBDM_BUSY)
1239 		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
1240 	ibdm.ibdm_busy |= IBDM_BUSY;
1241 	mutex_exit(&ibdm.ibdm_mutex);
1242 
1243 	ibdm_dump_sweep_fabric_timestamp(0);
1244 
1245 	/* Rescan the GID list for any removed GIDs for reprobe */
1246 	if (reprobe_flag)
1247 		ibdm_rescan_gidlist(NULL);
1248 
1249 	/*
1250 	 * Get list of all the ports reachable from the local known HCA
1251 	 * ports which are active
1252 	 */
1253 	mutex_enter(&ibdm.ibdm_hl_mutex);
1254 	for (ibdm_get_next_port(&hca_list, &port, 1); port;
1255 	    ibdm_get_next_port(&hca_list, &port, 1)) {
1256 		/*
1257 		 * Get PATHS to all the reachable ports from
1258 		 * SGID and update the global ibdm structure.
1259 		 */
1260 		new_paths = ibdm_get_reachable_ports(port, hca_list);
1261 		ibdm.ibdm_ngids += new_paths;
1262 	}
1263 	mutex_exit(&ibdm.ibdm_hl_mutex);
1264 
1265 	mutex_enter(&ibdm.ibdm_mutex);
1266 	ibdm.ibdm_ngid_probes_in_progress += ibdm.ibdm_ngids;
1267 	mutex_exit(&ibdm.ibdm_mutex);
1268 
1269 	/* Send a request to probe GIDs asynchronously. */
1270 	for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
1271 	    gid_info = gid_info->gl_next) {
1272 		mutex_enter(&gid_info->gl_mutex);
1273 		gid_info->gl_reprobe_flag = reprobe_flag;
1274 		mutex_exit(&gid_info->gl_mutex);
1275 
1276 		/* process newly encountered GIDs */
1277 		tid = taskq_dispatch(system_taskq, ibdm_probe_gid_thread,
1278 		    (void *)gid_info, TQ_NOSLEEP);
1279 		IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: gid_info = %p"
1280 		    " taskq_id = %x", gid_info, tid);
1281 		/* taskq failed to dispatch call it directly */
1282 		if (tid == NULL)
1283 			ibdm_probe_gid_thread((void *)gid_info);
1284 	}
1285 
1286 	mutex_enter(&ibdm.ibdm_mutex);
1287 	ibdm_wait_probe_completion();
1288 
1289 	/*
1290 	 * Update the properties, if reprobe_flag is set
1291 	 * Skip if gl_reprobe_flag is set, this will be
1292 	 * a re-inserted / new GID, for which notifications
1293 	 * have already been send.
1294 	 */
1295 	if (reprobe_flag) {
1296 		for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
1297 		    gid_info = gid_info->gl_next) {
1298 			if (gid_info->gl_iou == NULL)
1299 				continue;
1300 			if (gid_info->gl_reprobe_flag) {
1301 				gid_info->gl_reprobe_flag = 0;
1302 				continue;
1303 			}
1304 
1305 			niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
1306 			for (ii = 0; ii < niocs; ii++) {
1307 				ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii);
1308 				if (ioc)
1309 					ibdm_reprobe_update_port_srv(ioc,
1310 					    gid_info);
1311 			}
1312 		}
1313 	} else if (ibdm.ibdm_prev_iou) {
1314 		ibdm_ioc_info_t	*ioc_list;
1315 
1316 		/*
1317 		 * Get the list of IOCs which have changed.
1318 		 * If any IOCs have changed, Notify IBNexus
1319 		 */
1320 		ibdm.ibdm_prev_iou = 0;
1321 		ioc_list = ibdm_handle_prev_iou();
1322 		if (ioc_list) {
1323 			if (ibdm.ibdm_ibnex_callback != NULL) {
1324 				(*ibdm.ibdm_ibnex_callback)(
1325 				    (void *)ioc_list,
1326 				    IBDM_EVENT_IOC_PROP_UPDATE);
1327 			}
1328 		}
1329 	}
1330 
1331 	ibdm_dump_sweep_fabric_timestamp(1);
1332 
1333 	ibdm.ibdm_busy &= ~IBDM_BUSY;
1334 	cv_broadcast(&ibdm.ibdm_busy_cv);
1335 	IBTF_DPRINTF_L5("ibdm", "\tsweep_fabric: EXIT");
1336 }
1337 
1338 
1339 /*
1340  * ibdm_probe_gid_thread:
1341  *	thread that does the actual work for sweeping the fabric
1342  *	for a given GID
1343  */
1344 static void
1345 ibdm_probe_gid_thread(void *args)
1346 {
1347 	int			reprobe_flag;
1348 	ib_guid_t		node_guid;
1349 	ib_guid_t		port_guid;
1350 	ibdm_dp_gidinfo_t	*gid_info;
1351 
1352 	gid_info = (ibdm_dp_gidinfo_t *)args;
1353 	reprobe_flag = gid_info->gl_reprobe_flag;
1354 	IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: gid_info = %p, flag = %d",
1355 	    gid_info, reprobe_flag);
1356 	ASSERT(gid_info != NULL);
1357 	ASSERT(gid_info->gl_pending_cmds == 0);
1358 
1359 	if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE &&
1360 	    reprobe_flag == 0) {
1361 		/*
1362 		 * This GID may have been already probed. Send
1363 		 * in a CLP to check if IOUnitInfo changed?
1364 		 * Explicitly set gl_reprobe_flag to 0 so that
1365 		 * IBnex is not notified on completion
1366 		 */
1367 		if (gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) {
1368 			IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: "
1369 			    "get new IOCs information");
1370 			mutex_enter(&gid_info->gl_mutex);
1371 			gid_info->gl_pending_cmds++;
1372 			gid_info->gl_state = IBDM_GET_IOUNITINFO;
1373 			gid_info->gl_reprobe_flag = 0;
1374 			mutex_exit(&gid_info->gl_mutex);
1375 			if (ibdm_send_iounitinfo(gid_info) != IBDM_SUCCESS) {
1376 				mutex_enter(&gid_info->gl_mutex);
1377 				gid_info->gl_pending_cmds = 0;
1378 				mutex_exit(&gid_info->gl_mutex);
1379 				mutex_enter(&ibdm.ibdm_mutex);
1380 				--ibdm.ibdm_ngid_probes_in_progress;
1381 				ibdm_wakeup_probe_gid_cv();
1382 				mutex_exit(&ibdm.ibdm_mutex);
1383 			}
1384 		} else {
1385 			mutex_enter(&ibdm.ibdm_mutex);
1386 			--ibdm.ibdm_ngid_probes_in_progress;
1387 			ibdm_wakeup_probe_gid_cv();
1388 			mutex_exit(&ibdm.ibdm_mutex);
1389 		}
1390 		return;
1391 	} else if (reprobe_flag && gid_info->gl_state ==
1392 	    IBDM_GID_PROBING_COMPLETE) {
1393 		/*
1394 		 * Reprobe all IOCs for the GID which has completed
1395 		 * probe. Skip other port GIDs to same IOU.
1396 		 * Explicitly set gl_reprobe_flag to 0 so that
1397 		 * IBnex is not notified on completion
1398 		 */
1399 		ibdm_ioc_info_t *ioc_info;
1400 		uint8_t		niocs, ii;
1401 
1402 		ASSERT(gid_info->gl_iou);
1403 		mutex_enter(&gid_info->gl_mutex);
1404 		niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
1405 		gid_info->gl_state = IBDM_GET_IOC_DETAILS;
1406 		gid_info->gl_pending_cmds += niocs;
1407 		gid_info->gl_reprobe_flag = 0;
1408 		mutex_exit(&gid_info->gl_mutex);
1409 		for (ii = 0; ii < niocs; ii++) {
1410 			uchar_t			slot_info;
1411 			ib_dm_io_unitinfo_t	*giou_info;
1412 
1413 			/*
1414 			 * Check whether IOC is present in the slot
1415 			 * Series of nibbles (in the field
1416 			 * iou_ctrl_list) represents a slot in the
1417 			 * IOU.
1418 			 * Byte format: 76543210
1419 			 * Bits 0-3 of first byte represent Slot 2
1420 			 * bits 4-7 of first byte represent slot 1,
1421 			 * bits 0-3 of second byte represent slot 4
1422 			 * and so on
1423 			 * Each 4-bit nibble has the following meaning
1424 			 * 0x0 : IOC not installed
1425 			 * 0x1 : IOC is present
1426 			 * 0xf : Slot does not exist
1427 			 * and all other values are reserved.
1428 			 */
1429 			ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii);
1430 			giou_info = &gid_info->gl_iou->iou_info;
1431 			slot_info = giou_info->iou_ctrl_list[(ii/2)];
1432 			if ((ii % 2) == 0)
1433 				slot_info = (slot_info >> 4);
1434 
1435 			if ((slot_info & 0xf) != 1) {
1436 				ioc_info->ioc_state =
1437 				    IBDM_IOC_STATE_PROBE_FAILED;
1438 				ibdm_gid_decr_pending(gid_info);
1439 				continue;
1440 			}
1441 
1442 			if (ibdm_send_ioc_profile(gid_info, ii) !=
1443 			    IBDM_SUCCESS) {
1444 				ibdm_gid_decr_pending(gid_info);
1445 			}
1446 		}
1447 
1448 		return;
1449 	} else if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) {
1450 		mutex_enter(&ibdm.ibdm_mutex);
1451 		--ibdm.ibdm_ngid_probes_in_progress;
1452 		ibdm_wakeup_probe_gid_cv();
1453 		mutex_exit(&ibdm.ibdm_mutex);
1454 		return;
1455 	}
1456 
1457 	mutex_enter(&gid_info->gl_mutex);
1458 	gid_info->gl_pending_cmds++;
1459 	gid_info->gl_state = IBDM_GET_CLASSPORTINFO;
1460 	mutex_exit(&gid_info->gl_mutex);
1461 
1462 	/*
1463 	 * Check whether the destination GID supports DM agents. If
1464 	 * not, stop probing the GID and continue with the next GID
1465 	 * in the list.
1466 	 */
1467 	if (ibdm_is_dev_mgt_supported(gid_info) != IBDM_SUCCESS) {
1468 		mutex_enter(&gid_info->gl_mutex);
1469 		gid_info->gl_pending_cmds = 0;
1470 		gid_info->gl_state = IBDM_GID_PROBING_FAILED;
1471 		mutex_exit(&gid_info->gl_mutex);
1472 		ibdm_delete_glhca_list(gid_info);
1473 		mutex_enter(&ibdm.ibdm_mutex);
1474 		--ibdm.ibdm_ngid_probes_in_progress;
1475 		ibdm_wakeup_probe_gid_cv();
1476 		mutex_exit(&ibdm.ibdm_mutex);
1477 		return;
1478 	}
1479 
1480 	/* Get the nodeguid and portguid of the port */
1481 	if (ibdm_get_node_port_guids(gid_info->gl_sa_hdl, gid_info->gl_dlid,
1482 	    &node_guid, &port_guid) != IBDM_SUCCESS) {
1483 		mutex_enter(&gid_info->gl_mutex);
1484 		gid_info->gl_pending_cmds = 0;
1485 		gid_info->gl_state = IBDM_GID_PROBING_FAILED;
1486 		mutex_exit(&gid_info->gl_mutex);
1487 		ibdm_delete_glhca_list(gid_info);
1488 		mutex_enter(&ibdm.ibdm_mutex);
1489 		--ibdm.ibdm_ngid_probes_in_progress;
1490 		ibdm_wakeup_probe_gid_cv();
1491 		mutex_exit(&ibdm.ibdm_mutex);
1492 		return;
1493 	}
1494 
1495 	/*
1496 	 * Check whether we already knew about this NodeGuid
1497 	 * If so, do not probe the GID and continue with the
1498 	 * next  GID  in the gid  list. Set the GID state to
1499 	 * probing done.
1500 	 */
1501 	mutex_enter(&ibdm.ibdm_mutex);
1502 	gid_info->gl_nodeguid = node_guid;
1503 	gid_info->gl_portguid = port_guid;
1504 	if (ibdm_check_dest_nodeguid(gid_info) != NULL) {
1505 		mutex_exit(&ibdm.ibdm_mutex);
1506 		mutex_enter(&gid_info->gl_mutex);
1507 		gid_info->gl_pending_cmds = 0;
1508 		gid_info->gl_state = IBDM_GID_PROBING_SKIPPED;
1509 		mutex_exit(&gid_info->gl_mutex);
1510 		ibdm_delete_glhca_list(gid_info);
1511 		mutex_enter(&ibdm.ibdm_mutex);
1512 		--ibdm.ibdm_ngid_probes_in_progress;
1513 		ibdm_wakeup_probe_gid_cv();
1514 		mutex_exit(&ibdm.ibdm_mutex);
1515 		return;
1516 	}
1517 	ibdm_add_to_gl_gid(gid_info, gid_info);
1518 	mutex_exit(&ibdm.ibdm_mutex);
1519 
1520 	/*
1521 	 * New or reinserted GID : Enable notification to IBnex
1522 	 */
1523 	mutex_enter(&gid_info->gl_mutex);
1524 	gid_info->gl_reprobe_flag = 1;
1525 	mutex_exit(&gid_info->gl_mutex);
1526 
1527 	/*
1528 	 * Send ClassPortInfo request to the GID asynchronously.
1529 	 */
1530 	if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) {
1531 		mutex_enter(&gid_info->gl_mutex);
1532 		gid_info->gl_state = IBDM_GID_PROBING_FAILED;
1533 		gid_info->gl_pending_cmds = 0;
1534 		mutex_exit(&gid_info->gl_mutex);
1535 		ibdm_delete_glhca_list(gid_info);
1536 		mutex_enter(&ibdm.ibdm_mutex);
1537 		--ibdm.ibdm_ngid_probes_in_progress;
1538 		ibdm_wakeup_probe_gid_cv();
1539 		mutex_exit(&ibdm.ibdm_mutex);
1540 		return;
1541 	}
1542 }
1543 
1544 
1545 /*
1546  * ibdm_check_dest_nodeguid
1547  *	Searches for the NodeGuid in the GID list
1548  *	Returns matching gid_info if found and otherwise NULL
1549  *
1550  *	This function is called to handle new GIDs discovered
1551  *	during device sweep / probe or for GID_AVAILABLE event.
1552  *
1553  *	Parameter :
1554  *		gid_info	GID to check
1555  */
1556 static ibdm_dp_gidinfo_t *
1557 ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *gid_info)
1558 {
1559 	ibdm_dp_gidinfo_t	*gid_list;
1560 	ibdm_gid_t		*tmp;
1561 
1562 	IBTF_DPRINTF_L4("ibdm", "\tcheck_dest_nodeguid");
1563 
1564 	gid_list = ibdm.ibdm_dp_gidlist_head;
1565 	while (gid_list) {
1566 		if ((gid_list != gid_info) &&
1567 		    (gid_info->gl_nodeguid == gid_list->gl_nodeguid)) {
1568 			IBTF_DPRINTF_L4("ibdm",
1569 			    "\tcheck_dest_nodeguid: NodeGuid is present");
1570 
1571 			/* Add to gid_list */
1572 			tmp = kmem_zalloc(sizeof (ibdm_gid_t),
1573 			    KM_SLEEP);
1574 			tmp->gid_dgid_hi = gid_info->gl_dgid_hi;
1575 			tmp->gid_dgid_lo = gid_info->gl_dgid_lo;
1576 			tmp->gid_next = gid_list->gl_gid;
1577 			gid_list->gl_gid = tmp;
1578 			gid_list->gl_ngids++;
1579 			return (gid_list);
1580 		}
1581 
1582 		gid_list = gid_list->gl_next;
1583 	}
1584 
1585 	return (NULL);
1586 }
1587 
1588 
1589 /*
1590  * ibdm_is_dev_mgt_supported
1591  *	Get the PortInfo attribute (SA Query)
1592  *	Check "CompatabilityMask" field in the Portinfo.
1593  *	Return IBDM_SUCCESS if DM MAD's supported (if bit 19 set)
1594  *	by the port, otherwise IBDM_FAILURE
1595  */
1596 static int
1597 ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *gid_info)
1598 {
1599 	int			ret;
1600 	size_t			length = 0;
1601 	sa_portinfo_record_t	req, *resp = NULL;
1602 	ibmf_saa_access_args_t	qargs;
1603 
1604 	bzero(&req, sizeof (sa_portinfo_record_t));
1605 	req.EndportLID	= gid_info->gl_dlid;
1606 
1607 	qargs.sq_attr_id	= SA_PORTINFORECORD_ATTRID;
1608 	qargs.sq_access_type	= IBMF_SAA_RETRIEVE;
1609 	qargs.sq_component_mask = SA_PORTINFO_COMPMASK_PORTLID;
1610 	qargs.sq_template	= &req;
1611 	qargs.sq_callback	= NULL;
1612 	qargs.sq_callback_arg	= NULL;
1613 
1614 	ret = ibmf_sa_access(gid_info->gl_sa_hdl,
1615 	    &qargs, 0, &length, (void **)&resp);
1616 
1617 	if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) {
1618 		IBTF_DPRINTF_L2("ibdm", "\tis_dev_mgt_supported:"
1619 		    "failed to get PORTINFO attribute %d", ret);
1620 		return (IBDM_FAILURE);
1621 	}
1622 
1623 	if (resp->PortInfo.CapabilityMask & SM_CAP_MASK_IS_DM_SUPPD) {
1624 		IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: SUPPD !!");
1625 		ret = IBDM_SUCCESS;
1626 	} else {
1627 		IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: "
1628 		    "Not SUPPD !!, cap 0x%x", resp->PortInfo.CapabilityMask);
1629 		ret = IBDM_FAILURE;
1630 	}
1631 	kmem_free(resp, length);
1632 	return (ret);
1633 }
1634 
1635 
1636 /*
1637  * ibdm_get_node_port_guids()
1638  *	Get the NodeInfoRecord of the port
1639  *	Save NodeGuid and PortGUID values in the GID list structure.
1640  *	Return IBDM_SUCCESS/IBDM_FAILURE
1641  */
1642 static int
1643 ibdm_get_node_port_guids(ibmf_saa_handle_t sa_hdl, ib_lid_t dlid,
1644     ib_guid_t *node_guid, ib_guid_t *port_guid)
1645 {
1646 	int			ret;
1647 	size_t			length = 0;
1648 	sa_node_record_t	req, *resp = NULL;
1649 	ibmf_saa_access_args_t	qargs;
1650 
1651 	IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids");
1652 
1653 	bzero(&req, sizeof (sa_node_record_t));
1654 	req.LID = dlid;
1655 
1656 	qargs.sq_attr_id	= SA_NODERECORD_ATTRID;
1657 	qargs.sq_access_type	= IBMF_SAA_RETRIEVE;
1658 	qargs.sq_component_mask = SA_NODEINFO_COMPMASK_NODELID;
1659 	qargs.sq_template	= &req;
1660 	qargs.sq_callback	= NULL;
1661 	qargs.sq_callback_arg	= NULL;
1662 
1663 	ret = ibmf_sa_access(sa_hdl, &qargs, 0, &length, (void **)&resp);
1664 	if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) {
1665 		IBTF_DPRINTF_L2("ibdm", "\tget_node_port_guids:"
1666 		    " SA Retrieve Failed: %d", ret);
1667 		return (IBDM_FAILURE);
1668 	}
1669 	IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids: NodeGuid %llx Port"
1670 	    "GUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.NodeGUID);
1671 
1672 	*node_guid = resp->NodeInfo.NodeGUID;
1673 	*port_guid = resp->NodeInfo.PortGUID;
1674 	kmem_free(resp, length);
1675 	return (IBDM_SUCCESS);
1676 }
1677 
1678 
1679 /*
1680  * ibdm_get_reachable_ports()
1681  *	Get list of the destination GID (and its path  records) by
1682  *	querying the SA access.
1683  *
1684  *	Returns Number paths
1685  */
1686 static int
1687 ibdm_get_reachable_ports(ibdm_port_attr_t *portinfo, ibdm_hca_list_t *hca)
1688 {
1689 	uint_t			ii, jj, nrecs;
1690 	uint_t			npaths = 0;
1691 	size_t			length;
1692 	ib_gid_t		sgid;
1693 	ibdm_pkey_tbl_t		*pkey_tbl;
1694 	sa_path_record_t	*result;
1695 	sa_path_record_t	*precp;
1696 	ibdm_dp_gidinfo_t	*gid_info;
1697 
1698 	ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
1699 	IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: portinfo %p", portinfo);
1700 
1701 	sgid.gid_prefix = portinfo->pa_sn_prefix;
1702 	sgid.gid_guid	= portinfo->pa_port_guid;
1703 
1704 	/* get reversible paths */
1705 	if (portinfo->pa_sa_hdl && ibmf_saa_paths_from_gid(portinfo->pa_sa_hdl,
1706 	    sgid, IBMF_SAA_PKEY_WC, B_TRUE, 0, &nrecs, &length, &result)
1707 	    != IBMF_SUCCESS) {
1708 		IBTF_DPRINTF_L2("ibdm",
1709 		    "\tget_reachable_ports: Getting path records failed");
1710 		return (0);
1711 	}
1712 
1713 	for (ii = 0; ii < nrecs; ii++) {
1714 		precp = &result[ii];
1715 		if ((gid_info = ibdm_check_dgid(precp->DGID.gid_guid,
1716 		    precp->DGID.gid_prefix)) != NULL) {
1717 			IBTF_DPRINTF_L5("ibdm", "\tget_reachable_ports: "
1718 			    "Already exists nrecs %d, ii %d", nrecs, ii);
1719 			ibdm_addto_glhcalist(gid_info, hca);
1720 			continue;
1721 		}
1722 		/*
1723 		 * This is a new GID. Allocate a GID structure and
1724 		 * initialize the structure
1725 		 * gl_state is initialized to IBDM_GID_PROBE_NOT_DONE (0)
1726 		 * by kmem_zalloc call
1727 		 */
1728 		gid_info = kmem_zalloc(sizeof (ibdm_dp_gidinfo_t), KM_SLEEP);
1729 		mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL);
1730 		gid_info->gl_dgid_hi		= precp->DGID.gid_prefix;
1731 		gid_info->gl_dgid_lo		= precp->DGID.gid_guid;
1732 		gid_info->gl_sgid_hi		= precp->SGID.gid_prefix;
1733 		gid_info->gl_sgid_lo		= precp->SGID.gid_guid;
1734 		gid_info->gl_p_key		= precp->P_Key;
1735 		gid_info->gl_sa_hdl		= portinfo->pa_sa_hdl;
1736 		gid_info->gl_ibmf_hdl		= portinfo->pa_ibmf_hdl;
1737 		gid_info->gl_slid		= precp->SLID;
1738 		gid_info->gl_dlid		= precp->DLID;
1739 		gid_info->gl_transactionID	= (++ibdm.ibdm_transactionID)
1740 		    << IBDM_GID_TRANSACTIONID_SHIFT;
1741 		gid_info->gl_min_transactionID  = gid_info->gl_transactionID;
1742 		gid_info->gl_max_transactionID  = (ibdm.ibdm_transactionID +1)
1743 		    << IBDM_GID_TRANSACTIONID_SHIFT;
1744 		ibdm_addto_glhcalist(gid_info,  hca);
1745 
1746 		ibdm_dump_path_info(precp);
1747 
1748 		gid_info->gl_qp_hdl = NULL;
1749 		ASSERT(portinfo->pa_pkey_tbl != NULL &&
1750 		    portinfo->pa_npkeys != 0);
1751 
1752 		for (jj = 0; jj < portinfo->pa_npkeys; jj++) {
1753 			pkey_tbl = &portinfo->pa_pkey_tbl[jj];
1754 			if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) &&
1755 			    (pkey_tbl->pt_qp_hdl != NULL)) {
1756 				gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl;
1757 				break;
1758 			}
1759 		}
1760 
1761 		/*
1762 		 * QP handle for GID not initialized. No matching Pkey
1763 		 * was found!! ibdm should *not* hit this case. Flag an
1764 		 * error and drop the GID if ibdm does encounter this.
1765 		 */
1766 		if (gid_info->gl_qp_hdl == NULL) {
1767 			IBTF_DPRINTF_L2(ibdm_string,
1768 			    "\tget_reachable_ports: No matching Pkey");
1769 			ibdm_delete_gidinfo(gid_info);
1770 			continue;
1771 		}
1772 		if (ibdm.ibdm_dp_gidlist_head == NULL) {
1773 			ibdm.ibdm_dp_gidlist_head = gid_info;
1774 			ibdm.ibdm_dp_gidlist_tail = gid_info;
1775 		} else {
1776 			ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info;
1777 			gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail;
1778 			ibdm.ibdm_dp_gidlist_tail = gid_info;
1779 		}
1780 		npaths++;
1781 	}
1782 	kmem_free(result, length);
1783 	IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: npaths = %d", npaths);
1784 	return (npaths);
1785 }
1786 
1787 
1788 /*
1789  * ibdm_check_dgid()
1790  *	Look in the global list to check whether we know this DGID already
1791  *	Return IBDM_GID_PRESENT/IBDM_GID_NOT_PRESENT
1792  */
1793 static ibdm_dp_gidinfo_t *
1794 ibdm_check_dgid(ib_guid_t guid, ib_sn_prefix_t prefix)
1795 {
1796 	ibdm_dp_gidinfo_t	*gid_list;
1797 
1798 	for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
1799 	    gid_list = gid_list->gl_next) {
1800 		if ((guid == gid_list->gl_dgid_lo) &&
1801 		    (prefix == gid_list->gl_dgid_hi)) {
1802 			break;
1803 		}
1804 	}
1805 	return (gid_list);
1806 }
1807 
1808 
1809 /*
1810  * ibdm_find_gid()
1811  *	Look in the global list to find a GID entry with matching
1812  *	port & node GUID.
1813  *	Return pointer to gidinfo if found, else return NULL
1814  */
1815 static ibdm_dp_gidinfo_t *
1816 ibdm_find_gid(ib_guid_t nodeguid, ib_guid_t portguid)
1817 {
1818 	ibdm_dp_gidinfo_t	*gid_list;
1819 
1820 	IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid(%llx, %llx)\n",
1821 	    nodeguid, portguid);
1822 
1823 	for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
1824 	    gid_list = gid_list->gl_next) {
1825 		if ((portguid == gid_list->gl_portguid) &&
1826 		    (nodeguid == gid_list->gl_nodeguid)) {
1827 			break;
1828 		}
1829 	}
1830 
1831 	IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid : returned %p\n",
1832 	    gid_list);
1833 	return (gid_list);
1834 }
1835 
1836 
1837 /*
1838  * ibdm_send_classportinfo()
1839  *	Send classportinfo request. When the request is completed
1840  *	IBMF calls ibdm_classportinfo_cb routine to inform about
1841  *	the completion.
1842  *	Returns IBDM_SUCCESS/IBDM_FAILURE
1843  */
1844 static int
1845 ibdm_send_classportinfo(ibdm_dp_gidinfo_t *gid_info)
1846 {
1847 	ibmf_msg_t		*msg;
1848 	ib_mad_hdr_t		*hdr;
1849 	ibdm_timeout_cb_args_t	*cb_args;
1850 
1851 	IBTF_DPRINTF_L4("ibdm",
1852 	    "\tsend_classportinfo: gid info 0x%p", gid_info);
1853 
1854 	/*
1855 	 * Send command to get classportinfo attribute. Allocate a IBMF
1856 	 * packet and initialize the packet.
1857 	 */
1858 	if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
1859 	    &msg) != IBMF_SUCCESS) {
1860 		IBTF_DPRINTF_L4("ibdm", "\tsend_classportinfo: pkt alloc fail");
1861 		return (IBDM_FAILURE);
1862 	}
1863 
1864 	ibdm_alloc_send_buffers(msg);
1865 
1866 	msg->im_local_addr.ia_local_lid		= gid_info->gl_slid;
1867 	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
1868 	msg->im_local_addr.ia_remote_qno	= 1;
1869 	msg->im_local_addr.ia_p_key		= gid_info->gl_p_key;
1870 	msg->im_local_addr.ia_q_key		= IB_GSI_QKEY;
1871 
1872 	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
1873 	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
1874 	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
1875 	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
1876 	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
1877 	hdr->Status		= 0;
1878 	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
1879 	hdr->AttributeID	= h2b16(IB_DM_ATTR_CLASSPORTINFO);
1880 	hdr->AttributeModifier	= 0;
1881 
1882 	cb_args = &gid_info->gl_cpi_cb_args;
1883 	cb_args->cb_gid_info = gid_info;
1884 	cb_args->cb_retry_count	= ibdm_dft_retry_cnt;
1885 	cb_args->cb_req_type = IBDM_REQ_TYPE_CLASSPORTINFO;
1886 
1887 	gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
1888 	    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
1889 
1890 	IBTF_DPRINTF_L5("ibdm", "\tsend_classportinfo: "
1891 	    "timeout id %x", gid_info->gl_timeout_id);
1892 
1893 	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
1894 	    msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
1895 		IBTF_DPRINTF_L2("ibdm",
1896 		    "\tsend_classportinfo: ibmf send failed");
1897 		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
1898 	}
1899 
1900 	return (IBDM_SUCCESS);
1901 }
1902 
1903 
1904 /*
1905  * ibdm_handle_classportinfo()
1906  *	Invoked by the IBMF when the classportinfo request is completed.
1907  */
1908 static void
1909 ibdm_handle_classportinfo(ibmf_handle_t ibmf_hdl,
1910     ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
1911 {
1912 	void			*data;
1913 	timeout_id_t		timeout_id;
1914 	ib_mad_hdr_t		*hdr;
1915 	ibdm_mad_classportinfo_t *cpi;
1916 
1917 	IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo:ibmf hdl "
1918 	    "%p msg %p gid info %p", ibmf_hdl, msg, gid_info);
1919 
1920 	if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO) {
1921 		IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo: "
1922 		    "Not a ClassPortInfo resp");
1923 		*flag |= IBDM_IBMF_PKT_UNEXP_RESP;
1924 		return;
1925 	}
1926 
1927 	/*
1928 	 * Verify whether timeout handler is created/active.
1929 	 * If created/ active,  cancel the timeout  handler
1930 	 */
1931 	mutex_enter(&gid_info->gl_mutex);
1932 	ibdm_bump_transactionID(gid_info);
1933 	if (gid_info->gl_state != IBDM_GET_CLASSPORTINFO) {
1934 		IBTF_DPRINTF_L2("ibdm", "\thandle_classportinfo:DUP resp");
1935 		*flag |= IBDM_IBMF_PKT_DUP_RESP;
1936 		mutex_exit(&gid_info->gl_mutex);
1937 		return;
1938 	}
1939 	gid_info->gl_iou_cb_args.cb_req_type = 0;
1940 	if (gid_info->gl_timeout_id) {
1941 		timeout_id = gid_info->gl_timeout_id;
1942 		mutex_exit(&gid_info->gl_mutex);
1943 		IBTF_DPRINTF_L5("ibdm", "handle_ioclassportinfo: "
1944 		    "gl_timeout_id = 0x%x", timeout_id);
1945 		if (untimeout(timeout_id) == -1) {
1946 			IBTF_DPRINTF_L2("ibdm", "handle_classportinfo: "
1947 			    "untimeout gl_timeout_id failed");
1948 		}
1949 		mutex_enter(&gid_info->gl_mutex);
1950 		gid_info->gl_timeout_id = 0;
1951 	}
1952 	gid_info->gl_state = IBDM_GET_IOUNITINFO;
1953 	gid_info->gl_pending_cmds++;
1954 	mutex_exit(&gid_info->gl_mutex);
1955 
1956 	data = msg->im_msgbufs_recv.im_bufs_cl_data;
1957 	cpi = (ibdm_mad_classportinfo_t *)data;
1958 
1959 	/*
1960 	 * Cache the "RespTimeValue" and redirection information in the
1961 	 * global gid list data structure. This cached information will
1962 	 * be used to send any further requests to the GID.
1963 	 */
1964 	gid_info->gl_resp_timeout	=
1965 		(b2h32(cpi->RespTimeValue) & 0x1F);
1966 
1967 	gid_info->gl_redirected		= ((IBDM_IN_IBMFMSG_STATUS(msg) &
1968 	    MAD_STATUS_REDIRECT_REQUIRED) ? B_TRUE: B_FALSE);
1969 	gid_info->gl_redirect_dlid	= b2h16(cpi->RedirectLID);
1970 	gid_info->gl_redirect_QP	= (b2h32(cpi->RedirectQP) & 0xffffff);
1971 	gid_info->gl_redirect_pkey	= b2h16(cpi->RedirectP_Key);
1972 	gid_info->gl_redirect_qkey	= b2h32(cpi->RedirectQ_Key);
1973 	gid_info->gl_redirectGID_hi	= b2h64(cpi->RedirectGID_hi);
1974 	gid_info->gl_redirectGID_lo	= b2h64(cpi->RedirectGID_lo);
1975 
1976 	ibdm_dump_classportinfo(cpi);
1977 
1978 	/*
1979 	 * Send IOUnitInfo request
1980 	 * Reuse previously allocated IBMF packet for sending ClassPortInfo
1981 	 * Check whether DM agent on the remote node requested redirection
1982 	 * If so, send the request to the redirect DGID/DLID/PKEY/QP.
1983 	 */
1984 	ibdm_alloc_send_buffers(msg);
1985 	msg->im_local_addr.ia_local_lid	= gid_info->gl_slid;
1986 	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
1987 
1988 	if (gid_info->gl_redirected == B_TRUE) {
1989 		if (gid_info->gl_redirect_dlid != 0) {
1990 			msg->im_local_addr.ia_remote_lid =
1991 					gid_info->gl_redirect_dlid;
1992 		}
1993 		msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
1994 		msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
1995 		msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
1996 	} else {
1997 		msg->im_local_addr.ia_remote_qno = 1;
1998 		msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
1999 		msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
2000 	}
2001 
2002 	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
2003 	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
2004 	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
2005 	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
2006 	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
2007 	hdr->Status		= 0;
2008 	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
2009 	hdr->AttributeID	= h2b16(IB_DM_ATTR_IO_UNITINFO);
2010 	hdr->AttributeModifier	= 0;
2011 
2012 	gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO;
2013 	gid_info->gl_iou_cb_args.cb_gid_info = gid_info;
2014 	gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt;
2015 
2016 	gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2017 	    &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2018 
2019 	IBTF_DPRINTF_L5("ibdm", "handle_classportinfo:"
2020 	    "timeout %x", gid_info->gl_timeout_id);
2021 
2022 	if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, NULL,
2023 	    ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) != IBMF_SUCCESS) {
2024 		IBTF_DPRINTF_L2("ibdm",
2025 		    "\thandle_classportinfo: msg transport failed");
2026 		ibdm_ibmf_send_cb(ibmf_hdl, msg, &gid_info->gl_iou_cb_args);
2027 	}
2028 	(*flag) |= IBDM_IBMF_PKT_REUSED;
2029 }
2030 
2031 
2032 /*
2033  * ibdm_send_iounitinfo:
2034  *	Sends a DM request to get IOU unitinfo.
2035  */
2036 static int
2037 ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *gid_info)
2038 {
2039 	ibmf_msg_t	*msg;
2040 	ib_mad_hdr_t	*hdr;
2041 
2042 	IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: gid info 0x%p", gid_info);
2043 
2044 	/*
2045 	 * Send command to get iounitinfo attribute. Allocate a IBMF
2046 	 * packet and initialize the packet.
2047 	 */
2048 	if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, &msg) !=
2049 	    IBMF_SUCCESS) {
2050 		IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: pkt alloc fail");
2051 		return (IBDM_FAILURE);
2052 	}
2053 
2054 	mutex_enter(&gid_info->gl_mutex);
2055 	ibdm_bump_transactionID(gid_info);
2056 	mutex_exit(&gid_info->gl_mutex);
2057 
2058 
2059 	ibdm_alloc_send_buffers(msg);
2060 	msg->im_local_addr.ia_local_lid		= gid_info->gl_slid;
2061 	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
2062 	msg->im_local_addr.ia_remote_qno	= 1;
2063 	msg->im_local_addr.ia_p_key		= gid_info->gl_p_key;
2064 	msg->im_local_addr.ia_q_key		= IB_GSI_QKEY;
2065 
2066 	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
2067 	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
2068 	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
2069 	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
2070 	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
2071 	hdr->Status		= 0;
2072 	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
2073 	hdr->AttributeID	= h2b16(IB_DM_ATTR_IO_UNITINFO);
2074 	hdr->AttributeModifier	= 0;
2075 
2076 	gid_info->gl_iou_cb_args.cb_gid_info = gid_info;
2077 	gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt;
2078 	gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO;
2079 
2080 	gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2081 	    &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2082 
2083 	IBTF_DPRINTF_L5("ibdm", "send_iouunitinfo:"
2084 	    "timeout %x", gid_info->gl_timeout_id);
2085 
2086 	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg,
2087 	    NULL, ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) !=
2088 	    IBMF_SUCCESS) {
2089 		IBTF_DPRINTF_L2("ibdm", "\tsend_iounitinfo: ibmf send failed");
2090 		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl,
2091 		    msg, &gid_info->gl_iou_cb_args);
2092 	}
2093 	return (IBDM_SUCCESS);
2094 }
2095 
2096 /*
2097  * ibdm_handle_iounitinfo()
2098  *	Invoked by the IBMF when IO Unitinfo request is completed.
2099  */
2100 static void
2101 ibdm_handle_iounitinfo(ibmf_handle_t ibmf_hdl,
2102     ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
2103 {
2104 	int			ii, first = B_TRUE;
2105 	int			num_iocs;
2106 	size_t			size;
2107 	uchar_t			slot_info;
2108 	timeout_id_t		timeout_id;
2109 	ib_mad_hdr_t		*hdr;
2110 	ibdm_ioc_info_t		*ioc_info;
2111 	ib_dm_io_unitinfo_t	*iou_info;
2112 	ib_dm_io_unitinfo_t	*giou_info;
2113 	ibdm_timeout_cb_args_t	*cb_args;
2114 
2115 	IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo:"
2116 	    " ibmf hdl %p pkt %p gid info %p", ibmf_hdl, msg, gid_info);
2117 
2118 	if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_IO_UNITINFO) {
2119 		IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: "
2120 		    "Unexpected response");
2121 		(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
2122 		return;
2123 	}
2124 
2125 	mutex_enter(&gid_info->gl_mutex);
2126 	if (gid_info->gl_state != IBDM_GET_IOUNITINFO) {
2127 		IBTF_DPRINTF_L4("ibdm",
2128 		    "\thandle_iounitinfo: DUP resp");
2129 		mutex_exit(&gid_info->gl_mutex);
2130 		(*flag) = IBDM_IBMF_PKT_DUP_RESP;
2131 		return;
2132 	}
2133 	gid_info->gl_iou_cb_args.cb_req_type = 0;
2134 	if (gid_info->gl_timeout_id) {
2135 		timeout_id = gid_info->gl_timeout_id;
2136 		mutex_exit(&gid_info->gl_mutex);
2137 		IBTF_DPRINTF_L5("ibdm", "handle_iounitinfo: "
2138 		    "gl_timeout_id = 0x%x", timeout_id);
2139 		if (untimeout(timeout_id) == -1) {
2140 			IBTF_DPRINTF_L2("ibdm", "handle_iounitinfo: "
2141 			    "untimeout gl_timeout_id failed");
2142 		}
2143 		mutex_enter(&gid_info->gl_mutex);
2144 		gid_info->gl_timeout_id = 0;
2145 	}
2146 	gid_info->gl_state = IBDM_GET_IOC_DETAILS;
2147 
2148 	iou_info = IBDM_IN_IBMFMSG2IOU(msg);
2149 	ibdm_dump_iounitinfo(iou_info);
2150 	num_iocs = iou_info->iou_num_ctrl_slots;
2151 	/*
2152 	 * check if number of IOCs reported is zero? if yes, return.
2153 	 * when num_iocs are reported zero internal IOC database needs
2154 	 * to be updated. To ensure that save the number of IOCs in
2155 	 * the new field "gl_num_iocs". Use a new field instead of
2156 	 * "giou_info->iou_num_ctrl_slots" as that would prevent
2157 	 * an unnecessary kmem_alloc/kmem_free when num_iocs is 0.
2158 	 */
2159 	if (num_iocs == 0 && gid_info->gl_num_iocs == 0) {
2160 		IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: no IOC's");
2161 		mutex_exit(&gid_info->gl_mutex);
2162 		return;
2163 	}
2164 	IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: num_iocs = %d", num_iocs);
2165 
2166 	/*
2167 	 * if there is an existing gl_iou (IOU has been probed before)
2168 	 * check if the "iou_changeid" is same as saved entry in
2169 	 * "giou_info->iou_changeid".
2170 	 * (note: this logic can prevent IOC enumeration if a given
2171 	 * vendor doesn't support setting iou_changeid field for its IOU)
2172 	 *
2173 	 * if there is an existing gl_iou and iou_changeid has changed :
2174 	 * free up existing gl_iou info and its related structures.
2175 	 * reallocate gl_iou info all over again.
2176 	 * if we donot free this up; then this leads to memory leaks
2177 	 */
2178 	if (gid_info->gl_iou) {
2179 		giou_info = &gid_info->gl_iou->iou_info;
2180 		if (b2h16(iou_info->iou_changeid) ==
2181 		    giou_info->iou_changeid) {
2182 			IBTF_DPRINTF_L3("ibdm",
2183 			    "\thandle_iounitinfo: no IOCs changed");
2184 			gid_info->gl_state = IBDM_GID_PROBING_COMPLETE;
2185 			mutex_exit(&gid_info->gl_mutex);
2186 			return;
2187 		}
2188 
2189 		/*
2190 		 * Store the iou info as prev_iou to be used after
2191 		 * sweep is done.
2192 		 */
2193 		ASSERT(gid_info->gl_prev_iou == NULL);
2194 		IBTF_DPRINTF_L4(ibdm_string,
2195 		    "\thandle_iounitinfo: setting gl_prev_iou %p",
2196 		    gid_info->gl_prev_iou);
2197 		gid_info->gl_prev_iou = gid_info->gl_iou;
2198 		ibdm.ibdm_prev_iou = 1;
2199 		gid_info->gl_iou = NULL;
2200 	}
2201 
2202 	size = sizeof (ibdm_iou_info_t) + num_iocs * sizeof (ibdm_ioc_info_t);
2203 	gid_info->gl_iou = (ibdm_iou_info_t *)kmem_zalloc(size, KM_SLEEP);
2204 	giou_info = &gid_info->gl_iou->iou_info;
2205 	gid_info->gl_iou->iou_ioc_info = (ibdm_ioc_info_t *)
2206 		((char *)gid_info->gl_iou + sizeof (ibdm_iou_info_t));
2207 
2208 	giou_info->iou_num_ctrl_slots	= gid_info->gl_num_iocs	= num_iocs;
2209 	giou_info->iou_flag		= iou_info->iou_flag;
2210 	bcopy(iou_info->iou_ctrl_list, giou_info->iou_ctrl_list, 128);
2211 	giou_info->iou_changeid	= b2h16(iou_info->iou_changeid);
2212 	gid_info->gl_pending_cmds += num_iocs;
2213 	gid_info->gl_pending_cmds += 1; /* for diag code */
2214 	mutex_exit(&gid_info->gl_mutex);
2215 
2216 	if (ibdm_get_diagcode(gid_info, 0) != IBDM_SUCCESS) {
2217 		mutex_enter(&gid_info->gl_mutex);
2218 		gid_info->gl_pending_cmds--;
2219 		mutex_exit(&gid_info->gl_mutex);
2220 	}
2221 	/*
2222 	 * Parallelize getting IOC controller profiles from here.
2223 	 * Allocate IBMF packets and send commands to get IOC profile for
2224 	 * each IOC present on the IOU.
2225 	 */
2226 	for (ii = 0; ii < num_iocs; ii++) {
2227 		/*
2228 		 * Check whether IOC is present in the slot
2229 		 * Series of nibbles (in the field iou_ctrl_list) represents
2230 		 * a slot in the IOU.
2231 		 * Byte format: 76543210
2232 		 * Bits 0-3 of first byte represent Slot 2
2233 		 * bits 4-7 of first byte represent slot 1,
2234 		 * bits 0-3 of second byte represent slot 4 and so on
2235 		 * Each 4-bit nibble has the following meaning
2236 		 * 0x0 : IOC not installed
2237 		 * 0x1 : IOC is present
2238 		 * 0xf : Slot does not exist
2239 		 * and all other values are reserved.
2240 		 */
2241 		ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii);
2242 		slot_info = giou_info->iou_ctrl_list[(ii/2)];
2243 		if ((ii % 2) == 0)
2244 			slot_info = (slot_info >> 4);
2245 
2246 		if ((slot_info & 0xf) != 1) {
2247 			IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: "
2248 			    "No IOC is present in the slot = %d", ii);
2249 			ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_FAILED;
2250 			mutex_enter(&gid_info->gl_mutex);
2251 			gid_info->gl_pending_cmds--;
2252 			mutex_exit(&gid_info->gl_mutex);
2253 			continue;
2254 		}
2255 
2256 		mutex_enter(&gid_info->gl_mutex);
2257 		ibdm_bump_transactionID(gid_info);
2258 		mutex_exit(&gid_info->gl_mutex);
2259 
2260 		/*
2261 		 * Re use the already allocated packet (for IOUnitinfo) to
2262 		 * send the first IOC controller attribute. Allocate new
2263 		 * IBMF packets for the rest of the IOC's
2264 		 */
2265 		if (first != B_TRUE) {
2266 			msg = NULL;
2267 			if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP,
2268 			    &msg) != IBMF_SUCCESS) {
2269 				IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: "
2270 				    "IBMF packet allocation failed");
2271 				mutex_enter(&gid_info->gl_mutex);
2272 				gid_info->gl_pending_cmds--;
2273 				mutex_exit(&gid_info->gl_mutex);
2274 				continue;
2275 			}
2276 
2277 		}
2278 
2279 		/* allocate send buffers for all messages */
2280 		ibdm_alloc_send_buffers(msg);
2281 
2282 		msg->im_local_addr.ia_local_lid	= gid_info->gl_slid;
2283 		msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
2284 		if (gid_info->gl_redirected == B_TRUE) {
2285 			if (gid_info->gl_redirect_dlid != 0) {
2286 				msg->im_local_addr.ia_remote_lid =
2287 						gid_info->gl_redirect_dlid;
2288 			}
2289 			msg->im_local_addr.ia_remote_qno =
2290 						gid_info->gl_redirect_QP;
2291 			msg->im_local_addr.ia_p_key =
2292 						gid_info->gl_redirect_pkey;
2293 			msg->im_local_addr.ia_q_key =
2294 						gid_info->gl_redirect_qkey;
2295 		} else {
2296 			msg->im_local_addr.ia_remote_qno = 1;
2297 			msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
2298 			msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
2299 		}
2300 
2301 		hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
2302 		hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
2303 		hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
2304 		hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
2305 		hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
2306 		hdr->Status		= 0;
2307 		hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
2308 		hdr->AttributeID	= h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE);
2309 		hdr->AttributeModifier 	= h2b32(ii + 1);
2310 
2311 		ioc_info->ioc_state	= IBDM_IOC_STATE_PROBE_INVALID;
2312 		cb_args			= &ioc_info->ioc_cb_args;
2313 		cb_args->cb_gid_info	= gid_info;
2314 		cb_args->cb_retry_count	= ibdm_dft_retry_cnt;
2315 		cb_args->cb_req_type	= IBDM_REQ_TYPE_IOCINFO;
2316 		cb_args->cb_ioc_num	= ii;
2317 
2318 		ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2319 		    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2320 
2321 		IBTF_DPRINTF_L5("ibdm", "\thandle_iounitinfo:"
2322 		    "timeout 0x%x, ioc_num %d", ioc_info->ioc_timeout_id, ii);
2323 
2324 		if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg,
2325 		    NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
2326 			IBTF_DPRINTF_L2("ibdm",
2327 			    "\thandle_iounitinfo: msg transport failed");
2328 			ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args);
2329 		}
2330 		(*flag) |= IBDM_IBMF_PKT_REUSED;
2331 		first = B_FALSE;
2332 		gid_info->gl_iou->iou_niocs_probe_in_progress++;
2333 	}
2334 }
2335 
2336 
2337 /*
2338  * ibdm_handle_ioc_profile()
2339  *	Invoked by the IBMF when the IOCControllerProfile request
2340  *	gets completed
2341  */
2342 static void
2343 ibdm_handle_ioc_profile(ibmf_handle_t ibmf_hdl,
2344     ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
2345 {
2346 	int				first = B_TRUE, reprobe = 0;
2347 	uint_t				ii, ioc_no, srv_start;
2348 	uint_t				nserv_entries;
2349 	timeout_id_t			timeout_id;
2350 	ib_mad_hdr_t			*hdr;
2351 	ibdm_ioc_info_t			*ioc_info;
2352 	ibdm_timeout_cb_args_t		*cb_args;
2353 	ib_dm_ioc_ctrl_profile_t	*ioc, *gioc;
2354 
2355 	IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:"
2356 	    " ibmf hdl %p msg %p gid info %p", ibmf_hdl, msg, gid_info);
2357 
2358 	ioc = IBDM_IN_IBMFMSG2IOC(msg);
2359 	/*
2360 	 * Check whether we know this IOC already
2361 	 * This will return NULL if reprobe is in progress
2362 	 * IBDM_IOC_STATE_REPROBE_PROGRESS will be set.
2363 	 * Do not hold mutexes here.
2364 	 */
2365 	if (ibdm_is_ioc_present(ioc->ioc_guid, gid_info, flag) != NULL) {
2366 		IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:"
2367 			"IOC guid %llx is present", ioc->ioc_guid);
2368 		return;
2369 	}
2370 	ioc_no = IBDM_IN_IBMFMSG_ATTRMOD(msg);
2371 	IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile: ioc_no = %d", ioc_no-1);
2372 
2373 	/* Make sure that IOC index is with the valid range */
2374 	if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) {
2375 		IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: "
2376 		    "IOC index Out of range, index %d", ioc);
2377 		(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
2378 		return;
2379 	}
2380 	ioc_info = &gid_info->gl_iou->iou_ioc_info[ioc_no - 1];
2381 	ioc_info->ioc_iou_info = gid_info->gl_iou;
2382 
2383 	mutex_enter(&gid_info->gl_mutex);
2384 	if (ioc_info->ioc_state == IBDM_IOC_STATE_REPROBE_PROGRESS) {
2385 		reprobe = 1;
2386 		ioc_info->ioc_prev_serv = ioc_info->ioc_serv;
2387 		ioc_info->ioc_serv = NULL;
2388 		ioc_info->ioc_prev_serv_cnt =
2389 		    ioc_info->ioc_profile.ioc_service_entries;
2390 	} else if (ioc_info->ioc_state != IBDM_IOC_STATE_PROBE_INVALID) {
2391 		IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: DUP response"
2392 		    "ioc %d, ioc_state %x", ioc_no - 1, ioc_info->ioc_state);
2393 		mutex_exit(&gid_info->gl_mutex);
2394 		(*flag) |= IBDM_IBMF_PKT_DUP_RESP;
2395 		return;
2396 	}
2397 	ioc_info->ioc_cb_args.cb_req_type = 0;
2398 	if (ioc_info->ioc_timeout_id) {
2399 		timeout_id = ioc_info->ioc_timeout_id;
2400 		mutex_exit(&gid_info->gl_mutex);
2401 		IBTF_DPRINTF_L5("ibdm", "handle_ioc_profile: "
2402 		    "ioc_timeout_id = 0x%x", timeout_id);
2403 		if (untimeout(timeout_id) == -1) {
2404 			IBTF_DPRINTF_L2("ibdm", "handle_ioc_profile: "
2405 			    "untimeout ioc_timeout_id failed");
2406 		}
2407 		mutex_enter(&gid_info->gl_mutex);
2408 		ioc_info->ioc_timeout_id = 0;
2409 	}
2410 
2411 	ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_SUCCESS;
2412 	if (reprobe == 0) {
2413 		ioc_info->ioc_iou_guid = gid_info->gl_nodeguid;
2414 		ioc_info->ioc_nodeguid = gid_info->gl_nodeguid;
2415 	}
2416 
2417 	/*
2418 	 * Save all the IOC information in the global structures.
2419 	 * Note the wire format is Big Endian and the Sparc process also
2420 	 * big endian. So, there is no need to convert the data fields
2421 	 * The conversion routines used below are ineffective on Sparc
2422 	 * machines where as they will be effective on little endian
2423 	 * machines such as Intel processors.
2424 	 */
2425 	gioc = (ib_dm_ioc_ctrl_profile_t *)&ioc_info->ioc_profile;
2426 
2427 	/*
2428 	 * Restrict updates to onlyport GIDs and service entries during reprobe
2429 	 */
2430 	if (reprobe == 0) {
2431 		gioc->ioc_guid			= b2h64(ioc->ioc_guid);
2432 		gioc->ioc_vendorid		=
2433 		    ((b2h32(ioc->ioc_vendorid) & IB_DM_VENDORID_MASK)
2434 		    >> IB_DM_VENDORID_SHIFT);
2435 		gioc->ioc_deviceid		= b2h32(ioc->ioc_deviceid);
2436 		gioc->ioc_device_ver		= b2h16(ioc->ioc_device_ver);
2437 		gioc->ioc_subsys_vendorid	=
2438 		    ((b2h32(ioc->ioc_subsys_vendorid) & IB_DM_VENDORID_MASK)
2439 		    >> IB_DM_VENDORID_SHIFT);
2440 		gioc->ioc_subsys_id		= b2h32(ioc->ioc_subsys_id);
2441 		gioc->ioc_io_class		= b2h16(ioc->ioc_io_class);
2442 		gioc->ioc_io_subclass		= b2h16(ioc->ioc_io_subclass);
2443 		gioc->ioc_protocol		= b2h16(ioc->ioc_protocol);
2444 		gioc->ioc_protocol_ver		= b2h16(ioc->ioc_protocol_ver);
2445 		gioc->ioc_send_msg_qdepth	=
2446 		    b2h16(ioc->ioc_send_msg_qdepth);
2447 		gioc->ioc_rdma_read_qdepth	=
2448 		    b2h16(ioc->ioc_rdma_read_qdepth);
2449 		gioc->ioc_send_msg_sz		= b2h32(ioc->ioc_send_msg_sz);
2450 		gioc->ioc_rdma_xfer_sz		= b2h32(ioc->ioc_rdma_xfer_sz);
2451 		gioc->ioc_ctrl_opcap_mask	= ioc->ioc_ctrl_opcap_mask;
2452 		bcopy(ioc->ioc_id_string, gioc->ioc_id_string,
2453 		    IB_DM_IOC_ID_STRING_LEN);
2454 
2455 		ioc_info->ioc_iou_diagcode = gid_info->gl_iou->iou_diagcode;
2456 		ioc_info->ioc_iou_dc_valid = gid_info->gl_iou->iou_dc_valid;
2457 		ioc_info->ioc_diagdeviceid = (IB_DM_IOU_DEVICEID_MASK &
2458 		    gid_info->gl_iou->iou_info.iou_flag) ? B_TRUE : B_FALSE;
2459 
2460 		if (ioc_info->ioc_diagdeviceid == B_TRUE)
2461 			gid_info->gl_pending_cmds++;
2462 	}
2463 	gioc->ioc_service_entries	= ioc->ioc_service_entries;
2464 	gid_info->gl_pending_cmds += (gioc->ioc_service_entries/4);
2465 	if (gioc->ioc_service_entries % 4)
2466 		gid_info->gl_pending_cmds++;
2467 
2468 	mutex_exit(&gid_info->gl_mutex);
2469 
2470 	ibdm_dump_ioc_profile(gioc);
2471 
2472 	if ((ioc_info->ioc_diagdeviceid == B_TRUE) && (reprobe == 0)) {
2473 		if (ibdm_get_diagcode(gid_info, ioc_no) != IBDM_SUCCESS) {
2474 			mutex_enter(&gid_info->gl_mutex);
2475 			gid_info->gl_pending_cmds--;
2476 			mutex_exit(&gid_info->gl_mutex);
2477 		}
2478 	}
2479 	ioc_info->ioc_serv = (ibdm_srvents_info_t *)kmem_zalloc(
2480 	    (gioc->ioc_service_entries * sizeof (ibdm_srvents_info_t)),
2481 	    KM_SLEEP);
2482 
2483 	/*
2484 	 * In one single request, maximum number of requests that can be
2485 	 * obtained is 4. If number of service entries are more than four,
2486 	 * calculate number requests needed and send them parallelly.
2487 	 */
2488 	nserv_entries = ioc->ioc_service_entries;
2489 	ii = 0;
2490 	while (nserv_entries) {
2491 		mutex_enter(&gid_info->gl_mutex);
2492 		ibdm_bump_transactionID(gid_info);
2493 		mutex_exit(&gid_info->gl_mutex);
2494 
2495 		if (first != B_TRUE) {
2496 			if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP,
2497 			    &msg) != IBMF_SUCCESS) {
2498 				continue;
2499 			}
2500 
2501 		}
2502 		ibdm_alloc_send_buffers(msg);
2503 		msg->im_local_addr.ia_local_lid	= gid_info->gl_slid;
2504 		msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
2505 		if (gid_info->gl_redirected == B_TRUE) {
2506 			if (gid_info->gl_redirect_dlid != 0) {
2507 				msg->im_local_addr.ia_remote_lid =
2508 				    gid_info->gl_redirect_dlid;
2509 			}
2510 			msg->im_local_addr.ia_remote_qno =
2511 			    gid_info->gl_redirect_QP;
2512 			msg->im_local_addr.ia_p_key =
2513 			    gid_info->gl_redirect_pkey;
2514 			msg->im_local_addr.ia_q_key =
2515 			    gid_info->gl_redirect_qkey;
2516 		} else {
2517 			msg->im_local_addr.ia_remote_qno = 1;
2518 			msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
2519 			msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
2520 		}
2521 
2522 		hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
2523 		hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
2524 		hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
2525 		hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
2526 		hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
2527 		hdr->Status		= 0;
2528 		hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
2529 		hdr->AttributeID	= h2b16(IB_DM_ATTR_SERVICE_ENTRIES);
2530 
2531 		srv_start = ii * 4;
2532 		cb_args = &ioc_info->ioc_serv[srv_start].se_cb_args;
2533 		cb_args->cb_gid_info	= gid_info;
2534 		cb_args->cb_retry_count	= ibdm_dft_retry_cnt;
2535 		cb_args->cb_req_type	= IBDM_REQ_TYPE_SRVENTS;
2536 		cb_args->cb_srvents_start = srv_start;
2537 		cb_args->cb_ioc_num	= ioc_no - 1;
2538 
2539 		if (nserv_entries >= IBDM_MAX_SERV_ENTRIES_PER_REQ) {
2540 			nserv_entries -= IBDM_MAX_SERV_ENTRIES_PER_REQ;
2541 			cb_args->cb_srvents_end = (cb_args->cb_srvents_start +
2542 			    IBDM_MAX_SERV_ENTRIES_PER_REQ - 1);
2543 		} else {
2544 			cb_args->cb_srvents_end =
2545 			    (cb_args->cb_srvents_start + nserv_entries - 1);
2546 			nserv_entries = 0;
2547 		}
2548 		ibdm_fill_srv_attr_mod(hdr, cb_args);
2549 
2550 		ioc_info->ioc_serv[srv_start].se_timeout_id = timeout(
2551 		    ibdm_pkt_timeout_hdlr, cb_args,
2552 		    IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2553 
2554 		IBTF_DPRINTF_L5("ibdm", "\thandle_ioc_profile:"
2555 		    "timeout %x, ioc %d srv %d",
2556 		    ioc_info->ioc_serv[srv_start].se_timeout_id,
2557 		    ioc_no - 1, srv_start);
2558 
2559 		if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg,
2560 		    NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
2561 			IBTF_DPRINTF_L2("ibdm",
2562 			    "\thandle_ioc_profile: msg send failed");
2563 			ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args);
2564 		}
2565 		(*flag) |= IBDM_IBMF_PKT_REUSED;
2566 		first = B_FALSE;
2567 		ii++;
2568 	}
2569 }
2570 
2571 
2572 /*
2573  * ibdm_handle_srventry_mad()
2574  */
2575 static void
2576 ibdm_handle_srventry_mad(ibmf_msg_t *msg,
2577     ibdm_dp_gidinfo_t *gid_info, int *flag)
2578 {
2579 	uint_t			ii, ioc_no, attrmod;
2580 	uint_t			nentries, start, end;
2581 	timeout_id_t		timeout_id;
2582 	ib_dm_srv_t		*srv_ents;
2583 	ibdm_ioc_info_t		*ioc_info;
2584 	ibdm_srvents_info_t	*gsrv_ents;
2585 
2586 	IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad:"
2587 	    " IBMF msg %p gid info %p", msg, gid_info);
2588 
2589 	srv_ents = IBDM_IN_IBMFMSG2SRVENT(msg);
2590 	/*
2591 	 * Get the start and end index of the service entries
2592 	 * Upper 16 bits identify the IOC
2593 	 * Lower 16 bits specify the range of service entries
2594 	 * 	LSB specifies (Big endian) end of the range
2595 	 * 	MSB specifies (Big endian) start of the range
2596 	 */
2597 	attrmod = IBDM_IN_IBMFMSG_ATTRMOD(msg);
2598 	ioc_no	= ((attrmod >> 16) & IBDM_16_BIT_MASK);
2599 	end	= ((attrmod >> 8) & IBDM_8_BIT_MASK);
2600 	start	= (attrmod & IBDM_8_BIT_MASK);
2601 
2602 	/* Make sure that IOC index is with the valid range */
2603 	if ((ioc_no < 1) |
2604 	    (ioc_no > gid_info->gl_iou->iou_info.iou_num_ctrl_slots)) {
2605 		IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: "
2606 		    "IOC index Out of range, index %d", ioc_no);
2607 		(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
2608 		return;
2609 	}
2610 	ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1));
2611 
2612 	/*
2613 	 * Make sure that the "start" and "end" service indexes are
2614 	 * with in the valid range
2615 	 */
2616 	nentries = ioc_info->ioc_profile.ioc_service_entries;
2617 	if ((start > end) | (start >= nentries) | (end >= nentries)) {
2618 		IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: "
2619 		    "Attr modifier 0x%x, #Serv entries %d", attrmod, nentries);
2620 		(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
2621 		return;
2622 	}
2623 	gsrv_ents = &ioc_info->ioc_serv[start];
2624 	mutex_enter(&gid_info->gl_mutex);
2625 	if (gsrv_ents->se_state != IBDM_SE_INVALID) {
2626 		IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: "
2627 			"already known, ioc %d, srv %d, se_state %x",
2628 		    ioc_no - 1, start, gsrv_ents->se_state);
2629 		mutex_exit(&gid_info->gl_mutex);
2630 		(*flag) |= IBDM_IBMF_PKT_DUP_RESP;
2631 		return;
2632 	}
2633 	ioc_info->ioc_serv[start].se_cb_args.cb_req_type = 0;
2634 	if (ioc_info->ioc_serv[start].se_timeout_id) {
2635 		IBTF_DPRINTF_L2("ibdm",
2636 		    "\thandle_srventry_mad: ioc %d start %d", ioc_no, start);
2637 		timeout_id = ioc_info->ioc_serv[start].se_timeout_id;
2638 		mutex_exit(&gid_info->gl_mutex);
2639 		IBTF_DPRINTF_L5("ibdm", "handle_srverntry_mad: "
2640 		    "se_timeout_id = 0x%x", timeout_id);
2641 		if (untimeout(timeout_id) == -1) {
2642 			IBTF_DPRINTF_L2("ibdm", "handle_srventry_mad: "
2643 			    "untimeout se_timeout_id failed");
2644 		}
2645 		mutex_enter(&gid_info->gl_mutex);
2646 		ioc_info->ioc_serv[start].se_timeout_id = 0;
2647 	}
2648 
2649 	gsrv_ents->se_state = IBDM_SE_VALID;
2650 	mutex_exit(&gid_info->gl_mutex);
2651 	for (ii = start; ii <= end; ii++, srv_ents++, gsrv_ents++) {
2652 		gsrv_ents->se_attr.srv_id = b2h64(srv_ents->srv_id);
2653 		bcopy(srv_ents->srv_name,
2654 		    gsrv_ents->se_attr.srv_name, IB_DM_MAX_SVC_NAME_LEN);
2655 		ibdm_dump_service_entries(&gsrv_ents->se_attr);
2656 	}
2657 }
2658 
2659 
2660 /*
2661  * ibdm_get_diagcode:
2662  *	Send request to get IOU/IOC diag code
2663  *	Returns IBDM_SUCCESS/IBDM_FAILURE
2664  */
2665 static int
2666 ibdm_get_diagcode(ibdm_dp_gidinfo_t *gid_info, int attr)
2667 {
2668 	ibmf_msg_t		*msg;
2669 	ib_mad_hdr_t		*hdr;
2670 	ibdm_ioc_info_t		*ioc;
2671 	ibdm_timeout_cb_args_t	*cb_args;
2672 	timeout_id_t		*timeout_id;
2673 
2674 	IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: gid info %p, attr = %d",
2675 	    gid_info, attr);
2676 
2677 	if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
2678 	    &msg) != IBMF_SUCCESS) {
2679 		IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: pkt alloc fail");
2680 		return (IBDM_FAILURE);
2681 	}
2682 
2683 	ibdm_alloc_send_buffers(msg);
2684 
2685 	mutex_enter(&gid_info->gl_mutex);
2686 	ibdm_bump_transactionID(gid_info);
2687 	mutex_exit(&gid_info->gl_mutex);
2688 
2689 	msg->im_local_addr.ia_local_lid	= gid_info->gl_slid;
2690 	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
2691 	if (gid_info->gl_redirected == B_TRUE) {
2692 		if (gid_info->gl_redirect_dlid != 0) {
2693 			msg->im_local_addr.ia_remote_lid =
2694 			    gid_info->gl_redirect_dlid;
2695 		}
2696 
2697 		msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
2698 		msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
2699 		msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
2700 	} else {
2701 		msg->im_local_addr.ia_remote_qno = 1;
2702 		msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
2703 		msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
2704 	}
2705 
2706 	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
2707 	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
2708 	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
2709 	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
2710 	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
2711 	hdr->Status		= 0;
2712 	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
2713 
2714 	hdr->AttributeID	= h2b16(IB_DM_ATTR_DIAG_CODE);
2715 	hdr->AttributeModifier	= h2b32(attr);
2716 
2717 	if (attr == 0) {
2718 		cb_args = &gid_info->gl_iou_cb_args;
2719 		gid_info->gl_iou->iou_dc_valid = B_FALSE;
2720 		cb_args->cb_ioc_num	= 0;
2721 		cb_args->cb_req_type	= IBDM_REQ_TYPE_IOU_DIAGCODE;
2722 		timeout_id = &gid_info->gl_timeout_id;
2723 	} else {
2724 		ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attr - 1));
2725 		ioc->ioc_dc_valid = B_FALSE;
2726 		cb_args = &ioc->ioc_dc_cb_args;
2727 		cb_args->cb_ioc_num	= attr - 1;
2728 		cb_args->cb_req_type	= IBDM_REQ_TYPE_IOC_DIAGCODE;
2729 		timeout_id = &ioc->ioc_dc_timeout_id;
2730 	}
2731 	cb_args->cb_gid_info	= gid_info;
2732 	cb_args->cb_retry_count	= ibdm_dft_retry_cnt;
2733 	cb_args->cb_srvents_start = 0;
2734 
2735 
2736 	*timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2737 	    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2738 
2739 	IBTF_DPRINTF_L5("ibdm", "\tget_diagcode:"
2740 	    "timeout %x, ioc %d", *timeout_id, cb_args->cb_ioc_num);
2741 
2742 	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
2743 	    msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
2744 		IBTF_DPRINTF_L2("ibdm", "\tget_diagcode: ibmf send failed");
2745 		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
2746 	}
2747 	return (IBDM_SUCCESS);
2748 }
2749 
2750 /*
2751  * ibdm_handle_diagcode:
2752  *	Process the DiagCode MAD response and update local DM
2753  *	data structure.
2754  */
2755 static void
2756 ibdm_handle_diagcode(ibmf_msg_t *ibmf_msg,
2757     ibdm_dp_gidinfo_t *gid_info, int *flag)
2758 {
2759 	uint16_t	attrmod, *diagcode;
2760 	ibdm_iou_info_t	*iou;
2761 	ibdm_ioc_info_t	*ioc;
2762 	timeout_id_t	timeout_id;
2763 	ibdm_timeout_cb_args_t	*cb_args;
2764 
2765 	diagcode = (uint16_t *)ibmf_msg->im_msgbufs_recv.im_bufs_cl_data;
2766 
2767 	mutex_enter(&gid_info->gl_mutex);
2768 	attrmod = IBDM_IN_IBMFMSG_ATTRMOD(ibmf_msg);
2769 	iou = gid_info->gl_iou;
2770 	if (attrmod == 0) {
2771 		if (iou->iou_dc_valid != B_FALSE) {
2772 			(*flag) |= IBDM_IBMF_PKT_DUP_RESP;
2773 			IBTF_DPRINTF_L4("ibdm",
2774 			    "\thandle_diagcode: Duplicate IOU DiagCode");
2775 			mutex_exit(&gid_info->gl_mutex);
2776 			return;
2777 		}
2778 		cb_args = &gid_info->gl_iou_cb_args;
2779 		cb_args->cb_req_type = 0;
2780 		iou->iou_diagcode = b2h16(*diagcode);
2781 		iou->iou_dc_valid = B_TRUE;
2782 		if (gid_info->gl_timeout_id) {
2783 			timeout_id = gid_info->gl_timeout_id;
2784 			mutex_exit(&gid_info->gl_mutex);
2785 			IBTF_DPRINTF_L5("ibdm", "\thandle_diagcode: "
2786 			    "gl_timeout_id = 0x%x", timeout_id);
2787 			if (untimeout(timeout_id) == -1) {
2788 				IBTF_DPRINTF_L2("ibdm", "handle_diagcode: "
2789 				    "untimeout gl_timeout_id failed");
2790 			}
2791 			mutex_enter(&gid_info->gl_mutex);
2792 			gid_info->gl_timeout_id = 0;
2793 		}
2794 	} else {
2795 		ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod - 1));
2796 		if (ioc->ioc_dc_valid != B_FALSE) {
2797 			(*flag) |= IBDM_IBMF_PKT_DUP_RESP;
2798 			IBTF_DPRINTF_L4("ibdm",
2799 			    "\thandle_diagcode: Duplicate IOC DiagCode");
2800 			mutex_exit(&gid_info->gl_mutex);
2801 			return;
2802 		}
2803 		cb_args = &ioc->ioc_dc_cb_args;
2804 		cb_args->cb_req_type = 0;
2805 		ioc->ioc_diagcode = b2h16(*diagcode);
2806 		ioc->ioc_dc_valid = B_TRUE;
2807 		timeout_id = iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id;
2808 		if (timeout_id) {
2809 			mutex_exit(&gid_info->gl_mutex);
2810 			IBTF_DPRINTF_L5("ibdm", "handle_diagcode: "
2811 			    "timeout_id = 0x%x", timeout_id);
2812 			if (untimeout(timeout_id) == -1) {
2813 				IBTF_DPRINTF_L2("ibdm", "\thandle_diagcode: "
2814 				    "untimeout ioc_dc_timeout_id failed");
2815 			}
2816 			mutex_enter(&gid_info->gl_mutex);
2817 			iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id = 0;
2818 		}
2819 	}
2820 	mutex_exit(&gid_info->gl_mutex);
2821 
2822 	IBTF_DPRINTF_L4("ibdm", "\thandle_diagcode: DiagCode : 0x%x"
2823 	    "attrmod : 0x%x", b2h16(*diagcode), attrmod);
2824 }
2825 
2826 
2827 /*
2828  * ibdm_is_ioc_present()
2829  *	Return ibdm_ioc_info_t if IOC guid is found in the global gid list
2830  */
2831 static ibdm_ioc_info_t *
2832 ibdm_is_ioc_present(ib_guid_t ioc_guid,
2833     ibdm_dp_gidinfo_t *gid_info, int *flag)
2834 {
2835 	int				ii;
2836 	ibdm_ioc_info_t			*ioc;
2837 	ibdm_dp_gidinfo_t		*head;
2838 	ib_dm_io_unitinfo_t		*iou;
2839 
2840 	mutex_enter(&ibdm.ibdm_mutex);
2841 	head = ibdm.ibdm_dp_gidlist_head;
2842 	while (head) {
2843 		mutex_enter(&head->gl_mutex);
2844 		if (head->gl_iou == NULL) {
2845 			mutex_exit(&head->gl_mutex);
2846 			head = head->gl_next;
2847 			continue;
2848 		}
2849 		iou = &head->gl_iou->iou_info;
2850 		for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) {
2851 			ioc = IBDM_GIDINFO2IOCINFO(head, ii);
2852 			if ((ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) &&
2853 			    (ioc->ioc_profile.ioc_guid == ioc_guid)) {
2854 				if (gid_info == head) {
2855 					*flag |= IBDM_IBMF_PKT_DUP_RESP;
2856 				} else if (ibdm_check_dgid(head->gl_dgid_lo,
2857 				    head->gl_dgid_hi) != NULL) {
2858 					IBTF_DPRINTF_L4("ibdm", "\tis_ioc_"
2859 					    "present: gid not present");
2860 					ibdm_add_to_gl_gid(gid_info, head);
2861 				}
2862 				mutex_exit(&head->gl_mutex);
2863 				mutex_exit(&ibdm.ibdm_mutex);
2864 				return (ioc);
2865 			}
2866 		}
2867 		mutex_exit(&head->gl_mutex);
2868 		head = head->gl_next;
2869 	}
2870 	mutex_exit(&ibdm.ibdm_mutex);
2871 	return (NULL);
2872 }
2873 
2874 
2875 /*
2876  * ibdm_ibmf_send_cb()
2877  *	IBMF invokes this callback routine after posting the DM MAD to
2878  *	the HCA.
2879  */
2880 /*ARGSUSED*/
2881 static void
2882 ibdm_ibmf_send_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *ibmf_msg, void *arg)
2883 {
2884 	ibdm_dump_ibmf_msg(ibmf_msg, 1);
2885 	ibdm_free_send_buffers(ibmf_msg);
2886 	if (ibmf_free_msg(ibmf_hdl, &ibmf_msg) != IBMF_SUCCESS) {
2887 		IBTF_DPRINTF_L4("ibdm",
2888 		    "\tibmf_send_cb: IBMF free msg failed");
2889 	}
2890 }
2891 
2892 
2893 /*
2894  * ibdm_ibmf_recv_cb()
2895  *	Invoked by the IBMF when a response to the one of the DM requests
2896  *	is received.
2897  */
2898 /*ARGSUSED*/
2899 static void
2900 ibdm_ibmf_recv_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg)
2901 {
2902 	ibdm_taskq_args_t	*taskq_args;
2903 
2904 	/*
2905 	 * If the taskq enable is set then dispatch a taskq to process
2906 	 * the MAD, otherwise just process it on this thread
2907 	 */
2908 	if (ibdm_taskq_enable != IBDM_ENABLE_TASKQ_HANDLING) {
2909 		ibdm_process_incoming_mad(ibmf_hdl, msg, arg);
2910 		return;
2911 	}
2912 
2913 	/*
2914 	 * create a taskq and dispatch it to process the incoming MAD
2915 	 */
2916 	taskq_args = kmem_alloc(sizeof (ibdm_taskq_args_t), KM_NOSLEEP);
2917 	if (taskq_args == NULL) {
2918 		IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: kmem_alloc failed for"
2919 		    "taskq_args");
2920 		if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
2921 			IBTF_DPRINTF_L4("ibmf_recv_cb",
2922 			    "\tibmf_recv_cb: IBMF free msg failed");
2923 		}
2924 		return;
2925 	}
2926 	taskq_args->tq_ibmf_handle = ibmf_hdl;
2927 	taskq_args->tq_ibmf_msg = msg;
2928 	taskq_args->tq_args = arg;
2929 
2930 	if (taskq_dispatch(system_taskq, ibdm_recv_incoming_mad, taskq_args,
2931 	    TQ_NOSLEEP) == 0) {
2932 		IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: taskq_dispatch failed");
2933 		if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
2934 			IBTF_DPRINTF_L4("ibmf_recv_cb",
2935 			    "\tibmf_recv_cb: IBMF free msg failed");
2936 		}
2937 		kmem_free(taskq_args, sizeof (ibdm_taskq_args_t));
2938 		return;
2939 	}
2940 
2941 	/* taskq_args are deleted in ibdm_recv_incoming_mad() */
2942 }
2943 
2944 
2945 void
2946 ibdm_recv_incoming_mad(void *args)
2947 {
2948 	ibdm_taskq_args_t	*taskq_args;
2949 
2950 	taskq_args = (ibdm_taskq_args_t *)args;
2951 
2952 	IBTF_DPRINTF_L4("ibdm", "\tibdm_recv_incoming_mad: "
2953 	    "Processing incoming MAD via taskq");
2954 
2955 	ibdm_process_incoming_mad(taskq_args->tq_ibmf_handle,
2956 	    taskq_args->tq_ibmf_msg, taskq_args->tq_args);
2957 
2958 	kmem_free(taskq_args, sizeof (ibdm_taskq_args_t));
2959 }
2960 
2961 
2962 /*
2963  * Calls ibdm_process_incoming_mad with all function arguments  extracted
2964  * from args
2965  */
2966 /*ARGSUSED*/
2967 static void
2968 ibdm_process_incoming_mad(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg)
2969 {
2970 	int			flag = 0;
2971 	int			ret;
2972 	uint64_t		transaction_id;
2973 	ib_mad_hdr_t		*hdr;
2974 	ibdm_dp_gidinfo_t	*gid_info = NULL;
2975 
2976 	IBTF_DPRINTF_L4("ibdm",
2977 	    "\tprocess_incoming_mad: ibmf hdl %p pkt %p", ibmf_hdl, msg);
2978 	ibdm_dump_ibmf_msg(msg, 0);
2979 
2980 	/*
2981 	 * IBMF calls this routine for every DM MAD that arrives at this port.
2982 	 * But we handle only the responses for requests we sent. We drop all
2983 	 * the DM packets that does not have response bit set in the MAD
2984 	 * header(this eliminates all the requests sent to this port).
2985 	 * We handle only DM class version 1 MAD's
2986 	 */
2987 	hdr = IBDM_IN_IBMFMSG_MADHDR(msg);
2988 	if (ibdm_verify_mad_status(hdr) != IBDM_SUCCESS) {
2989 		if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
2990 			IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: "
2991 			    "IBMF free msg failed DM request drop it");
2992 		}
2993 		return;
2994 	}
2995 
2996 	transaction_id = b2h64(hdr->TransactionID);
2997 
2998 	mutex_enter(&ibdm.ibdm_mutex);
2999 	gid_info = ibdm.ibdm_dp_gidlist_head;
3000 	while (gid_info) {
3001 		if ((gid_info->gl_transactionID  &
3002 		    IBDM_GID_TRANSACTIONID_MASK) ==
3003 		    (transaction_id & IBDM_GID_TRANSACTIONID_MASK))
3004 			break;
3005 		gid_info = gid_info->gl_next;
3006 	}
3007 	mutex_exit(&ibdm.ibdm_mutex);
3008 
3009 	if (gid_info == NULL) {
3010 		/* Drop the packet */
3011 		IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: transaction ID"
3012 		    " does not match: 0x%llx", transaction_id);
3013 		if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3014 			IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
3015 			    "IBMF free msg failed DM request drop it");
3016 		}
3017 		return;
3018 	}
3019 
3020 	/* Handle redirection for all the MAD's, except ClassPortInfo */
3021 	if (((IBDM_IN_IBMFMSG_STATUS(msg) & MAD_STATUS_REDIRECT_REQUIRED)) &&
3022 		(IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO)) {
3023 		ret = ibdm_handle_redirection(msg, gid_info, &flag);
3024 		if (ret == IBDM_SUCCESS) {
3025 			return;
3026 		}
3027 	} else {
3028 		uint_t gl_state;
3029 
3030 		mutex_enter(&gid_info->gl_mutex);
3031 		gl_state = gid_info->gl_state;
3032 		mutex_exit(&gid_info->gl_mutex);
3033 
3034 		switch (gl_state) {
3035 		case IBDM_GET_CLASSPORTINFO:
3036 			ibdm_handle_classportinfo(
3037 			    ibmf_hdl, msg, gid_info, &flag);
3038 			break;
3039 
3040 		case IBDM_GET_IOUNITINFO:
3041 			ibdm_handle_iounitinfo(ibmf_hdl, msg, gid_info, &flag);
3042 			break;
3043 
3044 		case IBDM_GET_IOC_DETAILS:
3045 			switch (IBDM_IN_IBMFMSG_ATTR(msg)) {
3046 
3047 			case IB_DM_ATTR_SERVICE_ENTRIES:
3048 				ibdm_handle_srventry_mad(msg, gid_info, &flag);
3049 				break;
3050 
3051 			case IB_DM_ATTR_IOC_CTRL_PROFILE:
3052 				ibdm_handle_ioc_profile(
3053 				    ibmf_hdl, msg, gid_info, &flag);
3054 				break;
3055 
3056 			case IB_DM_ATTR_DIAG_CODE:
3057 				ibdm_handle_diagcode(msg, gid_info, &flag);
3058 				break;
3059 
3060 			default:
3061 				IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
3062 				    "Error state, wrong attribute :-(");
3063 				(void) ibmf_free_msg(ibmf_hdl, &msg);
3064 				return;
3065 			}
3066 			break;
3067 		default:
3068 			IBTF_DPRINTF_L2("ibdm",
3069 			    "process_incoming_mad: Dropping the packet"
3070 			    " gl_state %x", gl_state);
3071 			if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3072 				IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
3073 				    "IBMF free msg failed DM request drop it");
3074 			}
3075 			return;
3076 		}
3077 	}
3078 
3079 	if ((flag & IBDM_IBMF_PKT_DUP_RESP) ||
3080 	    (flag & IBDM_IBMF_PKT_UNEXP_RESP)) {
3081 		IBTF_DPRINTF_L2("ibdm",
3082 		    "\tprocess_incoming_mad:Dup/unexp resp : 0x%x", flag);
3083 		if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3084 			IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
3085 			    "IBMF free msg failed DM request drop it");
3086 		}
3087 		return;
3088 	}
3089 
3090 	mutex_enter(&gid_info->gl_mutex);
3091 	if (gid_info->gl_pending_cmds < 1) {
3092 		IBTF_DPRINTF_L2("ibdm",
3093 		    "\tprocess_incoming_mad: pending commands negative");
3094 	}
3095 	if (--gid_info->gl_pending_cmds) {
3096 		IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: "
3097 		    "gid_info %p pending cmds %d",
3098 		    gid_info, gid_info->gl_pending_cmds);
3099 		mutex_exit(&gid_info->gl_mutex);
3100 	} else {
3101 		IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: Probing DONE");
3102 		gid_info->gl_state = IBDM_GID_PROBING_COMPLETE;
3103 		mutex_exit(&gid_info->gl_mutex);
3104 		ibdm_notify_newgid_iocs(gid_info);
3105 		mutex_enter(&ibdm.ibdm_mutex);
3106 		if (--ibdm.ibdm_ngid_probes_in_progress == 0) {
3107 			IBTF_DPRINTF_L4("ibdm",
3108 			    "\tprocess_incoming_mad: Wakeup");
3109 			ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS;
3110 			cv_broadcast(&ibdm.ibdm_probe_cv);
3111 		}
3112 		mutex_exit(&ibdm.ibdm_mutex);
3113 	}
3114 
3115 	/*
3116 	 * Do not deallocate the IBMF packet if atleast one request
3117 	 * is posted. IBMF packet is reused.
3118 	 */
3119 	if (!(flag & IBDM_IBMF_PKT_REUSED)) {
3120 		if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3121 			IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: "
3122 			    "IBMF free msg failed DM request drop it");
3123 		}
3124 	}
3125 }
3126 
3127 
3128 /*
3129  * ibdm_verify_mad_status()
3130  *	Verifies the MAD status
3131  *	Returns IBDM_SUCCESS if status is correct
3132  *	Returns IBDM_FAILURE for bogus MAD status
3133  */
3134 static int
3135 ibdm_verify_mad_status(ib_mad_hdr_t *hdr)
3136 {
3137 	int	ret = 0;
3138 
3139 	if ((hdr->R_Method != IB_DM_DEVMGT_METHOD_GET_RESP) ||
3140 	    (hdr->ClassVersion != IB_DM_CLASS_VERSION_1)) {
3141 		return (IBDM_FAILURE);
3142 	}
3143 
3144 	if (b2h16(hdr->Status) == 0)
3145 		ret = IBDM_SUCCESS;
3146 	else if ((b2h16(hdr->Status) & 0x1f) == MAD_STATUS_REDIRECT_REQUIRED)
3147 		ret = IBDM_SUCCESS;
3148 	else {
3149 		IBTF_DPRINTF_L2("ibdm",
3150 		    "\tverify_mad_status: Status : 0x%x", b2h16(hdr->Status));
3151 		ret = IBDM_FAILURE;
3152 	}
3153 	return (ret);
3154 }
3155 
3156 
3157 
3158 /*
3159  * ibdm_handle_redirection()
3160  *	Returns IBDM_SUCCESS/IBDM_FAILURE
3161  */
3162 static int
3163 ibdm_handle_redirection(ibmf_msg_t *msg,
3164     ibdm_dp_gidinfo_t *gid_info, int *flag)
3165 {
3166 	int			attrmod, ioc_no, start;
3167 	void			*data;
3168 	timeout_id_t		*timeout_id;
3169 	ib_mad_hdr_t		*hdr;
3170 	ibdm_ioc_info_t		*ioc = NULL;
3171 	ibdm_timeout_cb_args_t	*cb_args;
3172 	ibdm_mad_classportinfo_t	*cpi;
3173 
3174 	IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Enter");
3175 	mutex_enter(&gid_info->gl_mutex);
3176 	switch (gid_info->gl_state) {
3177 	case IBDM_GET_IOUNITINFO:
3178 		cb_args		= &gid_info->gl_iou_cb_args;
3179 		timeout_id	= &gid_info->gl_timeout_id;
3180 		break;
3181 
3182 	case IBDM_GET_IOC_DETAILS:
3183 		attrmod	= IBDM_IN_IBMFMSG_ATTRMOD(msg);
3184 		switch (IBDM_IN_IBMFMSG_ATTR(msg)) {
3185 
3186 		case IB_DM_ATTR_DIAG_CODE:
3187 			if (attrmod == 0) {
3188 				cb_args = &gid_info->gl_iou_cb_args;
3189 				timeout_id = &gid_info->gl_timeout_id;
3190 				break;
3191 			}
3192 			if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) {
3193 				IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
3194 				    "IOC# Out of range %d", attrmod);
3195 				(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3196 				mutex_exit(&gid_info->gl_mutex);
3197 				return (IBDM_FAILURE);
3198 			}
3199 			ioc	= IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1));
3200 			cb_args = &ioc->ioc_dc_cb_args;
3201 			timeout_id = &ioc->ioc_dc_timeout_id;
3202 			break;
3203 
3204 		case IB_DM_ATTR_IOC_CTRL_PROFILE:
3205 			if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) {
3206 				IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
3207 				    "IOC# Out of range %d", attrmod);
3208 				(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3209 				mutex_exit(&gid_info->gl_mutex);
3210 				return (IBDM_FAILURE);
3211 			}
3212 			ioc	= IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1));
3213 			cb_args = &ioc->ioc_cb_args;
3214 			timeout_id = &ioc->ioc_timeout_id;
3215 			break;
3216 
3217 		case IB_DM_ATTR_SERVICE_ENTRIES:
3218 			ioc_no	= ((attrmod >> 16) & IBDM_16_BIT_MASK);
3219 			if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) {
3220 				IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
3221 				    "IOC# Out of range %d", ioc_no);
3222 				(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3223 				mutex_exit(&gid_info->gl_mutex);
3224 				return (IBDM_FAILURE);
3225 			}
3226 			start 	= (attrmod & IBDM_8_BIT_MASK);
3227 			ioc	= IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1));
3228 			if (start > ioc->ioc_profile.ioc_service_entries) {
3229 				IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
3230 				    " SE index Out of range %d", start);
3231 				(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3232 				mutex_exit(&gid_info->gl_mutex);
3233 				return (IBDM_FAILURE);
3234 			}
3235 			cb_args = &ioc->ioc_serv[start].se_cb_args;
3236 			timeout_id = &ioc->ioc_serv[start].se_timeout_id;
3237 			break;
3238 
3239 		default:
3240 			/* ERROR State */
3241 			IBTF_DPRINTF_L2("ibdm",
3242 			    "\thandle_redirection: wrong attribute :-(");
3243 			(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3244 			mutex_exit(&gid_info->gl_mutex);
3245 			return (IBDM_FAILURE);
3246 		}
3247 		break;
3248 	default:
3249 		/* ERROR State */
3250 		IBTF_DPRINTF_L2("ibdm",
3251 		    "\thandle_redirection: Error state :-(");
3252 		(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3253 		mutex_exit(&gid_info->gl_mutex);
3254 		return (IBDM_FAILURE);
3255 	}
3256 	if ((*timeout_id) != 0) {
3257 		mutex_exit(&gid_info->gl_mutex);
3258 		if (untimeout(*timeout_id) == -1) {
3259 			IBTF_DPRINTF_L2("ibdm", "\thandle_redirection: "
3260 			    "untimeout failed %x", *timeout_id);
3261 		} else {
3262 			IBTF_DPRINTF_L5("ibdm",
3263 			    "\thandle_redirection: timeout %x", *timeout_id);
3264 		}
3265 		mutex_enter(&gid_info->gl_mutex);
3266 		*timeout_id = 0;
3267 	}
3268 
3269 	data = msg->im_msgbufs_recv.im_bufs_cl_data;
3270 	cpi = (ibdm_mad_classportinfo_t *)data;
3271 
3272 	gid_info->gl_resp_timeout	=
3273 		(b2h32(cpi->RespTimeValue) & 0x1F);
3274 
3275 	gid_info->gl_redirected		= B_TRUE;
3276 	gid_info->gl_redirect_dlid	= b2h16(cpi->RedirectLID);
3277 	gid_info->gl_redirect_QP	= (b2h32(cpi->RedirectQP) & 0xffffff);
3278 	gid_info->gl_redirect_pkey	= b2h16(cpi->RedirectP_Key);
3279 	gid_info->gl_redirect_qkey	= b2h32(cpi->RedirectQ_Key);
3280 	gid_info->gl_redirectGID_hi	= b2h64(cpi->RedirectGID_hi);
3281 	gid_info->gl_redirectGID_lo	= b2h64(cpi->RedirectGID_lo);
3282 
3283 	if (gid_info->gl_redirect_dlid != 0) {
3284 		msg->im_local_addr.ia_remote_lid =
3285 				gid_info->gl_redirect_dlid;
3286 	}
3287 	ibdm_bump_transactionID(gid_info);
3288 	mutex_exit(&gid_info->gl_mutex);
3289 
3290 	ibdm_alloc_send_buffers(msg);
3291 
3292 	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
3293 	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
3294 	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
3295 	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
3296 	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
3297 	hdr->Status		= 0;
3298 	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
3299 	hdr->AttributeID	=
3300 		msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID;
3301 	hdr->AttributeModifier	=
3302 		msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeModifier;
3303 
3304 	msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
3305 	msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
3306 	msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
3307 
3308 	*timeout_id = timeout(ibdm_pkt_timeout_hdlr,
3309 	    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
3310 
3311 	IBTF_DPRINTF_L5("ibdm", "\thandle_redirect:"
3312 	    "timeout %x", *timeout_id);
3313 
3314 	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
3315 	    msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
3316 		IBTF_DPRINTF_L4("ibdm", "\thandle_redirection:"
3317 		    "message transport failed");
3318 		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
3319 	}
3320 	(*flag) |= IBDM_IBMF_PKT_REUSED;
3321 	IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Exit");
3322 	return (IBDM_SUCCESS);
3323 }
3324 
3325 
3326 /*
3327  * ibdm_pkt_timeout_hdlr
3328  *	This  timeout  handler is  registed for every  IBMF  packet that is
3329  *	sent through the IBMF.  It gets called when no response is received
3330  *	within the specified time for the packet. No retries for the failed
3331  *	commands  currently.  Drops the failed  IBMF packet and  update the
3332  *	pending list commands.
3333  */
3334 static void
3335 ibdm_pkt_timeout_hdlr(void *arg)
3336 {
3337 	int			probe_done = B_FALSE;
3338 	ibdm_iou_info_t		*iou;
3339 	ibdm_ioc_info_t		*ioc;
3340 	ibdm_timeout_cb_args_t	*cb_args = arg;
3341 	ibdm_dp_gidinfo_t	*gid_info;
3342 	int			srv_ent;
3343 	uint_t			new_gl_state;
3344 
3345 	IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: gid_info: %p "
3346 	    "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
3347 	    cb_args->cb_req_type, cb_args->cb_ioc_num,
3348 	    cb_args->cb_srvents_start);
3349 
3350 	gid_info = cb_args->cb_gid_info;
3351 	mutex_enter(&gid_info->gl_mutex);
3352 
3353 	if ((gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) ||
3354 	    (cb_args->cb_req_type == 0)) {
3355 
3356 		IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: req completed"
3357 		    "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_req_type,
3358 		    cb_args->cb_ioc_num, cb_args->cb_srvents_start);
3359 
3360 		if (gid_info->gl_timeout_id)
3361 			gid_info->gl_timeout_id = 0;
3362 		mutex_exit(&gid_info->gl_mutex);
3363 		return;
3364 	}
3365 	if (cb_args->cb_retry_count) {
3366 		cb_args->cb_retry_count--;
3367 		if (ibdm_retry_command(cb_args) == IBDM_SUCCESS) {
3368 			if (gid_info->gl_timeout_id)
3369 				gid_info->gl_timeout_id = 0;
3370 			mutex_exit(&gid_info->gl_mutex);
3371 			return;
3372 		}
3373 		cb_args->cb_retry_count = 0;
3374 	}
3375 
3376 	IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: command failed: gid %p"
3377 	    " rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
3378 	    cb_args->cb_req_type, cb_args->cb_ioc_num,
3379 	    cb_args->cb_srvents_start);
3380 
3381 	new_gl_state = IBDM_GID_PROBING_COMPLETE;
3382 	switch (cb_args->cb_req_type) {
3383 
3384 	case IBDM_REQ_TYPE_CLASSPORTINFO:
3385 	case IBDM_REQ_TYPE_IOUINFO:
3386 		new_gl_state = IBDM_GID_PROBING_FAILED;
3387 		if (--gid_info->gl_pending_cmds == 0)
3388 			probe_done = B_TRUE;
3389 		if (gid_info->gl_timeout_id)
3390 			gid_info->gl_timeout_id = 0;
3391 		mutex_exit(&gid_info->gl_mutex);
3392 		ibdm_delete_glhca_list(gid_info);
3393 		mutex_enter(&gid_info->gl_mutex);
3394 		break;
3395 	case IBDM_REQ_TYPE_IOCINFO:
3396 		iou = gid_info->gl_iou;
3397 		ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num];
3398 		ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED;
3399 		if (--gid_info->gl_pending_cmds == 0)
3400 			probe_done = B_TRUE;
3401 #ifndef __lock_lint
3402 		if (ioc->ioc_timeout_id)
3403 			ioc->ioc_timeout_id = 0;
3404 #endif
3405 		break;
3406 	case IBDM_REQ_TYPE_SRVENTS:
3407 		iou = gid_info->gl_iou;
3408 		ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num];
3409 		ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED;
3410 		if (--gid_info->gl_pending_cmds == 0)
3411 			probe_done = B_TRUE;
3412 		srv_ent = cb_args->cb_srvents_start;
3413 #ifndef __lock_lint
3414 		if (ioc->ioc_serv[srv_ent].se_timeout_id)
3415 			ioc->ioc_serv[srv_ent].se_timeout_id = 0;
3416 #endif
3417 		break;
3418 	case IBDM_REQ_TYPE_IOU_DIAGCODE:
3419 		iou = gid_info->gl_iou;
3420 		iou->iou_dc_valid = B_FALSE;
3421 		if (--gid_info->gl_pending_cmds == 0)
3422 			probe_done = B_TRUE;
3423 		if (gid_info->gl_timeout_id)
3424 			gid_info->gl_timeout_id = 0;
3425 		break;
3426 	case IBDM_REQ_TYPE_IOC_DIAGCODE:
3427 		iou = gid_info->gl_iou;
3428 		ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num];
3429 		ioc->ioc_dc_valid = B_FALSE;
3430 		if (--gid_info->gl_pending_cmds == 0)
3431 			probe_done = B_TRUE;
3432 #ifndef __lock_lint
3433 		if (ioc->ioc_dc_timeout_id)
3434 			ioc->ioc_dc_timeout_id = 0;
3435 #endif
3436 		break;
3437 	}
3438 	if (probe_done == B_TRUE) {
3439 		gid_info->gl_state = new_gl_state;
3440 		mutex_exit(&gid_info->gl_mutex);
3441 		ibdm_notify_newgid_iocs(gid_info);
3442 		mutex_enter(&ibdm.ibdm_mutex);
3443 		if (--ibdm.ibdm_ngid_probes_in_progress == 0) {
3444 			IBTF_DPRINTF_L4("ibdm", "\tpkt_timeout_hdlr: Wakeup");
3445 			ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS;
3446 			cv_broadcast(&ibdm.ibdm_probe_cv);
3447 		}
3448 		mutex_exit(&ibdm.ibdm_mutex);
3449 	} else
3450 		mutex_exit(&gid_info->gl_mutex);
3451 }
3452 
3453 
3454 /*
3455  * ibdm_retry_command()
3456  *	Retries the failed command.
3457  *	Returns IBDM_FAILURE/IBDM_SUCCESS
3458  */
3459 static int
3460 ibdm_retry_command(ibdm_timeout_cb_args_t *cb_args)
3461 {
3462 	int			ret;
3463 	ibmf_msg_t		*msg;
3464 	ib_mad_hdr_t		*hdr;
3465 	ibdm_dp_gidinfo_t	*gid_info = cb_args->cb_gid_info;
3466 	timeout_id_t		*timeout_id;
3467 	ibdm_ioc_info_t		*ioc;
3468 	int			ioc_no;
3469 
3470 	IBTF_DPRINTF_L2("ibdm", "\tretry_command: gid_info: %p "
3471 	    "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
3472 	    cb_args->cb_req_type, cb_args->cb_ioc_num,
3473 	    cb_args->cb_srvents_start);
3474 
3475 	ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP, &msg);
3476 
3477 
3478 	/*
3479 	 * Reset the gid if alloc_msg failed with BAD_HANDLE
3480 	 * ibdm_reset_gidinfo reinits the gid_info
3481 	 */
3482 	if (ret == IBMF_BAD_HANDLE) {
3483 		IBTF_DPRINTF_L3(ibdm_string, "\tretry_command: gid %p hdl bad",
3484 		    gid_info);
3485 
3486 		mutex_exit(&gid_info->gl_mutex);
3487 		ibdm_reset_gidinfo(gid_info);
3488 		mutex_enter(&gid_info->gl_mutex);
3489 
3490 		/* Retry alloc */
3491 		ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP,
3492 		    &msg);
3493 	}
3494 
3495 	if (ret != IBDM_SUCCESS) {
3496 		IBTF_DPRINTF_L2("ibdm", "\tretry_command: alloc failed: %p "
3497 		    "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
3498 		    cb_args->cb_req_type, cb_args->cb_ioc_num,
3499 		    cb_args->cb_srvents_start);
3500 		return (IBDM_FAILURE);
3501 	}
3502 
3503 	ibdm_alloc_send_buffers(msg);
3504 
3505 	ibdm_bump_transactionID(gid_info);
3506 
3507 	msg->im_local_addr.ia_local_lid	= gid_info->gl_slid;
3508 	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
3509 	if (gid_info->gl_redirected == B_TRUE) {
3510 		if (gid_info->gl_redirect_dlid != 0) {
3511 			msg->im_local_addr.ia_remote_lid =
3512 					gid_info->gl_redirect_dlid;
3513 		}
3514 		msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
3515 		msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
3516 		msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
3517 	} else {
3518 		msg->im_local_addr.ia_remote_qno = 1;
3519 		msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
3520 		msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
3521 	}
3522 	hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
3523 	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
3524 	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
3525 	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
3526 	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
3527 	hdr->Status		= 0;
3528 	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
3529 
3530 	switch (cb_args->cb_req_type) {
3531 	case IBDM_REQ_TYPE_CLASSPORTINFO:
3532 		hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO);
3533 		hdr->AttributeModifier = 0;
3534 		timeout_id = &gid_info->gl_timeout_id;
3535 		break;
3536 	case IBDM_REQ_TYPE_IOUINFO:
3537 		hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO);
3538 		hdr->AttributeModifier = 0;
3539 		timeout_id = &gid_info->gl_timeout_id;
3540 		break;
3541 	case IBDM_REQ_TYPE_IOCINFO:
3542 		hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE);
3543 		hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1);
3544 		ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num);
3545 		timeout_id = &ioc->ioc_timeout_id;
3546 		break;
3547 	case IBDM_REQ_TYPE_SRVENTS:
3548 		hdr->AttributeID = h2b16(IB_DM_ATTR_SERVICE_ENTRIES);
3549 		ibdm_fill_srv_attr_mod(hdr, cb_args);
3550 		ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num);
3551 		timeout_id =
3552 		    &ioc->ioc_serv[cb_args->cb_srvents_start].se_timeout_id;
3553 		break;
3554 	case IBDM_REQ_TYPE_IOU_DIAGCODE:
3555 		hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE);
3556 		hdr->AttributeModifier = 0;
3557 		timeout_id = &gid_info->gl_timeout_id;
3558 		break;
3559 	case IBDM_REQ_TYPE_IOC_DIAGCODE:
3560 		hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE);
3561 		hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1);
3562 		ioc_no = cb_args->cb_ioc_num;
3563 		ioc = &gid_info->gl_iou->iou_ioc_info[ioc_no];
3564 		timeout_id = &ioc->ioc_dc_timeout_id;
3565 		break;
3566 	}
3567 
3568 	mutex_exit(&gid_info->gl_mutex);
3569 
3570 	*timeout_id = timeout(ibdm_pkt_timeout_hdlr,
3571 	    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
3572 
3573 	IBTF_DPRINTF_L5("ibdm", "\tretry_command: %p,%x,%d,%d:"
3574 	    "timeout %x", cb_args->cb_req_type, cb_args->cb_ioc_num,
3575 	    cb_args->cb_srvents_start, *timeout_id);
3576 
3577 	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl,
3578 	    gid_info->gl_qp_hdl, msg, NULL, ibdm_ibmf_send_cb,
3579 	    cb_args, 0) != IBMF_SUCCESS) {
3580 		IBTF_DPRINTF_L2("ibdm", "\tretry_command: send failed: %p "
3581 		    "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
3582 		    cb_args->cb_req_type, cb_args->cb_ioc_num,
3583 		    cb_args->cb_srvents_start);
3584 		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
3585 	}
3586 	mutex_enter(&gid_info->gl_mutex);
3587 	return (IBDM_SUCCESS);
3588 }
3589 
3590 
3591 /*
3592  * ibdm_update_ioc_port_gidlist()
3593  */
3594 static void
3595 ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *dest,
3596     ibdm_dp_gidinfo_t *gid_info)
3597 {
3598 	int		ii, ngid_ents;
3599 	ibdm_gid_t	*tmp;
3600 	ibdm_hca_list_t	*gid_hca_head, *temp;
3601 	ibdm_hca_list_t	*ioc_head = NULL;
3602 
3603 	IBTF_DPRINTF_L5("ibdm", "\tupdate_ioc_port_gidlist: Enter");
3604 
3605 	ngid_ents = gid_info->gl_ngids;
3606 	dest->ioc_nportgids = ngid_ents;
3607 	dest->ioc_gid_list = kmem_zalloc(sizeof (ibdm_gid_t) *
3608 	    ngid_ents, KM_SLEEP);
3609 	tmp = gid_info->gl_gid;
3610 	for (ii = 0; (ii < ngid_ents) && (tmp); ii++) {
3611 		dest->ioc_gid_list[ii].gid_dgid_hi = tmp->gid_dgid_hi;
3612 		dest->ioc_gid_list[ii].gid_dgid_lo = tmp->gid_dgid_lo;
3613 		tmp = tmp->gid_next;
3614 	}
3615 
3616 	gid_hca_head = gid_info->gl_hca_list;
3617 	while (gid_hca_head) {
3618 		temp = ibdm_dup_hca_attr(gid_hca_head);
3619 		temp->hl_next = ioc_head;
3620 		ioc_head = temp;
3621 		gid_hca_head = gid_hca_head->hl_next;
3622 	}
3623 	dest->ioc_hca_list = ioc_head;
3624 }
3625 
3626 
3627 /*
3628  * ibdm_alloc_send_buffers()
3629  *	Allocates memory for the IBMF send buffer
3630  */
3631 static void
3632 ibdm_alloc_send_buffers(ibmf_msg_t *msgp)
3633 {
3634 	msgp->im_msgbufs_send.im_bufs_mad_hdr =
3635 	    kmem_zalloc(IBDM_MAD_SIZE, KM_SLEEP);
3636 	msgp->im_msgbufs_send.im_bufs_cl_data = (uchar_t *)
3637 	    msgp->im_msgbufs_send.im_bufs_mad_hdr + sizeof (ib_mad_hdr_t);
3638 	msgp->im_msgbufs_send.im_bufs_cl_data_len =
3639 	    IBDM_MAD_SIZE - sizeof (ib_mad_hdr_t);
3640 }
3641 
3642 
3643 /*
3644  * ibdm_alloc_send_buffers()
3645  *	De-allocates memory for the IBMF send buffer
3646  */
3647 static void
3648 ibdm_free_send_buffers(ibmf_msg_t *msgp)
3649 {
3650 	if (msgp->im_msgbufs_send.im_bufs_mad_hdr != NULL)
3651 		kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr, IBDM_MAD_SIZE);
3652 }
3653 
3654 /*
3655  * ibdm_probe_ioc()
3656  *  	1. Gets the node records for the port GUID. This detects all the port
3657  *  		to the IOU.
3658  *	2. Selectively probes all the IOC, given it's node GUID
3659  *	3. In case of reprobe, only the IOC to be reprobed is send the IOC
3660  *		Controller Profile asynchronously
3661  */
3662 /*ARGSUSED*/
3663 static void
3664 ibdm_probe_ioc(ib_guid_t nodeguid, ib_guid_t ioc_guid, int reprobe_flag)
3665 {
3666 	int			ii, nrecords;
3667 	size_t			nr_len = 0, pi_len = 0;
3668 	ib_gid_t		sgid, dgid;
3669 	ibdm_hca_list_t		*hca_list = NULL;
3670 	sa_node_record_t	*nr, *tmp;
3671 	ibdm_port_attr_t	*port = NULL;
3672 	ibdm_dp_gidinfo_t	*reprobe_gid, *new_gid, *node_gid;
3673 	ibdm_dp_gidinfo_t	*temp_gidinfo;
3674 	ibdm_gid_t		*temp_gid;
3675 	sa_portinfo_record_t	*pi;
3676 
3677 	IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc(%llx, %llx, %x): Begin",
3678 	    nodeguid, ioc_guid, reprobe_flag);
3679 
3680 	/* Rescan the GID list for any removed GIDs for reprobe */
3681 	if (reprobe_flag)
3682 		ibdm_rescan_gidlist(&ioc_guid);
3683 
3684 	mutex_enter(&ibdm.ibdm_hl_mutex);
3685 	for (ibdm_get_next_port(&hca_list, &port, 1); port;
3686 	    ibdm_get_next_port(&hca_list, &port, 1)) {
3687 		reprobe_gid = new_gid = node_gid = NULL;
3688 
3689 		nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len, nodeguid);
3690 		if (nr == NULL) {
3691 			IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc: no records");
3692 			continue;
3693 		}
3694 		nrecords = (nr_len / sizeof (sa_node_record_t));
3695 		for (tmp = nr, ii = 0;  (ii < nrecords); ii++, tmp++) {
3696 			pi = ibdm_get_portinfo(
3697 			    port->pa_sa_hdl, &pi_len, tmp->LID);
3698 
3699 			if ((pi) && (pi->PortInfo.CapabilityMask &
3700 			    SM_CAP_MASK_IS_DM_SUPPD)) {
3701 				/*
3702 				 * For reprobes: Check if GID, already in
3703 				 * the list. If so, set the state to SKIPPED
3704 				 */
3705 				if (((temp_gidinfo = ibdm_find_gid(nodeguid,
3706 				    tmp->NodeInfo.PortGUID)) != NULL) &&
3707 				    temp_gidinfo->gl_state ==
3708 				    IBDM_GID_PROBING_COMPLETE) {
3709 					ASSERT(reprobe_gid == NULL);
3710 					ibdm_addto_glhcalist(temp_gidinfo,
3711 					    hca_list);
3712 					reprobe_gid = temp_gidinfo;
3713 					kmem_free(pi, pi_len);
3714 					continue;
3715 				} else if (temp_gidinfo != NULL) {
3716 					kmem_free(pi, pi_len);
3717 					ibdm_addto_glhcalist(temp_gidinfo,
3718 					    hca_list);
3719 					continue;
3720 				}
3721 
3722 				IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : "
3723 				    "create_gid : prefix %llx, guid %llx\n",
3724 				    pi->PortInfo.GidPrefix,
3725 				    tmp->NodeInfo.PortGUID);
3726 
3727 				sgid.gid_prefix = port->pa_sn_prefix;
3728 				sgid.gid_guid = port->pa_port_guid;
3729 				dgid.gid_prefix = pi->PortInfo.GidPrefix;
3730 				dgid.gid_guid = tmp->NodeInfo.PortGUID;
3731 				new_gid = ibdm_create_gid_info(port, sgid,
3732 				    dgid);
3733 				if (new_gid == NULL) {
3734 					IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
3735 					    "create_gid_info failed\n");
3736 					kmem_free(pi, pi_len);
3737 					continue;
3738 				}
3739 				if (node_gid == NULL) {
3740 					node_gid = new_gid;
3741 					ibdm_add_to_gl_gid(node_gid, node_gid);
3742 				} else {
3743 					IBTF_DPRINTF_L4("ibdm",
3744 					    "\tprobe_ioc: new gid");
3745 					temp_gid = kmem_zalloc(
3746 					    sizeof (ibdm_gid_t), KM_SLEEP);
3747 					temp_gid->gid_dgid_hi =
3748 					    new_gid->gl_dgid_hi;
3749 					temp_gid->gid_dgid_lo =
3750 					    new_gid->gl_dgid_lo;
3751 					temp_gid->gid_next = node_gid->gl_gid;
3752 					node_gid->gl_gid = temp_gid;
3753 					node_gid->gl_ngids++;
3754 				}
3755 				new_gid->gl_nodeguid = nodeguid;
3756 				new_gid->gl_portguid = dgid.gid_guid;
3757 				ibdm_addto_glhcalist(new_gid, hca_list);
3758 
3759 				/*
3760 				 * Set the state to skipped as all these
3761 				 * gids point to the same node.
3762 				 * We (re)probe only one GID below and reset
3763 				 * state appropriately
3764 				 */
3765 				new_gid->gl_state = IBDM_GID_PROBING_SKIPPED;
3766 				kmem_free(pi, pi_len);
3767 			}
3768 		}
3769 		kmem_free(nr, nr_len);
3770 
3771 		IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : reprobe_flag %d "
3772 		    "reprobe_gid %p new_gid %p node_gid %p",
3773 		    reprobe_flag, reprobe_gid, new_gid, node_gid);
3774 
3775 		if (reprobe_flag != 0 && reprobe_gid != NULL) {
3776 			int	niocs, jj;
3777 			ibdm_ioc_info_t *tmp_ioc;
3778 			int ioc_matched = 0;
3779 
3780 			mutex_exit(&ibdm.ibdm_hl_mutex);
3781 			mutex_enter(&reprobe_gid->gl_mutex);
3782 			reprobe_gid->gl_state = IBDM_GET_IOC_DETAILS;
3783 			niocs =
3784 			    reprobe_gid->gl_iou->iou_info.iou_num_ctrl_slots;
3785 			reprobe_gid->gl_pending_cmds++;
3786 			mutex_exit(&reprobe_gid->gl_mutex);
3787 
3788 			for (jj = 0; jj < niocs; jj++) {
3789 				tmp_ioc =
3790 				    IBDM_GIDINFO2IOCINFO(reprobe_gid, jj);
3791 				if (tmp_ioc->ioc_profile.ioc_guid != ioc_guid)
3792 					continue;
3793 
3794 				ioc_matched = 1;
3795 
3796 				/*
3797 				 * Explicitly set gl_reprobe_flag to 0 so that
3798 				 * IBnex is not notified on completion
3799 				 */
3800 				mutex_enter(&reprobe_gid->gl_mutex);
3801 				reprobe_gid->gl_reprobe_flag = 0;
3802 				mutex_exit(&reprobe_gid->gl_mutex);
3803 
3804 				mutex_enter(&ibdm.ibdm_mutex);
3805 				ibdm.ibdm_ngid_probes_in_progress++;
3806 				mutex_exit(&ibdm.ibdm_mutex);
3807 				if (ibdm_send_ioc_profile(reprobe_gid, jj) !=
3808 				    IBDM_SUCCESS) {
3809 					IBTF_DPRINTF_L4("ibdm",
3810 					    "\tprobe_ioc: "
3811 					    "send_ioc_profile failed "
3812 					    "for ioc %d", jj);
3813 					ibdm_gid_decr_pending(reprobe_gid);
3814 					break;
3815 				}
3816 				mutex_enter(&ibdm.ibdm_mutex);
3817 				ibdm_wait_probe_completion();
3818 				mutex_exit(&ibdm.ibdm_mutex);
3819 				break;
3820 			}
3821 			if (ioc_matched == 0)
3822 				ibdm_gid_decr_pending(reprobe_gid);
3823 			else {
3824 				mutex_enter(&ibdm.ibdm_hl_mutex);
3825 				break;
3826 			}
3827 		} else if (new_gid != NULL) {
3828 			mutex_exit(&ibdm.ibdm_hl_mutex);
3829 			node_gid = node_gid ? node_gid : new_gid;
3830 
3831 			/*
3832 			 * New or reinserted GID : Enable notification
3833 			 * to IBnex
3834 			 */
3835 			mutex_enter(&node_gid->gl_mutex);
3836 			node_gid->gl_reprobe_flag = 1;
3837 			mutex_exit(&node_gid->gl_mutex);
3838 
3839 			ibdm_probe_gid(node_gid);
3840 
3841 			mutex_enter(&ibdm.ibdm_hl_mutex);
3842 		}
3843 	}
3844 	mutex_exit(&ibdm.ibdm_hl_mutex);
3845 	IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : End\n");
3846 }
3847 
3848 
3849 /*
3850  * ibdm_probe_gid()
3851  *	Selectively probes the GID
3852  */
3853 static void
3854 ibdm_probe_gid(ibdm_dp_gidinfo_t *gid_info)
3855 {
3856 	IBTF_DPRINTF_L4("ibdm", "\tprobe_gid:");
3857 	mutex_enter(&gid_info->gl_mutex);
3858 	gid_info->gl_pending_cmds++;
3859 	gid_info->gl_state = IBDM_GET_CLASSPORTINFO;
3860 	mutex_exit(&gid_info->gl_mutex);
3861 	if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) {
3862 		mutex_enter(&gid_info->gl_mutex);
3863 		gid_info->gl_state = IBDM_GID_PROBING_FAILED;
3864 		--gid_info->gl_pending_cmds;
3865 		mutex_exit(&gid_info->gl_mutex);
3866 		ibdm_delete_glhca_list(gid_info);
3867 		gid_info = gid_info->gl_next;
3868 		return;
3869 	}
3870 	mutex_enter(&ibdm.ibdm_mutex);
3871 	ibdm.ibdm_ngid_probes_in_progress++;
3872 	gid_info = gid_info->gl_next;
3873 
3874 	ibdm_wait_probe_completion();
3875 	mutex_exit(&ibdm.ibdm_mutex);
3876 	IBTF_DPRINTF_L4("ibdm", "\tprobe_gid: Wakeup signal received");
3877 }
3878 
3879 
3880 /*
3881  * ibdm_create_gid_info()
3882  *	Allocates a gid_info structure and initializes
3883  *	Returns pointer to the structure on success
3884  *	and NULL on failure
3885  */
3886 static ibdm_dp_gidinfo_t *
3887 ibdm_create_gid_info(ibdm_port_attr_t *port, ib_gid_t sgid, ib_gid_t dgid)
3888 {
3889 	uint8_t			ii, npaths;
3890 	sa_path_record_t	*path;
3891 	size_t			len;
3892 	ibdm_pkey_tbl_t		*pkey_tbl;
3893 	ibdm_dp_gidinfo_t	*gid_info = NULL;
3894 	int			ret;
3895 
3896 	IBTF_DPRINTF_L4("ibdm", "\tcreate_gid_info: Begin");
3897 	npaths = 1;
3898 
3899 	/* query for reversible paths */
3900 	if (port->pa_sa_hdl)
3901 		ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl,
3902 		    sgid, dgid, IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0,
3903 		    &len, &path);
3904 	else
3905 		return (NULL);
3906 
3907 	if (ret == IBMF_SUCCESS && path) {
3908 		ibdm_dump_path_info(path);
3909 
3910 		gid_info = kmem_zalloc(
3911 		    sizeof (ibdm_dp_gidinfo_t), KM_SLEEP);
3912 		mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL);
3913 		gid_info->gl_dgid_hi		= path->DGID.gid_prefix;
3914 		gid_info->gl_dgid_lo		= path->DGID.gid_guid;
3915 		gid_info->gl_sgid_hi		= path->SGID.gid_prefix;
3916 		gid_info->gl_sgid_lo		= path->SGID.gid_guid;
3917 		gid_info->gl_p_key		= path->P_Key;
3918 		gid_info->gl_sa_hdl		= port->pa_sa_hdl;
3919 		gid_info->gl_ibmf_hdl		= port->pa_ibmf_hdl;
3920 		gid_info->gl_slid		= path->SLID;
3921 		gid_info->gl_dlid		= path->DLID;
3922 		gid_info->gl_transactionID	= (++ibdm.ibdm_transactionID)
3923 		    << IBDM_GID_TRANSACTIONID_SHIFT;
3924 		gid_info->gl_min_transactionID  = gid_info->gl_transactionID;
3925 		gid_info->gl_max_transactionID  = (ibdm.ibdm_transactionID +1)
3926 		    << IBDM_GID_TRANSACTIONID_SHIFT;
3927 
3928 		gid_info->gl_qp_hdl = IBMF_QP_HANDLE_DEFAULT;
3929 		for (ii = 0; ii < port->pa_npkeys; ii++) {
3930 			if (port->pa_pkey_tbl == NULL)
3931 				break;
3932 
3933 			pkey_tbl = &port->pa_pkey_tbl[ii];
3934 			if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) &&
3935 			    (pkey_tbl->pt_qp_hdl != NULL)) {
3936 				gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl;
3937 				break;
3938 			}
3939 		}
3940 		kmem_free(path, len);
3941 
3942 		/*
3943 		 * QP handle for GID not initialized. No matching Pkey
3944 		 * was found!! ibdm should *not* hit this case. Flag an
3945 		 * error and drop the GID if ibdm does encounter this.
3946 		 */
3947 		if (gid_info->gl_qp_hdl == NULL) {
3948 			IBTF_DPRINTF_L2(ibdm_string,
3949 			    "\tcreate_gid_info: No matching Pkey");
3950 			ibdm_delete_gidinfo(gid_info);
3951 			return (NULL);
3952 		}
3953 
3954 		ibdm.ibdm_ngids++;
3955 		if (ibdm.ibdm_dp_gidlist_head == NULL) {
3956 			ibdm.ibdm_dp_gidlist_head = gid_info;
3957 			ibdm.ibdm_dp_gidlist_tail = gid_info;
3958 		} else {
3959 			ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info;
3960 			gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail;
3961 			ibdm.ibdm_dp_gidlist_tail = gid_info;
3962 		}
3963 	}
3964 
3965 	return (gid_info);
3966 }
3967 
3968 
3969 /*
3970  * ibdm_get_node_records
3971  *	Sends a SA query to get the NODE record
3972  *	Returns pointer to the sa_node_record_t on success
3973  *	and NULL on failure
3974  */
3975 static sa_node_record_t *
3976 ibdm_get_node_records(ibmf_saa_handle_t sa_hdl, size_t *length, ib_guid_t guid)
3977 {
3978 	sa_node_record_t	req, *resp = NULL;
3979 	ibmf_saa_access_args_t	args;
3980 	int			ret;
3981 
3982 	IBTF_DPRINTF_L4("ibdm", "\tget_node_records: Begin");
3983 
3984 	bzero(&req, sizeof (sa_node_record_t));
3985 	req.NodeInfo.NodeGUID = guid;
3986 
3987 	args.sq_attr_id		= SA_NODERECORD_ATTRID;
3988 	args.sq_access_type 	= IBMF_SAA_RETRIEVE;
3989 	args.sq_component_mask 	= SA_NODEINFO_COMPMASK_NODEGUID;
3990 	args.sq_template	= &req;
3991 	args.sq_callback	= NULL;
3992 	args.sq_callback_arg 	= NULL;
3993 
3994 	ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp);
3995 	if (ret != IBMF_SUCCESS) {
3996 		IBTF_DPRINTF_L2("ibdm", "\tget_node_records:"
3997 		    " SA Retrieve Failed: %d", ret);
3998 		return (NULL);
3999 	}
4000 	if ((resp == NULL) || (*length == 0)) {
4001 		IBTF_DPRINTF_L2("ibdm", "\tget_node_records: No records");
4002 		return (NULL);
4003 	}
4004 
4005 	IBTF_DPRINTF_L4("ibdm", "\tget_node_records: NodeGuid %llx "
4006 	    "PortGUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.PortGUID);
4007 
4008 	return (resp);
4009 }
4010 
4011 
4012 /*
4013  * ibdm_get_portinfo()
4014  *	Sends a SA query to get the PortInfo record
4015  *	Returns pointer to the sa_portinfo_record_t on success
4016  *	and NULL on failure
4017  */
4018 static sa_portinfo_record_t *
4019 ibdm_get_portinfo(ibmf_saa_handle_t sa_hdl, size_t *length, ib_lid_t lid)
4020 {
4021 	sa_portinfo_record_t	req, *resp = NULL;
4022 	ibmf_saa_access_args_t	args;
4023 	int			ret;
4024 
4025 	IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: Begin");
4026 
4027 	bzero(&req, sizeof (sa_portinfo_record_t));
4028 	req.EndportLID	= lid;
4029 
4030 	args.sq_attr_id		= SA_PORTINFORECORD_ATTRID;
4031 	args.sq_access_type	= IBMF_SAA_RETRIEVE;
4032 	args.sq_component_mask	= SA_PORTINFO_COMPMASK_PORTLID;
4033 	args.sq_template	= &req;
4034 	args.sq_callback	= NULL;
4035 	args.sq_callback_arg	= NULL;
4036 
4037 	ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp);
4038 	if (ret != IBMF_SUCCESS) {
4039 		IBTF_DPRINTF_L2("ibdm", "\tget_portinfo:"
4040 		    " SA Retrieve Failed: 0x%X", ret);
4041 		return (NULL);
4042 	}
4043 	if ((*length == 0) || (resp == NULL))
4044 		return (NULL);
4045 
4046 	IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: GidPrefix %llx Cap 0x%x",
4047 	    resp->PortInfo.GidPrefix, resp->PortInfo.CapabilityMask);
4048 	return (resp);
4049 }
4050 
4051 
4052 /*
4053  * ibdm_ibnex_register_callback
4054  *	IB nexus callback routine for HCA attach and detach notification
4055  */
4056 void
4057 ibdm_ibnex_register_callback(ibdm_callback_t ibnex_dm_callback)
4058 {
4059 	IBTF_DPRINTF_L4("ibdm", "\tibnex_register_callbacks");
4060 	mutex_enter(&ibdm.ibdm_ibnex_mutex);
4061 	ibdm.ibdm_ibnex_callback = ibnex_dm_callback;
4062 	mutex_exit(&ibdm.ibdm_ibnex_mutex);
4063 }
4064 
4065 
4066 /*
4067  * ibdm_ibnex_unregister_callbacks
4068  */
4069 void
4070 ibdm_ibnex_unregister_callback()
4071 {
4072 	IBTF_DPRINTF_L4("ibdm", "\tibnex_unregister_callbacks");
4073 	mutex_enter(&ibdm.ibdm_ibnex_mutex);
4074 	ibdm.ibdm_ibnex_callback = NULL;
4075 	mutex_exit(&ibdm.ibdm_ibnex_mutex);
4076 }
4077 
4078 
4079 /*
4080  * ibdm_ibnex_get_waittime()
4081  *	Calculates the wait time based on the last HCA attach time
4082  */
4083 time_t
4084 ibdm_ibnex_get_waittime(ib_guid_t hca_guid, int *dft_wait)
4085 {
4086 	int		ii;
4087 	time_t		temp, wait_time = 0;
4088 	ibdm_hca_list_t	*hca;
4089 
4090 	IBTF_DPRINTF_L4("ibdm", "\tibnex_get_waittime hcaguid:%llx"
4091 	    "\tport settling time %d", hca_guid, *dft_wait);
4092 
4093 	mutex_enter(&ibdm.ibdm_hl_mutex);
4094 	hca = ibdm.ibdm_hca_list_head;
4095 
4096 	if (hca_guid) {
4097 		for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4098 			if ((hca_guid == hca->hl_hca_guid) &&
4099 			    (hca->hl_nports != hca->hl_nports_active)) {
4100 				wait_time =
4101 					ddi_get_time() - hca->hl_attach_time;
4102 				wait_time = ((wait_time >= *dft_wait) ?
4103 					0 : (*dft_wait - wait_time));
4104 				break;
4105 			}
4106 			hca = hca->hl_next;
4107 		}
4108 		mutex_exit(&ibdm.ibdm_hl_mutex);
4109 		IBTF_DPRINTF_L4("ibdm", "\tibnex_get_waittime %llx", wait_time);
4110 		return (wait_time);
4111 	}
4112 
4113 	for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4114 		if (hca->hl_nports != hca->hl_nports_active) {
4115 			temp = ddi_get_time() - hca->hl_attach_time;
4116 			temp = ((temp >= *dft_wait) ? 0 : (*dft_wait - temp));
4117 			wait_time = (temp > wait_time) ? temp : wait_time;
4118 		}
4119 	}
4120 	mutex_exit(&ibdm.ibdm_hl_mutex);
4121 	IBTF_DPRINTF_L4("ibdm", "\tibnex_get_waittime %llx", wait_time);
4122 	return (wait_time);
4123 }
4124 
4125 
4126 /*
4127  * ibdm_ibnex_probe_hcaport
4128  *	Probes the presence of HCA port (with HCA dip and port number)
4129  *	Returns port attributes structure on SUCCESS
4130  */
4131 ibdm_port_attr_t *
4132 ibdm_ibnex_probe_hcaport(ib_guid_t hca_guid, uint8_t port_num)
4133 {
4134 	int			ii, jj;
4135 	ibdm_hca_list_t		*hca_list;
4136 	ibdm_port_attr_t	*port_attr;
4137 
4138 	IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_hcaport:");
4139 
4140 	mutex_enter(&ibdm.ibdm_hl_mutex);
4141 	hca_list = ibdm.ibdm_hca_list_head;
4142 	for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4143 		if (hca_list->hl_hca_guid == hca_guid) {
4144 			for (jj = 0; jj < hca_list->hl_nports; jj++) {
4145 				if (hca_list->hl_port_attr[jj].pa_port_num ==
4146 				    port_num) {
4147 					break;
4148 				}
4149 			}
4150 			if (jj != hca_list->hl_nports)
4151 				break;
4152 		}
4153 		hca_list = hca_list->hl_next;
4154 	}
4155 	if (ii == ibdm.ibdm_hca_count) {
4156 		IBTF_DPRINTF_L2("ibdm", "\tibnex_probe_hcaport: not found");
4157 		mutex_exit(&ibdm.ibdm_hl_mutex);
4158 		return (NULL);
4159 	}
4160 	port_attr = (ibdm_port_attr_t *)kmem_zalloc(
4161 	    sizeof (ibdm_port_attr_t), KM_SLEEP);
4162 	bcopy((char *)&hca_list->hl_port_attr[jj],
4163 	    port_attr, sizeof (ibdm_port_attr_t));
4164 	ibdm_update_port_attr(port_attr);
4165 
4166 	mutex_exit(&ibdm.ibdm_hl_mutex);
4167 	return (port_attr);
4168 }
4169 
4170 
4171 /*
4172  * ibdm_ibnex_get_port_attrs
4173  *	Scan all HCAs for a matching port_guid.
4174  *	Returns "port attributes" structure on success.
4175  */
4176 ibdm_port_attr_t *
4177 ibdm_ibnex_get_port_attrs(ib_guid_t port_guid)
4178 {
4179 	int			ii, jj;
4180 	ibdm_hca_list_t		*hca_list;
4181 	ibdm_port_attr_t	*port_attr;
4182 
4183 	IBTF_DPRINTF_L4("ibdm", "\tibnex_get_port_attrs:");
4184 
4185 	mutex_enter(&ibdm.ibdm_hl_mutex);
4186 	hca_list = ibdm.ibdm_hca_list_head;
4187 
4188 	for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4189 		for (jj = 0; jj < hca_list->hl_nports; jj++) {
4190 			if (hca_list->hl_port_attr[jj].pa_port_guid ==
4191 				    port_guid) {
4192 				break;
4193 			}
4194 		}
4195 		if (jj != hca_list->hl_nports)
4196 			break;
4197 		hca_list = hca_list->hl_next;
4198 	}
4199 
4200 	if (ii == ibdm.ibdm_hca_count) {
4201 		IBTF_DPRINTF_L2("ibdm", "\tibnex_get_port_attrs: not found");
4202 		mutex_exit(&ibdm.ibdm_hl_mutex);
4203 		return (NULL);
4204 	}
4205 
4206 	port_attr = (ibdm_port_attr_t *)kmem_alloc(sizeof (ibdm_port_attr_t),
4207 	    KM_SLEEP);
4208 	bcopy((char *)&hca_list->hl_port_attr[jj], port_attr,
4209 	    sizeof (ibdm_port_attr_t));
4210 	ibdm_update_port_attr(port_attr);
4211 
4212 	mutex_exit(&ibdm.ibdm_hl_mutex);
4213 	return (port_attr);
4214 }
4215 
4216 
4217 /*
4218  * ibdm_ibnex_free_port_attr()
4219  */
4220 void
4221 ibdm_ibnex_free_port_attr(ibdm_port_attr_t *port_attr)
4222 {
4223 	IBTF_DPRINTF_L4("ibdm", "\tibnex_free_port_attr:");
4224 	if (port_attr) {
4225 		if (port_attr->pa_pkey_tbl != NULL) {
4226 			kmem_free(port_attr->pa_pkey_tbl,
4227 			    (port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t)));
4228 		}
4229 		kmem_free(port_attr, sizeof (ibdm_port_attr_t));
4230 	}
4231 }
4232 
4233 
4234 /*
4235  * ibdm_ibnex_get_hca_list()
4236  *	Returns portinfo for all the port for all the HCA's
4237  */
4238 void
4239 ibdm_ibnex_get_hca_list(ibdm_hca_list_t **hca, int *count)
4240 {
4241 	ibdm_hca_list_t		*head = NULL, *temp, *temp1;
4242 	int			ii;
4243 
4244 	IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_list:");
4245 
4246 	mutex_enter(&ibdm.ibdm_hl_mutex);
4247 	temp = ibdm.ibdm_hca_list_head;
4248 	for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4249 		temp1 = ibdm_dup_hca_attr(temp);
4250 		temp1->hl_next = head;
4251 		head = temp1;
4252 		temp = temp->hl_next;
4253 	}
4254 	*count = ibdm.ibdm_hca_count;
4255 	*hca = head;
4256 	mutex_exit(&ibdm.ibdm_hl_mutex);
4257 }
4258 
4259 
4260 /*
4261  * ibdm_ibnex_get_hca_info_by_guid()
4262  */
4263 ibdm_hca_list_t	*
4264 ibdm_ibnex_get_hca_info_by_guid(ib_guid_t hca_guid)
4265 {
4266 	ibdm_hca_list_t		*head = NULL, *hca = NULL;
4267 
4268 	IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip");
4269 
4270 	mutex_enter(&ibdm.ibdm_hl_mutex);
4271 	head = ibdm.ibdm_hca_list_head;
4272 	while (head) {
4273 		if (head->hl_hca_guid == hca_guid) {
4274 			hca = ibdm_dup_hca_attr(head);
4275 			hca->hl_next = NULL;
4276 			break;
4277 		}
4278 		head = head->hl_next;
4279 	}
4280 	mutex_exit(&ibdm.ibdm_hl_mutex);
4281 	IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip %p", hca);
4282 	return (hca);
4283 }
4284 
4285 
4286 /*
4287  * ibdm_dup_hca_attr()
4288  *	Allocate a new HCA attribute strucuture and initialize
4289  *	hca attribute structure with the incoming HCA attributes
4290  *	returned the allocated hca attributes.
4291  */
4292 static ibdm_hca_list_t *
4293 ibdm_dup_hca_attr(ibdm_hca_list_t *in_hca)
4294 {
4295 	int			len;
4296 	ibdm_hca_list_t		*out_hca;
4297 
4298 	len = sizeof (ibdm_hca_list_t) +
4299 	    (in_hca->hl_nports * sizeof (ibdm_port_attr_t));
4300 	IBTF_DPRINTF_L4("ibdm", "\tdup_hca_attr len %d", len);
4301 	out_hca = (ibdm_hca_list_t *)kmem_alloc(len, KM_SLEEP);
4302 	bcopy((char *)in_hca,
4303 		(char *)out_hca, sizeof (ibdm_hca_list_t));
4304 	if (in_hca->hl_nports) {
4305 		out_hca->hl_port_attr = (ibdm_port_attr_t *)
4306 			((char *)out_hca + sizeof (ibdm_hca_list_t));
4307 		bcopy((char *)in_hca->hl_port_attr,
4308 			(char *)out_hca->hl_port_attr,
4309 			(in_hca->hl_nports * sizeof (ibdm_port_attr_t)));
4310 		for (len = 0; len < out_hca->hl_nports; len++)
4311 			ibdm_update_port_attr(&out_hca->hl_port_attr[len]);
4312 	}
4313 	return (out_hca);
4314 }
4315 
4316 
4317 /*
4318  * ibdm_ibnex_free_hca_list()
4319  *	Free one/more HCA lists
4320  */
4321 void
4322 ibdm_ibnex_free_hca_list(ibdm_hca_list_t *hca_list)
4323 {
4324 	int			ii;
4325 	size_t			len;
4326 	ibdm_hca_list_t 	*temp;
4327 	ibdm_port_attr_t	*port;
4328 
4329 	IBTF_DPRINTF_L4("ibdm", "\tibnex_free_hca_list:");
4330 	ASSERT(hca_list);
4331 	while (hca_list) {
4332 		temp = hca_list;
4333 		hca_list = hca_list->hl_next;
4334 		for (ii = 0; ii < temp->hl_nports; ii++) {
4335 			port = &temp->hl_port_attr[ii];
4336 			len = (port->pa_npkeys * sizeof (ibdm_pkey_tbl_t));
4337 			if (len != 0)
4338 				kmem_free(port->pa_pkey_tbl, len);
4339 		}
4340 		len = sizeof (ibdm_hca_list_t) + (temp->hl_nports *
4341 			sizeof (ibdm_port_attr_t));
4342 		kmem_free(temp, len);
4343 	}
4344 }
4345 
4346 
4347 /*
4348  * ibdm_ibnex_probe_iocguid()
4349  *	Probes the IOC on the fabric and returns the IOC information
4350  *	if present. Otherwise, NULL is returned
4351  */
4352 /* ARGSUSED */
4353 ibdm_ioc_info_t *
4354 ibdm_ibnex_probe_ioc(ib_guid_t iou, ib_guid_t ioc_guid, int reprobe_flag)
4355 {
4356 	int			k;
4357 	ibdm_ioc_info_t		*ioc_info;
4358 	ibdm_dp_gidinfo_t	*gid_info;
4359 
4360 	IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_ioc: (%llX, %llX, %d) Begin",
4361 	    iou, ioc_guid, reprobe_flag);
4362 	/* Check whether we know this already */
4363 	ioc_info = ibdm_ibnex_get_ioc_info(ioc_guid);
4364 	if (ioc_info == NULL) {
4365 		mutex_enter(&ibdm.ibdm_mutex);
4366 		while (ibdm.ibdm_busy & IBDM_BUSY)
4367 			cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
4368 		ibdm.ibdm_busy |= IBDM_BUSY;
4369 		mutex_exit(&ibdm.ibdm_mutex);
4370 		ibdm_probe_ioc(iou, ioc_guid, 0);
4371 		mutex_enter(&ibdm.ibdm_mutex);
4372 		ibdm.ibdm_busy &= ~IBDM_BUSY;
4373 		cv_broadcast(&ibdm.ibdm_busy_cv);
4374 		mutex_exit(&ibdm.ibdm_mutex);
4375 		ioc_info = ibdm_ibnex_get_ioc_info(ioc_guid);
4376 	} else if (reprobe_flag) {	/* Handle Reprobe for the IOC */
4377 		/* Free the ioc_list before reprobe; and cancel any timers */
4378 		mutex_enter(&ibdm.ibdm_mutex);
4379 		if (ioc_info->ioc_timeout_id) {
4380 			IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: "
4381 			    "ioc_timeout_id = 0x%x",
4382 			    ioc_info->ioc_timeout_id);
4383 			if (untimeout(ioc_info->ioc_timeout_id) == -1) {
4384 				IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
4385 				    "untimeout ioc_timeout_id failed");
4386 			}
4387 			ioc_info->ioc_timeout_id = 0;
4388 		}
4389 		if (ioc_info->ioc_dc_timeout_id) {
4390 			IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: "
4391 			    "ioc_dc_timeout_id = 0x%x",
4392 			    ioc_info->ioc_dc_timeout_id);
4393 			if (untimeout(ioc_info->ioc_dc_timeout_id) == -1) {
4394 				IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
4395 				    "untimeout ioc_dc_timeout_id failed");
4396 			}
4397 			ioc_info->ioc_dc_timeout_id = 0;
4398 		}
4399 		for (k = 0; k < ioc_info->ioc_profile.ioc_service_entries; k++)
4400 			if (ioc_info->ioc_serv[k].se_timeout_id) {
4401 				IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: "
4402 				    "ioc_info->ioc_serv[k].se_timeout_id = %x",
4403 				    k, ioc_info->ioc_serv[k].se_timeout_id);
4404 				if (untimeout(ioc_info->ioc_serv[k].
4405 				    se_timeout_id) == -1) {
4406 					IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
4407 					    "untimeout se_timeout_id %d "
4408 					    "failed", k);
4409 				}
4410 				ioc_info->ioc_serv[k].se_timeout_id = 0;
4411 			}
4412 		mutex_exit(&ibdm.ibdm_mutex);
4413 		ibdm_ibnex_free_ioc_list(ioc_info);
4414 
4415 		mutex_enter(&ibdm.ibdm_mutex);
4416 		while (ibdm.ibdm_busy & IBDM_BUSY)
4417 			cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
4418 		ibdm.ibdm_busy |= IBDM_BUSY;
4419 		mutex_exit(&ibdm.ibdm_mutex);
4420 
4421 		ibdm_probe_ioc(iou, ioc_guid, 1);
4422 
4423 		/*
4424 		 * Skip if gl_reprobe_flag is set, this will be
4425 		 * a re-inserted / new GID, for which notifications
4426 		 * have already been send.
4427 		 */
4428 		for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
4429 		    gid_info = gid_info->gl_next) {
4430 			uint8_t			ii, niocs;
4431 			ibdm_ioc_info_t		*ioc;
4432 
4433 			if (gid_info->gl_iou == NULL)
4434 				continue;
4435 
4436 			if (gid_info->gl_reprobe_flag) {
4437 				gid_info->gl_reprobe_flag = 0;
4438 				continue;
4439 			}
4440 
4441 			niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
4442 			for (ii = 0; ii < niocs; ii++) {
4443 				ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii);
4444 				if (ioc->ioc_profile.ioc_guid == ioc_guid) {
4445 					mutex_enter(&ibdm.ibdm_mutex);
4446 					ibdm_reprobe_update_port_srv(ioc,
4447 					    gid_info);
4448 					mutex_exit(&ibdm.ibdm_mutex);
4449 				}
4450 			}
4451 		}
4452 		mutex_enter(&ibdm.ibdm_mutex);
4453 		ibdm.ibdm_busy &= ~IBDM_BUSY;
4454 		cv_broadcast(&ibdm.ibdm_busy_cv);
4455 		mutex_exit(&ibdm.ibdm_mutex);
4456 
4457 		ioc_info = ibdm_ibnex_get_ioc_info(ioc_guid);
4458 	}
4459 	return (ioc_info);
4460 }
4461 
4462 
4463 /*
4464  * ibdm_ibnex_get_ioc_info()
4465  *	Returns pointer to ibdm_ioc_info_t if it finds
4466  *	matching record for the ioc_guid, otherwise NULL
4467  *	is returned
4468  */
4469 ibdm_ioc_info_t *
4470 ibdm_ibnex_get_ioc_info(ib_guid_t ioc_guid)
4471 {
4472 	int			ii;
4473 	ibdm_ioc_info_t		*ioc = NULL, *tmp = NULL;
4474 	ibdm_dp_gidinfo_t	*gid_list;
4475 	ib_dm_io_unitinfo_t	*iou;
4476 
4477 	IBTF_DPRINTF_L4("ibdm", "\tibnex_get_ioc_info: GUID %llx", ioc_guid);
4478 
4479 	mutex_enter(&ibdm.ibdm_mutex);
4480 	while (ibdm.ibdm_busy & IBDM_BUSY)
4481 		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
4482 	ibdm.ibdm_busy |= IBDM_BUSY;
4483 
4484 	gid_list = ibdm.ibdm_dp_gidlist_head;
4485 	while (gid_list) {
4486 		mutex_enter(&gid_list->gl_mutex);
4487 		if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) {
4488 			mutex_exit(&gid_list->gl_mutex);
4489 			gid_list = gid_list->gl_next;
4490 			continue;
4491 		}
4492 		if (gid_list->gl_iou == NULL) {
4493 			IBTF_DPRINTF_L2("ibdm",
4494 			    "\tget_ioc_info: No IOU info");
4495 			mutex_exit(&gid_list->gl_mutex);
4496 			gid_list = gid_list->gl_next;
4497 			continue;
4498 		}
4499 		iou = &gid_list->gl_iou->iou_info;
4500 		for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) {
4501 			tmp = IBDM_GIDINFO2IOCINFO(gid_list, ii);
4502 			if ((tmp->ioc_profile.ioc_guid == ioc_guid) &&
4503 			    (tmp->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS)) {
4504 				ioc = ibdm_dup_ioc_info(tmp, gid_list);
4505 				mutex_exit(&gid_list->gl_mutex);
4506 				ibdm.ibdm_busy &= ~IBDM_BUSY;
4507 				cv_broadcast(&ibdm.ibdm_busy_cv);
4508 				mutex_exit(&ibdm.ibdm_mutex);
4509 				IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: End");
4510 				return (ioc);
4511 			}
4512 		}
4513 		if (ii == iou->iou_num_ctrl_slots)
4514 			ioc = NULL;
4515 
4516 		mutex_exit(&gid_list->gl_mutex);
4517 		gid_list = gid_list->gl_next;
4518 	}
4519 
4520 	ibdm.ibdm_busy &= ~IBDM_BUSY;
4521 	cv_broadcast(&ibdm.ibdm_busy_cv);
4522 	mutex_exit(&ibdm.ibdm_mutex);
4523 	IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: failure End");
4524 	return (ioc);
4525 }
4526 
4527 
4528 /*
4529  * ibdm_ibnex_get_ioc_count()
4530  *	Returns number of ibdm_ioc_info_t it finds
4531  */
4532 int
4533 ibdm_ibnex_get_ioc_count(void)
4534 {
4535 	int			count = 0, k;
4536 	ibdm_ioc_info_t		*ioc;
4537 	ibdm_dp_gidinfo_t	*gid_list;
4538 
4539 	mutex_enter(&ibdm.ibdm_mutex);
4540 	ibdm_sweep_fabric(0);
4541 
4542 	while (ibdm.ibdm_busy & IBDM_BUSY)
4543 		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
4544 	ibdm.ibdm_busy |= IBDM_BUSY;
4545 
4546 	for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
4547 	    gid_list = gid_list->gl_next) {
4548 		mutex_enter(&gid_list->gl_mutex);
4549 		if ((gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) ||
4550 		    (gid_list->gl_iou == NULL)) {
4551 			mutex_exit(&gid_list->gl_mutex);
4552 			continue;
4553 		}
4554 		for (k = 0; k < gid_list->gl_iou->iou_info.iou_num_ctrl_slots;
4555 		    k++) {
4556 			ioc = IBDM_GIDINFO2IOCINFO(gid_list, k);
4557 			if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS)
4558 				++count;
4559 		}
4560 		mutex_exit(&gid_list->gl_mutex);
4561 	}
4562 	ibdm.ibdm_busy &= ~IBDM_BUSY;
4563 	cv_broadcast(&ibdm.ibdm_busy_cv);
4564 	mutex_exit(&ibdm.ibdm_mutex);
4565 
4566 	IBTF_DPRINTF_L4("ibdm", "\tget_ioc_count: count = %d", count);
4567 	return (count);
4568 }
4569 
4570 
4571 /*
4572  * ibdm_ibnex_get_ioc_list()
4573  *	Returns information about all the IOCs present on the fabric.
4574  *	Reprobes the IOCs and the GID list if list_flag is set to REPROBE_ALL.
4575  *	Does not sweep fabric if DONOT_PROBE is set
4576  */
4577 ibdm_ioc_info_t *
4578 ibdm_ibnex_get_ioc_list(ibdm_ibnex_get_ioclist_mtd_t list_flag)
4579 {
4580 	int			ii;
4581 	ibdm_ioc_info_t		*ioc_list = NULL, *tmp, *ioc;
4582 	ibdm_dp_gidinfo_t	*gid_list;
4583 	ib_dm_io_unitinfo_t	*iou;
4584 
4585 	IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: Enter");
4586 
4587 	mutex_enter(&ibdm.ibdm_mutex);
4588 	if (list_flag != IBDM_IBNEX_DONOT_PROBE)
4589 		ibdm_sweep_fabric(list_flag == IBDM_IBNEX_REPROBE_ALL);
4590 
4591 	while (ibdm.ibdm_busy & IBDM_BUSY)
4592 		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
4593 	ibdm.ibdm_busy |= IBDM_BUSY;
4594 
4595 	gid_list = ibdm.ibdm_dp_gidlist_head;
4596 	while (gid_list) {
4597 		mutex_enter(&gid_list->gl_mutex);
4598 		if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) {
4599 			mutex_exit(&gid_list->gl_mutex);
4600 			gid_list = gid_list->gl_next;
4601 			continue;
4602 		}
4603 		if (gid_list->gl_iou == NULL) {
4604 			IBTF_DPRINTF_L2("ibdm",
4605 			    "\tget_ioc_list: No IOU info");
4606 			mutex_exit(&gid_list->gl_mutex);
4607 			gid_list = gid_list->gl_next;
4608 			continue;
4609 		}
4610 		iou = &gid_list->gl_iou->iou_info;
4611 		for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) {
4612 			ioc = IBDM_GIDINFO2IOCINFO(gid_list, ii);
4613 			if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) {
4614 				tmp = ibdm_dup_ioc_info(ioc, gid_list);
4615 				tmp->ioc_next = ioc_list;
4616 				ioc_list = tmp;
4617 			}
4618 		}
4619 		mutex_exit(&gid_list->gl_mutex);
4620 		gid_list = gid_list->gl_next;
4621 	}
4622 	ibdm.ibdm_busy &= ~IBDM_BUSY;
4623 	cv_broadcast(&ibdm.ibdm_busy_cv);
4624 	mutex_exit(&ibdm.ibdm_mutex);
4625 
4626 	IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: End");
4627 	return (ioc_list);
4628 }
4629 
4630 /*
4631  * ibdm_dup_ioc_info()
4632  *	Duplicate the IOC information and return the IOC
4633  *	information.
4634  */
4635 static ibdm_ioc_info_t *
4636 ibdm_dup_ioc_info(ibdm_ioc_info_t *in_ioc, ibdm_dp_gidinfo_t *gid_list)
4637 {
4638 	ibdm_ioc_info_t	*out_ioc;
4639 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*out_ioc));
4640 
4641 	out_ioc = kmem_alloc(sizeof (ibdm_ioc_info_t), KM_SLEEP);
4642 	bcopy(in_ioc, out_ioc, sizeof (ibdm_ioc_info_t));
4643 	ibdm_update_ioc_port_gidlist(out_ioc, gid_list);
4644 	out_ioc->ioc_iou_dc_valid = gid_list->gl_iou->iou_dc_valid;
4645 	out_ioc->ioc_iou_diagcode = gid_list->gl_iou->iou_diagcode;
4646 
4647 	return (out_ioc);
4648 }
4649 
4650 
4651 /*
4652  * ibdm_free_ioc_list()
4653  *	Deallocate memory for IOC list structure
4654  */
4655 void
4656 ibdm_ibnex_free_ioc_list(ibdm_ioc_info_t *ioc)
4657 {
4658 	ibdm_ioc_info_t *temp;
4659 
4660 	IBTF_DPRINTF_L4("ibdm", "\tibnex_free_ioc_list:");
4661 	while (ioc) {
4662 		temp = ioc;
4663 		ioc = ioc->ioc_next;
4664 		kmem_free(temp->ioc_gid_list,
4665 		    (sizeof (ibdm_gid_t) * temp->ioc_nportgids));
4666 		if (temp->ioc_hca_list)
4667 			ibdm_ibnex_free_hca_list(temp->ioc_hca_list);
4668 		kmem_free(temp, sizeof (ibdm_ioc_info_t));
4669 	}
4670 }
4671 
4672 
4673 /*
4674  * ibdm_ibnex_update_pkey_tbls
4675  *	Updates the DM P_Key database.
4676  *	NOTE: Two cases are handled here: P_Key being added or removed.
4677  *
4678  * Arguments		: NONE
4679  * Return Values	: NONE
4680  */
4681 void
4682 ibdm_ibnex_update_pkey_tbls(void)
4683 {
4684 	int			h, pp, pidx;
4685 	uint_t			nports;
4686 	uint_t			size;
4687 	ib_pkey_t		new_pkey;
4688 	ib_pkey_t		*orig_pkey;
4689 	ibdm_hca_list_t		*hca_list;
4690 	ibdm_port_attr_t	*port;
4691 	ibt_hca_portinfo_t	*pinfop;
4692 
4693 	IBTF_DPRINTF_L4("ibdm", "\tibnex_update_pkey_tbls:");
4694 
4695 	mutex_enter(&ibdm.ibdm_hl_mutex);
4696 	hca_list = ibdm.ibdm_hca_list_head;
4697 
4698 	for (h = 0; h < ibdm.ibdm_hca_count; h++) {
4699 
4700 		/* This updates P_Key Tables for all ports of this HCA */
4701 		(void) ibt_query_hca_ports(hca_list->hl_hca_hdl, 0, &pinfop,
4702 		    &nports, &size);
4703 
4704 		/* number of ports shouldn't have changed */
4705 		ASSERT(nports == hca_list->hl_nports);
4706 
4707 		for (pp = 0; pp < hca_list->hl_nports; pp++) {
4708 			port = &hca_list->hl_port_attr[pp];
4709 
4710 			/*
4711 			 * First figure out the P_Keys from IBTL.
4712 			 * Three things could have happened:
4713 			 *	New P_Keys added
4714 			 *	Existing P_Keys removed
4715 			 *	Both of the above two
4716 			 *
4717 			 * Loop through the P_Key Indices and check if a
4718 			 * give P_Key_Ix matches that of the one seen by
4719 			 * IBDM. If they match no action is needed.
4720 			 *
4721 			 * If they don't match:
4722 			 *	1. if orig_pkey is invalid and new_pkey is valid
4723 			 *		---> add new_pkey to DM database
4724 			 *	2. if orig_pkey is valid and new_pkey is invalid
4725 			 *		---> remove orig_pkey from DM database
4726 			 *	3. if orig_pkey and new_pkey are both valid:
4727 			 *		---> remov orig_pkey from DM database
4728 			 *		---> add new_pkey to DM database
4729 			 *	4. if orig_pkey and new_pkey are both invalid:
4730 			 *		---> do nothing. Updated DM database.
4731 			 */
4732 
4733 			for (pidx = 0; pidx < port->pa_npkeys; pidx++) {
4734 				new_pkey = pinfop[pp].p_pkey_tbl[pidx];
4735 				orig_pkey = &port->pa_pkey_tbl[pidx].pt_pkey;
4736 
4737 				/* keys match - do nothing */
4738 				if (*orig_pkey == new_pkey)
4739 					continue;
4740 
4741 				if (IBDM_INVALID_PKEY(*orig_pkey) &&
4742 				    !IBDM_INVALID_PKEY(new_pkey)) {
4743 					/* P_Key was added */
4744 					IBTF_DPRINTF_L5("ibdm",
4745 					    "\tibnex_update_pkey_tbls: new "
4746 					    "P_Key added = 0x%x", new_pkey);
4747 					*orig_pkey = new_pkey;
4748 					ibdm_port_attr_ibmf_init(port,
4749 					    new_pkey, pp);
4750 				} else if (!IBDM_INVALID_PKEY(*orig_pkey) &&
4751 				    IBDM_INVALID_PKEY(new_pkey)) {
4752 					/* P_Key was removed */
4753 					IBTF_DPRINTF_L5("ibdm",
4754 					    "\tibnex_update_pkey_tbls: P_Key "
4755 					    "removed = 0x%x", *orig_pkey);
4756 					*orig_pkey = new_pkey;
4757 					(void) ibdm_port_attr_ibmf_fini(port,
4758 					    pidx);
4759 				} else if (!IBDM_INVALID_PKEY(*orig_pkey) &&
4760 				    !IBDM_INVALID_PKEY(new_pkey)) {
4761 					/* P_Key were replaced */
4762 					IBTF_DPRINTF_L5("ibdm",
4763 					    "\tibnex_update_pkey_tbls: P_Key "
4764 					    "replaced 0x%x with 0x%x",
4765 					    *orig_pkey, new_pkey);
4766 					(void) ibdm_port_attr_ibmf_fini(port,
4767 					    pidx);
4768 					*orig_pkey = new_pkey;
4769 					ibdm_port_attr_ibmf_init(port,
4770 					    new_pkey, pp);
4771 				} else {
4772 					/*
4773 					 * P_Keys are invalid
4774 					 * set anyway to reflect if
4775 					 * INVALID_FULL was changed to
4776 					 * INVALID_LIMITED or vice-versa.
4777 					 */
4778 					*orig_pkey = new_pkey;
4779 				} /* end of else */
4780 
4781 			} /* loop of p_key index */
4782 
4783 		} /* loop of #ports of HCA */
4784 
4785 		ibt_free_portinfo(pinfop, size);
4786 		hca_list = hca_list->hl_next;
4787 
4788 	} /* loop for all HCAs in the system */
4789 
4790 	mutex_exit(&ibdm.ibdm_hl_mutex);
4791 }
4792 
4793 
4794 /*
4795  * ibdm_send_ioc_profile()
4796  *	Send IOC Controller Profile request. When the request is completed
4797  *	IBMF calls ibdm_process_incoming_mad routine to inform about
4798  *	the completion.
4799  */
4800 static int
4801 ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *gid_info, uint8_t ioc_no)
4802 {
4803 	ibmf_msg_t		*msg;
4804 	ib_mad_hdr_t	*hdr;
4805 	ibdm_ioc_info_t	*ioc_info = &(gid_info->gl_iou->iou_ioc_info[ioc_no]);
4806 	ibdm_timeout_cb_args_t	*cb_args;
4807 
4808 	IBTF_DPRINTF_L4("ibdm", "\tsend_ioc_profile: "
4809 	    "gid info 0x%p, ioc_no = %d", gid_info, ioc_no);
4810 
4811 	/*
4812 	 * Send command to get IOC profile.
4813 	 * Allocate a IBMF packet and initialize the packet.
4814 	 */
4815 	if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
4816 	    &msg) != IBMF_SUCCESS) {
4817 		IBTF_DPRINTF_L4("ibdm", "\tsend_ioc_profile: pkt alloc fail");
4818 		return (IBDM_FAILURE);
4819 	}
4820 
4821 	ibdm_alloc_send_buffers(msg);
4822 
4823 	mutex_enter(&gid_info->gl_mutex);
4824 	ibdm_bump_transactionID(gid_info);
4825 	mutex_exit(&gid_info->gl_mutex);
4826 
4827 	msg->im_local_addr.ia_local_lid	= gid_info->gl_slid;
4828 	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
4829 	if (gid_info->gl_redirected == B_TRUE) {
4830 		if (gid_info->gl_redirect_dlid != 0) {
4831 			msg->im_local_addr.ia_remote_lid =
4832 			    gid_info->gl_redirect_dlid;
4833 		}
4834 		msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
4835 		msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
4836 		msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
4837 	} else {
4838 		msg->im_local_addr.ia_remote_qno = 1;
4839 		msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
4840 		msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
4841 	}
4842 
4843 	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
4844 	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
4845 	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
4846 	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
4847 	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
4848 	hdr->Status		= 0;
4849 	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
4850 	hdr->AttributeID	= h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE);
4851 	hdr->AttributeModifier 	= h2b32(ioc_no + 1);
4852 
4853 	ioc_info->ioc_state	= IBDM_IOC_STATE_REPROBE_PROGRESS;
4854 	cb_args			= &ioc_info->ioc_cb_args;
4855 	cb_args->cb_gid_info	= gid_info;
4856 	cb_args->cb_retry_count	= ibdm_dft_retry_cnt;
4857 	cb_args->cb_req_type	= IBDM_REQ_TYPE_IOCINFO;
4858 	cb_args->cb_ioc_num	= ioc_no;
4859 
4860 	ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
4861 	    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
4862 
4863 	IBTF_DPRINTF_L5("ibdm", "\tsend_ioc_profile:"
4864 	    "timeout %x", ioc_info->ioc_timeout_id);
4865 
4866 	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg,
4867 	    NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
4868 		IBTF_DPRINTF_L2("ibdm",
4869 		    "\tsend_ioc_profile: msg transport failed");
4870 		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
4871 	}
4872 	ioc_info->ioc_state = IBDM_IOC_STATE_REPROBE_PROGRESS;
4873 	return (IBDM_SUCCESS);
4874 }
4875 
4876 
4877 /*
4878  * ibdm_port_reachable
4879  *	Sends a SA query to get the NODE record for port GUID
4880  *	Returns IBDM_SUCCESS if the port GID is reachable
4881  */
4882 static int
4883 ibdm_port_reachable(ibmf_saa_handle_t sa_hdl, ib_guid_t guid,
4884     ib_guid_t *node_guid)
4885 {
4886 	sa_node_record_t	req, *resp = NULL;
4887 	ibmf_saa_access_args_t	args;
4888 	int			ret;
4889 	size_t		length;
4890 
4891 	IBTF_DPRINTF_L4("ibdm", "\tport_reachable: port_guid %llx",
4892 	    guid);
4893 
4894 	bzero(&req, sizeof (sa_node_record_t));
4895 	req.NodeInfo.PortGUID = guid;
4896 
4897 	args.sq_attr_id		= SA_NODERECORD_ATTRID;
4898 	args.sq_access_type 	= IBMF_SAA_RETRIEVE;
4899 	args.sq_component_mask 	= SA_NODEINFO_COMPMASK_PORTGUID;
4900 	args.sq_template	= &req;
4901 	args.sq_callback	= NULL;
4902 	args.sq_callback_arg 	= NULL;
4903 
4904 	ret = ibmf_sa_access(sa_hdl, &args, 0, &length, (void **) &resp);
4905 	if (ret != IBMF_SUCCESS) {
4906 		IBTF_DPRINTF_L2("ibdm", "\tport_reachable:"
4907 		    " SA Retrieve Failed: %d", ret);
4908 		return (IBDM_FAILURE);
4909 	}
4910 
4911 	if ((resp == NULL) || (length == 0)) {
4912 		IBTF_DPRINTF_L2("ibdm", "\tport_reachable: No records");
4913 		return (IBDM_FAILURE);
4914 	}
4915 
4916 	if (node_guid != NULL)
4917 		*node_guid = resp->NodeInfo.NodeGUID;
4918 
4919 	kmem_free(resp, length);
4920 
4921 	return (IBDM_SUCCESS);
4922 }
4923 
4924 /*
4925  * Update the gidlist for all affected IOCs when GID becomes
4926  * available/unavailable.
4927  *
4928  * Parameters :
4929  *	gidinfo - Incoming / Outgoing GID.
4930  *	add_flag - 1 for GID added, 0 for GID removed.
4931  *		- (-1) : IOC gid list updated, ioc_list required.
4932  *
4933  * This function gets the GID for the node GUID corresponding to the
4934  * port GID. Gets the IOU info
4935  */
4936 static ibdm_ioc_info_t *
4937 ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *gid_info, int avail_flag)
4938 {
4939 	ibdm_dp_gidinfo_t	*node_gid = NULL;
4940 	uint8_t	niocs, ii;
4941 	ibdm_ioc_info_t	*ioc, *ioc_list = NULL, *tmp;
4942 
4943 	IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist");
4944 
4945 	switch (avail_flag) {
4946 		case 1 :
4947 			node_gid = ibdm_check_dest_nodeguid(gid_info);
4948 			break;
4949 		case 0 :
4950 			node_gid = ibdm_handle_gid_rm(gid_info);
4951 			break;
4952 		case -1 :
4953 			node_gid = gid_info;
4954 			break;
4955 		default :
4956 			break;
4957 	}
4958 
4959 	if (node_gid == NULL) {
4960 		IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist: "
4961 		    "No node GID found, port gid 0x%p, avail_flag %d",
4962 		    gid_info, avail_flag);
4963 		return (NULL);
4964 	}
4965 
4966 	mutex_enter(&node_gid->gl_mutex);
4967 	if ((node_gid->gl_state != IBDM_GID_PROBING_COMPLETE &&
4968 	    node_gid->gl_state != IBDM_GID_PROBING_SKIPPED) ||
4969 	    node_gid->gl_iou == NULL) {
4970 		IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist "
4971 		    "gl_state %x, gl_iou %p", node_gid->gl_state,
4972 		    node_gid->gl_iou);
4973 		mutex_exit(&node_gid->gl_mutex);
4974 		return (NULL);
4975 	}
4976 
4977 	niocs = node_gid->gl_iou->iou_info.iou_num_ctrl_slots;
4978 	IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : niocs %x",
4979 	    niocs);
4980 	for (ii = 0; ii < niocs; ii++) {
4981 		ioc = IBDM_GIDINFO2IOCINFO(node_gid, ii);
4982 		/*
4983 		 * Skip IOCs for which probe is not complete or
4984 		 * reprobe is progress
4985 		 */
4986 		if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) {
4987 			tmp = ibdm_dup_ioc_info(ioc, node_gid);
4988 			tmp->ioc_info_updated.ib_gid_prop_updated = 1;
4989 			tmp->ioc_next = ioc_list;
4990 			ioc_list = tmp;
4991 		}
4992 	}
4993 	mutex_exit(&node_gid->gl_mutex);
4994 
4995 	IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : return %p",
4996 	    ioc_list);
4997 	return (ioc_list);
4998 }
4999 
5000 /*
5001  * ibdm_saa_event_cb :
5002  *	Event handling which does *not* require ibdm_hl_mutex to be
5003  *	held are executed in the same thread. This is to prevent
5004  *	deadlocks with HCA port down notifications which hold the
5005  *	ibdm_hl_mutex.
5006  *
5007  *	GID_AVAILABLE event is handled here. A taskq is spawned to
5008  *	handle GID_UNAVAILABLE.
5009  *
5010  *	A new mutex ibdm_ibnex_mutex has been introduced to protect
5011  *	ibnex_callback. This has been done to prevent any possible
5012  *	deadlock (described above) while handling GID_AVAILABLE.
5013  *
5014  *	IBMF calls the event callback for a HCA port. The SA handle
5015  *	for this port would be valid, till the callback returns.
5016  *	IBDM calling IBDM using the above SA handle should be valid.
5017  *
5018  *	IBDM will additionally  check (SA handle != NULL), before
5019  *	calling IBMF.
5020  */
5021 /*ARGSUSED*/
5022 static void
5023 ibdm_saa_event_cb(ibmf_saa_handle_t ibmf_saa_handle,
5024     ibmf_saa_subnet_event_t ibmf_saa_event,
5025     ibmf_saa_event_details_t *event_details, void *callback_arg)
5026 {
5027 	ibdm_saa_event_arg_t *event_arg;
5028 	ib_gid_t		sgid, dgid;
5029 	ibdm_port_attr_t	*hca_port;
5030 	ibdm_dp_gidinfo_t	*gid_info, *node_gid_info = NULL;
5031 	ib_guid_t		nodeguid;
5032 
5033 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg));
5034 
5035 	hca_port = (ibdm_port_attr_t *)callback_arg;
5036 
5037 	IBTF_DPRINTF_L4("ibdm", "\tsaa_event_cb(%x, %x, %x, %x)\n",
5038 	    ibmf_saa_handle, ibmf_saa_event, event_details,
5039 	    callback_arg);
5040 #ifdef DEBUG
5041 	if (ibdm_ignore_saa_event)
5042 		return;
5043 #endif
5044 
5045 	if (ibmf_saa_event == IBMF_SAA_EVENT_GID_AVAILABLE) {
5046 		/*
5047 		 * Ensure no other probe / sweep fabric is in
5048 		 * progress.
5049 		 */
5050 		mutex_enter(&ibdm.ibdm_mutex);
5051 		while (ibdm.ibdm_busy & IBDM_BUSY)
5052 			cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5053 		ibdm.ibdm_busy |= IBDM_BUSY;
5054 		mutex_exit(&ibdm.ibdm_mutex);
5055 
5056 		/*
5057 		 * If we already know about this GID, return.
5058 		 * GID_AVAILABLE may be reported for multiple HCA
5059 		 * ports.
5060 		 */
5061 		if ((ibdm_check_dgid(event_details->ie_gid.gid_guid,
5062 		    event_details->ie_gid.gid_prefix))  != NULL) {
5063 			mutex_enter(&ibdm.ibdm_mutex);
5064 			ibdm.ibdm_busy &= ~IBDM_BUSY;
5065 			cv_broadcast(&ibdm.ibdm_busy_cv);
5066 			mutex_exit(&ibdm.ibdm_mutex);
5067 			return;
5068 		}
5069 
5070 		IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) "
5071 		    "Insertion notified",
5072 		    event_details->ie_gid.gid_prefix,
5073 		    event_details->ie_gid.gid_guid);
5074 
5075 		/* This is a new gid, insert it to GID list */
5076 		sgid.gid_prefix = hca_port->pa_sn_prefix;
5077 		sgid.gid_guid = hca_port->pa_port_guid;
5078 		dgid.gid_prefix = event_details->ie_gid.gid_prefix;
5079 		dgid.gid_guid = event_details->ie_gid.gid_guid;
5080 		gid_info = ibdm_create_gid_info(hca_port, sgid, dgid);
5081 		if (gid_info == NULL) {
5082 			IBTF_DPRINTF_L4("ibdm", "\tGID_AVAILABLE: "
5083 			    "create_gid_info returned NULL");
5084 			mutex_enter(&ibdm.ibdm_mutex);
5085 			ibdm.ibdm_busy &= ~IBDM_BUSY;
5086 			cv_broadcast(&ibdm.ibdm_busy_cv);
5087 			mutex_exit(&ibdm.ibdm_mutex);
5088 			return;
5089 		}
5090 		mutex_enter(&gid_info->gl_mutex);
5091 		gid_info->gl_state = IBDM_GID_PROBING_SKIPPED;
5092 		mutex_exit(&gid_info->gl_mutex);
5093 
5094 		/* Get the node GUID */
5095 		if (ibdm_port_reachable(ibmf_saa_handle, dgid.gid_guid,
5096 		    &nodeguid) != IBDM_SUCCESS) {
5097 			/*
5098 			 * Set the state to PROBE_NOT_DONE for the
5099 			 * next sweep to probe it
5100 			 */
5101 			IBTF_DPRINTF_L2("ibdm", "\tsaa_event_taskq: "
5102 			    "Skipping GID : port GUID not found");
5103 			mutex_enter(&gid_info->gl_mutex);
5104 			gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE;
5105 			mutex_exit(&gid_info->gl_mutex);
5106 			mutex_enter(&ibdm.ibdm_mutex);
5107 			ibdm.ibdm_busy &= ~IBDM_BUSY;
5108 			cv_broadcast(&ibdm.ibdm_busy_cv);
5109 			mutex_exit(&ibdm.ibdm_mutex);
5110 			return;
5111 		}
5112 
5113 		gid_info->gl_nodeguid = nodeguid;
5114 		gid_info->gl_portguid = dgid.gid_guid;
5115 
5116 		/*
5117 		 * Get the gid info with the same node GUID.
5118 		 */
5119 		mutex_enter(&ibdm.ibdm_mutex);
5120 		node_gid_info = ibdm.ibdm_dp_gidlist_head;
5121 		while (node_gid_info) {
5122 			if (node_gid_info->gl_nodeguid ==
5123 			    gid_info->gl_nodeguid &&
5124 			    node_gid_info->gl_iou != NULL) {
5125 				break;
5126 			}
5127 			node_gid_info = node_gid_info->gl_next;
5128 		}
5129 		mutex_exit(&ibdm.ibdm_mutex);
5130 
5131 		/*
5132 		 * Handling a new GID requires filling of gl_hca_list.
5133 		 * This require ibdm hca_list to be parsed and hence
5134 		 * holding the ibdm_hl_mutex. Spawning a new thread to
5135 		 * handle this.
5136 		 */
5137 		if (node_gid_info == NULL) {
5138 			if (taskq_dispatch(system_taskq,
5139 			    ibdm_saa_handle_new_gid, (void *)gid_info,
5140 			    TQ_NOSLEEP) == NULL) {
5141 				IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: "
5142 				    "new_gid taskq_dispatch failed");
5143 				return;
5144 			}
5145 		}
5146 
5147 		mutex_enter(&ibdm.ibdm_mutex);
5148 		ibdm.ibdm_busy &= ~IBDM_BUSY;
5149 		cv_broadcast(&ibdm.ibdm_busy_cv);
5150 		mutex_exit(&ibdm.ibdm_mutex);
5151 		return;
5152 	}
5153 
5154 	if (ibmf_saa_event != IBMF_SAA_EVENT_GID_UNAVAILABLE)
5155 		return;
5156 
5157 	event_arg = (ibdm_saa_event_arg_t *)kmem_alloc(
5158 	    sizeof (ibdm_saa_event_arg_t), KM_SLEEP);
5159 	event_arg->ibmf_saa_handle = ibmf_saa_handle;
5160 	event_arg->ibmf_saa_event = ibmf_saa_event;
5161 	bcopy(event_details, &event_arg->event_details,
5162 	    sizeof (ibmf_saa_event_details_t));
5163 	event_arg->callback_arg = callback_arg;
5164 
5165 	if (taskq_dispatch(system_taskq, ibdm_saa_event_taskq,
5166 	    (void *)event_arg, TQ_NOSLEEP) == NULL) {
5167 		IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: "
5168 		    "taskq_dispatch failed");
5169 		ibdm_free_saa_event_arg(event_arg);
5170 		return;
5171 	}
5172 }
5173 
5174 /*
5175  * Handle a new GID discovered by GID_AVAILABLE saa event.
5176  */
5177 void
5178 ibdm_saa_handle_new_gid(void *arg)
5179 {
5180 	ibdm_dp_gidinfo_t	*gid_info;
5181 	ibdm_hca_list_t		*hca_list = NULL;
5182 	ibdm_port_attr_t	*port = NULL;
5183 	ibdm_ioc_info_t		*ioc_list = NULL;
5184 
5185 	IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid(%p)", arg);
5186 
5187 	gid_info = (ibdm_dp_gidinfo_t *)arg;
5188 
5189 	/*
5190 	 * Ensure that no other sweep / probe has completed
5191 	 * probing this gid.
5192 	 */
5193 	mutex_enter(&gid_info->gl_mutex);
5194 	if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) {
5195 		mutex_exit(&gid_info->gl_mutex);
5196 		return;
5197 	}
5198 	mutex_exit(&gid_info->gl_mutex);
5199 
5200 	/*
5201 	 * Parse HCAs to fill gl_hca_list
5202 	 */
5203 	mutex_enter(&ibdm.ibdm_hl_mutex);
5204 	for (ibdm_get_next_port(&hca_list, &port, 1); port;
5205 	    ibdm_get_next_port(&hca_list, &port, 1)) {
5206 		if (ibdm_port_reachable(port->pa_sa_hdl,
5207 		    gid_info->gl_portguid, NULL) ==
5208 		    IBDM_SUCCESS) {
5209 			ibdm_addto_glhcalist(gid_info, hca_list);
5210 		}
5211 	}
5212 	mutex_exit(&ibdm.ibdm_hl_mutex);
5213 
5214 	/*
5215 	 * Ensure no other probe / sweep fabric is in
5216 	 * progress.
5217 	 */
5218 	mutex_enter(&ibdm.ibdm_mutex);
5219 	while (ibdm.ibdm_busy & IBDM_BUSY)
5220 		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5221 	ibdm.ibdm_busy |= IBDM_BUSY;
5222 	mutex_exit(&ibdm.ibdm_mutex);
5223 
5224 	/*
5225 	 * New IOU probe it, to check if new IOCs
5226 	 */
5227 	IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid: "
5228 	    "new GID : probing");
5229 	mutex_enter(&ibdm.ibdm_mutex);
5230 	ibdm.ibdm_ngid_probes_in_progress++;
5231 	mutex_exit(&ibdm.ibdm_mutex);
5232 	mutex_enter(&gid_info->gl_mutex);
5233 	gid_info->gl_reprobe_flag = 0;
5234 	gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE;
5235 	mutex_exit(&gid_info->gl_mutex);
5236 	ibdm_probe_gid_thread((void *)gid_info);
5237 
5238 	mutex_enter(&ibdm.ibdm_mutex);
5239 	ibdm_wait_probe_completion();
5240 	mutex_exit(&ibdm.ibdm_mutex);
5241 
5242 	if (gid_info->gl_iou == NULL) {
5243 		mutex_enter(&ibdm.ibdm_mutex);
5244 		ibdm.ibdm_busy &= ~IBDM_BUSY;
5245 		cv_broadcast(&ibdm.ibdm_busy_cv);
5246 		mutex_exit(&ibdm.ibdm_mutex);
5247 		return;
5248 	}
5249 
5250 	/*
5251 	 * Update GID list in all IOCs affected by this
5252 	 */
5253 	ioc_list = ibdm_update_ioc_gidlist(gid_info, 1);
5254 
5255 	/*
5256 	 * Pass on the IOCs with updated GIDs to IBnexus
5257 	 */
5258 	if (ioc_list) {
5259 		mutex_enter(&ibdm.ibdm_ibnex_mutex);
5260 		if (ibdm.ibdm_ibnex_callback != NULL) {
5261 			(*ibdm.ibdm_ibnex_callback)((void *)ioc_list,
5262 			    IBDM_EVENT_IOC_PROP_UPDATE);
5263 		}
5264 		mutex_exit(&ibdm.ibdm_ibnex_mutex);
5265 	}
5266 
5267 	mutex_enter(&ibdm.ibdm_mutex);
5268 	ibdm.ibdm_busy &= ~IBDM_BUSY;
5269 	cv_broadcast(&ibdm.ibdm_busy_cv);
5270 	mutex_exit(&ibdm.ibdm_mutex);
5271 }
5272 
5273 /*
5274  * ibdm_saa_event_taskq :
5275  *	GID_UNAVAILABLE Event handling requires ibdm_hl_mutex to be
5276  *	held. The GID_UNAVAILABLE handling is done in a taskq to
5277  *	prevent deadlocks with HCA port down notifications which hold
5278  *	ibdm_hl_mutex.
5279  */
5280 void
5281 ibdm_saa_event_taskq(void *arg)
5282 {
5283 	ibdm_saa_event_arg_t *event_arg;
5284 	ibmf_saa_handle_t ibmf_saa_handle;
5285 	ibmf_saa_subnet_event_t ibmf_saa_event;
5286 	ibmf_saa_event_details_t *event_details;
5287 	void *callback_arg;
5288 
5289 	ibdm_dp_gidinfo_t	*gid_info;
5290 	ibdm_port_attr_t	*hca_port, *port = NULL;
5291 	ibdm_hca_list_t		*hca_list = NULL;
5292 	int	sa_handle_valid = 0;
5293 	ibdm_ioc_info_t		*ioc_list = NULL;
5294 
5295 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg));
5296 
5297 	event_arg = (ibdm_saa_event_arg_t *)arg;
5298 	ibmf_saa_handle = event_arg->ibmf_saa_handle;
5299 	ibmf_saa_event = event_arg->ibmf_saa_event;
5300 	event_details = &event_arg->event_details;
5301 	callback_arg = event_arg->callback_arg;
5302 
5303 	ASSERT(callback_arg != NULL);
5304 	ASSERT(ibmf_saa_event == IBMF_SAA_EVENT_GID_UNAVAILABLE);
5305 	IBTF_DPRINTF_L4("ibdm", "\tsaa_event_taskq(%x, %x, %x, %x)",
5306 	    ibmf_saa_handle, ibmf_saa_event, event_details,
5307 	    callback_arg);
5308 
5309 	hca_port = (ibdm_port_attr_t *)callback_arg;
5310 
5311 	/* Check if the port_attr is still valid */
5312 	mutex_enter(&ibdm.ibdm_hl_mutex);
5313 	for (ibdm_get_next_port(&hca_list, &port, 0); port;
5314 	    ibdm_get_next_port(&hca_list, &port, 0)) {
5315 		if (port == hca_port && port->pa_port_guid ==
5316 		    hca_port->pa_port_guid) {
5317 			if (ibmf_saa_handle == hca_port->pa_sa_hdl)
5318 				sa_handle_valid = 1;
5319 			break;
5320 		}
5321 	}
5322 	mutex_exit(&ibdm.ibdm_hl_mutex);
5323 	if (sa_handle_valid == 0) {
5324 		ibdm_free_saa_event_arg(event_arg);
5325 		return;
5326 	}
5327 
5328 	if (hca_port && (hca_port->pa_sa_hdl == NULL ||
5329 	    ibmf_saa_handle != hca_port->pa_sa_hdl)) {
5330 		ibdm_free_saa_event_arg(event_arg);
5331 		return;
5332 	}
5333 	hca_list = NULL;
5334 	port = NULL;
5335 
5336 	/*
5337 	 * Check if the GID is visible to other HCA ports.
5338 	 * Return if so.
5339 	 */
5340 	mutex_enter(&ibdm.ibdm_hl_mutex);
5341 	for (ibdm_get_next_port(&hca_list, &port, 1); port;
5342 	    ibdm_get_next_port(&hca_list, &port, 1)) {
5343 		if (ibdm_port_reachable(port->pa_sa_hdl,
5344 		    event_details->ie_gid.gid_guid, NULL) ==
5345 		    IBDM_SUCCESS) {
5346 			mutex_exit(&ibdm.ibdm_hl_mutex);
5347 			ibdm_free_saa_event_arg(event_arg);
5348 			return;
5349 		}
5350 	}
5351 	mutex_exit(&ibdm.ibdm_hl_mutex);
5352 
5353 	/*
5354 	 * Ensure no other probe / sweep fabric is in
5355 	 * progress.
5356 	 */
5357 	mutex_enter(&ibdm.ibdm_mutex);
5358 	while (ibdm.ibdm_busy & IBDM_BUSY)
5359 		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5360 	ibdm.ibdm_busy |= IBDM_BUSY;
5361 	mutex_exit(&ibdm.ibdm_mutex);
5362 
5363 	/*
5364 	 * If this GID is no longer in GID list, return
5365 	 * GID_UNAVAILABLE may be reported for multiple HCA
5366 	 * ports.
5367 	 */
5368 	mutex_enter(&ibdm.ibdm_mutex);
5369 	gid_info = ibdm.ibdm_dp_gidlist_head;
5370 	while (gid_info) {
5371 		if (gid_info->gl_portguid ==
5372 		    event_details->ie_gid.gid_guid) {
5373 			break;
5374 		}
5375 		gid_info = gid_info->gl_next;
5376 	}
5377 	mutex_exit(&ibdm.ibdm_mutex);
5378 	if (gid_info == NULL) {
5379 		mutex_enter(&ibdm.ibdm_mutex);
5380 		ibdm.ibdm_busy &= ~IBDM_BUSY;
5381 		cv_broadcast(&ibdm.ibdm_busy_cv);
5382 		mutex_exit(&ibdm.ibdm_mutex);
5383 		ibdm_free_saa_event_arg(event_arg);
5384 		return;
5385 	}
5386 
5387 	IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) "
5388 	    "Unavailable notification",
5389 	    event_details->ie_gid.gid_prefix,
5390 	    event_details->ie_gid.gid_guid);
5391 
5392 	/*
5393 	 * Update GID list in all IOCs affected by this
5394 	 */
5395 	if (gid_info->gl_state == IBDM_GID_PROBING_SKIPPED ||
5396 	    gid_info->gl_state == IBDM_GID_PROBING_COMPLETE)
5397 		ioc_list = ibdm_update_ioc_gidlist(gid_info, 0);
5398 
5399 	/*
5400 	 * Remove GID from the global GID list
5401 	 * Handle the case where all port GIDs for an
5402 	 * IOU have been hot-removed. Check both gid_info
5403 	 * & ioc_info for checking ngids.
5404 	 */
5405 	mutex_enter(&ibdm.ibdm_mutex);
5406 	if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) {
5407 		mutex_enter(&gid_info->gl_mutex);
5408 		(void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou);
5409 		mutex_exit(&gid_info->gl_mutex);
5410 	}
5411 	if (gid_info->gl_prev != NULL)
5412 		gid_info->gl_prev->gl_next = gid_info->gl_next;
5413 	if (gid_info->gl_next != NULL)
5414 		gid_info->gl_next->gl_prev = gid_info->gl_prev;
5415 
5416 	if (gid_info == ibdm.ibdm_dp_gidlist_head)
5417 		ibdm.ibdm_dp_gidlist_head = gid_info->gl_next;
5418 	if (gid_info == ibdm.ibdm_dp_gidlist_tail)
5419 		ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev;
5420 	ibdm.ibdm_ngids--;
5421 
5422 	ibdm.ibdm_busy &= ~IBDM_BUSY;
5423 	cv_broadcast(&ibdm.ibdm_busy_cv);
5424 	mutex_exit(&ibdm.ibdm_mutex);
5425 
5426 	mutex_destroy(&gid_info->gl_mutex);
5427 	kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t));
5428 
5429 	/*
5430 	 * Pass on the IOCs with updated GIDs to IBnexus
5431 	 */
5432 	if (ioc_list) {
5433 		IBTF_DPRINTF_L4("ibdm", "\tGID_UNAVAILABLE "
5434 		    "IOC_PROP_UPDATE for %p\n", ioc_list);
5435 		mutex_enter(&ibdm.ibdm_ibnex_mutex);
5436 		if (ibdm.ibdm_ibnex_callback != NULL) {
5437 			(*ibdm.ibdm_ibnex_callback)((void *)
5438 		    ioc_list, IBDM_EVENT_IOC_PROP_UPDATE);
5439 		}
5440 		mutex_exit(&ibdm.ibdm_ibnex_mutex);
5441 	}
5442 
5443 	ibdm_free_saa_event_arg(event_arg);
5444 }
5445 
5446 
5447 static int
5448 ibdm_cmp_gid_list(ibdm_gid_t *new, ibdm_gid_t *prev)
5449 {
5450 	ibdm_gid_t		*scan_new, *scan_prev;
5451 	int	cmp_failed = 0;
5452 
5453 	ASSERT(new != NULL);
5454 	ASSERT(prev != NULL);
5455 
5456 	/*
5457 	 * Search for each new gid anywhere in the prev GID list.
5458 	 * Note that the gid list could have been re-ordered.
5459 	 */
5460 	for (scan_new = new; scan_new; scan_new = scan_new->gid_next) {
5461 		for (scan_prev = prev, cmp_failed = 1; scan_prev;
5462 		    scan_prev = scan_prev->gid_next) {
5463 			if (scan_prev->gid_dgid_hi == scan_new->gid_dgid_hi &&
5464 			    scan_prev->gid_dgid_lo == scan_new->gid_dgid_lo) {
5465 				cmp_failed = 0;
5466 				break;
5467 			}
5468 		}
5469 
5470 		if (cmp_failed)
5471 			return (1);
5472 	}
5473 	return (0);
5474 }
5475 
5476 /*
5477  * This is always called in a single thread
5478  * This function updates the gid_list and serv_list of IOC
5479  * The current gid_list is in ioc_info_t(contains only port
5480  * guids for which probe is done) & gidinfo_t(other port gids)
5481  * The gids in both locations are used for comparision.
5482  */
5483 static void
5484 ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *ioc, ibdm_dp_gidinfo_t *gidinfo)
5485 {
5486 	ibdm_gid_t		*cur_gid_list;
5487 	uint_t			cur_nportgids;
5488 
5489 	ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
5490 
5491 	ioc->ioc_info_updated.ib_prop_updated = 0;
5492 
5493 
5494 	/* Current GID list in gid_info only */
5495 	cur_gid_list = gidinfo->gl_gid;
5496 	cur_nportgids = gidinfo->gl_ngids;
5497 
5498 	if (ioc->ioc_prev_serv_cnt !=
5499 	    ioc->ioc_profile.ioc_service_entries ||
5500 	    ibdm_serv_cmp(&ioc->ioc_serv[0], &ioc->ioc_prev_serv[0],
5501 	    ioc->ioc_prev_serv_cnt))
5502 		ioc->ioc_info_updated.ib_srv_prop_updated = 1;
5503 
5504 	if (ioc->ioc_prev_nportgids != cur_nportgids ||
5505 	    ioc->ioc_prev_gid_list == NULL || cur_gid_list == NULL) {
5506 		ioc->ioc_info_updated.ib_gid_prop_updated = 1;
5507 	} else if (ibdm_cmp_gid_list(ioc->ioc_prev_gid_list, cur_gid_list)) {
5508 		ioc->ioc_info_updated.ib_gid_prop_updated = 1;
5509 	}
5510 
5511 	/* Zero out previous entries */
5512 	ibdm_free_gid_list(ioc->ioc_prev_gid_list);
5513 	if (ioc->ioc_prev_serv)
5514 		kmem_free(ioc->ioc_prev_serv, ioc->ioc_prev_serv_cnt *
5515 		    sizeof (ibdm_srvents_info_t));
5516 	ioc->ioc_prev_serv_cnt = 0;
5517 	ioc->ioc_prev_nportgids = 0;
5518 	ioc->ioc_prev_serv = NULL;
5519 	ioc->ioc_prev_gid_list = NULL;
5520 }
5521 
5522 /*
5523  * Handle GID removal. This returns gid_info of an GID for the same
5524  * node GUID, if found.  For an GID with IOU information, the same
5525  * gid_info is returned if no gid_info with same node_guid is found.
5526  */
5527 static ibdm_dp_gidinfo_t *
5528 ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *rm_gid)
5529 {
5530 	ibdm_dp_gidinfo_t	*gid_list;
5531 
5532 	IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm(0x%p)", rm_gid);
5533 
5534 	if (rm_gid->gl_iou == NULL) {
5535 		IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm NO iou");
5536 		/*
5537 		 * Search for a GID with same node_guid and
5538 		 * gl_iou != NULL
5539 		 */
5540 		for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
5541 		    gid_list = gid_list->gl_next) {
5542 			if (gid_list->gl_iou != NULL && (gid_list->gl_nodeguid
5543 			    == rm_gid->gl_nodeguid))
5544 				break;
5545 		}
5546 
5547 		if (gid_list)
5548 			ibdm_rmfrom_glgid_list(gid_list, rm_gid);
5549 
5550 		IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list);
5551 		return (gid_list);
5552 	} else {
5553 		/*
5554 		 * Search for a GID with same node_guid and
5555 		 * gl_iou == NULL
5556 		 */
5557 		IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm with iou");
5558 		for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
5559 		    gid_list = gid_list->gl_next) {
5560 			if (gid_list->gl_iou == NULL && (gid_list->gl_nodeguid
5561 			    == rm_gid->gl_nodeguid))
5562 				break;
5563 		}
5564 
5565 		if (gid_list) {
5566 			/*
5567 			 * Copy the following fields from rm_gid :
5568 			 *	1. gl_state
5569 			 *	2. gl_iou
5570 			 *	3. gl_gid & gl_ngids
5571 			 *
5572 			 * Note :	Function is synchronized by
5573 			 *			ibdm_busy flag.
5574 			 *
5575 			 * Note :	Redirect info is initialized if
5576 			 *			any MADs for the GID fail
5577 			 */
5578 			IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm "
5579 			    "copying info to GID with gl_iou != NULl");
5580 			gid_list->gl_state = rm_gid->gl_state;
5581 			gid_list->gl_iou = rm_gid->gl_iou;
5582 			gid_list->gl_gid = rm_gid->gl_gid;
5583 			gid_list->gl_ngids = rm_gid->gl_ngids;
5584 
5585 			/* Remove the GID from gl_gid list */
5586 			ibdm_rmfrom_glgid_list(gid_list, rm_gid);
5587 		} else {
5588 			/*
5589 			 * Handle a case where all GIDs to the IOU have
5590 			 * been removed.
5591 			 */
5592 			IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm 0 GID "
5593 			    "to IOU");
5594 
5595 			ibdm_rmfrom_glgid_list(rm_gid, rm_gid);
5596 			return (rm_gid);
5597 		}
5598 		IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list);
5599 		return (gid_list);
5600 	}
5601 }
5602 
5603 static void
5604 ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *gid_info,
5605     ibdm_dp_gidinfo_t *rm_gid)
5606 {
5607 	ibdm_gid_t 		*tmp, *prev;
5608 
5609 	IBTF_DPRINTF_L4("ibdm", "\trmfrom_glgid (%p, %p)",
5610 	    gid_info, rm_gid);
5611 
5612 	for (tmp = gid_info->gl_gid, prev = NULL; tmp; ) {
5613 		if (tmp->gid_dgid_hi == rm_gid->gl_dgid_hi &&
5614 		    tmp->gid_dgid_lo == rm_gid->gl_dgid_lo) {
5615 			if (prev == NULL)
5616 				gid_info->gl_gid = tmp->gid_next;
5617 			else
5618 				prev->gid_next = tmp->gid_next;
5619 
5620 			kmem_free(tmp, sizeof (ibdm_gid_t));
5621 			gid_info->gl_ngids--;
5622 			break;
5623 		} else {
5624 			prev = tmp;
5625 			tmp = tmp->gid_next;
5626 		}
5627 	}
5628 }
5629 
5630 static void
5631 ibdm_addto_gidlist(ibdm_gid_t **src_ptr, ibdm_gid_t *dest)
5632 {
5633 	ibdm_gid_t *head = NULL, *new, *tail;
5634 
5635 	/* First copy the destination */
5636 	for (; dest; dest = dest->gid_next) {
5637 		new = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP);
5638 		new->gid_dgid_hi = dest->gid_dgid_hi;
5639 		new->gid_dgid_lo = dest->gid_dgid_lo;
5640 		new->gid_next = head;
5641 		head = new;
5642 	}
5643 
5644 	/* Insert this to the source */
5645 	if (*src_ptr == NULL)
5646 		*src_ptr = head;
5647 	else {
5648 		for (tail = *src_ptr; tail->gid_next != NULL;
5649 		    tail = tail->gid_next)
5650 			;
5651 
5652 		tail->gid_next = head;
5653 	}
5654 }
5655 
5656 static void
5657 ibdm_free_gid_list(ibdm_gid_t	*head)
5658 {
5659 	ibdm_gid_t	*delete;
5660 
5661 	for (delete = head; delete; ) {
5662 		head = delete->gid_next;
5663 		kmem_free(delete, sizeof (ibdm_gid_t));
5664 		delete = head;
5665 	}
5666 }
5667 
5668 /*
5669  * This function rescans the DM capable GIDs (gl_state is
5670  * GID_PROBE_COMPLETE or IBDM_GID_PROBING_SKIPPED.This
5671  * basically checks if the DM capable GID is reachable. If
5672  * not this is handled the same way as GID_UNAVAILABLE,
5673  * except that notifications are not send to IBnexus.
5674  *
5675  * This function also initializes the ioc_prev_list for
5676  * a particular IOC (when called from probe_ioc, with
5677  * ioc_guidp != NULL) or all IOCs for the gid (called from
5678  * sweep_fabric, ioc_guidp == NULL).
5679  */
5680 static void
5681 ibdm_rescan_gidlist(ib_guid_t *ioc_guidp)
5682 {
5683 	ibdm_dp_gidinfo_t	*gid_info, *tmp;
5684 	int ii, niocs, found;
5685 	ibdm_hca_list_t *hca_list = NULL;
5686 	ibdm_port_attr_t *port = NULL;
5687 	ibdm_ioc_info_t *ioc_list;
5688 
5689 	for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) {
5690 		found = 0;
5691 		if (gid_info->gl_state != IBDM_GID_PROBING_SKIPPED &&
5692 		    gid_info->gl_state != IBDM_GID_PROBING_COMPLETE) {
5693 			gid_info = gid_info->gl_next;
5694 			continue;
5695 		}
5696 
5697 		/*
5698 		 * Check if the GID is visible to any HCA ports.
5699 		 * Return if so.
5700 		 */
5701 		mutex_enter(&ibdm.ibdm_hl_mutex);
5702 		for (ibdm_get_next_port(&hca_list, &port, 1); port;
5703 		    ibdm_get_next_port(&hca_list, &port, 1)) {
5704 			if (ibdm_port_reachable(port->pa_sa_hdl,
5705 			    gid_info->gl_dgid_lo, NULL) == IBDM_SUCCESS) {
5706 				found = 1;
5707 				break;
5708 			}
5709 		}
5710 		mutex_exit(&ibdm.ibdm_hl_mutex);
5711 
5712 		if (found) {
5713 			if (gid_info->gl_iou == NULL) {
5714 				gid_info = gid_info->gl_next;
5715 				continue;
5716 			}
5717 
5718 			/* Intialize the ioc_prev_gid_list */
5719 			niocs =
5720 			    gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
5721 			for (ii = 0; ii < niocs; ii++) {
5722 				ioc_list = IBDM_GIDINFO2IOCINFO(gid_info, ii);
5723 
5724 				if (ioc_guidp == NULL || (*ioc_guidp ==
5725 				    ioc_list->ioc_profile.ioc_guid)) {
5726 					/* Add info of GIDs in gid_info also */
5727 					ibdm_addto_gidlist(
5728 					    &ioc_list->ioc_prev_gid_list,
5729 					    gid_info->gl_gid);
5730 					ioc_list->ioc_prev_nportgids =
5731 					    gid_info->gl_ngids;
5732 				}
5733 			}
5734 			gid_info = gid_info->gl_next;
5735 			continue;
5736 		}
5737 
5738 		IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist "
5739 		    "deleted port GUID %llx",
5740 		    gid_info->gl_dgid_lo);
5741 
5742 		/*
5743 		 * Update GID list in all IOCs affected by this
5744 		 */
5745 		ioc_list = ibdm_update_ioc_gidlist(gid_info, 0);
5746 
5747 		/*
5748 		 * Remove GID from the global GID list
5749 		 * Handle the case where all port GIDs for an
5750 		 * IOU have been hot-removed.
5751 		 */
5752 		mutex_enter(&ibdm.ibdm_mutex);
5753 		if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) {
5754 			mutex_enter(&gid_info->gl_mutex);
5755 			(void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou);
5756 			mutex_exit(&gid_info->gl_mutex);
5757 		}
5758 		tmp = gid_info->gl_next;
5759 		if (gid_info->gl_prev != NULL)
5760 			gid_info->gl_prev->gl_next = gid_info->gl_next;
5761 		if (gid_info->gl_next != NULL)
5762 			gid_info->gl_next->gl_prev = gid_info->gl_prev;
5763 
5764 		if (gid_info == ibdm.ibdm_dp_gidlist_head)
5765 			ibdm.ibdm_dp_gidlist_head = gid_info->gl_next;
5766 		if (gid_info == ibdm.ibdm_dp_gidlist_tail)
5767 			ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev;
5768 		ibdm.ibdm_ngids--;
5769 
5770 		mutex_destroy(&gid_info->gl_mutex);
5771 		kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t));
5772 		gid_info = tmp;
5773 
5774 		mutex_exit(&ibdm.ibdm_mutex);
5775 
5776 		/*
5777 		 * Pass on the IOCs with updated GIDs to IBnexus
5778 		 */
5779 		if (ioc_list) {
5780 			IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist "
5781 			    "IOC_PROP_UPDATE for %p\n", ioc_list);
5782 			mutex_enter(&ibdm.ibdm_ibnex_mutex);
5783 			if (ibdm.ibdm_ibnex_callback != NULL) {
5784 				(*ibdm.ibdm_ibnex_callback)((void *)
5785 				    ioc_list, IBDM_EVENT_IOC_PROP_UPDATE);
5786 			}
5787 			mutex_exit(&ibdm.ibdm_ibnex_mutex);
5788 		}
5789 	}
5790 }
5791 
5792 /*
5793  * This function notifies IBnex of IOCs on this GID.
5794  * Notification is for GIDs with gl_reprobe_flag set.
5795  * The flag is set when IOC probe / fabric sweep
5796  * probes a GID starting from CLASS port info.
5797  *
5798  * IBnexus will have information of a reconnected IOC
5799  * if it had probed it before. If this is a new IOC,
5800  * IBnexus ignores the notification.
5801  *
5802  * This function should be called with no locks held.
5803  */
5804 static void
5805 ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *gid_info)
5806 {
5807 	ibdm_ioc_info_t	*ioc_list;
5808 
5809 	if (gid_info->gl_reprobe_flag == 0 ||
5810 	    gid_info->gl_iou == NULL)
5811 		return;
5812 
5813 	ioc_list = ibdm_update_ioc_gidlist(gid_info, -1);
5814 
5815 	/*
5816 	 * Pass on the IOCs with updated GIDs to IBnexus
5817 	 */
5818 	if (ioc_list) {
5819 		mutex_enter(&ibdm.ibdm_ibnex_mutex);
5820 		if (ibdm.ibdm_ibnex_callback != NULL) {
5821 			(*ibdm.ibdm_ibnex_callback)((void *)ioc_list,
5822 			    IBDM_EVENT_IOC_PROP_UPDATE);
5823 		}
5824 		mutex_exit(&ibdm.ibdm_ibnex_mutex);
5825 	}
5826 }
5827 
5828 
5829 static void
5830 ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *arg)
5831 {
5832 	if (arg != NULL)
5833 		kmem_free(arg, sizeof (ibdm_saa_event_arg_t));
5834 }
5835 
5836 /*
5837  * This function parses the list of HCAs and HCA ports
5838  * to return the port_attr of the next HCA port. A port
5839  * connected to IB fabric (port_state active) is returned,
5840  * if connected_flag is set.
5841  */
5842 static void
5843 ibdm_get_next_port(ibdm_hca_list_t **inp_hcap,
5844     ibdm_port_attr_t **inp_portp, int connect_flag)
5845 {
5846 	int ii;
5847 	ibdm_port_attr_t *port, *next_port = NULL;
5848 	ibdm_port_attr_t *inp_port;
5849 	ibdm_hca_list_t	 *hca_list;
5850 	int found = 0;
5851 
5852 	ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
5853 	IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port(%p, %p, %x)",
5854 	    inp_hcap, inp_portp, connect_flag);
5855 
5856 	hca_list = *inp_hcap;
5857 	inp_port = *inp_portp;
5858 
5859 	if (hca_list == NULL)
5860 		hca_list = ibdm.ibdm_hca_list_head;
5861 
5862 	for (; hca_list; hca_list = hca_list->hl_next) {
5863 		for (ii = 0; ii < hca_list->hl_nports; ii++) {
5864 			port = &hca_list->hl_port_attr[ii];
5865 
5866 			/*
5867 			 * inp_port != NULL;
5868 			 * 	Skip till we find the matching port
5869 			 */
5870 			if (inp_port && !found) {
5871 				if (inp_port == port)
5872 					found = 1;
5873 				continue;
5874 			}
5875 
5876 			if (!connect_flag) {
5877 				next_port = port;
5878 				break;
5879 			}
5880 
5881 			if (port->pa_sa_hdl == NULL)
5882 				ibdm_initialize_port(port);
5883 			if (port->pa_sa_hdl == NULL)
5884 				(void) ibdm_fini_port(port);
5885 			else if (next_port == NULL &&
5886 			    port->pa_sa_hdl != NULL &&
5887 			    port->pa_state == IBT_PORT_ACTIVE) {
5888 				next_port = port;
5889 				break;
5890 			}
5891 		}
5892 
5893 		if (next_port)
5894 			break;
5895 	}
5896 
5897 	IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port : "
5898 	    "returns hca_list %p port %p", hca_list, next_port);
5899 	*inp_hcap = hca_list;
5900 	*inp_portp = next_port;
5901 }
5902 
5903 static void
5904 ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *nodegid, ibdm_dp_gidinfo_t *addgid)
5905 {
5906 	ibdm_gid_t	*tmp;
5907 
5908 	tmp = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP);
5909 	tmp->gid_dgid_hi = addgid->gl_dgid_hi;
5910 	tmp->gid_dgid_lo = addgid->gl_dgid_lo;
5911 
5912 	mutex_enter(&nodegid->gl_mutex);
5913 	tmp->gid_next = nodegid->gl_gid;
5914 	nodegid->gl_gid = tmp;
5915 	nodegid->gl_ngids++;
5916 	mutex_exit(&nodegid->gl_mutex);
5917 }
5918 
5919 static void
5920 ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *gid_info,
5921     ibdm_hca_list_t *hca)
5922 {
5923 	ibdm_hca_list_t		*head, *prev = NULL, *temp;
5924 
5925 	IBTF_DPRINTF_L4(ibdm_string, "\taddto_glhcalist(%p, %p) "
5926 	    ": gl_hca_list %p", gid_info, hca, gid_info->gl_hca_list);
5927 	ASSERT(!MUTEX_HELD(&gid_info->gl_mutex));
5928 	mutex_enter(&gid_info->gl_mutex);
5929 	head = gid_info->gl_hca_list;
5930 	if (head == NULL) {
5931 		head = ibdm_dup_hca_attr(hca);
5932 		head->hl_next = NULL;
5933 		gid_info->gl_hca_list = head;
5934 		mutex_exit(&gid_info->gl_mutex);
5935 		IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: "
5936 		    "gid %p, gl_hca_list %p", gid_info,
5937 		    gid_info->gl_hca_list);
5938 		return;
5939 	}
5940 
5941 	/* Check if already in the list */
5942 	while (head) {
5943 		if (head->hl_hca_guid == hca->hl_hca_guid) {
5944 			mutex_exit(&gid_info->gl_mutex);
5945 			IBTF_DPRINTF_L4(ibdm_string,
5946 			    "\taddto_glhcalist : gid %p hca %p dup",
5947 			    gid_info, hca);
5948 			return;
5949 		}
5950 		prev = head;
5951 		head = head->hl_next;
5952 	}
5953 
5954 	/* Add this HCA to gl_hca_list */
5955 	temp =  ibdm_dup_hca_attr(hca);
5956 	temp->hl_next = NULL;
5957 	prev->hl_next = temp;
5958 
5959 	mutex_exit(&gid_info->gl_mutex);
5960 
5961 	IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: "
5962 	    "gid %p, gl_hca_list %p", gid_info, gid_info->gl_hca_list);
5963 }
5964 
5965 static void
5966 ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *gid_info)
5967 {
5968 	ASSERT(!MUTEX_HELD(&gid_info->gl_mutex));
5969 	ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex));
5970 
5971 	mutex_enter(&gid_info->gl_mutex);
5972 	if (gid_info->gl_hca_list)
5973 		ibdm_ibnex_free_hca_list(gid_info->gl_hca_list);
5974 	gid_info->gl_hca_list = NULL;
5975 	mutex_exit(&gid_info->gl_mutex);
5976 }
5977 
5978 
5979 static void
5980 ibdm_reset_all_dgids(ibmf_saa_handle_t port_sa_hdl)
5981 {
5982 	IBTF_DPRINTF_L4(ibdm_string, "\treset_all_dgids(%X)",
5983 	    port_sa_hdl);
5984 
5985 	ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex));
5986 	ASSERT(!MUTEX_HELD(&ibdm.ibdm_hl_mutex));
5987 
5988 	/* Check : Not busy in another probe / sweep */
5989 	mutex_enter(&ibdm.ibdm_mutex);
5990 	if ((ibdm.ibdm_busy & IBDM_BUSY) == 0) {
5991 		ibdm_dp_gidinfo_t	*gid_info;
5992 
5993 		ibdm.ibdm_busy |= IBDM_BUSY;
5994 		mutex_exit(&ibdm.ibdm_mutex);
5995 
5996 		/*
5997 		 * Check if any GID is using the SA & IBMF handle
5998 		 * of HCA port going down. Reset ibdm_dp_gidinfo_t
5999 		 * using another HCA port which can reach the GID.
6000 		 * This is for DM capable GIDs only, no need to do
6001 		 * this for others
6002 		 *
6003 		 * Delete the GID if no alternate HCA port to reach
6004 		 * it is found.
6005 		 */
6006 		for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) {
6007 			ibdm_dp_gidinfo_t *tmp;
6008 
6009 			IBTF_DPRINTF_L4(ibdm_string, "\tevent_hdlr "
6010 			    "checking gidinfo %p", gid_info);
6011 
6012 			if (gid_info->gl_sa_hdl == port_sa_hdl) {
6013 				IBTF_DPRINTF_L3(ibdm_string,
6014 				    "\tevent_hdlr: down HCA port hdl "
6015 				    "matches gid %p", gid_info);
6016 
6017 				/*
6018 				 * The non-DM GIDs can come back
6019 				 * with a new subnet prefix, when
6020 				 * the HCA port commes up again. To
6021 				 * avoid issues, delete non-DM
6022 				 * capable GIDs, if the gid was
6023 				 * discovered using the HCA port
6024 				 * going down. This is ensured by
6025 				 * setting gl_disconnected to 1.
6026 				 */
6027 				if (gid_info->gl_nodeguid == 0)
6028 					gid_info->gl_disconnected = 1;
6029 				else
6030 					ibdm_reset_gidinfo(gid_info);
6031 
6032 				if (gid_info->gl_disconnected) {
6033 					IBTF_DPRINTF_L3(ibdm_string,
6034 					    "\tevent_hdlr: deleting"
6035 					    " gid %p", gid_info);
6036 					tmp = gid_info;
6037 					gid_info = gid_info->gl_next;
6038 					ibdm_delete_gidinfo(tmp);
6039 				} else
6040 					gid_info = gid_info->gl_next;
6041 			} else
6042 				gid_info = gid_info->gl_next;
6043 		}
6044 
6045 		mutex_enter(&ibdm.ibdm_mutex);
6046 		ibdm.ibdm_busy &= ~IBDM_BUSY;
6047 		cv_signal(&ibdm.ibdm_busy_cv);
6048 	}
6049 	mutex_exit(&ibdm.ibdm_mutex);
6050 }
6051 
6052 static void
6053 ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *gidinfo)
6054 {
6055 	ibdm_hca_list_t	*hca_list = NULL;
6056 	ibdm_port_attr_t	*port = NULL;
6057 	int	gid_reinited = 0;
6058 	sa_node_record_t	*nr, *tmp;
6059 	sa_portinfo_record_t	*pi;
6060 	size_t	nr_len = 0, pi_len = 0;
6061 	size_t	path_len;
6062 	ib_gid_t	sgid, dgid;
6063 	int	ret, ii, nrecords;
6064 	sa_path_record_t	*path;
6065 	uint8_t	npaths = 1;
6066 	ibdm_pkey_tbl_t		*pkey_tbl;
6067 
6068 	IBTF_DPRINTF_L4(ibdm_string, "\treset_gidinfo(%p)", gidinfo);
6069 
6070 	/*
6071 	 * Get list of all the ports reachable from the local known HCA
6072 	 * ports which are active
6073 	 */
6074 	mutex_enter(&ibdm.ibdm_hl_mutex);
6075 	for (ibdm_get_next_port(&hca_list, &port, 1); port;
6076 	    ibdm_get_next_port(&hca_list, &port, 1)) {
6077 
6078 
6079 		/*
6080 		 * Get the path and re-populate the gidinfo.
6081 		 * Getting the path is the same probe_ioc
6082 		 * Init the gid info as in ibdm_create_gidinfo()
6083 		 */
6084 		nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len,
6085 		    gidinfo->gl_nodeguid);
6086 		if (nr == NULL) {
6087 			IBTF_DPRINTF_L4(ibdm_string,
6088 			    "\treset_gidinfo : no records");
6089 			continue;
6090 		}
6091 
6092 		nrecords = (nr_len / sizeof (sa_node_record_t));
6093 		for (tmp = nr, ii = 0;  (ii < nrecords); ii++, tmp++) {
6094 			if (tmp->NodeInfo.PortGUID == gidinfo->gl_portguid)
6095 				break;
6096 		}
6097 
6098 		if (ii == nrecords) {
6099 			IBTF_DPRINTF_L4(ibdm_string,
6100 			    "\treset_gidinfo : no record for portguid");
6101 			kmem_free(nr, nr_len);
6102 			continue;
6103 		}
6104 
6105 		pi = ibdm_get_portinfo(port->pa_sa_hdl, &pi_len, tmp->LID);
6106 		if (pi == NULL) {
6107 			IBTF_DPRINTF_L4(ibdm_string,
6108 			    "\treset_gidinfo : no portinfo");
6109 			kmem_free(nr, nr_len);
6110 			continue;
6111 		}
6112 
6113 		sgid.gid_prefix = port->pa_sn_prefix;
6114 		sgid.gid_guid = port->pa_port_guid;
6115 		dgid.gid_prefix = pi->PortInfo.GidPrefix;
6116 		dgid.gid_guid = tmp->NodeInfo.PortGUID;
6117 
6118 		ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl, sgid, dgid,
6119 		    IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0, &path_len, &path);
6120 
6121 		if ((ret != IBMF_SUCCESS) || path == NULL) {
6122 			IBTF_DPRINTF_L4(ibdm_string,
6123 			    "\treset_gidinfo : no paths");
6124 			kmem_free(pi, pi_len);
6125 			kmem_free(nr, nr_len);
6126 			continue;
6127 		}
6128 
6129 		gidinfo->gl_dgid_hi	= path->DGID.gid_prefix;
6130 		gidinfo->gl_dgid_lo	= path->DGID.gid_guid;
6131 		gidinfo->gl_sgid_hi	= path->SGID.gid_prefix;
6132 		gidinfo->gl_sgid_lo	= path->SGID.gid_guid;
6133 		gidinfo->gl_p_key	= path->P_Key;
6134 		gidinfo->gl_sa_hdl	= port->pa_sa_hdl;
6135 		gidinfo->gl_ibmf_hdl	= port->pa_ibmf_hdl;
6136 		gidinfo->gl_slid	= path->SLID;
6137 		gidinfo->gl_dlid	= path->DLID;
6138 		/* Reset redirect info, next MAD will set if redirected */
6139 		gidinfo->gl_redirected = 0;
6140 
6141 		gidinfo->gl_qp_hdl = IBMF_QP_HANDLE_DEFAULT;
6142 		for (ii = 0; ii < port->pa_npkeys; ii++) {
6143 			if (port->pa_pkey_tbl == NULL)
6144 				break;
6145 
6146 			pkey_tbl = &port->pa_pkey_tbl[ii];
6147 			if ((gidinfo->gl_p_key == pkey_tbl->pt_pkey) &&
6148 			    (pkey_tbl->pt_qp_hdl != NULL)) {
6149 				gidinfo->gl_qp_hdl = pkey_tbl->pt_qp_hdl;
6150 				break;
6151 			}
6152 		}
6153 
6154 		if (gidinfo->gl_qp_hdl == NULL)
6155 			IBTF_DPRINTF_L2(ibdm_string,
6156 			    "\treset_gid_info: No matching Pkey");
6157 		else
6158 			gid_reinited = 1;
6159 
6160 		kmem_free(path, path_len);
6161 		kmem_free(pi, pi_len);
6162 		kmem_free(nr, nr_len);
6163 		break;
6164 	}
6165 	mutex_exit(&ibdm.ibdm_hl_mutex);
6166 
6167 	if (!gid_reinited)
6168 		gidinfo->gl_disconnected = 1;
6169 }
6170 
6171 static void
6172 ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *gidinfo)
6173 {
6174 	ibdm_ioc_info_t *ioc_list;
6175 	int	in_gidlist = 0;
6176 
6177 	/*
6178 	 * Check if gidinfo has been inserted into the
6179 	 * ibdm_dp_gidlist_head list. gl_next or gl_prev
6180 	 * != NULL, if gidinfo is the list.
6181 	 */
6182 	if (gidinfo->gl_prev != NULL ||
6183 	    gidinfo->gl_next != NULL ||
6184 	    ibdm.ibdm_dp_gidlist_head == gidinfo)
6185 		in_gidlist = 1;
6186 
6187 	ioc_list = ibdm_update_ioc_gidlist(gidinfo, 0);
6188 
6189 	/*
6190 	 * Remove GID from the global GID list
6191 	 * Handle the case where all port GIDs for an
6192 	 * IOU have been hot-removed.
6193 	 */
6194 	mutex_enter(&ibdm.ibdm_mutex);
6195 	if (gidinfo->gl_iou != NULL && gidinfo->gl_ngids == 0) {
6196 		mutex_enter(&gidinfo->gl_mutex);
6197 		(void) ibdm_free_iou_info(gidinfo, &gidinfo->gl_iou);
6198 		mutex_exit(&gidinfo->gl_mutex);
6199 	}
6200 
6201 	/* Delete gl_hca_list */
6202 	mutex_exit(&ibdm.ibdm_mutex);
6203 	ibdm_delete_glhca_list(gidinfo);
6204 	mutex_enter(&ibdm.ibdm_mutex);
6205 
6206 	if (in_gidlist) {
6207 		if (gidinfo->gl_prev != NULL)
6208 			gidinfo->gl_prev->gl_next = gidinfo->gl_next;
6209 		if (gidinfo->gl_next != NULL)
6210 			gidinfo->gl_next->gl_prev = gidinfo->gl_prev;
6211 
6212 		if (gidinfo == ibdm.ibdm_dp_gidlist_head)
6213 			ibdm.ibdm_dp_gidlist_head = gidinfo->gl_next;
6214 		if (gidinfo == ibdm.ibdm_dp_gidlist_tail)
6215 			ibdm.ibdm_dp_gidlist_tail = gidinfo->gl_prev;
6216 		ibdm.ibdm_ngids--;
6217 	}
6218 	mutex_exit(&ibdm.ibdm_mutex);
6219 
6220 	mutex_destroy(&gidinfo->gl_mutex);
6221 	kmem_free(gidinfo, sizeof (ibdm_dp_gidinfo_t));
6222 
6223 	/*
6224 	 * Pass on the IOCs with updated GIDs to IBnexus
6225 	 */
6226 	if (ioc_list) {
6227 		IBTF_DPRINTF_L4("ibdm", "\tdelete_gidinfo "
6228 		    "IOC_PROP_UPDATE for %p\n", ioc_list);
6229 		mutex_enter(&ibdm.ibdm_ibnex_mutex);
6230 		if (ibdm.ibdm_ibnex_callback != NULL) {
6231 			(*ibdm.ibdm_ibnex_callback)((void *)
6232 			    ioc_list, IBDM_EVENT_IOC_PROP_UPDATE);
6233 		}
6234 		mutex_exit(&ibdm.ibdm_ibnex_mutex);
6235 	}
6236 }
6237 
6238 
6239 static void
6240 ibdm_fill_srv_attr_mod(ib_mad_hdr_t *hdr, ibdm_timeout_cb_args_t *cb_args)
6241 {
6242 	uint32_t	attr_mod;
6243 
6244 	attr_mod = (cb_args->cb_ioc_num + 1) << 16;
6245 	attr_mod |= cb_args->cb_srvents_start;
6246 	attr_mod |= (cb_args->cb_srvents_end) << 8;
6247 	hdr->AttributeModifier = h2b32(attr_mod);
6248 }
6249 
6250 static void
6251 ibdm_bump_transactionID(ibdm_dp_gidinfo_t *gid_info)
6252 {
6253 	ASSERT(MUTEX_HELD(&gid_info->gl_mutex));
6254 	gid_info->gl_transactionID++;
6255 	if (gid_info->gl_transactionID == gid_info->gl_max_transactionID) {
6256 		IBTF_DPRINTF_L4(ibdm_string,
6257 		    "\tbump_transactionID(%p), wrapup", gid_info);
6258 		gid_info->gl_transactionID = gid_info->gl_min_transactionID;
6259 	}
6260 }
6261 
6262 /*
6263  * gl_prev_iou is set for *non-reprobe* sweeep requests, which
6264  * detected that ChangeID in IOU info has changed. The service
6265  * entry also may have changed. Check if service entry in IOC
6266  * has changed wrt the prev iou, if so notify to IB Nexus.
6267  */
6268 static ibdm_ioc_info_t *
6269 ibdm_handle_prev_iou()
6270 {
6271 	ibdm_dp_gidinfo_t *gid_info;
6272 	ibdm_ioc_info_t	*ioc_list_head = NULL, *ioc_list;
6273 	ibdm_ioc_info_t	*prev_ioc, *ioc;
6274 	int		ii, jj, niocs, prev_niocs;
6275 
6276 	ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
6277 
6278 	IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iou enter");
6279 	for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
6280 	    gid_info = gid_info->gl_next) {
6281 		if (gid_info->gl_prev_iou == NULL)
6282 			continue;
6283 
6284 		IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iou gid %p",
6285 		    gid_info);
6286 		niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
6287 		prev_niocs =
6288 		    gid_info->gl_prev_iou->iou_info.iou_num_ctrl_slots;
6289 		for (ii = 0; ii < niocs; ii++) {
6290 			ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii);
6291 
6292 			/* Find matching IOC */
6293 			for (jj = 0; jj < prev_niocs; jj++) {
6294 				prev_ioc = (ibdm_ioc_info_t *)
6295 				    &gid_info->gl_prev_iou->iou_ioc_info[jj];
6296 				if (prev_ioc->ioc_profile.ioc_guid ==
6297 				    ioc->ioc_profile.ioc_guid)
6298 					break;
6299 			}
6300 			if (jj == prev_niocs)
6301 				prev_ioc = NULL;
6302 			if (ioc == NULL || prev_ioc == NULL)
6303 				continue;
6304 			if ((ioc->ioc_profile.ioc_service_entries !=
6305 			    prev_ioc->ioc_profile.ioc_service_entries) ||
6306 			    ibdm_serv_cmp(&ioc->ioc_serv[0],
6307 			    &prev_ioc->ioc_serv[0],
6308 			    ioc->ioc_profile.ioc_service_entries) != 0) {
6309 				IBTF_DPRINTF_L4(ibdm_string,
6310 				    "/thandle_prev_iou modified IOC: "
6311 				    "current ioc %p, old ioc %p",
6312 				    ioc, prev_ioc);
6313 				ioc_list = ibdm_dup_ioc_info(ioc, gid_info);
6314 				ioc_list->ioc_info_updated.ib_prop_updated
6315 				    = 0;
6316 				ioc_list->ioc_info_updated.ib_srv_prop_updated
6317 				    = 1;
6318 
6319 				if (ioc_list_head == NULL)
6320 					ioc_list_head = ioc_list;
6321 				else {
6322 					ioc_list_head->ioc_next = ioc_list;
6323 					ioc_list_head = ioc_list;
6324 				}
6325 			}
6326 		}
6327 
6328 		mutex_enter(&gid_info->gl_mutex);
6329 		(void) ibdm_free_iou_info(gid_info, &gid_info->gl_prev_iou);
6330 		mutex_exit(&gid_info->gl_mutex);
6331 	}
6332 	IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iouret %p",
6333 	    ioc_list_head);
6334 	return (ioc_list_head);
6335 }
6336 
6337 /*
6338  * Compares two service entries lists, returns 0 if same, returns 1
6339  * if no match.
6340  */
6341 static int
6342 ibdm_serv_cmp(ibdm_srvents_info_t *serv1, ibdm_srvents_info_t *serv2,
6343     int nserv)
6344 {
6345 	int	ii;
6346 
6347 	IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: enter");
6348 	for (ii = 0; ii < nserv; ii++, serv1++, serv2++) {
6349 		if (serv1->se_attr.srv_id != serv2->se_attr.srv_id ||
6350 			bcmp(serv1->se_attr.srv_name,
6351 			    serv2->se_attr.srv_name,
6352 			    IB_DM_MAX_SVC_NAME_LEN) != 0) {
6353 			IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: ret 1");
6354 			return (1);
6355 		}
6356 	}
6357 	IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: ret 0");
6358 	return (0);
6359 }
6360 
6361 /* For debugging purpose only */
6362 #ifdef	DEBUG
6363 void
6364 ibdm_dump_ibmf_msg(ibmf_msg_t *ibmf_msg, int flag)
6365 {
6366 	ib_mad_hdr_t	*mad_hdr;
6367 
6368 	IBTF_DPRINTF_L4("ibdm", "\t\t(IBMF_PKT): Local address info");
6369 	IBTF_DPRINTF_L4("ibdm", "\t\t            ------------------");
6370 
6371 	IBTF_DPRINTF_L4("ibdm", "\tLocal Lid  : 0x%x\tRemote Lid : 0x%x"
6372 	    " Remote Qp  : 0x%x", ibmf_msg->im_local_addr.ia_local_lid,
6373 	    ibmf_msg->im_local_addr.ia_remote_lid,
6374 	    ibmf_msg->im_local_addr.ia_remote_qno);
6375 	IBTF_DPRINTF_L4("ibdm", "\tP_key      : 0x%x\tQ_key      : 0x%x",
6376 	    ibmf_msg->im_local_addr.ia_p_key, ibmf_msg->im_local_addr.ia_q_key);
6377 
6378 	if (flag)
6379 		mad_hdr = (ib_mad_hdr_t *)IBDM_OUT_IBMFMSG_MADHDR(ibmf_msg);
6380 	else
6381 		mad_hdr = IBDM_IN_IBMFMSG_MADHDR(ibmf_msg);
6382 
6383 	IBTF_DPRINTF_L4("ibdm", "\t\t MAD Header info");
6384 	IBTF_DPRINTF_L4("ibdm", "\t\t ---------------");
6385 
6386 	IBTF_DPRINTF_L4("ibdm", "\tBase version  : 0x%x"
6387 	    "\tMgmt Class : 0x%x", mad_hdr->BaseVersion, mad_hdr->MgmtClass);
6388 	IBTF_DPRINTF_L4("ibdm", "\tClass version : 0x%x"
6389 	    "\tR Method           : 0x%x",
6390 	    mad_hdr->ClassVersion, mad_hdr->R_Method);
6391 	IBTF_DPRINTF_L4("ibdm", "\tMAD  Status   : 0x%x"
6392 	    "\tTransaction ID     : 0x%llx",
6393 	    mad_hdr->Status, mad_hdr->TransactionID);
6394 	IBTF_DPRINTF_L4("ibdm", "\t Attribute ID  : 0x%x"
6395 	    "\tAttribute Modified : 0x%lx",
6396 	    mad_hdr->AttributeID, mad_hdr->AttributeModifier);
6397 }
6398 
6399 void
6400 ibdm_dump_path_info(sa_path_record_t *path)
6401 {
6402 	IBTF_DPRINTF_L4("ibdm", "\t\t Path information");
6403 	IBTF_DPRINTF_L4("ibdm", "\t\t ----------------");
6404 
6405 	IBTF_DPRINTF_L4("ibdm", "\t DGID hi  : %llx\tDGID lo  : %llx",
6406 	    path->DGID.gid_prefix, path->DGID.gid_guid);
6407 	IBTF_DPRINTF_L4("ibdm", "\t SGID hi  : %llx\tSGID lo  : %llx",
6408 	    path->SGID.gid_prefix, path->SGID.gid_guid);
6409 	IBTF_DPRINTF_L4("ibdm", "\t SLID     : %x\tDlID     : %x",
6410 	    path->SLID, path->DLID);
6411 	IBTF_DPRINTF_L4("ibdm", "\t P Key    : %x", path->P_Key);
6412 }
6413 
6414 
6415 void
6416 ibdm_dump_classportinfo(ibdm_mad_classportinfo_t *classportinfo)
6417 {
6418 	IBTF_DPRINTF_L4("ibdm", "\t\t CLASSPORT INFO");
6419 	IBTF_DPRINTF_L4("ibdm", "\t\t --------------");
6420 
6421 	IBTF_DPRINTF_L4("ibdm", "\t Response Time Value : 0x%x",
6422 	    ((b2h32(classportinfo->RespTimeValue)) & 0x1F));
6423 
6424 	IBTF_DPRINTF_L4("ibdm", "\t Redirected QP       : 0x%x",
6425 	    (b2h32(classportinfo->RedirectQP)));
6426 	IBTF_DPRINTF_L4("ibdm", "\t Redirected P KEY    : 0x%x",
6427 	    b2h16(classportinfo->RedirectP_Key));
6428 	IBTF_DPRINTF_L4("ibdm", "\t Redirected Q KEY    : 0x%x",
6429 	    b2h16(classportinfo->RedirectQ_Key));
6430 	IBTF_DPRINTF_L4("ibdm", "\t Redirected GID hi   : 0x%x",
6431 	    b2h64(classportinfo->RedirectGID_hi));
6432 	IBTF_DPRINTF_L4("ibdm", "\t Redirected GID lo   : 0x%x",
6433 	    b2h64(classportinfo->RedirectGID_lo));
6434 }
6435 
6436 
6437 void
6438 ibdm_dump_iounitinfo(ib_dm_io_unitinfo_t *iou_info)
6439 {
6440 	IBTF_DPRINTF_L4("ibdm", "\t\t I/O UnitInfo");
6441 	IBTF_DPRINTF_L4("ibdm", "\t\t ------------");
6442 
6443 	IBTF_DPRINTF_L4("ibdm", "\tChange ID            : 0x%x",
6444 	    b2h16(iou_info->iou_changeid));
6445 	IBTF_DPRINTF_L4("ibdm", "\t#of ctrl slots       : %d",
6446 	    iou_info->iou_num_ctrl_slots);
6447 	IBTF_DPRINTF_L4("ibdm", "\tIOU flag             : 0x%x",
6448 	    iou_info->iou_flag);
6449 	IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 0   : 0x%x",
6450 	    iou_info->iou_ctrl_list[0]);
6451 	IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 1   : 0x%x",
6452 	    iou_info->iou_ctrl_list[1]);
6453 	IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 2   : 0x%x",
6454 	    iou_info->iou_ctrl_list[2]);
6455 }
6456 
6457 
6458 void
6459 ibdm_dump_ioc_profile(ib_dm_ioc_ctrl_profile_t *ioc)
6460 {
6461 	IBTF_DPRINTF_L4("ibdm", "\t\t IOC Controller Profile");
6462 	IBTF_DPRINTF_L4("ibdm", "\t\t ----------------------");
6463 
6464 	IBTF_DPRINTF_L4("ibdm", "\tIOC Guid    : %llx", ioc->ioc_guid);
6465 	IBTF_DPRINTF_L4("ibdm", "\tVendorID    : 0x%x", ioc->ioc_vendorid);
6466 	IBTF_DPRINTF_L4("ibdm", "\tDevice Id   : 0x%x", ioc->ioc_deviceid);
6467 	IBTF_DPRINTF_L4("ibdm", "\tDevice Ver  : 0x%x", ioc->ioc_device_ver);
6468 	IBTF_DPRINTF_L4("ibdm", "\tSubsys ID   : 0x%x", ioc->ioc_subsys_id);
6469 	IBTF_DPRINTF_L4("ibdm", "\tIO class    : 0x%x", ioc->ioc_io_class);
6470 	IBTF_DPRINTF_L4("ibdm", "\tIO subclass : 0x%x", ioc->ioc_io_subclass);
6471 	IBTF_DPRINTF_L4("ibdm", "\tProtocol    : 0x%x", ioc->ioc_protocol);
6472 	IBTF_DPRINTF_L4("ibdm", "\tProtocolV   : 0x%x", ioc->ioc_protocol_ver);
6473 	IBTF_DPRINTF_L4("ibdm", "\tmsg qdepth  : %d", ioc->ioc_send_msg_qdepth);
6474 	IBTF_DPRINTF_L4("ibdm", "\trdma qdepth : %d",
6475 						ioc->ioc_rdma_read_qdepth);
6476 	IBTF_DPRINTF_L4("ibdm", "\tsndmsg sz   : %d", ioc->ioc_send_msg_sz);
6477 	IBTF_DPRINTF_L4("ibdm", "\trdma xfersz : %d", ioc->ioc_rdma_xfer_sz);
6478 	IBTF_DPRINTF_L4("ibdm", "\topcal mask  : 0x%x",
6479 						ioc->ioc_ctrl_opcap_mask);
6480 	IBTF_DPRINTF_L4("ibdm", "\tsrventries  : %x", ioc->ioc_service_entries);
6481 }
6482 
6483 
6484 void
6485 ibdm_dump_service_entries(ib_dm_srv_t *srv_ents)
6486 {
6487 	IBTF_DPRINTF_L4("ibdm",
6488 	    "\thandle_srventry_mad: service id : %llx", srv_ents->srv_id);
6489 
6490 	IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad: "
6491 	    "Service Name : %s", srv_ents->srv_name);
6492 }
6493 
6494 int ibdm_allow_sweep_fabric_timestamp = 1;
6495 
6496 void
6497 ibdm_dump_sweep_fabric_timestamp(int flag)
6498 {
6499 	static hrtime_t x;
6500 	if (flag) {
6501 		if (ibdm_allow_sweep_fabric_timestamp) {
6502 			IBTF_DPRINTF_L4("ibdm", "\tTime taken to complete "
6503 			    "sweep %lld ms", ((gethrtime() - x)/ 1000000));
6504 		}
6505 		x = 0;
6506 	} else
6507 		x = gethrtime();
6508 }
6509 #endif
6510