xref: /illumos-gate/usr/src/uts/common/io/ib/mgt/ibdm/ibdm.c (revision e4ac07a33b07da3635a7a16ac8fccab07c33380a)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 /*
29  * ibdm.c
30  *
31  * This file contains the InifiniBand Device Manager (IBDM) support functions.
32  * IB nexus driver will only be the client for the IBDM module.
33  *
34  * IBDM registers with IBTF for HCA arrival/removal notification.
35  * IBDM registers with SA access to send DM MADs to discover the IOC's behind
36  * the IOU's.
37  *
38  * IB nexus driver registers with IBDM to find the information about the
39  * HCA's and IOC's (behind the IOU) present on the IB fabric.
40  */
41 
42 #include <sys/systm.h>
43 #include <sys/taskq.h>
44 #include <sys/ib/mgt/ibdm/ibdm_impl.h>
45 #include <sys/modctl.h>
46 
47 /* Function Prototype declarations */
48 static int	ibdm_free_iou_info(ibdm_dp_gidinfo_t *);
49 static int	ibdm_fini(void);
50 static int	ibdm_init(void);
51 static int	ibdm_get_reachable_ports(ibdm_port_attr_t *,
52 			ibdm_hca_list_t *);
53 static ibdm_dp_gidinfo_t *ibdm_check_dgid(ib_guid_t, ib_sn_prefix_t);
54 static ibdm_dp_gidinfo_t *ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *);
55 static int	ibdm_send_classportinfo(ibdm_dp_gidinfo_t *);
56 static int	ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *);
57 static int	ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *);
58 static int	ibdm_get_node_port_guids(ibmf_saa_handle_t, ib_lid_t,
59 		    ib_guid_t *, ib_guid_t *);
60 static int	ibdm_retry_command(ibdm_timeout_cb_args_t *);
61 static int	ibdm_get_diagcode(ibdm_dp_gidinfo_t *, int);
62 static int	ibdm_verify_mad_status(ib_mad_hdr_t *);
63 static int	ibdm_handle_redirection(ibmf_msg_t *,
64 		    ibdm_dp_gidinfo_t *, int *);
65 static void	ibdm_wait_probe_completion(void);
66 static void	ibdm_sweep_fabric(int);
67 static void	ibdm_probe_gid_thread(void *);
68 static void	ibdm_wakeup_probe_gid_cv(void);
69 static void	ibdm_port_attr_ibmf_init(ibdm_port_attr_t *, ib_pkey_t, int);
70 static int	ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *, int);
71 static void	ibdm_update_port_attr(ibdm_port_attr_t *);
72 static void	ibdm_handle_hca_attach(ib_guid_t);
73 static void	ibdm_handle_srventry_mad(ibmf_msg_t *,
74 		    ibdm_dp_gidinfo_t *, int *);
75 static void	ibdm_ibmf_recv_cb(ibmf_handle_t, ibmf_msg_t *, void *);
76 static void	ibdm_recv_incoming_mad(void *);
77 static void	ibdm_process_incoming_mad(ibmf_handle_t, ibmf_msg_t *, void *);
78 static void	ibdm_ibmf_send_cb(ibmf_handle_t, ibmf_msg_t *, void *);
79 static void	ibdm_pkt_timeout_hdlr(void *arg);
80 static void	ibdm_initialize_port(ibdm_port_attr_t *);
81 static void	ibdm_handle_diagcode(ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
82 static void	ibdm_probe_gid(ibdm_dp_gidinfo_t *);
83 static void	ibdm_alloc_send_buffers(ibmf_msg_t *);
84 static void	ibdm_free_send_buffers(ibmf_msg_t *);
85 static void	ibdm_handle_hca_detach(ib_guid_t);
86 static int	ibdm_fini_port(ibdm_port_attr_t *);
87 static int	ibdm_uninit_hca(ibdm_hca_list_t *);
88 static void	ibdm_handle_iounitinfo(ibmf_handle_t,
89 		    ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
90 static void	ibdm_handle_ioc_profile(ibmf_handle_t,
91 		    ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
92 static void	ibdm_event_hdlr(void *, ibt_hca_hdl_t,
93 		    ibt_async_code_t, ibt_async_event_t *);
94 static void	ibdm_handle_classportinfo(ibmf_handle_t,
95 		    ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
96 static void	ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *,
97 		    ibdm_dp_gidinfo_t *);
98 
99 static ibdm_hca_list_t		*ibdm_dup_hca_attr(ibdm_hca_list_t *);
100 static ibdm_ioc_info_t		*ibdm_dup_ioc_info(ibdm_ioc_info_t *,
101 				    ibdm_dp_gidinfo_t *gid_list);
102 static void			ibdm_probe_ioc(ib_guid_t, ib_guid_t, int);
103 static ibdm_ioc_info_t		*ibdm_is_ioc_present(ib_guid_t,
104 				    ibdm_dp_gidinfo_t *, int *);
105 static ibdm_port_attr_t		*ibdm_get_port_attr(ibt_async_event_t *,
106 				    ibdm_hca_list_t **);
107 static sa_node_record_t		*ibdm_get_node_records(ibmf_saa_handle_t,
108 				    size_t *, ib_guid_t);
109 static sa_portinfo_record_t	*ibdm_get_portinfo(ibmf_saa_handle_t, size_t *,
110 				    ib_lid_t);
111 static ibdm_dp_gidinfo_t	*ibdm_create_gid_info(ibdm_port_attr_t *,
112 				    ib_gid_t, ib_gid_t);
113 static ibdm_dp_gidinfo_t	*ibdm_find_gid(ib_guid_t, ib_guid_t);
114 static int	ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *, uint8_t);
115 static ibdm_ioc_info_t	*ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *, int);
116 static void	ibdm_saa_event_cb(ibmf_saa_handle_t, ibmf_saa_subnet_event_t,
117 		    ibmf_saa_event_details_t *, void *);
118 static void	ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *,
119     ibdm_dp_gidinfo_t *);
120 static ibdm_dp_gidinfo_t *ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *);
121 static void ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *,
122     ibdm_dp_gidinfo_t *);
123 static void ibdm_addto_gidlist(ibdm_gid_t **, ibdm_gid_t *);
124 static void ibdm_free_gid_list(ibdm_gid_t *);
125 static void ibdm_rescan_gidlist(ib_guid_t *ioc_guid);
126 static void ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *);
127 static void ibdm_saa_event_taskq(void *);
128 static void ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *);
129 static void ibdm_get_next_port(ibdm_hca_list_t **,
130     ibdm_port_attr_t **, int);
131 static void ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *,
132     ibdm_dp_gidinfo_t *);
133 static void ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *,
134     ibdm_hca_list_t *);
135 static void ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *);
136 static void ibdm_saa_handle_new_gid(void *);
137 static void ibdm_reset_all_dgids(ibmf_saa_handle_t);
138 static void ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *);
139 static void ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *);
140 static void ibdm_fill_srv_attr_mod(ib_mad_hdr_t *, ibdm_timeout_cb_args_t *);
141 static void ibdm_bump_transactionID(ibdm_dp_gidinfo_t *);
142 
143 int	ibdm_dft_timeout	= IBDM_DFT_TIMEOUT;
144 int	ibdm_dft_retry_cnt	= IBDM_DFT_NRETRIES;
145 #ifdef DEBUG
146 int	ibdm_ignore_saa_event = 0;
147 #endif
148 
149 /* Modload support */
150 static struct modlmisc ibdm_modlmisc	= {
151 	&mod_miscops,
152 	"InfiniBand Device Manager %I%",
153 };
154 
155 struct modlinkage ibdm_modlinkage = {
156 	MODREV_1,
157 	(void *)&ibdm_modlmisc,
158 	NULL
159 };
160 
161 static ibt_clnt_modinfo_t ibdm_ibt_modinfo = {
162 	IBTI_V2,
163 	IBT_DM,
164 	ibdm_event_hdlr,
165 	NULL,
166 	"ibdm"
167 };
168 
169 /* Global variables */
170 ibdm_t	ibdm;
171 int	ibdm_taskq_enable = IBDM_ENABLE_TASKQ_HANDLING;
172 char	*ibdm_string = "ibdm";
173 
174 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by cv",
175     ibdm.ibdm_dp_gidlist_head))
176 
177 /*
178  * _init
179  *	Loadable module init, called before any other module.
180  *	Initialize mutex
181  *	Register with IBTF
182  */
183 int
184 _init(void)
185 {
186 	int		err;
187 
188 	IBTF_DPRINTF_L4("ibdm", "\t_init: addr of ibdm %p", &ibdm);
189 
190 	if ((err = ibdm_init()) != IBDM_SUCCESS) {
191 		IBTF_DPRINTF_L2("ibdm", "_init: ibdm_init failed 0x%x", err);
192 		(void) ibdm_fini();
193 		return (DDI_FAILURE);
194 	}
195 
196 	if ((err = mod_install(&ibdm_modlinkage)) != 0) {
197 		IBTF_DPRINTF_L2("ibdm", "_init: mod_install failed 0x%x", err);
198 		(void) ibdm_fini();
199 	}
200 	return (err);
201 }
202 
203 
204 int
205 _fini(void)
206 {
207 	int err;
208 
209 	if ((err = ibdm_fini()) != IBDM_SUCCESS) {
210 		IBTF_DPRINTF_L2("ibdm", "_fini: ibdm_fini failed 0x%x", err);
211 		(void) ibdm_init();
212 		return (EBUSY);
213 	}
214 
215 	if ((err = mod_remove(&ibdm_modlinkage)) != 0) {
216 		IBTF_DPRINTF_L2("ibdm", "_fini: mod_remove failed 0x%x", err);
217 		(void) ibdm_init();
218 	}
219 	return (err);
220 }
221 
222 
223 int
224 _info(struct modinfo *modinfop)
225 {
226 	return (mod_info(&ibdm_modlinkage, modinfop));
227 }
228 
229 
230 /*
231  * ibdm_init():
232  * 	Register with IBTF
233  *	Allocate memory for the HCAs
234  *	Allocate minor-nodes for the HCAs
235  */
236 static int
237 ibdm_init(void)
238 {
239 	int			i, hca_count;
240 	ib_guid_t		*hca_guids;
241 	ibt_status_t		status;
242 
243 	IBTF_DPRINTF_L4("ibdm", "\tibdm_init:");
244 	if (!(ibdm.ibdm_state & IBDM_LOCKS_ALLOCED)) {
245 		mutex_init(&ibdm.ibdm_mutex, NULL, MUTEX_DEFAULT, NULL);
246 		mutex_init(&ibdm.ibdm_hl_mutex, NULL, MUTEX_DEFAULT, NULL);
247 		mutex_init(&ibdm.ibdm_ibnex_mutex, NULL, MUTEX_DEFAULT, NULL);
248 		mutex_enter(&ibdm.ibdm_mutex);
249 		ibdm.ibdm_state |= IBDM_LOCKS_ALLOCED;
250 	}
251 
252 	if (!(ibdm.ibdm_state & IBDM_IBT_ATTACHED)) {
253 		if ((status = ibt_attach(&ibdm_ibt_modinfo, NULL, NULL,
254 		    (void *)&ibdm.ibdm_ibt_clnt_hdl)) != IBT_SUCCESS) {
255 			IBTF_DPRINTF_L2("ibdm", "ibdm_init: ibt_attach "
256 			    "failed %x", status);
257 			mutex_exit(&ibdm.ibdm_mutex);
258 			return (IBDM_FAILURE);
259 		}
260 
261 		ibdm.ibdm_state |= IBDM_IBT_ATTACHED;
262 		mutex_exit(&ibdm.ibdm_mutex);
263 	}
264 
265 
266 	if (!(ibdm.ibdm_state & IBDM_HCA_ATTACHED)) {
267 		hca_count = ibt_get_hca_list(&hca_guids);
268 		IBTF_DPRINTF_L4("ibdm", "ibdm_init: num_hcas = %d", hca_count);
269 		for (i = 0; i < hca_count; i++)
270 			(void) ibdm_handle_hca_attach(hca_guids[i]);
271 		if (hca_count)
272 			ibt_free_hca_list(hca_guids, hca_count);
273 
274 		mutex_enter(&ibdm.ibdm_mutex);
275 		ibdm.ibdm_state |= IBDM_HCA_ATTACHED;
276 		mutex_exit(&ibdm.ibdm_mutex);
277 	}
278 
279 	if (!(ibdm.ibdm_state & IBDM_CVS_ALLOCED)) {
280 		cv_init(&ibdm.ibdm_probe_cv, NULL, CV_DRIVER, NULL);
281 		cv_init(&ibdm.ibdm_busy_cv, NULL, CV_DRIVER, NULL);
282 		mutex_enter(&ibdm.ibdm_mutex);
283 		ibdm.ibdm_state |= IBDM_CVS_ALLOCED;
284 		mutex_exit(&ibdm.ibdm_mutex);
285 	}
286 	return (IBDM_SUCCESS);
287 }
288 
289 
290 static int
291 ibdm_free_iou_info(ibdm_dp_gidinfo_t *gid_info)
292 {
293 	int			ii, k, niocs;
294 	size_t			size;
295 	ibdm_gid_t		*delete, *head;
296 	timeout_id_t		timeout_id;
297 	ibdm_ioc_info_t		*ioc;
298 
299 	ASSERT(mutex_owned(&gid_info->gl_mutex));
300 	if (gid_info->gl_iou == NULL) {
301 		IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: No IOU");
302 		return (0);
303 	}
304 
305 	niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
306 	IBTF_DPRINTF_L4("ibdm", "\tfree_iou_info: gid_info = %p, niocs %d",
307 	    gid_info, niocs);
308 
309 	for (ii = 0; ii < niocs; ii++) {
310 		ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii);
311 
312 		/* handle the case where an ioc_timeout_id is scheduled */
313 		if (ioc->ioc_timeout_id) {
314 			timeout_id = ioc->ioc_timeout_id;
315 			mutex_exit(&gid_info->gl_mutex);
316 			IBTF_DPRINTF_L5("ibdm", "free_iou_info: "
317 			    "ioc_timeout_id = 0x%x", timeout_id);
318 			if (untimeout(timeout_id) == -1) {
319 				IBTF_DPRINTF_L2("ibdm", "free_iou_info: "
320 				    "untimeout ioc_timeout_id failed");
321 				mutex_enter(&gid_info->gl_mutex);
322 				return (-1);
323 			}
324 			mutex_enter(&gid_info->gl_mutex);
325 			ioc->ioc_timeout_id = 0;
326 		}
327 
328 		/* handle the case where an ioc_dc_timeout_id is scheduled */
329 		if (ioc->ioc_dc_timeout_id) {
330 			timeout_id = ioc->ioc_dc_timeout_id;
331 			mutex_exit(&gid_info->gl_mutex);
332 			IBTF_DPRINTF_L5("ibdm", "free_iou_info: "
333 			    "ioc_dc_timeout_id = 0x%x", timeout_id);
334 			if (untimeout(timeout_id) == -1) {
335 				IBTF_DPRINTF_L2("ibdm", "free_iou_info: "
336 				    "untimeout ioc_dc_timeout_id failed");
337 				mutex_enter(&gid_info->gl_mutex);
338 				return (-1);
339 			}
340 			mutex_enter(&gid_info->gl_mutex);
341 			ioc->ioc_dc_timeout_id = 0;
342 		}
343 
344 		/* handle the case where serv[k].se_timeout_id is scheduled */
345 		for (k = 0; k < ioc->ioc_profile.ioc_service_entries; k++) {
346 			if (ioc->ioc_serv[k].se_timeout_id) {
347 				timeout_id = ioc->ioc_serv[k].se_timeout_id;
348 				mutex_exit(&gid_info->gl_mutex);
349 				IBTF_DPRINTF_L5("ibdm", "free_iou_info: "
350 				    "ioc->ioc_serv[%d].se_timeout_id = 0x%x",
351 				    k, timeout_id);
352 				if (untimeout(timeout_id) == -1) {
353 					IBTF_DPRINTF_L2("ibdm", "free_iou_info:"
354 					    " untimeout se_timeout_id failed");
355 					mutex_enter(&gid_info->gl_mutex);
356 					return (-1);
357 				}
358 				mutex_enter(&gid_info->gl_mutex);
359 				ioc->ioc_serv[k].se_timeout_id = 0;
360 			}
361 		}
362 
363 		/* delete GID list */
364 		head = ioc->ioc_gid_list;
365 		while (head) {
366 			IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: "
367 			    "Deleting gid_list struct %p", head);
368 			delete = head;
369 			head = head->gid_next;
370 			kmem_free(delete, sizeof (ibdm_gid_t));
371 		}
372 		ioc->ioc_gid_list = NULL;
373 
374 		/* delete ioc_serv */
375 		size = ioc->ioc_profile.ioc_service_entries *
376 		    sizeof (ibdm_srvents_info_t);
377 		if (ioc->ioc_serv && size) {
378 			kmem_free(ioc->ioc_serv, size);
379 			ioc->ioc_serv = NULL;
380 		}
381 	}
382 
383 	IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: deleting IOU & IOC");
384 	size = sizeof (ibdm_iou_info_t) + niocs * sizeof (ibdm_ioc_info_t);
385 	kmem_free(gid_info->gl_iou, size);
386 	gid_info->gl_iou = NULL;
387 	return (0);
388 }
389 
390 
391 /*
392  * ibdm_fini():
393  * 	Un-register with IBTF
394  *	De allocate memory for the GID info
395  */
396 static int
397 ibdm_fini()
398 {
399 	int			ii;
400 	ibdm_hca_list_t		*hca_list, *temp;
401 	ibdm_dp_gidinfo_t	*gid_info, *tmp;
402 	ibdm_gid_t		*head, *delete;
403 
404 	IBTF_DPRINTF_L4("ibdm", "\tibdm_fini");
405 
406 	mutex_enter(&ibdm.ibdm_hl_mutex);
407 	if (ibdm.ibdm_state & IBDM_IBT_ATTACHED) {
408 		if (ibt_detach(ibdm.ibdm_ibt_clnt_hdl) != IBT_SUCCESS) {
409 			IBTF_DPRINTF_L2("ibdm", "\t_fini: ibt_detach failed");
410 			mutex_exit(&ibdm.ibdm_hl_mutex);
411 			return (IBDM_FAILURE);
412 		}
413 		ibdm.ibdm_state &= ~IBDM_IBT_ATTACHED;
414 		ibdm.ibdm_ibt_clnt_hdl = NULL;
415 	}
416 
417 	hca_list = ibdm.ibdm_hca_list_head;
418 	IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: nhcas %d", ibdm.ibdm_hca_count);
419 	for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
420 		temp = hca_list;
421 		hca_list = hca_list->hl_next;
422 		IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: hca %p", temp);
423 		if (ibdm_uninit_hca(temp) != IBDM_SUCCESS) {
424 			IBTF_DPRINTF_L2("ibdm", "\tibdm_fini: "
425 			    "uninit_hca %p failed", temp);
426 			mutex_exit(&ibdm.ibdm_hl_mutex);
427 			return (IBDM_FAILURE);
428 		}
429 	}
430 	mutex_exit(&ibdm.ibdm_hl_mutex);
431 
432 	mutex_enter(&ibdm.ibdm_mutex);
433 	if (ibdm.ibdm_state & IBDM_HCA_ATTACHED)
434 		ibdm.ibdm_state &= ~IBDM_HCA_ATTACHED;
435 
436 	gid_info = ibdm.ibdm_dp_gidlist_head;
437 	while (gid_info) {
438 		mutex_enter(&gid_info->gl_mutex);
439 		(void) ibdm_free_iou_info(gid_info);
440 		mutex_exit(&gid_info->gl_mutex);
441 		ibdm_delete_glhca_list(gid_info);
442 
443 		tmp = gid_info;
444 		gid_info = gid_info->gl_next;
445 		mutex_destroy(&tmp->gl_mutex);
446 		head = tmp->gl_gid;
447 		while (head) {
448 			IBTF_DPRINTF_L4("ibdm",
449 			    "\tibdm_fini: Deleting gid structs");
450 			delete = head;
451 			head = head->gid_next;
452 			kmem_free(delete, sizeof (ibdm_gid_t));
453 		}
454 		kmem_free(tmp, sizeof (ibdm_dp_gidinfo_t));
455 	}
456 	mutex_exit(&ibdm.ibdm_mutex);
457 
458 	if (ibdm.ibdm_state & IBDM_LOCKS_ALLOCED) {
459 		ibdm.ibdm_state &= ~IBDM_LOCKS_ALLOCED;
460 		mutex_destroy(&ibdm.ibdm_mutex);
461 		mutex_destroy(&ibdm.ibdm_hl_mutex);
462 		mutex_destroy(&ibdm.ibdm_ibnex_mutex);
463 	}
464 	if (ibdm.ibdm_state & IBDM_CVS_ALLOCED) {
465 		ibdm.ibdm_state &= ~IBDM_CVS_ALLOCED;
466 		cv_destroy(&ibdm.ibdm_probe_cv);
467 		cv_destroy(&ibdm.ibdm_busy_cv);
468 	}
469 	return (IBDM_SUCCESS);
470 }
471 
472 
473 /*
474  * ibdm_event_hdlr()
475  *
476  *	IBDM registers  this asynchronous event handler at the time of
477  *	ibt_attach. IBDM support the following async events. For other
478  *	event, simply returns success.
479  *	IBT_HCA_ATTACH_EVENT:
480  *		Retrieves the  information about all the port that are
481  *		present on this HCA,  allocates  the  port  attributes
482  *		structure  and calls IB  nexus  callback  routine with
483  *		the port attributes structure as an input argument.
484  *	IBT_HCA_DETACH_EVENT:
485  *		Retrieves the information about all the ports that are
486  *		present on  this HCA and  calls IB nexus callback with
487  *		port guid as an argument
488  *	IBT_EVENT_PORT_UP:
489  *		Register with IBMF and SA access
490  *		Setup IBMF receive callback routine
491  *	IBT_EVENT_PORT_DOWN:
492  *		Un-Register with IBMF and SA access
493  *		Teardown IBMF receive callback routine
494  */
495 /*ARGSUSED*/
496 static void
497 ibdm_event_hdlr(void *clnt_hdl,
498     ibt_hca_hdl_t hca_hdl, ibt_async_code_t code, ibt_async_event_t *event)
499 {
500 	ibdm_hca_list_t		*hca_list;
501 	ibdm_port_attr_t	*port;
502 	ibmf_saa_handle_t	port_sa_hdl;
503 
504 	IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: async code 0x%x", code);
505 
506 	switch (code) {
507 	case IBT_HCA_ATTACH_EVENT:	/* New HCA registered with IBTF */
508 		ibdm_handle_hca_attach(event->ev_hca_guid);
509 		break;
510 
511 	case IBT_HCA_DETACH_EVENT:	/* HCA unregistered with IBTF */
512 		ibdm_handle_hca_detach(event->ev_hca_guid);
513 		mutex_enter(&ibdm.ibdm_ibnex_mutex);
514 		if (ibdm.ibdm_ibnex_callback != NULL) {
515 			(*ibdm.ibdm_ibnex_callback)((void *)
516 			    &event->ev_hca_guid, IBDM_EVENT_HCA_REMOVED);
517 		}
518 		mutex_exit(&ibdm.ibdm_ibnex_mutex);
519 		break;
520 
521 	case IBT_EVENT_PORT_UP:
522 		IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_UP");
523 		mutex_enter(&ibdm.ibdm_hl_mutex);
524 		port = ibdm_get_port_attr(event, &hca_list);
525 		if (port == NULL) {
526 			IBTF_DPRINTF_L2("ibdm",
527 				"\tevent_hdlr: HCA not present");
528 			mutex_exit(&ibdm.ibdm_hl_mutex);
529 			break;
530 		}
531 		ibdm_initialize_port(port);
532 		hca_list->hl_nports_active++;
533 		mutex_exit(&ibdm.ibdm_hl_mutex);
534 		break;
535 
536 	case IBT_ERROR_PORT_DOWN:
537 		IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_DOWN");
538 		mutex_enter(&ibdm.ibdm_hl_mutex);
539 		port = ibdm_get_port_attr(event, &hca_list);
540 		if (port == NULL) {
541 			IBTF_DPRINTF_L2("ibdm",
542 				"\tevent_hdlr: HCA not present");
543 			mutex_exit(&ibdm.ibdm_hl_mutex);
544 			break;
545 		}
546 		hca_list->hl_nports_active--;
547 		port_sa_hdl = port->pa_sa_hdl;
548 		(void) ibdm_fini_port(port);
549 		mutex_exit(&ibdm.ibdm_hl_mutex);
550 		ibdm_reset_all_dgids(port_sa_hdl);
551 		break;
552 
553 	default:		/* Ignore all other events/errors */
554 		break;
555 	}
556 }
557 
558 
559 /*
560  * ibdm_initialize_port()
561  *	Register with IBMF
562  *	Register with SA access
563  *	Register a receive callback routine with IBMF. IBMF invokes
564  *	this routine whenever a MAD arrives at this port.
565  *	Update the port attributes
566  */
567 static void
568 ibdm_initialize_port(ibdm_port_attr_t *port)
569 {
570 	int				ii;
571 	uint_t				nports, size;
572 	uint_t				pkey_idx;
573 	ib_pkey_t			pkey;
574 	ibt_hca_portinfo_t		*pinfop;
575 	ibmf_register_info_t		ibmf_reg;
576 	ibmf_saa_subnet_event_args_t	event_args;
577 
578 	IBTF_DPRINTF_L4("ibdm", "\tinitialize_port:");
579 	ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
580 
581 	/* Check whether the port is active */
582 	if (ibt_get_port_state(port->pa_hca_hdl, port->pa_port_num, NULL,
583 	    NULL) != IBT_SUCCESS)
584 		return;
585 
586 	if (port->pa_sa_hdl != NULL)
587 		return;
588 
589 	if (ibt_query_hca_ports(port->pa_hca_hdl, port->pa_port_num,
590 	    &pinfop, &nports, &size) != IBT_SUCCESS) {
591 		/* This should not occur */
592 		port->pa_npkeys		= 0;
593 		port->pa_pkey_tbl	= NULL;
594 		return;
595 	}
596 	port->pa_sn_prefix = pinfop->p_sgid_tbl[0].gid_prefix;
597 
598 	port->pa_state		= pinfop->p_linkstate;
599 	port->pa_npkeys		= pinfop->p_pkey_tbl_sz;
600 	port->pa_pkey_tbl	= (ibdm_pkey_tbl_t *)kmem_zalloc(
601 	    port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP);
602 
603 	for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++)
604 		port->pa_pkey_tbl[pkey_idx].pt_pkey =
605 		    pinfop->p_pkey_tbl[pkey_idx];
606 
607 	ibt_free_portinfo(pinfop, size);
608 
609 	event_args.is_event_callback = ibdm_saa_event_cb;
610 	event_args.is_event_callback_arg = port;
611 	if (ibmf_sa_session_open(port->pa_port_guid, 0, &event_args,
612 	    IBMF_VERSION, 0, &port->pa_sa_hdl) != IBMF_SUCCESS) {
613 		IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: "
614 		    "sa access registration failed");
615 		return;
616 	}
617 	ibmf_reg.ir_ci_guid		= port->pa_hca_guid;
618 	ibmf_reg.ir_port_num		= port->pa_port_num;
619 	ibmf_reg.ir_client_class	= DEV_MGT_MANAGER;
620 
621 	if (ibmf_register(&ibmf_reg, IBMF_VERSION, 0, NULL, NULL,
622 	    &port->pa_ibmf_hdl, &port->pa_ibmf_caps) != IBMF_SUCCESS) {
623 		IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: "
624 		    "IBMF registration failed");
625 		(void) ibdm_fini_port(port);
626 		return;
627 	}
628 	if (ibmf_setup_async_cb(port->pa_ibmf_hdl, IBMF_QP_HANDLE_DEFAULT,
629 	    ibdm_ibmf_recv_cb, 0, 0) != IBMF_SUCCESS) {
630 		IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: "
631 		    "IBMF setup recv cb failed");
632 		(void) ibdm_fini_port(port);
633 		return;
634 	}
635 
636 	for (ii = 0; ii < port->pa_npkeys; ii++) {
637 		pkey = port->pa_pkey_tbl[ii].pt_pkey;
638 		if (IBDM_INVALID_PKEY(pkey)) {
639 			port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
640 			continue;
641 		}
642 		ibdm_port_attr_ibmf_init(port, pkey, ii);
643 	}
644 }
645 
646 
647 /*
648  * ibdm_port_attr_ibmf_init:
649  *	With IBMF - Alloc QP Handle and Setup Async callback
650  */
651 static void
652 ibdm_port_attr_ibmf_init(ibdm_port_attr_t *port, ib_pkey_t pkey, int ii)
653 {
654 	int ret;
655 
656 	if ((ret = ibmf_alloc_qp(port->pa_ibmf_hdl, pkey, IB_GSI_QKEY,
657 	    IBMF_ALT_QP_MAD_NO_RMPP, &port->pa_pkey_tbl[ii].pt_qp_hdl)) !=
658 	    IBMF_SUCCESS) {
659 		IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: "
660 		    "IBMF failed to alloc qp %d", ret);
661 		port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
662 		return;
663 	}
664 
665 	IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_init: QP handle is %p",
666 	    port->pa_ibmf_hdl);
667 
668 	if ((ret = ibmf_setup_async_cb(port->pa_ibmf_hdl,
669 	    port->pa_pkey_tbl[ii].pt_qp_hdl, ibdm_ibmf_recv_cb, 0, 0)) !=
670 	    IBMF_SUCCESS) {
671 		IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: "
672 		    "IBMF setup recv cb failed %d", ret);
673 		(void) ibmf_free_qp(port->pa_ibmf_hdl,
674 		    &port->pa_pkey_tbl[ii].pt_qp_hdl, 0);
675 		port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
676 	}
677 }
678 
679 
680 /*
681  * ibdm_get_port_attr()
682  *	Get port attributes from HCA guid and port number
683  *	Return pointer to ibdm_port_attr_t on Success
684  *	and NULL on failure
685  */
686 static ibdm_port_attr_t *
687 ibdm_get_port_attr(ibt_async_event_t *event, ibdm_hca_list_t **retval)
688 {
689 	ibdm_hca_list_t		*hca_list;
690 	ibdm_port_attr_t	*port_attr;
691 	int			ii;
692 
693 	IBTF_DPRINTF_L4("ibdm", "\tget_port_attr: port# %d", event->ev_port);
694 	ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
695 	hca_list = ibdm.ibdm_hca_list_head;
696 	while (hca_list) {
697 		if (hca_list->hl_hca_guid == event->ev_hca_guid) {
698 			for (ii = 0; ii < hca_list->hl_nports; ii++) {
699 				port_attr = &hca_list->hl_port_attr[ii];
700 				if (port_attr->pa_port_num == event->ev_port) {
701 					*retval = hca_list;
702 					return (port_attr);
703 				}
704 			}
705 		}
706 		hca_list = hca_list->hl_next;
707 	}
708 	return (NULL);
709 }
710 
711 
712 /*
713  * ibdm_update_port_attr()
714  *	Update the port attributes
715  */
716 static void
717 ibdm_update_port_attr(ibdm_port_attr_t *port)
718 {
719 	uint_t			nports, size;
720 	uint_t			pkey_idx;
721 	ibt_hca_portinfo_t	*portinfop;
722 
723 	IBTF_DPRINTF_L4("ibdm", "\tupdate_port_attr: Begin");
724 	if (ibt_query_hca_ports(port->pa_hca_hdl,
725 	    port->pa_port_num, &portinfop, &nports, &size) != IBT_SUCCESS) {
726 		/* This should not occur */
727 		port->pa_npkeys		= 0;
728 		port->pa_pkey_tbl	= NULL;
729 		return;
730 	}
731 	port->pa_sn_prefix = portinfop->p_sgid_tbl[0].gid_prefix;
732 
733 	port->pa_state		= portinfop->p_linkstate;
734 
735 	/*
736 	 * PKey information in portinfo valid only if port is
737 	 * ACTIVE. Bail out if not.
738 	 */
739 	if (port->pa_state != IBT_PORT_ACTIVE) {
740 		port->pa_npkeys		= 0;
741 		port->pa_pkey_tbl	= NULL;
742 		ibt_free_portinfo(portinfop, size);
743 		return;
744 	}
745 
746 	port->pa_npkeys		= portinfop->p_pkey_tbl_sz;
747 	port->pa_pkey_tbl	= (ibdm_pkey_tbl_t *)kmem_zalloc(
748 	    port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP);
749 
750 	for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++) {
751 		port->pa_pkey_tbl[pkey_idx].pt_pkey =
752 		    portinfop->p_pkey_tbl[pkey_idx];
753 	}
754 	ibt_free_portinfo(portinfop, size);
755 }
756 
757 
758 /*
759  * ibdm_handle_hca_attach()
760  */
761 static void
762 ibdm_handle_hca_attach(ib_guid_t hca_guid)
763 {
764 	uint_t			size;
765 	uint_t			ii, nports;
766 	ibt_status_t		status;
767 	ibt_hca_hdl_t		hca_hdl;
768 	ibt_hca_attr_t		*hca_attr;
769 	ibdm_hca_list_t		*hca_list, *temp;
770 	ibdm_port_attr_t	*port_attr;
771 	ibt_hca_portinfo_t	*portinfop;
772 
773 	IBTF_DPRINTF_L4("ibdm",
774 	    "\thandle_hca_attach: hca_guid = 0x%llX", hca_guid);
775 
776 	/* open the HCA first */
777 	if ((status = ibt_open_hca(ibdm.ibdm_ibt_clnt_hdl, hca_guid,
778 	    &hca_hdl)) != IBT_SUCCESS) {
779 		IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: "
780 		    "open_hca failed, status 0x%x", status);
781 		return;
782 	}
783 
784 	hca_attr = (ibt_hca_attr_t *)
785 	    kmem_alloc(sizeof (ibt_hca_attr_t), KM_SLEEP);
786 	/* ibt_query_hca always returns IBT_SUCCESS */
787 	(void) ibt_query_hca(hca_hdl, hca_attr);
788 
789 	IBTF_DPRINTF_L4("ibdm", "\tvid: 0x%x, pid: 0x%x, ver: 0x%x,"
790 	    " #ports: %d", hca_attr->hca_vendor_id, hca_attr->hca_device_id,
791 	    hca_attr->hca_version_id, hca_attr->hca_nports);
792 
793 	if ((status = ibt_query_hca_ports(hca_hdl, 0, &portinfop, &nports,
794 	    &size)) != IBT_SUCCESS) {
795 		IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: "
796 		    "ibt_query_hca_ports failed, status 0x%x", status);
797 		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
798 		(void) ibt_close_hca(hca_hdl);
799 		return;
800 	}
801 	hca_list = (ibdm_hca_list_t *)
802 	    kmem_zalloc((sizeof (ibdm_hca_list_t)), KM_SLEEP);
803 	hca_list->hl_port_attr = (ibdm_port_attr_t *)kmem_zalloc(
804 	    (sizeof (ibdm_port_attr_t) * hca_attr->hca_nports), KM_SLEEP);
805 	hca_list->hl_hca_guid = hca_attr->hca_node_guid;
806 	hca_list->hl_nports = hca_attr->hca_nports;
807 	hca_list->hl_attach_time = ddi_get_time();
808 	hca_list->hl_hca_hdl = hca_hdl;
809 
810 	/*
811 	 * Init a dummy port attribute for the HCA node
812 	 * This is for Per-HCA Node. Initialize port_attr :
813 	 * 	hca_guid & port_guid -> hca_guid
814 	 *	npkeys, pkey_tbl is NULL
815 	 *	port_num, sn_prefix is 0
816 	 *	vendorid, product_id, dev_version from HCA
817 	 *	pa_state is IBT_PORT_ACTIVE
818 	 */
819 	hca_list->hl_hca_port_attr = (ibdm_port_attr_t *)kmem_zalloc(
820 	    sizeof (ibdm_port_attr_t), KM_SLEEP);
821 	port_attr = hca_list->hl_hca_port_attr;
822 	port_attr->pa_vendorid  = hca_attr->hca_vendor_id;
823 	port_attr->pa_productid	= hca_attr->hca_device_id;
824 	port_attr->pa_dev_version = hca_attr->hca_version_id;
825 	port_attr->pa_hca_guid	= hca_attr->hca_node_guid;
826 	port_attr->pa_hca_hdl	= hca_list->hl_hca_hdl;
827 	port_attr->pa_port_guid	= hca_attr->hca_node_guid;
828 	port_attr->pa_state	= IBT_PORT_ACTIVE;
829 
830 
831 	for (ii = 0; ii < nports; ii++) {
832 		port_attr		= &hca_list->hl_port_attr[ii];
833 		port_attr->pa_vendorid	= hca_attr->hca_vendor_id;
834 		port_attr->pa_productid	= hca_attr->hca_device_id;
835 		port_attr->pa_dev_version = hca_attr->hca_version_id;
836 		port_attr->pa_hca_guid	= hca_attr->hca_node_guid;
837 		port_attr->pa_hca_hdl	= hca_list->hl_hca_hdl;
838 		port_attr->pa_port_guid	= portinfop[ii].p_sgid_tbl->gid_guid;
839 		port_attr->pa_sn_prefix	= portinfop[ii].p_sgid_tbl->gid_prefix;
840 		port_attr->pa_port_num	= portinfop[ii].p_port_num;
841 		port_attr->pa_state	= portinfop[ii].p_linkstate;
842 
843 		/*
844 		 * Register with IBMF, SA access when the port is in
845 		 * ACTIVE state. Also register a callback routine
846 		 * with IBMF to receive incoming DM MAD's.
847 		 * The IBDM event handler takes care of registration of
848 		 * port which are not active.
849 		 */
850 		IBTF_DPRINTF_L4("ibdm",
851 		    "\thandle_hca_attach: port guid %llx Port state 0x%x",
852 		    port_attr->pa_port_guid, portinfop[ii].p_linkstate);
853 
854 		if (portinfop[ii].p_linkstate == IBT_PORT_ACTIVE) {
855 			mutex_enter(&ibdm.ibdm_hl_mutex);
856 			hca_list->hl_nports_active++;
857 			ibdm_initialize_port(port_attr);
858 			mutex_exit(&ibdm.ibdm_hl_mutex);
859 		}
860 	}
861 	mutex_enter(&ibdm.ibdm_hl_mutex);
862 	for (temp = ibdm.ibdm_hca_list_head; temp; temp = temp->hl_next) {
863 		if (temp->hl_hca_guid == hca_guid) {
864 			IBTF_DPRINTF_L2("ibdm", "hca_attach: HCA %llX "
865 			    "already seen by IBDM", hca_guid);
866 			mutex_exit(&ibdm.ibdm_hl_mutex);
867 			(void) ibdm_uninit_hca(hca_list);
868 			return;
869 		}
870 	}
871 	ibdm.ibdm_hca_count++;
872 	if (ibdm.ibdm_hca_list_head == NULL) {
873 		ibdm.ibdm_hca_list_head = hca_list;
874 		ibdm.ibdm_hca_list_tail = hca_list;
875 	} else {
876 		ibdm.ibdm_hca_list_tail->hl_next = hca_list;
877 		ibdm.ibdm_hca_list_tail = hca_list;
878 	}
879 	mutex_exit(&ibdm.ibdm_hl_mutex);
880 	mutex_enter(&ibdm.ibdm_ibnex_mutex);
881 	if (ibdm.ibdm_ibnex_callback != NULL) {
882 		(*ibdm.ibdm_ibnex_callback)((void *)
883 		    &hca_guid, IBDM_EVENT_HCA_ADDED);
884 	}
885 	mutex_exit(&ibdm.ibdm_ibnex_mutex);
886 
887 	kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
888 	ibt_free_portinfo(portinfop, size);
889 }
890 
891 
892 /*
893  * ibdm_handle_hca_detach()
894  */
895 static void
896 ibdm_handle_hca_detach(ib_guid_t hca_guid)
897 {
898 	ibdm_hca_list_t		*head, *prev = NULL;
899 	ibdm_dp_gidinfo_t	*gidinfo;
900 
901 	IBTF_DPRINTF_L4("ibdm",
902 	    "\thandle_hca_detach: hca_guid = 0x%llx", hca_guid);
903 
904 	/* Make sure no probes are running */
905 	mutex_enter(&ibdm.ibdm_mutex);
906 	while (ibdm.ibdm_busy & IBDM_BUSY)
907 		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
908 	ibdm.ibdm_busy |= IBDM_BUSY;
909 	mutex_exit(&ibdm.ibdm_mutex);
910 
911 	mutex_enter(&ibdm.ibdm_hl_mutex);
912 	head = ibdm.ibdm_hca_list_head;
913 	while (head) {
914 		if (head->hl_hca_guid == hca_guid) {
915 			if (prev == NULL)
916 				ibdm.ibdm_hca_list_head = head->hl_next;
917 			else
918 				prev->hl_next = head->hl_next;
919 			ibdm.ibdm_hca_count--;
920 			break;
921 		}
922 		prev = head;
923 		head = head->hl_next;
924 	}
925 	mutex_exit(&ibdm.ibdm_hl_mutex);
926 	if (ibdm_uninit_hca(head) != IBDM_SUCCESS)
927 		(void) ibdm_handle_hca_attach(hca_guid);
928 
929 	/*
930 	 * Now clean up the HCA lists in the gidlist.
931 	 */
932 	for (gidinfo = ibdm.ibdm_dp_gidlist_head; gidinfo; gidinfo =
933 	    gidinfo->gl_next) {
934 		prev = NULL;
935 		head = gidinfo->gl_hca_list;
936 		while (head) {
937 			if (head->hl_hca_guid == hca_guid) {
938 				if (prev == NULL)
939 					gidinfo->gl_hca_list =
940 					    head->hl_next;
941 				else
942 					prev->hl_next = head->hl_next;
943 
944 				break;
945 			}
946 			prev = head;
947 			head = head->hl_next;
948 		}
949 	}
950 
951 	mutex_enter(&ibdm.ibdm_mutex);
952 	ibdm.ibdm_busy &= ~IBDM_BUSY;
953 	cv_broadcast(&ibdm.ibdm_busy_cv);
954 	mutex_exit(&ibdm.ibdm_mutex);
955 }
956 
957 
958 static int
959 ibdm_uninit_hca(ibdm_hca_list_t *head)
960 {
961 	int			ii;
962 	ibdm_port_attr_t	*port_attr;
963 
964 	for (ii = 0; ii < head->hl_nports; ii++) {
965 		port_attr = &head->hl_port_attr[ii];
966 		if (ibdm_fini_port(port_attr) != IBDM_SUCCESS) {
967 			IBTF_DPRINTF_L2("ibdm", "uninit_hca: HCA %p port 0x%x "
968 			    "ibdm_fini_port() failed", head, ii);
969 			return (IBDM_FAILURE);
970 		}
971 	}
972 	if (head->hl_hca_hdl)
973 		if (ibt_close_hca(head->hl_hca_hdl) != IBT_SUCCESS)
974 			return (IBDM_FAILURE);
975 	kmem_free(head->hl_port_attr,
976 	    head->hl_nports * sizeof (ibdm_port_attr_t));
977 	kmem_free(head->hl_hca_port_attr, sizeof (ibdm_port_attr_t));
978 	kmem_free(head, sizeof (ibdm_hca_list_t));
979 	return (IBDM_SUCCESS);
980 }
981 
982 
983 /*
984  * For each port on the HCA,
985  *	1) Teardown IBMF receive callback function
986  *	2) Unregister with IBMF
987  *	3) Unregister with SA access
988  */
989 static int
990 ibdm_fini_port(ibdm_port_attr_t *port_attr)
991 {
992 	int	ii, ibmf_status;
993 
994 	for (ii = 0; ii < port_attr->pa_npkeys; ii++) {
995 		if (port_attr->pa_pkey_tbl == NULL)
996 			break;
997 		if (!port_attr->pa_pkey_tbl[ii].pt_qp_hdl)
998 			continue;
999 		if (ibdm_port_attr_ibmf_fini(port_attr, ii) != IBDM_SUCCESS) {
1000 			IBTF_DPRINTF_L4("ibdm", "\tfini_port: "
1001 			    "ibdm_port_attr_ibmf_fini failed for "
1002 			    "port pkey 0x%x", ii);
1003 			return (IBDM_FAILURE);
1004 		}
1005 	}
1006 
1007 	if (port_attr->pa_ibmf_hdl) {
1008 		ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl,
1009 		    IBMF_QP_HANDLE_DEFAULT, 0);
1010 		if (ibmf_status != IBMF_SUCCESS) {
1011 			IBTF_DPRINTF_L4("ibdm", "\tfini_port: "
1012 			    "ibmf_tear_down_async_cb failed %d", ibmf_status);
1013 			return (IBDM_FAILURE);
1014 		}
1015 
1016 		ibmf_status = ibmf_unregister(&port_attr->pa_ibmf_hdl, 0);
1017 		if (ibmf_status != IBMF_SUCCESS) {
1018 			IBTF_DPRINTF_L4("ibdm", "\tfini_port: "
1019 			    "ibmf_unregister failed %d", ibmf_status);
1020 			return (IBDM_FAILURE);
1021 		}
1022 
1023 		port_attr->pa_ibmf_hdl = NULL;
1024 	}
1025 
1026 	if (port_attr->pa_sa_hdl) {
1027 		ibmf_status = ibmf_sa_session_close(&port_attr->pa_sa_hdl, 0);
1028 		if (ibmf_status != IBMF_SUCCESS) {
1029 			IBTF_DPRINTF_L4("ibdm", "\tfini_port: "
1030 			    "ibmf_sa_session_close failed %d", ibmf_status);
1031 			return (IBDM_FAILURE);
1032 		}
1033 		port_attr->pa_sa_hdl = NULL;
1034 	}
1035 
1036 	if (port_attr->pa_pkey_tbl != NULL) {
1037 		kmem_free(port_attr->pa_pkey_tbl,
1038 		    port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t));
1039 		port_attr->pa_pkey_tbl = NULL;
1040 		port_attr->pa_npkeys = 0;
1041 	}
1042 
1043 	return (IBDM_SUCCESS);
1044 }
1045 
1046 
1047 /*
1048  * ibdm_port_attr_ibmf_fini:
1049  *	With IBMF - Tear down Async callback and free QP Handle
1050  */
1051 static int
1052 ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *port_attr, int ii)
1053 {
1054 	int ibmf_status;
1055 
1056 	IBTF_DPRINTF_L5("ibdm", "\tport_attr_ibmf_fini:");
1057 
1058 	if (port_attr->pa_pkey_tbl[ii].pt_qp_hdl) {
1059 		ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl,
1060 		    port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0);
1061 		if (ibmf_status != IBMF_SUCCESS) {
1062 			IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: "
1063 			    "ibmf_tear_down_async_cb failed %d", ibmf_status);
1064 			return (IBDM_FAILURE);
1065 		}
1066 		ibmf_status = ibmf_free_qp(port_attr->pa_ibmf_hdl,
1067 		    &port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0);
1068 		if (ibmf_status != IBMF_SUCCESS) {
1069 			IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: "
1070 			    "ibmf_free_qp failed %d", ibmf_status);
1071 			return (IBDM_FAILURE);
1072 		}
1073 		port_attr->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
1074 	}
1075 	return (IBDM_SUCCESS);
1076 }
1077 
1078 
1079 /*
1080  * ibdm_gid_decr_pending:
1081  *	decrement gl_pending_cmds. If zero wakeup sleeping threads
1082  */
1083 static void
1084 ibdm_gid_decr_pending(ibdm_dp_gidinfo_t *gidinfo)
1085 {
1086 	mutex_enter(&ibdm.ibdm_mutex);
1087 	mutex_enter(&gidinfo->gl_mutex);
1088 	if (--gidinfo->gl_pending_cmds == 0) {
1089 		/*
1090 		 * Handle DGID getting removed.
1091 		 */
1092 		if (gidinfo->gl_disconnected) {
1093 			mutex_exit(&gidinfo->gl_mutex);
1094 			mutex_exit(&ibdm.ibdm_mutex);
1095 
1096 			IBTF_DPRINTF_L3(ibdm_string, "\tgid_decr_pending: "
1097 			    "gidinfo %p hot removal", gidinfo);
1098 			ibdm_delete_gidinfo(gidinfo);
1099 
1100 			mutex_enter(&ibdm.ibdm_mutex);
1101 			ibdm.ibdm_ngid_probes_in_progress--;
1102 			ibdm_wait_probe_completion();
1103 			mutex_exit(&ibdm.ibdm_mutex);
1104 			return;
1105 		}
1106 		mutex_exit(&gidinfo->gl_mutex);
1107 		mutex_exit(&ibdm.ibdm_mutex);
1108 		ibdm_notify_newgid_iocs(gidinfo);
1109 		mutex_enter(&ibdm.ibdm_mutex);
1110 		mutex_enter(&gidinfo->gl_mutex);
1111 
1112 		ibdm.ibdm_ngid_probes_in_progress--;
1113 		ibdm_wait_probe_completion();
1114 	}
1115 	mutex_exit(&gidinfo->gl_mutex);
1116 	mutex_exit(&ibdm.ibdm_mutex);
1117 }
1118 
1119 
1120 /*
1121  * ibdm_wait_probe_completion:
1122  *	wait for probing to complete
1123  */
1124 static void
1125 ibdm_wait_probe_completion(void)
1126 {
1127 	ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
1128 	if (ibdm.ibdm_ngid_probes_in_progress) {
1129 		IBTF_DPRINTF_L4("ibdm",	"\twait for probe complete");
1130 		ibdm.ibdm_busy |= IBDM_PROBE_IN_PROGRESS;
1131 		while (ibdm.ibdm_busy & IBDM_PROBE_IN_PROGRESS)
1132 			cv_wait(&ibdm.ibdm_probe_cv, &ibdm.ibdm_mutex);
1133 	}
1134 }
1135 
1136 
1137 /*
1138  * ibdm_wakeup_probe_gid_cv:
1139  *	wakeup waiting threads (based on ibdm_ngid_probes_in_progress)
1140  */
1141 static void
1142 ibdm_wakeup_probe_gid_cv(void)
1143 {
1144 	ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
1145 	if (!ibdm.ibdm_ngid_probes_in_progress) {
1146 		IBTF_DPRINTF_L4("ibdm", "wakeup_probe_gid_thread: Wakeup");
1147 		ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS;
1148 		cv_broadcast(&ibdm.ibdm_probe_cv);
1149 	}
1150 
1151 }
1152 
1153 
1154 /*
1155  * ibdm_sweep_fabric(reprobe_flag)
1156  *	Find all possible Managed IOU's and their IOC's that are visible
1157  *	to the host. The algorithm used is as follows
1158  *
1159  *	Send a "bus walk" request for each port on the host HCA to SA access
1160  *	SA returns complete set of GID's that are reachable from
1161  *	source port. This is done in parallel.
1162  *
1163  *	Initialize GID state to IBDM_GID_PROBE_NOT_DONE
1164  *
1165  *	Sort the GID list and eliminate duplicate GID's
1166  *		1) Use DGID for sorting
1167  *		2) use PortGuid for sorting
1168  *			Send SA query to retrieve NodeRecord and
1169  *			extract PortGuid from that.
1170  *
1171  *	Set GID state to IBDM_GID_PROBE_FAILED to all the ports that dont
1172  *	support DM MAD's
1173  *		Send a "Portinfo" query to get the port capabilities and
1174  *		then check for DM MAD's support
1175  *
1176  *	Send "ClassPortInfo" request for all the GID's in parallel,
1177  *	set the GID state to IBDM_GET_CLASSPORTINFO and wait on the
1178  *	cv_signal to complete.
1179  *
1180  *	When DM agent on the remote GID sends back the response, IBMF
1181  *	invokes DM callback routine.
1182  *
1183  *	If the response is proper, send "IOUnitInfo" request and set
1184  *	GID state to IBDM_GET_IOUNITINFO.
1185  *
1186  *	If the response is proper, send "IocProfileInfo" request to
1187  *	all the IOC simultaneously and set GID state to IBDM_GET_IOC_DETAILS.
1188  *
1189  *	Send request to get Service entries simultaneously
1190  *
1191  *	Signal the waiting thread when received response for all the commands.
1192  *
1193  *	Set the GID state to IBDM_GID_PROBE_FAILED when received a error
1194  *	response during the probing period.
1195  *
1196  *	Note:
1197  *	ibdm.ibdm_ngid_probes_in_progress and ibdm_gid_list_t:gl_pending_cmds
1198  *	keep track of number commands in progress at any point of time.
1199  *	MAD transaction ID is used to identify a particular GID
1200  *	TBD: Consider registering the IBMF receive callback on demand
1201  *
1202  *	Note: This routine must be called with ibdm.ibdm_mutex held
1203  *	TBD: Re probe the failure GID (for certain failures) when requested
1204  *	     for fabric sweep next time
1205  *
1206  *	Parameters : If reprobe_flag is set, All IOCs will be reprobed.
1207  */
1208 static void
1209 ibdm_sweep_fabric(int reprobe_flag)
1210 {
1211 	int			ii;
1212 	int			new_paths = 0;
1213 	uint8_t			niocs;
1214 	taskqid_t		tid;
1215 	ibdm_ioc_info_t		*ioc;
1216 	ibdm_hca_list_t		*hca_list = NULL;
1217 	ibdm_port_attr_t	*port = NULL;
1218 	ibdm_dp_gidinfo_t 	*gid_info;
1219 
1220 	IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: Enter");
1221 	ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
1222 
1223 	/*
1224 	 * Check whether a sweep already in progress. If so, just
1225 	 * wait for the fabric sweep to complete
1226 	 */
1227 	while (ibdm.ibdm_busy & IBDM_BUSY)
1228 		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
1229 	ibdm.ibdm_busy |= IBDM_BUSY;
1230 	mutex_exit(&ibdm.ibdm_mutex);
1231 
1232 	ibdm_dump_sweep_fabric_timestamp(0);
1233 
1234 	/* Rescan the GID list for any removed GIDs for reprobe */
1235 	if (reprobe_flag)
1236 		ibdm_rescan_gidlist(NULL);
1237 
1238 	/*
1239 	 * Get list of all the ports reachable from the local known HCA
1240 	 * ports which are active
1241 	 */
1242 	mutex_enter(&ibdm.ibdm_hl_mutex);
1243 	for (ibdm_get_next_port(&hca_list, &port, 1); port;
1244 	    ibdm_get_next_port(&hca_list, &port, 1)) {
1245 		/*
1246 		 * Get PATHS to all the reachable ports from
1247 		 * SGID and update the global ibdm structure.
1248 		 */
1249 		new_paths = ibdm_get_reachable_ports(port, hca_list);
1250 		ibdm.ibdm_ngids += new_paths;
1251 	}
1252 	mutex_exit(&ibdm.ibdm_hl_mutex);
1253 
1254 	mutex_enter(&ibdm.ibdm_mutex);
1255 	ibdm.ibdm_ngid_probes_in_progress += ibdm.ibdm_ngids;
1256 	mutex_exit(&ibdm.ibdm_mutex);
1257 
1258 	/* Send a request to probe GIDs asynchronously. */
1259 	for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
1260 	    gid_info = gid_info->gl_next) {
1261 		mutex_enter(&gid_info->gl_mutex);
1262 		gid_info->gl_reprobe_flag = reprobe_flag;
1263 		mutex_exit(&gid_info->gl_mutex);
1264 
1265 		/* process newly encountered GIDs */
1266 		tid = taskq_dispatch(system_taskq, ibdm_probe_gid_thread,
1267 		    (void *)gid_info, TQ_NOSLEEP);
1268 		IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: gid_info = %p"
1269 		    " taskq_id = %x", gid_info, tid);
1270 		/* taskq failed to dispatch call it directly */
1271 		if (tid == NULL)
1272 			ibdm_probe_gid_thread((void *)gid_info);
1273 	}
1274 
1275 	mutex_enter(&ibdm.ibdm_mutex);
1276 	ibdm_wait_probe_completion();
1277 
1278 	/*
1279 	 * Update the properties, if reprobe_flag is set
1280 	 * Skip if gl_reprobe_flag is set, this will be
1281 	 * a re-inserted / new GID, for which notifications
1282 	 * have already been send.
1283 	 */
1284 	if (reprobe_flag) {
1285 		for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
1286 		    gid_info = gid_info->gl_next) {
1287 			if (gid_info->gl_iou == NULL)
1288 				continue;
1289 			if (gid_info->gl_reprobe_flag) {
1290 				gid_info->gl_reprobe_flag = 0;
1291 				continue;
1292 			}
1293 
1294 			niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
1295 			for (ii = 0; ii < niocs; ii++) {
1296 				ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii);
1297 				if (ioc)
1298 					ibdm_reprobe_update_port_srv(ioc,
1299 					    gid_info);
1300 			}
1301 		}
1302 	}
1303 	ibdm_dump_sweep_fabric_timestamp(1);
1304 
1305 	ibdm.ibdm_busy &= ~IBDM_BUSY;
1306 	cv_broadcast(&ibdm.ibdm_busy_cv);
1307 	IBTF_DPRINTF_L5("ibdm", "\tsweep_fabric: EXIT");
1308 }
1309 
1310 
1311 /*
1312  * ibdm_probe_gid_thread:
1313  *	thread that does the actual work for sweeping the fabric
1314  *	for a given GID
1315  */
1316 static void
1317 ibdm_probe_gid_thread(void *args)
1318 {
1319 	int			reprobe_flag;
1320 	ib_guid_t		node_guid;
1321 	ib_guid_t		port_guid;
1322 	ibdm_dp_gidinfo_t	*gid_info;
1323 
1324 	gid_info = (ibdm_dp_gidinfo_t *)args;
1325 	reprobe_flag = gid_info->gl_reprobe_flag;
1326 	IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: gid_info = %p, flag = %d",
1327 	    gid_info, reprobe_flag);
1328 	ASSERT(gid_info != NULL);
1329 	ASSERT(gid_info->gl_pending_cmds == 0);
1330 
1331 	if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE &&
1332 	    reprobe_flag == 0) {
1333 		/*
1334 		 * This GID may have been already probed. Send
1335 		 * in a CLP to check if IOUnitInfo changed?
1336 		 * Explicitly set gl_reprobe_flag to 0 so that
1337 		 * IBnex is not notified on completion
1338 		 */
1339 		if (gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) {
1340 			IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: "
1341 			    "get new IOCs information");
1342 			mutex_enter(&gid_info->gl_mutex);
1343 			gid_info->gl_pending_cmds++;
1344 			gid_info->gl_state = IBDM_GET_IOUNITINFO;
1345 			gid_info->gl_reprobe_flag = 0;
1346 			mutex_exit(&gid_info->gl_mutex);
1347 			if (ibdm_send_iounitinfo(gid_info) != IBDM_SUCCESS) {
1348 				mutex_enter(&gid_info->gl_mutex);
1349 				gid_info->gl_pending_cmds = 0;
1350 				mutex_exit(&gid_info->gl_mutex);
1351 				mutex_enter(&ibdm.ibdm_mutex);
1352 				--ibdm.ibdm_ngid_probes_in_progress;
1353 				ibdm_wakeup_probe_gid_cv();
1354 				mutex_exit(&ibdm.ibdm_mutex);
1355 			}
1356 		} else {
1357 			mutex_enter(&ibdm.ibdm_mutex);
1358 			--ibdm.ibdm_ngid_probes_in_progress;
1359 			ibdm_wakeup_probe_gid_cv();
1360 			mutex_exit(&ibdm.ibdm_mutex);
1361 		}
1362 		return;
1363 	} else if (reprobe_flag && gid_info->gl_state ==
1364 	    IBDM_GID_PROBING_COMPLETE) {
1365 		/*
1366 		 * Reprobe all IOCs for the GID which has completed
1367 		 * probe. Skip other port GIDs to same IOU.
1368 		 * Explicitly set gl_reprobe_flag to 0 so that
1369 		 * IBnex is not notified on completion
1370 		 */
1371 		ibdm_ioc_info_t *ioc_info;
1372 		uint8_t		niocs, ii;
1373 
1374 		ASSERT(gid_info->gl_iou);
1375 		mutex_enter(&gid_info->gl_mutex);
1376 		niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
1377 		gid_info->gl_state = IBDM_GET_IOC_DETAILS;
1378 		gid_info->gl_pending_cmds += niocs;
1379 		gid_info->gl_reprobe_flag = 0;
1380 		mutex_exit(&gid_info->gl_mutex);
1381 		for (ii = 0; ii < niocs; ii++) {
1382 			uchar_t			slot_info;
1383 			ib_dm_io_unitinfo_t	*giou_info;
1384 
1385 			/*
1386 			 * Check whether IOC is present in the slot
1387 			 * Series of nibbles (in the field
1388 			 * iou_ctrl_list) represents a slot in the
1389 			 * IOU.
1390 			 * Byte format: 76543210
1391 			 * Bits 0-3 of first byte represent Slot 2
1392 			 * bits 4-7 of first byte represent slot 1,
1393 			 * bits 0-3 of second byte represent slot 4
1394 			 * and so on
1395 			 * Each 4-bit nibble has the following meaning
1396 			 * 0x0 : IOC not installed
1397 			 * 0x1 : IOC is present
1398 			 * 0xf : Slot does not exist
1399 			 * and all other values are reserved.
1400 			 */
1401 			ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii);
1402 			giou_info = &gid_info->gl_iou->iou_info;
1403 			slot_info = giou_info->iou_ctrl_list[(ii/2)];
1404 			if ((ii % 2) == 0)
1405 				slot_info = (slot_info >> 4);
1406 
1407 			if ((slot_info & 0xf) != 1) {
1408 				ioc_info->ioc_state =
1409 				    IBDM_IOC_STATE_PROBE_FAILED;
1410 				ibdm_gid_decr_pending(gid_info);
1411 				continue;
1412 			}
1413 
1414 			if (ibdm_send_ioc_profile(gid_info, ii) !=
1415 			    IBDM_SUCCESS) {
1416 				ibdm_gid_decr_pending(gid_info);
1417 			}
1418 		}
1419 
1420 		return;
1421 	} else if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) {
1422 		mutex_enter(&ibdm.ibdm_mutex);
1423 		--ibdm.ibdm_ngid_probes_in_progress;
1424 		ibdm_wakeup_probe_gid_cv();
1425 		mutex_exit(&ibdm.ibdm_mutex);
1426 		return;
1427 	}
1428 
1429 	mutex_enter(&gid_info->gl_mutex);
1430 	gid_info->gl_pending_cmds++;
1431 	gid_info->gl_state = IBDM_GET_CLASSPORTINFO;
1432 	mutex_exit(&gid_info->gl_mutex);
1433 
1434 	/*
1435 	 * Check whether the destination GID supports DM agents. If
1436 	 * not, stop probing the GID and continue with the next GID
1437 	 * in the list.
1438 	 */
1439 	if (ibdm_is_dev_mgt_supported(gid_info) != IBDM_SUCCESS) {
1440 		mutex_enter(&gid_info->gl_mutex);
1441 		gid_info->gl_pending_cmds = 0;
1442 		gid_info->gl_state = IBDM_GID_PROBING_FAILED;
1443 		mutex_exit(&gid_info->gl_mutex);
1444 		ibdm_delete_glhca_list(gid_info);
1445 		mutex_enter(&ibdm.ibdm_mutex);
1446 		--ibdm.ibdm_ngid_probes_in_progress;
1447 		ibdm_wakeup_probe_gid_cv();
1448 		mutex_exit(&ibdm.ibdm_mutex);
1449 		return;
1450 	}
1451 
1452 	/* Get the nodeguid and portguid of the port */
1453 	if (ibdm_get_node_port_guids(gid_info->gl_sa_hdl, gid_info->gl_dlid,
1454 	    &node_guid, &port_guid) != IBDM_SUCCESS) {
1455 		mutex_enter(&gid_info->gl_mutex);
1456 		gid_info->gl_pending_cmds = 0;
1457 		gid_info->gl_state = IBDM_GID_PROBING_FAILED;
1458 		mutex_exit(&gid_info->gl_mutex);
1459 		ibdm_delete_glhca_list(gid_info);
1460 		mutex_enter(&ibdm.ibdm_mutex);
1461 		--ibdm.ibdm_ngid_probes_in_progress;
1462 		ibdm_wakeup_probe_gid_cv();
1463 		mutex_exit(&ibdm.ibdm_mutex);
1464 		return;
1465 	}
1466 
1467 	/*
1468 	 * Check whether we already knew about this NodeGuid
1469 	 * If so, do not probe the GID and continue with the
1470 	 * next  GID  in the gid  list. Set the GID state to
1471 	 * probing done.
1472 	 */
1473 	mutex_enter(&ibdm.ibdm_mutex);
1474 	gid_info->gl_nodeguid = node_guid;
1475 	gid_info->gl_portguid = port_guid;
1476 	if (ibdm_check_dest_nodeguid(gid_info) != NULL) {
1477 		mutex_exit(&ibdm.ibdm_mutex);
1478 		mutex_enter(&gid_info->gl_mutex);
1479 		gid_info->gl_pending_cmds = 0;
1480 		gid_info->gl_state = IBDM_GID_PROBING_SKIPPED;
1481 		mutex_exit(&gid_info->gl_mutex);
1482 		ibdm_delete_glhca_list(gid_info);
1483 		mutex_enter(&ibdm.ibdm_mutex);
1484 		--ibdm.ibdm_ngid_probes_in_progress;
1485 		ibdm_wakeup_probe_gid_cv();
1486 		mutex_exit(&ibdm.ibdm_mutex);
1487 		return;
1488 	}
1489 	ibdm_add_to_gl_gid(gid_info, gid_info);
1490 	mutex_exit(&ibdm.ibdm_mutex);
1491 
1492 	/*
1493 	 * New or reinserted GID : Enable notification to IBnex
1494 	 */
1495 	mutex_enter(&gid_info->gl_mutex);
1496 	gid_info->gl_reprobe_flag = 1;
1497 	mutex_exit(&gid_info->gl_mutex);
1498 
1499 	/*
1500 	 * Send ClassPortInfo request to the GID asynchronously.
1501 	 */
1502 	if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) {
1503 		mutex_enter(&gid_info->gl_mutex);
1504 		gid_info->gl_state = IBDM_GID_PROBING_FAILED;
1505 		gid_info->gl_pending_cmds = 0;
1506 		mutex_exit(&gid_info->gl_mutex);
1507 		ibdm_delete_glhca_list(gid_info);
1508 		mutex_enter(&ibdm.ibdm_mutex);
1509 		--ibdm.ibdm_ngid_probes_in_progress;
1510 		ibdm_wakeup_probe_gid_cv();
1511 		mutex_exit(&ibdm.ibdm_mutex);
1512 		return;
1513 	}
1514 }
1515 
1516 
1517 /*
1518  * ibdm_check_dest_nodeguid
1519  *	Searches for the NodeGuid in the GID list
1520  *	Returns matching gid_info if found and otherwise NULL
1521  *
1522  *	This function is called to handle new GIDs discovered
1523  *	during device sweep / probe or for GID_AVAILABLE event.
1524  *
1525  *	Parameter :
1526  *		gid_info	GID to check
1527  */
1528 static ibdm_dp_gidinfo_t *
1529 ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *gid_info)
1530 {
1531 	ibdm_dp_gidinfo_t	*gid_list;
1532 	ibdm_gid_t		*tmp;
1533 
1534 	IBTF_DPRINTF_L4("ibdm", "\tcheck_dest_nodeguid");
1535 
1536 	gid_list = ibdm.ibdm_dp_gidlist_head;
1537 	while (gid_list) {
1538 		if ((gid_list != gid_info) &&
1539 		    (gid_info->gl_nodeguid == gid_list->gl_nodeguid)) {
1540 			IBTF_DPRINTF_L4("ibdm",
1541 			    "\tcheck_dest_nodeguid: NodeGuid is present");
1542 
1543 			/* Add to gid_list */
1544 			tmp = kmem_zalloc(sizeof (ibdm_gid_t),
1545 			    KM_SLEEP);
1546 			tmp->gid_dgid_hi = gid_info->gl_dgid_hi;
1547 			tmp->gid_dgid_lo = gid_info->gl_dgid_lo;
1548 			tmp->gid_next = gid_list->gl_gid;
1549 			gid_list->gl_gid = tmp;
1550 			gid_list->gl_ngids++;
1551 			return (gid_list);
1552 		}
1553 
1554 		gid_list = gid_list->gl_next;
1555 	}
1556 
1557 	return (NULL);
1558 }
1559 
1560 
1561 /*
1562  * ibdm_is_dev_mgt_supported
1563  *	Get the PortInfo attribute (SA Query)
1564  *	Check "CompatabilityMask" field in the Portinfo.
1565  *	Return IBDM_SUCCESS if DM MAD's supported (if bit 19 set)
1566  *	by the port, otherwise IBDM_FAILURE
1567  */
1568 static int
1569 ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *gid_info)
1570 {
1571 	int			ret;
1572 	size_t			length = 0;
1573 	sa_portinfo_record_t	req, *resp = NULL;
1574 	ibmf_saa_access_args_t	qargs;
1575 
1576 	bzero(&req, sizeof (sa_portinfo_record_t));
1577 	req.EndportLID	= gid_info->gl_dlid;
1578 
1579 	qargs.sq_attr_id	= SA_PORTINFORECORD_ATTRID;
1580 	qargs.sq_access_type	= IBMF_SAA_RETRIEVE;
1581 	qargs.sq_component_mask = SA_PORTINFO_COMPMASK_PORTLID;
1582 	qargs.sq_template	= &req;
1583 	qargs.sq_callback	= NULL;
1584 	qargs.sq_callback_arg	= NULL;
1585 
1586 	ret = ibmf_sa_access(gid_info->gl_sa_hdl,
1587 	    &qargs, 0, &length, (void **)&resp);
1588 
1589 	if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) {
1590 		IBTF_DPRINTF_L2("ibdm", "\tis_dev_mgt_supported:"
1591 		    "failed to get PORTINFO attribute %d", ret);
1592 		return (IBDM_FAILURE);
1593 	}
1594 
1595 	if (resp->PortInfo.CapabilityMask & SM_CAP_MASK_IS_DM_SUPPD) {
1596 		IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: SUPPD !!");
1597 		ret = IBDM_SUCCESS;
1598 	} else {
1599 		IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: "
1600 		    "Not SUPPD !!, cap 0x%x", resp->PortInfo.CapabilityMask);
1601 		ret = IBDM_FAILURE;
1602 	}
1603 	kmem_free(resp, length);
1604 	return (ret);
1605 }
1606 
1607 
1608 /*
1609  * ibdm_get_node_port_guids()
1610  *	Get the NodeInfoRecord of the port
1611  *	Save NodeGuid and PortGUID values in the GID list structure.
1612  *	Return IBDM_SUCCESS/IBDM_FAILURE
1613  */
1614 static int
1615 ibdm_get_node_port_guids(ibmf_saa_handle_t sa_hdl, ib_lid_t dlid,
1616     ib_guid_t *node_guid, ib_guid_t *port_guid)
1617 {
1618 	int			ret;
1619 	size_t			length = 0;
1620 	sa_node_record_t	req, *resp = NULL;
1621 	ibmf_saa_access_args_t	qargs;
1622 
1623 	IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids");
1624 
1625 	bzero(&req, sizeof (sa_node_record_t));
1626 	req.LID = dlid;
1627 
1628 	qargs.sq_attr_id	= SA_NODERECORD_ATTRID;
1629 	qargs.sq_access_type	= IBMF_SAA_RETRIEVE;
1630 	qargs.sq_component_mask = SA_NODEINFO_COMPMASK_NODELID;
1631 	qargs.sq_template	= &req;
1632 	qargs.sq_callback	= NULL;
1633 	qargs.sq_callback_arg	= NULL;
1634 
1635 	ret = ibmf_sa_access(sa_hdl, &qargs, 0, &length, (void **)&resp);
1636 	if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) {
1637 		IBTF_DPRINTF_L2("ibdm", "\tget_node_port_guids:"
1638 		    " SA Retrieve Failed: %d", ret);
1639 		return (IBDM_FAILURE);
1640 	}
1641 	IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids: NodeGuid %llx Port"
1642 	    "GUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.NodeGUID);
1643 
1644 	*node_guid = resp->NodeInfo.NodeGUID;
1645 	*port_guid = resp->NodeInfo.PortGUID;
1646 	kmem_free(resp, length);
1647 	return (IBDM_SUCCESS);
1648 }
1649 
1650 
1651 /*
1652  * ibdm_get_reachable_ports()
1653  *	Get list of the destination GID (and its path  records) by
1654  *	querying the SA access.
1655  *
1656  *	Returns Number paths
1657  */
1658 static int
1659 ibdm_get_reachable_ports(ibdm_port_attr_t *portinfo, ibdm_hca_list_t *hca)
1660 {
1661 	uint_t			ii, jj, nrecs;
1662 	uint_t			npaths = 0;
1663 	size_t			length;
1664 	ib_gid_t		sgid;
1665 	ibdm_pkey_tbl_t		*pkey_tbl;
1666 	sa_path_record_t	*result;
1667 	sa_path_record_t	*precp;
1668 	ibdm_dp_gidinfo_t	*gid_info;
1669 
1670 	ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
1671 	IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: portinfo %p", portinfo);
1672 
1673 	sgid.gid_prefix = portinfo->pa_sn_prefix;
1674 	sgid.gid_guid	= portinfo->pa_port_guid;
1675 
1676 	/* get reversible paths */
1677 	if (portinfo->pa_sa_hdl && ibmf_saa_paths_from_gid(portinfo->pa_sa_hdl,
1678 	    sgid, IBMF_SAA_PKEY_WC, B_TRUE, 0, &nrecs, &length, &result)
1679 	    != IBMF_SUCCESS) {
1680 		IBTF_DPRINTF_L2("ibdm",
1681 		    "\tget_reachable_ports: Getting path records failed");
1682 		return (0);
1683 	}
1684 
1685 	for (ii = 0; ii < nrecs; ii++) {
1686 		precp = &result[ii];
1687 		if ((gid_info = ibdm_check_dgid(precp->DGID.gid_guid,
1688 		    precp->DGID.gid_prefix)) != NULL) {
1689 			IBTF_DPRINTF_L2("ibdm", "\tget_reachable_ports: "
1690 			    "Already exists nrecs %d, ii %d", nrecs, ii);
1691 			ibdm_addto_glhcalist(gid_info, hca);
1692 			continue;
1693 		}
1694 		/*
1695 		 * This is a new GID. Allocate a GID structure and
1696 		 * initialize the structure
1697 		 * gl_state is initialized to IBDM_GID_PROBE_NOT_DONE (0)
1698 		 * by kmem_zalloc call
1699 		 */
1700 		gid_info = kmem_zalloc(sizeof (ibdm_dp_gidinfo_t), KM_SLEEP);
1701 		mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL);
1702 		gid_info->gl_dgid_hi		= precp->DGID.gid_prefix;
1703 		gid_info->gl_dgid_lo		= precp->DGID.gid_guid;
1704 		gid_info->gl_sgid_hi		= precp->SGID.gid_prefix;
1705 		gid_info->gl_sgid_lo		= precp->SGID.gid_guid;
1706 		gid_info->gl_p_key		= precp->P_Key;
1707 		gid_info->gl_sa_hdl		= portinfo->pa_sa_hdl;
1708 		gid_info->gl_ibmf_hdl		= portinfo->pa_ibmf_hdl;
1709 		gid_info->gl_slid		= precp->SLID;
1710 		gid_info->gl_dlid		= precp->DLID;
1711 		gid_info->gl_transactionID	= (++ibdm.ibdm_transactionID)
1712 		    << IBDM_GID_TRANSACTIONID_SHIFT;
1713 		gid_info->gl_min_transactionID  = gid_info->gl_transactionID;
1714 		gid_info->gl_max_transactionID  = (ibdm.ibdm_transactionID +1)
1715 		    << IBDM_GID_TRANSACTIONID_SHIFT;
1716 		ibdm_addto_glhcalist(gid_info,  hca);
1717 
1718 		ibdm_dump_path_info(precp);
1719 
1720 		gid_info->gl_qp_hdl = NULL;
1721 		ASSERT(portinfo->pa_pkey_tbl != NULL &&
1722 		    portinfo->pa_npkeys != 0);
1723 
1724 		for (jj = 0; jj < portinfo->pa_npkeys; jj++) {
1725 			pkey_tbl = &portinfo->pa_pkey_tbl[jj];
1726 			if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) &&
1727 			    (pkey_tbl->pt_qp_hdl != NULL)) {
1728 				gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl;
1729 				break;
1730 			}
1731 		}
1732 
1733 		/*
1734 		 * QP handle for GID not initialized. No matching Pkey
1735 		 * was found!! ibdm should *not* hit this case. Flag an
1736 		 * error and drop the GID if ibdm does encounter this.
1737 		 */
1738 		if (gid_info->gl_qp_hdl == NULL) {
1739 			IBTF_DPRINTF_L2(ibdm_string,
1740 			    "\tget_reachable_ports: No matching Pkey");
1741 			ibdm_delete_gidinfo(gid_info);
1742 			continue;
1743 		}
1744 		if (ibdm.ibdm_dp_gidlist_head == NULL) {
1745 			ibdm.ibdm_dp_gidlist_head = gid_info;
1746 			ibdm.ibdm_dp_gidlist_tail = gid_info;
1747 		} else {
1748 			ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info;
1749 			gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail;
1750 			ibdm.ibdm_dp_gidlist_tail = gid_info;
1751 		}
1752 		npaths++;
1753 	}
1754 	kmem_free(result, length);
1755 	IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: npaths = %d", npaths);
1756 	return (npaths);
1757 }
1758 
1759 
1760 /*
1761  * ibdm_check_dgid()
1762  *	Look in the global list to check whether we know this DGID already
1763  *	Return IBDM_GID_PRESENT/IBDM_GID_NOT_PRESENT
1764  */
1765 static ibdm_dp_gidinfo_t *
1766 ibdm_check_dgid(ib_guid_t guid, ib_sn_prefix_t prefix)
1767 {
1768 	ibdm_dp_gidinfo_t	*gid_list;
1769 
1770 	for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
1771 	    gid_list = gid_list->gl_next) {
1772 		if ((guid == gid_list->gl_dgid_lo) &&
1773 		    (prefix == gid_list->gl_dgid_hi)) {
1774 			break;
1775 		}
1776 	}
1777 	return (gid_list);
1778 }
1779 
1780 
1781 /*
1782  * ibdm_find_gid()
1783  *	Look in the global list to find a GID entry with matching
1784  *	port & node GUID.
1785  *	Return pointer to gidinfo if found, else return NULL
1786  */
1787 static ibdm_dp_gidinfo_t *
1788 ibdm_find_gid(ib_guid_t nodeguid, ib_guid_t portguid)
1789 {
1790 	ibdm_dp_gidinfo_t	*gid_list;
1791 
1792 	IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid(%llx, %llx)\n",
1793 	    nodeguid, portguid);
1794 
1795 	for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
1796 	    gid_list = gid_list->gl_next) {
1797 		if ((portguid == gid_list->gl_portguid) &&
1798 		    (nodeguid == gid_list->gl_nodeguid)) {
1799 			break;
1800 		}
1801 	}
1802 
1803 	IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid : returned %p\n",
1804 	    gid_list);
1805 	return (gid_list);
1806 }
1807 
1808 
1809 /*
1810  * ibdm_send_classportinfo()
1811  *	Send classportinfo request. When the request is completed
1812  *	IBMF calls ibdm_classportinfo_cb routine to inform about
1813  *	the completion.
1814  *	Returns IBDM_SUCCESS/IBDM_FAILURE
1815  */
1816 static int
1817 ibdm_send_classportinfo(ibdm_dp_gidinfo_t *gid_info)
1818 {
1819 	ibmf_msg_t		*msg;
1820 	ib_mad_hdr_t		*hdr;
1821 	ibdm_timeout_cb_args_t	*cb_args;
1822 
1823 	IBTF_DPRINTF_L4("ibdm",
1824 	    "\tsend_classportinfo: gid info 0x%p", gid_info);
1825 
1826 	/*
1827 	 * Send command to get classportinfo attribute. Allocate a IBMF
1828 	 * packet and initialize the packet.
1829 	 */
1830 	if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
1831 	    &msg) != IBMF_SUCCESS) {
1832 		IBTF_DPRINTF_L4("ibdm", "\tsend_classportinfo: pkt alloc fail");
1833 		return (IBDM_FAILURE);
1834 	}
1835 
1836 	ibdm_alloc_send_buffers(msg);
1837 
1838 	msg->im_local_addr.ia_local_lid		= gid_info->gl_slid;
1839 	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
1840 	msg->im_local_addr.ia_remote_qno	= 1;
1841 	msg->im_local_addr.ia_p_key		= gid_info->gl_p_key;
1842 	msg->im_local_addr.ia_q_key		= IB_GSI_QKEY;
1843 
1844 	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
1845 	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
1846 	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
1847 	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
1848 	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
1849 	hdr->Status		= 0;
1850 	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
1851 	hdr->AttributeID	= h2b16(IB_DM_ATTR_CLASSPORTINFO);
1852 	hdr->AttributeModifier	= 0;
1853 
1854 	cb_args = &gid_info->gl_cpi_cb_args;
1855 	cb_args->cb_gid_info = gid_info;
1856 	cb_args->cb_retry_count	= ibdm_dft_retry_cnt;
1857 	cb_args->cb_req_type = IBDM_REQ_TYPE_CLASSPORTINFO;
1858 
1859 	gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
1860 	    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
1861 
1862 	IBTF_DPRINTF_L5("ibdm", "\tsend_classportinfo: "
1863 	    "timeout id %x", gid_info->gl_timeout_id);
1864 
1865 	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
1866 	    msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
1867 		IBTF_DPRINTF_L2("ibdm",
1868 		    "\tsend_classportinfo: ibmf send failed");
1869 		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
1870 	}
1871 
1872 	return (IBDM_SUCCESS);
1873 }
1874 
1875 
1876 /*
1877  * ibdm_handle_classportinfo()
1878  *	Invoked by the IBMF when the classportinfo request is completed.
1879  */
1880 static void
1881 ibdm_handle_classportinfo(ibmf_handle_t ibmf_hdl,
1882     ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
1883 {
1884 	void			*data;
1885 	timeout_id_t		timeout_id;
1886 	ib_mad_hdr_t		*hdr;
1887 	ibdm_mad_classportinfo_t *cpi;
1888 
1889 	IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo:ibmf hdl "
1890 	    "%p msg %p gid info %p", ibmf_hdl, msg, gid_info);
1891 
1892 	if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO) {
1893 		IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo: "
1894 		    "Not a ClassPortInfo resp");
1895 		*flag |= IBDM_IBMF_PKT_UNEXP_RESP;
1896 		return;
1897 	}
1898 
1899 	/*
1900 	 * Verify whether timeout handler is created/active.
1901 	 * If created/ active,  cancel the timeout  handler
1902 	 */
1903 	mutex_enter(&gid_info->gl_mutex);
1904 	ibdm_bump_transactionID(gid_info);
1905 	if (gid_info->gl_state != IBDM_GET_CLASSPORTINFO) {
1906 		IBTF_DPRINTF_L2("ibdm", "\thandle_classportinfo:DUP resp");
1907 		*flag |= IBDM_IBMF_PKT_DUP_RESP;
1908 		mutex_exit(&gid_info->gl_mutex);
1909 		return;
1910 	}
1911 	gid_info->gl_iou_cb_args.cb_req_type = 0;
1912 	if (gid_info->gl_timeout_id) {
1913 		timeout_id = gid_info->gl_timeout_id;
1914 		mutex_exit(&gid_info->gl_mutex);
1915 		IBTF_DPRINTF_L5("ibdm", "handle_ioclassportinfo: "
1916 		    "gl_timeout_id = 0x%x", timeout_id);
1917 		if (untimeout(timeout_id) == -1) {
1918 			IBTF_DPRINTF_L2("ibdm", "handle_classportinfo: "
1919 			    "untimeout gl_timeout_id failed");
1920 		}
1921 		mutex_enter(&gid_info->gl_mutex);
1922 		gid_info->gl_timeout_id = 0;
1923 	}
1924 	gid_info->gl_state = IBDM_GET_IOUNITINFO;
1925 	gid_info->gl_pending_cmds++;
1926 	mutex_exit(&gid_info->gl_mutex);
1927 
1928 	data = msg->im_msgbufs_recv.im_bufs_cl_data;
1929 	cpi = (ibdm_mad_classportinfo_t *)data;
1930 
1931 	/*
1932 	 * Cache the "RespTimeValue" and redirection information in the
1933 	 * global gid list data structure. This cached information will
1934 	 * be used to send any further requests to the GID.
1935 	 */
1936 	gid_info->gl_resp_timeout	=
1937 		(b2h32(cpi->RespTimeValue) & 0x1F);
1938 
1939 	gid_info->gl_redirected		= ((IBDM_IN_IBMFMSG_STATUS(msg) &
1940 	    MAD_STATUS_REDIRECT_REQUIRED) ? B_TRUE: B_FALSE);
1941 	gid_info->gl_redirect_dlid	= b2h16(cpi->RedirectLID);
1942 	gid_info->gl_redirect_QP	= (b2h32(cpi->RedirectQP) & 0xffffff);
1943 	gid_info->gl_redirect_pkey	= b2h16(cpi->RedirectP_Key);
1944 	gid_info->gl_redirect_qkey	= b2h32(cpi->RedirectQ_Key);
1945 	gid_info->gl_redirectGID_hi	= b2h64(cpi->RedirectGID_hi);
1946 	gid_info->gl_redirectGID_lo	= b2h64(cpi->RedirectGID_lo);
1947 
1948 	ibdm_dump_classportinfo(cpi);
1949 
1950 	/*
1951 	 * Send IOUnitInfo request
1952 	 * Reuse previously allocated IBMF packet for sending ClassPortInfo
1953 	 * Check whether DM agent on the remote node requested redirection
1954 	 * If so, send the request to the redirect DGID/DLID/PKEY/QP.
1955 	 */
1956 	ibdm_alloc_send_buffers(msg);
1957 	msg->im_local_addr.ia_local_lid	= gid_info->gl_slid;
1958 	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
1959 
1960 	if (gid_info->gl_redirected == B_TRUE) {
1961 		if (gid_info->gl_redirect_dlid != 0) {
1962 			msg->im_local_addr.ia_remote_lid =
1963 					gid_info->gl_redirect_dlid;
1964 		}
1965 		msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
1966 		msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
1967 		msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
1968 	} else {
1969 		msg->im_local_addr.ia_remote_qno = 1;
1970 		msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
1971 		msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
1972 	}
1973 
1974 	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
1975 	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
1976 	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
1977 	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
1978 	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
1979 	hdr->Status		= 0;
1980 	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
1981 	hdr->AttributeID	= h2b16(IB_DM_ATTR_IO_UNITINFO);
1982 	hdr->AttributeModifier	= 0;
1983 
1984 	gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO;
1985 	gid_info->gl_iou_cb_args.cb_gid_info = gid_info;
1986 	gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt;
1987 
1988 	gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
1989 	    &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
1990 
1991 	IBTF_DPRINTF_L5("ibdm", "handle_classportinfo:"
1992 	    "timeout %x", gid_info->gl_timeout_id);
1993 
1994 	if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, NULL,
1995 	    ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) != IBMF_SUCCESS) {
1996 		IBTF_DPRINTF_L2("ibdm",
1997 		    "\thandle_classportinfo: msg transport failed");
1998 		ibdm_ibmf_send_cb(ibmf_hdl, msg, &gid_info->gl_iou_cb_args);
1999 	}
2000 	(*flag) |= IBDM_IBMF_PKT_REUSED;
2001 }
2002 
2003 
2004 /*
2005  * ibdm_send_iounitinfo:
2006  *	Sends a DM request to get IOU unitinfo.
2007  */
2008 static int
2009 ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *gid_info)
2010 {
2011 	ibmf_msg_t	*msg;
2012 	ib_mad_hdr_t	*hdr;
2013 
2014 	IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: gid info 0x%p", gid_info);
2015 
2016 	/*
2017 	 * Send command to get iounitinfo attribute. Allocate a IBMF
2018 	 * packet and initialize the packet.
2019 	 */
2020 	if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, &msg) !=
2021 	    IBMF_SUCCESS) {
2022 		IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: pkt alloc fail");
2023 		return (IBDM_FAILURE);
2024 	}
2025 
2026 	mutex_enter(&gid_info->gl_mutex);
2027 	ibdm_bump_transactionID(gid_info);
2028 	mutex_exit(&gid_info->gl_mutex);
2029 
2030 
2031 	ibdm_alloc_send_buffers(msg);
2032 	msg->im_local_addr.ia_local_lid		= gid_info->gl_slid;
2033 	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
2034 	msg->im_local_addr.ia_remote_qno	= 1;
2035 	msg->im_local_addr.ia_p_key		= gid_info->gl_p_key;
2036 	msg->im_local_addr.ia_q_key		= IB_GSI_QKEY;
2037 
2038 	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
2039 	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
2040 	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
2041 	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
2042 	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
2043 	hdr->Status		= 0;
2044 	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
2045 	hdr->AttributeID	= h2b16(IB_DM_ATTR_IO_UNITINFO);
2046 	hdr->AttributeModifier	= 0;
2047 
2048 	gid_info->gl_iou_cb_args.cb_gid_info = gid_info;
2049 	gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt;
2050 	gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO;
2051 
2052 	gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2053 	    &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2054 
2055 	IBTF_DPRINTF_L5("ibdm", "send_iouunitinfo:"
2056 	    "timeout %x", gid_info->gl_timeout_id);
2057 
2058 	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg,
2059 	    NULL, ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) !=
2060 	    IBMF_SUCCESS) {
2061 		IBTF_DPRINTF_L2("ibdm", "\tsend_iounitinfo: ibmf send failed");
2062 		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl,
2063 		    msg, &gid_info->gl_iou_cb_args);
2064 	}
2065 	return (IBDM_SUCCESS);
2066 }
2067 
2068 /*
2069  * ibdm_handle_iounitinfo()
2070  *	Invoked by the IBMF when IO Unitinfo request is completed.
2071  */
2072 static void
2073 ibdm_handle_iounitinfo(ibmf_handle_t ibmf_hdl,
2074     ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
2075 {
2076 	int			ii, first = B_TRUE;
2077 	int			num_iocs;
2078 	size_t			size;
2079 	uchar_t			slot_info;
2080 	timeout_id_t		timeout_id;
2081 	ib_mad_hdr_t		*hdr;
2082 	ibdm_ioc_info_t		*ioc_info;
2083 	ib_dm_io_unitinfo_t	*iou_info;
2084 	ib_dm_io_unitinfo_t	*giou_info;
2085 	ibdm_timeout_cb_args_t	*cb_args;
2086 
2087 	IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo:"
2088 	    " ibmf hdl %p pkt %p gid info %p", ibmf_hdl, msg, gid_info);
2089 
2090 	if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_IO_UNITINFO) {
2091 		IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: "
2092 		    "Unexpected response");
2093 		(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
2094 		return;
2095 	}
2096 
2097 	mutex_enter(&gid_info->gl_mutex);
2098 	if (gid_info->gl_state != IBDM_GET_IOUNITINFO) {
2099 		IBTF_DPRINTF_L4("ibdm",
2100 		    "\thandle_iounitinfo: DUP resp");
2101 		mutex_exit(&gid_info->gl_mutex);
2102 		(*flag) = IBDM_IBMF_PKT_DUP_RESP;
2103 		return;
2104 	}
2105 	gid_info->gl_iou_cb_args.cb_req_type = 0;
2106 	if (gid_info->gl_timeout_id) {
2107 		timeout_id = gid_info->gl_timeout_id;
2108 		mutex_exit(&gid_info->gl_mutex);
2109 		IBTF_DPRINTF_L5("ibdm", "handle_iounitinfo: "
2110 		    "gl_timeout_id = 0x%x", timeout_id);
2111 		if (untimeout(timeout_id) == -1) {
2112 			IBTF_DPRINTF_L2("ibdm", "handle_iounitinfo: "
2113 			    "untimeout gl_timeout_id failed");
2114 		}
2115 		mutex_enter(&gid_info->gl_mutex);
2116 		gid_info->gl_timeout_id = 0;
2117 	}
2118 	gid_info->gl_state = IBDM_GET_IOC_DETAILS;
2119 
2120 	iou_info = IBDM_IN_IBMFMSG2IOU(msg);
2121 	ibdm_dump_iounitinfo(iou_info);
2122 	num_iocs = iou_info->iou_num_ctrl_slots;
2123 	/*
2124 	 * check if number of IOCs reported is zero? if yes, return.
2125 	 * when num_iocs are reported zero internal IOC database needs
2126 	 * to be updated. To ensure that save the number of IOCs in
2127 	 * the new field "gl_num_iocs". Use a new field instead of
2128 	 * "giou_info->iou_num_ctrl_slots" as that would prevent
2129 	 * an unnecessary kmem_alloc/kmem_free when num_iocs is 0.
2130 	 */
2131 	if (num_iocs == 0 && gid_info->gl_num_iocs == 0) {
2132 		IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: no IOC's");
2133 		mutex_exit(&gid_info->gl_mutex);
2134 		return;
2135 	}
2136 	IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: num_iocs = %d", num_iocs);
2137 
2138 	/*
2139 	 * if there is an existing gl_iou (IOU has been probed before)
2140 	 * check if the "iou_changeid" is same as saved entry in
2141 	 * "giou_info->iou_changeid".
2142 	 * (note: this logic can prevent IOC enumeration if a given
2143 	 * vendor doesn't support setting iou_changeid field for its IOU)
2144 	 *
2145 	 * if there is an existing gl_iou and iou_changeid has changed :
2146 	 * free up existing gl_iou info and its related structures.
2147 	 * reallocate gl_iou info all over again.
2148 	 * if we donot free this up; then this leads to memory leaks
2149 	 */
2150 	if (gid_info->gl_iou) {
2151 		giou_info = &gid_info->gl_iou->iou_info;
2152 		if (iou_info->iou_changeid == giou_info->iou_changeid) {
2153 			IBTF_DPRINTF_L3("ibdm",
2154 			    "\thandle_iounitinfo: no IOCs changed");
2155 			gid_info->gl_state = IBDM_GID_PROBING_COMPLETE;
2156 			mutex_exit(&gid_info->gl_mutex);
2157 			return;
2158 		}
2159 		if (ibdm_free_iou_info(gid_info)) {
2160 			IBTF_DPRINTF_L3("ibdm",
2161 			    "\thandle_iounitinfo: failed to cleanup resources");
2162 			gid_info->gl_state = IBDM_GID_PROBING_COMPLETE;
2163 			mutex_exit(&gid_info->gl_mutex);
2164 			return;
2165 		}
2166 	}
2167 
2168 	size = sizeof (ibdm_iou_info_t) + num_iocs * sizeof (ibdm_ioc_info_t);
2169 	gid_info->gl_iou = (ibdm_iou_info_t *)kmem_zalloc(size, KM_SLEEP);
2170 	giou_info = &gid_info->gl_iou->iou_info;
2171 	gid_info->gl_iou->iou_ioc_info = (ibdm_ioc_info_t *)
2172 		((char *)gid_info->gl_iou + sizeof (ibdm_iou_info_t));
2173 
2174 	giou_info->iou_num_ctrl_slots	= gid_info->gl_num_iocs	= num_iocs;
2175 	giou_info->iou_flag		= iou_info->iou_flag;
2176 	bcopy(iou_info->iou_ctrl_list, giou_info->iou_ctrl_list, 128);
2177 	giou_info->iou_changeid	= b2h16(iou_info->iou_changeid);
2178 	gid_info->gl_pending_cmds += num_iocs;
2179 	gid_info->gl_pending_cmds += 1; /* for diag code */
2180 	mutex_exit(&gid_info->gl_mutex);
2181 
2182 	if (ibdm_get_diagcode(gid_info, 0) != IBDM_SUCCESS) {
2183 		mutex_enter(&gid_info->gl_mutex);
2184 		gid_info->gl_pending_cmds--;
2185 		mutex_exit(&gid_info->gl_mutex);
2186 	}
2187 	/*
2188 	 * Parallelize getting IOC controller profiles from here.
2189 	 * Allocate IBMF packets and send commands to get IOC profile for
2190 	 * each IOC present on the IOU.
2191 	 */
2192 	for (ii = 0; ii < num_iocs; ii++) {
2193 		/*
2194 		 * Check whether IOC is present in the slot
2195 		 * Series of nibbles (in the field iou_ctrl_list) represents
2196 		 * a slot in the IOU.
2197 		 * Byte format: 76543210
2198 		 * Bits 0-3 of first byte represent Slot 2
2199 		 * bits 4-7 of first byte represent slot 1,
2200 		 * bits 0-3 of second byte represent slot 4 and so on
2201 		 * Each 4-bit nibble has the following meaning
2202 		 * 0x0 : IOC not installed
2203 		 * 0x1 : IOC is present
2204 		 * 0xf : Slot does not exist
2205 		 * and all other values are reserved.
2206 		 */
2207 		ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii);
2208 		slot_info = giou_info->iou_ctrl_list[(ii/2)];
2209 		if ((ii % 2) == 0)
2210 			slot_info = (slot_info >> 4);
2211 
2212 		if ((slot_info & 0xf) != 1) {
2213 			IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: "
2214 			    "No IOC is present in the slot = %d", ii);
2215 			ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_FAILED;
2216 			mutex_enter(&gid_info->gl_mutex);
2217 			gid_info->gl_pending_cmds--;
2218 			mutex_exit(&gid_info->gl_mutex);
2219 			continue;
2220 		}
2221 
2222 		mutex_enter(&gid_info->gl_mutex);
2223 		ibdm_bump_transactionID(gid_info);
2224 		mutex_exit(&gid_info->gl_mutex);
2225 
2226 		/*
2227 		 * Re use the already allocated packet (for IOUnitinfo) to
2228 		 * send the first IOC controller attribute. Allocate new
2229 		 * IBMF packets for the rest of the IOC's
2230 		 */
2231 		if (first != B_TRUE) {
2232 			msg = NULL;
2233 			if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP,
2234 			    &msg) != IBMF_SUCCESS) {
2235 				IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: "
2236 				    "IBMF packet allocation failed");
2237 				mutex_enter(&gid_info->gl_mutex);
2238 				gid_info->gl_pending_cmds--;
2239 				mutex_exit(&gid_info->gl_mutex);
2240 				continue;
2241 			}
2242 
2243 		}
2244 
2245 		/* allocate send buffers for all messages */
2246 		ibdm_alloc_send_buffers(msg);
2247 
2248 		msg->im_local_addr.ia_local_lid	= gid_info->gl_slid;
2249 		msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
2250 		if (gid_info->gl_redirected == B_TRUE) {
2251 			if (gid_info->gl_redirect_dlid != 0) {
2252 				msg->im_local_addr.ia_remote_lid =
2253 						gid_info->gl_redirect_dlid;
2254 			}
2255 			msg->im_local_addr.ia_remote_qno =
2256 						gid_info->gl_redirect_QP;
2257 			msg->im_local_addr.ia_p_key =
2258 						gid_info->gl_redirect_pkey;
2259 			msg->im_local_addr.ia_q_key =
2260 						gid_info->gl_redirect_qkey;
2261 		} else {
2262 			msg->im_local_addr.ia_remote_qno = 1;
2263 			msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
2264 			msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
2265 		}
2266 
2267 		hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
2268 		hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
2269 		hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
2270 		hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
2271 		hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
2272 		hdr->Status		= 0;
2273 		hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
2274 		hdr->AttributeID	= h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE);
2275 		hdr->AttributeModifier 	= h2b32(ii + 1);
2276 
2277 		ioc_info->ioc_state	= IBDM_IOC_STATE_PROBE_INVALID;
2278 		cb_args			= &ioc_info->ioc_cb_args;
2279 		cb_args->cb_gid_info	= gid_info;
2280 		cb_args->cb_retry_count	= ibdm_dft_retry_cnt;
2281 		cb_args->cb_req_type	= IBDM_REQ_TYPE_IOCINFO;
2282 		cb_args->cb_ioc_num	= ii;
2283 
2284 		ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2285 		    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2286 
2287 		IBTF_DPRINTF_L5("ibdm", "\thandle_iounitinfo:"
2288 		    "timeout 0x%x, ioc_num %d", ioc_info->ioc_timeout_id, ii);
2289 
2290 		if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg,
2291 		    NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
2292 			IBTF_DPRINTF_L2("ibdm",
2293 			    "\thandle_iounitinfo: msg transport failed");
2294 			ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args);
2295 		}
2296 		(*flag) |= IBDM_IBMF_PKT_REUSED;
2297 		first = B_FALSE;
2298 		gid_info->gl_iou->iou_niocs_probe_in_progress++;
2299 	}
2300 }
2301 
2302 
2303 /*
2304  * ibdm_handle_ioc_profile()
2305  *	Invoked by the IBMF when the IOCControllerProfile request
2306  *	gets completed
2307  */
2308 static void
2309 ibdm_handle_ioc_profile(ibmf_handle_t ibmf_hdl,
2310     ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
2311 {
2312 	int				first = B_TRUE, reprobe = 0;
2313 	uint_t				ii, ioc_no, srv_start;
2314 	uint_t				nserv_entries;
2315 	timeout_id_t			timeout_id;
2316 	ib_mad_hdr_t			*hdr;
2317 	ibdm_ioc_info_t			*ioc_info;
2318 	ibdm_timeout_cb_args_t		*cb_args;
2319 	ib_dm_ioc_ctrl_profile_t	*ioc, *gioc;
2320 
2321 	IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:"
2322 	    " ibmf hdl %p msg %p gid info %p", ibmf_hdl, msg, gid_info);
2323 
2324 	ioc = IBDM_IN_IBMFMSG2IOC(msg);
2325 	/*
2326 	 * Check whether we know this IOC already
2327 	 * This will return NULL if reprobe is in progress
2328 	 * IBDM_IOC_STATE_REPROBE_PROGRESS will be set.
2329 	 * Do not hold mutexes here.
2330 	 */
2331 	if (ibdm_is_ioc_present(ioc->ioc_guid, gid_info, flag) != NULL) {
2332 		IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:"
2333 			"IOC guid %llx is present", ioc->ioc_guid);
2334 		return;
2335 	}
2336 	ioc_no = IBDM_IN_IBMFMSG_ATTRMOD(msg);
2337 	IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile: ioc_no = %d", ioc_no-1);
2338 
2339 	/* Make sure that IOC index is with the valid range */
2340 	if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) {
2341 		IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: "
2342 		    "IOC index Out of range, index %d", ioc);
2343 		(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
2344 		return;
2345 	}
2346 	ioc_info = &gid_info->gl_iou->iou_ioc_info[ioc_no - 1];
2347 	ioc_info->ioc_iou_info = gid_info->gl_iou;
2348 
2349 	mutex_enter(&gid_info->gl_mutex);
2350 	if (ioc_info->ioc_state == IBDM_IOC_STATE_REPROBE_PROGRESS) {
2351 		reprobe = 1;
2352 		ioc_info->ioc_prev_serv = ioc_info->ioc_serv;
2353 		ioc_info->ioc_serv = NULL;
2354 		ioc_info->ioc_prev_serv_cnt =
2355 		    ioc_info->ioc_profile.ioc_service_entries;
2356 	} else if (ioc_info->ioc_state != IBDM_IOC_STATE_PROBE_INVALID) {
2357 		IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: DUP response"
2358 		    "ioc %d, ioc_state %x", ioc_no - 1, ioc_info->ioc_state);
2359 		mutex_exit(&gid_info->gl_mutex);
2360 		(*flag) |= IBDM_IBMF_PKT_DUP_RESP;
2361 		return;
2362 	}
2363 	ioc_info->ioc_cb_args.cb_req_type = 0;
2364 	if (ioc_info->ioc_timeout_id) {
2365 		timeout_id = ioc_info->ioc_timeout_id;
2366 		mutex_exit(&gid_info->gl_mutex);
2367 		IBTF_DPRINTF_L5("ibdm", "handle_ioc_profile: "
2368 		    "ioc_timeout_id = 0x%x", timeout_id);
2369 		if (untimeout(timeout_id) == -1) {
2370 			IBTF_DPRINTF_L2("ibdm", "handle_ioc_profile: "
2371 			    "untimeout ioc_timeout_id failed");
2372 		}
2373 		mutex_enter(&gid_info->gl_mutex);
2374 		ioc_info->ioc_timeout_id = 0;
2375 	}
2376 
2377 	ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_SUCCESS;
2378 	if (reprobe == 0) {
2379 		ioc_info->ioc_iou_guid = gid_info->gl_nodeguid;
2380 		ioc_info->ioc_nodeguid = gid_info->gl_nodeguid;
2381 	}
2382 
2383 	/*
2384 	 * Save all the IOC information in the global structures.
2385 	 * Note the wire format is Big Endian and the Sparc process also
2386 	 * big endian. So, there is no need to convert the data fields
2387 	 * The conversion routines used below are ineffective on Sparc
2388 	 * machines where as they will be effective on little endian
2389 	 * machines such as Intel processors.
2390 	 */
2391 	gioc = (ib_dm_ioc_ctrl_profile_t *)&ioc_info->ioc_profile;
2392 
2393 	/*
2394 	 * Restrict updates to onlyport GIDs and service entries during reprobe
2395 	 */
2396 	if (reprobe == 0) {
2397 		gioc->ioc_guid			= b2h64(ioc->ioc_guid);
2398 		gioc->ioc_vendorid		=
2399 		    ((b2h32(ioc->ioc_vendorid) & IB_DM_VENDORID_MASK)
2400 		    >> IB_DM_VENDORID_SHIFT);
2401 		gioc->ioc_deviceid		= b2h32(ioc->ioc_deviceid);
2402 		gioc->ioc_device_ver		= b2h16(ioc->ioc_device_ver);
2403 		gioc->ioc_subsys_vendorid	=
2404 		    ((b2h32(ioc->ioc_subsys_vendorid) & IB_DM_VENDORID_MASK)
2405 		    >> IB_DM_VENDORID_SHIFT);
2406 		gioc->ioc_subsys_id		= b2h32(ioc->ioc_subsys_id);
2407 		gioc->ioc_io_class		= b2h16(ioc->ioc_io_class);
2408 		gioc->ioc_io_subclass		= b2h16(ioc->ioc_io_subclass);
2409 		gioc->ioc_protocol		= b2h16(ioc->ioc_protocol);
2410 		gioc->ioc_protocol_ver		= b2h16(ioc->ioc_protocol_ver);
2411 		gioc->ioc_send_msg_qdepth	=
2412 		    b2h16(ioc->ioc_send_msg_qdepth);
2413 		gioc->ioc_rdma_read_qdepth	=
2414 		    b2h16(ioc->ioc_rdma_read_qdepth);
2415 		gioc->ioc_send_msg_sz		= b2h32(ioc->ioc_send_msg_sz);
2416 		gioc->ioc_rdma_xfer_sz		= b2h32(ioc->ioc_rdma_xfer_sz);
2417 		gioc->ioc_ctrl_opcap_mask	= ioc->ioc_ctrl_opcap_mask;
2418 		bcopy(ioc->ioc_id_string, gioc->ioc_id_string,
2419 		    IB_DM_IOC_ID_STRING_LEN);
2420 
2421 		ioc_info->ioc_iou_diagcode = gid_info->gl_iou->iou_diagcode;
2422 		ioc_info->ioc_iou_dc_valid = gid_info->gl_iou->iou_dc_valid;
2423 		ioc_info->ioc_diagdeviceid = (IB_DM_IOU_DEVICEID_MASK &
2424 		    gid_info->gl_iou->iou_info.iou_flag) ? B_TRUE : B_FALSE;
2425 
2426 		if (ioc_info->ioc_diagdeviceid == B_TRUE)
2427 			gid_info->gl_pending_cmds++;
2428 	}
2429 	gioc->ioc_service_entries	= ioc->ioc_service_entries;
2430 	gid_info->gl_pending_cmds += (gioc->ioc_service_entries/4);
2431 	if (gioc->ioc_service_entries % 4)
2432 		gid_info->gl_pending_cmds++;
2433 
2434 	mutex_exit(&gid_info->gl_mutex);
2435 
2436 	ibdm_dump_ioc_profile(gioc);
2437 
2438 	if ((ioc_info->ioc_diagdeviceid == B_TRUE) && (reprobe == 0)) {
2439 		if (ibdm_get_diagcode(gid_info, ioc_no) != IBDM_SUCCESS) {
2440 			mutex_enter(&gid_info->gl_mutex);
2441 			gid_info->gl_pending_cmds--;
2442 			mutex_exit(&gid_info->gl_mutex);
2443 		}
2444 	}
2445 	ioc_info->ioc_serv = (ibdm_srvents_info_t *)kmem_zalloc(
2446 	    (gioc->ioc_service_entries * sizeof (ibdm_srvents_info_t)),
2447 	    KM_SLEEP);
2448 
2449 	/*
2450 	 * In one single request, maximum number of requests that can be
2451 	 * obtained is 4. If number of service entries are more than four,
2452 	 * calculate number requests needed and send them parallelly.
2453 	 */
2454 	nserv_entries = ioc->ioc_service_entries;
2455 	ii = 0;
2456 	while (nserv_entries) {
2457 		mutex_enter(&gid_info->gl_mutex);
2458 		ibdm_bump_transactionID(gid_info);
2459 		mutex_exit(&gid_info->gl_mutex);
2460 
2461 		if (first != B_TRUE) {
2462 			if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP,
2463 			    &msg) != IBMF_SUCCESS) {
2464 				continue;
2465 			}
2466 
2467 		}
2468 		ibdm_alloc_send_buffers(msg);
2469 		msg->im_local_addr.ia_local_lid	= gid_info->gl_slid;
2470 		msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
2471 		if (gid_info->gl_redirected == B_TRUE) {
2472 			if (gid_info->gl_redirect_dlid != 0) {
2473 				msg->im_local_addr.ia_remote_lid =
2474 				    gid_info->gl_redirect_dlid;
2475 			}
2476 			msg->im_local_addr.ia_remote_qno =
2477 			    gid_info->gl_redirect_QP;
2478 			msg->im_local_addr.ia_p_key =
2479 			    gid_info->gl_redirect_pkey;
2480 			msg->im_local_addr.ia_q_key =
2481 			    gid_info->gl_redirect_qkey;
2482 		} else {
2483 			msg->im_local_addr.ia_remote_qno = 1;
2484 			msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
2485 			msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
2486 		}
2487 
2488 		hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
2489 		hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
2490 		hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
2491 		hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
2492 		hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
2493 		hdr->Status		= 0;
2494 		hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
2495 		hdr->AttributeID	= h2b16(IB_DM_ATTR_SERVICE_ENTRIES);
2496 
2497 		srv_start = ii * 4;
2498 		cb_args = &ioc_info->ioc_serv[srv_start].se_cb_args;
2499 		cb_args->cb_gid_info	= gid_info;
2500 		cb_args->cb_retry_count	= ibdm_dft_retry_cnt;
2501 		cb_args->cb_req_type	= IBDM_REQ_TYPE_SRVENTS;
2502 		cb_args->cb_srvents_start = srv_start;
2503 		cb_args->cb_ioc_num	= ioc_no - 1;
2504 
2505 		if (nserv_entries >= IBDM_MAX_SERV_ENTRIES_PER_REQ) {
2506 			nserv_entries -= IBDM_MAX_SERV_ENTRIES_PER_REQ;
2507 			cb_args->cb_srvents_end = (cb_args->cb_srvents_start +
2508 			    IBDM_MAX_SERV_ENTRIES_PER_REQ - 1);
2509 		} else {
2510 			cb_args->cb_srvents_end =
2511 			    (cb_args->cb_srvents_start + nserv_entries - 1);
2512 			nserv_entries = 0;
2513 		}
2514 		ibdm_fill_srv_attr_mod(hdr, cb_args);
2515 
2516 		ioc_info->ioc_serv[srv_start].se_timeout_id = timeout(
2517 		    ibdm_pkt_timeout_hdlr, cb_args,
2518 		    IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2519 
2520 		IBTF_DPRINTF_L5("ibdm", "\thandle_ioc_profile:"
2521 		    "timeout %x, ioc %d srv %d",
2522 		    ioc_info->ioc_serv[srv_start].se_timeout_id,
2523 		    ioc_no - 1, srv_start);
2524 
2525 		if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg,
2526 		    NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
2527 			IBTF_DPRINTF_L2("ibdm",
2528 			    "\thandle_ioc_profile: msg send failed");
2529 			ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args);
2530 		}
2531 		(*flag) |= IBDM_IBMF_PKT_REUSED;
2532 		first = B_FALSE;
2533 		ii++;
2534 	}
2535 }
2536 
2537 
2538 /*
2539  * ibdm_handle_srventry_mad()
2540  */
2541 static void
2542 ibdm_handle_srventry_mad(ibmf_msg_t *msg,
2543     ibdm_dp_gidinfo_t *gid_info, int *flag)
2544 {
2545 	uint_t			ii, ioc_no, attrmod;
2546 	uint_t			nentries, start, end;
2547 	timeout_id_t		timeout_id;
2548 	ib_dm_srv_t		*srv_ents;
2549 	ibdm_ioc_info_t		*ioc_info;
2550 	ibdm_srvents_info_t	*gsrv_ents;
2551 
2552 	IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad:"
2553 	    " IBMF msg %p gid info %p", msg, gid_info);
2554 
2555 	srv_ents = IBDM_IN_IBMFMSG2SRVENT(msg);
2556 	/*
2557 	 * Get the start and end index of the service entries
2558 	 * Upper 16 bits identify the IOC
2559 	 * Lower 16 bits specify the range of service entries
2560 	 * 	LSB specifies (Big endian) end of the range
2561 	 * 	MSB specifies (Big endian) start of the range
2562 	 */
2563 	attrmod = IBDM_IN_IBMFMSG_ATTRMOD(msg);
2564 	ioc_no	= ((attrmod >> 16) & IBDM_16_BIT_MASK);
2565 	end	= ((attrmod >> 8) & IBDM_8_BIT_MASK);
2566 	start	= (attrmod & IBDM_8_BIT_MASK);
2567 
2568 	/* Make sure that IOC index is with the valid range */
2569 	if ((ioc_no < 1) |
2570 	    (ioc_no > gid_info->gl_iou->iou_info.iou_num_ctrl_slots)) {
2571 		IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: "
2572 		    "IOC index Out of range, index %d", ioc_no);
2573 		(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
2574 		return;
2575 	}
2576 	ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1));
2577 
2578 	/*
2579 	 * Make sure that the "start" and "end" service indexes are
2580 	 * with in the valid range
2581 	 */
2582 	nentries = ioc_info->ioc_profile.ioc_service_entries;
2583 	if ((start > end) | (start >= nentries) | (end >= nentries)) {
2584 		IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: "
2585 		    "Attr modifier 0x%x, #Serv entries %d", attrmod, nentries);
2586 		(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
2587 		return;
2588 	}
2589 	gsrv_ents = &ioc_info->ioc_serv[start];
2590 	mutex_enter(&gid_info->gl_mutex);
2591 	if (gsrv_ents->se_state != IBDM_SE_INVALID) {
2592 		IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: "
2593 			"already known, ioc %d, srv %d, se_state %x",
2594 		    ioc_no - 1, start, gsrv_ents->se_state);
2595 		mutex_exit(&gid_info->gl_mutex);
2596 		(*flag) |= IBDM_IBMF_PKT_DUP_RESP;
2597 		return;
2598 	}
2599 	ioc_info->ioc_serv[start].se_cb_args.cb_req_type = 0;
2600 	if (ioc_info->ioc_serv[start].se_timeout_id) {
2601 		IBTF_DPRINTF_L2("ibdm",
2602 		    "\thandle_srventry_mad: ioc %d start %d", ioc_no, start);
2603 		timeout_id = ioc_info->ioc_serv[start].se_timeout_id;
2604 		mutex_exit(&gid_info->gl_mutex);
2605 		IBTF_DPRINTF_L5("ibdm", "handle_srverntry_mad: "
2606 		    "se_timeout_id = 0x%x", timeout_id);
2607 		if (untimeout(timeout_id) == -1) {
2608 			IBTF_DPRINTF_L2("ibdm", "handle_srventry_mad: "
2609 			    "untimeout se_timeout_id failed");
2610 		}
2611 		mutex_enter(&gid_info->gl_mutex);
2612 		ioc_info->ioc_serv[start].se_timeout_id = 0;
2613 	}
2614 
2615 	gsrv_ents->se_state = IBDM_SE_VALID;
2616 	mutex_exit(&gid_info->gl_mutex);
2617 	for (ii = start; ii <= end; ii++, srv_ents++, gsrv_ents++) {
2618 		gsrv_ents->se_attr.srv_id = b2h64(srv_ents->srv_id);
2619 		bcopy(srv_ents->srv_name,
2620 		    gsrv_ents->se_attr.srv_name, IB_DM_MAX_SVC_NAME_LEN);
2621 		ibdm_dump_service_entries(&gsrv_ents->se_attr);
2622 	}
2623 }
2624 
2625 
2626 /*
2627  * ibdm_get_diagcode:
2628  *	Send request to get IOU/IOC diag code
2629  *	Returns IBDM_SUCCESS/IBDM_FAILURE
2630  */
2631 static int
2632 ibdm_get_diagcode(ibdm_dp_gidinfo_t *gid_info, int attr)
2633 {
2634 	ibmf_msg_t		*msg;
2635 	ib_mad_hdr_t		*hdr;
2636 	ibdm_ioc_info_t		*ioc;
2637 	ibdm_timeout_cb_args_t	*cb_args;
2638 	timeout_id_t		*timeout_id;
2639 
2640 	IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: gid info %p, attr = %d",
2641 	    gid_info, attr);
2642 
2643 	if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
2644 	    &msg) != IBMF_SUCCESS) {
2645 		IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: pkt alloc fail");
2646 		return (IBDM_FAILURE);
2647 	}
2648 
2649 	ibdm_alloc_send_buffers(msg);
2650 
2651 	mutex_enter(&gid_info->gl_mutex);
2652 	ibdm_bump_transactionID(gid_info);
2653 	mutex_exit(&gid_info->gl_mutex);
2654 
2655 	msg->im_local_addr.ia_local_lid	= gid_info->gl_slid;
2656 	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
2657 	if (gid_info->gl_redirected == B_TRUE) {
2658 		if (gid_info->gl_redirect_dlid != 0) {
2659 			msg->im_local_addr.ia_remote_lid =
2660 			    gid_info->gl_redirect_dlid;
2661 		}
2662 
2663 		msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
2664 		msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
2665 		msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
2666 	} else {
2667 		msg->im_local_addr.ia_remote_qno = 1;
2668 		msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
2669 		msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
2670 	}
2671 
2672 	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
2673 	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
2674 	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
2675 	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
2676 	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
2677 	hdr->Status		= 0;
2678 	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
2679 
2680 	hdr->AttributeID	= h2b16(IB_DM_ATTR_DIAG_CODE);
2681 	hdr->AttributeModifier	= h2b32(attr);
2682 
2683 	if (attr == 0) {
2684 		cb_args = &gid_info->gl_iou_cb_args;
2685 		gid_info->gl_iou->iou_dc_valid = B_FALSE;
2686 		cb_args->cb_ioc_num	= 0;
2687 		cb_args->cb_req_type	= IBDM_REQ_TYPE_IOU_DIAGCODE;
2688 		timeout_id = &gid_info->gl_timeout_id;
2689 	} else {
2690 		ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attr - 1));
2691 		ioc->ioc_dc_valid = B_FALSE;
2692 		cb_args = &ioc->ioc_dc_cb_args;
2693 		cb_args->cb_ioc_num	= attr - 1;
2694 		cb_args->cb_req_type	= IBDM_REQ_TYPE_IOC_DIAGCODE;
2695 		timeout_id = &ioc->ioc_dc_timeout_id;
2696 	}
2697 	cb_args->cb_gid_info	= gid_info;
2698 	cb_args->cb_retry_count	= ibdm_dft_retry_cnt;
2699 	cb_args->cb_srvents_start = 0;
2700 
2701 
2702 	*timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2703 	    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2704 
2705 	IBTF_DPRINTF_L5("ibdm", "\tget_diagcode:"
2706 	    "timeout %x, ioc %d", *timeout_id, cb_args->cb_ioc_num);
2707 
2708 	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
2709 	    msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
2710 		IBTF_DPRINTF_L2("ibdm", "\tget_diagcode: ibmf send failed");
2711 		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
2712 	}
2713 	return (IBDM_SUCCESS);
2714 }
2715 
2716 /*
2717  * ibdm_handle_diagcode:
2718  *	Process the DiagCode MAD response and update local DM
2719  *	data structure.
2720  */
2721 static void
2722 ibdm_handle_diagcode(ibmf_msg_t *ibmf_msg,
2723     ibdm_dp_gidinfo_t *gid_info, int *flag)
2724 {
2725 	uint16_t	attrmod, *diagcode;
2726 	ibdm_iou_info_t	*iou;
2727 	ibdm_ioc_info_t	*ioc;
2728 	timeout_id_t	timeout_id;
2729 	ibdm_timeout_cb_args_t	*cb_args;
2730 
2731 	diagcode = (uint16_t *)ibmf_msg->im_msgbufs_recv.im_bufs_cl_data;
2732 
2733 	mutex_enter(&gid_info->gl_mutex);
2734 	attrmod = IBDM_IN_IBMFMSG_ATTRMOD(ibmf_msg);
2735 	iou = gid_info->gl_iou;
2736 	if (attrmod == 0) {
2737 		if (iou->iou_dc_valid != B_FALSE) {
2738 			(*flag) |= IBDM_IBMF_PKT_DUP_RESP;
2739 			IBTF_DPRINTF_L4("ibdm",
2740 			    "\thandle_diagcode: Duplicate IOU DiagCode");
2741 			mutex_exit(&gid_info->gl_mutex);
2742 			return;
2743 		}
2744 		cb_args = &gid_info->gl_iou_cb_args;
2745 		cb_args->cb_req_type = 0;
2746 		iou->iou_diagcode = b2h16(*diagcode);
2747 		iou->iou_dc_valid = B_TRUE;
2748 		if (gid_info->gl_timeout_id) {
2749 			timeout_id = gid_info->gl_timeout_id;
2750 			mutex_exit(&gid_info->gl_mutex);
2751 			IBTF_DPRINTF_L5("ibdm", "\thandle_diagcode: "
2752 			    "gl_timeout_id = 0x%x", timeout_id);
2753 			if (untimeout(timeout_id) == -1) {
2754 				IBTF_DPRINTF_L2("ibdm", "handle_diagcode: "
2755 				    "untimeout gl_timeout_id failed");
2756 			}
2757 			mutex_enter(&gid_info->gl_mutex);
2758 			gid_info->gl_timeout_id = 0;
2759 		}
2760 	} else {
2761 		ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod - 1));
2762 		if (ioc->ioc_dc_valid != B_FALSE) {
2763 			(*flag) |= IBDM_IBMF_PKT_DUP_RESP;
2764 			IBTF_DPRINTF_L4("ibdm",
2765 			    "\thandle_diagcode: Duplicate IOC DiagCode");
2766 			mutex_exit(&gid_info->gl_mutex);
2767 			return;
2768 		}
2769 		cb_args = &ioc->ioc_dc_cb_args;
2770 		cb_args->cb_req_type = 0;
2771 		ioc->ioc_diagcode = b2h16(*diagcode);
2772 		ioc->ioc_dc_valid = B_TRUE;
2773 		timeout_id = iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id;
2774 		if (timeout_id) {
2775 			mutex_exit(&gid_info->gl_mutex);
2776 			IBTF_DPRINTF_L5("ibdm", "handle_diagcode: "
2777 			    "timeout_id = 0x%x", timeout_id);
2778 			if (untimeout(timeout_id) == -1) {
2779 				IBTF_DPRINTF_L2("ibdm", "\thandle_diagcode: "
2780 				    "untimeout ioc_dc_timeout_id failed");
2781 			}
2782 			mutex_enter(&gid_info->gl_mutex);
2783 			iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id = 0;
2784 		}
2785 	}
2786 	mutex_exit(&gid_info->gl_mutex);
2787 
2788 	IBTF_DPRINTF_L4("ibdm", "\thandle_diagcode: DiagCode : 0x%x"
2789 	    "attrmod : 0x%x", b2h16(*diagcode), attrmod);
2790 }
2791 
2792 
2793 /*
2794  * ibdm_is_ioc_present()
2795  *	Return ibdm_ioc_info_t if IOC guid is found in the global gid list
2796  */
2797 static ibdm_ioc_info_t *
2798 ibdm_is_ioc_present(ib_guid_t ioc_guid,
2799     ibdm_dp_gidinfo_t *gid_info, int *flag)
2800 {
2801 	int				ii;
2802 	ibdm_ioc_info_t			*ioc;
2803 	ibdm_dp_gidinfo_t		*head;
2804 	ib_dm_io_unitinfo_t		*iou;
2805 
2806 	mutex_enter(&ibdm.ibdm_mutex);
2807 	head = ibdm.ibdm_dp_gidlist_head;
2808 	while (head) {
2809 		mutex_enter(&head->gl_mutex);
2810 		if (head->gl_iou == NULL) {
2811 			mutex_exit(&head->gl_mutex);
2812 			head = head->gl_next;
2813 			continue;
2814 		}
2815 		iou = &head->gl_iou->iou_info;
2816 		for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) {
2817 			ioc = IBDM_GIDINFO2IOCINFO(head, ii);
2818 			if ((ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) &&
2819 			    (ioc->ioc_profile.ioc_guid == ioc_guid)) {
2820 				if (gid_info == head) {
2821 					*flag |= IBDM_IBMF_PKT_DUP_RESP;
2822 				} else if (ibdm_check_dgid(head->gl_dgid_lo,
2823 				    head->gl_dgid_hi) != NULL) {
2824 					IBTF_DPRINTF_L4("ibdm", "\tis_ioc_"
2825 					    "present: gid not present");
2826 					ibdm_add_to_gl_gid(gid_info, head);
2827 				}
2828 				mutex_exit(&head->gl_mutex);
2829 				mutex_exit(&ibdm.ibdm_mutex);
2830 				return (ioc);
2831 			}
2832 		}
2833 		mutex_exit(&head->gl_mutex);
2834 		head = head->gl_next;
2835 	}
2836 	mutex_exit(&ibdm.ibdm_mutex);
2837 	return (NULL);
2838 }
2839 
2840 
2841 /*
2842  * ibdm_ibmf_send_cb()
2843  *	IBMF invokes this callback routine after posting the DM MAD to
2844  *	the HCA.
2845  */
2846 /*ARGSUSED*/
2847 static void
2848 ibdm_ibmf_send_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *ibmf_msg, void *arg)
2849 {
2850 	ibdm_dump_ibmf_msg(ibmf_msg, 1);
2851 	ibdm_free_send_buffers(ibmf_msg);
2852 	if (ibmf_free_msg(ibmf_hdl, &ibmf_msg) != IBMF_SUCCESS) {
2853 		IBTF_DPRINTF_L4("ibdm",
2854 		    "\tibmf_send_cb: IBMF free msg failed");
2855 	}
2856 }
2857 
2858 
2859 /*
2860  * ibdm_ibmf_recv_cb()
2861  *	Invoked by the IBMF when a response to the one of the DM requests
2862  *	is received.
2863  */
2864 /*ARGSUSED*/
2865 static void
2866 ibdm_ibmf_recv_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg)
2867 {
2868 	ibdm_taskq_args_t	*taskq_args;
2869 
2870 	/*
2871 	 * If the taskq enable is set then dispatch a taskq to process
2872 	 * the MAD, otherwise just process it on this thread
2873 	 */
2874 	if (ibdm_taskq_enable != IBDM_ENABLE_TASKQ_HANDLING) {
2875 		ibdm_process_incoming_mad(ibmf_hdl, msg, arg);
2876 		return;
2877 	}
2878 
2879 	/*
2880 	 * create a taskq and dispatch it to process the incoming MAD
2881 	 */
2882 	taskq_args = kmem_alloc(sizeof (ibdm_taskq_args_t), KM_NOSLEEP);
2883 	if (taskq_args == NULL) {
2884 		IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: kmem_alloc failed for"
2885 		    "taskq_args");
2886 		if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
2887 			IBTF_DPRINTF_L4("ibmf_recv_cb",
2888 			    "\tibmf_recv_cb: IBMF free msg failed");
2889 		}
2890 		return;
2891 	}
2892 	taskq_args->tq_ibmf_handle = ibmf_hdl;
2893 	taskq_args->tq_ibmf_msg = msg;
2894 	taskq_args->tq_args = arg;
2895 
2896 	if (taskq_dispatch(system_taskq, ibdm_recv_incoming_mad, taskq_args,
2897 	    TQ_NOSLEEP) == 0) {
2898 		IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: taskq_dispatch failed");
2899 		if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
2900 			IBTF_DPRINTF_L4("ibmf_recv_cb",
2901 			    "\tibmf_recv_cb: IBMF free msg failed");
2902 		}
2903 		kmem_free(taskq_args, sizeof (ibdm_taskq_args_t));
2904 		return;
2905 	}
2906 
2907 	/* taskq_args are deleted in ibdm_recv_incoming_mad() */
2908 }
2909 
2910 
2911 void
2912 ibdm_recv_incoming_mad(void *args)
2913 {
2914 	ibdm_taskq_args_t	*taskq_args;
2915 
2916 	taskq_args = (ibdm_taskq_args_t *)args;
2917 
2918 	IBTF_DPRINTF_L4("ibdm", "\tibdm_recv_incoming_mad: "
2919 	    "Processing incoming MAD via taskq");
2920 
2921 	ibdm_process_incoming_mad(taskq_args->tq_ibmf_handle,
2922 	    taskq_args->tq_ibmf_msg, taskq_args->tq_args);
2923 
2924 	kmem_free(taskq_args, sizeof (ibdm_taskq_args_t));
2925 }
2926 
2927 
2928 /*
2929  * Calls ibdm_process_incoming_mad with all function arguments  extracted
2930  * from args
2931  */
2932 /*ARGSUSED*/
2933 static void
2934 ibdm_process_incoming_mad(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg)
2935 {
2936 	int			flag = 0;
2937 	int			ret;
2938 	uint64_t		transaction_id;
2939 	ib_mad_hdr_t		*hdr;
2940 	ibdm_dp_gidinfo_t	*gid_info = NULL;
2941 
2942 	IBTF_DPRINTF_L4("ibdm",
2943 	    "\tprocess_incoming_mad: ibmf hdl %p pkt %p", ibmf_hdl, msg);
2944 	ibdm_dump_ibmf_msg(msg, 0);
2945 
2946 	/*
2947 	 * IBMF calls this routine for every DM MAD that arrives at this port.
2948 	 * But we handle only the responses for requests we sent. We drop all
2949 	 * the DM packets that does not have response bit set in the MAD
2950 	 * header(this eliminates all the requests sent to this port).
2951 	 * We handle only DM class version 1 MAD's
2952 	 */
2953 	hdr = IBDM_IN_IBMFMSG_MADHDR(msg);
2954 	if (ibdm_verify_mad_status(hdr) != IBDM_SUCCESS) {
2955 		if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
2956 			IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: "
2957 			    "IBMF free msg failed DM request drop it");
2958 		}
2959 		return;
2960 	}
2961 
2962 	transaction_id = b2h64(hdr->TransactionID);
2963 
2964 	mutex_enter(&ibdm.ibdm_mutex);
2965 	gid_info = ibdm.ibdm_dp_gidlist_head;
2966 	while (gid_info) {
2967 		if ((gid_info->gl_transactionID  &
2968 		    IBDM_GID_TRANSACTIONID_MASK) ==
2969 		    (transaction_id & IBDM_GID_TRANSACTIONID_MASK))
2970 			break;
2971 		gid_info = gid_info->gl_next;
2972 	}
2973 	mutex_exit(&ibdm.ibdm_mutex);
2974 
2975 	if (gid_info == NULL) {
2976 		/* Drop the packet */
2977 		IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: transaction ID"
2978 		    " does not match: 0x%llx", transaction_id);
2979 		if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
2980 			IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
2981 			    "IBMF free msg failed DM request drop it");
2982 		}
2983 		return;
2984 	}
2985 
2986 	/* Handle redirection for all the MAD's, except ClassPortInfo */
2987 	if (((IBDM_IN_IBMFMSG_STATUS(msg) & MAD_STATUS_REDIRECT_REQUIRED)) &&
2988 		(IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO)) {
2989 		ret = ibdm_handle_redirection(msg, gid_info, &flag);
2990 		if (ret == IBDM_SUCCESS) {
2991 			return;
2992 		}
2993 	} else {
2994 		uint_t gl_state;
2995 
2996 		mutex_enter(&gid_info->gl_mutex);
2997 		gl_state = gid_info->gl_state;
2998 		mutex_exit(&gid_info->gl_mutex);
2999 
3000 		switch (gl_state) {
3001 		case IBDM_GET_CLASSPORTINFO:
3002 			ibdm_handle_classportinfo(
3003 			    ibmf_hdl, msg, gid_info, &flag);
3004 			break;
3005 
3006 		case IBDM_GET_IOUNITINFO:
3007 			ibdm_handle_iounitinfo(ibmf_hdl, msg, gid_info, &flag);
3008 			break;
3009 
3010 		case IBDM_GET_IOC_DETAILS:
3011 			switch (IBDM_IN_IBMFMSG_ATTR(msg)) {
3012 
3013 			case IB_DM_ATTR_SERVICE_ENTRIES:
3014 				ibdm_handle_srventry_mad(msg, gid_info, &flag);
3015 				break;
3016 
3017 			case IB_DM_ATTR_IOC_CTRL_PROFILE:
3018 				ibdm_handle_ioc_profile(
3019 				    ibmf_hdl, msg, gid_info, &flag);
3020 				break;
3021 
3022 			case IB_DM_ATTR_DIAG_CODE:
3023 				ibdm_handle_diagcode(msg, gid_info, &flag);
3024 				break;
3025 
3026 			default:
3027 				IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
3028 				    "Error state, wrong attribute :-(");
3029 				(void) ibmf_free_msg(ibmf_hdl, &msg);
3030 				return;
3031 			}
3032 			break;
3033 		default:
3034 			IBTF_DPRINTF_L2("ibdm",
3035 			    "process_incoming_mad: Dropping the packet"
3036 			    " gl_state %x", gl_state);
3037 			if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3038 				IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
3039 				    "IBMF free msg failed DM request drop it");
3040 			}
3041 			return;
3042 		}
3043 	}
3044 
3045 	if ((flag & IBDM_IBMF_PKT_DUP_RESP) ||
3046 	    (flag & IBDM_IBMF_PKT_UNEXP_RESP)) {
3047 		IBTF_DPRINTF_L2("ibdm",
3048 		    "\tprocess_incoming_mad:Dup/unexp resp : 0x%x", flag);
3049 		if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3050 			IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
3051 			    "IBMF free msg failed DM request drop it");
3052 		}
3053 		return;
3054 	}
3055 
3056 	mutex_enter(&gid_info->gl_mutex);
3057 	if (gid_info->gl_pending_cmds < 1) {
3058 		IBTF_DPRINTF_L2("ibdm",
3059 		    "\tprocess_incoming_mad: pending commands negative");
3060 	}
3061 	if (--gid_info->gl_pending_cmds) {
3062 		IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: "
3063 		    "gid_info %p pending cmds %d",
3064 		    gid_info, gid_info->gl_pending_cmds);
3065 		mutex_exit(&gid_info->gl_mutex);
3066 	} else {
3067 		IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: Probing DONE");
3068 		gid_info->gl_state = IBDM_GID_PROBING_COMPLETE;
3069 		mutex_exit(&gid_info->gl_mutex);
3070 		ibdm_notify_newgid_iocs(gid_info);
3071 		mutex_enter(&ibdm.ibdm_mutex);
3072 		if (--ibdm.ibdm_ngid_probes_in_progress == 0) {
3073 			IBTF_DPRINTF_L4("ibdm",
3074 			    "\tprocess_incoming_mad: Wakeup");
3075 			ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS;
3076 			cv_broadcast(&ibdm.ibdm_probe_cv);
3077 		}
3078 		mutex_exit(&ibdm.ibdm_mutex);
3079 	}
3080 
3081 	/*
3082 	 * Do not deallocate the IBMF packet if atleast one request
3083 	 * is posted. IBMF packet is reused.
3084 	 */
3085 	if (!(flag & IBDM_IBMF_PKT_REUSED)) {
3086 		if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3087 			IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: "
3088 			    "IBMF free msg failed DM request drop it");
3089 		}
3090 	}
3091 }
3092 
3093 
3094 /*
3095  * ibdm_verify_mad_status()
3096  *	Verifies the MAD status
3097  *	Returns IBDM_SUCCESS if status is correct
3098  *	Returns IBDM_FAILURE for bogus MAD status
3099  */
3100 static int
3101 ibdm_verify_mad_status(ib_mad_hdr_t *hdr)
3102 {
3103 	int	ret = 0;
3104 
3105 	if ((hdr->R_Method != IB_DM_DEVMGT_METHOD_GET_RESP) ||
3106 	    (hdr->ClassVersion != IB_DM_CLASS_VERSION_1)) {
3107 		return (IBDM_FAILURE);
3108 	}
3109 
3110 	if (b2h16(hdr->Status) == 0)
3111 		ret = IBDM_SUCCESS;
3112 	else if ((b2h16(hdr->Status) & 0x1f) == MAD_STATUS_REDIRECT_REQUIRED)
3113 		ret = IBDM_SUCCESS;
3114 	else {
3115 		IBTF_DPRINTF_L4("ibdm",
3116 		    "\tverify_mad_status: Stauts : 0x%x", b2h16(hdr->Status));
3117 		ret = IBDM_FAILURE;
3118 	}
3119 	return (ret);
3120 }
3121 
3122 
3123 
3124 /*
3125  * ibdm_handle_redirection()
3126  *	Returns IBDM_SUCCESS/IBDM_FAILURE
3127  */
3128 static int
3129 ibdm_handle_redirection(ibmf_msg_t *msg,
3130     ibdm_dp_gidinfo_t *gid_info, int *flag)
3131 {
3132 	int			attrmod, ioc_no, start;
3133 	void			*data;
3134 	timeout_id_t		*timeout_id;
3135 	ib_mad_hdr_t		*hdr;
3136 	ibdm_ioc_info_t		*ioc = NULL;
3137 	ibdm_timeout_cb_args_t	*cb_args;
3138 	ibdm_mad_classportinfo_t	*cpi;
3139 
3140 	IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Enter");
3141 	mutex_enter(&gid_info->gl_mutex);
3142 	switch (gid_info->gl_state) {
3143 	case IBDM_GET_IOUNITINFO:
3144 		cb_args		= &gid_info->gl_iou_cb_args;
3145 		timeout_id	= &gid_info->gl_timeout_id;
3146 		break;
3147 
3148 	case IBDM_GET_IOC_DETAILS:
3149 		attrmod	= IBDM_IN_IBMFMSG_ATTRMOD(msg);
3150 		switch (IBDM_IN_IBMFMSG_ATTR(msg)) {
3151 
3152 		case IB_DM_ATTR_DIAG_CODE:
3153 			if (attrmod == 0) {
3154 				cb_args = &gid_info->gl_iou_cb_args;
3155 				timeout_id = &gid_info->gl_timeout_id;
3156 				break;
3157 			}
3158 			if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) {
3159 				IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
3160 				    "IOC# Out of range %d", attrmod);
3161 				(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3162 				mutex_exit(&gid_info->gl_mutex);
3163 				return (IBDM_FAILURE);
3164 			}
3165 			ioc	= IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1));
3166 			cb_args = &ioc->ioc_dc_cb_args;
3167 			timeout_id = &ioc->ioc_dc_timeout_id;
3168 			break;
3169 
3170 		case IB_DM_ATTR_IOC_CTRL_PROFILE:
3171 			if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) {
3172 				IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
3173 				    "IOC# Out of range %d", attrmod);
3174 				(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3175 				mutex_exit(&gid_info->gl_mutex);
3176 				return (IBDM_FAILURE);
3177 			}
3178 			ioc	= IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1));
3179 			cb_args = &ioc->ioc_cb_args;
3180 			timeout_id = &ioc->ioc_timeout_id;
3181 			break;
3182 
3183 		case IB_DM_ATTR_SERVICE_ENTRIES:
3184 			ioc_no	= ((attrmod >> 16) & IBDM_16_BIT_MASK);
3185 			if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) {
3186 				IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
3187 				    "IOC# Out of range %d", ioc_no);
3188 				(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3189 				mutex_exit(&gid_info->gl_mutex);
3190 				return (IBDM_FAILURE);
3191 			}
3192 			start 	= (attrmod & IBDM_8_BIT_MASK);
3193 			ioc	= IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1));
3194 			if (start > ioc->ioc_profile.ioc_service_entries) {
3195 				IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
3196 				    " SE index Out of range %d", start);
3197 				(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3198 				mutex_exit(&gid_info->gl_mutex);
3199 				return (IBDM_FAILURE);
3200 			}
3201 			cb_args = &ioc->ioc_serv[start].se_cb_args;
3202 			timeout_id = &ioc->ioc_serv[start].se_timeout_id;
3203 			break;
3204 
3205 		default:
3206 			/* ERROR State */
3207 			IBTF_DPRINTF_L2("ibdm",
3208 			    "\thandle_redirection: wrong attribute :-(");
3209 			(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3210 			mutex_exit(&gid_info->gl_mutex);
3211 			return (IBDM_FAILURE);
3212 		}
3213 		break;
3214 	default:
3215 		/* ERROR State */
3216 		IBTF_DPRINTF_L2("ibdm",
3217 		    "\thandle_redirection: Error state :-(");
3218 		(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3219 		mutex_exit(&gid_info->gl_mutex);
3220 		return (IBDM_FAILURE);
3221 	}
3222 	if ((*timeout_id) != 0) {
3223 		mutex_exit(&gid_info->gl_mutex);
3224 		if (untimeout(*timeout_id) == -1) {
3225 			IBTF_DPRINTF_L2("ibdm", "\thandle_redirection: "
3226 			    "untimeout failed %x", *timeout_id);
3227 		} else {
3228 			IBTF_DPRINTF_L5("ibdm",
3229 			    "\thandle_redirection: timeout %x", *timeout_id);
3230 		}
3231 		mutex_enter(&gid_info->gl_mutex);
3232 		*timeout_id = 0;
3233 	}
3234 
3235 	data = msg->im_msgbufs_recv.im_bufs_cl_data;
3236 	cpi = (ibdm_mad_classportinfo_t *)data;
3237 
3238 	gid_info->gl_resp_timeout	=
3239 		(b2h32(cpi->RespTimeValue) & 0x1F);
3240 
3241 	gid_info->gl_redirected		= B_TRUE;
3242 	gid_info->gl_redirect_dlid	= b2h16(cpi->RedirectLID);
3243 	gid_info->gl_redirect_QP	= (b2h32(cpi->RedirectQP) & 0xffffff);
3244 	gid_info->gl_redirect_pkey	= b2h16(cpi->RedirectP_Key);
3245 	gid_info->gl_redirect_qkey	= b2h32(cpi->RedirectQ_Key);
3246 	gid_info->gl_redirectGID_hi	= b2h64(cpi->RedirectGID_hi);
3247 	gid_info->gl_redirectGID_lo	= b2h64(cpi->RedirectGID_lo);
3248 
3249 	if (gid_info->gl_redirect_dlid != 0) {
3250 		msg->im_local_addr.ia_remote_lid =
3251 				gid_info->gl_redirect_dlid;
3252 	}
3253 	ibdm_bump_transactionID(gid_info);
3254 	mutex_exit(&gid_info->gl_mutex);
3255 
3256 	ibdm_alloc_send_buffers(msg);
3257 
3258 	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
3259 	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
3260 	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
3261 	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
3262 	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
3263 	hdr->Status		= 0;
3264 	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
3265 	hdr->AttributeID	=
3266 		msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID;
3267 	hdr->AttributeModifier	=
3268 		msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeModifier;
3269 
3270 	msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
3271 	msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
3272 	msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
3273 
3274 	*timeout_id = timeout(ibdm_pkt_timeout_hdlr,
3275 	    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
3276 
3277 	IBTF_DPRINTF_L5("ibdm", "\thandle_redirect:"
3278 	    "timeout %x", *timeout_id);
3279 
3280 	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
3281 	    msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
3282 		IBTF_DPRINTF_L4("ibdm", "\thandle_redirection:"
3283 		    "message transport failed");
3284 		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
3285 	}
3286 	(*flag) |= IBDM_IBMF_PKT_REUSED;
3287 	IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Exit");
3288 	return (IBDM_SUCCESS);
3289 }
3290 
3291 
3292 /*
3293  * ibdm_pkt_timeout_hdlr
3294  *	This  timeout  handler is  registed for every  IBMF  packet that is
3295  *	sent through the IBMF.  It gets called when no response is received
3296  *	within the specified time for the packet. No retries for the failed
3297  *	commands  currently.  Drops the failed  IBMF packet and  update the
3298  *	pending list commands.
3299  */
3300 static void
3301 ibdm_pkt_timeout_hdlr(void *arg)
3302 {
3303 	int			probe_done = B_FALSE;
3304 	ibdm_iou_info_t		*iou;
3305 	ibdm_ioc_info_t		*ioc;
3306 	ibdm_timeout_cb_args_t	*cb_args = arg;
3307 	ibdm_dp_gidinfo_t	*gid_info;
3308 	int			srv_ent;
3309 	uint_t			new_gl_state;
3310 
3311 	IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: gid_info: %p "
3312 	    "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
3313 	    cb_args->cb_req_type, cb_args->cb_ioc_num,
3314 	    cb_args->cb_srvents_start);
3315 
3316 	gid_info = cb_args->cb_gid_info;
3317 	mutex_enter(&gid_info->gl_mutex);
3318 
3319 	if ((gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) ||
3320 	    (cb_args->cb_req_type == 0)) {
3321 
3322 		IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: req completed"
3323 		    "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_req_type,
3324 		    cb_args->cb_ioc_num, cb_args->cb_srvents_start);
3325 
3326 		if (gid_info->gl_timeout_id)
3327 			gid_info->gl_timeout_id = 0;
3328 		mutex_exit(&gid_info->gl_mutex);
3329 		return;
3330 	}
3331 	if (cb_args->cb_retry_count) {
3332 		cb_args->cb_retry_count--;
3333 		if (ibdm_retry_command(cb_args) == IBDM_SUCCESS) {
3334 			if (gid_info->gl_timeout_id)
3335 				gid_info->gl_timeout_id = 0;
3336 			mutex_exit(&gid_info->gl_mutex);
3337 			return;
3338 		}
3339 		cb_args->cb_retry_count = 0;
3340 	}
3341 
3342 	IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: command failed: gid %p"
3343 	    " rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
3344 	    cb_args->cb_req_type, cb_args->cb_ioc_num,
3345 	    cb_args->cb_srvents_start);
3346 
3347 	new_gl_state = IBDM_GID_PROBING_COMPLETE;
3348 	switch (cb_args->cb_req_type) {
3349 
3350 	case IBDM_REQ_TYPE_CLASSPORTINFO:
3351 	case IBDM_REQ_TYPE_IOUINFO:
3352 		new_gl_state = IBDM_GID_PROBING_FAILED;
3353 		if (--gid_info->gl_pending_cmds == 0)
3354 			probe_done = B_TRUE;
3355 		if (gid_info->gl_timeout_id)
3356 			gid_info->gl_timeout_id = 0;
3357 		mutex_exit(&gid_info->gl_mutex);
3358 		ibdm_delete_glhca_list(gid_info);
3359 		mutex_enter(&gid_info->gl_mutex);
3360 		break;
3361 	case IBDM_REQ_TYPE_IOCINFO:
3362 		iou = gid_info->gl_iou;
3363 		ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num];
3364 		ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED;
3365 		if (--gid_info->gl_pending_cmds == 0)
3366 			probe_done = B_TRUE;
3367 #ifndef __lock_lint
3368 		if (ioc->ioc_timeout_id)
3369 			ioc->ioc_timeout_id = 0;
3370 #endif
3371 		break;
3372 	case IBDM_REQ_TYPE_SRVENTS:
3373 		iou = gid_info->gl_iou;
3374 		ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num];
3375 		ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED;
3376 		if (--gid_info->gl_pending_cmds == 0)
3377 			probe_done = B_TRUE;
3378 		srv_ent = cb_args->cb_srvents_start;
3379 #ifndef __lock_lint
3380 		if (ioc->ioc_serv[srv_ent].se_timeout_id)
3381 			ioc->ioc_serv[srv_ent].se_timeout_id = 0;
3382 #endif
3383 		break;
3384 	case IBDM_REQ_TYPE_IOU_DIAGCODE:
3385 		iou = gid_info->gl_iou;
3386 		iou->iou_dc_valid = B_FALSE;
3387 		if (--gid_info->gl_pending_cmds == 0)
3388 			probe_done = B_TRUE;
3389 		if (gid_info->gl_timeout_id)
3390 			gid_info->gl_timeout_id = 0;
3391 		break;
3392 	case IBDM_REQ_TYPE_IOC_DIAGCODE:
3393 		iou = gid_info->gl_iou;
3394 		ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num];
3395 		ioc->ioc_dc_valid = B_FALSE;
3396 		if (--gid_info->gl_pending_cmds == 0)
3397 			probe_done = B_TRUE;
3398 #ifndef __lock_lint
3399 		if (ioc->ioc_dc_timeout_id)
3400 			ioc->ioc_dc_timeout_id = 0;
3401 #endif
3402 		break;
3403 	}
3404 	if (probe_done == B_TRUE) {
3405 		gid_info->gl_state = new_gl_state;
3406 		mutex_exit(&gid_info->gl_mutex);
3407 		ibdm_notify_newgid_iocs(gid_info);
3408 		mutex_enter(&ibdm.ibdm_mutex);
3409 		if (--ibdm.ibdm_ngid_probes_in_progress == 0) {
3410 			IBTF_DPRINTF_L4("ibdm", "\tpkt_timeout_hdlr: Wakeup");
3411 			ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS;
3412 			cv_broadcast(&ibdm.ibdm_probe_cv);
3413 		}
3414 		mutex_exit(&ibdm.ibdm_mutex);
3415 	} else
3416 		mutex_exit(&gid_info->gl_mutex);
3417 }
3418 
3419 
3420 /*
3421  * ibdm_retry_command()
3422  *	Retries the failed command.
3423  *	Returns IBDM_FAILURE/IBDM_SUCCESS
3424  */
3425 static int
3426 ibdm_retry_command(ibdm_timeout_cb_args_t *cb_args)
3427 {
3428 	int			ret, rval = IBDM_SUCCESS;
3429 	ibmf_msg_t		*msg;
3430 	ib_mad_hdr_t		*hdr;
3431 	ibdm_dp_gidinfo_t	*gid_info = cb_args->cb_gid_info;
3432 	timeout_id_t		*timeout_id;
3433 	ibdm_ioc_info_t		*ioc;
3434 	int			ioc_no;
3435 
3436 	IBTF_DPRINTF_L2("ibdm", "\tretry_command: gid_info: %p "
3437 	    "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
3438 	    cb_args->cb_req_type, cb_args->cb_ioc_num,
3439 	    cb_args->cb_srvents_start);
3440 
3441 	ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP, &msg);
3442 
3443 
3444 	/*
3445 	 * Reset the gid if alloc_msg failed with BAD_HANDLE
3446 	 * ibdm_reset_gidinfo reinits the gid_info
3447 	 */
3448 	if (ret == IBMF_BAD_HANDLE) {
3449 		IBTF_DPRINTF_L3(ibdm_string, "\tretry_command: gid %p hdl bad",
3450 		    gid_info);
3451 
3452 		mutex_exit(&gid_info->gl_mutex);
3453 		ibdm_reset_gidinfo(gid_info);
3454 		mutex_enter(&gid_info->gl_mutex);
3455 
3456 		/* Retry alloc */
3457 		ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP,
3458 		    &msg);
3459 	}
3460 
3461 	if (ret != IBDM_SUCCESS) {
3462 		IBTF_DPRINTF_L2("ibdm", "\tretry_command: alloc failed: %p "
3463 		    "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
3464 		    cb_args->cb_req_type, cb_args->cb_ioc_num,
3465 		    cb_args->cb_srvents_start);
3466 		return (IBDM_FAILURE);
3467 	}
3468 
3469 	ibdm_alloc_send_buffers(msg);
3470 
3471 	ibdm_bump_transactionID(gid_info);
3472 
3473 	msg->im_local_addr.ia_local_lid	= gid_info->gl_slid;
3474 	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
3475 	if (gid_info->gl_redirected == B_TRUE) {
3476 		if (gid_info->gl_redirect_dlid != 0) {
3477 			msg->im_local_addr.ia_remote_lid =
3478 					gid_info->gl_redirect_dlid;
3479 		}
3480 		msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
3481 		msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
3482 		msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
3483 	} else {
3484 		msg->im_local_addr.ia_remote_qno = 1;
3485 		msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
3486 		msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
3487 	}
3488 	hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
3489 	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
3490 	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
3491 	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
3492 	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
3493 	hdr->Status		= 0;
3494 	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
3495 
3496 	switch (cb_args->cb_req_type) {
3497 	case IBDM_REQ_TYPE_CLASSPORTINFO:
3498 		hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO);
3499 		hdr->AttributeModifier = 0;
3500 		timeout_id = &gid_info->gl_timeout_id;
3501 		break;
3502 	case IBDM_REQ_TYPE_IOUINFO:
3503 		hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO);
3504 		hdr->AttributeModifier = 0;
3505 		timeout_id = &gid_info->gl_timeout_id;
3506 		break;
3507 	case IBDM_REQ_TYPE_IOCINFO:
3508 		hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE);
3509 		hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1);
3510 		ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num);
3511 		timeout_id = &ioc->ioc_timeout_id;
3512 		break;
3513 	case IBDM_REQ_TYPE_SRVENTS:
3514 		hdr->AttributeID = h2b16(IB_DM_ATTR_SERVICE_ENTRIES);
3515 		ibdm_fill_srv_attr_mod(hdr, cb_args);
3516 		ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num);
3517 		timeout_id =
3518 		    &ioc->ioc_serv[cb_args->cb_srvents_start].se_timeout_id;
3519 		break;
3520 	case IBDM_REQ_TYPE_IOU_DIAGCODE:
3521 		hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE);
3522 		hdr->AttributeModifier = 0;
3523 		timeout_id = &gid_info->gl_timeout_id;
3524 		break;
3525 	case IBDM_REQ_TYPE_IOC_DIAGCODE:
3526 		hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE);
3527 		hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1);
3528 		ioc_no = cb_args->cb_ioc_num;
3529 		ioc = &gid_info->gl_iou->iou_ioc_info[ioc_no];
3530 		timeout_id = &ioc->ioc_dc_timeout_id;
3531 		break;
3532 	}
3533 
3534 	mutex_exit(&gid_info->gl_mutex);
3535 
3536 	*timeout_id = timeout(ibdm_pkt_timeout_hdlr,
3537 	    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
3538 
3539 	IBTF_DPRINTF_L5("ibdm", "\tretry_command: %p,%x,%d,%d:"
3540 	    "timeout %x", cb_args->cb_req_type, cb_args->cb_ioc_num,
3541 	    cb_args->cb_srvents_start, *timeout_id);
3542 
3543 	if ((rval = ibmf_msg_transport(gid_info->gl_ibmf_hdl,
3544 	    gid_info->gl_qp_hdl, msg, NULL, ibdm_ibmf_send_cb,
3545 	    cb_args, 0)) != IBMF_SUCCESS) {
3546 		IBTF_DPRINTF_L2("ibdm", "\tretry_command: send failed: %p "
3547 		    "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
3548 		    cb_args->cb_req_type, cb_args->cb_ioc_num,
3549 		    cb_args->cb_srvents_start);
3550 		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
3551 		rval = IBDM_FAILURE;
3552 	}
3553 	mutex_enter(&gid_info->gl_mutex);
3554 	return (rval);
3555 }
3556 
3557 
3558 /*
3559  * ibdm_update_ioc_port_gidlist()
3560  */
3561 static void
3562 ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *dest,
3563     ibdm_dp_gidinfo_t *gid_info)
3564 {
3565 	int		ii, ngid_ents;
3566 	ibdm_gid_t	*tmp;
3567 	ibdm_hca_list_t	*gid_hca_head, *temp;
3568 	ibdm_hca_list_t	*ioc_head = NULL;
3569 
3570 	IBTF_DPRINTF_L5("ibdm", "\tupdate_ioc_port_gidlist: Enter");
3571 
3572 	ngid_ents = gid_info->gl_ngids;
3573 	dest->ioc_nportgids = ngid_ents;
3574 	dest->ioc_gid_list = kmem_zalloc(sizeof (ibdm_gid_t) *
3575 	    ngid_ents, KM_SLEEP);
3576 	tmp = gid_info->gl_gid;
3577 	for (ii = 0; (ii < ngid_ents) && (tmp); ii++) {
3578 		dest->ioc_gid_list[ii].gid_dgid_hi = tmp->gid_dgid_hi;
3579 		dest->ioc_gid_list[ii].gid_dgid_lo = tmp->gid_dgid_lo;
3580 		tmp = tmp->gid_next;
3581 	}
3582 
3583 	gid_hca_head = gid_info->gl_hca_list;
3584 	while (gid_hca_head) {
3585 		temp = ibdm_dup_hca_attr(gid_hca_head);
3586 		temp->hl_next = ioc_head;
3587 		ioc_head = temp;
3588 		gid_hca_head = gid_hca_head->hl_next;
3589 	}
3590 	dest->ioc_hca_list = ioc_head;
3591 }
3592 
3593 
3594 /*
3595  * ibdm_alloc_send_buffers()
3596  *	Allocates memory for the IBMF send buffer
3597  */
3598 static void
3599 ibdm_alloc_send_buffers(ibmf_msg_t *msgp)
3600 {
3601 	msgp->im_msgbufs_send.im_bufs_mad_hdr =
3602 	    kmem_zalloc(IBDM_MAD_SIZE, KM_SLEEP);
3603 	msgp->im_msgbufs_send.im_bufs_cl_data = (uchar_t *)
3604 	    msgp->im_msgbufs_send.im_bufs_mad_hdr + sizeof (ib_mad_hdr_t);
3605 	msgp->im_msgbufs_send.im_bufs_cl_data_len =
3606 	    IBDM_MAD_SIZE - sizeof (ib_mad_hdr_t);
3607 }
3608 
3609 
3610 /*
3611  * ibdm_alloc_send_buffers()
3612  *	De-allocates memory for the IBMF send buffer
3613  */
3614 static void
3615 ibdm_free_send_buffers(ibmf_msg_t *msgp)
3616 {
3617 	if (msgp->im_msgbufs_send.im_bufs_mad_hdr != NULL)
3618 		kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr, IBDM_MAD_SIZE);
3619 }
3620 
3621 /*
3622  * ibdm_probe_ioc()
3623  *  	1. Gets the node records for the port GUID. This detects all the port
3624  *  		to the IOU.
3625  *	2. Selectively probes all the IOC, given it's node GUID
3626  *	3. In case of reprobe, only the IOC to be reprobed is send the IOC
3627  *		Controller Profile asynchronously
3628  */
3629 /*ARGSUSED*/
3630 static void
3631 ibdm_probe_ioc(ib_guid_t nodeguid, ib_guid_t ioc_guid, int reprobe_flag)
3632 {
3633 	int			ii, nrecords;
3634 	size_t			nr_len = 0, pi_len = 0;
3635 	ib_gid_t		sgid, dgid;
3636 	ibdm_hca_list_t		*hca_list = NULL;
3637 	sa_node_record_t	*nr, *tmp;
3638 	ibdm_port_attr_t	*port = NULL;
3639 	ibdm_dp_gidinfo_t	*reprobe_gid, *new_gid, *node_gid;
3640 	ibdm_dp_gidinfo_t	*temp_gidinfo;
3641 	ibdm_gid_t		*temp_gid;
3642 	sa_portinfo_record_t	*pi;
3643 
3644 	IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc(%x, %x, %x): Begin",
3645 	    nodeguid, ioc_guid, reprobe_flag);
3646 
3647 	/* Rescan the GID list for any removed GIDs for reprobe */
3648 	if (reprobe_flag)
3649 		ibdm_rescan_gidlist(&ioc_guid);
3650 
3651 	mutex_enter(&ibdm.ibdm_hl_mutex);
3652 	for (ibdm_get_next_port(&hca_list, &port, 1); port;
3653 	    ibdm_get_next_port(&hca_list, &port, 1)) {
3654 		reprobe_gid = new_gid = node_gid = NULL;
3655 
3656 		nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len, nodeguid);
3657 		if (nr == NULL) {
3658 			IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc: no records");
3659 			continue;
3660 		}
3661 		nrecords = (nr_len / sizeof (sa_node_record_t));
3662 		for (tmp = nr, ii = 0;  (ii < nrecords); ii++, tmp++) {
3663 			pi = ibdm_get_portinfo(
3664 			    port->pa_sa_hdl, &pi_len, tmp->LID);
3665 
3666 			if ((pi) && (pi->PortInfo.CapabilityMask &
3667 			    SM_CAP_MASK_IS_DM_SUPPD)) {
3668 				/*
3669 				 * For reprobes: Check if GID, already in
3670 				 * the list. If so, set the state to SKIPPED
3671 				 */
3672 				if (((temp_gidinfo = ibdm_find_gid(nodeguid,
3673 				    tmp->NodeInfo.PortGUID)) != NULL) &&
3674 				    temp_gidinfo->gl_state ==
3675 				    IBDM_GID_PROBING_COMPLETE) {
3676 					ASSERT(reprobe_gid == NULL);
3677 					ibdm_addto_glhcalist(temp_gidinfo,
3678 					    hca_list);
3679 					reprobe_gid = temp_gidinfo;
3680 					kmem_free(pi, pi_len);
3681 					continue;
3682 				} else if (temp_gidinfo != NULL) {
3683 					kmem_free(pi, pi_len);
3684 					ibdm_addto_glhcalist(temp_gidinfo,
3685 					    hca_list);
3686 					continue;
3687 				}
3688 
3689 				IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : "
3690 				    "create_gid : prefix %llx, guid %llx\n",
3691 				    pi->PortInfo.GidPrefix,
3692 				    tmp->NodeInfo.PortGUID);
3693 
3694 				sgid.gid_prefix = port->pa_sn_prefix;
3695 				sgid.gid_guid = port->pa_port_guid;
3696 				dgid.gid_prefix = pi->PortInfo.GidPrefix;
3697 				dgid.gid_guid = tmp->NodeInfo.PortGUID;
3698 				new_gid = ibdm_create_gid_info(port, sgid,
3699 				    dgid);
3700 				if (new_gid == NULL) {
3701 					IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
3702 					    "create_gid_info failed\n");
3703 					kmem_free(pi, pi_len);
3704 					continue;
3705 				}
3706 				if (node_gid == NULL) {
3707 					node_gid = new_gid;
3708 					ibdm_add_to_gl_gid(node_gid, node_gid);
3709 				} else {
3710 					IBTF_DPRINTF_L4("ibdm",
3711 					    "\tprobe_ioc: new gid");
3712 					temp_gid = kmem_zalloc(
3713 					    sizeof (ibdm_gid_t), KM_SLEEP);
3714 					temp_gid->gid_dgid_hi =
3715 					    new_gid->gl_dgid_hi;
3716 					temp_gid->gid_dgid_lo =
3717 					    new_gid->gl_dgid_lo;
3718 					temp_gid->gid_next = node_gid->gl_gid;
3719 					node_gid->gl_gid = temp_gid;
3720 					node_gid->gl_ngids++;
3721 				}
3722 				new_gid->gl_nodeguid = nodeguid;
3723 				new_gid->gl_portguid = dgid.gid_guid;
3724 				ibdm_addto_glhcalist(new_gid, hca_list);
3725 
3726 				/*
3727 				 * Set the state to skipped as all these
3728 				 * gids point to the same node.
3729 				 * We (re)probe only one GID below and reset
3730 				 * state appropriately
3731 				 */
3732 				new_gid->gl_state = IBDM_GID_PROBING_SKIPPED;
3733 				kmem_free(pi, pi_len);
3734 			}
3735 		}
3736 		kmem_free(nr, nr_len);
3737 
3738 		IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : reprobe_flag %d "
3739 		    "reprobe_gid %p new_gid %p node_gid %p",
3740 		    reprobe_flag, reprobe_gid, new_gid, node_gid);
3741 
3742 		if (reprobe_flag != 0 && reprobe_gid != NULL) {
3743 			int	niocs, jj;
3744 			ibdm_ioc_info_t *tmp_ioc;
3745 			int ioc_matched = 0;
3746 
3747 			mutex_exit(&ibdm.ibdm_hl_mutex);
3748 			mutex_enter(&reprobe_gid->gl_mutex);
3749 			reprobe_gid->gl_state = IBDM_GET_IOC_DETAILS;
3750 			niocs =
3751 			    reprobe_gid->gl_iou->iou_info.iou_num_ctrl_slots;
3752 			reprobe_gid->gl_pending_cmds++;
3753 			mutex_exit(&reprobe_gid->gl_mutex);
3754 
3755 			for (jj = 0; jj < niocs; jj++) {
3756 				tmp_ioc =
3757 				    IBDM_GIDINFO2IOCINFO(reprobe_gid, jj);
3758 				if (tmp_ioc->ioc_profile.ioc_guid != ioc_guid)
3759 					continue;
3760 
3761 				ioc_matched = 1;
3762 
3763 				/*
3764 				 * Explicitly set gl_reprobe_flag to 0 so that
3765 				 * IBnex is not notified on completion
3766 				 */
3767 				mutex_enter(&reprobe_gid->gl_mutex);
3768 				reprobe_gid->gl_reprobe_flag = 0;
3769 				mutex_exit(&reprobe_gid->gl_mutex);
3770 
3771 				mutex_enter(&ibdm.ibdm_mutex);
3772 				ibdm.ibdm_ngid_probes_in_progress++;
3773 				mutex_exit(&ibdm.ibdm_mutex);
3774 				if (ibdm_send_ioc_profile(reprobe_gid, jj) !=
3775 				    IBDM_SUCCESS) {
3776 					IBTF_DPRINTF_L4("ibdm",
3777 					    "\tprobe_ioc: "
3778 					    "send_ioc_profile failed "
3779 					    "for ioc %d", jj);
3780 					ibdm_gid_decr_pending(reprobe_gid);
3781 					break;
3782 				}
3783 				mutex_enter(&ibdm.ibdm_mutex);
3784 				ibdm_wait_probe_completion();
3785 				mutex_exit(&ibdm.ibdm_mutex);
3786 				break;
3787 			}
3788 			if (ioc_matched == 0)
3789 				ibdm_gid_decr_pending(reprobe_gid);
3790 			else {
3791 				mutex_enter(&ibdm.ibdm_hl_mutex);
3792 				break;
3793 			}
3794 		} else if (new_gid != NULL) {
3795 			mutex_exit(&ibdm.ibdm_hl_mutex);
3796 			node_gid = node_gid ? node_gid : new_gid;
3797 
3798 			/*
3799 			 * New or reinserted GID : Enable notification
3800 			 * to IBnex
3801 			 */
3802 			mutex_enter(&node_gid->gl_mutex);
3803 			node_gid->gl_reprobe_flag = 1;
3804 			mutex_exit(&node_gid->gl_mutex);
3805 
3806 			ibdm_probe_gid(node_gid);
3807 
3808 			mutex_enter(&ibdm.ibdm_hl_mutex);
3809 		} else {
3810 			IBTF_DPRINTF_L2("ibdm", "\tibdm_probe_ioc "
3811 			    "Invalid state!");
3812 		}
3813 
3814 	}
3815 	mutex_exit(&ibdm.ibdm_hl_mutex);
3816 	IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : End\n");
3817 }
3818 
3819 
3820 /*
3821  * ibdm_probe_gid()
3822  *	Selectively probes the GID
3823  */
3824 static void
3825 ibdm_probe_gid(ibdm_dp_gidinfo_t *gid_info)
3826 {
3827 	IBTF_DPRINTF_L4("ibdm", "\tprobe_gid:");
3828 	mutex_enter(&gid_info->gl_mutex);
3829 	gid_info->gl_pending_cmds++;
3830 	gid_info->gl_state = IBDM_GET_CLASSPORTINFO;
3831 	mutex_exit(&gid_info->gl_mutex);
3832 	if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) {
3833 		mutex_enter(&gid_info->gl_mutex);
3834 		gid_info->gl_state = IBDM_GID_PROBING_FAILED;
3835 		--gid_info->gl_pending_cmds;
3836 		mutex_exit(&gid_info->gl_mutex);
3837 		ibdm_delete_glhca_list(gid_info);
3838 		gid_info = gid_info->gl_next;
3839 		return;
3840 	}
3841 	mutex_enter(&ibdm.ibdm_mutex);
3842 	ibdm.ibdm_ngid_probes_in_progress++;
3843 	gid_info = gid_info->gl_next;
3844 
3845 	ibdm_wait_probe_completion();
3846 	mutex_exit(&ibdm.ibdm_mutex);
3847 	IBTF_DPRINTF_L4("ibdm", "\tprobe_gid: Wakeup signal received");
3848 }
3849 
3850 
3851 /*
3852  * ibdm_create_gid_info()
3853  *	Allocates a gid_info structure and initializes
3854  *	Returns pointer to the structure on success
3855  *	and NULL on failure
3856  */
3857 static ibdm_dp_gidinfo_t *
3858 ibdm_create_gid_info(ibdm_port_attr_t *port, ib_gid_t sgid, ib_gid_t dgid)
3859 {
3860 	uint8_t			ii, npaths;
3861 	sa_path_record_t	*path;
3862 	size_t			len;
3863 	ibdm_pkey_tbl_t		*pkey_tbl;
3864 	ibdm_dp_gidinfo_t	*gid_info = NULL;
3865 	int			ret;
3866 
3867 	IBTF_DPRINTF_L4("ibdm", "\tcreate_gid_info: Begin");
3868 	npaths = 1;
3869 
3870 	/* query for reversible paths */
3871 	if (port->pa_sa_hdl)
3872 		ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl,
3873 		    sgid, dgid, IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0,
3874 		    &len, &path);
3875 	else
3876 		return (NULL);
3877 
3878 	if (ret == IBMF_SUCCESS && path) {
3879 		ibdm_dump_path_info(path);
3880 
3881 		gid_info = kmem_zalloc(
3882 		    sizeof (ibdm_dp_gidinfo_t), KM_SLEEP);
3883 		mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL);
3884 		gid_info->gl_dgid_hi		= path->DGID.gid_prefix;
3885 		gid_info->gl_dgid_lo		= path->DGID.gid_guid;
3886 		gid_info->gl_sgid_hi		= path->SGID.gid_prefix;
3887 		gid_info->gl_sgid_lo		= path->SGID.gid_guid;
3888 		gid_info->gl_p_key		= path->P_Key;
3889 		gid_info->gl_sa_hdl		= port->pa_sa_hdl;
3890 		gid_info->gl_ibmf_hdl		= port->pa_ibmf_hdl;
3891 		gid_info->gl_slid		= path->SLID;
3892 		gid_info->gl_dlid		= path->DLID;
3893 		gid_info->gl_transactionID	= (++ibdm.ibdm_transactionID)
3894 		    << IBDM_GID_TRANSACTIONID_SHIFT;
3895 		gid_info->gl_min_transactionID  = gid_info->gl_transactionID;
3896 		gid_info->gl_max_transactionID  = (ibdm.ibdm_transactionID +1)
3897 		    << IBDM_GID_TRANSACTIONID_SHIFT;
3898 
3899 		gid_info->gl_qp_hdl = IBMF_QP_HANDLE_DEFAULT;
3900 		for (ii = 0; ii < port->pa_npkeys; ii++) {
3901 			if (port->pa_pkey_tbl == NULL)
3902 				break;
3903 
3904 			pkey_tbl = &port->pa_pkey_tbl[ii];
3905 			if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) &&
3906 			    (pkey_tbl->pt_qp_hdl != NULL)) {
3907 				gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl;
3908 				break;
3909 			}
3910 		}
3911 		kmem_free(path, len);
3912 
3913 		/*
3914 		 * QP handle for GID not initialized. No matching Pkey
3915 		 * was found!! ibdm should *not* hit this case. Flag an
3916 		 * error and drop the GID if ibdm does encounter this.
3917 		 */
3918 		if (gid_info->gl_qp_hdl == NULL) {
3919 			IBTF_DPRINTF_L2(ibdm_string,
3920 			    "\tcreate_gid_info: No matching Pkey");
3921 			ibdm_delete_gidinfo(gid_info);
3922 			return (NULL);
3923 		}
3924 
3925 		ibdm.ibdm_ngids++;
3926 		if (ibdm.ibdm_dp_gidlist_head == NULL) {
3927 			ibdm.ibdm_dp_gidlist_head = gid_info;
3928 			ibdm.ibdm_dp_gidlist_tail = gid_info;
3929 		} else {
3930 			ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info;
3931 			gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail;
3932 			ibdm.ibdm_dp_gidlist_tail = gid_info;
3933 		}
3934 	}
3935 
3936 	return (gid_info);
3937 }
3938 
3939 
3940 /*
3941  * ibdm_get_node_records
3942  *	Sends a SA query to get the NODE record
3943  *	Returns pointer to the sa_node_record_t on success
3944  *	and NULL on failure
3945  */
3946 static sa_node_record_t *
3947 ibdm_get_node_records(ibmf_saa_handle_t sa_hdl, size_t *length, ib_guid_t guid)
3948 {
3949 	sa_node_record_t	req, *resp = NULL;
3950 	ibmf_saa_access_args_t	args;
3951 	int			ret;
3952 
3953 	IBTF_DPRINTF_L4("ibdm", "\tget_node_records: Begin");
3954 
3955 	bzero(&req, sizeof (sa_node_record_t));
3956 	req.NodeInfo.NodeGUID = guid;
3957 
3958 	args.sq_attr_id		= SA_NODERECORD_ATTRID;
3959 	args.sq_access_type 	= IBMF_SAA_RETRIEVE;
3960 	args.sq_component_mask 	= SA_NODEINFO_COMPMASK_NODEGUID;
3961 	args.sq_template	= &req;
3962 	args.sq_callback	= NULL;
3963 	args.sq_callback_arg 	= NULL;
3964 
3965 	ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp);
3966 	if (ret != IBMF_SUCCESS) {
3967 		IBTF_DPRINTF_L2("ibdm", "\tget_node_records:"
3968 		    " SA Retrieve Failed: %d", ret);
3969 		return (NULL);
3970 	}
3971 	if ((resp == NULL) || (*length == 0)) {
3972 		IBTF_DPRINTF_L2("ibdm", "\tget_node_records: No records");
3973 		return (NULL);
3974 	}
3975 
3976 	IBTF_DPRINTF_L4("ibdm", "\tget_node_records: NodeGuid %llx "
3977 	    "PortGUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.PortGUID);
3978 
3979 	return (resp);
3980 }
3981 
3982 
3983 /*
3984  * ibdm_get_portinfo()
3985  *	Sends a SA query to get the PortInfo record
3986  *	Returns pointer to the sa_portinfo_record_t on success
3987  *	and NULL on failure
3988  */
3989 static sa_portinfo_record_t *
3990 ibdm_get_portinfo(ibmf_saa_handle_t sa_hdl, size_t *length, ib_lid_t lid)
3991 {
3992 	sa_portinfo_record_t	req, *resp = NULL;
3993 	ibmf_saa_access_args_t	args;
3994 	int			ret;
3995 
3996 	IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: Begin");
3997 
3998 	bzero(&req, sizeof (sa_portinfo_record_t));
3999 	req.EndportLID	= lid;
4000 
4001 	args.sq_attr_id		= SA_PORTINFORECORD_ATTRID;
4002 	args.sq_access_type	= IBMF_SAA_RETRIEVE;
4003 	args.sq_component_mask	= SA_PORTINFO_COMPMASK_PORTLID;
4004 	args.sq_template	= &req;
4005 	args.sq_callback	= NULL;
4006 	args.sq_callback_arg	= NULL;
4007 
4008 	ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp);
4009 	if (ret != IBMF_SUCCESS) {
4010 		IBTF_DPRINTF_L2("ibdm", "\tget_portinfo:"
4011 		    " SA Retrieve Failed: 0x%X", ret);
4012 		return (NULL);
4013 	}
4014 	if ((*length == 0) || (resp == NULL))
4015 		return (NULL);
4016 
4017 	IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: GidPrefix %llx Cap 0x%x",
4018 	    resp->PortInfo.GidPrefix, resp->PortInfo.CapabilityMask);
4019 	return (resp);
4020 }
4021 
4022 
4023 /*
4024  * ibdm_ibnex_register_callback
4025  *	IB nexus callback routine for HCA attach and detach notification
4026  */
4027 void
4028 ibdm_ibnex_register_callback(ibdm_callback_t ibnex_dm_callback)
4029 {
4030 	IBTF_DPRINTF_L4("ibdm", "\tibnex_register_callbacks");
4031 	mutex_enter(&ibdm.ibdm_ibnex_mutex);
4032 	ibdm.ibdm_ibnex_callback = ibnex_dm_callback;
4033 	mutex_exit(&ibdm.ibdm_ibnex_mutex);
4034 }
4035 
4036 
4037 /*
4038  * ibdm_ibnex_unregister_callbacks
4039  */
4040 void
4041 ibdm_ibnex_unregister_callback()
4042 {
4043 	IBTF_DPRINTF_L4("ibdm", "\tibnex_unregister_callbacks");
4044 	mutex_enter(&ibdm.ibdm_ibnex_mutex);
4045 	ibdm.ibdm_ibnex_callback = NULL;
4046 	mutex_exit(&ibdm.ibdm_ibnex_mutex);
4047 }
4048 
4049 
4050 /*
4051  * ibdm_ibnex_get_waittime()
4052  *	Calculates the wait time based on the last HCA attach time
4053  */
4054 time_t
4055 ibdm_ibnex_get_waittime(ib_guid_t hca_guid, int *dft_wait)
4056 {
4057 	int		ii;
4058 	time_t		temp, wait_time = 0;
4059 	ibdm_hca_list_t	*hca;
4060 
4061 	IBTF_DPRINTF_L4("ibdm", "\tibnex_get_waittime hcaguid:%llx"
4062 	    "\tport settling time %d", hca_guid, *dft_wait);
4063 
4064 	mutex_enter(&ibdm.ibdm_hl_mutex);
4065 	hca = ibdm.ibdm_hca_list_head;
4066 
4067 	if (hca_guid) {
4068 		for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4069 			if ((hca_guid == hca->hl_hca_guid) &&
4070 			    (hca->hl_nports != hca->hl_nports_active)) {
4071 				wait_time =
4072 					ddi_get_time() - hca->hl_attach_time;
4073 				wait_time = ((wait_time >= *dft_wait) ?
4074 					0 : (*dft_wait - wait_time));
4075 				break;
4076 			}
4077 			hca = hca->hl_next;
4078 		}
4079 		mutex_exit(&ibdm.ibdm_hl_mutex);
4080 		IBTF_DPRINTF_L4("ibdm", "\tibnex_get_waittime %llx", wait_time);
4081 		return (wait_time);
4082 	}
4083 
4084 	for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4085 		if (hca->hl_nports != hca->hl_nports_active) {
4086 			temp = ddi_get_time() - hca->hl_attach_time;
4087 			temp = ((temp >= *dft_wait) ? 0 : (*dft_wait - temp));
4088 			wait_time = (temp > wait_time) ? temp : wait_time;
4089 		}
4090 	}
4091 	mutex_exit(&ibdm.ibdm_hl_mutex);
4092 	IBTF_DPRINTF_L4("ibdm", "\tibnex_get_waittime %llx", wait_time);
4093 	return (wait_time);
4094 }
4095 
4096 
4097 /*
4098  * ibdm_ibnex_probe_hcaport
4099  *	Probes the presence of HCA port (with HCA dip and port number)
4100  *	Returns port attributes structure on SUCCESS
4101  */
4102 ibdm_port_attr_t *
4103 ibdm_ibnex_probe_hcaport(ib_guid_t hca_guid, uint8_t port_num)
4104 {
4105 	int			ii, jj;
4106 	ibdm_hca_list_t		*hca_list;
4107 	ibdm_port_attr_t	*port_attr;
4108 
4109 	IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_hcaport:");
4110 
4111 	mutex_enter(&ibdm.ibdm_hl_mutex);
4112 	hca_list = ibdm.ibdm_hca_list_head;
4113 	for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4114 		if (hca_list->hl_hca_guid == hca_guid) {
4115 			for (jj = 0; jj < hca_list->hl_nports; jj++) {
4116 				if (hca_list->hl_port_attr[jj].pa_port_num ==
4117 				    port_num) {
4118 					break;
4119 				}
4120 			}
4121 			if (jj != hca_list->hl_nports)
4122 				break;
4123 		}
4124 		hca_list = hca_list->hl_next;
4125 	}
4126 	if (ii == ibdm.ibdm_hca_count) {
4127 		IBTF_DPRINTF_L2("ibdm", "\tibnex_probe_hcaport: not found");
4128 		mutex_exit(&ibdm.ibdm_hl_mutex);
4129 		return (NULL);
4130 	}
4131 	port_attr = (ibdm_port_attr_t *)kmem_zalloc(
4132 	    sizeof (ibdm_port_attr_t), KM_SLEEP);
4133 	bcopy((char *)&hca_list->hl_port_attr[jj],
4134 	    port_attr, sizeof (ibdm_port_attr_t));
4135 	ibdm_update_port_attr(port_attr);
4136 
4137 	mutex_exit(&ibdm.ibdm_hl_mutex);
4138 	return (port_attr);
4139 }
4140 
4141 
4142 /*
4143  * ibdm_ibnex_get_port_attrs
4144  *	Scan all HCAs for a matching port_guid.
4145  *	Returns "port attributes" structure on success.
4146  */
4147 ibdm_port_attr_t *
4148 ibdm_ibnex_get_port_attrs(ib_guid_t port_guid)
4149 {
4150 	int			ii, jj;
4151 	ibdm_hca_list_t		*hca_list;
4152 	ibdm_port_attr_t	*port_attr;
4153 
4154 	IBTF_DPRINTF_L4("ibdm", "\tibnex_get_port_attrs:");
4155 
4156 	mutex_enter(&ibdm.ibdm_hl_mutex);
4157 	hca_list = ibdm.ibdm_hca_list_head;
4158 
4159 	for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4160 		for (jj = 0; jj < hca_list->hl_nports; jj++) {
4161 			if (hca_list->hl_port_attr[jj].pa_port_guid ==
4162 				    port_guid) {
4163 				break;
4164 			}
4165 		}
4166 		if (jj != hca_list->hl_nports)
4167 			break;
4168 		hca_list = hca_list->hl_next;
4169 	}
4170 
4171 	if (ii == ibdm.ibdm_hca_count) {
4172 		IBTF_DPRINTF_L2("ibdm", "\tibnex_get_port_attrs: not found");
4173 		mutex_exit(&ibdm.ibdm_hl_mutex);
4174 		return (NULL);
4175 	}
4176 
4177 	port_attr = (ibdm_port_attr_t *)kmem_alloc(sizeof (ibdm_port_attr_t),
4178 	    KM_SLEEP);
4179 	bcopy((char *)&hca_list->hl_port_attr[jj], port_attr,
4180 	    sizeof (ibdm_port_attr_t));
4181 	ibdm_update_port_attr(port_attr);
4182 
4183 	mutex_exit(&ibdm.ibdm_hl_mutex);
4184 	return (port_attr);
4185 }
4186 
4187 
4188 /*
4189  * ibdm_ibnex_free_port_attr()
4190  */
4191 void
4192 ibdm_ibnex_free_port_attr(ibdm_port_attr_t *port_attr)
4193 {
4194 	IBTF_DPRINTF_L4("ibdm", "\tibnex_free_port_attr:");
4195 	if (port_attr) {
4196 		if (port_attr->pa_pkey_tbl != NULL) {
4197 			kmem_free(port_attr->pa_pkey_tbl,
4198 			    (port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t)));
4199 		}
4200 		kmem_free(port_attr, sizeof (ibdm_port_attr_t));
4201 	}
4202 }
4203 
4204 
4205 /*
4206  * ibdm_ibnex_get_hca_list()
4207  *	Returns portinfo for all the port for all the HCA's
4208  */
4209 void
4210 ibdm_ibnex_get_hca_list(ibdm_hca_list_t **hca, int *count)
4211 {
4212 	ibdm_hca_list_t		*head = NULL, *temp, *temp1;
4213 	int			ii;
4214 
4215 	IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_list:");
4216 
4217 	mutex_enter(&ibdm.ibdm_hl_mutex);
4218 	temp = ibdm.ibdm_hca_list_head;
4219 	for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4220 		temp1 = ibdm_dup_hca_attr(temp);
4221 		temp1->hl_next = head;
4222 		head = temp1;
4223 		temp = temp->hl_next;
4224 	}
4225 	*count = ibdm.ibdm_hca_count;
4226 	*hca = head;
4227 	mutex_exit(&ibdm.ibdm_hl_mutex);
4228 }
4229 
4230 
4231 /*
4232  * ibdm_ibnex_get_hca_info_by_guid()
4233  */
4234 ibdm_hca_list_t	*
4235 ibdm_ibnex_get_hca_info_by_guid(ib_guid_t hca_guid)
4236 {
4237 	ibdm_hca_list_t		*head = NULL, *hca = NULL;
4238 
4239 	IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip");
4240 
4241 	mutex_enter(&ibdm.ibdm_hl_mutex);
4242 	head = ibdm.ibdm_hca_list_head;
4243 	while (head) {
4244 		if (head->hl_hca_guid == hca_guid) {
4245 			hca = ibdm_dup_hca_attr(head);
4246 			hca->hl_next = NULL;
4247 			break;
4248 		}
4249 		head = head->hl_next;
4250 	}
4251 	mutex_exit(&ibdm.ibdm_hl_mutex);
4252 	IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip %p", hca);
4253 	return (hca);
4254 }
4255 
4256 
4257 /*
4258  * ibdm_dup_hca_attr()
4259  *	Allocate a new HCA attribute strucuture and initialize
4260  *	hca attribute structure with the incoming HCA attributes
4261  *	returned the allocated hca attributes.
4262  */
4263 static ibdm_hca_list_t *
4264 ibdm_dup_hca_attr(ibdm_hca_list_t *in_hca)
4265 {
4266 	int			len;
4267 	ibdm_hca_list_t		*out_hca;
4268 
4269 	len = sizeof (ibdm_hca_list_t) +
4270 	    (in_hca->hl_nports * sizeof (ibdm_port_attr_t));
4271 	IBTF_DPRINTF_L4("ibdm", "\tdup_hca_attr len %d", len);
4272 	out_hca = (ibdm_hca_list_t *)kmem_alloc(len, KM_SLEEP);
4273 	bcopy((char *)in_hca,
4274 		(char *)out_hca, sizeof (ibdm_hca_list_t));
4275 	if (in_hca->hl_nports) {
4276 		out_hca->hl_port_attr = (ibdm_port_attr_t *)
4277 			((char *)out_hca + sizeof (ibdm_hca_list_t));
4278 		bcopy((char *)in_hca->hl_port_attr,
4279 			(char *)out_hca->hl_port_attr,
4280 			(in_hca->hl_nports * sizeof (ibdm_port_attr_t)));
4281 		for (len = 0; len < out_hca->hl_nports; len++)
4282 			ibdm_update_port_attr(&out_hca->hl_port_attr[len]);
4283 	}
4284 	return (out_hca);
4285 }
4286 
4287 
4288 /*
4289  * ibdm_ibnex_free_hca_list()
4290  *	Free one/more HCA lists
4291  */
4292 void
4293 ibdm_ibnex_free_hca_list(ibdm_hca_list_t *hca_list)
4294 {
4295 	int			ii, len;
4296 	ibdm_hca_list_t 	*temp;
4297 	ibdm_port_attr_t	*port;
4298 
4299 	IBTF_DPRINTF_L4("ibdm", "\tibnex_free_hca_list:");
4300 	ASSERT(hca_list);
4301 	while (hca_list) {
4302 		temp = hca_list;
4303 		hca_list = hca_list->hl_next;
4304 		for (ii = 0; ii < temp->hl_nports; ii++) {
4305 			port = &temp->hl_port_attr[ii];
4306 			len = (port->pa_npkeys * sizeof (ibdm_pkey_tbl_t));
4307 			if (len != 0)
4308 				kmem_free(port->pa_pkey_tbl, len);
4309 		}
4310 		len = sizeof (ibdm_hca_list_t) + (temp->hl_nports *
4311 			sizeof (ibdm_port_attr_t));
4312 		kmem_free(temp, len);
4313 	}
4314 }
4315 
4316 
4317 /*
4318  * ibdm_ibnex_probe_iocguid()
4319  *	Probes the IOC on the fabric and returns the IOC information
4320  *	if present. Otherwise, NULL is returned
4321  */
4322 /* ARGSUSED */
4323 ibdm_ioc_info_t *
4324 ibdm_ibnex_probe_ioc(ib_guid_t iou, ib_guid_t ioc_guid, int reprobe_flag)
4325 {
4326 	int			k;
4327 	ibdm_ioc_info_t		*ioc_info;
4328 	ibdm_dp_gidinfo_t	*gid_info;
4329 
4330 	IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_ioc: (%llX, %llX, %d) Begin",
4331 	    iou, ioc_guid, reprobe_flag);
4332 	/* Check whether we know this already */
4333 	ioc_info = ibdm_ibnex_get_ioc_info(ioc_guid);
4334 	if (ioc_info == NULL) {
4335 		mutex_enter(&ibdm.ibdm_mutex);
4336 		while (ibdm.ibdm_busy & IBDM_BUSY)
4337 			cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
4338 		ibdm.ibdm_busy |= IBDM_BUSY;
4339 		mutex_exit(&ibdm.ibdm_mutex);
4340 		ibdm_probe_ioc(iou, ioc_guid, 0);
4341 		mutex_enter(&ibdm.ibdm_mutex);
4342 		ibdm.ibdm_busy &= ~IBDM_BUSY;
4343 		cv_broadcast(&ibdm.ibdm_busy_cv);
4344 		mutex_exit(&ibdm.ibdm_mutex);
4345 		ioc_info = ibdm_ibnex_get_ioc_info(ioc_guid);
4346 	} else if (reprobe_flag) {	/* Handle Reprobe for the IOC */
4347 		/* Free the ioc_list before reprobe; and cancel any timers */
4348 		mutex_enter(&ibdm.ibdm_mutex);
4349 		if (ioc_info->ioc_timeout_id) {
4350 			IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: "
4351 			    "ioc_timeout_id = 0x%x",
4352 			    ioc_info->ioc_timeout_id);
4353 			if (untimeout(ioc_info->ioc_timeout_id) == -1) {
4354 				IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
4355 				    "untimeout ioc_timeout_id failed");
4356 			}
4357 			ioc_info->ioc_timeout_id = 0;
4358 		}
4359 		if (ioc_info->ioc_dc_timeout_id) {
4360 			IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: "
4361 			    "ioc_dc_timeout_id = 0x%x",
4362 			    ioc_info->ioc_dc_timeout_id);
4363 			if (untimeout(ioc_info->ioc_dc_timeout_id) == -1) {
4364 				IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
4365 				    "untimeout ioc_dc_timeout_id failed");
4366 			}
4367 			ioc_info->ioc_dc_timeout_id = 0;
4368 		}
4369 		for (k = 0; k < ioc_info->ioc_profile.ioc_service_entries; k++)
4370 			if (ioc_info->ioc_serv[k].se_timeout_id) {
4371 				IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: "
4372 				    "ioc_info->ioc_serv[k].se_timeout_id = %x",
4373 				    k, ioc_info->ioc_serv[k].se_timeout_id);
4374 				if (untimeout(ioc_info->ioc_serv[k].
4375 				    se_timeout_id) == -1) {
4376 					IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
4377 					    "untimeout se_timeout_id %d "
4378 					    "failed", k);
4379 				}
4380 				ioc_info->ioc_serv[k].se_timeout_id = 0;
4381 			}
4382 		mutex_exit(&ibdm.ibdm_mutex);
4383 		ibdm_ibnex_free_ioc_list(ioc_info);
4384 
4385 		mutex_enter(&ibdm.ibdm_mutex);
4386 		while (ibdm.ibdm_busy & IBDM_BUSY)
4387 			cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
4388 		ibdm.ibdm_busy |= IBDM_BUSY;
4389 		mutex_exit(&ibdm.ibdm_mutex);
4390 
4391 		ibdm_probe_ioc(iou, ioc_guid, 1);
4392 
4393 		/*
4394 		 * Skip if gl_reprobe_flag is set, this will be
4395 		 * a re-inserted / new GID, for which notifications
4396 		 * have already been send.
4397 		 */
4398 		for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
4399 		    gid_info = gid_info->gl_next) {
4400 			uint8_t			ii, niocs;
4401 			ibdm_ioc_info_t		*ioc;
4402 
4403 			if (gid_info->gl_iou == NULL)
4404 				continue;
4405 
4406 			if (gid_info->gl_reprobe_flag) {
4407 				gid_info->gl_reprobe_flag = 0;
4408 				continue;
4409 			}
4410 
4411 			niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
4412 			for (ii = 0; ii < niocs; ii++) {
4413 				ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii);
4414 				if (ioc->ioc_profile.ioc_guid == ioc_guid) {
4415 					mutex_enter(&ibdm.ibdm_mutex);
4416 					ibdm_reprobe_update_port_srv(ioc,
4417 					    gid_info);
4418 					mutex_exit(&ibdm.ibdm_mutex);
4419 				}
4420 			}
4421 		}
4422 		mutex_enter(&ibdm.ibdm_mutex);
4423 		ibdm.ibdm_busy &= ~IBDM_BUSY;
4424 		cv_broadcast(&ibdm.ibdm_busy_cv);
4425 		mutex_exit(&ibdm.ibdm_mutex);
4426 
4427 		ioc_info = ibdm_ibnex_get_ioc_info(ioc_guid);
4428 	}
4429 	return (ioc_info);
4430 }
4431 
4432 
4433 /*
4434  * ibdm_ibnex_get_ioc_info()
4435  *	Returns pointer to ibdm_ioc_info_t if it finds
4436  *	matching record for the ioc_guid, otherwise NULL
4437  *	is returned
4438  */
4439 ibdm_ioc_info_t *
4440 ibdm_ibnex_get_ioc_info(ib_guid_t ioc_guid)
4441 {
4442 	int			ii;
4443 	ibdm_ioc_info_t		*ioc = NULL, *tmp = NULL;
4444 	ibdm_dp_gidinfo_t	*gid_list;
4445 	ib_dm_io_unitinfo_t	*iou;
4446 
4447 	IBTF_DPRINTF_L4("ibdm", "\tibnex_get_ioc_info: GUID %llx", ioc_guid);
4448 
4449 	mutex_enter(&ibdm.ibdm_mutex);
4450 	while (ibdm.ibdm_busy & IBDM_BUSY)
4451 		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
4452 	ibdm.ibdm_busy |= IBDM_BUSY;
4453 
4454 	gid_list = ibdm.ibdm_dp_gidlist_head;
4455 	while (gid_list) {
4456 		mutex_enter(&gid_list->gl_mutex);
4457 		if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) {
4458 			mutex_exit(&gid_list->gl_mutex);
4459 			gid_list = gid_list->gl_next;
4460 			continue;
4461 		}
4462 		if (gid_list->gl_iou == NULL) {
4463 			IBTF_DPRINTF_L2("ibdm",
4464 			    "\tget_ioc_info: No IOU info");
4465 			mutex_exit(&gid_list->gl_mutex);
4466 			gid_list = gid_list->gl_next;
4467 			continue;
4468 		}
4469 		iou = &gid_list->gl_iou->iou_info;
4470 		for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) {
4471 			tmp = IBDM_GIDINFO2IOCINFO(gid_list, ii);
4472 			if ((tmp->ioc_profile.ioc_guid == ioc_guid) &&
4473 			    (tmp->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS)) {
4474 				ioc = ibdm_dup_ioc_info(tmp, gid_list);
4475 				mutex_exit(&gid_list->gl_mutex);
4476 				ibdm.ibdm_busy &= ~IBDM_BUSY;
4477 				cv_broadcast(&ibdm.ibdm_busy_cv);
4478 				mutex_exit(&ibdm.ibdm_mutex);
4479 				IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: End");
4480 				return (ioc);
4481 			}
4482 		}
4483 		if (ii == iou->iou_num_ctrl_slots)
4484 			ioc = NULL;
4485 
4486 		mutex_exit(&gid_list->gl_mutex);
4487 		gid_list = gid_list->gl_next;
4488 	}
4489 
4490 	ibdm.ibdm_busy &= ~IBDM_BUSY;
4491 	cv_broadcast(&ibdm.ibdm_busy_cv);
4492 	mutex_exit(&ibdm.ibdm_mutex);
4493 	IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: failure End");
4494 	return (ioc);
4495 }
4496 
4497 
4498 /*
4499  * ibdm_ibnex_get_ioc_count()
4500  *	Returns number of ibdm_ioc_info_t it finds
4501  */
4502 int
4503 ibdm_ibnex_get_ioc_count(void)
4504 {
4505 	int			count = 0, k;
4506 	ibdm_ioc_info_t		*ioc;
4507 	ibdm_dp_gidinfo_t	*gid_list;
4508 
4509 	mutex_enter(&ibdm.ibdm_mutex);
4510 	ibdm_sweep_fabric(0);
4511 
4512 	while (ibdm.ibdm_busy & IBDM_BUSY)
4513 		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
4514 	ibdm.ibdm_busy |= IBDM_BUSY;
4515 
4516 	for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
4517 	    gid_list = gid_list->gl_next) {
4518 		mutex_enter(&gid_list->gl_mutex);
4519 		if ((gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) ||
4520 		    (gid_list->gl_iou == NULL)) {
4521 			mutex_exit(&gid_list->gl_mutex);
4522 			continue;
4523 		}
4524 		for (k = 0; k < gid_list->gl_iou->iou_info.iou_num_ctrl_slots;
4525 		    k++) {
4526 			ioc = IBDM_GIDINFO2IOCINFO(gid_list, k);
4527 			if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS)
4528 				++count;
4529 		}
4530 		mutex_exit(&gid_list->gl_mutex);
4531 	}
4532 	ibdm.ibdm_busy &= ~IBDM_BUSY;
4533 	cv_broadcast(&ibdm.ibdm_busy_cv);
4534 	mutex_exit(&ibdm.ibdm_mutex);
4535 
4536 	IBTF_DPRINTF_L4("ibdm", "\tget_ioc_count: count = %d", count);
4537 	return (count);
4538 }
4539 
4540 
4541 /*
4542  * ibdm_ibnex_get_ioc_list()
4543  *	Returns information about all the IOCs present on the fabric.
4544  *	Reprobes the IOCs and the GID list if list_flag is set to REPROBE_ALL.
4545  *	Does not sweep fabric if DONOT_PROBE is set
4546  */
4547 ibdm_ioc_info_t *
4548 ibdm_ibnex_get_ioc_list(ibdm_ibnex_get_ioclist_mtd_t list_flag)
4549 {
4550 	int			ii;
4551 	ibdm_ioc_info_t		*ioc_list = NULL, *tmp, *ioc;
4552 	ibdm_dp_gidinfo_t	*gid_list;
4553 	ib_dm_io_unitinfo_t	*iou;
4554 
4555 	IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: Enter");
4556 
4557 	mutex_enter(&ibdm.ibdm_mutex);
4558 	if (list_flag != IBDM_IBNEX_DONOT_PROBE)
4559 		ibdm_sweep_fabric(list_flag == IBDM_IBNEX_REPROBE_ALL);
4560 
4561 	while (ibdm.ibdm_busy & IBDM_BUSY)
4562 		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
4563 	ibdm.ibdm_busy |= IBDM_BUSY;
4564 
4565 	gid_list = ibdm.ibdm_dp_gidlist_head;
4566 	while (gid_list) {
4567 		mutex_enter(&gid_list->gl_mutex);
4568 		if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) {
4569 			mutex_exit(&gid_list->gl_mutex);
4570 			gid_list = gid_list->gl_next;
4571 			continue;
4572 		}
4573 		if (gid_list->gl_iou == NULL) {
4574 			IBTF_DPRINTF_L2("ibdm",
4575 			    "\tget_ioc_list: No IOU info");
4576 			mutex_exit(&gid_list->gl_mutex);
4577 			gid_list = gid_list->gl_next;
4578 			continue;
4579 		}
4580 		iou = &gid_list->gl_iou->iou_info;
4581 		for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) {
4582 			ioc = IBDM_GIDINFO2IOCINFO(gid_list, ii);
4583 			if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) {
4584 				tmp = ibdm_dup_ioc_info(ioc, gid_list);
4585 				tmp->ioc_next = ioc_list;
4586 				ioc_list = tmp;
4587 			}
4588 		}
4589 		mutex_exit(&gid_list->gl_mutex);
4590 		gid_list = gid_list->gl_next;
4591 	}
4592 	ibdm.ibdm_busy &= ~IBDM_BUSY;
4593 	cv_broadcast(&ibdm.ibdm_busy_cv);
4594 	mutex_exit(&ibdm.ibdm_mutex);
4595 
4596 	IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: End");
4597 	return (ioc_list);
4598 }
4599 
4600 /*
4601  * ibdm_dup_ioc_info()
4602  *	Duplicate the IOC information and return the IOC
4603  *	information.
4604  */
4605 static ibdm_ioc_info_t *
4606 ibdm_dup_ioc_info(ibdm_ioc_info_t *in_ioc, ibdm_dp_gidinfo_t *gid_list)
4607 {
4608 	ibdm_ioc_info_t	*out_ioc;
4609 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*out_ioc));
4610 
4611 	out_ioc = kmem_alloc(sizeof (ibdm_ioc_info_t), KM_SLEEP);
4612 	bcopy(in_ioc, out_ioc, sizeof (ibdm_ioc_info_t));
4613 	ibdm_update_ioc_port_gidlist(out_ioc, gid_list);
4614 	out_ioc->ioc_iou_dc_valid = gid_list->gl_iou->iou_dc_valid;
4615 	out_ioc->ioc_iou_diagcode = gid_list->gl_iou->iou_diagcode;
4616 
4617 	return (out_ioc);
4618 }
4619 
4620 
4621 /*
4622  * ibdm_free_ioc_list()
4623  *	Deallocate memory for IOC list structure
4624  */
4625 void
4626 ibdm_ibnex_free_ioc_list(ibdm_ioc_info_t *ioc)
4627 {
4628 	ibdm_ioc_info_t *temp;
4629 
4630 	IBTF_DPRINTF_L4("ibdm", "\tibnex_free_ioc_list:");
4631 	while (ioc) {
4632 		temp = ioc;
4633 		ioc = ioc->ioc_next;
4634 		kmem_free(temp->ioc_gid_list,
4635 		    (sizeof (ibdm_gid_t) * temp->ioc_nportgids));
4636 		if (temp->ioc_hca_list)
4637 			ibdm_ibnex_free_hca_list(temp->ioc_hca_list);
4638 		kmem_free(temp, sizeof (ibdm_ioc_info_t));
4639 	}
4640 }
4641 
4642 
4643 /*
4644  * ibdm_ibnex_update_pkey_tbls
4645  *	Updates the DM P_Key database.
4646  *	NOTE: Two cases are handled here: P_Key being added or removed.
4647  *
4648  * Arguments		: NONE
4649  * Return Values	: NONE
4650  */
4651 void
4652 ibdm_ibnex_update_pkey_tbls(void)
4653 {
4654 	int			h, pp, pidx;
4655 	uint_t			nports;
4656 	uint_t			size;
4657 	ib_pkey_t		new_pkey;
4658 	ib_pkey_t		*orig_pkey;
4659 	ibdm_hca_list_t		*hca_list;
4660 	ibdm_port_attr_t	*port;
4661 	ibt_hca_portinfo_t	*pinfop;
4662 
4663 	IBTF_DPRINTF_L4("ibdm", "\tibnex_update_pkey_tbls:");
4664 
4665 	mutex_enter(&ibdm.ibdm_hl_mutex);
4666 	hca_list = ibdm.ibdm_hca_list_head;
4667 
4668 	for (h = 0; h < ibdm.ibdm_hca_count; h++) {
4669 
4670 		/* This updates P_Key Tables for all ports of this HCA */
4671 		(void) ibt_query_hca_ports(hca_list->hl_hca_hdl, 0, &pinfop,
4672 		    &nports, &size);
4673 
4674 		/* number of ports shouldn't have changed */
4675 		ASSERT(nports == hca_list->hl_nports);
4676 
4677 		for (pp = 0; pp < hca_list->hl_nports; pp++) {
4678 			port = &hca_list->hl_port_attr[pp];
4679 
4680 			/*
4681 			 * First figure out the P_Keys from IBTL.
4682 			 * Three things could have happened:
4683 			 *	New P_Keys added
4684 			 *	Existing P_Keys removed
4685 			 *	Both of the above two
4686 			 *
4687 			 * Loop through the P_Key Indices and check if a
4688 			 * give P_Key_Ix matches that of the one seen by
4689 			 * IBDM. If they match no action is needed.
4690 			 *
4691 			 * If they don't match:
4692 			 *	1. if orig_pkey is invalid and new_pkey is valid
4693 			 *		---> add new_pkey to DM database
4694 			 *	2. if orig_pkey is valid and new_pkey is invalid
4695 			 *		---> remove orig_pkey from DM database
4696 			 *	3. if orig_pkey and new_pkey are both valid:
4697 			 *		---> remov orig_pkey from DM database
4698 			 *		---> add new_pkey to DM database
4699 			 *	4. if orig_pkey and new_pkey are both invalid:
4700 			 *		---> do nothing. Updated DM database.
4701 			 */
4702 
4703 			for (pidx = 0; pidx < port->pa_npkeys; pidx++) {
4704 				new_pkey = pinfop[pp].p_pkey_tbl[pidx];
4705 				orig_pkey = &port->pa_pkey_tbl[pidx].pt_pkey;
4706 
4707 				/* keys match - do nothing */
4708 				if (*orig_pkey == new_pkey)
4709 					continue;
4710 
4711 				if (IBDM_INVALID_PKEY(*orig_pkey) &&
4712 				    !IBDM_INVALID_PKEY(new_pkey)) {
4713 					/* P_Key was added */
4714 					IBTF_DPRINTF_L5("ibdm",
4715 					    "\tibnex_update_pkey_tbls: new "
4716 					    "P_Key added = 0x%x", new_pkey);
4717 					*orig_pkey = new_pkey;
4718 					ibdm_port_attr_ibmf_init(port,
4719 					    new_pkey, pp);
4720 				} else if (!IBDM_INVALID_PKEY(*orig_pkey) &&
4721 				    IBDM_INVALID_PKEY(new_pkey)) {
4722 					/* P_Key was removed */
4723 					IBTF_DPRINTF_L5("ibdm",
4724 					    "\tibnex_update_pkey_tbls: P_Key "
4725 					    "removed = 0x%x", *orig_pkey);
4726 					*orig_pkey = new_pkey;
4727 					(void) ibdm_port_attr_ibmf_fini(port,
4728 					    pidx);
4729 				} else if (!IBDM_INVALID_PKEY(*orig_pkey) &&
4730 				    !IBDM_INVALID_PKEY(new_pkey)) {
4731 					/* P_Key were replaced */
4732 					IBTF_DPRINTF_L5("ibdm",
4733 					    "\tibnex_update_pkey_tbls: P_Key "
4734 					    "replaced 0x%x with 0x%x",
4735 					    *orig_pkey, new_pkey);
4736 					(void) ibdm_port_attr_ibmf_fini(port,
4737 					    pidx);
4738 					*orig_pkey = new_pkey;
4739 					ibdm_port_attr_ibmf_init(port,
4740 					    new_pkey, pp);
4741 				} else {
4742 					/*
4743 					 * P_Keys are invalid
4744 					 * set anyway to reflect if
4745 					 * INVALID_FULL was changed to
4746 					 * INVALID_LIMITED or vice-versa.
4747 					 */
4748 					*orig_pkey = new_pkey;
4749 				} /* end of else */
4750 
4751 			} /* loop of p_key index */
4752 
4753 		} /* loop of #ports of HCA */
4754 
4755 		ibt_free_portinfo(pinfop, size);
4756 		hca_list = hca_list->hl_next;
4757 
4758 	} /* loop for all HCAs in the system */
4759 
4760 	mutex_exit(&ibdm.ibdm_hl_mutex);
4761 }
4762 
4763 
4764 /*
4765  * ibdm_send_ioc_profile()
4766  *	Send IOC Controller Profile request. When the request is completed
4767  *	IBMF calls ibdm_process_incoming_mad routine to inform about
4768  *	the completion.
4769  */
4770 static int
4771 ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *gid_info, uint8_t ioc_no)
4772 {
4773 	ibmf_msg_t		*msg;
4774 	ib_mad_hdr_t	*hdr;
4775 	ibdm_ioc_info_t	*ioc_info = &(gid_info->gl_iou->iou_ioc_info[ioc_no]);
4776 	ibdm_timeout_cb_args_t	*cb_args;
4777 
4778 	IBTF_DPRINTF_L4("ibdm", "\tsend_ioc_profile: "
4779 	    "gid info 0x%p, ioc_no = %d", gid_info, ioc_no);
4780 
4781 	/*
4782 	 * Send command to get IOC profile.
4783 	 * Allocate a IBMF packet and initialize the packet.
4784 	 */
4785 	if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
4786 	    &msg) != IBMF_SUCCESS) {
4787 		IBTF_DPRINTF_L4("ibdm", "\tsend_ioc_profile: pkt alloc fail");
4788 		return (IBDM_FAILURE);
4789 	}
4790 
4791 	ibdm_alloc_send_buffers(msg);
4792 
4793 	mutex_enter(&gid_info->gl_mutex);
4794 	ibdm_bump_transactionID(gid_info);
4795 	mutex_exit(&gid_info->gl_mutex);
4796 
4797 	msg->im_local_addr.ia_local_lid	= gid_info->gl_slid;
4798 	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
4799 	if (gid_info->gl_redirected == B_TRUE) {
4800 		if (gid_info->gl_redirect_dlid != 0) {
4801 			msg->im_local_addr.ia_remote_lid =
4802 			    gid_info->gl_redirect_dlid;
4803 		}
4804 		msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
4805 		msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
4806 		msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
4807 	} else {
4808 		msg->im_local_addr.ia_remote_qno = 1;
4809 		msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
4810 		msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
4811 	}
4812 
4813 	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
4814 	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
4815 	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
4816 	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
4817 	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
4818 	hdr->Status		= 0;
4819 	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
4820 	hdr->AttributeID	= h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE);
4821 	hdr->AttributeModifier 	= h2b32(ioc_no + 1);
4822 
4823 	ioc_info->ioc_state	= IBDM_IOC_STATE_REPROBE_PROGRESS;
4824 	cb_args			= &ioc_info->ioc_cb_args;
4825 	cb_args->cb_gid_info	= gid_info;
4826 	cb_args->cb_retry_count	= ibdm_dft_retry_cnt;
4827 	cb_args->cb_req_type	= IBDM_REQ_TYPE_IOCINFO;
4828 	cb_args->cb_ioc_num	= ioc_no;
4829 
4830 	ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
4831 	    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
4832 
4833 	IBTF_DPRINTF_L5("ibdm", "\tsend_ioc_profile:"
4834 	    "timeout %x", ioc_info->ioc_timeout_id);
4835 
4836 	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg,
4837 	    NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
4838 		IBTF_DPRINTF_L2("ibdm",
4839 		    "\tsend_ioc_profile: msg transport failed");
4840 		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
4841 	}
4842 	ioc_info->ioc_state = IBDM_IOC_STATE_REPROBE_PROGRESS;
4843 	return (IBDM_SUCCESS);
4844 }
4845 
4846 
4847 /*
4848  * ibdm_port_reachable
4849  *	Sends a SA query to get the NODE record for port GUID
4850  *	Returns IBDM_SUCCESS if the port GID is reachable
4851  */
4852 static int
4853 ibdm_port_reachable(ibmf_saa_handle_t sa_hdl, ib_guid_t guid,
4854     ib_guid_t *node_guid)
4855 {
4856 	sa_node_record_t	req, *resp = NULL;
4857 	ibmf_saa_access_args_t	args;
4858 	int			ret;
4859 	size_t		length;
4860 
4861 	IBTF_DPRINTF_L4("ibdm", "\tport_reachable: port_guid %llx",
4862 	    guid);
4863 
4864 	bzero(&req, sizeof (sa_node_record_t));
4865 	req.NodeInfo.PortGUID = guid;
4866 
4867 	args.sq_attr_id		= SA_NODERECORD_ATTRID;
4868 	args.sq_access_type 	= IBMF_SAA_RETRIEVE;
4869 	args.sq_component_mask 	= SA_NODEINFO_COMPMASK_PORTGUID;
4870 	args.sq_template	= &req;
4871 	args.sq_callback	= NULL;
4872 	args.sq_callback_arg 	= NULL;
4873 
4874 	ret = ibmf_sa_access(sa_hdl, &args, 0, &length, (void **) &resp);
4875 	if (ret != IBMF_SUCCESS) {
4876 		IBTF_DPRINTF_L2("ibdm", "\tport_reachable:"
4877 		    " SA Retrieve Failed: %d", ret);
4878 		return (IBDM_FAILURE);
4879 	}
4880 
4881 	if ((resp == NULL) || (length == 0)) {
4882 		IBTF_DPRINTF_L2("ibdm", "\tport_reachable: No records");
4883 		return (IBDM_FAILURE);
4884 	}
4885 
4886 	if (node_guid != NULL)
4887 		*node_guid = resp->NodeInfo.NodeGUID;
4888 
4889 	kmem_free(resp, length);
4890 
4891 	return (IBDM_SUCCESS);
4892 }
4893 
4894 /*
4895  * Update the gidlist for all affected IOCs when GID becomes
4896  * available/unavailable.
4897  *
4898  * Parameters :
4899  *	gidinfo - Incoming / Outgoing GID.
4900  *	add_flag - 1 for GID added, 0 for GID removed.
4901  *		- (-1) : IOC gid list updated, ioc_list required.
4902  *
4903  * This function gets the GID for the node GUID corresponding to the
4904  * port GID. Gets the IOU info
4905  */
4906 static ibdm_ioc_info_t *
4907 ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *gid_info, int avail_flag)
4908 {
4909 	ibdm_dp_gidinfo_t	*node_gid = NULL;
4910 	uint8_t	niocs, ii;
4911 	ibdm_ioc_info_t	*ioc, *ioc_list = NULL, *tmp;
4912 
4913 	IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist");
4914 
4915 	switch (avail_flag) {
4916 		case 1 :
4917 			node_gid = ibdm_check_dest_nodeguid(gid_info);
4918 			break;
4919 		case 0 :
4920 			node_gid = ibdm_handle_gid_rm(gid_info);
4921 			break;
4922 		case -1 :
4923 			node_gid = gid_info;
4924 			break;
4925 		default :
4926 			break;
4927 	}
4928 
4929 	if (node_gid == NULL) {
4930 		IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist: "
4931 		    "No node GID found, port gid 0x%p, avail_flag %d",
4932 		    gid_info, avail_flag);
4933 		return (NULL);
4934 	}
4935 
4936 	mutex_enter(&node_gid->gl_mutex);
4937 	if ((node_gid->gl_state != IBDM_GID_PROBING_COMPLETE &&
4938 	    node_gid->gl_state != IBDM_GID_PROBING_SKIPPED) ||
4939 	    node_gid->gl_iou == NULL) {
4940 		IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist "
4941 		    "gl_state %x, gl_iou %p", node_gid->gl_state,
4942 		    node_gid->gl_iou);
4943 		mutex_exit(&node_gid->gl_mutex);
4944 		return (NULL);
4945 	}
4946 
4947 	niocs = node_gid->gl_iou->iou_info.iou_num_ctrl_slots;
4948 	IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : niocs %x",
4949 	    niocs);
4950 	for (ii = 0; ii < niocs; ii++) {
4951 		ioc = IBDM_GIDINFO2IOCINFO(node_gid, ii);
4952 		/*
4953 		 * Skip IOCs for which probe is not complete or
4954 		 * reprobe is progress
4955 		 */
4956 		if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) {
4957 			tmp = ibdm_dup_ioc_info(ioc, node_gid);
4958 			tmp->ioc_info_updated.ib_gid_prop_updated = 1;
4959 			tmp->ioc_next = ioc_list;
4960 			ioc_list = tmp;
4961 		}
4962 	}
4963 	mutex_exit(&node_gid->gl_mutex);
4964 
4965 	IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : return %p",
4966 	    ioc_list);
4967 	return (ioc_list);
4968 }
4969 
4970 /*
4971  * ibdm_saa_event_cb :
4972  *	Event handling which does *not* require ibdm_hl_mutex to be
4973  *	held are executed in the same thread. This is to prevent
4974  *	deadlocks with HCA port down notifications which hold the
4975  *	ibdm_hl_mutex.
4976  *
4977  *	GID_AVAILABLE event is handled here. A taskq is spawned to
4978  *	handle GID_UNAVAILABLE.
4979  *
4980  *	A new mutex ibdm_ibnex_mutex has been introduced to protect
4981  *	ibnex_callback. This has been done to prevent any possible
4982  *	deadlock (described above) while handling GID_AVAILABLE.
4983  *
4984  *	IBMF calls the event callback for a HCA port. The SA handle
4985  *	for this port would be valid, till the callback returns.
4986  *	IBDM calling IBDM using the above SA handle should be valid.
4987  *
4988  *	IBDM will additionally  check (SA handle != NULL), before
4989  *	calling IBMF.
4990  */
4991 /*ARGSUSED*/
4992 static void
4993 ibdm_saa_event_cb(ibmf_saa_handle_t ibmf_saa_handle,
4994     ibmf_saa_subnet_event_t ibmf_saa_event,
4995     ibmf_saa_event_details_t *event_details, void *callback_arg)
4996 {
4997 	ibdm_saa_event_arg_t *event_arg;
4998 	ib_gid_t		sgid, dgid;
4999 	ibdm_port_attr_t	*hca_port;
5000 	ibdm_dp_gidinfo_t	*gid_info, *node_gid_info = NULL;
5001 	ib_guid_t		nodeguid;
5002 
5003 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg));
5004 
5005 	hca_port = (ibdm_port_attr_t *)callback_arg;
5006 
5007 	IBTF_DPRINTF_L4("ibdm", "\tsaa_event_cb(%x, %x, %x, %x)\n",
5008 	    ibmf_saa_handle, ibmf_saa_event, event_details,
5009 	    callback_arg);
5010 #ifdef DEBUG
5011 	if (ibdm_ignore_saa_event)
5012 		return;
5013 #endif
5014 
5015 	if (ibmf_saa_event == IBMF_SAA_EVENT_GID_AVAILABLE) {
5016 		/*
5017 		 * Ensure no other probe / sweep fabric is in
5018 		 * progress.
5019 		 */
5020 		mutex_enter(&ibdm.ibdm_mutex);
5021 		while (ibdm.ibdm_busy & IBDM_BUSY)
5022 			cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5023 		ibdm.ibdm_busy |= IBDM_BUSY;
5024 		mutex_exit(&ibdm.ibdm_mutex);
5025 
5026 		/*
5027 		 * If we already know about this GID, return.
5028 		 * GID_AVAILABLE may be reported for multiple HCA
5029 		 * ports.
5030 		 */
5031 		if ((ibdm_check_dgid(event_details->ie_gid.gid_guid,
5032 		    event_details->ie_gid.gid_prefix))  != NULL) {
5033 			mutex_enter(&ibdm.ibdm_mutex);
5034 			ibdm.ibdm_busy &= ~IBDM_BUSY;
5035 			cv_broadcast(&ibdm.ibdm_busy_cv);
5036 			mutex_exit(&ibdm.ibdm_mutex);
5037 			return;
5038 		}
5039 
5040 		IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) "
5041 		    "Insertion notified",
5042 		    event_details->ie_gid.gid_prefix,
5043 		    event_details->ie_gid.gid_guid);
5044 
5045 		/* This is a new gid, insert it to GID list */
5046 		sgid.gid_prefix = hca_port->pa_sn_prefix;
5047 		sgid.gid_guid = hca_port->pa_port_guid;
5048 		dgid.gid_prefix = event_details->ie_gid.gid_prefix;
5049 		dgid.gid_guid = event_details->ie_gid.gid_guid;
5050 		gid_info = ibdm_create_gid_info(hca_port, sgid, dgid);
5051 		if (gid_info == NULL) {
5052 			IBTF_DPRINTF_L4("ibdm", "\tGID_AVAILABLE: "
5053 			    "create_gid_info returned NULL");
5054 			mutex_enter(&ibdm.ibdm_mutex);
5055 			ibdm.ibdm_busy &= ~IBDM_BUSY;
5056 			cv_broadcast(&ibdm.ibdm_busy_cv);
5057 			mutex_exit(&ibdm.ibdm_mutex);
5058 			return;
5059 		}
5060 		mutex_enter(&gid_info->gl_mutex);
5061 		gid_info->gl_state = IBDM_GID_PROBING_SKIPPED;
5062 		mutex_exit(&gid_info->gl_mutex);
5063 
5064 		/* Get the node GUID */
5065 		if (ibdm_port_reachable(ibmf_saa_handle, dgid.gid_guid,
5066 		    &nodeguid) != IBDM_SUCCESS) {
5067 			/*
5068 			 * Set the state to PROBE_NOT_DONE for the
5069 			 * next sweep to probe it
5070 			 */
5071 			IBTF_DPRINTF_L2("ibdm", "\tsaa_event_taskq: "
5072 			    "Skipping GID : port GUID not found");
5073 			mutex_enter(&gid_info->gl_mutex);
5074 			gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE;
5075 			mutex_exit(&gid_info->gl_mutex);
5076 			mutex_enter(&ibdm.ibdm_mutex);
5077 			ibdm.ibdm_busy &= ~IBDM_BUSY;
5078 			cv_broadcast(&ibdm.ibdm_busy_cv);
5079 			mutex_exit(&ibdm.ibdm_mutex);
5080 			return;
5081 		}
5082 
5083 		gid_info->gl_nodeguid = nodeguid;
5084 		gid_info->gl_portguid = dgid.gid_guid;
5085 
5086 		/*
5087 		 * Get the gid info with the same node GUID.
5088 		 */
5089 		mutex_enter(&ibdm.ibdm_mutex);
5090 		node_gid_info = ibdm.ibdm_dp_gidlist_head;
5091 		while (node_gid_info) {
5092 			if (node_gid_info->gl_nodeguid ==
5093 			    gid_info->gl_nodeguid &&
5094 			    node_gid_info->gl_iou != NULL) {
5095 				break;
5096 			}
5097 			node_gid_info = node_gid_info->gl_next;
5098 		}
5099 		mutex_exit(&ibdm.ibdm_mutex);
5100 
5101 		/*
5102 		 * Handling a new GID requires filling of gl_hca_list.
5103 		 * This require ibdm hca_list to be parsed and hence
5104 		 * holding the ibdm_hl_mutex. Spawning a new thread to
5105 		 * handle this.
5106 		 */
5107 		if (node_gid_info == NULL) {
5108 			if (taskq_dispatch(system_taskq,
5109 			    ibdm_saa_handle_new_gid, (void *)gid_info,
5110 			    TQ_NOSLEEP) == NULL) {
5111 				IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: "
5112 				    "new_gid taskq_dispatch failed");
5113 				return;
5114 			}
5115 		}
5116 
5117 		mutex_enter(&ibdm.ibdm_mutex);
5118 		ibdm.ibdm_busy &= ~IBDM_BUSY;
5119 		cv_broadcast(&ibdm.ibdm_busy_cv);
5120 		mutex_exit(&ibdm.ibdm_mutex);
5121 		return;
5122 	}
5123 
5124 	if (ibmf_saa_event != IBMF_SAA_EVENT_GID_UNAVAILABLE)
5125 		return;
5126 
5127 	event_arg = (ibdm_saa_event_arg_t *)kmem_alloc(
5128 	    sizeof (ibdm_saa_event_arg_t), KM_SLEEP);
5129 	event_arg->ibmf_saa_handle = ibmf_saa_handle;
5130 	event_arg->ibmf_saa_event = ibmf_saa_event;
5131 	bcopy(event_details, &event_arg->event_details,
5132 	    sizeof (ibmf_saa_event_details_t));
5133 	event_arg->callback_arg = callback_arg;
5134 
5135 	if (taskq_dispatch(system_taskq, ibdm_saa_event_taskq,
5136 	    (void *)event_arg, TQ_NOSLEEP) == NULL) {
5137 		IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: "
5138 		    "taskq_dispatch failed");
5139 		ibdm_free_saa_event_arg(event_arg);
5140 		return;
5141 	}
5142 }
5143 
5144 /*
5145  * Handle a new GID discovered by GID_AVAILABLE saa event.
5146  */
5147 void
5148 ibdm_saa_handle_new_gid(void *arg)
5149 {
5150 	ibdm_dp_gidinfo_t	*gid_info;
5151 	ibdm_hca_list_t		*hca_list = NULL;
5152 	ibdm_port_attr_t	*port = NULL;
5153 	ibdm_ioc_info_t		*ioc_list = NULL;
5154 
5155 	IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid(%p)", arg);
5156 
5157 	gid_info = (ibdm_dp_gidinfo_t *)arg;
5158 
5159 	/*
5160 	 * Ensure that no other sweep / probe has completed
5161 	 * probing this gid.
5162 	 */
5163 	mutex_enter(&gid_info->gl_mutex);
5164 	if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) {
5165 		mutex_exit(&gid_info->gl_mutex);
5166 		return;
5167 	}
5168 	mutex_exit(&gid_info->gl_mutex);
5169 
5170 	/*
5171 	 * Parse HCAs to fill gl_hca_list
5172 	 */
5173 	mutex_enter(&ibdm.ibdm_hl_mutex);
5174 	for (ibdm_get_next_port(&hca_list, &port, 1); port;
5175 	    ibdm_get_next_port(&hca_list, &port, 1)) {
5176 		if (ibdm_port_reachable(port->pa_sa_hdl,
5177 		    gid_info->gl_portguid, NULL) ==
5178 		    IBDM_SUCCESS) {
5179 			ibdm_addto_glhcalist(gid_info, hca_list);
5180 		}
5181 	}
5182 	mutex_exit(&ibdm.ibdm_hl_mutex);
5183 
5184 	/*
5185 	 * Ensure no other probe / sweep fabric is in
5186 	 * progress.
5187 	 */
5188 	mutex_enter(&ibdm.ibdm_mutex);
5189 	while (ibdm.ibdm_busy & IBDM_BUSY)
5190 		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5191 	ibdm.ibdm_busy |= IBDM_BUSY;
5192 	mutex_exit(&ibdm.ibdm_mutex);
5193 
5194 	/*
5195 	 * New IOU probe it, to check if new IOCs
5196 	 */
5197 	IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid: "
5198 	    "new GID : probing");
5199 	mutex_enter(&ibdm.ibdm_mutex);
5200 	ibdm.ibdm_ngid_probes_in_progress++;
5201 	mutex_exit(&ibdm.ibdm_mutex);
5202 	mutex_enter(&gid_info->gl_mutex);
5203 	gid_info->gl_reprobe_flag = 0;
5204 	gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE;
5205 	mutex_exit(&gid_info->gl_mutex);
5206 	ibdm_probe_gid_thread((void *)gid_info);
5207 
5208 	mutex_enter(&ibdm.ibdm_mutex);
5209 	ibdm_wait_probe_completion();
5210 	mutex_exit(&ibdm.ibdm_mutex);
5211 
5212 	if (gid_info->gl_iou == NULL) {
5213 		mutex_enter(&ibdm.ibdm_mutex);
5214 		ibdm.ibdm_busy &= ~IBDM_BUSY;
5215 		cv_broadcast(&ibdm.ibdm_busy_cv);
5216 		mutex_exit(&ibdm.ibdm_mutex);
5217 		return;
5218 	}
5219 
5220 	/*
5221 	 * Update GID list in all IOCs affected by this
5222 	 */
5223 	ioc_list = ibdm_update_ioc_gidlist(gid_info, 1);
5224 
5225 	/*
5226 	 * Pass on the IOCs with updated GIDs to IBnexus
5227 	 */
5228 	if (ioc_list) {
5229 		mutex_enter(&ibdm.ibdm_ibnex_mutex);
5230 		if (ibdm.ibdm_ibnex_callback != NULL) {
5231 			(*ibdm.ibdm_ibnex_callback)((void *)ioc_list,
5232 			    IBDM_EVENT_IOC_PROP_UPDATE);
5233 		}
5234 		mutex_exit(&ibdm.ibdm_ibnex_mutex);
5235 	}
5236 
5237 	mutex_enter(&ibdm.ibdm_mutex);
5238 	ibdm.ibdm_busy &= ~IBDM_BUSY;
5239 	cv_broadcast(&ibdm.ibdm_busy_cv);
5240 	mutex_exit(&ibdm.ibdm_mutex);
5241 }
5242 
5243 /*
5244  * ibdm_saa_event_taskq :
5245  *	GID_UNAVAILABLE Event handling requires ibdm_hl_mutex to be
5246  *	held. The GID_UNAVAILABLE handling is done in a taskq to
5247  *	prevent deadlocks with HCA port down notifications which hold
5248  *	ibdm_hl_mutex.
5249  */
5250 void
5251 ibdm_saa_event_taskq(void *arg)
5252 {
5253 	ibdm_saa_event_arg_t *event_arg;
5254 	ibmf_saa_handle_t ibmf_saa_handle;
5255 	ibmf_saa_subnet_event_t ibmf_saa_event;
5256 	ibmf_saa_event_details_t *event_details;
5257 	void *callback_arg;
5258 
5259 	ibdm_dp_gidinfo_t	*gid_info;
5260 	ibdm_port_attr_t	*hca_port, *port = NULL;
5261 	ibdm_hca_list_t		*hca_list = NULL;
5262 	int	sa_handle_valid = 0;
5263 	ibdm_ioc_info_t		*ioc_list = NULL;
5264 
5265 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg));
5266 
5267 	event_arg = (ibdm_saa_event_arg_t *)arg;
5268 	ibmf_saa_handle = event_arg->ibmf_saa_handle;
5269 	ibmf_saa_event = event_arg->ibmf_saa_event;
5270 	event_details = &event_arg->event_details;
5271 	callback_arg = event_arg->callback_arg;
5272 
5273 	ASSERT(callback_arg != NULL);
5274 	ASSERT(ibmf_saa_event == IBMF_SAA_EVENT_GID_UNAVAILABLE);
5275 	IBTF_DPRINTF_L4("ibdm", "\tsaa_event_taskq(%x, %x, %x, %x)",
5276 	    ibmf_saa_handle, ibmf_saa_event, event_details,
5277 	    callback_arg);
5278 
5279 	hca_port = (ibdm_port_attr_t *)callback_arg;
5280 
5281 	/* Check if the port_attr is still valid */
5282 	mutex_enter(&ibdm.ibdm_hl_mutex);
5283 	for (ibdm_get_next_port(&hca_list, &port, 0); port;
5284 	    ibdm_get_next_port(&hca_list, &port, 0)) {
5285 		if (port == hca_port && port->pa_port_guid ==
5286 		    hca_port->pa_port_guid) {
5287 			if (ibmf_saa_handle == hca_port->pa_sa_hdl)
5288 				sa_handle_valid = 1;
5289 			break;
5290 		}
5291 	}
5292 	mutex_exit(&ibdm.ibdm_hl_mutex);
5293 	if (sa_handle_valid == 0) {
5294 		ibdm_free_saa_event_arg(event_arg);
5295 		return;
5296 	}
5297 
5298 	if (hca_port && (hca_port->pa_sa_hdl == NULL ||
5299 	    ibmf_saa_handle != hca_port->pa_sa_hdl)) {
5300 		ibdm_free_saa_event_arg(event_arg);
5301 		return;
5302 	}
5303 	hca_list = NULL;
5304 	port = NULL;
5305 
5306 	/*
5307 	 * Check if the GID is visible to other HCA ports.
5308 	 * Return if so.
5309 	 */
5310 	mutex_enter(&ibdm.ibdm_hl_mutex);
5311 	for (ibdm_get_next_port(&hca_list, &port, 1); port;
5312 	    ibdm_get_next_port(&hca_list, &port, 1)) {
5313 		if (ibdm_port_reachable(port->pa_sa_hdl,
5314 		    event_details->ie_gid.gid_guid, NULL) ==
5315 		    IBDM_SUCCESS) {
5316 			mutex_exit(&ibdm.ibdm_hl_mutex);
5317 			ibdm_free_saa_event_arg(event_arg);
5318 			return;
5319 		}
5320 	}
5321 	mutex_exit(&ibdm.ibdm_hl_mutex);
5322 
5323 	/*
5324 	 * Ensure no other probe / sweep fabric is in
5325 	 * progress.
5326 	 */
5327 	mutex_enter(&ibdm.ibdm_mutex);
5328 	while (ibdm.ibdm_busy & IBDM_BUSY)
5329 		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5330 	ibdm.ibdm_busy |= IBDM_BUSY;
5331 	mutex_exit(&ibdm.ibdm_mutex);
5332 
5333 	/*
5334 	 * If this GID is no longer in GID list, return
5335 	 * GID_UNAVAILABLE may be reported for multiple HCA
5336 	 * ports.
5337 	 */
5338 	mutex_enter(&ibdm.ibdm_mutex);
5339 	gid_info = ibdm.ibdm_dp_gidlist_head;
5340 	while (gid_info) {
5341 		if (gid_info->gl_portguid ==
5342 		    event_details->ie_gid.gid_guid) {
5343 			break;
5344 		}
5345 		gid_info = gid_info->gl_next;
5346 	}
5347 	mutex_exit(&ibdm.ibdm_mutex);
5348 	if (gid_info == NULL) {
5349 		mutex_enter(&ibdm.ibdm_mutex);
5350 		ibdm.ibdm_busy &= ~IBDM_BUSY;
5351 		cv_broadcast(&ibdm.ibdm_busy_cv);
5352 		mutex_exit(&ibdm.ibdm_mutex);
5353 		ibdm_free_saa_event_arg(event_arg);
5354 		return;
5355 	}
5356 
5357 	IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) "
5358 	    "Unavailable notification",
5359 	    event_details->ie_gid.gid_prefix,
5360 	    event_details->ie_gid.gid_guid);
5361 
5362 	/*
5363 	 * Update GID list in all IOCs affected by this
5364 	 */
5365 	if (gid_info->gl_state == IBDM_GID_PROBING_SKIPPED ||
5366 	    gid_info->gl_state == IBDM_GID_PROBING_COMPLETE)
5367 		ioc_list = ibdm_update_ioc_gidlist(gid_info, 0);
5368 
5369 	/*
5370 	 * Remove GID from the global GID list
5371 	 * Handle the case where all port GIDs for an
5372 	 * IOU have been hot-removed. Check both gid_info
5373 	 * & ioc_info for checking ngids.
5374 	 */
5375 	mutex_enter(&ibdm.ibdm_mutex);
5376 	if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) {
5377 		mutex_enter(&gid_info->gl_mutex);
5378 		(void) ibdm_free_iou_info(gid_info);
5379 		mutex_exit(&gid_info->gl_mutex);
5380 	}
5381 	if (gid_info->gl_prev != NULL)
5382 		gid_info->gl_prev->gl_next = gid_info->gl_next;
5383 	if (gid_info->gl_next != NULL)
5384 		gid_info->gl_next->gl_prev = gid_info->gl_prev;
5385 
5386 	if (gid_info == ibdm.ibdm_dp_gidlist_head)
5387 		ibdm.ibdm_dp_gidlist_head = gid_info->gl_next;
5388 	if (gid_info == ibdm.ibdm_dp_gidlist_tail)
5389 		ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev;
5390 	ibdm.ibdm_ngids--;
5391 
5392 	ibdm.ibdm_busy &= ~IBDM_BUSY;
5393 	cv_broadcast(&ibdm.ibdm_busy_cv);
5394 	mutex_exit(&ibdm.ibdm_mutex);
5395 
5396 	mutex_destroy(&gid_info->gl_mutex);
5397 	kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t));
5398 
5399 	/*
5400 	 * Pass on the IOCs with updated GIDs to IBnexus
5401 	 */
5402 	if (ioc_list) {
5403 		IBTF_DPRINTF_L4("ibdm", "\tGID_UNAVAILABLE "
5404 		    "IOC_PROP_UPDATE for %p\n", ioc_list);
5405 		mutex_enter(&ibdm.ibdm_ibnex_mutex);
5406 		if (ibdm.ibdm_ibnex_callback != NULL) {
5407 			(*ibdm.ibdm_ibnex_callback)((void *)
5408 		    ioc_list, IBDM_EVENT_IOC_PROP_UPDATE);
5409 		}
5410 		mutex_exit(&ibdm.ibdm_ibnex_mutex);
5411 	}
5412 
5413 	ibdm_free_saa_event_arg(event_arg);
5414 }
5415 
5416 
5417 static int
5418 ibdm_cmp_gid_list(ibdm_gid_t *new, ibdm_gid_t *prev)
5419 {
5420 	ibdm_gid_t		*scan_new, *scan_prev;
5421 	int	cmp_failed = 0;
5422 
5423 	ASSERT(new != NULL);
5424 	ASSERT(prev != NULL);
5425 
5426 	/*
5427 	 * Search for each new gid anywhere in the prev GID list.
5428 	 * Note that the gid list could have been re-ordered.
5429 	 */
5430 	for (scan_new = new; scan_new; scan_new = scan_new->gid_next) {
5431 		for (scan_prev = prev, cmp_failed = 1; scan_prev;
5432 		    scan_prev = scan_prev->gid_next) {
5433 			if (scan_prev->gid_dgid_hi == scan_new->gid_dgid_hi &&
5434 			    scan_prev->gid_dgid_lo == scan_new->gid_dgid_lo) {
5435 				cmp_failed = 0;
5436 				break;
5437 			}
5438 		}
5439 
5440 		if (cmp_failed)
5441 			return (1);
5442 	}
5443 	return (0);
5444 }
5445 
5446 /*
5447  * This is always called in a single thread
5448  * This function updates the gid_list and serv_list of IOC
5449  * The current gid_list is in ioc_info_t(contains only port
5450  * guids for which probe is done) & gidinfo_t(other port gids)
5451  * The gids in both locations are used for comparision.
5452  */
5453 static void
5454 ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *ioc, ibdm_dp_gidinfo_t *gidinfo)
5455 {
5456 	ibdm_gid_t		*cur_gid_list;
5457 	uint_t			cur_nportgids;
5458 
5459 	ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
5460 
5461 	ioc->ioc_info_updated.ib_prop_updated = 0;
5462 
5463 
5464 	/* Current GID list in gid_info only */
5465 	cur_gid_list = gidinfo->gl_gid;
5466 	cur_nportgids = gidinfo->gl_ngids;
5467 
5468 	/*
5469 	 * Service entry names and IDs are not compared currently.
5470 	 * This may require change.
5471 	 */
5472 	if (ioc->ioc_prev_serv_cnt != ioc->ioc_profile.ioc_service_entries)
5473 		ioc->ioc_info_updated.ib_srv_prop_updated = 1;
5474 
5475 	if (ioc->ioc_prev_nportgids != cur_nportgids ||
5476 	    ioc->ioc_prev_gid_list == NULL || cur_gid_list == NULL) {
5477 		ioc->ioc_info_updated.ib_gid_prop_updated = 1;
5478 	} else if (ibdm_cmp_gid_list(ioc->ioc_prev_gid_list, cur_gid_list)) {
5479 		ioc->ioc_info_updated.ib_gid_prop_updated = 1;
5480 	}
5481 
5482 	/* Zero out previous entries */
5483 	ibdm_free_gid_list(ioc->ioc_prev_gid_list);
5484 	if (ioc->ioc_prev_serv)
5485 		kmem_free(ioc->ioc_prev_serv, ioc->ioc_prev_serv_cnt *
5486 		    sizeof (ibdm_srvents_info_t));
5487 	ioc->ioc_prev_serv_cnt = 0;
5488 	ioc->ioc_prev_nportgids = 0;
5489 	ioc->ioc_prev_serv = NULL;
5490 	ioc->ioc_prev_gid_list = NULL;
5491 }
5492 
5493 /*
5494  * Handle GID removal. This returns gid_info of an GID for the same
5495  * node GUID, if found.  For an GID with IOU information, the same
5496  * gid_info is returned if no gid_info with same node_guid is found.
5497  */
5498 static ibdm_dp_gidinfo_t *
5499 ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *rm_gid)
5500 {
5501 	ibdm_dp_gidinfo_t	*gid_list;
5502 
5503 	IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm(0x%p)", rm_gid);
5504 
5505 	if (rm_gid->gl_iou == NULL) {
5506 		IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm NO iou");
5507 		/*
5508 		 * Search for a GID with same node_guid and
5509 		 * gl_iou != NULL
5510 		 */
5511 		for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
5512 		    gid_list = gid_list->gl_next) {
5513 			if (gid_list->gl_iou != NULL && (gid_list->gl_nodeguid
5514 			    == rm_gid->gl_nodeguid))
5515 				break;
5516 		}
5517 
5518 		if (gid_list)
5519 			ibdm_rmfrom_glgid_list(gid_list, rm_gid);
5520 
5521 		IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list);
5522 		return (gid_list);
5523 	} else {
5524 		/*
5525 		 * Search for a GID with same node_guid and
5526 		 * gl_iou == NULL
5527 		 */
5528 		IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm with iou");
5529 		for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
5530 		    gid_list = gid_list->gl_next) {
5531 			if (gid_list->gl_iou == NULL && (gid_list->gl_nodeguid
5532 			    == rm_gid->gl_nodeguid))
5533 				break;
5534 		}
5535 
5536 		if (gid_list) {
5537 			/*
5538 			 * Copy the following fields from rm_gid :
5539 			 *	1. gl_state
5540 			 *	2. gl_iou
5541 			 *	3. gl_gid & gl_ngids
5542 			 *
5543 			 * Note :	Function is synchronized by
5544 			 *			ibdm_busy flag.
5545 			 *
5546 			 * Note :	Redirect info is initialized if
5547 			 *			any MADs for the GID fail
5548 			 */
5549 			IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm "
5550 			    "copying info to GID with gl_iou != NULl");
5551 			gid_list->gl_state = rm_gid->gl_state;
5552 			gid_list->gl_iou = rm_gid->gl_iou;
5553 			gid_list->gl_gid = rm_gid->gl_gid;
5554 			gid_list->gl_ngids = rm_gid->gl_ngids;
5555 
5556 			/* Remove the GID from gl_gid list */
5557 			ibdm_rmfrom_glgid_list(gid_list, rm_gid);
5558 		} else {
5559 			/*
5560 			 * Handle a case where all GIDs to the IOU have
5561 			 * been removed.
5562 			 */
5563 			IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm 0 GID "
5564 			    "to IOU");
5565 
5566 			ibdm_rmfrom_glgid_list(rm_gid, rm_gid);
5567 			return (rm_gid);
5568 		}
5569 		IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list);
5570 		return (gid_list);
5571 	}
5572 }
5573 
5574 static void
5575 ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *gid_info,
5576     ibdm_dp_gidinfo_t *rm_gid)
5577 {
5578 	ibdm_gid_t 		*tmp, *prev;
5579 
5580 	IBTF_DPRINTF_L4("ibdm", "\trmfrom_glgid (%p, %p)",
5581 	    gid_info, rm_gid);
5582 
5583 	for (tmp = gid_info->gl_gid, prev = NULL; tmp; ) {
5584 		if (tmp->gid_dgid_hi == rm_gid->gl_dgid_hi &&
5585 		    tmp->gid_dgid_lo == rm_gid->gl_dgid_lo) {
5586 			if (prev == NULL)
5587 				gid_info->gl_gid = tmp->gid_next;
5588 			else
5589 				prev->gid_next = tmp->gid_next;
5590 
5591 			kmem_free(tmp, sizeof (ibdm_gid_t));
5592 			gid_info->gl_ngids--;
5593 			break;
5594 		} else {
5595 			prev = tmp;
5596 			tmp = tmp->gid_next;
5597 		}
5598 	}
5599 }
5600 
5601 static void
5602 ibdm_addto_gidlist(ibdm_gid_t **src_ptr, ibdm_gid_t *dest)
5603 {
5604 	ibdm_gid_t *head = NULL, *new, *tail;
5605 
5606 	/* First copy the destination */
5607 	for (; dest; dest = dest->gid_next) {
5608 		new = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP);
5609 		new->gid_dgid_hi = dest->gid_dgid_hi;
5610 		new->gid_dgid_lo = dest->gid_dgid_lo;
5611 		new->gid_next = head;
5612 		head = new;
5613 	}
5614 
5615 	/* Insert this to the source */
5616 	if (*src_ptr == NULL)
5617 		*src_ptr = head;
5618 	else {
5619 		for (tail = *src_ptr; tail->gid_next != NULL;
5620 		    tail = tail->gid_next)
5621 			;
5622 
5623 		tail->gid_next = head;
5624 	}
5625 }
5626 
5627 static void
5628 ibdm_free_gid_list(ibdm_gid_t	*head)
5629 {
5630 	ibdm_gid_t	*delete;
5631 
5632 	for (delete = head; delete; ) {
5633 		head = delete->gid_next;
5634 		kmem_free(delete, sizeof (ibdm_gid_t));
5635 		delete = head;
5636 	}
5637 }
5638 
5639 /*
5640  * This function rescans the DM capable GIDs (gl_state is
5641  * GID_PROBE_COMPLETE or IBDM_GID_PROBING_SKIPPED.This
5642  * basically checks if the DM capable GID is reachable. If
5643  * not this is handled the same way as GID_UNAVAILABLE,
5644  * except that notifications are not send to IBnexus.
5645  *
5646  * This function also initializes the ioc_prev_list for
5647  * a particular IOC (when called from probe_ioc, with
5648  * ioc_guidp != NULL) or all IOCs for the gid (called from
5649  * sweep_fabric, ioc_guidp == NULL).
5650  */
5651 static void
5652 ibdm_rescan_gidlist(ib_guid_t *ioc_guidp)
5653 {
5654 	ibdm_dp_gidinfo_t	*gid_info, *tmp;
5655 	int ii, niocs, found;
5656 	ibdm_hca_list_t *hca_list = NULL;
5657 	ibdm_port_attr_t *port = NULL;
5658 	ibdm_ioc_info_t *ioc_list;
5659 
5660 	for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) {
5661 		found = 0;
5662 		if (gid_info->gl_state != IBDM_GID_PROBING_SKIPPED &&
5663 		    gid_info->gl_state != IBDM_GID_PROBING_COMPLETE) {
5664 			gid_info = gid_info->gl_next;
5665 			continue;
5666 		}
5667 
5668 		/*
5669 		 * Check if the GID is visible to any HCA ports.
5670 		 * Return if so.
5671 		 */
5672 		mutex_enter(&ibdm.ibdm_hl_mutex);
5673 		for (ibdm_get_next_port(&hca_list, &port, 1); port;
5674 		    ibdm_get_next_port(&hca_list, &port, 1)) {
5675 			if (ibdm_port_reachable(port->pa_sa_hdl,
5676 			    gid_info->gl_dgid_lo, NULL) == IBDM_SUCCESS) {
5677 				found = 1;
5678 				break;
5679 			}
5680 		}
5681 		mutex_exit(&ibdm.ibdm_hl_mutex);
5682 
5683 		if (found) {
5684 			if (gid_info->gl_iou == NULL) {
5685 				gid_info = gid_info->gl_next;
5686 				continue;
5687 			}
5688 
5689 			/* Intialize the ioc_prev_gid_list */
5690 			niocs =
5691 			    gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
5692 			for (ii = 0; ii < niocs; ii++) {
5693 				ioc_list = IBDM_GIDINFO2IOCINFO(gid_info, ii);
5694 
5695 				if (ioc_guidp == NULL || (*ioc_guidp ==
5696 				    ioc_list->ioc_profile.ioc_guid)) {
5697 					/* Add info of GIDs in gid_info also */
5698 					ibdm_addto_gidlist(
5699 					    &ioc_list->ioc_prev_gid_list,
5700 					    gid_info->gl_gid);
5701 					ioc_list->ioc_prev_nportgids =
5702 					    gid_info->gl_ngids;
5703 				}
5704 			}
5705 			gid_info = gid_info->gl_next;
5706 			continue;
5707 		}
5708 
5709 		IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist "
5710 		    "deleted port GUID %llx",
5711 		    gid_info->gl_dgid_lo);
5712 
5713 		/*
5714 		 * Update GID list in all IOCs affected by this
5715 		 */
5716 		ioc_list = ibdm_update_ioc_gidlist(gid_info, 0);
5717 
5718 		/*
5719 		 * Remove GID from the global GID list
5720 		 * Handle the case where all port GIDs for an
5721 		 * IOU have been hot-removed.
5722 		 */
5723 		mutex_enter(&ibdm.ibdm_mutex);
5724 		if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) {
5725 			mutex_enter(&gid_info->gl_mutex);
5726 			(void) ibdm_free_iou_info(gid_info);
5727 			mutex_exit(&gid_info->gl_mutex);
5728 		}
5729 		tmp = gid_info->gl_next;
5730 		if (gid_info->gl_prev != NULL)
5731 			gid_info->gl_prev->gl_next = gid_info->gl_next;
5732 		if (gid_info->gl_next != NULL)
5733 			gid_info->gl_next->gl_prev = gid_info->gl_prev;
5734 
5735 		if (gid_info == ibdm.ibdm_dp_gidlist_head)
5736 			ibdm.ibdm_dp_gidlist_head = gid_info->gl_next;
5737 		if (gid_info == ibdm.ibdm_dp_gidlist_tail)
5738 			ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev;
5739 		ibdm.ibdm_ngids--;
5740 
5741 		mutex_destroy(&gid_info->gl_mutex);
5742 		kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t));
5743 		gid_info = tmp;
5744 
5745 		mutex_exit(&ibdm.ibdm_mutex);
5746 
5747 		/*
5748 		 * Pass on the IOCs with updated GIDs to IBnexus
5749 		 */
5750 		if (ioc_list) {
5751 			IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist "
5752 			    "IOC_PROP_UPDATE for %p\n", ioc_list);
5753 			mutex_enter(&ibdm.ibdm_ibnex_mutex);
5754 			if (ibdm.ibdm_ibnex_callback != NULL) {
5755 				(*ibdm.ibdm_ibnex_callback)((void *)
5756 				    ioc_list, IBDM_EVENT_IOC_PROP_UPDATE);
5757 			}
5758 			mutex_exit(&ibdm.ibdm_ibnex_mutex);
5759 		}
5760 	}
5761 }
5762 
5763 /*
5764  * This function notifies IBnex of IOCs on this GID.
5765  * Notification is for GIDs with gl_reprobe_flag set.
5766  * The flag is set when IOC probe / fabric sweep
5767  * probes a GID starting from CLASS port info.
5768  *
5769  * IBnexus will have information of a reconnected IOC
5770  * if it had probed it before. If this is a new IOC,
5771  * IBnexus ignores the notification.
5772  *
5773  * This function should be called with no locks held.
5774  */
5775 static void
5776 ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *gid_info)
5777 {
5778 	ibdm_ioc_info_t	*ioc_list;
5779 
5780 	if (gid_info->gl_reprobe_flag == 0 ||
5781 	    gid_info->gl_iou == NULL)
5782 		return;
5783 
5784 	ioc_list = ibdm_update_ioc_gidlist(gid_info, -1);
5785 
5786 	/*
5787 	 * Pass on the IOCs with updated GIDs to IBnexus
5788 	 */
5789 	if (ioc_list) {
5790 		mutex_enter(&ibdm.ibdm_ibnex_mutex);
5791 		if (ibdm.ibdm_ibnex_callback != NULL) {
5792 			(*ibdm.ibdm_ibnex_callback)((void *)ioc_list,
5793 			    IBDM_EVENT_IOC_PROP_UPDATE);
5794 		}
5795 		mutex_exit(&ibdm.ibdm_ibnex_mutex);
5796 	}
5797 }
5798 
5799 
5800 static void
5801 ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *arg)
5802 {
5803 	if (arg != NULL)
5804 		kmem_free(arg, sizeof (ibdm_saa_event_arg_t));
5805 }
5806 
5807 /*
5808  * This function parses the list of HCAs and HCA ports
5809  * to return the port_attr of the next HCA port. A port
5810  * connected to IB fabric (port_state active) is returned,
5811  * if connected_flag is set.
5812  */
5813 static void
5814 ibdm_get_next_port(ibdm_hca_list_t **inp_hcap,
5815     ibdm_port_attr_t **inp_portp, int connect_flag)
5816 {
5817 	int ii;
5818 	ibdm_port_attr_t *port, *next_port = NULL;
5819 	ibdm_port_attr_t *inp_port;
5820 	ibdm_hca_list_t	 *hca_list;
5821 	int found = 0;
5822 
5823 	ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
5824 	IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port(%p, %p, %x)",
5825 	    inp_hcap, inp_portp, connect_flag);
5826 
5827 	hca_list = *inp_hcap;
5828 	inp_port = *inp_portp;
5829 
5830 	if (hca_list == NULL)
5831 		hca_list = ibdm.ibdm_hca_list_head;
5832 
5833 	for (; hca_list; hca_list = hca_list->hl_next) {
5834 		for (ii = 0; ii < hca_list->hl_nports; ii++) {
5835 			port = &hca_list->hl_port_attr[ii];
5836 
5837 			/*
5838 			 * inp_port != NULL;
5839 			 * 	Skip till we find the matching port
5840 			 */
5841 			if (inp_port && !found) {
5842 				if (inp_port == port)
5843 					found = 1;
5844 				continue;
5845 			}
5846 
5847 			if (!connect_flag) {
5848 				next_port = port;
5849 				break;
5850 			}
5851 
5852 			if (port->pa_sa_hdl == NULL)
5853 				ibdm_initialize_port(port);
5854 			if (port->pa_sa_hdl == NULL)
5855 				(void) ibdm_fini_port(port);
5856 			else if (next_port == NULL &&
5857 			    port->pa_sa_hdl != NULL &&
5858 			    port->pa_state == IBT_PORT_ACTIVE) {
5859 				next_port = port;
5860 				break;
5861 			}
5862 		}
5863 
5864 		if (next_port)
5865 			break;
5866 	}
5867 
5868 	IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port : "
5869 	    "returns hca_list %p port %p", hca_list, next_port);
5870 	*inp_hcap = hca_list;
5871 	*inp_portp = next_port;
5872 }
5873 
5874 static void
5875 ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *nodegid, ibdm_dp_gidinfo_t *addgid)
5876 {
5877 	ibdm_gid_t	*tmp;
5878 
5879 	tmp = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP);
5880 	tmp->gid_dgid_hi = addgid->gl_dgid_hi;
5881 	tmp->gid_dgid_lo = addgid->gl_dgid_lo;
5882 
5883 	mutex_enter(&nodegid->gl_mutex);
5884 	tmp->gid_next = nodegid->gl_gid;
5885 	nodegid->gl_gid = tmp;
5886 	nodegid->gl_ngids++;
5887 	mutex_exit(&nodegid->gl_mutex);
5888 }
5889 
5890 static void
5891 ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *gid_info,
5892     ibdm_hca_list_t *hca)
5893 {
5894 	ibdm_hca_list_t		*head, *prev = NULL, *temp;
5895 
5896 	IBTF_DPRINTF_L4(ibdm_string, "\taddto_glhcalist(%p, %p) "
5897 	    ": gl_hca_list %p", gid_info, hca, gid_info->gl_hca_list);
5898 	ASSERT(!MUTEX_HELD(&gid_info->gl_mutex));
5899 	mutex_enter(&gid_info->gl_mutex);
5900 	head = gid_info->gl_hca_list;
5901 	if (head == NULL) {
5902 		head = ibdm_dup_hca_attr(hca);
5903 		head->hl_next = NULL;
5904 		gid_info->gl_hca_list = head;
5905 		mutex_exit(&gid_info->gl_mutex);
5906 		IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: "
5907 		    "gid %p, gl_hca_list %p", gid_info,
5908 		    gid_info->gl_hca_list);
5909 		return;
5910 	}
5911 
5912 	/* Check if already in the list */
5913 	while (head) {
5914 		if (head->hl_hca_guid == hca->hl_hca_guid) {
5915 			mutex_exit(&gid_info->gl_mutex);
5916 			IBTF_DPRINTF_L4(ibdm_string,
5917 			    "\taddto_glhcalist : gid %x hca %x dup",
5918 			    gid_info, hca);
5919 			return;
5920 		}
5921 		prev = head;
5922 		head = head->hl_next;
5923 	}
5924 
5925 	/* Add this HCA to gl_hca_list */
5926 	temp =  ibdm_dup_hca_attr(hca);
5927 	temp->hl_next = NULL;
5928 	prev->hl_next = temp;
5929 
5930 	mutex_exit(&gid_info->gl_mutex);
5931 
5932 	IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: "
5933 	    "gid %p, gl_hca_list %p", gid_info, gid_info->gl_hca_list);
5934 }
5935 
5936 static void
5937 ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *gid_info)
5938 {
5939 	ASSERT(!MUTEX_HELD(&gid_info->gl_mutex));
5940 	ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex));
5941 
5942 	mutex_enter(&gid_info->gl_mutex);
5943 	if (gid_info->gl_hca_list)
5944 		ibdm_ibnex_free_hca_list(gid_info->gl_hca_list);
5945 	gid_info->gl_hca_list = NULL;
5946 	mutex_exit(&gid_info->gl_mutex);
5947 }
5948 
5949 
5950 static void
5951 ibdm_reset_all_dgids(ibmf_saa_handle_t port_sa_hdl)
5952 {
5953 	IBTF_DPRINTF_L4(ibdm_string, "\treset_all_dgids(%X)",
5954 	    port_sa_hdl);
5955 
5956 	ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex));
5957 	ASSERT(!MUTEX_HELD(&ibdm.ibdm_hl_mutex));
5958 
5959 	/* Check : Not busy in another probe / sweep */
5960 	mutex_enter(&ibdm.ibdm_mutex);
5961 	if ((ibdm.ibdm_busy & IBDM_BUSY) == 0) {
5962 		ibdm_dp_gidinfo_t	*gid_info;
5963 
5964 		ibdm.ibdm_busy |= IBDM_BUSY;
5965 		mutex_exit(&ibdm.ibdm_mutex);
5966 
5967 		/*
5968 		 * Check if any GID is using the SA & IBMF handle
5969 		 * of HCA port going down. Reset ibdm_dp_gidinfo_t
5970 		 * using another HCA port which can reach the GID.
5971 		 * This is for DM capable GIDs only, no need to do
5972 		 * this for others
5973 		 *
5974 		 * Delete the GID if no alternate HCA port to reach
5975 		 * it is found.
5976 		 */
5977 		for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) {
5978 			ibdm_dp_gidinfo_t *tmp;
5979 
5980 			IBTF_DPRINTF_L4(ibdm_string, "\tevent_hdlr "
5981 			    "checking gidinfo %p", gid_info);
5982 
5983 			if (gid_info->gl_nodeguid != 0 &&
5984 			    gid_info->gl_sa_hdl == port_sa_hdl) {
5985 				IBTF_DPRINTF_L3(ibdm_string,
5986 				    "\tevent_hdlr: down HCA port hdl "
5987 				    "matches gid %p", gid_info);
5988 
5989 				ibdm_reset_gidinfo(gid_info);
5990 				if (gid_info->gl_disconnected) {
5991 					IBTF_DPRINTF_L3(ibdm_string,
5992 					    "\tevent_hdlr: deleting"
5993 					    " gid %p", gid_info);
5994 					tmp = gid_info;
5995 					gid_info = gid_info->gl_next;
5996 					ibdm_delete_gidinfo(tmp);
5997 				} else
5998 					gid_info = gid_info->gl_next;
5999 			} else
6000 				gid_info = gid_info->gl_next;
6001 		}
6002 
6003 		mutex_enter(&ibdm.ibdm_mutex);
6004 		ibdm.ibdm_busy &= ~IBDM_BUSY;
6005 		cv_signal(&ibdm.ibdm_busy_cv);
6006 	}
6007 	mutex_exit(&ibdm.ibdm_mutex);
6008 }
6009 
6010 static void
6011 ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *gidinfo)
6012 {
6013 	ibdm_hca_list_t	*hca_list = NULL;
6014 	ibdm_port_attr_t	*port = NULL;
6015 	int	gid_reinited = 0;
6016 	sa_node_record_t	*nr, *tmp;
6017 	sa_portinfo_record_t	*pi;
6018 	size_t	nr_len = 0, pi_len = 0;
6019 	size_t	path_len;
6020 	ib_gid_t	sgid, dgid;
6021 	int	ret, ii, nrecords;
6022 	sa_path_record_t	*path;
6023 	uint8_t	npaths = 1;
6024 
6025 	IBTF_DPRINTF_L4(ibdm_string, "\treset_gidinfo(%p)", gidinfo);
6026 
6027 	/*
6028 	 * Get list of all the ports reachable from the local known HCA
6029 	 * ports which are active
6030 	 */
6031 	mutex_enter(&ibdm.ibdm_hl_mutex);
6032 	for (ibdm_get_next_port(&hca_list, &port, 1); port;
6033 	    ibdm_get_next_port(&hca_list, &port, 1)) {
6034 
6035 
6036 		/*
6037 		 * Get the path and re-populate the gidinfo.
6038 		 * Getting the path is the same probe_ioc
6039 		 * Init the gid info as in ibdm_create_gidinfo()
6040 		 */
6041 		nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len,
6042 		    gidinfo->gl_nodeguid);
6043 		if (nr == NULL) {
6044 			IBTF_DPRINTF_L4(ibdm_string,
6045 			    "\treset_gidinfo : no records");
6046 			continue;
6047 		}
6048 
6049 		nrecords = (nr_len / sizeof (sa_node_record_t));
6050 		for (tmp = nr, ii = 0;  (ii < nrecords); ii++, tmp++) {
6051 			if (tmp->NodeInfo.PortGUID == gidinfo->gl_portguid)
6052 				break;
6053 		}
6054 
6055 		if (ii == nrecords) {
6056 			IBTF_DPRINTF_L4(ibdm_string,
6057 			    "\treset_gidinfo : no record for portguid");
6058 			kmem_free(nr, nr_len);
6059 			continue;
6060 		}
6061 
6062 		pi = ibdm_get_portinfo(port->pa_sa_hdl, &pi_len, tmp->LID);
6063 		if (pi == NULL) {
6064 			IBTF_DPRINTF_L4(ibdm_string,
6065 			    "\treset_gidinfo : no portinfo");
6066 			kmem_free(nr, nr_len);
6067 			continue;
6068 		}
6069 
6070 		sgid.gid_prefix = port->pa_sn_prefix;
6071 		sgid.gid_guid = port->pa_port_guid;
6072 		dgid.gid_prefix = pi->PortInfo.GidPrefix;
6073 		dgid.gid_guid = tmp->NodeInfo.PortGUID;
6074 
6075 		ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl, sgid, dgid,
6076 		    IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0, &path_len, &path);
6077 
6078 		if ((ret != IBMF_SUCCESS) || path == NULL) {
6079 			IBTF_DPRINTF_L4(ibdm_string,
6080 			    "\treset_gidinfo : no paths");
6081 			kmem_free(pi, pi_len);
6082 			kmem_free(nr, nr_len);
6083 			continue;
6084 		}
6085 
6086 		gidinfo->gl_dgid_hi	= path->DGID.gid_prefix;
6087 		gidinfo->gl_dgid_lo	= path->DGID.gid_guid;
6088 		gidinfo->gl_sgid_hi	= path->SGID.gid_prefix;
6089 		gidinfo->gl_sgid_lo	= path->SGID.gid_guid;
6090 		gidinfo->gl_p_key		= path->P_Key;
6091 		gidinfo->gl_sa_hdl		= port->pa_sa_hdl;
6092 		gidinfo->gl_ibmf_hdl	= port->pa_ibmf_hdl;
6093 		gidinfo->gl_slid		= path->SLID;
6094 		gidinfo->gl_dlid		= path->DLID;
6095 
6096 		/* Reset redirect info, next MAD will set if redirected */
6097 		gidinfo->gl_redirected = 0;
6098 
6099 		gid_reinited = 1;
6100 
6101 		kmem_free(path, path_len);
6102 		kmem_free(pi, pi_len);
6103 		kmem_free(nr, nr_len);
6104 		break;
6105 	}
6106 	mutex_exit(&ibdm.ibdm_hl_mutex);
6107 
6108 	if (!gid_reinited)
6109 		gidinfo->gl_disconnected = 1;
6110 }
6111 
6112 static void
6113 ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *gidinfo)
6114 {
6115 	ibdm_ioc_info_t *ioc_list;
6116 	int	in_gidlist = 0;
6117 
6118 	/*
6119 	 * Check if gidinfo has been inserted into the
6120 	 * ibdm_dp_gidlist_head list. gl_next or gl_prev
6121 	 * != NULL, if gidinfo is the list.
6122 	 */
6123 	if (gidinfo->gl_prev != NULL ||
6124 	    gidinfo->gl_next != NULL)
6125 		in_gidlist = 1;
6126 
6127 	ioc_list = ibdm_update_ioc_gidlist(gidinfo, 0);
6128 
6129 	/*
6130 	 * Remove GID from the global GID list
6131 	 * Handle the case where all port GIDs for an
6132 	 * IOU have been hot-removed.
6133 	 */
6134 	mutex_enter(&ibdm.ibdm_mutex);
6135 	if (gidinfo->gl_iou != NULL && gidinfo->gl_ngids == 0) {
6136 		mutex_enter(&gidinfo->gl_mutex);
6137 		(void) ibdm_free_iou_info(gidinfo);
6138 		mutex_exit(&gidinfo->gl_mutex);
6139 	}
6140 
6141 	if (in_gidlist) {
6142 		if (gidinfo->gl_prev != NULL)
6143 			gidinfo->gl_prev->gl_next = gidinfo->gl_next;
6144 		if (gidinfo->gl_next != NULL)
6145 			gidinfo->gl_next->gl_prev = gidinfo->gl_prev;
6146 
6147 		if (gidinfo == ibdm.ibdm_dp_gidlist_head)
6148 			ibdm.ibdm_dp_gidlist_head = gidinfo->gl_next;
6149 		if (gidinfo == ibdm.ibdm_dp_gidlist_tail)
6150 			ibdm.ibdm_dp_gidlist_tail = gidinfo->gl_prev;
6151 		ibdm.ibdm_ngids--;
6152 	}
6153 	mutex_exit(&ibdm.ibdm_mutex);
6154 
6155 	mutex_destroy(&gidinfo->gl_mutex);
6156 	kmem_free(gidinfo, sizeof (ibdm_dp_gidinfo_t));
6157 
6158 	/*
6159 	 * Pass on the IOCs with updated GIDs to IBnexus
6160 	 */
6161 	if (ioc_list) {
6162 		IBTF_DPRINTF_L4("ibdm", "\tdelete_gidinfo "
6163 		    "IOC_PROP_UPDATE for %p\n", ioc_list);
6164 		mutex_enter(&ibdm.ibdm_ibnex_mutex);
6165 		if (ibdm.ibdm_ibnex_callback != NULL) {
6166 			(*ibdm.ibdm_ibnex_callback)((void *)
6167 			    ioc_list, IBDM_EVENT_IOC_PROP_UPDATE);
6168 		}
6169 		mutex_exit(&ibdm.ibdm_ibnex_mutex);
6170 	}
6171 }
6172 
6173 
6174 static void
6175 ibdm_fill_srv_attr_mod(ib_mad_hdr_t *hdr, ibdm_timeout_cb_args_t *cb_args)
6176 {
6177 	uint32_t	attr_mod;
6178 
6179 	attr_mod = (cb_args->cb_ioc_num + 1) << 16;
6180 	attr_mod |= cb_args->cb_srvents_start;
6181 	attr_mod |= (cb_args->cb_srvents_end) << 8;
6182 	hdr->AttributeModifier = h2b32(attr_mod);
6183 }
6184 
6185 static void
6186 ibdm_bump_transactionID(ibdm_dp_gidinfo_t *gid_info)
6187 {
6188 	ASSERT(MUTEX_HELD(&gid_info->gl_mutex));
6189 	gid_info->gl_transactionID++;
6190 	if (gid_info->gl_transactionID == gid_info->gl_max_transactionID) {
6191 		IBTF_DPRINTF_L4(ibdm_string,
6192 		    "\tbump_transactionID(%p), wrapup", gid_info);
6193 		gid_info->gl_transactionID = gid_info->gl_min_transactionID;
6194 	}
6195 }
6196 
6197 /* For debugging purpose only */
6198 #ifdef	DEBUG
6199 void
6200 ibdm_dump_ibmf_msg(ibmf_msg_t *ibmf_msg, int flag)
6201 {
6202 	ib_mad_hdr_t	*mad_hdr;
6203 
6204 	IBTF_DPRINTF_L4("ibdm", "\t\t(IBMF_PKT): Local address info");
6205 	IBTF_DPRINTF_L4("ibdm", "\t\t            ------------------");
6206 
6207 	IBTF_DPRINTF_L4("ibdm", "\tLocal Lid  : 0x%x\tRemote Lid : 0x%x"
6208 	    " Remote Qp  : 0x%x", ibmf_msg->im_local_addr.ia_local_lid,
6209 	    ibmf_msg->im_local_addr.ia_remote_lid,
6210 	    ibmf_msg->im_local_addr.ia_remote_qno);
6211 	IBTF_DPRINTF_L4("ibdm", "\tP_key      : 0x%x\tQ_key      : 0x%x",
6212 	    ibmf_msg->im_local_addr.ia_p_key, ibmf_msg->im_local_addr.ia_q_key);
6213 
6214 	if (flag)
6215 		mad_hdr = (ib_mad_hdr_t *)IBDM_OUT_IBMFMSG_MADHDR(ibmf_msg);
6216 	else
6217 		mad_hdr = IBDM_IN_IBMFMSG_MADHDR(ibmf_msg);
6218 
6219 	IBTF_DPRINTF_L4("ibdm", "\t\t MAD Header info");
6220 	IBTF_DPRINTF_L4("ibdm", "\t\t ---------------");
6221 
6222 	IBTF_DPRINTF_L4("ibdm", "\tBase version  : 0x%x"
6223 	    "\tMgmt Class : 0x%x", mad_hdr->BaseVersion, mad_hdr->MgmtClass);
6224 	IBTF_DPRINTF_L4("ibdm", "\tClass version : 0x%x"
6225 	    "\tR Method           : 0x%x",
6226 	    mad_hdr->ClassVersion, mad_hdr->R_Method);
6227 	IBTF_DPRINTF_L4("ibdm", "\tMAD  Status   : 0x%x"
6228 	    "\tTransaction ID     : 0x%llx",
6229 	    mad_hdr->Status, mad_hdr->TransactionID);
6230 	IBTF_DPRINTF_L4("ibdm", "\t Attribute ID  : 0x%x"
6231 	    "\tAttribute Modified : 0x%lx",
6232 	    mad_hdr->AttributeID, mad_hdr->AttributeModifier);
6233 }
6234 
6235 void
6236 ibdm_dump_path_info(sa_path_record_t *path)
6237 {
6238 	IBTF_DPRINTF_L4("ibdm", "\t\t Path information");
6239 	IBTF_DPRINTF_L4("ibdm", "\t\t ----------------");
6240 
6241 	IBTF_DPRINTF_L4("ibdm", "\t DGID hi  : %llx\tDGID lo  : %llx",
6242 	    path->DGID.gid_prefix, path->DGID.gid_guid);
6243 	IBTF_DPRINTF_L4("ibdm", "\t SGID hi  : %llx\tSGID lo  : %llx",
6244 	    path->SGID.gid_prefix, path->SGID.gid_guid);
6245 	IBTF_DPRINTF_L4("ibdm", "\t SLID     : %x\tDlID     : %x",
6246 	    path->SLID, path->DLID);
6247 	IBTF_DPRINTF_L4("ibdm", "\t P Key    : %x", path->P_Key);
6248 }
6249 
6250 
6251 void
6252 ibdm_dump_classportinfo(ibdm_mad_classportinfo_t *classportinfo)
6253 {
6254 	IBTF_DPRINTF_L4("ibdm", "\t\t CLASSPORT INFO");
6255 	IBTF_DPRINTF_L4("ibdm", "\t\t --------------");
6256 
6257 	IBTF_DPRINTF_L4("ibdm", "\t Response Time Value : 0x%x",
6258 	    ((b2h32(classportinfo->RespTimeValue)) & 0x1F));
6259 
6260 	IBTF_DPRINTF_L4("ibdm", "\t Redirected QP       : 0x%x",
6261 	    (b2h32(classportinfo->RedirectQP)));
6262 	IBTF_DPRINTF_L4("ibdm", "\t Redirected P KEY    : 0x%x",
6263 	    b2h16(classportinfo->RedirectP_Key));
6264 	IBTF_DPRINTF_L4("ibdm", "\t Redirected Q KEY    : 0x%x",
6265 	    b2h16(classportinfo->RedirectQ_Key));
6266 	IBTF_DPRINTF_L4("ibdm", "\t Redirected GID hi   : 0x%x",
6267 	    b2h64(classportinfo->RedirectGID_hi));
6268 	IBTF_DPRINTF_L4("ibdm", "\t Redirected GID lo   : 0x%x",
6269 	    b2h64(classportinfo->RedirectGID_lo));
6270 }
6271 
6272 
6273 void
6274 ibdm_dump_iounitinfo(ib_dm_io_unitinfo_t *iou_info)
6275 {
6276 	IBTF_DPRINTF_L4("ibdm", "\t\t I/O UnitInfo");
6277 	IBTF_DPRINTF_L4("ibdm", "\t\t ------------");
6278 
6279 	IBTF_DPRINTF_L4("ibdm", "\tChange ID            : 0x%x",
6280 	    b2h16(iou_info->iou_changeid));
6281 	IBTF_DPRINTF_L4("ibdm", "\t#of ctrl slots       : %d",
6282 	    iou_info->iou_num_ctrl_slots);
6283 	IBTF_DPRINTF_L4("ibdm", "\tIOU flag             : 0x%x",
6284 	    iou_info->iou_flag);
6285 	IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 0   : 0x%x",
6286 	    iou_info->iou_ctrl_list[0]);
6287 	IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 1   : 0x%x",
6288 	    iou_info->iou_ctrl_list[1]);
6289 	IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 2   : 0x%x",
6290 	    iou_info->iou_ctrl_list[2]);
6291 }
6292 
6293 
6294 void
6295 ibdm_dump_ioc_profile(ib_dm_ioc_ctrl_profile_t *ioc)
6296 {
6297 	IBTF_DPRINTF_L4("ibdm", "\t\t IOC Controller Profile");
6298 	IBTF_DPRINTF_L4("ibdm", "\t\t ----------------------");
6299 
6300 	IBTF_DPRINTF_L4("ibdm", "\tIOC Guid    : %llx", ioc->ioc_guid);
6301 	IBTF_DPRINTF_L4("ibdm", "\tVendorID    : 0x%x", ioc->ioc_vendorid);
6302 	IBTF_DPRINTF_L4("ibdm", "\tDevice Id   : 0x%x", ioc->ioc_deviceid);
6303 	IBTF_DPRINTF_L4("ibdm", "\tDevice Ver  : 0x%x", ioc->ioc_device_ver);
6304 	IBTF_DPRINTF_L4("ibdm", "\tSubsys ID   : 0x%x", ioc->ioc_subsys_id);
6305 	IBTF_DPRINTF_L4("ibdm", "\tIO class    : 0x%x", ioc->ioc_io_class);
6306 	IBTF_DPRINTF_L4("ibdm", "\tIO subclass : 0x%x", ioc->ioc_io_subclass);
6307 	IBTF_DPRINTF_L4("ibdm", "\tProtocol    : 0x%x", ioc->ioc_protocol);
6308 	IBTF_DPRINTF_L4("ibdm", "\tProtocolV   : 0x%x", ioc->ioc_protocol_ver);
6309 	IBTF_DPRINTF_L4("ibdm", "\tmsg qdepth  : %d", ioc->ioc_send_msg_qdepth);
6310 	IBTF_DPRINTF_L4("ibdm", "\trdma qdepth : %d",
6311 						ioc->ioc_rdma_read_qdepth);
6312 	IBTF_DPRINTF_L4("ibdm", "\tsndmsg sz   : %d", ioc->ioc_send_msg_sz);
6313 	IBTF_DPRINTF_L4("ibdm", "\trdma xfersz : %d", ioc->ioc_rdma_xfer_sz);
6314 	IBTF_DPRINTF_L4("ibdm", "\topcal mask  : 0x%x",
6315 						ioc->ioc_ctrl_opcap_mask);
6316 	IBTF_DPRINTF_L4("ibdm", "\tsrventries  : %x", ioc->ioc_service_entries);
6317 }
6318 
6319 
6320 void
6321 ibdm_dump_service_entries(ib_dm_srv_t *srv_ents)
6322 {
6323 	IBTF_DPRINTF_L4("ibdm",
6324 	    "\thandle_srventry_mad: service id : %llx", srv_ents->srv_id);
6325 
6326 	IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad: "
6327 	    "Service Name : %s", srv_ents->srv_name);
6328 }
6329 
6330 int ibdm_allow_sweep_fabric_timestamp = 1;
6331 
6332 void
6333 ibdm_dump_sweep_fabric_timestamp(int flag)
6334 {
6335 	static hrtime_t x;
6336 	if (flag) {
6337 		if (ibdm_allow_sweep_fabric_timestamp) {
6338 			IBTF_DPRINTF_L4("ibdm", "\tTime taken to complete "
6339 			    "sweep %lld ms", ((gethrtime() - x)/ 1000000));
6340 		}
6341 		x = 0;
6342 	} else
6343 		x = gethrtime();
6344 }
6345 #endif
6346