xref: /illumos-gate/usr/src/uts/common/io/ib/adapters/hermon/hermon.c (revision 9c865d645a5c60028aed172eea31ca36d81a2ff1)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*
28  * hermon.c
29  *    Hermon (InfiniBand) HCA Driver attach/detach Routines
30  *
31  *    Implements all the routines necessary for the attach, setup,
32  *    initialization (and subsequent possible teardown and detach) of the
33  *    Hermon InfiniBand HCA driver.
34  */
35 
36 #include <sys/types.h>
37 #include <sys/file.h>
38 #include <sys/open.h>
39 #include <sys/conf.h>
40 #include <sys/ddi.h>
41 #include <sys/sunddi.h>
42 #include <sys/modctl.h>
43 #include <sys/stat.h>
44 #include <sys/pci.h>
45 #include <sys/pci_cap.h>
46 #include <sys/bitmap.h>
47 #include <sys/policy.h>
48 
49 #include <sys/ib/adapters/hermon/hermon.h>
50 
51 /* The following works around a problem in pre-2_7_000 firmware. */
52 #define	HERMON_FW_WORKAROUND
53 
54 int hermon_verbose = 0;
55 
56 /* Hermon HCA State Pointer */
57 void *hermon_statep;
58 
59 int debug_vpd = 0;
60 
61 /* Disable the internal error-check polling thread */
62 int hermon_no_inter_err_chk = 0;
63 
64 /*
65  * The Hermon "userland resource database" is common to instances of the
66  * Hermon HCA driver.  This structure "hermon_userland_rsrc_db" contains all
67  * the necessary information to maintain it.
68  */
69 hermon_umap_db_t hermon_userland_rsrc_db;
70 
71 static int hermon_attach(dev_info_t *, ddi_attach_cmd_t);
72 static int hermon_detach(dev_info_t *, ddi_detach_cmd_t);
73 static int hermon_open(dev_t *, int, int, cred_t *);
74 static int hermon_close(dev_t, int, int, cred_t *);
75 static int hermon_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
76 
77 static int hermon_drv_init(hermon_state_t *state, dev_info_t *dip,
78     int instance);
79 static void hermon_drv_fini(hermon_state_t *state);
80 static void hermon_drv_fini2(hermon_state_t *state);
81 static int hermon_isr_init(hermon_state_t *state);
82 static void hermon_isr_fini(hermon_state_t *state);
83 
84 static int hermon_hw_init(hermon_state_t *state);
85 
86 static void hermon_hw_fini(hermon_state_t *state,
87     hermon_drv_cleanup_level_t cleanup);
88 static int hermon_soft_state_init(hermon_state_t *state);
89 static void hermon_soft_state_fini(hermon_state_t *state);
90 static int hermon_icm_config_setup(hermon_state_t *state,
91     hermon_hw_initqueryhca_t *inithca);
92 static void hermon_icm_tables_init(hermon_state_t *state);
93 static void hermon_icm_tables_fini(hermon_state_t *state);
94 static int hermon_icm_dma_init(hermon_state_t *state);
95 static void hermon_icm_dma_fini(hermon_state_t *state);
96 static void hermon_inithca_set(hermon_state_t *state,
97     hermon_hw_initqueryhca_t *inithca);
98 static int hermon_hca_port_init(hermon_state_t *state);
99 static int hermon_hca_ports_shutdown(hermon_state_t *state, uint_t num_init);
100 static int hermon_internal_uarpg_init(hermon_state_t *state);
101 static void hermon_internal_uarpg_fini(hermon_state_t *state);
102 static int hermon_special_qp_contexts_reserve(hermon_state_t *state);
103 static void hermon_special_qp_contexts_unreserve(hermon_state_t *state);
104 static int hermon_sw_reset(hermon_state_t *state);
105 static int hermon_mcg_init(hermon_state_t *state);
106 static void hermon_mcg_fini(hermon_state_t *state);
107 static int hermon_fw_version_check(hermon_state_t *state);
108 static void hermon_device_info_report(hermon_state_t *state);
109 static int hermon_pci_capability_list(hermon_state_t *state,
110     ddi_acc_handle_t hdl);
111 static void hermon_pci_capability_vpd(hermon_state_t *state,
112     ddi_acc_handle_t hdl, uint_t offset);
113 static int hermon_pci_read_vpd(ddi_acc_handle_t hdl, uint_t offset,
114     uint32_t addr, uint32_t *data);
115 static int hermon_intr_or_msi_init(hermon_state_t *state);
116 static int hermon_add_intrs(hermon_state_t *state, int intr_type);
117 static int hermon_intr_or_msi_fini(hermon_state_t *state);
118 void hermon_pci_capability_msix(hermon_state_t *state, ddi_acc_handle_t hdl,
119     uint_t offset);
120 
121 static uint64_t hermon_size_icm(hermon_state_t *state);
122 
123 /* X86 fastreboot support */
124 static ushort_t get_msix_ctrl(dev_info_t *);
125 static size_t get_msix_tbl_size(dev_info_t *);
126 static size_t get_msix_pba_size(dev_info_t *);
127 static void hermon_set_msix_info(hermon_state_t *);
128 static int hermon_intr_disable(hermon_state_t *);
129 static int hermon_quiesce(dev_info_t *);
130 
131 
132 /* Character/Block Operations */
133 static struct cb_ops hermon_cb_ops = {
134 	hermon_open,		/* open */
135 	hermon_close,		/* close */
136 	nodev,			/* strategy (block) */
137 	nodev,			/* print (block) */
138 	nodev,			/* dump (block) */
139 	nodev,			/* read */
140 	nodev,			/* write */
141 	hermon_ioctl,		/* ioctl */
142 	hermon_devmap,		/* devmap */
143 	NULL,			/* mmap */
144 	nodev,			/* segmap */
145 	nochpoll,		/* chpoll */
146 	ddi_prop_op,		/* prop_op */
147 	NULL,			/* streams */
148 	D_NEW | D_MP |
149 	D_64BIT | /* D_HOTPLUG | */
150 	D_DEVMAP,		/* flags */
151 	CB_REV			/* rev */
152 };
153 
154 /* Driver Operations */
155 static struct dev_ops hermon_ops = {
156 	DEVO_REV,		/* struct rev */
157 	0,			/* refcnt */
158 	hermon_getinfo,		/* getinfo */
159 	nulldev,		/* identify */
160 	nulldev,		/* probe */
161 	hermon_attach,		/* attach */
162 	hermon_detach,		/* detach */
163 	nodev,			/* reset */
164 	&hermon_cb_ops,		/* cb_ops */
165 	NULL,			/* bus_ops */
166 	nodev,			/* power */
167 	hermon_quiesce,		/* devo_quiesce */
168 };
169 
170 /* Module Driver Info */
171 static struct modldrv hermon_modldrv = {
172 	&mod_driverops,
173 	"ConnectX IB Driver",
174 	&hermon_ops
175 };
176 
177 /* Module Linkage */
178 static struct modlinkage hermon_modlinkage = {
179 	MODREV_1,
180 	&hermon_modldrv,
181 	NULL
182 };
183 
184 /*
185  * This extern refers to the ibc_operations_t function vector that is defined
186  * in the hermon_ci.c file.
187  */
188 extern ibc_operations_t	hermon_ibc_ops;
189 
190 /*
191  * _init()
192  */
193 int
194 _init()
195 {
196 	int	status;
197 
198 	status = ddi_soft_state_init(&hermon_statep, sizeof (hermon_state_t),
199 	    (size_t)HERMON_INITIAL_STATES);
200 	if (status != 0) {
201 		return (status);
202 	}
203 
204 	status = ibc_init(&hermon_modlinkage);
205 	if (status != 0) {
206 		ddi_soft_state_fini(&hermon_statep);
207 		return (status);
208 	}
209 
210 	status = mod_install(&hermon_modlinkage);
211 	if (status != 0) {
212 		ibc_fini(&hermon_modlinkage);
213 		ddi_soft_state_fini(&hermon_statep);
214 		return (status);
215 	}
216 
217 	/* Initialize the Hermon "userland resources database" */
218 	hermon_umap_db_init();
219 
220 	return (status);
221 }
222 
223 
224 /*
225  * _info()
226  */
227 int
228 _info(struct modinfo *modinfop)
229 {
230 	int	status;
231 
232 	status = mod_info(&hermon_modlinkage, modinfop);
233 	return (status);
234 }
235 
236 
237 /*
238  * _fini()
239  */
240 int
241 _fini()
242 {
243 	int	status;
244 
245 	status = mod_remove(&hermon_modlinkage);
246 	if (status != 0) {
247 		return (status);
248 	}
249 
250 	/* Destroy the Hermon "userland resources database" */
251 	hermon_umap_db_fini();
252 
253 	ibc_fini(&hermon_modlinkage);
254 	ddi_soft_state_fini(&hermon_statep);
255 
256 	return (status);
257 }
258 
259 
260 /*
261  * hermon_getinfo()
262  */
263 /* ARGSUSED */
264 static int
265 hermon_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
266 {
267 	dev_t		dev;
268 	hermon_state_t 	*state;
269 	minor_t		instance;
270 
271 	switch (cmd) {
272 	case DDI_INFO_DEVT2DEVINFO:
273 		dev = (dev_t)arg;
274 		instance = HERMON_DEV_INSTANCE(dev);
275 		state = ddi_get_soft_state(hermon_statep, instance);
276 		if (state == NULL) {
277 			return (DDI_FAILURE);
278 		}
279 		*result = (void *)state->hs_dip;
280 		return (DDI_SUCCESS);
281 
282 	case DDI_INFO_DEVT2INSTANCE:
283 		dev = (dev_t)arg;
284 		instance = HERMON_DEV_INSTANCE(dev);
285 		*result = (void *)(uintptr_t)instance;
286 		return (DDI_SUCCESS);
287 
288 	default:
289 		break;
290 	}
291 
292 	return (DDI_FAILURE);
293 }
294 
295 
296 /*
297  * hermon_open()
298  */
299 /* ARGSUSED */
300 static int
301 hermon_open(dev_t *devp, int flag, int otyp, cred_t *credp)
302 {
303 	hermon_state_t		*state;
304 	hermon_rsrc_t 		*rsrcp;
305 	hermon_umap_db_entry_t	*umapdb, *umapdb2;
306 	minor_t			instance;
307 	uint64_t		key, value;
308 	uint_t			hr_indx;
309 	dev_t			dev;
310 	int			status;
311 
312 	instance = HERMON_DEV_INSTANCE(*devp);
313 	state = ddi_get_soft_state(hermon_statep, instance);
314 	if (state == NULL) {
315 		return (ENXIO);
316 	}
317 
318 	/*
319 	 * Only allow driver to be opened for character access, and verify
320 	 * whether exclusive access is allowed.
321 	 */
322 	if ((otyp != OTYP_CHR) || ((flag & FEXCL) &&
323 	    secpolicy_excl_open(credp) != 0)) {
324 		return (EINVAL);
325 	}
326 
327 	/*
328 	 * Search for the current process PID in the "userland resources
329 	 * database".  If it is not found, then attempt to allocate a UAR
330 	 * page and add the ("key", "value") pair to the database.
331 	 * Note:  As a last step we always return a devp appropriate for
332 	 * the open.  Either we return a new minor number (based on the
333 	 * instance and the UAR page index) or we return the current minor
334 	 * number for the given client process.
335 	 *
336 	 * We also add an entry to the database to allow for lookup from
337 	 * "dev_t" to the current process PID.  This is necessary because,
338 	 * under certain circumstance, the process PID that calls the Hermon
339 	 * close() entry point may not be the same as the one who called
340 	 * open().  Specifically, this can happen if a child process calls
341 	 * the Hermon's open() entry point, gets a UAR page, maps it out (using
342 	 * mmap()), and then exits without calling munmap().  Because mmap()
343 	 * adds a reference to the file descriptor, at the exit of the child
344 	 * process the file descriptor is "inherited" by the parent (and will
345 	 * be close()'d by the parent's PID only when it exits).
346 	 *
347 	 * Note: We use the hermon_umap_db_find_nolock() and
348 	 * hermon_umap_db_add_nolock() database access routines below (with
349 	 * an explicit mutex_enter of the database lock - "hdl_umapdb_lock")
350 	 * to ensure that the multiple accesses (in this case searching for,
351 	 * and then adding _two_ database entries) can be done atomically.
352 	 */
353 	key = ddi_get_pid();
354 	mutex_enter(&hermon_userland_rsrc_db.hdl_umapdb_lock);
355 	status = hermon_umap_db_find_nolock(instance, key,
356 	    MLNX_UMAP_UARPG_RSRC, &value, 0, NULL);
357 	if (status != DDI_SUCCESS) {
358 		/*
359 		 * If we are in 'maintenance mode', we cannot alloc a UAR page.
360 		 * But we still need some rsrcp value, and a mostly unique
361 		 * hr_indx value.  So we set rsrcp to NULL for maintenance
362 		 * mode, and use a rolling count for hr_indx.  The field
363 		 * 'hs_open_hr_indx' is used only in this maintenance mode
364 		 * condition.
365 		 *
366 		 * Otherwise, if we are in operational mode then we allocate
367 		 * the UAR page as normal, and use the rsrcp value and tr_indx
368 		 * value from that allocation.
369 		 */
370 		if (!HERMON_IS_OPERATIONAL(state->hs_operational_mode)) {
371 			rsrcp = NULL;
372 			hr_indx = state->hs_open_ar_indx++;
373 		} else {
374 			/* Allocate a new UAR page for this process */
375 			status = hermon_rsrc_alloc(state, HERMON_UARPG, 1,
376 			    HERMON_NOSLEEP, &rsrcp);
377 			if (status != DDI_SUCCESS) {
378 				mutex_exit(
379 				    &hermon_userland_rsrc_db.hdl_umapdb_lock);
380 				return (EAGAIN);
381 			}
382 
383 			hr_indx = rsrcp->hr_indx;
384 		}
385 
386 		/*
387 		 * Allocate an entry to track the UAR page resource in the
388 		 * "userland resources database".
389 		 */
390 		umapdb = hermon_umap_db_alloc(instance, key,
391 		    MLNX_UMAP_UARPG_RSRC, (uint64_t)(uintptr_t)rsrcp);
392 		if (umapdb == NULL) {
393 			mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
394 			/* If in "maintenance mode", don't free the rsrc */
395 			if (HERMON_IS_OPERATIONAL(state->hs_operational_mode)) {
396 				hermon_rsrc_free(state, &rsrcp);
397 			}
398 			return (EAGAIN);
399 		}
400 
401 		/*
402 		 * Create a new device number.  Minor number is a function of
403 		 * the UAR page index (15 bits) and the device instance number
404 		 * (3 bits).
405 		 */
406 		dev = makedevice(getmajor(*devp), (hr_indx <<
407 		    HERMON_MINORNUM_SHIFT) | instance);
408 
409 		/*
410 		 * Allocate another entry in the "userland resources database"
411 		 * to track the association of the device number (above) to
412 		 * the current process ID (in "key").
413 		 */
414 		umapdb2 = hermon_umap_db_alloc(instance, dev,
415 		    MLNX_UMAP_PID_RSRC, (uint64_t)key);
416 		if (umapdb2 == NULL) {
417 			mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
418 			hermon_umap_db_free(umapdb);
419 			/* If in "maintenance mode", don't free the rsrc */
420 			if (HERMON_IS_OPERATIONAL(state->hs_operational_mode)) {
421 				hermon_rsrc_free(state, &rsrcp);
422 			}
423 			return (EAGAIN);
424 		}
425 
426 		/* Add the entries to the database */
427 		hermon_umap_db_add_nolock(umapdb);
428 		hermon_umap_db_add_nolock(umapdb2);
429 
430 	} else {
431 		/*
432 		 * Return the same device number as on the original open()
433 		 * call.  This was calculated as a function of the UAR page
434 		 * index (top 16 bits) and the device instance number
435 		 */
436 		rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
437 		dev = makedevice(getmajor(*devp), (rsrcp->hr_indx <<
438 		    HERMON_MINORNUM_SHIFT) | instance);
439 	}
440 	mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
441 
442 	*devp = dev;
443 
444 	return (0);
445 }
446 
447 
448 /*
449  * hermon_close()
450  */
451 /* ARGSUSED */
452 static int
453 hermon_close(dev_t dev, int flag, int otyp, cred_t *credp)
454 {
455 	hermon_state_t		*state;
456 	hermon_rsrc_t		*rsrcp;
457 	hermon_umap_db_entry_t	*umapdb;
458 	hermon_umap_db_priv_t	*priv;
459 	minor_t			instance;
460 	uint64_t		key, value;
461 	int			status, reset_status = 0;
462 
463 	instance = HERMON_DEV_INSTANCE(dev);
464 	state = ddi_get_soft_state(hermon_statep, instance);
465 	if (state == NULL) {
466 		return (ENXIO);
467 	}
468 
469 	/*
470 	 * Search for "dev_t" in the "userland resources database".  As
471 	 * explained above in hermon_open(), we can't depend on using the
472 	 * current process ID here to do the lookup because the process
473 	 * that ultimately closes may not be the same one who opened
474 	 * (because of inheritance).
475 	 * So we lookup the "dev_t" (which points to the PID of the process
476 	 * that opened), and we remove the entry from the database (and free
477 	 * it up).  Then we do another query based on the PID value.  And when
478 	 * we find that database entry, we free it up too and then free the
479 	 * Hermon UAR page resource.
480 	 *
481 	 * Note: We use the hermon_umap_db_find_nolock() database access
482 	 * routine below (with an explicit mutex_enter of the database lock)
483 	 * to ensure that the multiple accesses (which attempt to remove the
484 	 * two database entries) can be done atomically.
485 	 *
486 	 * This works the same in both maintenance mode and HCA mode, except
487 	 * for the call to hermon_rsrc_free().  In the case of maintenance mode,
488 	 * this call is not needed, as it was not allocated in hermon_open()
489 	 * above.
490 	 */
491 	key = dev;
492 	mutex_enter(&hermon_userland_rsrc_db.hdl_umapdb_lock);
493 	status = hermon_umap_db_find_nolock(instance, key, MLNX_UMAP_PID_RSRC,
494 	    &value, HERMON_UMAP_DB_REMOVE, &umapdb);
495 	if (status == DDI_SUCCESS) {
496 		/*
497 		 * If the "hdb_priv" field is non-NULL, it indicates that
498 		 * some "on close" handling is still necessary.  Call
499 		 * hermon_umap_db_handle_onclose_cb() to do the handling (i.e.
500 		 * to invoke all the registered callbacks).  Then free up
501 		 * the resources associated with "hdb_priv" and continue
502 		 * closing.
503 		 */
504 		priv = (hermon_umap_db_priv_t *)umapdb->hdbe_common.hdb_priv;
505 		if (priv != NULL) {
506 			reset_status = hermon_umap_db_handle_onclose_cb(priv);
507 			kmem_free(priv, sizeof (hermon_umap_db_priv_t));
508 			umapdb->hdbe_common.hdb_priv = (void *)NULL;
509 		}
510 
511 		hermon_umap_db_free(umapdb);
512 
513 		/*
514 		 * Now do another lookup using PID as the key (copy it from
515 		 * "value").  When this lookup is complete, the "value" field
516 		 * will contain the hermon_rsrc_t pointer for the UAR page
517 		 * resource.
518 		 */
519 		key = value;
520 		status = hermon_umap_db_find_nolock(instance, key,
521 		    MLNX_UMAP_UARPG_RSRC, &value, HERMON_UMAP_DB_REMOVE,
522 		    &umapdb);
523 		if (status == DDI_SUCCESS) {
524 			hermon_umap_db_free(umapdb);
525 			/* If in "maintenance mode", don't free the rsrc */
526 			if (HERMON_IS_OPERATIONAL(state->hs_operational_mode)) {
527 				rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
528 				hermon_rsrc_free(state, &rsrcp);
529 			}
530 		}
531 	}
532 	mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
533 	return (reset_status);
534 }
535 
536 
537 /*
538  * hermon_attach()
539  *    Context: Only called from attach() path context
540  */
541 static int
542 hermon_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
543 {
544 	hermon_state_t	*state;
545 	ibc_clnt_hdl_t	tmp_ibtfpriv;
546 	ibc_status_t	ibc_status;
547 	int		instance;
548 	int		status;
549 
550 #ifdef __lock_lint
551 	(void) hermon_quiesce(dip);
552 #endif
553 
554 	switch (cmd) {
555 	case DDI_ATTACH:
556 		instance = ddi_get_instance(dip);
557 		status = ddi_soft_state_zalloc(hermon_statep, instance);
558 		if (status != DDI_SUCCESS) {
559 			cmn_err(CE_NOTE, "hermon%d: driver failed to attach: "
560 			    "attach_ssz_fail", instance);
561 			goto fail_attach_nomsg;
562 
563 		}
564 		state = ddi_get_soft_state(hermon_statep, instance);
565 		if (state == NULL) {
566 			ddi_soft_state_free(hermon_statep, instance);
567 			cmn_err(CE_NOTE, "hermon%d: driver failed to attach: "
568 			    "attach_gss_fail", instance);
569 			goto fail_attach_nomsg;
570 		}
571 
572 		/* clear the attach error buffer */
573 		HERMON_ATTACH_MSG_INIT(state->hs_attach_buf);
574 
575 		/* Save away devinfo and instance before hermon_fm_init() */
576 		state->hs_dip = dip;
577 		state->hs_instance = instance;
578 
579 		hermon_fm_init(state);
580 
581 		/*
582 		 * Initialize Hermon driver and hardware.
583 		 *
584 		 * Note: If this initialization fails we may still wish to
585 		 * create a device node and remain operational so that Hermon
586 		 * firmware can be updated/flashed (i.e. "maintenance mode").
587 		 * If this is the case, then "hs_operational_mode" will be
588 		 * equal to HERMON_MAINTENANCE_MODE.  We will not attempt to
589 		 * attach to the IBTF or register with the IBMF (i.e. no
590 		 * InfiniBand interfaces will be enabled).
591 		 */
592 		status = hermon_drv_init(state, dip, instance);
593 		if ((status != DDI_SUCCESS) &&
594 		    (HERMON_IS_OPERATIONAL(state->hs_operational_mode))) {
595 			goto fail_attach;
596 		}
597 
598 		/*
599 		 * Change the Hermon FM mode
600 		 */
601 		if ((hermon_get_state(state) & HCA_PIO_FM) &&
602 		    HERMON_IS_OPERATIONAL(state->hs_operational_mode)) {
603 			/*
604 			 * Now we wait for 50ms to give an opportunity
605 			 * to Solaris FMA so that HW errors can be notified.
606 			 * Then check if there are HW errors or not. If
607 			 * a HW error is detected, the Hermon attachment
608 			 * must be failed.
609 			 */
610 			delay(drv_usectohz(50000));
611 			if (hermon_init_failure(state)) {
612 				hermon_drv_fini(state);
613 				HERMON_WARNING(state, "unable to "
614 				    "attach Hermon due to a HW error");
615 				HERMON_ATTACH_MSG(state->hs_attach_buf,
616 				    "hermon_attach_failure");
617 				goto fail_attach;
618 			}
619 
620 			/*
621 			 * There seems no HW errors during the attachment,
622 			 * so let's change the Hermon FM state to the
623 			 * ereport only mode.
624 			 */
625 			if (hermon_fm_ereport_init(state) != DDI_SUCCESS) {
626 				/* unwind the resources */
627 				hermon_drv_fini(state);
628 				HERMON_ATTACH_MSG(state->hs_attach_buf,
629 				    "hermon_attach_failure");
630 				goto fail_attach;
631 			}
632 		}
633 
634 		/* Create the minor node for device */
635 		status = ddi_create_minor_node(dip, "devctl", S_IFCHR, instance,
636 		    DDI_PSEUDO, 0);
637 		if (status != DDI_SUCCESS) {
638 			hermon_drv_fini(state);
639 			HERMON_ATTACH_MSG(state->hs_attach_buf,
640 			    "attach_create_mn_fail");
641 			goto fail_attach;
642 		}
643 
644 		/*
645 		 * If we are in "maintenance mode", then we don't want to
646 		 * register with the IBTF.  All InfiniBand interfaces are
647 		 * uninitialized, and the device is only capable of handling
648 		 * requests to update/flash firmware (or test/debug requests).
649 		 */
650 		if (HERMON_IS_OPERATIONAL(state->hs_operational_mode)) {
651 			cmn_err(CE_NOTE, "!Hermon is operational\n");
652 
653 			/* Attach to InfiniBand Transport Framework (IBTF) */
654 			ibc_status = ibc_attach(&tmp_ibtfpriv,
655 			    &state->hs_ibtfinfo);
656 			if (ibc_status != IBC_SUCCESS) {
657 				cmn_err(CE_CONT, "hermon_attach: ibc_attach "
658 				    "failed\n");
659 				ddi_remove_minor_node(dip, "devctl");
660 				hermon_drv_fini(state);
661 				HERMON_ATTACH_MSG(state->hs_attach_buf,
662 				    "attach_ibcattach_fail");
663 				goto fail_attach;
664 			}
665 
666 			/*
667 			 * Now that we've successfully attached to the IBTF,
668 			 * we enable all appropriate asynch and CQ events to
669 			 * be forwarded to the IBTF.
670 			 */
671 			HERMON_ENABLE_IBTF_CALLB(state, tmp_ibtfpriv);
672 
673 			ibc_post_attach(state->hs_ibtfpriv);
674 
675 			/* Register agents with IB Mgmt Framework (IBMF) */
676 			status = hermon_agent_handlers_init(state);
677 			if (status != DDI_SUCCESS) {
678 				(void) ibc_pre_detach(tmp_ibtfpriv, DDI_DETACH);
679 				HERMON_QUIESCE_IBTF_CALLB(state);
680 				if (state->hs_in_evcallb != 0) {
681 					HERMON_WARNING(state, "unable to "
682 					    "quiesce Hermon IBTF callbacks");
683 				}
684 				ibc_detach(tmp_ibtfpriv);
685 				ddi_remove_minor_node(dip, "devctl");
686 				hermon_drv_fini(state);
687 				HERMON_ATTACH_MSG(state->hs_attach_buf,
688 				    "attach_agentinit_fail");
689 				goto fail_attach;
690 			}
691 		}
692 
693 		/* Report attach in maintenance mode, if appropriate */
694 		if (!(HERMON_IS_OPERATIONAL(state->hs_operational_mode))) {
695 			cmn_err(CE_NOTE, "hermon%d: driver attached "
696 			    "(for maintenance mode only)", state->hs_instance);
697 			hermon_fm_ereport(state, HCA_IBA_ERR, HCA_ERR_DEGRADED);
698 		}
699 
700 		/* Report that driver was loaded */
701 		ddi_report_dev(dip);
702 
703 		/* Send device information to log file */
704 		hermon_device_info_report(state);
705 
706 		/* DEBUG PRINT */
707 		cmn_err(CE_CONT, "!Hermon attach complete\n");
708 		return (DDI_SUCCESS);
709 
710 	case DDI_RESUME:
711 		/* Add code here for DDI_RESUME XXX */
712 		return (DDI_FAILURE);
713 
714 	default:
715 		cmn_err(CE_WARN, "hermon_attach: unknown cmd (0x%x)\n", cmd);
716 		break;
717 	}
718 
719 fail_attach:
720 	cmn_err(CE_NOTE, "hermon%d: driver failed to attach: %s", instance,
721 	    state->hs_attach_buf);
722 	if (hermon_get_state(state) & HCA_EREPORT_FM) {
723 		hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
724 	}
725 	hermon_drv_fini2(state);
726 	hermon_fm_fini(state);
727 	ddi_soft_state_free(hermon_statep, instance);
728 
729 fail_attach_nomsg:
730 	return (DDI_FAILURE);
731 }
732 
733 
734 /*
735  * hermon_detach()
736  *    Context: Only called from detach() path context
737  */
738 static int
739 hermon_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
740 {
741 	hermon_state_t	*state;
742 	ibc_clnt_hdl_t	tmp_ibtfpriv;
743 	ibc_status_t	ibc_status;
744 	int		instance, status;
745 
746 	instance = ddi_get_instance(dip);
747 	state = ddi_get_soft_state(hermon_statep, instance);
748 	if (state == NULL) {
749 		return (DDI_FAILURE);
750 	}
751 
752 	switch (cmd) {
753 	case DDI_DETACH:
754 		/*
755 		 * If we are in "maintenance mode", then we do not want to
756 		 * do teardown for any of the InfiniBand interfaces.
757 		 * Specifically, this means not detaching from IBTF (we never
758 		 * attached to begin with) and not deregistering from IBMF.
759 		 */
760 		if (HERMON_IS_OPERATIONAL(state->hs_operational_mode)) {
761 			/* Unregister agents from IB Mgmt Framework (IBMF) */
762 			status = hermon_agent_handlers_fini(state);
763 			if (status != DDI_SUCCESS) {
764 				return (DDI_FAILURE);
765 			}
766 
767 			/*
768 			 * Attempt the "pre-detach" from InfiniBand Transport
769 			 * Framework (IBTF).  At this point the IBTF is still
770 			 * capable of handling incoming asynch and completion
771 			 * events.  This "pre-detach" is primarily a mechanism
772 			 * to notify the appropriate IBTF clients that the
773 			 * HCA is being removed/offlined.
774 			 */
775 			ibc_status = ibc_pre_detach(state->hs_ibtfpriv, cmd);
776 			if (ibc_status != IBC_SUCCESS) {
777 				status = hermon_agent_handlers_init(state);
778 				if (status != DDI_SUCCESS) {
779 					HERMON_WARNING(state, "failed to "
780 					    "restart Hermon agents");
781 				}
782 				return (DDI_FAILURE);
783 			}
784 
785 			/*
786 			 * Before we can fully detach from the IBTF we need to
787 			 * ensure that we have handled all outstanding event
788 			 * callbacks.  This is accomplished by quiescing the
789 			 * event callback mechanism.  Note: if we are unable
790 			 * to successfully quiesce the callbacks, then this is
791 			 * an indication that something has probably gone
792 			 * seriously wrong.  We print out a warning, but
793 			 * continue.
794 			 */
795 			tmp_ibtfpriv = state->hs_ibtfpriv;
796 			HERMON_QUIESCE_IBTF_CALLB(state);
797 			if (state->hs_in_evcallb != 0) {
798 				HERMON_WARNING(state, "unable to quiesce "
799 				    "Hermon IBTF callbacks");
800 			}
801 
802 			/* Complete the detach from the IBTF */
803 			ibc_detach(tmp_ibtfpriv);
804 		}
805 
806 		/* Remove the minor node for device */
807 		ddi_remove_minor_node(dip, "devctl");
808 
809 		/*
810 		 * Only call hermon_drv_fini() if we are in Hermon HCA mode.
811 		 * (Because if we are in "maintenance mode", then we never
812 		 * successfully finished init.)  Only report successful
813 		 * detach for normal HCA mode.
814 		 */
815 		if (HERMON_IS_OPERATIONAL(state->hs_operational_mode)) {
816 			/* Cleanup driver resources and shutdown hardware */
817 			hermon_drv_fini(state);
818 			cmn_err(CE_CONT, "!Hermon driver successfully "
819 			    "detached\n");
820 		}
821 
822 		hermon_drv_fini2(state);
823 		hermon_fm_fini(state);
824 		ddi_soft_state_free(hermon_statep, instance);
825 
826 		return (DDI_SUCCESS);
827 
828 	case DDI_SUSPEND:
829 		/* Add code here for DDI_SUSPEND XXX */
830 		return (DDI_FAILURE);
831 
832 	default:
833 		cmn_err(CE_WARN, "hermon_detach: unknown cmd (0x%x)\n", cmd);
834 		break;
835 	}
836 
837 	return (DDI_FAILURE);
838 }
839 
840 /*
841  * hermon_dma_attr_init()
842  *    Context: Can be called from interrupt or base context.
843  */
844 
845 /* ARGSUSED */
846 void
847 hermon_dma_attr_init(hermon_state_t *state, ddi_dma_attr_t *dma_attr)
848 {
849 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dma_attr))
850 
851 	dma_attr->dma_attr_version	= DMA_ATTR_V0;
852 	dma_attr->dma_attr_addr_lo	= 0;
853 	dma_attr->dma_attr_addr_hi	= 0xFFFFFFFFFFFFFFFFull;
854 	dma_attr->dma_attr_count_max	= 0xFFFFFFFFFFFFFFFFull;
855 	dma_attr->dma_attr_align	= HERMON_PAGESIZE;  /* default 4K */
856 	dma_attr->dma_attr_burstsizes	= 0x3FF;
857 	dma_attr->dma_attr_minxfer	= 1;
858 	dma_attr->dma_attr_maxxfer	= 0xFFFFFFFFFFFFFFFFull;
859 	dma_attr->dma_attr_seg		= 0xFFFFFFFFFFFFFFFFull;
860 	dma_attr->dma_attr_sgllen	= 0x7FFFFFFF;
861 	dma_attr->dma_attr_granular	= 1;
862 	dma_attr->dma_attr_flags	= 0;
863 }
864 
865 /*
866  * hermon_dma_alloc()
867  *    Context: Can be called from base context.
868  */
869 int
870 hermon_dma_alloc(hermon_state_t *state, hermon_dma_info_t *dma_info,
871     uint16_t opcode)
872 {
873 	ddi_dma_handle_t	dma_hdl;
874 	ddi_dma_attr_t		dma_attr;
875 	ddi_acc_handle_t	acc_hdl;
876 	ddi_dma_cookie_t	cookie;
877 	uint64_t		kaddr;
878 	uint64_t		real_len;
879 	uint_t			ccount;
880 	int			status;
881 
882 	hermon_dma_attr_init(state, &dma_attr);
883 
884 	/* Allocate a DMA handle */
885 	status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr, DDI_DMA_SLEEP,
886 	    NULL, &dma_hdl);
887 	if (status != DDI_SUCCESS) {
888 		IBTF_DPRINTF_L2("DMA", "alloc handle failed: %d", status);
889 		cmn_err(CE_CONT, "DMA alloc handle failed(status %d)", status);
890 		return (DDI_FAILURE);
891 	}
892 
893 	/* Allocate DMA memory */
894 	status = ddi_dma_mem_alloc(dma_hdl, dma_info->length,
895 	    &state->hs_reg_accattr, DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
896 	    (caddr_t *)&kaddr, (size_t *)&real_len, &acc_hdl);
897 	if (status != DDI_SUCCESS) {
898 		ddi_dma_free_handle(&dma_hdl);
899 		IBTF_DPRINTF_L2("DMA", "memory alloc failed: %d", status);
900 		cmn_err(CE_CONT, "DMA memory alloc failed(status %d)", status);
901 		return (DDI_FAILURE);
902 	}
903 	bzero((caddr_t)(uintptr_t)kaddr, real_len);
904 
905 	/* Bind the memory to the handle */
906 	status = ddi_dma_addr_bind_handle(dma_hdl, NULL,
907 	    (caddr_t)(uintptr_t)kaddr, (size_t)real_len, DDI_DMA_RDWR |
908 	    DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, &cookie, &ccount);
909 	if (status != DDI_SUCCESS) {
910 		ddi_dma_mem_free(&acc_hdl);
911 		ddi_dma_free_handle(&dma_hdl);
912 		IBTF_DPRINTF_L2("DMA", "bind handle failed: %d", status);
913 		cmn_err(CE_CONT, "DMA bind handle failed(status %d)", status);
914 		return (DDI_FAILURE);
915 	}
916 
917 	/* Package the hermon_dma_info contents and return */
918 	dma_info->vaddr   = kaddr;
919 	dma_info->dma_hdl = dma_hdl;
920 	dma_info->acc_hdl = acc_hdl;
921 
922 	/* Pass the mapping information to the firmware */
923 	status = hermon_map_cmd_post(state, dma_info, opcode, cookie, ccount);
924 	if (status != DDI_SUCCESS) {
925 		char *s;
926 		hermon_dma_free(dma_info);
927 		switch (opcode) {
928 		case MAP_ICM:
929 			s = "MAP_ICM";
930 			break;
931 		case MAP_FA:
932 			s = "MAP_FA";
933 			break;
934 		case MAP_ICM_AUX:
935 			s = "MAP_ICM_AUX";
936 			break;
937 		default:
938 			s = "UNKNOWN";
939 		}
940 		cmn_err(CE_NOTE, "Map cmd '%s' failed, status %08x\n",
941 		    s, status);
942 		return (DDI_FAILURE);
943 	}
944 
945 	return (DDI_SUCCESS);
946 }
947 
948 /*
949  * hermon_dma_free()
950  *    Context: Can be called from base context.
951  */
952 void
953 hermon_dma_free(hermon_dma_info_t *info)
954 {
955 	/* Unbind the handles and free the memory */
956 	(void) ddi_dma_unbind_handle(info->dma_hdl);
957 	ddi_dma_mem_free(&info->acc_hdl);
958 	ddi_dma_free_handle(&info->dma_hdl);
959 }
960 
961 /* These macros are valid for use only in hermon_icm_alloc/hermon_icm_free. */
962 #define	HERMON_ICM_ALLOC(rsrc) \
963 	hermon_icm_alloc(state, rsrc, index1, index2)
964 #define	HERMON_ICM_FREE(rsrc) \
965 	hermon_icm_free(state, rsrc, index1, index2)
966 
967 /*
968  * hermon_icm_alloc()
969  *    Context: Can be called from base context.
970  *
971  * Only one thread can be here for a given hermon_rsrc_type_t "type".
972  */
973 int
974 hermon_icm_alloc(hermon_state_t *state, hermon_rsrc_type_t type,
975     uint32_t index1, uint32_t index2)
976 {
977 	hermon_icm_table_t	*icm;
978 	hermon_dma_info_t	*dma_info;
979 	uint8_t			*bitmap;
980 	int			status;
981 
982 	if (hermon_verbose) {
983 		IBTF_DPRINTF_L2("hermon", "hermon_icm_alloc: rsrc_type (0x%x) "
984 		    "index1/2 (0x%x/0x%x)", type, index1, index2);
985 	}
986 
987 	icm = &state->hs_icm[type];
988 
989 	switch (type) {
990 	case HERMON_QPC:
991 		status = HERMON_ICM_ALLOC(HERMON_CMPT_QPC);
992 		if (status != DDI_SUCCESS) {
993 			return (status);
994 		}
995 		status = HERMON_ICM_ALLOC(HERMON_RDB);
996 		if (status != DDI_SUCCESS) {	/* undo icm_alloc's */
997 			HERMON_ICM_FREE(HERMON_CMPT_QPC);
998 			return (status);
999 		}
1000 		status = HERMON_ICM_ALLOC(HERMON_ALTC);
1001 		if (status != DDI_SUCCESS) {	/* undo icm_alloc's */
1002 			HERMON_ICM_FREE(HERMON_RDB);
1003 			HERMON_ICM_FREE(HERMON_CMPT_QPC);
1004 			return (status);
1005 		}
1006 		status = HERMON_ICM_ALLOC(HERMON_AUXC);
1007 		if (status != DDI_SUCCESS) {	/* undo icm_alloc's */
1008 			HERMON_ICM_FREE(HERMON_ALTC);
1009 			HERMON_ICM_FREE(HERMON_RDB);
1010 			HERMON_ICM_FREE(HERMON_CMPT_QPC);
1011 			return (status);
1012 		}
1013 		break;
1014 	case HERMON_SRQC:
1015 		status = HERMON_ICM_ALLOC(HERMON_CMPT_SRQC);
1016 		if (status != DDI_SUCCESS) {
1017 			return (status);
1018 		}
1019 		break;
1020 	case HERMON_CQC:
1021 		status = HERMON_ICM_ALLOC(HERMON_CMPT_CQC);
1022 		if (status != DDI_SUCCESS) {
1023 			return (status);
1024 		}
1025 		break;
1026 	case HERMON_EQC:
1027 		status = HERMON_ICM_ALLOC(HERMON_CMPT_EQC);
1028 		if (status != DDI_SUCCESS) {	/* undo icm_alloc's */
1029 			return (status);
1030 		}
1031 		break;
1032 	}
1033 
1034 	/* ensure existence of bitmap and dmainfo, sets "dma_info" */
1035 	hermon_bitmap(bitmap, dma_info, icm, index1);
1036 
1037 	/* Set up the DMA handle for allocation and mapping */
1038 	dma_info = icm->icm_dma[index1] + index2;
1039 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dma_info))
1040 	dma_info->length  = icm->span << icm->log_object_size;
1041 	dma_info->icmaddr = icm->icm_baseaddr +
1042 	    (((index1 << icm->split_shift) +
1043 	    (index2 << icm->span_shift)) << icm->log_object_size);
1044 
1045 	if (hermon_verbose) {
1046 		IBTF_DPRINTF_L2("hermon", "alloc DMA: "
1047 		    "rsrc (0x%x) index (%x, %x) "
1048 		    "icm_addr/len (%llx/%x) bitmap %p", type, index1, index2,
1049 		    (longlong_t)dma_info->icmaddr, dma_info->length, bitmap);
1050 	}
1051 
1052 	/* Allocate and map memory for this span */
1053 	status = hermon_dma_alloc(state, dma_info, MAP_ICM);
1054 	if (status != DDI_SUCCESS) {
1055 		IBTF_DPRINTF_L2("hermon", "hermon_icm_alloc: DMA "
1056 		    "allocation failed, status 0x%x", status);
1057 		switch (type) {
1058 		case HERMON_QPC:
1059 			HERMON_ICM_FREE(HERMON_AUXC);
1060 			HERMON_ICM_FREE(HERMON_ALTC);
1061 			HERMON_ICM_FREE(HERMON_RDB);
1062 			HERMON_ICM_FREE(HERMON_CMPT_QPC);
1063 			break;
1064 		case HERMON_SRQC:
1065 			HERMON_ICM_FREE(HERMON_CMPT_SRQC);
1066 			break;
1067 		case HERMON_CQC:
1068 			HERMON_ICM_FREE(HERMON_CMPT_CQC);
1069 			break;
1070 		case HERMON_EQC:
1071 			HERMON_ICM_FREE(HERMON_CMPT_EQC);
1072 			break;
1073 		}
1074 
1075 		return (DDI_FAILURE);
1076 	}
1077 	if (hermon_verbose) {
1078 		IBTF_DPRINTF_L2("hermon", "hermon_icm_alloc: mapping ICM: "
1079 		    "rsrc_type (0x%x) index (0x%x, 0x%x) alloc length (0x%x) "
1080 		    "icm_addr (0x%lx)", type, index1, index2, dma_info->length,
1081 		    dma_info->icmaddr);
1082 	}
1083 
1084 	/* Set the bit for this slot in the table bitmap */
1085 	HERMON_BMAP_BIT_SET(icm->icm_bitmap[index1], index2);
1086 
1087 	return (DDI_SUCCESS);
1088 }
1089 
1090 /*
1091  * hermon_icm_free()
1092  *    Context: Can be called from base context.
1093  *
1094  * ICM resources have been successfully returned from hermon_icm_alloc().
1095  * Associated dma_info is no longer in use.  Free the ICM backing memory.
1096  */
1097 void
1098 hermon_icm_free(hermon_state_t *state, hermon_rsrc_type_t type,
1099     uint32_t index1, uint32_t index2)
1100 {
1101 	hermon_icm_table_t	*icm;
1102 	hermon_dma_info_t	*dma_info;
1103 	int			status;
1104 
1105 	icm = &state->hs_icm[type];
1106 	ASSERT(icm->icm_dma[index1][index2].icm_refcnt == 0);
1107 
1108 	if (hermon_verbose) {
1109 		IBTF_DPRINTF_L2("hermon", "hermon_icm_free: rsrc_type (0x%x) "
1110 		    "index (0x%x, 0x%x)", type, index1, index2);
1111 	}
1112 
1113 	dma_info = icm->icm_dma[index1] + index2;
1114 
1115 	/* The following only happens if attach() is failing. */
1116 	if (dma_info == NULL)
1117 		return;
1118 
1119 	/* Unmap the ICM allocation, then free the backing DMA memory */
1120 	status = hermon_unmap_icm_cmd_post(state, dma_info);
1121 	if (status != DDI_SUCCESS) {
1122 		HERMON_WARNING(state, "UNMAP_ICM failure");
1123 	}
1124 	hermon_dma_free(dma_info);
1125 
1126 	/* Clear the bit in the ICM table bitmap */
1127 	HERMON_BMAP_BIT_CLR(icm->icm_bitmap[index1], index2);
1128 
1129 	switch (type) {
1130 	case HERMON_QPC:
1131 		HERMON_ICM_FREE(HERMON_AUXC);
1132 		HERMON_ICM_FREE(HERMON_ALTC);
1133 		HERMON_ICM_FREE(HERMON_RDB);
1134 		HERMON_ICM_FREE(HERMON_CMPT_QPC);
1135 		break;
1136 	case HERMON_SRQC:
1137 		HERMON_ICM_FREE(HERMON_CMPT_SRQC);
1138 		break;
1139 	case HERMON_CQC:
1140 		HERMON_ICM_FREE(HERMON_CMPT_CQC);
1141 		break;
1142 	case HERMON_EQC:
1143 		HERMON_ICM_FREE(HERMON_CMPT_EQC);
1144 		break;
1145 
1146 	}
1147 }
1148 
1149 /*
1150  * hermon_drv_init()
1151  *    Context: Only called from attach() path context
1152  */
1153 /* ARGSUSED */
1154 static int
1155 hermon_drv_init(hermon_state_t *state, dev_info_t *dip, int instance)
1156 {
1157 	int	status;
1158 
1159 	/*
1160 	 * Check and set the operational mode of the device. If the driver is
1161 	 * bound to the Hermon device in "maintenance mode", then this generally
1162 	 * means that either the device has been specifically jumpered to
1163 	 * start in this mode or the firmware boot process has failed to
1164 	 * successfully load either the primary or the secondary firmware
1165 	 * image.
1166 	 */
1167 	if (HERMON_IS_HCA_MODE(state->hs_dip)) {
1168 		state->hs_operational_mode = HERMON_HCA_MODE;
1169 		state->hs_cfg_profile_setting = HERMON_CFG_MEMFREE;
1170 
1171 	} else if (HERMON_IS_MAINTENANCE_MODE(state->hs_dip)) {
1172 		HERMON_FMANOTE(state, HERMON_FMA_MAINT);
1173 		state->hs_operational_mode = HERMON_MAINTENANCE_MODE;
1174 		state->hs_fm_degraded_reason = HCA_FW_MISC; /* not fw reason */
1175 		return (DDI_FAILURE);
1176 
1177 	} else {
1178 		state->hs_operational_mode = 0;	/* invalid operational mode */
1179 		HERMON_FMANOTE(state, HERMON_FMA_PCIID);
1180 		HERMON_WARNING(state, "unexpected device type detected");
1181 		return (DDI_FAILURE);
1182 	}
1183 
1184 	/*
1185 	 * Initialize the Hermon hardware.
1186 	 *
1187 	 * Note:  If this routine returns an error, it is often a reasonably
1188 	 * good indication that something Hermon firmware-related has caused
1189 	 * the failure or some HW related errors have caused the failure.
1190 	 * (also there are few possibilities that SW (e.g. SW resource
1191 	 * shortage) can cause the failure, but the majority case is due to
1192 	 * either a firmware related error or a HW related one) In order to
1193 	 * give the user an opportunity (if desired) to update or reflash
1194 	 * the Hermon firmware image, we set "hs_operational_mode" flag
1195 	 * (described above) to indicate that we wish to enter maintenance
1196 	 * mode in case of the firmware-related issue.
1197 	 */
1198 	status = hermon_hw_init(state);
1199 	if (status != DDI_SUCCESS) {
1200 		cmn_err(CE_NOTE, "hermon%d: error during attach: %s", instance,
1201 		    state->hs_attach_buf);
1202 		return (DDI_FAILURE);
1203 	}
1204 
1205 	/*
1206 	 * Now that the ISR has been setup, arm all the EQs for event
1207 	 * generation.
1208 	 */
1209 
1210 	status = hermon_eq_arm_all(state);
1211 	if (status != DDI_SUCCESS) {
1212 		cmn_err(CE_NOTE, "EQ Arm All failed\n");
1213 		hermon_hw_fini(state, HERMON_DRV_CLEANUP_ALL);
1214 		return (DDI_FAILURE);
1215 	}
1216 
1217 	/* test interrupts and event queues */
1218 	status = hermon_nop_post(state, 0x0, 0x0);
1219 	if (status != DDI_SUCCESS) {
1220 		cmn_err(CE_NOTE, "Interrupts/EQs failed\n");
1221 		hermon_hw_fini(state, HERMON_DRV_CLEANUP_ALL);
1222 		return (DDI_FAILURE);
1223 	}
1224 
1225 	/* Initialize Hermon softstate */
1226 	status = hermon_soft_state_init(state);
1227 	if (status != DDI_SUCCESS) {
1228 		cmn_err(CE_NOTE, "Failed to init soft state\n");
1229 		hermon_hw_fini(state, HERMON_DRV_CLEANUP_ALL);
1230 		return (DDI_FAILURE);
1231 	}
1232 
1233 	return (DDI_SUCCESS);
1234 }
1235 
1236 
1237 /*
1238  * hermon_drv_fini()
1239  *    Context: Only called from attach() and/or detach() path contexts
1240  */
1241 static void
1242 hermon_drv_fini(hermon_state_t *state)
1243 {
1244 	/* Cleanup Hermon softstate */
1245 	hermon_soft_state_fini(state);
1246 
1247 	/* Cleanup Hermon resources and shutdown hardware */
1248 	hermon_hw_fini(state, HERMON_DRV_CLEANUP_ALL);
1249 }
1250 
1251 
1252 /*
1253  * hermon_drv_fini2()
1254  *    Context: Only called from attach() and/or detach() path contexts
1255  */
1256 static void
1257 hermon_drv_fini2(hermon_state_t *state)
1258 {
1259 	if (state->hs_fm_poll_thread) {
1260 		ddi_periodic_delete(state->hs_fm_poll_thread);
1261 		state->hs_fm_poll_thread = NULL;
1262 	}
1263 
1264 	/* HERMON_DRV_CLEANUP_LEVEL1 */
1265 	if (state->hs_fm_cmdhdl) {
1266 		hermon_regs_map_free(state, &state->hs_fm_cmdhdl);
1267 		state->hs_fm_cmdhdl = NULL;
1268 	}
1269 
1270 	if (state->hs_reg_cmdhdl) {
1271 		ddi_regs_map_free(&state->hs_reg_cmdhdl);
1272 		state->hs_reg_cmdhdl = NULL;
1273 	}
1274 
1275 	/* HERMON_DRV_CLEANUP_LEVEL0 */
1276 	if (state->hs_msix_tbl_entries) {
1277 		kmem_free(state->hs_msix_tbl_entries,
1278 		    state->hs_msix_tbl_size);
1279 		state->hs_msix_tbl_entries = NULL;
1280 	}
1281 
1282 	if (state->hs_msix_pba_entries) {
1283 		kmem_free(state->hs_msix_pba_entries,
1284 		    state->hs_msix_pba_size);
1285 		state->hs_msix_pba_entries = NULL;
1286 	}
1287 
1288 	if (state->hs_fm_msix_tblhdl) {
1289 		hermon_regs_map_free(state, &state->hs_fm_msix_tblhdl);
1290 		state->hs_fm_msix_tblhdl = NULL;
1291 	}
1292 
1293 	if (state->hs_reg_msix_tblhdl) {
1294 		ddi_regs_map_free(&state->hs_reg_msix_tblhdl);
1295 		state->hs_reg_msix_tblhdl = NULL;
1296 	}
1297 
1298 	if (state->hs_fm_msix_pbahdl) {
1299 		hermon_regs_map_free(state, &state->hs_fm_msix_pbahdl);
1300 		state->hs_fm_msix_pbahdl = NULL;
1301 	}
1302 
1303 	if (state->hs_reg_msix_pbahdl) {
1304 		ddi_regs_map_free(&state->hs_reg_msix_pbahdl);
1305 		state->hs_reg_msix_pbahdl = NULL;
1306 	}
1307 
1308 	if (state->hs_fm_pcihdl) {
1309 		hermon_pci_config_teardown(state, &state->hs_fm_pcihdl);
1310 		state->hs_fm_pcihdl = NULL;
1311 	}
1312 
1313 	if (state->hs_reg_pcihdl) {
1314 		pci_config_teardown(&state->hs_reg_pcihdl);
1315 		state->hs_reg_pcihdl = NULL;
1316 	}
1317 }
1318 
1319 
1320 /*
1321  * hermon_isr_init()
1322  *    Context: Only called from attach() path context
1323  */
1324 static int
1325 hermon_isr_init(hermon_state_t *state)
1326 {
1327 	int	status;
1328 	int	intr;
1329 
1330 	for (intr = 0; intr < state->hs_intrmsi_allocd; intr++) {
1331 
1332 		/*
1333 		 * Add a handler for the interrupt or MSI
1334 		 */
1335 		status = ddi_intr_add_handler(state->hs_intrmsi_hdl[intr],
1336 		    hermon_isr, (caddr_t)state, (void *)(uintptr_t)intr);
1337 		if (status  != DDI_SUCCESS) {
1338 			return (DDI_FAILURE);
1339 		}
1340 
1341 		/*
1342 		 * Enable the software interrupt.  Note: depending on the value
1343 		 * returned in the capability flag, we have to call either
1344 		 * ddi_intr_block_enable() or ddi_intr_enable().
1345 		 */
1346 		if (state->hs_intrmsi_cap & DDI_INTR_FLAG_BLOCK) {
1347 			status = ddi_intr_block_enable(
1348 			    &state->hs_intrmsi_hdl[intr], 1);
1349 			if (status != DDI_SUCCESS) {
1350 				return (DDI_FAILURE);
1351 			}
1352 		} else {
1353 			status = ddi_intr_enable(state->hs_intrmsi_hdl[intr]);
1354 			if (status != DDI_SUCCESS) {
1355 				return (DDI_FAILURE);
1356 			}
1357 		}
1358 	}
1359 
1360 	/*
1361 	 * Now that the ISR has been enabled, defer arm_all  EQs for event
1362 	 * generation until later, in case MSIX is enabled
1363 	 */
1364 	return (DDI_SUCCESS);
1365 }
1366 
1367 
1368 /*
1369  * hermon_isr_fini()
1370  *    Context: Only called from attach() and/or detach() path contexts
1371  */
1372 static void
1373 hermon_isr_fini(hermon_state_t *state)
1374 {
1375 	int	intr;
1376 
1377 	for (intr = 0; intr < state->hs_intrmsi_allocd; intr++) {
1378 		/* Disable the software interrupt */
1379 		if (state->hs_intrmsi_cap & DDI_INTR_FLAG_BLOCK) {
1380 			(void) ddi_intr_block_disable(
1381 			    &state->hs_intrmsi_hdl[intr], 1);
1382 		} else {
1383 			(void) ddi_intr_disable(state->hs_intrmsi_hdl[intr]);
1384 		}
1385 
1386 		/*
1387 		 * Remove the software handler for the interrupt or MSI
1388 		 */
1389 		(void) ddi_intr_remove_handler(state->hs_intrmsi_hdl[intr]);
1390 	}
1391 }
1392 
1393 
1394 /*
1395  * Sum of ICM configured values:
1396  *     cMPT, dMPT, MTT, QPC, SRQC, RDB, CQC, ALTC, AUXC, EQC, MCG
1397  *
1398  */
1399 static uint64_t
1400 hermon_size_icm(hermon_state_t *state)
1401 {
1402 	hermon_hw_querydevlim_t	*devlim;
1403 	hermon_cfg_profile_t	*cfg;
1404 	uint64_t		num_cmpts, num_dmpts, num_mtts;
1405 	uint64_t		num_qpcs, num_srqc, num_rdbs;
1406 #ifndef HERMON_FW_WORKAROUND
1407 	uint64_t		num_auxc;
1408 #endif
1409 	uint64_t		num_cqcs, num_altc;
1410 	uint64_t		num_eqcs, num_mcgs;
1411 	uint64_t		size;
1412 
1413 	devlim = &state->hs_devlim;
1414 	cfg = state->hs_cfg_profile;
1415 	/* number of respective entries */
1416 	num_cmpts = (uint64_t)0x1 << cfg->cp_log_num_cmpt;
1417 	num_mtts = (uint64_t)0x1 << cfg->cp_log_num_mtt;
1418 	num_dmpts = (uint64_t)0x1 << cfg->cp_log_num_dmpt;
1419 	num_qpcs = (uint64_t)0x1 << cfg->cp_log_num_qp;
1420 	num_srqc = (uint64_t)0x1 << cfg->cp_log_num_srq;
1421 	num_rdbs = (uint64_t)0x1 << cfg->cp_log_num_rdb;
1422 	num_cqcs = (uint64_t)0x1 << cfg->cp_log_num_cq;
1423 	num_altc = (uint64_t)0x1 << cfg->cp_log_num_qp;
1424 #ifndef HERMON_FW_WORKAROUND
1425 	num_auxc = (uint64_t)0x1 << cfg->cp_log_num_qp;
1426 #endif
1427 	num_eqcs = (uint64_t)0x1 << cfg->cp_log_num_eq;
1428 	num_mcgs = (uint64_t)0x1 << cfg->cp_log_num_mcg;
1429 
1430 	size =
1431 	    num_cmpts 	* devlim->cmpt_entry_sz +
1432 	    num_dmpts	* devlim->dmpt_entry_sz +
1433 	    num_mtts	* devlim->mtt_entry_sz +
1434 	    num_qpcs	* devlim->qpc_entry_sz +
1435 	    num_srqc	* devlim->srq_entry_sz +
1436 	    num_rdbs	* devlim->rdmardc_entry_sz +
1437 	    num_cqcs	* devlim->cqc_entry_sz +
1438 	    num_altc	* devlim->altc_entry_sz +
1439 #ifdef HERMON_FW_WORKAROUND
1440 	    0x80000000ull +
1441 #else
1442 	    num_auxc	* devlim->aux_entry_sz	+
1443 #endif
1444 	    num_eqcs	* devlim->eqc_entry_sz +
1445 	    num_mcgs	* HERMON_MCGMEM_SZ(state);
1446 	return (size);
1447 }
1448 
1449 
1450 /*
1451  * hermon_hw_init()
1452  *    Context: Only called from attach() path context
1453  */
1454 static int
1455 hermon_hw_init(hermon_state_t *state)
1456 {
1457 	hermon_drv_cleanup_level_t	cleanup;
1458 	sm_nodeinfo_t			nodeinfo;
1459 	uint64_t			clr_intr_offset;
1460 	int				status;
1461 	uint32_t			fw_size;	/* in page */
1462 	uint64_t			offset;
1463 
1464 	/* This is where driver initialization begins */
1465 	cleanup = HERMON_DRV_CLEANUP_LEVEL0;
1466 
1467 	/* Setup device access attributes */
1468 	state->hs_reg_accattr.devacc_attr_version = DDI_DEVICE_ATTR_V0;
1469 	state->hs_reg_accattr.devacc_attr_endian_flags = DDI_STRUCTURE_BE_ACC;
1470 	state->hs_reg_accattr.devacc_attr_dataorder = DDI_STRICTORDER_ACC;
1471 	state->hs_reg_accattr.devacc_attr_access = DDI_DEFAULT_ACC;
1472 
1473 	/* Setup fma-protected access attributes */
1474 	state->hs_fm_accattr.devacc_attr_version =
1475 	    hermon_devacc_attr_version(state);
1476 	state->hs_fm_accattr.devacc_attr_endian_flags = DDI_STRUCTURE_BE_ACC;
1477 	state->hs_fm_accattr.devacc_attr_dataorder = DDI_STRICTORDER_ACC;
1478 	/* set acc err protection type */
1479 	state->hs_fm_accattr.devacc_attr_access =
1480 	    hermon_devacc_attr_access(state);
1481 
1482 	/* Setup for PCI config read/write of HCA device */
1483 	status = hermon_pci_config_setup(state, &state->hs_fm_pcihdl);
1484 	if (status != DDI_SUCCESS) {
1485 		hermon_hw_fini(state, cleanup);
1486 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1487 		    "hw_init_PCI_config_space_regmap_fail");
1488 		/* This case is not the degraded one */
1489 		return (DDI_FAILURE);
1490 	}
1491 
1492 	/* Map PCI config space and MSI-X tables/pba */
1493 	hermon_set_msix_info(state);
1494 
1495 	/* Map in Hermon registers (CMD, UAR, MSIX) and setup offsets */
1496 	status = hermon_regs_map_setup(state, HERMON_CMD_BAR,
1497 	    &state->hs_reg_cmd_baseaddr, 0, 0, &state->hs_fm_accattr,
1498 	    &state->hs_fm_cmdhdl);
1499 	if (status != DDI_SUCCESS) {
1500 		hermon_hw_fini(state, cleanup);
1501 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1502 		    "hw_init_CMD_BAR_regmap_fail");
1503 		/* This case is not the degraded one */
1504 		return (DDI_FAILURE);
1505 	}
1506 
1507 	cleanup = HERMON_DRV_CLEANUP_LEVEL1;
1508 	/*
1509 	 * We defer UAR-BAR mapping until later.  Need to know if
1510 	 * blueflame mapping is to be done, and don't know that until after
1511 	 * we get the dev_caps, so do it right after that
1512 	 */
1513 
1514 	/*
1515 	 * There is a third BAR defined for Hermon - it is for MSIX
1516 	 *
1517 	 * Will need to explore it's possible need/use w/ Mellanox
1518 	 * [es] Temporary mapping maybe
1519 	 */
1520 
1521 #ifdef HERMON_SUPPORTS_MSIX_BAR
1522 	status = ddi_regs_map_setup(state->hs_dip, HERMON_MSIX_BAR,
1523 	    &state->hs_reg_msi_baseaddr, 0, 0, &state->hs_reg_accattr,
1524 	    &state->hs_reg_msihdl);
1525 	if (status != DDI_SUCCESS) {
1526 		hermon_hw_fini(state, cleanup);
1527 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1528 		    "hw_init_MSIX_BAR_regmap_fail");
1529 		/* This case is not the degraded one */
1530 		return (DDI_FAILURE);
1531 	}
1532 #endif
1533 
1534 	cleanup = HERMON_DRV_CLEANUP_LEVEL2;
1535 
1536 	/*
1537 	 * Save interesting registers away. The offsets of the first two
1538 	 * here (HCR and sw_reset) are detailed in the PRM, the others are
1539 	 * derived from values in the QUERY_FW output, so we'll save them
1540 	 * off later.
1541 	 */
1542 	/* Host Command Register (HCR) */
1543 	state->hs_cmd_regs.hcr = (hermon_hw_hcr_t *)
1544 	    ((uintptr_t)state->hs_reg_cmd_baseaddr + HERMON_CMD_HCR_OFFSET);
1545 	state->hs_cmd_toggle = 0;	/* initialize it for use */
1546 
1547 	/* Software Reset register (sw_reset) and semaphore */
1548 	state->hs_cmd_regs.sw_reset = (uint32_t *)
1549 	    ((uintptr_t)state->hs_reg_cmd_baseaddr +
1550 	    HERMON_CMD_SW_RESET_OFFSET);
1551 	state->hs_cmd_regs.sw_semaphore = (uint32_t *)
1552 	    ((uintptr_t)state->hs_reg_cmd_baseaddr +
1553 	    HERMON_CMD_SW_SEMAPHORE_OFFSET);
1554 
1555 	/* Retrieve PCI device, vendor and rev IDs */
1556 	state->hs_vendor_id	 = HERMON_GET_VENDOR_ID(state->hs_dip);
1557 	state->hs_device_id	 = HERMON_GET_DEVICE_ID(state->hs_dip);
1558 	state->hs_revision_id	 = HERMON_GET_REVISION_ID(state->hs_dip);
1559 
1560 	/* make sure init'd before we start filling things in */
1561 	bzero(&state->hs_hcaparams, sizeof (struct hermon_hw_initqueryhca_s));
1562 
1563 	/* Initialize the Phase1 configuration profile */
1564 	status = hermon_cfg_profile_init_phase1(state);
1565 	if (status != DDI_SUCCESS) {
1566 		hermon_hw_fini(state, cleanup);
1567 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1568 		    "hw_init_cfginit1_fail");
1569 		/* This case is not the degraded one */
1570 		return (DDI_FAILURE);
1571 	}
1572 	cleanup = HERMON_DRV_CLEANUP_LEVEL3;
1573 
1574 	/* Do a software reset of the adapter to ensure proper state */
1575 	status = hermon_sw_reset(state);
1576 	if (status != HERMON_CMD_SUCCESS) {
1577 		hermon_hw_fini(state, cleanup);
1578 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1579 		    "hw_init_sw_reset_fail");
1580 		/* This case is not the degraded one */
1581 		return (DDI_FAILURE);
1582 	}
1583 
1584 	/* Initialize mailboxes */
1585 	status = hermon_rsrc_init_phase1(state);
1586 	if (status != DDI_SUCCESS) {
1587 		hermon_hw_fini(state, cleanup);
1588 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1589 		    "hw_init_rsrcinit1_fail");
1590 		/* This case is not the degraded one */
1591 		return (DDI_FAILURE);
1592 	}
1593 	cleanup = HERMON_DRV_CLEANUP_LEVEL4;
1594 
1595 	/* Post QUERY_FW */
1596 	status = hermon_cmn_query_cmd_post(state, QUERY_FW, 0, 0, &state->hs_fw,
1597 	    sizeof (hermon_hw_queryfw_t), HERMON_CMD_NOSLEEP_SPIN);
1598 	if (status != HERMON_CMD_SUCCESS) {
1599 		cmn_err(CE_NOTE, "QUERY_FW command failed: %08x\n", status);
1600 		hermon_hw_fini(state, cleanup);
1601 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1602 		    "hw_init_query_fw_cmd_fail");
1603 		/* This case is not the degraded one */
1604 		return (DDI_FAILURE);
1605 	}
1606 
1607 	/* Validate what/that HERMON FW version is appropriate */
1608 
1609 	status = hermon_fw_version_check(state);
1610 	if (status != DDI_SUCCESS) {
1611 		HERMON_FMANOTE(state, HERMON_FMA_FWVER);
1612 		if (state->hs_operational_mode == HERMON_HCA_MODE) {
1613 			cmn_err(CE_CONT, "Unsupported Hermon FW version: "
1614 			    "expected: %04d.%04d.%04d, "
1615 			    "actual: %04d.%04d.%04d\n",
1616 			    HERMON_FW_VER_MAJOR,
1617 			    HERMON_FW_VER_MINOR,
1618 			    HERMON_FW_VER_SUBMINOR,
1619 			    state->hs_fw.fw_rev_major,
1620 			    state->hs_fw.fw_rev_minor,
1621 			    state->hs_fw.fw_rev_subminor);
1622 		} else {
1623 			cmn_err(CE_CONT, "Unsupported FW version: "
1624 			    "%04d.%04d.%04d\n",
1625 			    state->hs_fw.fw_rev_major,
1626 			    state->hs_fw.fw_rev_minor,
1627 			    state->hs_fw.fw_rev_subminor);
1628 		}
1629 		state->hs_operational_mode = HERMON_MAINTENANCE_MODE;
1630 		state->hs_fm_degraded_reason = HCA_FW_MISMATCH;
1631 		hermon_hw_fini(state, cleanup);
1632 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1633 		    "hw_init_checkfwver_fail");
1634 		/* This case is the degraded one */
1635 		return (HERMON_CMD_BAD_NVMEM);
1636 	}
1637 
1638 	/*
1639 	 * Save off the rest of the interesting registers that we'll be using.
1640 	 * Setup the offsets for the other registers.
1641 	 */
1642 
1643 	/*
1644 	 * Hermon does the intr_offset from the BAR - technically should get the
1645 	 * BAR info from the response, but PRM says it's from BAR0-1, which is
1646 	 * for us the CMD BAR
1647 	 */
1648 
1649 	clr_intr_offset	 = state->hs_fw.clr_intr_offs & HERMON_CMD_OFFSET_MASK;
1650 
1651 	/* Save Clear Interrupt address */
1652 	state->hs_cmd_regs.clr_intr = (uint64_t *)
1653 	    (uintptr_t)(state->hs_reg_cmd_baseaddr + clr_intr_offset);
1654 
1655 	/*
1656 	 * Set the error buffer also into the structure - used in hermon_event.c
1657 	 * to check for internal error on the HCA, not reported in eqe or
1658 	 * (necessarily) by interrupt
1659 	 */
1660 	state->hs_cmd_regs.fw_err_buf = (uint32_t *)(uintptr_t)
1661 	    (state->hs_reg_cmd_baseaddr + state->hs_fw.error_buf_addr);
1662 
1663 	/*
1664 	 * Invoke a polling thread to check the error buffer periodically.
1665 	 */
1666 	if (!hermon_no_inter_err_chk) {
1667 		state->hs_fm_poll_thread = ddi_periodic_add(
1668 		    hermon_inter_err_chk, (void *)state, FM_POLL_INTERVAL,
1669 		    DDI_IPL_0);
1670 	}
1671 
1672 	cleanup = HERMON_DRV_CLEANUP_LEVEL5;
1673 
1674 	/*
1675 	 * Allocate, map, and run the HCA Firmware.
1676 	 */
1677 
1678 	/* Allocate memory for the firmware to load into and map it */
1679 
1680 	/* get next higher power of 2 */
1681 	fw_size = 1 << highbit(state->hs_fw.fw_pages);
1682 	state->hs_fw_dma.length = fw_size << HERMON_PAGESHIFT;
1683 	status = hermon_dma_alloc(state, &state->hs_fw_dma, MAP_FA);
1684 	if (status != DDI_SUCCESS) {
1685 		cmn_err(CE_NOTE, "FW alloc failed\n");
1686 		hermon_hw_fini(state, cleanup);
1687 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1688 		    "hw_init_dma_alloc_fw_fail");
1689 		/* This case is not the degraded one */
1690 		return (DDI_FAILURE);
1691 	}
1692 
1693 	cleanup = HERMON_DRV_CLEANUP_LEVEL6;
1694 
1695 	/* Invoke the RUN_FW cmd to run the firmware */
1696 	status = hermon_run_fw_cmd_post(state);
1697 	if (status != DDI_SUCCESS) {
1698 		cmn_err(CE_NOTE, "RUN_FW command failed: 0x%08x\n", status);
1699 		if (status == HERMON_CMD_BAD_NVMEM) {
1700 			state->hs_operational_mode = HERMON_MAINTENANCE_MODE;
1701 			state->hs_fm_degraded_reason = HCA_FW_CORRUPT;
1702 		}
1703 		hermon_hw_fini(state, cleanup);
1704 		HERMON_ATTACH_MSG(state->hs_attach_buf, "hw_init_run_fw_fail");
1705 		/*
1706 		 * If the status is HERMON_CMD_BAD_NVMEM, it's likely the
1707 		 * firmware is corrupted, so the mode falls into the
1708 		 * maintenance mode.
1709 		 */
1710 		return (status == HERMON_CMD_BAD_NVMEM ? HERMON_CMD_BAD_NVMEM :
1711 		    DDI_FAILURE);
1712 	}
1713 
1714 
1715 	/*
1716 	 * QUERY DEVICE LIMITS/CAPABILITIES
1717 	 * NOTE - in Hermon, the command is changed to QUERY_DEV_CAP,
1718 	 * but for familiarity we have kept the structure name the
1719 	 * same as Tavor/Arbel
1720 	 */
1721 
1722 	status = hermon_cmn_query_cmd_post(state, QUERY_DEV_CAP, 0, 0,
1723 	    &state->hs_devlim, sizeof (hermon_hw_querydevlim_t),
1724 	    HERMON_CMD_NOSLEEP_SPIN);
1725 	if (status != HERMON_CMD_SUCCESS) {
1726 		cmn_err(CE_NOTE, "QUERY_DEV_CAP command failed: 0x%08x\n",
1727 		    status);
1728 		hermon_hw_fini(state, cleanup);
1729 		HERMON_ATTACH_MSG(state->hs_attach_buf, "hw_init_devcap_fail");
1730 		/* This case is not the degraded one */
1731 		return (DDI_FAILURE);
1732 	}
1733 
1734 	state->hs_devlim.num_rsvd_eq = max(state->hs_devlim.num_rsvd_eq,
1735 	    (4 * state->hs_devlim.num_rsvd_uar));	/* lesser of resvd's */
1736 
1737 	/* now we have enough info to map in the UAR BAR */
1738 	/*
1739 	 * First, we figure out how to map the BAR for UAR - use only half if
1740 	 * BlueFlame is enabled - in that case the mapped length is 1/2 the
1741 	 * log_max_uar_sz (max__uar - 1) * 1MB ( +20).
1742 	 */
1743 
1744 	if (state->hs_devlim.blu_flm) {		/* Blue Flame Enabled */
1745 		offset = (uint64_t)1 << (state->hs_devlim.log_max_uar_sz + 20);
1746 	} else {
1747 		offset = 0;	/* a zero length means map the whole thing */
1748 	}
1749 	status = hermon_regs_map_setup(state, HERMON_UAR_BAR,
1750 	    &state->hs_reg_uar_baseaddr, 0, offset, &state->hs_fm_accattr,
1751 	    &state->hs_fm_uarhdl);
1752 	if (status != DDI_SUCCESS) {
1753 		HERMON_ATTACH_MSG(state->hs_attach_buf, "UAR BAR mapping");
1754 		/* This case is not the degraded one */
1755 		return (DDI_FAILURE);
1756 	}
1757 
1758 	/* and if BlueFlame is enabled, map the other half there */
1759 	if (state->hs_devlim.blu_flm) {		/* Blue Flame Enabled */
1760 		offset = (uint64_t)1 << (state->hs_devlim.log_max_uar_sz + 20);
1761 		status = ddi_regs_map_setup(state->hs_dip, HERMON_UAR_BAR,
1762 		    &state->hs_reg_bf_baseaddr, offset, offset,
1763 		    &state->hs_reg_accattr, &state->hs_reg_bfhdl);
1764 		if (status != DDI_SUCCESS) {
1765 			HERMON_ATTACH_MSG(state->hs_attach_buf,
1766 			    "BlueFlame BAR mapping");
1767 			/* This case is not the degraded one */
1768 			return (DDI_FAILURE);
1769 		}
1770 		/* This will be used in hw_fini if we fail to init. */
1771 		state->hs_bf_offset = offset;
1772 	}
1773 	cleanup = HERMON_DRV_CLEANUP_LEVEL7;
1774 
1775 	/* Hermon has a couple of things needed for phase 2 in query port */
1776 
1777 	status = hermon_cmn_query_cmd_post(state, QUERY_PORT, 0, 0x01,
1778 	    &state->hs_queryport, sizeof (hermon_hw_query_port_t),
1779 	    HERMON_CMD_NOSLEEP_SPIN);
1780 	if (status != HERMON_CMD_SUCCESS) {
1781 		cmn_err(CE_NOTE, "QUERY_PORT command failed: 0x%08x\n",
1782 		    status);
1783 		hermon_hw_fini(state, cleanup);
1784 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1785 		    "hw_init_queryport_fail");
1786 		/* This case is not the degraded one */
1787 		return (DDI_FAILURE);
1788 	}
1789 
1790 	/* Initialize the Phase2 Hermon configuration profile */
1791 	status = hermon_cfg_profile_init_phase2(state);
1792 	if (status != DDI_SUCCESS) {
1793 		cmn_err(CE_NOTE, "CFG phase 2 failed: 0x%08x\n", status);
1794 		hermon_hw_fini(state, cleanup);
1795 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1796 		    "hw_init_cfginit2_fail");
1797 		/* This case is not the degraded one */
1798 		return (DDI_FAILURE);
1799 	}
1800 
1801 	/* Determine and set the ICM size */
1802 	state->hs_icm_sz = hermon_size_icm(state);
1803 	status		 = hermon_set_icm_size_cmd_post(state);
1804 	if (status != DDI_SUCCESS) {
1805 		cmn_err(CE_NOTE, "Hermon: SET_ICM_SIZE cmd failed: 0x%08x\n",
1806 		    status);
1807 		hermon_hw_fini(state, cleanup);
1808 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1809 		    "hw_init_seticmsz_fail");
1810 		/* This case is not the degraded one */
1811 		return (DDI_FAILURE);
1812 	}
1813 	/* alloc icm aux physical memory and map it */
1814 
1815 	state->hs_icma_dma.length = 1 << highbit(state->hs_icma_sz);
1816 
1817 	status = hermon_dma_alloc(state, &state->hs_icma_dma, MAP_ICM_AUX);
1818 	if (status != DDI_SUCCESS) {
1819 		cmn_err(CE_NOTE, "failed to alloc (0x%llx) bytes for ICMA\n",
1820 		    (longlong_t)state->hs_icma_dma.length);
1821 		hermon_hw_fini(state, cleanup);
1822 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1823 		    "hw_init_dma_alloc_icm_aux_fail");
1824 		/* This case is not the degraded one */
1825 		return (DDI_FAILURE);
1826 	}
1827 	cleanup = HERMON_DRV_CLEANUP_LEVEL8;
1828 
1829 	cleanup = HERMON_DRV_CLEANUP_LEVEL9;
1830 
1831 	/* Allocate an array of structures to house the ICM tables */
1832 	state->hs_icm = kmem_zalloc(HERMON_NUM_ICM_RESOURCES *
1833 	    sizeof (hermon_icm_table_t), KM_SLEEP);
1834 
1835 	/* Set up the ICM address space and the INIT_HCA command input */
1836 	status = hermon_icm_config_setup(state, &state->hs_hcaparams);
1837 	if (status != HERMON_CMD_SUCCESS) {
1838 		cmn_err(CE_NOTE, "ICM configuration failed\n");
1839 		hermon_hw_fini(state, cleanup);
1840 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1841 		    "hw_init_icm_config_setup_fail");
1842 		/* This case is not the degraded one */
1843 		return (DDI_FAILURE);
1844 	}
1845 	cleanup = HERMON_DRV_CLEANUP_LEVEL10;
1846 
1847 	/* Initialize the adapter with the INIT_HCA cmd */
1848 	status = hermon_init_hca_cmd_post(state, &state->hs_hcaparams,
1849 	    HERMON_CMD_NOSLEEP_SPIN);
1850 	if (status != HERMON_CMD_SUCCESS) {
1851 		cmn_err(CE_NOTE, "INIT_HCA command failed: %08x\n", status);
1852 		hermon_hw_fini(state, cleanup);
1853 		HERMON_ATTACH_MSG(state->hs_attach_buf, "hw_init_hca_fail");
1854 		/* This case is not the degraded one */
1855 		return (DDI_FAILURE);
1856 	}
1857 	cleanup = HERMON_DRV_CLEANUP_LEVEL11;
1858 
1859 	/* Enter the second phase of init for Hermon configuration/resources */
1860 	status = hermon_rsrc_init_phase2(state);
1861 	if (status != DDI_SUCCESS) {
1862 		hermon_hw_fini(state, cleanup);
1863 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1864 		    "hw_init_rsrcinit2_fail");
1865 		/* This case is not the degraded one */
1866 		return (DDI_FAILURE);
1867 	}
1868 	cleanup = HERMON_DRV_CLEANUP_LEVEL12;
1869 
1870 	/* Query the adapter via QUERY_ADAPTER */
1871 	status = hermon_cmn_query_cmd_post(state, QUERY_ADAPTER, 0, 0,
1872 	    &state->hs_adapter, sizeof (hermon_hw_queryadapter_t),
1873 	    HERMON_CMD_NOSLEEP_SPIN);
1874 	if (status != HERMON_CMD_SUCCESS) {
1875 		cmn_err(CE_NOTE, "Hermon: QUERY_ADAPTER command failed: %08x\n",
1876 		    status);
1877 		hermon_hw_fini(state, cleanup);
1878 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1879 		    "hw_init_query_adapter_fail");
1880 		/* This case is not the degraded one */
1881 		return (DDI_FAILURE);
1882 	}
1883 
1884 	/* Allocate protection domain (PD) for Hermon internal use */
1885 	status = hermon_pd_alloc(state, &state->hs_pdhdl_internal,
1886 	    HERMON_SLEEP);
1887 	if (status != DDI_SUCCESS) {
1888 		cmn_err(CE_NOTE, "failed to alloc internal PD\n");
1889 		hermon_hw_fini(state, cleanup);
1890 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1891 		    "hw_init_internal_pd_alloc_fail");
1892 		/* This case is not the degraded one */
1893 		return (DDI_FAILURE);
1894 	}
1895 	cleanup = HERMON_DRV_CLEANUP_LEVEL13;
1896 
1897 	/* Setup UAR page for kernel use */
1898 	status = hermon_internal_uarpg_init(state);
1899 	if (status != DDI_SUCCESS) {
1900 		cmn_err(CE_NOTE, "failed to setup internal UAR\n");
1901 		hermon_hw_fini(state, cleanup);
1902 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1903 		    "hw_init_internal_uarpg_alloc_fail");
1904 		/* This case is not the degraded one */
1905 		return (DDI_FAILURE);
1906 	}
1907 	cleanup = HERMON_DRV_CLEANUP_LEVEL14;
1908 
1909 	/* Query and initialize the Hermon interrupt/MSI information */
1910 	status = hermon_intr_or_msi_init(state);
1911 	if (status != DDI_SUCCESS) {
1912 		cmn_err(CE_NOTE, "failed to setup INTR/MSI\n");
1913 		hermon_hw_fini(state, cleanup);
1914 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1915 		    "hw_init_intr_or_msi_init_fail");
1916 		/* This case is not the degraded one */
1917 		return (DDI_FAILURE);
1918 	}
1919 	cleanup = HERMON_DRV_CLEANUP_LEVEL15;
1920 
1921 	status = hermon_isr_init(state);	/* set up the isr */
1922 	if (status != DDI_SUCCESS) {
1923 		cmn_err(CE_NOTE, "failed to init isr\n");
1924 		hermon_hw_fini(state, cleanup);
1925 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1926 		    "hw_init_isrinit_fail");
1927 		/* This case is not the degraded one */
1928 		return (DDI_FAILURE);
1929 	}
1930 	cleanup = HERMON_DRV_CLEANUP_LEVEL16;
1931 
1932 	/* Setup the event queues */
1933 	status = hermon_eq_init_all(state);
1934 	if (status != DDI_SUCCESS) {
1935 		cmn_err(CE_NOTE, "failed to init EQs\n");
1936 		hermon_hw_fini(state, cleanup);
1937 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1938 		    "hw_init_eqinitall_fail");
1939 		/* This case is not the degraded one */
1940 		return (DDI_FAILURE);
1941 	}
1942 	cleanup = HERMON_DRV_CLEANUP_LEVEL17;
1943 
1944 
1945 
1946 	/* Reserve contexts for QP0 and QP1 */
1947 	status = hermon_special_qp_contexts_reserve(state);
1948 	if (status != DDI_SUCCESS) {
1949 		cmn_err(CE_NOTE, "failed to init special QPs\n");
1950 		hermon_hw_fini(state, cleanup);
1951 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1952 		    "hw_init_rsrv_sqp_fail");
1953 		/* This case is not the degraded one */
1954 		return (DDI_FAILURE);
1955 	}
1956 	cleanup = HERMON_DRV_CLEANUP_LEVEL18;
1957 
1958 	/* Initialize for multicast group handling */
1959 	status = hermon_mcg_init(state);
1960 	if (status != DDI_SUCCESS) {
1961 		cmn_err(CE_NOTE, "failed to init multicast\n");
1962 		hermon_hw_fini(state, cleanup);
1963 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1964 		    "hw_init_mcg_init_fail");
1965 		/* This case is not the degraded one */
1966 		return (DDI_FAILURE);
1967 	}
1968 	cleanup = HERMON_DRV_CLEANUP_LEVEL19;
1969 
1970 	/* Initialize the Hermon IB port(s) */
1971 	status = hermon_hca_port_init(state);
1972 	if (status != DDI_SUCCESS) {
1973 		cmn_err(CE_NOTE, "failed to init HCA Port\n");
1974 		hermon_hw_fini(state, cleanup);
1975 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1976 		    "hw_init_hca_port_init_fail");
1977 		/* This case is not the degraded one */
1978 		return (DDI_FAILURE);
1979 	}
1980 
1981 	cleanup = HERMON_DRV_CLEANUP_ALL;
1982 
1983 	/* Determine NodeGUID and SystemImageGUID */
1984 	status = hermon_getnodeinfo_cmd_post(state, HERMON_CMD_NOSLEEP_SPIN,
1985 	    &nodeinfo);
1986 	if (status != HERMON_CMD_SUCCESS) {
1987 		cmn_err(CE_NOTE, "GetNodeInfo command failed: %08x\n", status);
1988 		hermon_hw_fini(state, cleanup);
1989 		HERMON_ATTACH_MSG(state->hs_attach_buf,
1990 		    "hw_init_getnodeinfo_cmd_fail");
1991 		/* This case is not the degraded one */
1992 		return (DDI_FAILURE);
1993 	}
1994 
1995 	/*
1996 	 * If the NodeGUID value was set in OBP properties, then we use that
1997 	 * value.  But we still print a message if the value we queried from
1998 	 * firmware does not match this value.
1999 	 *
2000 	 * Otherwise if OBP value is not set then we use the value from
2001 	 * firmware unconditionally.
2002 	 */
2003 	if (state->hs_cfg_profile->cp_nodeguid) {
2004 		state->hs_nodeguid   = state->hs_cfg_profile->cp_nodeguid;
2005 	} else {
2006 		state->hs_nodeguid = nodeinfo.NodeGUID;
2007 	}
2008 
2009 	if (state->hs_nodeguid != nodeinfo.NodeGUID) {
2010 		cmn_err(CE_NOTE, "!NodeGUID value queried from firmware "
2011 		    "does not match value set by device property");
2012 	}
2013 
2014 	/*
2015 	 * If the SystemImageGUID value was set in OBP properties, then we use
2016 	 * that value.  But we still print a message if the value we queried
2017 	 * from firmware does not match this value.
2018 	 *
2019 	 * Otherwise if OBP value is not set then we use the value from
2020 	 * firmware unconditionally.
2021 	 */
2022 	if (state->hs_cfg_profile->cp_sysimgguid) {
2023 		state->hs_sysimgguid = state->hs_cfg_profile->cp_sysimgguid;
2024 	} else {
2025 		state->hs_sysimgguid = nodeinfo.SystemImageGUID;
2026 	}
2027 
2028 	if (state->hs_sysimgguid != nodeinfo.SystemImageGUID) {
2029 		cmn_err(CE_NOTE, "!SystemImageGUID value queried from firmware "
2030 		    "does not match value set by device property");
2031 	}
2032 
2033 	/* Get NodeDescription */
2034 	status = hermon_getnodedesc_cmd_post(state, HERMON_CMD_NOSLEEP_SPIN,
2035 	    (sm_nodedesc_t *)&state->hs_nodedesc);
2036 	if (status != HERMON_CMD_SUCCESS) {
2037 		cmn_err(CE_CONT, "GetNodeDesc command failed: %08x\n", status);
2038 		hermon_hw_fini(state, cleanup);
2039 		HERMON_ATTACH_MSG(state->hs_attach_buf,
2040 		    "hw_init_getnodedesc_cmd_fail");
2041 		/* This case is not the degraded one */
2042 		return (DDI_FAILURE);
2043 	}
2044 
2045 	return (DDI_SUCCESS);
2046 }
2047 
2048 
2049 /*
2050  * hermon_hw_fini()
2051  *    Context: Only called from attach() and/or detach() path contexts
2052  */
2053 static void
2054 hermon_hw_fini(hermon_state_t *state, hermon_drv_cleanup_level_t cleanup)
2055 {
2056 	uint_t		num_ports;
2057 	int		i, status;
2058 
2059 
2060 	/*
2061 	 * JBDB - We might not want to run these returns in all cases of
2062 	 * Bad News. We should still attempt to free all of the DMA memory
2063 	 * resources...  This needs to be worked last, after all allocations
2064 	 * are implemented. For now, and possibly for later, this works.
2065 	 */
2066 
2067 	switch (cleanup) {
2068 	/*
2069 	 * If we add more driver initialization steps that should be cleaned
2070 	 * up here, we need to ensure that HERMON_DRV_CLEANUP_ALL is still the
2071 	 * first entry (i.e. corresponds to the last init step).
2072 	 */
2073 	case HERMON_DRV_CLEANUP_ALL:
2074 		/* Shutdown the Hermon IB port(s) */
2075 		num_ports = state->hs_cfg_profile->cp_num_ports;
2076 		(void) hermon_hca_ports_shutdown(state, num_ports);
2077 		/* FALLTHROUGH */
2078 
2079 	case HERMON_DRV_CLEANUP_LEVEL19:
2080 		/* Teardown resources used for multicast group handling */
2081 		hermon_mcg_fini(state);
2082 		/* FALLTHROUGH */
2083 
2084 	case HERMON_DRV_CLEANUP_LEVEL18:
2085 		/* Unreserve the special QP contexts */
2086 		hermon_special_qp_contexts_unreserve(state);
2087 		/* FALLTHROUGH */
2088 
2089 	case HERMON_DRV_CLEANUP_LEVEL17:
2090 		/*
2091 		 * Attempt to teardown all event queues (EQ).  If we fail
2092 		 * here then print a warning message and return.  Something
2093 		 * (either in HW or SW) has gone seriously wrong.
2094 		 */
2095 		status = hermon_eq_fini_all(state);
2096 		if (status != DDI_SUCCESS) {
2097 			HERMON_WARNING(state, "failed to teardown EQs");
2098 			return;
2099 		}
2100 		/* FALLTHROUGH */
2101 	case HERMON_DRV_CLEANUP_LEVEL16:
2102 		/* Teardown Hermon interrupts */
2103 		hermon_isr_fini(state);
2104 		/* FALLTHROUGH */
2105 
2106 	case HERMON_DRV_CLEANUP_LEVEL15:
2107 		status = hermon_intr_or_msi_fini(state);
2108 		if (status != DDI_SUCCESS) {
2109 			HERMON_WARNING(state, "failed to free intr/MSI");
2110 			return;
2111 		}
2112 		/* FALLTHROUGH */
2113 
2114 	case HERMON_DRV_CLEANUP_LEVEL14:
2115 		/* Free the resources for the Hermon internal UAR pages */
2116 		hermon_internal_uarpg_fini(state);
2117 		/* FALLTHROUGH */
2118 
2119 	case HERMON_DRV_CLEANUP_LEVEL13:
2120 		/*
2121 		 * Free the PD that was used internally by Hermon software.  If
2122 		 * we fail here then print a warning and return.  Something
2123 		 * (probably software-related, but perhaps HW) has gone wrong.
2124 		 */
2125 		status = hermon_pd_free(state, &state->hs_pdhdl_internal);
2126 		if (status != DDI_SUCCESS) {
2127 			HERMON_WARNING(state, "failed to free internal PD");
2128 			return;
2129 		}
2130 		/* FALLTHROUGH */
2131 
2132 	case HERMON_DRV_CLEANUP_LEVEL12:
2133 		/* Cleanup all the phase2 resources first */
2134 		hermon_rsrc_fini(state, HERMON_RSRC_CLEANUP_ALL);
2135 		/* FALLTHROUGH */
2136 
2137 	case HERMON_DRV_CLEANUP_LEVEL11:
2138 		/* LEVEL11 is after INIT_HCA */
2139 		/* FALLTHROUGH */
2140 
2141 
2142 	case HERMON_DRV_CLEANUP_LEVEL10:
2143 		/*
2144 		 * Unmap the ICM memory area with UNMAP_ICM command.
2145 		 */
2146 		status = hermon_unmap_icm_cmd_post(state, NULL);
2147 		if (status != DDI_SUCCESS) {
2148 			cmn_err(CE_WARN,
2149 			    "hermon_hw_fini: failed to unmap ICM\n");
2150 		}
2151 
2152 		/* Free the initial ICM DMA handles */
2153 		hermon_icm_dma_fini(state);
2154 
2155 		/* Free the ICM table structures */
2156 		hermon_icm_tables_fini(state);
2157 
2158 		/* Free the ICM table handles */
2159 		kmem_free(state->hs_icm, HERMON_NUM_ICM_RESOURCES *
2160 		    sizeof (hermon_icm_table_t));
2161 
2162 		/* FALLTHROUGH */
2163 
2164 	case HERMON_DRV_CLEANUP_LEVEL9:
2165 		/*
2166 		 * Unmap the ICM Aux memory area with UNMAP_ICM_AUX command.
2167 		 */
2168 		status = hermon_unmap_icm_aux_cmd_post(state);
2169 		if (status != HERMON_CMD_SUCCESS) {
2170 			cmn_err(CE_NOTE,
2171 			    "hermon_hw_fini: failed to unmap ICMA\n");
2172 		}
2173 		/* FALLTHROUGH */
2174 
2175 	case HERMON_DRV_CLEANUP_LEVEL8:
2176 		/*
2177 		 * Deallocate ICM Aux DMA memory.
2178 		 */
2179 		hermon_dma_free(&state->hs_icma_dma);
2180 		/* FALLTHROUGH */
2181 
2182 	case HERMON_DRV_CLEANUP_LEVEL7:
2183 		if (state->hs_fm_uarhdl) {
2184 			hermon_regs_map_free(state, &state->hs_fm_uarhdl);
2185 			state->hs_fm_uarhdl = NULL;
2186 		}
2187 
2188 		if (state->hs_reg_uarhdl) {
2189 			ddi_regs_map_free(&state->hs_reg_uarhdl);
2190 			state->hs_reg_uarhdl = NULL;
2191 		}
2192 
2193 		if (state->hs_bf_offset != 0 && state->hs_reg_bfhdl) {
2194 			ddi_regs_map_free(&state->hs_reg_bfhdl);
2195 			state->hs_reg_bfhdl = NULL;
2196 		}
2197 
2198 		for (i = 0; i < HERMON_MAX_PORTS; i++) {
2199 			if (state->hs_pkey[i]) {
2200 				kmem_free(state->hs_pkey[i], (1 <<
2201 				    state->hs_cfg_profile->cp_log_max_pkeytbl) *
2202 				    sizeof (ib_pkey_t));
2203 				state->hs_pkey[i] = NULL;
2204 			}
2205 			if (state->hs_guid[i]) {
2206 				kmem_free(state->hs_guid[i], (1 <<
2207 				    state->hs_cfg_profile->cp_log_max_gidtbl) *
2208 				    sizeof (ib_guid_t));
2209 				state->hs_guid[i] = NULL;
2210 			}
2211 		}
2212 		/* FALLTHROUGH */
2213 
2214 	case HERMON_DRV_CLEANUP_LEVEL6:
2215 		/*
2216 		 * Unmap the firmware memory area with UNMAP_FA command.
2217 		 */
2218 		status = hermon_unmap_fa_cmd_post(state);
2219 
2220 		if (status != HERMON_CMD_SUCCESS) {
2221 			cmn_err(CE_NOTE,
2222 			    "hermon_hw_fini: failed to unmap FW\n");
2223 		}
2224 
2225 		/*
2226 		 * Deallocate firmware DMA memory.
2227 		 */
2228 		hermon_dma_free(&state->hs_fw_dma);
2229 		/* FALLTHROUGH */
2230 
2231 	case HERMON_DRV_CLEANUP_LEVEL5:
2232 		/* stop the poll thread */
2233 		if (state->hs_fm_poll_thread) {
2234 			ddi_periodic_delete(state->hs_fm_poll_thread);
2235 			state->hs_fm_poll_thread = NULL;
2236 		}
2237 		/* FALLTHROUGH */
2238 
2239 	case HERMON_DRV_CLEANUP_LEVEL4:
2240 		/* Then cleanup the phase1 resources */
2241 		hermon_rsrc_fini(state, HERMON_RSRC_CLEANUP_PHASE1_COMPLETE);
2242 		/* FALLTHROUGH */
2243 
2244 	case HERMON_DRV_CLEANUP_LEVEL3:
2245 		/* Teardown any resources allocated for the config profile */
2246 		hermon_cfg_profile_fini(state);
2247 		/* FALLTHROUGH */
2248 
2249 	case HERMON_DRV_CLEANUP_LEVEL2:
2250 #ifdef HERMON_SUPPORTS_MSIX_BAR
2251 		/*
2252 		 * unmap 3rd BAR, MSIX BAR
2253 		 */
2254 		if (state->hs_reg_msihdl) {
2255 			ddi_regs_map_free(&state->hs_reg_msihdl);
2256 			state->hs_reg_msihdl = NULL;
2257 		}
2258 		/* FALLTHROUGH */
2259 #endif
2260 	case HERMON_DRV_CLEANUP_LEVEL1:
2261 	case HERMON_DRV_CLEANUP_LEVEL0:
2262 		/*
2263 		 * LEVEL1 and LEVEL0 resources are freed in
2264 		 * hermon_drv_fini2().
2265 		 */
2266 		break;
2267 
2268 	default:
2269 		HERMON_WARNING(state, "unexpected driver cleanup level");
2270 		return;
2271 	}
2272 }
2273 
2274 
2275 /*
2276  * hermon_soft_state_init()
2277  *    Context: Only called from attach() path context
2278  */
2279 static int
2280 hermon_soft_state_init(hermon_state_t *state)
2281 {
2282 	ibt_hca_attr_t		*hca_attr;
2283 	uint64_t		maxval, val;
2284 	ibt_hca_flags_t		caps = IBT_HCA_NO_FLAGS;
2285 	ibt_hca_flags2_t	caps2 = IBT_HCA2_NO_FLAGS;
2286 	int			status;
2287 	int			max_send_wqe_bytes;
2288 	int			max_recv_wqe_bytes;
2289 
2290 	/*
2291 	 * The ibc_hca_info_t struct is passed to the IBTF.  This is the
2292 	 * routine where we initialize it.  Many of the init values come from
2293 	 * either configuration variables or successful queries of the Hermon
2294 	 * hardware abilities
2295 	 */
2296 	state->hs_ibtfinfo.hca_ci_vers	= IBCI_V3;
2297 	state->hs_ibtfinfo.hca_dip	= state->hs_dip;
2298 	state->hs_ibtfinfo.hca_handle	= (ibc_hca_hdl_t)state;
2299 	state->hs_ibtfinfo.hca_ops	= &hermon_ibc_ops;
2300 
2301 	hca_attr = kmem_zalloc(sizeof (ibt_hca_attr_t), KM_SLEEP);
2302 	state->hs_ibtfinfo.hca_attr = hca_attr;
2303 
2304 	hca_attr->hca_fw_major_version = state->hs_fw.fw_rev_major;
2305 	hca_attr->hca_fw_minor_version = state->hs_fw.fw_rev_minor;
2306 	hca_attr->hca_fw_micro_version = state->hs_fw.fw_rev_subminor;
2307 
2308 	/* CQ interrupt moderation maximums - each limited to 16 bits */
2309 	hca_attr->hca_max_cq_mod_count = 0xFFFF;
2310 	hca_attr->hca_max_cq_mod_usec = 0xFFFF;
2311 
2312 	/* CQ relocation to other EQs - change when multiple MSI-Xs are used */
2313 	hca_attr->hca_max_cq_handlers = 1;
2314 
2315 	/*
2316 	 * Determine HCA capabilities:
2317 	 * No default support for IBT_HCA_RD, IBT_HCA_RAW_MULTICAST,
2318 	 *    IBT_HCA_ATOMICS_GLOBAL, IBT_HCA_RESIZE_CHAN, IBT_HCA_INIT_TYPE,
2319 	 *    or IBT_HCA_SHUTDOWN_PORT
2320 	 * But IBT_HCA_AH_PORT_CHECK, IBT_HCA_SQD_RTS_PORT, IBT_HCA_SI_GUID,
2321 	 *    IBT_HCA_RNR_NAK, IBT_HCA_CURRENT_QP_STATE, IBT_HCA_PORT_UP,
2322 	 *    IBT_HCA_SRQ, IBT_HCA_RESIZE_SRQ and IBT_HCA_FMR are always
2323 	 *    supported
2324 	 * All other features are conditionally supported, depending on the
2325 	 *    status return by the Hermon HCA in QUERY_DEV_LIM.
2326 	 */
2327 	if (state->hs_devlim.ud_multi) {
2328 		caps |= IBT_HCA_UD_MULTICAST;
2329 	}
2330 	if (state->hs_devlim.atomic) {
2331 		caps |= IBT_HCA_ATOMICS_HCA;
2332 	}
2333 	if (state->hs_devlim.apm) {
2334 		caps |= IBT_HCA_AUTO_PATH_MIG;
2335 	}
2336 	if (state->hs_devlim.pkey_v) {
2337 		caps |= IBT_HCA_PKEY_CNTR;
2338 	}
2339 	if (state->hs_devlim.qkey_v) {
2340 		caps |= IBT_HCA_QKEY_CNTR;
2341 	}
2342 	if (state->hs_devlim.ipoib_cksm) {
2343 		caps |= IBT_HCA_CKSUM_FULL;
2344 		caps2 |= IBT_HCA2_IP_CLASS;
2345 	}
2346 	if (state->hs_devlim.mod_wr_srq) {
2347 		caps |= IBT_HCA_RESIZE_SRQ;
2348 	}
2349 	if (state->hs_devlim.lif) {
2350 		caps |= IBT_HCA_LOCAL_INVAL_FENCE;
2351 	}
2352 	if (state->hs_devlim.reserved_lkey) {
2353 		caps2 |= IBT_HCA2_RES_LKEY;
2354 		hca_attr->hca_reserved_lkey = state->hs_devlim.rsv_lkey;
2355 	}
2356 	if (state->hs_devlim.local_inv && state->hs_devlim.remote_inv &&
2357 	    state->hs_devlim.fast_reg_wr) {	/* fw needs to be >= 2.6.636 */
2358 		if (state->hs_fw.fw_rev_major > 2)
2359 			caps2 |= IBT_HCA2_MEM_MGT_EXT;
2360 		else if (state->hs_fw.fw_rev_major == 2)
2361 			if (state->hs_fw.fw_rev_minor > 6)
2362 				caps2 |= IBT_HCA2_MEM_MGT_EXT;
2363 			else if (state->hs_fw.fw_rev_minor == 6)
2364 				if (state->hs_fw.fw_rev_subminor >= 636)
2365 					caps2 |= IBT_HCA2_MEM_MGT_EXT;
2366 	}
2367 	if (state->hs_devlim.mps) {
2368 		caps |= IBT_HCA_ZERO_BASED_VA;
2369 	}
2370 	if (state->hs_devlim.zb) {
2371 		caps |= IBT_HCA_MULT_PAGE_SZ_MR;
2372 	}
2373 	caps |= (IBT_HCA_AH_PORT_CHECK | IBT_HCA_SQD_SQD_PORT |
2374 	    IBT_HCA_SI_GUID | IBT_HCA_RNR_NAK | IBT_HCA_CURRENT_QP_STATE |
2375 	    IBT_HCA_PORT_UP | IBT_HCA_RC_SRQ | IBT_HCA_UD_SRQ | IBT_HCA_FMR);
2376 
2377 	if (state->hs_devlim.log_max_gso_sz) {
2378 		hca_attr->hca_max_lso_size =
2379 		    (1 << state->hs_devlim.log_max_gso_sz);
2380 		/* 64 = ctrl & datagram seg, 4 = LSO seg, 16 = 1 SGL */
2381 		hca_attr->hca_max_lso_hdr_size =
2382 		    state->hs_devlim.max_desc_sz_sq - (64 + 4 + 16);
2383 	}
2384 
2385 	caps |= IBT_HCA_WQE_SIZE_INFO;
2386 	max_send_wqe_bytes = state->hs_devlim.max_desc_sz_sq;
2387 	max_recv_wqe_bytes = state->hs_devlim.max_desc_sz_rq;
2388 	hca_attr->hca_ud_send_sgl_sz = (max_send_wqe_bytes / 16) - 4;
2389 	hca_attr->hca_conn_send_sgl_sz = (max_send_wqe_bytes / 16) - 1;
2390 	hca_attr->hca_conn_rdma_sgl_overhead = 1;
2391 	hca_attr->hca_recv_sgl_sz = max_recv_wqe_bytes / 16;
2392 
2393 	/* We choose not to support "inline" unless it improves performance */
2394 	hca_attr->hca_max_inline_size = 0;
2395 	hca_attr->hca_ud_send_inline_sz = 0;
2396 	hca_attr->hca_conn_send_inline_sz = 0;
2397 	hca_attr->hca_conn_rdmaw_inline_overhead = 4;
2398 
2399 	hca_attr->hca_flags = caps;
2400 	hca_attr->hca_flags2 = caps2;
2401 
2402 	/*
2403 	 * Set hca_attr's IDs
2404 	 */
2405 	hca_attr->hca_vendor_id	 = state->hs_vendor_id;
2406 	hca_attr->hca_device_id	 = state->hs_device_id;
2407 	hca_attr->hca_version_id = state->hs_revision_id;
2408 
2409 	/*
2410 	 * Determine number of available QPs and max QP size.  Number of
2411 	 * available QPs is determined by subtracting the number of
2412 	 * "reserved QPs" (i.e. reserved for firmware use) from the
2413 	 * total number configured.
2414 	 */
2415 	val = ((uint64_t)1 << state->hs_cfg_profile->cp_log_num_qp);
2416 	hca_attr->hca_max_qp = val - ((uint64_t)1 <<
2417 	    state->hs_devlim.log_rsvd_qp);
2418 	maxval	= ((uint64_t)1 << state->hs_devlim.log_max_qp_sz);
2419 	val	= ((uint64_t)1 << state->hs_cfg_profile->cp_log_max_qp_sz);
2420 	if (val > maxval) {
2421 		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2422 		HERMON_ATTACH_MSG(state->hs_attach_buf,
2423 		    "soft_state_init_maxqpsz_toobig_fail");
2424 		return (DDI_FAILURE);
2425 	}
2426 	/* we need to reduce this by the max space needed for headroom */
2427 	hca_attr->hca_max_qp_sz = (uint_t)val - (HERMON_QP_OH_SIZE >>
2428 	    HERMON_QP_WQE_LOG_MINIMUM) - 1;
2429 
2430 	/*
2431 	 * Determine max scatter-gather size in WQEs. The HCA has split
2432 	 * the max sgl into rec'v Q and send Q values. Use the least.
2433 	 *
2434 	 * This is mainly useful for legacy clients.  Smart clients
2435 	 * such as IPoIB will use the IBT_HCA_WQE_SIZE_INFO sgl info.
2436 	 */
2437 	if (state->hs_devlim.max_sg_rq <= state->hs_devlim.max_sg_sq) {
2438 		maxval = state->hs_devlim.max_sg_rq;
2439 	} else {
2440 		maxval = state->hs_devlim.max_sg_sq;
2441 	}
2442 	val	= state->hs_cfg_profile->cp_wqe_max_sgl;
2443 	if (val > maxval) {
2444 		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2445 		HERMON_ATTACH_MSG(state->hs_attach_buf,
2446 		    "soft_state_init_toomanysgl_fail");
2447 		return (DDI_FAILURE);
2448 	}
2449 	/* If the rounded value for max SGL is too large, cap it */
2450 	if (state->hs_cfg_profile->cp_wqe_real_max_sgl > maxval) {
2451 		state->hs_cfg_profile->cp_wqe_real_max_sgl = (uint32_t)maxval;
2452 		val = maxval;
2453 	} else {
2454 		val = state->hs_cfg_profile->cp_wqe_real_max_sgl;
2455 	}
2456 
2457 	hca_attr->hca_max_sgl	 = (uint_t)val;
2458 	hca_attr->hca_max_rd_sgl = 0;	/* zero because RD is unsupported */
2459 
2460 	/*
2461 	 * Determine number of available CQs and max CQ size. Number of
2462 	 * available CQs is determined by subtracting the number of
2463 	 * "reserved CQs" (i.e. reserved for firmware use) from the
2464 	 * total number configured.
2465 	 */
2466 	val = ((uint64_t)1 << state->hs_cfg_profile->cp_log_num_cq);
2467 	hca_attr->hca_max_cq = val - ((uint64_t)1 <<
2468 	    state->hs_devlim.log_rsvd_cq);
2469 	maxval	= ((uint64_t)1 << state->hs_devlim.log_max_cq_sz);
2470 	val	= ((uint64_t)1 << state->hs_cfg_profile->cp_log_max_cq_sz) - 1;
2471 	if (val > maxval) {
2472 		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2473 		HERMON_ATTACH_MSG(state->hs_attach_buf,
2474 		    "soft_state_init_maxcqsz_toobig_fail");
2475 		return (DDI_FAILURE);
2476 	}
2477 	hca_attr->hca_max_cq_sz = (uint_t)val;
2478 
2479 	/*
2480 	 * Determine number of available SRQs and max SRQ size. Number of
2481 	 * available SRQs is determined by subtracting the number of
2482 	 * "reserved SRQs" (i.e. reserved for firmware use) from the
2483 	 * total number configured.
2484 	 */
2485 	val = ((uint64_t)1 << state->hs_cfg_profile->cp_log_num_srq);
2486 	hca_attr->hca_max_srqs = val - ((uint64_t)1 <<
2487 	    state->hs_devlim.log_rsvd_srq);
2488 	maxval  = ((uint64_t)1 << state->hs_devlim.log_max_srq_sz);
2489 	val	= ((uint64_t)1 << state->hs_cfg_profile->cp_log_max_srq_sz);
2490 
2491 	if (val > maxval) {
2492 		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2493 		HERMON_ATTACH_MSG(state->hs_attach_buf,
2494 		    "soft_state_init_maxsrqsz_toobig_fail");
2495 		return (DDI_FAILURE);
2496 	}
2497 	hca_attr->hca_max_srqs_sz = (uint_t)val;
2498 
2499 	val	= hca_attr->hca_recv_sgl_sz - 1; /* SRQ has a list link */
2500 	maxval	= state->hs_devlim.max_sg_rq - 1;
2501 	if (val > maxval) {
2502 		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2503 		HERMON_ATTACH_MSG(state->hs_attach_buf,
2504 		    "soft_state_init_toomanysrqsgl_fail");
2505 		return (DDI_FAILURE);
2506 	}
2507 	hca_attr->hca_max_srq_sgl = (uint_t)val;
2508 
2509 	/*
2510 	 * Determine supported HCA page sizes
2511 	 * XXX
2512 	 * For now we simply return the system pagesize as the only supported
2513 	 * pagesize
2514 	 */
2515 	hca_attr->hca_page_sz = ((PAGESIZE == (1 << 13)) ? IBT_PAGE_8K :
2516 	    IBT_PAGE_4K);
2517 
2518 	/*
2519 	 * Determine number of available MemReg, MemWin, and their max size.
2520 	 * Number of available MRs and MWs is determined by subtracting
2521 	 * the number of "reserved MPTs" (i.e. reserved for firmware use)
2522 	 * from the total number configured for each.
2523 	 */
2524 	val = ((uint64_t)1 << state->hs_cfg_profile->cp_log_num_dmpt);
2525 	hca_attr->hca_max_memr	  = val - ((uint64_t)1 <<
2526 	    state->hs_devlim.log_rsvd_dmpt);
2527 	hca_attr->hca_max_mem_win = state->hs_devlim.mem_win ? (val -
2528 	    ((uint64_t)1 << state->hs_devlim.log_rsvd_dmpt)) : 0;
2529 	maxval	= state->hs_devlim.log_max_mrw_sz;
2530 	val	= state->hs_cfg_profile->cp_log_max_mrw_sz;
2531 	if (val > maxval) {
2532 		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2533 		HERMON_ATTACH_MSG(state->hs_attach_buf,
2534 		    "soft_state_init_maxmrwsz_toobig_fail");
2535 		return (DDI_FAILURE);
2536 	}
2537 	hca_attr->hca_max_memr_len = ((uint64_t)1 << val);
2538 
2539 	/* Determine RDMA/Atomic properties */
2540 	val = ((uint64_t)1 << state->hs_cfg_profile->cp_log_num_rdb);
2541 	hca_attr->hca_max_rsc = (uint_t)val;
2542 	val = state->hs_cfg_profile->cp_hca_max_rdma_in_qp;
2543 	hca_attr->hca_max_rdma_in_qp  = (uint8_t)val;
2544 	val = state->hs_cfg_profile->cp_hca_max_rdma_out_qp;
2545 	hca_attr->hca_max_rdma_out_qp = (uint8_t)val;
2546 	hca_attr->hca_max_rdma_in_ee  = 0;
2547 	hca_attr->hca_max_rdma_out_ee = 0;
2548 
2549 	/*
2550 	 * Determine maximum number of raw IPv6 and Ether QPs.  Set to 0
2551 	 * because neither type of raw QP is supported
2552 	 */
2553 	hca_attr->hca_max_ipv6_qp  = 0;
2554 	hca_attr->hca_max_ether_qp = 0;
2555 
2556 	/* Determine max number of MCGs and max QP-per-MCG */
2557 	val = ((uint64_t)1 << state->hs_cfg_profile->cp_log_num_qp);
2558 	hca_attr->hca_max_mcg_qps   = (uint_t)val;
2559 	val = ((uint64_t)1 << state->hs_cfg_profile->cp_log_num_mcg);
2560 	hca_attr->hca_max_mcg	    = (uint_t)val;
2561 	val = state->hs_cfg_profile->cp_num_qp_per_mcg;
2562 	hca_attr->hca_max_qp_per_mcg = (uint_t)val;
2563 
2564 	/* Determine max number partitions (i.e. PKeys) */
2565 	maxval	= ((uint64_t)state->hs_cfg_profile->cp_num_ports <<
2566 	    state->hs_queryport.log_max_pkey);
2567 	val	= ((uint64_t)state->hs_cfg_profile->cp_num_ports <<
2568 	    state->hs_cfg_profile->cp_log_max_pkeytbl);
2569 
2570 	if (val > maxval) {
2571 		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2572 		HERMON_ATTACH_MSG(state->hs_attach_buf,
2573 		    "soft_state_init_toomanypkey_fail");
2574 		return (DDI_FAILURE);
2575 	}
2576 	hca_attr->hca_max_partitions = (uint16_t)val;
2577 
2578 	/* Determine number of ports */
2579 	maxval = state->hs_devlim.num_ports;
2580 	val = state->hs_cfg_profile->cp_num_ports;
2581 	if ((val > maxval) || (val == 0)) {
2582 		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2583 		HERMON_ATTACH_MSG(state->hs_attach_buf,
2584 		    "soft_state_init_toomanyports_fail");
2585 		return (DDI_FAILURE);
2586 	}
2587 	hca_attr->hca_nports = (uint8_t)val;
2588 
2589 	/* Copy NodeGUID and SystemImageGUID from softstate */
2590 	hca_attr->hca_node_guid = state->hs_nodeguid;
2591 	hca_attr->hca_si_guid	= state->hs_sysimgguid;
2592 
2593 	/*
2594 	 * Determine local ACK delay.  Use the value suggested by the Hermon
2595 	 * hardware (from the QUERY_DEV_CAP command)
2596 	 */
2597 	hca_attr->hca_local_ack_delay = state->hs_devlim.ca_ack_delay;
2598 
2599 	/* Determine max SGID table and PKey table sizes */
2600 	val	= ((uint64_t)1 << state->hs_cfg_profile->cp_log_max_gidtbl);
2601 	hca_attr->hca_max_port_sgid_tbl_sz = (uint_t)val;
2602 	val	= ((uint64_t)1 << state->hs_cfg_profile->cp_log_max_pkeytbl);
2603 	hca_attr->hca_max_port_pkey_tbl_sz = (uint16_t)val;
2604 
2605 	/* Determine max number of PDs */
2606 	maxval	= ((uint64_t)1 << state->hs_devlim.log_max_pd);
2607 	val	= ((uint64_t)1 << state->hs_cfg_profile->cp_log_num_pd);
2608 	if (val > maxval) {
2609 		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2610 		HERMON_ATTACH_MSG(state->hs_attach_buf,
2611 		    "soft_state_init_toomanypd_fail");
2612 		return (DDI_FAILURE);
2613 	}
2614 	hca_attr->hca_max_pd = (uint_t)val;
2615 
2616 	/* Determine max number of Address Handles (NOT IN ARBEL or HERMON) */
2617 	hca_attr->hca_max_ah = 0;
2618 
2619 	/* No RDDs or EECs (since Reliable Datagram is not supported) */
2620 	hca_attr->hca_max_rdd = 0;
2621 	hca_attr->hca_max_eec = 0;
2622 
2623 	/* Initialize lock for reserved UAR page access */
2624 	mutex_init(&state->hs_uar_lock, NULL, MUTEX_DRIVER,
2625 	    DDI_INTR_PRI(state->hs_intrmsi_pri));
2626 
2627 	/* Initialize the flash fields */
2628 	state->hs_fw_flashstarted = 0;
2629 	mutex_init(&state->hs_fw_flashlock, NULL, MUTEX_DRIVER,
2630 	    DDI_INTR_PRI(state->hs_intrmsi_pri));
2631 
2632 	/* Initialize the lock for the info ioctl */
2633 	mutex_init(&state->hs_info_lock, NULL, MUTEX_DRIVER,
2634 	    DDI_INTR_PRI(state->hs_intrmsi_pri));
2635 
2636 	/* Initialize the AVL tree for QP number support */
2637 	hermon_qpn_avl_init(state);
2638 
2639 	/* Initialize the kstat info structure */
2640 	status = hermon_kstat_init(state);
2641 	if (status != DDI_SUCCESS) {
2642 		hermon_qpn_avl_fini(state);
2643 		mutex_destroy(&state->hs_info_lock);
2644 		mutex_destroy(&state->hs_fw_flashlock);
2645 		mutex_destroy(&state->hs_uar_lock);
2646 		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2647 		HERMON_ATTACH_MSG(state->hs_attach_buf,
2648 		    "soft_state_init_kstatinit_fail");
2649 		return (DDI_FAILURE);
2650 	}
2651 
2652 	return (DDI_SUCCESS);
2653 }
2654 
2655 
2656 /*
2657  * hermon_soft_state_fini()
2658  *    Context: Called only from detach() path context
2659  */
2660 static void
2661 hermon_soft_state_fini(hermon_state_t *state)
2662 {
2663 
2664 	/* Teardown the kstat info */
2665 	hermon_kstat_fini(state);
2666 
2667 	/* Teardown the AVL tree for QP number support */
2668 	hermon_qpn_avl_fini(state);
2669 
2670 	/* Free up info ioctl mutex */
2671 	mutex_destroy(&state->hs_info_lock);
2672 
2673 	/* Free up flash mutex */
2674 	mutex_destroy(&state->hs_fw_flashlock);
2675 
2676 	/* Free up the UAR page access mutex */
2677 	mutex_destroy(&state->hs_uar_lock);
2678 
2679 	/* Free up the hca_attr struct */
2680 	kmem_free(state->hs_ibtfinfo.hca_attr, sizeof (ibt_hca_attr_t));
2681 
2682 }
2683 
2684 /*
2685  * hermon_icm_config_setup()
2686  *    Context: Only called from attach() path context
2687  */
2688 static int
2689 hermon_icm_config_setup(hermon_state_t *state,
2690     hermon_hw_initqueryhca_t *inithca)
2691 {
2692 	hermon_hw_querydevlim_t	*devlim;
2693 	hermon_cfg_profile_t	*cfg;
2694 	hermon_icm_table_t	*icm_p[HERMON_NUM_ICM_RESOURCES];
2695 	hermon_icm_table_t	*icm;
2696 	hermon_icm_table_t	*tmp;
2697 	uint64_t		icm_addr;
2698 	uint64_t		icm_size;
2699 	int			status, i, j;
2700 
2701 
2702 	/* Bring in local devlims, cfg_profile and hs_icm table list */
2703 	devlim = &state->hs_devlim;
2704 	cfg = state->hs_cfg_profile;
2705 	icm = state->hs_icm;
2706 
2707 	/*
2708 	 * Assign each ICM table's entry size from data in the devlims,
2709 	 * except for RDB and MCG sizes, which are not returned in devlims
2710 	 * but do have a fixed size, and the UAR context entry size, which
2711 	 * we determine. For this, we use the "cp_num_pgs_per_uce" value
2712 	 * from our hs_cfg_profile.
2713 	 */
2714 	icm[HERMON_CMPT].object_size	= devlim->cmpt_entry_sz;
2715 	icm[HERMON_CMPT_QPC].object_size	= devlim->cmpt_entry_sz;
2716 	icm[HERMON_CMPT_SRQC].object_size	= devlim->cmpt_entry_sz;
2717 	icm[HERMON_CMPT_CQC].object_size	= devlim->cmpt_entry_sz;
2718 	icm[HERMON_CMPT_EQC].object_size	= devlim->cmpt_entry_sz;
2719 	icm[HERMON_MTT].object_size	= devlim->mtt_entry_sz;
2720 	icm[HERMON_DMPT].object_size	= devlim->dmpt_entry_sz;
2721 	icm[HERMON_QPC].object_size	= devlim->qpc_entry_sz;
2722 	icm[HERMON_CQC].object_size	= devlim->cqc_entry_sz;
2723 	icm[HERMON_SRQC].object_size	= devlim->srq_entry_sz;
2724 	icm[HERMON_EQC].object_size	= devlim->eqc_entry_sz;
2725 	icm[HERMON_RDB].object_size	= devlim->rdmardc_entry_sz *
2726 	    cfg->cp_hca_max_rdma_in_qp;
2727 	icm[HERMON_MCG].object_size	= HERMON_MCGMEM_SZ(state);
2728 	icm[HERMON_ALTC].object_size	= devlim->altc_entry_sz;
2729 	icm[HERMON_AUXC].object_size	= devlim->aux_entry_sz;
2730 
2731 	/* Assign each ICM table's log2 number of entries */
2732 	icm[HERMON_CMPT].log_num_entries = cfg->cp_log_num_cmpt;
2733 	icm[HERMON_CMPT_QPC].log_num_entries = cfg->cp_log_num_qp;
2734 	icm[HERMON_CMPT_SRQC].log_num_entries = cfg->cp_log_num_srq;
2735 	icm[HERMON_CMPT_CQC].log_num_entries = cfg->cp_log_num_cq;
2736 	icm[HERMON_CMPT_EQC].log_num_entries = HERMON_NUM_EQ_SHIFT;
2737 	icm[HERMON_MTT].log_num_entries	= cfg->cp_log_num_mtt;
2738 	icm[HERMON_DMPT].log_num_entries = cfg->cp_log_num_dmpt;
2739 	icm[HERMON_QPC].log_num_entries	= cfg->cp_log_num_qp;
2740 	icm[HERMON_SRQC].log_num_entries = cfg->cp_log_num_srq;
2741 	icm[HERMON_CQC].log_num_entries	= cfg->cp_log_num_cq;
2742 	icm[HERMON_EQC].log_num_entries	= HERMON_NUM_EQ_SHIFT;
2743 	icm[HERMON_RDB].log_num_entries	= cfg->cp_log_num_qp;
2744 	icm[HERMON_MCG].log_num_entries	= cfg->cp_log_num_mcg;
2745 	icm[HERMON_ALTC].log_num_entries = cfg->cp_log_num_qp;
2746 	icm[HERMON_AUXC].log_num_entries = cfg->cp_log_num_qp;
2747 
2748 	/* Initialize the ICM tables */
2749 	hermon_icm_tables_init(state);
2750 
2751 	/*
2752 	 * ICM tables must be aligned on their size in the ICM address
2753 	 * space. So, here we order the tables from largest total table
2754 	 * size to the smallest. All tables are a power of 2 in size, so
2755 	 * this will ensure that all tables are aligned on their own size
2756 	 * without wasting space in the ICM.
2757 	 *
2758 	 * In order to easily set the ICM addresses without needing to
2759 	 * worry about the ordering of our table indices as relates to
2760 	 * the hermon_rsrc_type_t enum, we will use a list of pointers
2761 	 * representing the tables for the sort, then assign ICM addresses
2762 	 * below using it.
2763 	 */
2764 	for (i = 0; i < HERMON_NUM_ICM_RESOURCES; i++) {
2765 		icm_p[i] = &icm[i];
2766 	}
2767 	for (i = HERMON_NUM_ICM_RESOURCES; i > 0; i--) {
2768 		switch (i) {
2769 		case HERMON_CMPT_QPC:
2770 		case HERMON_CMPT_SRQC:
2771 		case HERMON_CMPT_CQC:
2772 		case HERMON_CMPT_EQC:
2773 			continue;
2774 		}
2775 		for (j = 1; j < i; j++) {
2776 			if (icm_p[j]->table_size > icm_p[j - 1]->table_size) {
2777 				tmp		= icm_p[j];
2778 				icm_p[j]	= icm_p[j - 1];
2779 				icm_p[j - 1]	= tmp;
2780 			}
2781 		}
2782 	}
2783 
2784 	/* Initialize the ICM address and ICM size */
2785 	icm_addr = icm_size = 0;
2786 
2787 	/*
2788 	 * Set the ICM base address of each table, using our sorted
2789 	 * list of pointers from above.
2790 	 */
2791 	for (i = 0; i < HERMON_NUM_ICM_RESOURCES; i++) {
2792 		j = icm_p[i]->icm_type;
2793 		switch (j) {
2794 		case HERMON_CMPT_QPC:
2795 		case HERMON_CMPT_SRQC:
2796 		case HERMON_CMPT_CQC:
2797 		case HERMON_CMPT_EQC:
2798 			continue;
2799 		}
2800 		if (icm[j].table_size) {
2801 			/*
2802 			 * Set the ICM base address in the table, save the
2803 			 * ICM offset in the rsrc pool and increment the
2804 			 * total ICM allocation.
2805 			 */
2806 			icm[j].icm_baseaddr = icm_addr;
2807 			if (hermon_verbose) {
2808 				IBTF_DPRINTF_L2("ICMADDR", "rsrc %x @ %p"
2809 				    " size %llx", j, icm[j].icm_baseaddr,
2810 				    icm[j].table_size);
2811 			}
2812 			icm_size += icm[j].table_size;
2813 		}
2814 
2815 		/* Verify that we don't exceed maximum ICM size */
2816 		if (icm_size > devlim->max_icm_size) {
2817 			/* free the ICM table memory resources */
2818 			hermon_icm_tables_fini(state);
2819 			cmn_err(CE_WARN, "ICM configuration exceeds maximum "
2820 			    "configuration: max (0x%lx) requested (0x%lx)\n",
2821 			    (ulong_t)devlim->max_icm_size, (ulong_t)icm_size);
2822 			HERMON_ATTACH_MSG(state->hs_attach_buf,
2823 			    "icm_config_toobig_fail");
2824 			return (DDI_FAILURE);
2825 		}
2826 
2827 		/* assign address to the 4 pieces of the CMPT */
2828 		if (j == HERMON_CMPT) {
2829 			uint64_t cmpt_size = icm[j].table_size >> 2;
2830 #define	init_cmpt_icm_baseaddr(rsrc, indx)				\
2831 	icm[rsrc].icm_baseaddr	= icm_addr + (indx * cmpt_size);
2832 			init_cmpt_icm_baseaddr(HERMON_CMPT_QPC, 0);
2833 			init_cmpt_icm_baseaddr(HERMON_CMPT_SRQC, 1);
2834 			init_cmpt_icm_baseaddr(HERMON_CMPT_CQC, 2);
2835 			init_cmpt_icm_baseaddr(HERMON_CMPT_EQC, 3);
2836 		}
2837 
2838 		/* Increment the ICM address for the next table */
2839 		icm_addr += icm[j].table_size;
2840 	}
2841 
2842 	/* Populate the structure for the INIT_HCA command */
2843 	hermon_inithca_set(state, inithca);
2844 
2845 	/*
2846 	 * Prior to invoking INIT_HCA, we must have ICM memory in place
2847 	 * for the reserved objects in each table. We will allocate and map
2848 	 * this initial ICM memory here. Note that given the assignment
2849 	 * of span_size above, tables that are smaller or equal in total
2850 	 * size to the default span_size will be mapped in full.
2851 	 */
2852 	status = hermon_icm_dma_init(state);
2853 	if (status != DDI_SUCCESS) {
2854 		/* free the ICM table memory resources */
2855 		hermon_icm_tables_fini(state);
2856 		HERMON_WARNING(state, "Failed to allocate initial ICM");
2857 		HERMON_ATTACH_MSG(state->hs_attach_buf,
2858 		    "icm_config_dma_init_fail");
2859 		return (DDI_FAILURE);
2860 	}
2861 
2862 	return (DDI_SUCCESS);
2863 }
2864 
2865 /*
2866  * hermon_inithca_set()
2867  *    Context: Only called from attach() path context
2868  */
2869 static void
2870 hermon_inithca_set(hermon_state_t *state, hermon_hw_initqueryhca_t *inithca)
2871 {
2872 	hermon_cfg_profile_t	*cfg;
2873 	hermon_icm_table_t	*icm;
2874 	int			i;
2875 
2876 
2877 	/* Populate the INIT_HCA structure */
2878 	icm = state->hs_icm;
2879 	cfg = state->hs_cfg_profile;
2880 
2881 	/* set version */
2882 	inithca->version = 0x02;	/* PRM 0.36 */
2883 	/* set cacheline - log2 in 16-byte chunks */
2884 	inithca->log2_cacheline = 0x2;	/* optimized for 64 byte cache */
2885 
2886 	/* we need to update the inithca info with thie UAR info too */
2887 	inithca->uar.log_max_uars = highbit(cfg->cp_log_num_uar);
2888 	inithca->uar.uar_pg_sz = PAGESHIFT - HERMON_PAGESHIFT;
2889 
2890 	/* Set endianess */
2891 #ifdef	_LITTLE_ENDIAN
2892 	inithca->big_endian	= 0;
2893 #else
2894 	inithca->big_endian	= 1;
2895 #endif
2896 
2897 	/* Port Checking is on by default */
2898 	inithca->udav_port_chk	= HERMON_UDAV_PORTCHK_ENABLED;
2899 
2900 	/* Enable IPoIB checksum */
2901 	if (state->hs_devlim.ipoib_cksm)
2902 		inithca->chsum_en = 1;
2903 
2904 	/* Set each ICM table's attributes */
2905 	for (i = 0; i < HERMON_NUM_ICM_RESOURCES; i++) {
2906 		switch (icm[i].icm_type) {
2907 		case HERMON_CMPT:
2908 			inithca->tpt.cmpt_baseaddr = icm[i].icm_baseaddr;
2909 			break;
2910 
2911 		case HERMON_MTT:
2912 			inithca->tpt.mtt_baseaddr = icm[i].icm_baseaddr;
2913 			break;
2914 
2915 		case HERMON_DMPT:
2916 			inithca->tpt.dmpt_baseaddr = icm[i].icm_baseaddr;
2917 			inithca->tpt.log_dmpt_sz   = icm[i].log_num_entries;
2918 			inithca->tpt.pgfault_rnr_to = 0; /* just in case */
2919 			break;
2920 
2921 		case HERMON_QPC:
2922 			inithca->context.log_num_qp = icm[i].log_num_entries;
2923 			inithca->context.qpc_baseaddr_h =
2924 			    icm[i].icm_baseaddr >> 32;
2925 			inithca->context.qpc_baseaddr_l =
2926 			    (icm[i].icm_baseaddr & 0xFFFFFFFF) >> 5;
2927 			break;
2928 
2929 		case HERMON_CQC:
2930 			inithca->context.log_num_cq = icm[i].log_num_entries;
2931 			inithca->context.cqc_baseaddr_h =
2932 			    icm[i].icm_baseaddr >> 32;
2933 			inithca->context.cqc_baseaddr_l =
2934 			    (icm[i].icm_baseaddr & 0xFFFFFFFF) >> 5;
2935 			break;
2936 
2937 		case HERMON_SRQC:
2938 			inithca->context.log_num_srq = icm[i].log_num_entries;
2939 			inithca->context.srqc_baseaddr_h =
2940 			    icm[i].icm_baseaddr >> 32;
2941 			inithca->context.srqc_baseaddr_l =
2942 			    (icm[i].icm_baseaddr & 0xFFFFFFFF) >> 5;
2943 			break;
2944 
2945 		case HERMON_EQC:
2946 			inithca->context.log_num_eq = icm[i].log_num_entries;
2947 			inithca->context.eqc_baseaddr_h =
2948 			    icm[i].icm_baseaddr >> 32;
2949 			inithca->context.eqc_baseaddr_l =
2950 			    (icm[i].icm_baseaddr & 0xFFFFFFFF) >> 5;
2951 			break;
2952 
2953 		case HERMON_RDB:
2954 			inithca->context.rdmardc_baseaddr_h =
2955 			    icm[i].icm_baseaddr >> 32;
2956 			inithca->context.rdmardc_baseaddr_l =
2957 			    (icm[i].icm_baseaddr & 0xFFFFFFFF) >> 5;
2958 			inithca->context.log_num_rdmardc =
2959 			    icm[i].log_num_entries;
2960 			break;
2961 
2962 		case HERMON_MCG:
2963 			inithca->multi.mc_baseaddr    = icm[i].icm_baseaddr;
2964 			inithca->multi.log_mc_tbl_sz  = icm[i].log_num_entries;
2965 			inithca->multi.log_mc_tbl_ent =
2966 			    highbit(HERMON_MCGMEM_SZ(state)) - 1;
2967 			inithca->multi.log_mc_tbl_hash_sz =
2968 			    cfg->cp_log_num_mcg_hash;
2969 			inithca->multi.mc_hash_fn = HERMON_MCG_DEFAULT_HASH_FN;
2970 			break;
2971 
2972 		case HERMON_ALTC:
2973 			inithca->context.altc_baseaddr = icm[i].icm_baseaddr;
2974 			break;
2975 
2976 		case HERMON_AUXC:
2977 			inithca->context.auxc_baseaddr = icm[i].icm_baseaddr;
2978 			break;
2979 
2980 		default:
2981 			break;
2982 
2983 		}
2984 	}
2985 
2986 }
2987 
2988 /*
2989  * hermon_icm_tables_init()
2990  *    Context: Only called from attach() path context
2991  *
2992  * Dynamic ICM breaks the various ICM tables into "span_size" chunks
2993  * to enable allocation of backing memory on demand.  Arbel used a
2994  * fixed size ARBEL_ICM_SPAN_SIZE (initially was 512KB) as the
2995  * span_size for all ICM chunks.  Hermon has other considerations,
2996  * so the span_size used differs from Arbel.
2997  *
2998  * The basic considerations for why Hermon differs are:
2999  *
3000  *	1) ICM memory is in units of HERMON pages.
3001  *
3002  *	2) The AUXC table is approximately 1 byte per QP.
3003  *
3004  *	3) ICM memory for AUXC, ALTC, and RDB is allocated when
3005  *	the ICM memory for the corresponding QPC is allocated.
3006  *
3007  *	4) ICM memory for the CMPT corresponding to the various primary
3008  *	resources (QPC, SRQC, CQC, and EQC) is allocated when the ICM
3009  *	memory for the primary resource is allocated.
3010  *
3011  * One HERMON page (4KB) would typically map 4K QPs worth of AUXC.
3012  * So, the minimum chunk for the various QPC related ICM memory should
3013  * all be allocated to support the 4K QPs.  Currently, this means the
3014  * amount of memory for the various QP chunks is:
3015  *
3016  *	QPC	256*4K bytes
3017  *	RDB	128*4K bytes
3018  *	CMPT	 64*4K bytes
3019  *	ALTC	 64*4K bytes
3020  *	AUXC	  1*4K bytes
3021  *
3022  * The span_size chosen for the QP resource is 4KB of AUXC entries,
3023  * or 1 HERMON_PAGESIZE worth, which is the minimum ICM mapping size.
3024  *
3025  * Other ICM resources can have their span_size be more arbitrary.
3026  * This is 4K (HERMON_ICM_SPAN), except for MTTs because they are tiny.
3027  */
3028 
3029 /* macro to make the code below cleaner */
3030 #define	init_dependent(rsrc, dep)				\
3031 	icm[dep].span		= icm[rsrc].span;		\
3032 	icm[dep].num_spans	= icm[rsrc].num_spans;		\
3033 	icm[dep].split_shift	= icm[rsrc].split_shift;	\
3034 	icm[dep].span_mask	= icm[rsrc].span_mask;		\
3035 	icm[dep].span_shift	= icm[rsrc].span_shift;		\
3036 	icm[dep].rsrc_mask	= icm[rsrc].rsrc_mask;		\
3037 	if (hermon_verbose) {					\
3038 		IBTF_DPRINTF_L2("hermon", "tables_init: "	\
3039 		    "rsrc (0x%x) size (0x%lx) span (0x%x) "	\
3040 		    "num_spans (0x%x)", dep, icm[dep].table_size, \
3041 		    icm[dep].span, icm[dep].num_spans);		\
3042 		IBTF_DPRINTF_L2("hermon", "tables_init: "	\
3043 		    "span_shift (0x%x) split_shift (0x%x)",	\
3044 		    icm[dep].span_shift, icm[dep].split_shift);	\
3045 		IBTF_DPRINTF_L2("hermon", "tables_init: "	\
3046 		    "span_mask (0x%x)  rsrc_mask   (0x%x)",	\
3047 		    icm[dep].span_mask, icm[dep].rsrc_mask);	\
3048 	}
3049 
3050 static void
3051 hermon_icm_tables_init(hermon_state_t *state)
3052 {
3053 	hermon_icm_table_t	*icm;
3054 	int			i, k;
3055 	uint32_t		per_split;
3056 
3057 
3058 	icm = state->hs_icm;
3059 
3060 	for (i = 0; i < HERMON_NUM_ICM_RESOURCES; i++) {
3061 		icm[i].icm_type		= i;
3062 		icm[i].num_entries	= 1 << icm[i].log_num_entries;
3063 		icm[i].log_object_size	= highbit(icm[i].object_size) - 1;
3064 		icm[i].table_size	= icm[i].num_entries <<
3065 		    icm[i].log_object_size;
3066 
3067 		/* deal with "dependent" resource types */
3068 		switch (i) {
3069 		case HERMON_AUXC:
3070 #ifdef HERMON_FW_WORKAROUND
3071 			icm[i].table_size = 0x80000000ull;
3072 			/* FALLTHROUGH */
3073 #endif
3074 		case HERMON_CMPT_QPC:
3075 		case HERMON_RDB:
3076 		case HERMON_ALTC:
3077 			init_dependent(HERMON_QPC, i);
3078 			continue;
3079 		case HERMON_CMPT_SRQC:
3080 			init_dependent(HERMON_SRQC, i);
3081 			continue;
3082 		case HERMON_CMPT_CQC:
3083 			init_dependent(HERMON_CQC, i);
3084 			continue;
3085 		case HERMON_CMPT_EQC:
3086 			init_dependent(HERMON_EQC, i);
3087 			continue;
3088 		}
3089 
3090 		icm[i].span = HERMON_ICM_SPAN;	/* default #rsrc's in 1 span */
3091 		if (i == HERMON_MTT) /* Alloc enough MTTs to map 256MB */
3092 			icm[i].span = HERMON_ICM_SPAN * 16;
3093 		icm[i].num_spans = icm[i].num_entries / icm[i].span;
3094 		if (icm[i].num_spans == 0) {
3095 			icm[i].span = icm[i].num_entries;
3096 			per_split = 1;
3097 			icm[i].num_spans = icm[i].num_entries / icm[i].span;
3098 		} else {
3099 			per_split = icm[i].num_spans / HERMON_ICM_SPLIT;
3100 			if (per_split == 0) {
3101 				per_split = 1;
3102 			}
3103 		}
3104 		if (hermon_verbose)
3105 			IBTF_DPRINTF_L2("ICM", "rsrc %x  span %x  num_spans %x",
3106 			    i, icm[i].span, icm[i].num_spans);
3107 
3108 		/*
3109 		 * Ensure a minimum table size of an ICM page, and a
3110 		 * maximum span size of the ICM table size.  This ensures
3111 		 * that we don't have less than an ICM page to map, which is
3112 		 * impossible, and that we will map an entire table at
3113 		 * once if it's total size is less than the span size.
3114 		 */
3115 		icm[i].table_size = max(icm[i].table_size, HERMON_PAGESIZE);
3116 
3117 		icm[i].span_shift = 0;
3118 		for (k = icm[i].span; k != 1; k >>= 1)
3119 			icm[i].span_shift++;
3120 		icm[i].split_shift = icm[i].span_shift;
3121 		for (k = per_split; k != 1; k >>= 1)
3122 			icm[i].split_shift++;
3123 		icm[i].span_mask = (1 << icm[i].split_shift) -
3124 		    (1 << icm[i].span_shift);
3125 		icm[i].rsrc_mask = (1 << icm[i].span_shift) - 1;
3126 
3127 
3128 		/* Initialize the table lock */
3129 		mutex_init(&icm[i].icm_table_lock, NULL, MUTEX_DRIVER,
3130 		    DDI_INTR_PRI(state->hs_intrmsi_pri));
3131 		cv_init(&icm[i].icm_table_cv, NULL, CV_DRIVER, NULL);
3132 
3133 		if (hermon_verbose) {
3134 			IBTF_DPRINTF_L2("hermon", "tables_init: "
3135 			    "rsrc (0x%x) size (0x%lx)", i, icm[i].table_size);
3136 			IBTF_DPRINTF_L2("hermon", "tables_init: "
3137 			    "span (0x%x) num_spans (0x%x)",
3138 			    icm[i].span, icm[i].num_spans);
3139 			IBTF_DPRINTF_L2("hermon", "tables_init: "
3140 			    "span_shift (0x%x) split_shift (0x%x)",
3141 			    icm[i].span_shift, icm[i].split_shift);
3142 			IBTF_DPRINTF_L2("hermon", "tables_init: "
3143 			    "span_mask (0x%x)  rsrc_mask   (0x%x)",
3144 			    icm[i].span_mask, icm[i].rsrc_mask);
3145 		}
3146 	}
3147 
3148 }
3149 
3150 /*
3151  * hermon_icm_tables_fini()
3152  *    Context: Only called from attach() path context
3153  *
3154  * Clean up all icm_tables.  Free the bitmap and dma_info arrays.
3155  */
3156 static void
3157 hermon_icm_tables_fini(hermon_state_t *state)
3158 {
3159 	hermon_icm_table_t	*icm;
3160 	int			nspans;
3161 	int			i, j;
3162 
3163 
3164 	icm = state->hs_icm;
3165 
3166 	for (i = 0; i < HERMON_NUM_ICM_RESOURCES; i++) {
3167 
3168 		mutex_enter(&icm[i].icm_table_lock);
3169 		nspans = icm[i].num_spans;
3170 
3171 		for (j = 0; j < HERMON_ICM_SPLIT; j++) {
3172 			if (icm[i].icm_dma[j])
3173 				/* Free the ICM DMA slots */
3174 				kmem_free(icm[i].icm_dma[j],
3175 				    nspans * sizeof (hermon_dma_info_t));
3176 
3177 			if (icm[i].icm_bitmap[j])
3178 				/* Free the table bitmap */
3179 				kmem_free(icm[i].icm_bitmap[j],
3180 				    (nspans + 7) / 8);
3181 		}
3182 		/* Destroy the table lock */
3183 		cv_destroy(&icm[i].icm_table_cv);
3184 		mutex_exit(&icm[i].icm_table_lock);
3185 		mutex_destroy(&icm[i].icm_table_lock);
3186 	}
3187 
3188 }
3189 
3190 /*
3191  * hermon_icm_dma_init()
3192  *    Context: Only called from attach() path context
3193  */
3194 static int
3195 hermon_icm_dma_init(hermon_state_t *state)
3196 {
3197 	hermon_icm_table_t	*icm;
3198 	hermon_rsrc_type_t	type;
3199 	int			status;
3200 
3201 
3202 	/*
3203 	 * This routine will allocate initial ICM DMA resources for ICM
3204 	 * tables that have reserved ICM objects. This is the only routine
3205 	 * where we should have to allocate ICM outside of hermon_rsrc_alloc().
3206 	 * We need to allocate ICM here explicitly, rather than in
3207 	 * hermon_rsrc_alloc(), because we've not yet completed the resource
3208 	 * pool initialization. When the resource pools are initialized
3209 	 * (in hermon_rsrc_init_phase2(), see hermon_rsrc.c for more
3210 	 * information), resource preallocations will be invoked to match
3211 	 * the ICM allocations seen here. We will then be able to use the
3212 	 * normal allocation path.  Note we don't need to set a refcnt on
3213 	 * these initial allocations because that will be done in the calls
3214 	 * to hermon_rsrc_alloc() from hermon_hw_entries_init() for the
3215 	 * "prealloc" objects (see hermon_rsrc.c for more information).
3216 	 */
3217 	for (type = 0; type < HERMON_NUM_ICM_RESOURCES; type++) {
3218 
3219 		/* ICM for these is allocated within hermon_icm_alloc() */
3220 		switch (type) {
3221 		case HERMON_CMPT:
3222 		case HERMON_CMPT_QPC:
3223 		case HERMON_CMPT_SRQC:
3224 		case HERMON_CMPT_CQC:
3225 		case HERMON_CMPT_EQC:
3226 		case HERMON_AUXC:
3227 		case HERMON_ALTC:
3228 		case HERMON_RDB:
3229 			continue;
3230 		}
3231 
3232 		icm = &state->hs_icm[type];
3233 
3234 		mutex_enter(&icm->icm_table_lock);
3235 		status = hermon_icm_alloc(state, type, 0, 0);
3236 		mutex_exit(&icm->icm_table_lock);
3237 		if (status != DDI_SUCCESS) {
3238 			while (type--) {
3239 				icm = &state->hs_icm[type];
3240 				mutex_enter(&icm->icm_table_lock);
3241 				hermon_icm_free(state, type, 0, 0);
3242 				mutex_exit(&icm->icm_table_lock);
3243 			}
3244 			return (DDI_FAILURE);
3245 		}
3246 
3247 		if (hermon_verbose) {
3248 			IBTF_DPRINTF_L2("hermon", "hermon_icm_dma_init: "
3249 			    "table (0x%x) index (0x%x) allocated", type, 0);
3250 		}
3251 	}
3252 
3253 	return (DDI_SUCCESS);
3254 }
3255 
3256 /*
3257  * hermon_icm_dma_fini()
3258  *    Context: Only called from attach() path context
3259  *
3260  * ICM has been completely unmapped.  We just free the memory here.
3261  */
3262 static void
3263 hermon_icm_dma_fini(hermon_state_t *state)
3264 {
3265 	hermon_icm_table_t	*icm;
3266 	hermon_dma_info_t	*dma_info;
3267 	hermon_rsrc_type_t	type;
3268 	int			index1, index2;
3269 
3270 
3271 	for (type = 0; type < HERMON_NUM_ICM_RESOURCES; type++) {
3272 		icm = &state->hs_icm[type];
3273 		for (index1 = 0; index1 < HERMON_ICM_SPLIT; index1++) {
3274 			dma_info = icm->icm_dma[index1];
3275 			if (dma_info == NULL)
3276 				continue;
3277 			for (index2 = 0; index2 < icm->num_spans; index2++) {
3278 				if (dma_info[index2].dma_hdl)
3279 					hermon_dma_free(&dma_info[index2]);
3280 				dma_info[index2].dma_hdl = NULL;
3281 			}
3282 		}
3283 	}
3284 
3285 }
3286 
3287 /*
3288  * hermon_hca_port_init()
3289  *    Context: Only called from attach() path context
3290  */
3291 static int
3292 hermon_hca_port_init(hermon_state_t *state)
3293 {
3294 	hermon_hw_set_port_t	*portinits, *initport;
3295 	hermon_cfg_profile_t	*cfgprof;
3296 	uint_t			num_ports;
3297 	int			i = 0, status;
3298 	uint64_t		maxval, val;
3299 	uint64_t		sysimgguid, nodeguid, portguid;
3300 
3301 
3302 	cfgprof = state->hs_cfg_profile;
3303 
3304 	/* Get number of HCA ports */
3305 	num_ports = cfgprof->cp_num_ports;
3306 
3307 	/* Allocate space for Hermon set port  struct(s) */
3308 	portinits = (hermon_hw_set_port_t *)kmem_zalloc(num_ports *
3309 	    sizeof (hermon_hw_set_port_t), KM_SLEEP);
3310 
3311 
3312 
3313 	/* Post commands to initialize each Hermon HCA port */
3314 	/*
3315 	 * In Hermon, the process is different than in previous HCAs.
3316 	 * Here, you have to:
3317 	 *	QUERY_PORT - to get basic information from the HCA
3318 	 *	set the fields accordingly
3319 	 *	SET_PORT - to change/set everything as desired
3320 	 *	INIT_PORT - to bring the port up
3321 	 *
3322 	 * Needs to be done for each port in turn
3323 	 */
3324 
3325 	for (i = 0; i < num_ports; i++) {
3326 		bzero(&state->hs_queryport, sizeof (hermon_hw_query_port_t));
3327 		status = hermon_cmn_query_cmd_post(state, QUERY_PORT, 0,
3328 		    (i + 1), &state->hs_queryport,
3329 		    sizeof (hermon_hw_query_port_t), HERMON_CMD_NOSLEEP_SPIN);
3330 		if (status != HERMON_CMD_SUCCESS) {
3331 			cmn_err(CE_CONT, "Hermon: QUERY_PORT (port %02d) "
3332 			    "command failed: %08x\n", i + 1, status);
3333 			goto init_ports_fail;
3334 		}
3335 		initport = &portinits[i];
3336 		state->hs_initport = &portinits[i];
3337 
3338 		bzero(initport, sizeof (hermon_hw_query_port_t));
3339 
3340 		/*
3341 		 * Determine whether we need to override the firmware's
3342 		 * default SystemImageGUID setting.
3343 		 */
3344 		sysimgguid = cfgprof->cp_sysimgguid;
3345 		if (sysimgguid != 0) {
3346 			initport->sig		= 1;
3347 			initport->sys_img_guid	= sysimgguid;
3348 		}
3349 
3350 		/*
3351 		 * Determine whether we need to override the firmware's
3352 		 * default NodeGUID setting.
3353 		 */
3354 		nodeguid = cfgprof->cp_nodeguid;
3355 		if (nodeguid != 0) {
3356 			initport->ng		= 1;
3357 			initport->node_guid	= nodeguid;
3358 		}
3359 
3360 		/*
3361 		 * Determine whether we need to override the firmware's
3362 		 * default PortGUID setting.
3363 		 */
3364 		portguid = cfgprof->cp_portguid[i];
3365 		if (portguid != 0) {
3366 			initport->g0		= 1;
3367 			initport->guid0		= portguid;
3368 		}
3369 
3370 		/* Validate max MTU size */
3371 		maxval  = state->hs_queryport.ib_mtu;
3372 		val	= cfgprof->cp_max_mtu;
3373 		if (val > maxval) {
3374 			goto init_ports_fail;
3375 		}
3376 
3377 		/* Validate the max port width */
3378 		maxval  = state->hs_queryport.ib_port_wid;
3379 		val	= cfgprof->cp_max_port_width;
3380 		if (val > maxval) {
3381 			goto init_ports_fail;
3382 		}
3383 
3384 		/* Validate max VL cap size */
3385 		maxval  = state->hs_queryport.max_vl;
3386 		val	= cfgprof->cp_max_vlcap;
3387 		if (val > maxval) {
3388 			goto init_ports_fail;
3389 		}
3390 
3391 		/* Validate max GID table size */
3392 		maxval  = ((uint64_t)1 << state->hs_queryport.log_max_gid);
3393 		val	= ((uint64_t)1 << cfgprof->cp_log_max_gidtbl);
3394 		if (val > maxval) {
3395 			goto init_ports_fail;
3396 		}
3397 		initport->max_guid = (uint16_t)val;
3398 		initport->mg = 1;
3399 
3400 		/* Validate max PKey table size */
3401 		maxval	= ((uint64_t)1 << state->hs_queryport.log_max_pkey);
3402 		val	= ((uint64_t)1 << cfgprof->cp_log_max_pkeytbl);
3403 		if (val > maxval) {
3404 			goto init_ports_fail;
3405 		}
3406 		initport->max_pkey = (uint16_t)val;
3407 		initport->mp = 1;
3408 		/*
3409 		 * Post the SET_PORT cmd to Hermon firmware. This sets
3410 		 * the parameters of the port.
3411 		 */
3412 		status = hermon_set_port_cmd_post(state, initport, i + 1,
3413 		    HERMON_CMD_NOSLEEP_SPIN);
3414 		if (status != HERMON_CMD_SUCCESS) {
3415 			cmn_err(CE_CONT, "Hermon: SET_PORT (port %02d) command "
3416 			    "failed: %08x\n", i + 1, status);
3417 			goto init_ports_fail;
3418 		}
3419 		/* issue another SET_PORT cmd - performance fix/workaround */
3420 		/* XXX - need to discuss with Mellanox */
3421 		bzero(initport, sizeof (hermon_hw_query_port_t));
3422 		initport->cap_mask = 0x02500868;
3423 		status = hermon_set_port_cmd_post(state, initport, i + 1,
3424 		    HERMON_CMD_NOSLEEP_SPIN);
3425 		if (status != HERMON_CMD_SUCCESS) {
3426 			cmn_err(CE_CONT, "Hermon: SET_PORT (port %02d) command "
3427 			    "failed: %08x\n", i + 1, status);
3428 			goto init_ports_fail;
3429 		}
3430 	}
3431 
3432 	/*
3433 	 * Finally, do the INIT_PORT for each port in turn
3434 	 * When this command completes, the corresponding Hermon port
3435 	 * will be physically "Up" and initialized.
3436 	 */
3437 	for (i = 0; i < num_ports; i++) {
3438 		status = hermon_init_port_cmd_post(state, i + 1,
3439 		    HERMON_CMD_NOSLEEP_SPIN);
3440 		if (status != HERMON_CMD_SUCCESS) {
3441 			cmn_err(CE_CONT, "Hermon: INIT_PORT (port %02d) "
3442 			    "comman failed: %08x\n", i + 1, status);
3443 			goto init_ports_fail;
3444 		}
3445 	}
3446 
3447 	/* Free up the memory for Hermon port init struct(s), return success */
3448 	kmem_free(portinits, num_ports * sizeof (hermon_hw_set_port_t));
3449 	return (DDI_SUCCESS);
3450 
3451 init_ports_fail:
3452 	/*
3453 	 * Free up the memory for Hermon port init struct(s), shutdown any
3454 	 * successfully initialized ports, and return failure
3455 	 */
3456 	kmem_free(portinits, num_ports * sizeof (hermon_hw_set_port_t));
3457 	(void) hermon_hca_ports_shutdown(state, i);
3458 
3459 	return (DDI_FAILURE);
3460 }
3461 
3462 
3463 /*
3464  * hermon_hca_ports_shutdown()
3465  *    Context: Only called from attach() and/or detach() path contexts
3466  */
3467 static int
3468 hermon_hca_ports_shutdown(hermon_state_t *state, uint_t num_init)
3469 {
3470 	int	i, status;
3471 
3472 	/*
3473 	 * Post commands to shutdown all init'd Hermon HCA ports.  Note: if
3474 	 * any of these commands fail for any reason, it would be entirely
3475 	 * unexpected and probably indicative a serious problem (HW or SW).
3476 	 * Although we do return void from this function, this type of failure
3477 	 * should not go unreported.  That is why we have the warning message.
3478 	 */
3479 	for (i = 0; i < num_init; i++) {
3480 		status = hermon_close_port_cmd_post(state, i + 1,
3481 		    HERMON_CMD_NOSLEEP_SPIN);
3482 		if (status != HERMON_CMD_SUCCESS) {
3483 			HERMON_WARNING(state, "failed to shutdown HCA port");
3484 			return (status);
3485 		}
3486 	}
3487 	return (HERMON_CMD_SUCCESS);
3488 }
3489 
3490 
3491 /*
3492  * hermon_internal_uarpg_init
3493  *    Context: Only called from attach() path context
3494  */
3495 static int
3496 hermon_internal_uarpg_init(hermon_state_t *state)
3497 {
3498 	int	status;
3499 	hermon_dbr_info_t 	*info;
3500 
3501 	/*
3502 	 * Allocate the UAR page for kernel use. This UAR page is
3503 	 * the privileged UAR page through which all kernel generated
3504 	 * doorbells will be rung. There are a number of UAR pages
3505 	 * reserved by hardware at the front of the UAR BAR, indicated
3506 	 * by DEVCAP.num_rsvd_uar, which we have already allocated. So,
3507 	 * the kernel page, or UAR page index num_rsvd_uar, will be
3508 	 * allocated here for kernel use.
3509 	 */
3510 
3511 	status = hermon_rsrc_alloc(state, HERMON_UARPG, 1, HERMON_SLEEP,
3512 	    &state->hs_uarkpg_rsrc);
3513 	if (status != DDI_SUCCESS) {
3514 		return (DDI_FAILURE);
3515 	}
3516 
3517 	/* Setup pointer to kernel UAR page */
3518 	state->hs_uar = (hermon_hw_uar_t *)state->hs_uarkpg_rsrc->hr_addr;
3519 
3520 	/* need to set up DBr tracking as well */
3521 	status = hermon_dbr_page_alloc(state, &info);
3522 	if (status != DDI_SUCCESS) {
3523 		return (DDI_FAILURE);
3524 	}
3525 	state->hs_kern_dbr = info;
3526 	return (DDI_SUCCESS);
3527 }
3528 
3529 
3530 /*
3531  * hermon_internal_uarpg_fini
3532  *    Context: Only called from attach() and/or detach() path contexts
3533  */
3534 static void
3535 hermon_internal_uarpg_fini(hermon_state_t *state)
3536 {
3537 	/* Free up Hermon UAR page #1 (kernel driver doorbells) */
3538 	hermon_rsrc_free(state, &state->hs_uarkpg_rsrc);
3539 }
3540 
3541 
3542 /*
3543  * hermon_special_qp_contexts_reserve()
3544  *    Context: Only called from attach() path context
3545  */
3546 static int
3547 hermon_special_qp_contexts_reserve(hermon_state_t *state)
3548 {
3549 	hermon_rsrc_t	*qp0_rsrc, *qp1_rsrc, *qp_resvd;
3550 	int		status;
3551 
3552 	/* Initialize the lock used for special QP rsrc management */
3553 	mutex_init(&state->hs_spec_qplock, NULL, MUTEX_DRIVER,
3554 	    DDI_INTR_PRI(state->hs_intrmsi_pri));
3555 
3556 	/*
3557 	 * Reserve contexts for QP0.  These QP contexts will be setup to
3558 	 * act as aliases for the real QP0.  Note: We are required to grab
3559 	 * two QPs (one per port) even if we are operating in single-port
3560 	 * mode.
3561 	 */
3562 	status = hermon_rsrc_alloc(state, HERMON_QPC, 2,
3563 	    HERMON_SLEEP, &qp0_rsrc);
3564 	if (status != DDI_SUCCESS) {
3565 		mutex_destroy(&state->hs_spec_qplock);
3566 		return (DDI_FAILURE);
3567 	}
3568 	state->hs_spec_qp0 = qp0_rsrc;
3569 
3570 	/*
3571 	 * Reserve contexts for QP1.  These QP contexts will be setup to
3572 	 * act as aliases for the real QP1.  Note: We are required to grab
3573 	 * two QPs (one per port) even if we are operating in single-port
3574 	 * mode.
3575 	 */
3576 	status = hermon_rsrc_alloc(state, HERMON_QPC, 2,
3577 	    HERMON_SLEEP, &qp1_rsrc);
3578 	if (status != DDI_SUCCESS) {
3579 		hermon_rsrc_free(state, &qp0_rsrc);
3580 		mutex_destroy(&state->hs_spec_qplock);
3581 		return (DDI_FAILURE);
3582 	}
3583 	state->hs_spec_qp1 = qp1_rsrc;
3584 
3585 	status = hermon_rsrc_alloc(state, HERMON_QPC, 4,
3586 	    HERMON_SLEEP, &qp_resvd);
3587 	if (status != DDI_SUCCESS) {
3588 		hermon_rsrc_free(state, &qp1_rsrc);
3589 		hermon_rsrc_free(state, &qp0_rsrc);
3590 		mutex_destroy(&state->hs_spec_qplock);
3591 		return (DDI_FAILURE);
3592 	}
3593 	state->hs_spec_qp_unused = qp_resvd;
3594 
3595 	return (DDI_SUCCESS);
3596 }
3597 
3598 
3599 /*
3600  * hermon_special_qp_contexts_unreserve()
3601  *    Context: Only called from attach() and/or detach() path contexts
3602  */
3603 static void
3604 hermon_special_qp_contexts_unreserve(hermon_state_t *state)
3605 {
3606 
3607 	/* Unreserve contexts for spec_qp_unused */
3608 	hermon_rsrc_free(state, &state->hs_spec_qp_unused);
3609 
3610 	/* Unreserve contexts for QP1 */
3611 	hermon_rsrc_free(state, &state->hs_spec_qp1);
3612 
3613 	/* Unreserve contexts for QP0 */
3614 	hermon_rsrc_free(state, &state->hs_spec_qp0);
3615 
3616 	/* Destroy the lock used for special QP rsrc management */
3617 	mutex_destroy(&state->hs_spec_qplock);
3618 
3619 }
3620 
3621 
3622 /*
3623  * hermon_sw_reset()
3624  *    Context: Currently called only from attach() path context
3625  */
3626 static int
3627 hermon_sw_reset(hermon_state_t *state)
3628 {
3629 	ddi_acc_handle_t	hdl = hermon_get_pcihdl(state);
3630 	ddi_acc_handle_t	cmdhdl = hermon_get_cmdhdl(state);
3631 	uint32_t		reset_delay;
3632 	int			status, i;
3633 	uint32_t		sem;
3634 	uint_t			offset;
3635 	uint32_t		data32;		/* for devctl & linkctl */
3636 	int			loopcnt;
3637 
3638 	/* initialize the FMA retry loop */
3639 	hermon_pio_init(fm_loop_cnt, fm_status, fm_test);
3640 	hermon_pio_init(fm_loop_cnt2, fm_status2, fm_test2);
3641 
3642 	/*
3643 	 * If the configured software reset delay is set to zero, then we
3644 	 * will not attempt a software reset of the Hermon device.
3645 	 */
3646 	reset_delay = state->hs_cfg_profile->cp_sw_reset_delay;
3647 	if (reset_delay == 0) {
3648 		return (DDI_SUCCESS);
3649 	}
3650 
3651 	/* the FMA retry loop starts. */
3652 	hermon_pio_start(state, cmdhdl, pio_error, fm_loop_cnt, fm_status,
3653 	    fm_test);
3654 	hermon_pio_start(state, hdl, pio_error2, fm_loop_cnt2, fm_status2,
3655 	    fm_test2);
3656 
3657 	/* Query the PCI capabilities of the HCA device */
3658 	/* but don't process the VPD until after reset */
3659 	status = hermon_pci_capability_list(state, hdl);
3660 	if (status != DDI_SUCCESS) {
3661 		cmn_err(CE_NOTE, "failed to get pci capabilities list(0x%x)\n",
3662 		    status);
3663 		return (DDI_FAILURE);
3664 	}
3665 
3666 	/*
3667 	 * Read all PCI config info (reg0...reg63).  Note: According to the
3668 	 * Hermon software reset application note, we should not read or
3669 	 * restore the values in reg22 and reg23.
3670 	 * NOTE:  For Hermon (and Arbel too) it says to restore the command
3671 	 * register LAST, and technically, you need to restore the
3672 	 * PCIE Capability "device control" and "link control" (word-sized,
3673 	 * at offsets 0x08 and 0x10 from the capbility ID respectively).
3674 	 * We hold off restoring the command register - offset 0x4 - till last
3675 	 */
3676 
3677 	/* 1st, wait for the semaphore assure accessibility - per PRM */
3678 	status = -1;
3679 	for (i = 0; i < NANOSEC/MICROSEC /* 1sec timeout */; i++) {
3680 		sem = ddi_get32(cmdhdl, state->hs_cmd_regs.sw_semaphore);
3681 		if (sem == 0) {
3682 			status = 0;
3683 			break;
3684 		}
3685 		drv_usecwait(1);
3686 	}
3687 
3688 	/* Check if timeout happens */
3689 	if (status == -1) {
3690 		/*
3691 		 * Remove this acc handle from Hermon, then log
3692 		 * the error.
3693 		 */
3694 		hermon_pci_config_teardown(state, &hdl);
3695 
3696 		cmn_err(CE_WARN, "hermon_sw_reset timeout: "
3697 		    "failed to get the semaphore(0x%p)\n",
3698 		    (void *)state->hs_cmd_regs.sw_semaphore);
3699 
3700 		hermon_fm_ereport(state, HCA_IBA_ERR, HCA_ERR_NON_FATAL);
3701 		return (DDI_FAILURE);
3702 	}
3703 
3704 	for (i = 0; i < HERMON_SW_RESET_NUMREGS; i++) {
3705 		if ((i != HERMON_SW_RESET_REG22_RSVD) &&
3706 		    (i != HERMON_SW_RESET_REG23_RSVD)) {
3707 			state->hs_cfg_data[i]  = pci_config_get32(hdl, i << 2);
3708 		}
3709 	}
3710 
3711 	/*
3712 	 * Perform the software reset (by writing 1 at offset 0xF0010)
3713 	 */
3714 	ddi_put32(cmdhdl, state->hs_cmd_regs.sw_reset, HERMON_SW_RESET_START);
3715 
3716 	/*
3717 	 * This delay is required so as not to cause a panic here. If the
3718 	 * device is accessed too soon after reset it will not respond to
3719 	 * config cycles, causing a Master Abort and panic.
3720 	 */
3721 	drv_usecwait(reset_delay);
3722 
3723 	/*
3724 	 * Poll waiting for the device to finish resetting.
3725 	 */
3726 	loopcnt = 100;	/* 100 times @ 100 usec - total delay 10 msec */
3727 	while ((pci_config_get32(hdl, 0) & 0x0000FFFF) != PCI_VENID_MLX) {
3728 		drv_usecwait(HERMON_SW_RESET_POLL_DELAY);
3729 		if (--loopcnt == 0)
3730 			break;	/* just in case, break and go on */
3731 	}
3732 	if (loopcnt == 0)
3733 		cmn_err(CE_CONT, "!Never see VEND_ID - read == %X",
3734 		    pci_config_get32(hdl, 0));
3735 
3736 	/*
3737 	 * Restore the config info
3738 	 */
3739 	for (i = 0; i < HERMON_SW_RESET_NUMREGS; i++) {
3740 		if (i == 1) continue;	/* skip the status/ctrl reg */
3741 		if ((i != HERMON_SW_RESET_REG22_RSVD) &&
3742 		    (i != HERMON_SW_RESET_REG23_RSVD)) {
3743 			pci_config_put32(hdl, i << 2, state->hs_cfg_data[i]);
3744 		}
3745 	}
3746 
3747 	/*
3748 	 * PCI Express Capability - we saved during capability list, and
3749 	 * we'll restore them here.
3750 	 */
3751 	offset = state->hs_pci_cap_offset;
3752 	data32 = state->hs_pci_cap_devctl;
3753 	pci_config_put32(hdl, offset + HERMON_PCI_CAP_DEV_OFFS, data32);
3754 	data32 = state->hs_pci_cap_lnkctl;
3755 	pci_config_put32(hdl, offset + HERMON_PCI_CAP_LNK_OFFS, data32);
3756 
3757 	pci_config_put32(hdl, 0x04, (state->hs_cfg_data[1] | 0x0006));
3758 
3759 	/* the FMA retry loop ends. */
3760 	hermon_pio_end(state, hdl, pio_error2, fm_loop_cnt2, fm_status2,
3761 	    fm_test2);
3762 	hermon_pio_end(state, cmdhdl, pio_error, fm_loop_cnt, fm_status,
3763 	    fm_test);
3764 
3765 	return (DDI_SUCCESS);
3766 
3767 pio_error2:
3768 	/* fall through */
3769 pio_error:
3770 	hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_NON_FATAL);
3771 	return (DDI_FAILURE);
3772 }
3773 
3774 
3775 /*
3776  * hermon_mcg_init()
3777  *    Context: Only called from attach() path context
3778  */
3779 static int
3780 hermon_mcg_init(hermon_state_t *state)
3781 {
3782 	uint_t		mcg_tmp_sz;
3783 
3784 
3785 	/*
3786 	 * Allocate space for the MCG temporary copy buffer.  This is
3787 	 * used by the Attach/Detach Multicast Group code
3788 	 */
3789 	mcg_tmp_sz = HERMON_MCGMEM_SZ(state);
3790 	state->hs_mcgtmp = kmem_zalloc(mcg_tmp_sz, KM_SLEEP);
3791 
3792 	/*
3793 	 * Initialize the multicast group mutex.  This ensures atomic
3794 	 * access to add, modify, and remove entries in the multicast
3795 	 * group hash lists.
3796 	 */
3797 	mutex_init(&state->hs_mcglock, NULL, MUTEX_DRIVER,
3798 	    DDI_INTR_PRI(state->hs_intrmsi_pri));
3799 
3800 	return (DDI_SUCCESS);
3801 }
3802 
3803 
3804 /*
3805  * hermon_mcg_fini()
3806  *    Context: Only called from attach() and/or detach() path contexts
3807  */
3808 static void
3809 hermon_mcg_fini(hermon_state_t *state)
3810 {
3811 	uint_t		mcg_tmp_sz;
3812 
3813 
3814 	/* Free up the space used for the MCG temporary copy buffer */
3815 	mcg_tmp_sz = HERMON_MCGMEM_SZ(state);
3816 	kmem_free(state->hs_mcgtmp, mcg_tmp_sz);
3817 
3818 	/* Destroy the multicast group mutex */
3819 	mutex_destroy(&state->hs_mcglock);
3820 
3821 }
3822 
3823 
3824 /*
3825  * hermon_fw_version_check()
3826  *    Context: Only called from attach() path context
3827  */
3828 static int
3829 hermon_fw_version_check(hermon_state_t *state)
3830 {
3831 
3832 	uint_t	hermon_fw_ver_major;
3833 	uint_t	hermon_fw_ver_minor;
3834 	uint_t	hermon_fw_ver_subminor;
3835 
3836 #ifdef FMA_TEST
3837 	if (hermon_test_num == -1) {
3838 		return (DDI_FAILURE);
3839 	}
3840 #endif
3841 
3842 	/*
3843 	 * Depending on which version of driver we have attached, and which
3844 	 * HCA we've attached, the firmware version checks will be different.
3845 	 * We set up the comparison values for both Arbel and Sinai HCAs.
3846 	 */
3847 	switch (state->hs_operational_mode) {
3848 	case HERMON_HCA_MODE:
3849 		hermon_fw_ver_major = HERMON_FW_VER_MAJOR;
3850 		hermon_fw_ver_minor = HERMON_FW_VER_MINOR;
3851 		hermon_fw_ver_subminor = HERMON_FW_VER_SUBMINOR;
3852 		break;
3853 
3854 	default:
3855 		return (DDI_FAILURE);
3856 	}
3857 
3858 	/*
3859 	 * If FW revision major number is less than acceptable,
3860 	 * return failure, else if greater return success.  If
3861 	 * the major numbers are equal than check the minor number
3862 	 */
3863 	if (state->hs_fw.fw_rev_major < hermon_fw_ver_major) {
3864 		return (DDI_FAILURE);
3865 	} else if (state->hs_fw.fw_rev_major > hermon_fw_ver_major) {
3866 		return (DDI_SUCCESS);
3867 	}
3868 
3869 	/*
3870 	 * Do the same check as above, except for minor revision numbers
3871 	 * If the minor numbers are equal than check the subminor number
3872 	 */
3873 	if (state->hs_fw.fw_rev_minor < hermon_fw_ver_minor) {
3874 		return (DDI_FAILURE);
3875 	} else if (state->hs_fw.fw_rev_minor > hermon_fw_ver_minor) {
3876 		return (DDI_SUCCESS);
3877 	}
3878 
3879 	/*
3880 	 * Once again we do the same check as above, except for the subminor
3881 	 * revision number.  If the subminor numbers are equal here, then
3882 	 * these are the same firmware version, return success
3883 	 */
3884 	if (state->hs_fw.fw_rev_subminor < hermon_fw_ver_subminor) {
3885 		return (DDI_FAILURE);
3886 	} else if (state->hs_fw.fw_rev_subminor > hermon_fw_ver_subminor) {
3887 		return (DDI_SUCCESS);
3888 	}
3889 
3890 	return (DDI_SUCCESS);
3891 }
3892 
3893 
3894 /*
3895  * hermon_device_info_report()
3896  *    Context: Only called from attach() path context
3897  */
3898 static void
3899 hermon_device_info_report(hermon_state_t *state)
3900 {
3901 
3902 	cmn_err(CE_CONT, "?hermon%d: FW ver: %04d.%04d.%04d, "
3903 	    "HW rev: %02d\n", state->hs_instance, state->hs_fw.fw_rev_major,
3904 	    state->hs_fw.fw_rev_minor, state->hs_fw.fw_rev_subminor,
3905 	    state->hs_revision_id);
3906 	cmn_err(CE_CONT, "?hermon%d: %64s (0x%016" PRIx64 ")\n",
3907 	    state->hs_instance, state->hs_nodedesc, state->hs_nodeguid);
3908 
3909 }
3910 
3911 
3912 /*
3913  * hermon_pci_capability_list()
3914  *    Context: Only called from attach() path context
3915  */
3916 static int
3917 hermon_pci_capability_list(hermon_state_t *state, ddi_acc_handle_t hdl)
3918 {
3919 	uint_t		offset, data;
3920 	uint32_t	data32;
3921 
3922 	state->hs_pci_cap_offset = 0;		/* make sure it's cleared */
3923 
3924 	/*
3925 	 * Check for the "PCI Capabilities" bit in the "Status Register".
3926 	 * Bit 4 in this register indicates the presence of a "PCI
3927 	 * Capabilities" list.
3928 	 *
3929 	 * PCI-Express requires this bit to be set to 1.
3930 	 */
3931 	data = pci_config_get16(hdl, 0x06);
3932 	if ((data & 0x10) == 0) {
3933 		return (DDI_FAILURE);
3934 	}
3935 
3936 	/*
3937 	 * Starting from offset 0x34 in PCI config space, find the
3938 	 * head of "PCI capabilities" list, and walk the list.  If
3939 	 * capabilities of a known type are encountered (e.g.
3940 	 * "PCI-X Capability"), then call the appropriate handler
3941 	 * function.
3942 	 */
3943 	offset = pci_config_get8(hdl, 0x34);
3944 	while (offset != 0x0) {
3945 		data = pci_config_get8(hdl, offset);
3946 		/*
3947 		 * Check for known capability types.  Hermon has the
3948 		 * following:
3949 		 *    o Power Mgmt	 (0x02)
3950 		 *    o VPD Capability   (0x03)
3951 		 *    o PCI-E Capability (0x10)
3952 		 *    o MSIX Capability  (0x11)
3953 		 */
3954 		switch (data) {
3955 		case 0x01:
3956 			/* power mgmt handling */
3957 			break;
3958 		case 0x03:
3959 
3960 /*
3961  * Reading the PCIe VPD is inconsistent - that is, sometimes causes
3962  * problems on (mostly) X64, though we've also seen problems w/ Sparc
3963  * and Tavor --- so, for now until it's root caused, don't try and
3964  * read it
3965  */
3966 #ifdef HERMON_VPD_WORKS
3967 			hermon_pci_capability_vpd(state, hdl, offset);
3968 #else
3969 			delay(100);
3970 			hermon_pci_capability_vpd(state, hdl, offset);
3971 #endif
3972 			break;
3973 		case 0x10:
3974 			/*
3975 			 * PCI Express Capability - save offset & contents
3976 			 * for later in reset
3977 			 */
3978 			state->hs_pci_cap_offset = offset;
3979 			data32 = pci_config_get32(hdl,
3980 			    offset + HERMON_PCI_CAP_DEV_OFFS);
3981 			state->hs_pci_cap_devctl = data32;
3982 			data32 = pci_config_get32(hdl,
3983 			    offset + HERMON_PCI_CAP_LNK_OFFS);
3984 			state->hs_pci_cap_lnkctl = data32;
3985 			break;
3986 		case 0x11:
3987 			/*
3988 			 * MSIX support - nothing to do, taken care of in the
3989 			 * MSI/MSIX interrupt frameworkd
3990 			 */
3991 			break;
3992 		default:
3993 			/* just go on to the next */
3994 			break;
3995 		}
3996 
3997 		/* Get offset of next entry in list */
3998 		offset = pci_config_get8(hdl, offset + 1);
3999 	}
4000 
4001 	return (DDI_SUCCESS);
4002 }
4003 
4004 /*
4005  * hermon_pci_read_vpd()
4006  *    Context: Only called from attach() path context
4007  *    utility routine for hermon_pci_capability_vpd()
4008  */
4009 static int
4010 hermon_pci_read_vpd(ddi_acc_handle_t hdl, uint_t offset, uint32_t addr,
4011     uint32_t *data)
4012 {
4013 	int		retry = 40;  /* retry counter for EEPROM poll */
4014 	uint32_t	val;
4015 	int		vpd_addr = offset + 2;
4016 	int		vpd_data = offset + 4;
4017 
4018 	/*
4019 	 * In order to read a 32-bit value from VPD, we are to write down
4020 	 * the address (offset in the VPD itself) to the address register.
4021 	 * To signal the read, we also clear bit 31.  We then poll on bit 31
4022 	 * and when it is set, we can then read our 4 bytes from the data
4023 	 * register.
4024 	 */
4025 	(void) pci_config_put32(hdl, offset, addr << 16);
4026 	do {
4027 		drv_usecwait(1000);
4028 		val = pci_config_get16(hdl, vpd_addr);
4029 		if (val & 0x8000) {		/* flag bit set */
4030 			*data = pci_config_get32(hdl, vpd_data);
4031 			return (DDI_SUCCESS);
4032 		}
4033 	} while (--retry);
4034 	/* read of flag failed write one message but count the failures */
4035 	if (debug_vpd == 0)
4036 		cmn_err(CE_NOTE,
4037 		    "!Failed to see flag bit after VPD addr write\n");
4038 	debug_vpd++;
4039 
4040 
4041 vpd_read_fail:
4042 	return (DDI_FAILURE);
4043 }
4044 
4045 
4046 
4047 /*
4048  *   hermon_pci_capability_vpd()
4049  *    Context: Only called from attach() path context
4050  */
4051 static void
4052 hermon_pci_capability_vpd(hermon_state_t *state, ddi_acc_handle_t hdl,
4053     uint_t offset)
4054 {
4055 	uint8_t			name_length;
4056 	uint8_t			pn_length;
4057 	int			i, err = 0;
4058 	int			vpd_str_id = 0;
4059 	int			vpd_ro_desc;
4060 	int			vpd_ro_pn_desc;
4061 #ifdef _BIG_ENDIAN
4062 	uint32_t		data32;
4063 #endif /* _BIG_ENDIAN */
4064 	union {
4065 		uint32_t	vpd_int[HERMON_VPD_HDR_DWSIZE];
4066 		uchar_t		vpd_char[HERMON_VPD_HDR_BSIZE];
4067 	} vpd;
4068 
4069 
4070 	/*
4071 	 * Read in the Vital Product Data (VPD) to the extend needed
4072 	 * by the fwflash utility
4073 	 */
4074 	for (i = 0; i < HERMON_VPD_HDR_DWSIZE; i++) {
4075 		err = hermon_pci_read_vpd(hdl, offset, i << 2, &vpd.vpd_int[i]);
4076 		if (err != DDI_SUCCESS) {
4077 			cmn_err(CE_NOTE, "!VPD read failed\n");
4078 			goto out;
4079 		}
4080 	}
4081 
4082 #ifdef _BIG_ENDIAN
4083 	/* Need to swap bytes for big endian. */
4084 	for (i = 0; i < HERMON_VPD_HDR_DWSIZE; i++) {
4085 		data32 = vpd.vpd_int[i];
4086 		vpd.vpd_char[(i << 2) + 3] =
4087 		    (uchar_t)((data32 & 0xFF000000) >> 24);
4088 		vpd.vpd_char[(i << 2) + 2] =
4089 		    (uchar_t)((data32 & 0x00FF0000) >> 16);
4090 		vpd.vpd_char[(i << 2) + 1] =
4091 		    (uchar_t)((data32 & 0x0000FF00) >> 8);
4092 		vpd.vpd_char[i << 2] = (uchar_t)(data32 & 0x000000FF);
4093 	}
4094 #endif	/* _BIG_ENDIAN */
4095 
4096 	/* Check for VPD String ID Tag */
4097 	if (vpd.vpd_char[vpd_str_id] == 0x82) {
4098 		/* get the product name */
4099 		name_length = (uint8_t)vpd.vpd_char[vpd_str_id + 1];
4100 		if (name_length > sizeof (state->hs_hca_name)) {
4101 			cmn_err(CE_NOTE, "!VPD name too large (0x%x)\n",
4102 			    name_length);
4103 			goto out;
4104 		}
4105 		(void) memcpy(state->hs_hca_name, &vpd.vpd_char[vpd_str_id + 3],
4106 		    name_length);
4107 		state->hs_hca_name[name_length] = 0;
4108 
4109 		/* get the part number */
4110 		vpd_ro_desc = name_length + 3; /* read-only tag location */
4111 		vpd_ro_pn_desc = vpd_ro_desc + 3; /* P/N keyword location */
4112 
4113 		/* Verify read-only tag and Part Number keyword. */
4114 		if (vpd.vpd_char[vpd_ro_desc] != 0x90 ||
4115 		    (vpd.vpd_char[vpd_ro_pn_desc] != 'P' &&
4116 		    vpd.vpd_char[vpd_ro_pn_desc + 1] != 'N')) {
4117 			cmn_err(CE_NOTE, "!VPD Part Number not found\n");
4118 			goto out;
4119 		}
4120 
4121 		pn_length = (uint8_t)vpd.vpd_char[vpd_ro_pn_desc + 2];
4122 		if (pn_length > sizeof (state->hs_hca_pn)) {
4123 			cmn_err(CE_NOTE, "!VPD part number too large (0x%x)\n",
4124 			    name_length);
4125 			goto out;
4126 		}
4127 		(void) memcpy(state->hs_hca_pn,
4128 		    &vpd.vpd_char[vpd_ro_pn_desc + 3],
4129 		    pn_length);
4130 		state->hs_hca_pn[pn_length] = 0;
4131 		state->hs_hca_pn_len = pn_length;
4132 		cmn_err(CE_CONT, "!vpd %s\n", state->hs_hca_pn);
4133 	} else {
4134 		/* Wrong VPD String ID Tag */
4135 		cmn_err(CE_NOTE, "!VPD String ID Tag not found, tag: %02x\n",
4136 		    vpd.vpd_char[0]);
4137 		goto out;
4138 	}
4139 	return;
4140 out:
4141 	state->hs_hca_pn_len = 0;
4142 }
4143 
4144 
4145 
4146 /*
4147  * hermon_intr_or_msi_init()
4148  *    Context: Only called from attach() path context
4149  */
4150 static int
4151 hermon_intr_or_msi_init(hermon_state_t *state)
4152 {
4153 	int	status;
4154 
4155 
4156 	/* Query for the list of supported interrupt event types */
4157 	status = ddi_intr_get_supported_types(state->hs_dip,
4158 	    &state->hs_intr_types_avail);
4159 	if (status != DDI_SUCCESS) {
4160 		return (DDI_FAILURE);
4161 	}
4162 
4163 	/*
4164 	 * If Hermon supports MSI-X in this system (and, if it
4165 	 * hasn't been overridden by a configuration variable), then
4166 	 * the default behavior is to use a single MSI-X.  Otherwise,
4167 	 * fallback to using legacy interrupts.  Also, if MSI-X is chosen,
4168 	 * but fails for whatever reasons, then next try MSI
4169 	 */
4170 	if ((state->hs_cfg_profile->cp_use_msi_if_avail != 0) &&
4171 	    (state->hs_intr_types_avail & DDI_INTR_TYPE_MSIX)) {
4172 		status = hermon_add_intrs(state, DDI_INTR_TYPE_MSIX);
4173 		if (status == DDI_SUCCESS) {
4174 			state->hs_intr_type_chosen = DDI_INTR_TYPE_MSIX;
4175 			return (DDI_SUCCESS);
4176 		}
4177 	}
4178 
4179 	/*
4180 	 * If Hermon supports MSI in this system (and, if it
4181 	 * hasn't been overridden by a configuration variable), then
4182 	 * the default behavior is to use a single MSIX.  Otherwise,
4183 	 * fallback to using legacy interrupts.  Also, if MSI is chosen,
4184 	 * but fails for whatever reasons, then fallback to using legacy
4185 	 * interrupts.
4186 	 */
4187 	if ((state->hs_cfg_profile->cp_use_msi_if_avail != 0) &&
4188 	    (state->hs_intr_types_avail & DDI_INTR_TYPE_MSI)) {
4189 		status = hermon_add_intrs(state, DDI_INTR_TYPE_MSI);
4190 		if (status == DDI_SUCCESS) {
4191 			state->hs_intr_type_chosen = DDI_INTR_TYPE_MSI;
4192 			return (DDI_SUCCESS);
4193 		}
4194 	}
4195 
4196 	/*
4197 	 * MSI interrupt allocation failed, or was not available.  Fallback to
4198 	 * legacy interrupt support.
4199 	 */
4200 	if (state->hs_intr_types_avail & DDI_INTR_TYPE_FIXED) {
4201 		status = hermon_add_intrs(state, DDI_INTR_TYPE_FIXED);
4202 		if (status == DDI_SUCCESS) {
4203 			state->hs_intr_type_chosen = DDI_INTR_TYPE_FIXED;
4204 			return (DDI_SUCCESS);
4205 		}
4206 	}
4207 
4208 	/*
4209 	 * None of MSI, MSI-X, nor legacy interrupts were successful.
4210 	 * Return failure.
4211 	 */
4212 	return (DDI_FAILURE);
4213 }
4214 
4215 /*
4216  * hermon_add_intrs()
4217  *    Context: Only called from attach() patch context
4218  */
4219 static int
4220 hermon_add_intrs(hermon_state_t *state, int intr_type)
4221 {
4222 	int	status;
4223 
4224 
4225 	/* Get number of interrupts/MSI supported */
4226 	status = ddi_intr_get_nintrs(state->hs_dip, intr_type,
4227 	    &state->hs_intrmsi_count);
4228 	if (status != DDI_SUCCESS) {
4229 		return (DDI_FAILURE);
4230 	}
4231 
4232 	/* Get number of available interrupts/MSI */
4233 	status = ddi_intr_get_navail(state->hs_dip, intr_type,
4234 	    &state->hs_intrmsi_avail);
4235 	if (status != DDI_SUCCESS) {
4236 		return (DDI_FAILURE);
4237 	}
4238 
4239 	/* Ensure that we have at least one (1) usable MSI or interrupt */
4240 	if ((state->hs_intrmsi_avail < 1) || (state->hs_intrmsi_count < 1)) {
4241 		return (DDI_FAILURE);
4242 	}
4243 
4244 	/* Attempt to allocate the maximum #interrupt/MSI handles */
4245 	status = ddi_intr_alloc(state->hs_dip, &state->hs_intrmsi_hdl[0],
4246 	    intr_type, 0, min(HERMON_MSIX_MAX, state->hs_intrmsi_avail),
4247 	    &state->hs_intrmsi_allocd, DDI_INTR_ALLOC_NORMAL);
4248 	if (status != DDI_SUCCESS) {
4249 		return (DDI_FAILURE);
4250 	}
4251 
4252 	/* Ensure that we have allocated at least one (1) MSI or interrupt */
4253 	if (state->hs_intrmsi_allocd < 1) {
4254 		return (DDI_FAILURE);
4255 	}
4256 	state->hs_eq_dist = state->hs_intrmsi_allocd - 1; /* start at 0 */
4257 
4258 	/*
4259 	 * Extract the priority for the allocated interrupt/MSI.  This
4260 	 * will be used later when initializing certain mutexes.
4261 	 */
4262 	status = ddi_intr_get_pri(state->hs_intrmsi_hdl[0],
4263 	    &state->hs_intrmsi_pri);
4264 	if (status != DDI_SUCCESS) {
4265 		/* Free the allocated interrupt/MSI handle */
4266 		(void) ddi_intr_free(state->hs_intrmsi_hdl[0]);
4267 
4268 		return (DDI_FAILURE);
4269 	}
4270 
4271 	/* Make sure the interrupt/MSI priority is below 'high level' */
4272 	if (state->hs_intrmsi_pri >= ddi_intr_get_hilevel_pri()) {
4273 		/* Free the allocated interrupt/MSI handle */
4274 		(void) ddi_intr_free(state->hs_intrmsi_hdl[0]);
4275 
4276 		return (DDI_FAILURE);
4277 	}
4278 
4279 	/* Get add'l capability information regarding interrupt/MSI */
4280 	status = ddi_intr_get_cap(state->hs_intrmsi_hdl[0],
4281 	    &state->hs_intrmsi_cap);
4282 	if (status != DDI_SUCCESS) {
4283 		/* Free the allocated interrupt/MSI handle */
4284 		(void) ddi_intr_free(state->hs_intrmsi_hdl[0]);
4285 
4286 		return (DDI_FAILURE);
4287 	}
4288 
4289 	return (DDI_SUCCESS);
4290 }
4291 
4292 
4293 /*
4294  * hermon_intr_or_msi_fini()
4295  *    Context: Only called from attach() and/or detach() path contexts
4296  */
4297 static int
4298 hermon_intr_or_msi_fini(hermon_state_t *state)
4299 {
4300 	int	status;
4301 	int	intr;
4302 
4303 	for (intr = 0; intr < state->hs_intrmsi_allocd; intr++) {
4304 
4305 		/* Free the allocated interrupt/MSI handle */
4306 		status = ddi_intr_free(state->hs_intrmsi_hdl[intr]);
4307 		if (status != DDI_SUCCESS) {
4308 			return (DDI_FAILURE);
4309 		}
4310 	}
4311 	return (DDI_SUCCESS);
4312 }
4313 
4314 
4315 /*ARGSUSED*/
4316 void
4317 hermon_pci_capability_msix(hermon_state_t *state, ddi_acc_handle_t hdl,
4318     uint_t offset)
4319 {
4320 	uint32_t	msix_data;
4321 	uint16_t	msg_cntr;
4322 	uint32_t	t_offset;	/* table offset */
4323 	uint32_t	t_bir;
4324 	uint32_t	p_offset;	/* pba */
4325 	uint32_t	p_bir;
4326 	int		t_size;		/* size in entries - each is 4 dwords */
4327 
4328 	/* come in with offset pointing at the capability structure */
4329 
4330 	msix_data = pci_config_get32(hdl, offset);
4331 	cmn_err(CE_CONT, "Full cap structure dword = %X\n", msix_data);
4332 	msg_cntr =  pci_config_get16(hdl, offset+2);
4333 	cmn_err(CE_CONT, "MSIX msg_control = %X\n", msg_cntr);
4334 	offset += 4;
4335 	msix_data = pci_config_get32(hdl, offset);	/* table info */
4336 	t_offset = (msix_data & 0xFFF8) >> 3;
4337 	t_bir = msix_data & 0x07;
4338 	offset += 4;
4339 	cmn_err(CE_CONT, "  table %X --offset = %X, bir(bar) = %X\n",
4340 	    msix_data, t_offset, t_bir);
4341 	msix_data = pci_config_get32(hdl, offset);	/* PBA info */
4342 	p_offset = (msix_data & 0xFFF8) >> 3;
4343 	p_bir = msix_data & 0x07;
4344 
4345 	cmn_err(CE_CONT, "  PBA   %X --offset = %X, bir(bar) = %X\n",
4346 	    msix_data, p_offset, p_bir);
4347 	t_size = msg_cntr & 0x7FF;		/* low eleven bits */
4348 	cmn_err(CE_CONT, "    table size = %X entries\n", t_size);
4349 
4350 	offset = t_offset;		/* reuse this for offset from BAR */
4351 #ifdef HERMON_SUPPORTS_MSIX_BAR
4352 	cmn_err(CE_CONT, "First 2 table entries behind BAR2 \n");
4353 	for (i = 0; i < 2; i++) {
4354 		for (j = 0; j < 4; j++, offset += 4) {
4355 			msix_data = ddi_get32(state->hs_reg_msihdl,
4356 			    (uint32_t *)((uintptr_t)state->hs_reg_msi_baseaddr
4357 			    + offset));
4358 			cmn_err(CE_CONT, "MSI table entry %d, dword %d == %X\n",
4359 			    i, j, msix_data);
4360 		}
4361 	}
4362 #endif
4363 
4364 }
4365 
4366 /*
4367  * X86 fastreboot support functions.
4368  * These functions are used to save/restore MSI-X table/PBA and also
4369  * to disable MSI-X interrupts in hermon_quiesce().
4370  */
4371 
4372 /* Return the message control for MSI-X */
4373 static ushort_t
4374 get_msix_ctrl(dev_info_t *dip)
4375 {
4376 	ushort_t msix_ctrl = 0, caps_ctrl = 0;
4377 	hermon_state_t *state = ddi_get_soft_state(hermon_statep,
4378 	    DEVI(dip)->devi_instance);
4379 	ddi_acc_handle_t pci_cfg_hdl = hermon_get_pcihdl(state);
4380 	ASSERT(pci_cfg_hdl != NULL);
4381 
4382 	if ((PCI_CAP_LOCATE(pci_cfg_hdl,
4383 	    PCI_CAP_ID_MSI_X, &caps_ctrl) == DDI_SUCCESS)) {
4384 		if ((msix_ctrl = PCI_CAP_GET16(pci_cfg_hdl, NULL, caps_ctrl,
4385 		    PCI_MSIX_CTRL)) == PCI_CAP_EINVAL16)
4386 			return (0);
4387 	}
4388 	ASSERT(msix_ctrl != 0);
4389 
4390 	return (msix_ctrl);
4391 }
4392 
4393 /* Return the MSI-X table size */
4394 static size_t
4395 get_msix_tbl_size(dev_info_t *dip)
4396 {
4397 	ushort_t msix_ctrl = get_msix_ctrl(dip);
4398 	ASSERT(msix_ctrl != 0);
4399 
4400 	return (((msix_ctrl & PCI_MSIX_TBL_SIZE_MASK) + 1) *
4401 	    PCI_MSIX_VECTOR_SIZE);
4402 }
4403 
4404 /* Return the MSI-X PBA size */
4405 static size_t
4406 get_msix_pba_size(dev_info_t *dip)
4407 {
4408 	ushort_t msix_ctrl = get_msix_ctrl(dip);
4409 	ASSERT(msix_ctrl != 0);
4410 
4411 	return (((msix_ctrl & PCI_MSIX_TBL_SIZE_MASK) + 64) / 64 * 8);
4412 }
4413 
4414 /* Set up the MSI-X table/PBA save area */
4415 static void
4416 hermon_set_msix_info(hermon_state_t *state)
4417 {
4418 	uint_t			rnumber, breg, nregs;
4419 	ushort_t		caps_ctrl, msix_ctrl;
4420 	pci_regspec_t		*rp;
4421 	int			reg_size, addr_space, offset, *regs_list, i;
4422 
4423 	/*
4424 	 * MSI-X BIR Index Table:
4425 	 * BAR indicator register (BIR) to Base Address register.
4426 	 */
4427 	uchar_t pci_msix_bir_index[8] = {0x10, 0x14, 0x18, 0x1c,
4428 	    0x20, 0x24, 0xff, 0xff};
4429 
4430 	/* Fastreboot data access  attribute */
4431 	ddi_device_acc_attr_t	dev_attr = {
4432 		0,				/* version */
4433 		DDI_STRUCTURE_LE_ACC,
4434 		DDI_STRICTORDER_ACC,		/* attr access */
4435 		0
4436 	};
4437 
4438 	ddi_acc_handle_t pci_cfg_hdl = hermon_get_pcihdl(state);
4439 	ASSERT(pci_cfg_hdl != NULL);
4440 
4441 	if ((PCI_CAP_LOCATE(pci_cfg_hdl,
4442 	    PCI_CAP_ID_MSI_X, &caps_ctrl) == DDI_SUCCESS)) {
4443 		if ((msix_ctrl = PCI_CAP_GET16(pci_cfg_hdl, NULL, caps_ctrl,
4444 		    PCI_MSIX_CTRL)) == PCI_CAP_EINVAL16)
4445 			return;
4446 	}
4447 	ASSERT(msix_ctrl != 0);
4448 
4449 	state->hs_msix_tbl_offset = PCI_CAP_GET32(pci_cfg_hdl, NULL, caps_ctrl,
4450 	    PCI_MSIX_TBL_OFFSET);
4451 
4452 	/* Get the BIR for MSI-X table */
4453 	breg = pci_msix_bir_index[state->hs_msix_tbl_offset &
4454 	    PCI_MSIX_TBL_BIR_MASK];
4455 	ASSERT(breg != 0xFF);
4456 
4457 	/* Set the MSI-X table offset */
4458 	state->hs_msix_tbl_offset = state->hs_msix_tbl_offset &
4459 	    ~PCI_MSIX_TBL_BIR_MASK;
4460 
4461 	/* Set the MSI-X table size */
4462 	state->hs_msix_tbl_size = ((msix_ctrl & PCI_MSIX_TBL_SIZE_MASK) + 1) *
4463 	    PCI_MSIX_VECTOR_SIZE;
4464 
4465 	if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, state->hs_dip,
4466 	    DDI_PROP_DONTPASS, "reg", (int **)&regs_list, &nregs) !=
4467 	    DDI_PROP_SUCCESS) {
4468 		return;
4469 	}
4470 	reg_size = sizeof (pci_regspec_t) / sizeof (int);
4471 
4472 	/* Check the register number for MSI-X table */
4473 	for (i = 1, rnumber = 0; i < nregs/reg_size; i++) {
4474 		rp = (pci_regspec_t *)&regs_list[i * reg_size];
4475 		addr_space = rp->pci_phys_hi & PCI_ADDR_MASK;
4476 		offset = PCI_REG_REG_G(rp->pci_phys_hi);
4477 
4478 		if ((offset == breg) && ((addr_space == PCI_ADDR_MEM32) ||
4479 		    (addr_space == PCI_ADDR_MEM64))) {
4480 			rnumber = i;
4481 			break;
4482 		}
4483 	}
4484 	ASSERT(rnumber != 0);
4485 	state->hs_msix_tbl_rnumber = rnumber;
4486 
4487 	/* Set device attribute version and access according to Hermon FM */
4488 	dev_attr.devacc_attr_version = hermon_devacc_attr_version(state);
4489 	dev_attr.devacc_attr_access = hermon_devacc_attr_access(state);
4490 
4491 	/* Map the entire MSI-X vector table */
4492 	if (hermon_regs_map_setup(state, state->hs_msix_tbl_rnumber,
4493 	    (caddr_t *)&state->hs_msix_tbl_addr, state->hs_msix_tbl_offset,
4494 	    state->hs_msix_tbl_size, &dev_attr,
4495 	    &state->hs_fm_msix_tblhdl) != DDI_SUCCESS) {
4496 		return;
4497 	}
4498 
4499 	state->hs_msix_pba_offset = PCI_CAP_GET32(pci_cfg_hdl, NULL, caps_ctrl,
4500 	    PCI_MSIX_PBA_OFFSET);
4501 
4502 	/* Get the BIR for MSI-X PBA */
4503 	breg = pci_msix_bir_index[state->hs_msix_pba_offset &
4504 	    PCI_MSIX_PBA_BIR_MASK];
4505 	ASSERT(breg != 0xFF);
4506 
4507 	/* Set the MSI-X PBA offset */
4508 	state->hs_msix_pba_offset = state->hs_msix_pba_offset &
4509 	    ~PCI_MSIX_PBA_BIR_MASK;
4510 
4511 	/* Set the MSI-X PBA size */
4512 	state->hs_msix_pba_size =
4513 	    ((msix_ctrl & PCI_MSIX_TBL_SIZE_MASK) + 64) / 64 * 8;
4514 
4515 	/* Check the register number for MSI-X PBA */
4516 	for (i = 1, rnumber = 0; i < nregs/reg_size; i++) {
4517 		rp = (pci_regspec_t *)&regs_list[i * reg_size];
4518 		addr_space = rp->pci_phys_hi & PCI_ADDR_MASK;
4519 		offset = PCI_REG_REG_G(rp->pci_phys_hi);
4520 
4521 		if ((offset == breg) && ((addr_space == PCI_ADDR_MEM32) ||
4522 		    (addr_space == PCI_ADDR_MEM64))) {
4523 			rnumber = i;
4524 			break;
4525 		}
4526 	}
4527 	ASSERT(rnumber != 0);
4528 	state->hs_msix_pba_rnumber = rnumber;
4529 
4530 	/* Map in the MSI-X Pending Bit Array */
4531 	if (hermon_regs_map_setup(state, state->hs_msix_pba_rnumber,
4532 	    (caddr_t *)&state->hs_msix_pba_addr, state->hs_msix_pba_offset,
4533 	    state->hs_msix_pba_size, &dev_attr,
4534 	    &state->hs_fm_msix_pbahdl) != DDI_SUCCESS) {
4535 		hermon_regs_map_free(state, &state->hs_fm_msix_tblhdl);
4536 		state->hs_fm_msix_tblhdl = NULL;
4537 		return;
4538 	}
4539 
4540 	/* Set the MSI-X table save area */
4541 	state->hs_msix_tbl_entries = kmem_alloc(state->hs_msix_tbl_size,
4542 	    KM_SLEEP);
4543 
4544 	/* Set the MSI-X PBA save area */
4545 	state->hs_msix_pba_entries = kmem_alloc(state->hs_msix_pba_size,
4546 	    KM_SLEEP);
4547 }
4548 
4549 /* Disable Hermon interrupts */
4550 static int
4551 hermon_intr_disable(hermon_state_t *state)
4552 {
4553 	ushort_t msix_ctrl = 0, caps_ctrl = 0;
4554 	ddi_acc_handle_t pci_cfg_hdl = hermon_get_pcihdl(state);
4555 	ddi_acc_handle_t msix_tblhdl = hermon_get_msix_tblhdl(state);
4556 	int i, j;
4557 	ASSERT(pci_cfg_hdl != NULL && msix_tblhdl != NULL);
4558 	ASSERT(state->hs_intr_types_avail &
4559 	    (DDI_INTR_TYPE_FIXED | DDI_INTR_TYPE_MSI | DDI_INTR_TYPE_MSIX));
4560 
4561 	/*
4562 	 * Check if MSI-X interrupts are used. If so, disable MSI-X interupts.
4563 	 * If not, since Hermon doesn't support MSI interrupts, assuming the
4564 	 * legacy interrupt is used instead, disable the legacy interrupt.
4565 	 */
4566 	if ((state->hs_cfg_profile->cp_use_msi_if_avail != 0) &&
4567 	    (state->hs_intr_types_avail & DDI_INTR_TYPE_MSIX)) {
4568 
4569 		if ((PCI_CAP_LOCATE(pci_cfg_hdl,
4570 		    PCI_CAP_ID_MSI_X, &caps_ctrl) == DDI_SUCCESS)) {
4571 			if ((msix_ctrl = PCI_CAP_GET16(pci_cfg_hdl, NULL,
4572 			    caps_ctrl, PCI_MSIX_CTRL)) == PCI_CAP_EINVAL16)
4573 				return (DDI_FAILURE);
4574 		}
4575 		ASSERT(msix_ctrl != 0);
4576 
4577 		if (!(msix_ctrl & PCI_MSIX_ENABLE_BIT))
4578 			return (DDI_SUCCESS);
4579 
4580 		/* Clear all inums in MSI-X table */
4581 		for (i = 0; i < get_msix_tbl_size(state->hs_dip);
4582 		    i += PCI_MSIX_VECTOR_SIZE) {
4583 			for (j = 0; j < PCI_MSIX_VECTOR_SIZE; j += 4) {
4584 				char *addr = state->hs_msix_tbl_addr + i + j;
4585 				ddi_put32(msix_tblhdl,
4586 				    (uint32_t *)(uintptr_t)addr, 0x0);
4587 			}
4588 		}
4589 
4590 		/* Disable MSI-X interrupts */
4591 		msix_ctrl &= ~PCI_MSIX_ENABLE_BIT;
4592 		PCI_CAP_PUT16(pci_cfg_hdl, NULL, caps_ctrl, PCI_MSIX_CTRL,
4593 		    msix_ctrl);
4594 
4595 	} else {
4596 		uint16_t cmdreg = pci_config_get16(pci_cfg_hdl, PCI_CONF_COMM);
4597 		ASSERT(state->hs_intr_types_avail & DDI_INTR_TYPE_FIXED);
4598 
4599 		/* Disable the legacy interrupts */
4600 		cmdreg |= PCI_COMM_INTX_DISABLE;
4601 		pci_config_put16(pci_cfg_hdl, PCI_CONF_COMM, cmdreg);
4602 	}
4603 
4604 	return (DDI_SUCCESS);
4605 }
4606 
4607 /* Hermon quiesce(9F) entry */
4608 static int
4609 hermon_quiesce(dev_info_t *dip)
4610 {
4611 	hermon_state_t *state = ddi_get_soft_state(hermon_statep,
4612 	    DEVI(dip)->devi_instance);
4613 	ddi_acc_handle_t pcihdl = hermon_get_pcihdl(state);
4614 	ddi_acc_handle_t cmdhdl = hermon_get_cmdhdl(state);
4615 	ddi_acc_handle_t msix_tbl_hdl = hermon_get_msix_tblhdl(state);
4616 	ddi_acc_handle_t msix_pba_hdl = hermon_get_msix_pbahdl(state);
4617 	uint32_t sem, reset_delay = state->hs_cfg_profile->cp_sw_reset_delay;
4618 	uint64_t data64;
4619 	uint32_t data32;
4620 	int status, i, j, loopcnt;
4621 	uint_t offset;
4622 
4623 	ASSERT(state != NULL);
4624 
4625 	/* start fastreboot */
4626 	state->hs_quiescing = B_TRUE;
4627 
4628 	/* If it's in maintenance mode, do nothing but return with SUCCESS */
4629 	if (!HERMON_IS_OPERATIONAL(state->hs_operational_mode)) {
4630 		return (DDI_SUCCESS);
4631 	}
4632 
4633 	/* suppress Hermon FM ereports */
4634 	if (hermon_get_state(state) & HCA_EREPORT_FM) {
4635 		hermon_clr_state_nolock(state, HCA_EREPORT_FM);
4636 	}
4637 
4638 	/* Shutdown HCA ports */
4639 	if (hermon_hca_ports_shutdown(state,
4640 	    state->hs_cfg_profile->cp_num_ports) != HERMON_CMD_SUCCESS) {
4641 		state->hs_quiescing = B_FALSE;
4642 		return (DDI_FAILURE);
4643 	}
4644 
4645 	/* Close HCA */
4646 	if (hermon_close_hca_cmd_post(state, HERMON_CMD_NOSLEEP_SPIN) !=
4647 	    HERMON_CMD_SUCCESS) {
4648 		state->hs_quiescing = B_FALSE;
4649 		return (DDI_FAILURE);
4650 	}
4651 
4652 	/* Disable interrupts */
4653 	if (hermon_intr_disable(state) != DDI_SUCCESS) {
4654 		state->hs_quiescing = B_FALSE;
4655 		return (DDI_FAILURE);
4656 	}
4657 
4658 	/*
4659 	 * Query the PCI capabilities of the HCA device, but don't process
4660 	 * the VPD until after reset.
4661 	 */
4662 	if (hermon_pci_capability_list(state, pcihdl) != DDI_SUCCESS) {
4663 		state->hs_quiescing = B_FALSE;
4664 		return (DDI_FAILURE);
4665 	}
4666 
4667 	/*
4668 	 * Read all PCI config info (reg0...reg63).  Note: According to the
4669 	 * Hermon software reset application note, we should not read or
4670 	 * restore the values in reg22 and reg23.
4671 	 * NOTE:  For Hermon (and Arbel too) it says to restore the command
4672 	 * register LAST, and technically, you need to restore the
4673 	 * PCIE Capability "device control" and "link control" (word-sized,
4674 	 * at offsets 0x08 and 0x10 from the capbility ID respectively).
4675 	 * We hold off restoring the command register - offset 0x4 - till last
4676 	 */
4677 
4678 	/* 1st, wait for the semaphore assure accessibility - per PRM */
4679 	status = -1;
4680 	for (i = 0; i < NANOSEC/MICROSEC /* 1sec timeout */; i++) {
4681 		sem = ddi_get32(cmdhdl, state->hs_cmd_regs.sw_semaphore);
4682 		if (sem == 0) {
4683 			status = 0;
4684 			break;
4685 		}
4686 		drv_usecwait(1);
4687 	}
4688 
4689 	/* Check if timeout happens */
4690 	if (status == -1) {
4691 		state->hs_quiescing = B_FALSE;
4692 		return (DDI_FAILURE);
4693 	}
4694 
4695 	/* MSI-X interrupts are used, save the MSI-X table */
4696 	if (msix_tbl_hdl && msix_pba_hdl) {
4697 		/* save MSI-X table */
4698 		for (i = 0; i < get_msix_tbl_size(state->hs_dip);
4699 		    i += PCI_MSIX_VECTOR_SIZE) {
4700 			for (j = 0; j < PCI_MSIX_VECTOR_SIZE; j += 4) {
4701 				char *addr = state->hs_msix_tbl_addr + i + j;
4702 				data32 = ddi_get32(msix_tbl_hdl,
4703 				    (uint32_t *)(uintptr_t)addr);
4704 				*(uint32_t *)(uintptr_t)(state->
4705 				    hs_msix_tbl_entries + i + j) = data32;
4706 			}
4707 		}
4708 		/* save MSI-X PBA */
4709 		for (i = 0; i < get_msix_pba_size(state->hs_dip); i += 8) {
4710 			char *addr = state->hs_msix_pba_addr + i;
4711 			data64 = ddi_get64(msix_pba_hdl,
4712 			    (uint64_t *)(uintptr_t)addr);
4713 			*(uint64_t *)(uintptr_t)(state->
4714 			    hs_msix_pba_entries + i) = data64;
4715 		}
4716 	}
4717 
4718 	/* save PCI config space */
4719 	for (i = 0; i < HERMON_SW_RESET_NUMREGS; i++) {
4720 		if ((i != HERMON_SW_RESET_REG22_RSVD) &&
4721 		    (i != HERMON_SW_RESET_REG23_RSVD)) {
4722 			state->hs_cfg_data[i]  =
4723 			    pci_config_get32(pcihdl, i << 2);
4724 		}
4725 	}
4726 
4727 	/* SW-reset HCA */
4728 	ddi_put32(cmdhdl, state->hs_cmd_regs.sw_reset, HERMON_SW_RESET_START);
4729 
4730 	/*
4731 	 * This delay is required so as not to cause a panic here. If the
4732 	 * device is accessed too soon after reset it will not respond to
4733 	 * config cycles, causing a Master Abort and panic.
4734 	 */
4735 	drv_usecwait(reset_delay);
4736 
4737 	/* Poll waiting for the device to finish resetting */
4738 	loopcnt = 100;	/* 100 times @ 100 usec - total delay 10 msec */
4739 	while ((pci_config_get32(pcihdl, 0) & 0x0000FFFF) != PCI_VENID_MLX) {
4740 		drv_usecwait(HERMON_SW_RESET_POLL_DELAY);
4741 		if (--loopcnt == 0)
4742 			break;	/* just in case, break and go on */
4743 	}
4744 	if (loopcnt == 0) {
4745 		state->hs_quiescing = B_FALSE;
4746 		return (DDI_FAILURE);
4747 	}
4748 
4749 	/* Restore the config info */
4750 	for (i = 0; i < HERMON_SW_RESET_NUMREGS; i++) {
4751 		if (i == 1) continue;	/* skip the status/ctrl reg */
4752 		if ((i != HERMON_SW_RESET_REG22_RSVD) &&
4753 		    (i != HERMON_SW_RESET_REG23_RSVD)) {
4754 			pci_config_put32(pcihdl, i << 2, state->hs_cfg_data[i]);
4755 		}
4756 	}
4757 
4758 	/* If MSI-X interrupts are used, restore the MSI-X table */
4759 	if (msix_tbl_hdl && msix_pba_hdl) {
4760 		/* restore MSI-X PBA */
4761 		for (i = 0; i < get_msix_pba_size(state->hs_dip); i += 8) {
4762 			char *addr = state->hs_msix_pba_addr + i;
4763 			data64 = *(uint64_t *)(uintptr_t)
4764 			    (state->hs_msix_pba_entries + i);
4765 			ddi_put64(msix_pba_hdl,
4766 			    (uint64_t *)(uintptr_t)addr, data64);
4767 		}
4768 		/* restore MSI-X table */
4769 		for (i = 0; i < get_msix_tbl_size(state->hs_dip);
4770 		    i += PCI_MSIX_VECTOR_SIZE) {
4771 			for (j = 0; j < PCI_MSIX_VECTOR_SIZE; j += 4) {
4772 				char *addr = state->hs_msix_tbl_addr + i + j;
4773 				data32 = *(uint32_t *)(uintptr_t)
4774 				    (state->hs_msix_tbl_entries + i + j);
4775 				ddi_put32(msix_tbl_hdl,
4776 				    (uint32_t *)(uintptr_t)addr, data32);
4777 			}
4778 		}
4779 	}
4780 
4781 	/*
4782 	 * PCI Express Capability - we saved during capability list, and
4783 	 * we'll restore them here.
4784 	 */
4785 	offset = state->hs_pci_cap_offset;
4786 	data32 = state->hs_pci_cap_devctl;
4787 	pci_config_put32(pcihdl, offset + HERMON_PCI_CAP_DEV_OFFS, data32);
4788 	data32 = state->hs_pci_cap_lnkctl;
4789 	pci_config_put32(pcihdl, offset + HERMON_PCI_CAP_LNK_OFFS, data32);
4790 
4791 	/* restore the command register */
4792 	pci_config_put32(pcihdl, 0x04, (state->hs_cfg_data[1] | 0x0006));
4793 
4794 	return (DDI_SUCCESS);
4795 }
4796