xref: /illumos-gate/usr/src/uts/common/io/scsi/adapters/pmcs/pmcs_attach.c (revision b46556d07aa5469d61a8b3e3067bb7c3c26f0a7c)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 #include <sys/scsi/adapters/pmcs/pmcs.h>
25 
26 #define	PMCS_DRIVER_VERSION	"pmcs HBA device driver"
27 
28 static	char	*pmcs_driver_rev = PMCS_DRIVER_VERSION;
29 
30 /*
31  * Non-DDI Compliant stuff
32  */
33 extern char hw_serial[];
34 
35 /*
36  * Global driver data
37  */
38 void *pmcs_softc_state = NULL;
39 void *pmcs_iport_softstate = NULL;
40 
41 /*
42  * Tracing and Logging info
43  */
44 pmcs_tbuf_t *pmcs_tbuf = NULL;
45 uint32_t pmcs_tbuf_num_elems = 0;
46 pmcs_tbuf_t *pmcs_tbuf_ptr;
47 uint32_t pmcs_tbuf_idx = 0;
48 boolean_t pmcs_tbuf_wrap = B_FALSE;
49 kmutex_t pmcs_trace_lock;
50 
51 /*
52  * If pmcs_force_syslog value is non-zero, all messages put in the trace log
53  * will also be sent to system log.
54  */
55 int pmcs_force_syslog = 0;
56 int pmcs_console = 0;
57 
58 /*
59  * External References
60  */
61 extern int ncpus_online;
62 
63 /*
64  * Local static data
65  */
66 static int fwlog_level = 3;
67 static int physpeed = PHY_LINK_ALL;
68 static int phymode = PHY_LM_AUTO;
69 static int block_mask = 0;
70 static int phymap_stable_usec = 3 * MICROSEC;
71 static int iportmap_stable_usec = 2 * MICROSEC;
72 static int iportmap_csync_usec = 20 * MICROSEC;
73 
74 #ifdef DEBUG
75 static int debug_mask = 1;
76 #else
77 static int debug_mask = 0;
78 #endif
79 
80 #ifdef DISABLE_MSIX
81 static int disable_msix = 1;
82 #else
83 static int disable_msix = 0;
84 #endif
85 
86 #ifdef DISABLE_MSI
87 static int disable_msi = 1;
88 #else
89 static int disable_msi = 0;
90 #endif
91 
92 /*
93  * DEBUG: testing: allow detach with an active port:
94  *
95  * # echo 'detach_driver_unconfig/W 10'		| mdb -kw
96  * # echo 'scsi_hba_bus_unconfig_remove/W 1'	| mdb -kw
97  * # echo 'pmcs`detach_with_active_port/W 1'	| mdb -kw
98  * # modunload -i <pmcs_driver_index>
99  */
100 static int detach_with_active_port = 0;
101 
102 static uint16_t maxqdepth = 0xfffe;
103 
104 /*
105  * Local prototypes
106  */
107 static int pmcs_attach(dev_info_t *, ddi_attach_cmd_t);
108 static int pmcs_detach(dev_info_t *, ddi_detach_cmd_t);
109 static int pmcs_unattach(pmcs_hw_t *);
110 static int pmcs_iport_unattach(pmcs_iport_t *);
111 static int pmcs_add_more_chunks(pmcs_hw_t *, unsigned long);
112 static void pmcs_watchdog(void *);
113 static int pmcs_setup_intr(pmcs_hw_t *);
114 static int pmcs_teardown_intr(pmcs_hw_t *);
115 
116 static uint_t pmcs_nonio_ix(caddr_t, caddr_t);
117 static uint_t pmcs_general_ix(caddr_t, caddr_t);
118 static uint_t pmcs_event_ix(caddr_t, caddr_t);
119 static uint_t pmcs_iodone_ix(caddr_t, caddr_t);
120 static uint_t pmcs_fatal_ix(caddr_t, caddr_t);
121 static uint_t pmcs_all_intr(caddr_t, caddr_t);
122 static int pmcs_quiesce(dev_info_t *dip);
123 static boolean_t pmcs_fabricate_wwid(pmcs_hw_t *);
124 
125 static void pmcs_create_all_phy_stats(pmcs_iport_t *);
126 int pmcs_update_phy_stats(kstat_t *, int);
127 
128 static void pmcs_fm_fini(pmcs_hw_t *pwp);
129 static void pmcs_fm_init(pmcs_hw_t *pwp);
130 static int pmcs_fm_error_cb(dev_info_t *dip,
131     ddi_fm_error_t *err, const void *impl_data);
132 
133 /*
134  * Local configuration data
135  */
136 static struct dev_ops pmcs_ops = {
137 	DEVO_REV,		/* devo_rev, */
138 	0,			/* refcnt */
139 	ddi_no_info,		/* info */
140 	nulldev,		/* identify */
141 	nulldev,		/* probe */
142 	pmcs_attach,		/* attach */
143 	pmcs_detach,		/* detach */
144 	nodev,			/* reset */
145 	NULL,			/* driver operations */
146 	NULL,			/* bus operations */
147 	ddi_power,		/* power management */
148 	pmcs_quiesce		/* quiesce */
149 };
150 
151 static struct modldrv modldrv = {
152 	&mod_driverops,
153 	PMCS_DRIVER_VERSION,
154 	&pmcs_ops,	/* driver ops */
155 };
156 static struct modlinkage modlinkage = {
157 	MODREV_1, &modldrv, NULL
158 };
159 
160 const ddi_dma_attr_t pmcs_dattr = {
161 	DMA_ATTR_V0,			/* dma_attr version	*/
162 	0x0000000000000000ull,		/* dma_attr_addr_lo	*/
163 	0xFFFFFFFFFFFFFFFFull,		/* dma_attr_addr_hi	*/
164 	0x00000000FFFFFFFFull,		/* dma_attr_count_max	*/
165 	0x0000000000000001ull,		/* dma_attr_align	*/
166 	0x00000078,			/* dma_attr_burstsizes	*/
167 	0x00000001,			/* dma_attr_minxfer	*/
168 	0x00000000FFFFFFFFull,		/* dma_attr_maxxfer	*/
169 	0x00000000FFFFFFFFull,		/* dma_attr_seg		*/
170 	1,				/* dma_attr_sgllen	*/
171 	512,				/* dma_attr_granular	*/
172 	0				/* dma_attr_flags	*/
173 };
174 
175 static ddi_device_acc_attr_t rattr = {
176 	DDI_DEVICE_ATTR_V1,
177 	DDI_STRUCTURE_LE_ACC,
178 	DDI_STRICTORDER_ACC,
179 	DDI_DEFAULT_ACC
180 };
181 
182 
183 /*
184  * Attach/Detach functions
185  */
186 
187 int
188 _init(void)
189 {
190 	int ret;
191 
192 	ret = ddi_soft_state_init(&pmcs_softc_state, sizeof (pmcs_hw_t), 1);
193 	if (ret != 0) {
194 		cmn_err(CE_WARN, "?soft state init failed for pmcs");
195 		return (ret);
196 	}
197 
198 	if ((ret = scsi_hba_init(&modlinkage)) != 0) {
199 		cmn_err(CE_WARN, "?scsi_hba_init failed for pmcs");
200 		ddi_soft_state_fini(&pmcs_softc_state);
201 		return (ret);
202 	}
203 
204 	/*
205 	 * Allocate soft state for iports
206 	 */
207 	ret = ddi_soft_state_init(&pmcs_iport_softstate,
208 	    sizeof (pmcs_iport_t), 2);
209 	if (ret != 0) {
210 		cmn_err(CE_WARN, "?iport soft state init failed for pmcs");
211 		ddi_soft_state_fini(&pmcs_softc_state);
212 		return (ret);
213 	}
214 
215 	ret = mod_install(&modlinkage);
216 	if (ret != 0) {
217 		cmn_err(CE_WARN, "?mod_install failed for pmcs (%d)", ret);
218 		scsi_hba_fini(&modlinkage);
219 		ddi_soft_state_fini(&pmcs_iport_softstate);
220 		ddi_soft_state_fini(&pmcs_softc_state);
221 		return (ret);
222 	}
223 
224 	/* Initialize the global trace lock */
225 	mutex_init(&pmcs_trace_lock, NULL, MUTEX_DRIVER, NULL);
226 
227 	return (0);
228 }
229 
230 int
231 _fini(void)
232 {
233 	int ret;
234 	if ((ret = mod_remove(&modlinkage)) != 0) {
235 		return (ret);
236 	}
237 	scsi_hba_fini(&modlinkage);
238 
239 	/* Free pmcs log buffer and destroy the global lock */
240 	if (pmcs_tbuf) {
241 		kmem_free(pmcs_tbuf,
242 		    pmcs_tbuf_num_elems * sizeof (pmcs_tbuf_t));
243 		pmcs_tbuf = NULL;
244 	}
245 	mutex_destroy(&pmcs_trace_lock);
246 
247 	ddi_soft_state_fini(&pmcs_iport_softstate);
248 	ddi_soft_state_fini(&pmcs_softc_state);
249 	return (0);
250 }
251 
252 int
253 _info(struct modinfo *modinfop)
254 {
255 	return (mod_info(&modlinkage, modinfop));
256 }
257 
258 static int
259 pmcs_iport_attach(dev_info_t *dip)
260 {
261 	pmcs_iport_t		*iport;
262 	pmcs_hw_t		*pwp;
263 	scsi_hba_tran_t		*tran;
264 	void			*ua_priv = NULL;
265 	char			*iport_ua;
266 	char			*init_port;
267 	int			hba_inst;
268 	int			inst;
269 
270 	hba_inst = ddi_get_instance(ddi_get_parent(dip));
271 	inst = ddi_get_instance(dip);
272 
273 	pwp = ddi_get_soft_state(pmcs_softc_state, hba_inst);
274 	if (pwp == NULL) {
275 		cmn_err(CE_WARN, "%s: No HBA softstate for instance %d",
276 		    __func__, inst);
277 		return (DDI_FAILURE);
278 	}
279 
280 	if ((pwp->state == STATE_UNPROBING) || (pwp->state == STATE_DEAD)) {
281 		return (DDI_FAILURE);
282 	}
283 
284 	if ((iport_ua = scsi_hba_iport_unit_address(dip)) == NULL) {
285 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
286 		    "%s: invoked with NULL unit address, inst (%d)",
287 		    __func__, inst);
288 		return (DDI_FAILURE);
289 	}
290 
291 	if (ddi_soft_state_zalloc(pmcs_iport_softstate, inst) != DDI_SUCCESS) {
292 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
293 		    "Failed to alloc soft state for iport %d", inst);
294 		return (DDI_FAILURE);
295 	}
296 
297 	iport = ddi_get_soft_state(pmcs_iport_softstate, inst);
298 	if (iport == NULL) {
299 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
300 		    "cannot get iport soft state");
301 		goto iport_attach_fail1;
302 	}
303 
304 	mutex_init(&iport->lock, NULL, MUTEX_DRIVER,
305 	    DDI_INTR_PRI(pwp->intr_pri));
306 	cv_init(&iport->refcnt_cv, NULL, CV_DEFAULT, NULL);
307 	cv_init(&iport->smp_cv, NULL, CV_DEFAULT, NULL);
308 	mutex_init(&iport->refcnt_lock, NULL, MUTEX_DRIVER,
309 	    DDI_INTR_PRI(pwp->intr_pri));
310 	mutex_init(&iport->smp_lock, NULL, MUTEX_DRIVER,
311 	    DDI_INTR_PRI(pwp->intr_pri));
312 
313 	/* Set some data on the iport handle */
314 	iport->dip = dip;
315 	iport->pwp = pwp;
316 
317 	/* Dup the UA into the iport handle */
318 	iport->ua = strdup(iport_ua);
319 
320 	tran = (scsi_hba_tran_t *)ddi_get_driver_private(dip);
321 	tran->tran_hba_private = iport;
322 
323 	list_create(&iport->phys, sizeof (pmcs_phy_t),
324 	    offsetof(pmcs_phy_t, list_node));
325 
326 	/*
327 	 * If our unit address is active in the phymap, configure our
328 	 * iport's phylist.
329 	 */
330 	mutex_enter(&iport->lock);
331 	ua_priv = sas_phymap_lookup_uapriv(pwp->hss_phymap, iport->ua);
332 	if (ua_priv) {
333 		/* Non-NULL private data indicates the unit address is active */
334 		iport->ua_state = UA_ACTIVE;
335 		if (pmcs_iport_configure_phys(iport) != DDI_SUCCESS) {
336 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
337 			    "%s: failed to "
338 			    "configure phys on iport handle (0x%p), "
339 			    " unit address [%s]", __func__,
340 			    (void *)iport, iport_ua);
341 			mutex_exit(&iport->lock);
342 			goto iport_attach_fail2;
343 		}
344 	} else {
345 		iport->ua_state = UA_INACTIVE;
346 	}
347 	mutex_exit(&iport->lock);
348 
349 	/* Allocate string-based soft state pool for targets */
350 	iport->tgt_sstate = NULL;
351 	if (ddi_soft_state_bystr_init(&iport->tgt_sstate,
352 	    sizeof (pmcs_xscsi_t), PMCS_TGT_SSTATE_SZ) != 0) {
353 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
354 		    "cannot get iport tgt soft state");
355 		goto iport_attach_fail2;
356 	}
357 
358 	/* Create this iport's target map */
359 	if (pmcs_iport_tgtmap_create(iport) == B_FALSE) {
360 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
361 		    "Failed to create tgtmap on iport %d", inst);
362 		goto iport_attach_fail3;
363 	}
364 
365 	/* Set up the 'initiator-port' DDI property on this iport */
366 	init_port = kmem_zalloc(PMCS_MAX_UA_SIZE, KM_SLEEP);
367 	if (pwp->separate_ports) {
368 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
369 		    "%s: separate ports not supported", __func__);
370 	} else {
371 		/* Set initiator-port value to the HBA's base WWN */
372 		(void) scsi_wwn_to_wwnstr(pwp->sas_wwns[0], 1,
373 		    init_port);
374 	}
375 
376 	mutex_enter(&iport->lock);
377 	pmcs_smhba_add_iport_prop(iport, DATA_TYPE_STRING,
378 	    SCSI_ADDR_PROP_INITIATOR_PORT, init_port);
379 	kmem_free(init_port, PMCS_MAX_UA_SIZE);
380 
381 	/* Set up a 'num-phys' DDI property for the iport node */
382 	pmcs_smhba_add_iport_prop(iport, DATA_TYPE_INT32, PMCS_NUM_PHYS,
383 	    &iport->nphy);
384 	mutex_exit(&iport->lock);
385 
386 	/* Create kstats for each of the phys in this port */
387 	pmcs_create_all_phy_stats(iport);
388 
389 	/*
390 	 * Insert this iport handle into our list and set
391 	 * iports_attached on the HBA node.
392 	 */
393 	rw_enter(&pwp->iports_lock, RW_WRITER);
394 	ASSERT(!list_link_active(&iport->list_node));
395 	list_insert_tail(&pwp->iports, iport);
396 	pwp->iports_attached = 1;
397 	pwp->num_iports++;
398 	rw_exit(&pwp->iports_lock);
399 
400 	pmcs_prt(pwp, PMCS_PRT_DEBUG_IPORT, NULL, NULL,
401 	    "iport%d attached", inst);
402 	ddi_report_dev(dip);
403 	return (DDI_SUCCESS);
404 
405 	/* teardown and fail */
406 iport_attach_fail3:
407 	ddi_soft_state_bystr_fini(&iport->tgt_sstate);
408 iport_attach_fail2:
409 	list_destroy(&iport->phys);
410 	strfree(iport->ua);
411 	mutex_destroy(&iport->refcnt_lock);
412 	mutex_destroy(&iport->smp_lock);
413 	cv_destroy(&iport->refcnt_cv);
414 	cv_destroy(&iport->smp_cv);
415 	mutex_destroy(&iport->lock);
416 iport_attach_fail1:
417 	ddi_soft_state_free(pmcs_iport_softstate, inst);
418 	return (DDI_FAILURE);
419 }
420 
421 static int
422 pmcs_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
423 {
424 	scsi_hba_tran_t *tran;
425 	char chiprev, *fwsupport, hw_rev[24], fw_rev[24];
426 	off_t set3size;
427 	int inst, i;
428 	int sm_hba = 1;
429 	int protocol = 0;
430 	int num_phys = 0;
431 	pmcs_hw_t *pwp;
432 	pmcs_phy_t *phyp;
433 	uint32_t num_threads;
434 	char buf[64];
435 	char *fwl_file;
436 
437 	switch (cmd) {
438 	case DDI_ATTACH:
439 		break;
440 
441 	case DDI_PM_RESUME:
442 	case DDI_RESUME:
443 		tran = (scsi_hba_tran_t *)ddi_get_driver_private(dip);
444 		if (!tran) {
445 			return (DDI_FAILURE);
446 		}
447 		/* No DDI_?_RESUME on iport nodes */
448 		if (scsi_hba_iport_unit_address(dip) != NULL) {
449 			return (DDI_SUCCESS);
450 		}
451 		pwp = TRAN2PMC(tran);
452 		if (pwp == NULL) {
453 			return (DDI_FAILURE);
454 		}
455 
456 		mutex_enter(&pwp->lock);
457 		pwp->suspended = 0;
458 		if (pwp->tq) {
459 			ddi_taskq_resume(pwp->tq);
460 		}
461 		mutex_exit(&pwp->lock);
462 		return (DDI_SUCCESS);
463 
464 	default:
465 		return (DDI_FAILURE);
466 	}
467 
468 	/*
469 	 * If this is an iport node, invoke iport attach.
470 	 */
471 	if (scsi_hba_iport_unit_address(dip) != NULL) {
472 		return (pmcs_iport_attach(dip));
473 	}
474 
475 	/*
476 	 * From here on is attach for the HBA node
477 	 */
478 
479 #ifdef	DEBUG
480 	/*
481 	 * Check to see if this unit is to be disabled.  We can't disable
482 	 * on a per-iport node.  It's either the entire HBA or nothing.
483 	 */
484 	(void) snprintf(buf, sizeof (buf),
485 	    "disable-instance-%d", ddi_get_instance(dip));
486 	if (ddi_prop_get_int(DDI_DEV_T_ANY, dip,
487 	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, buf, 0)) {
488 		cmn_err(CE_NOTE, "pmcs%d: disabled by configuration",
489 		    ddi_get_instance(dip));
490 		return (DDI_FAILURE);
491 	}
492 #endif
493 
494 	/*
495 	 * Allocate softstate
496 	 */
497 	inst = ddi_get_instance(dip);
498 	if (ddi_soft_state_zalloc(pmcs_softc_state, inst) != DDI_SUCCESS) {
499 		cmn_err(CE_WARN, "pmcs%d: Failed to alloc soft state", inst);
500 		return (DDI_FAILURE);
501 	}
502 
503 	pwp = ddi_get_soft_state(pmcs_softc_state, inst);
504 	if (pwp == NULL) {
505 		cmn_err(CE_WARN, "pmcs%d: cannot get soft state", inst);
506 		ddi_soft_state_free(pmcs_softc_state, inst);
507 		return (DDI_FAILURE);
508 	}
509 	pwp->dip = dip;
510 	STAILQ_INIT(&pwp->dq);
511 	STAILQ_INIT(&pwp->cq);
512 	STAILQ_INIT(&pwp->wf);
513 	STAILQ_INIT(&pwp->pf);
514 
515 	/*
516 	 * Create the list for iports and init its lock.
517 	 */
518 	list_create(&pwp->iports, sizeof (pmcs_iport_t),
519 	    offsetof(pmcs_iport_t, list_node));
520 	rw_init(&pwp->iports_lock, NULL, RW_DRIVER, NULL);
521 
522 	pwp->state = STATE_PROBING;
523 
524 	/*
525 	 * Get driver.conf properties
526 	 */
527 	pwp->debug_mask = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
528 	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-debug-mask",
529 	    debug_mask);
530 	pwp->phyid_block_mask = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
531 	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-phyid-block-mask",
532 	    block_mask);
533 	pwp->physpeed = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
534 	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-physpeed", physpeed);
535 	pwp->phymode = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
536 	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-phymode", phymode);
537 	pwp->fwlog = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
538 	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-fwlog", fwlog_level);
539 	if (pwp->fwlog > PMCS_FWLOG_MAX) {
540 		pwp->fwlog = PMCS_FWLOG_MAX;
541 	}
542 	if ((ddi_prop_lookup_string(DDI_DEV_T_ANY, dip, 0, "pmcs-fwlogfile",
543 	    &fwl_file) == DDI_SUCCESS)) {
544 		if (snprintf(pwp->fwlogfile_aap1, MAXPATHLEN, "%s%d-aap1.0",
545 		    fwl_file, ddi_get_instance(dip)) > MAXPATHLEN) {
546 			pwp->fwlogfile_aap1[0] = '\0';
547 			pwp->fwlogfile_iop[0] = '\0';
548 		} else if (snprintf(pwp->fwlogfile_iop, MAXPATHLEN,
549 		    "%s%d-iop.0", fwl_file,
550 		    ddi_get_instance(dip)) > MAXPATHLEN) {
551 			pwp->fwlogfile_aap1[0] = '\0';
552 			pwp->fwlogfile_iop[0] = '\0';
553 		}
554 		ddi_prop_free(fwl_file);
555 	} else {
556 		pwp->fwlogfile_aap1[0] = '\0';
557 		pwp->fwlogfile_iop[0] = '\0';
558 	}
559 
560 	pwp->open_retry_interval = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
561 	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-open-retry-interval",
562 	    OPEN_RETRY_INTERVAL_DEF);
563 	if (pwp->open_retry_interval > OPEN_RETRY_INTERVAL_MAX) {
564 		pwp->open_retry_interval = OPEN_RETRY_INTERVAL_MAX;
565 	}
566 
567 	mutex_enter(&pmcs_trace_lock);
568 	if (pmcs_tbuf == NULL) {
569 		/* Allocate trace buffer */
570 		pmcs_tbuf_num_elems = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
571 		    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-tbuf-num-elems",
572 		    PMCS_TBUF_NUM_ELEMS_DEF);
573 		if ((pmcs_tbuf_num_elems == DDI_PROP_NOT_FOUND) ||
574 		    (pmcs_tbuf_num_elems == 0)) {
575 			pmcs_tbuf_num_elems = PMCS_TBUF_NUM_ELEMS_DEF;
576 		}
577 
578 		pmcs_tbuf = kmem_zalloc(pmcs_tbuf_num_elems *
579 		    sizeof (pmcs_tbuf_t), KM_SLEEP);
580 		pmcs_tbuf_ptr = pmcs_tbuf;
581 		pmcs_tbuf_idx = 0;
582 	}
583 	mutex_exit(&pmcs_trace_lock);
584 
585 	if (pwp->fwlog && strlen(pwp->fwlogfile_aap1) > 0) {
586 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
587 		    "%s: firmware event log files: %s, %s", __func__,
588 		    pwp->fwlogfile_aap1, pwp->fwlogfile_iop);
589 		pwp->fwlog_file = 1;
590 	} else {
591 		if (pwp->fwlog == 0) {
592 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
593 			    "%s: No firmware event log will be written "
594 			    "(event log disabled)", __func__);
595 		} else {
596 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
597 			    "%s: No firmware event log will be written "
598 			    "(no filename configured - too long?)", __func__);
599 		}
600 		pwp->fwlog_file = 0;
601 	}
602 
603 	disable_msix = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
604 	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-disable-msix",
605 	    disable_msix);
606 	disable_msi = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
607 	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-disable-msi",
608 	    disable_msi);
609 	maxqdepth = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
610 	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-maxqdepth", maxqdepth);
611 	pwp->fw_force_update = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
612 	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-fw-force-update", 0);
613 	if (pwp->fw_force_update == 0) {
614 		pwp->fw_disable_update = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
615 		    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM,
616 		    "pmcs-fw-disable-update", 0);
617 	}
618 	pwp->ioq_depth = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
619 	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-num-io-qentries",
620 	    PMCS_NQENTRY);
621 
622 	/*
623 	 * Initialize FMA
624 	 */
625 	pwp->dev_acc_attr = pwp->reg_acc_attr = rattr;
626 	pwp->iqp_dma_attr = pwp->oqp_dma_attr =
627 	    pwp->regdump_dma_attr = pwp->cip_dma_attr =
628 	    pwp->fwlog_dma_attr = pmcs_dattr;
629 	pwp->fm_capabilities = ddi_getprop(DDI_DEV_T_ANY, pwp->dip,
630 	    DDI_PROP_NOTPROM | DDI_PROP_DONTPASS, "fm-capable",
631 	    DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
632 	    DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
633 	pmcs_fm_init(pwp);
634 
635 	/*
636 	 * Map registers
637 	 */
638 	if (pci_config_setup(dip, &pwp->pci_acc_handle)) {
639 		pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL,
640 		    "pci config setup failed");
641 		ddi_soft_state_free(pmcs_softc_state, inst);
642 		return (DDI_FAILURE);
643 	}
644 
645 	/*
646 	 * Get the size of register set 3.
647 	 */
648 	if (ddi_dev_regsize(dip, PMCS_REGSET_3, &set3size) != DDI_SUCCESS) {
649 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
650 		    "unable to get size of register set %d", PMCS_REGSET_3);
651 		pci_config_teardown(&pwp->pci_acc_handle);
652 		ddi_soft_state_free(pmcs_softc_state, inst);
653 		return (DDI_FAILURE);
654 	}
655 
656 	/*
657 	 * Map registers
658 	 */
659 	pwp->reg_acc_attr.devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC;
660 
661 	if (ddi_regs_map_setup(dip, PMCS_REGSET_0, (caddr_t *)&pwp->msg_regs,
662 	    0, 0, &pwp->reg_acc_attr, &pwp->msg_acc_handle)) {
663 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
664 		    "failed to map Message Unit registers");
665 		pci_config_teardown(&pwp->pci_acc_handle);
666 		ddi_soft_state_free(pmcs_softc_state, inst);
667 		return (DDI_FAILURE);
668 	}
669 
670 	if (ddi_regs_map_setup(dip, PMCS_REGSET_1, (caddr_t *)&pwp->top_regs,
671 	    0, 0, &pwp->reg_acc_attr, &pwp->top_acc_handle)) {
672 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
673 		    "failed to map TOP registers");
674 		ddi_regs_map_free(&pwp->msg_acc_handle);
675 		pci_config_teardown(&pwp->pci_acc_handle);
676 		ddi_soft_state_free(pmcs_softc_state, inst);
677 		return (DDI_FAILURE);
678 	}
679 
680 	if (ddi_regs_map_setup(dip, PMCS_REGSET_2, (caddr_t *)&pwp->gsm_regs,
681 	    0, 0, &pwp->reg_acc_attr, &pwp->gsm_acc_handle)) {
682 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
683 		    "failed to map GSM registers");
684 		ddi_regs_map_free(&pwp->top_acc_handle);
685 		ddi_regs_map_free(&pwp->msg_acc_handle);
686 		pci_config_teardown(&pwp->pci_acc_handle);
687 		ddi_soft_state_free(pmcs_softc_state, inst);
688 		return (DDI_FAILURE);
689 	}
690 
691 	if (ddi_regs_map_setup(dip, PMCS_REGSET_3, (caddr_t *)&pwp->mpi_regs,
692 	    0, 0, &pwp->reg_acc_attr, &pwp->mpi_acc_handle)) {
693 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
694 		    "failed to map MPI registers");
695 		ddi_regs_map_free(&pwp->top_acc_handle);
696 		ddi_regs_map_free(&pwp->gsm_acc_handle);
697 		ddi_regs_map_free(&pwp->msg_acc_handle);
698 		pci_config_teardown(&pwp->pci_acc_handle);
699 		ddi_soft_state_free(pmcs_softc_state, inst);
700 		return (DDI_FAILURE);
701 	}
702 	pwp->mpibar =
703 	    (((5U << 2) + 0x10) << PMCS_MSGU_MPI_BAR_SHIFT) | set3size;
704 
705 	/*
706 	 * Make sure we can support this card.
707 	 */
708 	pwp->chiprev = pmcs_rd_topunit(pwp, PMCS_DEVICE_REVISION);
709 
710 	switch (pwp->chiprev) {
711 	case PMCS_PM8001_REV_A:
712 	case PMCS_PM8001_REV_B:
713 		pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL,
714 		    "Rev A/B Card no longer supported");
715 		goto failure;
716 	case PMCS_PM8001_REV_C:
717 		break;
718 	default:
719 		pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL,
720 		    "Unknown chip revision (%d)", pwp->chiprev);
721 		goto failure;
722 	}
723 
724 	/*
725 	 * Allocate DMA addressable area for Inbound and Outbound Queue indices
726 	 * that the chip needs to access plus a space for scratch usage
727 	 */
728 	pwp->cip_dma_attr.dma_attr_align = sizeof (uint32_t);
729 	if (pmcs_dma_setup(pwp, &pwp->cip_dma_attr, &pwp->cip_acchdls,
730 	    &pwp->cip_handles, ptob(1), (caddr_t *)&pwp->cip,
731 	    &pwp->ciaddr) == B_FALSE) {
732 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
733 		    "Failed to setup DMA for index/scratch");
734 		goto failure;
735 	}
736 
737 	bzero(pwp->cip, ptob(1));
738 	pwp->scratch = &pwp->cip[PMCS_INDICES_SIZE];
739 	pwp->scratch_dma = pwp->ciaddr + PMCS_INDICES_SIZE;
740 
741 	/*
742 	 * Allocate DMA S/G list chunks
743 	 */
744 	(void) pmcs_add_more_chunks(pwp, ptob(1) * PMCS_MIN_CHUNK_PAGES);
745 
746 	/*
747 	 * Allocate a DMA addressable area for the firmware log (if needed)
748 	 */
749 	if (pwp->fwlog) {
750 		/*
751 		 * Align to event log header and entry size
752 		 */
753 		pwp->fwlog_dma_attr.dma_attr_align = 32;
754 		if (pmcs_dma_setup(pwp, &pwp->fwlog_dma_attr,
755 		    &pwp->fwlog_acchdl,
756 		    &pwp->fwlog_hndl, PMCS_FWLOG_SIZE,
757 		    (caddr_t *)&pwp->fwlogp,
758 		    &pwp->fwaddr) == B_FALSE) {
759 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
760 			    "Failed to setup DMA for fwlog area");
761 			pwp->fwlog = 0;
762 		} else {
763 			bzero(pwp->fwlogp, PMCS_FWLOG_SIZE);
764 			pwp->fwlogp_aap1 = (pmcs_fw_event_hdr_t *)pwp->fwlogp;
765 			pwp->fwlogp_iop = (pmcs_fw_event_hdr_t *)((void *)
766 			    ((caddr_t)pwp->fwlogp + (PMCS_FWLOG_SIZE / 2)));
767 		}
768 	}
769 
770 	if (pwp->flash_chunk_addr == 0) {
771 		pwp->regdump_dma_attr.dma_attr_align = PMCS_FLASH_CHUNK_SIZE;
772 		if (pmcs_dma_setup(pwp, &pwp->regdump_dma_attr,
773 		    &pwp->regdump_acchdl,
774 		    &pwp->regdump_hndl, PMCS_FLASH_CHUNK_SIZE,
775 		    (caddr_t *)&pwp->flash_chunkp, &pwp->flash_chunk_addr) ==
776 		    B_FALSE) {
777 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
778 			    "Failed to setup DMA for register dump area");
779 			goto failure;
780 		}
781 		bzero(pwp->flash_chunkp, PMCS_FLASH_CHUNK_SIZE);
782 	}
783 
784 	/*
785 	 * More bits of local initialization...
786 	 */
787 	pwp->tq = ddi_taskq_create(dip, "_tq", 4, TASKQ_DEFAULTPRI, 0);
788 	if (pwp->tq == NULL) {
789 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
790 		    "unable to create worker taskq");
791 		goto failure;
792 	}
793 
794 	/*
795 	 * Cache of structures for dealing with I/O completion callbacks.
796 	 */
797 	(void) snprintf(buf, sizeof (buf), "pmcs_iocomp_cb_cache%d", inst);
798 	pwp->iocomp_cb_cache = kmem_cache_create(buf,
799 	    sizeof (pmcs_iocomp_cb_t), 16, NULL, NULL, NULL, NULL, NULL, 0);
800 
801 	/*
802 	 * Cache of PHY structures
803 	 */
804 	(void) snprintf(buf, sizeof (buf), "pmcs_phy_cache%d", inst);
805 	pwp->phy_cache = kmem_cache_create(buf, sizeof (pmcs_phy_t), 8,
806 	    pmcs_phy_constructor, pmcs_phy_destructor, NULL, (void *)pwp,
807 	    NULL, 0);
808 
809 	/*
810 	 * Allocate space for the I/O completion threads
811 	 */
812 	num_threads = ncpus_online;
813 	if (num_threads > PMCS_MAX_CQ_THREADS) {
814 		num_threads = PMCS_MAX_CQ_THREADS;
815 	}
816 
817 	pwp->cq_info.cq_threads = num_threads;
818 	pwp->cq_info.cq_thr_info = kmem_zalloc(
819 	    sizeof (pmcs_cq_thr_info_t) * pwp->cq_info.cq_threads, KM_SLEEP);
820 	pwp->cq_info.cq_next_disp_thr = 0;
821 	pwp->cq_info.cq_stop = B_FALSE;
822 
823 	/*
824 	 * Set the quantum value in clock ticks for the I/O interrupt
825 	 * coalescing timer.
826 	 */
827 	pwp->io_intr_coal.quantum = drv_usectohz(PMCS_QUANTUM_TIME_USECS);
828 
829 	/*
830 	 * We have a delicate dance here. We need to set up
831 	 * interrupts so we know how to set up some OQC
832 	 * tables. However, while we're setting up table
833 	 * access, we may need to flash new firmware and
834 	 * reset the card, which will take some finessing.
835 	 */
836 
837 	/*
838 	 * Set up interrupts here.
839 	 */
840 	switch (pmcs_setup_intr(pwp)) {
841 	case 0:
842 		break;
843 	case EIO:
844 		pwp->stuck = 1;
845 		/* FALLTHROUGH */
846 	default:
847 		goto failure;
848 	}
849 
850 	/*
851 	 * Set these up now becuase they are used to initialize the OQC tables.
852 	 *
853 	 * If we have MSI or MSI-X interrupts set up and we have enough
854 	 * vectors for each OQ, the Outbound Queue vectors can all be the
855 	 * same as the appropriate interrupt routine will have been called
856 	 * and the doorbell register automatically cleared.
857 	 * This keeps us from having to check the Outbound Doorbell register
858 	 * when the routines for these interrupts are called.
859 	 *
860 	 * If we have Legacy INT-X interrupts set up or we didn't have enough
861 	 * MSI/MSI-X vectors to uniquely identify each OQ, we point these
862 	 * vectors to the bits we would like to have set in the Outbound
863 	 * Doorbell register because pmcs_all_intr will read the doorbell
864 	 * register to find out why we have an interrupt and write the
865 	 * corresponding 'clear' bit for that interrupt.
866 	 */
867 
868 	switch (pwp->intr_cnt) {
869 	case 1:
870 		/*
871 		 * Only one vector, so we must check all OQs for MSI.  For
872 		 * INT-X, there's only one vector anyway, so we can just
873 		 * use the outbound queue bits to keep from having to
874 		 * check each queue for each interrupt.
875 		 */
876 		if (pwp->int_type == PMCS_INT_FIXED) {
877 			pwp->oqvec[PMCS_OQ_IODONE] = PMCS_OQ_IODONE;
878 			pwp->oqvec[PMCS_OQ_GENERAL] = PMCS_OQ_GENERAL;
879 			pwp->oqvec[PMCS_OQ_EVENTS] = PMCS_OQ_EVENTS;
880 		} else {
881 			pwp->oqvec[PMCS_OQ_IODONE] = PMCS_OQ_IODONE;
882 			pwp->oqvec[PMCS_OQ_GENERAL] = PMCS_OQ_IODONE;
883 			pwp->oqvec[PMCS_OQ_EVENTS] = PMCS_OQ_IODONE;
884 		}
885 		break;
886 	case 2:
887 		/* With 2, we can at least isolate IODONE */
888 		pwp->oqvec[PMCS_OQ_IODONE] = PMCS_OQ_IODONE;
889 		pwp->oqvec[PMCS_OQ_GENERAL] = PMCS_OQ_GENERAL;
890 		pwp->oqvec[PMCS_OQ_EVENTS] = PMCS_OQ_GENERAL;
891 		break;
892 	case 4:
893 		/* With 4 vectors, everybody gets one */
894 		pwp->oqvec[PMCS_OQ_IODONE] = PMCS_OQ_IODONE;
895 		pwp->oqvec[PMCS_OQ_GENERAL] = PMCS_OQ_GENERAL;
896 		pwp->oqvec[PMCS_OQ_EVENTS] = PMCS_OQ_EVENTS;
897 		break;
898 	}
899 
900 	/*
901 	 * Do the first part of setup
902 	 */
903 	if (pmcs_setup(pwp)) {
904 		goto failure;
905 	}
906 	pmcs_report_fwversion(pwp);
907 
908 	/*
909 	 * Now do some additonal allocations based upon information
910 	 * gathered during MPI setup.
911 	 */
912 	pwp->root_phys = kmem_zalloc(pwp->nphy * sizeof (pmcs_phy_t), KM_SLEEP);
913 	ASSERT(pwp->nphy < SAS2_PHYNUM_MAX);
914 	phyp = pwp->root_phys;
915 	for (i = 0; i < pwp->nphy; i++) {
916 		if (i < pwp->nphy-1) {
917 			phyp->sibling = (phyp + 1);
918 		}
919 		mutex_init(&phyp->phy_lock, NULL, MUTEX_DRIVER,
920 		    DDI_INTR_PRI(pwp->intr_pri));
921 		phyp->phynum = i & SAS2_PHYNUM_MASK;
922 		pmcs_phy_name(pwp, phyp, phyp->path, sizeof (phyp->path));
923 		phyp->pwp = pwp;
924 		phyp->device_id = PMCS_INVALID_DEVICE_ID;
925 		phyp->portid = PMCS_PHY_INVALID_PORT_ID;
926 		phyp++;
927 	}
928 
929 	pwp->work = kmem_zalloc(pwp->max_cmd * sizeof (pmcwork_t), KM_SLEEP);
930 	for (i = 0; i < pwp->max_cmd; i++) {
931 		pmcwork_t *pwrk = &pwp->work[i];
932 		mutex_init(&pwrk->lock, NULL, MUTEX_DRIVER,
933 		    DDI_INTR_PRI(pwp->intr_pri));
934 		cv_init(&pwrk->sleep_cv, NULL, CV_DRIVER, NULL);
935 		STAILQ_INSERT_TAIL(&pwp->wf, pwrk, next);
936 
937 	}
938 	pwp->targets = (pmcs_xscsi_t **)
939 	    kmem_zalloc(pwp->max_dev * sizeof (pmcs_xscsi_t *), KM_SLEEP);
940 
941 	pwp->iqpt = (pmcs_iqp_trace_t *)
942 	    kmem_zalloc(sizeof (pmcs_iqp_trace_t), KM_SLEEP);
943 	pwp->iqpt->head = kmem_zalloc(PMCS_IQP_TRACE_BUFFER_SIZE, KM_SLEEP);
944 	pwp->iqpt->curpos = pwp->iqpt->head;
945 	pwp->iqpt->size_left = PMCS_IQP_TRACE_BUFFER_SIZE;
946 
947 	/*
948 	 * Start MPI communication.
949 	 */
950 	if (pmcs_start_mpi(pwp)) {
951 		if (pmcs_soft_reset(pwp, B_FALSE)) {
952 			goto failure;
953 		}
954 		pwp->last_reset_reason = PMCS_LAST_RST_ATTACH;
955 	}
956 
957 	/*
958 	 * Do some initial acceptance tests.
959 	 * This tests interrupts and queues.
960 	 */
961 	if (pmcs_echo_test(pwp)) {
962 		goto failure;
963 	}
964 
965 	/* Read VPD - if it exists */
966 	if (pmcs_get_nvmd(pwp, PMCS_NVMD_VPD, PMCIN_NVMD_VPD, 0, NULL, 0)) {
967 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
968 		    "%s: Unable to read VPD: "
969 		    "attempting to fabricate", __func__);
970 		/*
971 		 * When we release, this must goto failure and the call
972 		 * to pmcs_fabricate_wwid is removed.
973 		 */
974 		/* goto failure; */
975 		if (!pmcs_fabricate_wwid(pwp)) {
976 			goto failure;
977 		}
978 	}
979 
980 	/*
981 	 * We're now officially running
982 	 */
983 	pwp->state = STATE_RUNNING;
984 
985 	/*
986 	 * Check firmware versions and load new firmware
987 	 * if needed and reset.
988 	 */
989 	if (pmcs_firmware_update(pwp)) {
990 		pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL,
991 		    "%s: Firmware update failed", __func__);
992 		goto failure;
993 	}
994 
995 	/*
996 	 * Create completion threads.
997 	 */
998 	for (i = 0; i < pwp->cq_info.cq_threads; i++) {
999 		pwp->cq_info.cq_thr_info[i].cq_pwp = pwp;
1000 		pwp->cq_info.cq_thr_info[i].cq_thread =
1001 		    thread_create(NULL, 0, pmcs_scsa_cq_run,
1002 		    &pwp->cq_info.cq_thr_info[i], 0, &p0, TS_RUN, minclsyspri);
1003 	}
1004 
1005 	/*
1006 	 * Create one thread to deal with the updating of the interrupt
1007 	 * coalescing timer.
1008 	 */
1009 	pwp->ict_thread = thread_create(NULL, 0, pmcs_check_intr_coal,
1010 	    pwp, 0, &p0, TS_RUN, minclsyspri);
1011 
1012 	/*
1013 	 * Kick off the watchdog
1014 	 */
1015 	pwp->wdhandle = timeout(pmcs_watchdog, pwp,
1016 	    drv_usectohz(PMCS_WATCH_INTERVAL));
1017 	/*
1018 	 * Do the SCSI attachment code (before starting phys)
1019 	 */
1020 	if (pmcs_scsa_init(pwp, &pmcs_dattr)) {
1021 		goto failure;
1022 	}
1023 	pwp->hba_attached = 1;
1024 
1025 	/* Check all acc & dma handles allocated in attach */
1026 	if (pmcs_check_acc_dma_handle(pwp)) {
1027 		ddi_fm_service_impact(pwp->dip, DDI_SERVICE_LOST);
1028 		goto failure;
1029 	}
1030 
1031 	/*
1032 	 * Create the iportmap for this HBA instance
1033 	 */
1034 	if (scsi_hba_iportmap_create(dip, iportmap_csync_usec,
1035 	    iportmap_stable_usec, &pwp->hss_iportmap) != DDI_SUCCESS) {
1036 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1037 		    "%s: pmcs%d iportmap_create failed", __func__, inst);
1038 		goto failure;
1039 	}
1040 	ASSERT(pwp->hss_iportmap);
1041 
1042 	/*
1043 	 * Create the phymap for this HBA instance
1044 	 */
1045 	if (sas_phymap_create(dip, phymap_stable_usec, PHYMAP_MODE_SIMPLE, NULL,
1046 	    pwp, pmcs_phymap_activate, pmcs_phymap_deactivate,
1047 	    &pwp->hss_phymap) != DDI_SUCCESS) {
1048 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1049 		    "%s: pmcs%d phymap_create failed", __func__, inst);
1050 		goto failure;
1051 	}
1052 	ASSERT(pwp->hss_phymap);
1053 
1054 	/*
1055 	 * Start the PHYs.
1056 	 */
1057 	if (pmcs_start_phys(pwp)) {
1058 		goto failure;
1059 	}
1060 
1061 	/*
1062 	 * From this point on, we can't fail.
1063 	 */
1064 	ddi_report_dev(dip);
1065 
1066 	/* SM-HBA */
1067 	pmcs_smhba_add_hba_prop(pwp, DATA_TYPE_INT32, PMCS_SMHBA_SUPPORTED,
1068 	    &sm_hba);
1069 
1070 	/* SM-HBA */
1071 	pmcs_smhba_add_hba_prop(pwp, DATA_TYPE_STRING, PMCS_DRV_VERSION,
1072 	    pmcs_driver_rev);
1073 
1074 	/* SM-HBA */
1075 	chiprev = 'A' + pwp->chiprev;
1076 	(void) snprintf(hw_rev, 2, "%s", &chiprev);
1077 	pmcs_smhba_add_hba_prop(pwp, DATA_TYPE_STRING, PMCS_HWARE_VERSION,
1078 	    hw_rev);
1079 
1080 	/* SM-HBA */
1081 	switch (PMCS_FW_TYPE(pwp)) {
1082 	case PMCS_FW_TYPE_RELEASED:
1083 		fwsupport = "Released";
1084 		break;
1085 	case PMCS_FW_TYPE_DEVELOPMENT:
1086 		fwsupport = "Development";
1087 		break;
1088 	case PMCS_FW_TYPE_ALPHA:
1089 		fwsupport = "Alpha";
1090 		break;
1091 	case PMCS_FW_TYPE_BETA:
1092 		fwsupport = "Beta";
1093 		break;
1094 	default:
1095 		fwsupport = "Special";
1096 		break;
1097 	}
1098 	(void) snprintf(fw_rev, sizeof (fw_rev), "%x.%x.%x %s",
1099 	    PMCS_FW_MAJOR(pwp), PMCS_FW_MINOR(pwp), PMCS_FW_MICRO(pwp),
1100 	    fwsupport);
1101 	pmcs_smhba_add_hba_prop(pwp, DATA_TYPE_STRING, PMCS_FWARE_VERSION,
1102 	    fw_rev);
1103 
1104 	/* SM-HBA */
1105 	num_phys = pwp->nphy;
1106 	pmcs_smhba_add_hba_prop(pwp, DATA_TYPE_INT32, PMCS_NUM_PHYS_HBA,
1107 	    &num_phys);
1108 
1109 	/* SM-HBA */
1110 	protocol = SAS_SSP_SUPPORT | SAS_SATA_SUPPORT | SAS_SMP_SUPPORT;
1111 	pmcs_smhba_add_hba_prop(pwp, DATA_TYPE_INT32, PMCS_SUPPORTED_PROTOCOL,
1112 	    &protocol);
1113 
1114 	/* Receptacle properties (FMA) */
1115 	pwp->recept_labels[0] = PMCS_RECEPT_LABEL_0;
1116 	pwp->recept_pm[0] = PMCS_RECEPT_PM_0;
1117 	pwp->recept_labels[1] = PMCS_RECEPT_LABEL_1;
1118 	pwp->recept_pm[1] = PMCS_RECEPT_PM_1;
1119 	if (ddi_prop_update_string_array(DDI_DEV_T_NONE, dip,
1120 	    SCSI_HBA_PROP_RECEPTACLE_LABEL, &pwp->recept_labels[0],
1121 	    PMCS_NUM_RECEPTACLES) != DDI_PROP_SUCCESS) {
1122 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1123 		    "%s: failed to create %s property", __func__,
1124 		    "receptacle-label");
1125 	}
1126 	if (ddi_prop_update_string_array(DDI_DEV_T_NONE, dip,
1127 	    SCSI_HBA_PROP_RECEPTACLE_PM, &pwp->recept_pm[0],
1128 	    PMCS_NUM_RECEPTACLES) != DDI_PROP_SUCCESS) {
1129 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1130 		    "%s: failed to create %s property", __func__,
1131 		    "receptacle-pm");
1132 	}
1133 
1134 	return (DDI_SUCCESS);
1135 
1136 failure:
1137 	if (pmcs_unattach(pwp)) {
1138 		pwp->stuck = 1;
1139 	}
1140 	return (DDI_FAILURE);
1141 }
1142 
1143 int
1144 pmcs_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
1145 {
1146 	int inst = ddi_get_instance(dip);
1147 	pmcs_iport_t	*iport = NULL;
1148 	pmcs_hw_t	*pwp = NULL;
1149 	scsi_hba_tran_t	*tran;
1150 
1151 	if (scsi_hba_iport_unit_address(dip) != NULL) {
1152 		/* iport node */
1153 		iport = ddi_get_soft_state(pmcs_iport_softstate, inst);
1154 		ASSERT(iport);
1155 		if (iport == NULL) {
1156 			return (DDI_FAILURE);
1157 		}
1158 		pwp = iport->pwp;
1159 	} else {
1160 		/* hba node */
1161 		pwp = (pmcs_hw_t *)ddi_get_soft_state(pmcs_softc_state, inst);
1162 		ASSERT(pwp);
1163 		if (pwp == NULL) {
1164 			return (DDI_FAILURE);
1165 		}
1166 	}
1167 	switch (cmd) {
1168 	case DDI_DETACH:
1169 		if (iport) {
1170 			/* iport detach */
1171 			if (pmcs_iport_unattach(iport)) {
1172 				return (DDI_FAILURE);
1173 			}
1174 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1175 			    "iport%d detached", inst);
1176 			return (DDI_SUCCESS);
1177 		} else {
1178 			/* HBA detach */
1179 			if (pmcs_unattach(pwp)) {
1180 				return (DDI_FAILURE);
1181 			}
1182 			return (DDI_SUCCESS);
1183 		}
1184 
1185 	case DDI_SUSPEND:
1186 	case DDI_PM_SUSPEND:
1187 		/* No DDI_SUSPEND on iport nodes */
1188 		if (iport) {
1189 			return (DDI_SUCCESS);
1190 		}
1191 
1192 		if (pwp->stuck) {
1193 			return (DDI_FAILURE);
1194 		}
1195 		tran = (scsi_hba_tran_t *)ddi_get_driver_private(dip);
1196 		if (!tran) {
1197 			return (DDI_FAILURE);
1198 		}
1199 
1200 		pwp = TRAN2PMC(tran);
1201 		if (pwp == NULL) {
1202 			return (DDI_FAILURE);
1203 		}
1204 		mutex_enter(&pwp->lock);
1205 		if (pwp->tq) {
1206 			ddi_taskq_suspend(pwp->tq);
1207 		}
1208 		pwp->suspended = 1;
1209 		mutex_exit(&pwp->lock);
1210 		pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, "PMC8X6G suspending");
1211 		return (DDI_SUCCESS);
1212 
1213 	default:
1214 		return (DDI_FAILURE);
1215 	}
1216 }
1217 
1218 static int
1219 pmcs_iport_unattach(pmcs_iport_t *iport)
1220 {
1221 	pmcs_hw_t	*pwp = iport->pwp;
1222 
1223 	/*
1224 	 * First, check if there are still any configured targets on this
1225 	 * iport.  If so, we fail detach.
1226 	 */
1227 	if (pmcs_iport_has_targets(pwp, iport)) {
1228 		pmcs_prt(pwp, PMCS_PRT_DEBUG_IPORT, NULL, NULL,
1229 		    "iport%d detach failure: iport has targets (luns)",
1230 		    ddi_get_instance(iport->dip));
1231 		return (DDI_FAILURE);
1232 	}
1233 
1234 	/*
1235 	 * Remove this iport from our list if it is inactive in the phymap.
1236 	 */
1237 	rw_enter(&pwp->iports_lock, RW_WRITER);
1238 	mutex_enter(&iport->lock);
1239 
1240 	if ((iport->ua_state == UA_ACTIVE) &&
1241 	    (detach_with_active_port == 0)) {
1242 		mutex_exit(&iport->lock);
1243 		rw_exit(&pwp->iports_lock);
1244 		pmcs_prt(pwp, PMCS_PRT_DEBUG_IPORT, NULL, NULL,
1245 		    "iport%d detach failure: "
1246 		    "iport unit address active in phymap",
1247 		    ddi_get_instance(iport->dip));
1248 		return (DDI_FAILURE);
1249 	}
1250 
1251 	/* If it's our only iport, clear iports_attached */
1252 	ASSERT(pwp->num_iports >= 1);
1253 	if (--pwp->num_iports == 0) {
1254 		pwp->iports_attached = 0;
1255 	}
1256 
1257 	ASSERT(list_link_active(&iport->list_node));
1258 	list_remove(&pwp->iports, iport);
1259 	rw_exit(&pwp->iports_lock);
1260 
1261 	/*
1262 	 * We have removed the iport handle from the HBA's iports list,
1263 	 * there will be no new references to it. Two things must be
1264 	 * guarded against here.  First, we could have PHY up events,
1265 	 * adding themselves to the iport->phys list and grabbing ref's
1266 	 * on our iport handle.  Second, we could have existing references
1267 	 * to this iport handle from a point in time prior to the list
1268 	 * removal above.
1269 	 *
1270 	 * So first, destroy the phys list. Remove any phys that have snuck
1271 	 * in after the phymap deactivate, dropping the refcnt accordingly.
1272 	 * If these PHYs are still up if and when the phymap reactivates
1273 	 * (i.e. when this iport reattaches), we'll populate the list with
1274 	 * them and bump the refcnt back up.
1275 	 */
1276 	pmcs_remove_phy_from_iport(iport, NULL);
1277 	ASSERT(list_is_empty(&iport->phys));
1278 	list_destroy(&iport->phys);
1279 	mutex_exit(&iport->lock);
1280 
1281 	/*
1282 	 * Second, wait for any other references to this iport to be
1283 	 * dropped, then continue teardown.
1284 	 */
1285 	mutex_enter(&iport->refcnt_lock);
1286 	while (iport->refcnt != 0) {
1287 		cv_wait(&iport->refcnt_cv, &iport->refcnt_lock);
1288 	}
1289 	mutex_exit(&iport->refcnt_lock);
1290 
1291 
1292 	/* Destroy the iport target map */
1293 	if (pmcs_iport_tgtmap_destroy(iport) == B_FALSE) {
1294 		return (DDI_FAILURE);
1295 	}
1296 
1297 	/* Free the tgt soft state */
1298 	if (iport->tgt_sstate != NULL) {
1299 		ddi_soft_state_bystr_fini(&iport->tgt_sstate);
1300 	}
1301 
1302 	/* Free our unit address string */
1303 	strfree(iport->ua);
1304 
1305 	/* Finish teardown and free the softstate */
1306 	mutex_destroy(&iport->refcnt_lock);
1307 	mutex_destroy(&iport->smp_lock);
1308 	ASSERT(iport->refcnt == 0);
1309 	cv_destroy(&iport->refcnt_cv);
1310 	cv_destroy(&iport->smp_cv);
1311 	mutex_destroy(&iport->lock);
1312 	ddi_soft_state_free(pmcs_iport_softstate, ddi_get_instance(iport->dip));
1313 
1314 	return (DDI_SUCCESS);
1315 }
1316 
1317 static int
1318 pmcs_unattach(pmcs_hw_t *pwp)
1319 {
1320 	int i;
1321 	enum pwpstate curstate;
1322 	pmcs_cq_thr_info_t *cqti;
1323 
1324 	/*
1325 	 * Tear down the interrupt infrastructure.
1326 	 */
1327 	if (pmcs_teardown_intr(pwp)) {
1328 		pwp->stuck = 1;
1329 	}
1330 	pwp->intr_cnt = 0;
1331 
1332 	/*
1333 	 * Grab a lock, if initted, to set state.
1334 	 */
1335 	if (pwp->locks_initted) {
1336 		mutex_enter(&pwp->lock);
1337 		if (pwp->state != STATE_DEAD) {
1338 			pwp->state = STATE_UNPROBING;
1339 		}
1340 		curstate = pwp->state;
1341 		mutex_exit(&pwp->lock);
1342 
1343 		/*
1344 		 * Stop the I/O completion threads.
1345 		 */
1346 		mutex_enter(&pwp->cq_lock);
1347 		pwp->cq_info.cq_stop = B_TRUE;
1348 		for (i = 0; i < pwp->cq_info.cq_threads; i++) {
1349 			if (pwp->cq_info.cq_thr_info[i].cq_thread) {
1350 				cqti = &pwp->cq_info.cq_thr_info[i];
1351 				mutex_enter(&cqti->cq_thr_lock);
1352 				cv_signal(&cqti->cq_cv);
1353 				mutex_exit(&cqti->cq_thr_lock);
1354 				mutex_exit(&pwp->cq_lock);
1355 				thread_join(cqti->cq_thread->t_did);
1356 				mutex_enter(&pwp->cq_lock);
1357 			}
1358 		}
1359 		mutex_exit(&pwp->cq_lock);
1360 		kmem_free(pwp->cq_info.cq_thr_info,
1361 		    sizeof (pmcs_cq_thr_info_t) * pwp->cq_info.cq_threads);
1362 
1363 		/*
1364 		 * Stop the interrupt coalescing timer thread
1365 		 */
1366 		if (pwp->ict_thread) {
1367 			mutex_enter(&pwp->ict_lock);
1368 			pwp->io_intr_coal.stop_thread = B_TRUE;
1369 			cv_signal(&pwp->ict_cv);
1370 			mutex_exit(&pwp->ict_lock);
1371 			thread_join(pwp->ict_thread->t_did);
1372 		}
1373 	} else {
1374 		if (pwp->state != STATE_DEAD) {
1375 			pwp->state = STATE_UNPROBING;
1376 		}
1377 		curstate = pwp->state;
1378 	}
1379 
1380 	/*
1381 	 * Make sure that any pending watchdog won't
1382 	 * be called from this point on out.
1383 	 */
1384 	(void) untimeout(pwp->wdhandle);
1385 	/*
1386 	 * After the above action, the watchdog
1387 	 * timer that starts up the worker task
1388 	 * may trigger but will exit immediately
1389 	 * on triggering.
1390 	 *
1391 	 * Now that this is done, we can destroy
1392 	 * the task queue, which will wait if we're
1393 	 * running something on it.
1394 	 */
1395 	if (pwp->tq) {
1396 		ddi_taskq_destroy(pwp->tq);
1397 		pwp->tq = NULL;
1398 	}
1399 
1400 	pmcs_fm_fini(pwp);
1401 
1402 	if (pwp->hba_attached) {
1403 		(void) scsi_hba_detach(pwp->dip);
1404 		pwp->hba_attached = 0;
1405 	}
1406 
1407 	/*
1408 	 * If the chip hasn't been marked dead, shut it down now
1409 	 * to bring it back to a known state without attempting
1410 	 * a soft reset.
1411 	 */
1412 	if (curstate != STATE_DEAD && pwp->locks_initted) {
1413 		/*
1414 		 * De-register all registered devices
1415 		 */
1416 		pmcs_deregister_devices(pwp, pwp->root_phys);
1417 
1418 		/*
1419 		 * Stop all the phys.
1420 		 */
1421 		pmcs_stop_phys(pwp);
1422 
1423 		/*
1424 		 * Shut Down Message Passing
1425 		 */
1426 		(void) pmcs_stop_mpi(pwp);
1427 
1428 		/*
1429 		 * Reset chip
1430 		 */
1431 		(void) pmcs_soft_reset(pwp, B_FALSE);
1432 		pwp->last_reset_reason = PMCS_LAST_RST_DETACH;
1433 	}
1434 
1435 	/*
1436 	 * Turn off interrupts on the chip
1437 	 */
1438 	if (pwp->mpi_acc_handle) {
1439 		pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_MASK, 0xffffffff);
1440 	}
1441 
1442 	if (pwp->hss_phymap != NULL) {
1443 		/* Destroy the phymap */
1444 		sas_phymap_destroy(pwp->hss_phymap);
1445 	}
1446 
1447 	if (pwp->hss_iportmap != NULL) {
1448 		/* Destroy the iportmap */
1449 		scsi_hba_iportmap_destroy(pwp->hss_iportmap);
1450 	}
1451 
1452 	/* Destroy the iports lock and list */
1453 	rw_destroy(&pwp->iports_lock);
1454 	ASSERT(list_is_empty(&pwp->iports));
1455 	list_destroy(&pwp->iports);
1456 
1457 	/*
1458 	 * Free DMA handles and associated consistent memory
1459 	 */
1460 	if (pwp->regdump_hndl) {
1461 		if (ddi_dma_unbind_handle(pwp->regdump_hndl) != DDI_SUCCESS) {
1462 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1463 			    "Condition check failed "
1464 			    "at %s():%d", __func__, __LINE__);
1465 		}
1466 		ddi_dma_free_handle(&pwp->regdump_hndl);
1467 		ddi_dma_mem_free(&pwp->regdump_acchdl);
1468 		pwp->regdump_hndl = 0;
1469 	}
1470 	if (pwp->fwlog_hndl) {
1471 		if (ddi_dma_unbind_handle(pwp->fwlog_hndl) != DDI_SUCCESS) {
1472 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1473 			    "Condition check failed "
1474 			    "at %s():%d", __func__, __LINE__);
1475 		}
1476 		ddi_dma_free_handle(&pwp->fwlog_hndl);
1477 		ddi_dma_mem_free(&pwp->fwlog_acchdl);
1478 		pwp->fwlog_hndl = 0;
1479 	}
1480 	if (pwp->cip_handles) {
1481 		if (ddi_dma_unbind_handle(pwp->cip_handles) != DDI_SUCCESS) {
1482 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1483 			    "Condition check failed "
1484 			    "at %s():%d", __func__, __LINE__);
1485 		}
1486 		ddi_dma_free_handle(&pwp->cip_handles);
1487 		ddi_dma_mem_free(&pwp->cip_acchdls);
1488 		pwp->cip_handles = 0;
1489 	}
1490 	for (i = 0; i < PMCS_NOQ; i++) {
1491 		if (pwp->oqp_handles[i]) {
1492 			if (ddi_dma_unbind_handle(pwp->oqp_handles[i]) !=
1493 			    DDI_SUCCESS) {
1494 				pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1495 				    "Condition check failed at %s():%d",
1496 				    __func__, __LINE__);
1497 			}
1498 			ddi_dma_free_handle(&pwp->oqp_handles[i]);
1499 			ddi_dma_mem_free(&pwp->oqp_acchdls[i]);
1500 			pwp->oqp_handles[i] = 0;
1501 		}
1502 	}
1503 	for (i = 0; i < PMCS_NIQ; i++) {
1504 		if (pwp->iqp_handles[i]) {
1505 			if (ddi_dma_unbind_handle(pwp->iqp_handles[i]) !=
1506 			    DDI_SUCCESS) {
1507 				pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1508 				    "Condition check failed at %s():%d",
1509 				    __func__, __LINE__);
1510 			}
1511 			ddi_dma_free_handle(&pwp->iqp_handles[i]);
1512 			ddi_dma_mem_free(&pwp->iqp_acchdls[i]);
1513 			pwp->iqp_handles[i] = 0;
1514 		}
1515 	}
1516 
1517 	pmcs_free_dma_chunklist(pwp);
1518 
1519 	/*
1520 	 * Unmap registers and destroy access handles
1521 	 */
1522 	if (pwp->mpi_acc_handle) {
1523 		ddi_regs_map_free(&pwp->mpi_acc_handle);
1524 		pwp->mpi_acc_handle = 0;
1525 	}
1526 	if (pwp->top_acc_handle) {
1527 		ddi_regs_map_free(&pwp->top_acc_handle);
1528 		pwp->top_acc_handle = 0;
1529 	}
1530 	if (pwp->gsm_acc_handle) {
1531 		ddi_regs_map_free(&pwp->gsm_acc_handle);
1532 		pwp->gsm_acc_handle = 0;
1533 	}
1534 	if (pwp->msg_acc_handle) {
1535 		ddi_regs_map_free(&pwp->msg_acc_handle);
1536 		pwp->msg_acc_handle = 0;
1537 	}
1538 	if (pwp->pci_acc_handle) {
1539 		pci_config_teardown(&pwp->pci_acc_handle);
1540 		pwp->pci_acc_handle = 0;
1541 	}
1542 
1543 	/*
1544 	 * Do memory allocation cleanup.
1545 	 */
1546 	while (pwp->dma_freelist) {
1547 		pmcs_dmachunk_t *this = pwp->dma_freelist;
1548 		pwp->dma_freelist = this->nxt;
1549 		kmem_free(this, sizeof (pmcs_dmachunk_t));
1550 	}
1551 
1552 	/*
1553 	 * Free pools
1554 	 */
1555 	if (pwp->iocomp_cb_cache) {
1556 		kmem_cache_destroy(pwp->iocomp_cb_cache);
1557 	}
1558 
1559 	/*
1560 	 * Free all PHYs (at level > 0), then free the cache
1561 	 */
1562 	pmcs_free_all_phys(pwp, pwp->root_phys);
1563 	if (pwp->phy_cache) {
1564 		kmem_cache_destroy(pwp->phy_cache);
1565 	}
1566 
1567 	/*
1568 	 * Free root PHYs
1569 	 */
1570 	if (pwp->root_phys) {
1571 		pmcs_phy_t *phyp = pwp->root_phys;
1572 		for (i = 0; i < pwp->nphy; i++) {
1573 			mutex_destroy(&phyp->phy_lock);
1574 			phyp = phyp->sibling;
1575 		}
1576 		kmem_free(pwp->root_phys, pwp->nphy * sizeof (pmcs_phy_t));
1577 		pwp->root_phys = NULL;
1578 		pwp->nphy = 0;
1579 	}
1580 
1581 	/* Free the targets list */
1582 	if (pwp->targets) {
1583 		kmem_free(pwp->targets,
1584 		    sizeof (pmcs_xscsi_t *) * pwp->max_dev);
1585 	}
1586 
1587 	/*
1588 	 * Free work structures
1589 	 */
1590 
1591 	if (pwp->work && pwp->max_cmd) {
1592 		for (i = 0; i < pwp->max_cmd; i++) {
1593 			pmcwork_t *pwrk = &pwp->work[i];
1594 			mutex_destroy(&pwrk->lock);
1595 			cv_destroy(&pwrk->sleep_cv);
1596 		}
1597 		kmem_free(pwp->work, sizeof (pmcwork_t) * pwp->max_cmd);
1598 		pwp->work = NULL;
1599 		pwp->max_cmd = 0;
1600 	}
1601 
1602 	/*
1603 	 * Do last property and SCSA cleanup
1604 	 */
1605 	if (pwp->smp_tran) {
1606 		smp_hba_tran_free(pwp->smp_tran);
1607 		pwp->smp_tran = NULL;
1608 	}
1609 	if (pwp->tran) {
1610 		scsi_hba_tran_free(pwp->tran);
1611 		pwp->tran = NULL;
1612 	}
1613 	if (pwp->reset_notify_listf) {
1614 		scsi_hba_reset_notify_tear_down(pwp->reset_notify_listf);
1615 		pwp->reset_notify_listf = NULL;
1616 	}
1617 	ddi_prop_remove_all(pwp->dip);
1618 	if (pwp->stuck) {
1619 		return (-1);
1620 	}
1621 
1622 	/* Free register dump area if allocated */
1623 	if (pwp->regdumpp) {
1624 		kmem_free(pwp->regdumpp, PMCS_REG_DUMP_SIZE);
1625 		pwp->regdumpp = NULL;
1626 	}
1627 	if (pwp->iqpt && pwp->iqpt->head) {
1628 		kmem_free(pwp->iqpt->head, PMCS_IQP_TRACE_BUFFER_SIZE);
1629 		pwp->iqpt->head = pwp->iqpt->curpos = NULL;
1630 	}
1631 	if (pwp->iqpt) {
1632 		kmem_free(pwp->iqpt, sizeof (pmcs_iqp_trace_t));
1633 		pwp->iqpt = NULL;
1634 	}
1635 
1636 	/* Destroy pwp's lock */
1637 	if (pwp->locks_initted) {
1638 		mutex_destroy(&pwp->lock);
1639 		mutex_destroy(&pwp->dma_lock);
1640 		mutex_destroy(&pwp->axil_lock);
1641 		mutex_destroy(&pwp->cq_lock);
1642 		mutex_destroy(&pwp->config_lock);
1643 		mutex_destroy(&pwp->ict_lock);
1644 		mutex_destroy(&pwp->wfree_lock);
1645 		mutex_destroy(&pwp->pfree_lock);
1646 		mutex_destroy(&pwp->dead_phylist_lock);
1647 #ifdef	DEBUG
1648 		mutex_destroy(&pwp->dbglock);
1649 #endif
1650 		cv_destroy(&pwp->config_cv);
1651 		cv_destroy(&pwp->ict_cv);
1652 		cv_destroy(&pwp->drain_cv);
1653 		pwp->locks_initted = 0;
1654 	}
1655 
1656 	ddi_soft_state_free(pmcs_softc_state, ddi_get_instance(pwp->dip));
1657 	return (0);
1658 }
1659 
1660 /*
1661  * quiesce (9E) entry point
1662  *
1663  * This function is called when the system is single-threaded at high PIL
1664  * with preemption disabled. Therefore, the function must not block/wait/sleep.
1665  *
1666  * Returns DDI_SUCCESS or DDI_FAILURE.
1667  *
1668  */
1669 static int
1670 pmcs_quiesce(dev_info_t *dip)
1671 {
1672 	pmcs_hw_t	*pwp;
1673 	scsi_hba_tran_t	*tran;
1674 
1675 	if ((tran = ddi_get_driver_private(dip)) == NULL)
1676 		return (DDI_SUCCESS);
1677 
1678 	/* No quiesce necessary on a per-iport basis */
1679 	if (scsi_hba_iport_unit_address(dip) != NULL) {
1680 		return (DDI_SUCCESS);
1681 	}
1682 
1683 	if ((pwp = TRAN2PMC(tran)) == NULL)
1684 		return (DDI_SUCCESS);
1685 
1686 	/* Stop MPI & Reset chip (no need to re-initialize) */
1687 	(void) pmcs_stop_mpi(pwp);
1688 	(void) pmcs_soft_reset(pwp, B_TRUE);
1689 	pwp->last_reset_reason = PMCS_LAST_RST_QUIESCE;
1690 
1691 	return (DDI_SUCCESS);
1692 }
1693 
1694 /*
1695  * Called with xp->statlock and PHY lock and scratch acquired.
1696  */
1697 static int
1698 pmcs_add_sata_device(pmcs_hw_t *pwp, pmcs_xscsi_t *xp)
1699 {
1700 	ata_identify_t *ati;
1701 	int result, i;
1702 	pmcs_phy_t *pptr;
1703 	uint16_t *a;
1704 	union {
1705 		uint8_t nsa[8];
1706 		uint16_t nsb[4];
1707 	} u;
1708 
1709 	/*
1710 	 * Safe defaults - use only if this target is brand new (i.e. doesn't
1711 	 * already have these settings configured)
1712 	 */
1713 	if (xp->capacity == 0) {
1714 		xp->capacity = (uint64_t)-1;
1715 		xp->ca = 1;
1716 		xp->qdepth = 1;
1717 		xp->pio = 1;
1718 	}
1719 
1720 	pptr = xp->phy;
1721 
1722 	/*
1723 	 * We only try and issue an IDENTIFY for first level
1724 	 * (direct attached) devices. We don't try and
1725 	 * set other quirks here (this will happen later,
1726 	 * if the device is fully configured)
1727 	 */
1728 	if (pptr->level) {
1729 		return (0);
1730 	}
1731 
1732 	mutex_exit(&xp->statlock);
1733 	result = pmcs_sata_identify(pwp, pptr);
1734 	mutex_enter(&xp->statlock);
1735 
1736 	if (result) {
1737 		return (result);
1738 	}
1739 	ati = pwp->scratch;
1740 	a = &ati->word108;
1741 	for (i = 0; i < 4; i++) {
1742 		u.nsb[i] = ddi_swap16(*a++);
1743 	}
1744 
1745 	/*
1746 	 * Check the returned data for being a valid (NAA=5) WWN.
1747 	 * If so, use that and override the SAS address we were
1748 	 * given at Link Up time.
1749 	 */
1750 	if ((u.nsa[0] >> 4) == 5) {
1751 		(void) memcpy(pptr->sas_address, u.nsa, 8);
1752 	}
1753 	pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
1754 	    "%s: %s has SAS ADDRESS " SAS_ADDR_FMT,
1755 	    __func__, pptr->path, SAS_ADDR_PRT(pptr->sas_address));
1756 	return (0);
1757 }
1758 
1759 /*
1760  * Called with PHY lock and target statlock held and scratch acquired
1761  */
1762 static boolean_t
1763 pmcs_add_new_device(pmcs_hw_t *pwp, pmcs_xscsi_t *target)
1764 {
1765 	ASSERT(target != NULL);
1766 	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, target, "%s: target = 0x%p",
1767 	    __func__, (void *) target);
1768 
1769 	switch (target->phy->dtype) {
1770 	case SATA:
1771 		if (pmcs_add_sata_device(pwp, target) != 0) {
1772 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, target->phy,
1773 			    target, "%s: add_sata_device failed for tgt 0x%p",
1774 			    __func__, (void *) target);
1775 			return (B_FALSE);
1776 		}
1777 		break;
1778 	case SAS:
1779 		target->qdepth = maxqdepth;
1780 		break;
1781 	case EXPANDER:
1782 		target->qdepth = 1;
1783 		break;
1784 	}
1785 
1786 	target->new = 0;
1787 	target->assigned = 1;
1788 	target->dev_state = PMCS_DEVICE_STATE_OPERATIONAL;
1789 	target->dtype = target->phy->dtype;
1790 
1791 	/*
1792 	 * Set the PHY's config stop time to 0.  This is one of the final
1793 	 * stops along the config path, so we're indicating that we
1794 	 * successfully configured the PHY.
1795 	 */
1796 	target->phy->config_stop = 0;
1797 
1798 	return (B_TRUE);
1799 }
1800 
1801 void
1802 pmcs_worker(void *arg)
1803 {
1804 	pmcs_hw_t *pwp = arg;
1805 	ulong_t work_flags;
1806 
1807 	DTRACE_PROBE2(pmcs__worker, ulong_t, pwp->work_flags, boolean_t,
1808 	    pwp->config_changed);
1809 
1810 	if (pwp->state != STATE_RUNNING) {
1811 		return;
1812 	}
1813 
1814 	work_flags = atomic_swap_ulong(&pwp->work_flags, 0);
1815 
1816 	if (work_flags & PMCS_WORK_FLAG_DUMP_REGS) {
1817 		mutex_enter(&pwp->lock);
1818 		pmcs_register_dump_int(pwp);
1819 		mutex_exit(&pwp->lock);
1820 	}
1821 
1822 	if (work_flags & PMCS_WORK_FLAG_SAS_HW_ACK) {
1823 		pmcs_ack_events(pwp);
1824 	}
1825 
1826 	if (work_flags & PMCS_WORK_FLAG_SPINUP_RELEASE) {
1827 		mutex_enter(&pwp->lock);
1828 		pmcs_spinup_release(pwp, NULL);
1829 		mutex_exit(&pwp->lock);
1830 	}
1831 
1832 	if (work_flags & PMCS_WORK_FLAG_SSP_EVT_RECOVERY) {
1833 		pmcs_ssp_event_recovery(pwp);
1834 	}
1835 
1836 	if (work_flags & PMCS_WORK_FLAG_DS_ERR_RECOVERY) {
1837 		pmcs_dev_state_recovery(pwp, NULL);
1838 	}
1839 
1840 	if (work_flags & PMCS_WORK_FLAG_DEREGISTER_DEV) {
1841 		pmcs_deregister_device_work(pwp, NULL);
1842 	}
1843 
1844 	if (work_flags & PMCS_WORK_FLAG_DISCOVER) {
1845 		pmcs_discover(pwp);
1846 	}
1847 
1848 	if (work_flags & PMCS_WORK_FLAG_ABORT_HANDLE) {
1849 		if (pmcs_abort_handler(pwp)) {
1850 			SCHEDULE_WORK(pwp, PMCS_WORK_ABORT_HANDLE);
1851 		}
1852 	}
1853 
1854 	if (work_flags & PMCS_WORK_FLAG_SATA_RUN) {
1855 		pmcs_sata_work(pwp);
1856 	}
1857 
1858 	if (work_flags & PMCS_WORK_FLAG_RUN_QUEUES) {
1859 		pmcs_scsa_wq_run(pwp);
1860 		mutex_enter(&pwp->lock);
1861 		PMCS_CQ_RUN(pwp);
1862 		mutex_exit(&pwp->lock);
1863 	}
1864 
1865 	if (work_flags & PMCS_WORK_FLAG_ADD_DMA_CHUNKS) {
1866 		if (pmcs_add_more_chunks(pwp,
1867 		    ptob(1) * PMCS_ADDTL_CHUNK_PAGES)) {
1868 			SCHEDULE_WORK(pwp, PMCS_WORK_ADD_DMA_CHUNKS);
1869 		} else {
1870 			SCHEDULE_WORK(pwp, PMCS_WORK_RUN_QUEUES);
1871 		}
1872 	}
1873 }
1874 
1875 static int
1876 pmcs_add_more_chunks(pmcs_hw_t *pwp, unsigned long nsize)
1877 {
1878 	pmcs_dmachunk_t *dc;
1879 	unsigned long dl;
1880 	pmcs_chunk_t	*pchunk = NULL;
1881 
1882 	pwp->cip_dma_attr.dma_attr_align = sizeof (uint32_t);
1883 
1884 	pchunk = kmem_zalloc(sizeof (pmcs_chunk_t), KM_SLEEP);
1885 	if (pchunk == NULL) {
1886 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1887 		    "Not enough memory for DMA chunks");
1888 		return (-1);
1889 	}
1890 
1891 	if (pmcs_dma_setup(pwp, &pwp->cip_dma_attr, &pchunk->acc_handle,
1892 	    &pchunk->dma_handle, nsize, (caddr_t *)&pchunk->addrp,
1893 	    &pchunk->dma_addr) == B_FALSE) {
1894 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1895 		    "Failed to setup DMA for chunks");
1896 		kmem_free(pchunk, sizeof (pmcs_chunk_t));
1897 		return (-1);
1898 	}
1899 
1900 	if ((pmcs_check_acc_handle(pchunk->acc_handle) != DDI_SUCCESS) ||
1901 	    (pmcs_check_dma_handle(pchunk->dma_handle) != DDI_SUCCESS)) {
1902 		ddi_fm_service_impact(pwp->dip, DDI_SERVICE_UNAFFECTED);
1903 		return (-1);
1904 	}
1905 
1906 	bzero(pchunk->addrp, nsize);
1907 	dc = NULL;
1908 	for (dl = 0; dl < (nsize / PMCS_SGL_CHUNKSZ); dl++) {
1909 		pmcs_dmachunk_t *tmp;
1910 		tmp = kmem_alloc(sizeof (pmcs_dmachunk_t), KM_SLEEP);
1911 		tmp->nxt = dc;
1912 		dc = tmp;
1913 	}
1914 	mutex_enter(&pwp->dma_lock);
1915 	pmcs_idma_chunks(pwp, dc, pchunk, nsize);
1916 	pwp->nchunks++;
1917 	mutex_exit(&pwp->dma_lock);
1918 	return (0);
1919 }
1920 
1921 static void
1922 pmcs_check_forward_progress(pmcs_hw_t *pwp)
1923 {
1924 	pmcwork_t	*wrkp;
1925 	uint32_t	*iqp;
1926 	uint32_t	cur_iqci;
1927 	uint32_t	cur_work_idx;
1928 	uint32_t	cur_msgu_tick;
1929 	uint32_t	cur_iop_tick;
1930 	int		i;
1931 
1932 	mutex_enter(&pwp->lock);
1933 
1934 	if (pwp->state == STATE_IN_RESET) {
1935 		mutex_exit(&pwp->lock);
1936 		return;
1937 	}
1938 
1939 	/*
1940 	 * Ensure that inbound work is getting picked up.  First, check to
1941 	 * see if new work has been posted.  If it has, ensure that the
1942 	 * work is moving forward by checking the consumer index and the
1943 	 * last_htag for the work being processed against what we saw last
1944 	 * time.  Note: we use the work structure's 'last_htag' because at
1945 	 * any given moment it could be freed back, thus clearing 'htag'
1946 	 * and setting 'last_htag' (see pmcs_pwork).
1947 	 */
1948 	for (i = 0; i < PMCS_NIQ; i++) {
1949 		cur_iqci = pmcs_rd_iqci(pwp, i);
1950 		iqp = &pwp->iqp[i][cur_iqci * (PMCS_QENTRY_SIZE >> 2)];
1951 		cur_work_idx = PMCS_TAG_INDEX(LE_32(*(iqp+1)));
1952 		wrkp = &pwp->work[cur_work_idx];
1953 		if (cur_iqci == pwp->shadow_iqpi[i]) {
1954 			pwp->last_iqci[i] = cur_iqci;
1955 			pwp->last_htag[i] = wrkp->last_htag;
1956 			continue;
1957 		}
1958 		if ((cur_iqci == pwp->last_iqci[i]) &&
1959 		    (wrkp->last_htag == pwp->last_htag[i])) {
1960 			pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL,
1961 			    "Inbound Queue stall detected, issuing reset");
1962 			goto hot_reset;
1963 		}
1964 		pwp->last_iqci[i] = cur_iqci;
1965 		pwp->last_htag[i] = wrkp->last_htag;
1966 	}
1967 
1968 	/*
1969 	 * Check heartbeat on both the MSGU and IOP.  It is unlikely that
1970 	 * we'd ever fail here, as the inbound queue monitoring code above
1971 	 * would detect a stall due to either of these elements being
1972 	 * stalled, but we might as well keep an eye on them.
1973 	 */
1974 	cur_msgu_tick = pmcs_rd_gst_tbl(pwp, PMCS_GST_MSGU_TICK);
1975 	if (cur_msgu_tick == pwp->last_msgu_tick) {
1976 		pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL,
1977 		    "Stall detected on MSGU, issuing reset");
1978 		goto hot_reset;
1979 	}
1980 	pwp->last_msgu_tick = cur_msgu_tick;
1981 
1982 	cur_iop_tick  = pmcs_rd_gst_tbl(pwp, PMCS_GST_IOP_TICK);
1983 	if (cur_iop_tick == pwp->last_iop_tick) {
1984 		pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL,
1985 		    "Stall detected on IOP, issuing reset");
1986 		goto hot_reset;
1987 	}
1988 	pwp->last_iop_tick = cur_iop_tick;
1989 
1990 	mutex_exit(&pwp->lock);
1991 	return;
1992 
1993 hot_reset:
1994 	pwp->state = STATE_DEAD;
1995 	/*
1996 	 * We've detected a stall. Attempt to recover service via hot
1997 	 * reset. In case of failure, pmcs_hot_reset() will handle the
1998 	 * failure and issue any required FM notifications.
1999 	 * See pmcs_subr.c for more details.
2000 	 */
2001 	if (pmcs_hot_reset(pwp)) {
2002 		pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL,
2003 		    "%s: hot reset failure", __func__);
2004 	} else {
2005 		pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL,
2006 		    "%s: hot reset complete", __func__);
2007 		pwp->last_reset_reason = PMCS_LAST_RST_STALL;
2008 	}
2009 	mutex_exit(&pwp->lock);
2010 }
2011 
2012 static void
2013 pmcs_check_commands(pmcs_hw_t *pwp)
2014 {
2015 	pmcs_cmd_t *sp;
2016 	size_t amt;
2017 	char path[32];
2018 	pmcwork_t *pwrk;
2019 	pmcs_xscsi_t *target;
2020 	pmcs_phy_t *phyp;
2021 	int rval;
2022 
2023 	for (pwrk = pwp->work; pwrk < &pwp->work[pwp->max_cmd]; pwrk++) {
2024 		mutex_enter(&pwrk->lock);
2025 
2026 		/*
2027 		 * If the command isn't active, we can't be timing it still.
2028 		 * Active means the tag is not free and the state is "on chip".
2029 		 */
2030 		if (!PMCS_COMMAND_ACTIVE(pwrk)) {
2031 			mutex_exit(&pwrk->lock);
2032 			continue;
2033 		}
2034 
2035 		/*
2036 		 * No timer active for this command.
2037 		 */
2038 		if (pwrk->timer == 0) {
2039 			mutex_exit(&pwrk->lock);
2040 			continue;
2041 		}
2042 
2043 		/*
2044 		 * Knock off bits for the time interval.
2045 		 */
2046 		if (pwrk->timer >= US2WT(PMCS_WATCH_INTERVAL)) {
2047 			pwrk->timer -= US2WT(PMCS_WATCH_INTERVAL);
2048 		} else {
2049 			pwrk->timer = 0;
2050 		}
2051 		if (pwrk->timer > 0) {
2052 			mutex_exit(&pwrk->lock);
2053 			continue;
2054 		}
2055 
2056 		/*
2057 		 * The command has now officially timed out.
2058 		 * Get the path for it. If it doesn't have
2059 		 * a phy pointer any more, it's really dead
2060 		 * and can just be put back on the free list.
2061 		 * There should *not* be any commands associated
2062 		 * with it any more.
2063 		 */
2064 		if (pwrk->phy == NULL) {
2065 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2066 			    "dead command with gone phy being recycled");
2067 			ASSERT(pwrk->xp == NULL);
2068 			pmcs_pwork(pwp, pwrk);
2069 			continue;
2070 		}
2071 		amt = sizeof (path);
2072 		amt = min(sizeof (pwrk->phy->path), amt);
2073 		(void) memcpy(path, pwrk->phy->path, amt);
2074 
2075 		/*
2076 		 * If this is a non-SCSA command, stop here. Eventually
2077 		 * we might do something with non-SCSA commands here-
2078 		 * but so far their timeout mechanisms are handled in
2079 		 * the WAIT_FOR macro.
2080 		 */
2081 		if (pwrk->xp == NULL) {
2082 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2083 			    "%s: non-SCSA cmd tag 0x%x timed out",
2084 			    path, pwrk->htag);
2085 			mutex_exit(&pwrk->lock);
2086 			continue;
2087 		}
2088 
2089 		sp = pwrk->arg;
2090 		ASSERT(sp != NULL);
2091 
2092 		/*
2093 		 * Mark it as timed out.
2094 		 */
2095 		CMD2PKT(sp)->pkt_reason = CMD_TIMEOUT;
2096 		CMD2PKT(sp)->pkt_statistics |= STAT_TIMEOUT;
2097 #ifdef	DEBUG
2098 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pwrk->phy, pwrk->xp,
2099 		    "%s: SCSA cmd tag 0x%x timed out (state %x) onwire=%d",
2100 		    path, pwrk->htag, pwrk->state, pwrk->onwire);
2101 #else
2102 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pwrk->phy, pwrk->xp,
2103 		    "%s: SCSA cmd tag 0x%x timed out (state %x)",
2104 		    path, pwrk->htag, pwrk->state);
2105 #endif
2106 		/*
2107 		 * Mark the work structure as timed out.
2108 		 */
2109 		pwrk->state = PMCS_WORK_STATE_TIMED_OUT;
2110 		phyp = pwrk->phy;
2111 		target = pwrk->xp;
2112 		ASSERT(target != NULL);
2113 		mutex_exit(&pwrk->lock);
2114 
2115 		pmcs_lock_phy(phyp);
2116 		mutex_enter(&target->statlock);
2117 
2118 		/*
2119 		 * No point attempting recovery if the device is gone
2120 		 */
2121 		if (target->dev_gone) {
2122 			mutex_exit(&target->statlock);
2123 			pmcs_unlock_phy(phyp);
2124 			pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, target,
2125 			    "%s: tgt(0x%p) is gone. Returning CMD_DEV_GONE "
2126 			    "for htag 0x%08x", __func__,
2127 			    (void *)target, pwrk->htag);
2128 			mutex_enter(&pwrk->lock);
2129 			if (!PMCS_COMMAND_DONE(pwrk)) {
2130 				/* Complete this command here */
2131 				pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, target,
2132 				    "%s: Completing cmd (htag 0x%08x) "
2133 				    "anyway", __func__, pwrk->htag);
2134 				pwrk->dead = 1;
2135 				CMD2PKT(sp)->pkt_reason = CMD_DEV_GONE;
2136 				CMD2PKT(sp)->pkt_state = STATE_GOT_BUS;
2137 				pmcs_complete_work_impl(pwp, pwrk, NULL, 0);
2138 			} else {
2139 				mutex_exit(&pwrk->lock);
2140 			}
2141 			continue;
2142 		}
2143 
2144 		mutex_exit(&target->statlock);
2145 		rval = pmcs_abort(pwp, phyp, pwrk->htag, 0, 1);
2146 		if (rval) {
2147 			pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, target,
2148 			    "%s: Bad status (%d) on abort of HTAG 0x%08x",
2149 			    __func__, rval, pwrk->htag);
2150 			pmcs_unlock_phy(phyp);
2151 			mutex_enter(&pwrk->lock);
2152 			if (!PMCS_COMMAND_DONE(pwrk)) {
2153 				/* Complete this command here */
2154 				pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, target,
2155 				    "%s: Completing cmd (htag 0x%08x) "
2156 				    "anyway", __func__, pwrk->htag);
2157 				if (target->dev_gone) {
2158 					pwrk->dead = 1;
2159 					CMD2PKT(sp)->pkt_reason = CMD_DEV_GONE;
2160 					CMD2PKT(sp)->pkt_state = STATE_GOT_BUS;
2161 				}
2162 				pmcs_complete_work_impl(pwp, pwrk, NULL, 0);
2163 			} else {
2164 				mutex_exit(&pwrk->lock);
2165 			}
2166 			pmcs_lock_phy(phyp);
2167 			/*
2168 			 * No need to reschedule ABORT if we get any other
2169 			 * status
2170 			 */
2171 			if (rval == ENOMEM) {
2172 				phyp->abort_sent = 0;
2173 				phyp->abort_pending = 1;
2174 				SCHEDULE_WORK(pwp, PMCS_WORK_ABORT_HANDLE);
2175 			}
2176 		}
2177 		pmcs_unlock_phy(phyp);
2178 	}
2179 	/*
2180 	 * Run any completions that may have been queued up.
2181 	 */
2182 	PMCS_CQ_RUN(pwp);
2183 }
2184 
2185 static void
2186 pmcs_watchdog(void *arg)
2187 {
2188 	pmcs_hw_t *pwp = arg;
2189 
2190 	DTRACE_PROBE2(pmcs__watchdog, ulong_t, pwp->work_flags, boolean_t,
2191 	    pwp->config_changed);
2192 
2193 	/*
2194 	 * Check forward progress on the chip
2195 	 */
2196 	if (++pwp->watchdog_count == PMCS_FWD_PROG_TRIGGER) {
2197 		pwp->watchdog_count = 0;
2198 		pmcs_check_forward_progress(pwp);
2199 	}
2200 
2201 	/*
2202 	 * Check to see if we need to kick discovery off again
2203 	 */
2204 	mutex_enter(&pwp->config_lock);
2205 	if (pwp->config_restart &&
2206 	    (ddi_get_lbolt() >= pwp->config_restart_time)) {
2207 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2208 		    "%s: Timer expired for re-enumeration: Start discovery",
2209 		    __func__);
2210 		pwp->config_restart = B_FALSE;
2211 		SCHEDULE_WORK(pwp, PMCS_WORK_DISCOVER);
2212 	}
2213 	mutex_exit(&pwp->config_lock);
2214 
2215 	mutex_enter(&pwp->lock);
2216 	if (pwp->state != STATE_RUNNING) {
2217 		mutex_exit(&pwp->lock);
2218 		return;
2219 	}
2220 
2221 	if (atomic_cas_ulong(&pwp->work_flags, 0, 0) != 0) {
2222 		if (ddi_taskq_dispatch(pwp->tq, pmcs_worker, pwp,
2223 		    DDI_NOSLEEP) != DDI_SUCCESS) {
2224 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2225 			    "Could not dispatch to worker thread");
2226 		}
2227 	}
2228 	pwp->wdhandle = timeout(pmcs_watchdog, pwp,
2229 	    drv_usectohz(PMCS_WATCH_INTERVAL));
2230 
2231 	mutex_exit(&pwp->lock);
2232 
2233 	pmcs_check_commands(pwp);
2234 	pmcs_handle_dead_phys(pwp);
2235 }
2236 
2237 static int
2238 pmcs_remove_ihandlers(pmcs_hw_t *pwp, int icnt)
2239 {
2240 	int i, r, rslt = 0;
2241 	for (i = 0; i < icnt; i++) {
2242 		r = ddi_intr_remove_handler(pwp->ih_table[i]);
2243 		if (r == DDI_SUCCESS) {
2244 			continue;
2245 		}
2246 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2247 		    "%s: unable to remove interrupt handler %d", __func__, i);
2248 		rslt = -1;
2249 		break;
2250 	}
2251 	return (rslt);
2252 }
2253 
2254 static int
2255 pmcs_disable_intrs(pmcs_hw_t *pwp, int icnt)
2256 {
2257 	if (pwp->intr_cap & DDI_INTR_FLAG_BLOCK) {
2258 		int r = ddi_intr_block_disable(&pwp->ih_table[0],
2259 		    pwp->intr_cnt);
2260 		if (r != DDI_SUCCESS) {
2261 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2262 			    "unable to disable interrupt block");
2263 			return (-1);
2264 		}
2265 	} else {
2266 		int i;
2267 		for (i = 0; i < icnt; i++) {
2268 			if (ddi_intr_disable(pwp->ih_table[i]) == DDI_SUCCESS) {
2269 				continue;
2270 			}
2271 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2272 			    "unable to disable interrupt %d", i);
2273 			return (-1);
2274 		}
2275 	}
2276 	return (0);
2277 }
2278 
2279 static int
2280 pmcs_free_intrs(pmcs_hw_t *pwp, int icnt)
2281 {
2282 	int i;
2283 	for (i = 0; i < icnt; i++) {
2284 		if (ddi_intr_free(pwp->ih_table[i]) == DDI_SUCCESS) {
2285 			continue;
2286 		}
2287 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2288 		    "unable to free interrupt %d", i);
2289 		return (-1);
2290 	}
2291 	kmem_free(pwp->ih_table, pwp->ih_table_size);
2292 	pwp->ih_table_size = 0;
2293 	return (0);
2294 }
2295 
2296 /*
2297  * Try to set up interrupts of type "type" with a minimum number of interrupts
2298  * of "min".
2299  */
2300 static void
2301 pmcs_setup_intr_impl(pmcs_hw_t *pwp, int type, int min)
2302 {
2303 	int rval, avail, count, actual, max;
2304 
2305 	rval = ddi_intr_get_nintrs(pwp->dip, type, &count);
2306 	if ((rval != DDI_SUCCESS) || (count < min)) {
2307 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2308 		    "%s: get_nintrs failed; type: %d rc: %d count: %d min: %d",
2309 		    __func__, type, rval, count, min);
2310 		return;
2311 	}
2312 
2313 	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2314 	    "%s: nintrs = %d for type: %d", __func__, count, type);
2315 
2316 	rval = ddi_intr_get_navail(pwp->dip, type, &avail);
2317 	if ((rval != DDI_SUCCESS) || (avail < min)) {
2318 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2319 		    "%s: get_navail failed; type: %d rc: %d avail: %d min: %d",
2320 		    __func__, type, rval, avail, min);
2321 		return;
2322 	}
2323 
2324 	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2325 	    "%s: navail = %d for type: %d", __func__, avail, type);
2326 
2327 	pwp->ih_table_size = avail * sizeof (ddi_intr_handle_t);
2328 	pwp->ih_table = kmem_alloc(pwp->ih_table_size, KM_SLEEP);
2329 
2330 	switch (type) {
2331 	case DDI_INTR_TYPE_MSIX:
2332 		pwp->int_type = PMCS_INT_MSIX;
2333 		max = PMCS_MAX_MSIX;
2334 		break;
2335 	case DDI_INTR_TYPE_MSI:
2336 		pwp->int_type = PMCS_INT_MSI;
2337 		max = PMCS_MAX_MSI;
2338 		break;
2339 	case DDI_INTR_TYPE_FIXED:
2340 	default:
2341 		pwp->int_type = PMCS_INT_FIXED;
2342 		max = PMCS_MAX_FIXED;
2343 		break;
2344 	}
2345 
2346 	rval = ddi_intr_alloc(pwp->dip, pwp->ih_table, type, 0, max, &actual,
2347 	    DDI_INTR_ALLOC_NORMAL);
2348 	if (rval != DDI_SUCCESS) {
2349 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2350 		    "%s: ddi_intr_alloc failed; type: %d rc: %d",
2351 		    __func__, type, rval);
2352 		kmem_free(pwp->ih_table, pwp->ih_table_size);
2353 		pwp->ih_table = NULL;
2354 		pwp->ih_table_size = 0;
2355 		pwp->intr_cnt = 0;
2356 		pwp->int_type = PMCS_INT_NONE;
2357 		return;
2358 	}
2359 
2360 	pwp->intr_cnt = actual;
2361 }
2362 
2363 /*
2364  * Set up interrupts.
2365  * We return one of three values:
2366  *
2367  * 0 - success
2368  * EAGAIN - failure to set up interrupts
2369  * EIO - "" + we're now stuck partly enabled
2370  *
2371  * If EIO is returned, we can't unload the driver.
2372  */
2373 static int
2374 pmcs_setup_intr(pmcs_hw_t *pwp)
2375 {
2376 	int i, r, itypes, oqv_count;
2377 	ddi_intr_handler_t **iv_table;
2378 	size_t iv_table_size;
2379 	uint_t pri;
2380 
2381 	if (ddi_intr_get_supported_types(pwp->dip, &itypes) != DDI_SUCCESS) {
2382 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2383 		    "cannot get interrupt types");
2384 		return (EAGAIN);
2385 	}
2386 
2387 	if (disable_msix) {
2388 		itypes &= ~DDI_INTR_TYPE_MSIX;
2389 	}
2390 	if (disable_msi) {
2391 		itypes &= ~DDI_INTR_TYPE_MSI;
2392 	}
2393 
2394 	/*
2395 	 * We won't know what firmware we're running until we call pmcs_setup,
2396 	 * and we can't call pmcs_setup until we establish interrupts.
2397 	 */
2398 
2399 	pwp->int_type = PMCS_INT_NONE;
2400 
2401 	/*
2402 	 * We want PMCS_MAX_MSIX vectors for MSI-X.  Anything less would be
2403 	 * uncivilized.
2404 	 */
2405 	if (itypes & DDI_INTR_TYPE_MSIX) {
2406 		pmcs_setup_intr_impl(pwp, DDI_INTR_TYPE_MSIX, PMCS_MAX_MSIX);
2407 		if (pwp->int_type == PMCS_INT_MSIX) {
2408 			itypes = 0;
2409 		}
2410 	}
2411 
2412 	if (itypes & DDI_INTR_TYPE_MSI) {
2413 		pmcs_setup_intr_impl(pwp, DDI_INTR_TYPE_MSI, 1);
2414 		if (pwp->int_type == PMCS_INT_MSI) {
2415 			itypes = 0;
2416 		}
2417 	}
2418 
2419 	if (itypes & DDI_INTR_TYPE_FIXED) {
2420 		pmcs_setup_intr_impl(pwp, DDI_INTR_TYPE_FIXED, 1);
2421 		if (pwp->int_type == PMCS_INT_FIXED) {
2422 			itypes = 0;
2423 		}
2424 	}
2425 
2426 	if (pwp->intr_cnt == 0) {
2427 		pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL,
2428 		    "No interrupts available");
2429 		return (EAGAIN);
2430 	}
2431 
2432 	iv_table_size = sizeof (ddi_intr_handler_t *) * pwp->intr_cnt;
2433 	iv_table = kmem_alloc(iv_table_size, KM_SLEEP);
2434 
2435 	/*
2436 	 * Get iblock cookie and add handlers.
2437 	 */
2438 	switch (pwp->intr_cnt) {
2439 	case 1:
2440 		iv_table[0] = pmcs_all_intr;
2441 		break;
2442 	case 2:
2443 		iv_table[0] = pmcs_iodone_ix;
2444 		iv_table[1] = pmcs_nonio_ix;
2445 		break;
2446 	case 4:
2447 		iv_table[PMCS_MSIX_GENERAL] = pmcs_general_ix;
2448 		iv_table[PMCS_MSIX_IODONE] = pmcs_iodone_ix;
2449 		iv_table[PMCS_MSIX_EVENTS] = pmcs_event_ix;
2450 		iv_table[PMCS_MSIX_FATAL] = pmcs_fatal_ix;
2451 		break;
2452 	default:
2453 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2454 		    "%s: intr_cnt = %d - unexpected", __func__, pwp->intr_cnt);
2455 		kmem_free(iv_table, iv_table_size);
2456 		return (EAGAIN);
2457 	}
2458 
2459 	for (i = 0; i < pwp->intr_cnt; i++) {
2460 		r = ddi_intr_add_handler(pwp->ih_table[i], iv_table[i],
2461 		    (caddr_t)pwp, NULL);
2462 		if (r != DDI_SUCCESS) {
2463 			kmem_free(iv_table, iv_table_size);
2464 			if (pmcs_remove_ihandlers(pwp, i)) {
2465 				return (EIO);
2466 			}
2467 			if (pmcs_free_intrs(pwp, i)) {
2468 				return (EIO);
2469 			}
2470 			pwp->intr_cnt = 0;
2471 			return (EAGAIN);
2472 		}
2473 	}
2474 
2475 	kmem_free(iv_table, iv_table_size);
2476 
2477 	if (ddi_intr_get_cap(pwp->ih_table[0], &pwp->intr_cap) != DDI_SUCCESS) {
2478 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2479 		    "unable to get int capabilities");
2480 		if (pmcs_remove_ihandlers(pwp, pwp->intr_cnt)) {
2481 			return (EIO);
2482 		}
2483 		if (pmcs_free_intrs(pwp, pwp->intr_cnt)) {
2484 			return (EIO);
2485 		}
2486 		pwp->intr_cnt = 0;
2487 		return (EAGAIN);
2488 	}
2489 
2490 	if (pwp->intr_cap & DDI_INTR_FLAG_BLOCK) {
2491 		r = ddi_intr_block_enable(&pwp->ih_table[0], pwp->intr_cnt);
2492 		if (r != DDI_SUCCESS) {
2493 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2494 			    "intr blk enable failed");
2495 			if (pmcs_remove_ihandlers(pwp, pwp->intr_cnt)) {
2496 				return (EIO);
2497 			}
2498 			if (pmcs_free_intrs(pwp, pwp->intr_cnt)) {
2499 				return (EIO);
2500 			}
2501 			pwp->intr_cnt = 0;
2502 			return (EFAULT);
2503 		}
2504 	} else {
2505 		for (i = 0; i < pwp->intr_cnt; i++) {
2506 			r = ddi_intr_enable(pwp->ih_table[i]);
2507 			if (r == DDI_SUCCESS) {
2508 				continue;
2509 			}
2510 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2511 			    "unable to enable interrupt %d", i);
2512 			if (pmcs_disable_intrs(pwp, i)) {
2513 				return (EIO);
2514 			}
2515 			if (pmcs_remove_ihandlers(pwp, pwp->intr_cnt)) {
2516 				return (EIO);
2517 			}
2518 			if (pmcs_free_intrs(pwp, pwp->intr_cnt)) {
2519 				return (EIO);
2520 			}
2521 			pwp->intr_cnt = 0;
2522 			return (EAGAIN);
2523 		}
2524 	}
2525 
2526 	/*
2527 	 * Set up locks.
2528 	 */
2529 	if (ddi_intr_get_pri(pwp->ih_table[0], &pri) != DDI_SUCCESS) {
2530 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2531 		    "unable to get interrupt priority");
2532 		if (pmcs_disable_intrs(pwp, pwp->intr_cnt)) {
2533 			return (EIO);
2534 		}
2535 		if (pmcs_remove_ihandlers(pwp, pwp->intr_cnt)) {
2536 			return (EIO);
2537 		}
2538 		if (pmcs_free_intrs(pwp, pwp->intr_cnt)) {
2539 			return (EIO);
2540 		}
2541 		pwp->intr_cnt = 0;
2542 		return (EAGAIN);
2543 	}
2544 
2545 	pwp->locks_initted = 1;
2546 	pwp->intr_pri = pri;
2547 	mutex_init(&pwp->lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2548 	mutex_init(&pwp->dma_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2549 	mutex_init(&pwp->axil_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2550 	mutex_init(&pwp->cq_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2551 	mutex_init(&pwp->ict_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2552 	mutex_init(&pwp->config_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2553 	mutex_init(&pwp->wfree_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2554 	mutex_init(&pwp->pfree_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2555 	mutex_init(&pwp->dead_phylist_lock, NULL, MUTEX_DRIVER,
2556 	    DDI_INTR_PRI(pri));
2557 #ifdef	DEBUG
2558 	mutex_init(&pwp->dbglock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2559 #endif
2560 	cv_init(&pwp->ict_cv, NULL, CV_DRIVER, NULL);
2561 	cv_init(&pwp->drain_cv, NULL, CV_DRIVER, NULL);
2562 	cv_init(&pwp->config_cv, NULL, CV_DRIVER, NULL);
2563 	for (i = 0; i < PMCS_NIQ; i++) {
2564 		mutex_init(&pwp->iqp_lock[i], NULL,
2565 		    MUTEX_DRIVER, DDI_INTR_PRI(pwp->intr_pri));
2566 	}
2567 	for (i = 0; i < pwp->cq_info.cq_threads; i++) {
2568 		mutex_init(&pwp->cq_info.cq_thr_info[i].cq_thr_lock, NULL,
2569 		    MUTEX_DRIVER, DDI_INTR_PRI(pwp->intr_pri));
2570 		cv_init(&pwp->cq_info.cq_thr_info[i].cq_cv, NULL,
2571 		    CV_DRIVER, NULL);
2572 	}
2573 
2574 	pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, "%d %s interrup%s configured",
2575 	    pwp->intr_cnt, (pwp->int_type == PMCS_INT_MSIX)? "MSI-X" :
2576 	    ((pwp->int_type == PMCS_INT_MSI)? "MSI" : "INT-X"),
2577 	    pwp->intr_cnt == 1? "t" : "ts");
2578 
2579 
2580 	/*
2581 	 * Enable Interrupts
2582 	 */
2583 	if (pwp->intr_cnt > PMCS_NOQ) {
2584 		oqv_count = pwp->intr_cnt;
2585 	} else {
2586 		oqv_count = PMCS_NOQ;
2587 	}
2588 	for (pri = 0xffffffff, i = 0; i < oqv_count; i++) {
2589 		pri ^= (1 << i);
2590 	}
2591 
2592 	mutex_enter(&pwp->lock);
2593 	pwp->intr_mask = pri;
2594 	pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_MASK, pwp->intr_mask);
2595 	pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR, 0xffffffff);
2596 	mutex_exit(&pwp->lock);
2597 
2598 	return (0);
2599 }
2600 
2601 static int
2602 pmcs_teardown_intr(pmcs_hw_t *pwp)
2603 {
2604 	if (pwp->intr_cnt) {
2605 		if (pmcs_disable_intrs(pwp, pwp->intr_cnt)) {
2606 			return (EIO);
2607 		}
2608 		if (pmcs_remove_ihandlers(pwp, pwp->intr_cnt)) {
2609 			return (EIO);
2610 		}
2611 		if (pmcs_free_intrs(pwp, pwp->intr_cnt)) {
2612 			return (EIO);
2613 		}
2614 		pwp->intr_cnt = 0;
2615 	}
2616 	return (0);
2617 }
2618 
2619 static uint_t
2620 pmcs_general_ix(caddr_t arg1, caddr_t arg2)
2621 {
2622 	pmcs_hw_t *pwp = (pmcs_hw_t *)((void *)arg1);
2623 	_NOTE(ARGUNUSED(arg2));
2624 	pmcs_general_intr(pwp);
2625 	return (DDI_INTR_CLAIMED);
2626 }
2627 
2628 static uint_t
2629 pmcs_event_ix(caddr_t arg1, caddr_t arg2)
2630 {
2631 	pmcs_hw_t *pwp = (pmcs_hw_t *)((void *)arg1);
2632 	_NOTE(ARGUNUSED(arg2));
2633 	pmcs_event_intr(pwp);
2634 	return (DDI_INTR_CLAIMED);
2635 }
2636 
2637 static uint_t
2638 pmcs_iodone_ix(caddr_t arg1, caddr_t arg2)
2639 {
2640 	_NOTE(ARGUNUSED(arg2));
2641 	pmcs_hw_t *pwp = (pmcs_hw_t *)((void *)arg1);
2642 
2643 	/*
2644 	 * It's possible that if we just turned interrupt coalescing off
2645 	 * (and thus, re-enabled auto clear for interrupts on the I/O outbound
2646 	 * queue) that there was an interrupt already pending.  We use
2647 	 * io_intr_coal.int_cleared to ensure that we still drop in here and
2648 	 * clear the appropriate interrupt bit one last time.
2649 	 */
2650 	mutex_enter(&pwp->ict_lock);
2651 	if (pwp->io_intr_coal.timer_on ||
2652 	    (pwp->io_intr_coal.int_cleared == B_FALSE)) {
2653 		pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR,
2654 		    (1 << PMCS_OQ_IODONE));
2655 		pwp->io_intr_coal.int_cleared = B_TRUE;
2656 	}
2657 	mutex_exit(&pwp->ict_lock);
2658 
2659 	pmcs_iodone_intr(pwp);
2660 
2661 	return (DDI_INTR_CLAIMED);
2662 }
2663 
2664 static uint_t
2665 pmcs_fatal_ix(caddr_t arg1, caddr_t arg2)
2666 {
2667 	pmcs_hw_t *pwp = (pmcs_hw_t *)((void *)arg1);
2668 	_NOTE(ARGUNUSED(arg2));
2669 	pmcs_fatal_handler(pwp);
2670 	return (DDI_INTR_CLAIMED);
2671 }
2672 
2673 static uint_t
2674 pmcs_nonio_ix(caddr_t arg1, caddr_t arg2)
2675 {
2676 	_NOTE(ARGUNUSED(arg2));
2677 	pmcs_hw_t *pwp = (void *)arg1;
2678 	uint32_t obdb = pmcs_rd_msgunit(pwp, PMCS_MSGU_OBDB);
2679 
2680 	/*
2681 	 * Check for Fatal Interrupts
2682 	 */
2683 	if (obdb & (1 << PMCS_FATAL_INTERRUPT)) {
2684 		pmcs_fatal_handler(pwp);
2685 		return (DDI_INTR_CLAIMED);
2686 	}
2687 
2688 	if (obdb & (1 << PMCS_OQ_GENERAL)) {
2689 		pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR,
2690 		    (1 << PMCS_OQ_GENERAL));
2691 		pmcs_general_intr(pwp);
2692 		pmcs_event_intr(pwp);
2693 	}
2694 
2695 	return (DDI_INTR_CLAIMED);
2696 }
2697 
2698 static uint_t
2699 pmcs_all_intr(caddr_t arg1, caddr_t arg2)
2700 {
2701 	_NOTE(ARGUNUSED(arg2));
2702 	pmcs_hw_t *pwp = (void *) arg1;
2703 	uint32_t obdb;
2704 	int handled = 0;
2705 
2706 	obdb = pmcs_rd_msgunit(pwp, PMCS_MSGU_OBDB);
2707 
2708 	/*
2709 	 * Check for Fatal Interrupts
2710 	 */
2711 	if (obdb & (1 << PMCS_FATAL_INTERRUPT)) {
2712 		pmcs_fatal_handler(pwp);
2713 		return (DDI_INTR_CLAIMED);
2714 	}
2715 
2716 	/*
2717 	 * Check for Outbound Queue service needed
2718 	 */
2719 	if (obdb & (1 << PMCS_OQ_IODONE)) {
2720 		pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR,
2721 		    (1 << PMCS_OQ_IODONE));
2722 		obdb ^= (1 << PMCS_OQ_IODONE);
2723 		handled++;
2724 		pmcs_iodone_intr(pwp);
2725 	}
2726 	if (obdb & (1 << PMCS_OQ_GENERAL)) {
2727 		pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR,
2728 		    (1 << PMCS_OQ_GENERAL));
2729 		obdb ^= (1 << PMCS_OQ_GENERAL);
2730 		handled++;
2731 		pmcs_general_intr(pwp);
2732 	}
2733 	if (obdb & (1 << PMCS_OQ_EVENTS)) {
2734 		pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR,
2735 		    (1 << PMCS_OQ_EVENTS));
2736 		obdb ^= (1 << PMCS_OQ_EVENTS);
2737 		handled++;
2738 		pmcs_event_intr(pwp);
2739 	}
2740 	if (obdb) {
2741 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2742 		    "interrupt bits not handled (0x%x)", obdb);
2743 		pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR, obdb);
2744 		handled++;
2745 	}
2746 	if (pwp->int_type == PMCS_INT_MSI) {
2747 		handled++;
2748 	}
2749 	return (handled? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED);
2750 }
2751 
2752 void
2753 pmcs_fatal_handler(pmcs_hw_t *pwp)
2754 {
2755 	pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL, "Fatal Interrupt caught");
2756 
2757 	mutex_enter(&pwp->lock);
2758 	pwp->state = STATE_DEAD;
2759 
2760 	/*
2761 	 * Attempt a hot reset. In case of failure, pmcs_hot_reset() will
2762 	 * handle the failure and issue any required FM notifications.
2763 	 * See pmcs_subr.c for more details.
2764 	 */
2765 	if (pmcs_hot_reset(pwp)) {
2766 		pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL,
2767 		    "%s: hot reset failure", __func__);
2768 	} else {
2769 		pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL,
2770 		    "%s: hot reset complete", __func__);
2771 		pwp->last_reset_reason = PMCS_LAST_RST_FATAL_ERROR;
2772 	}
2773 	mutex_exit(&pwp->lock);
2774 }
2775 
2776 /*
2777  * Called with PHY lock and target statlock held and scratch acquired.
2778  */
2779 boolean_t
2780 pmcs_assign_device(pmcs_hw_t *pwp, pmcs_xscsi_t *tgt)
2781 {
2782 	pmcs_phy_t *pptr = tgt->phy;
2783 
2784 	switch (pptr->dtype) {
2785 	case SAS:
2786 	case EXPANDER:
2787 		break;
2788 	case SATA:
2789 		tgt->ca = 1;
2790 		break;
2791 	default:
2792 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, tgt,
2793 		    "%s: Target %p has PHY %p with invalid dtype",
2794 		    __func__, (void *)tgt, (void *)pptr);
2795 		return (B_FALSE);
2796 	}
2797 
2798 	tgt->new = 1;
2799 	tgt->dev_gone = 0;
2800 	tgt->recover_wait = 0;
2801 
2802 	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, tgt,
2803 	    "%s: config %s vtgt %u for " SAS_ADDR_FMT, __func__,
2804 	    pptr->path, tgt->target_num, SAS_ADDR_PRT(pptr->sas_address));
2805 
2806 	if (pmcs_add_new_device(pwp, tgt) != B_TRUE) {
2807 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, tgt,
2808 		    "%s: Failed for vtgt %u / WWN " SAS_ADDR_FMT, __func__,
2809 		    tgt->target_num, SAS_ADDR_PRT(pptr->sas_address));
2810 		mutex_destroy(&tgt->statlock);
2811 		mutex_destroy(&tgt->wqlock);
2812 		mutex_destroy(&tgt->aqlock);
2813 		return (B_FALSE);
2814 	}
2815 
2816 	return (B_TRUE);
2817 }
2818 
2819 /*
2820  * Called with softstate lock held
2821  */
2822 void
2823 pmcs_remove_device(pmcs_hw_t *pwp, pmcs_phy_t *pptr)
2824 {
2825 	pmcs_xscsi_t *xp;
2826 	unsigned int vtgt;
2827 
2828 	ASSERT(mutex_owned(&pwp->lock));
2829 
2830 	for (vtgt = 0; vtgt < pwp->max_dev; vtgt++) {
2831 		xp = pwp->targets[vtgt];
2832 		if (xp == NULL) {
2833 			continue;
2834 		}
2835 
2836 		mutex_enter(&xp->statlock);
2837 		if (xp->phy == pptr) {
2838 			if (xp->new) {
2839 				xp->new = 0;
2840 				pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, xp,
2841 				    "cancel config of vtgt %u", vtgt);
2842 			} else {
2843 				pmcs_clear_xp(pwp, xp);
2844 				pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, xp,
2845 				    "Removed tgt 0x%p vtgt %u",
2846 				    (void *)xp, vtgt);
2847 			}
2848 			mutex_exit(&xp->statlock);
2849 			break;
2850 		}
2851 		mutex_exit(&xp->statlock);
2852 	}
2853 }
2854 
2855 void
2856 pmcs_prt_impl(pmcs_hw_t *pwp, pmcs_prt_level_t level,
2857     pmcs_phy_t *phyp, pmcs_xscsi_t *target, const char *fmt, ...)
2858 {
2859 	va_list	ap;
2860 	int written = 0;
2861 	char *ptr;
2862 	uint32_t elem_size = PMCS_TBUF_ELEM_SIZE - 1;
2863 	boolean_t system_log;
2864 	int system_log_level;
2865 	hrtime_t hrtimestamp;
2866 
2867 	switch (level) {
2868 	case PMCS_PRT_DEBUG_DEVEL:
2869 	case PMCS_PRT_DEBUG_DEV_STATE:
2870 	case PMCS_PRT_DEBUG_PHY_LOCKING:
2871 	case PMCS_PRT_DEBUG_SCSI_STATUS:
2872 	case PMCS_PRT_DEBUG_UNDERFLOW:
2873 	case PMCS_PRT_DEBUG_CONFIG:
2874 	case PMCS_PRT_DEBUG_IPORT:
2875 	case PMCS_PRT_DEBUG_MAP:
2876 	case PMCS_PRT_DEBUG3:
2877 	case PMCS_PRT_DEBUG2:
2878 	case PMCS_PRT_DEBUG1:
2879 	case PMCS_PRT_DEBUG:
2880 		system_log = B_FALSE;
2881 		break;
2882 	case PMCS_PRT_INFO:
2883 		system_log = B_TRUE;
2884 		system_log_level = CE_CONT;
2885 		break;
2886 	case PMCS_PRT_WARN:
2887 		system_log = B_TRUE;
2888 		system_log_level = CE_NOTE;
2889 		break;
2890 	case PMCS_PRT_ERR:
2891 		system_log = B_TRUE;
2892 		system_log_level = CE_WARN;
2893 		break;
2894 	default:
2895 		return;
2896 	}
2897 
2898 	mutex_enter(&pmcs_trace_lock);
2899 	hrtimestamp = gethrtime();
2900 	gethrestime(&pmcs_tbuf_ptr->timestamp);
2901 
2902 	if (pwp->fw_timestamp != 0) {
2903 		/* Calculate the approximate firmware time stamp... */
2904 		pmcs_tbuf_ptr->fw_timestamp = pwp->fw_timestamp +
2905 		    ((hrtimestamp - pwp->hrtimestamp) / PMCS_FWLOG_TIMER_DIV);
2906 	} else {
2907 		pmcs_tbuf_ptr->fw_timestamp = 0;
2908 	}
2909 
2910 	ptr = pmcs_tbuf_ptr->buf;
2911 
2912 	/*
2913 	 * Store the pertinent PHY and target information if there is any
2914 	 */
2915 	if (target == NULL) {
2916 		pmcs_tbuf_ptr->target_num = PMCS_INVALID_TARGET_NUM;
2917 		pmcs_tbuf_ptr->target_ua[0] = '\0';
2918 	} else {
2919 		pmcs_tbuf_ptr->target_num = target->target_num;
2920 		(void) strncpy(pmcs_tbuf_ptr->target_ua, target->ua,
2921 		    PMCS_TBUF_UA_MAX_SIZE);
2922 	}
2923 
2924 	if (phyp == NULL) {
2925 		(void) memset(pmcs_tbuf_ptr->phy_sas_address, 0, 8);
2926 		pmcs_tbuf_ptr->phy_path[0] = '\0';
2927 		pmcs_tbuf_ptr->phy_dtype = NOTHING;
2928 	} else {
2929 		(void) memcpy(pmcs_tbuf_ptr->phy_sas_address,
2930 		    phyp->sas_address, 8);
2931 		(void) strncpy(pmcs_tbuf_ptr->phy_path, phyp->path, 32);
2932 		pmcs_tbuf_ptr->phy_dtype = phyp->dtype;
2933 	}
2934 
2935 	written += snprintf(ptr, elem_size, "pmcs%d:%d: ",
2936 	    ddi_get_instance(pwp->dip), level);
2937 	ptr += strlen(ptr);
2938 	va_start(ap, fmt);
2939 	written += vsnprintf(ptr, elem_size - written, fmt, ap);
2940 	va_end(ap);
2941 	if (written > elem_size - 1) {
2942 		/* Indicate truncation */
2943 		pmcs_tbuf_ptr->buf[elem_size - 1] = '+';
2944 	}
2945 	if (++pmcs_tbuf_idx == pmcs_tbuf_num_elems) {
2946 		pmcs_tbuf_ptr = pmcs_tbuf;
2947 		pmcs_tbuf_wrap = B_TRUE;
2948 		pmcs_tbuf_idx = 0;
2949 	} else {
2950 		++pmcs_tbuf_ptr;
2951 	}
2952 	mutex_exit(&pmcs_trace_lock);
2953 
2954 	/*
2955 	 * When pmcs_force_syslog in non-zero, everything goes also
2956 	 * to syslog, at CE_CONT level.
2957 	 */
2958 	if (pmcs_force_syslog) {
2959 		system_log = B_TRUE;
2960 		system_log_level = CE_CONT;
2961 	}
2962 
2963 	/*
2964 	 * Anything that comes in with PMCS_PRT_INFO, WARN, or ERR also
2965 	 * goes to syslog.
2966 	 */
2967 	if (system_log) {
2968 		char local[196];
2969 
2970 		switch (system_log_level) {
2971 		case CE_CONT:
2972 			(void) snprintf(local, sizeof (local), "%sINFO: ",
2973 			    pmcs_console ? "" : "?");
2974 			break;
2975 		case CE_NOTE:
2976 		case CE_WARN:
2977 			local[0] = '\0';
2978 			break;
2979 		default:
2980 			return;
2981 		}
2982 
2983 		ptr = local;
2984 		ptr += strlen(local);
2985 		(void) snprintf(ptr, (sizeof (local)) -
2986 		    ((size_t)ptr - (size_t)local), "pmcs%d: ",
2987 		    ddi_get_instance(pwp->dip));
2988 		ptr += strlen(ptr);
2989 		va_start(ap, fmt);
2990 		(void) vsnprintf(ptr,
2991 		    (sizeof (local)) - ((size_t)ptr - (size_t)local), fmt, ap);
2992 		va_end(ap);
2993 		if (level == CE_CONT) {
2994 			(void) strlcat(local, "\n", sizeof (local));
2995 		}
2996 		cmn_err(system_log_level, local);
2997 	}
2998 
2999 }
3000 
3001 /*
3002  * pmcs_acquire_scratch
3003  *
3004  * If "wait" is true, the caller will wait until it can acquire the scratch.
3005  * This implies the caller needs to be in a context where spinning for an
3006  * indeterminate amount of time is acceptable.
3007  */
3008 int
3009 pmcs_acquire_scratch(pmcs_hw_t *pwp, boolean_t wait)
3010 {
3011 	int rval;
3012 
3013 	if (!wait) {
3014 		return (atomic_swap_8(&pwp->scratch_locked, 1));
3015 	}
3016 
3017 	/*
3018 	 * Caller will wait for scratch.
3019 	 */
3020 	while ((rval = atomic_swap_8(&pwp->scratch_locked, 1)) != 0) {
3021 		drv_usecwait(100);
3022 	}
3023 
3024 	return (rval);
3025 }
3026 
3027 void
3028 pmcs_release_scratch(pmcs_hw_t *pwp)
3029 {
3030 	pwp->scratch_locked = 0;
3031 }
3032 
3033 /* Called with iport_lock and phy lock held */
3034 void
3035 pmcs_create_one_phy_stats(pmcs_iport_t *iport, pmcs_phy_t *phyp)
3036 {
3037 	sas_phy_stats_t		*ps;
3038 	pmcs_hw_t		*pwp;
3039 	int			ndata;
3040 	char			ks_name[KSTAT_STRLEN];
3041 
3042 	ASSERT(mutex_owned(&iport->lock));
3043 	pwp = iport->pwp;
3044 	ASSERT(pwp != NULL);
3045 	ASSERT(mutex_owned(&phyp->phy_lock));
3046 
3047 	if (phyp->phy_stats != NULL) {
3048 		/*
3049 		 * Delete existing kstats with name containing
3050 		 * old iport instance# and allow creation of
3051 		 * new kstats with new iport instance# in the name.
3052 		 */
3053 		kstat_delete(phyp->phy_stats);
3054 	}
3055 
3056 	ndata = (sizeof (sas_phy_stats_t)/sizeof (kstat_named_t));
3057 
3058 	(void) snprintf(ks_name, sizeof (ks_name),
3059 	    "%s.%llx.%d.%d", ddi_driver_name(iport->dip),
3060 	    (longlong_t)pwp->sas_wwns[0],
3061 	    ddi_get_instance(iport->dip), phyp->phynum);
3062 
3063 	phyp->phy_stats = kstat_create("pmcs",
3064 	    ddi_get_instance(iport->dip), ks_name, KSTAT_SAS_PHY_CLASS,
3065 	    KSTAT_TYPE_NAMED, ndata, 0);
3066 
3067 	if (phyp->phy_stats == NULL) {
3068 		pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, NULL,
3069 		    "%s: Failed to create %s kstats for PHY(0x%p) at %s",
3070 		    __func__, ks_name, (void *)phyp, phyp->path);
3071 		return;
3072 	}
3073 
3074 	ps = (sas_phy_stats_t *)phyp->phy_stats->ks_data;
3075 
3076 	kstat_named_init(&ps->seconds_since_last_reset,
3077 	    "SecondsSinceLastReset", KSTAT_DATA_ULONGLONG);
3078 	kstat_named_init(&ps->tx_frames,
3079 	    "TxFrames", KSTAT_DATA_ULONGLONG);
3080 	kstat_named_init(&ps->rx_frames,
3081 	    "RxFrames", KSTAT_DATA_ULONGLONG);
3082 	kstat_named_init(&ps->tx_words,
3083 	    "TxWords", KSTAT_DATA_ULONGLONG);
3084 	kstat_named_init(&ps->rx_words,
3085 	    "RxWords", KSTAT_DATA_ULONGLONG);
3086 	kstat_named_init(&ps->invalid_dword_count,
3087 	    "InvalidDwordCount", KSTAT_DATA_ULONGLONG);
3088 	kstat_named_init(&ps->running_disparity_error_count,
3089 	    "RunningDisparityErrorCount", KSTAT_DATA_ULONGLONG);
3090 	kstat_named_init(&ps->loss_of_dword_sync_count,
3091 	    "LossofDwordSyncCount", KSTAT_DATA_ULONGLONG);
3092 	kstat_named_init(&ps->phy_reset_problem_count,
3093 	    "PhyResetProblemCount", KSTAT_DATA_ULONGLONG);
3094 
3095 	phyp->phy_stats->ks_private = phyp;
3096 	phyp->phy_stats->ks_update = pmcs_update_phy_stats;
3097 	kstat_install(phyp->phy_stats);
3098 }
3099 
3100 static void
3101 pmcs_create_all_phy_stats(pmcs_iport_t *iport)
3102 {
3103 	pmcs_hw_t		*pwp;
3104 	pmcs_phy_t		*phyp;
3105 
3106 	ASSERT(iport != NULL);
3107 	pwp = iport->pwp;
3108 	ASSERT(pwp != NULL);
3109 
3110 	mutex_enter(&iport->lock);
3111 
3112 	for (phyp = list_head(&iport->phys);
3113 	    phyp != NULL;
3114 	    phyp = list_next(&iport->phys, phyp)) {
3115 
3116 		mutex_enter(&phyp->phy_lock);
3117 		pmcs_create_one_phy_stats(iport, phyp);
3118 		mutex_exit(&phyp->phy_lock);
3119 	}
3120 
3121 	mutex_exit(&iport->lock);
3122 }
3123 
3124 int
3125 pmcs_update_phy_stats(kstat_t *ks, int rw)
3126 {
3127 	int		val, ret = DDI_FAILURE;
3128 	pmcs_phy_t	*pptr = (pmcs_phy_t *)ks->ks_private;
3129 	pmcs_hw_t	*pwp = pptr->pwp;
3130 	sas_phy_stats_t	*ps = ks->ks_data;
3131 
3132 	_NOTE(ARGUNUSED(rw));
3133 	ASSERT((pptr != NULL) && (pwp != NULL));
3134 
3135 	/*
3136 	 * We just want to lock against other invocations of kstat;
3137 	 * we don't need to pmcs_lock_phy() for this.
3138 	 */
3139 	mutex_enter(&pptr->phy_lock);
3140 
3141 	/* Get Stats from Chip */
3142 	val = pmcs_get_diag_report(pwp, PMCS_INVALID_DWORD_CNT, pptr->phynum);
3143 	if (val == DDI_FAILURE)
3144 		goto fail;
3145 	ps->invalid_dword_count.value.ull = (unsigned long long)val;
3146 
3147 	val = pmcs_get_diag_report(pwp, PMCS_DISPARITY_ERR_CNT, pptr->phynum);
3148 	if (val == DDI_FAILURE)
3149 		goto fail;
3150 	ps->running_disparity_error_count.value.ull = (unsigned long long)val;
3151 
3152 	val = pmcs_get_diag_report(pwp, PMCS_LOST_DWORD_SYNC_CNT, pptr->phynum);
3153 	if (val == DDI_FAILURE)
3154 		goto fail;
3155 	ps->loss_of_dword_sync_count.value.ull = (unsigned long long)val;
3156 
3157 	val = pmcs_get_diag_report(pwp, PMCS_RESET_FAILED_CNT, pptr->phynum);
3158 	if (val == DDI_FAILURE)
3159 		goto fail;
3160 	ps->phy_reset_problem_count.value.ull = (unsigned long long)val;
3161 
3162 	ret = DDI_SUCCESS;
3163 fail:
3164 	mutex_exit(&pptr->phy_lock);
3165 	return (ret);
3166 }
3167 
3168 /*ARGSUSED*/
3169 static int
3170 pmcs_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
3171 {
3172 	/*
3173 	 * as the driver can always deal with an error in any dma or
3174 	 * access handle, we can just return the fme_status value.
3175 	 */
3176 	pci_ereport_post(dip, err, NULL);
3177 	return (err->fme_status);
3178 }
3179 
3180 static void
3181 pmcs_fm_init(pmcs_hw_t *pwp)
3182 {
3183 	ddi_iblock_cookie_t	fm_ibc;
3184 
3185 	/* Only register with IO Fault Services if we have some capability */
3186 	if (pwp->fm_capabilities) {
3187 		/* Adjust access and dma attributes for FMA */
3188 		pwp->reg_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
3189 		pwp->iqp_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3190 		pwp->oqp_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3191 		pwp->cip_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3192 		pwp->fwlog_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3193 
3194 		/*
3195 		 * Register capabilities with IO Fault Services.
3196 		 */
3197 		ddi_fm_init(pwp->dip, &pwp->fm_capabilities, &fm_ibc);
3198 
3199 		/*
3200 		 * Initialize pci ereport capabilities if ereport
3201 		 * capable (should always be.)
3202 		 */
3203 		if (DDI_FM_EREPORT_CAP(pwp->fm_capabilities) ||
3204 		    DDI_FM_ERRCB_CAP(pwp->fm_capabilities)) {
3205 			pci_ereport_setup(pwp->dip);
3206 		}
3207 
3208 		/*
3209 		 * Register error callback if error callback capable.
3210 		 */
3211 		if (DDI_FM_ERRCB_CAP(pwp->fm_capabilities)) {
3212 			ddi_fm_handler_register(pwp->dip,
3213 			    pmcs_fm_error_cb, (void *) pwp);
3214 		}
3215 	}
3216 }
3217 
3218 static void
3219 pmcs_fm_fini(pmcs_hw_t *pwp)
3220 {
3221 	/* Only unregister FMA capabilities if registered */
3222 	if (pwp->fm_capabilities) {
3223 		/*
3224 		 * Un-register error callback if error callback capable.
3225 		 */
3226 		if (DDI_FM_ERRCB_CAP(pwp->fm_capabilities)) {
3227 			ddi_fm_handler_unregister(pwp->dip);
3228 		}
3229 
3230 		/*
3231 		 * Release any resources allocated by pci_ereport_setup()
3232 		 */
3233 		if (DDI_FM_EREPORT_CAP(pwp->fm_capabilities) ||
3234 		    DDI_FM_ERRCB_CAP(pwp->fm_capabilities)) {
3235 			pci_ereport_teardown(pwp->dip);
3236 		}
3237 
3238 		/* Unregister from IO Fault Services */
3239 		ddi_fm_fini(pwp->dip);
3240 
3241 		/* Adjust access and dma attributes for FMA */
3242 		pwp->reg_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC;
3243 		pwp->iqp_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3244 		pwp->oqp_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3245 		pwp->cip_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3246 		pwp->fwlog_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3247 	}
3248 }
3249 
3250 static boolean_t
3251 pmcs_fabricate_wwid(pmcs_hw_t *pwp)
3252 {
3253 	char *cp, c;
3254 	uint64_t adr;
3255 	int i;
3256 
3257 	cp = &c;
3258 	(void) ddi_strtoul(hw_serial, &cp, 10, (unsigned long *)&adr);
3259 
3260 	if (adr == 0) {
3261 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
3262 		    "%s: No serial number available to fabricate WWN",
3263 		    __func__);
3264 
3265 		adr = (uint64_t)gethrtime();
3266 	}
3267 
3268 	adr <<= 8;
3269 	adr |= ((uint64_t)ddi_get_instance(pwp->dip) << 52);
3270 	adr |= (5ULL << 60);
3271 
3272 	for (i = 0; i < PMCS_MAX_PORTS; i++) {
3273 		pwp->sas_wwns[i] = adr + i;
3274 	}
3275 
3276 	return (B_TRUE);
3277 }
3278