1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24#include <sys/scsi/adapters/pmcs/pmcs.h>
25
26#define	PMCS_DRIVER_VERSION	"pmcs HBA device driver"
27
28static	char	*pmcs_driver_rev = PMCS_DRIVER_VERSION;
29
30/*
31 * Non-DDI Compliant stuff
32 */
33extern char hw_serial[];
34
35/*
36 * Global driver data
37 */
38void *pmcs_softc_state = NULL;
39void *pmcs_iport_softstate = NULL;
40
41/*
42 * Tracing and Logging info
43 */
44pmcs_tbuf_t *pmcs_tbuf = NULL;
45uint32_t pmcs_tbuf_num_elems = 0;
46pmcs_tbuf_t *pmcs_tbuf_ptr;
47uint32_t pmcs_tbuf_idx = 0;
48boolean_t pmcs_tbuf_wrap = B_FALSE;
49kmutex_t pmcs_trace_lock;
50
51/*
52 * If pmcs_force_syslog value is non-zero, all messages put in the trace log
53 * will also be sent to system log.
54 */
55int pmcs_force_syslog = 0;
56int pmcs_console = 0;
57
58/*
59 * External References
60 */
61extern int ncpus_online;
62
63/*
64 * Local static data
65 */
66static int fwlog_level = 3;
67static int physpeed = PHY_LINK_ALL;
68static int phymode = PHY_LM_AUTO;
69static int block_mask = 0;
70static int phymap_stable_usec = 3 * MICROSEC;
71static int iportmap_stable_usec = 2 * MICROSEC;
72static int iportmap_csync_usec = 20 * MICROSEC;
73
74#ifdef DEBUG
75static int debug_mask = 1;
76#else
77static int debug_mask = 0;
78#endif
79
80#ifdef DISABLE_MSIX
81static int disable_msix = 1;
82#else
83static int disable_msix = 0;
84#endif
85
86#ifdef DISABLE_MSI
87static int disable_msi = 1;
88#else
89static int disable_msi = 0;
90#endif
91
92/*
93 * DEBUG: testing: allow detach with an active port:
94 *
95 * # echo 'detach_driver_unconfig/W 10'		| mdb -kw
96 * # echo 'scsi_hba_bus_unconfig_remove/W 1'	| mdb -kw
97 * # echo 'pmcs`detach_with_active_port/W 1'	| mdb -kw
98 * # modunload -i <pmcs_driver_index>
99 */
100static int detach_with_active_port = 0;
101
102static uint16_t maxqdepth = 0xfffe;
103
104/*
105 * Local prototypes
106 */
107static int pmcs_attach(dev_info_t *, ddi_attach_cmd_t);
108static int pmcs_detach(dev_info_t *, ddi_detach_cmd_t);
109static int pmcs_unattach(pmcs_hw_t *);
110static int pmcs_iport_unattach(pmcs_iport_t *);
111static int pmcs_add_more_chunks(pmcs_hw_t *, unsigned long);
112static void pmcs_watchdog(void *);
113static int pmcs_setup_intr(pmcs_hw_t *);
114static int pmcs_teardown_intr(pmcs_hw_t *);
115
116static uint_t pmcs_nonio_ix(caddr_t, caddr_t);
117static uint_t pmcs_general_ix(caddr_t, caddr_t);
118static uint_t pmcs_event_ix(caddr_t, caddr_t);
119static uint_t pmcs_iodone_ix(caddr_t, caddr_t);
120static uint_t pmcs_fatal_ix(caddr_t, caddr_t);
121static uint_t pmcs_all_intr(caddr_t, caddr_t);
122static int pmcs_quiesce(dev_info_t *dip);
123static boolean_t pmcs_fabricate_wwid(pmcs_hw_t *);
124
125static void pmcs_create_all_phy_stats(pmcs_iport_t *);
126int pmcs_update_phy_stats(kstat_t *, int);
127
128static void pmcs_fm_fini(pmcs_hw_t *pwp);
129static void pmcs_fm_init(pmcs_hw_t *pwp);
130static int pmcs_fm_error_cb(dev_info_t *dip,
131    ddi_fm_error_t *err, const void *impl_data);
132
133/*
134 * Local configuration data
135 */
136static struct dev_ops pmcs_ops = {
137	DEVO_REV,		/* devo_rev, */
138	0,			/* refcnt */
139	ddi_no_info,		/* info */
140	nulldev,		/* identify */
141	nulldev,		/* probe */
142	pmcs_attach,		/* attach */
143	pmcs_detach,		/* detach */
144	nodev,			/* reset */
145	NULL,			/* driver operations */
146	NULL,			/* bus operations */
147	ddi_power,		/* power management */
148	pmcs_quiesce		/* quiesce */
149};
150
151static struct modldrv modldrv = {
152	&mod_driverops,
153	PMCS_DRIVER_VERSION,
154	&pmcs_ops,	/* driver ops */
155};
156static struct modlinkage modlinkage = {
157	MODREV_1, &modldrv, NULL
158};
159
160const ddi_dma_attr_t pmcs_dattr = {
161	DMA_ATTR_V0,			/* dma_attr version	*/
162	0x0000000000000000ull,		/* dma_attr_addr_lo	*/
163	0xFFFFFFFFFFFFFFFFull,		/* dma_attr_addr_hi	*/
164	0x00000000FFFFFFFFull,		/* dma_attr_count_max	*/
165	0x0000000000000001ull,		/* dma_attr_align	*/
166	0x00000078,			/* dma_attr_burstsizes	*/
167	0x00000001,			/* dma_attr_minxfer	*/
168	0x00000000FFFFFFFFull,		/* dma_attr_maxxfer	*/
169	0x00000000FFFFFFFFull,		/* dma_attr_seg		*/
170	1,				/* dma_attr_sgllen	*/
171	512,				/* dma_attr_granular	*/
172	0				/* dma_attr_flags	*/
173};
174
175static ddi_device_acc_attr_t rattr = {
176	DDI_DEVICE_ATTR_V1,
177	DDI_STRUCTURE_LE_ACC,
178	DDI_STRICTORDER_ACC,
179	DDI_DEFAULT_ACC
180};
181
182
183/*
184 * Attach/Detach functions
185 */
186
187int
188_init(void)
189{
190	int ret;
191
192	ret = ddi_soft_state_init(&pmcs_softc_state, sizeof (pmcs_hw_t), 1);
193	if (ret != 0) {
194		cmn_err(CE_WARN, "?soft state init failed for pmcs");
195		return (ret);
196	}
197
198	if ((ret = scsi_hba_init(&modlinkage)) != 0) {
199		cmn_err(CE_WARN, "?scsi_hba_init failed for pmcs");
200		ddi_soft_state_fini(&pmcs_softc_state);
201		return (ret);
202	}
203
204	/*
205	 * Allocate soft state for iports
206	 */
207	ret = ddi_soft_state_init(&pmcs_iport_softstate,
208	    sizeof (pmcs_iport_t), 2);
209	if (ret != 0) {
210		cmn_err(CE_WARN, "?iport soft state init failed for pmcs");
211		ddi_soft_state_fini(&pmcs_softc_state);
212		return (ret);
213	}
214
215	ret = mod_install(&modlinkage);
216	if (ret != 0) {
217		cmn_err(CE_WARN, "?mod_install failed for pmcs (%d)", ret);
218		scsi_hba_fini(&modlinkage);
219		ddi_soft_state_fini(&pmcs_iport_softstate);
220		ddi_soft_state_fini(&pmcs_softc_state);
221		return (ret);
222	}
223
224	/* Initialize the global trace lock */
225	mutex_init(&pmcs_trace_lock, NULL, MUTEX_DRIVER, NULL);
226
227	return (0);
228}
229
230int
231_fini(void)
232{
233	int ret;
234	if ((ret = mod_remove(&modlinkage)) != 0) {
235		return (ret);
236	}
237	scsi_hba_fini(&modlinkage);
238
239	/* Free pmcs log buffer and destroy the global lock */
240	if (pmcs_tbuf) {
241		kmem_free(pmcs_tbuf,
242		    pmcs_tbuf_num_elems * sizeof (pmcs_tbuf_t));
243		pmcs_tbuf = NULL;
244	}
245	mutex_destroy(&pmcs_trace_lock);
246
247	ddi_soft_state_fini(&pmcs_iport_softstate);
248	ddi_soft_state_fini(&pmcs_softc_state);
249	return (0);
250}
251
252int
253_info(struct modinfo *modinfop)
254{
255	return (mod_info(&modlinkage, modinfop));
256}
257
258static int
259pmcs_iport_attach(dev_info_t *dip)
260{
261	pmcs_iport_t		*iport;
262	pmcs_hw_t		*pwp;
263	scsi_hba_tran_t		*tran;
264	void			*ua_priv = NULL;
265	char			*iport_ua;
266	char			*init_port;
267	int			hba_inst;
268	int			inst;
269
270	hba_inst = ddi_get_instance(ddi_get_parent(dip));
271	inst = ddi_get_instance(dip);
272
273	pwp = ddi_get_soft_state(pmcs_softc_state, hba_inst);
274	if (pwp == NULL) {
275		cmn_err(CE_WARN, "%s: No HBA softstate for instance %d",
276		    __func__, inst);
277		return (DDI_FAILURE);
278	}
279
280	if ((pwp->state == STATE_UNPROBING) || (pwp->state == STATE_DEAD)) {
281		return (DDI_FAILURE);
282	}
283
284	if ((iport_ua = scsi_hba_iport_unit_address(dip)) == NULL) {
285		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
286		    "%s: invoked with NULL unit address, inst (%d)",
287		    __func__, inst);
288		return (DDI_FAILURE);
289	}
290
291	if (ddi_soft_state_zalloc(pmcs_iport_softstate, inst) != DDI_SUCCESS) {
292		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
293		    "Failed to alloc soft state for iport %d", inst);
294		return (DDI_FAILURE);
295	}
296
297	iport = ddi_get_soft_state(pmcs_iport_softstate, inst);
298	if (iport == NULL) {
299		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
300		    "cannot get iport soft state");
301		goto iport_attach_fail1;
302	}
303
304	mutex_init(&iport->lock, NULL, MUTEX_DRIVER,
305	    DDI_INTR_PRI(pwp->intr_pri));
306	cv_init(&iport->refcnt_cv, NULL, CV_DEFAULT, NULL);
307	cv_init(&iport->smp_cv, NULL, CV_DEFAULT, NULL);
308	mutex_init(&iport->refcnt_lock, NULL, MUTEX_DRIVER,
309	    DDI_INTR_PRI(pwp->intr_pri));
310	mutex_init(&iport->smp_lock, NULL, MUTEX_DRIVER,
311	    DDI_INTR_PRI(pwp->intr_pri));
312
313	/* Set some data on the iport handle */
314	iport->dip = dip;
315	iport->pwp = pwp;
316
317	/* Dup the UA into the iport handle */
318	iport->ua = strdup(iport_ua);
319
320	tran = (scsi_hba_tran_t *)ddi_get_driver_private(dip);
321	tran->tran_hba_private = iport;
322
323	list_create(&iport->phys, sizeof (pmcs_phy_t),
324	    offsetof(pmcs_phy_t, list_node));
325
326	/*
327	 * If our unit address is active in the phymap, configure our
328	 * iport's phylist.
329	 */
330	mutex_enter(&iport->lock);
331	ua_priv = sas_phymap_lookup_uapriv(pwp->hss_phymap, iport->ua);
332	if (ua_priv) {
333		/* Non-NULL private data indicates the unit address is active */
334		iport->ua_state = UA_ACTIVE;
335		if (pmcs_iport_configure_phys(iport) != DDI_SUCCESS) {
336			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
337			    "%s: failed to "
338			    "configure phys on iport handle (0x%p), "
339			    " unit address [%s]", __func__,
340			    (void *)iport, iport_ua);
341			mutex_exit(&iport->lock);
342			goto iport_attach_fail2;
343		}
344	} else {
345		iport->ua_state = UA_INACTIVE;
346	}
347	mutex_exit(&iport->lock);
348
349	/* Allocate string-based soft state pool for targets */
350	iport->tgt_sstate = NULL;
351	if (ddi_soft_state_bystr_init(&iport->tgt_sstate,
352	    sizeof (pmcs_xscsi_t), PMCS_TGT_SSTATE_SZ) != 0) {
353		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
354		    "cannot get iport tgt soft state");
355		goto iport_attach_fail2;
356	}
357
358	/* Create this iport's target map */
359	if (pmcs_iport_tgtmap_create(iport) == B_FALSE) {
360		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
361		    "Failed to create tgtmap on iport %d", inst);
362		goto iport_attach_fail3;
363	}
364
365	/* Set up the 'initiator-port' DDI property on this iport */
366	init_port = kmem_zalloc(PMCS_MAX_UA_SIZE, KM_SLEEP);
367	if (pwp->separate_ports) {
368		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
369		    "%s: separate ports not supported", __func__);
370	} else {
371		/* Set initiator-port value to the HBA's base WWN */
372		(void) scsi_wwn_to_wwnstr(pwp->sas_wwns[0], 1,
373		    init_port);
374	}
375
376	mutex_enter(&iport->lock);
377	pmcs_smhba_add_iport_prop(iport, DATA_TYPE_STRING,
378	    SCSI_ADDR_PROP_INITIATOR_PORT, init_port);
379	kmem_free(init_port, PMCS_MAX_UA_SIZE);
380
381	/* Set up a 'num-phys' DDI property for the iport node */
382	pmcs_smhba_add_iport_prop(iport, DATA_TYPE_INT32, PMCS_NUM_PHYS,
383	    &iport->nphy);
384	mutex_exit(&iport->lock);
385
386	/* Create kstats for each of the phys in this port */
387	pmcs_create_all_phy_stats(iport);
388
389	/*
390	 * Insert this iport handle into our list and set
391	 * iports_attached on the HBA node.
392	 */
393	rw_enter(&pwp->iports_lock, RW_WRITER);
394	ASSERT(!list_link_active(&iport->list_node));
395	list_insert_tail(&pwp->iports, iport);
396	pwp->iports_attached = 1;
397	pwp->num_iports++;
398	rw_exit(&pwp->iports_lock);
399
400	pmcs_prt(pwp, PMCS_PRT_DEBUG_IPORT, NULL, NULL,
401	    "iport%d attached", inst);
402	ddi_report_dev(dip);
403	return (DDI_SUCCESS);
404
405	/* teardown and fail */
406iport_attach_fail3:
407	ddi_soft_state_bystr_fini(&iport->tgt_sstate);
408iport_attach_fail2:
409	list_destroy(&iport->phys);
410	strfree(iport->ua);
411	mutex_destroy(&iport->refcnt_lock);
412	mutex_destroy(&iport->smp_lock);
413	cv_destroy(&iport->refcnt_cv);
414	cv_destroy(&iport->smp_cv);
415	mutex_destroy(&iport->lock);
416iport_attach_fail1:
417	ddi_soft_state_free(pmcs_iport_softstate, inst);
418	return (DDI_FAILURE);
419}
420
421static int
422pmcs_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
423{
424	scsi_hba_tran_t *tran;
425	char chiprev, *fwsupport, hw_rev[24], fw_rev[24];
426	off_t set3size;
427	int inst, i;
428	int sm_hba = 1;
429	int protocol = 0;
430	int num_phys = 0;
431	pmcs_hw_t *pwp;
432	pmcs_phy_t *phyp;
433	uint32_t num_threads;
434	char buf[64];
435	char *fwl_file;
436
437	switch (cmd) {
438	case DDI_ATTACH:
439		break;
440
441	case DDI_PM_RESUME:
442	case DDI_RESUME:
443		tran = (scsi_hba_tran_t *)ddi_get_driver_private(dip);
444		if (!tran) {
445			return (DDI_FAILURE);
446		}
447		/* No DDI_?_RESUME on iport nodes */
448		if (scsi_hba_iport_unit_address(dip) != NULL) {
449			return (DDI_SUCCESS);
450		}
451		pwp = TRAN2PMC(tran);
452		if (pwp == NULL) {
453			return (DDI_FAILURE);
454		}
455
456		mutex_enter(&pwp->lock);
457		pwp->suspended = 0;
458		if (pwp->tq) {
459			ddi_taskq_resume(pwp->tq);
460		}
461		mutex_exit(&pwp->lock);
462		return (DDI_SUCCESS);
463
464	default:
465		return (DDI_FAILURE);
466	}
467
468	/*
469	 * If this is an iport node, invoke iport attach.
470	 */
471	if (scsi_hba_iport_unit_address(dip) != NULL) {
472		return (pmcs_iport_attach(dip));
473	}
474
475	/*
476	 * From here on is attach for the HBA node
477	 */
478
479#ifdef	DEBUG
480	/*
481	 * Check to see if this unit is to be disabled.  We can't disable
482	 * on a per-iport node.  It's either the entire HBA or nothing.
483	 */
484	(void) snprintf(buf, sizeof (buf),
485	    "disable-instance-%d", ddi_get_instance(dip));
486	if (ddi_prop_get_int(DDI_DEV_T_ANY, dip,
487	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, buf, 0)) {
488		cmn_err(CE_NOTE, "pmcs%d: disabled by configuration",
489		    ddi_get_instance(dip));
490		return (DDI_FAILURE);
491	}
492#endif
493
494	/*
495	 * Allocate softstate
496	 */
497	inst = ddi_get_instance(dip);
498	if (ddi_soft_state_zalloc(pmcs_softc_state, inst) != DDI_SUCCESS) {
499		cmn_err(CE_WARN, "pmcs%d: Failed to alloc soft state", inst);
500		return (DDI_FAILURE);
501	}
502
503	pwp = ddi_get_soft_state(pmcs_softc_state, inst);
504	if (pwp == NULL) {
505		cmn_err(CE_WARN, "pmcs%d: cannot get soft state", inst);
506		ddi_soft_state_free(pmcs_softc_state, inst);
507		return (DDI_FAILURE);
508	}
509	pwp->dip = dip;
510	STAILQ_INIT(&pwp->dq);
511	STAILQ_INIT(&pwp->cq);
512	STAILQ_INIT(&pwp->wf);
513	STAILQ_INIT(&pwp->pf);
514
515	/*
516	 * Create the list for iports and init its lock.
517	 */
518	list_create(&pwp->iports, sizeof (pmcs_iport_t),
519	    offsetof(pmcs_iport_t, list_node));
520	rw_init(&pwp->iports_lock, NULL, RW_DRIVER, NULL);
521
522	pwp->state = STATE_PROBING;
523
524	/*
525	 * Get driver.conf properties
526	 */
527	pwp->debug_mask = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
528	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-debug-mask",
529	    debug_mask);
530	pwp->phyid_block_mask = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
531	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-phyid-block-mask",
532	    block_mask);
533	pwp->physpeed = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
534	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-physpeed", physpeed);
535	pwp->phymode = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
536	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-phymode", phymode);
537	pwp->fwlog = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
538	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-fwlog", fwlog_level);
539	if (pwp->fwlog > PMCS_FWLOG_MAX) {
540		pwp->fwlog = PMCS_FWLOG_MAX;
541	}
542	if ((ddi_prop_lookup_string(DDI_DEV_T_ANY, dip, 0, "pmcs-fwlogfile",
543	    &fwl_file) == DDI_SUCCESS)) {
544		if (snprintf(pwp->fwlogfile_aap1, MAXPATHLEN, "%s%d-aap1.0",
545		    fwl_file, ddi_get_instance(dip)) > MAXPATHLEN) {
546			pwp->fwlogfile_aap1[0] = '\0';
547			pwp->fwlogfile_iop[0] = '\0';
548		} else if (snprintf(pwp->fwlogfile_iop, MAXPATHLEN,
549		    "%s%d-iop.0", fwl_file,
550		    ddi_get_instance(dip)) > MAXPATHLEN) {
551			pwp->fwlogfile_aap1[0] = '\0';
552			pwp->fwlogfile_iop[0] = '\0';
553		}
554		ddi_prop_free(fwl_file);
555	} else {
556		pwp->fwlogfile_aap1[0] = '\0';
557		pwp->fwlogfile_iop[0] = '\0';
558	}
559
560	pwp->open_retry_interval = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
561	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-open-retry-interval",
562	    OPEN_RETRY_INTERVAL_DEF);
563	if (pwp->open_retry_interval > OPEN_RETRY_INTERVAL_MAX) {
564		pwp->open_retry_interval = OPEN_RETRY_INTERVAL_MAX;
565	}
566
567	mutex_enter(&pmcs_trace_lock);
568	if (pmcs_tbuf == NULL) {
569		/* Allocate trace buffer */
570		pmcs_tbuf_num_elems = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
571		    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-tbuf-num-elems",
572		    PMCS_TBUF_NUM_ELEMS_DEF);
573		if ((pmcs_tbuf_num_elems == DDI_PROP_NOT_FOUND) ||
574		    (pmcs_tbuf_num_elems == 0)) {
575			pmcs_tbuf_num_elems = PMCS_TBUF_NUM_ELEMS_DEF;
576		}
577
578		pmcs_tbuf = kmem_zalloc(pmcs_tbuf_num_elems *
579		    sizeof (pmcs_tbuf_t), KM_SLEEP);
580		pmcs_tbuf_ptr = pmcs_tbuf;
581		pmcs_tbuf_idx = 0;
582	}
583	mutex_exit(&pmcs_trace_lock);
584
585	if (pwp->fwlog && strlen(pwp->fwlogfile_aap1) > 0) {
586		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
587		    "%s: firmware event log files: %s, %s", __func__,
588		    pwp->fwlogfile_aap1, pwp->fwlogfile_iop);
589		pwp->fwlog_file = 1;
590	} else {
591		if (pwp->fwlog == 0) {
592			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
593			    "%s: No firmware event log will be written "
594			    "(event log disabled)", __func__);
595		} else {
596			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
597			    "%s: No firmware event log will be written "
598			    "(no filename configured - too long?)", __func__);
599		}
600		pwp->fwlog_file = 0;
601	}
602
603	disable_msix = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
604	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-disable-msix",
605	    disable_msix);
606	disable_msi = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
607	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-disable-msi",
608	    disable_msi);
609	maxqdepth = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
610	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-maxqdepth", maxqdepth);
611	pwp->fw_force_update = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
612	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-fw-force-update", 0);
613	if (pwp->fw_force_update == 0) {
614		pwp->fw_disable_update = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
615		    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM,
616		    "pmcs-fw-disable-update", 0);
617	}
618	pwp->ioq_depth = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
619	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-num-io-qentries",
620	    PMCS_NQENTRY);
621
622	/*
623	 * Initialize FMA
624	 */
625	pwp->dev_acc_attr = pwp->reg_acc_attr = rattr;
626	pwp->iqp_dma_attr = pwp->oqp_dma_attr =
627	    pwp->regdump_dma_attr = pwp->cip_dma_attr =
628	    pwp->fwlog_dma_attr = pmcs_dattr;
629	pwp->fm_capabilities = ddi_getprop(DDI_DEV_T_ANY, pwp->dip,
630	    DDI_PROP_NOTPROM | DDI_PROP_DONTPASS, "fm-capable",
631	    DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
632	    DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
633	pmcs_fm_init(pwp);
634
635	/*
636	 * Map registers
637	 */
638	if (pci_config_setup(dip, &pwp->pci_acc_handle)) {
639		pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL,
640		    "pci config setup failed");
641		ddi_soft_state_free(pmcs_softc_state, inst);
642		return (DDI_FAILURE);
643	}
644
645	/*
646	 * Get the size of register set 3.
647	 */
648	if (ddi_dev_regsize(dip, PMCS_REGSET_3, &set3size) != DDI_SUCCESS) {
649		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
650		    "unable to get size of register set %d", PMCS_REGSET_3);
651		pci_config_teardown(&pwp->pci_acc_handle);
652		ddi_soft_state_free(pmcs_softc_state, inst);
653		return (DDI_FAILURE);
654	}
655
656	/*
657	 * Map registers
658	 */
659	pwp->reg_acc_attr.devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC;
660
661	if (ddi_regs_map_setup(dip, PMCS_REGSET_0, (caddr_t *)&pwp->msg_regs,
662	    0, 0, &pwp->reg_acc_attr, &pwp->msg_acc_handle)) {
663		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
664		    "failed to map Message Unit registers");
665		pci_config_teardown(&pwp->pci_acc_handle);
666		ddi_soft_state_free(pmcs_softc_state, inst);
667		return (DDI_FAILURE);
668	}
669
670	if (ddi_regs_map_setup(dip, PMCS_REGSET_1, (caddr_t *)&pwp->top_regs,
671	    0, 0, &pwp->reg_acc_attr, &pwp->top_acc_handle)) {
672		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
673		    "failed to map TOP registers");
674		ddi_regs_map_free(&pwp->msg_acc_handle);
675		pci_config_teardown(&pwp->pci_acc_handle);
676		ddi_soft_state_free(pmcs_softc_state, inst);
677		return (DDI_FAILURE);
678	}
679
680	if (ddi_regs_map_setup(dip, PMCS_REGSET_2, (caddr_t *)&pwp->gsm_regs,
681	    0, 0, &pwp->reg_acc_attr, &pwp->gsm_acc_handle)) {
682		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
683		    "failed to map GSM registers");
684		ddi_regs_map_free(&pwp->top_acc_handle);
685		ddi_regs_map_free(&pwp->msg_acc_handle);
686		pci_config_teardown(&pwp->pci_acc_handle);
687		ddi_soft_state_free(pmcs_softc_state, inst);
688		return (DDI_FAILURE);
689	}
690
691	if (ddi_regs_map_setup(dip, PMCS_REGSET_3, (caddr_t *)&pwp->mpi_regs,
692	    0, 0, &pwp->reg_acc_attr, &pwp->mpi_acc_handle)) {
693		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
694		    "failed to map MPI registers");
695		ddi_regs_map_free(&pwp->top_acc_handle);
696		ddi_regs_map_free(&pwp->gsm_acc_handle);
697		ddi_regs_map_free(&pwp->msg_acc_handle);
698		pci_config_teardown(&pwp->pci_acc_handle);
699		ddi_soft_state_free(pmcs_softc_state, inst);
700		return (DDI_FAILURE);
701	}
702	pwp->mpibar =
703	    (((5U << 2) + 0x10) << PMCS_MSGU_MPI_BAR_SHIFT) | set3size;
704
705	/*
706	 * Make sure we can support this card.
707	 */
708	pwp->chiprev = pmcs_rd_topunit(pwp, PMCS_DEVICE_REVISION);
709
710	switch (pwp->chiprev) {
711	case PMCS_PM8001_REV_A:
712	case PMCS_PM8001_REV_B:
713		pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL,
714		    "Rev A/B Card no longer supported");
715		goto failure;
716	case PMCS_PM8001_REV_C:
717		break;
718	default:
719		pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL,
720		    "Unknown chip revision (%d)", pwp->chiprev);
721		goto failure;
722	}
723
724	/*
725	 * Allocate DMA addressable area for Inbound and Outbound Queue indices
726	 * that the chip needs to access plus a space for scratch usage
727	 */
728	pwp->cip_dma_attr.dma_attr_align = sizeof (uint32_t);
729	if (pmcs_dma_setup(pwp, &pwp->cip_dma_attr, &pwp->cip_acchdls,
730	    &pwp->cip_handles, ptob(1), (caddr_t *)&pwp->cip,
731	    &pwp->ciaddr) == B_FALSE) {
732		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
733		    "Failed to setup DMA for index/scratch");
734		goto failure;
735	}
736
737	bzero(pwp->cip, ptob(1));
738	pwp->scratch = &pwp->cip[PMCS_INDICES_SIZE];
739	pwp->scratch_dma = pwp->ciaddr + PMCS_INDICES_SIZE;
740
741	/*
742	 * Allocate DMA S/G list chunks
743	 */
744	(void) pmcs_add_more_chunks(pwp, ptob(1) * PMCS_MIN_CHUNK_PAGES);
745
746	/*
747	 * Allocate a DMA addressable area for the firmware log (if needed)
748	 */
749	if (pwp->fwlog) {
750		/*
751		 * Align to event log header and entry size
752		 */
753		pwp->fwlog_dma_attr.dma_attr_align = 32;
754		if (pmcs_dma_setup(pwp, &pwp->fwlog_dma_attr,
755		    &pwp->fwlog_acchdl,
756		    &pwp->fwlog_hndl, PMCS_FWLOG_SIZE,
757		    (caddr_t *)&pwp->fwlogp,
758		    &pwp->fwaddr) == B_FALSE) {
759			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
760			    "Failed to setup DMA for fwlog area");
761			pwp->fwlog = 0;
762		} else {
763			bzero(pwp->fwlogp, PMCS_FWLOG_SIZE);
764			pwp->fwlogp_aap1 = (pmcs_fw_event_hdr_t *)pwp->fwlogp;
765			pwp->fwlogp_iop = (pmcs_fw_event_hdr_t *)((void *)
766			    ((caddr_t)pwp->fwlogp + (PMCS_FWLOG_SIZE / 2)));
767		}
768	}
769
770	if (pwp->flash_chunk_addr == 0) {
771		pwp->regdump_dma_attr.dma_attr_align = PMCS_FLASH_CHUNK_SIZE;
772		if (pmcs_dma_setup(pwp, &pwp->regdump_dma_attr,
773		    &pwp->regdump_acchdl,
774		    &pwp->regdump_hndl, PMCS_FLASH_CHUNK_SIZE,
775		    (caddr_t *)&pwp->flash_chunkp, &pwp->flash_chunk_addr) ==
776		    B_FALSE) {
777			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
778			    "Failed to setup DMA for register dump area");
779			goto failure;
780		}
781		bzero(pwp->flash_chunkp, PMCS_FLASH_CHUNK_SIZE);
782	}
783
784	/*
785	 * More bits of local initialization...
786	 */
787	pwp->tq = ddi_taskq_create(dip, "_tq", 4, TASKQ_DEFAULTPRI, 0);
788	if (pwp->tq == NULL) {
789		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
790		    "unable to create worker taskq");
791		goto failure;
792	}
793
794	/*
795	 * Cache of structures for dealing with I/O completion callbacks.
796	 */
797	(void) snprintf(buf, sizeof (buf), "pmcs_iocomp_cb_cache%d", inst);
798	pwp->iocomp_cb_cache = kmem_cache_create(buf,
799	    sizeof (pmcs_iocomp_cb_t), 16, NULL, NULL, NULL, NULL, NULL, 0);
800
801	/*
802	 * Cache of PHY structures
803	 */
804	(void) snprintf(buf, sizeof (buf), "pmcs_phy_cache%d", inst);
805	pwp->phy_cache = kmem_cache_create(buf, sizeof (pmcs_phy_t), 8,
806	    pmcs_phy_constructor, pmcs_phy_destructor, NULL, (void *)pwp,
807	    NULL, 0);
808
809	/*
810	 * Allocate space for the I/O completion threads
811	 */
812	num_threads = ncpus_online;
813	if (num_threads > PMCS_MAX_CQ_THREADS) {
814		num_threads = PMCS_MAX_CQ_THREADS;
815	}
816
817	pwp->cq_info.cq_threads = num_threads;
818	pwp->cq_info.cq_thr_info = kmem_zalloc(
819	    sizeof (pmcs_cq_thr_info_t) * pwp->cq_info.cq_threads, KM_SLEEP);
820	pwp->cq_info.cq_next_disp_thr = 0;
821	pwp->cq_info.cq_stop = B_FALSE;
822
823	/*
824	 * Set the quantum value in clock ticks for the I/O interrupt
825	 * coalescing timer.
826	 */
827	pwp->io_intr_coal.quantum = drv_usectohz(PMCS_QUANTUM_TIME_USECS);
828
829	/*
830	 * We have a delicate dance here. We need to set up
831	 * interrupts so we know how to set up some OQC
832	 * tables. However, while we're setting up table
833	 * access, we may need to flash new firmware and
834	 * reset the card, which will take some finessing.
835	 */
836
837	/*
838	 * Set up interrupts here.
839	 */
840	switch (pmcs_setup_intr(pwp)) {
841	case 0:
842		break;
843	case EIO:
844		pwp->stuck = 1;
845		/* FALLTHROUGH */
846	default:
847		goto failure;
848	}
849
850	/*
851	 * Set these up now becuase they are used to initialize the OQC tables.
852	 *
853	 * If we have MSI or MSI-X interrupts set up and we have enough
854	 * vectors for each OQ, the Outbound Queue vectors can all be the
855	 * same as the appropriate interrupt routine will have been called
856	 * and the doorbell register automatically cleared.
857	 * This keeps us from having to check the Outbound Doorbell register
858	 * when the routines for these interrupts are called.
859	 *
860	 * If we have Legacy INT-X interrupts set up or we didn't have enough
861	 * MSI/MSI-X vectors to uniquely identify each OQ, we point these
862	 * vectors to the bits we would like to have set in the Outbound
863	 * Doorbell register because pmcs_all_intr will read the doorbell
864	 * register to find out why we have an interrupt and write the
865	 * corresponding 'clear' bit for that interrupt.
866	 */
867
868	switch (pwp->intr_cnt) {
869	case 1:
870		/*
871		 * Only one vector, so we must check all OQs for MSI.  For
872		 * INT-X, there's only one vector anyway, so we can just
873		 * use the outbound queue bits to keep from having to
874		 * check each queue for each interrupt.
875		 */
876		if (pwp->int_type == PMCS_INT_FIXED) {
877			pwp->oqvec[PMCS_OQ_IODONE] = PMCS_OQ_IODONE;
878			pwp->oqvec[PMCS_OQ_GENERAL] = PMCS_OQ_GENERAL;
879			pwp->oqvec[PMCS_OQ_EVENTS] = PMCS_OQ_EVENTS;
880		} else {
881			pwp->oqvec[PMCS_OQ_IODONE] = PMCS_OQ_IODONE;
882			pwp->oqvec[PMCS_OQ_GENERAL] = PMCS_OQ_IODONE;
883			pwp->oqvec[PMCS_OQ_EVENTS] = PMCS_OQ_IODONE;
884		}
885		break;
886	case 2:
887		/* With 2, we can at least isolate IODONE */
888		pwp->oqvec[PMCS_OQ_IODONE] = PMCS_OQ_IODONE;
889		pwp->oqvec[PMCS_OQ_GENERAL] = PMCS_OQ_GENERAL;
890		pwp->oqvec[PMCS_OQ_EVENTS] = PMCS_OQ_GENERAL;
891		break;
892	case 4:
893		/* With 4 vectors, everybody gets one */
894		pwp->oqvec[PMCS_OQ_IODONE] = PMCS_OQ_IODONE;
895		pwp->oqvec[PMCS_OQ_GENERAL] = PMCS_OQ_GENERAL;
896		pwp->oqvec[PMCS_OQ_EVENTS] = PMCS_OQ_EVENTS;
897		break;
898	}
899
900	/*
901	 * Do the first part of setup
902	 */
903	if (pmcs_setup(pwp)) {
904		goto failure;
905	}
906	pmcs_report_fwversion(pwp);
907
908	/*
909	 * Now do some additonal allocations based upon information
910	 * gathered during MPI setup.
911	 */
912	pwp->root_phys = kmem_zalloc(pwp->nphy * sizeof (pmcs_phy_t), KM_SLEEP);
913	ASSERT(pwp->nphy < SAS2_PHYNUM_MAX);
914	phyp = pwp->root_phys;
915	for (i = 0; i < pwp->nphy; i++) {
916		if (i < pwp->nphy-1) {
917			phyp->sibling = (phyp + 1);
918		}
919		mutex_init(&phyp->phy_lock, NULL, MUTEX_DRIVER,
920		    DDI_INTR_PRI(pwp->intr_pri));
921		phyp->phynum = i & SAS2_PHYNUM_MASK;
922		pmcs_phy_name(pwp, phyp, phyp->path, sizeof (phyp->path));
923		phyp->pwp = pwp;
924		phyp->device_id = PMCS_INVALID_DEVICE_ID;
925		phyp->portid = PMCS_PHY_INVALID_PORT_ID;
926		phyp++;
927	}
928
929	pwp->work = kmem_zalloc(pwp->max_cmd * sizeof (pmcwork_t), KM_SLEEP);
930	for (i = 0; i < pwp->max_cmd; i++) {
931		pmcwork_t *pwrk = &pwp->work[i];
932		mutex_init(&pwrk->lock, NULL, MUTEX_DRIVER,
933		    DDI_INTR_PRI(pwp->intr_pri));
934		cv_init(&pwrk->sleep_cv, NULL, CV_DRIVER, NULL);
935		STAILQ_INSERT_TAIL(&pwp->wf, pwrk, next);
936
937	}
938	pwp->targets = (pmcs_xscsi_t **)
939	    kmem_zalloc(pwp->max_dev * sizeof (pmcs_xscsi_t *), KM_SLEEP);
940
941	pwp->iqpt = (pmcs_iqp_trace_t *)
942	    kmem_zalloc(sizeof (pmcs_iqp_trace_t), KM_SLEEP);
943	pwp->iqpt->head = kmem_zalloc(PMCS_IQP_TRACE_BUFFER_SIZE, KM_SLEEP);
944	pwp->iqpt->curpos = pwp->iqpt->head;
945	pwp->iqpt->size_left = PMCS_IQP_TRACE_BUFFER_SIZE;
946
947	/*
948	 * Start MPI communication.
949	 */
950	if (pmcs_start_mpi(pwp)) {
951		if (pmcs_soft_reset(pwp, B_FALSE)) {
952			goto failure;
953		}
954		pwp->last_reset_reason = PMCS_LAST_RST_ATTACH;
955	}
956
957	/*
958	 * Do some initial acceptance tests.
959	 * This tests interrupts and queues.
960	 */
961	if (pmcs_echo_test(pwp)) {
962		goto failure;
963	}
964
965	/* Read VPD - if it exists */
966	if (pmcs_get_nvmd(pwp, PMCS_NVMD_VPD, PMCIN_NVMD_VPD, 0, NULL, 0)) {
967		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
968		    "%s: Unable to read VPD: "
969		    "attempting to fabricate", __func__);
970		/*
971		 * When we release, this must goto failure and the call
972		 * to pmcs_fabricate_wwid is removed.
973		 */
974		/* goto failure; */
975		if (!pmcs_fabricate_wwid(pwp)) {
976			goto failure;
977		}
978	}
979
980	/*
981	 * We're now officially running
982	 */
983	pwp->state = STATE_RUNNING;
984
985	/*
986	 * Check firmware versions and load new firmware
987	 * if needed and reset.
988	 */
989	if (pmcs_firmware_update(pwp)) {
990		pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL,
991		    "%s: Firmware update failed", __func__);
992		goto failure;
993	}
994
995	/*
996	 * Create completion threads.
997	 */
998	for (i = 0; i < pwp->cq_info.cq_threads; i++) {
999		pwp->cq_info.cq_thr_info[i].cq_pwp = pwp;
1000		pwp->cq_info.cq_thr_info[i].cq_thread =
1001		    thread_create(NULL, 0, pmcs_scsa_cq_run,
1002		    &pwp->cq_info.cq_thr_info[i], 0, &p0, TS_RUN, minclsyspri);
1003	}
1004
1005	/*
1006	 * Create one thread to deal with the updating of the interrupt
1007	 * coalescing timer.
1008	 */
1009	pwp->ict_thread = thread_create(NULL, 0, pmcs_check_intr_coal,
1010	    pwp, 0, &p0, TS_RUN, minclsyspri);
1011
1012	/*
1013	 * Kick off the watchdog
1014	 */
1015	pwp->wdhandle = timeout(pmcs_watchdog, pwp,
1016	    drv_usectohz(PMCS_WATCH_INTERVAL));
1017	/*
1018	 * Do the SCSI attachment code (before starting phys)
1019	 */
1020	if (pmcs_scsa_init(pwp, &pmcs_dattr)) {
1021		goto failure;
1022	}
1023	pwp->hba_attached = 1;
1024
1025	/* Check all acc & dma handles allocated in attach */
1026	if (pmcs_check_acc_dma_handle(pwp)) {
1027		ddi_fm_service_impact(pwp->dip, DDI_SERVICE_LOST);
1028		goto failure;
1029	}
1030
1031	/*
1032	 * Create the iportmap for this HBA instance
1033	 */
1034	if (scsi_hba_iportmap_create(dip, iportmap_csync_usec,
1035	    iportmap_stable_usec, &pwp->hss_iportmap) != DDI_SUCCESS) {
1036		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1037		    "%s: pmcs%d iportmap_create failed", __func__, inst);
1038		goto failure;
1039	}
1040	ASSERT(pwp->hss_iportmap);
1041
1042	/*
1043	 * Create the phymap for this HBA instance
1044	 */
1045	if (sas_phymap_create(dip, phymap_stable_usec, PHYMAP_MODE_SIMPLE, NULL,
1046	    pwp, pmcs_phymap_activate, pmcs_phymap_deactivate,
1047	    &pwp->hss_phymap) != DDI_SUCCESS) {
1048		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1049		    "%s: pmcs%d phymap_create failed", __func__, inst);
1050		goto failure;
1051	}
1052	ASSERT(pwp->hss_phymap);
1053
1054	/*
1055	 * Start the PHYs.
1056	 */
1057	if (pmcs_start_phys(pwp)) {
1058		goto failure;
1059	}
1060
1061	/*
1062	 * From this point on, we can't fail.
1063	 */
1064	ddi_report_dev(dip);
1065
1066	/* SM-HBA */
1067	pmcs_smhba_add_hba_prop(pwp, DATA_TYPE_INT32, PMCS_SMHBA_SUPPORTED,
1068	    &sm_hba);
1069
1070	/* SM-HBA */
1071	pmcs_smhba_add_hba_prop(pwp, DATA_TYPE_STRING, PMCS_DRV_VERSION,
1072	    pmcs_driver_rev);
1073
1074	/* SM-HBA */
1075	chiprev = 'A' + pwp->chiprev;
1076	(void) snprintf(hw_rev, 2, "%s", &chiprev);
1077	pmcs_smhba_add_hba_prop(pwp, DATA_TYPE_STRING, PMCS_HWARE_VERSION,
1078	    hw_rev);
1079
1080	/* SM-HBA */
1081	switch (PMCS_FW_TYPE(pwp)) {
1082	case PMCS_FW_TYPE_RELEASED:
1083		fwsupport = "Released";
1084		break;
1085	case PMCS_FW_TYPE_DEVELOPMENT:
1086		fwsupport = "Development";
1087		break;
1088	case PMCS_FW_TYPE_ALPHA:
1089		fwsupport = "Alpha";
1090		break;
1091	case PMCS_FW_TYPE_BETA:
1092		fwsupport = "Beta";
1093		break;
1094	default:
1095		fwsupport = "Special";
1096		break;
1097	}
1098	(void) snprintf(fw_rev, sizeof (fw_rev), "%x.%x.%x %s",
1099	    PMCS_FW_MAJOR(pwp), PMCS_FW_MINOR(pwp), PMCS_FW_MICRO(pwp),
1100	    fwsupport);
1101	pmcs_smhba_add_hba_prop(pwp, DATA_TYPE_STRING, PMCS_FWARE_VERSION,
1102	    fw_rev);
1103
1104	/* SM-HBA */
1105	num_phys = pwp->nphy;
1106	pmcs_smhba_add_hba_prop(pwp, DATA_TYPE_INT32, PMCS_NUM_PHYS_HBA,
1107	    &num_phys);
1108
1109	/* SM-HBA */
1110	protocol = SAS_SSP_SUPPORT | SAS_SATA_SUPPORT | SAS_SMP_SUPPORT;
1111	pmcs_smhba_add_hba_prop(pwp, DATA_TYPE_INT32, PMCS_SUPPORTED_PROTOCOL,
1112	    &protocol);
1113
1114	/* Receptacle properties (FMA) */
1115	pwp->recept_labels[0] = PMCS_RECEPT_LABEL_0;
1116	pwp->recept_pm[0] = PMCS_RECEPT_PM_0;
1117	pwp->recept_labels[1] = PMCS_RECEPT_LABEL_1;
1118	pwp->recept_pm[1] = PMCS_RECEPT_PM_1;
1119	if (ddi_prop_update_string_array(DDI_DEV_T_NONE, dip,
1120	    SCSI_HBA_PROP_RECEPTACLE_LABEL, &pwp->recept_labels[0],
1121	    PMCS_NUM_RECEPTACLES) != DDI_PROP_SUCCESS) {
1122		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1123		    "%s: failed to create %s property", __func__,
1124		    "receptacle-label");
1125	}
1126	if (ddi_prop_update_string_array(DDI_DEV_T_NONE, dip,
1127	    SCSI_HBA_PROP_RECEPTACLE_PM, &pwp->recept_pm[0],
1128	    PMCS_NUM_RECEPTACLES) != DDI_PROP_SUCCESS) {
1129		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1130		    "%s: failed to create %s property", __func__,
1131		    "receptacle-pm");
1132	}
1133
1134	return (DDI_SUCCESS);
1135
1136failure:
1137	if (pmcs_unattach(pwp)) {
1138		pwp->stuck = 1;
1139	}
1140	return (DDI_FAILURE);
1141}
1142
1143int
1144pmcs_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
1145{
1146	int inst = ddi_get_instance(dip);
1147	pmcs_iport_t	*iport = NULL;
1148	pmcs_hw_t	*pwp = NULL;
1149	scsi_hba_tran_t	*tran;
1150
1151	if (scsi_hba_iport_unit_address(dip) != NULL) {
1152		/* iport node */
1153		iport = ddi_get_soft_state(pmcs_iport_softstate, inst);
1154		ASSERT(iport);
1155		if (iport == NULL) {
1156			return (DDI_FAILURE);
1157		}
1158		pwp = iport->pwp;
1159	} else {
1160		/* hba node */
1161		pwp = (pmcs_hw_t *)ddi_get_soft_state(pmcs_softc_state, inst);
1162		ASSERT(pwp);
1163		if (pwp == NULL) {
1164			return (DDI_FAILURE);
1165		}
1166	}
1167	switch (cmd) {
1168	case DDI_DETACH:
1169		if (iport) {
1170			/* iport detach */
1171			if (pmcs_iport_unattach(iport)) {
1172				return (DDI_FAILURE);
1173			}
1174			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1175			    "iport%d detached", inst);
1176			return (DDI_SUCCESS);
1177		} else {
1178			/* HBA detach */
1179			if (pmcs_unattach(pwp)) {
1180				return (DDI_FAILURE);
1181			}
1182			return (DDI_SUCCESS);
1183		}
1184
1185	case DDI_SUSPEND:
1186	case DDI_PM_SUSPEND:
1187		/* No DDI_SUSPEND on iport nodes */
1188		if (iport) {
1189			return (DDI_SUCCESS);
1190		}
1191
1192		if (pwp->stuck) {
1193			return (DDI_FAILURE);
1194		}
1195		tran = (scsi_hba_tran_t *)ddi_get_driver_private(dip);
1196		if (!tran) {
1197			return (DDI_FAILURE);
1198		}
1199
1200		pwp = TRAN2PMC(tran);
1201		if (pwp == NULL) {
1202			return (DDI_FAILURE);
1203		}
1204		mutex_enter(&pwp->lock);
1205		if (pwp->tq) {
1206			ddi_taskq_suspend(pwp->tq);
1207		}
1208		pwp->suspended = 1;
1209		mutex_exit(&pwp->lock);
1210		pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, "PMC8X6G suspending");
1211		return (DDI_SUCCESS);
1212
1213	default:
1214		return (DDI_FAILURE);
1215	}
1216}
1217
1218static int
1219pmcs_iport_unattach(pmcs_iport_t *iport)
1220{
1221	pmcs_hw_t	*pwp = iport->pwp;
1222
1223	/*
1224	 * First, check if there are still any configured targets on this
1225	 * iport.  If so, we fail detach.
1226	 */
1227	if (pmcs_iport_has_targets(pwp, iport)) {
1228		pmcs_prt(pwp, PMCS_PRT_DEBUG_IPORT, NULL, NULL,
1229		    "iport%d detach failure: iport has targets (luns)",
1230		    ddi_get_instance(iport->dip));
1231		return (DDI_FAILURE);
1232	}
1233
1234	/*
1235	 * Remove this iport from our list if it is inactive in the phymap.
1236	 */
1237	rw_enter(&pwp->iports_lock, RW_WRITER);
1238	mutex_enter(&iport->lock);
1239
1240	if ((iport->ua_state == UA_ACTIVE) &&
1241	    (detach_with_active_port == 0)) {
1242		mutex_exit(&iport->lock);
1243		rw_exit(&pwp->iports_lock);
1244		pmcs_prt(pwp, PMCS_PRT_DEBUG_IPORT, NULL, NULL,
1245		    "iport%d detach failure: "
1246		    "iport unit address active in phymap",
1247		    ddi_get_instance(iport->dip));
1248		return (DDI_FAILURE);
1249	}
1250
1251	/* If it's our only iport, clear iports_attached */
1252	ASSERT(pwp->num_iports >= 1);
1253	if (--pwp->num_iports == 0) {
1254		pwp->iports_attached = 0;
1255	}
1256
1257	ASSERT(list_link_active(&iport->list_node));
1258	list_remove(&pwp->iports, iport);
1259	rw_exit(&pwp->iports_lock);
1260
1261	/*
1262	 * We have removed the iport handle from the HBA's iports list,
1263	 * there will be no new references to it. Two things must be
1264	 * guarded against here.  First, we could have PHY up events,
1265	 * adding themselves to the iport->phys list and grabbing ref's
1266	 * on our iport handle.  Second, we could have existing references
1267	 * to this iport handle from a point in time prior to the list
1268	 * removal above.
1269	 *
1270	 * So first, destroy the phys list. Remove any phys that have snuck
1271	 * in after the phymap deactivate, dropping the refcnt accordingly.
1272	 * If these PHYs are still up if and when the phymap reactivates
1273	 * (i.e. when this iport reattaches), we'll populate the list with
1274	 * them and bump the refcnt back up.
1275	 */
1276	pmcs_remove_phy_from_iport(iport, NULL);
1277	ASSERT(list_is_empty(&iport->phys));
1278	list_destroy(&iport->phys);
1279	mutex_exit(&iport->lock);
1280
1281	/*
1282	 * Second, wait for any other references to this iport to be
1283	 * dropped, then continue teardown.
1284	 */
1285	mutex_enter(&iport->refcnt_lock);
1286	while (iport->refcnt != 0) {
1287		cv_wait(&iport->refcnt_cv, &iport->refcnt_lock);
1288	}
1289	mutex_exit(&iport->refcnt_lock);
1290
1291
1292	/* Destroy the iport target map */
1293	if (pmcs_iport_tgtmap_destroy(iport) == B_FALSE) {
1294		return (DDI_FAILURE);
1295	}
1296
1297	/* Free the tgt soft state */
1298	if (iport->tgt_sstate != NULL) {
1299		ddi_soft_state_bystr_fini(&iport->tgt_sstate);
1300	}
1301
1302	/* Free our unit address string */
1303	strfree(iport->ua);
1304
1305	/* Finish teardown and free the softstate */
1306	mutex_destroy(&iport->refcnt_lock);
1307	mutex_destroy(&iport->smp_lock);
1308	ASSERT(iport->refcnt == 0);
1309	cv_destroy(&iport->refcnt_cv);
1310	cv_destroy(&iport->smp_cv);
1311	mutex_destroy(&iport->lock);
1312	ddi_soft_state_free(pmcs_iport_softstate, ddi_get_instance(iport->dip));
1313
1314	return (DDI_SUCCESS);
1315}
1316
1317static int
1318pmcs_unattach(pmcs_hw_t *pwp)
1319{
1320	int i;
1321	enum pwpstate curstate;
1322	pmcs_cq_thr_info_t *cqti;
1323
1324	/*
1325	 * Tear down the interrupt infrastructure.
1326	 */
1327	if (pmcs_teardown_intr(pwp)) {
1328		pwp->stuck = 1;
1329	}
1330	pwp->intr_cnt = 0;
1331
1332	/*
1333	 * Grab a lock, if initted, to set state.
1334	 */
1335	if (pwp->locks_initted) {
1336		mutex_enter(&pwp->lock);
1337		if (pwp->state != STATE_DEAD) {
1338			pwp->state = STATE_UNPROBING;
1339		}
1340		curstate = pwp->state;
1341		mutex_exit(&pwp->lock);
1342
1343		/*
1344		 * Stop the I/O completion threads.
1345		 */
1346		mutex_enter(&pwp->cq_lock);
1347		pwp->cq_info.cq_stop = B_TRUE;
1348		for (i = 0; i < pwp->cq_info.cq_threads; i++) {
1349			if (pwp->cq_info.cq_thr_info[i].cq_thread) {
1350				cqti = &pwp->cq_info.cq_thr_info[i];
1351				mutex_enter(&cqti->cq_thr_lock);
1352				cv_signal(&cqti->cq_cv);
1353				mutex_exit(&cqti->cq_thr_lock);
1354				mutex_exit(&pwp->cq_lock);
1355				thread_join(cqti->cq_thread->t_did);
1356				mutex_enter(&pwp->cq_lock);
1357			}
1358		}
1359		mutex_exit(&pwp->cq_lock);
1360		kmem_free(pwp->cq_info.cq_thr_info,
1361		    sizeof (pmcs_cq_thr_info_t) * pwp->cq_info.cq_threads);
1362
1363		/*
1364		 * Stop the interrupt coalescing timer thread
1365		 */
1366		if (pwp->ict_thread) {
1367			mutex_enter(&pwp->ict_lock);
1368			pwp->io_intr_coal.stop_thread = B_TRUE;
1369			cv_signal(&pwp->ict_cv);
1370			mutex_exit(&pwp->ict_lock);
1371			thread_join(pwp->ict_thread->t_did);
1372		}
1373	} else {
1374		if (pwp->state != STATE_DEAD) {
1375			pwp->state = STATE_UNPROBING;
1376		}
1377		curstate = pwp->state;
1378	}
1379
1380	/*
1381	 * Make sure that any pending watchdog won't
1382	 * be called from this point on out.
1383	 */
1384	(void) untimeout(pwp->wdhandle);
1385	/*
1386	 * After the above action, the watchdog
1387	 * timer that starts up the worker task
1388	 * may trigger but will exit immediately
1389	 * on triggering.
1390	 *
1391	 * Now that this is done, we can destroy
1392	 * the task queue, which will wait if we're
1393	 * running something on it.
1394	 */
1395	if (pwp->tq) {
1396		ddi_taskq_destroy(pwp->tq);
1397		pwp->tq = NULL;
1398	}
1399
1400	pmcs_fm_fini(pwp);
1401
1402	if (pwp->hba_attached) {
1403		(void) scsi_hba_detach(pwp->dip);
1404		pwp->hba_attached = 0;
1405	}
1406
1407	/*
1408	 * If the chip hasn't been marked dead, shut it down now
1409	 * to bring it back to a known state without attempting
1410	 * a soft reset.
1411	 */
1412	if (curstate != STATE_DEAD && pwp->locks_initted) {
1413		/*
1414		 * De-register all registered devices
1415		 */
1416		pmcs_deregister_devices(pwp, pwp->root_phys);
1417
1418		/*
1419		 * Stop all the phys.
1420		 */
1421		pmcs_stop_phys(pwp);
1422
1423		/*
1424		 * Shut Down Message Passing
1425		 */
1426		(void) pmcs_stop_mpi(pwp);
1427
1428		/*
1429		 * Reset chip
1430		 */
1431		(void) pmcs_soft_reset(pwp, B_FALSE);
1432		pwp->last_reset_reason = PMCS_LAST_RST_DETACH;
1433	}
1434
1435	/*
1436	 * Turn off interrupts on the chip
1437	 */
1438	if (pwp->mpi_acc_handle) {
1439		pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_MASK, 0xffffffff);
1440	}
1441
1442	if (pwp->hss_phymap != NULL) {
1443		/* Destroy the phymap */
1444		sas_phymap_destroy(pwp->hss_phymap);
1445	}
1446
1447	if (pwp->hss_iportmap != NULL) {
1448		/* Destroy the iportmap */
1449		scsi_hba_iportmap_destroy(pwp->hss_iportmap);
1450	}
1451
1452	/* Destroy the iports lock and list */
1453	rw_destroy(&pwp->iports_lock);
1454	ASSERT(list_is_empty(&pwp->iports));
1455	list_destroy(&pwp->iports);
1456
1457	/*
1458	 * Free DMA handles and associated consistent memory
1459	 */
1460	if (pwp->regdump_hndl) {
1461		if (ddi_dma_unbind_handle(pwp->regdump_hndl) != DDI_SUCCESS) {
1462			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1463			    "Condition check failed "
1464			    "at %s():%d", __func__, __LINE__);
1465		}
1466		ddi_dma_free_handle(&pwp->regdump_hndl);
1467		ddi_dma_mem_free(&pwp->regdump_acchdl);
1468		pwp->regdump_hndl = 0;
1469	}
1470	if (pwp->fwlog_hndl) {
1471		if (ddi_dma_unbind_handle(pwp->fwlog_hndl) != DDI_SUCCESS) {
1472			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1473			    "Condition check failed "
1474			    "at %s():%d", __func__, __LINE__);
1475		}
1476		ddi_dma_free_handle(&pwp->fwlog_hndl);
1477		ddi_dma_mem_free(&pwp->fwlog_acchdl);
1478		pwp->fwlog_hndl = 0;
1479	}
1480	if (pwp->cip_handles) {
1481		if (ddi_dma_unbind_handle(pwp->cip_handles) != DDI_SUCCESS) {
1482			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1483			    "Condition check failed "
1484			    "at %s():%d", __func__, __LINE__);
1485		}
1486		ddi_dma_free_handle(&pwp->cip_handles);
1487		ddi_dma_mem_free(&pwp->cip_acchdls);
1488		pwp->cip_handles = 0;
1489	}
1490	for (i = 0; i < PMCS_NOQ; i++) {
1491		if (pwp->oqp_handles[i]) {
1492			if (ddi_dma_unbind_handle(pwp->oqp_handles[i]) !=
1493			    DDI_SUCCESS) {
1494				pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1495				    "Condition check failed at %s():%d",
1496				    __func__, __LINE__);
1497			}
1498			ddi_dma_free_handle(&pwp->oqp_handles[i]);
1499			ddi_dma_mem_free(&pwp->oqp_acchdls[i]);
1500			pwp->oqp_handles[i] = 0;
1501		}
1502	}
1503	for (i = 0; i < PMCS_NIQ; i++) {
1504		if (pwp->iqp_handles[i]) {
1505			if (ddi_dma_unbind_handle(pwp->iqp_handles[i]) !=
1506			    DDI_SUCCESS) {
1507				pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1508				    "Condition check failed at %s():%d",
1509				    __func__, __LINE__);
1510			}
1511			ddi_dma_free_handle(&pwp->iqp_handles[i]);
1512			ddi_dma_mem_free(&pwp->iqp_acchdls[i]);
1513			pwp->iqp_handles[i] = 0;
1514		}
1515	}
1516
1517	pmcs_free_dma_chunklist(pwp);
1518
1519	/*
1520	 * Unmap registers and destroy access handles
1521	 */
1522	if (pwp->mpi_acc_handle) {
1523		ddi_regs_map_free(&pwp->mpi_acc_handle);
1524		pwp->mpi_acc_handle = 0;
1525	}
1526	if (pwp->top_acc_handle) {
1527		ddi_regs_map_free(&pwp->top_acc_handle);
1528		pwp->top_acc_handle = 0;
1529	}
1530	if (pwp->gsm_acc_handle) {
1531		ddi_regs_map_free(&pwp->gsm_acc_handle);
1532		pwp->gsm_acc_handle = 0;
1533	}
1534	if (pwp->msg_acc_handle) {
1535		ddi_regs_map_free(&pwp->msg_acc_handle);
1536		pwp->msg_acc_handle = 0;
1537	}
1538	if (pwp->pci_acc_handle) {
1539		pci_config_teardown(&pwp->pci_acc_handle);
1540		pwp->pci_acc_handle = 0;
1541	}
1542
1543	/*
1544	 * Do memory allocation cleanup.
1545	 */
1546	while (pwp->dma_freelist) {
1547		pmcs_dmachunk_t *this = pwp->dma_freelist;
1548		pwp->dma_freelist = this->nxt;
1549		kmem_free(this, sizeof (pmcs_dmachunk_t));
1550	}
1551
1552	/*
1553	 * Free pools
1554	 */
1555	if (pwp->iocomp_cb_cache) {
1556		kmem_cache_destroy(pwp->iocomp_cb_cache);
1557	}
1558
1559	/*
1560	 * Free all PHYs (at level > 0), then free the cache
1561	 */
1562	pmcs_free_all_phys(pwp, pwp->root_phys);
1563	if (pwp->phy_cache) {
1564		kmem_cache_destroy(pwp->phy_cache);
1565	}
1566
1567	/*
1568	 * Free root PHYs
1569	 */
1570	if (pwp->root_phys) {
1571		pmcs_phy_t *phyp = pwp->root_phys;
1572		for (i = 0; i < pwp->nphy; i++) {
1573			mutex_destroy(&phyp->phy_lock);
1574			phyp = phyp->sibling;
1575		}
1576		kmem_free(pwp->root_phys, pwp->nphy * sizeof (pmcs_phy_t));
1577		pwp->root_phys = NULL;
1578		pwp->nphy = 0;
1579	}
1580
1581	/* Free the targets list */
1582	if (pwp->targets) {
1583		kmem_free(pwp->targets,
1584		    sizeof (pmcs_xscsi_t *) * pwp->max_dev);
1585	}
1586
1587	/*
1588	 * Free work structures
1589	 */
1590
1591	if (pwp->work && pwp->max_cmd) {
1592		for (i = 0; i < pwp->max_cmd; i++) {
1593			pmcwork_t *pwrk = &pwp->work[i];
1594			mutex_destroy(&pwrk->lock);
1595			cv_destroy(&pwrk->sleep_cv);
1596		}
1597		kmem_free(pwp->work, sizeof (pmcwork_t) * pwp->max_cmd);
1598		pwp->work = NULL;
1599		pwp->max_cmd = 0;
1600	}
1601
1602	/*
1603	 * Do last property and SCSA cleanup
1604	 */
1605	if (pwp->smp_tran) {
1606		smp_hba_tran_free(pwp->smp_tran);
1607		pwp->smp_tran = NULL;
1608	}
1609	if (pwp->tran) {
1610		scsi_hba_tran_free(pwp->tran);
1611		pwp->tran = NULL;
1612	}
1613	if (pwp->reset_notify_listf) {
1614		scsi_hba_reset_notify_tear_down(pwp->reset_notify_listf);
1615		pwp->reset_notify_listf = NULL;
1616	}
1617	ddi_prop_remove_all(pwp->dip);
1618	if (pwp->stuck) {
1619		return (-1);
1620	}
1621
1622	/* Free register dump area if allocated */
1623	if (pwp->regdumpp) {
1624		kmem_free(pwp->regdumpp, PMCS_REG_DUMP_SIZE);
1625		pwp->regdumpp = NULL;
1626	}
1627	if (pwp->iqpt && pwp->iqpt->head) {
1628		kmem_free(pwp->iqpt->head, PMCS_IQP_TRACE_BUFFER_SIZE);
1629		pwp->iqpt->head = pwp->iqpt->curpos = NULL;
1630	}
1631	if (pwp->iqpt) {
1632		kmem_free(pwp->iqpt, sizeof (pmcs_iqp_trace_t));
1633		pwp->iqpt = NULL;
1634	}
1635
1636	/* Destroy pwp's lock */
1637	if (pwp->locks_initted) {
1638		mutex_destroy(&pwp->lock);
1639		mutex_destroy(&pwp->dma_lock);
1640		mutex_destroy(&pwp->axil_lock);
1641		mutex_destroy(&pwp->cq_lock);
1642		mutex_destroy(&pwp->config_lock);
1643		mutex_destroy(&pwp->ict_lock);
1644		mutex_destroy(&pwp->wfree_lock);
1645		mutex_destroy(&pwp->pfree_lock);
1646		mutex_destroy(&pwp->dead_phylist_lock);
1647#ifdef	DEBUG
1648		mutex_destroy(&pwp->dbglock);
1649#endif
1650		cv_destroy(&pwp->config_cv);
1651		cv_destroy(&pwp->ict_cv);
1652		cv_destroy(&pwp->drain_cv);
1653		pwp->locks_initted = 0;
1654	}
1655
1656	ddi_soft_state_free(pmcs_softc_state, ddi_get_instance(pwp->dip));
1657	return (0);
1658}
1659
1660/*
1661 * quiesce (9E) entry point
1662 *
1663 * This function is called when the system is single-threaded at high PIL
1664 * with preemption disabled. Therefore, the function must not block/wait/sleep.
1665 *
1666 * Returns DDI_SUCCESS or DDI_FAILURE.
1667 *
1668 */
1669static int
1670pmcs_quiesce(dev_info_t *dip)
1671{
1672	pmcs_hw_t	*pwp;
1673	scsi_hba_tran_t	*tran;
1674
1675	if ((tran = ddi_get_driver_private(dip)) == NULL)
1676		return (DDI_SUCCESS);
1677
1678	/* No quiesce necessary on a per-iport basis */
1679	if (scsi_hba_iport_unit_address(dip) != NULL) {
1680		return (DDI_SUCCESS);
1681	}
1682
1683	if ((pwp = TRAN2PMC(tran)) == NULL)
1684		return (DDI_SUCCESS);
1685
1686	/* Stop MPI & Reset chip (no need to re-initialize) */
1687	(void) pmcs_stop_mpi(pwp);
1688	(void) pmcs_soft_reset(pwp, B_TRUE);
1689	pwp->last_reset_reason = PMCS_LAST_RST_QUIESCE;
1690
1691	return (DDI_SUCCESS);
1692}
1693
1694/*
1695 * Called with xp->statlock and PHY lock and scratch acquired.
1696 */
1697static int
1698pmcs_add_sata_device(pmcs_hw_t *pwp, pmcs_xscsi_t *xp)
1699{
1700	ata_identify_t *ati;
1701	int result, i;
1702	pmcs_phy_t *pptr;
1703	uint16_t *a;
1704	union {
1705		uint8_t nsa[8];
1706		uint16_t nsb[4];
1707	} u;
1708
1709	/*
1710	 * Safe defaults - use only if this target is brand new (i.e. doesn't
1711	 * already have these settings configured)
1712	 */
1713	if (xp->capacity == 0) {
1714		xp->capacity = (uint64_t)-1;
1715		xp->ca = 1;
1716		xp->qdepth = 1;
1717		xp->pio = 1;
1718	}
1719
1720	pptr = xp->phy;
1721
1722	/*
1723	 * We only try and issue an IDENTIFY for first level
1724	 * (direct attached) devices. We don't try and
1725	 * set other quirks here (this will happen later,
1726	 * if the device is fully configured)
1727	 */
1728	if (pptr->level) {
1729		return (0);
1730	}
1731
1732	mutex_exit(&xp->statlock);
1733	result = pmcs_sata_identify(pwp, pptr);
1734	mutex_enter(&xp->statlock);
1735
1736	if (result) {
1737		return (result);
1738	}
1739	ati = pwp->scratch;
1740	a = &ati->word108;
1741	for (i = 0; i < 4; i++) {
1742		u.nsb[i] = ddi_swap16(*a++);
1743	}
1744
1745	/*
1746	 * Check the returned data for being a valid (NAA=5) WWN.
1747	 * If so, use that and override the SAS address we were
1748	 * given at Link Up time.
1749	 */
1750	if ((u.nsa[0] >> 4) == 5) {
1751		(void) memcpy(pptr->sas_address, u.nsa, 8);
1752	}
1753	pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
1754	    "%s: %s has SAS ADDRESS " SAS_ADDR_FMT,
1755	    __func__, pptr->path, SAS_ADDR_PRT(pptr->sas_address));
1756	return (0);
1757}
1758
1759/*
1760 * Called with PHY lock and target statlock held and scratch acquired
1761 */
1762static boolean_t
1763pmcs_add_new_device(pmcs_hw_t *pwp, pmcs_xscsi_t *target)
1764{
1765	ASSERT(target != NULL);
1766	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, target, "%s: target = 0x%p",
1767	    __func__, (void *) target);
1768
1769	switch (target->phy->dtype) {
1770	case SATA:
1771		if (pmcs_add_sata_device(pwp, target) != 0) {
1772			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, target->phy,
1773			    target, "%s: add_sata_device failed for tgt 0x%p",
1774			    __func__, (void *) target);
1775			return (B_FALSE);
1776		}
1777		break;
1778	case SAS:
1779		target->qdepth = maxqdepth;
1780		break;
1781	case EXPANDER:
1782		target->qdepth = 1;
1783		break;
1784	}
1785
1786	target->new = 0;
1787	target->assigned = 1;
1788	target->dev_state = PMCS_DEVICE_STATE_OPERATIONAL;
1789	target->dtype = target->phy->dtype;
1790
1791	/*
1792	 * Set the PHY's config stop time to 0.  This is one of the final
1793	 * stops along the config path, so we're indicating that we
1794	 * successfully configured the PHY.
1795	 */
1796	target->phy->config_stop = 0;
1797
1798	return (B_TRUE);
1799}
1800
1801void
1802pmcs_worker(void *arg)
1803{
1804	pmcs_hw_t *pwp = arg;
1805	ulong_t work_flags;
1806
1807	DTRACE_PROBE2(pmcs__worker, ulong_t, pwp->work_flags, boolean_t,
1808	    pwp->config_changed);
1809
1810	if (pwp->state != STATE_RUNNING) {
1811		return;
1812	}
1813
1814	work_flags = atomic_swap_ulong(&pwp->work_flags, 0);
1815
1816	if (work_flags & PMCS_WORK_FLAG_DUMP_REGS) {
1817		mutex_enter(&pwp->lock);
1818		pmcs_register_dump_int(pwp);
1819		mutex_exit(&pwp->lock);
1820	}
1821
1822	if (work_flags & PMCS_WORK_FLAG_SAS_HW_ACK) {
1823		pmcs_ack_events(pwp);
1824	}
1825
1826	if (work_flags & PMCS_WORK_FLAG_SPINUP_RELEASE) {
1827		mutex_enter(&pwp->lock);
1828		pmcs_spinup_release(pwp, NULL);
1829		mutex_exit(&pwp->lock);
1830	}
1831
1832	if (work_flags & PMCS_WORK_FLAG_SSP_EVT_RECOVERY) {
1833		pmcs_ssp_event_recovery(pwp);
1834	}
1835
1836	if (work_flags & PMCS_WORK_FLAG_DS_ERR_RECOVERY) {
1837		pmcs_dev_state_recovery(pwp, NULL);
1838	}
1839
1840	if (work_flags & PMCS_WORK_FLAG_DEREGISTER_DEV) {
1841		pmcs_deregister_device_work(pwp, NULL);
1842	}
1843
1844	if (work_flags & PMCS_WORK_FLAG_DISCOVER) {
1845		pmcs_discover(pwp);
1846	}
1847
1848	if (work_flags & PMCS_WORK_FLAG_ABORT_HANDLE) {
1849		if (pmcs_abort_handler(pwp)) {
1850			SCHEDULE_WORK(pwp, PMCS_WORK_ABORT_HANDLE);
1851		}
1852	}
1853
1854	if (work_flags & PMCS_WORK_FLAG_SATA_RUN) {
1855		pmcs_sata_work(pwp);
1856	}
1857
1858	if (work_flags & PMCS_WORK_FLAG_RUN_QUEUES) {
1859		pmcs_scsa_wq_run(pwp);
1860		mutex_enter(&pwp->lock);
1861		PMCS_CQ_RUN(pwp);
1862		mutex_exit(&pwp->lock);
1863	}
1864
1865	if (work_flags & PMCS_WORK_FLAG_ADD_DMA_CHUNKS) {
1866		if (pmcs_add_more_chunks(pwp,
1867		    ptob(1) * PMCS_ADDTL_CHUNK_PAGES)) {
1868			SCHEDULE_WORK(pwp, PMCS_WORK_ADD_DMA_CHUNKS);
1869		} else {
1870			SCHEDULE_WORK(pwp, PMCS_WORK_RUN_QUEUES);
1871		}
1872	}
1873}
1874
1875static int
1876pmcs_add_more_chunks(pmcs_hw_t *pwp, unsigned long nsize)
1877{
1878	pmcs_dmachunk_t *dc;
1879	unsigned long dl;
1880	pmcs_chunk_t	*pchunk = NULL;
1881
1882	pwp->cip_dma_attr.dma_attr_align = sizeof (uint32_t);
1883
1884	pchunk = kmem_zalloc(sizeof (pmcs_chunk_t), KM_SLEEP);
1885	if (pchunk == NULL) {
1886		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1887		    "Not enough memory for DMA chunks");
1888		return (-1);
1889	}
1890
1891	if (pmcs_dma_setup(pwp, &pwp->cip_dma_attr, &pchunk->acc_handle,
1892	    &pchunk->dma_handle, nsize, (caddr_t *)&pchunk->addrp,
1893	    &pchunk->dma_addr) == B_FALSE) {
1894		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1895		    "Failed to setup DMA for chunks");
1896		kmem_free(pchunk, sizeof (pmcs_chunk_t));
1897		return (-1);
1898	}
1899
1900	if ((pmcs_check_acc_handle(pchunk->acc_handle) != DDI_SUCCESS) ||
1901	    (pmcs_check_dma_handle(pchunk->dma_handle) != DDI_SUCCESS)) {
1902		ddi_fm_service_impact(pwp->dip, DDI_SERVICE_UNAFFECTED);
1903		return (-1);
1904	}
1905
1906	bzero(pchunk->addrp, nsize);
1907	dc = NULL;
1908	for (dl = 0; dl < (nsize / PMCS_SGL_CHUNKSZ); dl++) {
1909		pmcs_dmachunk_t *tmp;
1910		tmp = kmem_alloc(sizeof (pmcs_dmachunk_t), KM_SLEEP);
1911		tmp->nxt = dc;
1912		dc = tmp;
1913	}
1914	mutex_enter(&pwp->dma_lock);
1915	pmcs_idma_chunks(pwp, dc, pchunk, nsize);
1916	pwp->nchunks++;
1917	mutex_exit(&pwp->dma_lock);
1918	return (0);
1919}
1920
1921static void
1922pmcs_check_forward_progress(pmcs_hw_t *pwp)
1923{
1924	pmcwork_t	*wrkp;
1925	uint32_t	*iqp;
1926	uint32_t	cur_iqci;
1927	uint32_t	cur_work_idx;
1928	uint32_t	cur_msgu_tick;
1929	uint32_t	cur_iop_tick;
1930	int		i;
1931
1932	mutex_enter(&pwp->lock);
1933
1934	if (pwp->state == STATE_IN_RESET) {
1935		mutex_exit(&pwp->lock);
1936		return;
1937	}
1938
1939	/*
1940	 * Ensure that inbound work is getting picked up.  First, check to
1941	 * see if new work has been posted.  If it has, ensure that the
1942	 * work is moving forward by checking the consumer index and the
1943	 * last_htag for the work being processed against what we saw last
1944	 * time.  Note: we use the work structure's 'last_htag' because at
1945	 * any given moment it could be freed back, thus clearing 'htag'
1946	 * and setting 'last_htag' (see pmcs_pwork).
1947	 */
1948	for (i = 0; i < PMCS_NIQ; i++) {
1949		cur_iqci = pmcs_rd_iqci(pwp, i);
1950		iqp = &pwp->iqp[i][cur_iqci * (PMCS_QENTRY_SIZE >> 2)];
1951		cur_work_idx = PMCS_TAG_INDEX(LE_32(*(iqp+1)));
1952		wrkp = &pwp->work[cur_work_idx];
1953		if (cur_iqci == pwp->shadow_iqpi[i]) {
1954			pwp->last_iqci[i] = cur_iqci;
1955			pwp->last_htag[i] = wrkp->last_htag;
1956			continue;
1957		}
1958		if ((cur_iqci == pwp->last_iqci[i]) &&
1959		    (wrkp->last_htag == pwp->last_htag[i])) {
1960			pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL,
1961			    "Inbound Queue stall detected, issuing reset");
1962			goto hot_reset;
1963		}
1964		pwp->last_iqci[i] = cur_iqci;
1965		pwp->last_htag[i] = wrkp->last_htag;
1966	}
1967
1968	/*
1969	 * Check heartbeat on both the MSGU and IOP.  It is unlikely that
1970	 * we'd ever fail here, as the inbound queue monitoring code above
1971	 * would detect a stall due to either of these elements being
1972	 * stalled, but we might as well keep an eye on them.
1973	 */
1974	cur_msgu_tick = pmcs_rd_gst_tbl(pwp, PMCS_GST_MSGU_TICK);
1975	if (cur_msgu_tick == pwp->last_msgu_tick) {
1976		pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL,
1977		    "Stall detected on MSGU, issuing reset");
1978		goto hot_reset;
1979	}
1980	pwp->last_msgu_tick = cur_msgu_tick;
1981
1982	cur_iop_tick  = pmcs_rd_gst_tbl(pwp, PMCS_GST_IOP_TICK);
1983	if (cur_iop_tick == pwp->last_iop_tick) {
1984		pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL,
1985		    "Stall detected on IOP, issuing reset");
1986		goto hot_reset;
1987	}
1988	pwp->last_iop_tick = cur_iop_tick;
1989
1990	mutex_exit(&pwp->lock);
1991	return;
1992
1993hot_reset:
1994	pwp->state = STATE_DEAD;
1995	/*
1996	 * We've detected a stall. Attempt to recover service via hot
1997	 * reset. In case of failure, pmcs_hot_reset() will handle the
1998	 * failure and issue any required FM notifications.
1999	 * See pmcs_subr.c for more details.
2000	 */
2001	if (pmcs_hot_reset(pwp)) {
2002		pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL,
2003		    "%s: hot reset failure", __func__);
2004	} else {
2005		pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL,
2006		    "%s: hot reset complete", __func__);
2007		pwp->last_reset_reason = PMCS_LAST_RST_STALL;
2008	}
2009	mutex_exit(&pwp->lock);
2010}
2011
2012static void
2013pmcs_check_commands(pmcs_hw_t *pwp)
2014{
2015	pmcs_cmd_t *sp;
2016	size_t amt;
2017	char path[32];
2018	pmcwork_t *pwrk;
2019	pmcs_xscsi_t *target;
2020	pmcs_phy_t *phyp;
2021	int rval;
2022
2023	for (pwrk = pwp->work; pwrk < &pwp->work[pwp->max_cmd]; pwrk++) {
2024		mutex_enter(&pwrk->lock);
2025
2026		/*
2027		 * If the command isn't active, we can't be timing it still.
2028		 * Active means the tag is not free and the state is "on chip".
2029		 */
2030		if (!PMCS_COMMAND_ACTIVE(pwrk)) {
2031			mutex_exit(&pwrk->lock);
2032			continue;
2033		}
2034
2035		/*
2036		 * No timer active for this command.
2037		 */
2038		if (pwrk->timer == 0) {
2039			mutex_exit(&pwrk->lock);
2040			continue;
2041		}
2042
2043		/*
2044		 * Knock off bits for the time interval.
2045		 */
2046		if (pwrk->timer >= US2WT(PMCS_WATCH_INTERVAL)) {
2047			pwrk->timer -= US2WT(PMCS_WATCH_INTERVAL);
2048		} else {
2049			pwrk->timer = 0;
2050		}
2051		if (pwrk->timer > 0) {
2052			mutex_exit(&pwrk->lock);
2053			continue;
2054		}
2055
2056		/*
2057		 * The command has now officially timed out.
2058		 * Get the path for it. If it doesn't have
2059		 * a phy pointer any more, it's really dead
2060		 * and can just be put back on the free list.
2061		 * There should *not* be any commands associated
2062		 * with it any more.
2063		 */
2064		if (pwrk->phy == NULL) {
2065			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2066			    "dead command with gone phy being recycled");
2067			ASSERT(pwrk->xp == NULL);
2068			pmcs_pwork(pwp, pwrk);
2069			continue;
2070		}
2071		amt = sizeof (path);
2072		amt = min(sizeof (pwrk->phy->path), amt);
2073		(void) memcpy(path, pwrk->phy->path, amt);
2074
2075		/*
2076		 * If this is a non-SCSA command, stop here. Eventually
2077		 * we might do something with non-SCSA commands here-
2078		 * but so far their timeout mechanisms are handled in
2079		 * the WAIT_FOR macro.
2080		 */
2081		if (pwrk->xp == NULL) {
2082			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2083			    "%s: non-SCSA cmd tag 0x%x timed out",
2084			    path, pwrk->htag);
2085			mutex_exit(&pwrk->lock);
2086			continue;
2087		}
2088
2089		sp = pwrk->arg;
2090		ASSERT(sp != NULL);
2091
2092		/*
2093		 * Mark it as timed out.
2094		 */
2095		CMD2PKT(sp)->pkt_reason = CMD_TIMEOUT;
2096		CMD2PKT(sp)->pkt_statistics |= STAT_TIMEOUT;
2097#ifdef	DEBUG
2098		pmcs_prt(pwp, PMCS_PRT_DEBUG, pwrk->phy, pwrk->xp,
2099		    "%s: SCSA cmd tag 0x%x timed out (state %x) onwire=%d",
2100		    path, pwrk->htag, pwrk->state, pwrk->onwire);
2101#else
2102		pmcs_prt(pwp, PMCS_PRT_DEBUG, pwrk->phy, pwrk->xp,
2103		    "%s: SCSA cmd tag 0x%x timed out (state %x)",
2104		    path, pwrk->htag, pwrk->state);
2105#endif
2106		/*
2107		 * Mark the work structure as timed out.
2108		 */
2109		pwrk->state = PMCS_WORK_STATE_TIMED_OUT;
2110		phyp = pwrk->phy;
2111		target = pwrk->xp;
2112		ASSERT(target != NULL);
2113		mutex_exit(&pwrk->lock);
2114
2115		pmcs_lock_phy(phyp);
2116		mutex_enter(&target->statlock);
2117
2118		/*
2119		 * No point attempting recovery if the device is gone
2120		 */
2121		if (target->dev_gone) {
2122			mutex_exit(&target->statlock);
2123			pmcs_unlock_phy(phyp);
2124			pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, target,
2125			    "%s: tgt(0x%p) is gone. Returning CMD_DEV_GONE "
2126			    "for htag 0x%08x", __func__,
2127			    (void *)target, pwrk->htag);
2128			mutex_enter(&pwrk->lock);
2129			if (!PMCS_COMMAND_DONE(pwrk)) {
2130				/* Complete this command here */
2131				pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, target,
2132				    "%s: Completing cmd (htag 0x%08x) "
2133				    "anyway", __func__, pwrk->htag);
2134				pwrk->dead = 1;
2135				CMD2PKT(sp)->pkt_reason = CMD_DEV_GONE;
2136				CMD2PKT(sp)->pkt_state = STATE_GOT_BUS;
2137				pmcs_complete_work_impl(pwp, pwrk, NULL, 0);
2138			} else {
2139				mutex_exit(&pwrk->lock);
2140			}
2141			continue;
2142		}
2143
2144		mutex_exit(&target->statlock);
2145		rval = pmcs_abort(pwp, phyp, pwrk->htag, 0, 1);
2146		if (rval) {
2147			pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, target,
2148			    "%s: Bad status (%d) on abort of HTAG 0x%08x",
2149			    __func__, rval, pwrk->htag);
2150			pmcs_unlock_phy(phyp);
2151			mutex_enter(&pwrk->lock);
2152			if (!PMCS_COMMAND_DONE(pwrk)) {
2153				/* Complete this command here */
2154				pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, target,
2155				    "%s: Completing cmd (htag 0x%08x) "
2156				    "anyway", __func__, pwrk->htag);
2157				if (target->dev_gone) {
2158					pwrk->dead = 1;
2159					CMD2PKT(sp)->pkt_reason = CMD_DEV_GONE;
2160					CMD2PKT(sp)->pkt_state = STATE_GOT_BUS;
2161				}
2162				pmcs_complete_work_impl(pwp, pwrk, NULL, 0);
2163			} else {
2164				mutex_exit(&pwrk->lock);
2165			}
2166			pmcs_lock_phy(phyp);
2167			/*
2168			 * No need to reschedule ABORT if we get any other
2169			 * status
2170			 */
2171			if (rval == ENOMEM) {
2172				phyp->abort_sent = 0;
2173				phyp->abort_pending = 1;
2174				SCHEDULE_WORK(pwp, PMCS_WORK_ABORT_HANDLE);
2175			}
2176		}
2177		pmcs_unlock_phy(phyp);
2178	}
2179	/*
2180	 * Run any completions that may have been queued up.
2181	 */
2182	PMCS_CQ_RUN(pwp);
2183}
2184
2185static void
2186pmcs_watchdog(void *arg)
2187{
2188	pmcs_hw_t *pwp = arg;
2189
2190	DTRACE_PROBE2(pmcs__watchdog, ulong_t, pwp->work_flags, boolean_t,
2191	    pwp->config_changed);
2192
2193	/*
2194	 * Check forward progress on the chip
2195	 */
2196	if (++pwp->watchdog_count == PMCS_FWD_PROG_TRIGGER) {
2197		pwp->watchdog_count = 0;
2198		pmcs_check_forward_progress(pwp);
2199	}
2200
2201	/*
2202	 * Check to see if we need to kick discovery off again
2203	 */
2204	mutex_enter(&pwp->config_lock);
2205	if (pwp->config_restart &&
2206	    (ddi_get_lbolt() >= pwp->config_restart_time)) {
2207		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2208		    "%s: Timer expired for re-enumeration: Start discovery",
2209		    __func__);
2210		pwp->config_restart = B_FALSE;
2211		SCHEDULE_WORK(pwp, PMCS_WORK_DISCOVER);
2212	}
2213	mutex_exit(&pwp->config_lock);
2214
2215	mutex_enter(&pwp->lock);
2216	if (pwp->state != STATE_RUNNING) {
2217		mutex_exit(&pwp->lock);
2218		return;
2219	}
2220
2221	if (atomic_cas_ulong(&pwp->work_flags, 0, 0) != 0) {
2222		if (ddi_taskq_dispatch(pwp->tq, pmcs_worker, pwp,
2223		    DDI_NOSLEEP) != DDI_SUCCESS) {
2224			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2225			    "Could not dispatch to worker thread");
2226		}
2227	}
2228	pwp->wdhandle = timeout(pmcs_watchdog, pwp,
2229	    drv_usectohz(PMCS_WATCH_INTERVAL));
2230
2231	mutex_exit(&pwp->lock);
2232
2233	pmcs_check_commands(pwp);
2234	pmcs_handle_dead_phys(pwp);
2235}
2236
2237static int
2238pmcs_remove_ihandlers(pmcs_hw_t *pwp, int icnt)
2239{
2240	int i, r, rslt = 0;
2241	for (i = 0; i < icnt; i++) {
2242		r = ddi_intr_remove_handler(pwp->ih_table[i]);
2243		if (r == DDI_SUCCESS) {
2244			continue;
2245		}
2246		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2247		    "%s: unable to remove interrupt handler %d", __func__, i);
2248		rslt = -1;
2249		break;
2250	}
2251	return (rslt);
2252}
2253
2254static int
2255pmcs_disable_intrs(pmcs_hw_t *pwp, int icnt)
2256{
2257	if (pwp->intr_cap & DDI_INTR_FLAG_BLOCK) {
2258		int r = ddi_intr_block_disable(&pwp->ih_table[0],
2259		    pwp->intr_cnt);
2260		if (r != DDI_SUCCESS) {
2261			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2262			    "unable to disable interrupt block");
2263			return (-1);
2264		}
2265	} else {
2266		int i;
2267		for (i = 0; i < icnt; i++) {
2268			if (ddi_intr_disable(pwp->ih_table[i]) == DDI_SUCCESS) {
2269				continue;
2270			}
2271			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2272			    "unable to disable interrupt %d", i);
2273			return (-1);
2274		}
2275	}
2276	return (0);
2277}
2278
2279static int
2280pmcs_free_intrs(pmcs_hw_t *pwp, int icnt)
2281{
2282	int i;
2283	for (i = 0; i < icnt; i++) {
2284		if (ddi_intr_free(pwp->ih_table[i]) == DDI_SUCCESS) {
2285			continue;
2286		}
2287		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2288		    "unable to free interrupt %d", i);
2289		return (-1);
2290	}
2291	kmem_free(pwp->ih_table, pwp->ih_table_size);
2292	pwp->ih_table_size = 0;
2293	return (0);
2294}
2295
2296/*
2297 * Try to set up interrupts of type "type" with a minimum number of interrupts
2298 * of "min".
2299 */
2300static void
2301pmcs_setup_intr_impl(pmcs_hw_t *pwp, int type, int min)
2302{
2303	int rval, avail, count, actual, max;
2304
2305	rval = ddi_intr_get_nintrs(pwp->dip, type, &count);
2306	if ((rval != DDI_SUCCESS) || (count < min)) {
2307		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2308		    "%s: get_nintrs failed; type: %d rc: %d count: %d min: %d",
2309		    __func__, type, rval, count, min);
2310		return;
2311	}
2312
2313	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2314	    "%s: nintrs = %d for type: %d", __func__, count, type);
2315
2316	rval = ddi_intr_get_navail(pwp->dip, type, &avail);
2317	if ((rval != DDI_SUCCESS) || (avail < min)) {
2318		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2319		    "%s: get_navail failed; type: %d rc: %d avail: %d min: %d",
2320		    __func__, type, rval, avail, min);
2321		return;
2322	}
2323
2324	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2325	    "%s: navail = %d for type: %d", __func__, avail, type);
2326
2327	pwp->ih_table_size = avail * sizeof (ddi_intr_handle_t);
2328	pwp->ih_table = kmem_alloc(pwp->ih_table_size, KM_SLEEP);
2329
2330	switch (type) {
2331	case DDI_INTR_TYPE_MSIX:
2332		pwp->int_type = PMCS_INT_MSIX;
2333		max = PMCS_MAX_MSIX;
2334		break;
2335	case DDI_INTR_TYPE_MSI:
2336		pwp->int_type = PMCS_INT_MSI;
2337		max = PMCS_MAX_MSI;
2338		break;
2339	case DDI_INTR_TYPE_FIXED:
2340	default:
2341		pwp->int_type = PMCS_INT_FIXED;
2342		max = PMCS_MAX_FIXED;
2343		break;
2344	}
2345
2346	rval = ddi_intr_alloc(pwp->dip, pwp->ih_table, type, 0, max, &actual,
2347	    DDI_INTR_ALLOC_NORMAL);
2348	if (rval != DDI_SUCCESS) {
2349		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2350		    "%s: ddi_intr_alloc failed; type: %d rc: %d",
2351		    __func__, type, rval);
2352		kmem_free(pwp->ih_table, pwp->ih_table_size);
2353		pwp->ih_table = NULL;
2354		pwp->ih_table_size = 0;
2355		pwp->intr_cnt = 0;
2356		pwp->int_type = PMCS_INT_NONE;
2357		return;
2358	}
2359
2360	pwp->intr_cnt = actual;
2361}
2362
2363/*
2364 * Set up interrupts.
2365 * We return one of three values:
2366 *
2367 * 0 - success
2368 * EAGAIN - failure to set up interrupts
2369 * EIO - "" + we're now stuck partly enabled
2370 *
2371 * If EIO is returned, we can't unload the driver.
2372 */
2373static int
2374pmcs_setup_intr(pmcs_hw_t *pwp)
2375{
2376	int i, r, itypes, oqv_count;
2377	ddi_intr_handler_t **iv_table;
2378	size_t iv_table_size;
2379	uint_t pri;
2380
2381	if (ddi_intr_get_supported_types(pwp->dip, &itypes) != DDI_SUCCESS) {
2382		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2383		    "cannot get interrupt types");
2384		return (EAGAIN);
2385	}
2386
2387	if (disable_msix) {
2388		itypes &= ~DDI_INTR_TYPE_MSIX;
2389	}
2390	if (disable_msi) {
2391		itypes &= ~DDI_INTR_TYPE_MSI;
2392	}
2393
2394	/*
2395	 * We won't know what firmware we're running until we call pmcs_setup,
2396	 * and we can't call pmcs_setup until we establish interrupts.
2397	 */
2398
2399	pwp->int_type = PMCS_INT_NONE;
2400
2401	/*
2402	 * We want PMCS_MAX_MSIX vectors for MSI-X.  Anything less would be
2403	 * uncivilized.
2404	 */
2405	if (itypes & DDI_INTR_TYPE_MSIX) {
2406		pmcs_setup_intr_impl(pwp, DDI_INTR_TYPE_MSIX, PMCS_MAX_MSIX);
2407		if (pwp->int_type == PMCS_INT_MSIX) {
2408			itypes = 0;
2409		}
2410	}
2411
2412	if (itypes & DDI_INTR_TYPE_MSI) {
2413		pmcs_setup_intr_impl(pwp, DDI_INTR_TYPE_MSI, 1);
2414		if (pwp->int_type == PMCS_INT_MSI) {
2415			itypes = 0;
2416		}
2417	}
2418
2419	if (itypes & DDI_INTR_TYPE_FIXED) {
2420		pmcs_setup_intr_impl(pwp, DDI_INTR_TYPE_FIXED, 1);
2421		if (pwp->int_type == PMCS_INT_FIXED) {
2422			itypes = 0;
2423		}
2424	}
2425
2426	if (pwp->intr_cnt == 0) {
2427		pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL,
2428		    "No interrupts available");
2429		return (EAGAIN);
2430	}
2431
2432	iv_table_size = sizeof (ddi_intr_handler_t *) * pwp->intr_cnt;
2433	iv_table = kmem_alloc(iv_table_size, KM_SLEEP);
2434
2435	/*
2436	 * Get iblock cookie and add handlers.
2437	 */
2438	switch (pwp->intr_cnt) {
2439	case 1:
2440		iv_table[0] = pmcs_all_intr;
2441		break;
2442	case 2:
2443		iv_table[0] = pmcs_iodone_ix;
2444		iv_table[1] = pmcs_nonio_ix;
2445		break;
2446	case 4:
2447		iv_table[PMCS_MSIX_GENERAL] = pmcs_general_ix;
2448		iv_table[PMCS_MSIX_IODONE] = pmcs_iodone_ix;
2449		iv_table[PMCS_MSIX_EVENTS] = pmcs_event_ix;
2450		iv_table[PMCS_MSIX_FATAL] = pmcs_fatal_ix;
2451		break;
2452	default:
2453		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2454		    "%s: intr_cnt = %d - unexpected", __func__, pwp->intr_cnt);
2455		kmem_free(iv_table, iv_table_size);
2456		return (EAGAIN);
2457	}
2458
2459	for (i = 0; i < pwp->intr_cnt; i++) {
2460		r = ddi_intr_add_handler(pwp->ih_table[i], iv_table[i],
2461		    (caddr_t)pwp, NULL);
2462		if (r != DDI_SUCCESS) {
2463			kmem_free(iv_table, iv_table_size);
2464			if (pmcs_remove_ihandlers(pwp, i)) {
2465				return (EIO);
2466			}
2467			if (pmcs_free_intrs(pwp, i)) {
2468				return (EIO);
2469			}
2470			pwp->intr_cnt = 0;
2471			return (EAGAIN);
2472		}
2473	}
2474
2475	kmem_free(iv_table, iv_table_size);
2476
2477	if (ddi_intr_get_cap(pwp->ih_table[0], &pwp->intr_cap) != DDI_SUCCESS) {
2478		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2479		    "unable to get int capabilities");
2480		if (pmcs_remove_ihandlers(pwp, pwp->intr_cnt)) {
2481			return (EIO);
2482		}
2483		if (pmcs_free_intrs(pwp, pwp->intr_cnt)) {
2484			return (EIO);
2485		}
2486		pwp->intr_cnt = 0;
2487		return (EAGAIN);
2488	}
2489
2490	if (pwp->intr_cap & DDI_INTR_FLAG_BLOCK) {
2491		r = ddi_intr_block_enable(&pwp->ih_table[0], pwp->intr_cnt);
2492		if (r != DDI_SUCCESS) {
2493			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2494			    "intr blk enable failed");
2495			if (pmcs_remove_ihandlers(pwp, pwp->intr_cnt)) {
2496				return (EIO);
2497			}
2498			if (pmcs_free_intrs(pwp, pwp->intr_cnt)) {
2499				return (EIO);
2500			}
2501			pwp->intr_cnt = 0;
2502			return (EFAULT);
2503		}
2504	} else {
2505		for (i = 0; i < pwp->intr_cnt; i++) {
2506			r = ddi_intr_enable(pwp->ih_table[i]);
2507			if (r == DDI_SUCCESS) {
2508				continue;
2509			}
2510			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2511			    "unable to enable interrupt %d", i);
2512			if (pmcs_disable_intrs(pwp, i)) {
2513				return (EIO);
2514			}
2515			if (pmcs_remove_ihandlers(pwp, pwp->intr_cnt)) {
2516				return (EIO);
2517			}
2518			if (pmcs_free_intrs(pwp, pwp->intr_cnt)) {
2519				return (EIO);
2520			}
2521			pwp->intr_cnt = 0;
2522			return (EAGAIN);
2523		}
2524	}
2525
2526	/*
2527	 * Set up locks.
2528	 */
2529	if (ddi_intr_get_pri(pwp->ih_table[0], &pri) != DDI_SUCCESS) {
2530		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2531		    "unable to get interrupt priority");
2532		if (pmcs_disable_intrs(pwp, pwp->intr_cnt)) {
2533			return (EIO);
2534		}
2535		if (pmcs_remove_ihandlers(pwp, pwp->intr_cnt)) {
2536			return (EIO);
2537		}
2538		if (pmcs_free_intrs(pwp, pwp->intr_cnt)) {
2539			return (EIO);
2540		}
2541		pwp->intr_cnt = 0;
2542		return (EAGAIN);
2543	}
2544
2545	pwp->locks_initted = 1;
2546	pwp->intr_pri = pri;
2547	mutex_init(&pwp->lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2548	mutex_init(&pwp->dma_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2549	mutex_init(&pwp->axil_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2550	mutex_init(&pwp->cq_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2551	mutex_init(&pwp->ict_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2552	mutex_init(&pwp->config_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2553	mutex_init(&pwp->wfree_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2554	mutex_init(&pwp->pfree_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2555	mutex_init(&pwp->dead_phylist_lock, NULL, MUTEX_DRIVER,
2556	    DDI_INTR_PRI(pri));
2557#ifdef	DEBUG
2558	mutex_init(&pwp->dbglock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2559#endif
2560	cv_init(&pwp->ict_cv, NULL, CV_DRIVER, NULL);
2561	cv_init(&pwp->drain_cv, NULL, CV_DRIVER, NULL);
2562	cv_init(&pwp->config_cv, NULL, CV_DRIVER, NULL);
2563	for (i = 0; i < PMCS_NIQ; i++) {
2564		mutex_init(&pwp->iqp_lock[i], NULL,
2565		    MUTEX_DRIVER, DDI_INTR_PRI(pwp->intr_pri));
2566	}
2567	for (i = 0; i < pwp->cq_info.cq_threads; i++) {
2568		mutex_init(&pwp->cq_info.cq_thr_info[i].cq_thr_lock, NULL,
2569		    MUTEX_DRIVER, DDI_INTR_PRI(pwp->intr_pri));
2570		cv_init(&pwp->cq_info.cq_thr_info[i].cq_cv, NULL,
2571		    CV_DRIVER, NULL);
2572	}
2573
2574	pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, "%d %s interrup%s configured",
2575	    pwp->intr_cnt, (pwp->int_type == PMCS_INT_MSIX)? "MSI-X" :
2576	    ((pwp->int_type == PMCS_INT_MSI)? "MSI" : "INT-X"),
2577	    pwp->intr_cnt == 1? "t" : "ts");
2578
2579
2580	/*
2581	 * Enable Interrupts
2582	 */
2583	if (pwp->intr_cnt > PMCS_NOQ) {
2584		oqv_count = pwp->intr_cnt;
2585	} else {
2586		oqv_count = PMCS_NOQ;
2587	}
2588	for (pri = 0xffffffff, i = 0; i < oqv_count; i++) {
2589		pri ^= (1 << i);
2590	}
2591
2592	mutex_enter(&pwp->lock);
2593	pwp->intr_mask = pri;
2594	pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_MASK, pwp->intr_mask);
2595	pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR, 0xffffffff);
2596	mutex_exit(&pwp->lock);
2597
2598	return (0);
2599}
2600
2601static int
2602pmcs_teardown_intr(pmcs_hw_t *pwp)
2603{
2604	if (pwp->intr_cnt) {
2605		if (pmcs_disable_intrs(pwp, pwp->intr_cnt)) {
2606			return (EIO);
2607		}
2608		if (pmcs_remove_ihandlers(pwp, pwp->intr_cnt)) {
2609			return (EIO);
2610		}
2611		if (pmcs_free_intrs(pwp, pwp->intr_cnt)) {
2612			return (EIO);
2613		}
2614		pwp->intr_cnt = 0;
2615	}
2616	return (0);
2617}
2618
2619static uint_t
2620pmcs_general_ix(caddr_t arg1, caddr_t arg2)
2621{
2622	pmcs_hw_t *pwp = (pmcs_hw_t *)((void *)arg1);
2623	_NOTE(ARGUNUSED(arg2));
2624	pmcs_general_intr(pwp);
2625	return (DDI_INTR_CLAIMED);
2626}
2627
2628static uint_t
2629pmcs_event_ix(caddr_t arg1, caddr_t arg2)
2630{
2631	pmcs_hw_t *pwp = (pmcs_hw_t *)((void *)arg1);
2632	_NOTE(ARGUNUSED(arg2));
2633	pmcs_event_intr(pwp);
2634	return (DDI_INTR_CLAIMED);
2635}
2636
2637static uint_t
2638pmcs_iodone_ix(caddr_t arg1, caddr_t arg2)
2639{
2640	_NOTE(ARGUNUSED(arg2));
2641	pmcs_hw_t *pwp = (pmcs_hw_t *)((void *)arg1);
2642
2643	/*
2644	 * It's possible that if we just turned interrupt coalescing off
2645	 * (and thus, re-enabled auto clear for interrupts on the I/O outbound
2646	 * queue) that there was an interrupt already pending.  We use
2647	 * io_intr_coal.int_cleared to ensure that we still drop in here and
2648	 * clear the appropriate interrupt bit one last time.
2649	 */
2650	mutex_enter(&pwp->ict_lock);
2651	if (pwp->io_intr_coal.timer_on ||
2652	    (pwp->io_intr_coal.int_cleared == B_FALSE)) {
2653		pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR,
2654		    (1 << PMCS_OQ_IODONE));
2655		pwp->io_intr_coal.int_cleared = B_TRUE;
2656	}
2657	mutex_exit(&pwp->ict_lock);
2658
2659	pmcs_iodone_intr(pwp);
2660
2661	return (DDI_INTR_CLAIMED);
2662}
2663
2664static uint_t
2665pmcs_fatal_ix(caddr_t arg1, caddr_t arg2)
2666{
2667	pmcs_hw_t *pwp = (pmcs_hw_t *)((void *)arg1);
2668	_NOTE(ARGUNUSED(arg2));
2669	pmcs_fatal_handler(pwp);
2670	return (DDI_INTR_CLAIMED);
2671}
2672
2673static uint_t
2674pmcs_nonio_ix(caddr_t arg1, caddr_t arg2)
2675{
2676	_NOTE(ARGUNUSED(arg2));
2677	pmcs_hw_t *pwp = (void *)arg1;
2678	uint32_t obdb = pmcs_rd_msgunit(pwp, PMCS_MSGU_OBDB);
2679
2680	/*
2681	 * Check for Fatal Interrupts
2682	 */
2683	if (obdb & (1 << PMCS_FATAL_INTERRUPT)) {
2684		pmcs_fatal_handler(pwp);
2685		return (DDI_INTR_CLAIMED);
2686	}
2687
2688	if (obdb & (1 << PMCS_OQ_GENERAL)) {
2689		pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR,
2690		    (1 << PMCS_OQ_GENERAL));
2691		pmcs_general_intr(pwp);
2692		pmcs_event_intr(pwp);
2693	}
2694
2695	return (DDI_INTR_CLAIMED);
2696}
2697
2698static uint_t
2699pmcs_all_intr(caddr_t arg1, caddr_t arg2)
2700{
2701	_NOTE(ARGUNUSED(arg2));
2702	pmcs_hw_t *pwp = (void *) arg1;
2703	uint32_t obdb;
2704	int handled = 0;
2705
2706	obdb = pmcs_rd_msgunit(pwp, PMCS_MSGU_OBDB);
2707
2708	/*
2709	 * Check for Fatal Interrupts
2710	 */
2711	if (obdb & (1 << PMCS_FATAL_INTERRUPT)) {
2712		pmcs_fatal_handler(pwp);
2713		return (DDI_INTR_CLAIMED);
2714	}
2715
2716	/*
2717	 * Check for Outbound Queue service needed
2718	 */
2719	if (obdb & (1 << PMCS_OQ_IODONE)) {
2720		pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR,
2721		    (1 << PMCS_OQ_IODONE));
2722		obdb ^= (1 << PMCS_OQ_IODONE);
2723		handled++;
2724		pmcs_iodone_intr(pwp);
2725	}
2726	if (obdb & (1 << PMCS_OQ_GENERAL)) {
2727		pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR,
2728		    (1 << PMCS_OQ_GENERAL));
2729		obdb ^= (1 << PMCS_OQ_GENERAL);
2730		handled++;
2731		pmcs_general_intr(pwp);
2732	}
2733	if (obdb & (1 << PMCS_OQ_EVENTS)) {
2734		pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR,
2735		    (1 << PMCS_OQ_EVENTS));
2736		obdb ^= (1 << PMCS_OQ_EVENTS);
2737		handled++;
2738		pmcs_event_intr(pwp);
2739	}
2740	if (obdb) {
2741		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2742		    "interrupt bits not handled (0x%x)", obdb);
2743		pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR, obdb);
2744		handled++;
2745	}
2746	if (pwp->int_type == PMCS_INT_MSI) {
2747		handled++;
2748	}
2749	return (handled? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED);
2750}
2751
2752void
2753pmcs_fatal_handler(pmcs_hw_t *pwp)
2754{
2755	pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL, "Fatal Interrupt caught");
2756
2757	mutex_enter(&pwp->lock);
2758	pwp->state = STATE_DEAD;
2759
2760	/*
2761	 * Attempt a hot reset. In case of failure, pmcs_hot_reset() will
2762	 * handle the failure and issue any required FM notifications.
2763	 * See pmcs_subr.c for more details.
2764	 */
2765	if (pmcs_hot_reset(pwp)) {
2766		pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL,
2767		    "%s: hot reset failure", __func__);
2768	} else {
2769		pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL,
2770		    "%s: hot reset complete", __func__);
2771		pwp->last_reset_reason = PMCS_LAST_RST_FATAL_ERROR;
2772	}
2773	mutex_exit(&pwp->lock);
2774}
2775
2776/*
2777 * Called with PHY lock and target statlock held and scratch acquired.
2778 */
2779boolean_t
2780pmcs_assign_device(pmcs_hw_t *pwp, pmcs_xscsi_t *tgt)
2781{
2782	pmcs_phy_t *pptr = tgt->phy;
2783
2784	switch (pptr->dtype) {
2785	case SAS:
2786	case EXPANDER:
2787		break;
2788	case SATA:
2789		tgt->ca = 1;
2790		break;
2791	default:
2792		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, tgt,
2793		    "%s: Target %p has PHY %p with invalid dtype",
2794		    __func__, (void *)tgt, (void *)pptr);
2795		return (B_FALSE);
2796	}
2797
2798	tgt->new = 1;
2799	tgt->dev_gone = 0;
2800	tgt->recover_wait = 0;
2801
2802	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, tgt,
2803	    "%s: config %s vtgt %u for " SAS_ADDR_FMT, __func__,
2804	    pptr->path, tgt->target_num, SAS_ADDR_PRT(pptr->sas_address));
2805
2806	if (pmcs_add_new_device(pwp, tgt) != B_TRUE) {
2807		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, tgt,
2808		    "%s: Failed for vtgt %u / WWN " SAS_ADDR_FMT, __func__,
2809		    tgt->target_num, SAS_ADDR_PRT(pptr->sas_address));
2810		mutex_destroy(&tgt->statlock);
2811		mutex_destroy(&tgt->wqlock);
2812		mutex_destroy(&tgt->aqlock);
2813		return (B_FALSE);
2814	}
2815
2816	return (B_TRUE);
2817}
2818
2819/*
2820 * Called with softstate lock held
2821 */
2822void
2823pmcs_remove_device(pmcs_hw_t *pwp, pmcs_phy_t *pptr)
2824{
2825	pmcs_xscsi_t *xp;
2826	unsigned int vtgt;
2827
2828	ASSERT(mutex_owned(&pwp->lock));
2829
2830	for (vtgt = 0; vtgt < pwp->max_dev; vtgt++) {
2831		xp = pwp->targets[vtgt];
2832		if (xp == NULL) {
2833			continue;
2834		}
2835
2836		mutex_enter(&xp->statlock);
2837		if (xp->phy == pptr) {
2838			if (xp->new) {
2839				xp->new = 0;
2840				pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, xp,
2841				    "cancel config of vtgt %u", vtgt);
2842			} else {
2843				pmcs_clear_xp(pwp, xp);
2844				pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, xp,
2845				    "Removed tgt 0x%p vtgt %u",
2846				    (void *)xp, vtgt);
2847			}
2848			mutex_exit(&xp->statlock);
2849			break;
2850		}
2851		mutex_exit(&xp->statlock);
2852	}
2853}
2854
2855void
2856pmcs_prt_impl(pmcs_hw_t *pwp, pmcs_prt_level_t level,
2857    pmcs_phy_t *phyp, pmcs_xscsi_t *target, const char *fmt, ...)
2858{
2859	va_list	ap;
2860	int written = 0;
2861	char *ptr;
2862	uint32_t elem_size = PMCS_TBUF_ELEM_SIZE - 1;
2863	boolean_t system_log;
2864	int system_log_level;
2865	hrtime_t hrtimestamp;
2866
2867	switch (level) {
2868	case PMCS_PRT_DEBUG_DEVEL:
2869	case PMCS_PRT_DEBUG_DEV_STATE:
2870	case PMCS_PRT_DEBUG_PHY_LOCKING:
2871	case PMCS_PRT_DEBUG_SCSI_STATUS:
2872	case PMCS_PRT_DEBUG_UNDERFLOW:
2873	case PMCS_PRT_DEBUG_CONFIG:
2874	case PMCS_PRT_DEBUG_IPORT:
2875	case PMCS_PRT_DEBUG_MAP:
2876	case PMCS_PRT_DEBUG3:
2877	case PMCS_PRT_DEBUG2:
2878	case PMCS_PRT_DEBUG1:
2879	case PMCS_PRT_DEBUG:
2880		system_log = B_FALSE;
2881		break;
2882	case PMCS_PRT_INFO:
2883		system_log = B_TRUE;
2884		system_log_level = CE_CONT;
2885		break;
2886	case PMCS_PRT_WARN:
2887		system_log = B_TRUE;
2888		system_log_level = CE_NOTE;
2889		break;
2890	case PMCS_PRT_ERR:
2891		system_log = B_TRUE;
2892		system_log_level = CE_WARN;
2893		break;
2894	default:
2895		return;
2896	}
2897
2898	mutex_enter(&pmcs_trace_lock);
2899	hrtimestamp = gethrtime();
2900	gethrestime(&pmcs_tbuf_ptr->timestamp);
2901
2902	if (pwp->fw_timestamp != 0) {
2903		/* Calculate the approximate firmware time stamp... */
2904		pmcs_tbuf_ptr->fw_timestamp = pwp->fw_timestamp +
2905		    ((hrtimestamp - pwp->hrtimestamp) / PMCS_FWLOG_TIMER_DIV);
2906	} else {
2907		pmcs_tbuf_ptr->fw_timestamp = 0;
2908	}
2909
2910	ptr = pmcs_tbuf_ptr->buf;
2911
2912	/*
2913	 * Store the pertinent PHY and target information if there is any
2914	 */
2915	if (target == NULL) {
2916		pmcs_tbuf_ptr->target_num = PMCS_INVALID_TARGET_NUM;
2917		pmcs_tbuf_ptr->target_ua[0] = '\0';
2918	} else {
2919		pmcs_tbuf_ptr->target_num = target->target_num;
2920		(void) strncpy(pmcs_tbuf_ptr->target_ua, target->ua,
2921		    PMCS_TBUF_UA_MAX_SIZE);
2922	}
2923
2924	if (phyp == NULL) {
2925		(void) memset(pmcs_tbuf_ptr->phy_sas_address, 0, 8);
2926		pmcs_tbuf_ptr->phy_path[0] = '\0';
2927		pmcs_tbuf_ptr->phy_dtype = NOTHING;
2928	} else {
2929		(void) memcpy(pmcs_tbuf_ptr->phy_sas_address,
2930		    phyp->sas_address, 8);
2931		(void) strncpy(pmcs_tbuf_ptr->phy_path, phyp->path, 32);
2932		pmcs_tbuf_ptr->phy_dtype = phyp->dtype;
2933	}
2934
2935	written += snprintf(ptr, elem_size, "pmcs%d:%d: ",
2936	    ddi_get_instance(pwp->dip), level);
2937	ptr += strlen(ptr);
2938	va_start(ap, fmt);
2939	written += vsnprintf(ptr, elem_size - written, fmt, ap);
2940	va_end(ap);
2941	if (written > elem_size - 1) {
2942		/* Indicate truncation */
2943		pmcs_tbuf_ptr->buf[elem_size - 1] = '+';
2944	}
2945	if (++pmcs_tbuf_idx == pmcs_tbuf_num_elems) {
2946		pmcs_tbuf_ptr = pmcs_tbuf;
2947		pmcs_tbuf_wrap = B_TRUE;
2948		pmcs_tbuf_idx = 0;
2949	} else {
2950		++pmcs_tbuf_ptr;
2951	}
2952	mutex_exit(&pmcs_trace_lock);
2953
2954	/*
2955	 * When pmcs_force_syslog in non-zero, everything goes also
2956	 * to syslog, at CE_CONT level.
2957	 */
2958	if (pmcs_force_syslog) {
2959		system_log = B_TRUE;
2960		system_log_level = CE_CONT;
2961	}
2962
2963	/*
2964	 * Anything that comes in with PMCS_PRT_INFO, WARN, or ERR also
2965	 * goes to syslog.
2966	 */
2967	if (system_log) {
2968		char local[196];
2969
2970		switch (system_log_level) {
2971		case CE_CONT:
2972			(void) snprintf(local, sizeof (local), "%sINFO: ",
2973			    pmcs_console ? "" : "?");
2974			break;
2975		case CE_NOTE:
2976		case CE_WARN:
2977			local[0] = '\0';
2978			break;
2979		default:
2980			return;
2981		}
2982
2983		ptr = local;
2984		ptr += strlen(local);
2985		(void) snprintf(ptr, (sizeof (local)) -
2986		    ((size_t)ptr - (size_t)local), "pmcs%d: ",
2987		    ddi_get_instance(pwp->dip));
2988		ptr += strlen(ptr);
2989		va_start(ap, fmt);
2990		(void) vsnprintf(ptr,
2991		    (sizeof (local)) - ((size_t)ptr - (size_t)local), fmt, ap);
2992		va_end(ap);
2993		if (level == CE_CONT) {
2994			(void) strlcat(local, "\n", sizeof (local));
2995		}
2996		cmn_err(system_log_level, local);
2997	}
2998
2999}
3000
3001/*
3002 * pmcs_acquire_scratch
3003 *
3004 * If "wait" is true, the caller will wait until it can acquire the scratch.
3005 * This implies the caller needs to be in a context where spinning for an
3006 * indeterminate amount of time is acceptable.
3007 */
3008int
3009pmcs_acquire_scratch(pmcs_hw_t *pwp, boolean_t wait)
3010{
3011	int rval;
3012
3013	if (!wait) {
3014		return (atomic_swap_8(&pwp->scratch_locked, 1));
3015	}
3016
3017	/*
3018	 * Caller will wait for scratch.
3019	 */
3020	while ((rval = atomic_swap_8(&pwp->scratch_locked, 1)) != 0) {
3021		drv_usecwait(100);
3022	}
3023
3024	return (rval);
3025}
3026
3027void
3028pmcs_release_scratch(pmcs_hw_t *pwp)
3029{
3030	pwp->scratch_locked = 0;
3031}
3032
3033/* Called with iport_lock and phy lock held */
3034void
3035pmcs_create_one_phy_stats(pmcs_iport_t *iport, pmcs_phy_t *phyp)
3036{
3037	sas_phy_stats_t		*ps;
3038	pmcs_hw_t		*pwp;
3039	int			ndata;
3040	char			ks_name[KSTAT_STRLEN];
3041
3042	ASSERT(mutex_owned(&iport->lock));
3043	pwp = iport->pwp;
3044	ASSERT(pwp != NULL);
3045	ASSERT(mutex_owned(&phyp->phy_lock));
3046
3047	if (phyp->phy_stats != NULL) {
3048		/*
3049		 * Delete existing kstats with name containing
3050		 * old iport instance# and allow creation of
3051		 * new kstats with new iport instance# in the name.
3052		 */
3053		kstat_delete(phyp->phy_stats);
3054	}
3055
3056	ndata = (sizeof (sas_phy_stats_t)/sizeof (kstat_named_t));
3057
3058	(void) snprintf(ks_name, sizeof (ks_name),
3059	    "%s.%llx.%d.%d", ddi_driver_name(iport->dip),
3060	    (longlong_t)pwp->sas_wwns[0],
3061	    ddi_get_instance(iport->dip), phyp->phynum);
3062
3063	phyp->phy_stats = kstat_create("pmcs",
3064	    ddi_get_instance(iport->dip), ks_name, KSTAT_SAS_PHY_CLASS,
3065	    KSTAT_TYPE_NAMED, ndata, 0);
3066
3067	if (phyp->phy_stats == NULL) {
3068		pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, NULL,
3069		    "%s: Failed to create %s kstats for PHY(0x%p) at %s",
3070		    __func__, ks_name, (void *)phyp, phyp->path);
3071		return;
3072	}
3073
3074	ps = (sas_phy_stats_t *)phyp->phy_stats->ks_data;
3075
3076	kstat_named_init(&ps->seconds_since_last_reset,
3077	    "SecondsSinceLastReset", KSTAT_DATA_ULONGLONG);
3078	kstat_named_init(&ps->tx_frames,
3079	    "TxFrames", KSTAT_DATA_ULONGLONG);
3080	kstat_named_init(&ps->rx_frames,
3081	    "RxFrames", KSTAT_DATA_ULONGLONG);
3082	kstat_named_init(&ps->tx_words,
3083	    "TxWords", KSTAT_DATA_ULONGLONG);
3084	kstat_named_init(&ps->rx_words,
3085	    "RxWords", KSTAT_DATA_ULONGLONG);
3086	kstat_named_init(&ps->invalid_dword_count,
3087	    "InvalidDwordCount", KSTAT_DATA_ULONGLONG);
3088	kstat_named_init(&ps->running_disparity_error_count,
3089	    "RunningDisparityErrorCount", KSTAT_DATA_ULONGLONG);
3090	kstat_named_init(&ps->loss_of_dword_sync_count,
3091	    "LossofDwordSyncCount", KSTAT_DATA_ULONGLONG);
3092	kstat_named_init(&ps->phy_reset_problem_count,
3093	    "PhyResetProblemCount", KSTAT_DATA_ULONGLONG);
3094
3095	phyp->phy_stats->ks_private = phyp;
3096	phyp->phy_stats->ks_update = pmcs_update_phy_stats;
3097	kstat_install(phyp->phy_stats);
3098}
3099
3100static void
3101pmcs_create_all_phy_stats(pmcs_iport_t *iport)
3102{
3103	pmcs_hw_t		*pwp;
3104	pmcs_phy_t		*phyp;
3105
3106	ASSERT(iport != NULL);
3107	pwp = iport->pwp;
3108	ASSERT(pwp != NULL);
3109
3110	mutex_enter(&iport->lock);
3111
3112	for (phyp = list_head(&iport->phys);
3113	    phyp != NULL;
3114	    phyp = list_next(&iport->phys, phyp)) {
3115
3116		mutex_enter(&phyp->phy_lock);
3117		pmcs_create_one_phy_stats(iport, phyp);
3118		mutex_exit(&phyp->phy_lock);
3119	}
3120
3121	mutex_exit(&iport->lock);
3122}
3123
3124int
3125pmcs_update_phy_stats(kstat_t *ks, int rw)
3126{
3127	int		val, ret = DDI_FAILURE;
3128	pmcs_phy_t	*pptr = (pmcs_phy_t *)ks->ks_private;
3129	pmcs_hw_t	*pwp = pptr->pwp;
3130	sas_phy_stats_t	*ps = ks->ks_data;
3131
3132	_NOTE(ARGUNUSED(rw));
3133	ASSERT((pptr != NULL) && (pwp != NULL));
3134
3135	/*
3136	 * We just want to lock against other invocations of kstat;
3137	 * we don't need to pmcs_lock_phy() for this.
3138	 */
3139	mutex_enter(&pptr->phy_lock);
3140
3141	/* Get Stats from Chip */
3142	val = pmcs_get_diag_report(pwp, PMCS_INVALID_DWORD_CNT, pptr->phynum);
3143	if (val == DDI_FAILURE)
3144		goto fail;
3145	ps->invalid_dword_count.value.ull = (unsigned long long)val;
3146
3147	val = pmcs_get_diag_report(pwp, PMCS_DISPARITY_ERR_CNT, pptr->phynum);
3148	if (val == DDI_FAILURE)
3149		goto fail;
3150	ps->running_disparity_error_count.value.ull = (unsigned long long)val;
3151
3152	val = pmcs_get_diag_report(pwp, PMCS_LOST_DWORD_SYNC_CNT, pptr->phynum);
3153	if (val == DDI_FAILURE)
3154		goto fail;
3155	ps->loss_of_dword_sync_count.value.ull = (unsigned long long)val;
3156
3157	val = pmcs_get_diag_report(pwp, PMCS_RESET_FAILED_CNT, pptr->phynum);
3158	if (val == DDI_FAILURE)
3159		goto fail;
3160	ps->phy_reset_problem_count.value.ull = (unsigned long long)val;
3161
3162	ret = DDI_SUCCESS;
3163fail:
3164	mutex_exit(&pptr->phy_lock);
3165	return (ret);
3166}
3167
3168/*ARGSUSED*/
3169static int
3170pmcs_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
3171{
3172	/*
3173	 * as the driver can always deal with an error in any dma or
3174	 * access handle, we can just return the fme_status value.
3175	 */
3176	pci_ereport_post(dip, err, NULL);
3177	return (err->fme_status);
3178}
3179
3180static void
3181pmcs_fm_init(pmcs_hw_t *pwp)
3182{
3183	ddi_iblock_cookie_t	fm_ibc;
3184
3185	/* Only register with IO Fault Services if we have some capability */
3186	if (pwp->fm_capabilities) {
3187		/* Adjust access and dma attributes for FMA */
3188		pwp->reg_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
3189		pwp->iqp_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3190		pwp->oqp_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3191		pwp->cip_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3192		pwp->fwlog_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3193
3194		/*
3195		 * Register capabilities with IO Fault Services.
3196		 */
3197		ddi_fm_init(pwp->dip, &pwp->fm_capabilities, &fm_ibc);
3198
3199		/*
3200		 * Initialize pci ereport capabilities if ereport
3201		 * capable (should always be.)
3202		 */
3203		if (DDI_FM_EREPORT_CAP(pwp->fm_capabilities) ||
3204		    DDI_FM_ERRCB_CAP(pwp->fm_capabilities)) {
3205			pci_ereport_setup(pwp->dip);
3206		}
3207
3208		/*
3209		 * Register error callback if error callback capable.
3210		 */
3211		if (DDI_FM_ERRCB_CAP(pwp->fm_capabilities)) {
3212			ddi_fm_handler_register(pwp->dip,
3213			    pmcs_fm_error_cb, (void *) pwp);
3214		}
3215	}
3216}
3217
3218static void
3219pmcs_fm_fini(pmcs_hw_t *pwp)
3220{
3221	/* Only unregister FMA capabilities if registered */
3222	if (pwp->fm_capabilities) {
3223		/*
3224		 * Un-register error callback if error callback capable.
3225		 */
3226		if (DDI_FM_ERRCB_CAP(pwp->fm_capabilities)) {
3227			ddi_fm_handler_unregister(pwp->dip);
3228		}
3229
3230		/*
3231		 * Release any resources allocated by pci_ereport_setup()
3232		 */
3233		if (DDI_FM_EREPORT_CAP(pwp->fm_capabilities) ||
3234		    DDI_FM_ERRCB_CAP(pwp->fm_capabilities)) {
3235			pci_ereport_teardown(pwp->dip);
3236		}
3237
3238		/* Unregister from IO Fault Services */
3239		ddi_fm_fini(pwp->dip);
3240
3241		/* Adjust access and dma attributes for FMA */
3242		pwp->reg_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC;
3243		pwp->iqp_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3244		pwp->oqp_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3245		pwp->cip_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3246		pwp->fwlog_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3247	}
3248}
3249
3250static boolean_t
3251pmcs_fabricate_wwid(pmcs_hw_t *pwp)
3252{
3253	char *cp, c;
3254	uint64_t adr;
3255	int i;
3256
3257	cp = &c;
3258	(void) ddi_strtoul(hw_serial, &cp, 10, (unsigned long *)&adr);
3259
3260	if (adr == 0) {
3261		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
3262		    "%s: No serial number available to fabricate WWN",
3263		    __func__);
3264
3265		adr = (uint64_t)gethrtime();
3266	}
3267
3268	adr <<= 8;
3269	adr |= ((uint64_t)ddi_get_instance(pwp->dip) << 52);
3270	adr |= (5ULL << 60);
3271
3272	for (i = 0; i < PMCS_MAX_PORTS; i++) {
3273		pwp->sas_wwns[i] = adr + i;
3274	}
3275
3276	return (B_TRUE);
3277}
3278