1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25/*
26 * This file contains functions that are called via interrupts.
27 */
28
29#include <sys/scsi/adapters/pmcs/pmcs.h>
30
31#ifdef	DEBUG
32#define	VALID_IOMB_CHECK(p, w, m, b, c)					\
33	if (!(w & PMCS_IOMB_VALID)) {					\
34		char l[64];						\
35		(void) snprintf(l, sizeof (l),				\
36		    "%s: INVALID IOMB (oq_ci=%u oq_pi=%u)", __func__, b, c); \
37		pmcs_print_entry(pwp, PMCS_PRT_DEBUG, l, m);		\
38		STEP_OQ_ENTRY(pwp, PMCS_OQ_EVENTS, b, 1);		\
39		continue;						\
40	}
41#define	WRONG_OBID_CHECK(pwp, w, q)	\
42	if (((w & PMCS_IOMB_OBID_MASK) >> PMCS_IOMB_OBID_SHIFT) != q) {	\
43		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,		\
44		    "%s: COMPLETION WITH WRONG OBID (0x%x)", __func__,	\
45		    (w & PMCS_IOMB_OBID_MASK) >> PMCS_IOMB_OBID_SHIFT);	\
46	}
47#else
48#define	VALID_IOMB_CHECK(a, b, c, d, e)
49#define	WRONG_OBID_CHECK(a, b, c)
50#endif
51
52#define	OQLIM_CHECK(p, l)				\
53	if (++l == (p)->ioq_depth) {			\
54		pmcs_prt(p, PMCS_PRT_DEBUG, NULL, NULL,	\
55		    "%s: possible ob queue overflow",	\
56		    __func__);				\
57		break;					\
58	}
59
60#define	COPY_OUTBOUND(p, w, l, n, a, x, q, c)				\
61	n = ((w & PMCS_IOMB_BC_MASK) >> PMCS_IOMB_BC_SHIFT);		\
62	a = PMCS_QENTRY_SIZE;						\
63	(void) memcpy(l, x, PMCS_QENTRY_SIZE);				\
64	if (n > 1) {							\
65		a <<= 1;						\
66		(void) memcpy(&l[PMCS_QENTRY_SIZE],			\
67		    GET_OQ_ENTRY(p, q, c, 1), PMCS_QENTRY_SIZE);	\
68	}								\
69	pmcs_prt(p, PMCS_PRT_DEBUG3, NULL, NULL,			\
70	    "%s: ptr %p ci %d w0 %x nbuf %d",				\
71	    __func__, (void *)x, ci, w0, n)
72
73#define	EVT_PRT(hwp, msg, phy)	\
74	pmcs_prt(hwp, PMCS_PRT_DEBUG, NULL, NULL, "Phy 0x%x: %s", phy, # msg)
75
76
77/*
78 * Map the link rate reported in the event to the SAS link rate value
79 */
80static uint8_t
81pmcs_link_rate(uint32_t event_link_rate)
82{
83	uint8_t sas_link_rate = 0;
84
85	switch (event_link_rate) {
86	case 1:
87		sas_link_rate = SAS_LINK_RATE_1_5GBIT;
88		break;
89	case 2:
90		sas_link_rate = SAS_LINK_RATE_3GBIT;
91		break;
92	case 4:
93		sas_link_rate = SAS_LINK_RATE_6GBIT;
94		break;
95	}
96
97	return (sas_link_rate);
98}
99
100/*
101 * Called with pwrk lock
102 */
103static void
104pmcs_complete_work(pmcs_hw_t *pwp, pmcwork_t *pwrk, uint32_t *iomb, size_t amt)
105{
106#ifdef	DEBUG
107	pwp->ltime[pwp->lti] = gethrtime();
108	pwp->ltags[pwp->lti++] = pwrk->htag;
109#endif
110	pwrk->htag |= PMCS_TAG_DONE;
111
112	/*
113	 * If the command has timed out, leave it in that state.
114	 */
115	if (pwrk->state != PMCS_WORK_STATE_TIMED_OUT) {
116		pwrk->state = PMCS_WORK_STATE_INTR;
117		pwrk->onwire = 0;
118	}
119
120	pmcs_complete_work_impl(pwp, pwrk, iomb, amt);
121}
122
123static void
124pmcs_work_not_found(pmcs_hw_t *pwp, uint32_t htag, uint32_t *iomb)
125{
126#ifdef	DEBUG
127	int i;
128	hrtime_t now;
129	char buf[64];
130
131	(void) snprintf(buf, sizeof (buf),
132	    "unable to find work structure for tag 0x%x", htag);
133
134	pmcs_print_entry(pwp, PMCS_PRT_DEBUG1, buf, iomb);
135	if (htag == 0) {
136		return;
137	}
138	now = gethrtime();
139	for (i = 0; i < 256; i++) {
140		mutex_enter(&pwp->dbglock);
141		if (pwp->ltags[i] == htag) {
142			pmcs_prt(pwp, PMCS_PRT_DEBUG1, NULL, NULL,
143			    "same tag already completed (%llu us ago)",
144			    (unsigned long long) (now - pwp->ltime[i]));
145		}
146		if (pwp->ftags[i] == htag) {
147			pmcs_prt(pwp, PMCS_PRT_DEBUG1, NULL, NULL,
148			    "same tag started (line %d) (%llu ns ago)",
149			    pwp->ftag_lines[i], (unsigned long long)
150			    (now - pwp->ftime[i]));
151		}
152		mutex_exit(&pwp->dbglock);
153	}
154#else
155	char buf[64];
156	(void) snprintf(buf, sizeof (buf),
157	    "unable to find work structure for tag 0x%x", htag);
158	pmcs_print_entry(pwp, PMCS_PRT_DEBUG1, buf, iomb);
159#endif
160}
161
162
163static void
164pmcs_process_io_completion(pmcs_hw_t *pwp, pmcs_iocomp_cb_t *ioccb, size_t amt)
165{
166	pmcwork_t *pwrk;
167	uint32_t tag_type;
168	uint32_t htag = LE_32(((uint32_t *)((void *)ioccb->iomb))[1]);
169
170	pwrk = pmcs_tag2wp(pwp, htag, B_FALSE);
171	if (pwrk == NULL) {
172		pmcs_work_not_found(pwp, htag, (void *)&ioccb->iomb);
173		kmem_cache_free(pwp->iocomp_cb_cache, ioccb);
174		return;
175	}
176
177	pwrk->htag |= PMCS_TAG_DONE;
178
179	/*
180	 * If the command has timed out, leave it in that state.
181	 */
182	if (pwrk->state != PMCS_WORK_STATE_TIMED_OUT) {
183		pwrk->state = PMCS_WORK_STATE_INTR;
184		pwrk->onwire = 0;
185	}
186
187	/*
188	 * Some SATA and SAS commands are run in "WAIT" mode.
189	 * We can tell this from the tag type. In this case,
190	 * we just do a wakeup (not a callback).
191	 */
192	tag_type = PMCS_TAG_TYPE(pwrk->htag);
193	if (tag_type == PMCS_TAG_TYPE_WAIT) {
194		ASSERT(PMCS_TAG_TYPE(pwrk->htag) == PMCS_TAG_TYPE_WAIT);
195		if (pwrk->arg && amt) {
196			(void) memcpy(pwrk->arg, ioccb->iomb, amt);
197		}
198		cv_signal(&pwrk->sleep_cv);
199		mutex_exit(&pwrk->lock);
200		kmem_cache_free(pwp->iocomp_cb_cache, ioccb);
201		return;
202	}
203	ASSERT(tag_type == PMCS_TAG_TYPE_CBACK);
204
205#ifdef	DEBUG
206	pwp->ltime[pwp->lti] = gethrtime();
207	pwp->ltags[pwp->lti++] = pwrk->htag;
208#endif
209
210	ioccb->pwrk = pwrk;
211
212	/*
213	 * Only update state to IOCOMPQ if we were in the INTR state.
214	 * Any other state (e.g. TIMED_OUT, ABORTED) needs to remain.
215	 */
216	if (pwrk->state == PMCS_WORK_STATE_INTR) {
217		pwrk->state = PMCS_WORK_STATE_IOCOMPQ;
218	}
219
220	mutex_enter(&pwp->cq_lock);
221	if (pwp->iocomp_cb_tail) {
222		pwp->iocomp_cb_tail->next = ioccb;
223		pwp->iocomp_cb_tail = ioccb;
224	} else {
225		pwp->iocomp_cb_head = ioccb;
226		pwp->iocomp_cb_tail = ioccb;
227	}
228	ioccb->next = NULL;
229	mutex_exit(&pwp->cq_lock);
230
231	mutex_exit(&pwrk->lock);
232	/* Completion queue will be run at end of pmcs_iodone_intr */
233}
234
235
236static void
237pmcs_process_completion(pmcs_hw_t *pwp, void *iomb, size_t amt)
238{
239	pmcwork_t *pwrk;
240	uint32_t htag = LE_32(((uint32_t *)iomb)[1]);
241
242	pwrk = pmcs_tag2wp(pwp, htag, B_FALSE);
243	if (pwrk == NULL) {
244		pmcs_work_not_found(pwp, htag, iomb);
245		return;
246	}
247
248	pmcs_complete_work(pwp, pwrk, iomb, amt);
249	/*
250	 * The pwrk lock is now released
251	 */
252}
253
254static void
255pmcs_kill_port(pmcs_hw_t *pwp, int portid)
256{
257	pmcs_phy_t *pptr = pwp->ports[portid];
258
259	if (pptr == NULL) {
260		return;
261	}
262
263	/*
264	 * Clear any subsidiary phys
265	 */
266	mutex_enter(&pwp->lock);
267
268	for (pptr = pwp->root_phys; pptr; pptr = pptr->sibling) {
269		pmcs_lock_phy(pptr);
270		if (pptr->link_rate && pptr->portid == portid &&
271		    pptr->subsidiary) {
272			pmcs_clear_phy(pwp, pptr);
273		}
274		pmcs_unlock_phy(pptr);
275	}
276
277	pptr = pwp->ports[portid];
278	pwp->ports[portid] = NULL;
279	mutex_exit(&pwp->lock);
280
281	pmcs_lock_phy(pptr);
282	pmcs_kill_changed(pwp, pptr, 0);
283	pmcs_unlock_phy(pptr);
284
285	RESTART_DISCOVERY(pwp);
286	pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, "PortID 0x%x Cleared", portid);
287}
288
289void
290pmcs_process_sas_hw_event(pmcs_hw_t *pwp, void *iomb, size_t amt)
291{
292	uint32_t w1 = LE_32(((uint32_t *)iomb)[1]);
293	uint32_t w3 = LE_32(((uint32_t *)iomb)[3]);
294	char buf[32];
295	uint8_t phynum = IOP_EVENT_PHYNUM(w1);
296	uint8_t portid = IOP_EVENT_PORTID(w1);
297	pmcs_iport_t *iport;
298	pmcs_phy_t *pptr, *subphy, *tphyp;
299	int need_ack = 0;
300	int primary;
301
302	switch (IOP_EVENT_EVENT(w1)) {
303	case IOP_EVENT_PHY_STOP_STATUS:
304		if (IOP_EVENT_STATUS(w1)) {
305			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
306			    "PORT %d failed to stop (0x%x)",
307			    phynum, IOP_EVENT_STATUS(w1));
308		} else {
309			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
310			    "PHY 0x%x Stopped", phynum);
311			mutex_enter(&pwp->lock);
312			pptr = pwp->root_phys + phynum;
313			pmcs_lock_phy(pptr);
314			mutex_exit(&pwp->lock);
315			if (pptr->configured) {
316				pmcs_kill_changed(pwp, pptr, 0);
317			} else {
318				pmcs_set_changed(pwp, pptr, B_TRUE, 0);
319			}
320			pmcs_unlock_phy(pptr);
321			RESTART_DISCOVERY(pwp);
322		}
323		/* Reposition htag to the 'expected' position. */
324		((uint32_t *)iomb)[1] = ((uint32_t *)iomb)[2];
325		pmcs_process_completion(pwp, iomb, amt);
326		break;
327	case IOP_EVENT_SAS_PHY_UP:
328	{
329		static const uint8_t sas_identify_af_endian_xfvec[] = {
330			0x5c, 0x5a, 0x56, 0x00
331		};
332		pmcs_phy_t *rp;
333		sas_identify_af_t af;
334		uint64_t phy_id, wwn;
335
336		/*
337		 * If we're not at running state, don't do anything
338		 */
339		mutex_enter(&pwp->lock);
340		if (pwp->state != STATE_RUNNING) {
341			mutex_exit(&pwp->lock);
342			break;
343		}
344		pptr = pwp->root_phys + phynum;
345		pmcs_lock_phy(pptr);
346
347		/*
348		 * No need to lock the primary root PHY.  It can never go
349		 * away, and we're only concerned with the port width and
350		 * the portid, both of which only ever change in this function.
351		 */
352		rp = pwp->ports[portid];
353
354		mutex_exit(&pwp->lock);
355
356		pmcs_endian_transform(pwp, &af, &((uint32_t *)iomb)[4],
357		    sas_identify_af_endian_xfvec);
358
359		/* Copy the remote address into our phy handle */
360		(void) memcpy(pptr->sas_address, af.sas_address, 8);
361		wwn = pmcs_barray2wwn(pptr->sas_address);
362		phy_id = (uint64_t)af.phy_identifier;
363
364		/*
365		 * Check to see if there is a PortID already active.
366		 */
367		if (rp) {
368			if (rp->portid != portid) {
369				pmcs_unlock_phy(pptr);
370				pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
371				    "PortID 0x%x: PHY 0x%x SAS LINK UP IS FOR "
372				    "A DIFFERENT PORTID 0x%x", rp->portid,
373				    phynum, portid);
374				break;
375			}
376
377			/*
378			 * If the dtype isn't NOTHING, then this is actually
379			 * the primary PHY for this port.  It probably went
380			 * down and came back up, so be sure not to mark it
381			 * as a subsidiary.
382			 */
383			if (pptr->dtype == NOTHING) {
384				pptr->subsidiary = 1;
385			}
386			pptr->link_rate =
387			    pmcs_link_rate(IOP_EVENT_LINK_RATE(w1));
388			pptr->portid = portid;
389			pptr->dead = 0;
390			pmcs_unlock_phy(pptr);
391
392			rp->width = IOP_EVENT_NPIP(w3);
393
394			/* Add this PHY to the phymap */
395			if (sas_phymap_phy_add(pwp->hss_phymap, phynum,
396			    pwp->sas_wwns[0], wwn) != DDI_SUCCESS) {
397				pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
398				    "Unable to add phy %u for 0x%" PRIx64 ".0x%"
399				    PRIx64, phynum, pwp->sas_wwns[rp->phynum],
400				    wwn);
401			}
402
403			/*
404			 * Get our iport, if attached, and set it up.  Update
405			 * the PHY's phymask props while we're locked.
406			 */
407			pmcs_lock_phy(pptr);
408			pmcs_update_phy_pm_props(pptr, (1ULL << phynum),
409			    (1ULL << phy_id), B_TRUE);
410			pmcs_unlock_phy(pptr);
411			iport = pmcs_get_iport_by_wwn(pwp, wwn);
412			if (iport) {
413				primary = !pptr->subsidiary;
414
415				mutex_enter(&iport->lock);
416				if (primary) {
417					iport->pptr = pptr;
418				}
419				if (iport->ua_state == UA_ACTIVE) {
420					pmcs_add_phy_to_iport(iport, pptr);
421					pptr->iport = iport;
422				}
423				mutex_exit(&iport->lock);
424				pmcs_rele_iport(iport);
425			}
426
427			pmcs_update_phy_pm_props(rp, (1ULL << phynum),
428			    (1ULL << phy_id), B_TRUE);
429			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
430			    "PortID 0x%x: PHY 0x%x SAS LINK UP WIDENS PORT "
431			    "TO %d PHYS", portid, phynum, rp->width);
432
433			break;
434		}
435
436		/*
437		 * Check to see if anything is here already
438		 */
439		if (pptr->dtype != NOTHING && pptr->configured) {
440			pmcs_unlock_phy(pptr);
441			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
442			    "PortID 0x%x: SAS PHY 0x%x UP HITS EXISTING "
443			    "CONFIGURED TREE", portid, phynum);
444			break;
445		}
446
447		if (af.address_frame_type != SAS_AF_IDENTIFY) {
448			pmcs_unlock_phy(pptr);
449			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
450			    "SAS link up on phy 0x%x, "
451			    "but unexpected frame type 0x%x found", phynum,
452			    af.address_frame_type);
453			break;
454		}
455		pptr->width = IOP_EVENT_NPIP(w3);
456		pptr->portid = portid;
457		pptr->dead = 0;
458		pptr->link_rate = pmcs_link_rate(IOP_EVENT_LINK_RATE(w1));
459
460		/*
461		 * Check to see whether this is an expander or an endpoint
462		 */
463		switch (af.device_type) {
464		case SAS_IF_DTYPE_ENDPOINT:
465			pptr->pend_dtype = SAS;
466			pptr->dtype = SAS;
467			break;
468		case SAS_IF_DTYPE_EDGE:
469		case SAS_IF_DTYPE_FANOUT:
470			pptr->pend_dtype = EXPANDER;
471			pptr->dtype = EXPANDER;
472			break;
473		default:
474			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
475			    "unknown device type 0x%x", af.device_type);
476			pptr->pend_dtype = NOTHING;
477			pptr->dtype = NOTHING;
478			break;
479		}
480
481		/*
482		 * If this is a direct-attached SAS drive, do the spinup
483		 * release now.
484		 */
485		if (pptr->dtype == SAS) {
486			pptr->spinup_hold = 1;
487			pmcs_spinup_release(pwp, pptr);
488			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
489			    "Release spinup hold on PHY 0x%x", phynum);
490		}
491
492		pmcs_set_changed(pwp, pptr, B_TRUE, 0);
493		if (pptr->width > 1) {
494			pmcs_prt(pwp, PMCS_PRT_INFO, pptr, NULL,
495			    "PortID 0x%x: PHY 0x%x SAS"
496			    " LINK UP @ %s Gb with %d phys/s", portid, phynum,
497			    pmcs_get_rate(pptr->link_rate), pptr->width);
498		} else {
499			pmcs_prt(pwp, PMCS_PRT_INFO, pptr, NULL,
500			    "PortID 0x%x: PHY 0x%x SAS"
501			    " LINK UP @ %s Gb/s", portid, phynum,
502			    pmcs_get_rate(pptr->link_rate));
503		}
504		pmcs_unlock_phy(pptr);
505
506		/* Add this PHY to the phymap */
507		if (sas_phymap_phy_add(pwp->hss_phymap, phynum,
508		    pwp->sas_wwns[0], wwn) != DDI_SUCCESS) {
509			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
510			    "Unable to add phy %u for 0x%" PRIx64 ".0x%"
511			    PRIx64, phynum, pwp->sas_wwns[pptr->phynum], wwn);
512		}
513
514		/* Get a pointer to our iport and set it up if attached */
515		iport = pmcs_get_iport_by_wwn(pwp, wwn);
516		if (iport) {
517			primary = !pptr->subsidiary;
518
519			mutex_enter(&iport->lock);
520			if (primary) {
521				iport->pptr = pptr;
522			}
523			if (iport->ua_state == UA_ACTIVE) {
524				pmcs_add_phy_to_iport(iport, pptr);
525				pptr->iport = iport;
526			}
527			mutex_exit(&iport->lock);
528			pmcs_rele_iport(iport);
529		}
530
531		pmcs_lock_phy(pptr);
532		pmcs_update_phy_pm_props(pptr, (1ULL << phynum),
533		    (1ULL << phy_id), B_TRUE);
534		pmcs_smhba_log_sysevent(pwp, ESC_SAS_PHY_EVENT,
535		    SAS_PHY_ONLINE, pptr);
536		pmcs_unlock_phy(pptr);
537
538		mutex_enter(&pwp->lock);
539		pwp->ports[portid] = pptr;
540		mutex_exit(&pwp->lock);
541		RESTART_DISCOVERY(pwp);
542
543		break;
544	}
545	case IOP_EVENT_SATA_PHY_UP: {
546		uint64_t wwn;
547		/*
548		 * If we're not at running state, don't do anything
549		 */
550		mutex_enter(&pwp->lock);
551		if (pwp->state != STATE_RUNNING) {
552			mutex_exit(&pwp->lock);
553			break;
554		}
555
556		/*
557		 * Check to see if anything is here already
558		 */
559		pmcs_lock_phy(pwp->root_phys + phynum);
560		pptr = pwp->root_phys + phynum;
561		mutex_exit(&pwp->lock);
562
563		if (pptr->dtype != NOTHING && pptr->configured) {
564			pmcs_unlock_phy(pptr);
565			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
566			    "PortID 0x%x: SATA PHY 0x%x"
567			    " UP HITS EXISTING CONFIGURED TREE",
568			    portid, phynum);
569			break;
570		}
571
572		pptr->width = 1;
573		pptr->dead = 0;
574
575		/*
576		 * Install the PHY number in the least significant byte
577		 * with a NAA=3 (locally assigned address) in the most
578		 * significant nubble.
579		 *
580		 * Later, we'll either use that or dig a
581		 * WWN out of words 108..111.
582		 */
583		pptr->sas_address[0] = 0x30;
584		pptr->sas_address[1] = 0;
585		pptr->sas_address[2] = 0;
586		pptr->sas_address[3] = 0;
587		pptr->sas_address[4] = 0;
588		pptr->sas_address[5] = 0;
589		pptr->sas_address[6] = 0;
590		pptr->sas_address[7] = phynum;
591		pptr->portid = portid;
592		pptr->link_rate = pmcs_link_rate(IOP_EVENT_LINK_RATE(w1));
593		pptr->dtype = SATA;
594		pmcs_set_changed(pwp, pptr, B_TRUE, 0);
595		pmcs_prt(pwp, PMCS_PRT_INFO, pptr, NULL,
596		    "PortID 0x%x: PHY 0x%x SATA LINK UP @ %s Gb/s",
597		    pptr->portid, phynum, pmcs_get_rate(pptr->link_rate));
598		wwn = pmcs_barray2wwn(pptr->sas_address);
599		pmcs_unlock_phy(pptr);
600
601		/* Add this PHY to the phymap */
602		if (sas_phymap_phy_add(pwp->hss_phymap, phynum,
603		    pwp->sas_wwns[0], wwn) != DDI_SUCCESS) {
604			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
605			    "Unable to add phy %u for 0x%" PRIx64 ".0x%"
606			    PRIx64, phynum, pwp->sas_wwns[pptr->phynum],
607			    wwn);
608		}
609
610		/* Get our iport, if attached, and set it up */
611		iport = pmcs_get_iport_by_wwn(pwp, wwn);
612		if (iport) {
613			mutex_enter(&iport->lock);
614			iport->pptr = pptr;
615			if (iport->ua_state == UA_ACTIVE) {
616				pmcs_add_phy_to_iport(iport, pptr);
617				pptr->iport = iport;
618				ASSERT(iport->nphy == 1);
619				iport->nphy = 1;
620			}
621			mutex_exit(&iport->lock);
622			pmcs_rele_iport(iport);
623		}
624
625		pmcs_lock_phy(pptr);
626		pmcs_update_phy_pm_props(pptr, (1ULL << phynum), 1ULL, B_TRUE);
627		pmcs_smhba_log_sysevent(pwp, ESC_SAS_PHY_EVENT,
628		    SAS_PHY_ONLINE, pptr);
629		pmcs_unlock_phy(pptr);
630
631		mutex_enter(&pwp->lock);
632		pwp->ports[pptr->portid] = pptr;
633		mutex_exit(&pwp->lock);
634		RESTART_DISCOVERY(pwp);
635		break;
636	}
637
638	case IOP_EVENT_SATA_SPINUP_HOLD:
639		tphyp = (pmcs_phy_t *)(pwp->root_phys + phynum);
640		/*
641		 * No need to lock the entire tree for this
642		 */
643		mutex_enter(&tphyp->phy_lock);
644		tphyp->spinup_hold = 1;
645		pmcs_spinup_release(pwp, tphyp);
646		mutex_exit(&tphyp->phy_lock);
647		break;
648	case IOP_EVENT_PHY_DOWN: {
649		uint64_t wwn;
650
651		/*
652		 * If we're not at running state, don't do anything
653		 */
654		mutex_enter(&pwp->lock);
655		if (pwp->state != STATE_RUNNING) {
656			mutex_exit(&pwp->lock);
657			break;
658		}
659		pptr = pwp->ports[portid];
660
661		subphy = pwp->root_phys + phynum;
662		/*
663		 * subphy is a pointer to the PHY corresponding to the incoming
664		 * event. pptr points to the primary PHY for the corresponding
665		 * port.  So, subphy and pptr may or may not be the same PHY,
666		 * but that doesn't change what we need to do with each.
667		 */
668		ASSERT(subphy);
669		mutex_exit(&pwp->lock);
670
671		if (pptr == NULL) {
672			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
673			    "PortID 0x%x: PHY 0x%x LINK DOWN- no portid ptr",
674			    portid, phynum);
675			break;
676		}
677		if (IOP_EVENT_PORT_STATE(w3) == IOP_EVENT_PS_NIL) {
678			pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL,
679			    "PortID 0x%x: PHY 0x%x NOT VALID YET",
680			    portid, phynum);
681			need_ack = 1;
682			break;
683		}
684		if (IOP_EVENT_PORT_STATE(w3) == IOP_EVENT_PS_IN_RESET) {
685			pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL,
686			    "PortID 0x%x: PHY 0x%x IN RESET",
687			    portid, phynum);
688			/* Entire port is down due to a host-initiated reset */
689			mutex_enter(&pptr->phy_lock);
690			/* Clear the phymask props in pptr */
691			pmcs_update_phy_pm_props(pptr, pptr->att_port_pm_tmp,
692			    pptr->tgt_port_pm_tmp, B_FALSE);
693			iport = pptr->iport;
694			mutex_exit(&pptr->phy_lock);
695			if (iport) {
696				mutex_enter(&iport->lock);
697				pmcs_iport_teardown_phys(iport);
698				mutex_exit(&iport->lock);
699			}
700
701			/* Clear down all PHYs in the port */
702			for (pptr = pwp->root_phys; pptr;
703			    pptr = pptr->sibling) {
704				pmcs_lock_phy(pptr);
705				if (pptr->portid == portid) {
706					pptr->pend_dtype = NOTHING;
707					pptr->dtype = NOTHING;
708					pptr->portid =
709					    PMCS_IPORT_INVALID_PORT_ID;
710					if (pptr->valid_device_id) {
711						pptr->deregister_wait = 1;
712					}
713				}
714				pmcs_unlock_phy(pptr);
715				SCHEDULE_WORK(pwp, PMCS_WORK_DEREGISTER_DEV);
716				(void) ddi_taskq_dispatch(pwp->tq, pmcs_worker,
717				    pwp, DDI_NOSLEEP);
718			}
719
720			break;
721		}
722
723		if (IOP_EVENT_PORT_STATE(w3) == IOP_EVENT_PS_LOSTCOMM) {
724			pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL,
725			    "PortID 0x%x: PHY 0x%x TEMPORARILY DOWN",
726			    portid, phynum);
727			need_ack = 1;
728			break;
729		}
730
731		if (IOP_EVENT_PORT_STATE(w3) == IOP_EVENT_PS_VALID) {
732
733			/*
734			 * This is not the last phy in the port, so if this
735			 * is the primary PHY, promote another PHY to primary.
736			 */
737			if (pptr == subphy) {
738				primary = !subphy->subsidiary;
739				ASSERT(primary);
740
741				tphyp = pptr;
742				pptr = pmcs_promote_next_phy(tphyp);
743
744				if (pptr) {
745					/* Update primary pptr in ports */
746					pwp->ports[portid] = pptr;
747					pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr,
748					    NULL, "PortID 0x%x: PHY 0x%x "
749					    "promoted to primary", portid,
750					    pptr->phynum);
751				} else {
752					pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr,
753					    NULL, "PortID 0x%x: PHY 0x%x: "
754					    "unable to promote phy", portid,
755					    phynum);
756				}
757			}
758
759			/*
760			 * Drop port width on the primary phy handle
761			 * No need to lock the entire tree for this
762			 */
763			mutex_enter(&pptr->phy_lock);
764			pmcs_update_phy_pm_props(pptr, subphy->att_port_pm_tmp,
765			    subphy->tgt_port_pm_tmp, B_FALSE);
766			pptr->width = IOP_EVENT_NPIP(w3);
767			mutex_exit(&pptr->phy_lock);
768
769			/* Clear the iport reference and portid on the subphy */
770			mutex_enter(&subphy->phy_lock);
771			iport = subphy->iport;
772			subphy->iport = NULL;
773			subphy->portid = PMCS_PHY_INVALID_PORT_ID;
774			subphy->pend_dtype = NOTHING;
775			subphy->dtype = NOTHING;
776			mutex_exit(&subphy->phy_lock);
777
778			/*
779			 * If the iport was set on this phy, decrement its
780			 * nphy count and remove this phy from the phys list.
781			 */
782			if (iport) {
783				mutex_enter(&iport->lock);
784				if (iport->ua_state == UA_ACTIVE) {
785					pmcs_remove_phy_from_iport(iport,
786					    subphy);
787				}
788				mutex_exit(&iport->lock);
789			}
790
791			pmcs_lock_phy(subphy);
792			wwn = pmcs_barray2wwn(pptr->sas_address);
793			if (subphy->subsidiary)
794				pmcs_clear_phy(pwp, subphy);
795			pmcs_unlock_phy(subphy);
796
797			/* Remove this PHY from the phymap */
798			if (sas_phymap_phy_rem(pwp->hss_phymap, phynum) !=
799			    DDI_SUCCESS) {
800				pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
801				    "Unable to remove phy %u for 0x%" PRIx64
802				    ".0x%" PRIx64, phynum,
803				    pwp->sas_wwns[pptr->phynum], wwn);
804			}
805
806			pmcs_prt(pwp, PMCS_PRT_INFO, pptr, NULL,
807			    "PortID 0x%x: PHY 0x%x LINK DOWN NARROWS PORT "
808			    "TO %d PHYS", portid, phynum, pptr->width);
809			break;
810		}
811		if (IOP_EVENT_PORT_STATE(w3) != IOP_EVENT_PS_INVALID) {
812			pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL,
813			    "PortID 0x%x: PHY 0x%x LINK DOWN NOT HANDLED "
814			    "(state 0x%x)", portid, phynum,
815			    IOP_EVENT_PORT_STATE(w3));
816			need_ack = 1;
817			break;
818		}
819		/* Remove this PHY from the phymap */
820		if (sas_phymap_phy_rem(pwp->hss_phymap, phynum) !=
821		    DDI_SUCCESS) {
822			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
823			    "Unable to remove phy %u for 0x%" PRIx64
824			    ".0x%" PRIx64, phynum,
825			    pwp->sas_wwns[pptr->phynum], wwn);
826		}
827
828		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
829		    "PortID 0x%x: PHY 0x%x LINK DOWN (port invalid)",
830		    portid, phynum);
831
832		/*
833		 * Last PHY on the port.
834		 * Assumption: pptr and subphy are both "valid".  In fact,
835		 * they should be one and the same.
836		 *
837		 * Drop port width on the primary phy handle
838		 * Report the event and clear its PHY pm props while we've
839		 * got the lock
840		 */
841		ASSERT(pptr == subphy);
842		mutex_enter(&pptr->phy_lock);
843		pptr->width = 0;
844		pmcs_update_phy_pm_props(pptr, pptr->att_port_pm_tmp,
845		    pptr->tgt_port_pm_tmp, B_FALSE);
846		pmcs_smhba_log_sysevent(pwp, ESC_SAS_PHY_EVENT,
847		    SAS_PHY_OFFLINE, pptr);
848		mutex_exit(&pptr->phy_lock);
849
850		/* Clear the iport reference and portid on the subphy */
851		pmcs_lock_phy(subphy);
852		iport = subphy->iport;
853		subphy->deregister_wait = 1;
854		subphy->iport = NULL;
855		subphy->portid = PMCS_PHY_INVALID_PORT_ID;
856		subphy->pend_dtype = NOTHING;
857		subphy->dtype = NOTHING;
858		pmcs_unlock_phy(subphy);
859		SCHEDULE_WORK(pwp, PMCS_WORK_DEREGISTER_DEV);
860		(void) ddi_taskq_dispatch(pwp->tq, pmcs_worker,
861		    pwp, DDI_NOSLEEP);
862
863		/*
864		 * If the iport was set on this phy, decrement its
865		 * nphy count and remove this phy from the phys list.
866		 * Also, clear the iport's pptr as this port is now
867		 * down.
868		 */
869		if (iport) {
870			mutex_enter(&iport->lock);
871			if (iport->ua_state == UA_ACTIVE) {
872				pmcs_remove_phy_from_iport(iport, subphy);
873				iport->pptr = NULL;
874				iport->ua_state = UA_PEND_DEACTIVATE;
875			}
876			mutex_exit(&iport->lock);
877		}
878
879		pmcs_lock_phy(subphy);
880		if (subphy->subsidiary)
881			pmcs_clear_phy(pwp, subphy);
882		pmcs_unlock_phy(subphy);
883
884		/*
885		 * Since we're now really dead, it's time to clean up.
886		 */
887		pmcs_kill_port(pwp, portid);
888		need_ack = 1;
889
890		break;
891	}
892	case IOP_EVENT_BROADCAST_CHANGE:
893		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
894		    "PortID 0x%x: PHY 0x%x Broadcast Change", portid, phynum);
895		need_ack = 1;
896		mutex_enter(&pwp->lock);
897		pptr = pwp->ports[portid];
898		if (pptr) {
899			pmcs_lock_phy(pptr);
900			if (pptr->phynum == phynum) {
901				pmcs_set_changed(pwp, pptr, B_TRUE, 0);
902			}
903			pmcs_smhba_log_sysevent(pwp, ESC_SAS_HBA_PORT_BROADCAST,
904			    SAS_PORT_BROADCAST_CHANGE, pptr);
905			pmcs_unlock_phy(pptr);
906		}
907		mutex_exit(&pwp->lock);
908		RESTART_DISCOVERY(pwp);
909		break;
910	case IOP_EVENT_BROADCAST_SES:
911		EVT_PRT(pwp, IOP_EVENT_BROADCAST_SES, phynum);
912		mutex_enter(&pwp->lock);
913		pptr = pwp->ports[portid];
914		mutex_exit(&pwp->lock);
915		if (pptr) {
916			pmcs_lock_phy(pptr);
917			pmcs_smhba_log_sysevent(pwp, ESC_SAS_HBA_PORT_BROADCAST,
918			    SAS_PORT_BROADCAST_SES, pptr);
919			pmcs_unlock_phy(pptr);
920		}
921		break;
922	case IOP_EVENT_PHY_ERR_INBOUND_CRC:
923	{
924		char buf[32];
925		(void) snprintf(buf, sizeof (buf), "Inbound PHY CRC error");
926		need_ack = 1;
927		break;
928	}
929	case IOP_EVENT_HARD_RESET_RECEIVED:
930		EVT_PRT(pwp, IOP_EVENT_HARD_RESET_RECEIVED, phynum);
931		break;
932	case IOP_EVENT_EVENT_ID_FRAME_TIMO:
933		EVT_PRT(pwp, IOP_EVENT_EVENT_ID_FRAME_TIMO, phynum);
934		break;
935	case IOP_EVENT_BROADCAST_EXP:
936		pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL,
937		    "PortID 0x%x: PHY 0x%x Broadcast Exp Change",
938		    portid, phynum);
939		/*
940		 * Comparing Section 6.8.1.4 of SMHBA (rev 7) spec and Section
941		 * 7.2.3 of SAS2 (Rev 15) spec,
942		 * _BROADCAST_EXPANDER event corresponds to _D01_4 primitive
943		 */
944		mutex_enter(&pwp->lock);
945		pptr = pwp->ports[portid];
946		mutex_exit(&pwp->lock);
947		if (pptr) {
948			pmcs_lock_phy(pptr);
949			pmcs_smhba_log_sysevent(pwp, ESC_SAS_HBA_PORT_BROADCAST,
950			    SAS_PORT_BROADCAST_D01_4, pptr);
951			pmcs_unlock_phy(pptr);
952		}
953		break;
954	case IOP_EVENT_PHY_START_STATUS:
955		switch (IOP_EVENT_STATUS(w1)) {
956		case IOP_PHY_START_OK:
957			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
958			    "PHY 0x%x Started", phynum);
959			break;
960		case IOP_PHY_START_ALREADY:
961			pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL,
962			    "PHY 0x%x Started (Already)", phynum);
963			break;
964		case IOP_PHY_START_INVALID:
965			pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL,
966			    "PHY 0x%x failed to start (invalid phy)", phynum);
967			break;
968		case IOP_PHY_START_ERROR:
969			pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL,
970			    "PHY 0x%x Start Error", phynum);
971			break;
972		default:
973			pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL,
974			    "PHY 0x%x failed to start (0x%x)", phynum,
975			    IOP_EVENT_STATUS(w1));
976			break;
977		}
978		/* Reposition htag to the 'expected' position. */
979		((uint32_t *)iomb)[1] = ((uint32_t *)iomb)[2];
980		pmcs_process_completion(pwp, iomb, amt);
981		break;
982	case IOP_EVENT_PHY_ERR_INVALID_DWORD:
983		need_ack = 1;
984		EVT_PRT(pwp, IOP_EVENT_PHY_ERR_INVALID_DWORD, phynum);
985		break;
986	case IOP_EVENT_PHY_ERR_DISPARITY_ERROR:
987		need_ack = 1;
988		EVT_PRT(pwp, IOP_EVENT_PHY_ERR_DISPARITY_ERROR, phynum);
989		break;
990	case IOP_EVENT_PHY_ERR_CODE_VIOLATION:
991		need_ack = 1;
992		EVT_PRT(pwp, IOP_EVENT_PHY_ERR_CODE_VIOLATION, phynum);
993		break;
994	case IOP_EVENT_PHY_ERR_LOSS_OF_DWORD_SYN:
995		need_ack = 1;
996		EVT_PRT(pwp, IOP_EVENT_PHY_ERR_LOSS_OF_DWORD_SYN, phynum);
997		break;
998	case IOP_EVENT_PHY_ERR_PHY_RESET_FAILD:
999		need_ack = 1;
1000		EVT_PRT(pwp, IOP_EVENT_PHY_ERR_PHY_RESET_FAILD, phynum);
1001		break;
1002	case IOP_EVENT_PORT_RECOVERY_TIMER_TMO:
1003		EVT_PRT(pwp, IOP_EVENT_PORT_RECOVERY_TIMER_TMO, phynum);
1004		break;
1005	case IOP_EVENT_PORT_RECOVER:
1006		EVT_PRT(pwp, IOP_EVENT_PORT_RECOVER, phynum);
1007		break;
1008	case IOP_EVENT_PORT_INVALID:
1009		mutex_enter(&pwp->lock);
1010		if (pwp->state != STATE_RUNNING) {
1011			mutex_exit(&pwp->lock);
1012			break;
1013		}
1014		mutex_exit(&pwp->lock);
1015		pmcs_kill_port(pwp, portid);
1016		pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL,
1017		    "PortID 0x%x: PORT Now Invalid", portid);
1018		break;
1019	case IOP_EVENT_PORT_RESET_TIMER_TMO:
1020		EVT_PRT(pwp, IOP_EVENT_PORT_RESET_TIMER_TMO, phynum);
1021		break;
1022	case IOP_EVENT_PORT_RESET_COMPLETE:
1023		EVT_PRT(pwp, IOP_EVENT_PORT_RESET_COMPLETE, phynum);
1024		break;
1025	case IOP_EVENT_BROADCAST_ASYNC_EVENT:
1026		EVT_PRT(pwp, IOP_EVENT_BROADCAST_ASYNC_EVENT, phynum);
1027		/*
1028		 * Comparing Section 6.8.1.4 of SMHBA (rev 7) spec and Section
1029		 * 7.2.3 of SAS2 (Rev 15) spec,
1030		 * _BROADCAST_ASYNC event corresponds to _D04_7 primitive
1031		 */
1032		mutex_enter(&pwp->lock);
1033		pptr = pwp->ports[portid];
1034		mutex_exit(&pwp->lock);
1035		if (pptr) {
1036			pmcs_lock_phy(pptr);
1037			pmcs_smhba_log_sysevent(pwp, ESC_SAS_HBA_PORT_BROADCAST,
1038			    SAS_PORT_BROADCAST_D04_7, pptr);
1039			pmcs_unlock_phy(pptr);
1040		}
1041		break;
1042	default:
1043		(void) snprintf(buf, sizeof (buf),
1044		    "unknown SAS H/W Event PHY 0x%x", phynum);
1045		pmcs_print_entry(pwp, PMCS_PRT_DEBUG, buf, iomb);
1046		break;
1047	}
1048	if (need_ack) {
1049		mutex_enter(&pwp->lock);
1050		/*
1051		 * Don't lock the entire tree for this.  Just grab the mutex
1052		 * on the root PHY.
1053		 */
1054		tphyp = pwp->root_phys + phynum;
1055		mutex_enter(&tphyp->phy_lock);
1056		tphyp->hw_event_ack = w1;
1057		mutex_exit(&tphyp->phy_lock);
1058		mutex_exit(&pwp->lock);
1059		pmcs_ack_events(pwp);
1060	}
1061}
1062
1063static void
1064pmcs_process_echo_completion(pmcs_hw_t *pwp, void *iomb, size_t amt)
1065{
1066	echo_test_t fred;
1067	pmcwork_t *pwrk;
1068	uint32_t *msg = iomb, htag = LE_32(msg[1]);
1069	pwrk = pmcs_tag2wp(pwp, htag, B_FALSE);
1070	if (pwrk) {
1071		(void) memcpy(&fred, &((uint32_t *)iomb)[2], sizeof (fred));
1072		fred.ptr[0]++;
1073		msg[2] = LE_32(PMCOUT_STATUS_OK);
1074		pmcs_complete_work(pwp, pwrk, msg, amt);
1075	} else {
1076		pmcs_print_entry(pwp, PMCS_PRT_DEBUG,
1077		    "ECHO completion with no work structure", iomb);
1078	}
1079}
1080
1081static void
1082pmcs_process_ssp_event(pmcs_hw_t *pwp, void *iomb, size_t amt)
1083{
1084	_NOTE(ARGUNUSED(amt));
1085	uint32_t status, htag, *w;
1086	pmcwork_t *pwrk;
1087	pmcs_phy_t *phyp = NULL;
1088	char *path;
1089
1090	w = iomb;
1091	htag = LE_32(w[1]);
1092	status = LE_32(w[2]);
1093
1094
1095	pwrk = pmcs_tag2wp(pwp, htag, B_FALSE);
1096	if (pwrk == NULL) {
1097		path = "????";
1098	} else {
1099		phyp = pwrk->phy;
1100		path = pwrk->phy->path;
1101	}
1102
1103	if (status != PMCOUT_STATUS_XFER_CMD_FRAME_ISSUED) {
1104		char buf[20];
1105		const char *emsg = pmcs_status_str(status);
1106
1107		if (emsg == NULL) {
1108			(void) snprintf(buf, sizeof (buf), "Status 0x%x",
1109			    status);
1110			emsg = buf;
1111		}
1112		pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, NULL, "%s: Bad SAS Status "
1113		    "(tag 0x%x) %s on %s", __func__, htag, emsg, path);
1114		if (pwrk != NULL) {
1115			/*
1116			 * There may be pending command on a target device.
1117			 * Or, it may be a double fault.
1118			 */
1119			pmcs_start_ssp_event_recovery(pwp, pwrk, iomb, amt);
1120		}
1121	} else {
1122		pmcs_prt(pwp, PMCS_PRT_DEBUG2, phyp, NULL,
1123		    "%s: tag %x put onto the wire for %s",
1124		    __func__, htag, path);
1125		if (pwrk) {
1126			pwrk->onwire = 1;
1127			mutex_exit(&pwrk->lock);
1128		}
1129	}
1130}
1131
1132static void
1133pmcs_process_sata_event(pmcs_hw_t *pwp, void *iomb, size_t amt)
1134{
1135	_NOTE(ARGUNUSED(amt));
1136	pmcwork_t *pwrk = NULL;
1137	pmcs_phy_t *pptr = NULL;
1138	uint32_t status, htag, *w;
1139	char *path = NULL;
1140
1141	w = iomb;
1142	htag = LE_32(w[1]);
1143	status = LE_32(w[2]);
1144
1145	/*
1146	 * If the status is PMCOUT_STATUS_XFER_ERROR_ABORTED_NCQ_MODE,
1147	 * we have to issue a READ LOG EXT ATA (page 0x10) command
1148	 * to the device. In this case, htag is not valid.
1149	 *
1150	 * If the status is PMCOUT_STATUS_XFER_CMD_FRAME_ISSUED, we're
1151	 * just noting that an I/O got put onto the wire.
1152	 *
1153	 * Othewise, other errors are indicative that things need to
1154	 * be aborted.
1155	 */
1156	if (htag) {
1157		pwrk = pmcs_tag2wp(pwp, htag, B_TRUE);
1158	}
1159
1160	if (pwrk) {
1161		pptr = pwrk->phy;
1162		path = pptr->path;
1163	} else {
1164		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, "%s: "
1165		    "cannot find work structure for SATA completion", __func__);
1166		return;
1167	}
1168
1169	if (status != PMCOUT_STATUS_XFER_CMD_FRAME_ISSUED) {
1170		char buf[20];
1171		const char *emsg = pmcs_status_str(status);
1172
1173		if (emsg == NULL) {
1174			(void) snprintf(buf, sizeof (buf), "Status 0x%x",
1175			    status);
1176			emsg = buf;
1177		}
1178		if (status == PMCOUT_STATUS_XFER_ERROR_ABORTED_NCQ_MODE) {
1179			pptr->need_rl_ext = 1;
1180			htag = 0;
1181		} else {
1182			pptr->abort_pending = 1;
1183		}
1184		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
1185		    "%s: Bad SATA Status (tag 0x%x) %s on %s",
1186		    __func__, htag, emsg, path);
1187		SCHEDULE_WORK(pwp, PMCS_WORK_ABORT_HANDLE);
1188		/*
1189		 * Unlike SSP devices, we let the abort we
1190		 * schedule above force the completion of
1191		 * problem commands.
1192		 */
1193		mutex_exit(&pwrk->lock);
1194	} else {
1195		pmcs_prt(pwp, PMCS_PRT_DEBUG2, pptr, NULL,
1196		    "%s: tag %x put onto the wire for %s",
1197		    __func__, htag, path);
1198		pwrk->onwire = 1;
1199		mutex_exit(&pwrk->lock);
1200	}
1201
1202	mutex_exit(&pptr->phy_lock);
1203}
1204
1205static void
1206pmcs_process_abort_completion(pmcs_hw_t *pwp, void *iomb, size_t amt)
1207{
1208	pmcs_phy_t *pptr;
1209	struct pmcwork *pwrk;
1210	uint32_t htag = LE_32(((uint32_t *)iomb)[1]);
1211	uint32_t status = LE_32(((uint32_t *)iomb)[2]);
1212	uint32_t scope = LE_32(((uint32_t *)iomb)[3]) & 0x1;
1213	char *path;
1214
1215	pwrk = pmcs_tag2wp(pwp, htag, B_TRUE);
1216	if (pwrk == NULL) {
1217		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1218		    "%s: cannot find work structure for ABORT", __func__);
1219		return;
1220	}
1221
1222	pptr = pwrk->phy;
1223	if (pptr) {
1224		pptr->abort_pending = 0;
1225		pptr->abort_sent = 0;
1226
1227		/*
1228		 * Don't do this if the status was ABORT_IN_PROGRESS and
1229		 * the scope bit was set
1230		 */
1231		if ((status != PMCOUT_STATUS_IO_ABORT_IN_PROGRESS) || !scope) {
1232			pptr->abort_all_start = 0;
1233			cv_signal(&pptr->abort_all_cv);
1234		}
1235		path = pptr->path;
1236		mutex_exit(&pptr->phy_lock);
1237	} else {
1238		path = "(no phy)";
1239	}
1240
1241	switch (status) {
1242	case PMCOUT_STATUS_OK:
1243		if (scope) {
1244			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
1245			    "%s: abort all succeeded for %s. (htag=0x%x)",
1246			    __func__, path, htag);
1247		} else {
1248			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
1249			    "%s: abort tag 0x%x succeeded for %s. (htag=0x%x)",
1250			    __func__, pwrk->abt_htag, path, htag);
1251		}
1252		break;
1253
1254	case PMCOUT_STATUS_IO_NOT_VALID:
1255		if (scope) {
1256			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
1257			    "%s: ABORT %s failed (DEV NOT VALID) for %s. "
1258			    "(htag=0x%x)", __func__, scope ? "all" : "tag",
1259			    path, htag);
1260		} else {
1261			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
1262			    "%s: ABORT %s failed (I/O NOT VALID) for %s. "
1263			    "(htag=0x%x)", __func__, scope ? "all" : "tag",
1264			    path, htag);
1265		}
1266		break;
1267
1268	case PMCOUT_STATUS_IO_ABORT_IN_PROGRESS:
1269		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, "%s: ABORT %s failed "
1270		    "for %s, htag 0x%x (ABORT IN PROGRESS)", __func__,
1271		    scope ? "all" : "tag", path, htag);
1272		break;
1273
1274	default:
1275		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, "%s: Unknown status "
1276		    "%d for ABORT %s, htag 0x%x, PHY %s", __func__, status,
1277		    scope ? "all" : "tag", htag, path);
1278		break;
1279	}
1280
1281	pmcs_complete_work(pwp, pwrk, iomb, amt);
1282}
1283
1284static void
1285pmcs_process_general_event(pmcs_hw_t *pwp, uint32_t *iomb)
1286{
1287	uint32_t htag;
1288	char local[60];
1289	struct pmcwork *pwrk;
1290	int i;
1291
1292	if (LE_32(iomb[1]) == INBOUND_IOMB_V_BIT_NOT_SET) {
1293		(void) snprintf(local, sizeof (local),
1294		    "VALID bit not set on INBOUND IOMB");
1295	} else if (LE_32(iomb[1]) ==
1296	    INBOUND_IOMB_OPC_NOT_SUPPORTED) {
1297		(void) snprintf(local, sizeof (local),
1298		    "opcode not set on inbound IOMB");
1299	} else {
1300		(void) snprintf(local, sizeof (local),
1301		    "unknown GENERAL EVENT status (0x%x)",
1302		    LE_32(iomb[1]));
1303	}
1304	/* Pull up bad IOMB into usual position */
1305	for (i = 0; i < PMCS_MSG_SIZE - 2; i++) {
1306		iomb[i] = iomb[i+2];
1307	}
1308	/* overwrite status with an error */
1309	iomb[2] = LE_32(PMCOUT_STATUS_PROG_ERROR);
1310	iomb[PMCS_MSG_SIZE - 2] = 0;
1311	iomb[PMCS_MSG_SIZE - 1] = 0;
1312	htag = LE_32(iomb[1]);
1313	pmcs_print_entry(pwp, PMCS_PRT_DEBUG, local, iomb);
1314	pwrk = pmcs_tag2wp(pwp, htag, B_FALSE);
1315	if (pwrk) {
1316		pmcs_complete_work(pwp, pwrk, iomb, PMCS_QENTRY_SIZE);
1317	}
1318}
1319
1320void
1321pmcs_general_intr(pmcs_hw_t *pwp)
1322{
1323	char local[PMCS_QENTRY_SIZE << 1];
1324	uint32_t w0, pi, ci;
1325	uint32_t *ptr, nbuf, lim = 0;
1326	size_t amt;
1327
1328	ci = pmcs_rd_oqci(pwp, PMCS_OQ_GENERAL);
1329	pi = pmcs_rd_oqpi(pwp, PMCS_OQ_GENERAL);
1330
1331	while (ci != pi) {
1332		OQLIM_CHECK(pwp, lim);
1333		ptr = GET_OQ_ENTRY(pwp, PMCS_OQ_GENERAL, ci, 0);
1334		w0 = LE_32(ptr[0]);
1335		VALID_IOMB_CHECK(pwp, w0, ptr, ci, pi);
1336		WRONG_OBID_CHECK(pwp, w0, PMCS_OQ_GENERAL);
1337		COPY_OUTBOUND(pwp, w0, local, nbuf, amt, ptr,
1338		    PMCS_OQ_GENERAL, ci);
1339
1340		switch (w0 & PMCS_IOMB_OPCODE_MASK) {
1341		case PMCOUT_SSP_COMPLETION:
1342			/*
1343			 * We only get SSP completion here for Task Management
1344			 * completions.
1345			 */
1346		case PMCOUT_SMP_COMPLETION:
1347		case PMCOUT_LOCAL_PHY_CONTROL:
1348		case PMCOUT_DEVICE_REGISTRATION:
1349		case PMCOUT_DEREGISTER_DEVICE_HANDLE:
1350		case PMCOUT_GET_NVMD_DATA:
1351		case PMCOUT_SET_NVMD_DATA:
1352		case PMCOUT_GET_DEVICE_STATE:
1353		case PMCOUT_SET_DEVICE_STATE:
1354			pmcs_process_completion(pwp, local, amt);
1355			break;
1356		case PMCOUT_SSP_ABORT:
1357		case PMCOUT_SATA_ABORT:
1358		case PMCOUT_SMP_ABORT:
1359			pmcs_process_abort_completion(pwp, local, amt);
1360			break;
1361		case PMCOUT_SSP_EVENT:
1362			pmcs_process_ssp_event(pwp, local, amt);
1363			break;
1364		case PMCOUT_ECHO:
1365			pmcs_process_echo_completion(pwp, local, amt);
1366			break;
1367		case PMCOUT_SAS_HW_EVENT_ACK_ACK:
1368			if (LE_32(ptr[2]) != SAS_HW_EVENT_ACK_OK) {
1369				pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1370				    "SAS H/W EVENT ACK/ACK Status=0x%b",
1371				    LE_32(ptr[2]), "\020\4InvParm\3"
1372				    "InvPort\2InvPhy\1InvSEA");
1373			}
1374			pmcs_process_completion(pwp, local, amt);
1375			break;
1376		case PMCOUT_SKIP_ENTRIES:
1377			pmcs_prt(pwp, PMCS_PRT_DEBUG3, NULL, NULL,
1378			    "%s: skip %d entries", __func__, nbuf);
1379			break;
1380		default:
1381			(void) snprintf(local, sizeof (local),
1382			    "%s: unhandled message", __func__);
1383			pmcs_print_entry(pwp, PMCS_PRT_DEBUG, local, ptr);
1384			break;
1385		}
1386		STEP_OQ_ENTRY(pwp, PMCS_OQ_GENERAL, ci, nbuf);
1387	}
1388	if (lim) {
1389		SYNC_OQ_ENTRY(pwp, PMCS_OQ_GENERAL, ci, pi);
1390	}
1391}
1392
1393/*
1394 * pmcs_check_intr_coal
1395 *
1396 * This function makes a determination on the dynamic value of the
1397 * interrupt coalescing timer register.  We only use this for I/O
1398 * completions.
1399 *
1400 * The basic algorithm is as follows:
1401 *
1402 * PMCS_MAX_IO_COMPS_PER_INTR: The maximum number of I/O completions per
1403 * I/O completion interrupt.  We won't increase the interrupt coalescing
1404 * timer if we're already processing this many completions per interrupt
1405 * beyond the threshold.
1406 *
1407 * Values in io_intr_coal structure:
1408 *
1409 * intr_latency: The average number of nsecs between interrupts during
1410 * the echo test.  Used to help determine whether to increase the coalescing
1411 * timer.
1412 *
1413 * intr_threshold: Calculated number of interrupts beyond which we may
1414 * increase the timer.  This value is calculated based on the calculated
1415 * interrupt latency during the ECHO test and the current value of the
1416 * coalescing timer.
1417 *
1418 * nsecs_between_intrs: Total number of nsecs between all the interrupts
1419 * in the current timeslice.
1420 *
1421 * last_io_comp: Time of the last I/O interrupt.
1422 *
1423 * num_io_completions: Number of I/O completions during the slice
1424 *
1425 * num_intrs: Number of I/O completion interrupts during the slice
1426 *
1427 * max_io_completions: Number of times we hit >= PMCS_MAX_IO_COMPS_PER_INTR
1428 * during interrupt processing.
1429 *
1430 * PMCS_MAX_IO_COMPS_LOWAT_SHIFT/HIWAT_SHIFT
1431 * Low and high marks used to determine whether we processed enough interrupts
1432 * that contained the maximum number of I/O completions to warrant increasing
1433 * the timer
1434 *
1435 * intr_coal_timer: The current value of the register (in usecs)
1436 *
1437 * timer_on: B_TRUE means we are using the timer
1438 *
1439 * The timer is increased if we processed more than intr_threshold interrupts
1440 * during the quantum and the number of interrupts containing the maximum
1441 * number of I/O completions is between PMCS_MAX_IO_COMPS_LOWAT_SHIFT and
1442 * _HIWAT_SHIFT
1443 *
1444 * If the average time between completions is greater than twice
1445 * the current timer value, the timer value is decreased.
1446 *
1447 * If we did not take any interrupts during a quantum, we turn the timer off.
1448 */
1449void
1450pmcs_check_intr_coal(void *arg)
1451{
1452	pmcs_hw_t	*pwp = (pmcs_hw_t *)arg;
1453	uint32_t	avg_nsecs;
1454	clock_t		lbolt, ret;
1455	pmcs_io_intr_coal_t *ici;
1456
1457	ici = &pwp->io_intr_coal;
1458	mutex_enter(&pwp->ict_lock);
1459	while (ici->stop_thread == B_FALSE) {
1460		/*
1461		 * Wait for next time quantum... collect stats
1462		 */
1463		lbolt = ddi_get_lbolt();
1464		while (ici->stop_thread == B_FALSE) {
1465			ret = cv_timedwait(&pwp->ict_cv, &pwp->ict_lock,
1466			    lbolt + ici->quantum);
1467			if (ret == -1) {
1468				break;
1469			}
1470		}
1471
1472		if (ici->stop_thread == B_TRUE) {
1473			continue;
1474		}
1475
1476		DTRACE_PROBE1(pmcs__check__intr__coal, pmcs_io_intr_coal_t *,
1477		    &pwp->io_intr_coal);
1478
1479		/*
1480		 * Determine whether to adjust timer
1481		 */
1482		if (ici->num_intrs == 0) {
1483			/*
1484			 * If timer is off, nothing more to do.
1485			 */
1486			if (!pwp->io_intr_coal.timer_on) {
1487				continue;
1488			}
1489
1490			/*
1491			 * No interrupts.  Turn off the timer.
1492			 */
1493			pmcs_wr_topunit(pwp, PMCS_INT_COALESCING_CONTROL, 0);
1494
1495			if (pwp->odb_auto_clear & (1 << PMCS_MSIX_IODONE)) {
1496				pmcs_wr_topunit(pwp, PMCS_OBDB_AUTO_CLR,
1497				    pwp->odb_auto_clear);
1498			}
1499
1500			ici->timer_on = B_FALSE;
1501			ici->max_io_completions = 0;
1502			ici->num_intrs = 0;
1503			ici->int_cleared = B_FALSE;
1504			ici->num_io_completions = 0;
1505			DTRACE_PROBE1(pmcs__intr__coalesce__timer__off,
1506			    pmcs_io_intr_coal_t *, ici);
1507			continue;
1508		}
1509
1510		avg_nsecs = ici->nsecs_between_intrs / ici->num_intrs;
1511
1512		if ((ici->num_intrs > ici->intr_threshold) &&
1513		    (ici->max_io_completions > (ici->num_intrs >>
1514		    PMCS_MAX_IO_COMPS_LOWAT_SHIFT)) &&
1515		    (ici->max_io_completions < (ici->num_intrs >>
1516		    PMCS_MAX_IO_COMPS_HIWAT_SHIFT))) {
1517			pmcs_set_intr_coal_timer(pwp, INCREASE_TIMER);
1518		} else if (avg_nsecs >
1519		    (ici->intr_coal_timer * 1000 * 2)) {
1520			pmcs_set_intr_coal_timer(pwp, DECREASE_TIMER);
1521		}
1522
1523		/*
1524		 * Reset values for new sampling period.
1525		 */
1526		ici->max_io_completions = 0;
1527		ici->nsecs_between_intrs = 0;
1528		ici->num_intrs = 0;
1529		ici->num_io_completions = 0;
1530
1531		/*
1532		 * If a firmware event log file is configured, check to see
1533		 * if it needs to be written to the file.  We do this here
1534		 * because writing to a file from a callout thread (i.e.
1535		 * from the watchdog timer) can cause livelocks.
1536		 */
1537		if (pwp->fwlog_file) {
1538			mutex_exit(&pwp->ict_lock);
1539			pmcs_gather_fwlog(pwp);
1540			mutex_enter(&pwp->ict_lock);
1541		}
1542	}
1543
1544	mutex_exit(&pwp->ict_lock);
1545	thread_exit();
1546}
1547
1548void
1549pmcs_iodone_intr(pmcs_hw_t *pwp)
1550{
1551	char local[PMCS_QENTRY_SIZE << 1];
1552	pmcs_iocomp_cb_t *ioccb;
1553	uint32_t w0, ci, pi, nbuf, lim = 0, niodone = 0, iomb_opcode;
1554	size_t amt;
1555	uint32_t *ptr;
1556	hrtime_t curtime = gethrtime();
1557
1558	ci = pmcs_rd_oqci(pwp, PMCS_OQ_IODONE);
1559	pi = pmcs_rd_oqpi(pwp, PMCS_OQ_IODONE);
1560
1561	while (ci != pi) {
1562		OQLIM_CHECK(pwp, lim);
1563		ptr = GET_OQ_ENTRY(pwp, PMCS_OQ_IODONE, ci, 0);
1564		w0 = LE_32(ptr[0]);
1565		VALID_IOMB_CHECK(pwp, w0, ptr, ci, pi);
1566		WRONG_OBID_CHECK(pwp, w0, PMCS_OQ_IODONE);
1567		iomb_opcode = (w0 & PMCS_IOMB_OPCODE_MASK);
1568
1569		if ((iomb_opcode == PMCOUT_SSP_COMPLETION) ||
1570		    (iomb_opcode == PMCOUT_SATA_COMPLETION)) {
1571			ioccb =
1572			    kmem_cache_alloc(pwp->iocomp_cb_cache, KM_NOSLEEP);
1573			if (ioccb == NULL) {
1574				pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL,
1575				    "%s: kmem_cache_alloc failed", __func__);
1576				break;
1577			}
1578
1579			COPY_OUTBOUND(pwp, w0, ioccb->iomb, nbuf, amt, ptr,
1580			    PMCS_OQ_IODONE, ci);
1581
1582			niodone++;
1583			pmcs_process_io_completion(pwp, ioccb, amt);
1584		} else {
1585			COPY_OUTBOUND(pwp, w0, local, nbuf, amt, ptr,
1586			    PMCS_OQ_IODONE, ci);
1587
1588			switch (iomb_opcode) {
1589			case PMCOUT_ECHO:
1590				pmcs_process_echo_completion(pwp, local, amt);
1591				break;
1592			case PMCOUT_SATA_EVENT:
1593				pmcs_process_sata_event(pwp, local, amt);
1594				break;
1595			case PMCOUT_SSP_EVENT:
1596				pmcs_process_ssp_event(pwp, local, amt);
1597				break;
1598			case PMCOUT_SKIP_ENTRIES:
1599				pmcs_prt(pwp, PMCS_PRT_DEBUG3, NULL, NULL,
1600				    "%s: skip %d entries", __func__, nbuf);
1601				break;
1602			default:
1603				(void) snprintf(local, sizeof (local),
1604				    "%s: unhandled message", __func__);
1605				pmcs_print_entry(pwp, PMCS_PRT_DEBUG, local,
1606				    ptr);
1607				break;
1608			}
1609		}
1610
1611		STEP_OQ_ENTRY(pwp, PMCS_OQ_IODONE, ci, nbuf);
1612	}
1613
1614	if (lim != 0) {
1615		SYNC_OQ_ENTRY(pwp, PMCS_OQ_IODONE, ci, pi);
1616	}
1617
1618	/*
1619	 * Update the interrupt coalescing timer check stats and run
1620	 * completions for queued up commands.
1621	 */
1622
1623	if (niodone > 0) {
1624		/*
1625		 * If we can't get the lock, then completions are either
1626		 * already running or will be scheduled to do so shortly.
1627		 */
1628		if (mutex_tryenter(&pwp->cq_lock) != 0) {
1629			PMCS_CQ_RUN_LOCKED(pwp);
1630			mutex_exit(&pwp->cq_lock);
1631		}
1632
1633		mutex_enter(&pwp->ict_lock);
1634		pwp->io_intr_coal.nsecs_between_intrs +=
1635		    curtime - pwp->io_intr_coal.last_io_comp;
1636		pwp->io_intr_coal.num_intrs++;
1637		pwp->io_intr_coal.num_io_completions += niodone;
1638		if (niodone >= PMCS_MAX_IO_COMPS_PER_INTR) {
1639			pwp->io_intr_coal.max_io_completions++;
1640		}
1641		pwp->io_intr_coal.last_io_comp = gethrtime();
1642		mutex_exit(&pwp->ict_lock);
1643	}
1644}
1645
1646void
1647pmcs_event_intr(pmcs_hw_t *pwp)
1648{
1649	char local[PMCS_QENTRY_SIZE << 1];
1650	uint32_t w0, ci, pi, nbuf, lim =  0;
1651	size_t amt;
1652	uint32_t *ptr;
1653
1654	ci = pmcs_rd_oqci(pwp, PMCS_OQ_EVENTS);
1655	pi = pmcs_rd_oqpi(pwp, PMCS_OQ_EVENTS);
1656
1657	while (ci != pi) {
1658		OQLIM_CHECK(pwp, lim);
1659		ptr = GET_OQ_ENTRY(pwp, PMCS_OQ_EVENTS, ci, 0);
1660		w0 = LE_32(ptr[0]);
1661		VALID_IOMB_CHECK(pwp, w0, ptr, ci, pi);
1662		WRONG_OBID_CHECK(pwp, w0, PMCS_OQ_EVENTS);
1663		COPY_OUTBOUND(pwp, w0, local, nbuf, amt, ptr,
1664		    PMCS_OQ_EVENTS, ci);
1665
1666		switch (w0 & PMCS_IOMB_OPCODE_MASK) {
1667		case PMCOUT_ECHO:
1668			pmcs_process_echo_completion(pwp, local, amt);
1669			break;
1670		case PMCOUT_SATA_EVENT:
1671			pmcs_process_sata_event(pwp, local, amt);
1672			break;
1673		case PMCOUT_SSP_EVENT:
1674			pmcs_process_ssp_event(pwp, local, amt);
1675			break;
1676		case PMCOUT_GENERAL_EVENT:
1677			pmcs_process_general_event(pwp, ptr);
1678			break;
1679		case PMCOUT_DEVICE_HANDLE_REMOVED:
1680		{
1681			uint32_t port = IOP_EVENT_PORTID(LE_32(ptr[1]));
1682			uint32_t did = LE_32(ptr[2]);
1683			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1684			    "PortID 0x%x device_id 0x%x removed", port, did);
1685			break;
1686		}
1687		case PMCOUT_SAS_HW_EVENT:
1688			if (nbuf > 1) {
1689				pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL,
1690				    "multiple SAS HW_EVENT (%d) responses "
1691				    "in EVENT OQ", nbuf);
1692			}
1693			pmcs_process_sas_hw_event(pwp, local, PMCS_QENTRY_SIZE);
1694			break;
1695		case PMCOUT_FW_FLASH_UPDATE:
1696		case PMCOUT_GET_TIME_STAMP:
1697		case PMCOUT_GET_DEVICE_STATE:
1698		case PMCOUT_SET_DEVICE_STATE:
1699		case PMCOUT_SAS_DIAG_EXECUTE:
1700			pmcs_process_completion(pwp, local, amt);
1701			break;
1702		case PMCOUT_SKIP_ENTRIES:
1703			pmcs_prt(pwp, PMCS_PRT_DEBUG3, NULL, NULL,
1704			    "%s: skip %d entries", __func__, nbuf);
1705			break;
1706		default:
1707			(void) snprintf(local, sizeof (local),
1708			    "%s: unhandled message", __func__);
1709			pmcs_print_entry(pwp, PMCS_PRT_DEBUG, local, ptr);
1710			break;
1711		}
1712		STEP_OQ_ENTRY(pwp, PMCS_OQ_EVENTS, ci, nbuf);
1713	}
1714	if (lim) {
1715		SYNC_OQ_ENTRY(pwp, PMCS_OQ_EVENTS, ci, pi);
1716	}
1717}
1718
1719void
1720pmcs_timed_out(pmcs_hw_t *pwp, uint32_t htag, const char *func)
1721{
1722#ifdef	DEBUG
1723	hrtime_t now = gethrtime();
1724	int i;
1725
1726	for (i = 0; i < 256; i++) {
1727		if (pwp->ftags[i] == htag) {
1728			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1729			    "Inbound msg (tag 0x%8x) timed out - "
1730			    "was started %llu ns ago in %s:%d",
1731			    htag, (unsigned long long) (now - pwp->ftime[i]),
1732			    func, pwp->ftag_lines[i]);
1733			return;
1734		}
1735	}
1736#endif
1737	pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1738	    "Inbound Message (tag 0x%08x) timed out- was started in %s",
1739	    htag, func);
1740}
1741