1993e3fafSRobert Mustacchi /*
2993e3fafSRobert Mustacchi  * This file and its contents are supplied under the terms of the
3993e3fafSRobert Mustacchi  * Common Development and Distribution License ("CDDL"), version 1.0.
4993e3fafSRobert Mustacchi  * You may only use this file in accordance with the terms of version
5993e3fafSRobert Mustacchi  * 1.0 of the CDDL.
6993e3fafSRobert Mustacchi  *
7993e3fafSRobert Mustacchi  * A full copy of the text of the CDDL should have accompanied this
8993e3fafSRobert Mustacchi  * source.  A copy of the CDDL is also available via the Internet at
9993e3fafSRobert Mustacchi  * http://www.illumos.org/license/CDDL.
10993e3fafSRobert Mustacchi  */
11993e3fafSRobert Mustacchi 
12993e3fafSRobert Mustacchi /*
132aba3acdSRobert Mustacchi  * Copyright (c) 2018, Joyent, Inc.
14ec82ef79SMatthias Scheler  * Copyright (c) 2019 by Western Digital Corporation
15*0ae0ab6fSJoshua M. Clulow  * Copyright 2022 Oxide Computer Company
16993e3fafSRobert Mustacchi  */
17993e3fafSRobert Mustacchi 
18993e3fafSRobert Mustacchi /*
19993e3fafSRobert Mustacchi  * illumos USB framework endpoints and functions for xHCI.
20993e3fafSRobert Mustacchi  *
21993e3fafSRobert Mustacchi  * Please see the big theory statement in xhci.c for more information.
22993e3fafSRobert Mustacchi  */
23993e3fafSRobert Mustacchi 
24993e3fafSRobert Mustacchi #include <sys/usb/hcd/xhci/xhci.h>
25993e3fafSRobert Mustacchi #include <sys/sysmacros.h>
26993e3fafSRobert Mustacchi #include <sys/strsun.h>
27993e3fafSRobert Mustacchi #include <sys/strsubr.h>
28993e3fafSRobert Mustacchi 
29ec82ef79SMatthias Scheler xhci_t *
xhci_hcdi_get_xhcip_from_dev(usba_device_t * ud)30993e3fafSRobert Mustacchi xhci_hcdi_get_xhcip_from_dev(usba_device_t *ud)
31993e3fafSRobert Mustacchi {
32993e3fafSRobert Mustacchi 	dev_info_t *dip = ud->usb_root_hub_dip;
33993e3fafSRobert Mustacchi 	xhci_t *xhcip = ddi_get_soft_state(xhci_soft_state,
34993e3fafSRobert Mustacchi 	    ddi_get_instance(dip));
35993e3fafSRobert Mustacchi 	VERIFY(xhcip != NULL);
36993e3fafSRobert Mustacchi 	return (xhcip);
37993e3fafSRobert Mustacchi }
38993e3fafSRobert Mustacchi 
39993e3fafSRobert Mustacchi static xhci_t *
xhci_hcdi_get_xhcip(usba_pipe_handle_data_t * ph)40993e3fafSRobert Mustacchi xhci_hcdi_get_xhcip(usba_pipe_handle_data_t *ph)
41993e3fafSRobert Mustacchi {
42993e3fafSRobert Mustacchi 	return (xhci_hcdi_get_xhcip_from_dev(ph->p_usba_device));
43993e3fafSRobert Mustacchi }
44993e3fafSRobert Mustacchi 
45993e3fafSRobert Mustacchi /*
46993e3fafSRobert Mustacchi  * While the xHCI hardware is capable of supporting power management, we don't
47993e3fafSRobert Mustacchi  * in the driver right now. Note, USBA doesn't seem to end up calling this entry
48993e3fafSRobert Mustacchi  * point.
49993e3fafSRobert Mustacchi  */
50993e3fafSRobert Mustacchi /* ARGSUSED */
51993e3fafSRobert Mustacchi static int
xhci_hcdi_pm_support(dev_info_t * dip)52993e3fafSRobert Mustacchi xhci_hcdi_pm_support(dev_info_t *dip)
53993e3fafSRobert Mustacchi {
54993e3fafSRobert Mustacchi 	return (USB_FAILURE);
55993e3fafSRobert Mustacchi }
56993e3fafSRobert Mustacchi 
57993e3fafSRobert Mustacchi static int
xhci_hcdi_pipe_open(usba_pipe_handle_data_t * ph,usb_flags_t usb_flags)58993e3fafSRobert Mustacchi xhci_hcdi_pipe_open(usba_pipe_handle_data_t *ph, usb_flags_t usb_flags)
59993e3fafSRobert Mustacchi {
60993e3fafSRobert Mustacchi 	xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
61993e3fafSRobert Mustacchi 	xhci_pipe_t *pipe;
62*0ae0ab6fSJoshua M. Clulow 	xhci_endpoint_t *xep = NULL;
63993e3fafSRobert Mustacchi 	xhci_device_t *xd;
64993e3fafSRobert Mustacchi 	int kmflags = usb_flags & USB_FLAGS_SLEEP ? KM_SLEEP : KM_NOSLEEP;
65993e3fafSRobert Mustacchi 	int ret;
66993e3fafSRobert Mustacchi 	uint_t epid;
67993e3fafSRobert Mustacchi 
68993e3fafSRobert Mustacchi 	mutex_enter(&xhcip->xhci_lock);
69993e3fafSRobert Mustacchi 	if (xhcip->xhci_state & XHCI_S_ERROR) {
70993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
71993e3fafSRobert Mustacchi 		return (USB_HC_HARDWARE_ERROR);
72993e3fafSRobert Mustacchi 	}
73993e3fafSRobert Mustacchi 	mutex_exit(&xhcip->xhci_lock);
74993e3fafSRobert Mustacchi 
75993e3fafSRobert Mustacchi 	/*
76993e3fafSRobert Mustacchi 	 * If we're here, something must be trying to open an already-opened
77993e3fafSRobert Mustacchi 	 * pipe which is bad news.
78993e3fafSRobert Mustacchi 	 */
79993e3fafSRobert Mustacchi 	if (ph->p_hcd_private != NULL) {
80993e3fafSRobert Mustacchi 		return (USB_FAILURE);
81993e3fafSRobert Mustacchi 	}
82993e3fafSRobert Mustacchi 
83993e3fafSRobert Mustacchi 	pipe = kmem_zalloc(sizeof (xhci_pipe_t), kmflags);
84993e3fafSRobert Mustacchi 	if (pipe == NULL) {
85993e3fafSRobert Mustacchi 		return (USB_NO_RESOURCES);
86993e3fafSRobert Mustacchi 	}
87993e3fafSRobert Mustacchi 	pipe->xp_opentime = gethrtime();
88993e3fafSRobert Mustacchi 	pipe->xp_pipe = ph;
89993e3fafSRobert Mustacchi 
90993e3fafSRobert Mustacchi 	/*
91993e3fafSRobert Mustacchi 	 * If this is the root hub, there's nothing special to do on open. Just
92993e3fafSRobert Mustacchi 	 * go ahead and allow it to be opened. All we have to do is add this to
93993e3fafSRobert Mustacchi 	 * the list of our tracking structures for open pipes.
94993e3fafSRobert Mustacchi 	 */
95993e3fafSRobert Mustacchi 	if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
96993e3fafSRobert Mustacchi 		xep = NULL;
97993e3fafSRobert Mustacchi 		goto add;
98993e3fafSRobert Mustacchi 	}
99993e3fafSRobert Mustacchi 
100993e3fafSRobert Mustacchi 	/*
101993e3fafSRobert Mustacchi 	 * Now that we're here, we're being asked to open up an endpoint of some
102993e3fafSRobert Mustacchi 	 * kind. Because we've already handled the case of the root hub,
103993e3fafSRobert Mustacchi 	 * everything should have a device.
104993e3fafSRobert Mustacchi 	 */
105993e3fafSRobert Mustacchi 	epid = xhci_endpoint_pipe_to_epid(ph);
106993e3fafSRobert Mustacchi 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
107993e3fafSRobert Mustacchi 	if (xd == NULL) {
108993e3fafSRobert Mustacchi 		xhci_error(xhcip, "!encountered endpoint (%d) without device "
109993e3fafSRobert Mustacchi 		    "during pipe open", epid);
110993e3fafSRobert Mustacchi 		kmem_free(pipe, sizeof (xhci_pipe_t));
111993e3fafSRobert Mustacchi 		return (USB_FAILURE);
112993e3fafSRobert Mustacchi 	}
113993e3fafSRobert Mustacchi 
114993e3fafSRobert Mustacchi 	/*
115993e3fafSRobert Mustacchi 	 * See if this endpoint exists or not, in general endpoints should not
116993e3fafSRobert Mustacchi 	 * exist except for the default control endpoint, which we don't tear
117993e3fafSRobert Mustacchi 	 * down until the device itself is cleaned up. Otherwise, a given pipe
118993e3fafSRobert Mustacchi 	 * can only be open once.
119993e3fafSRobert Mustacchi 	 */
120993e3fafSRobert Mustacchi 	mutex_enter(&xhcip->xhci_lock);
121993e3fafSRobert Mustacchi 	if (epid == XHCI_DEFAULT_ENDPOINT) {
122993e3fafSRobert Mustacchi 		xep = xd->xd_endpoints[epid];
123993e3fafSRobert Mustacchi 		VERIFY(xep != NULL);
124993e3fafSRobert Mustacchi 		VERIFY(xep->xep_pipe == NULL);
125993e3fafSRobert Mustacchi 		xep->xep_pipe = ph;
126993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
127993e3fafSRobert Mustacchi 		ret = xhci_endpoint_update_default(xhcip, xd, xep);
128993e3fafSRobert Mustacchi 		if (ret != USB_SUCCESS) {
129993e3fafSRobert Mustacchi 			kmem_free(pipe, sizeof (xhci_pipe_t));
130993e3fafSRobert Mustacchi 			return (ret);
131993e3fafSRobert Mustacchi 		}
132993e3fafSRobert Mustacchi 		goto add;
133993e3fafSRobert Mustacchi 	}
134993e3fafSRobert Mustacchi 
135*0ae0ab6fSJoshua M. Clulow 	/*
136*0ae0ab6fSJoshua M. Clulow 	 * If we're opening an endpoint other than the default control endpoint,
137*0ae0ab6fSJoshua M. Clulow 	 * then the device should have had a USB address assigned by the
138*0ae0ab6fSJoshua M. Clulow 	 * controller. Sanity check that before continuing.
139*0ae0ab6fSJoshua M. Clulow 	 */
140*0ae0ab6fSJoshua M. Clulow 	VERIFY(xd->xd_addressed == B_TRUE);
141*0ae0ab6fSJoshua M. Clulow 
142*0ae0ab6fSJoshua M. Clulow 	/*
143*0ae0ab6fSJoshua M. Clulow 	 * We may have already initialized the endpoint with a previous pipe
144*0ae0ab6fSJoshua M. Clulow 	 * open.
145*0ae0ab6fSJoshua M. Clulow 	 */
146*0ae0ab6fSJoshua M. Clulow 	if ((xep = xd->xd_endpoints[epid]) != NULL &&
147*0ae0ab6fSJoshua M. Clulow 	    (xep->xep_state & XHCI_ENDPOINT_OPEN)) {
148993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
149*0ae0ab6fSJoshua M. Clulow 
150993e3fafSRobert Mustacchi 		kmem_free(pipe, sizeof (xhci_pipe_t));
151993e3fafSRobert Mustacchi 		xhci_log(xhcip, "!asked to open endpoint %d on slot %d and "
152993e3fafSRobert Mustacchi 		    "port %d, but endpoint already exists", epid, xd->xd_slot,
153993e3fafSRobert Mustacchi 		    xd->xd_port);
154993e3fafSRobert Mustacchi 		return (USB_FAILURE);
155993e3fafSRobert Mustacchi 	}
156993e3fafSRobert Mustacchi 
157*0ae0ab6fSJoshua M. Clulow 	if (xep != NULL) {
158*0ae0ab6fSJoshua M. Clulow 		/*
159*0ae0ab6fSJoshua M. Clulow 		 * The endpoint is already initialized but is not presently
160*0ae0ab6fSJoshua M. Clulow 		 * open so we can take it over here.
161*0ae0ab6fSJoshua M. Clulow 		 */
162*0ae0ab6fSJoshua M. Clulow 		if ((ret = xhci_endpoint_reinit(xhcip, xd, xep, ph) != 0)) {
163*0ae0ab6fSJoshua M. Clulow 			mutex_exit(&xhcip->xhci_lock);
164*0ae0ab6fSJoshua M. Clulow 
165*0ae0ab6fSJoshua M. Clulow 			kmem_free(pipe, sizeof (xhci_pipe_t));
166*0ae0ab6fSJoshua M. Clulow 			xhci_log(xhcip, "!asked to reopen endpoint %d on "
167*0ae0ab6fSJoshua M. Clulow 			    "slot %d and port %d, but reinit failed (%d)",
168*0ae0ab6fSJoshua M. Clulow 			    epid, xd->xd_slot, xd->xd_port, ret);
169*0ae0ab6fSJoshua M. Clulow 			return (ret);
170*0ae0ab6fSJoshua M. Clulow 		}
171*0ae0ab6fSJoshua M. Clulow 
172*0ae0ab6fSJoshua M. Clulow 		/*
173*0ae0ab6fSJoshua M. Clulow 		 * We need to ensure the endpoint is stopped before we try to
174*0ae0ab6fSJoshua M. Clulow 		 * reset the transfer ring.
175*0ae0ab6fSJoshua M. Clulow 		 */
176*0ae0ab6fSJoshua M. Clulow 		xep->xep_state |= XHCI_ENDPOINT_QUIESCE;
177*0ae0ab6fSJoshua M. Clulow 		if ((ret = xhci_endpoint_quiesce(xhcip, xd, xep)) !=
178*0ae0ab6fSJoshua M. Clulow 		    USB_SUCCESS) {
179*0ae0ab6fSJoshua M. Clulow 			/*
180*0ae0ab6fSJoshua M. Clulow 			 * If we could not quiesce the endpoint, release it so
181*0ae0ab6fSJoshua M. Clulow 			 * that another open can try again.
182*0ae0ab6fSJoshua M. Clulow 			 */
183*0ae0ab6fSJoshua M. Clulow 			xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
184*0ae0ab6fSJoshua M. Clulow 			xhci_endpoint_release(xhcip, xep);
185*0ae0ab6fSJoshua M. Clulow 			mutex_exit(&xhcip->xhci_lock);
186*0ae0ab6fSJoshua M. Clulow 
187*0ae0ab6fSJoshua M. Clulow 			kmem_free(pipe, sizeof (xhci_pipe_t));
188*0ae0ab6fSJoshua M. Clulow 			xhci_log(xhcip, "!asked to reopen endpoint %d on "
189*0ae0ab6fSJoshua M. Clulow 			    "slot %d and port %d, but quiesce failed (%d)",
190*0ae0ab6fSJoshua M. Clulow 			    epid, xd->xd_slot, xd->xd_port, ret);
191*0ae0ab6fSJoshua M. Clulow 			return (ret);
192*0ae0ab6fSJoshua M. Clulow 		}
193*0ae0ab6fSJoshua M. Clulow 
194*0ae0ab6fSJoshua M. Clulow 		/*
195*0ae0ab6fSJoshua M. Clulow 		 * Reset the transfer ring dequeue pointer.  The initial
196*0ae0ab6fSJoshua M. Clulow 		 * Configure Endpoint command leaves the endpoint in the
197*0ae0ab6fSJoshua M. Clulow 		 * Running state (xHCI 1.2 / 4.6.6), so even though the ring is
198*0ae0ab6fSJoshua M. Clulow 		 * still empty we ring the doorbell to end up in the same state
199*0ae0ab6fSJoshua M. Clulow 		 * (Running but Inactive).
200*0ae0ab6fSJoshua M. Clulow 		 */
201*0ae0ab6fSJoshua M. Clulow 		mutex_exit(&xhcip->xhci_lock);
202*0ae0ab6fSJoshua M. Clulow 		if ((ret = xhci_command_set_tr_dequeue(xhcip, xd, xep)) != 0 ||
203*0ae0ab6fSJoshua M. Clulow 		    (ret = xhci_endpoint_ring(xhcip, xd, xep)) != 0) {
204*0ae0ab6fSJoshua M. Clulow 			mutex_enter(&xhcip->xhci_lock);
205*0ae0ab6fSJoshua M. Clulow 			xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
206*0ae0ab6fSJoshua M. Clulow 			xhci_endpoint_release(xhcip, xep);
207*0ae0ab6fSJoshua M. Clulow 			mutex_exit(&xhcip->xhci_lock);
208*0ae0ab6fSJoshua M. Clulow 
209*0ae0ab6fSJoshua M. Clulow 			kmem_free(pipe, sizeof (xhci_pipe_t));
210*0ae0ab6fSJoshua M. Clulow 			xhci_log(xhcip, "!asked to open endpoint %d on "
211*0ae0ab6fSJoshua M. Clulow 			    "slot %d and port %d, but restart failed (%d)",
212*0ae0ab6fSJoshua M. Clulow 			    epid, xd->xd_slot, xd->xd_port, ret);
213*0ae0ab6fSJoshua M. Clulow 			return (USB_FAILURE);
214*0ae0ab6fSJoshua M. Clulow 		}
215*0ae0ab6fSJoshua M. Clulow 		mutex_enter(&xhcip->xhci_lock);
216*0ae0ab6fSJoshua M. Clulow 		xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
217*0ae0ab6fSJoshua M. Clulow 		mutex_exit(&xhcip->xhci_lock);
218*0ae0ab6fSJoshua M. Clulow 
219*0ae0ab6fSJoshua M. Clulow 		goto add;
220993e3fafSRobert Mustacchi 	}
221993e3fafSRobert Mustacchi 
222993e3fafSRobert Mustacchi 	/*
223*0ae0ab6fSJoshua M. Clulow 	 * Okay, at this point we need to go create and set up an endpoint from
224*0ae0ab6fSJoshua M. Clulow 	 * scratch.  Once we're done, we'll try to install it and make sure
225*0ae0ab6fSJoshua M. Clulow 	 * that it doesn't conflict with something else going on.
226993e3fafSRobert Mustacchi 	 */
227993e3fafSRobert Mustacchi 	ret = xhci_endpoint_init(xhcip, xd, ph);
228993e3fafSRobert Mustacchi 	if (ret != 0) {
229993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
230993e3fafSRobert Mustacchi 		kmem_free(pipe, sizeof (xhci_pipe_t));
231993e3fafSRobert Mustacchi 		if (ret == EIO) {
232993e3fafSRobert Mustacchi 			xhci_error(xhcip, "failed to initialize endpoint %d "
233993e3fafSRobert Mustacchi 			    "on device slot %d and port %d: encountered fatal "
234993e3fafSRobert Mustacchi 			    "FM error, resetting device", epid, xd->xd_slot,
235993e3fafSRobert Mustacchi 			    xd->xd_port);
236993e3fafSRobert Mustacchi 			xhci_fm_runtime_reset(xhcip);
237993e3fafSRobert Mustacchi 		}
238993e3fafSRobert Mustacchi 		return (USB_HC_HARDWARE_ERROR);
239993e3fafSRobert Mustacchi 	}
240993e3fafSRobert Mustacchi 	xep = xd->xd_endpoints[epid];
241993e3fafSRobert Mustacchi 
242993e3fafSRobert Mustacchi 	mutex_enter(&xd->xd_imtx);
243993e3fafSRobert Mustacchi 	mutex_exit(&xhcip->xhci_lock);
244993e3fafSRobert Mustacchi 
245993e3fafSRobert Mustacchi 	/*
2462aba3acdSRobert Mustacchi 	 * Update the slot and input context for this endpoint. We make sure to
2472aba3acdSRobert Mustacchi 	 * always set the slot as having changed in the context field as the
2482aba3acdSRobert Mustacchi 	 * specification suggests we should and some hardware requires it.
249993e3fafSRobert Mustacchi 	 */
250993e3fafSRobert Mustacchi 	xd->xd_input->xic_drop_flags = LE_32(0);
2512aba3acdSRobert Mustacchi 	xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0) |
2522aba3acdSRobert Mustacchi 	    XHCI_INCTX_MASK_DCI(epid + 1));
253993e3fafSRobert Mustacchi 
254993e3fafSRobert Mustacchi 	if (epid + 1 > XHCI_SCTX_GET_DCI(LE_32(xd->xd_slotin->xsc_info))) {
255993e3fafSRobert Mustacchi 		uint32_t info;
256993e3fafSRobert Mustacchi 
257993e3fafSRobert Mustacchi 		info = xd->xd_slotin->xsc_info;
258993e3fafSRobert Mustacchi 		info &= ~XHCI_SCTX_DCI_MASK;
259993e3fafSRobert Mustacchi 		info |= XHCI_SCTX_SET_DCI(epid + 1);
260993e3fafSRobert Mustacchi 		xd->xd_slotin->xsc_info = info;
261993e3fafSRobert Mustacchi 	}
262993e3fafSRobert Mustacchi 
263993e3fafSRobert Mustacchi 	XHCI_DMA_SYNC(xd->xd_ictx, DDI_DMA_SYNC_FORDEV);
264993e3fafSRobert Mustacchi 	if (xhci_check_dma_handle(xhcip, &xd->xd_ictx) != DDI_FM_OK) {
265993e3fafSRobert Mustacchi 		mutex_exit(&xd->xd_imtx);
266993e3fafSRobert Mustacchi 		xhci_endpoint_fini(xd, epid);
267993e3fafSRobert Mustacchi 		kmem_free(pipe, sizeof (xhci_pipe_t));
268993e3fafSRobert Mustacchi 		xhci_error(xhcip, "failed to open pipe on endpoint %d of "
269993e3fafSRobert Mustacchi 		    "device with slot %d and port %d: encountered fatal FM "
270993e3fafSRobert Mustacchi 		    "error syncing device input context, resetting device",
271993e3fafSRobert Mustacchi 		    epid, xd->xd_slot, xd->xd_port);
272993e3fafSRobert Mustacchi 		xhci_fm_runtime_reset(xhcip);
273993e3fafSRobert Mustacchi 		return (USB_HC_HARDWARE_ERROR);
274993e3fafSRobert Mustacchi 	}
275993e3fafSRobert Mustacchi 
276993e3fafSRobert Mustacchi 	if ((ret = xhci_command_configure_endpoint(xhcip, xd)) != USB_SUCCESS) {
277993e3fafSRobert Mustacchi 		mutex_exit(&xd->xd_imtx);
278993e3fafSRobert Mustacchi 		xhci_endpoint_fini(xd, epid);
279993e3fafSRobert Mustacchi 		kmem_free(pipe, sizeof (xhci_pipe_t));
280993e3fafSRobert Mustacchi 		return (ret);
281993e3fafSRobert Mustacchi 	}
282993e3fafSRobert Mustacchi 
283993e3fafSRobert Mustacchi 	mutex_exit(&xd->xd_imtx);
284993e3fafSRobert Mustacchi add:
285993e3fafSRobert Mustacchi 	pipe->xp_ep = xep;
286993e3fafSRobert Mustacchi 	ph->p_hcd_private = (usb_opaque_t)pipe;
287993e3fafSRobert Mustacchi 	mutex_enter(&xhcip->xhci_lock);
288993e3fafSRobert Mustacchi 	list_insert_tail(&xhcip->xhci_usba.xa_pipes, pipe);
289993e3fafSRobert Mustacchi 	mutex_exit(&xhcip->xhci_lock);
290993e3fafSRobert Mustacchi 
291993e3fafSRobert Mustacchi 	return (USB_SUCCESS);
292993e3fafSRobert Mustacchi }
293993e3fafSRobert Mustacchi 
294993e3fafSRobert Mustacchi static void
xhci_hcdi_periodic_free(xhci_t * xhcip,xhci_pipe_t * xp)295993e3fafSRobert Mustacchi xhci_hcdi_periodic_free(xhci_t *xhcip, xhci_pipe_t *xp)
296993e3fafSRobert Mustacchi {
297993e3fafSRobert Mustacchi 	int i;
298993e3fafSRobert Mustacchi 	xhci_periodic_pipe_t *xpp = &xp->xp_periodic;
299993e3fafSRobert Mustacchi 
300993e3fafSRobert Mustacchi 	if (xpp->xpp_tsize == 0)
301993e3fafSRobert Mustacchi 		return;
302993e3fafSRobert Mustacchi 
303993e3fafSRobert Mustacchi 	for (i = 0; i < xpp->xpp_ntransfers; i++) {
304993e3fafSRobert Mustacchi 		if (xpp->xpp_transfers[i] == NULL)
305993e3fafSRobert Mustacchi 			continue;
306993e3fafSRobert Mustacchi 		xhci_transfer_free(xhcip, xpp->xpp_transfers[i]);
307993e3fafSRobert Mustacchi 		xpp->xpp_transfers[i] = NULL;
308993e3fafSRobert Mustacchi 	}
309993e3fafSRobert Mustacchi 
310993e3fafSRobert Mustacchi 	xpp->xpp_ntransfers = 0;
311993e3fafSRobert Mustacchi 	xpp->xpp_tsize = 0;
312993e3fafSRobert Mustacchi }
313993e3fafSRobert Mustacchi 
314993e3fafSRobert Mustacchi /*
315993e3fafSRobert Mustacchi  * Iterate over all transfers and free everything on the pipe. Once done, update
316993e3fafSRobert Mustacchi  * the ring to basically 'consume' everything. For periodic IN endpoints, we
317993e3fafSRobert Mustacchi  * need to handle this somewhat differently and actually close the original
318993e3fafSRobert Mustacchi  * request and not deallocate the related pieces as those exist for the lifetime
319993e3fafSRobert Mustacchi  * of the endpoint and are constantly reused.
320993e3fafSRobert Mustacchi  */
321993e3fafSRobert Mustacchi static void
xhci_hcdi_pipe_flush(xhci_t * xhcip,xhci_endpoint_t * xep,int intr_code)322993e3fafSRobert Mustacchi xhci_hcdi_pipe_flush(xhci_t *xhcip, xhci_endpoint_t *xep, int intr_code)
323993e3fafSRobert Mustacchi {
324993e3fafSRobert Mustacchi 	xhci_transfer_t *xt;
325993e3fafSRobert Mustacchi 
326993e3fafSRobert Mustacchi 	ASSERT(MUTEX_HELD(&xhcip->xhci_lock));
327993e3fafSRobert Mustacchi 
328993e3fafSRobert Mustacchi 	while ((xt = list_remove_head(&xep->xep_transfers)) != NULL) {
329993e3fafSRobert Mustacchi 		if (xhci_endpoint_is_periodic_in(xep) == B_FALSE) {
330993e3fafSRobert Mustacchi 			usba_hcdi_cb(xep->xep_pipe, xt->xt_usba_req,
331993e3fafSRobert Mustacchi 			    USB_CR_FLUSHED);
332993e3fafSRobert Mustacchi 			xhci_transfer_free(xhcip, xt);
333993e3fafSRobert Mustacchi 		}
334993e3fafSRobert Mustacchi 	}
335993e3fafSRobert Mustacchi 
336993e3fafSRobert Mustacchi 	if (xhci_endpoint_is_periodic_in(xep) == B_TRUE) {
337993e3fafSRobert Mustacchi 		xhci_pipe_t *xp = (xhci_pipe_t *)xep->xep_pipe->p_hcd_private;
338993e3fafSRobert Mustacchi 		xhci_periodic_pipe_t *xpp = &xp->xp_periodic;
339993e3fafSRobert Mustacchi 
340993e3fafSRobert Mustacchi 		if (xpp->xpp_usb_req != NULL) {
341993e3fafSRobert Mustacchi 			usba_hcdi_cb(xep->xep_pipe, xpp->xpp_usb_req,
342993e3fafSRobert Mustacchi 			    intr_code);
343993e3fafSRobert Mustacchi 			xpp->xpp_usb_req = NULL;
344993e3fafSRobert Mustacchi 		}
345993e3fafSRobert Mustacchi 	}
346993e3fafSRobert Mustacchi }
347993e3fafSRobert Mustacchi 
348993e3fafSRobert Mustacchi /*
349993e3fafSRobert Mustacchi  * We've been asked to terminate some set of regular I/O on an interrupt pipe.
350993e3fafSRobert Mustacchi  * If this is for the root device, e.g. the xhci driver itself, then we remove
351993e3fafSRobert Mustacchi  * our interrupt callback. Otherwise we stop the device for interrupt polling as
352993e3fafSRobert Mustacchi  * follows:
353993e3fafSRobert Mustacchi  *
354993e3fafSRobert Mustacchi  * 1. Issue a stop endpoint command
355993e3fafSRobert Mustacchi  * 2. Check to make sure that the endpoint stopped and reset it if needed.
356993e3fafSRobert Mustacchi  * 3. Any thing that gets resolved can callback in the interim.
357993e3fafSRobert Mustacchi  * 4. Ensure that nothing is scheduled on the ring
358993e3fafSRobert Mustacchi  * 5. Skip the contents of the ring and set the TR dequeue pointer.
359993e3fafSRobert Mustacchi  * 6. Return the original callback with a USB_CR_STOPPED_POLLING, NULL out the
360993e3fafSRobert Mustacchi  *    callback in the process.
361993e3fafSRobert Mustacchi  */
362993e3fafSRobert Mustacchi static int
xhci_hcdi_pipe_poll_fini(usba_pipe_handle_data_t * ph,boolean_t is_close)363993e3fafSRobert Mustacchi xhci_hcdi_pipe_poll_fini(usba_pipe_handle_data_t *ph, boolean_t is_close)
364993e3fafSRobert Mustacchi {
365993e3fafSRobert Mustacchi 	int ret;
366993e3fafSRobert Mustacchi 	uint_t epid;
367993e3fafSRobert Mustacchi 	xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
368993e3fafSRobert Mustacchi 	xhci_device_t *xd;
369993e3fafSRobert Mustacchi 	xhci_endpoint_t *xep;
370993e3fafSRobert Mustacchi 	xhci_pipe_t *xp;
371993e3fafSRobert Mustacchi 	xhci_periodic_pipe_t *xpp;
372993e3fafSRobert Mustacchi 	usb_opaque_t urp;
373993e3fafSRobert Mustacchi 
374993e3fafSRobert Mustacchi 	mutex_enter(&xhcip->xhci_lock);
375993e3fafSRobert Mustacchi 	if (xhcip->xhci_state & XHCI_S_ERROR) {
376993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
377993e3fafSRobert Mustacchi 		return (USB_HC_HARDWARE_ERROR);
378993e3fafSRobert Mustacchi 	}
379993e3fafSRobert Mustacchi 
380993e3fafSRobert Mustacchi 	if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
381993e3fafSRobert Mustacchi 		xhci_root_hub_intr_root_disable(xhcip);
382993e3fafSRobert Mustacchi 		ret = USB_SUCCESS;
383993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
384993e3fafSRobert Mustacchi 		return (ret);
385993e3fafSRobert Mustacchi 	}
386993e3fafSRobert Mustacchi 
387993e3fafSRobert Mustacchi 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
388993e3fafSRobert Mustacchi 	epid = xhci_endpoint_pipe_to_epid(ph);
389993e3fafSRobert Mustacchi 	if (xd->xd_endpoints[epid] == NULL) {
390993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
391993e3fafSRobert Mustacchi 		xhci_error(xhcip, "asked to stop intr polling on slot %d, "
392993e3fafSRobert Mustacchi 		    "port %d, endpoint: %d, but no endpoint structure",
393993e3fafSRobert Mustacchi 		    xd->xd_slot, xd->xd_port, epid);
394993e3fafSRobert Mustacchi 		return (USB_FAILURE);
395993e3fafSRobert Mustacchi 	}
396993e3fafSRobert Mustacchi 	xep = xd->xd_endpoints[epid];
397993e3fafSRobert Mustacchi 	xp = (xhci_pipe_t *)ph->p_hcd_private;
398993e3fafSRobert Mustacchi 	if (xp == NULL) {
399993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
400993e3fafSRobert Mustacchi 		xhci_error(xhcip, "asked to do finish polling on slot %d, "
401993e3fafSRobert Mustacchi 		    "port %d, endpoint: %d, but no pipe structure",
402993e3fafSRobert Mustacchi 		    xd->xd_slot, xd->xd_port, epid);
403993e3fafSRobert Mustacchi 		return (USB_FAILURE);
404993e3fafSRobert Mustacchi 	}
405993e3fafSRobert Mustacchi 	xpp = &xp->xp_periodic;
406993e3fafSRobert Mustacchi 
407993e3fafSRobert Mustacchi 	/*
408993e3fafSRobert Mustacchi 	 * Ensure that no other resets or time outs are going on right now.
409993e3fafSRobert Mustacchi 	 */
410*0ae0ab6fSJoshua M. Clulow 	xhci_endpoint_serialize(xhcip, xep);
411993e3fafSRobert Mustacchi 
412993e3fafSRobert Mustacchi 	if (xpp->xpp_poll_state == XHCI_PERIODIC_POLL_IDLE) {
413993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
414993e3fafSRobert Mustacchi 		return (USB_SUCCESS);
415993e3fafSRobert Mustacchi 	}
416993e3fafSRobert Mustacchi 
417993e3fafSRobert Mustacchi 	if (xpp->xpp_poll_state == XHCI_PERIODIC_POLL_STOPPING) {
418993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
419993e3fafSRobert Mustacchi 		return (USB_FAILURE);
420993e3fafSRobert Mustacchi 	}
421993e3fafSRobert Mustacchi 
422993e3fafSRobert Mustacchi 	xpp->xpp_poll_state = XHCI_PERIODIC_POLL_STOPPING;
423993e3fafSRobert Mustacchi 	xep->xep_state |= XHCI_ENDPOINT_QUIESCE;
424993e3fafSRobert Mustacchi 	ret = xhci_endpoint_quiesce(xhcip, xd, xep);
425993e3fafSRobert Mustacchi 	if (ret != USB_SUCCESS) {
426993e3fafSRobert Mustacchi 		xhci_error(xhcip, "!failed to quiesce endpoint on slot %d, "
427993e3fafSRobert Mustacchi 		    "port %d, endpoint: %d, failed with %d.",
428993e3fafSRobert Mustacchi 		    xd->xd_slot, xd->xd_port, epid, ret);
429993e3fafSRobert Mustacchi 		xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
430993e3fafSRobert Mustacchi 		cv_broadcast(&xep->xep_state_cv);
431993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
432993e3fafSRobert Mustacchi 		return (ret);
433993e3fafSRobert Mustacchi 	}
434993e3fafSRobert Mustacchi 
435993e3fafSRobert Mustacchi 	/*
436993e3fafSRobert Mustacchi 	 * Okay, we've stopped this ring time to wrap it all up. Remove all the
437993e3fafSRobert Mustacchi 	 * transfers, note they aren't freed like a pipe reset.
438993e3fafSRobert Mustacchi 	 */
439993e3fafSRobert Mustacchi 	while (list_is_empty(&xep->xep_transfers) == 0)
440993e3fafSRobert Mustacchi 		(void) list_remove_head(&xep->xep_transfers);
441993e3fafSRobert Mustacchi 	xhci_ring_skip(&xep->xep_ring);
442993e3fafSRobert Mustacchi 	mutex_exit(&xhcip->xhci_lock);
443993e3fafSRobert Mustacchi 
444993e3fafSRobert Mustacchi 	if ((ret = xhci_command_set_tr_dequeue(xhcip, xd, xep)) !=
445993e3fafSRobert Mustacchi 	    USB_SUCCESS) {
446993e3fafSRobert Mustacchi 		xhci_error(xhcip, "!failed to reset endpoint ring on slot %d, "
447993e3fafSRobert Mustacchi 		    "port %d, endpoint: %d, failed with %d.",
448993e3fafSRobert Mustacchi 		    xd->xd_slot, xd->xd_port, epid, ret);
449993e3fafSRobert Mustacchi 		mutex_enter(&xhcip->xhci_lock);
450993e3fafSRobert Mustacchi 		xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
451993e3fafSRobert Mustacchi 		cv_broadcast(&xep->xep_state_cv);
452993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
453993e3fafSRobert Mustacchi 		return (ret);
454993e3fafSRobert Mustacchi 	}
455993e3fafSRobert Mustacchi 
456993e3fafSRobert Mustacchi 	mutex_enter(&xhcip->xhci_lock);
457993e3fafSRobert Mustacchi 	urp = xpp->xpp_usb_req;
458993e3fafSRobert Mustacchi 	xpp->xpp_usb_req = NULL;
459993e3fafSRobert Mustacchi 	xpp->xpp_poll_state = XHCI_PERIODIC_POLL_IDLE;
460993e3fafSRobert Mustacchi 	xep->xep_state &= ~XHCI_ENDPOINT_PERIODIC;
461993e3fafSRobert Mustacchi 	mutex_exit(&xhcip->xhci_lock);
462993e3fafSRobert Mustacchi 
463993e3fafSRobert Mustacchi 	/*
464993e3fafSRobert Mustacchi 	 * It's possible that with a persistent pipe, we may not actually have
465993e3fafSRobert Mustacchi 	 * anything left to call back on, because we already had.
466993e3fafSRobert Mustacchi 	 */
467993e3fafSRobert Mustacchi 	if (urp != NULL) {
468993e3fafSRobert Mustacchi 		usba_hcdi_cb(ph, urp, is_close == B_TRUE ?
469993e3fafSRobert Mustacchi 		    USB_CR_PIPE_CLOSING : USB_CR_STOPPED_POLLING);
470993e3fafSRobert Mustacchi 	}
471993e3fafSRobert Mustacchi 
472993e3fafSRobert Mustacchi 	/*
473993e3fafSRobert Mustacchi 	 * Notify anything waiting for us that we're done quiescing this device.
474993e3fafSRobert Mustacchi 	 */
475993e3fafSRobert Mustacchi 	mutex_enter(&xhcip->xhci_lock);
476993e3fafSRobert Mustacchi 	xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
477993e3fafSRobert Mustacchi 	cv_broadcast(&xep->xep_state_cv);
478993e3fafSRobert Mustacchi 	mutex_exit(&xhcip->xhci_lock);
479993e3fafSRobert Mustacchi 
480993e3fafSRobert Mustacchi 	return (USB_SUCCESS);
481993e3fafSRobert Mustacchi 
482993e3fafSRobert Mustacchi }
483993e3fafSRobert Mustacchi 
484993e3fafSRobert Mustacchi /*
485993e3fafSRobert Mustacchi  * Tear down everything that we did in open. After this, the consumer of this
486993e3fafSRobert Mustacchi  * USB device is done.
487993e3fafSRobert Mustacchi  */
488993e3fafSRobert Mustacchi static int
xhci_hcdi_pipe_close(usba_pipe_handle_data_t * ph,usb_flags_t usb_flags)489993e3fafSRobert Mustacchi xhci_hcdi_pipe_close(usba_pipe_handle_data_t *ph, usb_flags_t usb_flags)
490993e3fafSRobert Mustacchi {
491993e3fafSRobert Mustacchi 	xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
492993e3fafSRobert Mustacchi 	xhci_pipe_t *xp;
493993e3fafSRobert Mustacchi 	xhci_device_t *xd;
494993e3fafSRobert Mustacchi 	xhci_endpoint_t *xep;
495*0ae0ab6fSJoshua M. Clulow 	int ret;
496993e3fafSRobert Mustacchi 	uint_t epid;
497993e3fafSRobert Mustacchi 
498993e3fafSRobert Mustacchi 	if ((ph->p_ep.bmAttributes & USB_EP_ATTR_MASK) == USB_EP_ATTR_INTR &&
499993e3fafSRobert Mustacchi 	    xhcip->xhci_usba.xa_intr_cb_ph != NULL) {
500993e3fafSRobert Mustacchi 		if ((ret = xhci_hcdi_pipe_poll_fini(ph, B_TRUE)) !=
501993e3fafSRobert Mustacchi 		    USB_SUCCESS) {
502993e3fafSRobert Mustacchi 			return (ret);
503993e3fafSRobert Mustacchi 		}
504993e3fafSRobert Mustacchi 	}
505993e3fafSRobert Mustacchi 
506993e3fafSRobert Mustacchi 	mutex_enter(&xhcip->xhci_lock);
507993e3fafSRobert Mustacchi 
508993e3fafSRobert Mustacchi 	xp = (xhci_pipe_t *)ph->p_hcd_private;
509993e3fafSRobert Mustacchi 	VERIFY(xp != NULL);
510993e3fafSRobert Mustacchi 
511993e3fafSRobert Mustacchi 	/*
512993e3fafSRobert Mustacchi 	 * The default endpoint is special. It is created and destroyed with the
513993e3fafSRobert Mustacchi 	 * device. So like with open, closing it is just state tracking. The
514993e3fafSRobert Mustacchi 	 * same is true for the root hub.
515993e3fafSRobert Mustacchi 	 */
516993e3fafSRobert Mustacchi 	if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR)
517993e3fafSRobert Mustacchi 		goto remove;
518993e3fafSRobert Mustacchi 
519993e3fafSRobert Mustacchi 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
520993e3fafSRobert Mustacchi 	epid = xhci_endpoint_pipe_to_epid(ph);
521993e3fafSRobert Mustacchi 	if (xd->xd_endpoints[epid] == NULL) {
522993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
523993e3fafSRobert Mustacchi 		xhci_error(xhcip, "asked to do close pipe on slot %d, "
524993e3fafSRobert Mustacchi 		    "port %d, endpoint: %d, but no endpoint structure",
525993e3fafSRobert Mustacchi 		    xd->xd_slot, xd->xd_port, epid);
526993e3fafSRobert Mustacchi 		return (USB_FAILURE);
527993e3fafSRobert Mustacchi 	}
528993e3fafSRobert Mustacchi 	xep = xd->xd_endpoints[epid];
529993e3fafSRobert Mustacchi 
530993e3fafSRobert Mustacchi 	if (xp->xp_ep != NULL && xp->xp_ep->xep_num == XHCI_DEFAULT_ENDPOINT) {
531993e3fafSRobert Mustacchi 		xep->xep_pipe = NULL;
532993e3fafSRobert Mustacchi 		goto remove;
533993e3fafSRobert Mustacchi 	}
534993e3fafSRobert Mustacchi 
535993e3fafSRobert Mustacchi 	/*
536*0ae0ab6fSJoshua M. Clulow 	 * We clean up the endpoint by stopping it and cancelling any transfers
537*0ae0ab6fSJoshua M. Clulow 	 * that were in flight at the time.  The endpoint is not unconfigured
538*0ae0ab6fSJoshua M. Clulow 	 * until the device is torn down later.
539993e3fafSRobert Mustacchi 	 */
540*0ae0ab6fSJoshua M. Clulow 	xhci_endpoint_timeout_cancel(xhcip, xep);
541*0ae0ab6fSJoshua M. Clulow 	xep->xep_state |= XHCI_ENDPOINT_QUIESCE;
542*0ae0ab6fSJoshua M. Clulow 	if ((ret = xhci_endpoint_quiesce(xhcip, xd, xep)) != USB_SUCCESS) {
543*0ae0ab6fSJoshua M. Clulow 		/*
544*0ae0ab6fSJoshua M. Clulow 		 * If we cannot stop the ring, it is not safe to proceed and we
545*0ae0ab6fSJoshua M. Clulow 		 * must keep the pipe open.
546*0ae0ab6fSJoshua M. Clulow 		 */
547*0ae0ab6fSJoshua M. Clulow 		xep->xep_state &=
548*0ae0ab6fSJoshua M. Clulow 		    ~(XHCI_ENDPOINT_TEARDOWN | XHCI_ENDPOINT_QUIESCE);
549*0ae0ab6fSJoshua M. Clulow 		cv_broadcast(&xep->xep_state_cv);
550*0ae0ab6fSJoshua M. Clulow 		mutex_exit(&xhcip->xhci_lock);
551*0ae0ab6fSJoshua M. Clulow 		xhci_error(xhcip, "asked to do close pipe on slot %d, "
552*0ae0ab6fSJoshua M. Clulow 		    "port %d, endpoint: %d, but quiesce failed %d",
553*0ae0ab6fSJoshua M. Clulow 		    xd->xd_slot, xd->xd_port, epid, ret);
554*0ae0ab6fSJoshua M. Clulow 		return (USB_FAILURE);
555*0ae0ab6fSJoshua M. Clulow 	}
556993e3fafSRobert Mustacchi 
557993e3fafSRobert Mustacchi 	/*
558*0ae0ab6fSJoshua M. Clulow 	 * Now that we've stopped the endpoint, see if we need to flush any
559993e3fafSRobert Mustacchi 	 * transfers.
560993e3fafSRobert Mustacchi 	 */
561993e3fafSRobert Mustacchi 	xhci_hcdi_pipe_flush(xhcip, xep, USB_CR_PIPE_CLOSING);
562993e3fafSRobert Mustacchi 	if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
563993e3fafSRobert Mustacchi 		xhci_hcdi_periodic_free(xhcip, xp);
564993e3fafSRobert Mustacchi 	}
565993e3fafSRobert Mustacchi 
566*0ae0ab6fSJoshua M. Clulow 	xhci_endpoint_release(xhcip, xep);
567993e3fafSRobert Mustacchi 
568993e3fafSRobert Mustacchi remove:
569993e3fafSRobert Mustacchi 	ph->p_hcd_private = NULL;
570993e3fafSRobert Mustacchi 	list_remove(&xhcip->xhci_usba.xa_pipes, xp);
571993e3fafSRobert Mustacchi 	kmem_free(xp, sizeof (xhci_pipe_t));
572993e3fafSRobert Mustacchi 
573993e3fafSRobert Mustacchi 	mutex_exit(&xhcip->xhci_lock);
574993e3fafSRobert Mustacchi 
575993e3fafSRobert Mustacchi 	return (USB_SUCCESS);
576993e3fafSRobert Mustacchi }
577993e3fafSRobert Mustacchi 
578993e3fafSRobert Mustacchi /*
579993e3fafSRobert Mustacchi  * We've been asked to reset a pipe aka an endpoint. This endpoint may be in an
580993e3fafSRobert Mustacchi  * arbitrary state, it may be running or it may be halted. In this case, we go
581993e3fafSRobert Mustacchi  * through and check whether or not we know it's been halted or not. If it has
582993e3fafSRobert Mustacchi  * not, then we stop the endpoint.
583993e3fafSRobert Mustacchi  *
584993e3fafSRobert Mustacchi  * Once the endpoint has been stopped, walk all transfers and go ahead and
585993e3fafSRobert Mustacchi  * basically return them as being flushed. Then finally set the dequeue point
586993e3fafSRobert Mustacchi  * for this endpoint.
587993e3fafSRobert Mustacchi  */
588993e3fafSRobert Mustacchi /* ARGSUSED */
589993e3fafSRobert Mustacchi static int
xhci_hcdi_pipe_reset(usba_pipe_handle_data_t * ph,usb_flags_t usb_flags)590993e3fafSRobert Mustacchi xhci_hcdi_pipe_reset(usba_pipe_handle_data_t *ph, usb_flags_t usb_flags)
591993e3fafSRobert Mustacchi {
592993e3fafSRobert Mustacchi 	xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
593993e3fafSRobert Mustacchi 	xhci_device_t *xd;
594993e3fafSRobert Mustacchi 	xhci_endpoint_t *xep;
595993e3fafSRobert Mustacchi 	uint_t epid;
596993e3fafSRobert Mustacchi 	int ret;
597993e3fafSRobert Mustacchi 
598993e3fafSRobert Mustacchi 	mutex_enter(&xhcip->xhci_lock);
599993e3fafSRobert Mustacchi 	if (xhcip->xhci_state & XHCI_S_ERROR) {
600993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
601993e3fafSRobert Mustacchi 		return (USB_HC_HARDWARE_ERROR);
602993e3fafSRobert Mustacchi 	}
603993e3fafSRobert Mustacchi 
604993e3fafSRobert Mustacchi 	if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
605993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
606993e3fafSRobert Mustacchi 		return (USB_NOT_SUPPORTED);
607993e3fafSRobert Mustacchi 	}
608993e3fafSRobert Mustacchi 
609993e3fafSRobert Mustacchi 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
610993e3fafSRobert Mustacchi 	epid = xhci_endpoint_pipe_to_epid(ph);
611993e3fafSRobert Mustacchi 	if (xd->xd_endpoints[epid] == NULL) {
612993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
613993e3fafSRobert Mustacchi 		xhci_error(xhcip, "asked to do reset pipe on slot %d, "
614993e3fafSRobert Mustacchi 		    "port %d, endpoint: %d, but no endpoint structure",
615993e3fafSRobert Mustacchi 		    xd->xd_slot, xd->xd_port, epid);
616993e3fafSRobert Mustacchi 		return (USB_FAILURE);
617993e3fafSRobert Mustacchi 	}
618993e3fafSRobert Mustacchi 
619993e3fafSRobert Mustacchi 	xep = xd->xd_endpoints[epid];
620993e3fafSRobert Mustacchi 
621993e3fafSRobert Mustacchi 	/*
622993e3fafSRobert Mustacchi 	 * Ensure that no other resets or time outs are going on right now.
623993e3fafSRobert Mustacchi 	 */
624*0ae0ab6fSJoshua M. Clulow 	xhci_endpoint_serialize(xhcip, xep);
625993e3fafSRobert Mustacchi 
626993e3fafSRobert Mustacchi 	xep->xep_state |= XHCI_ENDPOINT_QUIESCE;
627993e3fafSRobert Mustacchi 	ret = xhci_endpoint_quiesce(xhcip, xd, xep);
628993e3fafSRobert Mustacchi 	if (ret != USB_SUCCESS) {
629993e3fafSRobert Mustacchi 		/*
630993e3fafSRobert Mustacchi 		 * We failed to quiesce for some reason, remove the flag and let
631993e3fafSRobert Mustacchi 		 * someone else give it a shot.
632993e3fafSRobert Mustacchi 		 */
633993e3fafSRobert Mustacchi 		xhci_error(xhcip, "!failed to quiesce endpoint on slot %d, "
634993e3fafSRobert Mustacchi 		    "port %d, endpoint: %d, failed with %d.",
635993e3fafSRobert Mustacchi 		    xd->xd_slot, xd->xd_port, epid, ret);
636993e3fafSRobert Mustacchi 		xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
637993e3fafSRobert Mustacchi 		cv_broadcast(&xep->xep_state_cv);
638993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
639993e3fafSRobert Mustacchi 		return (ret);
640993e3fafSRobert Mustacchi 	}
641993e3fafSRobert Mustacchi 
642993e3fafSRobert Mustacchi 	xhci_ring_skip(&xep->xep_ring);
643993e3fafSRobert Mustacchi 
644993e3fafSRobert Mustacchi 	mutex_exit(&xhcip->xhci_lock);
645993e3fafSRobert Mustacchi 	if ((ret = xhci_command_set_tr_dequeue(xhcip, xd, xep)) !=
646993e3fafSRobert Mustacchi 	    USB_SUCCESS) {
647993e3fafSRobert Mustacchi 		xhci_error(xhcip, "!failed to reset endpoint ring on slot %d, "
648993e3fafSRobert Mustacchi 		    "port %d, endpoint: %d, failed setting ring dequeue with "
649993e3fafSRobert Mustacchi 		    "%d.", xd->xd_slot, xd->xd_port, epid, ret);
650993e3fafSRobert Mustacchi 		mutex_enter(&xhcip->xhci_lock);
651993e3fafSRobert Mustacchi 		xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
652993e3fafSRobert Mustacchi 		cv_broadcast(&xep->xep_state_cv);
653993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
654993e3fafSRobert Mustacchi 		return (ret);
655993e3fafSRobert Mustacchi 	}
656993e3fafSRobert Mustacchi 
657993e3fafSRobert Mustacchi 	mutex_enter(&xhcip->xhci_lock);
658993e3fafSRobert Mustacchi 	xhci_hcdi_pipe_flush(xhcip, xep, USB_CR_PIPE_RESET);
659993e3fafSRobert Mustacchi 
660993e3fafSRobert Mustacchi 	/*
661993e3fafSRobert Mustacchi 	 * We need to remove the periodic flag as part of resetting, as if this
662993e3fafSRobert Mustacchi 	 * was used for periodic activity, it no longer is and therefore can now
663993e3fafSRobert Mustacchi 	 * be used for such purposes.
664993e3fafSRobert Mustacchi 	 *
665993e3fafSRobert Mustacchi 	 * Notify anything waiting for us that we're done quiescing this device.
666993e3fafSRobert Mustacchi 	 */
667993e3fafSRobert Mustacchi 	xep->xep_state &= ~(XHCI_ENDPOINT_QUIESCE | XHCI_ENDPOINT_PERIODIC);
668993e3fafSRobert Mustacchi 	cv_broadcast(&xep->xep_state_cv);
669993e3fafSRobert Mustacchi 	mutex_exit(&xhcip->xhci_lock);
670993e3fafSRobert Mustacchi 
671993e3fafSRobert Mustacchi 	return (USB_SUCCESS);
672993e3fafSRobert Mustacchi }
673993e3fafSRobert Mustacchi 
674993e3fafSRobert Mustacchi /*
675993e3fafSRobert Mustacchi  * We're asked to reset or change the data toggle, which is used in a few cases.
676993e3fafSRobert Mustacchi  * However, there doesn't seem to be a good way to do this in xHCI as the data
677993e3fafSRobert Mustacchi  * toggle isn't exposed. It seems that dropping a reset endpoint would
678993e3fafSRobert Mustacchi  * theoretically do this; however, that can only be used when in the HALTED
679993e3fafSRobert Mustacchi  * state. As such, for now we just return.
680993e3fafSRobert Mustacchi  */
681993e3fafSRobert Mustacchi /* ARGSUSED */
682993e3fafSRobert Mustacchi void
xhci_hcdi_pipe_reset_data_toggle(usba_pipe_handle_data_t * pipe_handle)683993e3fafSRobert Mustacchi xhci_hcdi_pipe_reset_data_toggle(usba_pipe_handle_data_t *pipe_handle)
684993e3fafSRobert Mustacchi {
685993e3fafSRobert Mustacchi }
686993e3fafSRobert Mustacchi 
687993e3fafSRobert Mustacchi /*
688993e3fafSRobert Mustacchi  * We need to convert the USB request to an 8-byte little endian value. If we
689993e3fafSRobert Mustacchi  * didn't have to think about big endian systems, this would be fine.
690993e3fafSRobert Mustacchi  * Unfortunately, with them, this is a bit confusing. The problem is that if you
691993e3fafSRobert Mustacchi  * think of this as a struct layout, the order that we or things together
692993e3fafSRobert Mustacchi  * represents their byte layout. e.g. ctrl_bRequest is at offset 1 in the SETUP
693993e3fafSRobert Mustacchi  * STAGE trb. However, when it becomes a part of a 64-bit big endian number, if
694993e3fafSRobert Mustacchi  * ends up at byte 7, where as it needs to be at one. Hence why we do a final
695993e3fafSRobert Mustacchi  * LE_64 at the end of this, to convert this into the byte order that it's
696993e3fafSRobert Mustacchi  * expected to be in.
697993e3fafSRobert Mustacchi  */
698993e3fafSRobert Mustacchi static uint64_t
xhci_hcdi_ctrl_req_to_trb(usb_ctrl_req_t * ucrp)699993e3fafSRobert Mustacchi xhci_hcdi_ctrl_req_to_trb(usb_ctrl_req_t *ucrp)
700993e3fafSRobert Mustacchi {
701993e3fafSRobert Mustacchi 	uint64_t ret = ucrp->ctrl_bmRequestType |
702993e3fafSRobert Mustacchi 	    (ucrp->ctrl_bRequest << 8) |
703993e3fafSRobert Mustacchi 	    ((uint64_t)LE_16(ucrp->ctrl_wValue) << 16) |
704993e3fafSRobert Mustacchi 	    ((uint64_t)LE_16(ucrp->ctrl_wIndex) << 32) |
705993e3fafSRobert Mustacchi 	    ((uint64_t)LE_16(ucrp->ctrl_wLength) << 48);
706993e3fafSRobert Mustacchi 	return (LE_64(ret));
707993e3fafSRobert Mustacchi }
708993e3fafSRobert Mustacchi 
709993e3fafSRobert Mustacchi /*
710993e3fafSRobert Mustacchi  * USBA calls us in order to make a specific control type request to a device,
711993e3fafSRobert Mustacchi  * potentially even the root hub. If the request is for the root hub, then we
712993e3fafSRobert Mustacchi  * need to intercept this and cons up the requested data.
713993e3fafSRobert Mustacchi  */
714993e3fafSRobert Mustacchi static int
xhci_hcdi_pipe_ctrl_xfer(usba_pipe_handle_data_t * ph,usb_ctrl_req_t * ucrp,usb_flags_t usb_flags)715993e3fafSRobert Mustacchi xhci_hcdi_pipe_ctrl_xfer(usba_pipe_handle_data_t *ph, usb_ctrl_req_t *ucrp,
716993e3fafSRobert Mustacchi     usb_flags_t usb_flags)
717993e3fafSRobert Mustacchi {
718993e3fafSRobert Mustacchi 	int ret, statusdir, trt;
719993e3fafSRobert Mustacchi 	uint_t ep;
720993e3fafSRobert Mustacchi 	xhci_device_t *xd;
721993e3fafSRobert Mustacchi 	xhci_endpoint_t *xep;
722993e3fafSRobert Mustacchi 	xhci_transfer_t *xt;
723993e3fafSRobert Mustacchi 	boolean_t datain;
724993e3fafSRobert Mustacchi 
725993e3fafSRobert Mustacchi 	xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
726993e3fafSRobert Mustacchi 
727993e3fafSRobert Mustacchi 	mutex_enter(&xhcip->xhci_lock);
728993e3fafSRobert Mustacchi 	if (xhcip->xhci_state & XHCI_S_ERROR) {
729993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
730993e3fafSRobert Mustacchi 		return (USB_HC_HARDWARE_ERROR);
731993e3fafSRobert Mustacchi 	}
732993e3fafSRobert Mustacchi 
733993e3fafSRobert Mustacchi 	if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
734993e3fafSRobert Mustacchi 		ret = xhci_root_hub_ctrl_req(xhcip, ph, ucrp);
735993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
736993e3fafSRobert Mustacchi 		return (ret);
737993e3fafSRobert Mustacchi 	}
738993e3fafSRobert Mustacchi 
739993e3fafSRobert Mustacchi 	/*
740993e3fafSRobert Mustacchi 	 * Determine the device and endpoint.
741993e3fafSRobert Mustacchi 	 */
742993e3fafSRobert Mustacchi 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
743993e3fafSRobert Mustacchi 	ep = xhci_endpoint_pipe_to_epid(ph);
744993e3fafSRobert Mustacchi 	if (xd->xd_endpoints[ep] == NULL) {
745993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
746993e3fafSRobert Mustacchi 		xhci_error(xhcip, "asked to do control transfer on slot %d, "
747993e3fafSRobert Mustacchi 		    "port %d, endpoint: %d, but no endpoint structure",
748993e3fafSRobert Mustacchi 		    xd->xd_slot, xd->xd_port, ep);
749993e3fafSRobert Mustacchi 		return (USB_FAILURE);
750993e3fafSRobert Mustacchi 	}
751993e3fafSRobert Mustacchi 	xep = xd->xd_endpoints[ep];
752993e3fafSRobert Mustacchi 
753993e3fafSRobert Mustacchi 	/*
754993e3fafSRobert Mustacchi 	 * There are several types of requests that we have to handle in special
755993e3fafSRobert Mustacchi 	 * ways in xHCI. If we have one of those requests, then we don't
756993e3fafSRobert Mustacchi 	 * necessarily go through the normal path. These special cases are all
757993e3fafSRobert Mustacchi 	 * documented in xHCI 1.1 / 4.5.4.
758993e3fafSRobert Mustacchi 	 *
759993e3fafSRobert Mustacchi 	 * Looking at that, you may ask why aren't SET_CONFIGURATION and SET_IF
760993e3fafSRobert Mustacchi 	 * special cased here. This action is a little confusing by default. The
761993e3fafSRobert Mustacchi 	 * xHCI specification requires that we may need to issue a configure
762993e3fafSRobert Mustacchi 	 * endpoint command as part of this. However, the xHCI 1.1 / 4.5.4.2
763993e3fafSRobert Mustacchi 	 * states that we don't actually need to if nothing in the endpoint
764993e3fafSRobert Mustacchi 	 * configuration context has changed. Because nothing in it should have
765993e3fafSRobert Mustacchi 	 * changed as part of this, we don't need to do anything and instead
766993e3fafSRobert Mustacchi 	 * just can issue the request normally. We're also assuming in the
767993e3fafSRobert Mustacchi 	 * USB_REQ_SET_IF case that if something's changing the interface, the
768993e3fafSRobert Mustacchi 	 * non-default endpoint will have yet to be opened.
769993e3fafSRobert Mustacchi 	 */
770993e3fafSRobert Mustacchi 	if (ucrp->ctrl_bmRequestType == USB_DEV_REQ_HOST_TO_DEV &&
771993e3fafSRobert Mustacchi 	    ucrp->ctrl_bRequest == USB_REQ_SET_ADDRESS) {
772993e3fafSRobert Mustacchi 		/*
773993e3fafSRobert Mustacchi 		 * As we've defined an explicit set-address endpoint, we should
774993e3fafSRobert Mustacchi 		 * never call this function. If we get here, always fail.
775993e3fafSRobert Mustacchi 		 */
776993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
777993e3fafSRobert Mustacchi 		usba_hcdi_cb(ph, (usb_opaque_t)ucrp, USB_CR_NOT_SUPPORTED);
778993e3fafSRobert Mustacchi 		return (USB_SUCCESS);
779993e3fafSRobert Mustacchi 	}
780993e3fafSRobert Mustacchi 
781993e3fafSRobert Mustacchi 	mutex_exit(&xhcip->xhci_lock);
782993e3fafSRobert Mustacchi 
783993e3fafSRobert Mustacchi 	/*
784993e3fafSRobert Mustacchi 	 * Allocate the transfer memory, etc.
785993e3fafSRobert Mustacchi 	 */
786993e3fafSRobert Mustacchi 	xt = xhci_transfer_alloc(xhcip, xep, ucrp->ctrl_wLength, 2, usb_flags);
787993e3fafSRobert Mustacchi 	if (xt == NULL) {
788993e3fafSRobert Mustacchi 		return (USB_NO_RESOURCES);
789993e3fafSRobert Mustacchi 	}
790993e3fafSRobert Mustacchi 	xt->xt_usba_req = (usb_opaque_t)ucrp;
791993e3fafSRobert Mustacchi 	xt->xt_timeout = ucrp->ctrl_timeout;
792993e3fafSRobert Mustacchi 	if (xt->xt_timeout == 0) {
793993e3fafSRobert Mustacchi 		xt->xt_timeout = HCDI_DEFAULT_TIMEOUT;
794993e3fafSRobert Mustacchi 	}
795993e3fafSRobert Mustacchi 
796993e3fafSRobert Mustacchi 	if (ucrp->ctrl_wLength > 0) {
797993e3fafSRobert Mustacchi 		if ((ucrp->ctrl_bmRequestType & USB_DEV_REQ_DEV_TO_HOST) != 0) {
798993e3fafSRobert Mustacchi 			trt = XHCI_TRB_TRT_IN;
799993e3fafSRobert Mustacchi 			datain = B_TRUE;
800993e3fafSRobert Mustacchi 			statusdir = 0;
801993e3fafSRobert Mustacchi 		} else {
802993e3fafSRobert Mustacchi 			trt = XHCI_TRB_TRT_OUT;
803993e3fafSRobert Mustacchi 			datain = B_FALSE;
804993e3fafSRobert Mustacchi 			statusdir = XHCI_TRB_DIR_IN;
805993e3fafSRobert Mustacchi 
806993e3fafSRobert Mustacchi 			xhci_transfer_copy(xt, ucrp->ctrl_data->b_rptr,
807993e3fafSRobert Mustacchi 			    ucrp->ctrl_wLength, B_FALSE);
808993e3fafSRobert Mustacchi 			if (xhci_transfer_sync(xhcip, xt,
809993e3fafSRobert Mustacchi 			    DDI_DMA_SYNC_FORDEV) != DDI_FM_OK) {
810993e3fafSRobert Mustacchi 				xhci_transfer_free(xhcip, xt);
811993e3fafSRobert Mustacchi 				xhci_error(xhcip, "failed to synchronize ctrl "
812993e3fafSRobert Mustacchi 				    "transfer DMA memory on endpoint %u of "
813993e3fafSRobert Mustacchi 				    "device on slot %d and port %d: resetting "
814993e3fafSRobert Mustacchi 				    "device", xep->xep_num, xd->xd_slot,
815993e3fafSRobert Mustacchi 				    xd->xd_port);
816993e3fafSRobert Mustacchi 				xhci_fm_runtime_reset(xhcip);
817993e3fafSRobert Mustacchi 				return (USB_HC_HARDWARE_ERROR);
818993e3fafSRobert Mustacchi 			}
819993e3fafSRobert Mustacchi 		}
820993e3fafSRobert Mustacchi 	} else {
821993e3fafSRobert Mustacchi 		trt = 0;
822993e3fafSRobert Mustacchi 		datain = B_FALSE;
823993e3fafSRobert Mustacchi 		statusdir = XHCI_TRB_DIR_IN;
824993e3fafSRobert Mustacchi 	}
825993e3fafSRobert Mustacchi 
826993e3fafSRobert Mustacchi 	/*
827993e3fafSRobert Mustacchi 	 * We always fill in the required setup and status TRBs ourselves;
828993e3fafSRobert Mustacchi 	 * however, to minimize our knowledge about how the data has been split
829993e3fafSRobert Mustacchi 	 * across multiple DMA cookies in an SGL, we leave that to the transfer
830993e3fafSRobert Mustacchi 	 * logic to fill in.
831993e3fafSRobert Mustacchi 	 */
832993e3fafSRobert Mustacchi 	xt->xt_trbs[0].trb_addr = xhci_hcdi_ctrl_req_to_trb(ucrp);
833993e3fafSRobert Mustacchi 	xt->xt_trbs[0].trb_status = LE_32(XHCI_TRB_LEN(8) | XHCI_TRB_INTR(0));
834993e3fafSRobert Mustacchi 	xt->xt_trbs[0].trb_flags = LE_32(trt | XHCI_TRB_IDT |
835993e3fafSRobert Mustacchi 	    XHCI_TRB_TYPE_SETUP);
836993e3fafSRobert Mustacchi 
837993e3fafSRobert Mustacchi 	if (ucrp->ctrl_wLength > 0)
838993e3fafSRobert Mustacchi 		xhci_transfer_trb_fill_data(xep, xt, 1, datain);
839993e3fafSRobert Mustacchi 
840993e3fafSRobert Mustacchi 	xt->xt_trbs[xt->xt_ntrbs - 1].trb_addr = 0;
841993e3fafSRobert Mustacchi 	xt->xt_trbs[xt->xt_ntrbs - 1].trb_status = LE_32(XHCI_TRB_INTR(0));
842993e3fafSRobert Mustacchi 	xt->xt_trbs[xt->xt_ntrbs - 1].trb_flags = LE_32(XHCI_TRB_TYPE_STATUS |
843993e3fafSRobert Mustacchi 	    XHCI_TRB_IOC | statusdir);
844993e3fafSRobert Mustacchi 
8452aba3acdSRobert Mustacchi 
846993e3fafSRobert Mustacchi 	mutex_enter(&xhcip->xhci_lock);
847993e3fafSRobert Mustacchi 
848993e3fafSRobert Mustacchi 	/*
849993e3fafSRobert Mustacchi 	 * Schedule the transfer, allocating resources in the process.
850993e3fafSRobert Mustacchi 	 */
851993e3fafSRobert Mustacchi 	if (xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE) != 0) {
852993e3fafSRobert Mustacchi 		xhci_transfer_free(xhcip, xt);
853993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
854993e3fafSRobert Mustacchi 		return (USB_NO_RESOURCES);
855993e3fafSRobert Mustacchi 	}
856993e3fafSRobert Mustacchi 
857993e3fafSRobert Mustacchi 	mutex_exit(&xhcip->xhci_lock);
858993e3fafSRobert Mustacchi 
859993e3fafSRobert Mustacchi 	return (USB_SUCCESS);
860993e3fafSRobert Mustacchi }
861993e3fafSRobert Mustacchi 
862993e3fafSRobert Mustacchi /*
863993e3fafSRobert Mustacchi  * This request is trying to get the upper bound on the amount of data we're
864993e3fafSRobert Mustacchi  * willing transfer in one go. Note that this amount can be broken down into
865993e3fafSRobert Mustacchi  * multiple SGL entries, this interface doesn't particularly care about that.
866993e3fafSRobert Mustacchi  */
867993e3fafSRobert Mustacchi /* ARGSUSED */
868993e3fafSRobert Mustacchi static int
xhci_hcdi_bulk_transfer_size(usba_device_t * ud,size_t * sizep)869993e3fafSRobert Mustacchi xhci_hcdi_bulk_transfer_size(usba_device_t *ud, size_t *sizep)
870993e3fafSRobert Mustacchi {
871993e3fafSRobert Mustacchi 	if (sizep != NULL)
872993e3fafSRobert Mustacchi 		*sizep = XHCI_MAX_TRANSFER;
873993e3fafSRobert Mustacchi 	return (USB_SUCCESS);
874993e3fafSRobert Mustacchi }
875993e3fafSRobert Mustacchi 
876993e3fafSRobert Mustacchi /*
877993e3fafSRobert Mustacchi  * Perform a bulk transfer. This is a pretty straightforward action. We
878993e3fafSRobert Mustacchi  * basically just allocate the appropriate transfer and try to schedule it,
879993e3fafSRobert Mustacchi  * hoping there is enough space.
880993e3fafSRobert Mustacchi  */
881993e3fafSRobert Mustacchi static int
xhci_hcdi_pipe_bulk_xfer(usba_pipe_handle_data_t * ph,usb_bulk_req_t * ubrp,usb_flags_t usb_flags)882993e3fafSRobert Mustacchi xhci_hcdi_pipe_bulk_xfer(usba_pipe_handle_data_t *ph, usb_bulk_req_t *ubrp,
883993e3fafSRobert Mustacchi     usb_flags_t usb_flags)
884993e3fafSRobert Mustacchi {
885993e3fafSRobert Mustacchi 	uint_t epid;
886993e3fafSRobert Mustacchi 	xhci_device_t *xd;
887993e3fafSRobert Mustacchi 	xhci_endpoint_t *xep;
888993e3fafSRobert Mustacchi 	xhci_transfer_t *xt;
889993e3fafSRobert Mustacchi 	boolean_t datain;
890993e3fafSRobert Mustacchi 
891993e3fafSRobert Mustacchi 	xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
892993e3fafSRobert Mustacchi 
893993e3fafSRobert Mustacchi 	mutex_enter(&xhcip->xhci_lock);
894993e3fafSRobert Mustacchi 	if (xhcip->xhci_state & XHCI_S_ERROR) {
895993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
896993e3fafSRobert Mustacchi 		return (USB_HC_HARDWARE_ERROR);
897993e3fafSRobert Mustacchi 	}
898993e3fafSRobert Mustacchi 
899993e3fafSRobert Mustacchi 	if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
900993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
901993e3fafSRobert Mustacchi 		return (USB_NOT_SUPPORTED);
902993e3fafSRobert Mustacchi 	}
903993e3fafSRobert Mustacchi 
904993e3fafSRobert Mustacchi 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
905993e3fafSRobert Mustacchi 	epid = xhci_endpoint_pipe_to_epid(ph);
906993e3fafSRobert Mustacchi 	if (xd->xd_endpoints[epid] == NULL) {
907993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
908993e3fafSRobert Mustacchi 		xhci_error(xhcip, "asked to do control transfer on slot %d, "
909993e3fafSRobert Mustacchi 		    "port %d, endpoint: %d, but no endpoint structure",
910993e3fafSRobert Mustacchi 		    xd->xd_slot, xd->xd_port, epid);
911993e3fafSRobert Mustacchi 		return (USB_FAILURE);
912993e3fafSRobert Mustacchi 	}
913993e3fafSRobert Mustacchi 	xep = xd->xd_endpoints[epid];
914993e3fafSRobert Mustacchi 	mutex_exit(&xhcip->xhci_lock);
915993e3fafSRobert Mustacchi 
916993e3fafSRobert Mustacchi 	if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
917993e3fafSRobert Mustacchi 		datain = B_TRUE;
918993e3fafSRobert Mustacchi 	} else {
919993e3fafSRobert Mustacchi 		datain = B_FALSE;
920993e3fafSRobert Mustacchi 	}
921993e3fafSRobert Mustacchi 
922993e3fafSRobert Mustacchi 	xt = xhci_transfer_alloc(xhcip, xep, ubrp->bulk_len, 0, usb_flags);
923993e3fafSRobert Mustacchi 	if (xt == NULL) {
924993e3fafSRobert Mustacchi 		return (USB_NO_RESOURCES);
925993e3fafSRobert Mustacchi 	}
926993e3fafSRobert Mustacchi 	xt->xt_usba_req = (usb_opaque_t)ubrp;
927993e3fafSRobert Mustacchi 	xt->xt_timeout = ubrp->bulk_timeout;
928993e3fafSRobert Mustacchi 	if (xt->xt_timeout == 0) {
929993e3fafSRobert Mustacchi 		xt->xt_timeout = HCDI_DEFAULT_TIMEOUT;
930993e3fafSRobert Mustacchi 	}
931993e3fafSRobert Mustacchi 
932993e3fafSRobert Mustacchi 	if (ubrp->bulk_len > 0 && datain == B_FALSE) {
933993e3fafSRobert Mustacchi 		xhci_transfer_copy(xt, ubrp->bulk_data->b_rptr, ubrp->bulk_len,
934993e3fafSRobert Mustacchi 		    B_FALSE);
935993e3fafSRobert Mustacchi 		if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORDEV) !=
936993e3fafSRobert Mustacchi 		    DDI_FM_OK) {
937993e3fafSRobert Mustacchi 			xhci_transfer_free(xhcip, xt);
938993e3fafSRobert Mustacchi 			xhci_error(xhcip, "failed to synchronize bulk "
939993e3fafSRobert Mustacchi 			    "transfer DMA memory on endpoint %u of "
940993e3fafSRobert Mustacchi 			    "device on slot %d and port %d: resetting "
941993e3fafSRobert Mustacchi 			    "device", xep->xep_num, xd->xd_slot,
942993e3fafSRobert Mustacchi 			    xd->xd_port);
943993e3fafSRobert Mustacchi 			xhci_fm_runtime_reset(xhcip);
944993e3fafSRobert Mustacchi 			return (USB_HC_HARDWARE_ERROR);
945993e3fafSRobert Mustacchi 		}
946993e3fafSRobert Mustacchi 	}
947993e3fafSRobert Mustacchi 
948993e3fafSRobert Mustacchi 	xhci_transfer_trb_fill_data(xep, xt, 0, datain);
949993e3fafSRobert Mustacchi 	mutex_enter(&xhcip->xhci_lock);
950993e3fafSRobert Mustacchi 	if (xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE) != 0) {
951993e3fafSRobert Mustacchi 		xhci_transfer_free(xhcip, xt);
952993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
953993e3fafSRobert Mustacchi 		return (USB_NO_RESOURCES);
954993e3fafSRobert Mustacchi 	}
955993e3fafSRobert Mustacchi 	mutex_exit(&xhcip->xhci_lock);
956993e3fafSRobert Mustacchi 
957993e3fafSRobert Mustacchi 	return (USB_SUCCESS);
958993e3fafSRobert Mustacchi }
959993e3fafSRobert Mustacchi 
960993e3fafSRobert Mustacchi static void
xhci_hcdi_isoc_transfer_fill(xhci_device_t * xd,xhci_endpoint_t * xep,xhci_transfer_t * xt,usb_isoc_req_t * usrp)961993e3fafSRobert Mustacchi xhci_hcdi_isoc_transfer_fill(xhci_device_t *xd, xhci_endpoint_t *xep,
962993e3fafSRobert Mustacchi     xhci_transfer_t *xt, usb_isoc_req_t *usrp)
963993e3fafSRobert Mustacchi {
964993e3fafSRobert Mustacchi 	int i;
965993e3fafSRobert Mustacchi 	uintptr_t buf;
966993e3fafSRobert Mustacchi 
967993e3fafSRobert Mustacchi 	buf = xt->xt_buffer.xdb_cookies[0].dmac_laddress;
968993e3fafSRobert Mustacchi 	for (i = 0; i < usrp->isoc_pkts_count; i++) {
969993e3fafSRobert Mustacchi 		int flags;
970993e3fafSRobert Mustacchi 		uint_t tbc, tlbpc;
971993e3fafSRobert Mustacchi 
972993e3fafSRobert Mustacchi 		ushort_t len = usrp->isoc_pkt_descr[i].isoc_pkt_length;
973993e3fafSRobert Mustacchi 		xhci_trb_t *trb = &xt->xt_trbs[i];
974993e3fafSRobert Mustacchi 
975993e3fafSRobert Mustacchi 		trb->trb_addr = LE_64(buf);
976993e3fafSRobert Mustacchi 
977993e3fafSRobert Mustacchi 		/*
9782aba3acdSRobert Mustacchi 		 * Because we know that a single frame can have all of its data
9792aba3acdSRobert Mustacchi 		 * in a single instance, we know that we don't need to do
980993e3fafSRobert Mustacchi 		 * anything special here.
981993e3fafSRobert Mustacchi 		 */
982993e3fafSRobert Mustacchi 		trb->trb_status = LE_32(XHCI_TRB_LEN(len) | XHCI_TRB_TDREM(0) |
983993e3fafSRobert Mustacchi 		    XHCI_TRB_INTR(0));
984993e3fafSRobert Mustacchi 
985993e3fafSRobert Mustacchi 		/*
986993e3fafSRobert Mustacchi 		 * Always enable SIA to start the frame ASAP. We also always
987993e3fafSRobert Mustacchi 		 * enable an interrupt on a short packet. If this is the last
9882aba3acdSRobert Mustacchi 		 * trb, then we will set IOC. Each TRB created here is really
9892aba3acdSRobert Mustacchi 		 * its own TD. However, we only set an interrupt on the last
9902aba3acdSRobert Mustacchi 		 * entry to better deal with scheduling.
991993e3fafSRobert Mustacchi 		 */
992993e3fafSRobert Mustacchi 		flags = XHCI_TRB_SIA | XHCI_TRB_ISP | XHCI_TRB_SET_FRAME(0);
993993e3fafSRobert Mustacchi 		flags |= XHCI_TRB_TYPE_ISOCH;
994993e3fafSRobert Mustacchi 
995993e3fafSRobert Mustacchi 		if (i + 1 == usrp->isoc_pkts_count)
996993e3fafSRobert Mustacchi 			flags |= XHCI_TRB_IOC;
997993e3fafSRobert Mustacchi 
998993e3fafSRobert Mustacchi 		/*
999993e3fafSRobert Mustacchi 		 * Now we need to calculate the TBC and the TLBPC.
1000993e3fafSRobert Mustacchi 		 */
1001993e3fafSRobert Mustacchi 		xhci_transfer_calculate_isoc(xd, xep, len, &tbc, &tlbpc);
1002993e3fafSRobert Mustacchi 		flags |= XHCI_TRB_SET_TBC(tbc);
1003993e3fafSRobert Mustacchi 		flags |= XHCI_TRB_SET_TLBPC(tlbpc);
1004993e3fafSRobert Mustacchi 
1005993e3fafSRobert Mustacchi 		trb->trb_flags = LE_32(flags);
1006993e3fafSRobert Mustacchi 		buf += len;
1007993e3fafSRobert Mustacchi 
1008993e3fafSRobert Mustacchi 		/*
1009993e3fafSRobert Mustacchi 		 * Go through and copy the required data to our local copy of
1010993e3fafSRobert Mustacchi 		 * the isoc descriptor. By default, we assume that all data will
1011993e3fafSRobert Mustacchi 		 * be copied and the status set to OK. This mirrors the fact
1012993e3fafSRobert Mustacchi 		 * that we won't get a notification unless there's been an
1013993e3fafSRobert Mustacchi 		 * error or short packet transfer.
1014993e3fafSRobert Mustacchi 		 */
1015993e3fafSRobert Mustacchi 		xt->xt_isoc[i].isoc_pkt_length = len;
1016993e3fafSRobert Mustacchi 		xt->xt_isoc[i].isoc_pkt_actual_length = len;
1017993e3fafSRobert Mustacchi 		xt->xt_isoc[i].isoc_pkt_status = USB_CR_OK;
1018993e3fafSRobert Mustacchi 	}
1019993e3fafSRobert Mustacchi }
1020993e3fafSRobert Mustacchi 
1021993e3fafSRobert Mustacchi /*
1022993e3fafSRobert Mustacchi  * Initialize periodic IN requests (both interrupt and isochronous)
1023993e3fafSRobert Mustacchi  */
1024993e3fafSRobert Mustacchi static int
xhci_hcdi_periodic_init(xhci_t * xhcip,usba_pipe_handle_data_t * ph,usb_opaque_t usb_req,size_t len,int usb_flags)1025993e3fafSRobert Mustacchi xhci_hcdi_periodic_init(xhci_t *xhcip, usba_pipe_handle_data_t *ph,
1026993e3fafSRobert Mustacchi     usb_opaque_t usb_req, size_t len, int usb_flags)
1027993e3fafSRobert Mustacchi {
1028993e3fafSRobert Mustacchi 	int i, ret;
1029993e3fafSRobert Mustacchi 	uint_t epid;
1030993e3fafSRobert Mustacchi 	xhci_device_t *xd;
1031993e3fafSRobert Mustacchi 	xhci_endpoint_t *xep;
1032993e3fafSRobert Mustacchi 	xhci_pipe_t *xp;
1033993e3fafSRobert Mustacchi 	xhci_periodic_pipe_t *xpp;
1034993e3fafSRobert Mustacchi 
1035993e3fafSRobert Mustacchi 	mutex_enter(&xhcip->xhci_lock);
1036993e3fafSRobert Mustacchi 	if (xhcip->xhci_state & XHCI_S_ERROR) {
1037993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
1038993e3fafSRobert Mustacchi 		return (USB_HC_HARDWARE_ERROR);
1039993e3fafSRobert Mustacchi 	}
1040993e3fafSRobert Mustacchi 
1041993e3fafSRobert Mustacchi 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
1042993e3fafSRobert Mustacchi 	epid = xhci_endpoint_pipe_to_epid(ph);
1043993e3fafSRobert Mustacchi 	if (xd->xd_endpoints[epid] == NULL) {
1044993e3fafSRobert Mustacchi 		xhci_error(xhcip, "asked to do periodic transfer on slot %d, "
1045993e3fafSRobert Mustacchi 		    "port %d, endpoint: %d, but no endpoint structure",
1046993e3fafSRobert Mustacchi 		    xd->xd_slot, xd->xd_port, epid);
1047993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
1048993e3fafSRobert Mustacchi 		return (USB_FAILURE);
1049993e3fafSRobert Mustacchi 	}
1050993e3fafSRobert Mustacchi 	xep = xd->xd_endpoints[epid];
1051993e3fafSRobert Mustacchi 	xp = (xhci_pipe_t *)ph->p_hcd_private;
1052993e3fafSRobert Mustacchi 	if (xp == NULL) {
1053993e3fafSRobert Mustacchi 		xhci_error(xhcip, "asked to do periodic transfer on slot %d, "
1054993e3fafSRobert Mustacchi 		    "port %d, endpoint: %d, but no pipe structure",
1055993e3fafSRobert Mustacchi 		    xd->xd_slot, xd->xd_port, epid);
1056993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
1057993e3fafSRobert Mustacchi 		return (USB_FAILURE);
1058993e3fafSRobert Mustacchi 	}
1059993e3fafSRobert Mustacchi 	xpp = &xp->xp_periodic;
1060993e3fafSRobert Mustacchi 
1061993e3fafSRobert Mustacchi 	/*
1062993e3fafSRobert Mustacchi 	 * Only allow a single polling request at any given time.
1063993e3fafSRobert Mustacchi 	 */
1064993e3fafSRobert Mustacchi 	if (xpp->xpp_usb_req != NULL) {
1065993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
1066993e3fafSRobert Mustacchi 		return (USB_BUSY);
1067993e3fafSRobert Mustacchi 	}
1068993e3fafSRobert Mustacchi 
1069993e3fafSRobert Mustacchi 	/*
1070993e3fafSRobert Mustacchi 	 * We keep allocations around in case we restart polling, which most
1071993e3fafSRobert Mustacchi 	 * devices do (not really caring about a lost event). However, we don't
1072993e3fafSRobert Mustacchi 	 * support a driver changing that size on us, which it probably won't.
1073993e3fafSRobert Mustacchi 	 * If we stumble across driver that does, then this will need to become
1074993e3fafSRobert Mustacchi 	 * a lot more complicated.
1075993e3fafSRobert Mustacchi 	 */
1076993e3fafSRobert Mustacchi 	if (xpp->xpp_tsize > 0 && xpp->xpp_tsize < len) {
1077993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
1078993e3fafSRobert Mustacchi 		return (USB_INVALID_REQUEST);
1079993e3fafSRobert Mustacchi 	}
1080993e3fafSRobert Mustacchi 
1081993e3fafSRobert Mustacchi 	if (xpp->xpp_tsize == 0) {
1082993e3fafSRobert Mustacchi 		int ntrbs;
1083993e3fafSRobert Mustacchi 		int ntransfers;
1084993e3fafSRobert Mustacchi 
1085993e3fafSRobert Mustacchi 		/*
1086993e3fafSRobert Mustacchi 		 * What we allocate varies based on whether or not this is an
1087993e3fafSRobert Mustacchi 		 * isochronous or interrupt IN periodic.
1088993e3fafSRobert Mustacchi 		 */
1089993e3fafSRobert Mustacchi 		if (xep->xep_type == USB_EP_ATTR_INTR) {
1090993e3fafSRobert Mustacchi 			ntrbs = 0;
1091993e3fafSRobert Mustacchi 			ntransfers = XHCI_INTR_IN_NTRANSFERS;
1092993e3fafSRobert Mustacchi 		} else {
1093993e3fafSRobert Mustacchi 			usb_isoc_req_t *usrp;
1094993e3fafSRobert Mustacchi 			ASSERT(xep->xep_type == USB_EP_ATTR_ISOCH);
1095993e3fafSRobert Mustacchi 
1096993e3fafSRobert Mustacchi 			usrp = (usb_isoc_req_t *)usb_req;
1097993e3fafSRobert Mustacchi 			ntrbs = usrp->isoc_pkts_count;
1098993e3fafSRobert Mustacchi 			ntransfers = XHCI_ISOC_IN_NTRANSFERS;
1099993e3fafSRobert Mustacchi 		}
1100993e3fafSRobert Mustacchi 
1101993e3fafSRobert Mustacchi 		xpp->xpp_tsize = len;
1102993e3fafSRobert Mustacchi 		xpp->xpp_ntransfers = ntransfers;
1103993e3fafSRobert Mustacchi 
1104993e3fafSRobert Mustacchi 		for (i = 0; i < xpp->xpp_ntransfers; i++) {
1105993e3fafSRobert Mustacchi 			xhci_transfer_t *xt = xhci_transfer_alloc(xhcip, xep,
1106993e3fafSRobert Mustacchi 			    len, ntrbs, usb_flags);
1107993e3fafSRobert Mustacchi 			if (xt == NULL) {
1108993e3fafSRobert Mustacchi 				xhci_hcdi_periodic_free(xhcip, xp);
1109993e3fafSRobert Mustacchi 				mutex_exit(&xhcip->xhci_lock);
1110993e3fafSRobert Mustacchi 				return (USB_NO_RESOURCES);
1111993e3fafSRobert Mustacchi 			}
1112993e3fafSRobert Mustacchi 
1113993e3fafSRobert Mustacchi 			if (xep->xep_type == USB_EP_ATTR_INTR) {
1114993e3fafSRobert Mustacchi 				xhci_transfer_trb_fill_data(xep, xt, 0, B_TRUE);
1115993e3fafSRobert Mustacchi 			} else {
1116993e3fafSRobert Mustacchi 				usb_isoc_req_t *usrp;
1117993e3fafSRobert Mustacchi 				usrp = (usb_isoc_req_t *)usb_req;
1118993e3fafSRobert Mustacchi 				xhci_hcdi_isoc_transfer_fill(xd, xep, xt, usrp);
1119993e3fafSRobert Mustacchi 				xt->xt_data_tohost = B_TRUE;
1120993e3fafSRobert Mustacchi 			}
1121993e3fafSRobert Mustacchi 			xpp->xpp_transfers[i] = xt;
1122993e3fafSRobert Mustacchi 		}
1123993e3fafSRobert Mustacchi 	}
1124993e3fafSRobert Mustacchi 
1125993e3fafSRobert Mustacchi 	/*
1126993e3fafSRobert Mustacchi 	 * Mark the endpoint as periodic so we don't have timeouts at play.
1127993e3fafSRobert Mustacchi 	 */
1128993e3fafSRobert Mustacchi 	xep->xep_state |= XHCI_ENDPOINT_PERIODIC;
1129993e3fafSRobert Mustacchi 
1130993e3fafSRobert Mustacchi 	/*
1131993e3fafSRobert Mustacchi 	 * Now that we've allocated everything, go ahead and schedule them and
1132993e3fafSRobert Mustacchi 	 * kick off the ring.
1133993e3fafSRobert Mustacchi 	 */
1134993e3fafSRobert Mustacchi 	for (i = 0; i < xpp->xpp_ntransfers; i++) {
1135993e3fafSRobert Mustacchi 		int ret;
1136993e3fafSRobert Mustacchi 		ret = xhci_endpoint_schedule(xhcip, xd, xep,
1137993e3fafSRobert Mustacchi 		    xpp->xpp_transfers[i], B_FALSE);
1138993e3fafSRobert Mustacchi 		if (ret != 0) {
1139993e3fafSRobert Mustacchi 			(void) xhci_ring_reset(xhcip, &xep->xep_ring);
1140993e3fafSRobert Mustacchi 			xep->xep_state &= ~XHCI_ENDPOINT_PERIODIC;
1141993e3fafSRobert Mustacchi 			mutex_exit(&xhcip->xhci_lock);
1142993e3fafSRobert Mustacchi 			return (ret);
1143993e3fafSRobert Mustacchi 		}
1144993e3fafSRobert Mustacchi 	}
1145993e3fafSRobert Mustacchi 
1146993e3fafSRobert Mustacchi 	/*
1147993e3fafSRobert Mustacchi 	 * Don't worry about freeing memory, it'll be done when the endpoint
1148993e3fafSRobert Mustacchi 	 * closes and the whole system is reset.
1149993e3fafSRobert Mustacchi 	 */
1150993e3fafSRobert Mustacchi 	xpp->xpp_usb_req = usb_req;
1151993e3fafSRobert Mustacchi 	xpp->xpp_poll_state = XHCI_PERIODIC_POLL_ACTIVE;
1152993e3fafSRobert Mustacchi 
1153993e3fafSRobert Mustacchi 	ret = xhci_endpoint_ring(xhcip, xd, xep);
1154993e3fafSRobert Mustacchi 	mutex_exit(&xhcip->xhci_lock);
1155993e3fafSRobert Mustacchi 	return (ret);
1156993e3fafSRobert Mustacchi }
1157993e3fafSRobert Mustacchi 
1158993e3fafSRobert Mustacchi static int
xhci_hcdi_intr_oneshot(xhci_t * xhcip,usba_pipe_handle_data_t * ph,usb_intr_req_t * uirp,usb_flags_t usb_flags)1159993e3fafSRobert Mustacchi xhci_hcdi_intr_oneshot(xhci_t *xhcip, usba_pipe_handle_data_t *ph,
1160993e3fafSRobert Mustacchi     usb_intr_req_t *uirp, usb_flags_t usb_flags)
1161993e3fafSRobert Mustacchi {
1162993e3fafSRobert Mustacchi 	uint_t epid;
1163993e3fafSRobert Mustacchi 	xhci_device_t *xd;
1164993e3fafSRobert Mustacchi 	xhci_endpoint_t *xep;
1165993e3fafSRobert Mustacchi 	xhci_transfer_t *xt;
1166993e3fafSRobert Mustacchi 	boolean_t datain;
1167993e3fafSRobert Mustacchi 	mblk_t *mp = NULL;
1168993e3fafSRobert Mustacchi 
1169993e3fafSRobert Mustacchi 	mutex_enter(&xhcip->xhci_lock);
1170993e3fafSRobert Mustacchi 	if (xhcip->xhci_state & XHCI_S_ERROR) {
1171993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
1172993e3fafSRobert Mustacchi 		return (USB_HC_HARDWARE_ERROR);
1173993e3fafSRobert Mustacchi 	}
1174993e3fafSRobert Mustacchi 
1175993e3fafSRobert Mustacchi 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
1176993e3fafSRobert Mustacchi 	epid = xhci_endpoint_pipe_to_epid(ph);
1177993e3fafSRobert Mustacchi 	if (xd->xd_endpoints[epid] == NULL) {
1178993e3fafSRobert Mustacchi 		xhci_error(xhcip, "asked to do interrupt transfer on slot %d, "
1179993e3fafSRobert Mustacchi 		    "port %d, endpoint: %d, but no endpoint structure",
1180993e3fafSRobert Mustacchi 		    xd->xd_slot, xd->xd_port, epid);
1181993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
1182993e3fafSRobert Mustacchi 		return (USB_FAILURE);
1183993e3fafSRobert Mustacchi 	}
1184993e3fafSRobert Mustacchi 	xep = xd->xd_endpoints[epid];
1185993e3fafSRobert Mustacchi 
1186993e3fafSRobert Mustacchi 	mutex_exit(&xhcip->xhci_lock);
1187993e3fafSRobert Mustacchi 
1188993e3fafSRobert Mustacchi 	if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
1189993e3fafSRobert Mustacchi 		datain = B_TRUE;
1190993e3fafSRobert Mustacchi 	} else {
1191993e3fafSRobert Mustacchi 		datain = B_FALSE;
1192993e3fafSRobert Mustacchi 	}
1193993e3fafSRobert Mustacchi 
1194993e3fafSRobert Mustacchi 	xt = xhci_transfer_alloc(xhcip, xep, uirp->intr_len, 0, usb_flags);
1195993e3fafSRobert Mustacchi 	if (xt == NULL) {
1196993e3fafSRobert Mustacchi 		return (USB_NO_RESOURCES);
1197993e3fafSRobert Mustacchi 	}
1198993e3fafSRobert Mustacchi 
1199993e3fafSRobert Mustacchi 	xt->xt_usba_req = (usb_opaque_t)uirp;
1200993e3fafSRobert Mustacchi 	xt->xt_timeout = uirp->intr_timeout;
1201993e3fafSRobert Mustacchi 	if (xt->xt_timeout == 0) {
1202993e3fafSRobert Mustacchi 		xt->xt_timeout = HCDI_DEFAULT_TIMEOUT;
1203993e3fafSRobert Mustacchi 	}
1204993e3fafSRobert Mustacchi 
1205993e3fafSRobert Mustacchi 	/*
1206993e3fafSRobert Mustacchi 	 * Unlike other request types, USB Interrupt-IN requests aren't required
1207993e3fafSRobert Mustacchi 	 * to have allocated the message block for data. If they haven't, we
1208993e3fafSRobert Mustacchi 	 * take care of that now.
1209993e3fafSRobert Mustacchi 	 */
1210993e3fafSRobert Mustacchi 	if (uirp->intr_len > 0 && datain == B_TRUE && uirp->intr_data == NULL) {
1211993e3fafSRobert Mustacchi 		if (usb_flags & USB_FLAGS_SLEEP) {
1212993e3fafSRobert Mustacchi 			mp = allocb_wait(uirp->intr_len, BPRI_LO, STR_NOSIG,
1213993e3fafSRobert Mustacchi 			    NULL);
1214993e3fafSRobert Mustacchi 		} else {
1215993e3fafSRobert Mustacchi 			mp = allocb(uirp->intr_len, 0);
1216993e3fafSRobert Mustacchi 		}
1217993e3fafSRobert Mustacchi 		if (mp == NULL) {
1218993e3fafSRobert Mustacchi 			xhci_transfer_free(xhcip, xt);
1219993e3fafSRobert Mustacchi 			mutex_exit(&xhcip->xhci_lock);
1220993e3fafSRobert Mustacchi 			return (USB_NO_RESOURCES);
1221993e3fafSRobert Mustacchi 		}
1222993e3fafSRobert Mustacchi 		uirp->intr_data = mp;
1223993e3fafSRobert Mustacchi 	}
1224993e3fafSRobert Mustacchi 
1225993e3fafSRobert Mustacchi 	if (uirp->intr_len > 0 && datain == B_FALSE) {
1226993e3fafSRobert Mustacchi 		xhci_transfer_copy(xt, uirp->intr_data->b_rptr, uirp->intr_len,
1227993e3fafSRobert Mustacchi 		    B_FALSE);
1228993e3fafSRobert Mustacchi 		if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORDEV) !=
1229993e3fafSRobert Mustacchi 		    DDI_FM_OK) {
1230993e3fafSRobert Mustacchi 			xhci_transfer_free(xhcip, xt);
1231993e3fafSRobert Mustacchi 			xhci_error(xhcip, "failed to synchronize interrupt "
1232993e3fafSRobert Mustacchi 			    "transfer DMA memory on endpoint %u of "
1233993e3fafSRobert Mustacchi 			    "device on slot %d and port %d: resetting "
1234993e3fafSRobert Mustacchi 			    "device", xep->xep_num, xd->xd_slot,
1235993e3fafSRobert Mustacchi 			    xd->xd_port);
1236993e3fafSRobert Mustacchi 			xhci_fm_runtime_reset(xhcip);
1237993e3fafSRobert Mustacchi 			return (USB_HC_HARDWARE_ERROR);
1238993e3fafSRobert Mustacchi 		}
1239993e3fafSRobert Mustacchi 	}
1240993e3fafSRobert Mustacchi 
1241993e3fafSRobert Mustacchi 	xhci_transfer_trb_fill_data(xep, xt, 0, datain);
1242993e3fafSRobert Mustacchi 	mutex_enter(&xhcip->xhci_lock);
1243993e3fafSRobert Mustacchi 	if (xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE) != 0) {
1244993e3fafSRobert Mustacchi 		if (mp != NULL) {
1245993e3fafSRobert Mustacchi 			uirp->intr_data = NULL;
1246993e3fafSRobert Mustacchi 			freemsg(mp);
1247993e3fafSRobert Mustacchi 		}
1248993e3fafSRobert Mustacchi 		xhci_transfer_free(xhcip, xt);
1249993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
1250993e3fafSRobert Mustacchi 		return (USB_NO_RESOURCES);
1251993e3fafSRobert Mustacchi 	}
1252993e3fafSRobert Mustacchi 	mutex_exit(&xhcip->xhci_lock);
1253993e3fafSRobert Mustacchi 
1254993e3fafSRobert Mustacchi 	return (USB_SUCCESS);
1255993e3fafSRobert Mustacchi }
1256993e3fafSRobert Mustacchi 
1257993e3fafSRobert Mustacchi /*
1258993e3fafSRobert Mustacchi  * We've been asked to perform an interrupt transfer. When this is an interrupt
1259993e3fafSRobert Mustacchi  * IN endpoint, that means that the hcd is being asked to start polling on the
1260993e3fafSRobert Mustacchi  * endpoint. When the endpoint is the root hub, it effectively becomes synthetic
1261993e3fafSRobert Mustacchi  * polling.
1262993e3fafSRobert Mustacchi  *
1263993e3fafSRobert Mustacchi  * When we have an interrupt out endpoint, then this is just a single simple
1264993e3fafSRobert Mustacchi  * interrupt request that we send out and there isn't much special to do beyond
1265993e3fafSRobert Mustacchi  * the normal activity.
1266993e3fafSRobert Mustacchi  */
1267993e3fafSRobert Mustacchi static int
xhci_hcdi_pipe_intr_xfer(usba_pipe_handle_data_t * ph,usb_intr_req_t * uirp,usb_flags_t usb_flags)1268993e3fafSRobert Mustacchi xhci_hcdi_pipe_intr_xfer(usba_pipe_handle_data_t *ph, usb_intr_req_t *uirp,
1269993e3fafSRobert Mustacchi     usb_flags_t usb_flags)
1270993e3fafSRobert Mustacchi {
1271993e3fafSRobert Mustacchi 	int ret;
1272993e3fafSRobert Mustacchi 	xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
1273993e3fafSRobert Mustacchi 
1274993e3fafSRobert Mustacchi 	if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
1275993e3fafSRobert Mustacchi 		if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
1276993e3fafSRobert Mustacchi 			ret = xhci_root_hub_intr_root_enable(xhcip, ph, uirp);
1277993e3fafSRobert Mustacchi 		} else if (uirp->intr_attributes & USB_ATTRS_ONE_XFER) {
1278993e3fafSRobert Mustacchi 			ret = xhci_hcdi_intr_oneshot(xhcip, ph, uirp,
1279993e3fafSRobert Mustacchi 			    usb_flags);
1280993e3fafSRobert Mustacchi 		} else {
1281993e3fafSRobert Mustacchi 			ret = xhci_hcdi_periodic_init(xhcip, ph,
1282993e3fafSRobert Mustacchi 			    (usb_opaque_t)uirp, uirp->intr_len, usb_flags);
1283993e3fafSRobert Mustacchi 		}
1284993e3fafSRobert Mustacchi 	} else {
1285993e3fafSRobert Mustacchi 		if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
1286993e3fafSRobert Mustacchi 			return (USB_NOT_SUPPORTED);
1287993e3fafSRobert Mustacchi 		}
1288993e3fafSRobert Mustacchi 		ret = xhci_hcdi_intr_oneshot(xhcip, ph, uirp, usb_flags);
1289993e3fafSRobert Mustacchi 	}
1290993e3fafSRobert Mustacchi 
1291993e3fafSRobert Mustacchi 	return (ret);
1292993e3fafSRobert Mustacchi }
1293993e3fafSRobert Mustacchi 
1294993e3fafSRobert Mustacchi /* ARGSUSED */
1295993e3fafSRobert Mustacchi static int
xhci_hcdi_pipe_stop_intr_polling(usba_pipe_handle_data_t * ph,usb_flags_t usb_flags)1296993e3fafSRobert Mustacchi xhci_hcdi_pipe_stop_intr_polling(usba_pipe_handle_data_t *ph,
1297993e3fafSRobert Mustacchi     usb_flags_t usb_flags)
1298993e3fafSRobert Mustacchi {
1299993e3fafSRobert Mustacchi 	return (xhci_hcdi_pipe_poll_fini(ph, B_FALSE));
1300993e3fafSRobert Mustacchi }
1301993e3fafSRobert Mustacchi 
1302993e3fafSRobert Mustacchi static int
xhci_hcdi_isoc_periodic(xhci_t * xhcip,usba_pipe_handle_data_t * ph,usb_isoc_req_t * usrp,usb_flags_t usb_flags)1303993e3fafSRobert Mustacchi xhci_hcdi_isoc_periodic(xhci_t *xhcip, usba_pipe_handle_data_t *ph,
1304993e3fafSRobert Mustacchi     usb_isoc_req_t *usrp, usb_flags_t usb_flags)
1305993e3fafSRobert Mustacchi {
1306993e3fafSRobert Mustacchi 	int i;
1307993e3fafSRobert Mustacchi 	size_t count;
1308993e3fafSRobert Mustacchi 
1309993e3fafSRobert Mustacchi 	count = 0;
1310993e3fafSRobert Mustacchi 	for (i = 0; i < usrp->isoc_pkts_count; i++) {
1311993e3fafSRobert Mustacchi 		count += usrp->isoc_pkt_descr[i].isoc_pkt_length;
1312993e3fafSRobert Mustacchi 	}
1313993e3fafSRobert Mustacchi 
1314993e3fafSRobert Mustacchi 	return (xhci_hcdi_periodic_init(xhcip, ph, (usb_opaque_t)usrp, count,
1315993e3fafSRobert Mustacchi 	    usb_flags));
1316993e3fafSRobert Mustacchi }
1317993e3fafSRobert Mustacchi 
1318993e3fafSRobert Mustacchi /*
1319993e3fafSRobert Mustacchi  * This is used to create an isochronous request to send data out to the device.
1320993e3fafSRobert Mustacchi  * This is a single one shot request, it is not something that we'll have to
1321993e3fafSRobert Mustacchi  * repeat over and over.
1322993e3fafSRobert Mustacchi  */
1323993e3fafSRobert Mustacchi static int
xhci_hcdi_isoc_oneshot(xhci_t * xhcip,usba_pipe_handle_data_t * ph,usb_isoc_req_t * usrp,usb_flags_t usb_flags)1324993e3fafSRobert Mustacchi xhci_hcdi_isoc_oneshot(xhci_t *xhcip, usba_pipe_handle_data_t *ph,
1325993e3fafSRobert Mustacchi     usb_isoc_req_t *usrp, usb_flags_t usb_flags)
1326993e3fafSRobert Mustacchi {
1327993e3fafSRobert Mustacchi 	int i, ret;
1328993e3fafSRobert Mustacchi 	uint_t epid;
1329993e3fafSRobert Mustacchi 	size_t count, mblen;
1330993e3fafSRobert Mustacchi 	xhci_device_t *xd;
1331993e3fafSRobert Mustacchi 	xhci_endpoint_t *xep;
1332993e3fafSRobert Mustacchi 	xhci_transfer_t *xt;
1333993e3fafSRobert Mustacchi 
1334993e3fafSRobert Mustacchi 	count = 0;
1335993e3fafSRobert Mustacchi 	for (i = 0; i < usrp->isoc_pkts_count; i++) {
1336993e3fafSRobert Mustacchi 		count += usrp->isoc_pkt_descr[i].isoc_pkt_length;
1337993e3fafSRobert Mustacchi 	}
1338993e3fafSRobert Mustacchi 	mblen = MBLKL(usrp->isoc_data);
1339993e3fafSRobert Mustacchi 
1340993e3fafSRobert Mustacchi 	if (count != mblen) {
1341993e3fafSRobert Mustacchi 		return (USB_INVALID_ARGS);
1342993e3fafSRobert Mustacchi 	}
1343993e3fafSRobert Mustacchi 
1344993e3fafSRobert Mustacchi 	mutex_enter(&xhcip->xhci_lock);
1345993e3fafSRobert Mustacchi 	if (xhcip->xhci_state & XHCI_S_ERROR) {
1346993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
1347993e3fafSRobert Mustacchi 		return (USB_HC_HARDWARE_ERROR);
1348993e3fafSRobert Mustacchi 	}
1349993e3fafSRobert Mustacchi 
1350993e3fafSRobert Mustacchi 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
1351993e3fafSRobert Mustacchi 	epid = xhci_endpoint_pipe_to_epid(ph);
1352993e3fafSRobert Mustacchi 	if (xd->xd_endpoints[epid] == NULL) {
1353993e3fafSRobert Mustacchi 		xhci_error(xhcip, "asked to do isochronous transfer on slot "
1354993e3fafSRobert Mustacchi 		    "%d, port %d, endpoint: %d, but no endpoint structure",
1355993e3fafSRobert Mustacchi 		    xd->xd_slot, xd->xd_port, epid);
1356993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
1357993e3fafSRobert Mustacchi 		return (USB_FAILURE);
1358993e3fafSRobert Mustacchi 	}
1359993e3fafSRobert Mustacchi 	xep = xd->xd_endpoints[epid];
1360993e3fafSRobert Mustacchi 	mutex_exit(&xhcip->xhci_lock);
1361993e3fafSRobert Mustacchi 
1362993e3fafSRobert Mustacchi 	xt = xhci_transfer_alloc(xhcip, xep, mblen, usrp->isoc_pkts_count,
1363993e3fafSRobert Mustacchi 	    usb_flags);
1364993e3fafSRobert Mustacchi 	if (xt == NULL) {
1365993e3fafSRobert Mustacchi 		return (USB_NO_RESOURCES);
1366993e3fafSRobert Mustacchi 	}
1367993e3fafSRobert Mustacchi 	xt->xt_usba_req = (usb_opaque_t)usrp;
1368993e3fafSRobert Mustacchi 
1369993e3fafSRobert Mustacchi 	/*
1370993e3fafSRobert Mustacchi 	 * USBA doesn't provide any real way for a timeout to be defined for an
1371993e3fafSRobert Mustacchi 	 * isochronous event. However, since we technically aren't a periodic
1372993e3fafSRobert Mustacchi 	 * endpoint, go ahead and always set the default timeout. It's better
1373993e3fafSRobert Mustacchi 	 * than nothing.
1374993e3fafSRobert Mustacchi 	 */
1375993e3fafSRobert Mustacchi 	xt->xt_timeout = HCDI_DEFAULT_TIMEOUT;
1376993e3fafSRobert Mustacchi 
1377993e3fafSRobert Mustacchi 	xhci_transfer_copy(xt, usrp->isoc_data->b_rptr, mblen, B_FALSE);
1378993e3fafSRobert Mustacchi 	if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORDEV) != DDI_FM_OK) {
1379993e3fafSRobert Mustacchi 		xhci_transfer_free(xhcip, xt);
1380993e3fafSRobert Mustacchi 		xhci_error(xhcip, "failed to synchronize isochronous "
1381993e3fafSRobert Mustacchi 		    "transfer DMA memory on endpoint %u of "
1382993e3fafSRobert Mustacchi 		    "device on slot %d and port %d: resetting "
1383993e3fafSRobert Mustacchi 		    "device", xep->xep_num, xd->xd_slot,
1384993e3fafSRobert Mustacchi 		    xd->xd_port);
1385993e3fafSRobert Mustacchi 		xhci_fm_runtime_reset(xhcip);
1386993e3fafSRobert Mustacchi 		return (USB_HC_HARDWARE_ERROR);
1387993e3fafSRobert Mustacchi 	}
1388993e3fafSRobert Mustacchi 
1389993e3fafSRobert Mustacchi 	/*
1390993e3fafSRobert Mustacchi 	 * Fill in the ISOC data. Note, that we always use ASAP scheduling and
1391993e3fafSRobert Mustacchi 	 * we don't support specifying the frame at this time, for better or
1392993e3fafSRobert Mustacchi 	 * worse.
1393993e3fafSRobert Mustacchi 	 */
1394993e3fafSRobert Mustacchi 	xhci_hcdi_isoc_transfer_fill(xd, xep, xt, usrp);
1395993e3fafSRobert Mustacchi 
1396993e3fafSRobert Mustacchi 	mutex_enter(&xhcip->xhci_lock);
1397993e3fafSRobert Mustacchi 	ret = xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE);
1398993e3fafSRobert Mustacchi 	mutex_exit(&xhcip->xhci_lock);
1399993e3fafSRobert Mustacchi 
1400993e3fafSRobert Mustacchi 	return (ret);
1401993e3fafSRobert Mustacchi }
1402993e3fafSRobert Mustacchi 
1403993e3fafSRobert Mustacchi static int
xhci_hcdi_pipe_isoc_xfer(usba_pipe_handle_data_t * ph,usb_isoc_req_t * usrp,usb_flags_t usb_flags)1404993e3fafSRobert Mustacchi xhci_hcdi_pipe_isoc_xfer(usba_pipe_handle_data_t *ph, usb_isoc_req_t *usrp,
1405993e3fafSRobert Mustacchi     usb_flags_t usb_flags)
1406993e3fafSRobert Mustacchi {
1407993e3fafSRobert Mustacchi 	int ret;
1408993e3fafSRobert Mustacchi 	xhci_t *xhcip;
1409993e3fafSRobert Mustacchi 
1410993e3fafSRobert Mustacchi 	xhcip = xhci_hcdi_get_xhcip(ph);
1411993e3fafSRobert Mustacchi 
1412993e3fafSRobert Mustacchi 	/*
1413993e3fafSRobert Mustacchi 	 * We don't support isochronous transactions on the root hub at all.
1414993e3fafSRobert Mustacchi 	 * Always fail them if for some reason we end up here.
1415993e3fafSRobert Mustacchi 	 */
1416993e3fafSRobert Mustacchi 	if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
1417993e3fafSRobert Mustacchi 		return (USB_NOT_SUPPORTED);
1418993e3fafSRobert Mustacchi 	}
1419993e3fafSRobert Mustacchi 
1420993e3fafSRobert Mustacchi 	/*
1421993e3fafSRobert Mustacchi 	 * We do not support being asked to set the frame ID at this time. We
1422993e3fafSRobert Mustacchi 	 * require that everything specify the attribute
1423993e3fafSRobert Mustacchi 	 * USB_ATTRS_ISOC_XFER_ASAP.
1424993e3fafSRobert Mustacchi 	 */
1425993e3fafSRobert Mustacchi 	if (!(usrp->isoc_attributes & USB_ATTRS_ISOC_XFER_ASAP)) {
1426993e3fafSRobert Mustacchi 		return (USB_NOT_SUPPORTED);
1427993e3fafSRobert Mustacchi 	}
1428993e3fafSRobert Mustacchi 
1429993e3fafSRobert Mustacchi 	if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
1430993e3fafSRobert Mustacchi 		/*
1431993e3fafSRobert Mustacchi 		 * Note, there is no such thing as a non-periodic isochronous
1432993e3fafSRobert Mustacchi 		 * incoming transfer.
1433993e3fafSRobert Mustacchi 		 */
1434993e3fafSRobert Mustacchi 		ret = xhci_hcdi_isoc_periodic(xhcip, ph, usrp, usb_flags);
1435993e3fafSRobert Mustacchi 	} else {
1436993e3fafSRobert Mustacchi 		ret = xhci_hcdi_isoc_oneshot(xhcip, ph, usrp, usb_flags);
1437993e3fafSRobert Mustacchi 	}
1438993e3fafSRobert Mustacchi 
1439993e3fafSRobert Mustacchi 	return (ret);
1440993e3fafSRobert Mustacchi }
1441993e3fafSRobert Mustacchi 
1442993e3fafSRobert Mustacchi /* ARGSUSED */
1443993e3fafSRobert Mustacchi static int
xhci_hcdi_pipe_stop_isoc_polling(usba_pipe_handle_data_t * ph,usb_flags_t usb_flags)1444993e3fafSRobert Mustacchi xhci_hcdi_pipe_stop_isoc_polling(usba_pipe_handle_data_t *ph,
1445993e3fafSRobert Mustacchi     usb_flags_t usb_flags)
1446993e3fafSRobert Mustacchi {
1447993e3fafSRobert Mustacchi 	return (xhci_hcdi_pipe_poll_fini(ph, B_FALSE));
1448993e3fafSRobert Mustacchi }
1449993e3fafSRobert Mustacchi 
1450993e3fafSRobert Mustacchi /*
1451993e3fafSRobert Mustacchi  * This is asking us for the current frame number. The USBA expects this to
1452993e3fafSRobert Mustacchi  * actually be a bit of a fiction, as it tries to maintain a frame number well
1453993e3fafSRobert Mustacchi  * beyond what the hardware actually contains in its registers. Hardware
1454993e3fafSRobert Mustacchi  * basically has a 14-bit counter, whereas we need to have a constant amount of
1455993e3fafSRobert Mustacchi  * milliseconds.
1456993e3fafSRobert Mustacchi  *
1457993e3fafSRobert Mustacchi  * Today, no client drivers actually use this API and everyone specifies the
1458993e3fafSRobert Mustacchi  * attribute to say that we should schedule things ASAP. So until we have some
1459993e3fafSRobert Mustacchi  * real device that want this functionality, we're going to fail.
1460993e3fafSRobert Mustacchi  */
1461993e3fafSRobert Mustacchi /* ARGSUSED */
1462993e3fafSRobert Mustacchi static int
xhci_hcdi_get_current_frame_number(usba_device_t * usba_device,usb_frame_number_t * frame_number)1463993e3fafSRobert Mustacchi xhci_hcdi_get_current_frame_number(usba_device_t *usba_device,
1464993e3fafSRobert Mustacchi     usb_frame_number_t *frame_number)
1465993e3fafSRobert Mustacchi {
1466993e3fafSRobert Mustacchi 	return (USB_FAILURE);
1467993e3fafSRobert Mustacchi }
1468993e3fafSRobert Mustacchi 
1469993e3fafSRobert Mustacchi /*
1470993e3fafSRobert Mustacchi  * See the comments around the XHCI_ISOC_MAX_TRB macro for more information.
1471993e3fafSRobert Mustacchi  */
1472993e3fafSRobert Mustacchi /* ARGSUSED */
1473993e3fafSRobert Mustacchi static int
xhci_hcdi_get_max_isoc_pkts(usba_device_t * usba_device,uint_t * max_isoc_pkts_per_request)1474993e3fafSRobert Mustacchi xhci_hcdi_get_max_isoc_pkts(usba_device_t *usba_device,
1475993e3fafSRobert Mustacchi     uint_t *max_isoc_pkts_per_request)
1476993e3fafSRobert Mustacchi {
1477993e3fafSRobert Mustacchi 	*max_isoc_pkts_per_request = XHCI_ISOC_MAX_TRB;
1478993e3fafSRobert Mustacchi 	return (USB_SUCCESS);
1479993e3fafSRobert Mustacchi }
1480993e3fafSRobert Mustacchi 
1481993e3fafSRobert Mustacchi /*
1482993e3fafSRobert Mustacchi  * VERSION 2 ops and helpers
1483993e3fafSRobert Mustacchi  */
1484993e3fafSRobert Mustacchi 
1485993e3fafSRobert Mustacchi static void
xhci_hcdi_device_free(xhci_device_t * xd)1486993e3fafSRobert Mustacchi xhci_hcdi_device_free(xhci_device_t *xd)
1487993e3fafSRobert Mustacchi {
1488993e3fafSRobert Mustacchi 	xhci_dma_free(&xd->xd_ictx);
1489993e3fafSRobert Mustacchi 	xhci_dma_free(&xd->xd_octx);
1490993e3fafSRobert Mustacchi 	mutex_destroy(&xd->xd_imtx);
1491993e3fafSRobert Mustacchi 	kmem_free(xd, sizeof (xhci_device_t));
1492993e3fafSRobert Mustacchi }
1493993e3fafSRobert Mustacchi 
1494993e3fafSRobert Mustacchi /*
1495993e3fafSRobert Mustacchi  * Calculate the device's route string. In USB 3.0 the route string is a 20-bit
1496993e3fafSRobert Mustacchi  * number. Each four bits represent a port number attached to a deeper hub.
1497993e3fafSRobert Mustacchi  * Particularly it represents the port on that current hub that you need to go
1498993e3fafSRobert Mustacchi  * down to reach the next device. Bits 0-3 represent the first *external* hub.
1499993e3fafSRobert Mustacchi  * So a device connected to a root hub has a route string of zero. Imagine the
1500993e3fafSRobert Mustacchi  * following set of devices:
1501993e3fafSRobert Mustacchi  *
1502993e3fafSRobert Mustacchi  *               . port 2      . port 5
1503993e3fafSRobert Mustacchi  *               .             .
1504993e3fafSRobert Mustacchi  *  +----------+ .  +--------+ .  +-------+
1505993e3fafSRobert Mustacchi  *  | root hub |-*->| hub 1  |-*->| hub 2 |
1506993e3fafSRobert Mustacchi  *  +----------+    +--------+    +-------+
1507993e3fafSRobert Mustacchi  *       * . port 12    * . port 8    * . port 1
1508993e3fafSRobert Mustacchi  *       v              v             v
1509993e3fafSRobert Mustacchi  *   +-------+      +-------+     +-------+
1510993e3fafSRobert Mustacchi  *   | dev a |      | dev b |     | dev c |
1511993e3fafSRobert Mustacchi  *   +-------+      +-------+     +-------+
1512993e3fafSRobert Mustacchi  *
1513993e3fafSRobert Mustacchi  * So, based on the above diagram, device a should have a route string of 0,
1514993e3fafSRobert Mustacchi  * because it's directly connected to the root port. Device b would simply have
1515993e3fafSRobert Mustacchi  * a route string of 8. This is because it travels through one non-root hub, hub
1516993e3fafSRobert Mustacchi  * 1, and it does so on port 8. The root ports basically don't matter. Device c
1517993e3fafSRobert Mustacchi  * would then have a route string of 0x15, as it's first traversing through hub
1518993e3fafSRobert Mustacchi  * 1 on port 2 and then hub 2 on port 5.
1519993e3fafSRobert Mustacchi  *
1520993e3fafSRobert Mustacchi  * Finally, it's worth mentioning that because it's a four bit field, if for
1521993e3fafSRobert Mustacchi  * some reason a device has more than 15 ports, we just treat the value as 15.
1522993e3fafSRobert Mustacchi  *
1523993e3fafSRobert Mustacchi  * Note, as part of this, we also grab what port on the root hub this whole
1524993e3fafSRobert Mustacchi  * chain is on, as we're required to store that information in the slot context.
1525993e3fafSRobert Mustacchi  */
1526993e3fafSRobert Mustacchi static void
xhci_hcdi_device_route(usba_device_t * ud,uint32_t * routep,uint32_t * root_port)1527993e3fafSRobert Mustacchi xhci_hcdi_device_route(usba_device_t *ud, uint32_t *routep, uint32_t *root_port)
1528993e3fafSRobert Mustacchi {
1529993e3fafSRobert Mustacchi 	uint32_t route = 0;
1530993e3fafSRobert Mustacchi 	usba_device_t *hub = ud->usb_parent_hub;
1531993e3fafSRobert Mustacchi 	usba_device_t *port_dev = ud;
1532993e3fafSRobert Mustacchi 
1533993e3fafSRobert Mustacchi 	ASSERT(hub != NULL);
1534993e3fafSRobert Mustacchi 
1535993e3fafSRobert Mustacchi 	/*
1536993e3fafSRobert Mustacchi 	 * Iterate over every hub, updating the route as we go. When we
1537993e3fafSRobert Mustacchi 	 * encounter a hub without a parent, then we're at the root hub. At
1538993e3fafSRobert Mustacchi 	 * which point, the port we want is on port_dev (the child of hub).
1539993e3fafSRobert Mustacchi 	 */
1540993e3fafSRobert Mustacchi 	while (hub->usb_parent_hub != NULL) {
1541993e3fafSRobert Mustacchi 		uint32_t p;
1542993e3fafSRobert Mustacchi 
1543993e3fafSRobert Mustacchi 		p = port_dev->usb_port;
1544993e3fafSRobert Mustacchi 		if (p > 15)
1545993e3fafSRobert Mustacchi 			p = 15;
1546993e3fafSRobert Mustacchi 		route <<= 4;
1547993e3fafSRobert Mustacchi 		route |= p & 0xf;
1548993e3fafSRobert Mustacchi 		port_dev = hub;
1549993e3fafSRobert Mustacchi 		hub = hub->usb_parent_hub;
1550993e3fafSRobert Mustacchi 	}
1551993e3fafSRobert Mustacchi 
1552993e3fafSRobert Mustacchi 	ASSERT(port_dev->usb_parent_hub == hub);
1553993e3fafSRobert Mustacchi 	*root_port = port_dev->usb_port;
1554993e3fafSRobert Mustacchi 	*routep = XHCI_ROUTE_MASK(route);
1555993e3fafSRobert Mustacchi }
1556993e3fafSRobert Mustacchi 
1557993e3fafSRobert Mustacchi /*
1558993e3fafSRobert Mustacchi  * If a low or full speed device is behind a high-speed device that is not a
1559993e3fafSRobert Mustacchi  * root hub, then we must include the port and slot of that device. USBA already
1560993e3fafSRobert Mustacchi  * stores this device in the usb_hs_hub_usba_dev member.
1561993e3fafSRobert Mustacchi  */
1562993e3fafSRobert Mustacchi static uint32_t
xhci_hcdi_device_tt(usba_device_t * ud)1563993e3fafSRobert Mustacchi xhci_hcdi_device_tt(usba_device_t *ud)
1564993e3fafSRobert Mustacchi {
1565993e3fafSRobert Mustacchi 	uint32_t ret;
1566993e3fafSRobert Mustacchi 	xhci_device_t *xd;
1567993e3fafSRobert Mustacchi 
1568993e3fafSRobert Mustacchi 	if (ud->usb_port_status >= USBA_HIGH_SPEED_DEV)
1569993e3fafSRobert Mustacchi 		return (0);
1570993e3fafSRobert Mustacchi 
1571993e3fafSRobert Mustacchi 	if (ud->usb_hs_hub_usba_dev == NULL)
1572993e3fafSRobert Mustacchi 		return (0);
1573993e3fafSRobert Mustacchi 
1574993e3fafSRobert Mustacchi 	ASSERT(ud->usb_hs_hub_usba_dev != NULL);
1575993e3fafSRobert Mustacchi 	ASSERT(ud->usb_hs_hub_usba_dev->usb_parent_hub != NULL);
1576993e3fafSRobert Mustacchi 	xd = usba_hcdi_get_device_private(ud->usb_hs_hub_usba_dev);
1577993e3fafSRobert Mustacchi 	ASSERT(xd != NULL);
1578993e3fafSRobert Mustacchi 
1579993e3fafSRobert Mustacchi 	ret = XHCI_SCTX_SET_TT_HUB_SID(xd->xd_slot);
1580993e3fafSRobert Mustacchi 	ret |= XHCI_SCTX_SET_TT_PORT_NUM(ud->usb_hs_hub_usba_dev->usb_port);
1581993e3fafSRobert Mustacchi 
1582993e3fafSRobert Mustacchi 	return (ret);
1583993e3fafSRobert Mustacchi }
1584993e3fafSRobert Mustacchi 
1585993e3fafSRobert Mustacchi /*
1586993e3fafSRobert Mustacchi  * Initialize a new device. This allocates a device slot from the controller,
1587993e3fafSRobert Mustacchi  * which tranfers it to our control.
1588993e3fafSRobert Mustacchi  */
1589993e3fafSRobert Mustacchi static int
xhci_hcdi_device_init(usba_device_t * ud,usb_port_t port,void ** hcdpp)1590993e3fafSRobert Mustacchi xhci_hcdi_device_init(usba_device_t *ud, usb_port_t port, void **hcdpp)
1591993e3fafSRobert Mustacchi {
1592993e3fafSRobert Mustacchi 	int ret, i;
1593993e3fafSRobert Mustacchi 	xhci_device_t *xd;
1594993e3fafSRobert Mustacchi 	ddi_device_acc_attr_t acc;
1595993e3fafSRobert Mustacchi 	ddi_dma_attr_t attr;
1596993e3fafSRobert Mustacchi 	xhci_t *xhcip = xhci_hcdi_get_xhcip_from_dev(ud);
1597993e3fafSRobert Mustacchi 	size_t isize, osize, incr;
1598993e3fafSRobert Mustacchi 	uint32_t route, rp, info, info2, tt;
1599993e3fafSRobert Mustacchi 
1600993e3fafSRobert Mustacchi 	xd = kmem_zalloc(sizeof (xhci_device_t), KM_SLEEP);
1601993e3fafSRobert Mustacchi 	xd->xd_port = port;
1602993e3fafSRobert Mustacchi 	xd->xd_usbdev = ud;
1603993e3fafSRobert Mustacchi 	mutex_init(&xd->xd_imtx, NULL, MUTEX_DRIVER,
1604993e3fafSRobert Mustacchi 	    (void *)(uintptr_t)xhcip->xhci_intr_pri);
1605993e3fafSRobert Mustacchi 
1606993e3fafSRobert Mustacchi 	/*
1607993e3fafSRobert Mustacchi 	 * The size of the context structures is based upon the presence of the
1608993e3fafSRobert Mustacchi 	 * context flag which determines whether we have a 32-byte or 64-byte
1609993e3fafSRobert Mustacchi 	 * context. Note that the input context always has to account for the
1610993e3fafSRobert Mustacchi 	 * entire size of the xhci_input_contex_t, which is 32-bytes by default.
1611993e3fafSRobert Mustacchi 	 */
1612993e3fafSRobert Mustacchi 	if (xhcip->xhci_caps.xcap_flags & XCAP_CSZ) {
1613993e3fafSRobert Mustacchi 		incr = 64;
1614993e3fafSRobert Mustacchi 		osize = XHCI_DEVICE_CONTEXT_64;
1615993e3fafSRobert Mustacchi 		isize = XHCI_DEVICE_CONTEXT_64 + incr;
1616993e3fafSRobert Mustacchi 	} else {
1617993e3fafSRobert Mustacchi 		incr = 32;
1618993e3fafSRobert Mustacchi 		osize = XHCI_DEVICE_CONTEXT_32;
1619993e3fafSRobert Mustacchi 		isize = XHCI_DEVICE_CONTEXT_32 + incr;
1620993e3fafSRobert Mustacchi 	}
1621993e3fafSRobert Mustacchi 
1622993e3fafSRobert Mustacchi 	xhci_dma_acc_attr(xhcip, &acc);
1623993e3fafSRobert Mustacchi 	xhci_dma_dma_attr(xhcip, &attr);
1624993e3fafSRobert Mustacchi 	if (xhci_dma_alloc(xhcip, &xd->xd_ictx, &attr, &acc, B_TRUE,
1625993e3fafSRobert Mustacchi 	    isize, B_FALSE) == B_FALSE) {
1626993e3fafSRobert Mustacchi 		xhci_hcdi_device_free(xd);
1627993e3fafSRobert Mustacchi 		return (USB_NO_RESOURCES);
1628993e3fafSRobert Mustacchi 	}
1629993e3fafSRobert Mustacchi 
1630993e3fafSRobert Mustacchi 	xd->xd_input = (xhci_input_context_t *)xd->xd_ictx.xdb_va;
1631993e3fafSRobert Mustacchi 	xd->xd_slotin = (xhci_slot_context_t *)(xd->xd_ictx.xdb_va + incr);
1632993e3fafSRobert Mustacchi 	for (i = 0; i < XHCI_NUM_ENDPOINTS; i++) {
1633993e3fafSRobert Mustacchi 		xd->xd_endin[i] =
1634993e3fafSRobert Mustacchi 		    (xhci_endpoint_context_t *)(xd->xd_ictx.xdb_va +
1635993e3fafSRobert Mustacchi 		    (i + 2) * incr);
1636993e3fafSRobert Mustacchi 	}
1637993e3fafSRobert Mustacchi 
1638993e3fafSRobert Mustacchi 	if (xhci_dma_alloc(xhcip, &xd->xd_octx, &attr, &acc, B_TRUE,
1639993e3fafSRobert Mustacchi 	    osize, B_FALSE) == B_FALSE) {
1640993e3fafSRobert Mustacchi 		xhci_hcdi_device_free(xd);
1641993e3fafSRobert Mustacchi 		return (USB_NO_RESOURCES);
1642993e3fafSRobert Mustacchi 	}
1643993e3fafSRobert Mustacchi 	xd->xd_slotout = (xhci_slot_context_t *)xd->xd_octx.xdb_va;
1644993e3fafSRobert Mustacchi 	for (i = 0; i < XHCI_NUM_ENDPOINTS; i++) {
1645993e3fafSRobert Mustacchi 		xd->xd_endout[i] =
1646993e3fafSRobert Mustacchi 		    (xhci_endpoint_context_t *)(xd->xd_octx.xdb_va +
1647993e3fafSRobert Mustacchi 		    (i + 1) * incr);
1648993e3fafSRobert Mustacchi 	}
1649993e3fafSRobert Mustacchi 
1650993e3fafSRobert Mustacchi 	ret = xhci_command_enable_slot(xhcip, &xd->xd_slot);
1651993e3fafSRobert Mustacchi 	if (ret != USB_SUCCESS) {
1652993e3fafSRobert Mustacchi 		xhci_hcdi_device_free(xd);
1653993e3fafSRobert Mustacchi 		return (ret);
1654993e3fafSRobert Mustacchi 	}
1655993e3fafSRobert Mustacchi 
1656993e3fafSRobert Mustacchi 	/*
1657993e3fafSRobert Mustacchi 	 * These are the default slot context and the endpoint zero context that
1658993e3fafSRobert Mustacchi 	 * we're enabling. See 4.3.3.
1659993e3fafSRobert Mustacchi 	 */
1660993e3fafSRobert Mustacchi 	xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0) |
1661993e3fafSRobert Mustacchi 	    XHCI_INCTX_MASK_DCI(1));
1662993e3fafSRobert Mustacchi 
1663993e3fafSRobert Mustacchi 	/*
1664993e3fafSRobert Mustacchi 	 * Note, we never need to set the MTT bit as illumos never enables the
1665993e3fafSRobert Mustacchi 	 * alternate MTT interface.
1666993e3fafSRobert Mustacchi 	 */
1667993e3fafSRobert Mustacchi 	xhci_hcdi_device_route(ud, &route, &rp);
1668993e3fafSRobert Mustacchi 	info = XHCI_SCTX_SET_ROUTE(route) | XHCI_SCTX_SET_DCI(1);
1669993e3fafSRobert Mustacchi 	switch (ud->usb_port_status) {
1670993e3fafSRobert Mustacchi 	case USBA_LOW_SPEED_DEV:
1671993e3fafSRobert Mustacchi 		info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_LOW);
1672993e3fafSRobert Mustacchi 		break;
1673993e3fafSRobert Mustacchi 	case USBA_HIGH_SPEED_DEV:
1674993e3fafSRobert Mustacchi 		info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_HIGH);
1675993e3fafSRobert Mustacchi 		break;
1676993e3fafSRobert Mustacchi 	case USBA_FULL_SPEED_DEV:
1677993e3fafSRobert Mustacchi 		info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_FULL);
1678993e3fafSRobert Mustacchi 		break;
1679993e3fafSRobert Mustacchi 	case USBA_SUPER_SPEED_DEV:
1680993e3fafSRobert Mustacchi 	default:
1681993e3fafSRobert Mustacchi 		info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_SUPER);
1682993e3fafSRobert Mustacchi 		break;
1683993e3fafSRobert Mustacchi 	}
1684993e3fafSRobert Mustacchi 	info2 = XHCI_SCTX_SET_RHPORT(rp);
1685993e3fafSRobert Mustacchi 	tt = XHCI_SCTX_SET_IRQ_TARGET(0);
1686993e3fafSRobert Mustacchi 	tt |= xhci_hcdi_device_tt(ud);
1687993e3fafSRobert Mustacchi 
1688993e3fafSRobert Mustacchi 	xd->xd_slotin->xsc_info = LE_32(info);
1689993e3fafSRobert Mustacchi 	xd->xd_slotin->xsc_info2 = LE_32(info2);
1690993e3fafSRobert Mustacchi 	xd->xd_slotin->xsc_tt = LE_32(tt);
1691993e3fafSRobert Mustacchi 
1692993e3fafSRobert Mustacchi 	if ((ret = xhci_endpoint_init(xhcip, xd, NULL)) != 0) {
1693993e3fafSRobert Mustacchi 		(void) xhci_command_disable_slot(xhcip, xd->xd_slot);
1694993e3fafSRobert Mustacchi 		xhci_hcdi_device_free(xd);
1695993e3fafSRobert Mustacchi 		return (USB_HC_HARDWARE_ERROR);
1696993e3fafSRobert Mustacchi 	}
1697993e3fafSRobert Mustacchi 
1698993e3fafSRobert Mustacchi 	if (xhci_context_slot_output_init(xhcip, xd) != B_TRUE) {
1699993e3fafSRobert Mustacchi 		(void) xhci_command_disable_slot(xhcip, xd->xd_slot);
1700993e3fafSRobert Mustacchi 		xhci_endpoint_fini(xd, 0);
1701993e3fafSRobert Mustacchi 		xhci_hcdi_device_free(xd);
1702993e3fafSRobert Mustacchi 		return (USB_HC_HARDWARE_ERROR);
1703993e3fafSRobert Mustacchi 	}
1704993e3fafSRobert Mustacchi 
1705993e3fafSRobert Mustacchi 	if ((ret = xhci_command_set_address(xhcip, xd, B_TRUE)) != 0) {
1706993e3fafSRobert Mustacchi 		(void) xhci_command_disable_slot(xhcip, xd->xd_slot);
1707993e3fafSRobert Mustacchi 		xhci_context_slot_output_fini(xhcip, xd);
1708993e3fafSRobert Mustacchi 		xhci_endpoint_fini(xd, 0);
1709993e3fafSRobert Mustacchi 		xhci_hcdi_device_free(xd);
1710993e3fafSRobert Mustacchi 		return (ret);
1711993e3fafSRobert Mustacchi 	}
1712993e3fafSRobert Mustacchi 
1713993e3fafSRobert Mustacchi 	mutex_enter(&xhcip->xhci_lock);
1714993e3fafSRobert Mustacchi 	list_insert_tail(&xhcip->xhci_usba.xa_devices, xd);
1715993e3fafSRobert Mustacchi 	mutex_exit(&xhcip->xhci_lock);
1716993e3fafSRobert Mustacchi 
1717993e3fafSRobert Mustacchi 	*hcdpp = xd;
1718993e3fafSRobert Mustacchi 	return (ret);
1719993e3fafSRobert Mustacchi }
1720993e3fafSRobert Mustacchi 
1721993e3fafSRobert Mustacchi /*
1722*0ae0ab6fSJoshua M. Clulow  * We're tearing down a device now.
1723993e3fafSRobert Mustacchi  */
1724993e3fafSRobert Mustacchi static void
xhci_hcdi_device_fini(usba_device_t * ud,void * hcdp)1725993e3fafSRobert Mustacchi xhci_hcdi_device_fini(usba_device_t *ud, void *hcdp)
1726993e3fafSRobert Mustacchi {
1727993e3fafSRobert Mustacchi 	int ret;
1728993e3fafSRobert Mustacchi 	xhci_endpoint_t *xep;
1729993e3fafSRobert Mustacchi 	xhci_device_t *xd;
1730993e3fafSRobert Mustacchi 	xhci_t *xhcip;
1731993e3fafSRobert Mustacchi 
1732993e3fafSRobert Mustacchi 	/*
1733993e3fafSRobert Mustacchi 	 * Right now, it's theoretically possible that USBA may try and call
1734993e3fafSRobert Mustacchi 	 * us here even if we hadn't successfully finished the device_init()
1735993e3fafSRobert Mustacchi 	 * endpoint. We should probably modify the USBA to make sure that this
1736993e3fafSRobert Mustacchi 	 * can't happen.
1737993e3fafSRobert Mustacchi 	 */
1738993e3fafSRobert Mustacchi 	if (hcdp == NULL)
1739993e3fafSRobert Mustacchi 		return;
1740993e3fafSRobert Mustacchi 
1741993e3fafSRobert Mustacchi 	xd = hcdp;
1742993e3fafSRobert Mustacchi 	xhcip = xhci_hcdi_get_xhcip_from_dev(ud);
1743993e3fafSRobert Mustacchi 
1744993e3fafSRobert Mustacchi 	/*
1745993e3fafSRobert Mustacchi 	 * Make sure we have no timeout running on the default endpoint still.
1746993e3fafSRobert Mustacchi 	 */
1747993e3fafSRobert Mustacchi 	xep = xd->xd_endpoints[XHCI_DEFAULT_ENDPOINT];
1748993e3fafSRobert Mustacchi 	mutex_enter(&xhcip->xhci_lock);
1749993e3fafSRobert Mustacchi 	xep->xep_state |= XHCI_ENDPOINT_TEARDOWN;
1750993e3fafSRobert Mustacchi 	mutex_exit(&xhcip->xhci_lock);
1751993e3fafSRobert Mustacchi 	(void) untimeout(xep->xep_timeout);
1752993e3fafSRobert Mustacchi 
1753993e3fafSRobert Mustacchi 	/*
1754993e3fafSRobert Mustacchi 	 * Go ahead and disable the slot. There's no reason to do anything
1755993e3fafSRobert Mustacchi 	 * special about the default endpoint as it will be disabled as a part
1756993e3fafSRobert Mustacchi 	 * of the slot disabling. However, if this all fails, we'll leave this
1757993e3fafSRobert Mustacchi 	 * sitting here in a failed state, eating up a device slot. It is
1758993e3fafSRobert Mustacchi 	 * unlikely this will occur.
1759993e3fafSRobert Mustacchi 	 */
1760993e3fafSRobert Mustacchi 	ret = xhci_command_disable_slot(xhcip, xd->xd_slot);
1761993e3fafSRobert Mustacchi 	if (ret != USB_SUCCESS) {
1762993e3fafSRobert Mustacchi 		xhci_error(xhcip, "failed to disable slot %d: %d",
1763993e3fafSRobert Mustacchi 		    xd->xd_slot, ret);
1764993e3fafSRobert Mustacchi 		return;
1765993e3fafSRobert Mustacchi 	}
1766993e3fafSRobert Mustacchi 
1767993e3fafSRobert Mustacchi 	xhci_context_slot_output_fini(xhcip, xd);
1768*0ae0ab6fSJoshua M. Clulow 
1769*0ae0ab6fSJoshua M. Clulow 	/*
1770*0ae0ab6fSJoshua M. Clulow 	 * Once the slot is disabled, we can free any endpoints that were
1771*0ae0ab6fSJoshua M. Clulow 	 * opened.
1772*0ae0ab6fSJoshua M. Clulow 	 */
1773*0ae0ab6fSJoshua M. Clulow 	for (uint_t n = 0; n < XHCI_NUM_ENDPOINTS; n++) {
1774*0ae0ab6fSJoshua M. Clulow 		if (xd->xd_endpoints[n] != NULL) {
1775*0ae0ab6fSJoshua M. Clulow 			xhci_endpoint_fini(xd, n);
1776*0ae0ab6fSJoshua M. Clulow 		}
1777*0ae0ab6fSJoshua M. Clulow 	}
1778993e3fafSRobert Mustacchi 
1779993e3fafSRobert Mustacchi 	mutex_enter(&xhcip->xhci_lock);
1780993e3fafSRobert Mustacchi 	list_remove(&xhcip->xhci_usba.xa_devices, xd);
1781993e3fafSRobert Mustacchi 	mutex_exit(&xhcip->xhci_lock);
1782993e3fafSRobert Mustacchi 
1783993e3fafSRobert Mustacchi 	xhci_hcdi_device_free(xd);
1784993e3fafSRobert Mustacchi }
1785993e3fafSRobert Mustacchi 
1786993e3fafSRobert Mustacchi /*
1787993e3fafSRobert Mustacchi  * Synchronously attempt to set the device address. For xHCI this involves it
1788993e3fafSRobert Mustacchi  * deciding what address to use.
1789993e3fafSRobert Mustacchi  */
1790993e3fafSRobert Mustacchi static int
xhci_hcdi_device_address(usba_device_t * ud)1791993e3fafSRobert Mustacchi xhci_hcdi_device_address(usba_device_t *ud)
1792993e3fafSRobert Mustacchi {
1793993e3fafSRobert Mustacchi 	int ret;
1794993e3fafSRobert Mustacchi 	xhci_t *xhcip = xhci_hcdi_get_xhcip_from_dev(ud);
1795993e3fafSRobert Mustacchi 	xhci_device_t *xd = usba_hcdi_get_device_private(ud);
1796993e3fafSRobert Mustacchi 	xhci_endpoint_t *xep;
1797993e3fafSRobert Mustacchi 
1798993e3fafSRobert Mustacchi 	mutex_enter(&xhcip->xhci_lock);
1799993e3fafSRobert Mustacchi 
1800993e3fafSRobert Mustacchi 	/*
1801993e3fafSRobert Mustacchi 	 * This device may already be addressed from the perspective of the xhci
1802993e3fafSRobert Mustacchi 	 * controller. For example, the device this represents may have been
1803993e3fafSRobert Mustacchi 	 * unconfigured, which does not actually remove the slot or other
1804993e3fafSRobert Mustacchi 	 * information, merely tears down all the active use of it and the child
1805993e3fafSRobert Mustacchi 	 * driver. In such cases, if we're already addressed, just return
1806993e3fafSRobert Mustacchi 	 * success. The actual USB address is a fiction for USBA anyways.
1807993e3fafSRobert Mustacchi 	 */
1808993e3fafSRobert Mustacchi 	if (xd->xd_addressed == B_TRUE) {
1809993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
1810993e3fafSRobert Mustacchi 		return (USB_SUCCESS);
1811993e3fafSRobert Mustacchi 	}
1812993e3fafSRobert Mustacchi 
1813993e3fafSRobert Mustacchi 	ASSERT(xd->xd_addressed == B_FALSE);
1814993e3fafSRobert Mustacchi 	xd->xd_addressed = B_TRUE;
1815993e3fafSRobert Mustacchi 	VERIFY3P(xd->xd_endpoints[XHCI_DEFAULT_ENDPOINT], !=, NULL);
1816993e3fafSRobert Mustacchi 	xep = xd->xd_endpoints[XHCI_DEFAULT_ENDPOINT];
1817993e3fafSRobert Mustacchi 	mutex_exit(&xhcip->xhci_lock);
1818993e3fafSRobert Mustacchi 
1819993e3fafSRobert Mustacchi 	if ((ret = xhci_endpoint_setup_default_context(xhcip, xd, xep)) != 0) {
1820993e3fafSRobert Mustacchi 		ASSERT(ret == EIO);
1821993e3fafSRobert Mustacchi 		return (USB_HC_HARDWARE_ERROR);
1822993e3fafSRobert Mustacchi 	}
1823993e3fafSRobert Mustacchi 
1824993e3fafSRobert Mustacchi 	ret = xhci_command_set_address(xhcip, xd, B_FALSE);
1825993e3fafSRobert Mustacchi 
1826993e3fafSRobert Mustacchi 	if (ret != USB_SUCCESS) {
1827993e3fafSRobert Mustacchi 		mutex_enter(&xhcip->xhci_lock);
1828993e3fafSRobert Mustacchi 		xd->xd_addressed = B_FALSE;
1829993e3fafSRobert Mustacchi 		mutex_exit(&xhcip->xhci_lock);
1830993e3fafSRobert Mustacchi 	}
1831993e3fafSRobert Mustacchi 
1832993e3fafSRobert Mustacchi 	return (ret);
1833993e3fafSRobert Mustacchi }
1834993e3fafSRobert Mustacchi 
1835993e3fafSRobert Mustacchi /*
1836993e3fafSRobert Mustacchi  * This is called relatively early on in a hub's life time. At this point, it's
1837993e3fafSRobert Mustacchi  * descriptors have all been pulled and the default control pipe is still open.
1838993e3fafSRobert Mustacchi  * What we need to do is go through and update the slot context to indicate that
1839993e3fafSRobert Mustacchi  * this is a hub, otherwise, the controller will never let us speak to
1840993e3fafSRobert Mustacchi  * downstream ports.
1841993e3fafSRobert Mustacchi  */
1842993e3fafSRobert Mustacchi static int
xhci_hcdi_hub_update(usba_device_t * ud,uint8_t nports,uint8_t tt)1843993e3fafSRobert Mustacchi xhci_hcdi_hub_update(usba_device_t *ud, uint8_t nports, uint8_t tt)
1844993e3fafSRobert Mustacchi {
1845993e3fafSRobert Mustacchi 	int ret;
1846993e3fafSRobert Mustacchi 	xhci_t *xhcip = xhci_hcdi_get_xhcip_from_dev(ud);
1847993e3fafSRobert Mustacchi 	xhci_device_t *xd = usba_hcdi_get_device_private(ud);
1848993e3fafSRobert Mustacchi 
1849993e3fafSRobert Mustacchi 	if (xd == NULL)
1850993e3fafSRobert Mustacchi 		return (USB_FAILURE);
1851993e3fafSRobert Mustacchi 
1852993e3fafSRobert Mustacchi 	if (ud->usb_hubdi == NULL) {
1853993e3fafSRobert Mustacchi 		return (USB_FAILURE);
1854993e3fafSRobert Mustacchi 	}
1855993e3fafSRobert Mustacchi 
1856993e3fafSRobert Mustacchi 	mutex_enter(&xd->xd_imtx);
1857993e3fafSRobert Mustacchi 
1858993e3fafSRobert Mustacchi 	/*
1859993e3fafSRobert Mustacchi 	 * Note, that usba never sets the interface of a hub to Multi TT. Hence
1860993e3fafSRobert Mustacchi 	 * why we're never setting the MTT bit in xsc_info.
1861993e3fafSRobert Mustacchi 	 */
1862993e3fafSRobert Mustacchi 	xd->xd_slotin->xsc_info |= LE_32(XHCI_SCTX_SET_HUB(1));
1863993e3fafSRobert Mustacchi 	xd->xd_slotin->xsc_info2 |= LE_32(XHCI_SCTX_SET_NPORTS(nports));
1864993e3fafSRobert Mustacchi 	if (ud->usb_port_status == USBA_HIGH_SPEED_DEV)
1865993e3fafSRobert Mustacchi 		xd->xd_slotin->xsc_tt |= LE_32(XHCI_SCTX_SET_TT_THINK_TIME(tt));
1866993e3fafSRobert Mustacchi 
1867993e3fafSRobert Mustacchi 	/*
1868993e3fafSRobert Mustacchi 	 * We're only updating the slot context, no endpoint contexts should be
1869993e3fafSRobert Mustacchi 	 * touched.
1870993e3fafSRobert Mustacchi 	 */
1871993e3fafSRobert Mustacchi 	xd->xd_input->xic_drop_flags = LE_32(0);
1872993e3fafSRobert Mustacchi 	xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0));
1873993e3fafSRobert Mustacchi 
1874993e3fafSRobert Mustacchi 	ret = xhci_command_evaluate_context(xhcip, xd);
1875993e3fafSRobert Mustacchi 	mutex_exit(&xd->xd_imtx);
1876993e3fafSRobert Mustacchi 	return (ret);
1877993e3fafSRobert Mustacchi }
1878993e3fafSRobert Mustacchi 
1879993e3fafSRobert Mustacchi void
xhci_hcd_fini(xhci_t * xhcip)1880993e3fafSRobert Mustacchi xhci_hcd_fini(xhci_t *xhcip)
1881993e3fafSRobert Mustacchi {
1882993e3fafSRobert Mustacchi 	usba_hcdi_unregister(xhcip->xhci_dip);
1883993e3fafSRobert Mustacchi 	usba_free_hcdi_ops(xhcip->xhci_usba.xa_ops);
1884993e3fafSRobert Mustacchi 	list_destroy(&xhcip->xhci_usba.xa_pipes);
1885993e3fafSRobert Mustacchi 	list_destroy(&xhcip->xhci_usba.xa_devices);
1886993e3fafSRobert Mustacchi }
1887993e3fafSRobert Mustacchi 
1888993e3fafSRobert Mustacchi int
xhci_hcd_init(xhci_t * xhcip)1889993e3fafSRobert Mustacchi xhci_hcd_init(xhci_t *xhcip)
1890993e3fafSRobert Mustacchi {
1891993e3fafSRobert Mustacchi 	usba_hcdi_register_args_t hreg;
1892993e3fafSRobert Mustacchi 	usba_hcdi_ops_t *ops;
1893993e3fafSRobert Mustacchi 
1894993e3fafSRobert Mustacchi 	ops = usba_alloc_hcdi_ops();
1895993e3fafSRobert Mustacchi 	VERIFY(ops != NULL);
1896993e3fafSRobert Mustacchi 
1897993e3fafSRobert Mustacchi 	ops->usba_hcdi_ops_version = HCDI_OPS_VERSION;
1898993e3fafSRobert Mustacchi 	ops->usba_hcdi_dip = xhcip->xhci_dip;
1899993e3fafSRobert Mustacchi 
1900993e3fafSRobert Mustacchi 	ops->usba_hcdi_pm_support = xhci_hcdi_pm_support;
1901993e3fafSRobert Mustacchi 	ops->usba_hcdi_pipe_open = xhci_hcdi_pipe_open;
1902993e3fafSRobert Mustacchi 	ops->usba_hcdi_pipe_close = xhci_hcdi_pipe_close;
1903993e3fafSRobert Mustacchi 	ops->usba_hcdi_pipe_reset = xhci_hcdi_pipe_reset;
1904993e3fafSRobert Mustacchi 	ops->usba_hcdi_pipe_reset_data_toggle =
1905993e3fafSRobert Mustacchi 	    xhci_hcdi_pipe_reset_data_toggle;
1906993e3fafSRobert Mustacchi 	ops->usba_hcdi_pipe_ctrl_xfer = xhci_hcdi_pipe_ctrl_xfer;
1907993e3fafSRobert Mustacchi 	ops->usba_hcdi_bulk_transfer_size = xhci_hcdi_bulk_transfer_size;
1908993e3fafSRobert Mustacchi 	ops->usba_hcdi_pipe_bulk_xfer = xhci_hcdi_pipe_bulk_xfer;
1909993e3fafSRobert Mustacchi 	ops->usba_hcdi_pipe_intr_xfer = xhci_hcdi_pipe_intr_xfer;
1910993e3fafSRobert Mustacchi 	ops->usba_hcdi_pipe_stop_intr_polling =
1911993e3fafSRobert Mustacchi 	    xhci_hcdi_pipe_stop_intr_polling;
1912993e3fafSRobert Mustacchi 	ops->usba_hcdi_pipe_isoc_xfer = xhci_hcdi_pipe_isoc_xfer;
1913993e3fafSRobert Mustacchi 	ops->usba_hcdi_pipe_stop_isoc_polling =
1914993e3fafSRobert Mustacchi 	    xhci_hcdi_pipe_stop_isoc_polling;
1915993e3fafSRobert Mustacchi 	ops->usba_hcdi_get_current_frame_number =
1916993e3fafSRobert Mustacchi 	    xhci_hcdi_get_current_frame_number;
1917993e3fafSRobert Mustacchi 	ops->usba_hcdi_get_max_isoc_pkts = xhci_hcdi_get_max_isoc_pkts;
1918993e3fafSRobert Mustacchi 	ops->usba_hcdi_console_input_init = xhci_hcdi_console_input_init;
1919993e3fafSRobert Mustacchi 	ops->usba_hcdi_console_input_fini = xhci_hcdi_console_input_fini;
1920993e3fafSRobert Mustacchi 	ops->usba_hcdi_console_input_enter = xhci_hcdi_console_input_enter;
1921993e3fafSRobert Mustacchi 	ops->usba_hcdi_console_read = xhci_hcdi_console_read;
1922993e3fafSRobert Mustacchi 	ops->usba_hcdi_console_input_exit = xhci_hcdi_console_input_exit;
1923993e3fafSRobert Mustacchi 
1924993e3fafSRobert Mustacchi 	ops->usba_hcdi_console_output_init = xhci_hcdi_console_output_init;
1925993e3fafSRobert Mustacchi 	ops->usba_hcdi_console_output_fini = xhci_hcdi_console_output_fini;
1926993e3fafSRobert Mustacchi 	ops->usba_hcdi_console_output_enter = xhci_hcdi_console_output_enter;
1927993e3fafSRobert Mustacchi 	ops->usba_hcdi_console_write = xhci_hcdi_console_write;
1928993e3fafSRobert Mustacchi 	ops->usba_hcdi_console_output_exit = xhci_hcdi_console_output_exit;
1929993e3fafSRobert Mustacchi 
1930993e3fafSRobert Mustacchi 	ops->usba_hcdi_device_init = xhci_hcdi_device_init;
1931993e3fafSRobert Mustacchi 	ops->usba_hcdi_device_fini = xhci_hcdi_device_fini;
1932993e3fafSRobert Mustacchi 	ops->usba_hcdi_device_address = xhci_hcdi_device_address;
1933993e3fafSRobert Mustacchi 	ops->usba_hcdi_hub_update = xhci_hcdi_hub_update;
1934993e3fafSRobert Mustacchi 
1935993e3fafSRobert Mustacchi 	hreg.usba_hcdi_register_version = HCDI_REGISTER_VERSION;
1936993e3fafSRobert Mustacchi 	hreg.usba_hcdi_register_dip = xhcip->xhci_dip;
1937993e3fafSRobert Mustacchi 	hreg.usba_hcdi_register_ops = ops;
1938993e3fafSRobert Mustacchi 
1939993e3fafSRobert Mustacchi 	/*
1940993e3fafSRobert Mustacchi 	 * We're required to give xhci a set of DMA attributes that it may loan
1941993e3fafSRobert Mustacchi 	 * out to other devices. Therefore we'll be conservative with what we
1942993e3fafSRobert Mustacchi 	 * end up giving it.
1943993e3fafSRobert Mustacchi 	 */
1944993e3fafSRobert Mustacchi 	xhci_dma_dma_attr(xhcip, &xhcip->xhci_usba.xa_dma_attr);
1945993e3fafSRobert Mustacchi 	hreg.usba_hcdi_register_dma_attr = &xhcip->xhci_usba.xa_dma_attr;
1946993e3fafSRobert Mustacchi 
1947993e3fafSRobert Mustacchi 	hreg.usba_hcdi_register_iblock_cookie =
1948993e3fafSRobert Mustacchi 	    (ddi_iblock_cookie_t)(uintptr_t)xhcip->xhci_intr_pri;
1949993e3fafSRobert Mustacchi 
1950993e3fafSRobert Mustacchi 	if (usba_hcdi_register(&hreg, 0) != DDI_SUCCESS) {
1951993e3fafSRobert Mustacchi 		usba_free_hcdi_ops(ops);
1952993e3fafSRobert Mustacchi 		return (DDI_FAILURE);
1953993e3fafSRobert Mustacchi 	}
1954993e3fafSRobert Mustacchi 
1955993e3fafSRobert Mustacchi 	xhcip->xhci_usba.xa_ops = ops;
1956993e3fafSRobert Mustacchi 
1957993e3fafSRobert Mustacchi 	list_create(&xhcip->xhci_usba.xa_devices, sizeof (xhci_device_t),
1958993e3fafSRobert Mustacchi 	    offsetof(xhci_device_t, xd_link));
1959993e3fafSRobert Mustacchi 	list_create(&xhcip->xhci_usba.xa_pipes, sizeof (xhci_pipe_t),
1960993e3fafSRobert Mustacchi 	    offsetof(xhci_pipe_t, xp_link));
1961993e3fafSRobert Mustacchi 
1962993e3fafSRobert Mustacchi 
1963993e3fafSRobert Mustacchi 	return (DDI_SUCCESS);
1964993e3fafSRobert Mustacchi }
1965