1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source.  A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * Copyright (c) 2018, Joyent, Inc.
14  * Copyright (c) 2019 by Western Digital Corporation
15  */
16 
17 /*
18  * illumos USB framework endpoints and functions for xHCI.
19  *
20  * Please see the big theory statement in xhci.c for more information.
21  */
22 
23 #include <sys/usb/hcd/xhci/xhci.h>
24 #include <sys/sysmacros.h>
25 #include <sys/strsun.h>
26 #include <sys/strsubr.h>
27 
28 xhci_t *
xhci_hcdi_get_xhcip_from_dev(usba_device_t * ud)29 xhci_hcdi_get_xhcip_from_dev(usba_device_t *ud)
30 {
31 	dev_info_t *dip = ud->usb_root_hub_dip;
32 	xhci_t *xhcip = ddi_get_soft_state(xhci_soft_state,
33 	    ddi_get_instance(dip));
34 	VERIFY(xhcip != NULL);
35 	return (xhcip);
36 }
37 
38 static xhci_t *
xhci_hcdi_get_xhcip(usba_pipe_handle_data_t * ph)39 xhci_hcdi_get_xhcip(usba_pipe_handle_data_t *ph)
40 {
41 	return (xhci_hcdi_get_xhcip_from_dev(ph->p_usba_device));
42 }
43 
44 /*
45  * While the xHCI hardware is capable of supporting power management, we don't
46  * in the driver right now. Note, USBA doesn't seem to end up calling this entry
47  * point.
48  */
49 /* ARGSUSED */
50 static int
xhci_hcdi_pm_support(dev_info_t * dip)51 xhci_hcdi_pm_support(dev_info_t *dip)
52 {
53 	return (USB_FAILURE);
54 }
55 
56 static int
xhci_hcdi_pipe_open(usba_pipe_handle_data_t * ph,usb_flags_t usb_flags)57 xhci_hcdi_pipe_open(usba_pipe_handle_data_t *ph, usb_flags_t usb_flags)
58 {
59 	xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
60 	xhci_pipe_t *pipe;
61 	xhci_endpoint_t *xep;
62 	xhci_device_t *xd;
63 	int kmflags = usb_flags & USB_FLAGS_SLEEP ? KM_SLEEP : KM_NOSLEEP;
64 	int ret;
65 	uint_t epid;
66 
67 	mutex_enter(&xhcip->xhci_lock);
68 	if (xhcip->xhci_state & XHCI_S_ERROR) {
69 		mutex_exit(&xhcip->xhci_lock);
70 		return (USB_HC_HARDWARE_ERROR);
71 	}
72 	mutex_exit(&xhcip->xhci_lock);
73 
74 	/*
75 	 * If we're here, something must be trying to open an already-opened
76 	 * pipe which is bad news.
77 	 */
78 	if (ph->p_hcd_private != NULL) {
79 		return (USB_FAILURE);
80 	}
81 
82 	pipe = kmem_zalloc(sizeof (xhci_pipe_t), kmflags);
83 	if (pipe == NULL) {
84 		return (USB_NO_RESOURCES);
85 	}
86 	pipe->xp_opentime = gethrtime();
87 	pipe->xp_pipe = ph;
88 
89 	/*
90 	 * If this is the root hub, there's nothing special to do on open. Just
91 	 * go ahead and allow it to be opened. All we have to do is add this to
92 	 * the list of our tracking structures for open pipes.
93 	 */
94 	if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
95 		xep = NULL;
96 		goto add;
97 	}
98 
99 	/*
100 	 * Now that we're here, we're being asked to open up an endpoint of some
101 	 * kind. Because we've already handled the case of the root hub,
102 	 * everything should have a device.
103 	 */
104 	epid = xhci_endpoint_pipe_to_epid(ph);
105 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
106 	if (xd == NULL) {
107 		xhci_error(xhcip, "!encountered endpoint (%d) without device "
108 		    "during pipe open", epid);
109 		kmem_free(pipe, sizeof (xhci_pipe_t));
110 		return (USB_FAILURE);
111 	}
112 
113 	/*
114 	 * See if this endpoint exists or not, in general endpoints should not
115 	 * exist except for the default control endpoint, which we don't tear
116 	 * down until the device itself is cleaned up. Otherwise, a given pipe
117 	 * can only be open once.
118 	 */
119 	mutex_enter(&xhcip->xhci_lock);
120 	if (epid == XHCI_DEFAULT_ENDPOINT) {
121 		xep = xd->xd_endpoints[epid];
122 		VERIFY(xep != NULL);
123 		VERIFY(xep->xep_pipe == NULL);
124 		xep->xep_pipe = ph;
125 		mutex_exit(&xhcip->xhci_lock);
126 		ret = xhci_endpoint_update_default(xhcip, xd, xep);
127 		if (ret != USB_SUCCESS) {
128 			kmem_free(pipe, sizeof (xhci_pipe_t));
129 			return (ret);
130 		}
131 		goto add;
132 	}
133 
134 	if (xd->xd_endpoints[epid] != NULL) {
135 		mutex_exit(&xhcip->xhci_lock);
136 		kmem_free(pipe, sizeof (xhci_pipe_t));
137 		xhci_log(xhcip, "!asked to open endpoint %d on slot %d and "
138 		    "port %d, but endpoint already exists", epid, xd->xd_slot,
139 		    xd->xd_port);
140 		return (USB_FAILURE);
141 	}
142 
143 	/*
144 	 * If we're opening an endpoint other than the default control endpoint,
145 	 * then the device should have had a USB address assigned by the
146 	 * controller. Sanity check that before continuing.
147 	 */
148 	if (epid != XHCI_DEFAULT_ENDPOINT) {
149 		VERIFY(xd->xd_addressed == B_TRUE);
150 	}
151 
152 	/*
153 	 * Okay, at this point we need to go create and set up an endpoint.
154 	 * Once we're done, we'll try to install it and make sure that it
155 	 * doesn't conflict with something else going on.
156 	 */
157 	ret = xhci_endpoint_init(xhcip, xd, ph);
158 	if (ret != 0) {
159 		mutex_exit(&xhcip->xhci_lock);
160 		kmem_free(pipe, sizeof (xhci_pipe_t));
161 		if (ret == EIO) {
162 			xhci_error(xhcip, "failed to initialize endpoint %d "
163 			    "on device slot %d and port %d: encountered fatal "
164 			    "FM error, resetting device", epid, xd->xd_slot,
165 			    xd->xd_port);
166 			xhci_fm_runtime_reset(xhcip);
167 		}
168 		return (USB_HC_HARDWARE_ERROR);
169 	}
170 	xep = xd->xd_endpoints[epid];
171 
172 	mutex_enter(&xd->xd_imtx);
173 	mutex_exit(&xhcip->xhci_lock);
174 
175 	/*
176 	 * Update the slot and input context for this endpoint. We make sure to
177 	 * always set the slot as having changed in the context field as the
178 	 * specification suggests we should and some hardware requires it.
179 	 */
180 	xd->xd_input->xic_drop_flags = LE_32(0);
181 	xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0) |
182 	    XHCI_INCTX_MASK_DCI(epid + 1));
183 
184 	if (epid + 1 > XHCI_SCTX_GET_DCI(LE_32(xd->xd_slotin->xsc_info))) {
185 		uint32_t info;
186 
187 		info = xd->xd_slotin->xsc_info;
188 		info &= ~XHCI_SCTX_DCI_MASK;
189 		info |= XHCI_SCTX_SET_DCI(epid + 1);
190 		xd->xd_slotin->xsc_info = info;
191 	}
192 
193 	XHCI_DMA_SYNC(xd->xd_ictx, DDI_DMA_SYNC_FORDEV);
194 	if (xhci_check_dma_handle(xhcip, &xd->xd_ictx) != DDI_FM_OK) {
195 		mutex_exit(&xd->xd_imtx);
196 		xhci_endpoint_fini(xd, epid);
197 		kmem_free(pipe, sizeof (xhci_pipe_t));
198 		xhci_error(xhcip, "failed to open pipe on endpoint %d of "
199 		    "device with slot %d and port %d: encountered fatal FM "
200 		    "error syncing device input context, resetting device",
201 		    epid, xd->xd_slot, xd->xd_port);
202 		xhci_fm_runtime_reset(xhcip);
203 		return (USB_HC_HARDWARE_ERROR);
204 	}
205 
206 	if ((ret = xhci_command_configure_endpoint(xhcip, xd)) != USB_SUCCESS) {
207 		mutex_exit(&xd->xd_imtx);
208 		xhci_endpoint_fini(xd, epid);
209 		kmem_free(pipe, sizeof (xhci_pipe_t));
210 		return (ret);
211 	}
212 
213 	mutex_exit(&xd->xd_imtx);
214 add:
215 	pipe->xp_ep = xep;
216 	ph->p_hcd_private = (usb_opaque_t)pipe;
217 	mutex_enter(&xhcip->xhci_lock);
218 	list_insert_tail(&xhcip->xhci_usba.xa_pipes, pipe);
219 	mutex_exit(&xhcip->xhci_lock);
220 
221 	return (USB_SUCCESS);
222 }
223 
224 static void
xhci_hcdi_periodic_free(xhci_t * xhcip,xhci_pipe_t * xp)225 xhci_hcdi_periodic_free(xhci_t *xhcip, xhci_pipe_t *xp)
226 {
227 	int i;
228 	xhci_periodic_pipe_t *xpp = &xp->xp_periodic;
229 
230 	if (xpp->xpp_tsize == 0)
231 		return;
232 
233 	for (i = 0; i < xpp->xpp_ntransfers; i++) {
234 		if (xpp->xpp_transfers[i] == NULL)
235 			continue;
236 		xhci_transfer_free(xhcip, xpp->xpp_transfers[i]);
237 		xpp->xpp_transfers[i] = NULL;
238 	}
239 
240 	xpp->xpp_ntransfers = 0;
241 	xpp->xpp_tsize = 0;
242 }
243 
244 /*
245  * Iterate over all transfers and free everything on the pipe. Once done, update
246  * the ring to basically 'consume' everything. For periodic IN endpoints, we
247  * need to handle this somewhat differently and actually close the original
248  * request and not deallocate the related pieces as those exist for the lifetime
249  * of the endpoint and are constantly reused.
250  */
251 static void
xhci_hcdi_pipe_flush(xhci_t * xhcip,xhci_endpoint_t * xep,int intr_code)252 xhci_hcdi_pipe_flush(xhci_t *xhcip, xhci_endpoint_t *xep, int intr_code)
253 {
254 	xhci_transfer_t *xt;
255 
256 	ASSERT(MUTEX_HELD(&xhcip->xhci_lock));
257 
258 	while ((xt = list_remove_head(&xep->xep_transfers)) != NULL) {
259 		if (xhci_endpoint_is_periodic_in(xep) == B_FALSE) {
260 			usba_hcdi_cb(xep->xep_pipe, xt->xt_usba_req,
261 			    USB_CR_FLUSHED);
262 			xhci_transfer_free(xhcip, xt);
263 		}
264 	}
265 
266 	if (xhci_endpoint_is_periodic_in(xep) == B_TRUE) {
267 		xhci_pipe_t *xp = (xhci_pipe_t *)xep->xep_pipe->p_hcd_private;
268 		xhci_periodic_pipe_t *xpp = &xp->xp_periodic;
269 
270 		if (xpp->xpp_usb_req != NULL) {
271 			usba_hcdi_cb(xep->xep_pipe, xpp->xpp_usb_req,
272 			    intr_code);
273 			xpp->xpp_usb_req = NULL;
274 		}
275 	}
276 }
277 
278 /*
279  * We've been asked to terminate some set of regular I/O on an interrupt pipe.
280  * If this is for the root device, e.g. the xhci driver itself, then we remove
281  * our interrupt callback. Otherwise we stop the device for interrupt polling as
282  * follows:
283  *
284  * 1. Issue a stop endpoint command
285  * 2. Check to make sure that the endpoint stopped and reset it if needed.
286  * 3. Any thing that gets resolved can callback in the interim.
287  * 4. Ensure that nothing is scheduled on the ring
288  * 5. Skip the contents of the ring and set the TR dequeue pointer.
289  * 6. Return the original callback with a USB_CR_STOPPED_POLLING, NULL out the
290  *    callback in the process.
291  */
292 static int
xhci_hcdi_pipe_poll_fini(usba_pipe_handle_data_t * ph,boolean_t is_close)293 xhci_hcdi_pipe_poll_fini(usba_pipe_handle_data_t *ph, boolean_t is_close)
294 {
295 	int ret;
296 	uint_t epid;
297 	xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
298 	xhci_device_t *xd;
299 	xhci_endpoint_t *xep;
300 	xhci_pipe_t *xp;
301 	xhci_periodic_pipe_t *xpp;
302 	usb_opaque_t urp;
303 
304 	mutex_enter(&xhcip->xhci_lock);
305 	if (xhcip->xhci_state & XHCI_S_ERROR) {
306 		mutex_exit(&xhcip->xhci_lock);
307 		return (USB_HC_HARDWARE_ERROR);
308 	}
309 
310 	if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
311 		xhci_root_hub_intr_root_disable(xhcip);
312 		ret = USB_SUCCESS;
313 		mutex_exit(&xhcip->xhci_lock);
314 		return (ret);
315 	}
316 
317 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
318 	epid = xhci_endpoint_pipe_to_epid(ph);
319 	if (xd->xd_endpoints[epid] == NULL) {
320 		mutex_exit(&xhcip->xhci_lock);
321 		xhci_error(xhcip, "asked to stop intr polling on slot %d, "
322 		    "port %d, endpoint: %d, but no endpoint structure",
323 		    xd->xd_slot, xd->xd_port, epid);
324 		return (USB_FAILURE);
325 	}
326 	xep = xd->xd_endpoints[epid];
327 	xp = (xhci_pipe_t *)ph->p_hcd_private;
328 	if (xp == NULL) {
329 		mutex_exit(&xhcip->xhci_lock);
330 		xhci_error(xhcip, "asked to do finish polling on slot %d, "
331 		    "port %d, endpoint: %d, but no pipe structure",
332 		    xd->xd_slot, xd->xd_port, epid);
333 		return (USB_FAILURE);
334 	}
335 	xpp = &xp->xp_periodic;
336 
337 	/*
338 	 * Ensure that no other resets or time outs are going on right now.
339 	 */
340 	while ((xep->xep_state & (XHCI_ENDPOINT_SERIALIZE)) != 0) {
341 		cv_wait(&xep->xep_state_cv, &xhcip->xhci_lock);
342 	}
343 
344 	if (xpp->xpp_poll_state == XHCI_PERIODIC_POLL_IDLE) {
345 		mutex_exit(&xhcip->xhci_lock);
346 		return (USB_SUCCESS);
347 	}
348 
349 	if (xpp->xpp_poll_state == XHCI_PERIODIC_POLL_STOPPING) {
350 		mutex_exit(&xhcip->xhci_lock);
351 		return (USB_FAILURE);
352 	}
353 
354 	xpp->xpp_poll_state = XHCI_PERIODIC_POLL_STOPPING;
355 	xep->xep_state |= XHCI_ENDPOINT_QUIESCE;
356 	ret = xhci_endpoint_quiesce(xhcip, xd, xep);
357 	if (ret != USB_SUCCESS) {
358 		xhci_error(xhcip, "!failed to quiesce endpoint on slot %d, "
359 		    "port %d, endpoint: %d, failed with %d.",
360 		    xd->xd_slot, xd->xd_port, epid, ret);
361 		xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
362 		cv_broadcast(&xep->xep_state_cv);
363 		mutex_exit(&xhcip->xhci_lock);
364 		return (ret);
365 	}
366 
367 	/*
368 	 * Okay, we've stopped this ring time to wrap it all up. Remove all the
369 	 * transfers, note they aren't freed like a pipe reset.
370 	 */
371 	while (list_is_empty(&xep->xep_transfers) == 0)
372 		(void) list_remove_head(&xep->xep_transfers);
373 	xhci_ring_skip(&xep->xep_ring);
374 	mutex_exit(&xhcip->xhci_lock);
375 
376 	if ((ret = xhci_command_set_tr_dequeue(xhcip, xd, xep)) !=
377 	    USB_SUCCESS) {
378 		xhci_error(xhcip, "!failed to reset endpoint ring on slot %d, "
379 		    "port %d, endpoint: %d, failed with %d.",
380 		    xd->xd_slot, xd->xd_port, epid, ret);
381 		mutex_enter(&xhcip->xhci_lock);
382 		xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
383 		cv_broadcast(&xep->xep_state_cv);
384 		mutex_exit(&xhcip->xhci_lock);
385 		return (ret);
386 	}
387 
388 	mutex_enter(&xhcip->xhci_lock);
389 	urp = xpp->xpp_usb_req;
390 	xpp->xpp_usb_req = NULL;
391 	xpp->xpp_poll_state = XHCI_PERIODIC_POLL_IDLE;
392 	xep->xep_state &= ~XHCI_ENDPOINT_PERIODIC;
393 	mutex_exit(&xhcip->xhci_lock);
394 
395 	/*
396 	 * It's possible that with a persistent pipe, we may not actually have
397 	 * anything left to call back on, because we already had.
398 	 */
399 	if (urp != NULL) {
400 		usba_hcdi_cb(ph, urp, is_close == B_TRUE ?
401 		    USB_CR_PIPE_CLOSING : USB_CR_STOPPED_POLLING);
402 	}
403 
404 	/*
405 	 * Notify anything waiting for us that we're done quiescing this device.
406 	 */
407 	mutex_enter(&xhcip->xhci_lock);
408 	xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
409 	cv_broadcast(&xep->xep_state_cv);
410 	mutex_exit(&xhcip->xhci_lock);
411 
412 	return (USB_SUCCESS);
413 
414 }
415 
416 /*
417  * Tear down everything that we did in open. After this, the consumer of this
418  * USB device is done.
419  */
420 /* ARGSUSED */
421 static int
xhci_hcdi_pipe_close(usba_pipe_handle_data_t * ph,usb_flags_t usb_flags)422 xhci_hcdi_pipe_close(usba_pipe_handle_data_t *ph, usb_flags_t usb_flags)
423 {
424 	xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
425 	xhci_pipe_t *xp;
426 	xhci_device_t *xd;
427 	xhci_endpoint_t *xep;
428 	uint32_t info;
429 	int ret, i;
430 	uint_t epid;
431 
432 	if ((ph->p_ep.bmAttributes & USB_EP_ATTR_MASK) == USB_EP_ATTR_INTR &&
433 	    xhcip->xhci_usba.xa_intr_cb_ph != NULL) {
434 		if ((ret = xhci_hcdi_pipe_poll_fini(ph, B_TRUE)) !=
435 		    USB_SUCCESS) {
436 			return (ret);
437 		}
438 	}
439 
440 	mutex_enter(&xhcip->xhci_lock);
441 
442 	xp = (xhci_pipe_t *)ph->p_hcd_private;
443 	VERIFY(xp != NULL);
444 
445 	/*
446 	 * The default endpoint is special. It is created and destroyed with the
447 	 * device. So like with open, closing it is just state tracking. The
448 	 * same is true for the root hub.
449 	 */
450 	if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR)
451 		goto remove;
452 
453 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
454 	epid = xhci_endpoint_pipe_to_epid(ph);
455 	if (xd->xd_endpoints[epid] == NULL) {
456 		mutex_exit(&xhcip->xhci_lock);
457 		xhci_error(xhcip, "asked to do close pipe on slot %d, "
458 		    "port %d, endpoint: %d, but no endpoint structure",
459 		    xd->xd_slot, xd->xd_port, epid);
460 		return (USB_FAILURE);
461 	}
462 	xep = xd->xd_endpoints[epid];
463 
464 	if (xp->xp_ep != NULL && xp->xp_ep->xep_num == XHCI_DEFAULT_ENDPOINT) {
465 		xep->xep_pipe = NULL;
466 		goto remove;
467 	}
468 
469 	/*
470 	 * We need to clean up the endpoint. So the first thing we need to do is
471 	 * stop it with a configure endpoint command. Once it's stopped, we can
472 	 * free all associated resources.
473 	 */
474 	mutex_enter(&xd->xd_imtx);
475 
476 	/*
477 	 * Potentially update the slot input context about the current max
478 	 * endpoint. Make sure to set that the slot context is being updated
479 	 * here as it may be changing and some hardware requires it.
480 	 */
481 	xd->xd_input->xic_drop_flags = LE_32(XHCI_INCTX_MASK_DCI(epid + 1));
482 	xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0));
483 	for (i = XHCI_NUM_ENDPOINTS - 1; i >= 0; i--) {
484 		if (xd->xd_endpoints[i] != NULL &&
485 		    xd->xd_endpoints[i] != xep)
486 			break;
487 	}
488 	info = xd->xd_slotin->xsc_info;
489 	info &= ~XHCI_SCTX_DCI_MASK;
490 	info |= XHCI_SCTX_SET_DCI(i + 1);
491 	xd->xd_slotin->xsc_info = info;
492 
493 	/*
494 	 * Also zero out our context for this endpoint. Note that we don't
495 	 * bother with syncing DMA memory here as it's not required to be synced
496 	 * for this operation.
497 	 */
498 	bzero(xd->xd_endin[xep->xep_num], sizeof (xhci_endpoint_context_t));
499 
500 	/*
501 	 * Stop the device and kill our timeout. Note, it is safe to hold the
502 	 * device's input mutex across the untimeout, this lock should never be
503 	 * referenced by the timeout code.
504 	 */
505 	xep->xep_state |= XHCI_ENDPOINT_TEARDOWN;
506 	mutex_exit(&xhcip->xhci_lock);
507 	(void) untimeout(xep->xep_timeout);
508 
509 	ret = xhci_command_configure_endpoint(xhcip, xd);
510 	mutex_exit(&xd->xd_imtx);
511 	if (ret != USB_SUCCESS)
512 		return (ret);
513 	mutex_enter(&xhcip->xhci_lock);
514 
515 	/*
516 	 * Now that we've unconfigured the endpoint. See if we need to flush any
517 	 * transfers.
518 	 */
519 	xhci_hcdi_pipe_flush(xhcip, xep, USB_CR_PIPE_CLOSING);
520 	if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
521 		xhci_hcdi_periodic_free(xhcip, xp);
522 	}
523 
524 	xhci_endpoint_fini(xd, epid);
525 
526 remove:
527 	ph->p_hcd_private = NULL;
528 	list_remove(&xhcip->xhci_usba.xa_pipes, xp);
529 	kmem_free(xp, sizeof (xhci_pipe_t));
530 
531 	mutex_exit(&xhcip->xhci_lock);
532 
533 	return (USB_SUCCESS);
534 }
535 
536 /*
537  * We've been asked to reset a pipe aka an endpoint. This endpoint may be in an
538  * arbitrary state, it may be running or it may be halted. In this case, we go
539  * through and check whether or not we know it's been halted or not. If it has
540  * not, then we stop the endpoint.
541  *
542  * Once the endpoint has been stopped, walk all transfers and go ahead and
543  * basically return them as being flushed. Then finally set the dequeue point
544  * for this endpoint.
545  */
546 /* ARGSUSED */
547 static int
xhci_hcdi_pipe_reset(usba_pipe_handle_data_t * ph,usb_flags_t usb_flags)548 xhci_hcdi_pipe_reset(usba_pipe_handle_data_t *ph, usb_flags_t usb_flags)
549 {
550 	xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
551 	xhci_device_t *xd;
552 	xhci_endpoint_t *xep;
553 	uint_t epid;
554 	int ret;
555 
556 	mutex_enter(&xhcip->xhci_lock);
557 	if (xhcip->xhci_state & XHCI_S_ERROR) {
558 		mutex_exit(&xhcip->xhci_lock);
559 		return (USB_HC_HARDWARE_ERROR);
560 	}
561 
562 	if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
563 		mutex_exit(&xhcip->xhci_lock);
564 		return (USB_NOT_SUPPORTED);
565 	}
566 
567 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
568 	epid = xhci_endpoint_pipe_to_epid(ph);
569 	if (xd->xd_endpoints[epid] == NULL) {
570 		mutex_exit(&xhcip->xhci_lock);
571 		xhci_error(xhcip, "asked to do reset pipe on slot %d, "
572 		    "port %d, endpoint: %d, but no endpoint structure",
573 		    xd->xd_slot, xd->xd_port, epid);
574 		return (USB_FAILURE);
575 	}
576 
577 	xep = xd->xd_endpoints[epid];
578 
579 	/*
580 	 * Ensure that no other resets or time outs are going on right now.
581 	 */
582 	while ((xep->xep_state & (XHCI_ENDPOINT_SERIALIZE)) != 0) {
583 		cv_wait(&xep->xep_state_cv, &xhcip->xhci_lock);
584 	}
585 
586 	xep->xep_state |= XHCI_ENDPOINT_QUIESCE;
587 	ret = xhci_endpoint_quiesce(xhcip, xd, xep);
588 	if (ret != USB_SUCCESS) {
589 		/*
590 		 * We failed to quiesce for some reason, remove the flag and let
591 		 * someone else give it a shot.
592 		 */
593 		xhci_error(xhcip, "!failed to quiesce endpoint on slot %d, "
594 		    "port %d, endpoint: %d, failed with %d.",
595 		    xd->xd_slot, xd->xd_port, epid, ret);
596 		xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
597 		cv_broadcast(&xep->xep_state_cv);
598 		mutex_exit(&xhcip->xhci_lock);
599 		return (ret);
600 	}
601 
602 	xhci_ring_skip(&xep->xep_ring);
603 
604 	mutex_exit(&xhcip->xhci_lock);
605 	if ((ret = xhci_command_set_tr_dequeue(xhcip, xd, xep)) !=
606 	    USB_SUCCESS) {
607 		xhci_error(xhcip, "!failed to reset endpoint ring on slot %d, "
608 		    "port %d, endpoint: %d, failed setting ring dequeue with "
609 		    "%d.", xd->xd_slot, xd->xd_port, epid, ret);
610 		mutex_enter(&xhcip->xhci_lock);
611 		xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
612 		cv_broadcast(&xep->xep_state_cv);
613 		mutex_exit(&xhcip->xhci_lock);
614 		return (ret);
615 	}
616 
617 	mutex_enter(&xhcip->xhci_lock);
618 	xhci_hcdi_pipe_flush(xhcip, xep, USB_CR_PIPE_RESET);
619 
620 	/*
621 	 * We need to remove the periodic flag as part of resetting, as if this
622 	 * was used for periodic activity, it no longer is and therefore can now
623 	 * be used for such purposes.
624 	 *
625 	 * Notify anything waiting for us that we're done quiescing this device.
626 	 */
627 	xep->xep_state &= ~(XHCI_ENDPOINT_QUIESCE | XHCI_ENDPOINT_PERIODIC);
628 	cv_broadcast(&xep->xep_state_cv);
629 	mutex_exit(&xhcip->xhci_lock);
630 
631 	return (USB_SUCCESS);
632 }
633 
634 /*
635  * We're asked to reset or change the data toggle, which is used in a few cases.
636  * However, there doesn't seem to be a good way to do this in xHCI as the data
637  * toggle isn't exposed. It seems that dropping a reset endpoint would
638  * theoretically do this; however, that can only be used when in the HALTED
639  * state. As such, for now we just return.
640  */
641 /* ARGSUSED */
642 void
xhci_hcdi_pipe_reset_data_toggle(usba_pipe_handle_data_t * pipe_handle)643 xhci_hcdi_pipe_reset_data_toggle(usba_pipe_handle_data_t *pipe_handle)
644 {
645 }
646 
647 /*
648  * We need to convert the USB request to an 8-byte little endian value. If we
649  * didn't have to think about big endian systems, this would be fine.
650  * Unfortunately, with them, this is a bit confusing. The problem is that if you
651  * think of this as a struct layout, the order that we or things together
652  * represents their byte layout. e.g. ctrl_bRequest is at offset 1 in the SETUP
653  * STAGE trb. However, when it becomes a part of a 64-bit big endian number, if
654  * ends up at byte 7, where as it needs to be at one. Hence why we do a final
655  * LE_64 at the end of this, to convert this into the byte order that it's
656  * expected to be in.
657  */
658 static uint64_t
xhci_hcdi_ctrl_req_to_trb(usb_ctrl_req_t * ucrp)659 xhci_hcdi_ctrl_req_to_trb(usb_ctrl_req_t *ucrp)
660 {
661 	uint64_t ret = ucrp->ctrl_bmRequestType |
662 	    (ucrp->ctrl_bRequest << 8) |
663 	    ((uint64_t)LE_16(ucrp->ctrl_wValue) << 16) |
664 	    ((uint64_t)LE_16(ucrp->ctrl_wIndex) << 32) |
665 	    ((uint64_t)LE_16(ucrp->ctrl_wLength) << 48);
666 	return (LE_64(ret));
667 }
668 
669 /*
670  * USBA calls us in order to make a specific control type request to a device,
671  * potentially even the root hub. If the request is for the root hub, then we
672  * need to intercept this and cons up the requested data.
673  */
674 static int
xhci_hcdi_pipe_ctrl_xfer(usba_pipe_handle_data_t * ph,usb_ctrl_req_t * ucrp,usb_flags_t usb_flags)675 xhci_hcdi_pipe_ctrl_xfer(usba_pipe_handle_data_t *ph, usb_ctrl_req_t *ucrp,
676     usb_flags_t usb_flags)
677 {
678 	int ret, statusdir, trt;
679 	uint_t ep;
680 	xhci_device_t *xd;
681 	xhci_endpoint_t *xep;
682 	xhci_transfer_t *xt;
683 	boolean_t datain;
684 
685 	xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
686 
687 	mutex_enter(&xhcip->xhci_lock);
688 	if (xhcip->xhci_state & XHCI_S_ERROR) {
689 		mutex_exit(&xhcip->xhci_lock);
690 		return (USB_HC_HARDWARE_ERROR);
691 	}
692 
693 	if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
694 		ret = xhci_root_hub_ctrl_req(xhcip, ph, ucrp);
695 		mutex_exit(&xhcip->xhci_lock);
696 		return (ret);
697 	}
698 
699 	/*
700 	 * Determine the device and endpoint.
701 	 */
702 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
703 	ep = xhci_endpoint_pipe_to_epid(ph);
704 	if (xd->xd_endpoints[ep] == NULL) {
705 		mutex_exit(&xhcip->xhci_lock);
706 		xhci_error(xhcip, "asked to do control transfer on slot %d, "
707 		    "port %d, endpoint: %d, but no endpoint structure",
708 		    xd->xd_slot, xd->xd_port, ep);
709 		return (USB_FAILURE);
710 	}
711 	xep = xd->xd_endpoints[ep];
712 
713 	/*
714 	 * There are several types of requests that we have to handle in special
715 	 * ways in xHCI. If we have one of those requests, then we don't
716 	 * necessarily go through the normal path. These special cases are all
717 	 * documented in xHCI 1.1 / 4.5.4.
718 	 *
719 	 * Looking at that, you may ask why aren't SET_CONFIGURATION and SET_IF
720 	 * special cased here. This action is a little confusing by default. The
721 	 * xHCI specification requires that we may need to issue a configure
722 	 * endpoint command as part of this. However, the xHCI 1.1 / 4.5.4.2
723 	 * states that we don't actually need to if nothing in the endpoint
724 	 * configuration context has changed. Because nothing in it should have
725 	 * changed as part of this, we don't need to do anything and instead
726 	 * just can issue the request normally. We're also assuming in the
727 	 * USB_REQ_SET_IF case that if something's changing the interface, the
728 	 * non-default endpoint will have yet to be opened.
729 	 */
730 	if (ucrp->ctrl_bmRequestType == USB_DEV_REQ_HOST_TO_DEV &&
731 	    ucrp->ctrl_bRequest == USB_REQ_SET_ADDRESS) {
732 		/*
733 		 * As we've defined an explicit set-address endpoint, we should
734 		 * never call this function. If we get here, always fail.
735 		 */
736 		mutex_exit(&xhcip->xhci_lock);
737 		usba_hcdi_cb(ph, (usb_opaque_t)ucrp, USB_CR_NOT_SUPPORTED);
738 		return (USB_SUCCESS);
739 	}
740 
741 	mutex_exit(&xhcip->xhci_lock);
742 
743 	/*
744 	 * Allocate the transfer memory, etc.
745 	 */
746 	xt = xhci_transfer_alloc(xhcip, xep, ucrp->ctrl_wLength, 2, usb_flags);
747 	if (xt == NULL) {
748 		return (USB_NO_RESOURCES);
749 	}
750 	xt->xt_usba_req = (usb_opaque_t)ucrp;
751 	xt->xt_timeout = ucrp->ctrl_timeout;
752 	if (xt->xt_timeout == 0) {
753 		xt->xt_timeout = HCDI_DEFAULT_TIMEOUT;
754 	}
755 
756 	if (ucrp->ctrl_wLength > 0) {
757 		if ((ucrp->ctrl_bmRequestType & USB_DEV_REQ_DEV_TO_HOST) != 0) {
758 			trt = XHCI_TRB_TRT_IN;
759 			datain = B_TRUE;
760 			statusdir = 0;
761 		} else {
762 			trt = XHCI_TRB_TRT_OUT;
763 			datain = B_FALSE;
764 			statusdir = XHCI_TRB_DIR_IN;
765 
766 			xhci_transfer_copy(xt, ucrp->ctrl_data->b_rptr,
767 			    ucrp->ctrl_wLength, B_FALSE);
768 			if (xhci_transfer_sync(xhcip, xt,
769 			    DDI_DMA_SYNC_FORDEV) != DDI_FM_OK) {
770 				xhci_transfer_free(xhcip, xt);
771 				xhci_error(xhcip, "failed to synchronize ctrl "
772 				    "transfer DMA memory on endpoint %u of "
773 				    "device on slot %d and port %d: resetting "
774 				    "device", xep->xep_num, xd->xd_slot,
775 				    xd->xd_port);
776 				xhci_fm_runtime_reset(xhcip);
777 				return (USB_HC_HARDWARE_ERROR);
778 			}
779 		}
780 	} else {
781 		trt = 0;
782 		datain = B_FALSE;
783 		statusdir = XHCI_TRB_DIR_IN;
784 	}
785 
786 	/*
787 	 * We always fill in the required setup and status TRBs ourselves;
788 	 * however, to minimize our knowledge about how the data has been split
789 	 * across multiple DMA cookies in an SGL, we leave that to the transfer
790 	 * logic to fill in.
791 	 */
792 	xt->xt_trbs[0].trb_addr = xhci_hcdi_ctrl_req_to_trb(ucrp);
793 	xt->xt_trbs[0].trb_status = LE_32(XHCI_TRB_LEN(8) | XHCI_TRB_INTR(0));
794 	xt->xt_trbs[0].trb_flags = LE_32(trt | XHCI_TRB_IDT |
795 	    XHCI_TRB_TYPE_SETUP);
796 
797 	if (ucrp->ctrl_wLength > 0)
798 		xhci_transfer_trb_fill_data(xep, xt, 1, datain);
799 
800 	xt->xt_trbs[xt->xt_ntrbs - 1].trb_addr = 0;
801 	xt->xt_trbs[xt->xt_ntrbs - 1].trb_status = LE_32(XHCI_TRB_INTR(0));
802 	xt->xt_trbs[xt->xt_ntrbs - 1].trb_flags = LE_32(XHCI_TRB_TYPE_STATUS |
803 	    XHCI_TRB_IOC | statusdir);
804 
805 
806 	mutex_enter(&xhcip->xhci_lock);
807 
808 	/*
809 	 * Schedule the transfer, allocating resources in the process.
810 	 */
811 	if (xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE) != 0) {
812 		xhci_transfer_free(xhcip, xt);
813 		mutex_exit(&xhcip->xhci_lock);
814 		return (USB_NO_RESOURCES);
815 	}
816 
817 	mutex_exit(&xhcip->xhci_lock);
818 
819 	return (USB_SUCCESS);
820 }
821 
822 /*
823  * This request is trying to get the upper bound on the amount of data we're
824  * willing transfer in one go. Note that this amount can be broken down into
825  * multiple SGL entries, this interface doesn't particularly care about that.
826  */
827 /* ARGSUSED */
828 static int
xhci_hcdi_bulk_transfer_size(usba_device_t * ud,size_t * sizep)829 xhci_hcdi_bulk_transfer_size(usba_device_t *ud, size_t *sizep)
830 {
831 	if (sizep != NULL)
832 		*sizep = XHCI_MAX_TRANSFER;
833 	return (USB_SUCCESS);
834 }
835 
836 /*
837  * Perform a bulk transfer. This is a pretty straightforward action. We
838  * basically just allocate the appropriate transfer and try to schedule it,
839  * hoping there is enough space.
840  */
841 static int
xhci_hcdi_pipe_bulk_xfer(usba_pipe_handle_data_t * ph,usb_bulk_req_t * ubrp,usb_flags_t usb_flags)842 xhci_hcdi_pipe_bulk_xfer(usba_pipe_handle_data_t *ph, usb_bulk_req_t *ubrp,
843     usb_flags_t usb_flags)
844 {
845 	uint_t epid;
846 	xhci_device_t *xd;
847 	xhci_endpoint_t *xep;
848 	xhci_transfer_t *xt;
849 	boolean_t datain;
850 
851 	xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
852 
853 	mutex_enter(&xhcip->xhci_lock);
854 	if (xhcip->xhci_state & XHCI_S_ERROR) {
855 		mutex_exit(&xhcip->xhci_lock);
856 		return (USB_HC_HARDWARE_ERROR);
857 	}
858 
859 	if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
860 		mutex_exit(&xhcip->xhci_lock);
861 		return (USB_NOT_SUPPORTED);
862 	}
863 
864 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
865 	epid = xhci_endpoint_pipe_to_epid(ph);
866 	if (xd->xd_endpoints[epid] == NULL) {
867 		mutex_exit(&xhcip->xhci_lock);
868 		xhci_error(xhcip, "asked to do control transfer on slot %d, "
869 		    "port %d, endpoint: %d, but no endpoint structure",
870 		    xd->xd_slot, xd->xd_port, epid);
871 		return (USB_FAILURE);
872 	}
873 	xep = xd->xd_endpoints[epid];
874 	mutex_exit(&xhcip->xhci_lock);
875 
876 	if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
877 		datain = B_TRUE;
878 	} else {
879 		datain = B_FALSE;
880 	}
881 
882 	xt = xhci_transfer_alloc(xhcip, xep, ubrp->bulk_len, 0, usb_flags);
883 	if (xt == NULL) {
884 		return (USB_NO_RESOURCES);
885 	}
886 	xt->xt_usba_req = (usb_opaque_t)ubrp;
887 	xt->xt_timeout = ubrp->bulk_timeout;
888 	if (xt->xt_timeout == 0) {
889 		xt->xt_timeout = HCDI_DEFAULT_TIMEOUT;
890 	}
891 
892 	if (ubrp->bulk_len > 0 && datain == B_FALSE) {
893 		xhci_transfer_copy(xt, ubrp->bulk_data->b_rptr, ubrp->bulk_len,
894 		    B_FALSE);
895 		if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORDEV) !=
896 		    DDI_FM_OK) {
897 			xhci_transfer_free(xhcip, xt);
898 			xhci_error(xhcip, "failed to synchronize bulk "
899 			    "transfer DMA memory on endpoint %u of "
900 			    "device on slot %d and port %d: resetting "
901 			    "device", xep->xep_num, xd->xd_slot,
902 			    xd->xd_port);
903 			xhci_fm_runtime_reset(xhcip);
904 			return (USB_HC_HARDWARE_ERROR);
905 		}
906 	}
907 
908 	xhci_transfer_trb_fill_data(xep, xt, 0, datain);
909 	mutex_enter(&xhcip->xhci_lock);
910 	if (xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE) != 0) {
911 		xhci_transfer_free(xhcip, xt);
912 		mutex_exit(&xhcip->xhci_lock);
913 		return (USB_NO_RESOURCES);
914 	}
915 	mutex_exit(&xhcip->xhci_lock);
916 
917 	return (USB_SUCCESS);
918 }
919 
920 static void
xhci_hcdi_isoc_transfer_fill(xhci_device_t * xd,xhci_endpoint_t * xep,xhci_transfer_t * xt,usb_isoc_req_t * usrp)921 xhci_hcdi_isoc_transfer_fill(xhci_device_t *xd, xhci_endpoint_t *xep,
922     xhci_transfer_t *xt, usb_isoc_req_t *usrp)
923 {
924 	int i;
925 	uintptr_t buf;
926 
927 	buf = xt->xt_buffer.xdb_cookies[0].dmac_laddress;
928 	for (i = 0; i < usrp->isoc_pkts_count; i++) {
929 		int flags;
930 		uint_t tbc, tlbpc;
931 
932 		ushort_t len = usrp->isoc_pkt_descr[i].isoc_pkt_length;
933 		xhci_trb_t *trb = &xt->xt_trbs[i];
934 
935 		trb->trb_addr = LE_64(buf);
936 
937 		/*
938 		 * Because we know that a single frame can have all of its data
939 		 * in a single instance, we know that we don't need to do
940 		 * anything special here.
941 		 */
942 		trb->trb_status = LE_32(XHCI_TRB_LEN(len) | XHCI_TRB_TDREM(0) |
943 		    XHCI_TRB_INTR(0));
944 
945 		/*
946 		 * Always enable SIA to start the frame ASAP. We also always
947 		 * enable an interrupt on a short packet. If this is the last
948 		 * trb, then we will set IOC. Each TRB created here is really
949 		 * its own TD. However, we only set an interrupt on the last
950 		 * entry to better deal with scheduling.
951 		 */
952 		flags = XHCI_TRB_SIA | XHCI_TRB_ISP | XHCI_TRB_SET_FRAME(0);
953 		flags |= XHCI_TRB_TYPE_ISOCH;
954 
955 		if (i + 1 == usrp->isoc_pkts_count)
956 			flags |= XHCI_TRB_IOC;
957 
958 		/*
959 		 * Now we need to calculate the TBC and the TLBPC.
960 		 */
961 		xhci_transfer_calculate_isoc(xd, xep, len, &tbc, &tlbpc);
962 		flags |= XHCI_TRB_SET_TBC(tbc);
963 		flags |= XHCI_TRB_SET_TLBPC(tlbpc);
964 
965 		trb->trb_flags = LE_32(flags);
966 		buf += len;
967 
968 		/*
969 		 * Go through and copy the required data to our local copy of
970 		 * the isoc descriptor. By default, we assume that all data will
971 		 * be copied and the status set to OK. This mirrors the fact
972 		 * that we won't get a notification unless there's been an
973 		 * error or short packet transfer.
974 		 */
975 		xt->xt_isoc[i].isoc_pkt_length = len;
976 		xt->xt_isoc[i].isoc_pkt_actual_length = len;
977 		xt->xt_isoc[i].isoc_pkt_status = USB_CR_OK;
978 	}
979 }
980 
981 /*
982  * Initialize periodic IN requests (both interrupt and isochronous)
983  */
984 static int
xhci_hcdi_periodic_init(xhci_t * xhcip,usba_pipe_handle_data_t * ph,usb_opaque_t usb_req,size_t len,int usb_flags)985 xhci_hcdi_periodic_init(xhci_t *xhcip, usba_pipe_handle_data_t *ph,
986     usb_opaque_t usb_req, size_t len, int usb_flags)
987 {
988 	int i, ret;
989 	uint_t epid;
990 	xhci_device_t *xd;
991 	xhci_endpoint_t *xep;
992 	xhci_pipe_t *xp;
993 	xhci_periodic_pipe_t *xpp;
994 
995 	mutex_enter(&xhcip->xhci_lock);
996 	if (xhcip->xhci_state & XHCI_S_ERROR) {
997 		mutex_exit(&xhcip->xhci_lock);
998 		return (USB_HC_HARDWARE_ERROR);
999 	}
1000 
1001 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
1002 	epid = xhci_endpoint_pipe_to_epid(ph);
1003 	if (xd->xd_endpoints[epid] == NULL) {
1004 		xhci_error(xhcip, "asked to do periodic transfer on slot %d, "
1005 		    "port %d, endpoint: %d, but no endpoint structure",
1006 		    xd->xd_slot, xd->xd_port, epid);
1007 		mutex_exit(&xhcip->xhci_lock);
1008 		return (USB_FAILURE);
1009 	}
1010 	xep = xd->xd_endpoints[epid];
1011 	xp = (xhci_pipe_t *)ph->p_hcd_private;
1012 	if (xp == NULL) {
1013 		xhci_error(xhcip, "asked to do periodic transfer on slot %d, "
1014 		    "port %d, endpoint: %d, but no pipe structure",
1015 		    xd->xd_slot, xd->xd_port, epid);
1016 		mutex_exit(&xhcip->xhci_lock);
1017 		return (USB_FAILURE);
1018 	}
1019 	xpp = &xp->xp_periodic;
1020 
1021 	/*
1022 	 * Only allow a single polling request at any given time.
1023 	 */
1024 	if (xpp->xpp_usb_req != NULL) {
1025 		mutex_exit(&xhcip->xhci_lock);
1026 		return (USB_BUSY);
1027 	}
1028 
1029 	/*
1030 	 * We keep allocations around in case we restart polling, which most
1031 	 * devices do (not really caring about a lost event). However, we don't
1032 	 * support a driver changing that size on us, which it probably won't.
1033 	 * If we stumble across driver that does, then this will need to become
1034 	 * a lot more complicated.
1035 	 */
1036 	if (xpp->xpp_tsize > 0 && xpp->xpp_tsize < len) {
1037 		mutex_exit(&xhcip->xhci_lock);
1038 		return (USB_INVALID_REQUEST);
1039 	}
1040 
1041 	if (xpp->xpp_tsize == 0) {
1042 		int ntrbs;
1043 		int ntransfers;
1044 
1045 		/*
1046 		 * What we allocate varies based on whether or not this is an
1047 		 * isochronous or interrupt IN periodic.
1048 		 */
1049 		if (xep->xep_type == USB_EP_ATTR_INTR) {
1050 			ntrbs = 0;
1051 			ntransfers = XHCI_INTR_IN_NTRANSFERS;
1052 		} else {
1053 			usb_isoc_req_t *usrp;
1054 			ASSERT(xep->xep_type == USB_EP_ATTR_ISOCH);
1055 
1056 			usrp = (usb_isoc_req_t *)usb_req;
1057 			ntrbs = usrp->isoc_pkts_count;
1058 			ntransfers = XHCI_ISOC_IN_NTRANSFERS;
1059 		}
1060 
1061 		xpp->xpp_tsize = len;
1062 		xpp->xpp_ntransfers = ntransfers;
1063 
1064 		for (i = 0; i < xpp->xpp_ntransfers; i++) {
1065 			xhci_transfer_t *xt = xhci_transfer_alloc(xhcip, xep,
1066 			    len, ntrbs, usb_flags);
1067 			if (xt == NULL) {
1068 				xhci_hcdi_periodic_free(xhcip, xp);
1069 				mutex_exit(&xhcip->xhci_lock);
1070 				return (USB_NO_RESOURCES);
1071 			}
1072 
1073 			if (xep->xep_type == USB_EP_ATTR_INTR) {
1074 				xhci_transfer_trb_fill_data(xep, xt, 0, B_TRUE);
1075 			} else {
1076 				usb_isoc_req_t *usrp;
1077 				usrp = (usb_isoc_req_t *)usb_req;
1078 				xhci_hcdi_isoc_transfer_fill(xd, xep, xt, usrp);
1079 				xt->xt_data_tohost = B_TRUE;
1080 			}
1081 			xpp->xpp_transfers[i] = xt;
1082 		}
1083 	}
1084 
1085 	/*
1086 	 * Mark the endpoint as periodic so we don't have timeouts at play.
1087 	 */
1088 	xep->xep_state |= XHCI_ENDPOINT_PERIODIC;
1089 
1090 	/*
1091 	 * Now that we've allocated everything, go ahead and schedule them and
1092 	 * kick off the ring.
1093 	 */
1094 	for (i = 0; i < xpp->xpp_ntransfers; i++) {
1095 		int ret;
1096 		ret = xhci_endpoint_schedule(xhcip, xd, xep,
1097 		    xpp->xpp_transfers[i], B_FALSE);
1098 		if (ret != 0) {
1099 			(void) xhci_ring_reset(xhcip, &xep->xep_ring);
1100 			xep->xep_state &= ~XHCI_ENDPOINT_PERIODIC;
1101 			mutex_exit(&xhcip->xhci_lock);
1102 			return (ret);
1103 		}
1104 	}
1105 
1106 	/*
1107 	 * Don't worry about freeing memory, it'll be done when the endpoint
1108 	 * closes and the whole system is reset.
1109 	 */
1110 	xpp->xpp_usb_req = usb_req;
1111 	xpp->xpp_poll_state = XHCI_PERIODIC_POLL_ACTIVE;
1112 
1113 	ret = xhci_endpoint_ring(xhcip, xd, xep);
1114 	mutex_exit(&xhcip->xhci_lock);
1115 	return (ret);
1116 }
1117 
1118 static int
xhci_hcdi_intr_oneshot(xhci_t * xhcip,usba_pipe_handle_data_t * ph,usb_intr_req_t * uirp,usb_flags_t usb_flags)1119 xhci_hcdi_intr_oneshot(xhci_t *xhcip, usba_pipe_handle_data_t *ph,
1120     usb_intr_req_t *uirp, usb_flags_t usb_flags)
1121 {
1122 	uint_t epid;
1123 	xhci_device_t *xd;
1124 	xhci_endpoint_t *xep;
1125 	xhci_transfer_t *xt;
1126 	boolean_t datain;
1127 	mblk_t *mp = NULL;
1128 
1129 	mutex_enter(&xhcip->xhci_lock);
1130 	if (xhcip->xhci_state & XHCI_S_ERROR) {
1131 		mutex_exit(&xhcip->xhci_lock);
1132 		return (USB_HC_HARDWARE_ERROR);
1133 	}
1134 
1135 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
1136 	epid = xhci_endpoint_pipe_to_epid(ph);
1137 	if (xd->xd_endpoints[epid] == NULL) {
1138 		xhci_error(xhcip, "asked to do interrupt transfer on slot %d, "
1139 		    "port %d, endpoint: %d, but no endpoint structure",
1140 		    xd->xd_slot, xd->xd_port, epid);
1141 		mutex_exit(&xhcip->xhci_lock);
1142 		return (USB_FAILURE);
1143 	}
1144 	xep = xd->xd_endpoints[epid];
1145 
1146 	mutex_exit(&xhcip->xhci_lock);
1147 
1148 	if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
1149 		datain = B_TRUE;
1150 	} else {
1151 		datain = B_FALSE;
1152 	}
1153 
1154 	xt = xhci_transfer_alloc(xhcip, xep, uirp->intr_len, 0, usb_flags);
1155 	if (xt == NULL) {
1156 		return (USB_NO_RESOURCES);
1157 	}
1158 
1159 	xt->xt_usba_req = (usb_opaque_t)uirp;
1160 	xt->xt_timeout = uirp->intr_timeout;
1161 	if (xt->xt_timeout == 0) {
1162 		xt->xt_timeout = HCDI_DEFAULT_TIMEOUT;
1163 	}
1164 
1165 	/*
1166 	 * Unlike other request types, USB Interrupt-IN requests aren't required
1167 	 * to have allocated the message block for data. If they haven't, we
1168 	 * take care of that now.
1169 	 */
1170 	if (uirp->intr_len > 0 && datain == B_TRUE && uirp->intr_data == NULL) {
1171 		if (usb_flags & USB_FLAGS_SLEEP) {
1172 			mp = allocb_wait(uirp->intr_len, BPRI_LO, STR_NOSIG,
1173 			    NULL);
1174 		} else {
1175 			mp = allocb(uirp->intr_len, 0);
1176 		}
1177 		if (mp == NULL) {
1178 			xhci_transfer_free(xhcip, xt);
1179 			mutex_exit(&xhcip->xhci_lock);
1180 			return (USB_NO_RESOURCES);
1181 		}
1182 		uirp->intr_data = mp;
1183 	}
1184 
1185 	if (uirp->intr_len > 0 && datain == B_FALSE) {
1186 		xhci_transfer_copy(xt, uirp->intr_data->b_rptr, uirp->intr_len,
1187 		    B_FALSE);
1188 		if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORDEV) !=
1189 		    DDI_FM_OK) {
1190 			xhci_transfer_free(xhcip, xt);
1191 			xhci_error(xhcip, "failed to synchronize interrupt "
1192 			    "transfer DMA memory on endpoint %u of "
1193 			    "device on slot %d and port %d: resetting "
1194 			    "device", xep->xep_num, xd->xd_slot,
1195 			    xd->xd_port);
1196 			xhci_fm_runtime_reset(xhcip);
1197 			return (USB_HC_HARDWARE_ERROR);
1198 		}
1199 	}
1200 
1201 	xhci_transfer_trb_fill_data(xep, xt, 0, datain);
1202 	mutex_enter(&xhcip->xhci_lock);
1203 	if (xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE) != 0) {
1204 		if (mp != NULL) {
1205 			uirp->intr_data = NULL;
1206 			freemsg(mp);
1207 		}
1208 		xhci_transfer_free(xhcip, xt);
1209 		mutex_exit(&xhcip->xhci_lock);
1210 		return (USB_NO_RESOURCES);
1211 	}
1212 	mutex_exit(&xhcip->xhci_lock);
1213 
1214 	return (USB_SUCCESS);
1215 }
1216 
1217 /*
1218  * We've been asked to perform an interrupt transfer. When this is an interrupt
1219  * IN endpoint, that means that the hcd is being asked to start polling on the
1220  * endpoint. When the endpoint is the root hub, it effectively becomes synthetic
1221  * polling.
1222  *
1223  * When we have an interrupt out endpoint, then this is just a single simple
1224  * interrupt request that we send out and there isn't much special to do beyond
1225  * the normal activity.
1226  */
1227 static int
xhci_hcdi_pipe_intr_xfer(usba_pipe_handle_data_t * ph,usb_intr_req_t * uirp,usb_flags_t usb_flags)1228 xhci_hcdi_pipe_intr_xfer(usba_pipe_handle_data_t *ph, usb_intr_req_t *uirp,
1229     usb_flags_t usb_flags)
1230 {
1231 	int ret;
1232 	xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
1233 
1234 	if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
1235 		if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
1236 			ret = xhci_root_hub_intr_root_enable(xhcip, ph, uirp);
1237 		} else if (uirp->intr_attributes & USB_ATTRS_ONE_XFER) {
1238 			ret = xhci_hcdi_intr_oneshot(xhcip, ph, uirp,
1239 			    usb_flags);
1240 		} else {
1241 			ret = xhci_hcdi_periodic_init(xhcip, ph,
1242 			    (usb_opaque_t)uirp, uirp->intr_len, usb_flags);
1243 		}
1244 	} else {
1245 		if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
1246 			return (USB_NOT_SUPPORTED);
1247 		}
1248 		ret = xhci_hcdi_intr_oneshot(xhcip, ph, uirp, usb_flags);
1249 	}
1250 
1251 	return (ret);
1252 }
1253 
1254 /* ARGSUSED */
1255 static int
xhci_hcdi_pipe_stop_intr_polling(usba_pipe_handle_data_t * ph,usb_flags_t usb_flags)1256 xhci_hcdi_pipe_stop_intr_polling(usba_pipe_handle_data_t *ph,
1257     usb_flags_t usb_flags)
1258 {
1259 	return (xhci_hcdi_pipe_poll_fini(ph, B_FALSE));
1260 }
1261 
1262 static int
xhci_hcdi_isoc_periodic(xhci_t * xhcip,usba_pipe_handle_data_t * ph,usb_isoc_req_t * usrp,usb_flags_t usb_flags)1263 xhci_hcdi_isoc_periodic(xhci_t *xhcip, usba_pipe_handle_data_t *ph,
1264     usb_isoc_req_t *usrp, usb_flags_t usb_flags)
1265 {
1266 	int i;
1267 	size_t count;
1268 
1269 	count = 0;
1270 	for (i = 0; i < usrp->isoc_pkts_count; i++) {
1271 		count += usrp->isoc_pkt_descr[i].isoc_pkt_length;
1272 	}
1273 
1274 	return (xhci_hcdi_periodic_init(xhcip, ph, (usb_opaque_t)usrp, count,
1275 	    usb_flags));
1276 }
1277 
1278 /*
1279  * This is used to create an isochronous request to send data out to the device.
1280  * This is a single one shot request, it is not something that we'll have to
1281  * repeat over and over.
1282  */
1283 static int
xhci_hcdi_isoc_oneshot(xhci_t * xhcip,usba_pipe_handle_data_t * ph,usb_isoc_req_t * usrp,usb_flags_t usb_flags)1284 xhci_hcdi_isoc_oneshot(xhci_t *xhcip, usba_pipe_handle_data_t *ph,
1285     usb_isoc_req_t *usrp, usb_flags_t usb_flags)
1286 {
1287 	int i, ret;
1288 	uint_t epid;
1289 	size_t count, mblen;
1290 	xhci_device_t *xd;
1291 	xhci_endpoint_t *xep;
1292 	xhci_transfer_t *xt;
1293 
1294 	count = 0;
1295 	for (i = 0; i < usrp->isoc_pkts_count; i++) {
1296 		count += usrp->isoc_pkt_descr[i].isoc_pkt_length;
1297 	}
1298 	mblen = MBLKL(usrp->isoc_data);
1299 
1300 	if (count != mblen) {
1301 		return (USB_INVALID_ARGS);
1302 	}
1303 
1304 	mutex_enter(&xhcip->xhci_lock);
1305 	if (xhcip->xhci_state & XHCI_S_ERROR) {
1306 		mutex_exit(&xhcip->xhci_lock);
1307 		return (USB_HC_HARDWARE_ERROR);
1308 	}
1309 
1310 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
1311 	epid = xhci_endpoint_pipe_to_epid(ph);
1312 	if (xd->xd_endpoints[epid] == NULL) {
1313 		xhci_error(xhcip, "asked to do isochronous transfer on slot "
1314 		    "%d, port %d, endpoint: %d, but no endpoint structure",
1315 		    xd->xd_slot, xd->xd_port, epid);
1316 		mutex_exit(&xhcip->xhci_lock);
1317 		return (USB_FAILURE);
1318 	}
1319 	xep = xd->xd_endpoints[epid];
1320 	mutex_exit(&xhcip->xhci_lock);
1321 
1322 	xt = xhci_transfer_alloc(xhcip, xep, mblen, usrp->isoc_pkts_count,
1323 	    usb_flags);
1324 	if (xt == NULL) {
1325 		return (USB_NO_RESOURCES);
1326 	}
1327 	xt->xt_usba_req = (usb_opaque_t)usrp;
1328 
1329 	/*
1330 	 * USBA doesn't provide any real way for a timeout to be defined for an
1331 	 * isochronous event. However, since we technically aren't a periodic
1332 	 * endpoint, go ahead and always set the default timeout. It's better
1333 	 * than nothing.
1334 	 */
1335 	xt->xt_timeout = HCDI_DEFAULT_TIMEOUT;
1336 
1337 	xhci_transfer_copy(xt, usrp->isoc_data->b_rptr, mblen, B_FALSE);
1338 	if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORDEV) != DDI_FM_OK) {
1339 		xhci_transfer_free(xhcip, xt);
1340 		xhci_error(xhcip, "failed to synchronize isochronous "
1341 		    "transfer DMA memory on endpoint %u of "
1342 		    "device on slot %d and port %d: resetting "
1343 		    "device", xep->xep_num, xd->xd_slot,
1344 		    xd->xd_port);
1345 		xhci_fm_runtime_reset(xhcip);
1346 		return (USB_HC_HARDWARE_ERROR);
1347 	}
1348 
1349 	/*
1350 	 * Fill in the ISOC data. Note, that we always use ASAP scheduling and
1351 	 * we don't support specifying the frame at this time, for better or
1352 	 * worse.
1353 	 */
1354 	xhci_hcdi_isoc_transfer_fill(xd, xep, xt, usrp);
1355 
1356 	mutex_enter(&xhcip->xhci_lock);
1357 	ret = xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE);
1358 	mutex_exit(&xhcip->xhci_lock);
1359 
1360 	return (ret);
1361 }
1362 
1363 static int
xhci_hcdi_pipe_isoc_xfer(usba_pipe_handle_data_t * ph,usb_isoc_req_t * usrp,usb_flags_t usb_flags)1364 xhci_hcdi_pipe_isoc_xfer(usba_pipe_handle_data_t *ph, usb_isoc_req_t *usrp,
1365     usb_flags_t usb_flags)
1366 {
1367 	int ret;
1368 	xhci_t *xhcip;
1369 
1370 	xhcip = xhci_hcdi_get_xhcip(ph);
1371 
1372 	/*
1373 	 * We don't support isochronous transactions on the root hub at all.
1374 	 * Always fail them if for some reason we end up here.
1375 	 */
1376 	if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
1377 		return (USB_NOT_SUPPORTED);
1378 	}
1379 
1380 	/*
1381 	 * We do not support being asked to set the frame ID at this time. We
1382 	 * require that everything specify the attribute
1383 	 * USB_ATTRS_ISOC_XFER_ASAP.
1384 	 */
1385 	if (!(usrp->isoc_attributes & USB_ATTRS_ISOC_XFER_ASAP)) {
1386 		return (USB_NOT_SUPPORTED);
1387 	}
1388 
1389 	if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
1390 		/*
1391 		 * Note, there is no such thing as a non-periodic isochronous
1392 		 * incoming transfer.
1393 		 */
1394 		ret = xhci_hcdi_isoc_periodic(xhcip, ph, usrp, usb_flags);
1395 	} else {
1396 		ret = xhci_hcdi_isoc_oneshot(xhcip, ph, usrp, usb_flags);
1397 	}
1398 
1399 	return (ret);
1400 }
1401 
1402 /* ARGSUSED */
1403 static int
xhci_hcdi_pipe_stop_isoc_polling(usba_pipe_handle_data_t * ph,usb_flags_t usb_flags)1404 xhci_hcdi_pipe_stop_isoc_polling(usba_pipe_handle_data_t *ph,
1405     usb_flags_t usb_flags)
1406 {
1407 	return (xhci_hcdi_pipe_poll_fini(ph, B_FALSE));
1408 }
1409 
1410 /*
1411  * This is asking us for the current frame number. The USBA expects this to
1412  * actually be a bit of a fiction, as it tries to maintain a frame number well
1413  * beyond what the hardware actually contains in its registers. Hardware
1414  * basically has a 14-bit counter, whereas we need to have a constant amount of
1415  * milliseconds.
1416  *
1417  * Today, no client drivers actually use this API and everyone specifies the
1418  * attribute to say that we should schedule things ASAP. So until we have some
1419  * real device that want this functionality, we're going to fail.
1420  */
1421 /* ARGSUSED */
1422 static int
xhci_hcdi_get_current_frame_number(usba_device_t * usba_device,usb_frame_number_t * frame_number)1423 xhci_hcdi_get_current_frame_number(usba_device_t *usba_device,
1424     usb_frame_number_t *frame_number)
1425 {
1426 	return (USB_FAILURE);
1427 }
1428 
1429 /*
1430  * See the comments around the XHCI_ISOC_MAX_TRB macro for more information.
1431  */
1432 /* ARGSUSED */
1433 static int
xhci_hcdi_get_max_isoc_pkts(usba_device_t * usba_device,uint_t * max_isoc_pkts_per_request)1434 xhci_hcdi_get_max_isoc_pkts(usba_device_t *usba_device,
1435     uint_t *max_isoc_pkts_per_request)
1436 {
1437 	*max_isoc_pkts_per_request = XHCI_ISOC_MAX_TRB;
1438 	return (USB_SUCCESS);
1439 }
1440 
1441 /*
1442  * VERSION 2 ops and helpers
1443  */
1444 
1445 static void
xhci_hcdi_device_free(xhci_device_t * xd)1446 xhci_hcdi_device_free(xhci_device_t *xd)
1447 {
1448 	xhci_dma_free(&xd->xd_ictx);
1449 	xhci_dma_free(&xd->xd_octx);
1450 	mutex_destroy(&xd->xd_imtx);
1451 	kmem_free(xd, sizeof (xhci_device_t));
1452 }
1453 
1454 /*
1455  * Calculate the device's route string. In USB 3.0 the route string is a 20-bit
1456  * number. Each four bits represent a port number attached to a deeper hub.
1457  * Particularly it represents the port on that current hub that you need to go
1458  * down to reach the next device. Bits 0-3 represent the first *external* hub.
1459  * So a device connected to a root hub has a route string of zero. Imagine the
1460  * following set of devices:
1461  *
1462  *               . port 2      . port 5
1463  *               .             .
1464  *  +----------+ .  +--------+ .  +-------+
1465  *  | root hub |-*->| hub 1  |-*->| hub 2 |
1466  *  +----------+    +--------+    +-------+
1467  *       * . port 12    * . port 8    * . port 1
1468  *       v              v             v
1469  *   +-------+      +-------+     +-------+
1470  *   | dev a |      | dev b |     | dev c |
1471  *   +-------+      +-------+     +-------+
1472  *
1473  * So, based on the above diagram, device a should have a route string of 0,
1474  * because it's directly connected to the root port. Device b would simply have
1475  * a route string of 8. This is because it travels through one non-root hub, hub
1476  * 1, and it does so on port 8. The root ports basically don't matter. Device c
1477  * would then have a route string of 0x15, as it's first traversing through hub
1478  * 1 on port 2 and then hub 2 on port 5.
1479  *
1480  * Finally, it's worth mentioning that because it's a four bit field, if for
1481  * some reason a device has more than 15 ports, we just treat the value as 15.
1482  *
1483  * Note, as part of this, we also grab what port on the root hub this whole
1484  * chain is on, as we're required to store that information in the slot context.
1485  */
1486 static void
xhci_hcdi_device_route(usba_device_t * ud,uint32_t * routep,uint32_t * root_port)1487 xhci_hcdi_device_route(usba_device_t *ud, uint32_t *routep, uint32_t *root_port)
1488 {
1489 	uint32_t route = 0;
1490 	usba_device_t *hub = ud->usb_parent_hub;
1491 	usba_device_t *port_dev = ud;
1492 
1493 	ASSERT(hub != NULL);
1494 
1495 	/*
1496 	 * Iterate over every hub, updating the route as we go. When we
1497 	 * encounter a hub without a parent, then we're at the root hub. At
1498 	 * which point, the port we want is on port_dev (the child of hub).
1499 	 */
1500 	while (hub->usb_parent_hub != NULL) {
1501 		uint32_t p;
1502 
1503 		p = port_dev->usb_port;
1504 		if (p > 15)
1505 			p = 15;
1506 		route <<= 4;
1507 		route |= p & 0xf;
1508 		port_dev = hub;
1509 		hub = hub->usb_parent_hub;
1510 	}
1511 
1512 	ASSERT(port_dev->usb_parent_hub == hub);
1513 	*root_port = port_dev->usb_port;
1514 	*routep = XHCI_ROUTE_MASK(route);
1515 }
1516 
1517 /*
1518  * If a low or full speed device is behind a high-speed device that is not a
1519  * root hub, then we must include the port and slot of that device. USBA already
1520  * stores this device in the usb_hs_hub_usba_dev member.
1521  */
1522 static uint32_t
xhci_hcdi_device_tt(usba_device_t * ud)1523 xhci_hcdi_device_tt(usba_device_t *ud)
1524 {
1525 	uint32_t ret;
1526 	xhci_device_t *xd;
1527 
1528 	if (ud->usb_port_status >= USBA_HIGH_SPEED_DEV)
1529 		return (0);
1530 
1531 	if (ud->usb_hs_hub_usba_dev == NULL)
1532 		return (0);
1533 
1534 	ASSERT(ud->usb_hs_hub_usba_dev != NULL);
1535 	ASSERT(ud->usb_hs_hub_usba_dev->usb_parent_hub != NULL);
1536 	xd = usba_hcdi_get_device_private(ud->usb_hs_hub_usba_dev);
1537 	ASSERT(xd != NULL);
1538 
1539 	ret = XHCI_SCTX_SET_TT_HUB_SID(xd->xd_slot);
1540 	ret |= XHCI_SCTX_SET_TT_PORT_NUM(ud->usb_hs_hub_usba_dev->usb_port);
1541 
1542 	return (ret);
1543 }
1544 
1545 /*
1546  * Initialize a new device. This allocates a device slot from the controller,
1547  * which tranfers it to our control.
1548  */
1549 static int
xhci_hcdi_device_init(usba_device_t * ud,usb_port_t port,void ** hcdpp)1550 xhci_hcdi_device_init(usba_device_t *ud, usb_port_t port, void **hcdpp)
1551 {
1552 	int ret, i;
1553 	xhci_device_t *xd;
1554 	ddi_device_acc_attr_t acc;
1555 	ddi_dma_attr_t attr;
1556 	xhci_t *xhcip = xhci_hcdi_get_xhcip_from_dev(ud);
1557 	size_t isize, osize, incr;
1558 	uint32_t route, rp, info, info2, tt;
1559 
1560 	xd = kmem_zalloc(sizeof (xhci_device_t), KM_SLEEP);
1561 	xd->xd_port = port;
1562 	xd->xd_usbdev = ud;
1563 	mutex_init(&xd->xd_imtx, NULL, MUTEX_DRIVER,
1564 	    (void *)(uintptr_t)xhcip->xhci_intr_pri);
1565 
1566 	/*
1567 	 * The size of the context structures is based upon the presence of the
1568 	 * context flag which determines whether we have a 32-byte or 64-byte
1569 	 * context. Note that the input context always has to account for the
1570 	 * entire size of the xhci_input_contex_t, which is 32-bytes by default.
1571 	 */
1572 	if (xhcip->xhci_caps.xcap_flags & XCAP_CSZ) {
1573 		incr = 64;
1574 		osize = XHCI_DEVICE_CONTEXT_64;
1575 		isize = XHCI_DEVICE_CONTEXT_64 + incr;
1576 	} else {
1577 		incr = 32;
1578 		osize = XHCI_DEVICE_CONTEXT_32;
1579 		isize = XHCI_DEVICE_CONTEXT_32 + incr;
1580 	}
1581 
1582 	xhci_dma_acc_attr(xhcip, &acc);
1583 	xhci_dma_dma_attr(xhcip, &attr);
1584 	if (xhci_dma_alloc(xhcip, &xd->xd_ictx, &attr, &acc, B_TRUE,
1585 	    isize, B_FALSE) == B_FALSE) {
1586 		xhci_hcdi_device_free(xd);
1587 		return (USB_NO_RESOURCES);
1588 	}
1589 
1590 	xd->xd_input = (xhci_input_context_t *)xd->xd_ictx.xdb_va;
1591 	xd->xd_slotin = (xhci_slot_context_t *)(xd->xd_ictx.xdb_va + incr);
1592 	for (i = 0; i < XHCI_NUM_ENDPOINTS; i++) {
1593 		xd->xd_endin[i] =
1594 		    (xhci_endpoint_context_t *)(xd->xd_ictx.xdb_va +
1595 		    (i + 2) * incr);
1596 	}
1597 
1598 	if (xhci_dma_alloc(xhcip, &xd->xd_octx, &attr, &acc, B_TRUE,
1599 	    osize, B_FALSE) == B_FALSE) {
1600 		xhci_hcdi_device_free(xd);
1601 		return (USB_NO_RESOURCES);
1602 	}
1603 	xd->xd_slotout = (xhci_slot_context_t *)xd->xd_octx.xdb_va;
1604 	for (i = 0; i < XHCI_NUM_ENDPOINTS; i++) {
1605 		xd->xd_endout[i] =
1606 		    (xhci_endpoint_context_t *)(xd->xd_octx.xdb_va +
1607 		    (i + 1) * incr);
1608 	}
1609 
1610 	ret = xhci_command_enable_slot(xhcip, &xd->xd_slot);
1611 	if (ret != USB_SUCCESS) {
1612 		xhci_hcdi_device_free(xd);
1613 		return (ret);
1614 	}
1615 
1616 	/*
1617 	 * These are the default slot context and the endpoint zero context that
1618 	 * we're enabling. See 4.3.3.
1619 	 */
1620 	xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0) |
1621 	    XHCI_INCTX_MASK_DCI(1));
1622 
1623 	/*
1624 	 * Note, we never need to set the MTT bit as illumos never enables the
1625 	 * alternate MTT interface.
1626 	 */
1627 	xhci_hcdi_device_route(ud, &route, &rp);
1628 	info = XHCI_SCTX_SET_ROUTE(route) | XHCI_SCTX_SET_DCI(1);
1629 	switch (ud->usb_port_status) {
1630 	case USBA_LOW_SPEED_DEV:
1631 		info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_LOW);
1632 		break;
1633 	case USBA_HIGH_SPEED_DEV:
1634 		info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_HIGH);
1635 		break;
1636 	case USBA_FULL_SPEED_DEV:
1637 		info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_FULL);
1638 		break;
1639 	case USBA_SUPER_SPEED_DEV:
1640 	default:
1641 		info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_SUPER);
1642 		break;
1643 	}
1644 	info2 = XHCI_SCTX_SET_RHPORT(rp);
1645 	tt = XHCI_SCTX_SET_IRQ_TARGET(0);
1646 	tt |= xhci_hcdi_device_tt(ud);
1647 
1648 	xd->xd_slotin->xsc_info = LE_32(info);
1649 	xd->xd_slotin->xsc_info2 = LE_32(info2);
1650 	xd->xd_slotin->xsc_tt = LE_32(tt);
1651 
1652 	if ((ret = xhci_endpoint_init(xhcip, xd, NULL)) != 0) {
1653 		(void) xhci_command_disable_slot(xhcip, xd->xd_slot);
1654 		xhci_hcdi_device_free(xd);
1655 		return (USB_HC_HARDWARE_ERROR);
1656 	}
1657 
1658 	if (xhci_context_slot_output_init(xhcip, xd) != B_TRUE) {
1659 		(void) xhci_command_disable_slot(xhcip, xd->xd_slot);
1660 		xhci_endpoint_fini(xd, 0);
1661 		xhci_hcdi_device_free(xd);
1662 		return (USB_HC_HARDWARE_ERROR);
1663 	}
1664 
1665 	if ((ret = xhci_command_set_address(xhcip, xd, B_TRUE)) != 0) {
1666 		(void) xhci_command_disable_slot(xhcip, xd->xd_slot);
1667 		xhci_context_slot_output_fini(xhcip, xd);
1668 		xhci_endpoint_fini(xd, 0);
1669 		xhci_hcdi_device_free(xd);
1670 		return (ret);
1671 	}
1672 
1673 	mutex_enter(&xhcip->xhci_lock);
1674 	list_insert_tail(&xhcip->xhci_usba.xa_devices, xd);
1675 	mutex_exit(&xhcip->xhci_lock);
1676 
1677 	*hcdpp = xd;
1678 	return (ret);
1679 }
1680 
1681 /*
1682  * We're tearing down a device now. That means that the only endpoint context
1683  * that's still valid would be endpoint zero.
1684  */
1685 static void
xhci_hcdi_device_fini(usba_device_t * ud,void * hcdp)1686 xhci_hcdi_device_fini(usba_device_t *ud, void *hcdp)
1687 {
1688 	int ret;
1689 	xhci_endpoint_t *xep;
1690 	xhci_device_t *xd;
1691 	xhci_t *xhcip;
1692 
1693 	/*
1694 	 * Right now, it's theoretically possible that USBA may try and call
1695 	 * us here even if we hadn't successfully finished the device_init()
1696 	 * endpoint. We should probably modify the USBA to make sure that this
1697 	 * can't happen.
1698 	 */
1699 	if (hcdp == NULL)
1700 		return;
1701 
1702 	xd = hcdp;
1703 	xhcip = xhci_hcdi_get_xhcip_from_dev(ud);
1704 
1705 	/*
1706 	 * Make sure we have no timeout running on the default endpoint still.
1707 	 */
1708 	xep = xd->xd_endpoints[XHCI_DEFAULT_ENDPOINT];
1709 	mutex_enter(&xhcip->xhci_lock);
1710 	xep->xep_state |= XHCI_ENDPOINT_TEARDOWN;
1711 	mutex_exit(&xhcip->xhci_lock);
1712 	(void) untimeout(xep->xep_timeout);
1713 
1714 	/*
1715 	 * Go ahead and disable the slot. There's no reason to do anything
1716 	 * special about the default endpoint as it will be disabled as a part
1717 	 * of the slot disabling. However, if this all fails, we'll leave this
1718 	 * sitting here in a failed state, eating up a device slot. It is
1719 	 * unlikely this will occur.
1720 	 */
1721 	ret = xhci_command_disable_slot(xhcip, xd->xd_slot);
1722 	if (ret != USB_SUCCESS) {
1723 		xhci_error(xhcip, "failed to disable slot %d: %d",
1724 		    xd->xd_slot, ret);
1725 		return;
1726 	}
1727 
1728 	xhci_context_slot_output_fini(xhcip, xd);
1729 	xhci_endpoint_fini(xd, XHCI_DEFAULT_ENDPOINT);
1730 
1731 	mutex_enter(&xhcip->xhci_lock);
1732 	list_remove(&xhcip->xhci_usba.xa_devices, xd);
1733 	mutex_exit(&xhcip->xhci_lock);
1734 
1735 	xhci_hcdi_device_free(xd);
1736 }
1737 
1738 /*
1739  * Synchronously attempt to set the device address. For xHCI this involves it
1740  * deciding what address to use.
1741  */
1742 static int
xhci_hcdi_device_address(usba_device_t * ud)1743 xhci_hcdi_device_address(usba_device_t *ud)
1744 {
1745 	int ret;
1746 	xhci_t *xhcip = xhci_hcdi_get_xhcip_from_dev(ud);
1747 	xhci_device_t *xd = usba_hcdi_get_device_private(ud);
1748 	xhci_endpoint_t *xep;
1749 
1750 	mutex_enter(&xhcip->xhci_lock);
1751 
1752 	/*
1753 	 * This device may already be addressed from the perspective of the xhci
1754 	 * controller. For example, the device this represents may have been
1755 	 * unconfigured, which does not actually remove the slot or other
1756 	 * information, merely tears down all the active use of it and the child
1757 	 * driver. In such cases, if we're already addressed, just return
1758 	 * success. The actual USB address is a fiction for USBA anyways.
1759 	 */
1760 	if (xd->xd_addressed == B_TRUE) {
1761 		mutex_exit(&xhcip->xhci_lock);
1762 		return (USB_SUCCESS);
1763 	}
1764 
1765 	ASSERT(xd->xd_addressed == B_FALSE);
1766 	xd->xd_addressed = B_TRUE;
1767 	VERIFY3P(xd->xd_endpoints[XHCI_DEFAULT_ENDPOINT], !=, NULL);
1768 	xep = xd->xd_endpoints[XHCI_DEFAULT_ENDPOINT];
1769 	mutex_exit(&xhcip->xhci_lock);
1770 
1771 	if ((ret = xhci_endpoint_setup_default_context(xhcip, xd, xep)) != 0) {
1772 		ASSERT(ret == EIO);
1773 		return (USB_HC_HARDWARE_ERROR);
1774 	}
1775 
1776 	ret = xhci_command_set_address(xhcip, xd, B_FALSE);
1777 
1778 	if (ret != USB_SUCCESS) {
1779 		mutex_enter(&xhcip->xhci_lock);
1780 		xd->xd_addressed = B_FALSE;
1781 		mutex_exit(&xhcip->xhci_lock);
1782 	}
1783 
1784 	return (ret);
1785 }
1786 
1787 /*
1788  * This is called relatively early on in a hub's life time. At this point, it's
1789  * descriptors have all been pulled and the default control pipe is still open.
1790  * What we need to do is go through and update the slot context to indicate that
1791  * this is a hub, otherwise, the controller will never let us speak to
1792  * downstream ports.
1793  */
1794 static int
xhci_hcdi_hub_update(usba_device_t * ud,uint8_t nports,uint8_t tt)1795 xhci_hcdi_hub_update(usba_device_t *ud, uint8_t nports, uint8_t tt)
1796 {
1797 	int ret;
1798 	xhci_t *xhcip = xhci_hcdi_get_xhcip_from_dev(ud);
1799 	xhci_device_t *xd = usba_hcdi_get_device_private(ud);
1800 
1801 	if (xd == NULL)
1802 		return (USB_FAILURE);
1803 
1804 	if (ud->usb_hubdi == NULL) {
1805 		return (USB_FAILURE);
1806 	}
1807 
1808 	mutex_enter(&xd->xd_imtx);
1809 
1810 	/*
1811 	 * Note, that usba never sets the interface of a hub to Multi TT. Hence
1812 	 * why we're never setting the MTT bit in xsc_info.
1813 	 */
1814 	xd->xd_slotin->xsc_info |= LE_32(XHCI_SCTX_SET_HUB(1));
1815 	xd->xd_slotin->xsc_info2 |= LE_32(XHCI_SCTX_SET_NPORTS(nports));
1816 	if (ud->usb_port_status == USBA_HIGH_SPEED_DEV)
1817 		xd->xd_slotin->xsc_tt |= LE_32(XHCI_SCTX_SET_TT_THINK_TIME(tt));
1818 
1819 	/*
1820 	 * We're only updating the slot context, no endpoint contexts should be
1821 	 * touched.
1822 	 */
1823 	xd->xd_input->xic_drop_flags = LE_32(0);
1824 	xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0));
1825 
1826 	ret = xhci_command_evaluate_context(xhcip, xd);
1827 	mutex_exit(&xd->xd_imtx);
1828 	return (ret);
1829 }
1830 
1831 void
xhci_hcd_fini(xhci_t * xhcip)1832 xhci_hcd_fini(xhci_t *xhcip)
1833 {
1834 	usba_hcdi_unregister(xhcip->xhci_dip);
1835 	usba_free_hcdi_ops(xhcip->xhci_usba.xa_ops);
1836 	list_destroy(&xhcip->xhci_usba.xa_pipes);
1837 	list_destroy(&xhcip->xhci_usba.xa_devices);
1838 }
1839 
1840 int
xhci_hcd_init(xhci_t * xhcip)1841 xhci_hcd_init(xhci_t *xhcip)
1842 {
1843 	usba_hcdi_register_args_t hreg;
1844 	usba_hcdi_ops_t *ops;
1845 
1846 	ops = usba_alloc_hcdi_ops();
1847 	VERIFY(ops != NULL);
1848 
1849 	ops->usba_hcdi_ops_version = HCDI_OPS_VERSION;
1850 	ops->usba_hcdi_dip = xhcip->xhci_dip;
1851 
1852 	ops->usba_hcdi_pm_support = xhci_hcdi_pm_support;
1853 	ops->usba_hcdi_pipe_open = xhci_hcdi_pipe_open;
1854 	ops->usba_hcdi_pipe_close = xhci_hcdi_pipe_close;
1855 	ops->usba_hcdi_pipe_reset = xhci_hcdi_pipe_reset;
1856 	ops->usba_hcdi_pipe_reset_data_toggle =
1857 	    xhci_hcdi_pipe_reset_data_toggle;
1858 	ops->usba_hcdi_pipe_ctrl_xfer = xhci_hcdi_pipe_ctrl_xfer;
1859 	ops->usba_hcdi_bulk_transfer_size = xhci_hcdi_bulk_transfer_size;
1860 	ops->usba_hcdi_pipe_bulk_xfer = xhci_hcdi_pipe_bulk_xfer;
1861 	ops->usba_hcdi_pipe_intr_xfer = xhci_hcdi_pipe_intr_xfer;
1862 	ops->usba_hcdi_pipe_stop_intr_polling =
1863 	    xhci_hcdi_pipe_stop_intr_polling;
1864 	ops->usba_hcdi_pipe_isoc_xfer = xhci_hcdi_pipe_isoc_xfer;
1865 	ops->usba_hcdi_pipe_stop_isoc_polling =
1866 	    xhci_hcdi_pipe_stop_isoc_polling;
1867 	ops->usba_hcdi_get_current_frame_number =
1868 	    xhci_hcdi_get_current_frame_number;
1869 	ops->usba_hcdi_get_max_isoc_pkts = xhci_hcdi_get_max_isoc_pkts;
1870 	ops->usba_hcdi_console_input_init = xhci_hcdi_console_input_init;
1871 	ops->usba_hcdi_console_input_fini = xhci_hcdi_console_input_fini;
1872 	ops->usba_hcdi_console_input_enter = xhci_hcdi_console_input_enter;
1873 	ops->usba_hcdi_console_read = xhci_hcdi_console_read;
1874 	ops->usba_hcdi_console_input_exit = xhci_hcdi_console_input_exit;
1875 
1876 	ops->usba_hcdi_console_output_init = xhci_hcdi_console_output_init;
1877 	ops->usba_hcdi_console_output_fini = xhci_hcdi_console_output_fini;
1878 	ops->usba_hcdi_console_output_enter = xhci_hcdi_console_output_enter;
1879 	ops->usba_hcdi_console_write = xhci_hcdi_console_write;
1880 	ops->usba_hcdi_console_output_exit = xhci_hcdi_console_output_exit;
1881 
1882 	ops->usba_hcdi_device_init = xhci_hcdi_device_init;
1883 	ops->usba_hcdi_device_fini = xhci_hcdi_device_fini;
1884 	ops->usba_hcdi_device_address = xhci_hcdi_device_address;
1885 	ops->usba_hcdi_hub_update = xhci_hcdi_hub_update;
1886 
1887 	hreg.usba_hcdi_register_version = HCDI_REGISTER_VERSION;
1888 	hreg.usba_hcdi_register_dip = xhcip->xhci_dip;
1889 	hreg.usba_hcdi_register_ops = ops;
1890 
1891 	/*
1892 	 * We're required to give xhci a set of DMA attributes that it may loan
1893 	 * out to other devices. Therefore we'll be conservative with what we
1894 	 * end up giving it.
1895 	 */
1896 	xhci_dma_dma_attr(xhcip, &xhcip->xhci_usba.xa_dma_attr);
1897 	hreg.usba_hcdi_register_dma_attr = &xhcip->xhci_usba.xa_dma_attr;
1898 
1899 	hreg.usba_hcdi_register_iblock_cookie =
1900 	    (ddi_iblock_cookie_t)(uintptr_t)xhcip->xhci_intr_pri;
1901 
1902 	if (usba_hcdi_register(&hreg, 0) != DDI_SUCCESS) {
1903 		usba_free_hcdi_ops(ops);
1904 		return (DDI_FAILURE);
1905 	}
1906 
1907 	xhcip->xhci_usba.xa_ops = ops;
1908 
1909 	list_create(&xhcip->xhci_usba.xa_devices, sizeof (xhci_device_t),
1910 	    offsetof(xhci_device_t, xd_link));
1911 	list_create(&xhcip->xhci_usba.xa_pipes, sizeof (xhci_pipe_t),
1912 	    offsetof(xhci_pipe_t, xp_link));
1913 
1914 
1915 	return (DDI_SUCCESS);
1916 }
1917