1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright (c) 2018, Joyent, Inc.
14 * Copyright (c) 2019 by Western Digital Corporation
15 * Copyright 2022 Oxide Computer Company
16 */
17
18 /*
19 * illumos USB framework endpoints and functions for xHCI.
20 *
21 * Please see the big theory statement in xhci.c for more information.
22 */
23
24 #include <sys/usb/hcd/xhci/xhci.h>
25 #include <sys/sysmacros.h>
26 #include <sys/strsun.h>
27 #include <sys/strsubr.h>
28
29 xhci_t *
xhci_hcdi_get_xhcip_from_dev(usba_device_t * ud)30 xhci_hcdi_get_xhcip_from_dev(usba_device_t *ud)
31 {
32 dev_info_t *dip = ud->usb_root_hub_dip;
33 xhci_t *xhcip = ddi_get_soft_state(xhci_soft_state,
34 ddi_get_instance(dip));
35 VERIFY(xhcip != NULL);
36 return (xhcip);
37 }
38
39 static xhci_t *
xhci_hcdi_get_xhcip(usba_pipe_handle_data_t * ph)40 xhci_hcdi_get_xhcip(usba_pipe_handle_data_t *ph)
41 {
42 return (xhci_hcdi_get_xhcip_from_dev(ph->p_usba_device));
43 }
44
45 /*
46 * While the xHCI hardware is capable of supporting power management, we don't
47 * in the driver right now. Note, USBA doesn't seem to end up calling this entry
48 * point.
49 */
50 /* ARGSUSED */
51 static int
xhci_hcdi_pm_support(dev_info_t * dip)52 xhci_hcdi_pm_support(dev_info_t *dip)
53 {
54 return (USB_FAILURE);
55 }
56
57 static int
xhci_hcdi_pipe_open(usba_pipe_handle_data_t * ph,usb_flags_t usb_flags)58 xhci_hcdi_pipe_open(usba_pipe_handle_data_t *ph, usb_flags_t usb_flags)
59 {
60 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
61 xhci_pipe_t *pipe;
62 xhci_endpoint_t *xep = NULL;
63 xhci_device_t *xd;
64 int kmflags = usb_flags & USB_FLAGS_SLEEP ? KM_SLEEP : KM_NOSLEEP;
65 int ret;
66 uint_t epid;
67
68 mutex_enter(&xhcip->xhci_lock);
69 if (xhcip->xhci_state & XHCI_S_ERROR) {
70 mutex_exit(&xhcip->xhci_lock);
71 return (USB_HC_HARDWARE_ERROR);
72 }
73 mutex_exit(&xhcip->xhci_lock);
74
75 /*
76 * If we're here, something must be trying to open an already-opened
77 * pipe which is bad news.
78 */
79 if (ph->p_hcd_private != NULL) {
80 return (USB_FAILURE);
81 }
82
83 pipe = kmem_zalloc(sizeof (xhci_pipe_t), kmflags);
84 if (pipe == NULL) {
85 return (USB_NO_RESOURCES);
86 }
87 pipe->xp_opentime = gethrtime();
88 pipe->xp_pipe = ph;
89
90 /*
91 * If this is the root hub, there's nothing special to do on open. Just
92 * go ahead and allow it to be opened. All we have to do is add this to
93 * the list of our tracking structures for open pipes.
94 */
95 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
96 xep = NULL;
97 goto add;
98 }
99
100 /*
101 * Now that we're here, we're being asked to open up an endpoint of some
102 * kind. Because we've already handled the case of the root hub,
103 * everything should have a device.
104 */
105 epid = xhci_endpoint_pipe_to_epid(ph);
106 xd = usba_hcdi_get_device_private(ph->p_usba_device);
107 if (xd == NULL) {
108 xhci_error(xhcip, "!encountered endpoint (%d) without device "
109 "during pipe open", epid);
110 kmem_free(pipe, sizeof (xhci_pipe_t));
111 return (USB_FAILURE);
112 }
113
114 /*
115 * See if this endpoint exists or not, in general endpoints should not
116 * exist except for the default control endpoint, which we don't tear
117 * down until the device itself is cleaned up. Otherwise, a given pipe
118 * can only be open once.
119 */
120 mutex_enter(&xhcip->xhci_lock);
121 if (epid == XHCI_DEFAULT_ENDPOINT) {
122 xep = xd->xd_endpoints[epid];
123 VERIFY(xep != NULL);
124 VERIFY(xep->xep_pipe == NULL);
125 xep->xep_pipe = ph;
126 mutex_exit(&xhcip->xhci_lock);
127 ret = xhci_endpoint_update_default(xhcip, xd, xep);
128 if (ret != USB_SUCCESS) {
129 kmem_free(pipe, sizeof (xhci_pipe_t));
130 return (ret);
131 }
132 goto add;
133 }
134
135 /*
136 * If we're opening an endpoint other than the default control endpoint,
137 * then the device should have had a USB address assigned by the
138 * controller. Sanity check that before continuing.
139 */
140 VERIFY(xd->xd_addressed == B_TRUE);
141
142 /*
143 * We may have already initialized the endpoint with a previous pipe
144 * open.
145 */
146 if ((xep = xd->xd_endpoints[epid]) != NULL &&
147 (xep->xep_state & XHCI_ENDPOINT_OPEN)) {
148 mutex_exit(&xhcip->xhci_lock);
149
150 kmem_free(pipe, sizeof (xhci_pipe_t));
151 xhci_log(xhcip, "!asked to open endpoint %d on slot %d and "
152 "port %d, but endpoint already exists", epid, xd->xd_slot,
153 xd->xd_port);
154 return (USB_FAILURE);
155 }
156
157 if (xep != NULL) {
158 /*
159 * The endpoint is already initialized but is not presently
160 * open so we can take it over here.
161 */
162 if ((ret = xhci_endpoint_reinit(xhcip, xd, xep, ph) != 0)) {
163 mutex_exit(&xhcip->xhci_lock);
164
165 kmem_free(pipe, sizeof (xhci_pipe_t));
166 xhci_log(xhcip, "!asked to reopen endpoint %d on "
167 "slot %d and port %d, but reinit failed (%d)",
168 epid, xd->xd_slot, xd->xd_port, ret);
169 return (ret);
170 }
171
172 /*
173 * We need to ensure the endpoint is stopped before we try to
174 * reset the transfer ring.
175 */
176 xep->xep_state |= XHCI_ENDPOINT_QUIESCE;
177 if ((ret = xhci_endpoint_quiesce(xhcip, xd, xep)) !=
178 USB_SUCCESS) {
179 /*
180 * If we could not quiesce the endpoint, release it so
181 * that another open can try again.
182 */
183 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
184 xhci_endpoint_release(xhcip, xep);
185 mutex_exit(&xhcip->xhci_lock);
186
187 kmem_free(pipe, sizeof (xhci_pipe_t));
188 xhci_log(xhcip, "!asked to reopen endpoint %d on "
189 "slot %d and port %d, but quiesce failed (%d)",
190 epid, xd->xd_slot, xd->xd_port, ret);
191 return (ret);
192 }
193
194 /*
195 * Reset the transfer ring dequeue pointer. The initial
196 * Configure Endpoint command leaves the endpoint in the
197 * Running state (xHCI 1.2 / 4.6.6), so even though the ring is
198 * still empty we ring the doorbell to end up in the same state
199 * (Running but Inactive).
200 */
201 mutex_exit(&xhcip->xhci_lock);
202 if ((ret = xhci_command_set_tr_dequeue(xhcip, xd, xep)) != 0 ||
203 (ret = xhci_endpoint_ring(xhcip, xd, xep)) != 0) {
204 mutex_enter(&xhcip->xhci_lock);
205 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
206 xhci_endpoint_release(xhcip, xep);
207 mutex_exit(&xhcip->xhci_lock);
208
209 kmem_free(pipe, sizeof (xhci_pipe_t));
210 xhci_log(xhcip, "!asked to open endpoint %d on "
211 "slot %d and port %d, but restart failed (%d)",
212 epid, xd->xd_slot, xd->xd_port, ret);
213 return (USB_FAILURE);
214 }
215 mutex_enter(&xhcip->xhci_lock);
216 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
217 mutex_exit(&xhcip->xhci_lock);
218
219 goto add;
220 }
221
222 /*
223 * Okay, at this point we need to go create and set up an endpoint from
224 * scratch. Once we're done, we'll try to install it and make sure
225 * that it doesn't conflict with something else going on.
226 */
227 ret = xhci_endpoint_init(xhcip, xd, ph);
228 if (ret != 0) {
229 mutex_exit(&xhcip->xhci_lock);
230 kmem_free(pipe, sizeof (xhci_pipe_t));
231 if (ret == EIO) {
232 xhci_error(xhcip, "failed to initialize endpoint %d "
233 "on device slot %d and port %d: encountered fatal "
234 "FM error, resetting device", epid, xd->xd_slot,
235 xd->xd_port);
236 xhci_fm_runtime_reset(xhcip);
237 }
238 return (USB_HC_HARDWARE_ERROR);
239 }
240 xep = xd->xd_endpoints[epid];
241
242 mutex_enter(&xd->xd_imtx);
243 mutex_exit(&xhcip->xhci_lock);
244
245 /*
246 * Update the slot and input context for this endpoint. We make sure to
247 * always set the slot as having changed in the context field as the
248 * specification suggests we should and some hardware requires it.
249 */
250 xd->xd_input->xic_drop_flags = LE_32(0);
251 xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0) |
252 XHCI_INCTX_MASK_DCI(epid + 1));
253
254 if (epid + 1 > XHCI_SCTX_GET_DCI(LE_32(xd->xd_slotin->xsc_info))) {
255 uint32_t info;
256
257 info = xd->xd_slotin->xsc_info;
258 info &= ~XHCI_SCTX_DCI_MASK;
259 info |= XHCI_SCTX_SET_DCI(epid + 1);
260 xd->xd_slotin->xsc_info = info;
261 }
262
263 XHCI_DMA_SYNC(xd->xd_ictx, DDI_DMA_SYNC_FORDEV);
264 if (xhci_check_dma_handle(xhcip, &xd->xd_ictx) != DDI_FM_OK) {
265 mutex_exit(&xd->xd_imtx);
266 xhci_endpoint_fini(xd, epid);
267 kmem_free(pipe, sizeof (xhci_pipe_t));
268 xhci_error(xhcip, "failed to open pipe on endpoint %d of "
269 "device with slot %d and port %d: encountered fatal FM "
270 "error syncing device input context, resetting device",
271 epid, xd->xd_slot, xd->xd_port);
272 xhci_fm_runtime_reset(xhcip);
273 return (USB_HC_HARDWARE_ERROR);
274 }
275
276 if ((ret = xhci_command_configure_endpoint(xhcip, xd)) != USB_SUCCESS) {
277 mutex_exit(&xd->xd_imtx);
278 xhci_endpoint_fini(xd, epid);
279 kmem_free(pipe, sizeof (xhci_pipe_t));
280 return (ret);
281 }
282
283 mutex_exit(&xd->xd_imtx);
284 add:
285 pipe->xp_ep = xep;
286 ph->p_hcd_private = (usb_opaque_t)pipe;
287 mutex_enter(&xhcip->xhci_lock);
288 list_insert_tail(&xhcip->xhci_usba.xa_pipes, pipe);
289 mutex_exit(&xhcip->xhci_lock);
290
291 return (USB_SUCCESS);
292 }
293
294 static void
xhci_hcdi_periodic_free(xhci_t * xhcip,xhci_pipe_t * xp)295 xhci_hcdi_periodic_free(xhci_t *xhcip, xhci_pipe_t *xp)
296 {
297 int i;
298 xhci_periodic_pipe_t *xpp = &xp->xp_periodic;
299
300 if (xpp->xpp_tsize == 0)
301 return;
302
303 for (i = 0; i < xpp->xpp_ntransfers; i++) {
304 if (xpp->xpp_transfers[i] == NULL)
305 continue;
306 xhci_transfer_free(xhcip, xpp->xpp_transfers[i]);
307 xpp->xpp_transfers[i] = NULL;
308 }
309
310 xpp->xpp_ntransfers = 0;
311 xpp->xpp_tsize = 0;
312 }
313
314 /*
315 * Iterate over all transfers and free everything on the pipe. Once done, update
316 * the ring to basically 'consume' everything. For periodic IN endpoints, we
317 * need to handle this somewhat differently and actually close the original
318 * request and not deallocate the related pieces as those exist for the lifetime
319 * of the endpoint and are constantly reused.
320 */
321 static void
xhci_hcdi_pipe_flush(xhci_t * xhcip,xhci_endpoint_t * xep,int intr_code)322 xhci_hcdi_pipe_flush(xhci_t *xhcip, xhci_endpoint_t *xep, int intr_code)
323 {
324 xhci_transfer_t *xt;
325
326 ASSERT(MUTEX_HELD(&xhcip->xhci_lock));
327
328 while ((xt = list_remove_head(&xep->xep_transfers)) != NULL) {
329 if (xhci_endpoint_is_periodic_in(xep) == B_FALSE) {
330 usba_hcdi_cb(xep->xep_pipe, xt->xt_usba_req,
331 USB_CR_FLUSHED);
332 xhci_transfer_free(xhcip, xt);
333 }
334 }
335
336 if (xhci_endpoint_is_periodic_in(xep) == B_TRUE) {
337 xhci_pipe_t *xp = (xhci_pipe_t *)xep->xep_pipe->p_hcd_private;
338 xhci_periodic_pipe_t *xpp = &xp->xp_periodic;
339
340 if (xpp->xpp_usb_req != NULL) {
341 usba_hcdi_cb(xep->xep_pipe, xpp->xpp_usb_req,
342 intr_code);
343 xpp->xpp_usb_req = NULL;
344 }
345 }
346 }
347
348 /*
349 * We've been asked to terminate some set of regular I/O on an interrupt pipe.
350 * If this is for the root device, e.g. the xhci driver itself, then we remove
351 * our interrupt callback. Otherwise we stop the device for interrupt polling as
352 * follows:
353 *
354 * 1. Issue a stop endpoint command
355 * 2. Check to make sure that the endpoint stopped and reset it if needed.
356 * 3. Any thing that gets resolved can callback in the interim.
357 * 4. Ensure that nothing is scheduled on the ring
358 * 5. Skip the contents of the ring and set the TR dequeue pointer.
359 * 6. Return the original callback with a USB_CR_STOPPED_POLLING, NULL out the
360 * callback in the process.
361 */
362 static int
xhci_hcdi_pipe_poll_fini(usba_pipe_handle_data_t * ph,boolean_t is_close)363 xhci_hcdi_pipe_poll_fini(usba_pipe_handle_data_t *ph, boolean_t is_close)
364 {
365 int ret;
366 uint_t epid;
367 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
368 xhci_device_t *xd;
369 xhci_endpoint_t *xep;
370 xhci_pipe_t *xp;
371 xhci_periodic_pipe_t *xpp;
372 usb_opaque_t urp;
373
374 mutex_enter(&xhcip->xhci_lock);
375 if (xhcip->xhci_state & XHCI_S_ERROR) {
376 mutex_exit(&xhcip->xhci_lock);
377 return (USB_HC_HARDWARE_ERROR);
378 }
379
380 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
381 xhci_root_hub_intr_root_disable(xhcip);
382 ret = USB_SUCCESS;
383 mutex_exit(&xhcip->xhci_lock);
384 return (ret);
385 }
386
387 xd = usba_hcdi_get_device_private(ph->p_usba_device);
388 epid = xhci_endpoint_pipe_to_epid(ph);
389 if (xd->xd_endpoints[epid] == NULL) {
390 mutex_exit(&xhcip->xhci_lock);
391 xhci_error(xhcip, "asked to stop intr polling on slot %d, "
392 "port %d, endpoint: %d, but no endpoint structure",
393 xd->xd_slot, xd->xd_port, epid);
394 return (USB_FAILURE);
395 }
396 xep = xd->xd_endpoints[epid];
397 xp = (xhci_pipe_t *)ph->p_hcd_private;
398 if (xp == NULL) {
399 mutex_exit(&xhcip->xhci_lock);
400 xhci_error(xhcip, "asked to do finish polling on slot %d, "
401 "port %d, endpoint: %d, but no pipe structure",
402 xd->xd_slot, xd->xd_port, epid);
403 return (USB_FAILURE);
404 }
405 xpp = &xp->xp_periodic;
406
407 /*
408 * Ensure that no other resets or time outs are going on right now.
409 */
410 xhci_endpoint_serialize(xhcip, xep);
411
412 if (xpp->xpp_poll_state == XHCI_PERIODIC_POLL_IDLE) {
413 mutex_exit(&xhcip->xhci_lock);
414 return (USB_SUCCESS);
415 }
416
417 if (xpp->xpp_poll_state == XHCI_PERIODIC_POLL_STOPPING) {
418 mutex_exit(&xhcip->xhci_lock);
419 return (USB_FAILURE);
420 }
421
422 xpp->xpp_poll_state = XHCI_PERIODIC_POLL_STOPPING;
423 xep->xep_state |= XHCI_ENDPOINT_QUIESCE;
424 ret = xhci_endpoint_quiesce(xhcip, xd, xep);
425 if (ret != USB_SUCCESS) {
426 xhci_error(xhcip, "!failed to quiesce endpoint on slot %d, "
427 "port %d, endpoint: %d, failed with %d.",
428 xd->xd_slot, xd->xd_port, epid, ret);
429 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
430 cv_broadcast(&xep->xep_state_cv);
431 mutex_exit(&xhcip->xhci_lock);
432 return (ret);
433 }
434
435 /*
436 * Okay, we've stopped this ring time to wrap it all up. Remove all the
437 * transfers, note they aren't freed like a pipe reset.
438 */
439 while (list_is_empty(&xep->xep_transfers) == 0)
440 (void) list_remove_head(&xep->xep_transfers);
441 xhci_ring_skip(&xep->xep_ring);
442 mutex_exit(&xhcip->xhci_lock);
443
444 if ((ret = xhci_command_set_tr_dequeue(xhcip, xd, xep)) !=
445 USB_SUCCESS) {
446 xhci_error(xhcip, "!failed to reset endpoint ring on slot %d, "
447 "port %d, endpoint: %d, failed with %d.",
448 xd->xd_slot, xd->xd_port, epid, ret);
449 mutex_enter(&xhcip->xhci_lock);
450 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
451 cv_broadcast(&xep->xep_state_cv);
452 mutex_exit(&xhcip->xhci_lock);
453 return (ret);
454 }
455
456 mutex_enter(&xhcip->xhci_lock);
457 urp = xpp->xpp_usb_req;
458 xpp->xpp_usb_req = NULL;
459 xpp->xpp_poll_state = XHCI_PERIODIC_POLL_IDLE;
460 xep->xep_state &= ~XHCI_ENDPOINT_PERIODIC;
461 mutex_exit(&xhcip->xhci_lock);
462
463 /*
464 * It's possible that with a persistent pipe, we may not actually have
465 * anything left to call back on, because we already had.
466 */
467 if (urp != NULL) {
468 usba_hcdi_cb(ph, urp, is_close == B_TRUE ?
469 USB_CR_PIPE_CLOSING : USB_CR_STOPPED_POLLING);
470 }
471
472 /*
473 * Notify anything waiting for us that we're done quiescing this device.
474 */
475 mutex_enter(&xhcip->xhci_lock);
476 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
477 cv_broadcast(&xep->xep_state_cv);
478 mutex_exit(&xhcip->xhci_lock);
479
480 return (USB_SUCCESS);
481
482 }
483
484 /*
485 * Tear down everything that we did in open. After this, the consumer of this
486 * USB device is done.
487 */
488 static int
xhci_hcdi_pipe_close(usba_pipe_handle_data_t * ph,usb_flags_t usb_flags)489 xhci_hcdi_pipe_close(usba_pipe_handle_data_t *ph, usb_flags_t usb_flags)
490 {
491 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
492 xhci_pipe_t *xp;
493 xhci_device_t *xd;
494 xhci_endpoint_t *xep;
495 int ret;
496 uint_t epid;
497
498 if ((ph->p_ep.bmAttributes & USB_EP_ATTR_MASK) == USB_EP_ATTR_INTR &&
499 xhcip->xhci_usba.xa_intr_cb_ph != NULL) {
500 if ((ret = xhci_hcdi_pipe_poll_fini(ph, B_TRUE)) !=
501 USB_SUCCESS) {
502 return (ret);
503 }
504 }
505
506 mutex_enter(&xhcip->xhci_lock);
507
508 xp = (xhci_pipe_t *)ph->p_hcd_private;
509 VERIFY(xp != NULL);
510
511 /*
512 * The default endpoint is special. It is created and destroyed with the
513 * device. So like with open, closing it is just state tracking. The
514 * same is true for the root hub.
515 */
516 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR)
517 goto remove;
518
519 xd = usba_hcdi_get_device_private(ph->p_usba_device);
520 epid = xhci_endpoint_pipe_to_epid(ph);
521 if (xd->xd_endpoints[epid] == NULL) {
522 mutex_exit(&xhcip->xhci_lock);
523 xhci_error(xhcip, "asked to do close pipe on slot %d, "
524 "port %d, endpoint: %d, but no endpoint structure",
525 xd->xd_slot, xd->xd_port, epid);
526 return (USB_FAILURE);
527 }
528 xep = xd->xd_endpoints[epid];
529
530 if (xp->xp_ep != NULL && xp->xp_ep->xep_num == XHCI_DEFAULT_ENDPOINT) {
531 xep->xep_pipe = NULL;
532 goto remove;
533 }
534
535 /*
536 * We clean up the endpoint by stopping it and cancelling any transfers
537 * that were in flight at the time. The endpoint is not unconfigured
538 * until the device is torn down later.
539 */
540 xhci_endpoint_timeout_cancel(xhcip, xep);
541 xep->xep_state |= XHCI_ENDPOINT_QUIESCE;
542 if ((ret = xhci_endpoint_quiesce(xhcip, xd, xep)) != USB_SUCCESS) {
543 /*
544 * If we cannot stop the ring, it is not safe to proceed and we
545 * must keep the pipe open.
546 */
547 xep->xep_state &=
548 ~(XHCI_ENDPOINT_TEARDOWN | XHCI_ENDPOINT_QUIESCE);
549 cv_broadcast(&xep->xep_state_cv);
550 mutex_exit(&xhcip->xhci_lock);
551 xhci_error(xhcip, "asked to do close pipe on slot %d, "
552 "port %d, endpoint: %d, but quiesce failed %d",
553 xd->xd_slot, xd->xd_port, epid, ret);
554 return (USB_FAILURE);
555 }
556
557 /*
558 * Now that we've stopped the endpoint, see if we need to flush any
559 * transfers.
560 */
561 xhci_hcdi_pipe_flush(xhcip, xep, USB_CR_PIPE_CLOSING);
562 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
563 xhci_hcdi_periodic_free(xhcip, xp);
564 }
565
566 xhci_endpoint_release(xhcip, xep);
567
568 remove:
569 ph->p_hcd_private = NULL;
570 list_remove(&xhcip->xhci_usba.xa_pipes, xp);
571 kmem_free(xp, sizeof (xhci_pipe_t));
572
573 mutex_exit(&xhcip->xhci_lock);
574
575 return (USB_SUCCESS);
576 }
577
578 /*
579 * We've been asked to reset a pipe aka an endpoint. This endpoint may be in an
580 * arbitrary state, it may be running or it may be halted. In this case, we go
581 * through and check whether or not we know it's been halted or not. If it has
582 * not, then we stop the endpoint.
583 *
584 * Once the endpoint has been stopped, walk all transfers and go ahead and
585 * basically return them as being flushed. Then finally set the dequeue point
586 * for this endpoint.
587 */
588 /* ARGSUSED */
589 static int
xhci_hcdi_pipe_reset(usba_pipe_handle_data_t * ph,usb_flags_t usb_flags)590 xhci_hcdi_pipe_reset(usba_pipe_handle_data_t *ph, usb_flags_t usb_flags)
591 {
592 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
593 xhci_device_t *xd;
594 xhci_endpoint_t *xep;
595 uint_t epid;
596 int ret;
597
598 mutex_enter(&xhcip->xhci_lock);
599 if (xhcip->xhci_state & XHCI_S_ERROR) {
600 mutex_exit(&xhcip->xhci_lock);
601 return (USB_HC_HARDWARE_ERROR);
602 }
603
604 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
605 mutex_exit(&xhcip->xhci_lock);
606 return (USB_NOT_SUPPORTED);
607 }
608
609 xd = usba_hcdi_get_device_private(ph->p_usba_device);
610 epid = xhci_endpoint_pipe_to_epid(ph);
611 if (xd->xd_endpoints[epid] == NULL) {
612 mutex_exit(&xhcip->xhci_lock);
613 xhci_error(xhcip, "asked to do reset pipe on slot %d, "
614 "port %d, endpoint: %d, but no endpoint structure",
615 xd->xd_slot, xd->xd_port, epid);
616 return (USB_FAILURE);
617 }
618
619 xep = xd->xd_endpoints[epid];
620
621 /*
622 * Ensure that no other resets or time outs are going on right now.
623 */
624 xhci_endpoint_serialize(xhcip, xep);
625
626 xep->xep_state |= XHCI_ENDPOINT_QUIESCE;
627 ret = xhci_endpoint_quiesce(xhcip, xd, xep);
628 if (ret != USB_SUCCESS) {
629 /*
630 * We failed to quiesce for some reason, remove the flag and let
631 * someone else give it a shot.
632 */
633 xhci_error(xhcip, "!failed to quiesce endpoint on slot %d, "
634 "port %d, endpoint: %d, failed with %d.",
635 xd->xd_slot, xd->xd_port, epid, ret);
636 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
637 cv_broadcast(&xep->xep_state_cv);
638 mutex_exit(&xhcip->xhci_lock);
639 return (ret);
640 }
641
642 xhci_ring_skip(&xep->xep_ring);
643
644 mutex_exit(&xhcip->xhci_lock);
645 if ((ret = xhci_command_set_tr_dequeue(xhcip, xd, xep)) !=
646 USB_SUCCESS) {
647 xhci_error(xhcip, "!failed to reset endpoint ring on slot %d, "
648 "port %d, endpoint: %d, failed setting ring dequeue with "
649 "%d.", xd->xd_slot, xd->xd_port, epid, ret);
650 mutex_enter(&xhcip->xhci_lock);
651 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
652 cv_broadcast(&xep->xep_state_cv);
653 mutex_exit(&xhcip->xhci_lock);
654 return (ret);
655 }
656
657 mutex_enter(&xhcip->xhci_lock);
658 xhci_hcdi_pipe_flush(xhcip, xep, USB_CR_PIPE_RESET);
659
660 /*
661 * We need to remove the periodic flag as part of resetting, as if this
662 * was used for periodic activity, it no longer is and therefore can now
663 * be used for such purposes.
664 *
665 * Notify anything waiting for us that we're done quiescing this device.
666 */
667 xep->xep_state &= ~(XHCI_ENDPOINT_QUIESCE | XHCI_ENDPOINT_PERIODIC);
668 cv_broadcast(&xep->xep_state_cv);
669 mutex_exit(&xhcip->xhci_lock);
670
671 return (USB_SUCCESS);
672 }
673
674 /*
675 * We're asked to reset or change the data toggle, which is used in a few cases.
676 * However, there doesn't seem to be a good way to do this in xHCI as the data
677 * toggle isn't exposed. It seems that dropping a reset endpoint would
678 * theoretically do this; however, that can only be used when in the HALTED
679 * state. As such, for now we just return.
680 */
681 /* ARGSUSED */
682 void
xhci_hcdi_pipe_reset_data_toggle(usba_pipe_handle_data_t * pipe_handle)683 xhci_hcdi_pipe_reset_data_toggle(usba_pipe_handle_data_t *pipe_handle)
684 {
685 }
686
687 /*
688 * We need to convert the USB request to an 8-byte little endian value. If we
689 * didn't have to think about big endian systems, this would be fine.
690 * Unfortunately, with them, this is a bit confusing. The problem is that if you
691 * think of this as a struct layout, the order that we or things together
692 * represents their byte layout. e.g. ctrl_bRequest is at offset 1 in the SETUP
693 * STAGE trb. However, when it becomes a part of a 64-bit big endian number, if
694 * ends up at byte 7, where as it needs to be at one. Hence why we do a final
695 * LE_64 at the end of this, to convert this into the byte order that it's
696 * expected to be in.
697 */
698 static uint64_t
xhci_hcdi_ctrl_req_to_trb(usb_ctrl_req_t * ucrp)699 xhci_hcdi_ctrl_req_to_trb(usb_ctrl_req_t *ucrp)
700 {
701 uint64_t ret = ucrp->ctrl_bmRequestType |
702 (ucrp->ctrl_bRequest << 8) |
703 ((uint64_t)LE_16(ucrp->ctrl_wValue) << 16) |
704 ((uint64_t)LE_16(ucrp->ctrl_wIndex) << 32) |
705 ((uint64_t)LE_16(ucrp->ctrl_wLength) << 48);
706 return (LE_64(ret));
707 }
708
709 /*
710 * USBA calls us in order to make a specific control type request to a device,
711 * potentially even the root hub. If the request is for the root hub, then we
712 * need to intercept this and cons up the requested data.
713 */
714 static int
xhci_hcdi_pipe_ctrl_xfer(usba_pipe_handle_data_t * ph,usb_ctrl_req_t * ucrp,usb_flags_t usb_flags)715 xhci_hcdi_pipe_ctrl_xfer(usba_pipe_handle_data_t *ph, usb_ctrl_req_t *ucrp,
716 usb_flags_t usb_flags)
717 {
718 int ret, statusdir, trt;
719 uint_t ep;
720 xhci_device_t *xd;
721 xhci_endpoint_t *xep;
722 xhci_transfer_t *xt;
723 boolean_t datain;
724
725 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
726
727 mutex_enter(&xhcip->xhci_lock);
728 if (xhcip->xhci_state & XHCI_S_ERROR) {
729 mutex_exit(&xhcip->xhci_lock);
730 return (USB_HC_HARDWARE_ERROR);
731 }
732
733 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
734 ret = xhci_root_hub_ctrl_req(xhcip, ph, ucrp);
735 mutex_exit(&xhcip->xhci_lock);
736 return (ret);
737 }
738
739 /*
740 * Determine the device and endpoint.
741 */
742 xd = usba_hcdi_get_device_private(ph->p_usba_device);
743 ep = xhci_endpoint_pipe_to_epid(ph);
744 if (xd->xd_endpoints[ep] == NULL) {
745 mutex_exit(&xhcip->xhci_lock);
746 xhci_error(xhcip, "asked to do control transfer on slot %d, "
747 "port %d, endpoint: %d, but no endpoint structure",
748 xd->xd_slot, xd->xd_port, ep);
749 return (USB_FAILURE);
750 }
751 xep = xd->xd_endpoints[ep];
752
753 /*
754 * There are several types of requests that we have to handle in special
755 * ways in xHCI. If we have one of those requests, then we don't
756 * necessarily go through the normal path. These special cases are all
757 * documented in xHCI 1.1 / 4.5.4.
758 *
759 * Looking at that, you may ask why aren't SET_CONFIGURATION and SET_IF
760 * special cased here. This action is a little confusing by default. The
761 * xHCI specification requires that we may need to issue a configure
762 * endpoint command as part of this. However, the xHCI 1.1 / 4.5.4.2
763 * states that we don't actually need to if nothing in the endpoint
764 * configuration context has changed. Because nothing in it should have
765 * changed as part of this, we don't need to do anything and instead
766 * just can issue the request normally. We're also assuming in the
767 * USB_REQ_SET_IF case that if something's changing the interface, the
768 * non-default endpoint will have yet to be opened.
769 */
770 if (ucrp->ctrl_bmRequestType == USB_DEV_REQ_HOST_TO_DEV &&
771 ucrp->ctrl_bRequest == USB_REQ_SET_ADDRESS) {
772 /*
773 * As we've defined an explicit set-address endpoint, we should
774 * never call this function. If we get here, always fail.
775 */
776 mutex_exit(&xhcip->xhci_lock);
777 usba_hcdi_cb(ph, (usb_opaque_t)ucrp, USB_CR_NOT_SUPPORTED);
778 return (USB_SUCCESS);
779 }
780
781 mutex_exit(&xhcip->xhci_lock);
782
783 /*
784 * Allocate the transfer memory, etc.
785 */
786 xt = xhci_transfer_alloc(xhcip, xep, ucrp->ctrl_wLength, 2, usb_flags);
787 if (xt == NULL) {
788 return (USB_NO_RESOURCES);
789 }
790 xt->xt_usba_req = (usb_opaque_t)ucrp;
791 xt->xt_timeout = ucrp->ctrl_timeout;
792 if (xt->xt_timeout == 0) {
793 xt->xt_timeout = HCDI_DEFAULT_TIMEOUT;
794 }
795
796 if (ucrp->ctrl_wLength > 0) {
797 if ((ucrp->ctrl_bmRequestType & USB_DEV_REQ_DEV_TO_HOST) != 0) {
798 trt = XHCI_TRB_TRT_IN;
799 datain = B_TRUE;
800 statusdir = 0;
801 } else {
802 trt = XHCI_TRB_TRT_OUT;
803 datain = B_FALSE;
804 statusdir = XHCI_TRB_DIR_IN;
805
806 xhci_transfer_copy(xt, ucrp->ctrl_data->b_rptr,
807 ucrp->ctrl_wLength, B_FALSE);
808 if (xhci_transfer_sync(xhcip, xt,
809 DDI_DMA_SYNC_FORDEV) != DDI_FM_OK) {
810 xhci_transfer_free(xhcip, xt);
811 xhci_error(xhcip, "failed to synchronize ctrl "
812 "transfer DMA memory on endpoint %u of "
813 "device on slot %d and port %d: resetting "
814 "device", xep->xep_num, xd->xd_slot,
815 xd->xd_port);
816 xhci_fm_runtime_reset(xhcip);
817 return (USB_HC_HARDWARE_ERROR);
818 }
819 }
820 } else {
821 trt = 0;
822 datain = B_FALSE;
823 statusdir = XHCI_TRB_DIR_IN;
824 }
825
826 /*
827 * We always fill in the required setup and status TRBs ourselves;
828 * however, to minimize our knowledge about how the data has been split
829 * across multiple DMA cookies in an SGL, we leave that to the transfer
830 * logic to fill in.
831 */
832 xt->xt_trbs[0].trb_addr = xhci_hcdi_ctrl_req_to_trb(ucrp);
833 xt->xt_trbs[0].trb_status = LE_32(XHCI_TRB_LEN(8) | XHCI_TRB_INTR(0));
834 xt->xt_trbs[0].trb_flags = LE_32(trt | XHCI_TRB_IDT |
835 XHCI_TRB_TYPE_SETUP);
836
837 if (ucrp->ctrl_wLength > 0)
838 xhci_transfer_trb_fill_data(xep, xt, 1, datain);
839
840 xt->xt_trbs[xt->xt_ntrbs - 1].trb_addr = 0;
841 xt->xt_trbs[xt->xt_ntrbs - 1].trb_status = LE_32(XHCI_TRB_INTR(0));
842 xt->xt_trbs[xt->xt_ntrbs - 1].trb_flags = LE_32(XHCI_TRB_TYPE_STATUS |
843 XHCI_TRB_IOC | statusdir);
844
845
846 mutex_enter(&xhcip->xhci_lock);
847
848 /*
849 * Schedule the transfer, allocating resources in the process.
850 */
851 if (xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE) != 0) {
852 xhci_transfer_free(xhcip, xt);
853 mutex_exit(&xhcip->xhci_lock);
854 return (USB_NO_RESOURCES);
855 }
856
857 mutex_exit(&xhcip->xhci_lock);
858
859 return (USB_SUCCESS);
860 }
861
862 /*
863 * This request is trying to get the upper bound on the amount of data we're
864 * willing transfer in one go. Note that this amount can be broken down into
865 * multiple SGL entries, this interface doesn't particularly care about that.
866 */
867 /* ARGSUSED */
868 static int
xhci_hcdi_bulk_transfer_size(usba_device_t * ud,size_t * sizep)869 xhci_hcdi_bulk_transfer_size(usba_device_t *ud, size_t *sizep)
870 {
871 if (sizep != NULL)
872 *sizep = XHCI_MAX_TRANSFER;
873 return (USB_SUCCESS);
874 }
875
876 /*
877 * Perform a bulk transfer. This is a pretty straightforward action. We
878 * basically just allocate the appropriate transfer and try to schedule it,
879 * hoping there is enough space.
880 */
881 static int
xhci_hcdi_pipe_bulk_xfer(usba_pipe_handle_data_t * ph,usb_bulk_req_t * ubrp,usb_flags_t usb_flags)882 xhci_hcdi_pipe_bulk_xfer(usba_pipe_handle_data_t *ph, usb_bulk_req_t *ubrp,
883 usb_flags_t usb_flags)
884 {
885 uint_t epid;
886 xhci_device_t *xd;
887 xhci_endpoint_t *xep;
888 xhci_transfer_t *xt;
889 boolean_t datain;
890
891 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
892
893 mutex_enter(&xhcip->xhci_lock);
894 if (xhcip->xhci_state & XHCI_S_ERROR) {
895 mutex_exit(&xhcip->xhci_lock);
896 return (USB_HC_HARDWARE_ERROR);
897 }
898
899 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
900 mutex_exit(&xhcip->xhci_lock);
901 return (USB_NOT_SUPPORTED);
902 }
903
904 xd = usba_hcdi_get_device_private(ph->p_usba_device);
905 epid = xhci_endpoint_pipe_to_epid(ph);
906 if (xd->xd_endpoints[epid] == NULL) {
907 mutex_exit(&xhcip->xhci_lock);
908 xhci_error(xhcip, "asked to do control transfer on slot %d, "
909 "port %d, endpoint: %d, but no endpoint structure",
910 xd->xd_slot, xd->xd_port, epid);
911 return (USB_FAILURE);
912 }
913 xep = xd->xd_endpoints[epid];
914 mutex_exit(&xhcip->xhci_lock);
915
916 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
917 datain = B_TRUE;
918 } else {
919 datain = B_FALSE;
920 }
921
922 xt = xhci_transfer_alloc(xhcip, xep, ubrp->bulk_len, 0, usb_flags);
923 if (xt == NULL) {
924 return (USB_NO_RESOURCES);
925 }
926 xt->xt_usba_req = (usb_opaque_t)ubrp;
927 xt->xt_timeout = ubrp->bulk_timeout;
928 if (xt->xt_timeout == 0) {
929 xt->xt_timeout = HCDI_DEFAULT_TIMEOUT;
930 }
931
932 if (ubrp->bulk_len > 0 && datain == B_FALSE) {
933 xhci_transfer_copy(xt, ubrp->bulk_data->b_rptr, ubrp->bulk_len,
934 B_FALSE);
935 if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORDEV) !=
936 DDI_FM_OK) {
937 xhci_transfer_free(xhcip, xt);
938 xhci_error(xhcip, "failed to synchronize bulk "
939 "transfer DMA memory on endpoint %u of "
940 "device on slot %d and port %d: resetting "
941 "device", xep->xep_num, xd->xd_slot,
942 xd->xd_port);
943 xhci_fm_runtime_reset(xhcip);
944 return (USB_HC_HARDWARE_ERROR);
945 }
946 }
947
948 xhci_transfer_trb_fill_data(xep, xt, 0, datain);
949 mutex_enter(&xhcip->xhci_lock);
950 if (xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE) != 0) {
951 xhci_transfer_free(xhcip, xt);
952 mutex_exit(&xhcip->xhci_lock);
953 return (USB_NO_RESOURCES);
954 }
955 mutex_exit(&xhcip->xhci_lock);
956
957 return (USB_SUCCESS);
958 }
959
960 static void
xhci_hcdi_isoc_transfer_fill(xhci_device_t * xd,xhci_endpoint_t * xep,xhci_transfer_t * xt,usb_isoc_req_t * usrp)961 xhci_hcdi_isoc_transfer_fill(xhci_device_t *xd, xhci_endpoint_t *xep,
962 xhci_transfer_t *xt, usb_isoc_req_t *usrp)
963 {
964 int i;
965 uintptr_t buf;
966
967 buf = xt->xt_buffer.xdb_cookies[0].dmac_laddress;
968 for (i = 0; i < usrp->isoc_pkts_count; i++) {
969 int flags;
970 uint_t tbc, tlbpc;
971
972 ushort_t len = usrp->isoc_pkt_descr[i].isoc_pkt_length;
973 xhci_trb_t *trb = &xt->xt_trbs[i];
974
975 trb->trb_addr = LE_64(buf);
976
977 /*
978 * Because we know that a single frame can have all of its data
979 * in a single instance, we know that we don't need to do
980 * anything special here.
981 */
982 trb->trb_status = LE_32(XHCI_TRB_LEN(len) | XHCI_TRB_TDREM(0) |
983 XHCI_TRB_INTR(0));
984
985 /*
986 * Always enable SIA to start the frame ASAP. We also always
987 * enable an interrupt on a short packet. If this is the last
988 * trb, then we will set IOC. Each TRB created here is really
989 * its own TD. However, we only set an interrupt on the last
990 * entry to better deal with scheduling.
991 */
992 flags = XHCI_TRB_SIA | XHCI_TRB_ISP | XHCI_TRB_SET_FRAME(0);
993 flags |= XHCI_TRB_TYPE_ISOCH;
994
995 if (i + 1 == usrp->isoc_pkts_count)
996 flags |= XHCI_TRB_IOC;
997
998 /*
999 * Now we need to calculate the TBC and the TLBPC.
1000 */
1001 xhci_transfer_calculate_isoc(xd, xep, len, &tbc, &tlbpc);
1002 flags |= XHCI_TRB_SET_TBC(tbc);
1003 flags |= XHCI_TRB_SET_TLBPC(tlbpc);
1004
1005 trb->trb_flags = LE_32(flags);
1006 buf += len;
1007
1008 /*
1009 * Go through and copy the required data to our local copy of
1010 * the isoc descriptor. By default, we assume that all data will
1011 * be copied and the status set to OK. This mirrors the fact
1012 * that we won't get a notification unless there's been an
1013 * error or short packet transfer.
1014 */
1015 xt->xt_isoc[i].isoc_pkt_length = len;
1016 xt->xt_isoc[i].isoc_pkt_actual_length = len;
1017 xt->xt_isoc[i].isoc_pkt_status = USB_CR_OK;
1018 }
1019 }
1020
1021 /*
1022 * Initialize periodic IN requests (both interrupt and isochronous)
1023 */
1024 static int
xhci_hcdi_periodic_init(xhci_t * xhcip,usba_pipe_handle_data_t * ph,usb_opaque_t usb_req,size_t len,int usb_flags)1025 xhci_hcdi_periodic_init(xhci_t *xhcip, usba_pipe_handle_data_t *ph,
1026 usb_opaque_t usb_req, size_t len, int usb_flags)
1027 {
1028 int i, ret;
1029 uint_t epid;
1030 xhci_device_t *xd;
1031 xhci_endpoint_t *xep;
1032 xhci_pipe_t *xp;
1033 xhci_periodic_pipe_t *xpp;
1034
1035 mutex_enter(&xhcip->xhci_lock);
1036 if (xhcip->xhci_state & XHCI_S_ERROR) {
1037 mutex_exit(&xhcip->xhci_lock);
1038 return (USB_HC_HARDWARE_ERROR);
1039 }
1040
1041 xd = usba_hcdi_get_device_private(ph->p_usba_device);
1042 epid = xhci_endpoint_pipe_to_epid(ph);
1043 if (xd->xd_endpoints[epid] == NULL) {
1044 xhci_error(xhcip, "asked to do periodic transfer on slot %d, "
1045 "port %d, endpoint: %d, but no endpoint structure",
1046 xd->xd_slot, xd->xd_port, epid);
1047 mutex_exit(&xhcip->xhci_lock);
1048 return (USB_FAILURE);
1049 }
1050 xep = xd->xd_endpoints[epid];
1051 xp = (xhci_pipe_t *)ph->p_hcd_private;
1052 if (xp == NULL) {
1053 xhci_error(xhcip, "asked to do periodic transfer on slot %d, "
1054 "port %d, endpoint: %d, but no pipe structure",
1055 xd->xd_slot, xd->xd_port, epid);
1056 mutex_exit(&xhcip->xhci_lock);
1057 return (USB_FAILURE);
1058 }
1059 xpp = &xp->xp_periodic;
1060
1061 /*
1062 * Only allow a single polling request at any given time.
1063 */
1064 if (xpp->xpp_usb_req != NULL) {
1065 mutex_exit(&xhcip->xhci_lock);
1066 return (USB_BUSY);
1067 }
1068
1069 /*
1070 * We keep allocations around in case we restart polling, which most
1071 * devices do (not really caring about a lost event). However, we don't
1072 * support a driver changing that size on us, which it probably won't.
1073 * If we stumble across driver that does, then this will need to become
1074 * a lot more complicated.
1075 */
1076 if (xpp->xpp_tsize > 0 && xpp->xpp_tsize < len) {
1077 mutex_exit(&xhcip->xhci_lock);
1078 return (USB_INVALID_REQUEST);
1079 }
1080
1081 if (xpp->xpp_tsize == 0) {
1082 int ntrbs;
1083 int ntransfers;
1084
1085 /*
1086 * What we allocate varies based on whether or not this is an
1087 * isochronous or interrupt IN periodic.
1088 */
1089 if (xep->xep_type == USB_EP_ATTR_INTR) {
1090 ntrbs = 0;
1091 ntransfers = XHCI_INTR_IN_NTRANSFERS;
1092 } else {
1093 usb_isoc_req_t *usrp;
1094 ASSERT(xep->xep_type == USB_EP_ATTR_ISOCH);
1095
1096 usrp = (usb_isoc_req_t *)usb_req;
1097 ntrbs = usrp->isoc_pkts_count;
1098 ntransfers = XHCI_ISOC_IN_NTRANSFERS;
1099 }
1100
1101 xpp->xpp_tsize = len;
1102 xpp->xpp_ntransfers = ntransfers;
1103
1104 for (i = 0; i < xpp->xpp_ntransfers; i++) {
1105 xhci_transfer_t *xt = xhci_transfer_alloc(xhcip, xep,
1106 len, ntrbs, usb_flags);
1107 if (xt == NULL) {
1108 xhci_hcdi_periodic_free(xhcip, xp);
1109 mutex_exit(&xhcip->xhci_lock);
1110 return (USB_NO_RESOURCES);
1111 }
1112
1113 if (xep->xep_type == USB_EP_ATTR_INTR) {
1114 xhci_transfer_trb_fill_data(xep, xt, 0, B_TRUE);
1115 } else {
1116 usb_isoc_req_t *usrp;
1117 usrp = (usb_isoc_req_t *)usb_req;
1118 xhci_hcdi_isoc_transfer_fill(xd, xep, xt, usrp);
1119 xt->xt_data_tohost = B_TRUE;
1120 }
1121 xpp->xpp_transfers[i] = xt;
1122 }
1123 }
1124
1125 /*
1126 * Mark the endpoint as periodic so we don't have timeouts at play.
1127 */
1128 xep->xep_state |= XHCI_ENDPOINT_PERIODIC;
1129
1130 /*
1131 * Now that we've allocated everything, go ahead and schedule them and
1132 * kick off the ring.
1133 */
1134 for (i = 0; i < xpp->xpp_ntransfers; i++) {
1135 int ret;
1136 ret = xhci_endpoint_schedule(xhcip, xd, xep,
1137 xpp->xpp_transfers[i], B_FALSE);
1138 if (ret != 0) {
1139 (void) xhci_ring_reset(xhcip, &xep->xep_ring);
1140 xep->xep_state &= ~XHCI_ENDPOINT_PERIODIC;
1141 mutex_exit(&xhcip->xhci_lock);
1142 return (ret);
1143 }
1144 }
1145
1146 /*
1147 * Don't worry about freeing memory, it'll be done when the endpoint
1148 * closes and the whole system is reset.
1149 */
1150 xpp->xpp_usb_req = usb_req;
1151 xpp->xpp_poll_state = XHCI_PERIODIC_POLL_ACTIVE;
1152
1153 ret = xhci_endpoint_ring(xhcip, xd, xep);
1154 mutex_exit(&xhcip->xhci_lock);
1155 return (ret);
1156 }
1157
1158 static int
xhci_hcdi_intr_oneshot(xhci_t * xhcip,usba_pipe_handle_data_t * ph,usb_intr_req_t * uirp,usb_flags_t usb_flags)1159 xhci_hcdi_intr_oneshot(xhci_t *xhcip, usba_pipe_handle_data_t *ph,
1160 usb_intr_req_t *uirp, usb_flags_t usb_flags)
1161 {
1162 uint_t epid;
1163 xhci_device_t *xd;
1164 xhci_endpoint_t *xep;
1165 xhci_transfer_t *xt;
1166 boolean_t datain;
1167 mblk_t *mp = NULL;
1168
1169 mutex_enter(&xhcip->xhci_lock);
1170 if (xhcip->xhci_state & XHCI_S_ERROR) {
1171 mutex_exit(&xhcip->xhci_lock);
1172 return (USB_HC_HARDWARE_ERROR);
1173 }
1174
1175 xd = usba_hcdi_get_device_private(ph->p_usba_device);
1176 epid = xhci_endpoint_pipe_to_epid(ph);
1177 if (xd->xd_endpoints[epid] == NULL) {
1178 xhci_error(xhcip, "asked to do interrupt transfer on slot %d, "
1179 "port %d, endpoint: %d, but no endpoint structure",
1180 xd->xd_slot, xd->xd_port, epid);
1181 mutex_exit(&xhcip->xhci_lock);
1182 return (USB_FAILURE);
1183 }
1184 xep = xd->xd_endpoints[epid];
1185
1186 mutex_exit(&xhcip->xhci_lock);
1187
1188 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
1189 datain = B_TRUE;
1190 } else {
1191 datain = B_FALSE;
1192 }
1193
1194 xt = xhci_transfer_alloc(xhcip, xep, uirp->intr_len, 0, usb_flags);
1195 if (xt == NULL) {
1196 return (USB_NO_RESOURCES);
1197 }
1198
1199 xt->xt_usba_req = (usb_opaque_t)uirp;
1200 xt->xt_timeout = uirp->intr_timeout;
1201 if (xt->xt_timeout == 0) {
1202 xt->xt_timeout = HCDI_DEFAULT_TIMEOUT;
1203 }
1204
1205 /*
1206 * Unlike other request types, USB Interrupt-IN requests aren't required
1207 * to have allocated the message block for data. If they haven't, we
1208 * take care of that now.
1209 */
1210 if (uirp->intr_len > 0 && datain == B_TRUE && uirp->intr_data == NULL) {
1211 if (usb_flags & USB_FLAGS_SLEEP) {
1212 mp = allocb_wait(uirp->intr_len, BPRI_LO, STR_NOSIG,
1213 NULL);
1214 } else {
1215 mp = allocb(uirp->intr_len, 0);
1216 }
1217 if (mp == NULL) {
1218 xhci_transfer_free(xhcip, xt);
1219 mutex_exit(&xhcip->xhci_lock);
1220 return (USB_NO_RESOURCES);
1221 }
1222 uirp->intr_data = mp;
1223 }
1224
1225 if (uirp->intr_len > 0 && datain == B_FALSE) {
1226 xhci_transfer_copy(xt, uirp->intr_data->b_rptr, uirp->intr_len,
1227 B_FALSE);
1228 if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORDEV) !=
1229 DDI_FM_OK) {
1230 xhci_transfer_free(xhcip, xt);
1231 xhci_error(xhcip, "failed to synchronize interrupt "
1232 "transfer DMA memory on endpoint %u of "
1233 "device on slot %d and port %d: resetting "
1234 "device", xep->xep_num, xd->xd_slot,
1235 xd->xd_port);
1236 xhci_fm_runtime_reset(xhcip);
1237 return (USB_HC_HARDWARE_ERROR);
1238 }
1239 }
1240
1241 xhci_transfer_trb_fill_data(xep, xt, 0, datain);
1242 mutex_enter(&xhcip->xhci_lock);
1243 if (xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE) != 0) {
1244 if (mp != NULL) {
1245 uirp->intr_data = NULL;
1246 freemsg(mp);
1247 }
1248 xhci_transfer_free(xhcip, xt);
1249 mutex_exit(&xhcip->xhci_lock);
1250 return (USB_NO_RESOURCES);
1251 }
1252 mutex_exit(&xhcip->xhci_lock);
1253
1254 return (USB_SUCCESS);
1255 }
1256
1257 /*
1258 * We've been asked to perform an interrupt transfer. When this is an interrupt
1259 * IN endpoint, that means that the hcd is being asked to start polling on the
1260 * endpoint. When the endpoint is the root hub, it effectively becomes synthetic
1261 * polling.
1262 *
1263 * When we have an interrupt out endpoint, then this is just a single simple
1264 * interrupt request that we send out and there isn't much special to do beyond
1265 * the normal activity.
1266 */
1267 static int
xhci_hcdi_pipe_intr_xfer(usba_pipe_handle_data_t * ph,usb_intr_req_t * uirp,usb_flags_t usb_flags)1268 xhci_hcdi_pipe_intr_xfer(usba_pipe_handle_data_t *ph, usb_intr_req_t *uirp,
1269 usb_flags_t usb_flags)
1270 {
1271 int ret;
1272 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
1273
1274 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
1275 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
1276 ret = xhci_root_hub_intr_root_enable(xhcip, ph, uirp);
1277 } else if (uirp->intr_attributes & USB_ATTRS_ONE_XFER) {
1278 ret = xhci_hcdi_intr_oneshot(xhcip, ph, uirp,
1279 usb_flags);
1280 } else {
1281 ret = xhci_hcdi_periodic_init(xhcip, ph,
1282 (usb_opaque_t)uirp, uirp->intr_len, usb_flags);
1283 }
1284 } else {
1285 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
1286 return (USB_NOT_SUPPORTED);
1287 }
1288 ret = xhci_hcdi_intr_oneshot(xhcip, ph, uirp, usb_flags);
1289 }
1290
1291 return (ret);
1292 }
1293
1294 /* ARGSUSED */
1295 static int
xhci_hcdi_pipe_stop_intr_polling(usba_pipe_handle_data_t * ph,usb_flags_t usb_flags)1296 xhci_hcdi_pipe_stop_intr_polling(usba_pipe_handle_data_t *ph,
1297 usb_flags_t usb_flags)
1298 {
1299 return (xhci_hcdi_pipe_poll_fini(ph, B_FALSE));
1300 }
1301
1302 static int
xhci_hcdi_isoc_periodic(xhci_t * xhcip,usba_pipe_handle_data_t * ph,usb_isoc_req_t * usrp,usb_flags_t usb_flags)1303 xhci_hcdi_isoc_periodic(xhci_t *xhcip, usba_pipe_handle_data_t *ph,
1304 usb_isoc_req_t *usrp, usb_flags_t usb_flags)
1305 {
1306 int i;
1307 size_t count;
1308
1309 count = 0;
1310 for (i = 0; i < usrp->isoc_pkts_count; i++) {
1311 count += usrp->isoc_pkt_descr[i].isoc_pkt_length;
1312 }
1313
1314 return (xhci_hcdi_periodic_init(xhcip, ph, (usb_opaque_t)usrp, count,
1315 usb_flags));
1316 }
1317
1318 /*
1319 * This is used to create an isochronous request to send data out to the device.
1320 * This is a single one shot request, it is not something that we'll have to
1321 * repeat over and over.
1322 */
1323 static int
xhci_hcdi_isoc_oneshot(xhci_t * xhcip,usba_pipe_handle_data_t * ph,usb_isoc_req_t * usrp,usb_flags_t usb_flags)1324 xhci_hcdi_isoc_oneshot(xhci_t *xhcip, usba_pipe_handle_data_t *ph,
1325 usb_isoc_req_t *usrp, usb_flags_t usb_flags)
1326 {
1327 int i, ret;
1328 uint_t epid;
1329 size_t count, mblen;
1330 xhci_device_t *xd;
1331 xhci_endpoint_t *xep;
1332 xhci_transfer_t *xt;
1333
1334 count = 0;
1335 for (i = 0; i < usrp->isoc_pkts_count; i++) {
1336 count += usrp->isoc_pkt_descr[i].isoc_pkt_length;
1337 }
1338 mblen = MBLKL(usrp->isoc_data);
1339
1340 if (count != mblen) {
1341 return (USB_INVALID_ARGS);
1342 }
1343
1344 mutex_enter(&xhcip->xhci_lock);
1345 if (xhcip->xhci_state & XHCI_S_ERROR) {
1346 mutex_exit(&xhcip->xhci_lock);
1347 return (USB_HC_HARDWARE_ERROR);
1348 }
1349
1350 xd = usba_hcdi_get_device_private(ph->p_usba_device);
1351 epid = xhci_endpoint_pipe_to_epid(ph);
1352 if (xd->xd_endpoints[epid] == NULL) {
1353 xhci_error(xhcip, "asked to do isochronous transfer on slot "
1354 "%d, port %d, endpoint: %d, but no endpoint structure",
1355 xd->xd_slot, xd->xd_port, epid);
1356 mutex_exit(&xhcip->xhci_lock);
1357 return (USB_FAILURE);
1358 }
1359 xep = xd->xd_endpoints[epid];
1360 mutex_exit(&xhcip->xhci_lock);
1361
1362 xt = xhci_transfer_alloc(xhcip, xep, mblen, usrp->isoc_pkts_count,
1363 usb_flags);
1364 if (xt == NULL) {
1365 return (USB_NO_RESOURCES);
1366 }
1367 xt->xt_usba_req = (usb_opaque_t)usrp;
1368
1369 /*
1370 * USBA doesn't provide any real way for a timeout to be defined for an
1371 * isochronous event. However, since we technically aren't a periodic
1372 * endpoint, go ahead and always set the default timeout. It's better
1373 * than nothing.
1374 */
1375 xt->xt_timeout = HCDI_DEFAULT_TIMEOUT;
1376
1377 xhci_transfer_copy(xt, usrp->isoc_data->b_rptr, mblen, B_FALSE);
1378 if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORDEV) != DDI_FM_OK) {
1379 xhci_transfer_free(xhcip, xt);
1380 xhci_error(xhcip, "failed to synchronize isochronous "
1381 "transfer DMA memory on endpoint %u of "
1382 "device on slot %d and port %d: resetting "
1383 "device", xep->xep_num, xd->xd_slot,
1384 xd->xd_port);
1385 xhci_fm_runtime_reset(xhcip);
1386 return (USB_HC_HARDWARE_ERROR);
1387 }
1388
1389 /*
1390 * Fill in the ISOC data. Note, that we always use ASAP scheduling and
1391 * we don't support specifying the frame at this time, for better or
1392 * worse.
1393 */
1394 xhci_hcdi_isoc_transfer_fill(xd, xep, xt, usrp);
1395
1396 mutex_enter(&xhcip->xhci_lock);
1397 ret = xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE);
1398 mutex_exit(&xhcip->xhci_lock);
1399
1400 return (ret);
1401 }
1402
1403 static int
xhci_hcdi_pipe_isoc_xfer(usba_pipe_handle_data_t * ph,usb_isoc_req_t * usrp,usb_flags_t usb_flags)1404 xhci_hcdi_pipe_isoc_xfer(usba_pipe_handle_data_t *ph, usb_isoc_req_t *usrp,
1405 usb_flags_t usb_flags)
1406 {
1407 int ret;
1408 xhci_t *xhcip;
1409
1410 xhcip = xhci_hcdi_get_xhcip(ph);
1411
1412 /*
1413 * We don't support isochronous transactions on the root hub at all.
1414 * Always fail them if for some reason we end up here.
1415 */
1416 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
1417 return (USB_NOT_SUPPORTED);
1418 }
1419
1420 /*
1421 * We do not support being asked to set the frame ID at this time. We
1422 * require that everything specify the attribute
1423 * USB_ATTRS_ISOC_XFER_ASAP.
1424 */
1425 if (!(usrp->isoc_attributes & USB_ATTRS_ISOC_XFER_ASAP)) {
1426 return (USB_NOT_SUPPORTED);
1427 }
1428
1429 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
1430 /*
1431 * Note, there is no such thing as a non-periodic isochronous
1432 * incoming transfer.
1433 */
1434 ret = xhci_hcdi_isoc_periodic(xhcip, ph, usrp, usb_flags);
1435 } else {
1436 ret = xhci_hcdi_isoc_oneshot(xhcip, ph, usrp, usb_flags);
1437 }
1438
1439 return (ret);
1440 }
1441
1442 /* ARGSUSED */
1443 static int
xhci_hcdi_pipe_stop_isoc_polling(usba_pipe_handle_data_t * ph,usb_flags_t usb_flags)1444 xhci_hcdi_pipe_stop_isoc_polling(usba_pipe_handle_data_t *ph,
1445 usb_flags_t usb_flags)
1446 {
1447 return (xhci_hcdi_pipe_poll_fini(ph, B_FALSE));
1448 }
1449
1450 /*
1451 * This is asking us for the current frame number. The USBA expects this to
1452 * actually be a bit of a fiction, as it tries to maintain a frame number well
1453 * beyond what the hardware actually contains in its registers. Hardware
1454 * basically has a 14-bit counter, whereas we need to have a constant amount of
1455 * milliseconds.
1456 *
1457 * Today, no client drivers actually use this API and everyone specifies the
1458 * attribute to say that we should schedule things ASAP. So until we have some
1459 * real device that want this functionality, we're going to fail.
1460 */
1461 /* ARGSUSED */
1462 static int
xhci_hcdi_get_current_frame_number(usba_device_t * usba_device,usb_frame_number_t * frame_number)1463 xhci_hcdi_get_current_frame_number(usba_device_t *usba_device,
1464 usb_frame_number_t *frame_number)
1465 {
1466 return (USB_FAILURE);
1467 }
1468
1469 /*
1470 * See the comments around the XHCI_ISOC_MAX_TRB macro for more information.
1471 */
1472 /* ARGSUSED */
1473 static int
xhci_hcdi_get_max_isoc_pkts(usba_device_t * usba_device,uint_t * max_isoc_pkts_per_request)1474 xhci_hcdi_get_max_isoc_pkts(usba_device_t *usba_device,
1475 uint_t *max_isoc_pkts_per_request)
1476 {
1477 *max_isoc_pkts_per_request = XHCI_ISOC_MAX_TRB;
1478 return (USB_SUCCESS);
1479 }
1480
1481 /*
1482 * VERSION 2 ops and helpers
1483 */
1484
1485 static void
xhci_hcdi_device_free(xhci_device_t * xd)1486 xhci_hcdi_device_free(xhci_device_t *xd)
1487 {
1488 xhci_dma_free(&xd->xd_ictx);
1489 xhci_dma_free(&xd->xd_octx);
1490 mutex_destroy(&xd->xd_imtx);
1491 kmem_free(xd, sizeof (xhci_device_t));
1492 }
1493
1494 /*
1495 * Calculate the device's route string. In USB 3.0 the route string is a 20-bit
1496 * number. Each four bits represent a port number attached to a deeper hub.
1497 * Particularly it represents the port on that current hub that you need to go
1498 * down to reach the next device. Bits 0-3 represent the first *external* hub.
1499 * So a device connected to a root hub has a route string of zero. Imagine the
1500 * following set of devices:
1501 *
1502 * . port 2 . port 5
1503 * . .
1504 * +----------+ . +--------+ . +-------+
1505 * | root hub |-*->| hub 1 |-*->| hub 2 |
1506 * +----------+ +--------+ +-------+
1507 * * . port 12 * . port 8 * . port 1
1508 * v v v
1509 * +-------+ +-------+ +-------+
1510 * | dev a | | dev b | | dev c |
1511 * +-------+ +-------+ +-------+
1512 *
1513 * So, based on the above diagram, device a should have a route string of 0,
1514 * because it's directly connected to the root port. Device b would simply have
1515 * a route string of 8. This is because it travels through one non-root hub, hub
1516 * 1, and it does so on port 8. The root ports basically don't matter. Device c
1517 * would then have a route string of 0x15, as it's first traversing through hub
1518 * 1 on port 2 and then hub 2 on port 5.
1519 *
1520 * Finally, it's worth mentioning that because it's a four bit field, if for
1521 * some reason a device has more than 15 ports, we just treat the value as 15.
1522 *
1523 * Note, as part of this, we also grab what port on the root hub this whole
1524 * chain is on, as we're required to store that information in the slot context.
1525 */
1526 static void
xhci_hcdi_device_route(usba_device_t * ud,uint32_t * routep,uint32_t * root_port)1527 xhci_hcdi_device_route(usba_device_t *ud, uint32_t *routep, uint32_t *root_port)
1528 {
1529 uint32_t route = 0;
1530 usba_device_t *hub = ud->usb_parent_hub;
1531 usba_device_t *port_dev = ud;
1532
1533 ASSERT(hub != NULL);
1534
1535 /*
1536 * Iterate over every hub, updating the route as we go. When we
1537 * encounter a hub without a parent, then we're at the root hub. At
1538 * which point, the port we want is on port_dev (the child of hub).
1539 */
1540 while (hub->usb_parent_hub != NULL) {
1541 uint32_t p;
1542
1543 p = port_dev->usb_port;
1544 if (p > 15)
1545 p = 15;
1546 route <<= 4;
1547 route |= p & 0xf;
1548 port_dev = hub;
1549 hub = hub->usb_parent_hub;
1550 }
1551
1552 ASSERT(port_dev->usb_parent_hub == hub);
1553 *root_port = port_dev->usb_port;
1554 *routep = XHCI_ROUTE_MASK(route);
1555 }
1556
1557 /*
1558 * If a low or full speed device is behind a high-speed device that is not a
1559 * root hub, then we must include the port and slot of that device. USBA already
1560 * stores this device in the usb_hs_hub_usba_dev member.
1561 */
1562 static uint32_t
xhci_hcdi_device_tt(usba_device_t * ud)1563 xhci_hcdi_device_tt(usba_device_t *ud)
1564 {
1565 uint32_t ret;
1566 xhci_device_t *xd;
1567
1568 if (ud->usb_port_status >= USBA_HIGH_SPEED_DEV)
1569 return (0);
1570
1571 if (ud->usb_hs_hub_usba_dev == NULL)
1572 return (0);
1573
1574 ASSERT(ud->usb_hs_hub_usba_dev != NULL);
1575 ASSERT(ud->usb_hs_hub_usba_dev->usb_parent_hub != NULL);
1576 xd = usba_hcdi_get_device_private(ud->usb_hs_hub_usba_dev);
1577 ASSERT(xd != NULL);
1578
1579 ret = XHCI_SCTX_SET_TT_HUB_SID(xd->xd_slot);
1580 ret |= XHCI_SCTX_SET_TT_PORT_NUM(ud->usb_hs_hub_usba_dev->usb_port);
1581
1582 return (ret);
1583 }
1584
1585 /*
1586 * Initialize a new device. This allocates a device slot from the controller,
1587 * which tranfers it to our control.
1588 */
1589 static int
xhci_hcdi_device_init(usba_device_t * ud,usb_port_t port,void ** hcdpp)1590 xhci_hcdi_device_init(usba_device_t *ud, usb_port_t port, void **hcdpp)
1591 {
1592 int ret, i;
1593 xhci_device_t *xd;
1594 ddi_device_acc_attr_t acc;
1595 ddi_dma_attr_t attr;
1596 xhci_t *xhcip = xhci_hcdi_get_xhcip_from_dev(ud);
1597 size_t isize, osize, incr;
1598 uint32_t route, rp, info, info2, tt;
1599
1600 xd = kmem_zalloc(sizeof (xhci_device_t), KM_SLEEP);
1601 xd->xd_port = port;
1602 xd->xd_usbdev = ud;
1603 mutex_init(&xd->xd_imtx, NULL, MUTEX_DRIVER,
1604 (void *)(uintptr_t)xhcip->xhci_intr_pri);
1605
1606 /*
1607 * The size of the context structures is based upon the presence of the
1608 * context flag which determines whether we have a 32-byte or 64-byte
1609 * context. Note that the input context always has to account for the
1610 * entire size of the xhci_input_contex_t, which is 32-bytes by default.
1611 */
1612 if (xhcip->xhci_caps.xcap_flags & XCAP_CSZ) {
1613 incr = 64;
1614 osize = XHCI_DEVICE_CONTEXT_64;
1615 isize = XHCI_DEVICE_CONTEXT_64 + incr;
1616 } else {
1617 incr = 32;
1618 osize = XHCI_DEVICE_CONTEXT_32;
1619 isize = XHCI_DEVICE_CONTEXT_32 + incr;
1620 }
1621
1622 xhci_dma_acc_attr(xhcip, &acc);
1623 xhci_dma_dma_attr(xhcip, &attr);
1624 if (xhci_dma_alloc(xhcip, &xd->xd_ictx, &attr, &acc, B_TRUE,
1625 isize, B_FALSE) == B_FALSE) {
1626 xhci_hcdi_device_free(xd);
1627 return (USB_NO_RESOURCES);
1628 }
1629
1630 xd->xd_input = (xhci_input_context_t *)xd->xd_ictx.xdb_va;
1631 xd->xd_slotin = (xhci_slot_context_t *)(xd->xd_ictx.xdb_va + incr);
1632 for (i = 0; i < XHCI_NUM_ENDPOINTS; i++) {
1633 xd->xd_endin[i] =
1634 (xhci_endpoint_context_t *)(xd->xd_ictx.xdb_va +
1635 (i + 2) * incr);
1636 }
1637
1638 if (xhci_dma_alloc(xhcip, &xd->xd_octx, &attr, &acc, B_TRUE,
1639 osize, B_FALSE) == B_FALSE) {
1640 xhci_hcdi_device_free(xd);
1641 return (USB_NO_RESOURCES);
1642 }
1643 xd->xd_slotout = (xhci_slot_context_t *)xd->xd_octx.xdb_va;
1644 for (i = 0; i < XHCI_NUM_ENDPOINTS; i++) {
1645 xd->xd_endout[i] =
1646 (xhci_endpoint_context_t *)(xd->xd_octx.xdb_va +
1647 (i + 1) * incr);
1648 }
1649
1650 ret = xhci_command_enable_slot(xhcip, &xd->xd_slot);
1651 if (ret != USB_SUCCESS) {
1652 xhci_hcdi_device_free(xd);
1653 return (ret);
1654 }
1655
1656 /*
1657 * These are the default slot context and the endpoint zero context that
1658 * we're enabling. See 4.3.3.
1659 */
1660 xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0) |
1661 XHCI_INCTX_MASK_DCI(1));
1662
1663 /*
1664 * Note, we never need to set the MTT bit as illumos never enables the
1665 * alternate MTT interface.
1666 */
1667 xhci_hcdi_device_route(ud, &route, &rp);
1668 info = XHCI_SCTX_SET_ROUTE(route) | XHCI_SCTX_SET_DCI(1);
1669 switch (ud->usb_port_status) {
1670 case USBA_LOW_SPEED_DEV:
1671 info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_LOW);
1672 break;
1673 case USBA_HIGH_SPEED_DEV:
1674 info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_HIGH);
1675 break;
1676 case USBA_FULL_SPEED_DEV:
1677 info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_FULL);
1678 break;
1679 case USBA_SUPER_SPEED_DEV:
1680 default:
1681 info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_SUPER);
1682 break;
1683 }
1684 info2 = XHCI_SCTX_SET_RHPORT(rp);
1685 tt = XHCI_SCTX_SET_IRQ_TARGET(0);
1686 tt |= xhci_hcdi_device_tt(ud);
1687
1688 xd->xd_slotin->xsc_info = LE_32(info);
1689 xd->xd_slotin->xsc_info2 = LE_32(info2);
1690 xd->xd_slotin->xsc_tt = LE_32(tt);
1691
1692 if ((ret = xhci_endpoint_init(xhcip, xd, NULL)) != 0) {
1693 (void) xhci_command_disable_slot(xhcip, xd->xd_slot);
1694 xhci_hcdi_device_free(xd);
1695 return (USB_HC_HARDWARE_ERROR);
1696 }
1697
1698 if (xhci_context_slot_output_init(xhcip, xd) != B_TRUE) {
1699 (void) xhci_command_disable_slot(xhcip, xd->xd_slot);
1700 xhci_endpoint_fini(xd, 0);
1701 xhci_hcdi_device_free(xd);
1702 return (USB_HC_HARDWARE_ERROR);
1703 }
1704
1705 if ((ret = xhci_command_set_address(xhcip, xd, B_TRUE)) != 0) {
1706 (void) xhci_command_disable_slot(xhcip, xd->xd_slot);
1707 xhci_context_slot_output_fini(xhcip, xd);
1708 xhci_endpoint_fini(xd, 0);
1709 xhci_hcdi_device_free(xd);
1710 return (ret);
1711 }
1712
1713 mutex_enter(&xhcip->xhci_lock);
1714 list_insert_tail(&xhcip->xhci_usba.xa_devices, xd);
1715 mutex_exit(&xhcip->xhci_lock);
1716
1717 *hcdpp = xd;
1718 return (ret);
1719 }
1720
1721 /*
1722 * We're tearing down a device now.
1723 */
1724 static void
xhci_hcdi_device_fini(usba_device_t * ud,void * hcdp)1725 xhci_hcdi_device_fini(usba_device_t *ud, void *hcdp)
1726 {
1727 int ret;
1728 xhci_endpoint_t *xep;
1729 xhci_device_t *xd;
1730 xhci_t *xhcip;
1731
1732 /*
1733 * Right now, it's theoretically possible that USBA may try and call
1734 * us here even if we hadn't successfully finished the device_init()
1735 * endpoint. We should probably modify the USBA to make sure that this
1736 * can't happen.
1737 */
1738 if (hcdp == NULL)
1739 return;
1740
1741 xd = hcdp;
1742 xhcip = xhci_hcdi_get_xhcip_from_dev(ud);
1743
1744 /*
1745 * Make sure we have no timeout running on the default endpoint still.
1746 */
1747 xep = xd->xd_endpoints[XHCI_DEFAULT_ENDPOINT];
1748 mutex_enter(&xhcip->xhci_lock);
1749 xep->xep_state |= XHCI_ENDPOINT_TEARDOWN;
1750 mutex_exit(&xhcip->xhci_lock);
1751 (void) untimeout(xep->xep_timeout);
1752
1753 /*
1754 * Go ahead and disable the slot. There's no reason to do anything
1755 * special about the default endpoint as it will be disabled as a part
1756 * of the slot disabling. However, if this all fails, we'll leave this
1757 * sitting here in a failed state, eating up a device slot. It is
1758 * unlikely this will occur.
1759 */
1760 ret = xhci_command_disable_slot(xhcip, xd->xd_slot);
1761 if (ret != USB_SUCCESS) {
1762 xhci_error(xhcip, "failed to disable slot %d: %d",
1763 xd->xd_slot, ret);
1764 return;
1765 }
1766
1767 xhci_context_slot_output_fini(xhcip, xd);
1768
1769 /*
1770 * Once the slot is disabled, we can free any endpoints that were
1771 * opened.
1772 */
1773 for (uint_t n = 0; n < XHCI_NUM_ENDPOINTS; n++) {
1774 if (xd->xd_endpoints[n] != NULL) {
1775 xhci_endpoint_fini(xd, n);
1776 }
1777 }
1778
1779 mutex_enter(&xhcip->xhci_lock);
1780 list_remove(&xhcip->xhci_usba.xa_devices, xd);
1781 mutex_exit(&xhcip->xhci_lock);
1782
1783 xhci_hcdi_device_free(xd);
1784 }
1785
1786 /*
1787 * Synchronously attempt to set the device address. For xHCI this involves it
1788 * deciding what address to use.
1789 */
1790 static int
xhci_hcdi_device_address(usba_device_t * ud)1791 xhci_hcdi_device_address(usba_device_t *ud)
1792 {
1793 int ret;
1794 xhci_t *xhcip = xhci_hcdi_get_xhcip_from_dev(ud);
1795 xhci_device_t *xd = usba_hcdi_get_device_private(ud);
1796 xhci_endpoint_t *xep;
1797
1798 mutex_enter(&xhcip->xhci_lock);
1799
1800 /*
1801 * This device may already be addressed from the perspective of the xhci
1802 * controller. For example, the device this represents may have been
1803 * unconfigured, which does not actually remove the slot or other
1804 * information, merely tears down all the active use of it and the child
1805 * driver. In such cases, if we're already addressed, just return
1806 * success. The actual USB address is a fiction for USBA anyways.
1807 */
1808 if (xd->xd_addressed == B_TRUE) {
1809 mutex_exit(&xhcip->xhci_lock);
1810 return (USB_SUCCESS);
1811 }
1812
1813 ASSERT(xd->xd_addressed == B_FALSE);
1814 xd->xd_addressed = B_TRUE;
1815 VERIFY3P(xd->xd_endpoints[XHCI_DEFAULT_ENDPOINT], !=, NULL);
1816 xep = xd->xd_endpoints[XHCI_DEFAULT_ENDPOINT];
1817 mutex_exit(&xhcip->xhci_lock);
1818
1819 if ((ret = xhci_endpoint_setup_default_context(xhcip, xd, xep)) != 0) {
1820 ASSERT(ret == EIO);
1821 return (USB_HC_HARDWARE_ERROR);
1822 }
1823
1824 ret = xhci_command_set_address(xhcip, xd, B_FALSE);
1825
1826 if (ret != USB_SUCCESS) {
1827 mutex_enter(&xhcip->xhci_lock);
1828 xd->xd_addressed = B_FALSE;
1829 mutex_exit(&xhcip->xhci_lock);
1830 }
1831
1832 return (ret);
1833 }
1834
1835 /*
1836 * This is called relatively early on in a hub's life time. At this point, it's
1837 * descriptors have all been pulled and the default control pipe is still open.
1838 * What we need to do is go through and update the slot context to indicate that
1839 * this is a hub, otherwise, the controller will never let us speak to
1840 * downstream ports.
1841 */
1842 static int
xhci_hcdi_hub_update(usba_device_t * ud,uint8_t nports,uint8_t tt)1843 xhci_hcdi_hub_update(usba_device_t *ud, uint8_t nports, uint8_t tt)
1844 {
1845 int ret;
1846 xhci_t *xhcip = xhci_hcdi_get_xhcip_from_dev(ud);
1847 xhci_device_t *xd = usba_hcdi_get_device_private(ud);
1848
1849 if (xd == NULL)
1850 return (USB_FAILURE);
1851
1852 if (ud->usb_hubdi == NULL) {
1853 return (USB_FAILURE);
1854 }
1855
1856 mutex_enter(&xd->xd_imtx);
1857
1858 /*
1859 * Note, that usba never sets the interface of a hub to Multi TT. Hence
1860 * why we're never setting the MTT bit in xsc_info.
1861 */
1862 xd->xd_slotin->xsc_info |= LE_32(XHCI_SCTX_SET_HUB(1));
1863 xd->xd_slotin->xsc_info2 |= LE_32(XHCI_SCTX_SET_NPORTS(nports));
1864 if (ud->usb_port_status == USBA_HIGH_SPEED_DEV)
1865 xd->xd_slotin->xsc_tt |= LE_32(XHCI_SCTX_SET_TT_THINK_TIME(tt));
1866
1867 /*
1868 * We're only updating the slot context, no endpoint contexts should be
1869 * touched.
1870 */
1871 xd->xd_input->xic_drop_flags = LE_32(0);
1872 xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0));
1873
1874 ret = xhci_command_evaluate_context(xhcip, xd);
1875 mutex_exit(&xd->xd_imtx);
1876 return (ret);
1877 }
1878
1879 void
xhci_hcd_fini(xhci_t * xhcip)1880 xhci_hcd_fini(xhci_t *xhcip)
1881 {
1882 usba_hcdi_unregister(xhcip->xhci_dip);
1883 usba_free_hcdi_ops(xhcip->xhci_usba.xa_ops);
1884 list_destroy(&xhcip->xhci_usba.xa_pipes);
1885 list_destroy(&xhcip->xhci_usba.xa_devices);
1886 }
1887
1888 int
xhci_hcd_init(xhci_t * xhcip)1889 xhci_hcd_init(xhci_t *xhcip)
1890 {
1891 usba_hcdi_register_args_t hreg;
1892 usba_hcdi_ops_t *ops;
1893
1894 ops = usba_alloc_hcdi_ops();
1895 VERIFY(ops != NULL);
1896
1897 ops->usba_hcdi_ops_version = HCDI_OPS_VERSION;
1898 ops->usba_hcdi_dip = xhcip->xhci_dip;
1899
1900 ops->usba_hcdi_pm_support = xhci_hcdi_pm_support;
1901 ops->usba_hcdi_pipe_open = xhci_hcdi_pipe_open;
1902 ops->usba_hcdi_pipe_close = xhci_hcdi_pipe_close;
1903 ops->usba_hcdi_pipe_reset = xhci_hcdi_pipe_reset;
1904 ops->usba_hcdi_pipe_reset_data_toggle =
1905 xhci_hcdi_pipe_reset_data_toggle;
1906 ops->usba_hcdi_pipe_ctrl_xfer = xhci_hcdi_pipe_ctrl_xfer;
1907 ops->usba_hcdi_bulk_transfer_size = xhci_hcdi_bulk_transfer_size;
1908 ops->usba_hcdi_pipe_bulk_xfer = xhci_hcdi_pipe_bulk_xfer;
1909 ops->usba_hcdi_pipe_intr_xfer = xhci_hcdi_pipe_intr_xfer;
1910 ops->usba_hcdi_pipe_stop_intr_polling =
1911 xhci_hcdi_pipe_stop_intr_polling;
1912 ops->usba_hcdi_pipe_isoc_xfer = xhci_hcdi_pipe_isoc_xfer;
1913 ops->usba_hcdi_pipe_stop_isoc_polling =
1914 xhci_hcdi_pipe_stop_isoc_polling;
1915 ops->usba_hcdi_get_current_frame_number =
1916 xhci_hcdi_get_current_frame_number;
1917 ops->usba_hcdi_get_max_isoc_pkts = xhci_hcdi_get_max_isoc_pkts;
1918 ops->usba_hcdi_console_input_init = xhci_hcdi_console_input_init;
1919 ops->usba_hcdi_console_input_fini = xhci_hcdi_console_input_fini;
1920 ops->usba_hcdi_console_input_enter = xhci_hcdi_console_input_enter;
1921 ops->usba_hcdi_console_read = xhci_hcdi_console_read;
1922 ops->usba_hcdi_console_input_exit = xhci_hcdi_console_input_exit;
1923
1924 ops->usba_hcdi_console_output_init = xhci_hcdi_console_output_init;
1925 ops->usba_hcdi_console_output_fini = xhci_hcdi_console_output_fini;
1926 ops->usba_hcdi_console_output_enter = xhci_hcdi_console_output_enter;
1927 ops->usba_hcdi_console_write = xhci_hcdi_console_write;
1928 ops->usba_hcdi_console_output_exit = xhci_hcdi_console_output_exit;
1929
1930 ops->usba_hcdi_device_init = xhci_hcdi_device_init;
1931 ops->usba_hcdi_device_fini = xhci_hcdi_device_fini;
1932 ops->usba_hcdi_device_address = xhci_hcdi_device_address;
1933 ops->usba_hcdi_hub_update = xhci_hcdi_hub_update;
1934
1935 hreg.usba_hcdi_register_version = HCDI_REGISTER_VERSION;
1936 hreg.usba_hcdi_register_dip = xhcip->xhci_dip;
1937 hreg.usba_hcdi_register_ops = ops;
1938
1939 /*
1940 * We're required to give xhci a set of DMA attributes that it may loan
1941 * out to other devices. Therefore we'll be conservative with what we
1942 * end up giving it.
1943 */
1944 xhci_dma_dma_attr(xhcip, &xhcip->xhci_usba.xa_dma_attr);
1945 hreg.usba_hcdi_register_dma_attr = &xhcip->xhci_usba.xa_dma_attr;
1946
1947 hreg.usba_hcdi_register_iblock_cookie =
1948 (ddi_iblock_cookie_t)(uintptr_t)xhcip->xhci_intr_pri;
1949
1950 if (usba_hcdi_register(&hreg, 0) != DDI_SUCCESS) {
1951 usba_free_hcdi_ops(ops);
1952 return (DDI_FAILURE);
1953 }
1954
1955 xhcip->xhci_usba.xa_ops = ops;
1956
1957 list_create(&xhcip->xhci_usba.xa_devices, sizeof (xhci_device_t),
1958 offsetof(xhci_device_t, xd_link));
1959 list_create(&xhcip->xhci_usba.xa_pipes, sizeof (xhci_pipe_t),
1960 offsetof(xhci_pipe_t, xp_link));
1961
1962
1963 return (DDI_SUCCESS);
1964 }
1965