1/* $FreeBSD$ */
2/*-
3 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 *
5 * Copyright (c) 2010 Hans Petter Selasky. All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29/*
30 * USB eXtensible Host Controller Interface, a.k.a. USB 3.0 controller.
31 *
32 * The XHCI 1.0 spec can be found at
33 * http://www.intel.com/technology/usb/download/xHCI_Specification_for_USB.pdf
34 * and the USB 3.0 spec at
35 * http://www.usb.org/developers/docs/usb_30_spec_060910.zip
36 */
37
38/*
39 * A few words about the design implementation: This driver emulates
40 * the concept about TDs which is found in EHCI specification. This
41 * way we achieve that the USB controller drivers look similar to
42 * eachother which makes it easier to understand the code.
43 */
44
45#ifdef USB_GLOBAL_INCLUDE_FILE
46#include USB_GLOBAL_INCLUDE_FILE
47#else
48#include <sys/stdint.h>
49#include <sys/stddef.h>
50#include <sys/param.h>
51#include <sys/queue.h>
52#include <sys/types.h>
53#include <sys/systm.h>
54#include <sys/kernel.h>
55#include <sys/bus.h>
56#include <sys/module.h>
57#include <sys/lock.h>
58#include <sys/mutex.h>
59#include <sys/condvar.h>
60#include <sys/sysctl.h>
61#include <sys/sx.h>
62#include <sys/unistd.h>
63#include <sys/callout.h>
64#include <sys/malloc.h>
65#include <sys/priv.h>
66
67#include <dev/usb/usb.h>
68#include <dev/usb/usbdi.h>
69
70#define	USB_DEBUG_VAR xhcidebug
71
72#include <dev/usb/usb_core.h>
73#include <dev/usb/usb_debug.h>
74#include <dev/usb/usb_busdma.h>
75#include <dev/usb/usb_process.h>
76#include <dev/usb/usb_transfer.h>
77#include <dev/usb/usb_device.h>
78#include <dev/usb/usb_hub.h>
79#include <dev/usb/usb_util.h>
80
81#include <dev/usb/usb_controller.h>
82#include <dev/usb/usb_bus.h>
83#endif			/* USB_GLOBAL_INCLUDE_FILE */
84
85#include <dev/usb/controller/xhci.h>
86#include <dev/usb/controller/xhcireg.h>
87
88#define	XHCI_BUS2SC(bus) \
89   ((struct xhci_softc *)(((uint8_t *)(bus)) - \
90    ((uint8_t *)&(((struct xhci_softc *)0)->sc_bus))))
91
92static SYSCTL_NODE(_hw_usb, OID_AUTO, xhci, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
93    "USB XHCI");
94
95static int xhcistreams;
96SYSCTL_INT(_hw_usb_xhci, OID_AUTO, streams, CTLFLAG_RWTUN,
97    &xhcistreams, 0, "Set to enable streams mode support");
98
99static int xhcictlquirk = 1;
100SYSCTL_INT(_hw_usb_xhci, OID_AUTO, ctlquirk, CTLFLAG_RWTUN,
101    &xhcictlquirk, 0, "Set to enable control endpoint quirk");
102
103#ifdef USB_DEBUG
104static int xhcidebug;
105static int xhciroute;
106static int xhcipolling;
107static int xhcidma32;
108static int xhcictlstep;
109
110SYSCTL_INT(_hw_usb_xhci, OID_AUTO, debug, CTLFLAG_RWTUN,
111    &xhcidebug, 0, "Debug level");
112SYSCTL_INT(_hw_usb_xhci, OID_AUTO, xhci_port_route, CTLFLAG_RWTUN,
113    &xhciroute, 0, "Routing bitmap for switching EHCI ports to the XHCI controller");
114SYSCTL_INT(_hw_usb_xhci, OID_AUTO, use_polling, CTLFLAG_RWTUN,
115    &xhcipolling, 0, "Set to enable software interrupt polling for the XHCI controller");
116SYSCTL_INT(_hw_usb_xhci, OID_AUTO, dma32, CTLFLAG_RWTUN,
117    &xhcidma32, 0, "Set to only use 32-bit DMA for the XHCI controller");
118SYSCTL_INT(_hw_usb_xhci, OID_AUTO, ctlstep, CTLFLAG_RWTUN,
119    &xhcictlstep, 0, "Set to enable control endpoint status stage stepping");
120#else
121#define	xhciroute 0
122#define	xhcidma32 0
123#define	xhcictlstep 0
124#endif
125
126#define	XHCI_INTR_ENDPT 1
127
128struct xhci_std_temp {
129	struct xhci_softc	*sc;
130	struct usb_page_cache	*pc;
131	struct xhci_td		*td;
132	struct xhci_td		*td_next;
133	uint32_t		len;
134	uint32_t		offset;
135	uint32_t		max_packet_size;
136	uint32_t		average;
137	uint16_t		isoc_delta;
138	uint16_t		isoc_frame;
139	uint8_t			shortpkt;
140	uint8_t			multishort;
141	uint8_t			last_frame;
142	uint8_t			trb_type;
143	uint8_t			direction;
144	uint8_t			tbc;
145	uint8_t			tlbpc;
146	uint8_t			step_td;
147	uint8_t			do_isoc_sync;
148};
149
150static void	xhci_do_poll(struct usb_bus *);
151static void	xhci_device_done(struct usb_xfer *, usb_error_t);
152static void	xhci_root_intr(struct xhci_softc *);
153static void	xhci_free_device_ext(struct usb_device *);
154static struct xhci_endpoint_ext *xhci_get_endpoint_ext(struct usb_device *,
155		    struct usb_endpoint_descriptor *);
156static usb_proc_callback_t xhci_configure_msg;
157static usb_error_t xhci_configure_device(struct usb_device *);
158static usb_error_t xhci_configure_endpoint(struct usb_device *,
159		   struct usb_endpoint_descriptor *, struct xhci_endpoint_ext *,
160		   uint16_t, uint8_t, uint8_t, uint8_t, uint16_t, uint16_t,
161		   uint8_t);
162static usb_error_t xhci_configure_mask(struct usb_device *,
163		    uint32_t, uint8_t);
164static usb_error_t xhci_cmd_evaluate_ctx(struct xhci_softc *,
165		    uint64_t, uint8_t);
166static void xhci_endpoint_doorbell(struct usb_xfer *);
167static void xhci_ctx_set_le32(struct xhci_softc *sc, volatile uint32_t *ptr, uint32_t val);
168static uint32_t xhci_ctx_get_le32(struct xhci_softc *sc, volatile uint32_t *ptr);
169static void xhci_ctx_set_le64(struct xhci_softc *sc, volatile uint64_t *ptr, uint64_t val);
170#ifdef USB_DEBUG
171static uint64_t xhci_ctx_get_le64(struct xhci_softc *sc, volatile uint64_t *ptr);
172#endif
173
174static const struct usb_bus_methods xhci_bus_methods;
175
176#ifdef USB_DEBUG
177static void
178xhci_dump_trb(struct xhci_trb *trb)
179{
180	DPRINTFN(5, "trb = %p\n", trb);
181	DPRINTFN(5, "qwTrb0 = 0x%016llx\n", (long long)le64toh(trb->qwTrb0));
182	DPRINTFN(5, "dwTrb2 = 0x%08x\n", le32toh(trb->dwTrb2));
183	DPRINTFN(5, "dwTrb3 = 0x%08x\n", le32toh(trb->dwTrb3));
184}
185
186static void
187xhci_dump_endpoint(struct xhci_softc *sc, struct xhci_endp_ctx *pep)
188{
189	DPRINTFN(5, "pep = %p\n", pep);
190	DPRINTFN(5, "dwEpCtx0=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx0));
191	DPRINTFN(5, "dwEpCtx1=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx1));
192	DPRINTFN(5, "qwEpCtx2=0x%016llx\n", (long long)xhci_ctx_get_le64(sc, &pep->qwEpCtx2));
193	DPRINTFN(5, "dwEpCtx4=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx4));
194	DPRINTFN(5, "dwEpCtx5=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx5));
195	DPRINTFN(5, "dwEpCtx6=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx6));
196	DPRINTFN(5, "dwEpCtx7=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx7));
197}
198
199static void
200xhci_dump_device(struct xhci_softc *sc, struct xhci_slot_ctx *psl)
201{
202	DPRINTFN(5, "psl = %p\n", psl);
203	DPRINTFN(5, "dwSctx0=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx0));
204	DPRINTFN(5, "dwSctx1=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx1));
205	DPRINTFN(5, "dwSctx2=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx2));
206	DPRINTFN(5, "dwSctx3=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx3));
207}
208#endif
209
210uint8_t
211xhci_use_polling(void)
212{
213#ifdef USB_DEBUG
214	return (xhcipolling != 0);
215#else
216	return (0);
217#endif
218}
219
220static void
221xhci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb)
222{
223	struct xhci_softc *sc = XHCI_BUS2SC(bus);
224	uint16_t i;
225
226	cb(bus, &sc->sc_hw.root_pc, &sc->sc_hw.root_pg,
227	   sizeof(struct xhci_hw_root), XHCI_PAGE_SIZE);
228
229	cb(bus, &sc->sc_hw.ctx_pc, &sc->sc_hw.ctx_pg,
230	   sizeof(struct xhci_dev_ctx_addr), XHCI_PAGE_SIZE);
231
232	for (i = 0; i != sc->sc_noscratch; i++) {
233		cb(bus, &sc->sc_hw.scratch_pc[i], &sc->sc_hw.scratch_pg[i],
234		    XHCI_PAGE_SIZE, XHCI_PAGE_SIZE);
235	}
236}
237
238static void
239xhci_ctx_set_le32(struct xhci_softc *sc, volatile uint32_t *ptr, uint32_t val)
240{
241	if (sc->sc_ctx_is_64_byte) {
242		uint32_t offset;
243		/* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */
244		/* all contexts are initially 32-bytes */
245		offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U));
246		ptr = (volatile uint32_t *)(((volatile uint8_t *)ptr) + offset);
247	}
248	*ptr = htole32(val);
249}
250
251static uint32_t
252xhci_ctx_get_le32(struct xhci_softc *sc, volatile uint32_t *ptr)
253{
254	if (sc->sc_ctx_is_64_byte) {
255		uint32_t offset;
256		/* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */
257		/* all contexts are initially 32-bytes */
258		offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U));
259		ptr = (volatile uint32_t *)(((volatile uint8_t *)ptr) + offset);
260	}
261	return (le32toh(*ptr));
262}
263
264static void
265xhci_ctx_set_le64(struct xhci_softc *sc, volatile uint64_t *ptr, uint64_t val)
266{
267	if (sc->sc_ctx_is_64_byte) {
268		uint32_t offset;
269		/* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */
270		/* all contexts are initially 32-bytes */
271		offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U));
272		ptr = (volatile uint64_t *)(((volatile uint8_t *)ptr) + offset);
273	}
274	*ptr = htole64(val);
275}
276
277#ifdef USB_DEBUG
278static uint64_t
279xhci_ctx_get_le64(struct xhci_softc *sc, volatile uint64_t *ptr)
280{
281	if (sc->sc_ctx_is_64_byte) {
282		uint32_t offset;
283		/* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */
284		/* all contexts are initially 32-bytes */
285		offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U));
286		ptr = (volatile uint64_t *)(((volatile uint8_t *)ptr) + offset);
287	}
288	return (le64toh(*ptr));
289}
290#endif
291
292static int
293xhci_reset_command_queue_locked(struct xhci_softc *sc)
294{
295	struct usb_page_search buf_res;
296	struct xhci_hw_root *phwr;
297	uint64_t addr;
298	uint32_t temp;
299
300	DPRINTF("\n");
301
302	temp = XREAD4(sc, oper, XHCI_CRCR_LO);
303	if (temp & XHCI_CRCR_LO_CRR) {
304		DPRINTF("Command ring running\n");
305		temp &= ~(XHCI_CRCR_LO_CS | XHCI_CRCR_LO_CA);
306
307		/*
308		 * Try to abort the last command as per section
309		 * 4.6.1.2 "Aborting a Command" of the XHCI
310		 * specification:
311		 */
312
313		/* stop and cancel */
314		XWRITE4(sc, oper, XHCI_CRCR_LO, temp | XHCI_CRCR_LO_CS);
315		XWRITE4(sc, oper, XHCI_CRCR_HI, 0);
316
317		XWRITE4(sc, oper, XHCI_CRCR_LO, temp | XHCI_CRCR_LO_CA);
318		XWRITE4(sc, oper, XHCI_CRCR_HI, 0);
319
320 		/* wait 250ms */
321 		usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 4);
322
323		/* check if command ring is still running */
324		temp = XREAD4(sc, oper, XHCI_CRCR_LO);
325		if (temp & XHCI_CRCR_LO_CRR) {
326			DPRINTF("Comand ring still running\n");
327			return (USB_ERR_IOERROR);
328		}
329	}
330
331	/* reset command ring */
332	sc->sc_command_ccs = 1;
333	sc->sc_command_idx = 0;
334
335	usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res);
336
337	/* set up command ring control base address */
338	addr = buf_res.physaddr;
339	phwr = buf_res.buffer;
340	addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_commands[0];
341
342	DPRINTF("CRCR=0x%016llx\n", (unsigned long long)addr);
343
344	memset(phwr->hwr_commands, 0, sizeof(phwr->hwr_commands));
345	phwr->hwr_commands[XHCI_MAX_COMMANDS - 1].qwTrb0 = htole64(addr);
346
347	usb_pc_cpu_flush(&sc->sc_hw.root_pc);
348
349	XWRITE4(sc, oper, XHCI_CRCR_LO, ((uint32_t)addr) | XHCI_CRCR_LO_RCS);
350	XWRITE4(sc, oper, XHCI_CRCR_HI, (uint32_t)(addr >> 32));
351
352	return (0);
353}
354
355usb_error_t
356xhci_start_controller(struct xhci_softc *sc)
357{
358	struct usb_page_search buf_res;
359	struct xhci_hw_root *phwr;
360	struct xhci_dev_ctx_addr *pdctxa;
361	usb_error_t err;
362	uint64_t addr;
363	uint32_t temp;
364	uint16_t i;
365
366	DPRINTF("\n");
367
368	sc->sc_event_ccs = 1;
369	sc->sc_event_idx = 0;
370	sc->sc_command_ccs = 1;
371	sc->sc_command_idx = 0;
372
373	err = xhci_reset_controller(sc);
374	if (err)
375		return (err);
376
377	/* set up number of device slots */
378	DPRINTF("CONFIG=0x%08x -> 0x%08x\n",
379	    XREAD4(sc, oper, XHCI_CONFIG), sc->sc_noslot);
380
381	XWRITE4(sc, oper, XHCI_CONFIG, sc->sc_noslot);
382
383	temp = XREAD4(sc, oper, XHCI_USBSTS);
384
385	/* clear interrupts */
386	XWRITE4(sc, oper, XHCI_USBSTS, temp);
387	/* disable all device notifications */
388	XWRITE4(sc, oper, XHCI_DNCTRL, 0);
389
390	/* set up device context base address */
391	usbd_get_page(&sc->sc_hw.ctx_pc, 0, &buf_res);
392	pdctxa = buf_res.buffer;
393	memset(pdctxa, 0, sizeof(*pdctxa));
394
395	addr = buf_res.physaddr;
396	addr += (uintptr_t)&((struct xhci_dev_ctx_addr *)0)->qwSpBufPtr[0];
397
398	/* slot 0 points to the table of scratchpad pointers */
399	pdctxa->qwBaaDevCtxAddr[0] = htole64(addr);
400
401	for (i = 0; i != sc->sc_noscratch; i++) {
402		struct usb_page_search buf_scp;
403		usbd_get_page(&sc->sc_hw.scratch_pc[i], 0, &buf_scp);
404		pdctxa->qwSpBufPtr[i] = htole64((uint64_t)buf_scp.physaddr);
405	}
406
407	addr = buf_res.physaddr;
408
409	XWRITE4(sc, oper, XHCI_DCBAAP_LO, (uint32_t)addr);
410	XWRITE4(sc, oper, XHCI_DCBAAP_HI, (uint32_t)(addr >> 32));
411	XWRITE4(sc, oper, XHCI_DCBAAP_LO, (uint32_t)addr);
412	XWRITE4(sc, oper, XHCI_DCBAAP_HI, (uint32_t)(addr >> 32));
413
414	/* set up event table size */
415	DPRINTF("ERSTSZ=0x%08x -> 0x%08x\n",
416	    XREAD4(sc, runt, XHCI_ERSTSZ(0)), sc->sc_erst_max);
417
418	XWRITE4(sc, runt, XHCI_ERSTSZ(0), XHCI_ERSTS_SET(sc->sc_erst_max));
419
420	/* set up interrupt rate */
421	XWRITE4(sc, runt, XHCI_IMOD(0), sc->sc_imod_default);
422
423	usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res);
424
425	phwr = buf_res.buffer;
426	addr = buf_res.physaddr;
427	addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_events[0];
428
429	/* reset hardware root structure */
430	memset(phwr, 0, sizeof(*phwr));
431
432	phwr->hwr_ring_seg[0].qwEvrsTablePtr = htole64(addr);
433	phwr->hwr_ring_seg[0].dwEvrsTableSize = htole32(XHCI_MAX_EVENTS);
434
435	/*
436	 * PR 237666:
437	 *
438	 * According to the XHCI specification, the XWRITE4's to
439	 * XHCI_ERSTBA_LO and _HI lead to the XHCI to copy the
440	 * qwEvrsTablePtr and dwEvrsTableSize values above at that
441	 * time, as the XHCI initializes its event ring support. This
442	 * is before the event ring starts to pay attention to the
443	 * RUN/STOP bit. Thus, make sure the values are observable to
444	 * the XHCI before that point.
445	 */
446	usb_bus_mem_flush_all(&sc->sc_bus, &xhci_iterate_hw_softc);
447
448	DPRINTF("ERDP(0)=0x%016llx\n", (unsigned long long)addr);
449
450	XWRITE4(sc, runt, XHCI_ERDP_LO(0), (uint32_t)addr);
451	XWRITE4(sc, runt, XHCI_ERDP_HI(0), (uint32_t)(addr >> 32));
452
453	addr = buf_res.physaddr;
454
455	DPRINTF("ERSTBA(0)=0x%016llx\n", (unsigned long long)addr);
456
457	XWRITE4(sc, runt, XHCI_ERSTBA_LO(0), (uint32_t)addr);
458	XWRITE4(sc, runt, XHCI_ERSTBA_HI(0), (uint32_t)(addr >> 32));
459
460	/* set up interrupter registers */
461	temp = XREAD4(sc, runt, XHCI_IMAN(0));
462	temp |= XHCI_IMAN_INTR_ENA;
463	XWRITE4(sc, runt, XHCI_IMAN(0), temp);
464
465	/* set up command ring control base address */
466	addr = buf_res.physaddr;
467	addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_commands[0];
468
469	DPRINTF("CRCR=0x%016llx\n", (unsigned long long)addr);
470
471	XWRITE4(sc, oper, XHCI_CRCR_LO, ((uint32_t)addr) | XHCI_CRCR_LO_RCS);
472	XWRITE4(sc, oper, XHCI_CRCR_HI, (uint32_t)(addr >> 32));
473
474	phwr->hwr_commands[XHCI_MAX_COMMANDS - 1].qwTrb0 = htole64(addr);
475
476	usb_bus_mem_flush_all(&sc->sc_bus, &xhci_iterate_hw_softc);
477
478	/* Go! */
479	XWRITE4(sc, oper, XHCI_USBCMD, XHCI_CMD_RS |
480	    XHCI_CMD_INTE | XHCI_CMD_HSEE);
481
482	for (i = 0; i != 100; i++) {
483		usb_pause_mtx(NULL, hz / 100);
484		temp = XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_HCH;
485		if (!temp)
486			break;
487	}
488	if (temp) {
489		XWRITE4(sc, oper, XHCI_USBCMD, 0);
490		device_printf(sc->sc_bus.parent, "Run timeout.\n");
491		return (USB_ERR_IOERROR);
492	}
493
494	/* catch any lost interrupts */
495	xhci_do_poll(&sc->sc_bus);
496
497	if (sc->sc_port_route != NULL) {
498		/* Route all ports to the XHCI by default */
499		sc->sc_port_route(sc->sc_bus.parent,
500		    ~xhciroute, xhciroute);
501	}
502	return (0);
503}
504
505usb_error_t
506xhci_halt_controller(struct xhci_softc *sc)
507{
508	uint32_t temp;
509	uint16_t i;
510
511	DPRINTF("\n");
512
513	sc->sc_capa_off = 0;
514	sc->sc_oper_off = XREAD1(sc, capa, XHCI_CAPLENGTH);
515	sc->sc_runt_off = XREAD4(sc, capa, XHCI_RTSOFF) & ~0xF;
516	sc->sc_door_off = XREAD4(sc, capa, XHCI_DBOFF) & ~0x3;
517
518	/* Halt controller */
519	XWRITE4(sc, oper, XHCI_USBCMD, 0);
520
521	for (i = 0; i != 100; i++) {
522		usb_pause_mtx(NULL, hz / 100);
523		temp = XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_HCH;
524		if (temp)
525			break;
526	}
527
528	if (!temp) {
529		device_printf(sc->sc_bus.parent, "Controller halt timeout.\n");
530		return (USB_ERR_IOERROR);
531	}
532	return (0);
533}
534
535usb_error_t
536xhci_reset_controller(struct xhci_softc *sc)
537{
538	uint32_t temp = 0;
539	uint16_t i;
540
541	DPRINTF("\n");
542
543	/* Reset controller */
544	XWRITE4(sc, oper, XHCI_USBCMD, XHCI_CMD_HCRST);
545
546	for (i = 0; i != 100; i++) {
547		usb_pause_mtx(NULL, hz / 100);
548		temp = (XREAD4(sc, oper, XHCI_USBCMD) & XHCI_CMD_HCRST) |
549		    (XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_CNR);
550		if (!temp)
551			break;
552	}
553
554	if (temp) {
555		device_printf(sc->sc_bus.parent, "Controller "
556		    "reset timeout.\n");
557		return (USB_ERR_IOERROR);
558	}
559	return (0);
560}
561
562usb_error_t
563xhci_init(struct xhci_softc *sc, device_t self, uint8_t dma32)
564{
565	uint32_t temp;
566
567	DPRINTF("\n");
568
569	/* initialize some bus fields */
570	sc->sc_bus.parent = self;
571
572	/* set the bus revision */
573	sc->sc_bus.usbrev = USB_REV_3_0;
574
575	/* set up the bus struct */
576	sc->sc_bus.methods = &xhci_bus_methods;
577
578	/* set up devices array */
579	sc->sc_bus.devices = sc->sc_devices;
580	sc->sc_bus.devices_max = XHCI_MAX_DEVICES;
581
582	/* set default cycle state in case of early interrupts */
583	sc->sc_event_ccs = 1;
584	sc->sc_command_ccs = 1;
585
586	/* set up bus space offsets */
587	sc->sc_capa_off = 0;
588	sc->sc_oper_off = XREAD1(sc, capa, XHCI_CAPLENGTH);
589	sc->sc_runt_off = XREAD4(sc, capa, XHCI_RTSOFF) & ~0x1F;
590	sc->sc_door_off = XREAD4(sc, capa, XHCI_DBOFF) & ~0x3;
591
592	DPRINTF("CAPLENGTH=0x%x\n", sc->sc_oper_off);
593	DPRINTF("RUNTIMEOFFSET=0x%x\n", sc->sc_runt_off);
594	DPRINTF("DOOROFFSET=0x%x\n", sc->sc_door_off);
595
596	DPRINTF("xHCI version = 0x%04x\n", XREAD2(sc, capa, XHCI_HCIVERSION));
597
598	if (!(XREAD4(sc, oper, XHCI_PAGESIZE) & XHCI_PAGESIZE_4K)) {
599		device_printf(sc->sc_bus.parent, "Controller does "
600		    "not support 4K page size.\n");
601		return (ENXIO);
602	}
603
604	temp = XREAD4(sc, capa, XHCI_HCSPARAMS0);
605
606	DPRINTF("HCS0 = 0x%08x\n", temp);
607
608	/* set up context size */
609	if (XHCI_HCS0_CSZ(temp)) {
610		sc->sc_ctx_is_64_byte = 1;
611	} else {
612		sc->sc_ctx_is_64_byte = 0;
613	}
614
615	/* get DMA bits */
616	sc->sc_bus.dma_bits = (XHCI_HCS0_AC64(temp) &&
617	    xhcidma32 == 0 && dma32 == 0) ? 64 : 32;
618
619	device_printf(self, "%d bytes context size, %d-bit DMA\n",
620	    sc->sc_ctx_is_64_byte ? 64 : 32, (int)sc->sc_bus.dma_bits);
621
622	/* enable 64Kbyte control endpoint quirk */
623	sc->sc_bus.control_ep_quirk = (xhcictlquirk ? 1 : 0);
624
625	temp = XREAD4(sc, capa, XHCI_HCSPARAMS1);
626
627	/* get number of device slots */
628	sc->sc_noport = XHCI_HCS1_N_PORTS(temp);
629
630	if (sc->sc_noport == 0) {
631		device_printf(sc->sc_bus.parent, "Invalid number "
632		    "of ports: %u\n", sc->sc_noport);
633		return (ENXIO);
634	}
635
636	sc->sc_noport = sc->sc_noport;
637	sc->sc_noslot = XHCI_HCS1_DEVSLOT_MAX(temp);
638
639	DPRINTF("Max slots: %u\n", sc->sc_noslot);
640
641	if (sc->sc_noslot > XHCI_MAX_DEVICES)
642		sc->sc_noslot = XHCI_MAX_DEVICES;
643
644	temp = XREAD4(sc, capa, XHCI_HCSPARAMS2);
645
646	DPRINTF("HCS2=0x%08x\n", temp);
647
648	/* get number of scratchpads */
649	sc->sc_noscratch = XHCI_HCS2_SPB_MAX(temp);
650
651	if (sc->sc_noscratch > XHCI_MAX_SCRATCHPADS) {
652		device_printf(sc->sc_bus.parent, "XHCI request "
653		    "too many scratchpads\n");
654		return (ENOMEM);
655	}
656
657	DPRINTF("Max scratch: %u\n", sc->sc_noscratch);
658
659	/* get event table size */
660	sc->sc_erst_max = 1U << XHCI_HCS2_ERST_MAX(temp);
661	if (sc->sc_erst_max > XHCI_MAX_RSEG)
662		sc->sc_erst_max = XHCI_MAX_RSEG;
663
664	temp = XREAD4(sc, capa, XHCI_HCSPARAMS3);
665
666	/* get maximum exit latency */
667	sc->sc_exit_lat_max = XHCI_HCS3_U1_DEL(temp) +
668	    XHCI_HCS3_U2_DEL(temp) + 250 /* us */;
669
670	/* Check if we should use the default IMOD value. */
671	if (sc->sc_imod_default == 0)
672		sc->sc_imod_default = XHCI_IMOD_DEFAULT;
673
674	/* get all DMA memory */
675	if (usb_bus_mem_alloc_all(&sc->sc_bus,
676	    USB_GET_DMA_TAG(self), &xhci_iterate_hw_softc)) {
677		return (ENOMEM);
678	}
679
680	/* set up command queue mutex and condition varible */
681	cv_init(&sc->sc_cmd_cv, "CMDQ");
682	sx_init(&sc->sc_cmd_sx, "CMDQ lock");
683
684	sc->sc_config_msg[0].hdr.pm_callback = &xhci_configure_msg;
685	sc->sc_config_msg[0].bus = &sc->sc_bus;
686	sc->sc_config_msg[1].hdr.pm_callback = &xhci_configure_msg;
687	sc->sc_config_msg[1].bus = &sc->sc_bus;
688
689	return (0);
690}
691
692void
693xhci_uninit(struct xhci_softc *sc)
694{
695	/*
696	 * NOTE: At this point the control transfer process is gone
697	 * and "xhci_configure_msg" is no longer called. Consequently
698	 * waiting for the configuration messages to complete is not
699	 * needed.
700	 */
701	usb_bus_mem_free_all(&sc->sc_bus, &xhci_iterate_hw_softc);
702
703	cv_destroy(&sc->sc_cmd_cv);
704	sx_destroy(&sc->sc_cmd_sx);
705}
706
707static void
708xhci_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
709{
710	struct xhci_softc *sc = XHCI_BUS2SC(bus);
711
712	switch (state) {
713	case USB_HW_POWER_SUSPEND:
714		DPRINTF("Stopping the XHCI\n");
715		xhci_halt_controller(sc);
716		xhci_reset_controller(sc);
717		break;
718	case USB_HW_POWER_SHUTDOWN:
719		DPRINTF("Stopping the XHCI\n");
720		xhci_halt_controller(sc);
721		xhci_reset_controller(sc);
722		break;
723	case USB_HW_POWER_RESUME:
724		DPRINTF("Starting the XHCI\n");
725		xhci_start_controller(sc);
726		break;
727	default:
728		break;
729	}
730}
731
732static usb_error_t
733xhci_generic_done_sub(struct usb_xfer *xfer)
734{
735	struct xhci_td *td;
736	struct xhci_td *td_alt_next;
737	uint32_t len;
738	uint8_t status;
739
740	td = xfer->td_transfer_cache;
741	td_alt_next = td->alt_next;
742
743	if (xfer->aframes != xfer->nframes)
744		usbd_xfer_set_frame_len(xfer, xfer->aframes, 0);
745
746	while (1) {
747		usb_pc_cpu_invalidate(td->page_cache);
748
749		status = td->status;
750		len = td->remainder;
751
752		DPRINTFN(4, "xfer=%p[%u/%u] rem=%u/%u status=%u\n",
753		    xfer, (unsigned int)xfer->aframes,
754		    (unsigned int)xfer->nframes,
755		    (unsigned int)len, (unsigned int)td->len,
756		    (unsigned int)status);
757
758		/*
759	         * Verify the status length and
760		 * add the length to "frlengths[]":
761	         */
762		if (len > td->len) {
763			/* should not happen */
764			DPRINTF("Invalid status length, "
765			    "0x%04x/0x%04x bytes\n", len, td->len);
766			status = XHCI_TRB_ERROR_LENGTH;
767		} else if (xfer->aframes != xfer->nframes) {
768			xfer->frlengths[xfer->aframes] += td->len - len;
769		}
770		/* Check for last transfer */
771		if (((void *)td) == xfer->td_transfer_last) {
772			td = NULL;
773			break;
774		}
775		/* Check for transfer error */
776		if (status != XHCI_TRB_ERROR_SHORT_PKT &&
777		    status != XHCI_TRB_ERROR_SUCCESS) {
778			/* the transfer is finished */
779			td = NULL;
780			break;
781		}
782		/* Check for short transfer */
783		if (len > 0) {
784			if (xfer->flags_int.short_frames_ok ||
785			    xfer->flags_int.isochronous_xfr ||
786			    xfer->flags_int.control_xfr) {
787				/* follow alt next */
788				td = td->alt_next;
789			} else {
790				/* the transfer is finished */
791				td = NULL;
792			}
793			break;
794		}
795		td = td->obj_next;
796
797		if (td->alt_next != td_alt_next) {
798			/* this USB frame is complete */
799			break;
800		}
801	}
802
803	/* update transfer cache */
804
805	xfer->td_transfer_cache = td;
806
807	return ((status == XHCI_TRB_ERROR_STALL) ? USB_ERR_STALLED :
808	    (status != XHCI_TRB_ERROR_SHORT_PKT &&
809	    status != XHCI_TRB_ERROR_SUCCESS) ? USB_ERR_IOERROR :
810	    USB_ERR_NORMAL_COMPLETION);
811}
812
813static void
814xhci_generic_done(struct usb_xfer *xfer)
815{
816	usb_error_t err = 0;
817
818	DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
819	    xfer, xfer->endpoint);
820
821	/* reset scanner */
822
823	xfer->td_transfer_cache = xfer->td_transfer_first;
824
825	if (xfer->flags_int.control_xfr) {
826		if (xfer->flags_int.control_hdr)
827			err = xhci_generic_done_sub(xfer);
828
829		xfer->aframes = 1;
830
831		if (xfer->td_transfer_cache == NULL)
832			goto done;
833	}
834
835	while (xfer->aframes != xfer->nframes) {
836		err = xhci_generic_done_sub(xfer);
837		xfer->aframes++;
838
839		if (xfer->td_transfer_cache == NULL)
840			goto done;
841	}
842
843	if (xfer->flags_int.control_xfr &&
844	    !xfer->flags_int.control_act)
845		err = xhci_generic_done_sub(xfer);
846done:
847	/* transfer is complete */
848	xhci_device_done(xfer, err);
849}
850
851static void
852xhci_activate_transfer(struct usb_xfer *xfer)
853{
854	struct xhci_td *td;
855
856	td = xfer->td_transfer_cache;
857
858	usb_pc_cpu_invalidate(td->page_cache);
859
860	if (!(td->td_trb[0].dwTrb3 & htole32(XHCI_TRB_3_CYCLE_BIT))) {
861		/* activate the transfer */
862
863		td->td_trb[0].dwTrb3 |= htole32(XHCI_TRB_3_CYCLE_BIT);
864		usb_pc_cpu_flush(td->page_cache);
865
866		xhci_endpoint_doorbell(xfer);
867	}
868}
869
870static void
871xhci_skip_transfer(struct usb_xfer *xfer)
872{
873	struct xhci_td *td;
874	struct xhci_td *td_last;
875
876	td = xfer->td_transfer_cache;
877	td_last = xfer->td_transfer_last;
878
879	td = td->alt_next;
880
881	usb_pc_cpu_invalidate(td->page_cache);
882
883	if (!(td->td_trb[0].dwTrb3 & htole32(XHCI_TRB_3_CYCLE_BIT))) {
884		usb_pc_cpu_invalidate(td_last->page_cache);
885
886		/* copy LINK TRB to current waiting location */
887
888		td->td_trb[0].qwTrb0 = td_last->td_trb[td_last->ntrb].qwTrb0;
889		td->td_trb[0].dwTrb2 = td_last->td_trb[td_last->ntrb].dwTrb2;
890		usb_pc_cpu_flush(td->page_cache);
891
892		td->td_trb[0].dwTrb3 = td_last->td_trb[td_last->ntrb].dwTrb3;
893		usb_pc_cpu_flush(td->page_cache);
894
895		xhci_endpoint_doorbell(xfer);
896	}
897}
898
899/*------------------------------------------------------------------------*
900 *	xhci_check_transfer
901 *------------------------------------------------------------------------*/
902static void
903xhci_check_transfer(struct xhci_softc *sc, struct xhci_trb *trb)
904{
905	struct xhci_endpoint_ext *pepext;
906	int64_t offset;
907	uint64_t td_event;
908	uint32_t temp;
909	uint32_t remainder;
910	uint16_t stream_id = 0;
911	uint16_t i;
912	uint8_t status;
913	uint8_t halted;
914	uint8_t epno;
915	uint8_t index;
916
917	/* decode TRB */
918	td_event = le64toh(trb->qwTrb0);
919	temp = le32toh(trb->dwTrb2);
920
921	remainder = XHCI_TRB_2_REM_GET(temp);
922	status = XHCI_TRB_2_ERROR_GET(temp);
923
924	temp = le32toh(trb->dwTrb3);
925	epno = XHCI_TRB_3_EP_GET(temp);
926	index = XHCI_TRB_3_SLOT_GET(temp);
927
928	/* check if error means halted */
929	halted = (status != XHCI_TRB_ERROR_SHORT_PKT &&
930	    status != XHCI_TRB_ERROR_SUCCESS);
931
932	DPRINTF("slot=%u epno=%u remainder=%u status=%u\n",
933	    index, epno, remainder, status);
934
935	if (index > sc->sc_noslot) {
936		DPRINTF("Invalid slot.\n");
937		return;
938	}
939
940	if ((epno == 0) || (epno >= XHCI_MAX_ENDPOINTS)) {
941		DPRINTF("Invalid endpoint.\n");
942		return;
943	}
944
945	pepext = &sc->sc_hw.devs[index].endp[epno];
946
947	/* try to find the USB transfer that generated the event */
948	for (i = 0;; i++) {
949		struct usb_xfer *xfer;
950		struct xhci_td *td;
951
952		if (i == (XHCI_MAX_TRANSFERS - 1)) {
953			if (pepext->trb_ep_mode != USB_EP_MODE_STREAMS ||
954			    stream_id == (XHCI_MAX_STREAMS - 1))
955				break;
956			stream_id++;
957			i = 0;
958			DPRINTFN(5, "stream_id=%u\n", stream_id);
959		}
960
961		xfer = pepext->xfer[i + (XHCI_MAX_TRANSFERS * stream_id)];
962		if (xfer == NULL)
963			continue;
964
965		td = xfer->td_transfer_cache;
966
967		DPRINTFN(5, "Checking if 0x%016llx == (0x%016llx .. 0x%016llx)\n",
968			(long long)td_event,
969			(long long)td->td_self,
970			(long long)td->td_self + sizeof(td->td_trb));
971
972		/*
973		 * NOTE: Some XHCI implementations might not trigger
974		 * an event on the last LINK TRB so we need to
975		 * consider both the last and second last event
976		 * address as conditions for a successful transfer.
977		 *
978		 * NOTE: We assume that the XHCI will only trigger one
979		 * event per chain of TRBs.
980		 */
981
982		offset = td_event - td->td_self;
983
984		if (offset >= 0 &&
985		    offset < (int64_t)sizeof(td->td_trb)) {
986			usb_pc_cpu_invalidate(td->page_cache);
987
988			/* compute rest of remainder, if any */
989			for (i = (offset / 16) + 1; i < td->ntrb; i++) {
990				temp = le32toh(td->td_trb[i].dwTrb2);
991				remainder += XHCI_TRB_2_BYTES_GET(temp);
992			}
993
994			DPRINTFN(5, "New remainder: %u\n", remainder);
995
996			/* clear isochronous transfer errors */
997			if (xfer->flags_int.isochronous_xfr) {
998				if (halted) {
999					halted = 0;
1000					status = XHCI_TRB_ERROR_SUCCESS;
1001					remainder = td->len;
1002				}
1003			}
1004
1005			/* "td->remainder" is verified later */
1006			td->remainder = remainder;
1007			td->status = status;
1008
1009			usb_pc_cpu_flush(td->page_cache);
1010
1011			/*
1012			 * 1) Last transfer descriptor makes the
1013			 * transfer done
1014			 */
1015			if (((void *)td) == xfer->td_transfer_last) {
1016				DPRINTF("TD is last\n");
1017				xhci_generic_done(xfer);
1018				break;
1019			}
1020
1021			/*
1022			 * 2) Any kind of error makes the transfer
1023			 * done
1024			 */
1025			if (halted) {
1026				DPRINTF("TD has I/O error\n");
1027				xhci_generic_done(xfer);
1028				break;
1029			}
1030
1031			/*
1032			 * 3) If there is no alternate next transfer,
1033			 * a short packet also makes the transfer done
1034			 */
1035			if (td->remainder > 0) {
1036				if (td->alt_next == NULL) {
1037					DPRINTF(
1038					    "short TD has no alternate next\n");
1039					xhci_generic_done(xfer);
1040					break;
1041				}
1042				DPRINTF("TD has short pkt\n");
1043				if (xfer->flags_int.short_frames_ok ||
1044				    xfer->flags_int.isochronous_xfr ||
1045				    xfer->flags_int.control_xfr) {
1046					/* follow the alt next */
1047					xfer->td_transfer_cache = td->alt_next;
1048					xhci_activate_transfer(xfer);
1049					break;
1050				}
1051				xhci_skip_transfer(xfer);
1052				xhci_generic_done(xfer);
1053				break;
1054			}
1055
1056			/*
1057			 * 4) Transfer complete - go to next TD
1058			 */
1059			DPRINTF("Following next TD\n");
1060			xfer->td_transfer_cache = td->obj_next;
1061			xhci_activate_transfer(xfer);
1062			break;		/* there should only be one match */
1063		}
1064	}
1065}
1066
1067static int
1068xhci_check_command(struct xhci_softc *sc, struct xhci_trb *trb)
1069{
1070	if (sc->sc_cmd_addr == trb->qwTrb0) {
1071		DPRINTF("Received command event\n");
1072		sc->sc_cmd_result[0] = trb->dwTrb2;
1073		sc->sc_cmd_result[1] = trb->dwTrb3;
1074		cv_signal(&sc->sc_cmd_cv);
1075		return (1);	/* command match */
1076	}
1077	return (0);
1078}
1079
1080static int
1081xhci_interrupt_poll(struct xhci_softc *sc)
1082{
1083	struct usb_page_search buf_res;
1084	struct xhci_hw_root *phwr;
1085	uint64_t addr;
1086	uint32_t temp;
1087	int retval = 0;
1088	uint16_t i;
1089	uint8_t event;
1090	uint8_t j;
1091	uint8_t k;
1092	uint8_t t;
1093
1094	usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res);
1095
1096	phwr = buf_res.buffer;
1097
1098	/* Receive any events */
1099
1100	usb_pc_cpu_invalidate(&sc->sc_hw.root_pc);
1101
1102	i = sc->sc_event_idx;
1103	j = sc->sc_event_ccs;
1104	t = 2;
1105
1106	while (1) {
1107		temp = le32toh(phwr->hwr_events[i].dwTrb3);
1108
1109		k = (temp & XHCI_TRB_3_CYCLE_BIT) ? 1 : 0;
1110
1111		if (j != k)
1112			break;
1113
1114		event = XHCI_TRB_3_TYPE_GET(temp);
1115
1116		DPRINTFN(10, "event[%u] = %u (0x%016llx 0x%08lx 0x%08lx)\n",
1117		    i, event, (long long)le64toh(phwr->hwr_events[i].qwTrb0),
1118		    (long)le32toh(phwr->hwr_events[i].dwTrb2),
1119		    (long)le32toh(phwr->hwr_events[i].dwTrb3));
1120
1121		switch (event) {
1122		case XHCI_TRB_EVENT_TRANSFER:
1123			xhci_check_transfer(sc, &phwr->hwr_events[i]);
1124			break;
1125		case XHCI_TRB_EVENT_CMD_COMPLETE:
1126			retval |= xhci_check_command(sc, &phwr->hwr_events[i]);
1127			break;
1128		default:
1129			DPRINTF("Unhandled event = %u\n", event);
1130			break;
1131		}
1132
1133		i++;
1134
1135		if (i == XHCI_MAX_EVENTS) {
1136			i = 0;
1137			j ^= 1;
1138
1139			/* check for timeout */
1140			if (!--t)
1141				break;
1142		}
1143	}
1144
1145	sc->sc_event_idx = i;
1146	sc->sc_event_ccs = j;
1147
1148	/*
1149	 * NOTE: The Event Ring Dequeue Pointer Register is 64-bit
1150	 * latched. That means to activate the register we need to
1151	 * write both the low and high double word of the 64-bit
1152	 * register.
1153	 */
1154
1155	addr = buf_res.physaddr;
1156	addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_events[i];
1157
1158	/* try to clear busy bit */
1159	addr |= XHCI_ERDP_LO_BUSY;
1160
1161	XWRITE4(sc, runt, XHCI_ERDP_LO(0), (uint32_t)addr);
1162	XWRITE4(sc, runt, XHCI_ERDP_HI(0), (uint32_t)(addr >> 32));
1163
1164	return (retval);
1165}
1166
1167static usb_error_t
1168xhci_do_command(struct xhci_softc *sc, struct xhci_trb *trb,
1169    uint16_t timeout_ms)
1170{
1171	struct usb_page_search buf_res;
1172	struct xhci_hw_root *phwr;
1173	uint64_t addr;
1174	uint32_t temp;
1175	uint8_t i;
1176	uint8_t j;
1177	uint8_t timeout = 0;
1178	int err;
1179
1180	XHCI_CMD_ASSERT_LOCKED(sc);
1181
1182	/* get hardware root structure */
1183
1184	usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res);
1185
1186	phwr = buf_res.buffer;
1187
1188	/* Queue command */
1189
1190	USB_BUS_LOCK(&sc->sc_bus);
1191retry:
1192	i = sc->sc_command_idx;
1193	j = sc->sc_command_ccs;
1194
1195	DPRINTFN(10, "command[%u] = %u (0x%016llx, 0x%08lx, 0x%08lx)\n",
1196	    i, XHCI_TRB_3_TYPE_GET(le32toh(trb->dwTrb3)),
1197	    (long long)le64toh(trb->qwTrb0),
1198	    (long)le32toh(trb->dwTrb2),
1199	    (long)le32toh(trb->dwTrb3));
1200
1201	phwr->hwr_commands[i].qwTrb0 = trb->qwTrb0;
1202	phwr->hwr_commands[i].dwTrb2 = trb->dwTrb2;
1203
1204	usb_pc_cpu_flush(&sc->sc_hw.root_pc);
1205
1206	temp = trb->dwTrb3;
1207
1208	if (j)
1209		temp |= htole32(XHCI_TRB_3_CYCLE_BIT);
1210	else
1211		temp &= ~htole32(XHCI_TRB_3_CYCLE_BIT);
1212
1213	temp &= ~htole32(XHCI_TRB_3_TC_BIT);
1214
1215	phwr->hwr_commands[i].dwTrb3 = temp;
1216
1217	usb_pc_cpu_flush(&sc->sc_hw.root_pc);
1218
1219	addr = buf_res.physaddr;
1220	addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_commands[i];
1221
1222	sc->sc_cmd_addr = htole64(addr);
1223
1224	i++;
1225
1226	if (i == (XHCI_MAX_COMMANDS - 1)) {
1227		if (j) {
1228			temp = htole32(XHCI_TRB_3_TC_BIT |
1229			    XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) |
1230			    XHCI_TRB_3_CYCLE_BIT);
1231		} else {
1232			temp = htole32(XHCI_TRB_3_TC_BIT |
1233			    XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK));
1234		}
1235
1236		phwr->hwr_commands[i].dwTrb3 = temp;
1237
1238		usb_pc_cpu_flush(&sc->sc_hw.root_pc);
1239
1240		i = 0;
1241		j ^= 1;
1242	}
1243
1244	sc->sc_command_idx = i;
1245	sc->sc_command_ccs = j;
1246
1247	XWRITE4(sc, door, XHCI_DOORBELL(0), 0);
1248
1249	err = cv_timedwait(&sc->sc_cmd_cv, &sc->sc_bus.bus_mtx,
1250	    USB_MS_TO_TICKS(timeout_ms));
1251
1252	/*
1253	 * In some error cases event interrupts are not generated.
1254	 * Poll one time to see if the command has completed.
1255	 */
1256	if (err != 0 && xhci_interrupt_poll(sc) != 0) {
1257		DPRINTF("Command was completed when polling\n");
1258		err = 0;
1259	}
1260	if (err != 0) {
1261		DPRINTF("Command timeout!\n");
1262		/*
1263		 * After some weeks of continuous operation, it has
1264		 * been observed that the ASMedia Technology, ASM1042
1265		 * SuperSpeed USB Host Controller can suddenly stop
1266		 * accepting commands via the command queue. Try to
1267		 * first reset the command queue. If that fails do a
1268		 * host controller reset.
1269		 */
1270		if (timeout == 0 &&
1271		    xhci_reset_command_queue_locked(sc) == 0) {
1272			temp = le32toh(trb->dwTrb3);
1273
1274			/*
1275			 * Avoid infinite XHCI reset loops if the set
1276			 * address command fails to respond due to a
1277			 * non-enumerating device:
1278			 */
1279			if (XHCI_TRB_3_TYPE_GET(temp) == XHCI_TRB_TYPE_ADDRESS_DEVICE &&
1280			    (temp & XHCI_TRB_3_BSR_BIT) == 0) {
1281				DPRINTF("Set address timeout\n");
1282			} else {
1283				timeout = 1;
1284				goto retry;
1285			}
1286		} else {
1287			DPRINTF("Controller reset!\n");
1288			usb_bus_reset_async_locked(&sc->sc_bus);
1289		}
1290		err = USB_ERR_TIMEOUT;
1291		trb->dwTrb2 = 0;
1292		trb->dwTrb3 = 0;
1293	} else {
1294		temp = le32toh(sc->sc_cmd_result[0]);
1295		if (XHCI_TRB_2_ERROR_GET(temp) != XHCI_TRB_ERROR_SUCCESS)
1296			err = USB_ERR_IOERROR;
1297
1298		trb->dwTrb2 = sc->sc_cmd_result[0];
1299		trb->dwTrb3 = sc->sc_cmd_result[1];
1300	}
1301
1302	USB_BUS_UNLOCK(&sc->sc_bus);
1303
1304	return (err);
1305}
1306
1307#if 0
1308static usb_error_t
1309xhci_cmd_nop(struct xhci_softc *sc)
1310{
1311	struct xhci_trb trb;
1312	uint32_t temp;
1313
1314	DPRINTF("\n");
1315
1316	trb.qwTrb0 = 0;
1317	trb.dwTrb2 = 0;
1318	temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NOOP);
1319
1320	trb.dwTrb3 = htole32(temp);
1321
1322	return (xhci_do_command(sc, &trb, 100 /* ms */));
1323}
1324#endif
1325
1326static usb_error_t
1327xhci_cmd_enable_slot(struct xhci_softc *sc, uint8_t *pslot)
1328{
1329	struct xhci_trb trb;
1330	uint32_t temp;
1331	usb_error_t err;
1332
1333	DPRINTF("\n");
1334
1335	trb.qwTrb0 = 0;
1336	trb.dwTrb2 = 0;
1337	trb.dwTrb3 = htole32(XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ENABLE_SLOT));
1338
1339	err = xhci_do_command(sc, &trb, 100 /* ms */);
1340	if (err)
1341		goto done;
1342
1343	temp = le32toh(trb.dwTrb3);
1344
1345	*pslot = XHCI_TRB_3_SLOT_GET(temp);
1346
1347done:
1348	return (err);
1349}
1350
1351static usb_error_t
1352xhci_cmd_disable_slot(struct xhci_softc *sc, uint8_t slot_id)
1353{
1354	struct xhci_trb trb;
1355	uint32_t temp;
1356
1357	DPRINTF("\n");
1358
1359	trb.qwTrb0 = 0;
1360	trb.dwTrb2 = 0;
1361	temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DISABLE_SLOT) |
1362	    XHCI_TRB_3_SLOT_SET(slot_id);
1363
1364	trb.dwTrb3 = htole32(temp);
1365
1366	return (xhci_do_command(sc, &trb, 100 /* ms */));
1367}
1368
1369static usb_error_t
1370xhci_cmd_set_address(struct xhci_softc *sc, uint64_t input_ctx,
1371    uint8_t bsr, uint8_t slot_id)
1372{
1373	struct xhci_trb trb;
1374	uint32_t temp;
1375
1376	DPRINTF("\n");
1377
1378	trb.qwTrb0 = htole64(input_ctx);
1379	trb.dwTrb2 = 0;
1380	temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ADDRESS_DEVICE) |
1381	    XHCI_TRB_3_SLOT_SET(slot_id);
1382
1383	if (bsr)
1384		temp |= XHCI_TRB_3_BSR_BIT;
1385
1386	trb.dwTrb3 = htole32(temp);
1387
1388	return (xhci_do_command(sc, &trb, 500 /* ms */));
1389}
1390
1391static usb_error_t
1392xhci_set_address(struct usb_device *udev, struct mtx *mtx, uint16_t address)
1393{
1394	struct usb_page_search buf_inp;
1395	struct usb_page_search buf_dev;
1396	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
1397	struct xhci_hw_dev *hdev;
1398	struct xhci_dev_ctx *pdev;
1399	struct xhci_endpoint_ext *pepext;
1400	uint32_t temp;
1401	uint16_t mps;
1402	usb_error_t err;
1403	uint8_t index;
1404
1405	/* the root HUB case is not handled here */
1406	if (udev->parent_hub == NULL)
1407		return (USB_ERR_INVAL);
1408
1409	index = udev->controller_slot_id;
1410
1411	hdev = 	&sc->sc_hw.devs[index];
1412
1413	if (mtx != NULL)
1414		mtx_unlock(mtx);
1415
1416	XHCI_CMD_LOCK(sc);
1417
1418	switch (hdev->state) {
1419	case XHCI_ST_DEFAULT:
1420	case XHCI_ST_ENABLED:
1421
1422		hdev->state = XHCI_ST_ENABLED;
1423
1424		/* set configure mask to slot and EP0 */
1425		xhci_configure_mask(udev, 3, 0);
1426
1427		/* configure input slot context structure */
1428		err = xhci_configure_device(udev);
1429
1430		if (err != 0) {
1431			DPRINTF("Could not configure device\n");
1432			break;
1433		}
1434
1435		/* configure input endpoint context structure */
1436		switch (udev->speed) {
1437		case USB_SPEED_LOW:
1438		case USB_SPEED_FULL:
1439			mps = 8;
1440			break;
1441		case USB_SPEED_HIGH:
1442			mps = 64;
1443			break;
1444		default:
1445			mps = 512;
1446			break;
1447		}
1448
1449		pepext = xhci_get_endpoint_ext(udev,
1450		    &udev->ctrl_ep_desc);
1451
1452		/* ensure the control endpoint is setup again */
1453		USB_BUS_LOCK(udev->bus);
1454		pepext->trb_halted = 1;
1455		pepext->trb_running = 0;
1456		USB_BUS_UNLOCK(udev->bus);
1457
1458		err = xhci_configure_endpoint(udev,
1459		    &udev->ctrl_ep_desc, pepext,
1460		    0, 1, 1, 0, mps, mps, USB_EP_MODE_DEFAULT);
1461
1462		if (err != 0) {
1463			DPRINTF("Could not configure default endpoint\n");
1464			break;
1465		}
1466
1467		/* execute set address command */
1468		usbd_get_page(&hdev->input_pc, 0, &buf_inp);
1469
1470		err = xhci_cmd_set_address(sc, buf_inp.physaddr,
1471		    (address == 0), index);
1472
1473		if (err != 0) {
1474			temp = le32toh(sc->sc_cmd_result[0]);
1475			if (address == 0 && sc->sc_port_route != NULL &&
1476			    XHCI_TRB_2_ERROR_GET(temp) ==
1477			    XHCI_TRB_ERROR_PARAMETER) {
1478				/* LynxPoint XHCI - ports are not switchable */
1479				/* Un-route all ports from the XHCI */
1480				sc->sc_port_route(sc->sc_bus.parent, 0, ~0);
1481			}
1482			DPRINTF("Could not set address "
1483			    "for slot %u.\n", index);
1484			if (address != 0)
1485				break;
1486		}
1487
1488		/* update device address to new value */
1489
1490		usbd_get_page(&hdev->device_pc, 0, &buf_dev);
1491		pdev = buf_dev.buffer;
1492		usb_pc_cpu_invalidate(&hdev->device_pc);
1493
1494		temp = xhci_ctx_get_le32(sc, &pdev->ctx_slot.dwSctx3);
1495		udev->address = XHCI_SCTX_3_DEV_ADDR_GET(temp);
1496
1497		/* update device state to new value */
1498
1499		if (address != 0)
1500			hdev->state = XHCI_ST_ADDRESSED;
1501		else
1502			hdev->state = XHCI_ST_DEFAULT;
1503		break;
1504
1505	default:
1506		DPRINTF("Wrong state for set address.\n");
1507		err = USB_ERR_IOERROR;
1508		break;
1509	}
1510	XHCI_CMD_UNLOCK(sc);
1511
1512	if (mtx != NULL)
1513		mtx_lock(mtx);
1514
1515	return (err);
1516}
1517
1518static usb_error_t
1519xhci_cmd_configure_ep(struct xhci_softc *sc, uint64_t input_ctx,
1520    uint8_t deconfigure, uint8_t slot_id)
1521{
1522	struct xhci_trb trb;
1523	uint32_t temp;
1524
1525	DPRINTF("\n");
1526
1527	trb.qwTrb0 = htole64(input_ctx);
1528	trb.dwTrb2 = 0;
1529	temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP) |
1530	    XHCI_TRB_3_SLOT_SET(slot_id);
1531
1532	if (deconfigure)
1533		temp |= XHCI_TRB_3_DCEP_BIT;
1534
1535	trb.dwTrb3 = htole32(temp);
1536
1537	return (xhci_do_command(sc, &trb, 100 /* ms */));
1538}
1539
1540static usb_error_t
1541xhci_cmd_evaluate_ctx(struct xhci_softc *sc, uint64_t input_ctx,
1542    uint8_t slot_id)
1543{
1544	struct xhci_trb trb;
1545	uint32_t temp;
1546
1547	DPRINTF("\n");
1548
1549	trb.qwTrb0 = htole64(input_ctx);
1550	trb.dwTrb2 = 0;
1551	temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVALUATE_CTX) |
1552	    XHCI_TRB_3_SLOT_SET(slot_id);
1553	trb.dwTrb3 = htole32(temp);
1554
1555	return (xhci_do_command(sc, &trb, 100 /* ms */));
1556}
1557
1558static usb_error_t
1559xhci_cmd_reset_ep(struct xhci_softc *sc, uint8_t preserve,
1560    uint8_t ep_id, uint8_t slot_id)
1561{
1562	struct xhci_trb trb;
1563	uint32_t temp;
1564
1565	DPRINTF("\n");
1566
1567	trb.qwTrb0 = 0;
1568	trb.dwTrb2 = 0;
1569	temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_EP) |
1570	    XHCI_TRB_3_SLOT_SET(slot_id) |
1571	    XHCI_TRB_3_EP_SET(ep_id);
1572
1573	if (preserve)
1574		temp |= XHCI_TRB_3_PRSV_BIT;
1575
1576	trb.dwTrb3 = htole32(temp);
1577
1578	return (xhci_do_command(sc, &trb, 100 /* ms */));
1579}
1580
1581static usb_error_t
1582xhci_cmd_set_tr_dequeue_ptr(struct xhci_softc *sc, uint64_t dequeue_ptr,
1583    uint16_t stream_id, uint8_t ep_id, uint8_t slot_id)
1584{
1585	struct xhci_trb trb;
1586	uint32_t temp;
1587
1588	DPRINTF("\n");
1589
1590	trb.qwTrb0 = htole64(dequeue_ptr);
1591
1592	temp = XHCI_TRB_2_STREAM_SET(stream_id);
1593	trb.dwTrb2 = htole32(temp);
1594
1595	temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SET_TR_DEQUEUE) |
1596	    XHCI_TRB_3_SLOT_SET(slot_id) |
1597	    XHCI_TRB_3_EP_SET(ep_id);
1598	trb.dwTrb3 = htole32(temp);
1599
1600	return (xhci_do_command(sc, &trb, 100 /* ms */));
1601}
1602
1603static usb_error_t
1604xhci_cmd_stop_ep(struct xhci_softc *sc, uint8_t suspend,
1605    uint8_t ep_id, uint8_t slot_id)
1606{
1607	struct xhci_trb trb;
1608	uint32_t temp;
1609
1610	DPRINTF("\n");
1611
1612	trb.qwTrb0 = 0;
1613	trb.dwTrb2 = 0;
1614	temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STOP_EP) |
1615	    XHCI_TRB_3_SLOT_SET(slot_id) |
1616	    XHCI_TRB_3_EP_SET(ep_id);
1617
1618	if (suspend)
1619		temp |= XHCI_TRB_3_SUSP_EP_BIT;
1620
1621	trb.dwTrb3 = htole32(temp);
1622
1623	return (xhci_do_command(sc, &trb, 100 /* ms */));
1624}
1625
1626static usb_error_t
1627xhci_cmd_reset_dev(struct xhci_softc *sc, uint8_t slot_id)
1628{
1629	struct xhci_trb trb;
1630	uint32_t temp;
1631
1632	DPRINTF("\n");
1633
1634	trb.qwTrb0 = 0;
1635	trb.dwTrb2 = 0;
1636	temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_DEVICE) |
1637	    XHCI_TRB_3_SLOT_SET(slot_id);
1638
1639	trb.dwTrb3 = htole32(temp);
1640
1641	return (xhci_do_command(sc, &trb, 100 /* ms */));
1642}
1643
1644/*------------------------------------------------------------------------*
1645 *	xhci_interrupt - XHCI interrupt handler
1646 *------------------------------------------------------------------------*/
1647void
1648xhci_interrupt(struct xhci_softc *sc)
1649{
1650	uint32_t status;
1651	uint32_t temp;
1652
1653	USB_BUS_LOCK(&sc->sc_bus);
1654
1655	status = XREAD4(sc, oper, XHCI_USBSTS);
1656
1657	/* acknowledge interrupts, if any */
1658	if (status != 0) {
1659		XWRITE4(sc, oper, XHCI_USBSTS, status);
1660		DPRINTFN(16, "real interrupt (status=0x%08x)\n", status);
1661	}
1662
1663	temp = XREAD4(sc, runt, XHCI_IMAN(0));
1664
1665	/* force clearing of pending interrupts */
1666	if (temp & XHCI_IMAN_INTR_PEND)
1667		XWRITE4(sc, runt, XHCI_IMAN(0), temp);
1668
1669	/* check for event(s) */
1670	xhci_interrupt_poll(sc);
1671
1672	if (status & (XHCI_STS_PCD | XHCI_STS_HCH |
1673	    XHCI_STS_HSE | XHCI_STS_HCE)) {
1674		if (status & XHCI_STS_PCD) {
1675			xhci_root_intr(sc);
1676		}
1677
1678		if (status & XHCI_STS_HCH) {
1679			printf("%s: host controller halted\n",
1680			    __FUNCTION__);
1681		}
1682
1683		if (status & XHCI_STS_HSE) {
1684			printf("%s: host system error\n",
1685			    __FUNCTION__);
1686		}
1687
1688		if (status & XHCI_STS_HCE) {
1689			printf("%s: host controller error\n",
1690			   __FUNCTION__);
1691		}
1692	}
1693	USB_BUS_UNLOCK(&sc->sc_bus);
1694}
1695
1696/*------------------------------------------------------------------------*
1697 *	xhci_timeout - XHCI timeout handler
1698 *------------------------------------------------------------------------*/
1699static void
1700xhci_timeout(void *arg)
1701{
1702	struct usb_xfer *xfer = arg;
1703
1704	DPRINTF("xfer=%p\n", xfer);
1705
1706	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
1707
1708	/* transfer is transferred */
1709	xhci_device_done(xfer, USB_ERR_TIMEOUT);
1710}
1711
1712static void
1713xhci_do_poll(struct usb_bus *bus)
1714{
1715	struct xhci_softc *sc = XHCI_BUS2SC(bus);
1716
1717	USB_BUS_LOCK(&sc->sc_bus);
1718	xhci_interrupt_poll(sc);
1719	USB_BUS_UNLOCK(&sc->sc_bus);
1720}
1721
1722static void
1723xhci_setup_generic_chain_sub(struct xhci_std_temp *temp)
1724{
1725	struct usb_page_search buf_res;
1726	struct xhci_td *td;
1727	struct xhci_td *td_next;
1728	struct xhci_td *td_alt_next;
1729	struct xhci_td *td_first;
1730	uint32_t buf_offset;
1731	uint32_t average;
1732	uint32_t len_old;
1733	uint32_t npkt_off;
1734	uint32_t dword;
1735	uint8_t shortpkt_old;
1736	uint8_t precompute;
1737	uint8_t x;
1738
1739	td_alt_next = NULL;
1740	buf_offset = 0;
1741	shortpkt_old = temp->shortpkt;
1742	len_old = temp->len;
1743	npkt_off = 0;
1744	precompute = 1;
1745
1746restart:
1747
1748	td = temp->td;
1749	td_next = td_first = temp->td_next;
1750
1751	while (1) {
1752		if (temp->len == 0) {
1753			if (temp->shortpkt)
1754				break;
1755
1756			/* send a Zero Length Packet, ZLP, last */
1757
1758			temp->shortpkt = 1;
1759			average = 0;
1760
1761		} else {
1762			average = temp->average;
1763
1764			if (temp->len < average) {
1765				if (temp->len % temp->max_packet_size) {
1766					temp->shortpkt = 1;
1767				}
1768				average = temp->len;
1769			}
1770		}
1771
1772		if (td_next == NULL)
1773			panic("%s: out of XHCI transfer descriptors!", __FUNCTION__);
1774
1775		/* get next TD */
1776
1777		td = td_next;
1778		td_next = td->obj_next;
1779
1780		/* check if we are pre-computing */
1781
1782		if (precompute) {
1783			/* update remaining length */
1784
1785			temp->len -= average;
1786
1787			continue;
1788		}
1789		/* fill out current TD */
1790
1791		td->len = average;
1792		td->remainder = 0;
1793		td->status = 0;
1794
1795		/* update remaining length */
1796
1797		temp->len -= average;
1798
1799		/* reset TRB index */
1800
1801		x = 0;
1802
1803		if (temp->trb_type == XHCI_TRB_TYPE_SETUP_STAGE) {
1804			/* immediate data */
1805
1806			if (average > 8)
1807				average = 8;
1808
1809			td->td_trb[0].qwTrb0 = 0;
1810
1811			usbd_copy_out(temp->pc, temp->offset + buf_offset,
1812			   (uint8_t *)(uintptr_t)&td->td_trb[0].qwTrb0,
1813			   average);
1814
1815			dword = XHCI_TRB_2_BYTES_SET(8) |
1816			    XHCI_TRB_2_TDSZ_SET(0) |
1817			    XHCI_TRB_2_IRQ_SET(0);
1818
1819			td->td_trb[0].dwTrb2 = htole32(dword);
1820
1821			dword = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SETUP_STAGE) |
1822			  XHCI_TRB_3_IDT_BIT | XHCI_TRB_3_CYCLE_BIT;
1823
1824			/* check wLength */
1825			if (td->td_trb[0].qwTrb0 &
1826			   htole64(XHCI_TRB_0_WLENGTH_MASK)) {
1827				if (td->td_trb[0].qwTrb0 &
1828				    htole64(XHCI_TRB_0_DIR_IN_MASK))
1829					dword |= XHCI_TRB_3_TRT_IN;
1830				else
1831					dword |= XHCI_TRB_3_TRT_OUT;
1832			}
1833
1834			td->td_trb[0].dwTrb3 = htole32(dword);
1835#ifdef USB_DEBUG
1836			xhci_dump_trb(&td->td_trb[x]);
1837#endif
1838			x++;
1839
1840		} else do {
1841			uint32_t npkt;
1842
1843			/* fill out buffer pointers */
1844
1845			if (average == 0) {
1846				memset(&buf_res, 0, sizeof(buf_res));
1847			} else {
1848				usbd_get_page(temp->pc, temp->offset +
1849				    buf_offset, &buf_res);
1850
1851				/* get length to end of page */
1852				if (buf_res.length > average)
1853					buf_res.length = average;
1854
1855				/* check for maximum length */
1856				if (buf_res.length > XHCI_TD_PAGE_SIZE)
1857					buf_res.length = XHCI_TD_PAGE_SIZE;
1858
1859				npkt_off += buf_res.length;
1860			}
1861
1862			/* set up npkt */
1863			npkt = howmany(len_old - npkt_off,
1864				       temp->max_packet_size);
1865
1866			if (npkt == 0)
1867				npkt = 1;
1868			else if (npkt > 31)
1869				npkt = 31;
1870
1871			/* fill out TRB's */
1872			td->td_trb[x].qwTrb0 =
1873			    htole64((uint64_t)buf_res.physaddr);
1874
1875			dword =
1876			  XHCI_TRB_2_BYTES_SET(buf_res.length) |
1877			  XHCI_TRB_2_TDSZ_SET(npkt) |
1878			  XHCI_TRB_2_IRQ_SET(0);
1879
1880			td->td_trb[x].dwTrb2 = htole32(dword);
1881
1882			switch (temp->trb_type) {
1883			case XHCI_TRB_TYPE_ISOCH:
1884				dword = XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_CYCLE_BIT |
1885				    XHCI_TRB_3_TBC_SET(temp->tbc) |
1886				    XHCI_TRB_3_TLBPC_SET(temp->tlbpc);
1887				if (td != td_first) {
1888					dword |= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL);
1889				} else if (temp->do_isoc_sync != 0) {
1890					temp->do_isoc_sync = 0;
1891					/* wait until "isoc_frame" */
1892					dword |= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ISOCH) |
1893					    XHCI_TRB_3_FRID_SET(temp->isoc_frame / 8);
1894				} else {
1895					/* start data transfer at next interval */
1896					dword |= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ISOCH) |
1897					    XHCI_TRB_3_ISO_SIA_BIT;
1898				}
1899				if (temp->direction == UE_DIR_IN)
1900					dword |= XHCI_TRB_3_ISP_BIT;
1901				break;
1902			case XHCI_TRB_TYPE_DATA_STAGE:
1903				dword = XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_CYCLE_BIT |
1904				    XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DATA_STAGE);
1905				if (temp->direction == UE_DIR_IN)
1906					dword |= XHCI_TRB_3_DIR_IN | XHCI_TRB_3_ISP_BIT;
1907				/*
1908				 * Section 3.2.9 in the XHCI
1909				 * specification about control
1910				 * transfers says that we should use a
1911				 * normal-TRB if there are more TRBs
1912				 * extending the data-stage
1913				 * TRB. Update the "trb_type".
1914				 */
1915				temp->trb_type = XHCI_TRB_TYPE_NORMAL;
1916				break;
1917			case XHCI_TRB_TYPE_STATUS_STAGE:
1918				dword = XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_CYCLE_BIT |
1919				    XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STATUS_STAGE);
1920				if (temp->direction == UE_DIR_IN)
1921					dword |= XHCI_TRB_3_DIR_IN;
1922				break;
1923			default:	/* XHCI_TRB_TYPE_NORMAL */
1924				dword = XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_CYCLE_BIT |
1925				    XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL);
1926				if (temp->direction == UE_DIR_IN)
1927					dword |= XHCI_TRB_3_ISP_BIT;
1928				break;
1929			}
1930			td->td_trb[x].dwTrb3 = htole32(dword);
1931
1932			average -= buf_res.length;
1933			buf_offset += buf_res.length;
1934#ifdef USB_DEBUG
1935			xhci_dump_trb(&td->td_trb[x]);
1936#endif
1937			x++;
1938
1939		} while (average != 0);
1940
1941		td->td_trb[x-1].dwTrb3 |= htole32(XHCI_TRB_3_IOC_BIT);
1942
1943		/* store number of data TRB's */
1944
1945		td->ntrb = x;
1946
1947		DPRINTF("NTRB=%u\n", x);
1948
1949		/* fill out link TRB */
1950
1951		if (td_next != NULL) {
1952			/* link the current TD with the next one */
1953			td->td_trb[x].qwTrb0 = htole64((uint64_t)td_next->td_self);
1954			DPRINTF("LINK=0x%08llx\n", (long long)td_next->td_self);
1955		} else {
1956			/* this field will get updated later */
1957			DPRINTF("NOLINK\n");
1958		}
1959
1960		dword = XHCI_TRB_2_IRQ_SET(0);
1961
1962		td->td_trb[x].dwTrb2 = htole32(dword);
1963
1964		dword = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) |
1965		    XHCI_TRB_3_CYCLE_BIT | XHCI_TRB_3_IOC_BIT |
1966		    /*
1967		     * CHAIN-BIT: Ensure that a multi-TRB IN-endpoint
1968		     * frame only receives a single short packet event
1969		     * by setting the CHAIN bit in the LINK field. In
1970		     * addition some XHCI controllers have problems
1971		     * sending a ZLP unless the CHAIN-BIT is set in
1972		     * the LINK TRB.
1973		     */
1974		    XHCI_TRB_3_CHAIN_BIT;
1975
1976		td->td_trb[x].dwTrb3 = htole32(dword);
1977
1978		td->alt_next = td_alt_next;
1979#ifdef USB_DEBUG
1980		xhci_dump_trb(&td->td_trb[x]);
1981#endif
1982		usb_pc_cpu_flush(td->page_cache);
1983	}
1984
1985	if (precompute) {
1986		precompute = 0;
1987
1988		/* set up alt next pointer, if any */
1989		if (temp->last_frame) {
1990			td_alt_next = NULL;
1991		} else {
1992			/* we use this field internally */
1993			td_alt_next = td_next;
1994		}
1995
1996		/* restore */
1997		temp->shortpkt = shortpkt_old;
1998		temp->len = len_old;
1999		goto restart;
2000	}
2001
2002	/*
2003	 * Remove cycle bit from the first TRB if we are
2004	 * stepping them:
2005	 */
2006	if (temp->step_td != 0) {
2007		td_first->td_trb[0].dwTrb3 &= ~htole32(XHCI_TRB_3_CYCLE_BIT);
2008		usb_pc_cpu_flush(td_first->page_cache);
2009	}
2010
2011	/* clear TD SIZE to zero, hence this is the last TRB */
2012	/* remove chain bit because this is the last data TRB in the chain */
2013	td->td_trb[td->ntrb - 1].dwTrb2 &= ~htole32(XHCI_TRB_2_TDSZ_SET(31));
2014	td->td_trb[td->ntrb - 1].dwTrb3 &= ~htole32(XHCI_TRB_3_CHAIN_BIT);
2015	/* remove CHAIN-BIT from last LINK TRB */
2016	td->td_trb[td->ntrb].dwTrb3 &= ~htole32(XHCI_TRB_3_CHAIN_BIT);
2017
2018	usb_pc_cpu_flush(td->page_cache);
2019
2020	temp->td = td;
2021	temp->td_next = td_next;
2022}
2023
2024static void
2025xhci_setup_generic_chain(struct usb_xfer *xfer)
2026{
2027	struct xhci_std_temp temp;
2028	struct xhci_td *td;
2029	uint32_t x;
2030	uint32_t y;
2031	uint8_t mult;
2032
2033	temp.do_isoc_sync = 0;
2034	temp.step_td = 0;
2035	temp.tbc = 0;
2036	temp.tlbpc = 0;
2037	temp.average = xfer->max_hc_frame_size;
2038	temp.max_packet_size = xfer->max_packet_size;
2039	temp.sc = XHCI_BUS2SC(xfer->xroot->bus);
2040	temp.pc = NULL;
2041	temp.last_frame = 0;
2042	temp.offset = 0;
2043	temp.multishort = xfer->flags_int.isochronous_xfr ||
2044	    xfer->flags_int.control_xfr ||
2045	    xfer->flags_int.short_frames_ok;
2046
2047	/* toggle the DMA set we are using */
2048	xfer->flags_int.curr_dma_set ^= 1;
2049
2050	/* get next DMA set */
2051	td = xfer->td_start[xfer->flags_int.curr_dma_set];
2052
2053	temp.td = NULL;
2054	temp.td_next = td;
2055
2056	xfer->td_transfer_first = td;
2057	xfer->td_transfer_cache = td;
2058
2059	if (xfer->flags_int.isochronous_xfr) {
2060		uint8_t shift;
2061
2062		/* compute multiplier for ISOCHRONOUS transfers */
2063		mult = xfer->endpoint->ecomp ?
2064		    UE_GET_SS_ISO_MULT(xfer->endpoint->ecomp->bmAttributes)
2065		    : 0;
2066		/* check for USB 2.0 multiplier */
2067		if (mult == 0) {
2068			mult = (xfer->endpoint->edesc->
2069			    wMaxPacketSize[1] >> 3) & 3;
2070		}
2071		/* range check */
2072		if (mult > 2)
2073			mult = 3;
2074		else
2075			mult++;
2076
2077		x = XREAD4(temp.sc, runt, XHCI_MFINDEX);
2078
2079		DPRINTF("MFINDEX=0x%08x\n", x);
2080
2081		switch (usbd_get_speed(xfer->xroot->udev)) {
2082		case USB_SPEED_FULL:
2083			shift = 3;
2084			temp.isoc_delta = 8;	/* 1ms */
2085			x += temp.isoc_delta - 1;
2086			x &= ~(temp.isoc_delta - 1);
2087			break;
2088		default:
2089			shift = usbd_xfer_get_fps_shift(xfer);
2090			temp.isoc_delta = 1U << shift;
2091			x += temp.isoc_delta - 1;
2092			x &= ~(temp.isoc_delta - 1);
2093			/* simple frame load balancing */
2094			x += xfer->endpoint->usb_uframe;
2095			break;
2096		}
2097
2098		y = XHCI_MFINDEX_GET(x - xfer->endpoint->isoc_next);
2099
2100		if ((xfer->endpoint->is_synced == 0) ||
2101		    (y < (xfer->nframes << shift)) ||
2102		    (XHCI_MFINDEX_GET(-y) >= (128 * 8))) {
2103			/*
2104			 * If there is data underflow or the pipe
2105			 * queue is empty we schedule the transfer a
2106			 * few frames ahead of the current frame
2107			 * position. Else two isochronous transfers
2108			 * might overlap.
2109			 */
2110			xfer->endpoint->isoc_next = XHCI_MFINDEX_GET(x + (3 * 8));
2111			xfer->endpoint->is_synced = 1;
2112			temp.do_isoc_sync = 1;
2113
2114			DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2115		}
2116
2117		/* compute isochronous completion time */
2118
2119		y = XHCI_MFINDEX_GET(xfer->endpoint->isoc_next - (x & ~7));
2120
2121		xfer->isoc_time_complete =
2122		    usb_isoc_time_expand(&temp.sc->sc_bus, x / 8) +
2123		    (y / 8) + (((xfer->nframes << shift) + 7) / 8);
2124
2125		x = 0;
2126		temp.isoc_frame = xfer->endpoint->isoc_next;
2127		temp.trb_type = XHCI_TRB_TYPE_ISOCH;
2128
2129		xfer->endpoint->isoc_next += xfer->nframes << shift;
2130
2131	} else if (xfer->flags_int.control_xfr) {
2132		/* check if we should prepend a setup message */
2133
2134		if (xfer->flags_int.control_hdr) {
2135			temp.len = xfer->frlengths[0];
2136			temp.pc = xfer->frbuffers + 0;
2137			temp.shortpkt = temp.len ? 1 : 0;
2138			temp.trb_type = XHCI_TRB_TYPE_SETUP_STAGE;
2139			temp.direction = 0;
2140
2141			/* check for last frame */
2142			if (xfer->nframes == 1) {
2143				/* no STATUS stage yet, SETUP is last */
2144				if (xfer->flags_int.control_act)
2145					temp.last_frame = 1;
2146			}
2147
2148			xhci_setup_generic_chain_sub(&temp);
2149		}
2150		x = 1;
2151		mult = 1;
2152		temp.isoc_delta = 0;
2153		temp.isoc_frame = 0;
2154		temp.trb_type = xfer->flags_int.control_did_data ?
2155		    XHCI_TRB_TYPE_NORMAL : XHCI_TRB_TYPE_DATA_STAGE;
2156	} else {
2157		x = 0;
2158		mult = 1;
2159		temp.isoc_delta = 0;
2160		temp.isoc_frame = 0;
2161		temp.trb_type = XHCI_TRB_TYPE_NORMAL;
2162	}
2163
2164	if (x != xfer->nframes) {
2165                /* set up page_cache pointer */
2166                temp.pc = xfer->frbuffers + x;
2167		/* set endpoint direction */
2168		temp.direction = UE_GET_DIR(xfer->endpointno);
2169	}
2170
2171	while (x != xfer->nframes) {
2172		/* DATA0 / DATA1 message */
2173
2174		temp.len = xfer->frlengths[x];
2175		temp.step_td = ((xfer->endpointno & UE_DIR_IN) &&
2176		    x != 0 && temp.multishort == 0);
2177
2178		x++;
2179
2180		if (x == xfer->nframes) {
2181			if (xfer->flags_int.control_xfr) {
2182				/* no STATUS stage yet, DATA is last */
2183				if (xfer->flags_int.control_act)
2184					temp.last_frame = 1;
2185			} else {
2186				temp.last_frame = 1;
2187			}
2188		}
2189		if (temp.len == 0) {
2190			/* make sure that we send an USB packet */
2191
2192			temp.shortpkt = 0;
2193
2194			temp.tbc = 0;
2195			temp.tlbpc = mult - 1;
2196
2197		} else if (xfer->flags_int.isochronous_xfr) {
2198			uint8_t tdpc;
2199
2200			/*
2201			 * Isochronous transfers don't have short
2202			 * packet termination:
2203			 */
2204
2205			temp.shortpkt = 1;
2206
2207			/* isochronous transfers have a transfer limit */
2208
2209			if (temp.len > xfer->max_frame_size)
2210				temp.len = xfer->max_frame_size;
2211
2212			/* compute TD packet count */
2213			tdpc = howmany(temp.len, xfer->max_packet_size);
2214
2215			temp.tbc = howmany(tdpc, mult) - 1;
2216			temp.tlbpc = (tdpc % mult);
2217
2218			if (temp.tlbpc == 0)
2219				temp.tlbpc = mult - 1;
2220			else
2221				temp.tlbpc--;
2222		} else {
2223			/* regular data transfer */
2224
2225			temp.shortpkt = xfer->flags.force_short_xfer ? 0 : 1;
2226		}
2227
2228		xhci_setup_generic_chain_sub(&temp);
2229
2230		if (xfer->flags_int.isochronous_xfr) {
2231			temp.offset += xfer->frlengths[x - 1];
2232			temp.isoc_frame += temp.isoc_delta;
2233		} else {
2234			/* get next Page Cache pointer */
2235			temp.pc = xfer->frbuffers + x;
2236		}
2237	}
2238
2239	/* check if we should append a status stage */
2240
2241	if (xfer->flags_int.control_xfr &&
2242	    !xfer->flags_int.control_act) {
2243		/*
2244		 * Send a DATA1 message and invert the current
2245		 * endpoint direction.
2246		 */
2247		if (xhcictlstep || temp.sc->sc_ctlstep) {
2248			/*
2249			 * Some XHCI controllers will not delay the
2250			 * status stage until the next SOF. Force this
2251			 * behaviour to avoid failed control
2252			 * transfers.
2253			 */
2254			temp.step_td = (xfer->nframes != 0);
2255		} else {
2256			temp.step_td = 0;
2257		}
2258		temp.direction = UE_GET_DIR(xfer->endpointno) ^ UE_DIR_IN;
2259		temp.len = 0;
2260		temp.pc = NULL;
2261		temp.shortpkt = 0;
2262		temp.last_frame = 1;
2263		temp.trb_type = XHCI_TRB_TYPE_STATUS_STAGE;
2264
2265		xhci_setup_generic_chain_sub(&temp);
2266	}
2267
2268	td = temp.td;
2269
2270	/* must have at least one frame! */
2271
2272	xfer->td_transfer_last = td;
2273
2274	DPRINTF("first=%p last=%p\n", xfer->td_transfer_first, td);
2275}
2276
2277static void
2278xhci_set_slot_pointer(struct xhci_softc *sc, uint8_t index, uint64_t dev_addr)
2279{
2280	struct usb_page_search buf_res;
2281	struct xhci_dev_ctx_addr *pdctxa;
2282
2283	usbd_get_page(&sc->sc_hw.ctx_pc, 0, &buf_res);
2284
2285	pdctxa = buf_res.buffer;
2286
2287	DPRINTF("addr[%u]=0x%016llx\n", index, (long long)dev_addr);
2288
2289	pdctxa->qwBaaDevCtxAddr[index] = htole64(dev_addr);
2290
2291	usb_pc_cpu_flush(&sc->sc_hw.ctx_pc);
2292}
2293
2294static usb_error_t
2295xhci_configure_mask(struct usb_device *udev, uint32_t mask, uint8_t drop)
2296{
2297	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2298	struct usb_page_search buf_inp;
2299	struct xhci_input_dev_ctx *pinp;
2300	uint32_t temp;
2301	uint8_t index;
2302	uint8_t x;
2303
2304	index = udev->controller_slot_id;
2305
2306	usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp);
2307
2308	pinp = buf_inp.buffer;
2309
2310	if (drop) {
2311		mask &= XHCI_INCTX_NON_CTRL_MASK;
2312		xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx0, mask);
2313		xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx1, 0);
2314	} else {
2315		/*
2316		 * Some hardware requires that we drop the endpoint
2317		 * context before adding it again:
2318		 */
2319		xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx0,
2320		    mask & XHCI_INCTX_NON_CTRL_MASK);
2321
2322		/* Add new endpoint context */
2323		xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx1, mask);
2324
2325		/* find most significant set bit */
2326		for (x = 31; x != 1; x--) {
2327			if (mask & (1 << x))
2328				break;
2329		}
2330
2331		/* adjust */
2332		x--;
2333
2334		/* figure out the maximum number of contexts */
2335		if (x > sc->sc_hw.devs[index].context_num)
2336			sc->sc_hw.devs[index].context_num = x;
2337		else
2338			x = sc->sc_hw.devs[index].context_num;
2339
2340		/* update number of contexts */
2341		temp = xhci_ctx_get_le32(sc, &pinp->ctx_slot.dwSctx0);
2342		temp &= ~XHCI_SCTX_0_CTX_NUM_SET(31);
2343		temp |= XHCI_SCTX_0_CTX_NUM_SET(x + 1);
2344		xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx0, temp);
2345	}
2346	usb_pc_cpu_flush(&sc->sc_hw.devs[index].input_pc);
2347	return (0);
2348}
2349
2350static usb_error_t
2351xhci_configure_endpoint(struct usb_device *udev,
2352    struct usb_endpoint_descriptor *edesc, struct xhci_endpoint_ext *pepext,
2353    uint16_t interval, uint8_t max_packet_count,
2354    uint8_t mult, uint8_t fps_shift, uint16_t max_packet_size,
2355    uint16_t max_frame_size, uint8_t ep_mode)
2356{
2357	struct usb_page_search buf_inp;
2358	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2359	struct xhci_input_dev_ctx *pinp;
2360	uint64_t ring_addr = pepext->physaddr;
2361	uint32_t temp;
2362	uint8_t index;
2363	uint8_t epno;
2364	uint8_t type;
2365
2366	index = udev->controller_slot_id;
2367
2368	usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp);
2369
2370	pinp = buf_inp.buffer;
2371
2372	epno = edesc->bEndpointAddress;
2373	type = edesc->bmAttributes & UE_XFERTYPE;
2374
2375	if (type == UE_CONTROL)
2376		epno |= UE_DIR_IN;
2377
2378	epno = XHCI_EPNO2EPID(epno);
2379
2380 	if (epno == 0)
2381		return (USB_ERR_NO_PIPE);		/* invalid */
2382
2383	if (max_packet_count == 0)
2384		return (USB_ERR_BAD_BUFSIZE);
2385
2386	max_packet_count--;
2387
2388	if (mult == 0)
2389		return (USB_ERR_BAD_BUFSIZE);
2390
2391	/* store endpoint mode */
2392	pepext->trb_ep_mode = ep_mode;
2393	/* store bMaxPacketSize for control endpoints */
2394	pepext->trb_ep_maxp = edesc->wMaxPacketSize[0];
2395	usb_pc_cpu_flush(pepext->page_cache);
2396
2397	if (ep_mode == USB_EP_MODE_STREAMS) {
2398		temp = XHCI_EPCTX_0_EPSTATE_SET(0) |
2399		    XHCI_EPCTX_0_MAXP_STREAMS_SET(XHCI_MAX_STREAMS_LOG - 1) |
2400		    XHCI_EPCTX_0_LSA_SET(1);
2401
2402		ring_addr += sizeof(struct xhci_trb) *
2403		    XHCI_MAX_TRANSFERS * XHCI_MAX_STREAMS;
2404	} else {
2405		temp = XHCI_EPCTX_0_EPSTATE_SET(0) |
2406		    XHCI_EPCTX_0_MAXP_STREAMS_SET(0) |
2407		    XHCI_EPCTX_0_LSA_SET(0);
2408
2409		ring_addr |= XHCI_EPCTX_2_DCS_SET(1);
2410	}
2411
2412	switch (udev->speed) {
2413	case USB_SPEED_FULL:
2414	case USB_SPEED_LOW:
2415		/* 1ms -> 125us */
2416		fps_shift += 3;
2417		break;
2418	default:
2419		break;
2420	}
2421
2422	switch (type) {
2423	case UE_INTERRUPT:
2424		if (fps_shift > 3)
2425			fps_shift--;
2426		temp |= XHCI_EPCTX_0_IVAL_SET(fps_shift);
2427		break;
2428	case UE_ISOCHRONOUS:
2429		temp |= XHCI_EPCTX_0_IVAL_SET(fps_shift);
2430
2431		switch (udev->speed) {
2432		case USB_SPEED_SUPER:
2433			if (mult > 3)
2434				mult = 3;
2435			temp |= XHCI_EPCTX_0_MULT_SET(mult - 1);
2436			max_packet_count /= mult;
2437			break;
2438		default:
2439			break;
2440		}
2441		break;
2442	default:
2443		break;
2444	}
2445
2446	xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx0, temp);
2447
2448	temp =
2449	    XHCI_EPCTX_1_HID_SET(0) |
2450	    XHCI_EPCTX_1_MAXB_SET(max_packet_count) |
2451	    XHCI_EPCTX_1_MAXP_SIZE_SET(max_packet_size);
2452
2453	/*
2454	 * Always enable the "three strikes and you are gone" feature
2455	 * except for ISOCHRONOUS endpoints. This is suggested by
2456	 * section 4.3.3 in the XHCI specification about device slot
2457	 * initialisation.
2458	 */
2459	if (type != UE_ISOCHRONOUS)
2460		temp |= XHCI_EPCTX_1_CERR_SET(3);
2461
2462	switch (type) {
2463	case UE_CONTROL:
2464		temp |= XHCI_EPCTX_1_EPTYPE_SET(4);
2465		break;
2466	case UE_ISOCHRONOUS:
2467		temp |= XHCI_EPCTX_1_EPTYPE_SET(1);
2468		break;
2469	case UE_BULK:
2470		temp |= XHCI_EPCTX_1_EPTYPE_SET(2);
2471		break;
2472	default:
2473		temp |= XHCI_EPCTX_1_EPTYPE_SET(3);
2474		break;
2475	}
2476
2477	/* check for IN direction */
2478	if (epno & 1)
2479		temp |= XHCI_EPCTX_1_EPTYPE_SET(4);
2480
2481	xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx1, temp);
2482	xhci_ctx_set_le64(sc, &pinp->ctx_ep[epno - 1].qwEpCtx2, ring_addr);
2483
2484	switch (edesc->bmAttributes & UE_XFERTYPE) {
2485	case UE_INTERRUPT:
2486	case UE_ISOCHRONOUS:
2487		temp = XHCI_EPCTX_4_MAX_ESIT_PAYLOAD_SET(max_frame_size) |
2488		    XHCI_EPCTX_4_AVG_TRB_LEN_SET(MIN(XHCI_PAGE_SIZE,
2489		    max_frame_size));
2490		break;
2491	case UE_CONTROL:
2492		temp = XHCI_EPCTX_4_AVG_TRB_LEN_SET(8);
2493		break;
2494	default:
2495		temp = XHCI_EPCTX_4_AVG_TRB_LEN_SET(XHCI_PAGE_SIZE);
2496		break;
2497	}
2498
2499	xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx4, temp);
2500
2501#ifdef USB_DEBUG
2502	xhci_dump_endpoint(sc, &pinp->ctx_ep[epno - 1]);
2503#endif
2504	usb_pc_cpu_flush(&sc->sc_hw.devs[index].input_pc);
2505
2506	return (0);		/* success */
2507}
2508
2509static usb_error_t
2510xhci_configure_endpoint_by_xfer(struct usb_xfer *xfer)
2511{
2512	struct xhci_endpoint_ext *pepext;
2513	struct usb_endpoint_ss_comp_descriptor *ecomp;
2514	usb_stream_t x;
2515
2516	pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
2517	    xfer->endpoint->edesc);
2518
2519	ecomp = xfer->endpoint->ecomp;
2520
2521	for (x = 0; x != XHCI_MAX_STREAMS; x++) {
2522		uint64_t temp;
2523
2524		/* halt any transfers */
2525		pepext->trb[x * XHCI_MAX_TRANSFERS].dwTrb3 = 0;
2526
2527		/* compute start of TRB ring for stream "x" */
2528		temp = pepext->physaddr +
2529		    (x * XHCI_MAX_TRANSFERS * sizeof(struct xhci_trb)) +
2530		    XHCI_SCTX_0_SCT_SEC_TR_RING;
2531
2532		/* make tree structure */
2533		pepext->trb[(XHCI_MAX_TRANSFERS *
2534		    XHCI_MAX_STREAMS) + x].qwTrb0 = htole64(temp);
2535
2536		/* reserved fields */
2537		pepext->trb[(XHCI_MAX_TRANSFERS *
2538                    XHCI_MAX_STREAMS) + x].dwTrb2 = 0;
2539		pepext->trb[(XHCI_MAX_TRANSFERS *
2540		    XHCI_MAX_STREAMS) + x].dwTrb3 = 0;
2541	}
2542	usb_pc_cpu_flush(pepext->page_cache);
2543
2544	return (xhci_configure_endpoint(xfer->xroot->udev,
2545	    xfer->endpoint->edesc, pepext,
2546	    xfer->interval, xfer->max_packet_count,
2547	    (ecomp != NULL) ? UE_GET_SS_ISO_MULT(ecomp->bmAttributes) + 1 : 1,
2548	    usbd_xfer_get_fps_shift(xfer), xfer->max_packet_size,
2549	    xfer->max_frame_size, xfer->endpoint->ep_mode));
2550}
2551
2552static usb_error_t
2553xhci_configure_device(struct usb_device *udev)
2554{
2555	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2556	struct usb_page_search buf_inp;
2557	struct usb_page_cache *pcinp;
2558	struct xhci_input_dev_ctx *pinp;
2559	struct usb_device *hubdev;
2560	uint32_t temp;
2561	uint32_t route;
2562	uint32_t rh_port;
2563	uint8_t is_hub;
2564	uint8_t index;
2565	uint8_t depth;
2566
2567	index = udev->controller_slot_id;
2568
2569	DPRINTF("index=%u\n", index);
2570
2571	pcinp = &sc->sc_hw.devs[index].input_pc;
2572
2573	usbd_get_page(pcinp, 0, &buf_inp);
2574
2575	pinp = buf_inp.buffer;
2576
2577	rh_port = 0;
2578	route = 0;
2579
2580	/* figure out route string and root HUB port number */
2581
2582	for (hubdev = udev; hubdev != NULL; hubdev = hubdev->parent_hub) {
2583		if (hubdev->parent_hub == NULL)
2584			break;
2585
2586		depth = hubdev->parent_hub->depth;
2587
2588		/*
2589		 * NOTE: HS/FS/LS devices and the SS root HUB can have
2590		 * more than 15 ports
2591		 */
2592
2593		rh_port = hubdev->port_no;
2594
2595		if (depth == 0)
2596			break;
2597
2598		if (rh_port > 15)
2599			rh_port = 15;
2600
2601		if (depth < 6)
2602			route |= rh_port << (4 * (depth - 1));
2603	}
2604
2605	DPRINTF("Route=0x%08x\n", route);
2606
2607	temp = XHCI_SCTX_0_ROUTE_SET(route) |
2608	    XHCI_SCTX_0_CTX_NUM_SET(
2609	    sc->sc_hw.devs[index].context_num + 1);
2610
2611	switch (udev->speed) {
2612	case USB_SPEED_LOW:
2613		temp |= XHCI_SCTX_0_SPEED_SET(2);
2614		if (udev->parent_hs_hub != NULL &&
2615		    udev->parent_hs_hub->ddesc.bDeviceProtocol ==
2616		    UDPROTO_HSHUBMTT) {
2617			DPRINTF("Device inherits MTT\n");
2618			temp |= XHCI_SCTX_0_MTT_SET(1);
2619		}
2620		break;
2621	case USB_SPEED_HIGH:
2622		temp |= XHCI_SCTX_0_SPEED_SET(3);
2623		if (sc->sc_hw.devs[index].nports != 0 &&
2624		    udev->ddesc.bDeviceProtocol == UDPROTO_HSHUBMTT) {
2625			DPRINTF("HUB supports MTT\n");
2626			temp |= XHCI_SCTX_0_MTT_SET(1);
2627		}
2628		break;
2629	case USB_SPEED_FULL:
2630		temp |= XHCI_SCTX_0_SPEED_SET(1);
2631		if (udev->parent_hs_hub != NULL &&
2632		    udev->parent_hs_hub->ddesc.bDeviceProtocol ==
2633		    UDPROTO_HSHUBMTT) {
2634			DPRINTF("Device inherits MTT\n");
2635			temp |= XHCI_SCTX_0_MTT_SET(1);
2636		}
2637		break;
2638	default:
2639		temp |= XHCI_SCTX_0_SPEED_SET(4);
2640		break;
2641	}
2642
2643	is_hub = sc->sc_hw.devs[index].nports != 0 &&
2644	    (udev->speed == USB_SPEED_SUPER ||
2645	    udev->speed == USB_SPEED_HIGH);
2646
2647	if (is_hub)
2648		temp |= XHCI_SCTX_0_HUB_SET(1);
2649
2650	xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx0, temp);
2651
2652	temp = XHCI_SCTX_1_RH_PORT_SET(rh_port);
2653
2654	if (is_hub) {
2655		temp |= XHCI_SCTX_1_NUM_PORTS_SET(
2656		    sc->sc_hw.devs[index].nports);
2657	}
2658
2659	xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx1, temp);
2660
2661	temp = XHCI_SCTX_2_IRQ_TARGET_SET(0);
2662
2663	if (is_hub) {
2664		temp |= XHCI_SCTX_2_TT_THINK_TIME_SET(
2665		    sc->sc_hw.devs[index].tt);
2666	}
2667
2668	hubdev = udev->parent_hs_hub;
2669
2670	/* check if we should activate the transaction translator */
2671	switch (udev->speed) {
2672	case USB_SPEED_FULL:
2673	case USB_SPEED_LOW:
2674		if (hubdev != NULL) {
2675			temp |= XHCI_SCTX_2_TT_HUB_SID_SET(
2676			    hubdev->controller_slot_id);
2677			temp |= XHCI_SCTX_2_TT_PORT_NUM_SET(
2678			    udev->hs_port_no);
2679		}
2680		break;
2681	default:
2682		break;
2683	}
2684
2685	xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx2, temp);
2686
2687	/*
2688	 * These fields should be initialized to zero, according to
2689	 * XHCI section 6.2.2 - slot context:
2690	 */
2691	temp = XHCI_SCTX_3_DEV_ADDR_SET(0) |
2692	    XHCI_SCTX_3_SLOT_STATE_SET(0);
2693
2694	xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx3, temp);
2695
2696#ifdef USB_DEBUG
2697	xhci_dump_device(sc, &pinp->ctx_slot);
2698#endif
2699	usb_pc_cpu_flush(pcinp);
2700
2701	return (0);		/* success */
2702}
2703
2704static usb_error_t
2705xhci_alloc_device_ext(struct usb_device *udev)
2706{
2707	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2708	struct usb_page_search buf_dev;
2709	struct usb_page_search buf_ep;
2710	struct xhci_trb *trb;
2711	struct usb_page_cache *pc;
2712	struct usb_page *pg;
2713	uint64_t addr;
2714	uint8_t index;
2715	uint8_t i;
2716
2717	index = udev->controller_slot_id;
2718
2719	pc = &sc->sc_hw.devs[index].device_pc;
2720	pg = &sc->sc_hw.devs[index].device_pg;
2721
2722	/* need to initialize the page cache */
2723	pc->tag_parent = sc->sc_bus.dma_parent_tag;
2724
2725	if (usb_pc_alloc_mem(pc, pg, sc->sc_ctx_is_64_byte ?
2726	    (2 * sizeof(struct xhci_dev_ctx)) :
2727	    sizeof(struct xhci_dev_ctx), XHCI_PAGE_SIZE))
2728		goto error;
2729
2730	usbd_get_page(pc, 0, &buf_dev);
2731
2732	pc = &sc->sc_hw.devs[index].input_pc;
2733	pg = &sc->sc_hw.devs[index].input_pg;
2734
2735	/* need to initialize the page cache */
2736	pc->tag_parent = sc->sc_bus.dma_parent_tag;
2737
2738	if (usb_pc_alloc_mem(pc, pg, sc->sc_ctx_is_64_byte ?
2739	    (2 * sizeof(struct xhci_input_dev_ctx)) :
2740	    sizeof(struct xhci_input_dev_ctx), XHCI_PAGE_SIZE)) {
2741		goto error;
2742	}
2743
2744	/* initialize all endpoint LINK TRBs */
2745
2746	for (i = 0; i != XHCI_MAX_ENDPOINTS; i++) {
2747		pc = &sc->sc_hw.devs[index].endpoint_pc[i];
2748		pg = &sc->sc_hw.devs[index].endpoint_pg[i];
2749
2750		/* need to initialize the page cache */
2751		pc->tag_parent = sc->sc_bus.dma_parent_tag;
2752
2753		if (usb_pc_alloc_mem(pc, pg,
2754		    sizeof(struct xhci_dev_endpoint_trbs), XHCI_TRB_ALIGN)) {
2755			goto error;
2756		}
2757
2758		/* lookup endpoint TRB ring */
2759		usbd_get_page(pc, 0, &buf_ep);
2760
2761		/* get TRB pointer */
2762		trb = buf_ep.buffer;
2763		trb += XHCI_MAX_TRANSFERS - 1;
2764
2765		/* get TRB start address */
2766		addr = buf_ep.physaddr;
2767
2768		/* create LINK TRB */
2769		trb->qwTrb0 = htole64(addr);
2770		trb->dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0));
2771		trb->dwTrb3 = htole32(XHCI_TRB_3_CYCLE_BIT |
2772		    XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK));
2773
2774		usb_pc_cpu_flush(pc);
2775	}
2776
2777	xhci_set_slot_pointer(sc, index, buf_dev.physaddr);
2778
2779	return (0);
2780
2781error:
2782	xhci_free_device_ext(udev);
2783
2784	return (USB_ERR_NOMEM);
2785}
2786
2787static void
2788xhci_free_device_ext(struct usb_device *udev)
2789{
2790	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2791	uint8_t index;
2792	uint8_t i;
2793
2794	index = udev->controller_slot_id;
2795	xhci_set_slot_pointer(sc, index, 0);
2796
2797	usb_pc_free_mem(&sc->sc_hw.devs[index].device_pc);
2798	usb_pc_free_mem(&sc->sc_hw.devs[index].input_pc);
2799	for (i = 0; i != XHCI_MAX_ENDPOINTS; i++)
2800		usb_pc_free_mem(&sc->sc_hw.devs[index].endpoint_pc[i]);
2801}
2802
2803static struct xhci_endpoint_ext *
2804xhci_get_endpoint_ext(struct usb_device *udev, struct usb_endpoint_descriptor *edesc)
2805{
2806	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2807	struct xhci_endpoint_ext *pepext;
2808	struct usb_page_cache *pc;
2809	struct usb_page_search buf_ep;
2810	uint8_t epno;
2811	uint8_t index;
2812
2813	epno = edesc->bEndpointAddress;
2814	if ((edesc->bmAttributes & UE_XFERTYPE) == UE_CONTROL)
2815		epno |= UE_DIR_IN;
2816
2817	epno = XHCI_EPNO2EPID(epno);
2818
2819	index = udev->controller_slot_id;
2820
2821	pc = &sc->sc_hw.devs[index].endpoint_pc[epno];
2822
2823	usbd_get_page(pc, 0, &buf_ep);
2824
2825	pepext = &sc->sc_hw.devs[index].endp[epno];
2826	pepext->page_cache = pc;
2827	pepext->trb = buf_ep.buffer;
2828	pepext->physaddr = buf_ep.physaddr;
2829
2830	return (pepext);
2831}
2832
2833static void
2834xhci_endpoint_doorbell(struct usb_xfer *xfer)
2835{
2836	struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus);
2837	uint8_t epno;
2838	uint8_t index;
2839
2840	epno = xfer->endpointno;
2841	if (xfer->flags_int.control_xfr)
2842		epno |= UE_DIR_IN;
2843
2844	epno = XHCI_EPNO2EPID(epno);
2845	index = xfer->xroot->udev->controller_slot_id;
2846
2847	if (xfer->xroot->udev->flags.self_suspended == 0) {
2848		XWRITE4(sc, door, XHCI_DOORBELL(index),
2849		    epno | XHCI_DB_SID_SET(xfer->stream_id));
2850	}
2851}
2852
2853static void
2854xhci_transfer_remove(struct usb_xfer *xfer, usb_error_t error)
2855{
2856	struct xhci_endpoint_ext *pepext;
2857
2858	if (xfer->flags_int.bandwidth_reclaimed) {
2859		xfer->flags_int.bandwidth_reclaimed = 0;
2860
2861		pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
2862		    xfer->endpoint->edesc);
2863
2864		pepext->trb_used[xfer->stream_id]--;
2865
2866		pepext->xfer[xfer->qh_pos] = NULL;
2867
2868		if (error && pepext->trb_running != 0) {
2869			pepext->trb_halted = 1;
2870			pepext->trb_running = 0;
2871		}
2872	}
2873}
2874
2875static usb_error_t
2876xhci_transfer_insert(struct usb_xfer *xfer)
2877{
2878	struct xhci_td *td_first;
2879	struct xhci_td *td_last;
2880	struct xhci_trb *trb_link;
2881	struct xhci_endpoint_ext *pepext;
2882	uint64_t addr;
2883	usb_stream_t id;
2884	uint8_t i;
2885	uint8_t inext;
2886	uint8_t trb_limit;
2887
2888	DPRINTFN(8, "\n");
2889
2890	id = xfer->stream_id;
2891
2892	/* check if already inserted */
2893	if (xfer->flags_int.bandwidth_reclaimed) {
2894		DPRINTFN(8, "Already in schedule\n");
2895		return (0);
2896	}
2897
2898	pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
2899	    xfer->endpoint->edesc);
2900
2901	td_first = xfer->td_transfer_first;
2902	td_last = xfer->td_transfer_last;
2903	addr = pepext->physaddr;
2904
2905	switch (xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE) {
2906	case UE_CONTROL:
2907	case UE_INTERRUPT:
2908		/* single buffered */
2909		trb_limit = 1;
2910		break;
2911	default:
2912		/* multi buffered */
2913		trb_limit = (XHCI_MAX_TRANSFERS - 2);
2914		break;
2915	}
2916
2917	if (pepext->trb_used[id] >= trb_limit) {
2918		DPRINTFN(8, "Too many TDs queued.\n");
2919		return (USB_ERR_NOMEM);
2920	}
2921
2922	/* check if bMaxPacketSize changed */
2923	if (xfer->flags_int.control_xfr != 0 &&
2924	    pepext->trb_ep_maxp != xfer->endpoint->edesc->wMaxPacketSize[0]) {
2925		DPRINTFN(8, "Reconfigure control endpoint\n");
2926
2927		/* force driver to reconfigure endpoint */
2928		pepext->trb_halted = 1;
2929		pepext->trb_running = 0;
2930	}
2931
2932	/* check for stopped condition, after putting transfer on interrupt queue */
2933	if (pepext->trb_running == 0) {
2934		struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus);
2935
2936		DPRINTFN(8, "Not running\n");
2937
2938		/* start configuration */
2939		(void)usb_proc_msignal(USB_BUS_CONTROL_XFER_PROC(&sc->sc_bus),
2940		    &sc->sc_config_msg[0], &sc->sc_config_msg[1]);
2941		return (0);
2942	}
2943
2944	pepext->trb_used[id]++;
2945
2946	/* get current TRB index */
2947	i = pepext->trb_index[id];
2948
2949	/* get next TRB index */
2950	inext = (i + 1);
2951
2952	/* the last entry of the ring is a hardcoded link TRB */
2953	if (inext >= (XHCI_MAX_TRANSFERS - 1))
2954		inext = 0;
2955
2956	/* store next TRB index, before stream ID offset is added */
2957	pepext->trb_index[id] = inext;
2958
2959	/* offset for stream */
2960	i += id * XHCI_MAX_TRANSFERS;
2961	inext += id * XHCI_MAX_TRANSFERS;
2962
2963	/* compute terminating return address */
2964	addr += (inext * sizeof(struct xhci_trb));
2965
2966	/* compute link TRB pointer */
2967	trb_link = td_last->td_trb + td_last->ntrb;
2968
2969	/* update next pointer of last link TRB */
2970	trb_link->qwTrb0 = htole64(addr);
2971	trb_link->dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0));
2972	trb_link->dwTrb3 = htole32(XHCI_TRB_3_IOC_BIT |
2973	    XHCI_TRB_3_CYCLE_BIT |
2974	    XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK));
2975
2976#ifdef USB_DEBUG
2977	xhci_dump_trb(&td_last->td_trb[td_last->ntrb]);
2978#endif
2979	usb_pc_cpu_flush(td_last->page_cache);
2980
2981	/* write ahead chain end marker */
2982
2983	pepext->trb[inext].qwTrb0 = 0;
2984	pepext->trb[inext].dwTrb2 = 0;
2985	pepext->trb[inext].dwTrb3 = 0;
2986
2987	/* update next pointer of link TRB */
2988
2989	pepext->trb[i].qwTrb0 = htole64((uint64_t)td_first->td_self);
2990	pepext->trb[i].dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0));
2991
2992#ifdef USB_DEBUG
2993	xhci_dump_trb(&pepext->trb[i]);
2994#endif
2995	usb_pc_cpu_flush(pepext->page_cache);
2996
2997	/* toggle cycle bit which activates the transfer chain */
2998
2999	pepext->trb[i].dwTrb3 = htole32(XHCI_TRB_3_CYCLE_BIT |
3000	    XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK));
3001
3002	usb_pc_cpu_flush(pepext->page_cache);
3003
3004	DPRINTF("qh_pos = %u\n", i);
3005
3006	pepext->xfer[i] = xfer;
3007
3008	xfer->qh_pos = i;
3009
3010	xfer->flags_int.bandwidth_reclaimed = 1;
3011
3012	xhci_endpoint_doorbell(xfer);
3013
3014	return (0);
3015}
3016
3017static void
3018xhci_root_intr(struct xhci_softc *sc)
3019{
3020	uint16_t i;
3021
3022	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3023
3024	/* clear any old interrupt data */
3025	memset(sc->sc_hub_idata, 0, sizeof(sc->sc_hub_idata));
3026
3027	for (i = 1; i <= sc->sc_noport; i++) {
3028		/* pick out CHANGE bits from the status register */
3029		if (XREAD4(sc, oper, XHCI_PORTSC(i)) & (
3030		    XHCI_PS_CSC | XHCI_PS_PEC |
3031		    XHCI_PS_OCC | XHCI_PS_WRC |
3032		    XHCI_PS_PRC | XHCI_PS_PLC |
3033		    XHCI_PS_CEC)) {
3034			sc->sc_hub_idata[i / 8] |= 1 << (i % 8);
3035			DPRINTF("port %d changed\n", i);
3036		}
3037	}
3038	uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
3039	    sizeof(sc->sc_hub_idata));
3040}
3041
3042/*------------------------------------------------------------------------*
3043 *	xhci_device_done - XHCI done handler
3044 *
3045 * NOTE: This function can be called two times in a row on
3046 * the same USB transfer. From close and from interrupt.
3047 *------------------------------------------------------------------------*/
3048static void
3049xhci_device_done(struct usb_xfer *xfer, usb_error_t error)
3050{
3051	DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n",
3052	    xfer, xfer->endpoint, error);
3053
3054	/* remove transfer from HW queue */
3055	xhci_transfer_remove(xfer, error);
3056
3057	/* dequeue transfer and start next transfer */
3058	usbd_transfer_done(xfer, error);
3059}
3060
3061/*------------------------------------------------------------------------*
3062 * XHCI data transfer support (generic type)
3063 *------------------------------------------------------------------------*/
3064static void
3065xhci_device_generic_open(struct usb_xfer *xfer)
3066{
3067	if (xfer->flags_int.isochronous_xfr) {
3068		switch (xfer->xroot->udev->speed) {
3069		case USB_SPEED_FULL:
3070			break;
3071		default:
3072			usb_hs_bandwidth_alloc(xfer);
3073			break;
3074		}
3075	}
3076}
3077
3078static void
3079xhci_device_generic_close(struct usb_xfer *xfer)
3080{
3081	DPRINTF("\n");
3082
3083	xhci_device_done(xfer, USB_ERR_CANCELLED);
3084
3085	if (xfer->flags_int.isochronous_xfr) {
3086		switch (xfer->xroot->udev->speed) {
3087		case USB_SPEED_FULL:
3088			break;
3089		default:
3090			usb_hs_bandwidth_free(xfer);
3091			break;
3092		}
3093	}
3094}
3095
3096static void
3097xhci_device_generic_multi_enter(struct usb_endpoint *ep,
3098    usb_stream_t stream_id, struct usb_xfer *enter_xfer)
3099{
3100	struct usb_xfer *xfer;
3101
3102	/* check if there is a current transfer */
3103	xfer = ep->endpoint_q[stream_id].curr;
3104	if (xfer == NULL)
3105		return;
3106
3107	/*
3108	 * Check if the current transfer is started and then pickup
3109	 * the next one, if any. Else wait for next start event due to
3110	 * block on failure feature.
3111	 */
3112	if (!xfer->flags_int.bandwidth_reclaimed)
3113		return;
3114
3115	xfer = TAILQ_FIRST(&ep->endpoint_q[stream_id].head);
3116	if (xfer == NULL) {
3117		/*
3118		 * In case of enter we have to consider that the
3119		 * transfer is queued by the USB core after the enter
3120		 * method is called.
3121		 */
3122		xfer = enter_xfer;
3123
3124		if (xfer == NULL)
3125			return;
3126	}
3127
3128	/* try to multi buffer */
3129	xhci_transfer_insert(xfer);
3130}
3131
3132static void
3133xhci_device_generic_enter(struct usb_xfer *xfer)
3134{
3135	DPRINTF("\n");
3136
3137	/* set up TD's and QH */
3138	xhci_setup_generic_chain(xfer);
3139
3140	xhci_device_generic_multi_enter(xfer->endpoint,
3141	    xfer->stream_id, xfer);
3142}
3143
3144static void
3145xhci_device_generic_start(struct usb_xfer *xfer)
3146{
3147	DPRINTF("\n");
3148
3149	/* try to insert xfer on HW queue */
3150	xhci_transfer_insert(xfer);
3151
3152	/* try to multi buffer */
3153	xhci_device_generic_multi_enter(xfer->endpoint,
3154	    xfer->stream_id, NULL);
3155
3156	/* add transfer last on interrupt queue */
3157	usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
3158
3159	/* start timeout, if any */
3160	if (xfer->timeout != 0)
3161		usbd_transfer_timeout_ms(xfer, &xhci_timeout, xfer->timeout);
3162}
3163
3164static const struct usb_pipe_methods xhci_device_generic_methods =
3165{
3166	.open = xhci_device_generic_open,
3167	.close = xhci_device_generic_close,
3168	.enter = xhci_device_generic_enter,
3169	.start = xhci_device_generic_start,
3170};
3171
3172/*------------------------------------------------------------------------*
3173 * xhci root HUB support
3174 *------------------------------------------------------------------------*
3175 * Simulate a hardware HUB by handling all the necessary requests.
3176 *------------------------------------------------------------------------*/
3177#define	HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) }
3178
3179static const
3180struct usb_device_descriptor xhci_devd =
3181{
3182	.bLength = sizeof(xhci_devd),
3183	.bDescriptorType = UDESC_DEVICE,	/* type */
3184	HSETW(.bcdUSB, 0x0300),			/* USB version */
3185	.bDeviceClass = UDCLASS_HUB,		/* class */
3186	.bDeviceSubClass = UDSUBCLASS_HUB,	/* subclass */
3187	.bDeviceProtocol = UDPROTO_SSHUB,	/* protocol */
3188	.bMaxPacketSize = 9,			/* max packet size */
3189	HSETW(.idVendor, 0x0000),		/* vendor */
3190	HSETW(.idProduct, 0x0000),		/* product */
3191	HSETW(.bcdDevice, 0x0100),		/* device version */
3192	.iManufacturer = 1,
3193	.iProduct = 2,
3194	.iSerialNumber = 0,
3195	.bNumConfigurations = 1,		/* # of configurations */
3196};
3197
3198static const
3199struct xhci_bos_desc xhci_bosd = {
3200	.bosd = {
3201		.bLength = sizeof(xhci_bosd.bosd),
3202		.bDescriptorType = UDESC_BOS,
3203		HSETW(.wTotalLength, sizeof(xhci_bosd)),
3204		.bNumDeviceCaps = 3,
3205	},
3206	.usb2extd = {
3207		.bLength = sizeof(xhci_bosd.usb2extd),
3208		.bDescriptorType = 1,
3209		.bDevCapabilityType = 2,
3210		.bmAttributes[0] = 2,
3211	},
3212	.usbdcd = {
3213		.bLength = sizeof(xhci_bosd.usbdcd),
3214		.bDescriptorType = UDESC_DEVICE_CAPABILITY,
3215		.bDevCapabilityType = 3,
3216		.bmAttributes = 0, /* XXX */
3217		HSETW(.wSpeedsSupported, 0x000C),
3218		.bFunctionalitySupport = 8,
3219		.bU1DevExitLat = 255,	/* dummy - not used */
3220		.wU2DevExitLat = { 0x00, 0x08 },
3221	},
3222	.cidd = {
3223		.bLength = sizeof(xhci_bosd.cidd),
3224		.bDescriptorType = 1,
3225		.bDevCapabilityType = 4,
3226		.bReserved = 0,
3227		.bContainerID = 0, /* XXX */
3228	},
3229};
3230
3231static const
3232struct xhci_config_desc xhci_confd = {
3233	.confd = {
3234		.bLength = sizeof(xhci_confd.confd),
3235		.bDescriptorType = UDESC_CONFIG,
3236		.wTotalLength[0] = sizeof(xhci_confd),
3237		.bNumInterface = 1,
3238		.bConfigurationValue = 1,
3239		.iConfiguration = 0,
3240		.bmAttributes = UC_SELF_POWERED,
3241		.bMaxPower = 0		/* max power */
3242	},
3243	.ifcd = {
3244		.bLength = sizeof(xhci_confd.ifcd),
3245		.bDescriptorType = UDESC_INTERFACE,
3246		.bNumEndpoints = 1,
3247		.bInterfaceClass = UICLASS_HUB,
3248		.bInterfaceSubClass = UISUBCLASS_HUB,
3249		.bInterfaceProtocol = 0,
3250	},
3251	.endpd = {
3252		.bLength = sizeof(xhci_confd.endpd),
3253		.bDescriptorType = UDESC_ENDPOINT,
3254		.bEndpointAddress = UE_DIR_IN | XHCI_INTR_ENDPT,
3255		.bmAttributes = UE_INTERRUPT,
3256		.wMaxPacketSize[0] = 2,		/* max 15 ports */
3257		.bInterval = 255,
3258	},
3259	.endpcd = {
3260		.bLength = sizeof(xhci_confd.endpcd),
3261		.bDescriptorType = UDESC_ENDPOINT_SS_COMP,
3262		.bMaxBurst = 0,
3263		.bmAttributes = 0,
3264	},
3265};
3266
3267static const
3268struct usb_hub_ss_descriptor xhci_hubd = {
3269	.bLength = sizeof(xhci_hubd),
3270	.bDescriptorType = UDESC_SS_HUB,
3271};
3272
3273static usb_error_t
3274xhci_roothub_exec(struct usb_device *udev,
3275    struct usb_device_request *req, const void **pptr, uint16_t *plength)
3276{
3277	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
3278	const char *str_ptr;
3279	const void *ptr;
3280	uint32_t port;
3281	uint32_t v;
3282	uint16_t len;
3283	uint16_t i;
3284	uint16_t value;
3285	uint16_t index;
3286	uint8_t j;
3287	usb_error_t err;
3288
3289	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3290
3291	/* buffer reset */
3292	ptr = (const void *)&sc->sc_hub_desc;
3293	len = 0;
3294	err = 0;
3295
3296	value = UGETW(req->wValue);
3297	index = UGETW(req->wIndex);
3298
3299	DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x "
3300	    "wValue=0x%04x wIndex=0x%04x\n",
3301	    req->bmRequestType, req->bRequest,
3302	    UGETW(req->wLength), value, index);
3303
3304#define	C(x,y) ((x) | ((y) << 8))
3305	switch (C(req->bRequest, req->bmRequestType)) {
3306	case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
3307	case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
3308	case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
3309		/*
3310		 * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
3311		 * for the integrated root hub.
3312		 */
3313		break;
3314	case C(UR_GET_CONFIG, UT_READ_DEVICE):
3315		len = 1;
3316		sc->sc_hub_desc.temp[0] = sc->sc_conf;
3317		break;
3318	case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
3319		switch (value >> 8) {
3320		case UDESC_DEVICE:
3321			if ((value & 0xff) != 0) {
3322				err = USB_ERR_IOERROR;
3323				goto done;
3324			}
3325			len = sizeof(xhci_devd);
3326			ptr = (const void *)&xhci_devd;
3327			break;
3328
3329		case UDESC_BOS:
3330			if ((value & 0xff) != 0) {
3331				err = USB_ERR_IOERROR;
3332				goto done;
3333			}
3334			len = sizeof(xhci_bosd);
3335			ptr = (const void *)&xhci_bosd;
3336			break;
3337
3338		case UDESC_CONFIG:
3339			if ((value & 0xff) != 0) {
3340				err = USB_ERR_IOERROR;
3341				goto done;
3342			}
3343			len = sizeof(xhci_confd);
3344			ptr = (const void *)&xhci_confd;
3345			break;
3346
3347		case UDESC_STRING:
3348			switch (value & 0xff) {
3349			case 0:	/* Language table */
3350				str_ptr = "\001";
3351				break;
3352
3353			case 1:	/* Vendor */
3354				str_ptr = sc->sc_vendor;
3355				break;
3356
3357			case 2:	/* Product */
3358				str_ptr = "XHCI root HUB";
3359				break;
3360
3361			default:
3362				str_ptr = "";
3363				break;
3364			}
3365
3366			len = usb_make_str_desc(
3367			    sc->sc_hub_desc.temp,
3368			    sizeof(sc->sc_hub_desc.temp),
3369			    str_ptr);
3370			break;
3371
3372		default:
3373			err = USB_ERR_IOERROR;
3374			goto done;
3375		}
3376		break;
3377	case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
3378		len = 1;
3379		sc->sc_hub_desc.temp[0] = 0;
3380		break;
3381	case C(UR_GET_STATUS, UT_READ_DEVICE):
3382		len = 2;
3383		USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED);
3384		break;
3385	case C(UR_GET_STATUS, UT_READ_INTERFACE):
3386	case C(UR_GET_STATUS, UT_READ_ENDPOINT):
3387		len = 2;
3388		USETW(sc->sc_hub_desc.stat.wStatus, 0);
3389		break;
3390	case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
3391		if (value >= XHCI_MAX_DEVICES) {
3392			err = USB_ERR_IOERROR;
3393			goto done;
3394		}
3395		break;
3396	case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
3397		if (value != 0 && value != 1) {
3398			err = USB_ERR_IOERROR;
3399			goto done;
3400		}
3401		sc->sc_conf = value;
3402		break;
3403	case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
3404		break;
3405	case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
3406	case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
3407	case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
3408		err = USB_ERR_IOERROR;
3409		goto done;
3410	case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
3411		break;
3412	case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
3413		break;
3414		/* Hub requests */
3415	case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
3416		break;
3417	case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
3418		DPRINTFN(9, "UR_CLEAR_PORT_FEATURE\n");
3419
3420		if ((index < 1) ||
3421		    (index > sc->sc_noport)) {
3422			err = USB_ERR_IOERROR;
3423			goto done;
3424		}
3425		port = XHCI_PORTSC(index);
3426
3427		v = XREAD4(sc, oper, port);
3428		i = XHCI_PS_PLS_GET(v);
3429		v &= ~XHCI_PS_CLEAR;
3430
3431		switch (value) {
3432		case UHF_C_BH_PORT_RESET:
3433			XWRITE4(sc, oper, port, v | XHCI_PS_WRC);
3434			break;
3435		case UHF_C_PORT_CONFIG_ERROR:
3436			XWRITE4(sc, oper, port, v | XHCI_PS_CEC);
3437			break;
3438		case UHF_C_PORT_SUSPEND:
3439		case UHF_C_PORT_LINK_STATE:
3440			XWRITE4(sc, oper, port, v | XHCI_PS_PLC);
3441			break;
3442		case UHF_C_PORT_CONNECTION:
3443			XWRITE4(sc, oper, port, v | XHCI_PS_CSC);
3444			break;
3445		case UHF_C_PORT_ENABLE:
3446			XWRITE4(sc, oper, port, v | XHCI_PS_PEC);
3447			break;
3448		case UHF_C_PORT_OVER_CURRENT:
3449			XWRITE4(sc, oper, port, v | XHCI_PS_OCC);
3450			break;
3451		case UHF_C_PORT_RESET:
3452			XWRITE4(sc, oper, port, v | XHCI_PS_PRC);
3453			break;
3454		case UHF_PORT_ENABLE:
3455			XWRITE4(sc, oper, port, v | XHCI_PS_PED);
3456			break;
3457		case UHF_PORT_POWER:
3458			XWRITE4(sc, oper, port, v & ~XHCI_PS_PP);
3459			break;
3460		case UHF_PORT_INDICATOR:
3461			XWRITE4(sc, oper, port, v & ~XHCI_PS_PIC_SET(3));
3462			break;
3463		case UHF_PORT_SUSPEND:
3464
3465			/* U3 -> U15 */
3466			if (i == 3) {
3467				XWRITE4(sc, oper, port, v |
3468				    XHCI_PS_PLS_SET(0xF) | XHCI_PS_LWS);
3469			}
3470
3471			/* wait 20ms for resume sequence to complete */
3472			usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 50);
3473
3474			/* U0 */
3475			XWRITE4(sc, oper, port, v |
3476			    XHCI_PS_PLS_SET(0) | XHCI_PS_LWS);
3477			break;
3478		default:
3479			err = USB_ERR_IOERROR;
3480			goto done;
3481		}
3482		break;
3483
3484	case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
3485		if ((value & 0xff) != 0) {
3486			err = USB_ERR_IOERROR;
3487			goto done;
3488		}
3489
3490		v = XREAD4(sc, capa, XHCI_HCSPARAMS0);
3491
3492		sc->sc_hub_desc.hubd = xhci_hubd;
3493
3494		sc->sc_hub_desc.hubd.bNbrPorts = sc->sc_noport;
3495
3496		if (XHCI_HCS0_PPC(v))
3497			i = UHD_PWR_INDIVIDUAL;
3498		else
3499			i = UHD_PWR_GANGED;
3500
3501		if (XHCI_HCS0_PIND(v))
3502			i |= UHD_PORT_IND;
3503
3504		i |= UHD_OC_INDIVIDUAL;
3505
3506		USETW(sc->sc_hub_desc.hubd.wHubCharacteristics, i);
3507
3508		/* see XHCI section 5.4.9: */
3509		sc->sc_hub_desc.hubd.bPwrOn2PwrGood = 10;
3510
3511		for (j = 1; j <= sc->sc_noport; j++) {
3512			v = XREAD4(sc, oper, XHCI_PORTSC(j));
3513			if (v & XHCI_PS_DR) {
3514				sc->sc_hub_desc.hubd.
3515				    DeviceRemovable[j / 8] |= 1U << (j % 8);
3516			}
3517		}
3518		len = sc->sc_hub_desc.hubd.bLength;
3519		break;
3520
3521	case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
3522		len = 16;
3523		memset(sc->sc_hub_desc.temp, 0, 16);
3524		break;
3525
3526	case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
3527		DPRINTFN(9, "UR_GET_STATUS i=%d\n", index);
3528
3529		if ((index < 1) ||
3530		    (index > sc->sc_noport)) {
3531			err = USB_ERR_IOERROR;
3532			goto done;
3533		}
3534
3535		v = XREAD4(sc, oper, XHCI_PORTSC(index));
3536
3537		DPRINTFN(9, "port status=0x%08x\n", v);
3538
3539		i = UPS_PORT_LINK_STATE_SET(XHCI_PS_PLS_GET(v));
3540
3541		switch (XHCI_PS_SPEED_GET(v)) {
3542		case 3:
3543			i |= UPS_HIGH_SPEED;
3544			break;
3545		case 2:
3546			i |= UPS_LOW_SPEED;
3547			break;
3548		case 1:
3549			/* FULL speed */
3550			break;
3551		default:
3552			i |= UPS_OTHER_SPEED;
3553			break;
3554		}
3555
3556		if (v & XHCI_PS_CCS)
3557			i |= UPS_CURRENT_CONNECT_STATUS;
3558		if (v & XHCI_PS_PED)
3559			i |= UPS_PORT_ENABLED;
3560		if (v & XHCI_PS_OCA)
3561			i |= UPS_OVERCURRENT_INDICATOR;
3562		if (v & XHCI_PS_PR)
3563			i |= UPS_RESET;
3564#if 0
3565		if (v & XHCI_PS_PP)
3566			/* XXX undefined */
3567#endif
3568		USETW(sc->sc_hub_desc.ps.wPortStatus, i);
3569
3570		i = 0;
3571		if (v & XHCI_PS_CSC)
3572			i |= UPS_C_CONNECT_STATUS;
3573		if (v & XHCI_PS_PEC)
3574			i |= UPS_C_PORT_ENABLED;
3575		if (v & XHCI_PS_OCC)
3576			i |= UPS_C_OVERCURRENT_INDICATOR;
3577		if (v & XHCI_PS_WRC)
3578			i |= UPS_C_BH_PORT_RESET;
3579		if (v & XHCI_PS_PRC)
3580			i |= UPS_C_PORT_RESET;
3581		if (v & XHCI_PS_PLC)
3582			i |= UPS_C_PORT_LINK_STATE;
3583		if (v & XHCI_PS_CEC)
3584			i |= UPS_C_PORT_CONFIG_ERROR;
3585
3586		USETW(sc->sc_hub_desc.ps.wPortChange, i);
3587		len = sizeof(sc->sc_hub_desc.ps);
3588		break;
3589
3590	case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
3591		err = USB_ERR_IOERROR;
3592		goto done;
3593
3594	case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
3595		break;
3596
3597	case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
3598
3599		i = index >> 8;
3600		index &= 0x00FF;
3601
3602		if ((index < 1) ||
3603		    (index > sc->sc_noport)) {
3604			err = USB_ERR_IOERROR;
3605			goto done;
3606		}
3607
3608		port = XHCI_PORTSC(index);
3609		v = XREAD4(sc, oper, port) & ~XHCI_PS_CLEAR;
3610
3611		switch (value) {
3612		case UHF_PORT_U1_TIMEOUT:
3613			if (XHCI_PS_SPEED_GET(v) != 4) {
3614				err = USB_ERR_IOERROR;
3615				goto done;
3616			}
3617			port = XHCI_PORTPMSC(index);
3618			v = XREAD4(sc, oper, port);
3619			v &= ~XHCI_PM3_U1TO_SET(0xFF);
3620			v |= XHCI_PM3_U1TO_SET(i);
3621			XWRITE4(sc, oper, port, v);
3622			break;
3623		case UHF_PORT_U2_TIMEOUT:
3624			if (XHCI_PS_SPEED_GET(v) != 4) {
3625				err = USB_ERR_IOERROR;
3626				goto done;
3627			}
3628			port = XHCI_PORTPMSC(index);
3629			v = XREAD4(sc, oper, port);
3630			v &= ~XHCI_PM3_U2TO_SET(0xFF);
3631			v |= XHCI_PM3_U2TO_SET(i);
3632			XWRITE4(sc, oper, port, v);
3633			break;
3634		case UHF_BH_PORT_RESET:
3635			XWRITE4(sc, oper, port, v | XHCI_PS_WPR);
3636			break;
3637		case UHF_PORT_LINK_STATE:
3638			XWRITE4(sc, oper, port, v |
3639			    XHCI_PS_PLS_SET(i) | XHCI_PS_LWS);
3640			/* 4ms settle time */
3641			usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
3642			break;
3643		case UHF_PORT_ENABLE:
3644			DPRINTFN(3, "set port enable %d\n", index);
3645			break;
3646		case UHF_PORT_SUSPEND:
3647			DPRINTFN(6, "suspend port %u (LPM=%u)\n", index, i);
3648			j = XHCI_PS_SPEED_GET(v);
3649			if ((j < 1) || (j > 3)) {
3650				/* non-supported speed */
3651				err = USB_ERR_IOERROR;
3652				goto done;
3653			}
3654			XWRITE4(sc, oper, port, v |
3655			    XHCI_PS_PLS_SET(i ? 2 /* LPM */ : 3) | XHCI_PS_LWS);
3656			break;
3657		case UHF_PORT_RESET:
3658			DPRINTFN(6, "reset port %d\n", index);
3659			XWRITE4(sc, oper, port, v | XHCI_PS_PR);
3660			break;
3661		case UHF_PORT_POWER:
3662			DPRINTFN(3, "set port power %d\n", index);
3663			XWRITE4(sc, oper, port, v | XHCI_PS_PP);
3664			break;
3665		case UHF_PORT_TEST:
3666			DPRINTFN(3, "set port test %d\n", index);
3667			break;
3668		case UHF_PORT_INDICATOR:
3669			DPRINTFN(3, "set port indicator %d\n", index);
3670
3671			v &= ~XHCI_PS_PIC_SET(3);
3672			v |= XHCI_PS_PIC_SET(1);
3673
3674			XWRITE4(sc, oper, port, v);
3675			break;
3676		default:
3677			err = USB_ERR_IOERROR;
3678			goto done;
3679		}
3680		break;
3681
3682	case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
3683	case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
3684	case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
3685	case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
3686		break;
3687	default:
3688		err = USB_ERR_IOERROR;
3689		goto done;
3690	}
3691done:
3692	*plength = len;
3693	*pptr = ptr;
3694	return (err);
3695}
3696
3697static void
3698xhci_xfer_setup(struct usb_setup_params *parm)
3699{
3700	struct usb_page_search page_info;
3701	struct usb_page_cache *pc;
3702	struct usb_xfer *xfer;
3703	void *last_obj;
3704	uint32_t ntd;
3705	uint32_t n;
3706
3707	xfer = parm->curr_xfer;
3708
3709	/*
3710	 * The proof for the "ntd" formula is illustrated like this:
3711	 *
3712	 * +------------------------------------+
3713	 * |                                    |
3714	 * |         |remainder ->              |
3715	 * |   +-----+---+                      |
3716	 * |   | xxx | x | frm 0                |
3717	 * |   +-----+---++                     |
3718	 * |   | xxx | xx | frm 1               |
3719	 * |   +-----+----+                     |
3720	 * |            ...                     |
3721	 * +------------------------------------+
3722	 *
3723	 * "xxx" means a completely full USB transfer descriptor
3724	 *
3725	 * "x" and "xx" means a short USB packet
3726	 *
3727	 * For the remainder of an USB transfer modulo
3728	 * "max_data_length" we need two USB transfer descriptors.
3729	 * One to transfer the remaining data and one to finalise with
3730	 * a zero length packet in case the "force_short_xfer" flag is
3731	 * set. We only need two USB transfer descriptors in the case
3732	 * where the transfer length of the first one is a factor of
3733	 * "max_frame_size". The rest of the needed USB transfer
3734	 * descriptors is given by the buffer size divided by the
3735	 * maximum data payload.
3736	 */
3737	parm->hc_max_packet_size = 0x400;
3738	parm->hc_max_packet_count = 16 * 3;
3739	parm->hc_max_frame_size = XHCI_TD_PAYLOAD_MAX;
3740
3741	xfer->flags_int.bdma_enable = 1;
3742
3743	usbd_transfer_setup_sub(parm);
3744
3745	if (xfer->flags_int.isochronous_xfr) {
3746		ntd = ((1 * xfer->nframes)
3747		    + (xfer->max_data_length / xfer->max_hc_frame_size));
3748	} else if (xfer->flags_int.control_xfr) {
3749		ntd = ((2 * xfer->nframes) + 1	/* STATUS */
3750		    + (xfer->max_data_length / xfer->max_hc_frame_size));
3751	} else {
3752		ntd = ((2 * xfer->nframes)
3753		    + (xfer->max_data_length / xfer->max_hc_frame_size));
3754	}
3755
3756alloc_dma_set:
3757
3758	if (parm->err)
3759		return;
3760
3761	/*
3762	 * Allocate queue heads and transfer descriptors
3763	 */
3764	last_obj = NULL;
3765
3766	if (usbd_transfer_setup_sub_malloc(
3767	    parm, &pc, sizeof(struct xhci_td),
3768	    XHCI_TD_ALIGN, ntd)) {
3769		parm->err = USB_ERR_NOMEM;
3770		return;
3771	}
3772	if (parm->buf) {
3773		for (n = 0; n != ntd; n++) {
3774			struct xhci_td *td;
3775
3776			usbd_get_page(pc + n, 0, &page_info);
3777
3778			td = page_info.buffer;
3779
3780			/* init TD */
3781			td->td_self = page_info.physaddr;
3782			td->obj_next = last_obj;
3783			td->page_cache = pc + n;
3784
3785			last_obj = td;
3786
3787			usb_pc_cpu_flush(pc + n);
3788		}
3789	}
3790	xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj;
3791
3792	if (!xfer->flags_int.curr_dma_set) {
3793		xfer->flags_int.curr_dma_set = 1;
3794		goto alloc_dma_set;
3795	}
3796}
3797
3798static uint8_t
3799xhci_get_endpoint_state(struct usb_device *udev, uint8_t epno)
3800{
3801	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
3802	struct usb_page_search buf_dev;
3803	struct xhci_hw_dev *hdev;
3804	struct xhci_dev_ctx *pdev;
3805	uint32_t temp;
3806
3807	MPASS(epno != 0);
3808
3809	hdev =	&sc->sc_hw.devs[udev->controller_slot_id];
3810
3811	usbd_get_page(&hdev->device_pc, 0, &buf_dev);
3812	pdev = buf_dev.buffer;
3813	usb_pc_cpu_invalidate(&hdev->device_pc);
3814
3815	temp = xhci_ctx_get_le32(sc, &pdev->ctx_ep[epno - 1].dwEpCtx0);
3816
3817	return (XHCI_EPCTX_0_EPSTATE_GET(temp));
3818}
3819
3820static usb_error_t
3821xhci_configure_reset_endpoint(struct usb_xfer *xfer)
3822{
3823	struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus);
3824	struct usb_page_search buf_inp;
3825	struct usb_device *udev;
3826	struct xhci_endpoint_ext *pepext;
3827	struct usb_endpoint_descriptor *edesc;
3828	struct usb_page_cache *pcinp;
3829	usb_error_t err;
3830	usb_stream_t stream_id;
3831	uint32_t mask;
3832	uint8_t index;
3833	uint8_t epno;
3834
3835	pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
3836	    xfer->endpoint->edesc);
3837
3838	udev = xfer->xroot->udev;
3839	index = udev->controller_slot_id;
3840
3841	pcinp = &sc->sc_hw.devs[index].input_pc;
3842
3843	usbd_get_page(pcinp, 0, &buf_inp);
3844
3845	edesc = xfer->endpoint->edesc;
3846
3847	epno = edesc->bEndpointAddress;
3848	stream_id = xfer->stream_id;
3849
3850	if ((edesc->bmAttributes & UE_XFERTYPE) == UE_CONTROL)
3851		epno |= UE_DIR_IN;
3852
3853	epno = XHCI_EPNO2EPID(epno);
3854
3855 	if (epno == 0)
3856		return (USB_ERR_NO_PIPE);		/* invalid */
3857
3858	XHCI_CMD_LOCK(sc);
3859
3860	/* configure endpoint */
3861
3862	err = xhci_configure_endpoint_by_xfer(xfer);
3863
3864	if (err != 0) {
3865		XHCI_CMD_UNLOCK(sc);
3866		return (err);
3867	}
3868
3869	/*
3870	 * Get the endpoint into the stopped state according to the
3871	 * endpoint context state diagram in the XHCI specification:
3872	 */
3873	switch (xhci_get_endpoint_state(udev, epno)) {
3874	case XHCI_EPCTX_0_EPSTATE_STOPPED:
3875		break;
3876	case XHCI_EPCTX_0_EPSTATE_HALTED:
3877		err = xhci_cmd_reset_ep(sc, 0, epno, index);
3878		if (err != 0)
3879			DPRINTF("Could not reset endpoint %u\n", epno);
3880		break;
3881	default:
3882		err = xhci_cmd_stop_ep(sc, 0, epno, index);
3883		if (err != 0)
3884			DPRINTF("Could not stop endpoint %u\n", epno);
3885		break;
3886	}
3887
3888	err = xhci_cmd_set_tr_dequeue_ptr(sc,
3889	    (pepext->physaddr + (stream_id * sizeof(struct xhci_trb) *
3890	    XHCI_MAX_TRANSFERS)) | XHCI_EPCTX_2_DCS_SET(1),
3891	    stream_id, epno, index);
3892
3893	if (err != 0)
3894		DPRINTF("Could not set dequeue ptr for endpoint %u\n", epno);
3895
3896	/*
3897	 * Get the endpoint into the running state according to the
3898	 * endpoint context state diagram in the XHCI specification:
3899	 */
3900
3901	mask = (1U << epno);
3902	xhci_configure_mask(udev, mask | 1U, 0);
3903
3904	if (!(sc->sc_hw.devs[index].ep_configured & mask)) {
3905		sc->sc_hw.devs[index].ep_configured |= mask;
3906		err = xhci_cmd_configure_ep(sc, buf_inp.physaddr, 0, index);
3907	} else {
3908		err = xhci_cmd_evaluate_ctx(sc, buf_inp.physaddr, index);
3909	}
3910
3911	if (err != 0) {
3912		DPRINTF("Could not configure "
3913		    "endpoint %u at slot %u.\n", epno, index);
3914	}
3915	XHCI_CMD_UNLOCK(sc);
3916
3917	return (0);
3918}
3919
3920static void
3921xhci_xfer_unsetup(struct usb_xfer *xfer)
3922{
3923	return;
3924}
3925
3926static void
3927xhci_start_dma_delay(struct usb_xfer *xfer)
3928{
3929	struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus);
3930
3931	/* put transfer on interrupt queue (again) */
3932	usbd_transfer_enqueue(&sc->sc_bus.intr_q, xfer);
3933
3934	(void)usb_proc_msignal(USB_BUS_CONTROL_XFER_PROC(&sc->sc_bus),
3935	    &sc->sc_config_msg[0], &sc->sc_config_msg[1]);
3936}
3937
3938static void
3939xhci_configure_msg(struct usb_proc_msg *pm)
3940{
3941	struct xhci_softc *sc;
3942	struct xhci_endpoint_ext *pepext;
3943	struct usb_xfer *xfer;
3944
3945	sc = XHCI_BUS2SC(((struct usb_bus_msg *)pm)->bus);
3946
3947restart:
3948	TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3949		pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
3950		    xfer->endpoint->edesc);
3951
3952		if ((pepext->trb_halted != 0) ||
3953		    (pepext->trb_running == 0)) {
3954			uint16_t i;
3955
3956			/* clear halted and running */
3957			pepext->trb_halted = 0;
3958			pepext->trb_running = 0;
3959
3960			/* nuke remaining buffered transfers */
3961
3962			for (i = 0; i != (XHCI_MAX_TRANSFERS *
3963			    XHCI_MAX_STREAMS); i++) {
3964				/*
3965				 * NOTE: We need to use the timeout
3966				 * error code here else existing
3967				 * isochronous clients can get
3968				 * confused:
3969				 */
3970				if (pepext->xfer[i] != NULL) {
3971					xhci_device_done(pepext->xfer[i],
3972					    USB_ERR_TIMEOUT);
3973				}
3974			}
3975
3976			/*
3977			 * NOTE: The USB transfer cannot vanish in
3978			 * this state!
3979			 */
3980
3981			USB_BUS_UNLOCK(&sc->sc_bus);
3982
3983			xhci_configure_reset_endpoint(xfer);
3984
3985			USB_BUS_LOCK(&sc->sc_bus);
3986
3987			/* check if halted is still cleared */
3988			if (pepext->trb_halted == 0) {
3989				pepext->trb_running = 1;
3990				memset(pepext->trb_index, 0,
3991				    sizeof(pepext->trb_index));
3992			}
3993			goto restart;
3994		}
3995
3996		if (xfer->flags_int.did_dma_delay) {
3997			/* remove transfer from interrupt queue (again) */
3998			usbd_transfer_dequeue(xfer);
3999
4000			/* we are finally done */
4001			usb_dma_delay_done_cb(xfer);
4002
4003			/* queue changed - restart */
4004			goto restart;
4005		}
4006	}
4007
4008	TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
4009		/* try to insert xfer on HW queue */
4010		xhci_transfer_insert(xfer);
4011
4012		/* try to multi buffer */
4013		xhci_device_generic_multi_enter(xfer->endpoint,
4014		    xfer->stream_id, NULL);
4015	}
4016}
4017
4018static void
4019xhci_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
4020    struct usb_endpoint *ep)
4021{
4022	struct xhci_endpoint_ext *pepext;
4023
4024	DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d\n",
4025	    ep, udev->address, edesc->bEndpointAddress, udev->flags.usb_mode);
4026
4027	if (udev->parent_hub == NULL) {
4028		/* root HUB has special endpoint handling */
4029		return;
4030	}
4031
4032	ep->methods = &xhci_device_generic_methods;
4033
4034	pepext = xhci_get_endpoint_ext(udev, edesc);
4035
4036	USB_BUS_LOCK(udev->bus);
4037	pepext->trb_halted = 1;
4038	pepext->trb_running = 0;
4039	USB_BUS_UNLOCK(udev->bus);
4040}
4041
4042static void
4043xhci_ep_uninit(struct usb_device *udev, struct usb_endpoint *ep)
4044{
4045
4046}
4047
4048static void
4049xhci_ep_clear_stall(struct usb_device *udev, struct usb_endpoint *ep)
4050{
4051	struct xhci_endpoint_ext *pepext;
4052
4053	DPRINTF("\n");
4054
4055	if (udev->flags.usb_mode != USB_MODE_HOST) {
4056		/* not supported */
4057		return;
4058	}
4059	if (udev->parent_hub == NULL) {
4060		/* root HUB has special endpoint handling */
4061		return;
4062	}
4063
4064	pepext = xhci_get_endpoint_ext(udev, ep->edesc);
4065
4066	USB_BUS_LOCK(udev->bus);
4067	pepext->trb_halted = 1;
4068	pepext->trb_running = 0;
4069	USB_BUS_UNLOCK(udev->bus);
4070}
4071
4072static usb_error_t
4073xhci_device_init(struct usb_device *udev)
4074{
4075	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
4076	usb_error_t err;
4077	uint8_t temp;
4078
4079	/* no init for root HUB */
4080	if (udev->parent_hub == NULL)
4081		return (0);
4082
4083	XHCI_CMD_LOCK(sc);
4084
4085	/* set invalid default */
4086
4087	udev->controller_slot_id = sc->sc_noslot + 1;
4088
4089	/* try to get a new slot ID from the XHCI */
4090
4091	err = xhci_cmd_enable_slot(sc, &temp);
4092
4093	if (err) {
4094		XHCI_CMD_UNLOCK(sc);
4095		return (err);
4096	}
4097
4098	if (temp > sc->sc_noslot) {
4099		XHCI_CMD_UNLOCK(sc);
4100		return (USB_ERR_BAD_ADDRESS);
4101	}
4102
4103	if (sc->sc_hw.devs[temp].state != XHCI_ST_DISABLED) {
4104		DPRINTF("slot %u already allocated.\n", temp);
4105		XHCI_CMD_UNLOCK(sc);
4106		return (USB_ERR_BAD_ADDRESS);
4107	}
4108
4109	/* store slot ID for later reference */
4110
4111	udev->controller_slot_id = temp;
4112
4113	/* reset data structure */
4114
4115	memset(&sc->sc_hw.devs[temp], 0, sizeof(sc->sc_hw.devs[0]));
4116
4117	/* set mark slot allocated */
4118
4119	sc->sc_hw.devs[temp].state = XHCI_ST_ENABLED;
4120
4121	err = xhci_alloc_device_ext(udev);
4122
4123	XHCI_CMD_UNLOCK(sc);
4124
4125	/* get device into default state */
4126
4127	if (err == 0)
4128		err = xhci_set_address(udev, NULL, 0);
4129
4130	return (err);
4131}
4132
4133static void
4134xhci_device_uninit(struct usb_device *udev)
4135{
4136	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
4137	uint8_t index;
4138
4139	/* no init for root HUB */
4140	if (udev->parent_hub == NULL)
4141		return;
4142
4143	XHCI_CMD_LOCK(sc);
4144
4145	index = udev->controller_slot_id;
4146
4147	if (index <= sc->sc_noslot) {
4148		xhci_cmd_disable_slot(sc, index);
4149		sc->sc_hw.devs[index].state = XHCI_ST_DISABLED;
4150
4151		/* free device extension */
4152		xhci_free_device_ext(udev);
4153	}
4154
4155	XHCI_CMD_UNLOCK(sc);
4156}
4157
4158static void
4159xhci_get_dma_delay(struct usb_device *udev, uint32_t *pus)
4160{
4161	/*
4162	 * Wait until the hardware has finished any possible use of
4163	 * the transfer descriptor(s)
4164	 */
4165	*pus = 2048;			/* microseconds */
4166}
4167
4168static void
4169xhci_device_resume(struct usb_device *udev)
4170{
4171	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
4172	uint8_t index;
4173	uint8_t n;
4174	uint8_t p;
4175
4176	DPRINTF("\n");
4177
4178	/* check for root HUB */
4179	if (udev->parent_hub == NULL)
4180		return;
4181
4182	index = udev->controller_slot_id;
4183
4184	XHCI_CMD_LOCK(sc);
4185
4186	/* blindly resume all endpoints */
4187
4188	USB_BUS_LOCK(udev->bus);
4189
4190	for (n = 1; n != XHCI_MAX_ENDPOINTS; n++) {
4191		for (p = 0; p != XHCI_MAX_STREAMS; p++) {
4192			XWRITE4(sc, door, XHCI_DOORBELL(index),
4193			    n | XHCI_DB_SID_SET(p));
4194		}
4195	}
4196
4197	USB_BUS_UNLOCK(udev->bus);
4198
4199	XHCI_CMD_UNLOCK(sc);
4200}
4201
4202static void
4203xhci_device_suspend(struct usb_device *udev)
4204{
4205	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
4206	uint8_t index;
4207	uint8_t n;
4208	usb_error_t err;
4209
4210	DPRINTF("\n");
4211
4212	/* check for root HUB */
4213	if (udev->parent_hub == NULL)
4214		return;
4215
4216	index = udev->controller_slot_id;
4217
4218	XHCI_CMD_LOCK(sc);
4219
4220	/* blindly suspend all endpoints */
4221
4222	for (n = 1; n != XHCI_MAX_ENDPOINTS; n++) {
4223		err = xhci_cmd_stop_ep(sc, 1, n, index);
4224		if (err != 0) {
4225			DPRINTF("Failed to suspend endpoint "
4226			    "%u on slot %u (ignored).\n", n, index);
4227		}
4228	}
4229
4230	XHCI_CMD_UNLOCK(sc);
4231}
4232
4233static void
4234xhci_set_hw_power(struct usb_bus *bus)
4235{
4236	DPRINTF("\n");
4237}
4238
4239static void
4240xhci_device_state_change(struct usb_device *udev)
4241{
4242	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
4243	struct usb_page_search buf_inp;
4244	usb_error_t err;
4245	uint8_t index;
4246
4247	/* check for root HUB */
4248	if (udev->parent_hub == NULL)
4249		return;
4250
4251	index = udev->controller_slot_id;
4252
4253	DPRINTF("\n");
4254
4255	if (usb_get_device_state(udev) == USB_STATE_CONFIGURED) {
4256		err = uhub_query_info(udev, &sc->sc_hw.devs[index].nports,
4257		    &sc->sc_hw.devs[index].tt);
4258		if (err != 0)
4259			sc->sc_hw.devs[index].nports = 0;
4260	}
4261
4262	XHCI_CMD_LOCK(sc);
4263
4264	switch (usb_get_device_state(udev)) {
4265	case USB_STATE_POWERED:
4266		if (sc->sc_hw.devs[index].state == XHCI_ST_DEFAULT)
4267			break;
4268
4269		/* set default state */
4270		sc->sc_hw.devs[index].state = XHCI_ST_DEFAULT;
4271		sc->sc_hw.devs[index].ep_configured = 3U;
4272
4273		/* reset number of contexts */
4274		sc->sc_hw.devs[index].context_num = 0;
4275
4276		err = xhci_cmd_reset_dev(sc, index);
4277
4278		if (err != 0) {
4279			DPRINTF("Device reset failed "
4280			    "for slot %u.\n", index);
4281		}
4282		break;
4283
4284	case USB_STATE_ADDRESSED:
4285		if (sc->sc_hw.devs[index].state == XHCI_ST_ADDRESSED)
4286			break;
4287
4288		sc->sc_hw.devs[index].state = XHCI_ST_ADDRESSED;
4289		sc->sc_hw.devs[index].ep_configured = 3U;
4290
4291		/* set configure mask to slot only */
4292		xhci_configure_mask(udev, 1, 0);
4293
4294		/* deconfigure all endpoints, except EP0 */
4295		err = xhci_cmd_configure_ep(sc, 0, 1, index);
4296
4297		if (err) {
4298			DPRINTF("Failed to deconfigure "
4299			    "slot %u.\n", index);
4300		}
4301		break;
4302
4303	case USB_STATE_CONFIGURED:
4304		if (sc->sc_hw.devs[index].state == XHCI_ST_CONFIGURED) {
4305			/* deconfigure all endpoints, except EP0 */
4306			err = xhci_cmd_configure_ep(sc, 0, 1, index);
4307
4308			if (err) {
4309				DPRINTF("Failed to deconfigure "
4310				    "slot %u.\n", index);
4311			}
4312		}
4313
4314		/* set configured state */
4315		sc->sc_hw.devs[index].state = XHCI_ST_CONFIGURED;
4316		sc->sc_hw.devs[index].ep_configured = 3U;
4317
4318		/* reset number of contexts */
4319		sc->sc_hw.devs[index].context_num = 0;
4320
4321		usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp);
4322
4323		xhci_configure_mask(udev, 3, 0);
4324
4325		err = xhci_configure_device(udev);
4326		if (err != 0) {
4327			DPRINTF("Could not configure device "
4328			    "at slot %u.\n", index);
4329		}
4330
4331		err = xhci_cmd_evaluate_ctx(sc, buf_inp.physaddr, index);
4332		if (err != 0) {
4333			DPRINTF("Could not evaluate device "
4334			    "context at slot %u.\n", index);
4335		}
4336		break;
4337
4338	default:
4339		break;
4340	}
4341	XHCI_CMD_UNLOCK(sc);
4342}
4343
4344static usb_error_t
4345xhci_set_endpoint_mode(struct usb_device *udev, struct usb_endpoint *ep,
4346    uint8_t ep_mode)
4347{
4348	switch (ep_mode) {
4349	case USB_EP_MODE_DEFAULT:
4350		return (0);
4351	case USB_EP_MODE_STREAMS:
4352		if (xhcistreams == 0 ||
4353		    (ep->edesc->bmAttributes & UE_XFERTYPE) != UE_BULK ||
4354		    udev->speed != USB_SPEED_SUPER)
4355			return (USB_ERR_INVAL);
4356		return (0);
4357	default:
4358		return (USB_ERR_INVAL);
4359	}
4360}
4361
4362static const struct usb_bus_methods xhci_bus_methods = {
4363	.endpoint_init = xhci_ep_init,
4364	.endpoint_uninit = xhci_ep_uninit,
4365	.xfer_setup = xhci_xfer_setup,
4366	.xfer_unsetup = xhci_xfer_unsetup,
4367	.get_dma_delay = xhci_get_dma_delay,
4368	.device_init = xhci_device_init,
4369	.device_uninit = xhci_device_uninit,
4370	.device_resume = xhci_device_resume,
4371	.device_suspend = xhci_device_suspend,
4372	.set_hw_power = xhci_set_hw_power,
4373	.roothub_exec = xhci_roothub_exec,
4374	.xfer_poll = xhci_do_poll,
4375	.start_dma_delay = xhci_start_dma_delay,
4376	.set_address = xhci_set_address,
4377	.clear_stall = xhci_ep_clear_stall,
4378	.device_state_change = xhci_device_state_change,
4379	.set_hw_power_sleep = xhci_set_hw_power_sleep,
4380	.set_endpoint_mode = xhci_set_endpoint_mode,
4381};
4382