1993e3fafSRobert Mustacchi /*
2993e3fafSRobert Mustacchi * This file and its contents are supplied under the terms of the
3993e3fafSRobert Mustacchi * Common Development and Distribution License ("CDDL"), version 1.0.
4993e3fafSRobert Mustacchi * You may only use this file in accordance with the terms of version
5993e3fafSRobert Mustacchi * 1.0 of the CDDL.
6993e3fafSRobert Mustacchi *
7993e3fafSRobert Mustacchi * A full copy of the text of the CDDL should have accompanied this
8993e3fafSRobert Mustacchi * source. A copy of the CDDL is also available via the Internet at
9993e3fafSRobert Mustacchi * http://www.illumos.org/license/CDDL.
10993e3fafSRobert Mustacchi */
11993e3fafSRobert Mustacchi
12993e3fafSRobert Mustacchi /*
13*2aba3acdSRobert Mustacchi * Copyright (c) 2018, Joyent, Inc.
14993e3fafSRobert Mustacchi */
15993e3fafSRobert Mustacchi
16993e3fafSRobert Mustacchi /*
17993e3fafSRobert Mustacchi * -----------------------------
18993e3fafSRobert Mustacchi * xHCI Ring Management Routines
19993e3fafSRobert Mustacchi * -----------------------------
20993e3fafSRobert Mustacchi *
21993e3fafSRobert Mustacchi * There are three major different types of rings for xHCI, these are:
22993e3fafSRobert Mustacchi *
23993e3fafSRobert Mustacchi * 1) Command Rings
24993e3fafSRobert Mustacchi * 2) Event Rings
25993e3fafSRobert Mustacchi * 3) Transfer Rings
26993e3fafSRobert Mustacchi *
27993e3fafSRobert Mustacchi * Command and Transfer rings function in similar ways while the event rings are
28993e3fafSRobert Mustacchi * different. The difference comes in who is the consumer and who is the
29993e3fafSRobert Mustacchi * producer. In the case of command and transfer rings, the driver is the
30993e3fafSRobert Mustacchi * producer. For the event ring the driver is the consumer.
31993e3fafSRobert Mustacchi *
32993e3fafSRobert Mustacchi * Each ring in xhci has a synthetic head and tail register. Each entry in a
33993e3fafSRobert Mustacchi * ring has a bit that's often referred to as the 'Cycle bit'. The cycle bit is
34993e3fafSRobert Mustacchi * toggled as a means of saying that a given entry needs to be consumed.
35993e3fafSRobert Mustacchi *
36993e3fafSRobert Mustacchi * When a ring is created, all of the data in it is initialized to zero and the
37993e3fafSRobert Mustacchi * producer and consumer agree that when the cycle bit is toggled, the ownership
38993e3fafSRobert Mustacchi * of the entry is transfered from the producer to the consumer. For example,
39993e3fafSRobert Mustacchi * the command ring defaults to saying that a cycle bit of one is what indicates
40993e3fafSRobert Mustacchi * the command is owned by the hardware. So as the driver (the producer) fills
41993e3fafSRobert Mustacchi * in entries, the driver toggles the cycle bit from 0->1 as part of writing out
42993e3fafSRobert Mustacchi * the TRB. When the command ring's doorbell is rung, the hardware (the
43993e3fafSRobert Mustacchi * consumer) begins processing commands. It will process them until one of two
44993e3fafSRobert Mustacchi * things happens:
45993e3fafSRobert Mustacchi *
46993e3fafSRobert Mustacchi * 1) The hardware encounters an entry with the old cycle bit (0 in this case)
47993e3fafSRobert Mustacchi *
48993e3fafSRobert Mustacchi * 2) The hardware hits the last entry in the ring which is a special kind of
49993e3fafSRobert Mustacchi * entry called a LINK TRB.
50993e3fafSRobert Mustacchi *
51993e3fafSRobert Mustacchi * A LINK TRB has two purposes:
52993e3fafSRobert Mustacchi *
53993e3fafSRobert Mustacchi * 1) Indicate where processing should be redirected. This can potentially be to
54993e3fafSRobert Mustacchi * another memory segment; however, this driver always programs LINK TRBs to
55993e3fafSRobert Mustacchi * point back to the start of the ring.
56993e3fafSRobert Mustacchi *
57993e3fafSRobert Mustacchi * 2) Indicate whether or not the cycle bit should be changed. We always
58993e3fafSRobert Mustacchi * indicate that the cycle bit should be toggled when a LINK TRB is processed.
59993e3fafSRobert Mustacchi *
60993e3fafSRobert Mustacchi * In this same example, whereas the driver (the producer) would be setting the
61993e3fafSRobert Mustacchi * cycle to 1 to indicate that an entry is to be processed, the driver would now
62993e3fafSRobert Mustacchi * set it to 0. Similarly, the hardware (the consumer) would be looking for a
63993e3fafSRobert Mustacchi * 0 to determine whether or not it should process the entry.
64993e3fafSRobert Mustacchi *
65993e3fafSRobert Mustacchi * Currently, when the driver allocates rings, it always allocates a single page
66993e3fafSRobert Mustacchi * for the ring. The entire page is dedicated to ring use, which is determined
67993e3fafSRobert Mustacchi * based on the devices PAGESIZE register. The last entry in a given page is
68993e3fafSRobert Mustacchi * always configured as a LINK TRB. As each entry in a ring is 16 bytes, this
69993e3fafSRobert Mustacchi * gives us an average of 255 usable descriptors on x86 and 511 on SPARC, as
70993e3fafSRobert Mustacchi * PAGESIZE is 4k and 8k respectively.
71993e3fafSRobert Mustacchi *
72993e3fafSRobert Mustacchi * The driver is always the producer for all rings except for the event ring,
73993e3fafSRobert Mustacchi * where it is the consumer.
74993e3fafSRobert Mustacchi *
75993e3fafSRobert Mustacchi * ----------------------
76993e3fafSRobert Mustacchi * Head and Tail Pointers
77993e3fafSRobert Mustacchi * ----------------------
78993e3fafSRobert Mustacchi *
79993e3fafSRobert Mustacchi * Now, while we have the cycle bits for the ring explained, we still need to
80993e3fafSRobert Mustacchi * keep track of what we consider the head and tail pointers, what the xHCI
81993e3fafSRobert Mustacchi * specification calls enqueue (head) and dequeue (tail) pointers. Now, in all
82993e3fafSRobert Mustacchi * the cases here, the actual tracking of the head pointer is basically done by
83993e3fafSRobert Mustacchi * the cycle bit; however, we maintain an actual offset in the xhci_ring_t
84993e3fafSRobert Mustacchi * structure. The tail is usually less synthetic; however, it's up for different
85993e3fafSRobert Mustacchi * folks to maintain it.
86993e3fafSRobert Mustacchi *
87993e3fafSRobert Mustacchi * We handle the command and transfer rings the same way. The head pointer
88993e3fafSRobert Mustacchi * indicates where we should insert the next TRB to transfer. The tail pointer
89993e3fafSRobert Mustacchi * indicates the last thing that hardware has told us it has processed. If the
90993e3fafSRobert Mustacchi * head and tail point to the same index, then we know the ring is empty.
91993e3fafSRobert Mustacchi *
92993e3fafSRobert Mustacchi * We increment the head pointer whenever we insert an entry. Note that we do
93993e3fafSRobert Mustacchi * not tell hardware about this in any way, it's just maintained by the cycle
94993e3fafSRobert Mustacchi * bit. Then, we keep track of what hardware has processed in our tail pointer,
95993e3fafSRobert Mustacchi * incrementing it only when we have an interrupt that indicates that it's been
96993e3fafSRobert Mustacchi * processed.
97993e3fafSRobert Mustacchi *
98993e3fafSRobert Mustacchi * One oddity here is that we only get notified of this via the event ring. So
99993e3fafSRobert Mustacchi * when the event ring encounters this information, it needs to go back and
100993e3fafSRobert Mustacchi * increment our command and transfer ring tails after processing events.
101993e3fafSRobert Mustacchi *
102993e3fafSRobert Mustacchi * For the event ring, we handle things differently. We still initialize
103993e3fafSRobert Mustacchi * everything to zero; however, we start processing things and looking at cycle
104993e3fafSRobert Mustacchi * bits only when we get an interrupt from hardware. With the event ring, we do
105993e3fafSRobert Mustacchi * *not* maintain a head pointer (it's still in the structure, but unused). We
106993e3fafSRobert Mustacchi * always start processing at the tail pointer and use the cycle bit to indicate
107993e3fafSRobert Mustacchi * what we should process. Once we're done incrementing things, we go and notify
108993e3fafSRobert Mustacchi * the hardware of how far we got with this process by updating the tail for the
109993e3fafSRobert Mustacchi * event ring via a memory mapped register.
110993e3fafSRobert Mustacchi */
111993e3fafSRobert Mustacchi
112993e3fafSRobert Mustacchi #include <sys/usb/hcd/xhci/xhci.h>
113993e3fafSRobert Mustacchi
114993e3fafSRobert Mustacchi void
xhci_ring_free(xhci_ring_t * xrp)115993e3fafSRobert Mustacchi xhci_ring_free(xhci_ring_t *xrp)
116993e3fafSRobert Mustacchi {
117993e3fafSRobert Mustacchi if (xrp->xr_trb != NULL) {
118993e3fafSRobert Mustacchi xhci_dma_free(&xrp->xr_dma);
119993e3fafSRobert Mustacchi xrp->xr_trb = NULL;
120993e3fafSRobert Mustacchi }
121993e3fafSRobert Mustacchi xrp->xr_ntrb = 0;
122993e3fafSRobert Mustacchi xrp->xr_head = 0;
123993e3fafSRobert Mustacchi xrp->xr_tail = 0;
124993e3fafSRobert Mustacchi xrp->xr_cycle = 0;
125993e3fafSRobert Mustacchi }
126993e3fafSRobert Mustacchi
127993e3fafSRobert Mustacchi /*
128993e3fafSRobert Mustacchi * Initialize a ring that hasn't been used and set up its link pointer back to
129993e3fafSRobert Mustacchi * it.
130993e3fafSRobert Mustacchi */
131993e3fafSRobert Mustacchi int
xhci_ring_reset(xhci_t * xhcip,xhci_ring_t * xrp)132993e3fafSRobert Mustacchi xhci_ring_reset(xhci_t *xhcip, xhci_ring_t *xrp)
133993e3fafSRobert Mustacchi {
134993e3fafSRobert Mustacchi xhci_trb_t *ltrb;
135993e3fafSRobert Mustacchi
136993e3fafSRobert Mustacchi ASSERT(xrp->xr_trb != NULL);
137993e3fafSRobert Mustacchi
138993e3fafSRobert Mustacchi bzero(xrp->xr_trb, sizeof (xhci_trb_t) * xrp->xr_ntrb);
139993e3fafSRobert Mustacchi xrp->xr_head = 0;
140993e3fafSRobert Mustacchi xrp->xr_tail = 0;
141993e3fafSRobert Mustacchi xrp->xr_cycle = 1;
142993e3fafSRobert Mustacchi
143993e3fafSRobert Mustacchi /*
144993e3fafSRobert Mustacchi * Set up the link TRB back to ourselves.
145993e3fafSRobert Mustacchi */
146993e3fafSRobert Mustacchi ltrb = &xrp->xr_trb[xrp->xr_ntrb - 1];
147993e3fafSRobert Mustacchi ltrb->trb_addr = LE_64(xhci_dma_pa(&xrp->xr_dma));
148993e3fafSRobert Mustacchi ltrb->trb_flags = LE_32(XHCI_TRB_TYPE_LINK | XHCI_TRB_LINKSEG);
149993e3fafSRobert Mustacchi
150993e3fafSRobert Mustacchi XHCI_DMA_SYNC(xrp->xr_dma, DDI_DMA_SYNC_FORDEV);
151993e3fafSRobert Mustacchi if (xhci_check_dma_handle(xhcip, &xrp->xr_dma) != DDI_FM_OK) {
152993e3fafSRobert Mustacchi ddi_fm_service_impact(xhcip->xhci_dip, DDI_SERVICE_LOST);
153993e3fafSRobert Mustacchi return (EIO);
154993e3fafSRobert Mustacchi }
155993e3fafSRobert Mustacchi
156993e3fafSRobert Mustacchi return (0);
157993e3fafSRobert Mustacchi }
158993e3fafSRobert Mustacchi
159993e3fafSRobert Mustacchi int
xhci_ring_alloc(xhci_t * xhcip,xhci_ring_t * xrp)160993e3fafSRobert Mustacchi xhci_ring_alloc(xhci_t *xhcip, xhci_ring_t *xrp)
161993e3fafSRobert Mustacchi {
162993e3fafSRobert Mustacchi ddi_dma_attr_t attr;
163993e3fafSRobert Mustacchi ddi_device_acc_attr_t acc;
164993e3fafSRobert Mustacchi
165993e3fafSRobert Mustacchi /*
166993e3fafSRobert Mustacchi * We use a transfer attribute for the rings as they require 64-byte
167993e3fafSRobert Mustacchi * boundaries.
168993e3fafSRobert Mustacchi */
169993e3fafSRobert Mustacchi xhci_dma_acc_attr(xhcip, &acc);
170993e3fafSRobert Mustacchi xhci_dma_transfer_attr(xhcip, &attr, XHCI_DEF_DMA_SGL);
171993e3fafSRobert Mustacchi bzero(xrp, sizeof (xhci_ring_t));
172993e3fafSRobert Mustacchi if (xhci_dma_alloc(xhcip, &xrp->xr_dma, &attr, &acc, B_FALSE,
173993e3fafSRobert Mustacchi xhcip->xhci_caps.xcap_pagesize, B_FALSE) == B_FALSE)
174993e3fafSRobert Mustacchi return (ENOMEM);
175993e3fafSRobert Mustacchi xrp->xr_trb = (xhci_trb_t *)xrp->xr_dma.xdb_va;
176993e3fafSRobert Mustacchi xrp->xr_ntrb = xhcip->xhci_caps.xcap_pagesize / sizeof (xhci_trb_t);
177993e3fafSRobert Mustacchi return (0);
178993e3fafSRobert Mustacchi }
179993e3fafSRobert Mustacchi
180993e3fafSRobert Mustacchi /*
181993e3fafSRobert Mustacchi * Note, caller should have already synced our DMA memory. This should not be
182993e3fafSRobert Mustacchi * used for the command ring, as its cycle is maintained by the cycling of the
183993e3fafSRobert Mustacchi * head. This function is only used for managing the event ring.
184993e3fafSRobert Mustacchi */
185993e3fafSRobert Mustacchi xhci_trb_t *
xhci_ring_event_advance(xhci_ring_t * xrp)186993e3fafSRobert Mustacchi xhci_ring_event_advance(xhci_ring_t *xrp)
187993e3fafSRobert Mustacchi {
188993e3fafSRobert Mustacchi xhci_trb_t *trb = &xrp->xr_trb[xrp->xr_tail];
189993e3fafSRobert Mustacchi VERIFY(xrp->xr_tail < xrp->xr_ntrb);
190993e3fafSRobert Mustacchi
191993e3fafSRobert Mustacchi if (xrp->xr_cycle != (LE_32(trb->trb_flags) & XHCI_TRB_CYCLE))
192993e3fafSRobert Mustacchi return (NULL);
193993e3fafSRobert Mustacchi
194993e3fafSRobert Mustacchi /*
195993e3fafSRobert Mustacchi * The event ring does not use a link TRB. It instead always uses
196993e3fafSRobert Mustacchi * information based on the table to wrap. That means that the last
197993e3fafSRobert Mustacchi * entry is in fact going to contain data, so we shouldn't wrap and
198993e3fafSRobert Mustacchi * toggle the cycle until after we've processed that, in other words the
199993e3fafSRobert Mustacchi * tail equals the total number of entries.
200993e3fafSRobert Mustacchi */
201993e3fafSRobert Mustacchi xrp->xr_tail++;
202993e3fafSRobert Mustacchi if (xrp->xr_tail == xrp->xr_ntrb) {
203993e3fafSRobert Mustacchi xrp->xr_cycle ^= 1;
204993e3fafSRobert Mustacchi xrp->xr_tail = 0;
205993e3fafSRobert Mustacchi }
206993e3fafSRobert Mustacchi
207993e3fafSRobert Mustacchi return (trb);
208993e3fafSRobert Mustacchi }
209993e3fafSRobert Mustacchi
210993e3fafSRobert Mustacchi /*
211993e3fafSRobert Mustacchi * When processing the command ring, we're going to get a single event for each
212993e3fafSRobert Mustacchi * entry in it. As we've submitted things in order, we need to make sure that
213993e3fafSRobert Mustacchi * this address matches the DMA address that we'd expect of the current tail.
214993e3fafSRobert Mustacchi */
215993e3fafSRobert Mustacchi boolean_t
xhci_ring_trb_tail_valid(xhci_ring_t * xrp,uint64_t dma)216993e3fafSRobert Mustacchi xhci_ring_trb_tail_valid(xhci_ring_t *xrp, uint64_t dma)
217993e3fafSRobert Mustacchi {
218993e3fafSRobert Mustacchi uint64_t tail;
219993e3fafSRobert Mustacchi
220993e3fafSRobert Mustacchi tail = xhci_dma_pa(&xrp->xr_dma) + xrp->xr_tail * sizeof (xhci_trb_t);
221993e3fafSRobert Mustacchi return (dma == tail);
222993e3fafSRobert Mustacchi }
223993e3fafSRobert Mustacchi
224993e3fafSRobert Mustacchi /*
225993e3fafSRobert Mustacchi * A variant on the above that checks for a given message within a range of
226993e3fafSRobert Mustacchi * entries and returns the offset to it from the tail.
227993e3fafSRobert Mustacchi */
228993e3fafSRobert Mustacchi int
xhci_ring_trb_valid_range(xhci_ring_t * xrp,uint64_t dma,uint_t range)229993e3fafSRobert Mustacchi xhci_ring_trb_valid_range(xhci_ring_t *xrp, uint64_t dma, uint_t range)
230993e3fafSRobert Mustacchi {
231993e3fafSRobert Mustacchi uint_t i;
232993e3fafSRobert Mustacchi uint_t tail = xrp->xr_tail;
233993e3fafSRobert Mustacchi uint64_t taddr;
234993e3fafSRobert Mustacchi
235993e3fafSRobert Mustacchi VERIFY(range < xrp->xr_ntrb);
236993e3fafSRobert Mustacchi for (i = 0; i < range; i++) {
237993e3fafSRobert Mustacchi taddr = xhci_dma_pa(&xrp->xr_dma) + tail * sizeof (xhci_trb_t);
238993e3fafSRobert Mustacchi if (taddr == dma)
239993e3fafSRobert Mustacchi return (i);
240993e3fafSRobert Mustacchi
241993e3fafSRobert Mustacchi tail++;
242993e3fafSRobert Mustacchi if (tail == xrp->xr_ntrb - 1)
243993e3fafSRobert Mustacchi tail = 0;
244993e3fafSRobert Mustacchi }
245993e3fafSRobert Mustacchi
246993e3fafSRobert Mustacchi return (-1);
247993e3fafSRobert Mustacchi }
248993e3fafSRobert Mustacchi
249993e3fafSRobert Mustacchi /*
250993e3fafSRobert Mustacchi * Determine whether or not we have enough space for this request in a given
251993e3fafSRobert Mustacchi * ring for the given request. Note, we have to be a bit careful here and ensure
252993e3fafSRobert Mustacchi * that we properly handle cases where we cross the link TRB and that we don't
253993e3fafSRobert Mustacchi * count it.
254993e3fafSRobert Mustacchi *
255993e3fafSRobert Mustacchi * To determine if we have enough space for a given number of trbs, we need to
256993e3fafSRobert Mustacchi * logically advance the head pointer and make sure that we don't cross the tail
257993e3fafSRobert Mustacchi * pointer. In other words, if after advancement, head == tail, we're in
258993e3fafSRobert Mustacchi * trouble and don't have enough space.
259993e3fafSRobert Mustacchi */
260993e3fafSRobert Mustacchi boolean_t
xhci_ring_trb_space(xhci_ring_t * xrp,uint_t ntrb)261993e3fafSRobert Mustacchi xhci_ring_trb_space(xhci_ring_t *xrp, uint_t ntrb)
262993e3fafSRobert Mustacchi {
263993e3fafSRobert Mustacchi uint_t i;
264993e3fafSRobert Mustacchi uint_t head = xrp->xr_head;
265993e3fafSRobert Mustacchi
266993e3fafSRobert Mustacchi VERIFY(ntrb > 0);
267993e3fafSRobert Mustacchi /* We use < to ignore the link TRB */
268993e3fafSRobert Mustacchi VERIFY(ntrb < xrp->xr_ntrb);
269993e3fafSRobert Mustacchi
270993e3fafSRobert Mustacchi for (i = 0; i < ntrb; i++) {
271993e3fafSRobert Mustacchi head++;
272993e3fafSRobert Mustacchi if (head == xrp->xr_ntrb - 1) {
273993e3fafSRobert Mustacchi head = 0;
274993e3fafSRobert Mustacchi }
275993e3fafSRobert Mustacchi
276993e3fafSRobert Mustacchi if (head == xrp->xr_tail)
277993e3fafSRobert Mustacchi return (B_FALSE);
278993e3fafSRobert Mustacchi }
279993e3fafSRobert Mustacchi
280993e3fafSRobert Mustacchi return (B_TRUE);
281993e3fafSRobert Mustacchi }
282993e3fafSRobert Mustacchi
283993e3fafSRobert Mustacchi /*
284993e3fafSRobert Mustacchi * Fill in a TRB in the ring at offset trboff. If cycle is currently set to
285993e3fafSRobert Mustacchi * B_TRUE, then we fill in the appropriate cycle bit to tell the system to
286993e3fafSRobert Mustacchi * advance, otherwise we leave the existing cycle bit untouched so the system
287993e3fafSRobert Mustacchi * doesn't accidentally advance until we have everything filled in.
288993e3fafSRobert Mustacchi */
289993e3fafSRobert Mustacchi void
xhci_ring_trb_fill(xhci_ring_t * xrp,uint_t trboff,xhci_trb_t * host_trb,uint64_t * trb_pap,boolean_t put_cycle)290993e3fafSRobert Mustacchi xhci_ring_trb_fill(xhci_ring_t *xrp, uint_t trboff, xhci_trb_t *host_trb,
291*2aba3acdSRobert Mustacchi uint64_t *trb_pap, boolean_t put_cycle)
292993e3fafSRobert Mustacchi {
293993e3fafSRobert Mustacchi uint_t i;
294993e3fafSRobert Mustacchi uint32_t flags;
295993e3fafSRobert Mustacchi uint_t ent = xrp->xr_head;
296993e3fafSRobert Mustacchi uint8_t cycle = xrp->xr_cycle;
297993e3fafSRobert Mustacchi xhci_trb_t *trb;
298993e3fafSRobert Mustacchi
299993e3fafSRobert Mustacchi for (i = 0; i < trboff; i++) {
300993e3fafSRobert Mustacchi ent++;
301993e3fafSRobert Mustacchi if (ent == xrp->xr_ntrb - 1) {
302993e3fafSRobert Mustacchi ent = 0;
303993e3fafSRobert Mustacchi cycle ^= 1;
304993e3fafSRobert Mustacchi }
305993e3fafSRobert Mustacchi }
306993e3fafSRobert Mustacchi
307993e3fafSRobert Mustacchi /*
308993e3fafSRobert Mustacchi * If we're being asked to not update the cycle for it to be valid to be
309993e3fafSRobert Mustacchi * produced, we need to xor this once again to get to the inappropriate
310993e3fafSRobert Mustacchi * value.
311993e3fafSRobert Mustacchi */
312993e3fafSRobert Mustacchi if (put_cycle == B_FALSE)
313993e3fafSRobert Mustacchi cycle ^= 1;
314993e3fafSRobert Mustacchi
315993e3fafSRobert Mustacchi trb = &xrp->xr_trb[ent];
316993e3fafSRobert Mustacchi
317993e3fafSRobert Mustacchi trb->trb_addr = host_trb->trb_addr;
318993e3fafSRobert Mustacchi trb->trb_status = host_trb->trb_status;
319993e3fafSRobert Mustacchi flags = host_trb->trb_flags;
320993e3fafSRobert Mustacchi if (cycle == 0) {
321993e3fafSRobert Mustacchi flags &= ~LE_32(XHCI_TRB_CYCLE);
322993e3fafSRobert Mustacchi } else {
323993e3fafSRobert Mustacchi flags |= LE_32(XHCI_TRB_CYCLE);
324993e3fafSRobert Mustacchi }
325993e3fafSRobert Mustacchi
326993e3fafSRobert Mustacchi trb->trb_flags = flags;
327*2aba3acdSRobert Mustacchi
328*2aba3acdSRobert Mustacchi if (trb_pap != NULL) {
329*2aba3acdSRobert Mustacchi uint64_t pa;
330*2aba3acdSRobert Mustacchi
331*2aba3acdSRobert Mustacchi /*
332*2aba3acdSRobert Mustacchi * This logic only works if we have a single cookie address.
333*2aba3acdSRobert Mustacchi * However, this is prettty tightly assumed for rings through
334*2aba3acdSRobert Mustacchi * the xhci driver at this time.
335*2aba3acdSRobert Mustacchi */
336*2aba3acdSRobert Mustacchi ASSERT3U(xrp->xr_dma.xdb_ncookies, ==, 1);
337*2aba3acdSRobert Mustacchi pa = xrp->xr_dma.xdb_cookies[0].dmac_laddress;
338*2aba3acdSRobert Mustacchi pa += ((uintptr_t)trb - (uintptr_t)&xrp->xr_trb[0]);
339*2aba3acdSRobert Mustacchi *trb_pap = pa;
340*2aba3acdSRobert Mustacchi }
341993e3fafSRobert Mustacchi }
342993e3fafSRobert Mustacchi
343993e3fafSRobert Mustacchi /*
344993e3fafSRobert Mustacchi * Update our metadata for the ring and verify the cycle bit is correctly set
345993e3fafSRobert Mustacchi * for the first trb. It is expected that it is incorrectly set.
346993e3fafSRobert Mustacchi */
347993e3fafSRobert Mustacchi void
xhci_ring_trb_produce(xhci_ring_t * xrp,uint_t ntrb)348993e3fafSRobert Mustacchi xhci_ring_trb_produce(xhci_ring_t *xrp, uint_t ntrb)
349993e3fafSRobert Mustacchi {
350993e3fafSRobert Mustacchi uint_t i, ohead;
351993e3fafSRobert Mustacchi xhci_trb_t *trb;
352993e3fafSRobert Mustacchi
353993e3fafSRobert Mustacchi VERIFY(ntrb > 0);
354993e3fafSRobert Mustacchi
355993e3fafSRobert Mustacchi ohead = xrp->xr_head;
356993e3fafSRobert Mustacchi
357993e3fafSRobert Mustacchi /*
358993e3fafSRobert Mustacchi * As part of updating the head, we need to make sure we correctly
359993e3fafSRobert Mustacchi * update the cycle bit of the link TRB. So we always do this first
360993e3fafSRobert Mustacchi * before we update the old head, to try and get a consistent view of
361993e3fafSRobert Mustacchi * the cycle bit.
362993e3fafSRobert Mustacchi */
363993e3fafSRobert Mustacchi for (i = 0; i < ntrb; i++) {
364993e3fafSRobert Mustacchi xrp->xr_head++;
365993e3fafSRobert Mustacchi /*
366993e3fafSRobert Mustacchi * If we're updating the link TRB, we also need to make sure
367993e3fafSRobert Mustacchi * that the Chain bit is set if we're in the middle of a TD
368993e3fafSRobert Mustacchi * comprised of multiple TRDs. Thankfully the algorithmn here is
369993e3fafSRobert Mustacchi * simple: set it to the value of the previous TRB.
370993e3fafSRobert Mustacchi */
371993e3fafSRobert Mustacchi if (xrp->xr_head == xrp->xr_ntrb - 1) {
372993e3fafSRobert Mustacchi trb = &xrp->xr_trb[xrp->xr_ntrb - 1];
373993e3fafSRobert Mustacchi if (xrp->xr_trb[xrp->xr_ntrb - 2].trb_flags &
374993e3fafSRobert Mustacchi XHCI_TRB_CHAIN) {
375993e3fafSRobert Mustacchi trb->trb_flags |= XHCI_TRB_CHAIN;
376993e3fafSRobert Mustacchi } else {
377993e3fafSRobert Mustacchi trb->trb_flags &= ~XHCI_TRB_CHAIN;
378993e3fafSRobert Mustacchi
379993e3fafSRobert Mustacchi }
380993e3fafSRobert Mustacchi trb->trb_flags ^= LE_32(XHCI_TRB_CYCLE);
381993e3fafSRobert Mustacchi xrp->xr_cycle ^= 1;
382993e3fafSRobert Mustacchi xrp->xr_head = 0;
383993e3fafSRobert Mustacchi }
384993e3fafSRobert Mustacchi }
385993e3fafSRobert Mustacchi
386993e3fafSRobert Mustacchi trb = &xrp->xr_trb[ohead];
387993e3fafSRobert Mustacchi trb->trb_flags ^= LE_32(XHCI_TRB_CYCLE);
388993e3fafSRobert Mustacchi }
389993e3fafSRobert Mustacchi
390993e3fafSRobert Mustacchi /*
391993e3fafSRobert Mustacchi * This is a convenience wrapper for the single TRB case to make callers less
392993e3fafSRobert Mustacchi * likely to mess up some of the required semantics.
393993e3fafSRobert Mustacchi */
394993e3fafSRobert Mustacchi void
xhci_ring_trb_put(xhci_ring_t * xrp,xhci_trb_t * trb)395993e3fafSRobert Mustacchi xhci_ring_trb_put(xhci_ring_t *xrp, xhci_trb_t *trb)
396993e3fafSRobert Mustacchi {
397*2aba3acdSRobert Mustacchi xhci_ring_trb_fill(xrp, 0U, trb, NULL, B_FALSE);
398993e3fafSRobert Mustacchi xhci_ring_trb_produce(xrp, 1U);
399993e3fafSRobert Mustacchi }
400993e3fafSRobert Mustacchi
401993e3fafSRobert Mustacchi /*
402993e3fafSRobert Mustacchi * Update the tail pointer for a ring based on the DMA address of a consumed
403993e3fafSRobert Mustacchi * entry. Note, this entry indicates what we just processed, therefore we should
404993e3fafSRobert Mustacchi * bump the tail entry to the next one.
405993e3fafSRobert Mustacchi */
406993e3fafSRobert Mustacchi boolean_t
xhci_ring_trb_consumed(xhci_ring_t * xrp,uint64_t dma)407993e3fafSRobert Mustacchi xhci_ring_trb_consumed(xhci_ring_t *xrp, uint64_t dma)
408993e3fafSRobert Mustacchi {
409993e3fafSRobert Mustacchi uint64_t pa = xhci_dma_pa(&xrp->xr_dma);
410993e3fafSRobert Mustacchi uint64_t high = pa + xrp->xr_ntrb * sizeof (xhci_trb_t);
411993e3fafSRobert Mustacchi
412993e3fafSRobert Mustacchi if (dma < pa || dma >= high ||
413993e3fafSRobert Mustacchi dma % sizeof (xhci_trb_t) != 0)
414993e3fafSRobert Mustacchi return (B_FALSE);
415993e3fafSRobert Mustacchi
416993e3fafSRobert Mustacchi dma -= pa;
417993e3fafSRobert Mustacchi dma /= sizeof (xhci_trb_t);
418993e3fafSRobert Mustacchi
419993e3fafSRobert Mustacchi VERIFY(dma < xrp->xr_ntrb);
420993e3fafSRobert Mustacchi
421993e3fafSRobert Mustacchi xrp->xr_tail = dma + 1;
422993e3fafSRobert Mustacchi if (xrp->xr_tail == xrp->xr_ntrb - 1)
423993e3fafSRobert Mustacchi xrp->xr_tail = 0;
424993e3fafSRobert Mustacchi
425993e3fafSRobert Mustacchi return (B_TRUE);
426993e3fafSRobert Mustacchi }
427993e3fafSRobert Mustacchi
428993e3fafSRobert Mustacchi /*
429993e3fafSRobert Mustacchi * The ring represented here has been reset and we're being asked to basically
430993e3fafSRobert Mustacchi * skip all outstanding entries. Note, this shouldn't be used for the event
431993e3fafSRobert Mustacchi * ring. Because the cycle bit is toggled whenever the head moves past the link
432993e3fafSRobert Mustacchi * trb, the cycle bit is already correct. So in this case, it's really just a
433993e3fafSRobert Mustacchi * matter of setting the current tail equal to the head, at which point we
434993e3fafSRobert Mustacchi * consider things empty.
435993e3fafSRobert Mustacchi */
436993e3fafSRobert Mustacchi void
xhci_ring_skip(xhci_ring_t * xrp)437993e3fafSRobert Mustacchi xhci_ring_skip(xhci_ring_t *xrp)
438993e3fafSRobert Mustacchi {
439993e3fafSRobert Mustacchi xrp->xr_tail = xrp->xr_head;
440993e3fafSRobert Mustacchi }
441993e3fafSRobert Mustacchi
442993e3fafSRobert Mustacchi /*
443993e3fafSRobert Mustacchi * A variant on the normal skip. This basically just tells us to make sure that
444993e3fafSRobert Mustacchi * that everything this transfer represents has been skipped. Callers need to
445993e3fafSRobert Mustacchi * make sure that this is actually the first transfer in the ring. Like above,
446993e3fafSRobert Mustacchi * we don't need to touch the cycle bit.
447993e3fafSRobert Mustacchi */
448993e3fafSRobert Mustacchi void
xhci_ring_skip_transfer(xhci_ring_t * xrp,xhci_transfer_t * xt)449993e3fafSRobert Mustacchi xhci_ring_skip_transfer(xhci_ring_t *xrp, xhci_transfer_t *xt)
450993e3fafSRobert Mustacchi {
451993e3fafSRobert Mustacchi uint_t i;
452993e3fafSRobert Mustacchi
453993e3fafSRobert Mustacchi for (i = 0; i < xt->xt_ntrbs; i++) {
454993e3fafSRobert Mustacchi xrp->xr_tail++;
455993e3fafSRobert Mustacchi if (xrp->xr_tail == xrp->xr_ntrb - 1)
456993e3fafSRobert Mustacchi xrp->xr_tail = 0;
457993e3fafSRobert Mustacchi }
458993e3fafSRobert Mustacchi }
459