1993e3fafSRobert Mustacchi /*
2993e3fafSRobert Mustacchi * This file and its contents are supplied under the terms of the
3993e3fafSRobert Mustacchi * Common Development and Distribution License ("CDDL"), version 1.0.
4993e3fafSRobert Mustacchi * You may only use this file in accordance with the terms of version
5993e3fafSRobert Mustacchi * 1.0 of the CDDL.
6993e3fafSRobert Mustacchi *
7993e3fafSRobert Mustacchi * A full copy of the text of the CDDL should have accompanied this
8993e3fafSRobert Mustacchi * source. A copy of the CDDL is also available via the Internet at
9993e3fafSRobert Mustacchi * http://www.illumos.org/license/CDDL.
10993e3fafSRobert Mustacchi */
11993e3fafSRobert Mustacchi
12993e3fafSRobert Mustacchi /*
13*15ee7bdfSRobert Mustacchi * Copyright 2019, Joyent, Inc.
14993e3fafSRobert Mustacchi */
15993e3fafSRobert Mustacchi
16993e3fafSRobert Mustacchi /*
17993e3fafSRobert Mustacchi * xHCI DMA Management Routines
18993e3fafSRobert Mustacchi *
19993e3fafSRobert Mustacchi * Please see the big theory statement in xhci.c for more information.
20993e3fafSRobert Mustacchi */
21993e3fafSRobert Mustacchi
22993e3fafSRobert Mustacchi #include <sys/usb/hcd/xhci/xhci.h>
23993e3fafSRobert Mustacchi
24993e3fafSRobert Mustacchi int
xhci_check_dma_handle(xhci_t * xhcip,xhci_dma_buffer_t * xdb)25993e3fafSRobert Mustacchi xhci_check_dma_handle(xhci_t *xhcip, xhci_dma_buffer_t *xdb)
26993e3fafSRobert Mustacchi {
27993e3fafSRobert Mustacchi ddi_fm_error_t de;
28993e3fafSRobert Mustacchi
29993e3fafSRobert Mustacchi if (!DDI_FM_DMA_ERR_CAP(xhcip->xhci_fm_caps))
30993e3fafSRobert Mustacchi return (0);
31993e3fafSRobert Mustacchi
32993e3fafSRobert Mustacchi ddi_fm_dma_err_get(xdb->xdb_dma_handle, &de, DDI_FME_VERSION);
33993e3fafSRobert Mustacchi return (de.fme_status);
34993e3fafSRobert Mustacchi }
35993e3fafSRobert Mustacchi
36993e3fafSRobert Mustacchi void
xhci_dma_acc_attr(xhci_t * xhcip,ddi_device_acc_attr_t * accp)37993e3fafSRobert Mustacchi xhci_dma_acc_attr(xhci_t *xhcip, ddi_device_acc_attr_t *accp)
38993e3fafSRobert Mustacchi {
39993e3fafSRobert Mustacchi accp->devacc_attr_version = DDI_DEVICE_ATTR_V0;
40993e3fafSRobert Mustacchi accp->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
41993e3fafSRobert Mustacchi accp->devacc_attr_dataorder = DDI_STRICTORDER_ACC;
42993e3fafSRobert Mustacchi
43993e3fafSRobert Mustacchi if (DDI_FM_DMA_ERR_CAP(xhcip->xhci_fm_caps)) {
44993e3fafSRobert Mustacchi accp->devacc_attr_access = DDI_FLAGERR_ACC;
45993e3fafSRobert Mustacchi } else {
46993e3fafSRobert Mustacchi accp->devacc_attr_access = DDI_DEFAULT_ACC;
47993e3fafSRobert Mustacchi }
48993e3fafSRobert Mustacchi }
49993e3fafSRobert Mustacchi
50993e3fafSRobert Mustacchi /*
51993e3fafSRobert Mustacchi * These are DMA attributes that we assign when making a transfer. The SGL is
52993e3fafSRobert Mustacchi * variable and based on the caller, which varies based on the type of transfer
53993e3fafSRobert Mustacchi * we're doing.
54993e3fafSRobert Mustacchi */
55993e3fafSRobert Mustacchi void
xhci_dma_transfer_attr(xhci_t * xhcip,ddi_dma_attr_t * attrp,uint_t sgl)56993e3fafSRobert Mustacchi xhci_dma_transfer_attr(xhci_t *xhcip, ddi_dma_attr_t *attrp, uint_t sgl)
57993e3fafSRobert Mustacchi {
58993e3fafSRobert Mustacchi VERIFY3U(sgl, >, 0);
59993e3fafSRobert Mustacchi VERIFY3U(sgl, <=, XHCI_TRANSFER_DMA_SGL);
60993e3fafSRobert Mustacchi attrp->dma_attr_version = DMA_ATTR_V0;
61993e3fafSRobert Mustacchi
62993e3fafSRobert Mustacchi /*
63993e3fafSRobert Mustacchi * The range of data that we can use is based on what hardware supports.
64993e3fafSRobert Mustacchi */
65993e3fafSRobert Mustacchi attrp->dma_attr_addr_lo = 0x0;
66993e3fafSRobert Mustacchi if (xhcip->xhci_caps.xcap_flags & XCAP_AC64) {
67993e3fafSRobert Mustacchi attrp->dma_attr_addr_hi = UINT64_MAX;
68993e3fafSRobert Mustacchi } else {
69993e3fafSRobert Mustacchi attrp->dma_attr_addr_hi = UINT32_MAX;
70993e3fafSRobert Mustacchi }
71993e3fafSRobert Mustacchi
72993e3fafSRobert Mustacchi /*
73993e3fafSRobert Mustacchi * The count max indicates the total amount that will fit into one
74993e3fafSRobert Mustacchi * cookie, which is one TRB in our world. In other words 64k.
75993e3fafSRobert Mustacchi */
76993e3fafSRobert Mustacchi attrp->dma_attr_count_max = XHCI_TRB_MAX_TRANSFER;
77993e3fafSRobert Mustacchi
78993e3fafSRobert Mustacchi /*
79993e3fafSRobert Mustacchi * The alignment and segment are related. The alignment describes the
80993e3fafSRobert Mustacchi * alignment of the PA. The segment describes a boundary that the DMA
81*15ee7bdfSRobert Mustacchi * allocation cannot cross. In other words, for a given chunk of
82*15ee7bdfSRobert Mustacchi * allocated DMA memory, the allocated buffer cannot cross a 64k aligned
83*15ee7bdfSRobert Mustacchi * boundary. However, the physical address only needs to be aligned to
84*15ee7bdfSRobert Mustacchi * 64 bytes.
85993e3fafSRobert Mustacchi */
86993e3fafSRobert Mustacchi attrp->dma_attr_align = XHCI_DMA_ALIGN;
87993e3fafSRobert Mustacchi attrp->dma_attr_seg = XHCI_TRB_MAX_TRANSFER - 1;
88993e3fafSRobert Mustacchi
89993e3fafSRobert Mustacchi
90993e3fafSRobert Mustacchi attrp->dma_attr_burstsizes = 0xfff;
91993e3fafSRobert Mustacchi
92993e3fafSRobert Mustacchi /*
93993e3fafSRobert Mustacchi * This is the maximum we can send. Technically this is limited by the
94993e3fafSRobert Mustacchi * descriptors and not by hardware, hence why we use a large value for
95993e3fafSRobert Mustacchi * the max that'll be less than any memory allocation we ever throw at
96993e3fafSRobert Mustacchi * it.
97993e3fafSRobert Mustacchi */
98993e3fafSRobert Mustacchi attrp->dma_attr_minxfer = 0x1;
99993e3fafSRobert Mustacchi attrp->dma_attr_maxxfer = UINT32_MAX;
100993e3fafSRobert Mustacchi
101993e3fafSRobert Mustacchi /*
102993e3fafSRobert Mustacchi * This is determined by the caller.
103993e3fafSRobert Mustacchi */
104993e3fafSRobert Mustacchi attrp->dma_attr_sgllen = sgl;
105993e3fafSRobert Mustacchi
106993e3fafSRobert Mustacchi /*
107993e3fafSRobert Mustacchi * The granularity describes the addressing granularity. e.g. can things
108993e3fafSRobert Mustacchi * ask for chunks in units of this number of bytes. For PCI this should
109993e3fafSRobert Mustacchi * always be one.
110993e3fafSRobert Mustacchi */
111993e3fafSRobert Mustacchi attrp->dma_attr_granular = 1;
112993e3fafSRobert Mustacchi
113993e3fafSRobert Mustacchi if (DDI_FM_DMA_ERR_CAP(xhcip->xhci_fm_caps)) {
114993e3fafSRobert Mustacchi attrp->dma_attr_flags = DDI_DMA_FLAGERR;
115993e3fafSRobert Mustacchi } else {
116993e3fafSRobert Mustacchi attrp->dma_attr_flags = 0;
117993e3fafSRobert Mustacchi }
118993e3fafSRobert Mustacchi }
119993e3fafSRobert Mustacchi
120993e3fafSRobert Mustacchi /*
121993e3fafSRobert Mustacchi * This routine tries to create DMA attributes for normal allocations for data
122993e3fafSRobert Mustacchi * structures and the like. By default we use the same values as the transfer
123993e3fafSRobert Mustacchi * attributes, but have explicit comments about how they're different.
124993e3fafSRobert Mustacchi */
125993e3fafSRobert Mustacchi void
xhci_dma_dma_attr(xhci_t * xhcip,ddi_dma_attr_t * attrp)126993e3fafSRobert Mustacchi xhci_dma_dma_attr(xhci_t *xhcip, ddi_dma_attr_t *attrp)
127993e3fafSRobert Mustacchi {
128993e3fafSRobert Mustacchi /*
129993e3fafSRobert Mustacchi * Note, we always use a single SGL for these DMA allocations as these
130993e3fafSRobert Mustacchi * are used for small data structures.
131993e3fafSRobert Mustacchi */
132993e3fafSRobert Mustacchi xhci_dma_transfer_attr(xhcip, attrp, XHCI_DEF_DMA_SGL);
133993e3fafSRobert Mustacchi
134993e3fafSRobert Mustacchi /*
135993e3fafSRobert Mustacchi * The maximum size of any of these structures is 4k as opposed to the
136993e3fafSRobert Mustacchi * 64K max described above. Similarly the boundary requirement is
137993e3fafSRobert Mustacchi * reduced to 4k.
138993e3fafSRobert Mustacchi */
139993e3fafSRobert Mustacchi attrp->dma_attr_count_max = xhcip->xhci_caps.xcap_pagesize;
140993e3fafSRobert Mustacchi attrp->dma_attr_maxxfer = xhcip->xhci_caps.xcap_pagesize;
141993e3fafSRobert Mustacchi attrp->dma_attr_seg = xhcip->xhci_caps.xcap_pagesize - 1;
142993e3fafSRobert Mustacchi }
143993e3fafSRobert Mustacchi
144993e3fafSRobert Mustacchi /*
145993e3fafSRobert Mustacchi * Fill in attributes for a scratchpad entry. The scratchpad entries are
146993e3fafSRobert Mustacchi * somewhat different in so far as they are closest to a normal DMA attribute,
147993e3fafSRobert Mustacchi * except they have stricter alignments, needing to be page sized.
148993e3fafSRobert Mustacchi *
149993e3fafSRobert Mustacchi * In addition, because we never access this memory ourselves, we can just mark
150993e3fafSRobert Mustacchi * it all as relaxed ordering.
151993e3fafSRobert Mustacchi */
152993e3fafSRobert Mustacchi void
xhci_dma_scratchpad_attr(xhci_t * xhcip,ddi_dma_attr_t * attrp)153993e3fafSRobert Mustacchi xhci_dma_scratchpad_attr(xhci_t *xhcip, ddi_dma_attr_t *attrp)
154993e3fafSRobert Mustacchi {
155993e3fafSRobert Mustacchi xhci_dma_dma_attr(xhcip, attrp);
156993e3fafSRobert Mustacchi attrp->dma_attr_align = xhcip->xhci_caps.xcap_pagesize;
157993e3fafSRobert Mustacchi attrp->dma_attr_flags |= DDI_DMA_RELAXED_ORDERING;
158993e3fafSRobert Mustacchi }
159993e3fafSRobert Mustacchi
160993e3fafSRobert Mustacchi /*
161993e3fafSRobert Mustacchi * This should be used for the simple case of a single SGL entry, which is the
162993e3fafSRobert Mustacchi * vast majority of the non-transfer allocations.
163993e3fafSRobert Mustacchi */
164993e3fafSRobert Mustacchi uint64_t
xhci_dma_pa(xhci_dma_buffer_t * xdb)165993e3fafSRobert Mustacchi xhci_dma_pa(xhci_dma_buffer_t *xdb)
166993e3fafSRobert Mustacchi {
167993e3fafSRobert Mustacchi ASSERT(xdb->xdb_ncookies == 1);
168993e3fafSRobert Mustacchi return (xdb->xdb_cookies[0].dmac_laddress);
169993e3fafSRobert Mustacchi }
170993e3fafSRobert Mustacchi
171993e3fafSRobert Mustacchi void
xhci_dma_free(xhci_dma_buffer_t * xdb)172993e3fafSRobert Mustacchi xhci_dma_free(xhci_dma_buffer_t *xdb)
173993e3fafSRobert Mustacchi {
174993e3fafSRobert Mustacchi if (xdb->xdb_ncookies != 0) {
175993e3fafSRobert Mustacchi VERIFY(xdb->xdb_dma_handle != NULL);
176993e3fafSRobert Mustacchi (void) ddi_dma_unbind_handle(xdb->xdb_dma_handle);
177993e3fafSRobert Mustacchi xdb->xdb_ncookies = 0;
178993e3fafSRobert Mustacchi bzero(xdb->xdb_cookies, sizeof (ddi_dma_cookie_t) *
179993e3fafSRobert Mustacchi XHCI_TRANSFER_DMA_SGL);
180993e3fafSRobert Mustacchi xdb->xdb_len = 0;
181993e3fafSRobert Mustacchi }
182993e3fafSRobert Mustacchi
183993e3fafSRobert Mustacchi if (xdb->xdb_acc_handle != NULL) {
184993e3fafSRobert Mustacchi ddi_dma_mem_free(&xdb->xdb_acc_handle);
185993e3fafSRobert Mustacchi xdb->xdb_acc_handle = NULL;
186993e3fafSRobert Mustacchi xdb->xdb_va = NULL;
187993e3fafSRobert Mustacchi }
188993e3fafSRobert Mustacchi
189993e3fafSRobert Mustacchi if (xdb->xdb_dma_handle != NULL) {
190993e3fafSRobert Mustacchi ddi_dma_free_handle(&xdb->xdb_dma_handle);
191993e3fafSRobert Mustacchi xdb->xdb_dma_handle = NULL;
192993e3fafSRobert Mustacchi }
193993e3fafSRobert Mustacchi
194993e3fafSRobert Mustacchi ASSERT(xdb->xdb_va == NULL);
195993e3fafSRobert Mustacchi ASSERT(xdb->xdb_ncookies == 0);
196993e3fafSRobert Mustacchi ASSERT(xdb->xdb_cookies[0].dmac_laddress == 0);
197993e3fafSRobert Mustacchi ASSERT(xdb->xdb_len == 0);
198993e3fafSRobert Mustacchi }
199993e3fafSRobert Mustacchi
200993e3fafSRobert Mustacchi boolean_t
xhci_dma_alloc(xhci_t * xhcip,xhci_dma_buffer_t * xdb,ddi_dma_attr_t * attrp,ddi_device_acc_attr_t * accp,boolean_t zero,size_t size,boolean_t wait)201993e3fafSRobert Mustacchi xhci_dma_alloc(xhci_t *xhcip, xhci_dma_buffer_t *xdb,
202993e3fafSRobert Mustacchi ddi_dma_attr_t *attrp, ddi_device_acc_attr_t *accp, boolean_t zero,
203993e3fafSRobert Mustacchi size_t size, boolean_t wait)
204993e3fafSRobert Mustacchi {
205993e3fafSRobert Mustacchi int ret, i;
206993e3fafSRobert Mustacchi uint_t flags = DDI_DMA_CONSISTENT;
207993e3fafSRobert Mustacchi size_t len;
208993e3fafSRobert Mustacchi ddi_dma_cookie_t cookie;
209993e3fafSRobert Mustacchi uint_t ncookies;
210993e3fafSRobert Mustacchi int (*memcb)(caddr_t);
211993e3fafSRobert Mustacchi
212993e3fafSRobert Mustacchi if (wait == B_TRUE) {
213993e3fafSRobert Mustacchi memcb = DDI_DMA_SLEEP;
214993e3fafSRobert Mustacchi } else {
215993e3fafSRobert Mustacchi memcb = DDI_DMA_DONTWAIT;
216993e3fafSRobert Mustacchi }
217993e3fafSRobert Mustacchi
218993e3fafSRobert Mustacchi ret = ddi_dma_alloc_handle(xhcip->xhci_dip, attrp, memcb, NULL,
219993e3fafSRobert Mustacchi &xdb->xdb_dma_handle);
220993e3fafSRobert Mustacchi if (ret != 0) {
221993e3fafSRobert Mustacchi xhci_log(xhcip, "!failed to allocate DMA handle: %d", ret);
222993e3fafSRobert Mustacchi xdb->xdb_dma_handle = NULL;
223993e3fafSRobert Mustacchi return (B_FALSE);
224993e3fafSRobert Mustacchi }
225993e3fafSRobert Mustacchi
226993e3fafSRobert Mustacchi ret = ddi_dma_mem_alloc(xdb->xdb_dma_handle, size, accp, flags, memcb,
227993e3fafSRobert Mustacchi NULL, &xdb->xdb_va, &len, &xdb->xdb_acc_handle);
228993e3fafSRobert Mustacchi if (ret != DDI_SUCCESS) {
229993e3fafSRobert Mustacchi xhci_log(xhcip, "!failed to allocate DMA memory: %d", ret);
230993e3fafSRobert Mustacchi xdb->xdb_va = NULL;
231993e3fafSRobert Mustacchi xdb->xdb_acc_handle = NULL;
232993e3fafSRobert Mustacchi xhci_dma_free(xdb);
233993e3fafSRobert Mustacchi return (B_FALSE);
234993e3fafSRobert Mustacchi }
235993e3fafSRobert Mustacchi
236993e3fafSRobert Mustacchi if (zero == B_TRUE)
237993e3fafSRobert Mustacchi bzero(xdb->xdb_va, len);
238993e3fafSRobert Mustacchi
239993e3fafSRobert Mustacchi ret = ddi_dma_addr_bind_handle(xdb->xdb_dma_handle, NULL,
240993e3fafSRobert Mustacchi xdb->xdb_va, len, DDI_DMA_RDWR | flags, memcb, NULL, &cookie,
241993e3fafSRobert Mustacchi &ncookies);
242993e3fafSRobert Mustacchi if (ret != 0) {
243993e3fafSRobert Mustacchi xhci_log(xhcip, "!failed to bind DMA memory: %d", ret);
244993e3fafSRobert Mustacchi xhci_dma_free(xdb);
245993e3fafSRobert Mustacchi return (B_FALSE);
246993e3fafSRobert Mustacchi }
247993e3fafSRobert Mustacchi
248993e3fafSRobert Mustacchi /*
249993e3fafSRobert Mustacchi * Note we explicitly store the logical length of this allocation. The
250993e3fafSRobert Mustacchi * physical length is available via the cookies.
251993e3fafSRobert Mustacchi */
252993e3fafSRobert Mustacchi xdb->xdb_len = size;
253993e3fafSRobert Mustacchi xdb->xdb_ncookies = ncookies;
254993e3fafSRobert Mustacchi xdb->xdb_cookies[0] = cookie;
255993e3fafSRobert Mustacchi for (i = 1; i < ncookies; i++) {
256993e3fafSRobert Mustacchi ddi_dma_nextcookie(xdb->xdb_dma_handle, &xdb->xdb_cookies[i]);
257993e3fafSRobert Mustacchi }
258993e3fafSRobert Mustacchi
259993e3fafSRobert Mustacchi
260993e3fafSRobert Mustacchi return (B_TRUE);
261993e3fafSRobert Mustacchi }
262993e3fafSRobert Mustacchi
263993e3fafSRobert Mustacchi void
xhci_transfer_free(xhci_t * xhcip,xhci_transfer_t * xt)264993e3fafSRobert Mustacchi xhci_transfer_free(xhci_t *xhcip, xhci_transfer_t *xt)
265993e3fafSRobert Mustacchi {
266993e3fafSRobert Mustacchi if (xt == NULL)
267993e3fafSRobert Mustacchi return;
268993e3fafSRobert Mustacchi
269993e3fafSRobert Mustacchi VERIFY(xhcip != NULL);
270993e3fafSRobert Mustacchi xhci_dma_free(&xt->xt_buffer);
271993e3fafSRobert Mustacchi if (xt->xt_isoc != NULL) {
2722aba3acdSRobert Mustacchi ASSERT3U(xt->xt_ntrbs, >, 0);
273993e3fafSRobert Mustacchi kmem_free(xt->xt_isoc, sizeof (usb_isoc_pkt_descr_t) *
274993e3fafSRobert Mustacchi xt->xt_ntrbs);
275993e3fafSRobert Mustacchi xt->xt_isoc = NULL;
276993e3fafSRobert Mustacchi }
277993e3fafSRobert Mustacchi if (xt->xt_trbs != NULL) {
2782aba3acdSRobert Mustacchi ASSERT3U(xt->xt_ntrbs, >, 0);
279993e3fafSRobert Mustacchi kmem_free(xt->xt_trbs, sizeof (xhci_trb_t) * xt->xt_ntrbs);
280993e3fafSRobert Mustacchi xt->xt_trbs = NULL;
281993e3fafSRobert Mustacchi }
2822aba3acdSRobert Mustacchi if (xt->xt_trbs_pa != NULL) {
2832aba3acdSRobert Mustacchi ASSERT3U(xt->xt_ntrbs, >, 0);
2842aba3acdSRobert Mustacchi kmem_free(xt->xt_trbs_pa, sizeof (uint64_t) * xt->xt_ntrbs);
2852aba3acdSRobert Mustacchi xt->xt_trbs_pa = NULL;
2862aba3acdSRobert Mustacchi }
287993e3fafSRobert Mustacchi kmem_free(xt, sizeof (xhci_transfer_t));
288993e3fafSRobert Mustacchi }
289993e3fafSRobert Mustacchi
290993e3fafSRobert Mustacchi xhci_transfer_t *
xhci_transfer_alloc(xhci_t * xhcip,xhci_endpoint_t * xep,size_t size,uint_t trbs,int usb_flags)2912aba3acdSRobert Mustacchi xhci_transfer_alloc(xhci_t *xhcip, xhci_endpoint_t *xep, size_t size,
2922aba3acdSRobert Mustacchi uint_t trbs, int usb_flags)
293993e3fafSRobert Mustacchi {
294993e3fafSRobert Mustacchi int kmflags;
295993e3fafSRobert Mustacchi boolean_t dmawait;
296993e3fafSRobert Mustacchi xhci_transfer_t *xt;
297993e3fafSRobert Mustacchi ddi_device_acc_attr_t acc;
298993e3fafSRobert Mustacchi ddi_dma_attr_t attr;
299993e3fafSRobert Mustacchi
300993e3fafSRobert Mustacchi if (usb_flags & USB_FLAGS_SLEEP) {
301993e3fafSRobert Mustacchi kmflags = KM_SLEEP;
302993e3fafSRobert Mustacchi dmawait = B_TRUE;
303993e3fafSRobert Mustacchi } else {
304993e3fafSRobert Mustacchi kmflags = KM_NOSLEEP;
305993e3fafSRobert Mustacchi dmawait = B_FALSE;
306993e3fafSRobert Mustacchi }
307993e3fafSRobert Mustacchi
308993e3fafSRobert Mustacchi xt = kmem_zalloc(sizeof (xhci_transfer_t), kmflags);
309993e3fafSRobert Mustacchi if (xt == NULL)
310993e3fafSRobert Mustacchi return (NULL);
311993e3fafSRobert Mustacchi
312993e3fafSRobert Mustacchi if (size != 0) {
313993e3fafSRobert Mustacchi int sgl = XHCI_DEF_DMA_SGL;
314993e3fafSRobert Mustacchi
315993e3fafSRobert Mustacchi /*
316993e3fafSRobert Mustacchi * For BULK transfers, we always increase the number of SGL
317993e3fafSRobert Mustacchi * entries that we support to make things easier for the kernel.
318993e3fafSRobert Mustacchi * However, for control transfers, we currently opt to keep
319993e3fafSRobert Mustacchi * things a bit simpler and use our default of one SGL. There's
320993e3fafSRobert Mustacchi * no good technical reason for this, rather it just keeps
321993e3fafSRobert Mustacchi * things a bit easier.
322993e3fafSRobert Mustacchi *
323993e3fafSRobert Mustacchi * To simplify things, we don't use additional SGL entries for
324993e3fafSRobert Mustacchi * ISOC transfers. While this isn't the best, it isn't too far
325993e3fafSRobert Mustacchi * off from what ehci and co. have done before. If this becomes
326993e3fafSRobert Mustacchi * a technical issue, it's certainly possible to increase the
327993e3fafSRobert Mustacchi * SGL entry count.
3282aba3acdSRobert Mustacchi *
3292aba3acdSRobert Mustacchi * When we use the larger SGL count, we change our strategy for
3302aba3acdSRobert Mustacchi * being notified. In such a case we will opt to use an event
3312aba3acdSRobert Mustacchi * data packet. This helps deal with cases where some
3322aba3acdSRobert Mustacchi * controllers don't properly generate events for the last entry
3332aba3acdSRobert Mustacchi * in a TD with IOC when IOSP is set.
334993e3fafSRobert Mustacchi */
3352aba3acdSRobert Mustacchi if (xep->xep_type == USB_EP_ATTR_BULK) {
336993e3fafSRobert Mustacchi sgl = XHCI_TRANSFER_DMA_SGL;
3372aba3acdSRobert Mustacchi trbs++;
3382aba3acdSRobert Mustacchi }
339993e3fafSRobert Mustacchi
340993e3fafSRobert Mustacchi xhci_dma_acc_attr(xhcip, &acc);
341993e3fafSRobert Mustacchi xhci_dma_transfer_attr(xhcip, &attr, sgl);
342993e3fafSRobert Mustacchi if (xhci_dma_alloc(xhcip, &xt->xt_buffer, &attr, &acc, B_FALSE,
343993e3fafSRobert Mustacchi size, dmawait) == B_FALSE) {
344993e3fafSRobert Mustacchi kmem_free(xt, sizeof (xhci_transfer_t));
345993e3fafSRobert Mustacchi return (NULL);
346993e3fafSRobert Mustacchi }
347993e3fafSRobert Mustacchi
348993e3fafSRobert Mustacchi /*
349993e3fafSRobert Mustacchi * ISOC transfers are a bit special and don't need additional
350993e3fafSRobert Mustacchi * TRBs for data.
351993e3fafSRobert Mustacchi */
352993e3fafSRobert Mustacchi if (xep->xep_type != USB_EP_ATTR_ISOCH)
353993e3fafSRobert Mustacchi trbs += xt->xt_buffer.xdb_ncookies;
354993e3fafSRobert Mustacchi }
355993e3fafSRobert Mustacchi
356993e3fafSRobert Mustacchi xt->xt_trbs = kmem_zalloc(sizeof (xhci_trb_t) * trbs, kmflags);
357993e3fafSRobert Mustacchi if (xt->xt_trbs == NULL) {
358993e3fafSRobert Mustacchi xhci_dma_free(&xt->xt_buffer);
359993e3fafSRobert Mustacchi kmem_free(xt, sizeof (xhci_transfer_t));
360993e3fafSRobert Mustacchi return (NULL);
361993e3fafSRobert Mustacchi }
362993e3fafSRobert Mustacchi
3632aba3acdSRobert Mustacchi xt->xt_trbs_pa = kmem_zalloc(sizeof (uint64_t) * trbs, kmflags);
3642aba3acdSRobert Mustacchi if (xt->xt_trbs_pa == NULL) {
3652aba3acdSRobert Mustacchi kmem_free(xt->xt_trbs, sizeof (xhci_trb_t) * trbs);
3662aba3acdSRobert Mustacchi xhci_dma_free(&xt->xt_buffer);
3672aba3acdSRobert Mustacchi kmem_free(xt, sizeof (xhci_transfer_t));
3682aba3acdSRobert Mustacchi return (NULL);
3692aba3acdSRobert Mustacchi }
3702aba3acdSRobert Mustacchi
371993e3fafSRobert Mustacchi /*
372993e3fafSRobert Mustacchi * For ISOCH transfers, we need to also allocate the results data.
373993e3fafSRobert Mustacchi */
374993e3fafSRobert Mustacchi if (xep->xep_type == USB_EP_ATTR_ISOCH) {
375993e3fafSRobert Mustacchi xt->xt_isoc = kmem_zalloc(sizeof (usb_isoc_pkt_descr_t) * trbs,
376993e3fafSRobert Mustacchi kmflags);
377993e3fafSRobert Mustacchi if (xt->xt_isoc == NULL) {
3782aba3acdSRobert Mustacchi kmem_free(xt->xt_trbs_pa, sizeof (uint64_t) * trbs);
379993e3fafSRobert Mustacchi kmem_free(xt->xt_trbs, sizeof (xhci_trb_t) * trbs);
380993e3fafSRobert Mustacchi xhci_dma_free(&xt->xt_buffer);
381993e3fafSRobert Mustacchi kmem_free(xt, sizeof (xhci_transfer_t));
382993e3fafSRobert Mustacchi return (NULL);
383993e3fafSRobert Mustacchi }
384993e3fafSRobert Mustacchi }
385993e3fafSRobert Mustacchi
386993e3fafSRobert Mustacchi xt->xt_ntrbs = trbs;
387993e3fafSRobert Mustacchi xt->xt_cr = USB_CR_OK;
388993e3fafSRobert Mustacchi
389993e3fafSRobert Mustacchi return (xt);
390993e3fafSRobert Mustacchi }
391993e3fafSRobert Mustacchi
392993e3fafSRobert Mustacchi /*
393993e3fafSRobert Mustacchi * Abstract the notion of copying out to handle the case of multiple DMA
394993e3fafSRobert Mustacchi * cookies. If tobuf is true, we are copying to the kernel provided buffer,
395993e3fafSRobert Mustacchi * otherwise we're copying into the DMA memory.
396993e3fafSRobert Mustacchi */
397993e3fafSRobert Mustacchi void
xhci_transfer_copy(xhci_transfer_t * xt,void * buf,size_t len,boolean_t tobuf)398993e3fafSRobert Mustacchi xhci_transfer_copy(xhci_transfer_t *xt, void *buf, size_t len,
399993e3fafSRobert Mustacchi boolean_t tobuf)
400993e3fafSRobert Mustacchi {
401993e3fafSRobert Mustacchi void *dmabuf = xt->xt_buffer.xdb_va;
402993e3fafSRobert Mustacchi if (tobuf == B_TRUE)
403993e3fafSRobert Mustacchi bcopy(dmabuf, buf, len);
404993e3fafSRobert Mustacchi else
405993e3fafSRobert Mustacchi bcopy(buf, dmabuf, len);
406993e3fafSRobert Mustacchi }
407993e3fafSRobert Mustacchi
408993e3fafSRobert Mustacchi int
xhci_transfer_sync(xhci_t * xhcip,xhci_transfer_t * xt,uint_t type)409993e3fafSRobert Mustacchi xhci_transfer_sync(xhci_t *xhcip, xhci_transfer_t *xt, uint_t type)
410993e3fafSRobert Mustacchi {
411993e3fafSRobert Mustacchi XHCI_DMA_SYNC(xt->xt_buffer, type);
412993e3fafSRobert Mustacchi return (xhci_check_dma_handle(xhcip, &xt->xt_buffer));
413993e3fafSRobert Mustacchi }
414993e3fafSRobert Mustacchi
415993e3fafSRobert Mustacchi /*
416993e3fafSRobert Mustacchi * We're required to try and inform the xHCI controller about the number of data
417993e3fafSRobert Mustacchi * packets that are required. The algorithm to use is described in xHCI 1.1 /
418993e3fafSRobert Mustacchi * 4.11.2.4. While it might be tempting to just try and calculate the number of
419993e3fafSRobert Mustacchi * packets based on simple rounding of the remaining number of bytes, that
420993e3fafSRobert Mustacchi * misses a critical problem -- DMA boundaries may cause us to need additional
421993e3fafSRobert Mustacchi * packets that are missed initially. Consider a transfer made up of four
422993e3fafSRobert Mustacchi * different DMA buffers sized in bytes: 4096, 4096, 256, 256, with a 512 byte
423993e3fafSRobert Mustacchi * packet size.
424993e3fafSRobert Mustacchi *
425993e3fafSRobert Mustacchi * Remain 4608 512 256 0
426993e3fafSRobert Mustacchi * Bytes 4096 4096 256 256
427993e3fafSRobert Mustacchi * Naive TD 9 1 1 0
4282aba3acdSRobert Mustacchi * Act TD 10 2 1 0
429993e3fafSRobert Mustacchi *
430993e3fafSRobert Mustacchi * This means that the only safe way forward here is to work backwards and see
431993e3fafSRobert Mustacchi * how many we need to work up to this point.
432993e3fafSRobert Mustacchi */
4332aba3acdSRobert Mustacchi static uint_t
xhci_transfer_get_tdsize(xhci_transfer_t * xt,uint_t off,uint_t mps)434993e3fafSRobert Mustacchi xhci_transfer_get_tdsize(xhci_transfer_t *xt, uint_t off, uint_t mps)
435993e3fafSRobert Mustacchi {
436993e3fafSRobert Mustacchi int i;
437993e3fafSRobert Mustacchi uint_t npkt = 0;
438993e3fafSRobert Mustacchi
439993e3fafSRobert Mustacchi /*
440993e3fafSRobert Mustacchi * There are always zero packets for the last TRB.
441993e3fafSRobert Mustacchi */
442993e3fafSRobert Mustacchi ASSERT(xt->xt_buffer.xdb_ncookies > 0);
443993e3fafSRobert Mustacchi for (i = xt->xt_buffer.xdb_ncookies - 1; i > off; i--) {
4442aba3acdSRobert Mustacchi size_t len = roundup(xt->xt_buffer.xdb_cookies[i].dmac_size,
4452aba3acdSRobert Mustacchi mps);
446993e3fafSRobert Mustacchi npkt += len / mps;
447993e3fafSRobert Mustacchi }
448993e3fafSRobert Mustacchi
4492aba3acdSRobert Mustacchi /*
4502aba3acdSRobert Mustacchi * Make sure to clamp this value otherwise we risk truncation.
4512aba3acdSRobert Mustacchi */
4522aba3acdSRobert Mustacchi if (npkt >= XHCI_MAX_TDSIZE)
4532aba3acdSRobert Mustacchi return (XHCI_MAX_TDSIZE);
4542aba3acdSRobert Mustacchi
455993e3fafSRobert Mustacchi return (npkt);
456993e3fafSRobert Mustacchi }
457993e3fafSRobert Mustacchi
458993e3fafSRobert Mustacchi void
xhci_transfer_trb_fill_data(xhci_endpoint_t * xep,xhci_transfer_t * xt,int off,boolean_t in)459993e3fafSRobert Mustacchi xhci_transfer_trb_fill_data(xhci_endpoint_t *xep, xhci_transfer_t *xt, int off,
460993e3fafSRobert Mustacchi boolean_t in)
461993e3fafSRobert Mustacchi {
462993e3fafSRobert Mustacchi uint_t mps, tdsize, flags;
463993e3fafSRobert Mustacchi int i;
464993e3fafSRobert Mustacchi
465993e3fafSRobert Mustacchi VERIFY(xt->xt_buffer.xdb_ncookies > 0);
466993e3fafSRobert Mustacchi VERIFY(xep->xep_pipe != NULL);
467993e3fafSRobert Mustacchi VERIFY(off + xt->xt_buffer.xdb_ncookies <= xt->xt_ntrbs);
468993e3fafSRobert Mustacchi mps = xep->xep_pipe->p_ep.wMaxPacketSize;
469993e3fafSRobert Mustacchi
4702aba3acdSRobert Mustacchi if (in == B_TRUE) {
4712aba3acdSRobert Mustacchi xt->xt_data_tohost = B_TRUE;
4722aba3acdSRobert Mustacchi }
4732aba3acdSRobert Mustacchi
4742aba3acdSRobert Mustacchi /*
4752aba3acdSRobert Mustacchi * We assume that if we have a non-bulk endpoint, then we should only
4762aba3acdSRobert Mustacchi * have a single cookie. This falls out from the default SGL length that
4772aba3acdSRobert Mustacchi * we use for these other device types.
4782aba3acdSRobert Mustacchi */
4792aba3acdSRobert Mustacchi if (xep->xep_type != USB_EP_ATTR_BULK) {
4802aba3acdSRobert Mustacchi VERIFY3U(xt->xt_buffer.xdb_ncookies, ==, 1);
4812aba3acdSRobert Mustacchi }
4822aba3acdSRobert Mustacchi
483993e3fafSRobert Mustacchi for (i = 0; i < xt->xt_buffer.xdb_ncookies; i++) {
484993e3fafSRobert Mustacchi uint64_t pa, dmasz;
485993e3fafSRobert Mustacchi
486993e3fafSRobert Mustacchi pa = xt->xt_buffer.xdb_cookies[i].dmac_laddress;
487993e3fafSRobert Mustacchi dmasz = xt->xt_buffer.xdb_cookies[i].dmac_size;
488993e3fafSRobert Mustacchi
489993e3fafSRobert Mustacchi tdsize = xhci_transfer_get_tdsize(xt, i, mps);
490993e3fafSRobert Mustacchi
491993e3fafSRobert Mustacchi flags = XHCI_TRB_TYPE_NORMAL;
492993e3fafSRobert Mustacchi if (i == 0 && xep->xep_type == USB_EP_ATTR_CONTROL) {
493993e3fafSRobert Mustacchi flags = XHCI_TRB_TYPE_DATA;
494993e3fafSRobert Mustacchi if (in == B_TRUE)
495993e3fafSRobert Mustacchi flags |= XHCI_TRB_DIR_IN;
496993e3fafSRobert Mustacchi }
497993e3fafSRobert Mustacchi
498993e3fafSRobert Mustacchi /*
4992aba3acdSRobert Mustacchi * If we have more than one cookie, then we need to set chaining
5002aba3acdSRobert Mustacchi * on every TRB and the last TRB will turn into an event data
5012aba3acdSRobert Mustacchi * TRB. If we only have a single TRB, then we just set interrupt
5022aba3acdSRobert Mustacchi * on completion (IOC). There's no need to specifically set
5032aba3acdSRobert Mustacchi * interrupt on short packet (IOSP) in that case, as we'll
5042aba3acdSRobert Mustacchi * always get the event notification. We still need the chain
5052aba3acdSRobert Mustacchi * bit set on the last packet, so we can chain into the event
5062aba3acdSRobert Mustacchi * data. Even if all the data on a bulk endpoint (the only
5072aba3acdSRobert Mustacchi * endpoint type that uses chaining today) has only one cookie,
5082aba3acdSRobert Mustacchi * then we'll still schedule an event data block.
509993e3fafSRobert Mustacchi */
5102aba3acdSRobert Mustacchi if (xep->xep_type == USB_EP_ATTR_BULK ||
5112aba3acdSRobert Mustacchi xt->xt_buffer.xdb_ncookies > 1) {
512993e3fafSRobert Mustacchi flags |= XHCI_TRB_CHAIN;
513993e3fafSRobert Mustacchi }
514993e3fafSRobert Mustacchi
515993e3fafSRobert Mustacchi /*
5162aba3acdSRobert Mustacchi * What we set for the last TRB depends on the type of the
5172aba3acdSRobert Mustacchi * endpoint. If it's a bulk endpoint, then we have to set
5182aba3acdSRobert Mustacchi * evaluate next trb (ENT) so we successfully process the event
5192aba3acdSRobert Mustacchi * data TRB we'll set up. Otherwise, we need to make sure that
5202aba3acdSRobert Mustacchi * we set interrupt on completion, so we get the event. However,
5212aba3acdSRobert Mustacchi * we don't set the event on control endpoints, as the status
5222aba3acdSRobert Mustacchi * stage TD will be the one where we get the event. But, we do
5232aba3acdSRobert Mustacchi * still need an interrupt on short packet, because technically
5242aba3acdSRobert Mustacchi * the status stage is in its own TD.
525993e3fafSRobert Mustacchi */
5262aba3acdSRobert Mustacchi if (i + 1 == xt->xt_buffer.xdb_ncookies) {
5272aba3acdSRobert Mustacchi switch (xep->xep_type) {
5282aba3acdSRobert Mustacchi case USB_EP_ATTR_BULK:
5292aba3acdSRobert Mustacchi flags |= XHCI_TRB_ENT;
5302aba3acdSRobert Mustacchi break;
5312aba3acdSRobert Mustacchi case USB_EP_ATTR_CONTROL:
5322aba3acdSRobert Mustacchi flags |= XHCI_TRB_ISP;
5332aba3acdSRobert Mustacchi break;
5342aba3acdSRobert Mustacchi default:
5352aba3acdSRobert Mustacchi flags |= XHCI_TRB_IOC;
5362aba3acdSRobert Mustacchi break;
5372aba3acdSRobert Mustacchi }
538993e3fafSRobert Mustacchi }
539993e3fafSRobert Mustacchi
540993e3fafSRobert Mustacchi xt->xt_trbs[off + i].trb_addr = LE_64(pa);
541993e3fafSRobert Mustacchi xt->xt_trbs[off + i].trb_status = LE_32(XHCI_TRB_LEN(dmasz) |
542993e3fafSRobert Mustacchi XHCI_TRB_TDREM(tdsize) | XHCI_TRB_INTR(0));
543993e3fafSRobert Mustacchi xt->xt_trbs[off + i].trb_flags = LE_32(flags);
544993e3fafSRobert Mustacchi }
5452aba3acdSRobert Mustacchi
5462aba3acdSRobert Mustacchi /*
5472aba3acdSRobert Mustacchi * The last TRB in any bulk transfer is the Event Data TRB.
5482aba3acdSRobert Mustacchi */
5492aba3acdSRobert Mustacchi if (xep->xep_type == USB_EP_ATTR_BULK) {
5502aba3acdSRobert Mustacchi VERIFY(off + xt->xt_buffer.xdb_ncookies + 1 <= xt->xt_ntrbs);
5512aba3acdSRobert Mustacchi xt->xt_trbs[off + i].trb_addr = LE_64((uintptr_t)xt);
5522aba3acdSRobert Mustacchi xt->xt_trbs[off + i].trb_status = LE_32(XHCI_TRB_INTR(0));
5532aba3acdSRobert Mustacchi xt->xt_trbs[off + i].trb_flags = LE_32(XHCI_TRB_TYPE_EVENT |
5542aba3acdSRobert Mustacchi XHCI_TRB_IOC);
5552aba3acdSRobert Mustacchi }
556993e3fafSRobert Mustacchi }
557993e3fafSRobert Mustacchi
558993e3fafSRobert Mustacchi /*
559993e3fafSRobert Mustacchi * These are utility functions for isochronus transfers to help calculate the
560993e3fafSRobert Mustacchi * transfer burst count (TBC) and transfer last burst packet count (TLPBC)
561993e3fafSRobert Mustacchi * entries for an isochronus entry. See xHCI 1.1 / 4.11.2.3 for how to calculate
562993e3fafSRobert Mustacchi * them.
563993e3fafSRobert Mustacchi */
564993e3fafSRobert Mustacchi void
xhci_transfer_calculate_isoc(xhci_device_t * xd,xhci_endpoint_t * xep,uint_t trb_len,uint_t * tbc,uint_t * tlbpc)565993e3fafSRobert Mustacchi xhci_transfer_calculate_isoc(xhci_device_t *xd, xhci_endpoint_t *xep,
566993e3fafSRobert Mustacchi uint_t trb_len, uint_t *tbc, uint_t *tlbpc)
567993e3fafSRobert Mustacchi {
568993e3fafSRobert Mustacchi uint_t mps, tdpc, burst;
569993e3fafSRobert Mustacchi
570993e3fafSRobert Mustacchi /*
571993e3fafSRobert Mustacchi * Even if we're asked to send no data, that actually requires the
572993e3fafSRobert Mustacchi * equivalent of sending one byte of data.
573993e3fafSRobert Mustacchi */
574993e3fafSRobert Mustacchi if (trb_len == 0)
575993e3fafSRobert Mustacchi trb_len = 1;
576993e3fafSRobert Mustacchi
577993e3fafSRobert Mustacchi mps = XHCI_EPCTX_GET_MPS(xd->xd_endout[xep->xep_num]->xec_info2);
578993e3fafSRobert Mustacchi burst = XHCI_EPCTX_GET_MAXB(xd->xd_endout[xep->xep_num]->xec_info2);
579993e3fafSRobert Mustacchi
580993e3fafSRobert Mustacchi /*
581993e3fafSRobert Mustacchi * This is supposed to correspond to the Transfer Descriptor Packet
582993e3fafSRobert Mustacchi * Count from xHCI 1.1 / 4.14.1.
583993e3fafSRobert Mustacchi */
584993e3fafSRobert Mustacchi tdpc = howmany(trb_len, mps);
585993e3fafSRobert Mustacchi *tbc = howmany(tdpc, burst + 1) - 1;
586993e3fafSRobert Mustacchi
587993e3fafSRobert Mustacchi if ((tdpc % (burst + 1)) == 0)
588993e3fafSRobert Mustacchi *tlbpc = burst;
589993e3fafSRobert Mustacchi else
590993e3fafSRobert Mustacchi *tlbpc = (tdpc % (burst + 1)) - 1;
591993e3fafSRobert Mustacchi }
592