1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2020, The University of Queensland
14 * Copyright (c) 2018, Joyent, Inc.
15 */
16
17 /*
18 * DMA allocation and tear down routines.
19 */
20
21 #include <mlxcx.h>
22
23 void
mlxcx_dma_acc_attr(mlxcx_t * mlxp,ddi_device_acc_attr_t * accp)24 mlxcx_dma_acc_attr(mlxcx_t *mlxp, ddi_device_acc_attr_t *accp)
25 {
26 bzero(accp, sizeof (*accp));
27 accp->devacc_attr_version = DDI_DEVICE_ATTR_V0;
28 accp->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
29 accp->devacc_attr_dataorder = DDI_STRICTORDER_ACC;
30
31 if (DDI_FM_DMA_ERR_CAP(mlxp->mlx_fm_caps)) {
32 accp->devacc_attr_access = DDI_FLAGERR_ACC;
33 } else {
34 accp->devacc_attr_access = DDI_DEFAULT_ACC;
35 }
36 }
37
38 void
mlxcx_dma_page_attr(mlxcx_t * mlxp,ddi_dma_attr_t * attrp)39 mlxcx_dma_page_attr(mlxcx_t *mlxp, ddi_dma_attr_t *attrp)
40 {
41 bzero(attrp, sizeof (*attrp));
42 attrp->dma_attr_version = DMA_ATTR_V0;
43
44 /*
45 * This is a 64-bit PCIe device. We can use the entire address space.
46 */
47 attrp->dma_attr_addr_lo = 0x0;
48 attrp->dma_attr_addr_hi = UINT64_MAX;
49
50 /*
51 * The count max indicates the total amount that can fit into one
52 * cookie. Because we're creating a single page for tracking purposes,
53 * this can be a page in size. The alignment and segment are related to
54 * this same requirement. The alignment needs to be page aligned and the
55 * segment is the boundary that this can't cross, aka a 4k page.
56 */
57 attrp->dma_attr_count_max = MLXCX_CMD_DMA_PAGE_SIZE - 1;
58 attrp->dma_attr_align = MLXCX_CMD_DMA_PAGE_SIZE;
59 attrp->dma_attr_seg = MLXCX_CMD_DMA_PAGE_SIZE - 1;
60
61 attrp->dma_attr_burstsizes = 0xfff;
62
63 /*
64 * The minimum and and maximum sizes that we can send. We cap this based
65 * on the use of this, which is a page size.
66 */
67 attrp->dma_attr_minxfer = 0x1;
68 attrp->dma_attr_maxxfer = MLXCX_CMD_DMA_PAGE_SIZE;
69
70 /*
71 * This is supposed to be used for static data structures, therefore we
72 * keep this just to a page.
73 */
74 attrp->dma_attr_sgllen = 1;
75
76 /*
77 * The granularity describe the addressing graularity. That is, the
78 * hardware can ask for chunks in this units of bytes.
79 */
80 attrp->dma_attr_granular = MLXCX_CMD_DMA_PAGE_SIZE;
81
82 if (DDI_FM_DMA_ERR_CAP(mlxp->mlx_fm_caps)) {
83 attrp->dma_attr_flags = DDI_DMA_FLAGERR;
84 } else {
85 attrp->dma_attr_flags = 0;
86 }
87 }
88
89 /*
90 * DMA attributes for queue memory (EQ, CQ, WQ etc)
91 *
92 * These have to allocate in units of whole pages, but can be multiple
93 * pages and don't have to be physically contiguous.
94 */
95 void
mlxcx_dma_queue_attr(mlxcx_t * mlxp,ddi_dma_attr_t * attrp)96 mlxcx_dma_queue_attr(mlxcx_t *mlxp, ddi_dma_attr_t *attrp)
97 {
98 bzero(attrp, sizeof (*attrp));
99 attrp->dma_attr_version = DMA_ATTR_V0;
100
101 /*
102 * This is a 64-bit PCIe device. We can use the entire address space.
103 */
104 attrp->dma_attr_addr_lo = 0x0;
105 attrp->dma_attr_addr_hi = UINT64_MAX;
106
107 attrp->dma_attr_count_max = MLXCX_QUEUE_DMA_PAGE_SIZE - 1;
108
109 attrp->dma_attr_align = MLXCX_QUEUE_DMA_PAGE_SIZE;
110
111 attrp->dma_attr_burstsizes = 0xfff;
112
113 /*
114 * The minimum and and maximum sizes that we can send. We cap this based
115 * on the use of this, which is a page size.
116 */
117 attrp->dma_attr_minxfer = MLXCX_QUEUE_DMA_PAGE_SIZE;
118 attrp->dma_attr_maxxfer = UINT32_MAX;
119
120 attrp->dma_attr_seg = UINT64_MAX;
121
122 attrp->dma_attr_granular = MLXCX_QUEUE_DMA_PAGE_SIZE;
123
124 /* But we can have more than one. */
125 attrp->dma_attr_sgllen = MLXCX_CREATE_QUEUE_MAX_PAGES;
126
127 if (DDI_FM_DMA_ERR_CAP(mlxp->mlx_fm_caps)) {
128 attrp->dma_attr_flags = DDI_DMA_FLAGERR;
129 } else {
130 attrp->dma_attr_flags = 0;
131 }
132 }
133
134 /*
135 * DMA attributes for packet buffers
136 */
137 void
mlxcx_dma_buf_attr(mlxcx_t * mlxp,ddi_dma_attr_t * attrp)138 mlxcx_dma_buf_attr(mlxcx_t *mlxp, ddi_dma_attr_t *attrp)
139 {
140 bzero(attrp, sizeof (*attrp));
141 attrp->dma_attr_version = DMA_ATTR_V0;
142
143 /*
144 * This is a 64-bit PCIe device. We can use the entire address space.
145 */
146 attrp->dma_attr_addr_lo = 0x0;
147 attrp->dma_attr_addr_hi = UINT64_MAX;
148
149 /*
150 * Each scatter pointer has a 32-bit length field.
151 */
152 attrp->dma_attr_count_max = UINT32_MAX;
153
154 /*
155 * The PRM gives us no alignment requirements for scatter pointers,
156 * but it implies that units < 16bytes are a bad idea.
157 */
158 attrp->dma_attr_align = 16;
159 attrp->dma_attr_granular = 1;
160
161 attrp->dma_attr_burstsizes = 0xfff;
162
163 attrp->dma_attr_minxfer = 1;
164 attrp->dma_attr_maxxfer = UINT64_MAX;
165
166 attrp->dma_attr_seg = UINT64_MAX;
167
168 /*
169 * We choose how many scatter pointers we're allowed per packet when
170 * we set the recv queue stride. This macro is from mlxcx_reg.h where
171 * we fix that for all of our receive queues.
172 */
173 attrp->dma_attr_sgllen = MLXCX_RECVQ_MAX_PTRS;
174
175 if (DDI_FM_DMA_ERR_CAP(mlxp->mlx_fm_caps)) {
176 attrp->dma_attr_flags = DDI_DMA_FLAGERR;
177 } else {
178 attrp->dma_attr_flags = 0;
179 }
180 }
181
182 /*
183 * DMA attributes for queue doorbells
184 */
185 void
mlxcx_dma_qdbell_attr(mlxcx_t * mlxp,ddi_dma_attr_t * attrp)186 mlxcx_dma_qdbell_attr(mlxcx_t *mlxp, ddi_dma_attr_t *attrp)
187 {
188 bzero(attrp, sizeof (*attrp));
189 attrp->dma_attr_version = DMA_ATTR_V0;
190
191 /*
192 * This is a 64-bit PCIe device. We can use the entire address space.
193 */
194 attrp->dma_attr_addr_lo = 0x0;
195 attrp->dma_attr_addr_hi = UINT64_MAX;
196
197 /*
198 * Queue doorbells are always exactly 16 bytes in length, but
199 * the ddi_dma functions don't like such small values of count_max.
200 *
201 * We tell some lies here.
202 */
203 attrp->dma_attr_count_max = MLXCX_QUEUE_DMA_PAGE_SIZE - 1;
204 attrp->dma_attr_align = 8;
205 attrp->dma_attr_burstsizes = 0x8;
206 attrp->dma_attr_minxfer = 1;
207 attrp->dma_attr_maxxfer = UINT16_MAX;
208 attrp->dma_attr_seg = MLXCX_QUEUE_DMA_PAGE_SIZE - 1;
209 attrp->dma_attr_granular = 1;
210 attrp->dma_attr_sgllen = 1;
211
212 if (DDI_FM_DMA_ERR_CAP(mlxp->mlx_fm_caps)) {
213 attrp->dma_attr_flags = DDI_DMA_FLAGERR;
214 } else {
215 attrp->dma_attr_flags = 0;
216 }
217 }
218
219 void
mlxcx_dma_free(mlxcx_dma_buffer_t * mxdb)220 mlxcx_dma_free(mlxcx_dma_buffer_t *mxdb)
221 {
222 int ret;
223
224 if (mxdb->mxdb_flags & MLXCX_DMABUF_BOUND) {
225 VERIFY(mxdb->mxdb_dma_handle != NULL);
226 ret = ddi_dma_unbind_handle(mxdb->mxdb_dma_handle);
227 VERIFY3S(ret, ==, DDI_SUCCESS);
228 mxdb->mxdb_flags &= ~MLXCX_DMABUF_BOUND;
229 mxdb->mxdb_ncookies = 0;
230 }
231
232 if (mxdb->mxdb_flags & MLXCX_DMABUF_MEM_ALLOC) {
233 ddi_dma_mem_free(&mxdb->mxdb_acc_handle);
234 mxdb->mxdb_acc_handle = NULL;
235 mxdb->mxdb_va = NULL;
236 mxdb->mxdb_len = 0;
237 mxdb->mxdb_flags &= ~MLXCX_DMABUF_MEM_ALLOC;
238 }
239
240 if (mxdb->mxdb_flags & MLXCX_DMABUF_FOREIGN) {
241 /* The mblk will be freed separately */
242 mxdb->mxdb_va = NULL;
243 mxdb->mxdb_len = 0;
244 mxdb->mxdb_flags &= ~MLXCX_DMABUF_FOREIGN;
245 }
246
247 if (mxdb->mxdb_flags & MLXCX_DMABUF_HDL_ALLOC) {
248 ddi_dma_free_handle(&mxdb->mxdb_dma_handle);
249 mxdb->mxdb_dma_handle = NULL;
250 mxdb->mxdb_flags &= ~MLXCX_DMABUF_HDL_ALLOC;
251 }
252
253 ASSERT3U(mxdb->mxdb_flags, ==, 0);
254 ASSERT3P(mxdb->mxdb_dma_handle, ==, NULL);
255 ASSERT3P(mxdb->mxdb_va, ==, NULL);
256 ASSERT3U(mxdb->mxdb_len, ==, 0);
257 ASSERT3U(mxdb->mxdb_ncookies, ==, 0);
258 }
259
260 void
mlxcx_dma_unbind(mlxcx_t * mlxp,mlxcx_dma_buffer_t * mxdb)261 mlxcx_dma_unbind(mlxcx_t *mlxp, mlxcx_dma_buffer_t *mxdb)
262 {
263 int ret;
264
265 ASSERT(mxdb->mxdb_flags & MLXCX_DMABUF_HDL_ALLOC);
266 ASSERT(mxdb->mxdb_flags & MLXCX_DMABUF_BOUND);
267
268 if (mxdb->mxdb_flags & MLXCX_DMABUF_FOREIGN) {
269 /* The mblk will be freed separately */
270 mxdb->mxdb_va = NULL;
271 mxdb->mxdb_len = 0;
272 mxdb->mxdb_flags &= ~MLXCX_DMABUF_FOREIGN;
273 }
274
275 ret = ddi_dma_unbind_handle(mxdb->mxdb_dma_handle);
276 VERIFY3S(ret, ==, DDI_SUCCESS);
277 mxdb->mxdb_flags &= ~MLXCX_DMABUF_BOUND;
278 mxdb->mxdb_ncookies = 0;
279 }
280
281 boolean_t
mlxcx_dma_init(mlxcx_t * mlxp,mlxcx_dma_buffer_t * mxdb,ddi_dma_attr_t * attrp,boolean_t wait)282 mlxcx_dma_init(mlxcx_t *mlxp, mlxcx_dma_buffer_t *mxdb,
283 ddi_dma_attr_t *attrp, boolean_t wait)
284 {
285 int ret;
286 int (*memcb)(caddr_t);
287
288 if (wait == B_TRUE) {
289 memcb = DDI_DMA_SLEEP;
290 } else {
291 memcb = DDI_DMA_DONTWAIT;
292 }
293
294 ASSERT3S(mxdb->mxdb_flags, ==, 0);
295
296 ret = ddi_dma_alloc_handle(mlxp->mlx_dip, attrp, memcb, NULL,
297 &mxdb->mxdb_dma_handle);
298 if (ret != 0) {
299 mlxcx_warn(mlxp, "!failed to allocate DMA handle: %d", ret);
300 mxdb->mxdb_dma_handle = NULL;
301 return (B_FALSE);
302 }
303 mxdb->mxdb_flags |= MLXCX_DMABUF_HDL_ALLOC;
304
305 return (B_TRUE);
306 }
307
308 boolean_t
mlxcx_dma_bind_mblk(mlxcx_t * mlxp,mlxcx_dma_buffer_t * mxdb,const mblk_t * mp,size_t off,boolean_t wait)309 mlxcx_dma_bind_mblk(mlxcx_t *mlxp, mlxcx_dma_buffer_t *mxdb,
310 const mblk_t *mp, size_t off, boolean_t wait)
311 {
312 int ret;
313 uint_t flags = DDI_DMA_STREAMING;
314 int (*memcb)(caddr_t);
315
316 if (wait == B_TRUE) {
317 memcb = DDI_DMA_SLEEP;
318 } else {
319 memcb = DDI_DMA_DONTWAIT;
320 }
321
322 ASSERT(mxdb->mxdb_flags & MLXCX_DMABUF_HDL_ALLOC);
323 ASSERT0(mxdb->mxdb_flags &
324 (MLXCX_DMABUF_FOREIGN | MLXCX_DMABUF_MEM_ALLOC));
325 ASSERT0(mxdb->mxdb_flags & MLXCX_DMABUF_BOUND);
326
327 ASSERT3U(off, <=, MBLKL(mp));
328 mxdb->mxdb_va = (caddr_t)(mp->b_rptr + off);
329 mxdb->mxdb_len = MBLKL(mp) - off;
330 mxdb->mxdb_flags |= MLXCX_DMABUF_FOREIGN;
331
332 ret = ddi_dma_addr_bind_handle(mxdb->mxdb_dma_handle, NULL,
333 mxdb->mxdb_va, mxdb->mxdb_len, DDI_DMA_WRITE | flags, memcb, NULL,
334 NULL, NULL);
335 if (ret != DDI_DMA_MAPPED) {
336 mxdb->mxdb_va = NULL;
337 mxdb->mxdb_len = 0;
338 mxdb->mxdb_flags &= ~MLXCX_DMABUF_FOREIGN;
339 return (B_FALSE);
340 }
341 mxdb->mxdb_flags |= MLXCX_DMABUF_BOUND;
342 mxdb->mxdb_ncookies = ddi_dma_ncookies(mxdb->mxdb_dma_handle);
343
344 return (B_TRUE);
345 }
346
347 boolean_t
mlxcx_dma_alloc(mlxcx_t * mlxp,mlxcx_dma_buffer_t * mxdb,ddi_dma_attr_t * attrp,ddi_device_acc_attr_t * accp,boolean_t zero,size_t size,boolean_t wait)348 mlxcx_dma_alloc(mlxcx_t *mlxp, mlxcx_dma_buffer_t *mxdb,
349 ddi_dma_attr_t *attrp, ddi_device_acc_attr_t *accp, boolean_t zero,
350 size_t size, boolean_t wait)
351 {
352 int ret;
353 uint_t flags = DDI_DMA_CONSISTENT;
354 size_t len;
355 int (*memcb)(caddr_t);
356
357 if (wait == B_TRUE) {
358 memcb = DDI_DMA_SLEEP;
359 } else {
360 memcb = DDI_DMA_DONTWAIT;
361 }
362
363 ASSERT3U(mxdb->mxdb_flags, ==, 0);
364
365 ret = ddi_dma_alloc_handle(mlxp->mlx_dip, attrp, memcb, NULL,
366 &mxdb->mxdb_dma_handle);
367 if (ret != 0) {
368 mlxcx_warn(mlxp, "!failed to allocate DMA handle: %d", ret);
369 mxdb->mxdb_dma_handle = NULL;
370 return (B_FALSE);
371 }
372 mxdb->mxdb_flags |= MLXCX_DMABUF_HDL_ALLOC;
373
374 ret = ddi_dma_mem_alloc(mxdb->mxdb_dma_handle, size, accp, flags, memcb,
375 NULL, &mxdb->mxdb_va, &len, &mxdb->mxdb_acc_handle);
376 if (ret != DDI_SUCCESS) {
377 mlxcx_warn(mlxp, "!failed to allocate DMA memory: %d", ret);
378 mxdb->mxdb_va = NULL;
379 mxdb->mxdb_acc_handle = NULL;
380 mlxcx_dma_free(mxdb);
381 return (B_FALSE);
382 }
383 mxdb->mxdb_len = size;
384 mxdb->mxdb_flags |= MLXCX_DMABUF_MEM_ALLOC;
385
386 if (zero == B_TRUE)
387 bzero(mxdb->mxdb_va, len);
388
389 ret = ddi_dma_addr_bind_handle(mxdb->mxdb_dma_handle, NULL,
390 mxdb->mxdb_va, len, DDI_DMA_RDWR | flags, memcb, NULL, NULL,
391 NULL);
392 if (ret != 0) {
393 mlxcx_warn(mlxp, "!failed to bind DMA memory: %d", ret);
394 mlxcx_dma_free(mxdb);
395 return (B_FALSE);
396 }
397 mxdb->mxdb_flags |= MLXCX_DMABUF_BOUND;
398 mxdb->mxdb_ncookies = ddi_dma_ncookies(mxdb->mxdb_dma_handle);
399
400 return (B_TRUE);
401 }
402
403 boolean_t
mlxcx_dma_alloc_offset(mlxcx_t * mlxp,mlxcx_dma_buffer_t * mxdb,ddi_dma_attr_t * attrp,ddi_device_acc_attr_t * accp,boolean_t zero,size_t size,size_t offset,boolean_t wait)404 mlxcx_dma_alloc_offset(mlxcx_t *mlxp, mlxcx_dma_buffer_t *mxdb,
405 ddi_dma_attr_t *attrp, ddi_device_acc_attr_t *accp, boolean_t zero,
406 size_t size, size_t offset, boolean_t wait)
407 {
408 int ret;
409 uint_t flags = DDI_DMA_STREAMING;
410 size_t len;
411 int (*memcb)(caddr_t);
412
413 if (wait == B_TRUE) {
414 memcb = DDI_DMA_SLEEP;
415 } else {
416 memcb = DDI_DMA_DONTWAIT;
417 }
418
419 ASSERT3U(mxdb->mxdb_flags, ==, 0);
420
421 ret = ddi_dma_alloc_handle(mlxp->mlx_dip, attrp, memcb, NULL,
422 &mxdb->mxdb_dma_handle);
423 if (ret != 0) {
424 mlxcx_warn(mlxp, "!failed to allocate DMA handle: %d", ret);
425 mxdb->mxdb_dma_handle = NULL;
426 return (B_FALSE);
427 }
428 mxdb->mxdb_flags |= MLXCX_DMABUF_HDL_ALLOC;
429
430 ret = ddi_dma_mem_alloc(mxdb->mxdb_dma_handle, size + offset, accp,
431 flags, memcb, NULL, &mxdb->mxdb_va, &len, &mxdb->mxdb_acc_handle);
432 if (ret != DDI_SUCCESS) {
433 mlxcx_warn(mlxp, "!failed to allocate DMA memory: %d", ret);
434 mxdb->mxdb_va = NULL;
435 mxdb->mxdb_acc_handle = NULL;
436 mlxcx_dma_free(mxdb);
437 return (B_FALSE);
438 }
439
440 if (zero == B_TRUE)
441 bzero(mxdb->mxdb_va, len);
442
443 mxdb->mxdb_va += offset;
444 len -= offset;
445 mxdb->mxdb_len = len;
446 mxdb->mxdb_flags |= MLXCX_DMABUF_MEM_ALLOC;
447
448 ret = ddi_dma_addr_bind_handle(mxdb->mxdb_dma_handle, NULL,
449 mxdb->mxdb_va, len, DDI_DMA_RDWR | flags, memcb, NULL, NULL,
450 NULL);
451 if (ret != 0) {
452 mlxcx_warn(mlxp, "!failed to bind DMA memory: %d", ret);
453 mlxcx_dma_free(mxdb);
454 return (B_FALSE);
455 }
456 mxdb->mxdb_flags |= MLXCX_DMABUF_BOUND;
457 mxdb->mxdb_ncookies = ddi_dma_ncookies(mxdb->mxdb_dma_handle);
458
459 return (B_TRUE);
460 }
461