1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2019 Joyent, Inc. 14 */ 15 16 /* 17 * VIRTIO FRAMEWORK: DMA ROUTINES 18 * 19 * For design and usage documentation, see the comments in "virtio.h". 20 */ 21 22 #include <sys/conf.h> 23 #include <sys/kmem.h> 24 #include <sys/debug.h> 25 #include <sys/modctl.h> 26 #include <sys/autoconf.h> 27 #include <sys/ddi_impldefs.h> 28 #include <sys/ddi.h> 29 #include <sys/sunddi.h> 30 #include <sys/sunndi.h> 31 #include <sys/avintr.h> 32 #include <sys/spl.h> 33 #include <sys/promif.h> 34 #include <sys/list.h> 35 #include <sys/bootconf.h> 36 #include <sys/bootsvcs.h> 37 #include <sys/sysmacros.h> 38 #include <sys/pci.h> 39 40 #include "virtio.h" 41 #include "virtio_impl.h" 42 43 44 45 void 46 virtio_dma_sync(virtio_dma_t *vidma, int flag) 47 { 48 VERIFY0(ddi_dma_sync(vidma->vidma_dma_handle, 0, 0, flag)); 49 } 50 51 uint_t 52 virtio_dma_ncookies(virtio_dma_t *vidma) 53 { 54 return (vidma->vidma_dma_ncookies); 55 } 56 57 size_t 58 virtio_dma_size(virtio_dma_t *vidma) 59 { 60 return (vidma->vidma_size); 61 } 62 63 void * 64 virtio_dma_va(virtio_dma_t *vidma, size_t offset) 65 { 66 VERIFY3U(offset, <, vidma->vidma_size); 67 68 return (vidma->vidma_va + offset); 69 } 70 71 uint64_t 72 virtio_dma_cookie_pa(virtio_dma_t *vidma, uint_t cookie) 73 { 74 VERIFY3U(cookie, <, vidma->vidma_dma_ncookies); 75 76 return (vidma->vidma_dma_cookies[cookie].dmac_laddress); 77 } 78 79 size_t 80 virtio_dma_cookie_size(virtio_dma_t *vidma, uint_t cookie) 81 { 82 VERIFY3U(cookie, <, vidma->vidma_dma_ncookies); 83 84 return (vidma->vidma_dma_cookies[cookie].dmac_size); 85 } 86 87 int 88 virtio_dma_init_handle(virtio_t *vio, virtio_dma_t *vidma, 89 const ddi_dma_attr_t *attr, int kmflags) 90 { 91 int r; 92 dev_info_t *dip = vio->vio_dip; 93 94 VERIFY(kmflags == KM_SLEEP || kmflags == KM_NOSLEEP); 95 int (*dma_wait)(caddr_t) = (kmflags == KM_SLEEP) ? DDI_DMA_SLEEP : 96 DDI_DMA_DONTWAIT; 97 98 vidma->vidma_virtio = vio; 99 100 /* 101 * Ensure we don't try to allocate a second time using the same 102 * tracking object. 103 */ 104 VERIFY0(vidma->vidma_level); 105 106 if ((r = ddi_dma_alloc_handle(dip, (ddi_dma_attr_t *)attr, dma_wait, 107 NULL, &vidma->vidma_dma_handle)) != DDI_SUCCESS) { 108 dev_err(dip, CE_WARN, "DMA handle allocation failed (%x)", r); 109 goto fail; 110 } 111 vidma->vidma_level |= VIRTIO_DMALEVEL_HANDLE_ALLOC; 112 113 return (DDI_SUCCESS); 114 115 fail: 116 virtio_dma_fini(vidma); 117 return (DDI_FAILURE); 118 } 119 120 int 121 virtio_dma_init(virtio_t *vio, virtio_dma_t *vidma, size_t sz, 122 const ddi_dma_attr_t *attr, int dmaflags, int kmflags) 123 { 124 int r; 125 dev_info_t *dip = vio->vio_dip; 126 caddr_t va = NULL; 127 128 VERIFY(kmflags == KM_SLEEP || kmflags == KM_NOSLEEP); 129 int (*dma_wait)(caddr_t) = (kmflags == KM_SLEEP) ? DDI_DMA_SLEEP : 130 DDI_DMA_DONTWAIT; 131 132 if (virtio_dma_init_handle(vio, vidma, attr, kmflags) != 133 DDI_SUCCESS) { 134 goto fail; 135 } 136 137 if ((r = ddi_dma_mem_alloc(vidma->vidma_dma_handle, sz, 138 &virtio_acc_attr, 139 dmaflags & (DDI_DMA_STREAMING | DDI_DMA_CONSISTENT), 140 dma_wait, NULL, &va, &vidma->vidma_real_size, 141 &vidma->vidma_acc_handle)) != DDI_SUCCESS) { 142 dev_err(dip, CE_WARN, "DMA memory allocation failed (%x)", r); 143 goto fail; 144 } 145 vidma->vidma_level |= VIRTIO_DMALEVEL_MEMORY_ALLOC; 146 147 /* 148 * Zero the memory to avoid accidental exposure of arbitrary kernel 149 * memory. 150 */ 151 bzero(va, vidma->vidma_real_size); 152 153 if (virtio_dma_bind(vidma, va, sz, dmaflags, kmflags) != DDI_SUCCESS) { 154 goto fail; 155 } 156 157 return (DDI_SUCCESS); 158 159 fail: 160 virtio_dma_fini(vidma); 161 return (DDI_FAILURE); 162 } 163 164 int 165 virtio_dma_bind(virtio_dma_t *vidma, void *va, size_t sz, int dmaflags, 166 int kmflags) 167 { 168 int r; 169 dev_info_t *dip = vidma->vidma_virtio->vio_dip; 170 ddi_dma_cookie_t dmac; 171 172 VERIFY(kmflags == KM_SLEEP || kmflags == KM_NOSLEEP); 173 int (*dma_wait)(caddr_t) = (kmflags == KM_SLEEP) ? DDI_DMA_SLEEP : 174 DDI_DMA_DONTWAIT; 175 176 VERIFY(vidma->vidma_level & VIRTIO_DMALEVEL_HANDLE_ALLOC); 177 VERIFY(!(vidma->vidma_level & VIRTIO_DMALEVEL_HANDLE_BOUND)); 178 179 vidma->vidma_va = va; 180 vidma->vidma_size = sz; 181 182 if ((r = ddi_dma_addr_bind_handle(vidma->vidma_dma_handle, NULL, 183 vidma->vidma_va, vidma->vidma_size, dmaflags, dma_wait, NULL, 184 &dmac, &vidma->vidma_dma_ncookies)) != DDI_DMA_MAPPED) { 185 VERIFY3S(r, !=, DDI_DMA_PARTIAL_MAP); 186 dev_err(dip, CE_WARN, "DMA handle bind failed (%x)", r); 187 goto fail; 188 } 189 vidma->vidma_level |= VIRTIO_DMALEVEL_HANDLE_BOUND; 190 191 if ((vidma->vidma_dma_cookies = kmem_alloc( 192 vidma->vidma_dma_ncookies * sizeof (ddi_dma_cookie_t), 193 kmflags)) == NULL) { 194 dev_err(dip, CE_WARN, "DMA cookie array allocation failure"); 195 goto fail; 196 } 197 vidma->vidma_level |= VIRTIO_DMALEVEL_COOKIE_ARRAY; 198 199 vidma->vidma_dma_cookies[0] = dmac; 200 for (uint_t n = 1; n < vidma->vidma_dma_ncookies; n++) { 201 ddi_dma_nextcookie(vidma->vidma_dma_handle, 202 &vidma->vidma_dma_cookies[n]); 203 } 204 205 return (DDI_SUCCESS); 206 207 fail: 208 virtio_dma_unbind(vidma); 209 return (DDI_FAILURE); 210 } 211 212 virtio_dma_t * 213 virtio_dma_alloc(virtio_t *vio, size_t sz, const ddi_dma_attr_t *attr, 214 int dmaflags, int kmflags) 215 { 216 virtio_dma_t *vidma; 217 218 if ((vidma = kmem_zalloc(sizeof (*vidma), kmflags)) == NULL) { 219 return (NULL); 220 } 221 222 if (virtio_dma_init(vio, vidma, sz, attr, dmaflags, kmflags) != 223 DDI_SUCCESS) { 224 kmem_free(vidma, sizeof (*vidma)); 225 return (NULL); 226 } 227 228 return (vidma); 229 } 230 231 virtio_dma_t * 232 virtio_dma_alloc_nomem(virtio_t *vio, const ddi_dma_attr_t *attr, int kmflags) 233 { 234 virtio_dma_t *vidma; 235 236 if ((vidma = kmem_zalloc(sizeof (*vidma), kmflags)) == NULL) { 237 return (NULL); 238 } 239 240 if (virtio_dma_init_handle(vio, vidma, attr, kmflags) != DDI_SUCCESS) { 241 kmem_free(vidma, sizeof (*vidma)); 242 return (NULL); 243 } 244 245 return (vidma); 246 } 247 248 void 249 virtio_dma_fini(virtio_dma_t *vidma) 250 { 251 virtio_dma_unbind(vidma); 252 253 if (vidma->vidma_level & VIRTIO_DMALEVEL_MEMORY_ALLOC) { 254 ddi_dma_mem_free(&vidma->vidma_acc_handle); 255 256 vidma->vidma_level &= ~VIRTIO_DMALEVEL_MEMORY_ALLOC; 257 } 258 259 if (vidma->vidma_level & VIRTIO_DMALEVEL_HANDLE_ALLOC) { 260 ddi_dma_free_handle(&vidma->vidma_dma_handle); 261 262 vidma->vidma_level &= ~VIRTIO_DMALEVEL_HANDLE_ALLOC; 263 } 264 265 VERIFY0(vidma->vidma_level); 266 bzero(vidma, sizeof (*vidma)); 267 } 268 269 void 270 virtio_dma_unbind(virtio_dma_t *vidma) 271 { 272 if (vidma->vidma_level & VIRTIO_DMALEVEL_COOKIE_ARRAY) { 273 kmem_free(vidma->vidma_dma_cookies, 274 vidma->vidma_dma_ncookies * sizeof (ddi_dma_cookie_t)); 275 276 vidma->vidma_level &= ~VIRTIO_DMALEVEL_COOKIE_ARRAY; 277 } 278 279 if (vidma->vidma_level & VIRTIO_DMALEVEL_HANDLE_BOUND) { 280 VERIFY3U(ddi_dma_unbind_handle(vidma->vidma_dma_handle), ==, 281 DDI_SUCCESS); 282 283 vidma->vidma_level &= ~VIRTIO_DMALEVEL_HANDLE_BOUND; 284 } 285 286 vidma->vidma_va = 0; 287 vidma->vidma_size = 0; 288 } 289 290 void 291 virtio_dma_free(virtio_dma_t *vidma) 292 { 293 virtio_dma_fini(vidma); 294 kmem_free(vidma, sizeof (*vidma)); 295 } 296