1 /*
2 * Copyright (c) 2012, 2013, Oracle and/or its affiliates. All rights reserved.
3 */
4
5 /*
6 * Copyright (c) 2012, 2013, Intel Corporation. All rights reserved.
7 */
8
9 /*
10 * Copyright © 2010 Daniel Vetter
11 *
12 * Permission is hereby granted, free of charge, to any person obtaining a
13 * copy of this software and associated documentation files (the "Software"),
14 * to deal in the Software without restriction, including without limitation
15 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
16 * and/or sell copies of the Software, and to permit persons to whom the
17 * Software is furnished to do so, subject to the following conditions:
18 *
19 * The above copyright notice and this permission notice (including the next
20 * paragraph) shall be included in all copies or substantial portions of the
21 * Software.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
24 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
25 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
26 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
27 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
28 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
29 * IN THE SOFTWARE.
30 *
31 */
32
33 #include "drmP.h"
34 #include "drm.h"
35 #include "i915_drm.h"
36 #include "i915_drv.h"
37 #include "intel_drv.h"
38
39 typedef uint32_t gtt_pte_t;
40
41 /* PPGTT stuff */
42 #define GEN6_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0xff0))
43
44 #define GEN6_PDE_VALID (1 << 0)
45 /* gen6+ has bit 11-4 for physical addr bit 39-32 */
46 #define GEN6_PDE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr)
47
48 #define GEN6_PTE_VALID (1 << 0)
49 #define GEN6_PTE_UNCACHED (1 << 1)
50 #define HSW_PTE_UNCACHED (0)
51 #define GEN6_PTE_CACHE_LLC (2 << 1)
52 #define GEN6_PTE_CACHE_LLC_MLC (3 << 1)
53 #define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr)
54
gen6_pte_encode(struct drm_device * dev,uint64_t addr,enum i915_cache_level level)55 static gen6_gtt_pte_t gen6_pte_encode(struct drm_device *dev,
56 uint64_t addr,
57 enum i915_cache_level level)
58 {
59 gen6_gtt_pte_t pte = GEN6_PTE_VALID;
60 pte |= GEN6_PTE_ADDR_ENCODE(addr);
61
62 switch (level) {
63 case I915_CACHE_LLC_MLC:
64 pte |= GEN6_PTE_CACHE_LLC_MLC;
65 break;
66 case I915_CACHE_LLC:
67 pte |= GEN6_PTE_CACHE_LLC;
68 break;
69 case I915_CACHE_NONE:
70 pte |= GEN6_PTE_UNCACHED;
71 break;
72 default:
73 BUG();
74 }
75
76 return pte;
77 }
78
79 #define BYT_PTE_WRITEABLE (1 << 1)
80 #define BYT_PTE_SNOOPED_BY_CPU_CACHES (1 << 2)
81
byt_pte_encode(struct drm_device * dev,uint64_t addr,enum i915_cache_level level)82 static gen6_gtt_pte_t byt_pte_encode(struct drm_device *dev,
83 uint64_t addr,
84 enum i915_cache_level level)
85 {
86 gen6_gtt_pte_t pte = GEN6_PTE_VALID;
87 pte |= GEN6_PTE_ADDR_ENCODE(addr);
88
89 /* Mark the page as writeable. Other platforms don't have a
90 * setting for read-only/writable, so this matches that behavior.
91 */
92 pte |= BYT_PTE_WRITEABLE;
93
94 if (level != I915_CACHE_NONE)
95 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
96
97 return pte;
98 }
99
hsw_pte_encode(struct drm_device * dev,uint64_t addr,enum i915_cache_level level)100 static gen6_gtt_pte_t hsw_pte_encode(struct drm_device *dev,
101 uint64_t addr,
102 enum i915_cache_level level)
103 {
104 gen6_gtt_pte_t pte = GEN6_PTE_VALID;
105 pte |= GEN6_PTE_ADDR_ENCODE(addr);
106
107 if (level != I915_CACHE_NONE)
108 pte |= GEN6_PTE_CACHE_LLC;
109
110 return pte;
111 }
112
113 static ddi_dma_attr_t ppgt_dma_attr = {
114 DMA_ATTR_V0,
115 0, /* dma_attr_addr_lo */
116 0xffffffffU, /* dma_attr_addr_hi */
117 0xffffffffU, /* dma_attr_count_max */
118 4096, /* dma_attr_align */
119 0x1fffU, /* dma_attr_burstsizes */
120 1, /* dma_attr_minxfer */
121 0xffffffffU, /* dma_attr_maxxfer */
122 0xffffffffU, /* dma_attr_seg */
123 1, /* dma_attr_sgllen, variable */
124 4, /* dma_attr_granular */
125 DDI_DMA_FLAGERR, /* dma_attr_flags */
126 };
127 static ddi_device_acc_attr_t ppgt_acc_attr = {
128 DDI_DEVICE_ATTR_V0,
129 DDI_NEVERSWAP_ACC,
130 DDI_MERGING_OK_ACC,
131 DDI_FLAGERR_ACC
132 };
133
134 static int
i915_ppgtt_page_alloc(struct drm_device * dev,struct i915_hw_ppgtt * ppgtt,int pgcnt)135 i915_ppgtt_page_alloc(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt,
136 int pgcnt)
137 {
138 ddi_dma_cookie_t cookie;
139 uint_t cookie_cnt;
140 pgcnt_t real_pgcnt;
141 uint32_t paddr, cookie_end;
142 int i, n;
143
144 ppgt_dma_attr.dma_attr_sgllen = (int)pgcnt;
145
146 if (ddi_dma_alloc_handle(dev->devinfo, &ppgt_dma_attr,
147 DDI_DMA_DONTWAIT, NULL, &ppgtt->dma_hdl)) {
148 DRM_ERROR("i915_ppgtt_page_alloc: "
149 "ddi_dma_alloc_handle failed");
150 goto err1;
151 }
152 if (ddi_dma_mem_alloc(ppgtt->dma_hdl, ptob(pgcnt), &ppgt_acc_attr,
153 IOMEM_DATA_UC_WR_COMBINE, DDI_DMA_DONTWAIT, NULL,
154 &ppgtt->kaddr, &ppgtt->real_size, &ppgtt->acc_hdl)) {
155 DRM_ERROR("drm_gem_object_alloc: "
156 "ddi_dma_mem_alloc failed");
157 goto err2;
158 }
159 if (ddi_dma_addr_bind_handle(ppgtt->dma_hdl, NULL,
160 ppgtt->kaddr, ppgtt->real_size, DDI_DMA_RDWR,
161 DDI_DMA_DONTWAIT, NULL, &cookie, &cookie_cnt)
162 != DDI_DMA_MAPPED) {
163 DRM_ERROR("drm_gem_object_alloc: "
164 "ddi_dma_addr_bind_handle failed");
165 goto err3;
166 }
167
168 real_pgcnt = btopr(ppgtt->real_size);
169
170 ppgtt->pt_pages = kmem_zalloc(real_pgcnt * sizeof (pfn_t), KM_NOSLEEP);
171 if (ppgtt->pt_pages == NULL) {
172 DRM_DEBUG("pfnarray == NULL");
173 goto err4;
174 }
175
176 for (n = 0, i = 1; ; i++) {
177 for (paddr = cookie.dmac_address,
178 cookie_end = cookie.dmac_address + cookie.dmac_size;
179 paddr < cookie_end;
180 paddr += PAGESIZE) {
181 ppgtt->pt_pages[n++] = btop(paddr);
182 if (n >= real_pgcnt)
183 return (0);
184 }
185 if (i >= cookie_cnt)
186 break;
187 ddi_dma_nextcookie(ppgtt->dma_hdl, &cookie);
188 }
189
190 err4:
191 (void) ddi_dma_unbind_handle(ppgtt->dma_hdl);
192 err3:
193 ddi_dma_mem_free(&ppgtt->acc_hdl);
194 err2:
195 ddi_dma_free_handle(&ppgtt->dma_hdl);
196 err1:
197 return (-1);
198
199 }
200
gen6_write_pdes(struct i915_hw_ppgtt * ppgtt)201 static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)
202 {
203 struct drm_i915_private *dev_priv = ppgtt->dev->dev_private;
204 gen6_gtt_pte_t *pd_addr;
205 uint32_t pt_addr;
206 uint32_t pd_entry;
207 int i;
208
209 WARN_ON(ppgtt->pd_offset & 0x3f);
210 for (i = 0; i < ppgtt->num_pd_entries; i++) {
211 pd_addr = (gen6_gtt_pte_t *)(uintptr_t)((caddr_t)dev_priv->gtt.virtual_gtt
212 + ppgtt->pd_offset + i * sizeof(gtt_pte_t));
213
214
215 pt_addr = ppgtt->pt_pages[i] << PAGE_SHIFT;
216 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
217 pd_entry |= GEN6_PDE_VALID;
218
219 ddi_put32(dev_priv->gtt.gtt_mapping.acc_handle,
220 pd_addr, pd_entry);
221
222 }
223 (void) ddi_get32(dev_priv->gtt.gtt_mapping.acc_handle,
224 (gen6_gtt_pte_t *)(uintptr_t)((caddr_t)dev_priv->gtt.virtual_gtt + ppgtt->pd_offset));
225 }
226
gen6_ppgtt_enable(struct drm_device * dev)227 static int gen6_ppgtt_enable(struct drm_device *dev)
228 {
229 drm_i915_private_t *dev_priv = dev->dev_private;
230 uint32_t pd_offset;
231 struct intel_ring_buffer *ring;
232 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
233 int i;
234
235 BUG_ON(ppgtt->pd_offset & 0x3f);
236
237 gen6_write_pdes(ppgtt);
238
239 pd_offset = ppgtt->pd_offset;
240 pd_offset /= 64; /* in cachelines, */
241 pd_offset <<= 16;
242
243 if (INTEL_INFO(dev)->gen == 6) {
244 uint32_t ecochk, gab_ctl, ecobits;
245
246 ecobits = I915_READ(GAC_ECO_BITS);
247 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
248 ECOBITS_PPGTT_CACHE64B);
249
250 gab_ctl = I915_READ(GAB_CTL);
251 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
252
253 ecochk = I915_READ(GAM_ECOCHK);
254 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
255 ECOCHK_PPGTT_CACHE64B);
256 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
257 } else if (INTEL_INFO(dev)->gen >= 7) {
258 uint32_t ecochk, ecobits;
259
260 ecobits = I915_READ(GAC_ECO_BITS);
261 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
262
263 ecochk = I915_READ(GAM_ECOCHK);
264 if (IS_HASWELL(dev)) {
265 ecochk |= ECOCHK_PPGTT_WB_HSW;
266 } else {
267 ecochk |= ECOCHK_PPGTT_LLC_IVB;
268 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
269 }
270 I915_WRITE(GAM_ECOCHK, ecochk);
271 /* GFX_MODE is per-ring on gen7+ */
272 }
273
274 for_each_ring(ring, dev_priv, i) {
275 if (INTEL_INFO(dev)->gen >= 7)
276 I915_WRITE(RING_MODE_GEN7(ring),
277 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
278
279 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
280 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
281 }
282 return 0;
283 }
284
285 /* PPGTT support for Sandybdrige/Gen6 and later */
gen6_ppgtt_clear_range(struct i915_hw_ppgtt * ppgtt,unsigned first_entry,unsigned num_entries)286 static void gen6_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt,
287 unsigned first_entry,
288 unsigned num_entries)
289 {
290 gen6_gtt_pte_t *pt_vaddr, scratch_pte;
291 unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
292 unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
293 unsigned last_pte, i;
294
295 scratch_pte = ppgtt->pte_encode(ppgtt->dev,
296 ppgtt->scratch_page_paddr,
297 I915_CACHE_LLC);
298
299 while (num_entries) {
300 last_pte = first_pte + num_entries;
301 if (last_pte > I915_PPGTT_PT_ENTRIES)
302 last_pte = I915_PPGTT_PT_ENTRIES;
303
304 pt_vaddr = (uint32_t*)(uintptr_t)(ppgtt->kaddr + act_pt*PAGE_SIZE);
305
306 for (i = first_pte; i < last_pte; i++)
307 pt_vaddr[i] = scratch_pte;
308
309 num_entries -= last_pte - first_pte;
310 first_pte = 0;
311 act_pt++;
312 }
313 }
314
gen6_ppgtt_insert_entries(struct i915_hw_ppgtt * ppgtt,unsigned first_entry,unsigned num_entries,pfn_t * pages,enum i915_cache_level cache_level)315 static void gen6_ppgtt_insert_entries(struct i915_hw_ppgtt *ppgtt,
316 unsigned first_entry, unsigned num_entries,
317 pfn_t *pages, enum i915_cache_level cache_level)
318 {
319 gen6_gtt_pte_t *pt_vaddr;
320 unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
321 unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES;
322 unsigned i, j;
323
324 pt_vaddr = (gen6_gtt_pte_t *)(uintptr_t)(ppgtt->kaddr + act_pt*PAGE_SIZE);
325 for (i = first_entry, j = 0; i < ( first_entry + num_entries); i++, j++) {
326
327 pt_vaddr[act_pte] = ppgtt->pte_encode(ppgtt->dev, pages[j] << PAGE_SHIFT,
328 cache_level);
329 if (++act_pte == I915_PPGTT_PT_ENTRIES) {
330 act_pt++;
331 pt_vaddr = (gen6_gtt_pte_t *)(uintptr_t)(ppgtt->kaddr + act_pt*PAGE_SIZE);
332 act_pte = 0;
333
334 }
335 }
336 }
337
gen6_ppgtt_cleanup(struct i915_hw_ppgtt * ppgtt)338 static void gen6_ppgtt_cleanup(struct i915_hw_ppgtt *ppgtt)
339 {
340 kmem_free(ppgtt->pt_pages, btopr(ppgtt->real_size) * sizeof (pfn_t));
341
342 (void) ddi_dma_unbind_handle(ppgtt->dma_hdl);
343 ddi_dma_mem_free(&ppgtt->acc_hdl);
344 ddi_dma_free_handle(&ppgtt->dma_hdl);
345 kfree(ppgtt, sizeof(*ppgtt));
346 }
347
gen6_ppgtt_init(struct i915_hw_ppgtt * ppgtt)348 static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
349 {
350 struct drm_device *dev = ppgtt->dev;
351 struct drm_i915_private *dev_priv = dev->dev_private;
352 unsigned first_pd_entry_in_global_pt;
353 int ret = -ENOMEM;
354
355 /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
356 * entries. For aliasing ppgtt support we just steal them at the end for
357 * now. */
358 first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt);
359
360 if (IS_HASWELL(dev)) {
361 ppgtt->pte_encode = hsw_pte_encode;
362 } else if (IS_VALLEYVIEW(dev)) {
363 ppgtt->pte_encode = byt_pte_encode;
364 } else {
365 ppgtt->pte_encode = gen6_pte_encode;
366 }
367 ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES;
368 ppgtt->enable = gen6_ppgtt_enable;
369 ppgtt->clear_range = gen6_ppgtt_clear_range;
370 ppgtt->insert_entries = gen6_ppgtt_insert_entries;
371 ppgtt->cleanup = gen6_ppgtt_cleanup;
372 ret = i915_ppgtt_page_alloc(dev, ppgtt, ppgtt->num_pd_entries);
373 if (ret)
374 return (-ENOMEM);
375
376 ppgtt->clear_range(ppgtt, 0,
377 ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES);
378
379 ppgtt->pd_offset = first_pd_entry_in_global_pt * sizeof(gen6_gtt_pte_t);
380
381 return 0;
382
383 }
384
i915_gem_init_aliasing_ppgtt(struct drm_device * dev)385 static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
386 {
387 struct drm_i915_private *dev_priv = dev->dev_private;
388 struct i915_hw_ppgtt *ppgtt;
389 int ret = -EINVAL;
390
391 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
392 if (!ppgtt)
393 return -ENOMEM;
394
395 ppgtt->dev = dev;
396 ppgtt->scratch_page_paddr = ptob(dev_priv->gtt.scratch_page->pfnarray[0]);
397
398 if (INTEL_INFO(dev)->gen < 8)
399 ret = gen6_ppgtt_init(ppgtt);
400 else {
401 BUG();
402 DRM_ERROR("ppgtt is not supported");
403 }
404
405 if (ret)
406 kfree(ppgtt, sizeof(*ppgtt));
407 else
408 dev_priv->mm.aliasing_ppgtt = ppgtt;
409
410 return ret;
411 }
412
i915_gem_cleanup_aliasing_ppgtt(struct drm_device * dev)413 void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
414 {
415 struct drm_i915_private *dev_priv = dev->dev_private;
416 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
417
418 if (!ppgtt)
419 return;
420
421 ppgtt->cleanup(ppgtt);
422 dev_priv->mm.aliasing_ppgtt = NULL;
423 }
424
i915_ppgtt_bind_object(struct i915_hw_ppgtt * ppgtt,struct drm_i915_gem_object * obj,enum i915_cache_level cache_level)425 void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
426 struct drm_i915_gem_object *obj,
427 enum i915_cache_level cache_level)
428 {
429
430 ppgtt->insert_entries(ppgtt,
431 obj->gtt_space->start >> PAGE_SHIFT,
432 obj->base.size >> PAGE_SHIFT,
433 obj->base.pfnarray,
434 cache_level);
435 }
436
i915_ppgtt_unbind_object(struct i915_hw_ppgtt * ppgtt,struct drm_i915_gem_object * obj)437 void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
438 struct drm_i915_gem_object *obj)
439 {
440 ppgtt->clear_range(ppgtt,
441 obj->gtt_space->start >> PAGE_SHIFT,
442 obj->base.size >> PAGE_SHIFT);
443 }
444
i915_gem_restore_gtt_mappings(struct drm_device * dev)445 void i915_gem_restore_gtt_mappings(struct drm_device *dev)
446 {
447 struct drm_i915_private *dev_priv = dev->dev_private;
448 struct drm_i915_gem_object *obj;
449
450 /* First fill our portion of the GTT with scratch pages */
451 /*
452 i915_ggtt_clear_range(dev_priv->mm.gtt_start / PAGE_SIZE,
453 (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE);
454 */
455
456 list_for_each_entry(obj, struct drm_i915_gem_object, &dev_priv->mm.bound_list, global_list) {
457 i915_gem_clflush_object(obj);
458 i915_gem_gtt_bind_object(obj, obj->cache_level);
459 }
460
461 i915_gem_chipset_flush(dev);
462 }
463
i915_gem_gtt_prepare_object(struct drm_i915_gem_object * obj)464 int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
465 {
466 #if 0
467 if (obj->has_dma_mapping)
468 return 0;
469
470 if (!dma_map_sg(&obj->base.dev->pdev->dev,
471 obj->pages->sgl, obj->pages->nents,
472 PCI_DMA_BIDIRECTIONAL))
473 return -ENOSPC;
474
475 #endif
476 return 0;
477 }
478
479 /*
480 * Binds an object into the global gtt with the specified cache level. The object
481 * will be accessible to the GPU via commands whose operands reference offsets
482 * within the global GTT as well as accessible by the GPU through the GMADR
483 * mapped BAR (dev_priv->mm.gtt->gtt).
484 */
gen6_ggtt_insert_entries(struct drm_i915_gem_object * obj,enum i915_cache_level level)485 static void gen6_ggtt_insert_entries(struct drm_i915_gem_object *obj,
486 enum i915_cache_level level)
487 {
488 struct drm_device *dev = obj->base.dev;
489 struct drm_i915_private *dev_priv = dev->dev_private;
490 unsigned first_entry = obj->gtt_offset >> PAGE_SHIFT;
491 unsigned num_entries = obj->base.size / PAGE_SIZE;
492 /* LINTED */
493 const int max_entries = gtt_total_entries(dev_priv->gtt);
494 uint64_t page_addr;
495 gtt_pte_t *gtt_addr;
496 int i, j;
497
498 for (i = first_entry, j = 0; i < ( first_entry + num_entries); i++, j++) {
499 gtt_addr = (gtt_pte_t *)(uintptr_t)((caddr_t)dev_priv->gtt.virtual_gtt + i * sizeof(gtt_pte_t));
500 page_addr = obj->base.pfnarray[j] << PAGE_SHIFT;
501 ddi_put32(dev_priv->gtt.gtt_mapping.acc_handle,
502 gtt_addr,
503 dev_priv->gtt.pte_encode(dev, page_addr, level));
504 }
505
506 BUG_ON(i > max_entries);
507 BUG_ON(j != obj->base.size / PAGE_SIZE);
508
509 /* XXX: This serves as a posting read to make sure that the PTE has
510 * actually been updated. There is some concern that even though
511 * registers and PTEs are within the same BAR that they are potentially
512 * of NUMA access patterns. Therefore, even with the way we assume
513 * hardware should work, we must keep this posting read for paranoia.
514 */
515 if (i != 0) {
516 gtt_addr = (gtt_pte_t *)(uintptr_t)((caddr_t)dev_priv->gtt.virtual_gtt + (i-1)*sizeof(gtt_pte_t));
517 page_addr = obj->base.pfnarray[j-1] << PAGE_SHIFT;
518 WARN_ON(ddi_get32(dev_priv->gtt.gtt_mapping.acc_handle, gtt_addr) !=
519 dev_priv->gtt.pte_encode(dev, page_addr, level));
520 }
521
522 /* This next bit makes the above posting read even more important. We
523 * want to flush the TLBs only after we're certain all the PTE updates
524 * have finished.
525 */
526 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
527 POSTING_READ(GFX_FLSH_CNTL_GEN6);
528 }
529
gen6_ggtt_clear_range(struct drm_device * dev,struct drm_i915_gem_object * obj,uint32_t type)530 static void gen6_ggtt_clear_range(struct drm_device *dev,
531 struct drm_i915_gem_object *obj,
532 uint32_t type)
533 {
534 struct drm_i915_private *dev_priv = dev->dev_private;
535 gtt_pte_t scratch_pte;
536 unsigned first_entry = obj->gtt_offset >> PAGE_SHIFT;
537 unsigned num_entries = obj->base.size / PAGE_SIZE;
538 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
539 uint64_t scratch_page_addr = dev_priv->gtt.scratch_page->pfnarray[0] << PAGE_SHIFT;
540 gtt_pte_t *gtt_addr;
541 int i;
542
543 if (num_entries > max_entries) {
544 DRM_ERROR("First entry = %d; Num entries = %d (max=%d)\n",
545 first_entry, num_entries, max_entries);
546 num_entries = max_entries;
547 }
548
549 for (i = first_entry ; i < ( first_entry + num_entries); i++) {
550 gtt_addr = (gtt_pte_t *)(uintptr_t)((caddr_t)dev_priv->gtt.virtual_gtt + i * sizeof(gtt_pte_t));
551 scratch_pte = dev_priv->gtt.pte_encode(dev, scratch_page_addr, I915_CACHE_LLC);
552 ddi_put32(dev_priv->gtt.gtt_mapping.acc_handle,
553 gtt_addr, scratch_pte);
554 }
555 }
556
i915_ggtt_insert_entries(struct drm_i915_gem_object * obj,enum i915_cache_level cache_level)557 void i915_ggtt_insert_entries(struct drm_i915_gem_object *obj,
558 enum i915_cache_level cache_level)
559 {
560 struct drm_device *dev = obj->base.dev;
561
562 unsigned int agp_type = (cache_level == I915_CACHE_NONE) ?
563 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
564
565 (void) drm_agp_bind_pages(dev,
566 obj->base.pfnarray,
567 obj->base.size >> PAGE_SHIFT,
568 obj->gtt_offset,
569 agp_type);
570 }
571
i915_ggtt_clear_range(struct drm_device * dev,struct drm_i915_gem_object * obj,uint32_t type)572 static void i915_ggtt_clear_range(struct drm_device *dev,
573 struct drm_i915_gem_object *obj,
574 uint32_t type)
575 {
576 struct drm_i915_private *dev_priv = dev->dev_private;
577
578 if (!obj->agp_mem) {
579 (void) drm_agp_unbind_pages(dev, obj->base.pfnarray,
580 (obj->base.size / PAGE_SIZE), obj->gtt_offset,
581 dev_priv->gtt.scratch_page->pfnarray[0], type);
582 obj->agp_mem = -1;
583 }
584 return;
585
586 }
587
i915_gem_gtt_bind_object(struct drm_i915_gem_object * obj,enum i915_cache_level cache_level)588 void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
589 enum i915_cache_level cache_level)
590 {
591 struct drm_device *dev = obj->base.dev;
592 struct drm_i915_private *dev_priv = dev->dev_private;
593
594 dev_priv->gtt.gtt_insert_entries(obj, cache_level);
595 obj->has_global_gtt_mapping = 1;
596 }
597
i915_gem_gtt_unbind_object(struct drm_i915_gem_object * obj,uint32_t type)598 void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj, uint32_t type)
599 {
600 struct drm_device *dev = obj->base.dev;
601 struct drm_i915_private *dev_priv = dev->dev_private;
602
603 dev_priv->gtt.gtt_clear_range(dev, obj, type);
604
605 obj->has_global_gtt_mapping = 0;
606 }
607
i915_gem_gtt_finish_object(struct drm_i915_gem_object * obj)608 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
609 {
610 }
611
i915_gtt_color_adjust(struct drm_mm_node * node,unsigned long color,unsigned long * start,unsigned long * end)612 static void i915_gtt_color_adjust(struct drm_mm_node *node,
613 unsigned long color,
614 unsigned long *start,
615 unsigned long *end)
616 {
617 if (node->color != color)
618 *start += 4096;
619
620 if (!list_empty(&node->node_list)) {
621 node = list_entry(node->node_list.next,
622 struct drm_mm_node,
623 node_list);
624 if (node->allocated && node->color != color)
625 *end -= 4096;
626 }
627 }
628
i915_gem_setup_global_gtt(struct drm_device * dev,unsigned long start,unsigned long mappable_end,unsigned long end)629 void i915_gem_setup_global_gtt(struct drm_device *dev,
630 unsigned long start,
631 unsigned long mappable_end,
632 unsigned long end)
633 {
634 /* Let GEM Manage all of the aperture.
635 *
636 * However, leave one page at the end still bound to the scratch page.
637 * There are a number of places where the hardware apparently prefetches
638 * past the end of the object, and we've seen multiple hangs with the
639 * GPU head pointer stuck in a batchbuffer bound at the last page of the
640 * aperture. One page should be enough to keep any prefetching inside
641 * of the aperture.
642 */
643 drm_i915_private_t *dev_priv = dev->dev_private;
644 struct drm_i915_gem_object *obj;
645
646 /* Substract the guard page ... */
647 drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE);
648 if (!HAS_LLC(dev))
649 dev_priv->mm.gtt_space.color_adjust = i915_gtt_color_adjust;
650
651 /* Mark any preallocated objects as occupied */
652 list_for_each_entry(obj, struct drm_i915_gem_object,
653 &dev_priv->mm.bound_list, global_list) {
654 DRM_DEBUG_KMS("reserving preallocated space: %x + %zx\n",
655 obj->gtt_offset, obj->base.size);
656
657 BUG_ON(obj->gtt_space != I915_GTT_RESERVED);
658 obj->gtt_space = drm_mm_create_block(&dev_priv->mm.gtt_space,
659 obj->gtt_offset,
660 obj->base.size,
661 false);
662 obj->has_global_gtt_mapping = 1;
663 }
664
665 dev_priv->gtt.start = start;
666 dev_priv->gtt.total = end - start;
667 }
668
669 static bool
intel_enable_ppgtt(struct drm_device * dev)670 intel_enable_ppgtt(struct drm_device *dev)
671 {
672 if (i915_enable_ppgtt >= 0)
673 return i915_enable_ppgtt;
674
675 /* Disable ppgtt on SNB if VT-d is on. */
676 if (INTEL_INFO(dev)->gen == 6)
677 return false;
678
679 return true;
680 }
681
i915_gem_init_global_gtt(struct drm_device * dev)682 int i915_gem_init_global_gtt(struct drm_device *dev)
683 {
684 struct drm_i915_private *dev_priv = dev->dev_private;
685 unsigned long gtt_size, mappable_size;
686 int ret;
687
688 gtt_size = dev_priv->gtt.total;
689 mappable_size = dev_priv->gtt.mappable_end;
690
691 if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
692
693 if (INTEL_INFO(dev)->gen <= 7) {
694 /* PPGTT pdes are stolen from global gtt ptes, so shrink the
695 * aperture accordingly when using aliasing ppgtt. */
696 gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
697 }
698 }
699 i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
700
701 ret = i915_setup_scratch_page(dev);
702 if (ret)
703 return ret;
704
705 if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
706 ret = i915_gem_init_aliasing_ppgtt(dev);
707 if (!ret)
708 return 0;
709 DRM_ERROR("Aliased PPGTT setup failed %d\n", ret);
710 drm_mm_takedown(&dev_priv->mm.gtt_space);
711 gtt_size += I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
712 }
713 return 0;
714 }
715
716 /*
717 * Just to be safe against memory overwrite, let's allocate this with
718 * sizeof drm_i915_gem_object, even though we don't use the i915 part.
719 * This way, all i915 GEM objects are the same size.
720 */
721 int
i915_setup_scratch_page(struct drm_device * dev)722 i915_setup_scratch_page(struct drm_device *dev)
723 {
724 struct drm_i915_private *dev_priv = dev->dev_private;
725 int gen;
726
727 /* setup scratch page */
728 dev_priv->gtt.scratch_page =
729 kzalloc(sizeof(struct drm_i915_gem_object), GFP_KERNEL);
730 if (dev_priv->gtt.scratch_page == NULL) {
731 DRM_ERROR("setup_scratch_page: gem object init failed");
732 return (-ENOMEM);
733 }
734
735 if (IS_G33(dev))
736 gen = 33;
737 else
738 gen = INTEL_INFO(dev)->gen * 10;
739
740 if (drm_gem_object_init(dev, dev_priv->gtt.scratch_page, DRM_PAGE_SIZE, gen) != 0) {
741 kmem_free(dev_priv->gtt.scratch_page,
742 sizeof (struct drm_i915_gem_object));
743 dev_priv->gtt.scratch_page = NULL;
744 DRM_ERROR("setup_scratch_page: gem object init failed");
745 return (-ENOMEM);
746 }
747 (void) memset(dev_priv->gtt.scratch_page->kaddr, 0, DRM_PAGE_SIZE);
748
749 return 0;
750 }
751
752 void
i915_teardown_scratch_page(struct drm_device * dev)753 i915_teardown_scratch_page(struct drm_device *dev)
754 {
755 struct drm_i915_private *dev_priv = dev->dev_private;
756
757 if (dev_priv->gtt.scratch_page == NULL)
758 return;
759
760 drm_gem_object_release(dev_priv->gtt.scratch_page);
761 kmem_free(dev_priv->gtt.scratch_page,
762 sizeof (struct drm_i915_gem_object));
763 dev_priv->gtt.scratch_page = NULL;
764 }
765
gen6_get_total_gtt_size(u16 snb_gmch_ctl)766 static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
767 {
768 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
769 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
770 return snb_gmch_ctl << 20;
771 }
772
gen6_get_stolen_size(u16 snb_gmch_ctl)773 static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
774 {
775 snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
776 snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
777 return snb_gmch_ctl << 25; /* 32 MB units */
778 }
779
780 static ddi_device_acc_attr_t gtt_attr = {
781 DDI_DEVICE_ATTR_V0,
782 DDI_NEVERSWAP_ACC,
783 DDI_STRICTORDER_ACC,
784 };
785
786 #define GEN6_GTTMMADR 1
787 #define GEN6_GTT_OFFSET 0x200000
788
i915_gen6_gtt_ioremap(struct drm_local_map * map,struct drm_device * dev)789 static void i915_gen6_gtt_ioremap(struct drm_local_map *map, struct drm_device *dev)
790 {
791 caddr_t base;
792 int ret;
793
794 ret = ddi_regs_map_setup(dev->devinfo, GEN6_GTTMMADR,
795 (caddr_t *)&base, 0, 0,
796 >t_attr, &map->acc_handle);
797
798 if (ret != DDI_SUCCESS) {
799 DRM_ERROR("failed to map GTT");
800 }
801
802 map->handle = (void *)(base + GEN6_GTT_OFFSET);
803 }
804
gen6_gmch_probe(struct drm_device * dev,size_t * gtt_total,size_t * stolen)805 static int gen6_gmch_probe(struct drm_device *dev,
806 size_t *gtt_total,
807 size_t *stolen)
808 {
809 struct drm_i915_private *dev_priv = dev->dev_private;
810 unsigned int gtt_size;
811 u16 snb_gmch_ctl;
812
813 dev_priv->gtt.mappable_end = pci_resource_len(dev->pdev, 1);
814
815 /* 64/512MB is the current min/max we actually know of, but this is just
816 * a coarse sanity check.
817 */
818 if ((dev_priv->gtt.mappable_end < (64<<20) ||
819 (dev_priv->gtt.mappable_end > (512<<20)))) {
820 DRM_ERROR("Unknown GMADR size (%lx)\n",
821 dev_priv->gtt.mappable_end);
822 return -ENXIO;
823 }
824
825 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
826 gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
827
828 *stolen = gen6_get_stolen_size(snb_gmch_ctl);
829
830 *gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT;
831
832 /* For GEN6+ the PTEs for the ggtt live at 2MB + BAR0 */
833 dev_priv->gtt.gtt_mapping.offset = dev->agp_aperbase;
834 dev_priv->gtt.gtt_mapping.size = gtt_size;
835 dev_priv->gtt.gtt_mapping.type = 0;
836 dev_priv->gtt.gtt_mapping.flags = 0;
837 dev_priv->gtt.gtt_mapping.mtrr = 0;
838
839 i915_gen6_gtt_ioremap(&dev_priv->gtt.gtt_mapping, dev);
840
841 if (dev_priv->gtt.gtt_mapping.handle == NULL) {
842 DRM_ERROR("Failed to map the gtt page table");
843 return -ENOMEM;
844 }
845 dev_priv->gtt.virtual_gtt = (caddr_t)dev_priv->gtt.gtt_mapping.handle;
846
847 dev_priv->gtt.gtt_clear_range = gen6_ggtt_clear_range;
848 dev_priv->gtt.gtt_insert_entries = gen6_ggtt_insert_entries;
849
850 return 0;
851 }
852
gen6_gmch_remove(struct drm_device * dev)853 static void gen6_gmch_remove(struct drm_device *dev)
854 {
855 struct drm_i915_private *dev_priv = dev->dev_private;
856 drm_core_ioremapfree(&dev_priv->gtt.gtt_mapping, dev);
857 }
858
859 static unsigned int
intel_gtt_stolen_size(struct drm_device * dev)860 intel_gtt_stolen_size(struct drm_device *dev)
861 {
862 u16 gmch_ctrl;
863 u8 rdct;
864 int local = 0;
865 static const int ddt[4] = { 0, 16, 32, 64 };
866 unsigned int stolen_size = 0;
867 drm_i915_private_t *dev_priv = dev->dev_private;
868
869 if ( INTEL_INFO(dev)->gen == 1)
870 return 0; /* no stolen mem on i81x */
871
872 if (i915_bridge_dev_read_config_word(&dev_priv->bridge_dev, INTEL_GMCH_CTRL, &gmch_ctrl))
873 return 0;
874
875 if (dev->pdev->device == PCI_DEVICE_ID_INTEL_82830_HB ||
876 dev->pdev->device == PCI_DEVICE_ID_INTEL_82845G_HB) {
877 switch (gmch_ctrl & I830_GMCH_GMS_MASK) {
878 case I830_GMCH_GMS_STOLEN_512:
879 stolen_size = KB(512);
880 break;
881 case I830_GMCH_GMS_STOLEN_1024:
882 stolen_size = MB(1);
883 break;
884 case I830_GMCH_GMS_STOLEN_8192:
885 stolen_size = MB(8);
886 break;
887 case I830_GMCH_GMS_LOCAL:
888 rdct = I915_READ8(I830_RDRAM_CHANNEL_TYPE);
889 stolen_size = (I830_RDRAM_ND(rdct) + 1) *
890 MB(ddt[I830_RDRAM_DDT(rdct)]);
891 local = 1;
892 break;
893 default:
894 stolen_size = 0;
895 break;
896 }
897 } else {
898 switch (gmch_ctrl & INTEL_GMCH_GMS_MASK) {
899 case INTEL_855_GMCH_GMS_STOLEN_1M:
900 stolen_size = MB(1);
901 break;
902 case INTEL_855_GMCH_GMS_STOLEN_4M:
903 stolen_size = MB(4);
904 break;
905 case INTEL_855_GMCH_GMS_STOLEN_8M:
906 stolen_size = MB(8);
907 break;
908 case INTEL_855_GMCH_GMS_STOLEN_16M:
909 stolen_size = MB(16);
910 break;
911 case INTEL_855_GMCH_GMS_STOLEN_32M:
912 stolen_size = MB(32);
913 break;
914 case INTEL_915G_GMCH_GMS_STOLEN_48M:
915 stolen_size = MB(48);
916 break;
917 case INTEL_915G_GMCH_GMS_STOLEN_64M:
918 stolen_size = MB(64);
919 break;
920 case INTEL_GMCH_GMS_STOLEN_128M:
921 stolen_size = MB(128);
922 break;
923 case INTEL_GMCH_GMS_STOLEN_256M:
924 stolen_size = MB(256);
925 break;
926 case INTEL_GMCH_GMS_STOLEN_96M:
927 stolen_size = MB(96);
928 break;
929 case INTEL_GMCH_GMS_STOLEN_160M:
930 stolen_size = MB(160);
931 break;
932 case INTEL_GMCH_GMS_STOLEN_224M:
933 stolen_size = MB(224);
934 break;
935 case INTEL_GMCH_GMS_STOLEN_352M:
936 stolen_size = MB(352);
937 break;
938 default:
939 stolen_size = 0;
940 break;
941 }
942 }
943
944 if (stolen_size > 0) {
945 DRM_INFO("detected %dK %s memory\n",
946 stolen_size / KB(1), local ? "local" : "stolen");
947 } else {
948 DRM_INFO("no pre-allocated video memory detected\n");
949 stolen_size = 0;
950 }
951
952 return stolen_size;
953 }
954
955 static unsigned int
intel_gtt_mappable_entries(struct drm_device * dev)956 intel_gtt_mappable_entries(struct drm_device *dev)
957 {
958 unsigned int aperture_size;
959 drm_i915_private_t *dev_priv = dev->dev_private;
960
961 if (INTEL_INFO(dev)->gen == 1) {
962 u32 smram_miscc;
963
964 pci_read_config_dword(dev->pdev,
965 I810_SMRAM_MISCC, &smram_miscc);
966
967 if ((smram_miscc & I810_GFX_MEM_WIN_SIZE)
968 == I810_GFX_MEM_WIN_32M)
969 aperture_size = MB(32);
970 else
971 aperture_size = MB(64);
972 } else if (INTEL_INFO(dev)->gen == 2) {
973 u16 gmch_ctrl;
974
975 if (i915_bridge_dev_read_config_word(&dev_priv->bridge_dev, INTEL_GMCH_CTRL, &gmch_ctrl))
976 return 0;
977
978 if ((gmch_ctrl & INTEL_GMCH_MEM_MASK) == INTEL_GMCH_MEM_64M)
979 aperture_size = MB(64);
980 else
981 aperture_size = MB(128);
982 } else {
983 /* 9xx supports large sizes, just look at the length */
984 aperture_size = pci_resource_len(dev->pdev,
985 (IS_G33(dev) || IS_I945GM(dev)) ? 2 : 1); /* OSOL_i915 */
986 DRM_DEBUG("aperture_size=%d", aperture_size);
987 }
988
989 return aperture_size >> PAGE_SHIFT;
990 }
991
992 static void
i965_adjust_pgetbl_size(struct drm_device * dev,unsigned int size_flag)993 i965_adjust_pgetbl_size(struct drm_device *dev, unsigned int size_flag)
994 {
995 u32 pgetbl_ctl, pgetbl_ctl2;
996 drm_i915_private_t *dev_priv = dev->dev_private;
997
998 /* ensure that ppgtt is disabled */
999 pgetbl_ctl2 = I915_READ(I965_PGETBL_CTL2);
1000 pgetbl_ctl2 &= ~I810_PGETBL_ENABLED;
1001 I915_WRITE(pgetbl_ctl2, I965_PGETBL_CTL2);
1002
1003 /* write the new ggtt size */
1004 pgetbl_ctl = I915_READ(I810_PGETBL_CTL);
1005 pgetbl_ctl &= ~I965_PGETBL_SIZE_MASK;
1006 pgetbl_ctl |= size_flag;
1007 I915_WRITE(pgetbl_ctl, I810_PGETBL_CTL);
1008 }
1009
1010 static unsigned int
i965_gtt_total_entries(struct drm_device * dev)1011 i965_gtt_total_entries(struct drm_device *dev)
1012 {
1013 int size;
1014 u32 pgetbl_ctl;
1015 u16 gmch_ctl;
1016 drm_i915_private_t *dev_priv = dev->dev_private;
1017
1018 if (i915_bridge_dev_read_config_word(&dev_priv->bridge_dev, INTEL_GMCH_CTRL, &gmch_ctl))
1019 return 0;
1020
1021 if (INTEL_INFO(dev)->gen == 5) {
1022 switch (gmch_ctl & G4x_GMCH_SIZE_MASK) {
1023 case G4x_GMCH_SIZE_1M:
1024 case G4x_GMCH_SIZE_VT_1M:
1025 i965_adjust_pgetbl_size(dev, I965_PGETBL_SIZE_1MB);
1026 break;
1027 case G4x_GMCH_SIZE_VT_1_5M:
1028 i965_adjust_pgetbl_size(dev, I965_PGETBL_SIZE_1_5MB);
1029 break;
1030 case G4x_GMCH_SIZE_2M:
1031 case G4x_GMCH_SIZE_VT_2M:
1032 i965_adjust_pgetbl_size(dev, I965_PGETBL_SIZE_2MB);
1033 break;
1034 }
1035 }
1036
1037 pgetbl_ctl = I915_READ(I810_PGETBL_CTL);
1038
1039 switch (pgetbl_ctl & I965_PGETBL_SIZE_MASK) {
1040 case I965_PGETBL_SIZE_128KB:
1041 size = KB(128);
1042 break;
1043 case I965_PGETBL_SIZE_256KB:
1044 size = KB(256);
1045 break;
1046 case I965_PGETBL_SIZE_512KB:
1047 size = KB(512);
1048 break;
1049 /* GTT pagetable sizes bigger than 512KB are not possible on G33! */
1050 case I965_PGETBL_SIZE_1MB:
1051 size = KB(1024);
1052 break;
1053 case I965_PGETBL_SIZE_2MB:
1054 size = KB(2048);
1055 break;
1056 case I965_PGETBL_SIZE_1_5MB:
1057 size = KB(1024 + 512);
1058 break;
1059 default:
1060 DRM_DEBUG("unknown page table size, assuming 512KB");
1061 size = KB(512);
1062 }
1063
1064 return size/4;
1065 }
1066
1067 static unsigned int
intel_gtt_total_entries(struct drm_device * dev)1068 intel_gtt_total_entries(struct drm_device *dev)
1069 {
1070 drm_i915_private_t *dev_priv = dev->dev_private;
1071
1072 if (IS_G33(dev) || INTEL_INFO(dev)->gen == 4 || INTEL_INFO(dev)->gen == 5)
1073 return i965_gtt_total_entries(dev);
1074 else {
1075 /* On previous hardware, the GTT size was just what was
1076 * required to map the aperture.
1077 */
1078 return (dev_priv->gtt.mappable_end >> PAGE_SHIFT);
1079 }
1080 }
1081
i915_gmch_probe(struct drm_device * dev,size_t * gtt_total,size_t * stolen)1082 static int i915_gmch_probe(struct drm_device *dev,
1083 size_t *gtt_total,
1084 size_t *stolen)
1085 {
1086 struct drm_i915_private *dev_priv = dev->dev_private;
1087
1088 dev_priv->gtt.mappable_end = intel_gtt_mappable_entries(dev) << PAGE_SHIFT;
1089 *gtt_total = intel_gtt_total_entries(dev) << PAGE_SHIFT;
1090 *stolen = intel_gtt_stolen_size(dev);
1091
1092 dev_priv->gtt.gtt_clear_range = i915_ggtt_clear_range;
1093 dev_priv->gtt.gtt_insert_entries = i915_ggtt_insert_entries;
1094
1095 return 0;
1096 }
1097
i915_gmch_remove(struct drm_device * dev)1098 static void i915_gmch_remove(struct drm_device *dev)
1099 {
1100
1101 }
1102
i915_gem_gtt_init(struct drm_device * dev)1103 int i915_gem_gtt_init(struct drm_device *dev)
1104 {
1105 struct drm_i915_private *dev_priv = dev->dev_private;
1106 struct i915_gtt *gtt = &dev_priv->gtt;
1107 int ret;
1108
1109 gtt->mappable_base = dev->agp_aperbase;
1110
1111 if (INTEL_INFO(dev)->gen <= 5) {
1112 dev_priv->gtt.gtt_probe = i915_gmch_probe;
1113 dev_priv->gtt.gtt_remove = i915_gmch_remove;
1114 } else {
1115 dev_priv->gtt.gtt_probe = gen6_gmch_probe;
1116 dev_priv->gtt.gtt_remove = gen6_gmch_remove;
1117 if (IS_HASWELL(dev)) {
1118 dev_priv->gtt.pte_encode = hsw_pte_encode;
1119 } else if (IS_VALLEYVIEW(dev)) {
1120 dev_priv->gtt.pte_encode = byt_pte_encode;
1121 } else {
1122 dev_priv->gtt.pte_encode = gen6_pte_encode;
1123 }
1124 }
1125
1126 ret = dev_priv->gtt.gtt_probe(dev, &dev_priv->gtt.total,
1127 &dev_priv->gtt.stolen_size);
1128 if (ret)
1129 return ret;
1130
1131 /* GMADR is the PCI mmio aperture into the global GTT. */
1132 DRM_INFO("Memory usable by graphics device = %dM\n",
1133 dev_priv->gtt.total >> 20);
1134 DRM_DEBUG_DRIVER("GMADR size = %ldM\n",
1135 dev_priv->gtt.mappable_end >> 20);
1136 DRM_DEBUG_DRIVER("GTT stolen size = %dM\n",
1137 dev_priv->gtt.stolen_size >> 20);
1138
1139 return 0;
1140 }
1141
1142 void
intel_rw_gtt(struct drm_device * dev,size_t size,uint32_t gtt_offset,void * gttp,uint32_t type)1143 intel_rw_gtt(struct drm_device *dev,
1144 size_t size,
1145 uint32_t gtt_offset,
1146 void *gttp,
1147 uint32_t type)
1148 {
1149 struct drm_i915_private *dev_priv = dev->dev_private;
1150 unsigned first_entry = gtt_offset >> PAGE_SHIFT;
1151 unsigned num_entries = size / PAGE_SIZE;
1152 uint32_t *gtt_addr;
1153 uint32_t *old_gtt = (uint32_t *)gttp;
1154 int i, j;
1155
1156 if (INTEL_INFO(dev)->gen <= 5) {
1157 (void) drm_agp_rw_gtt(dev, size/PAGE_SIZE, gtt_offset,
1158 gttp, type);
1159 } else {
1160 if (type) {
1161 for (i = first_entry, j = 0; i < ( first_entry + num_entries); i++, j++) {
1162 gtt_addr = (uint32_t *)(uintptr_t)((caddr_t)dev_priv->gtt.virtual_gtt
1163 + i * sizeof(uint32_t));
1164 ddi_put32(dev_priv->gtt.gtt_mapping.acc_handle,
1165 gtt_addr, old_gtt[j]);
1166 }
1167 } else {
1168 for (i = first_entry; i < ( first_entry + num_entries); i++) {
1169 gtt_addr = (uint32_t *)(uintptr_t)((caddr_t)dev_priv->gtt.virtual_gtt
1170 + i * sizeof(uint32_t));
1171 old_gtt[i] = ddi_get32(dev_priv->gtt.gtt_mapping.acc_handle,gtt_addr);
1172 }
1173 }
1174 }
1175 }
1176
1177 void
i915_clean_gtt(struct drm_device * dev,size_t offset)1178 i915_clean_gtt(struct drm_device *dev, size_t offset)
1179 {
1180 struct drm_i915_private *dev_priv = dev->dev_private;
1181 unsigned first_entry = offset >> PAGE_SHIFT;
1182 unsigned num_entries = (dev_priv->gtt.total - offset - PAGE_SIZE) >> PAGE_SHIFT;
1183 pfn_t scratch_pfn = 0;
1184 gtt_pte_t *gtt_addr, scratch_pte;
1185 int i;
1186
1187 if (dev_priv->gtt.scratch_page != NULL)
1188 scratch_pfn = dev_priv->gtt.scratch_page->pfnarray[0];
1189
1190 if (INTEL_INFO(dev)->gen <= 5) {
1191 (void) drm_agp_unbind_pages(dev, NULL, num_entries,
1192 offset, scratch_pfn, 1);
1193 } else {
1194 if (scratch_pfn != 0) {
1195 uint64_t scratch_page_addr = scratch_pfn << PAGE_SHIFT;
1196 scratch_pte = gen6_pte_encode(dev, scratch_page_addr, I915_CACHE_LLC);
1197 } else {
1198 /* No scratch page? Use an invalid PTE. */
1199 scratch_pte = 0;
1200 }
1201
1202 for (i = first_entry ; i < ( first_entry + num_entries); i++) {
1203 gtt_addr = (gtt_pte_t *)(uintptr_t)((caddr_t)dev_priv->gtt.virtual_gtt
1204 + i * sizeof(gtt_pte_t));
1205 ddi_put32(dev_priv->gtt.gtt_mapping.acc_handle,
1206 gtt_addr, scratch_pte);
1207 }
1208 }
1209 }
1210
1211