17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate * CDDL HEADER START
37c478bd9Sstevel@tonic-gate *
47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the
51e2e7a75Shuah * Common Development and Distribution License (the "License").
61e2e7a75Shuah * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate *
87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate * and limitations under the License.
127c478bd9Sstevel@tonic-gate *
137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate *
197c478bd9Sstevel@tonic-gate * CDDL HEADER END
207c478bd9Sstevel@tonic-gate */
217c478bd9Sstevel@tonic-gate /*
2225312132Sbpramod * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
237c478bd9Sstevel@tonic-gate * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate */
257c478bd9Sstevel@tonic-gate
267c478bd9Sstevel@tonic-gate #include <sys/types.h>
277c478bd9Sstevel@tonic-gate #include <sys/systm.h>
287c478bd9Sstevel@tonic-gate #include <sys/archsystm.h>
297c478bd9Sstevel@tonic-gate #include <sys/machsystm.h>
307c478bd9Sstevel@tonic-gate #include <sys/t_lock.h>
317c478bd9Sstevel@tonic-gate #include <sys/vmem.h>
327c478bd9Sstevel@tonic-gate #include <sys/mman.h>
337c478bd9Sstevel@tonic-gate #include <sys/vm.h>
347c478bd9Sstevel@tonic-gate #include <sys/cpu.h>
357c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
367c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
377c478bd9Sstevel@tonic-gate #include <sys/atomic.h>
387c478bd9Sstevel@tonic-gate #include <vm/as.h>
397c478bd9Sstevel@tonic-gate #include <vm/hat.h>
407c478bd9Sstevel@tonic-gate #include <vm/as.h>
417c478bd9Sstevel@tonic-gate #include <vm/page.h>
427c478bd9Sstevel@tonic-gate #include <vm/seg.h>
437c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
4425312132Sbpramod #include <vm/seg_kpm.h>
457c478bd9Sstevel@tonic-gate #include <vm/hat_sfmmu.h>
467c478bd9Sstevel@tonic-gate #include <sys/debug.h>
477c478bd9Sstevel@tonic-gate #include <sys/cpu_module.h>
4825312132Sbpramod #include <sys/mem_cage.h>
497c478bd9Sstevel@tonic-gate
507c478bd9Sstevel@tonic-gate /*
517c478bd9Sstevel@tonic-gate * A quick way to generate a cache consistent address to map in a page.
527c478bd9Sstevel@tonic-gate * users: ppcopy, pagezero, /proc, dev/mem
537c478bd9Sstevel@tonic-gate *
547c478bd9Sstevel@tonic-gate * The ppmapin/ppmapout routines provide a quick way of generating a cache
557c478bd9Sstevel@tonic-gate * consistent address by reserving a given amount of kernel address space.
567c478bd9Sstevel@tonic-gate * The base is PPMAPBASE and its size is PPMAPSIZE. This memory is divided
577c478bd9Sstevel@tonic-gate * into x number of sets, where x is the number of colors for the virtual
587c478bd9Sstevel@tonic-gate * cache. The number of colors is how many times a page can be mapped
597c478bd9Sstevel@tonic-gate * simulatenously in the cache. For direct map caches this translates to
607c478bd9Sstevel@tonic-gate * the number of pages in the cache.
617c478bd9Sstevel@tonic-gate * Each set will be assigned a group of virtual pages from the reserved memory
627c478bd9Sstevel@tonic-gate * depending on its virtual color.
637c478bd9Sstevel@tonic-gate * When trying to assign a virtual address we will find out the color for the
647c478bd9Sstevel@tonic-gate * physical page in question (if applicable). Then we will try to find an
657c478bd9Sstevel@tonic-gate * available virtual page from the set of the appropiate color.
667c478bd9Sstevel@tonic-gate */
677c478bd9Sstevel@tonic-gate
687c478bd9Sstevel@tonic-gate #define clsettoarray(color, set) ((color * nsets) + set)
697c478bd9Sstevel@tonic-gate
707c478bd9Sstevel@tonic-gate int pp_slots = 4; /* small default, tuned by cpu module */
717c478bd9Sstevel@tonic-gate
727c478bd9Sstevel@tonic-gate /* tuned by cpu module, default is "safe" */
737c478bd9Sstevel@tonic-gate int pp_consistent_coloring = PPAGE_STORES_POLLUTE | PPAGE_LOADS_POLLUTE;
747c478bd9Sstevel@tonic-gate
757c478bd9Sstevel@tonic-gate static caddr_t ppmap_vaddrs[PPMAPSIZE / MMU_PAGESIZE];
767c478bd9Sstevel@tonic-gate static int nsets; /* number of sets */
777c478bd9Sstevel@tonic-gate static int ppmap_pages; /* generate align mask */
787c478bd9Sstevel@tonic-gate static int ppmap_shift; /* set selector */
797c478bd9Sstevel@tonic-gate
807c478bd9Sstevel@tonic-gate #ifdef PPDEBUG
817c478bd9Sstevel@tonic-gate #define MAXCOLORS 16 /* for debug only */
827c478bd9Sstevel@tonic-gate static int ppalloc_noslot = 0; /* # of allocations from kernelmap */
837c478bd9Sstevel@tonic-gate static int align_hits[MAXCOLORS];
847c478bd9Sstevel@tonic-gate static int pp_allocs; /* # of ppmapin requests */
857c478bd9Sstevel@tonic-gate #endif /* PPDEBUG */
867c478bd9Sstevel@tonic-gate
877c478bd9Sstevel@tonic-gate /*
887c478bd9Sstevel@tonic-gate * There are only 64 TLB entries on spitfire, 16 on cheetah
897c478bd9Sstevel@tonic-gate * (fully-associative TLB) so we allow the cpu module to tune the
907c478bd9Sstevel@tonic-gate * number to use here via pp_slots.
917c478bd9Sstevel@tonic-gate */
927c478bd9Sstevel@tonic-gate static struct ppmap_va {
937c478bd9Sstevel@tonic-gate caddr_t ppmap_slots[MAXPP_SLOTS];
947c478bd9Sstevel@tonic-gate } ppmap_va[NCPU];
957c478bd9Sstevel@tonic-gate
967c478bd9Sstevel@tonic-gate void
ppmapinit(void)977c478bd9Sstevel@tonic-gate ppmapinit(void)
987c478bd9Sstevel@tonic-gate {
997c478bd9Sstevel@tonic-gate int color, nset, setsize;
1007c478bd9Sstevel@tonic-gate caddr_t va;
1017c478bd9Sstevel@tonic-gate
1027c478bd9Sstevel@tonic-gate ASSERT(pp_slots <= MAXPP_SLOTS);
1037c478bd9Sstevel@tonic-gate
1047c478bd9Sstevel@tonic-gate va = (caddr_t)PPMAPBASE;
1057c478bd9Sstevel@tonic-gate if (cache & CACHE_VAC) {
1067c478bd9Sstevel@tonic-gate int a;
1077c478bd9Sstevel@tonic-gate
1087c478bd9Sstevel@tonic-gate ppmap_pages = mmu_btop(shm_alignment);
1097c478bd9Sstevel@tonic-gate nsets = PPMAPSIZE / shm_alignment;
1107c478bd9Sstevel@tonic-gate setsize = shm_alignment;
1117c478bd9Sstevel@tonic-gate ppmap_shift = MMU_PAGESHIFT;
1127c478bd9Sstevel@tonic-gate a = ppmap_pages;
1137c478bd9Sstevel@tonic-gate while (a >>= 1)
1147c478bd9Sstevel@tonic-gate ppmap_shift++;
1157c478bd9Sstevel@tonic-gate } else {
1167c478bd9Sstevel@tonic-gate /*
1177c478bd9Sstevel@tonic-gate * If we do not have a virtual indexed cache we simply
1187c478bd9Sstevel@tonic-gate * have only one set containing all pages.
1197c478bd9Sstevel@tonic-gate */
1207c478bd9Sstevel@tonic-gate ppmap_pages = 1;
1217c478bd9Sstevel@tonic-gate nsets = mmu_btop(PPMAPSIZE);
1227c478bd9Sstevel@tonic-gate setsize = MMU_PAGESIZE;
1237c478bd9Sstevel@tonic-gate ppmap_shift = MMU_PAGESHIFT;
1247c478bd9Sstevel@tonic-gate }
1257c478bd9Sstevel@tonic-gate for (color = 0; color < ppmap_pages; color++) {
1267c478bd9Sstevel@tonic-gate for (nset = 0; nset < nsets; nset++) {
1277c478bd9Sstevel@tonic-gate ppmap_vaddrs[clsettoarray(color, nset)] =
1287c478bd9Sstevel@tonic-gate (caddr_t)((uintptr_t)va + (nset * setsize));
1297c478bd9Sstevel@tonic-gate }
1307c478bd9Sstevel@tonic-gate va += MMU_PAGESIZE;
1317c478bd9Sstevel@tonic-gate }
1327c478bd9Sstevel@tonic-gate }
1337c478bd9Sstevel@tonic-gate
1347c478bd9Sstevel@tonic-gate /*
1357c478bd9Sstevel@tonic-gate * Allocate a cache consistent virtual address to map a page, pp,
1367c478bd9Sstevel@tonic-gate * with protection, vprot; and map it in the MMU, using the most
1377c478bd9Sstevel@tonic-gate * efficient means possible. The argument avoid is a virtual address
1387c478bd9Sstevel@tonic-gate * hint which when masked yields an offset into a virtual cache
1397c478bd9Sstevel@tonic-gate * that should be avoided when allocating an address to map in a
1407c478bd9Sstevel@tonic-gate * page. An avoid arg of -1 means you don't care, for instance pagezero.
1417c478bd9Sstevel@tonic-gate *
1427c478bd9Sstevel@tonic-gate * machine dependent, depends on virtual address space layout,
1437c478bd9Sstevel@tonic-gate * understands that all kernel addresses have bit 31 set.
1447c478bd9Sstevel@tonic-gate *
1457c478bd9Sstevel@tonic-gate * NOTE: For sun4 platforms the meaning of the hint argument is opposite from
1467c478bd9Sstevel@tonic-gate * that found in other architectures. In other architectures the hint
1477c478bd9Sstevel@tonic-gate * (called avoid) was used to ask ppmapin to NOT use the specified cache color.
1487c478bd9Sstevel@tonic-gate * This was used to avoid virtual cache trashing in the bcopy. Unfortunately
1497c478bd9Sstevel@tonic-gate * in the case of a COW, this later on caused a cache aliasing conflict. In
1507c478bd9Sstevel@tonic-gate * sun4, the bcopy routine uses the block ld/st instructions so we don't have
1517c478bd9Sstevel@tonic-gate * to worry about virtual cache trashing. Actually, by using the hint to choose
1527c478bd9Sstevel@tonic-gate * the right color we can almost guarantee a cache conflict will not occur.
1537c478bd9Sstevel@tonic-gate */
1547c478bd9Sstevel@tonic-gate
1557c478bd9Sstevel@tonic-gate caddr_t
ppmapin(page_t * pp,uint_t vprot,caddr_t hint)1567c478bd9Sstevel@tonic-gate ppmapin(page_t *pp, uint_t vprot, caddr_t hint)
1577c478bd9Sstevel@tonic-gate {
1587c478bd9Sstevel@tonic-gate int color, nset, index, start;
1597c478bd9Sstevel@tonic-gate caddr_t va;
1607c478bd9Sstevel@tonic-gate
1617c478bd9Sstevel@tonic-gate #ifdef PPDEBUG
1627c478bd9Sstevel@tonic-gate pp_allocs++;
1637c478bd9Sstevel@tonic-gate #endif /* PPDEBUG */
1647c478bd9Sstevel@tonic-gate if (cache & CACHE_VAC) {
1657c478bd9Sstevel@tonic-gate color = sfmmu_get_ppvcolor(pp);
1667c478bd9Sstevel@tonic-gate if (color == -1) {
1677c478bd9Sstevel@tonic-gate if ((intptr_t)hint != -1L) {
1687c478bd9Sstevel@tonic-gate color = addr_to_vcolor(hint);
1697c478bd9Sstevel@tonic-gate } else {
1707c478bd9Sstevel@tonic-gate color = addr_to_vcolor(mmu_ptob(pp->p_pagenum));
1717c478bd9Sstevel@tonic-gate }
1727c478bd9Sstevel@tonic-gate }
1737c478bd9Sstevel@tonic-gate
1747c478bd9Sstevel@tonic-gate } else {
1757c478bd9Sstevel@tonic-gate /*
1767c478bd9Sstevel@tonic-gate * For physical caches, we can pick any address we want.
1777c478bd9Sstevel@tonic-gate */
1787c478bd9Sstevel@tonic-gate color = 0;
1797c478bd9Sstevel@tonic-gate }
1807c478bd9Sstevel@tonic-gate
1817c478bd9Sstevel@tonic-gate start = color;
1827c478bd9Sstevel@tonic-gate do {
1837c478bd9Sstevel@tonic-gate for (nset = 0; nset < nsets; nset++) {
1847c478bd9Sstevel@tonic-gate index = clsettoarray(color, nset);
1857c478bd9Sstevel@tonic-gate va = ppmap_vaddrs[index];
1867c478bd9Sstevel@tonic-gate if (va != NULL) {
1877c478bd9Sstevel@tonic-gate #ifdef PPDEBUG
1887c478bd9Sstevel@tonic-gate align_hits[color]++;
1897c478bd9Sstevel@tonic-gate #endif /* PPDEBUG */
190*75d94465SJosef 'Jeff' Sipek if (atomic_cas_ptr(&ppmap_vaddrs[index],
1917c478bd9Sstevel@tonic-gate va, NULL) == va) {
1927c478bd9Sstevel@tonic-gate hat_memload(kas.a_hat, va, pp,
19325312132Sbpramod vprot | HAT_NOSYNC,
19425312132Sbpramod HAT_LOAD_LOCK);
1957c478bd9Sstevel@tonic-gate return (va);
1967c478bd9Sstevel@tonic-gate }
1977c478bd9Sstevel@tonic-gate }
1987c478bd9Sstevel@tonic-gate }
1997c478bd9Sstevel@tonic-gate /*
2007c478bd9Sstevel@tonic-gate * first pick didn't succeed, try another
2017c478bd9Sstevel@tonic-gate */
2027c478bd9Sstevel@tonic-gate if (++color == ppmap_pages)
2037c478bd9Sstevel@tonic-gate color = 0;
2047c478bd9Sstevel@tonic-gate } while (color != start);
2057c478bd9Sstevel@tonic-gate
2067c478bd9Sstevel@tonic-gate #ifdef PPDEBUG
2077c478bd9Sstevel@tonic-gate ppalloc_noslot++;
2087c478bd9Sstevel@tonic-gate #endif /* PPDEBUG */
2097c478bd9Sstevel@tonic-gate
2107c478bd9Sstevel@tonic-gate /*
2117c478bd9Sstevel@tonic-gate * No free slots; get a random one from the kernel heap area.
2127c478bd9Sstevel@tonic-gate */
2137c478bd9Sstevel@tonic-gate va = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP);
2147c478bd9Sstevel@tonic-gate
2157c478bd9Sstevel@tonic-gate hat_memload(kas.a_hat, va, pp, vprot | HAT_NOSYNC, HAT_LOAD_LOCK);
2167c478bd9Sstevel@tonic-gate
2177c478bd9Sstevel@tonic-gate return (va);
2187c478bd9Sstevel@tonic-gate
2197c478bd9Sstevel@tonic-gate }
2207c478bd9Sstevel@tonic-gate
2217c478bd9Sstevel@tonic-gate void
ppmapout(caddr_t va)2227c478bd9Sstevel@tonic-gate ppmapout(caddr_t va)
2237c478bd9Sstevel@tonic-gate {
2247c478bd9Sstevel@tonic-gate int color, nset, index;
2257c478bd9Sstevel@tonic-gate
2267c478bd9Sstevel@tonic-gate if (va >= kernelheap && va < ekernelheap) {
2277c478bd9Sstevel@tonic-gate /*
2287c478bd9Sstevel@tonic-gate * Space came from kernelmap, flush the page and
2297c478bd9Sstevel@tonic-gate * return the space.
2307c478bd9Sstevel@tonic-gate */
2317c478bd9Sstevel@tonic-gate hat_unload(kas.a_hat, va, PAGESIZE,
2327c478bd9Sstevel@tonic-gate (HAT_UNLOAD_NOSYNC | HAT_UNLOAD_UNLOCK));
2337c478bd9Sstevel@tonic-gate vmem_free(heap_arena, va, PAGESIZE);
2347c478bd9Sstevel@tonic-gate } else {
2357c478bd9Sstevel@tonic-gate /*
2367c478bd9Sstevel@tonic-gate * Space came from ppmap_vaddrs[], give it back.
2377c478bd9Sstevel@tonic-gate */
2387c478bd9Sstevel@tonic-gate color = addr_to_vcolor(va);
2397c478bd9Sstevel@tonic-gate ASSERT((cache & CACHE_VAC)? (color < ppmap_pages) : 1);
2407c478bd9Sstevel@tonic-gate
2417c478bd9Sstevel@tonic-gate nset = ((uintptr_t)va >> ppmap_shift) & (nsets - 1);
2427c478bd9Sstevel@tonic-gate index = clsettoarray(color, nset);
2437c478bd9Sstevel@tonic-gate hat_unload(kas.a_hat, va, PAGESIZE,
2447c478bd9Sstevel@tonic-gate (HAT_UNLOAD_NOSYNC | HAT_UNLOAD_UNLOCK));
2457c478bd9Sstevel@tonic-gate
2467c478bd9Sstevel@tonic-gate ASSERT(ppmap_vaddrs[index] == NULL);
2477c478bd9Sstevel@tonic-gate ppmap_vaddrs[index] = va;
2487c478bd9Sstevel@tonic-gate }
2497c478bd9Sstevel@tonic-gate }
2507c478bd9Sstevel@tonic-gate
2517c478bd9Sstevel@tonic-gate #ifdef DEBUG
2527c478bd9Sstevel@tonic-gate #define PP_STAT_ADD(stat) (stat)++
2537c478bd9Sstevel@tonic-gate uint_t pload, ploadfail;
2547c478bd9Sstevel@tonic-gate uint_t ppzero, ppzero_short;
2557c478bd9Sstevel@tonic-gate #else
2567c478bd9Sstevel@tonic-gate #define PP_STAT_ADD(stat)
2577c478bd9Sstevel@tonic-gate #endif /* DEBUG */
2587c478bd9Sstevel@tonic-gate
2597c478bd9Sstevel@tonic-gate /*
2607c478bd9Sstevel@tonic-gate * Find a slot in per CPU page copy area. Load up a locked TLB in the
2617c478bd9Sstevel@tonic-gate * running cpu. We don't call hat layer to load up the tte since the
2627c478bd9Sstevel@tonic-gate * mapping is only temporary. If the thread migrates it'll get a TLB
2637c478bd9Sstevel@tonic-gate * miss trap and TLB/TSB miss handler will panic since there is no
2647c478bd9Sstevel@tonic-gate * official hat record of this mapping.
2657c478bd9Sstevel@tonic-gate */
2667c478bd9Sstevel@tonic-gate static caddr_t
pp_load_tlb(processorid_t cpu,caddr_t ** pslot,page_t * pp,uint_t prot)2677c478bd9Sstevel@tonic-gate pp_load_tlb(processorid_t cpu, caddr_t **pslot, page_t *pp, uint_t prot)
2687c478bd9Sstevel@tonic-gate {
2697c478bd9Sstevel@tonic-gate struct ppmap_va *ppmap;
2707c478bd9Sstevel@tonic-gate tte_t tte;
2717c478bd9Sstevel@tonic-gate caddr_t *myslot;
2727c478bd9Sstevel@tonic-gate caddr_t va;
2737c478bd9Sstevel@tonic-gate long i, start, stride;
2747c478bd9Sstevel@tonic-gate int vcolor;
2757c478bd9Sstevel@tonic-gate uint_t flags, strict_flag;
2767c478bd9Sstevel@tonic-gate
2777c478bd9Sstevel@tonic-gate PP_STAT_ADD(pload);
2787c478bd9Sstevel@tonic-gate
2797c478bd9Sstevel@tonic-gate ppmap = &ppmap_va[cpu];
2807c478bd9Sstevel@tonic-gate va = (caddr_t)(PPMAP_FAST_BASE + (MMU_PAGESIZE * MAXPP_SLOTS) * cpu);
2817c478bd9Sstevel@tonic-gate myslot = ppmap->ppmap_slots;
2827c478bd9Sstevel@tonic-gate ASSERT(addr_to_vcolor(va) == 0);
2837c478bd9Sstevel@tonic-gate
2847c478bd9Sstevel@tonic-gate if (prot & TTE_HWWR_INT) {
2857c478bd9Sstevel@tonic-gate flags = PPAGE_STORE_VCOLORING | PPAGE_STORES_POLLUTE;
2867c478bd9Sstevel@tonic-gate strict_flag = PPAGE_STORES_POLLUTE;
2877c478bd9Sstevel@tonic-gate } else {
2887c478bd9Sstevel@tonic-gate flags = PPAGE_LOAD_VCOLORING | PPAGE_LOADS_POLLUTE;
2897c478bd9Sstevel@tonic-gate strict_flag = PPAGE_LOADS_POLLUTE;
2907c478bd9Sstevel@tonic-gate }
2917c478bd9Sstevel@tonic-gate
2927c478bd9Sstevel@tonic-gate /*
2937c478bd9Sstevel@tonic-gate * If consistent handling is required then keep the current
2947c478bd9Sstevel@tonic-gate * vcolor of the page. Furthermore, if loads or stores can
2957c478bd9Sstevel@tonic-gate * pollute the VAC then using a "new" page (unassigned vcolor)
2967c478bd9Sstevel@tonic-gate * won't work and we have to return a failure.
2977c478bd9Sstevel@tonic-gate */
2987c478bd9Sstevel@tonic-gate if (pp_consistent_coloring & flags) {
2997c478bd9Sstevel@tonic-gate vcolor = sfmmu_get_ppvcolor(pp);
3007c478bd9Sstevel@tonic-gate if ((vcolor == -1) &&
3017c478bd9Sstevel@tonic-gate (pp_consistent_coloring & strict_flag))
3027c478bd9Sstevel@tonic-gate return (NULL);
3037c478bd9Sstevel@tonic-gate /* else keep the current vcolor of the page */
3047c478bd9Sstevel@tonic-gate } else {
3057c478bd9Sstevel@tonic-gate vcolor = -1;
3067c478bd9Sstevel@tonic-gate }
3077c478bd9Sstevel@tonic-gate
3087c478bd9Sstevel@tonic-gate if (vcolor != -1) {
3097c478bd9Sstevel@tonic-gate va += MMU_PAGESIZE * vcolor;
3107c478bd9Sstevel@tonic-gate start = vcolor;
3117c478bd9Sstevel@tonic-gate stride = ppmap_pages; /* number of colors */
3127c478bd9Sstevel@tonic-gate myslot += vcolor;
3137c478bd9Sstevel@tonic-gate } else {
3147c478bd9Sstevel@tonic-gate start = 0;
3157c478bd9Sstevel@tonic-gate stride = 1;
3167c478bd9Sstevel@tonic-gate }
3177c478bd9Sstevel@tonic-gate
3187c478bd9Sstevel@tonic-gate for (i = start; i < pp_slots; i += stride) {
3197c478bd9Sstevel@tonic-gate if (*myslot == NULL) {
320*75d94465SJosef 'Jeff' Sipek if (atomic_cas_ptr(myslot, NULL, va) == NULL)
3217c478bd9Sstevel@tonic-gate break;
3227c478bd9Sstevel@tonic-gate }
3237c478bd9Sstevel@tonic-gate myslot += stride;
3247c478bd9Sstevel@tonic-gate va += MMU_PAGESIZE * stride;
3257c478bd9Sstevel@tonic-gate }
3267c478bd9Sstevel@tonic-gate
3277c478bd9Sstevel@tonic-gate if (i >= pp_slots) {
3287c478bd9Sstevel@tonic-gate PP_STAT_ADD(ploadfail);
3297c478bd9Sstevel@tonic-gate return (NULL);
3307c478bd9Sstevel@tonic-gate }
3317c478bd9Sstevel@tonic-gate
3327c478bd9Sstevel@tonic-gate ASSERT(vcolor == -1 || addr_to_vcolor(va) == vcolor);
3337c478bd9Sstevel@tonic-gate
3347c478bd9Sstevel@tonic-gate /*
3357c478bd9Sstevel@tonic-gate * Now we have a slot we can use, make the tte.
3367c478bd9Sstevel@tonic-gate */
3377c478bd9Sstevel@tonic-gate tte.tte_inthi = TTE_VALID_INT | TTE_PFN_INTHI(pp->p_pagenum);
3387c478bd9Sstevel@tonic-gate tte.tte_intlo = TTE_PFN_INTLO(pp->p_pagenum) | TTE_CP_INT |
3397c478bd9Sstevel@tonic-gate TTE_CV_INT | TTE_PRIV_INT | TTE_LCK_INT | prot;
3407c478bd9Sstevel@tonic-gate
3417c478bd9Sstevel@tonic-gate ASSERT(CPU->cpu_id == cpu);
3421e2e7a75Shuah sfmmu_dtlb_ld_kva(va, &tte);
3437c478bd9Sstevel@tonic-gate
3447c478bd9Sstevel@tonic-gate *pslot = myslot; /* Return ptr to the slot we used. */
3457c478bd9Sstevel@tonic-gate
3467c478bd9Sstevel@tonic-gate return (va);
3477c478bd9Sstevel@tonic-gate }
3487c478bd9Sstevel@tonic-gate
3497c478bd9Sstevel@tonic-gate static void
pp_unload_tlb(caddr_t * pslot,caddr_t va)3507c478bd9Sstevel@tonic-gate pp_unload_tlb(caddr_t *pslot, caddr_t va)
3517c478bd9Sstevel@tonic-gate {
3527c478bd9Sstevel@tonic-gate ASSERT(*pslot == va);
3537c478bd9Sstevel@tonic-gate
3541e2e7a75Shuah vtag_flushpage(va, (uint64_t)ksfmmup);
3557c478bd9Sstevel@tonic-gate *pslot = NULL; /* release the slot */
3567c478bd9Sstevel@tonic-gate }
3577c478bd9Sstevel@tonic-gate
3587c478bd9Sstevel@tonic-gate /*
3597c478bd9Sstevel@tonic-gate * Common copy routine which attempts to use hwblkpagecopy. If this routine
3607c478bd9Sstevel@tonic-gate * can't be used, failure (0) will be returned. Otherwise, a PAGESIZE page
3617c478bd9Sstevel@tonic-gate * will be copied and success (1) will be returned.
3627c478bd9Sstevel@tonic-gate */
3637c478bd9Sstevel@tonic-gate int
ppcopy_common(page_t * fm_pp,page_t * to_pp)3647c478bd9Sstevel@tonic-gate ppcopy_common(page_t *fm_pp, page_t *to_pp)
3657c478bd9Sstevel@tonic-gate {
3667c478bd9Sstevel@tonic-gate caddr_t fm_va, to_va;
3677c478bd9Sstevel@tonic-gate caddr_t *fm_slot, *to_slot;
3687c478bd9Sstevel@tonic-gate processorid_t cpu;
3698b464eb8Smec label_t ljb;
3708b464eb8Smec int ret = 1;
3717c478bd9Sstevel@tonic-gate
37225312132Sbpramod ASSERT(fm_pp != NULL && PAGE_LOCKED(fm_pp));
37325312132Sbpramod ASSERT(to_pp != NULL && PAGE_LOCKED(to_pp));
3747c478bd9Sstevel@tonic-gate
3757c478bd9Sstevel@tonic-gate /*
3767c478bd9Sstevel@tonic-gate * If we can't use VIS block loads and stores we can't use
3777c478bd9Sstevel@tonic-gate * pp_load_tlb/pp_unload_tlb due to the possibility of
3787c478bd9Sstevel@tonic-gate * d$ aliasing.
3797c478bd9Sstevel@tonic-gate */
3807c478bd9Sstevel@tonic-gate if (!use_hw_bcopy && (cache & CACHE_VAC))
3817c478bd9Sstevel@tonic-gate return (0);
3827c478bd9Sstevel@tonic-gate
3837c478bd9Sstevel@tonic-gate kpreempt_disable();
3847c478bd9Sstevel@tonic-gate cpu = CPU->cpu_id;
3857c478bd9Sstevel@tonic-gate fm_va = pp_load_tlb(cpu, &fm_slot, fm_pp, 0);
3867c478bd9Sstevel@tonic-gate if (fm_va == NULL) {
3877c478bd9Sstevel@tonic-gate kpreempt_enable();
3887c478bd9Sstevel@tonic-gate return (0);
3897c478bd9Sstevel@tonic-gate }
3907c478bd9Sstevel@tonic-gate to_va = pp_load_tlb(cpu, &to_slot, to_pp, TTE_HWWR_INT);
3917c478bd9Sstevel@tonic-gate if (to_va == NULL) {
3927c478bd9Sstevel@tonic-gate pp_unload_tlb(fm_slot, fm_va);
3937c478bd9Sstevel@tonic-gate kpreempt_enable();
3947c478bd9Sstevel@tonic-gate return (0);
3957c478bd9Sstevel@tonic-gate }
3968b464eb8Smec if (on_fault(&ljb)) {
3978b464eb8Smec ret = 0;
3988b464eb8Smec goto faulted;
3998b464eb8Smec }
4007c478bd9Sstevel@tonic-gate hwblkpagecopy(fm_va, to_va);
4018b464eb8Smec no_fault();
4028b464eb8Smec faulted:
4037c478bd9Sstevel@tonic-gate ASSERT(CPU->cpu_id == cpu);
4047c478bd9Sstevel@tonic-gate pp_unload_tlb(fm_slot, fm_va);
4057c478bd9Sstevel@tonic-gate pp_unload_tlb(to_slot, to_va);
4067c478bd9Sstevel@tonic-gate kpreempt_enable();
4078b464eb8Smec return (ret);
4087c478bd9Sstevel@tonic-gate }
4097c478bd9Sstevel@tonic-gate
4107c478bd9Sstevel@tonic-gate /*
4117c478bd9Sstevel@tonic-gate * Routine to copy kernel pages during relocation. It will copy one
4127c478bd9Sstevel@tonic-gate * PAGESIZE page to another PAGESIZE page. This function may be called
4137c478bd9Sstevel@tonic-gate * above LOCK_LEVEL so it should not grab any locks.
4147c478bd9Sstevel@tonic-gate */
4157c478bd9Sstevel@tonic-gate void
ppcopy_kernel__relocatable(page_t * fm_pp,page_t * to_pp)4167c478bd9Sstevel@tonic-gate ppcopy_kernel__relocatable(page_t *fm_pp, page_t *to_pp)
4177c478bd9Sstevel@tonic-gate {
4187c478bd9Sstevel@tonic-gate uint64_t fm_pa, to_pa;
4197c478bd9Sstevel@tonic-gate size_t nbytes;
4207c478bd9Sstevel@tonic-gate
4217c478bd9Sstevel@tonic-gate fm_pa = (uint64_t)(fm_pp->p_pagenum) << MMU_PAGESHIFT;
4227c478bd9Sstevel@tonic-gate to_pa = (uint64_t)(to_pp->p_pagenum) << MMU_PAGESHIFT;
4237c478bd9Sstevel@tonic-gate
4247c478bd9Sstevel@tonic-gate nbytes = MMU_PAGESIZE;
4257c478bd9Sstevel@tonic-gate
4267c478bd9Sstevel@tonic-gate for (; nbytes > 0; fm_pa += 32, to_pa += 32, nbytes -= 32)
4277c478bd9Sstevel@tonic-gate hw_pa_bcopy32(fm_pa, to_pa);
4287c478bd9Sstevel@tonic-gate }
4297c478bd9Sstevel@tonic-gate
4307c478bd9Sstevel@tonic-gate /*
4317c478bd9Sstevel@tonic-gate * Copy the data from the physical page represented by "frompp" to
4327c478bd9Sstevel@tonic-gate * that represented by "topp".
4337c478bd9Sstevel@tonic-gate *
4347c478bd9Sstevel@tonic-gate * Try to use per cpu mapping first, if that fails then call pp_mapin
4357c478bd9Sstevel@tonic-gate * to load it.
4368b464eb8Smec *
4378b464eb8Smec * Returns one on success or zero on some sort of fault while doing the copy.
4387c478bd9Sstevel@tonic-gate */
4398b464eb8Smec int
ppcopy(page_t * fm_pp,page_t * to_pp)4407c478bd9Sstevel@tonic-gate ppcopy(page_t *fm_pp, page_t *to_pp)
4417c478bd9Sstevel@tonic-gate {
4427c478bd9Sstevel@tonic-gate caddr_t fm_va, to_va;
4438b464eb8Smec label_t ljb;
4448b464eb8Smec int ret = 1;
44525312132Sbpramod boolean_t use_kpm = B_FALSE;
4467c478bd9Sstevel@tonic-gate
4477c478bd9Sstevel@tonic-gate /* Try the fast path first */
4487c478bd9Sstevel@tonic-gate if (ppcopy_common(fm_pp, to_pp))
4498b464eb8Smec return (1);
4507c478bd9Sstevel@tonic-gate
45125312132Sbpramod /*
45225312132Sbpramod * Try to map using KPM if enabled and we are the cageout thread.
45325312132Sbpramod * If it fails, fall back to ppmapin/ppmaput
45425312132Sbpramod */
45525312132Sbpramod
45625312132Sbpramod if (kpm_enable) {
45725312132Sbpramod if (curthread == kcage_cageout_thread)
45825312132Sbpramod use_kpm = B_TRUE;
45925312132Sbpramod }
46025312132Sbpramod
46125312132Sbpramod if (use_kpm) {
46225312132Sbpramod if ((fm_va = hat_kpm_mapin(fm_pp, NULL)) == NULL ||
46325312132Sbpramod (to_va = hat_kpm_mapin(to_pp, NULL)) == NULL) {
46425312132Sbpramod if (fm_va != NULL)
46525312132Sbpramod hat_kpm_mapout(fm_pp, NULL, fm_va);
46625312132Sbpramod use_kpm = B_FALSE;
46725312132Sbpramod }
46825312132Sbpramod }
46925312132Sbpramod
47025312132Sbpramod if (use_kpm == B_FALSE) {
47125312132Sbpramod /* do the slow path */
47225312132Sbpramod fm_va = ppmapin(fm_pp, PROT_READ, (caddr_t)-1);
47325312132Sbpramod to_va = ppmapin(to_pp, PROT_READ | PROT_WRITE, fm_va);
47425312132Sbpramod if (on_fault(&ljb)) {
47525312132Sbpramod ret = 0;
47625312132Sbpramod goto faulted;
47725312132Sbpramod }
4788b464eb8Smec }
4797c478bd9Sstevel@tonic-gate bcopy(fm_va, to_va, PAGESIZE);
4808b464eb8Smec no_fault();
4818b464eb8Smec faulted:
48225312132Sbpramod /* unmap */
48325312132Sbpramod if (use_kpm == B_TRUE) {
48425312132Sbpramod hat_kpm_mapout(fm_pp, NULL, fm_va);
48525312132Sbpramod hat_kpm_mapout(to_pp, NULL, to_va);
48625312132Sbpramod } else {
48725312132Sbpramod ppmapout(fm_va);
48825312132Sbpramod ppmapout(to_va);
48925312132Sbpramod }
4908b464eb8Smec return (ret);
4917c478bd9Sstevel@tonic-gate }
4927c478bd9Sstevel@tonic-gate
4937c478bd9Sstevel@tonic-gate /*
4947c478bd9Sstevel@tonic-gate * Zero the physical page from off to off + len given by `pp'
4957c478bd9Sstevel@tonic-gate * without changing the reference and modified bits of page.
4967c478bd9Sstevel@tonic-gate *
4977c478bd9Sstevel@tonic-gate * Again, we'll try per cpu mapping first.
4987c478bd9Sstevel@tonic-gate */
4997c478bd9Sstevel@tonic-gate void
pagezero(page_t * pp,uint_t off,uint_t len)5007c478bd9Sstevel@tonic-gate pagezero(page_t *pp, uint_t off, uint_t len)
5017c478bd9Sstevel@tonic-gate {
5027c478bd9Sstevel@tonic-gate caddr_t va;
5037c478bd9Sstevel@tonic-gate caddr_t *slot;
5047c478bd9Sstevel@tonic-gate int fast = 1;
5057c478bd9Sstevel@tonic-gate processorid_t cpu;
5067c478bd9Sstevel@tonic-gate extern int hwblkclr(void *, size_t);
5077c478bd9Sstevel@tonic-gate extern int use_hw_bzero;
5087c478bd9Sstevel@tonic-gate
5097c478bd9Sstevel@tonic-gate ASSERT((int)len > 0 && (int)off >= 0 && off + len <= PAGESIZE);
5107c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(pp));
5117c478bd9Sstevel@tonic-gate
5127c478bd9Sstevel@tonic-gate PP_STAT_ADD(ppzero);
5137c478bd9Sstevel@tonic-gate
5147c478bd9Sstevel@tonic-gate if (len != MMU_PAGESIZE || !use_hw_bzero) {
5157c478bd9Sstevel@tonic-gate /*
5167c478bd9Sstevel@tonic-gate * Since the fast path doesn't do anything about
5177c478bd9Sstevel@tonic-gate * VAC coloring, we make sure bcopy h/w will be used.
5187c478bd9Sstevel@tonic-gate */
5197c478bd9Sstevel@tonic-gate fast = 0;
5207c478bd9Sstevel@tonic-gate va = NULL;
5217c478bd9Sstevel@tonic-gate PP_STAT_ADD(ppzero_short);
5227c478bd9Sstevel@tonic-gate }
5237c478bd9Sstevel@tonic-gate
5247c478bd9Sstevel@tonic-gate kpreempt_disable();
5257c478bd9Sstevel@tonic-gate
5267c478bd9Sstevel@tonic-gate if (fast) {
5277c478bd9Sstevel@tonic-gate cpu = CPU->cpu_id;
5287c478bd9Sstevel@tonic-gate va = pp_load_tlb(cpu, &slot, pp, TTE_HWWR_INT);
5297c478bd9Sstevel@tonic-gate }
5307c478bd9Sstevel@tonic-gate
5317c478bd9Sstevel@tonic-gate if (va == NULL) {
5327c478bd9Sstevel@tonic-gate /*
5337c478bd9Sstevel@tonic-gate * We are here either length != MMU_PAGESIZE or pp_load_tlb()
5347c478bd9Sstevel@tonic-gate * returns NULL or use_hw_bzero is disabled.
5357c478bd9Sstevel@tonic-gate */
5367c478bd9Sstevel@tonic-gate va = ppmapin(pp, PROT_READ | PROT_WRITE, (caddr_t)-1);
5377c478bd9Sstevel@tonic-gate fast = 0;
5387c478bd9Sstevel@tonic-gate }
5397c478bd9Sstevel@tonic-gate
5407c478bd9Sstevel@tonic-gate if (hwblkclr(va + off, len)) {
5417c478bd9Sstevel@tonic-gate /*
5427c478bd9Sstevel@tonic-gate * We may not have used block commit asi.
5437c478bd9Sstevel@tonic-gate * So flush the I-$ manually
5447c478bd9Sstevel@tonic-gate */
5457c478bd9Sstevel@tonic-gate
5467c478bd9Sstevel@tonic-gate ASSERT(fast == 0);
5477c478bd9Sstevel@tonic-gate
5487c478bd9Sstevel@tonic-gate sync_icache(va + off, len);
5497c478bd9Sstevel@tonic-gate } else {
5507c478bd9Sstevel@tonic-gate /*
5517c478bd9Sstevel@tonic-gate * We have used blk commit, and flushed the I-$. However we
5527c478bd9Sstevel@tonic-gate * still may have an instruction in the pipeline. Only a flush
5537c478bd9Sstevel@tonic-gate * instruction will invalidate that.
5547c478bd9Sstevel@tonic-gate */
5557c478bd9Sstevel@tonic-gate doflush(va);
5567c478bd9Sstevel@tonic-gate }
5577c478bd9Sstevel@tonic-gate
5587c478bd9Sstevel@tonic-gate if (fast) {
5597c478bd9Sstevel@tonic-gate ASSERT(CPU->cpu_id == cpu);
5607c478bd9Sstevel@tonic-gate pp_unload_tlb(slot, va);
5617c478bd9Sstevel@tonic-gate } else {
5627c478bd9Sstevel@tonic-gate ppmapout(va);
5637c478bd9Sstevel@tonic-gate }
5647c478bd9Sstevel@tonic-gate
5657c478bd9Sstevel@tonic-gate kpreempt_enable();
5667c478bd9Sstevel@tonic-gate }
567