1fedab560Sae /*
2fedab560Sae * CDDL HEADER START
3fedab560Sae *
4fedab560Sae * The contents of this file are subject to the terms of the
5fedab560Sae * Common Development and Distribution License (the "License").
6fedab560Sae * You may not use this file except in compliance with the License.
7fedab560Sae *
8fedab560Sae * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9fedab560Sae * or http://www.opensolaris.org/os/licensing.
10fedab560Sae * See the License for the specific language governing permissions
11fedab560Sae * and limitations under the License.
12fedab560Sae *
13fedab560Sae * When distributing Covered Code, include this CDDL HEADER in each
14fedab560Sae * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15fedab560Sae * If applicable, add the following below this CDDL HEADER, with the
16fedab560Sae * fields enclosed by brackets "[]" replaced with your own identifying
17fedab560Sae * information: Portions Copyright [yyyy] [name of copyright owner]
18fedab560Sae *
19fedab560Sae * CDDL HEADER END
20fedab560Sae */
21fedab560Sae /*
22*d20abfaaSPavel Tatashin * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23fedab560Sae * Use is subject to license terms.
24fedab560Sae */
25fedab560Sae
26fedab560Sae /*
27fedab560Sae * Kernel Physical Mapping (segkpm) hat interface routines for sun4u.
28fedab560Sae */
29fedab560Sae
30fedab560Sae #include <sys/types.h>
31fedab560Sae #include <vm/hat.h>
32fedab560Sae #include <vm/hat_sfmmu.h>
33fedab560Sae #include <vm/page.h>
34fedab560Sae #include <sys/sysmacros.h>
35fedab560Sae #include <sys/cmn_err.h>
36fedab560Sae #include <sys/machsystm.h>
37fedab560Sae #include <vm/seg_kpm.h>
38fedab560Sae #include <sys/cpu_module.h>
39fedab560Sae #include <vm/mach_kpm.h>
40fedab560Sae
41fedab560Sae /* kpm prototypes */
42fedab560Sae static caddr_t sfmmu_kpm_mapin(page_t *);
43fedab560Sae static void sfmmu_kpm_mapout(page_t *, caddr_t);
44fedab560Sae static int sfmmu_kpme_lookup(struct kpme *, page_t *);
45fedab560Sae static void sfmmu_kpme_add(struct kpme *, page_t *);
46fedab560Sae static void sfmmu_kpme_sub(struct kpme *, page_t *);
47fedab560Sae static caddr_t sfmmu_kpm_getvaddr(page_t *, int *);
48fedab560Sae static int sfmmu_kpm_fault(caddr_t, struct memseg *, page_t *);
49fedab560Sae static int sfmmu_kpm_fault_small(caddr_t, struct memseg *, page_t *);
50fedab560Sae static void sfmmu_kpm_vac_conflict(page_t *, caddr_t);
51fedab560Sae void sfmmu_kpm_pageunload(page_t *);
52fedab560Sae void sfmmu_kpm_vac_unload(page_t *, caddr_t);
53fedab560Sae static void sfmmu_kpm_demap_large(caddr_t);
54fedab560Sae static void sfmmu_kpm_demap_small(caddr_t);
55fedab560Sae static void sfmmu_kpm_demap_tlbs(caddr_t);
56fedab560Sae void sfmmu_kpm_hme_unload(page_t *);
57fedab560Sae kpm_hlk_t *sfmmu_kpm_kpmp_enter(page_t *, pgcnt_t);
58fedab560Sae void sfmmu_kpm_kpmp_exit(kpm_hlk_t *kpmp);
59fedab560Sae void sfmmu_kpm_page_cache(page_t *, int, int);
60fedab560Sae
61*d20abfaaSPavel Tatashin extern uint_t vac_colors;
62*d20abfaaSPavel Tatashin
63fedab560Sae /*
64fedab560Sae * Kernel Physical Mapping (kpm) facility
65fedab560Sae */
66fedab560Sae
67fedab560Sae void
mach_kpm_init()68fedab560Sae mach_kpm_init()
69fedab560Sae {}
70fedab560Sae
71fedab560Sae /* -- hat_kpm interface section -- */
72fedab560Sae
73fedab560Sae /*
74fedab560Sae * Mapin a locked page and return the vaddr.
75fedab560Sae * When a kpme is provided by the caller it is added to
76fedab560Sae * the page p_kpmelist. The page to be mapped in must
77fedab560Sae * be at least read locked (p_selock).
78fedab560Sae */
79fedab560Sae caddr_t
hat_kpm_mapin(struct page * pp,struct kpme * kpme)80fedab560Sae hat_kpm_mapin(struct page *pp, struct kpme *kpme)
81fedab560Sae {
82fedab560Sae kmutex_t *pml;
83fedab560Sae caddr_t vaddr;
84fedab560Sae
85fedab560Sae if (kpm_enable == 0) {
86fedab560Sae cmn_err(CE_WARN, "hat_kpm_mapin: kpm_enable not set");
87fedab560Sae return ((caddr_t)NULL);
88fedab560Sae }
89fedab560Sae
90fedab560Sae if (pp == NULL || PAGE_LOCKED(pp) == 0) {
91fedab560Sae cmn_err(CE_WARN, "hat_kpm_mapin: pp zero or not locked");
92fedab560Sae return ((caddr_t)NULL);
93fedab560Sae }
94fedab560Sae
95fedab560Sae pml = sfmmu_mlist_enter(pp);
96fedab560Sae ASSERT(pp->p_kpmref >= 0);
97fedab560Sae
98fedab560Sae vaddr = (pp->p_kpmref == 0) ?
99444ce08eSDonghai Qiao sfmmu_kpm_mapin(pp) : hat_kpm_page2va(pp, 1);
100fedab560Sae
101fedab560Sae if (kpme != NULL) {
102fedab560Sae /*
103fedab560Sae * Tolerate multiple mapins for the same kpme to avoid
104fedab560Sae * the need for an extra serialization.
105fedab560Sae */
106fedab560Sae if ((sfmmu_kpme_lookup(kpme, pp)) == 0)
107fedab560Sae sfmmu_kpme_add(kpme, pp);
108fedab560Sae
109fedab560Sae ASSERT(pp->p_kpmref > 0);
110fedab560Sae
111fedab560Sae } else {
112fedab560Sae pp->p_kpmref++;
113fedab560Sae }
114fedab560Sae
115fedab560Sae sfmmu_mlist_exit(pml);
116fedab560Sae return (vaddr);
117fedab560Sae }
118fedab560Sae
119fedab560Sae /*
120fedab560Sae * Mapout a locked page.
121fedab560Sae * When a kpme is provided by the caller it is removed from
122fedab560Sae * the page p_kpmelist. The page to be mapped out must be at
123fedab560Sae * least read locked (p_selock).
124fedab560Sae * Note: The seg_kpm layer provides a mapout interface for the
125fedab560Sae * case that a kpme is used and the underlying page is unlocked.
126fedab560Sae * This can be used instead of calling this function directly.
127fedab560Sae */
128fedab560Sae void
hat_kpm_mapout(struct page * pp,struct kpme * kpme,caddr_t vaddr)129fedab560Sae hat_kpm_mapout(struct page *pp, struct kpme *kpme, caddr_t vaddr)
130fedab560Sae {
131fedab560Sae kmutex_t *pml;
132fedab560Sae
133fedab560Sae if (kpm_enable == 0) {
134fedab560Sae cmn_err(CE_WARN, "hat_kpm_mapout: kpm_enable not set");
135fedab560Sae return;
136fedab560Sae }
137fedab560Sae
138fedab560Sae if (IS_KPM_ADDR(vaddr) == 0) {
139fedab560Sae cmn_err(CE_WARN, "hat_kpm_mapout: no kpm address");
140fedab560Sae return;
141fedab560Sae }
142fedab560Sae
143fedab560Sae if (pp == NULL || PAGE_LOCKED(pp) == 0) {
144fedab560Sae cmn_err(CE_WARN, "hat_kpm_mapout: page zero or not locked");
145fedab560Sae return;
146fedab560Sae }
147fedab560Sae
148fedab560Sae if (kpme != NULL) {
149fedab560Sae ASSERT(pp == kpme->kpe_page);
150fedab560Sae pp = kpme->kpe_page;
151fedab560Sae pml = sfmmu_mlist_enter(pp);
152fedab560Sae
153fedab560Sae if (sfmmu_kpme_lookup(kpme, pp) == 0)
154fedab560Sae panic("hat_kpm_mapout: kpme not found pp=%p",
155444ce08eSDonghai Qiao (void *)pp);
156fedab560Sae
157fedab560Sae ASSERT(pp->p_kpmref > 0);
158fedab560Sae sfmmu_kpme_sub(kpme, pp);
159fedab560Sae
160fedab560Sae } else {
161fedab560Sae pml = sfmmu_mlist_enter(pp);
162fedab560Sae pp->p_kpmref--;
163fedab560Sae }
164fedab560Sae
165fedab560Sae ASSERT(pp->p_kpmref >= 0);
166fedab560Sae if (pp->p_kpmref == 0)
167fedab560Sae sfmmu_kpm_mapout(pp, vaddr);
168fedab560Sae
169fedab560Sae sfmmu_mlist_exit(pml);
170fedab560Sae }
171fedab560Sae
172*d20abfaaSPavel Tatashin /*
173*d20abfaaSPavel Tatashin * hat_kpm_mapin_pfn is used to obtain a kpm mapping for physical
174*d20abfaaSPavel Tatashin * memory addresses that are not described by a page_t. It can
175*d20abfaaSPavel Tatashin * only be supported if vac_colors=1, because there is no page_t
176*d20abfaaSPavel Tatashin * and corresponding kpm_page_t to track VAC conflicts. Currently,
177*d20abfaaSPavel Tatashin * this may not be used on pfn's backed by page_t's, because the
178*d20abfaaSPavel Tatashin * kpm state may not be consistent in hat_kpm_fault if the page is
179*d20abfaaSPavel Tatashin * mapped using both this routine and hat_kpm_mapin. KPM should be
180*d20abfaaSPavel Tatashin * cleaned up on sun4u/vac_colors=1 to be minimal as on sun4v.
181*d20abfaaSPavel Tatashin * The caller must only pass pfn's for valid physical addresses; violation
182*d20abfaaSPavel Tatashin * of this rule will cause panic.
183*d20abfaaSPavel Tatashin */
184*d20abfaaSPavel Tatashin caddr_t
hat_kpm_mapin_pfn(pfn_t pfn)185*d20abfaaSPavel Tatashin hat_kpm_mapin_pfn(pfn_t pfn)
186*d20abfaaSPavel Tatashin {
187*d20abfaaSPavel Tatashin caddr_t paddr, vaddr;
188*d20abfaaSPavel Tatashin tte_t tte;
189*d20abfaaSPavel Tatashin uint_t szc = kpm_smallpages ? TTE8K : TTE4M;
190*d20abfaaSPavel Tatashin uint_t shift = kpm_smallpages ? MMU_PAGESHIFT : MMU_PAGESHIFT4M;
191*d20abfaaSPavel Tatashin
192*d20abfaaSPavel Tatashin if (kpm_enable == 0 || vac_colors > 1 ||
193*d20abfaaSPavel Tatashin page_numtomemseg_nolock(pfn) != NULL)
194*d20abfaaSPavel Tatashin return ((caddr_t)NULL);
195*d20abfaaSPavel Tatashin
196*d20abfaaSPavel Tatashin paddr = (caddr_t)ptob(pfn);
197*d20abfaaSPavel Tatashin vaddr = (uintptr_t)kpm_vbase + paddr;
198*d20abfaaSPavel Tatashin
199*d20abfaaSPavel Tatashin KPM_TTE_VCACHED(tte.ll, pfn, szc);
200*d20abfaaSPavel Tatashin sfmmu_kpm_load_tsb(vaddr, &tte, shift);
201*d20abfaaSPavel Tatashin
202*d20abfaaSPavel Tatashin return (vaddr);
203*d20abfaaSPavel Tatashin }
204*d20abfaaSPavel Tatashin
205*d20abfaaSPavel Tatashin /*ARGSUSED*/
206*d20abfaaSPavel Tatashin void
hat_kpm_mapout_pfn(pfn_t pfn)207*d20abfaaSPavel Tatashin hat_kpm_mapout_pfn(pfn_t pfn)
208*d20abfaaSPavel Tatashin {
209*d20abfaaSPavel Tatashin /* empty */
210*d20abfaaSPavel Tatashin }
211*d20abfaaSPavel Tatashin
212fedab560Sae /*
213fedab560Sae * Return the kpm virtual address for the page at pp.
214fedab560Sae * If checkswap is non zero and the page is backed by a
215fedab560Sae * swap vnode the physical address is used rather than
216fedab560Sae * p_offset to determine the kpm region.
217fedab560Sae * Note: The function has to be used w/ extreme care. The
218fedab560Sae * stability of the page identity is in the responsibility
219fedab560Sae * of the caller.
220fedab560Sae */
221fedab560Sae /*ARGSUSED*/
222fedab560Sae caddr_t
hat_kpm_page2va(struct page * pp,int checkswap)223fedab560Sae hat_kpm_page2va(struct page *pp, int checkswap)
224fedab560Sae {
225fedab560Sae int vcolor, vcolor_pa;
226fedab560Sae uintptr_t paddr, vaddr;
227fedab560Sae
228fedab560Sae ASSERT(kpm_enable);
229fedab560Sae
230fedab560Sae paddr = ptob(pp->p_pagenum);
231fedab560Sae vcolor_pa = addr_to_vcolor(paddr);
232fedab560Sae
233fedab560Sae if (checkswap && pp->p_vnode && IS_SWAPFSVP(pp->p_vnode))
234fedab560Sae vcolor = (PP_ISNC(pp)) ? vcolor_pa : PP_GET_VCOLOR(pp);
235fedab560Sae else
236fedab560Sae vcolor = addr_to_vcolor(pp->p_offset);
237fedab560Sae
238fedab560Sae vaddr = (uintptr_t)kpm_vbase + paddr;
239fedab560Sae
240fedab560Sae if (vcolor_pa != vcolor) {
241fedab560Sae vaddr += ((uintptr_t)(vcolor - vcolor_pa) << MMU_PAGESHIFT);
242fedab560Sae vaddr += (vcolor_pa > vcolor) ?
243444ce08eSDonghai Qiao ((uintptr_t)vcolor_pa << kpm_size_shift) :
244444ce08eSDonghai Qiao ((uintptr_t)(vcolor - vcolor_pa) << kpm_size_shift);
245fedab560Sae }
246fedab560Sae
247fedab560Sae return ((caddr_t)vaddr);
248fedab560Sae }
249fedab560Sae
250fedab560Sae /*
251fedab560Sae * Return the page for the kpm virtual address vaddr.
252fedab560Sae * Caller is responsible for the kpm mapping and lock
253fedab560Sae * state of the page.
254fedab560Sae */
255fedab560Sae page_t *
hat_kpm_vaddr2page(caddr_t vaddr)256fedab560Sae hat_kpm_vaddr2page(caddr_t vaddr)
257fedab560Sae {
258fedab560Sae uintptr_t paddr;
259fedab560Sae pfn_t pfn;
260fedab560Sae
261fedab560Sae ASSERT(IS_KPM_ADDR(vaddr));
262fedab560Sae
263fedab560Sae SFMMU_KPM_VTOP(vaddr, paddr);
264fedab560Sae pfn = (pfn_t)btop(paddr);
265fedab560Sae
266fedab560Sae return (page_numtopp_nolock(pfn));
267fedab560Sae }
268fedab560Sae
269fedab560Sae /* page to kpm_page */
270fedab560Sae #define PP2KPMPG(pp, kp) { \
271fedab560Sae struct memseg *mseg; \
272fedab560Sae pgcnt_t inx; \
273fedab560Sae pfn_t pfn; \
274fedab560Sae \
275fedab560Sae pfn = pp->p_pagenum; \
276fedab560Sae mseg = page_numtomemseg_nolock(pfn); \
277fedab560Sae ASSERT(mseg); \
278fedab560Sae inx = ptokpmp(kpmptop(ptokpmp(pfn)) - mseg->kpm_pbase); \
279fedab560Sae ASSERT(inx < mseg->kpm_nkpmpgs); \
280fedab560Sae kp = &mseg->kpm_pages[inx]; \
281fedab560Sae }
282fedab560Sae
283fedab560Sae /* page to kpm_spage */
284fedab560Sae #define PP2KPMSPG(pp, ksp) { \
285fedab560Sae struct memseg *mseg; \
286fedab560Sae pgcnt_t inx; \
287fedab560Sae pfn_t pfn; \
288fedab560Sae \
289fedab560Sae pfn = pp->p_pagenum; \
290fedab560Sae mseg = page_numtomemseg_nolock(pfn); \
291fedab560Sae ASSERT(mseg); \
292fedab560Sae inx = pfn - mseg->kpm_pbase; \
293fedab560Sae ksp = &mseg->kpm_spages[inx]; \
294fedab560Sae }
295fedab560Sae
296fedab560Sae /*
297fedab560Sae * hat_kpm_fault is called from segkpm_fault when a kpm tsbmiss occurred
298fedab560Sae * which could not be resolved by the trap level tsbmiss handler for the
299fedab560Sae * following reasons:
300fedab560Sae * . The vaddr is in VAC alias range (always PAGESIZE mapping size).
301fedab560Sae * . The kpm (s)page range of vaddr is in a VAC alias prevention state.
302fedab560Sae * . tsbmiss handling at trap level is not desired (DEBUG kernel only,
303fedab560Sae * kpm_tsbmtl == 0).
304fedab560Sae */
305fedab560Sae int
hat_kpm_fault(struct hat * hat,caddr_t vaddr)306fedab560Sae hat_kpm_fault(struct hat *hat, caddr_t vaddr)
307fedab560Sae {
308fedab560Sae int error;
309fedab560Sae uintptr_t paddr;
310fedab560Sae pfn_t pfn;
311fedab560Sae struct memseg *mseg;
312fedab560Sae page_t *pp;
313fedab560Sae
314fedab560Sae if (kpm_enable == 0) {
315fedab560Sae cmn_err(CE_WARN, "hat_kpm_fault: kpm_enable not set");
316fedab560Sae return (ENOTSUP);
317fedab560Sae }
318fedab560Sae
319fedab560Sae ASSERT(hat == ksfmmup);
320fedab560Sae ASSERT(IS_KPM_ADDR(vaddr));
321fedab560Sae
322fedab560Sae SFMMU_KPM_VTOP(vaddr, paddr);
323fedab560Sae pfn = (pfn_t)btop(paddr);
324*d20abfaaSPavel Tatashin if ((mseg = page_numtomemseg_nolock(pfn)) != NULL) {
325*d20abfaaSPavel Tatashin pp = &mseg->pages[(pgcnt_t)(pfn - mseg->pages_base)];
326*d20abfaaSPavel Tatashin ASSERT((pfn_t)pp->p_pagenum == pfn);
327*d20abfaaSPavel Tatashin }
328fedab560Sae
329*d20abfaaSPavel Tatashin /*
330*d20abfaaSPavel Tatashin * hat_kpm_mapin_pfn may add a kpm translation for memory that falls
331*d20abfaaSPavel Tatashin * outside of memsegs. Check for this case and provide the translation
332*d20abfaaSPavel Tatashin * here.
333*d20abfaaSPavel Tatashin */
334*d20abfaaSPavel Tatashin if (vac_colors == 1 && mseg == NULL) {
335*d20abfaaSPavel Tatashin tte_t tte;
336*d20abfaaSPavel Tatashin uint_t szc = kpm_smallpages ? TTE8K : TTE4M;
337*d20abfaaSPavel Tatashin uint_t shift = kpm_smallpages ? MMU_PAGESHIFT : MMU_PAGESHIFT4M;
338*d20abfaaSPavel Tatashin
339*d20abfaaSPavel Tatashin ASSERT(address_in_memlist(phys_install, paddr, 1));
340*d20abfaaSPavel Tatashin KPM_TTE_VCACHED(tte.ll, pfn, szc);
341*d20abfaaSPavel Tatashin sfmmu_kpm_load_tsb(vaddr, &tte, shift);
342*d20abfaaSPavel Tatashin error = 0;
343*d20abfaaSPavel Tatashin } else if (mseg == NULL || !PAGE_LOCKED(pp))
344*d20abfaaSPavel Tatashin error = EFAULT;
345*d20abfaaSPavel Tatashin else if (kpm_smallpages == 0)
346fedab560Sae error = sfmmu_kpm_fault(vaddr, mseg, pp);
347fedab560Sae else
348fedab560Sae error = sfmmu_kpm_fault_small(vaddr, mseg, pp);
349fedab560Sae
350fedab560Sae return (error);
351fedab560Sae }
352fedab560Sae
353fedab560Sae /*
354fedab560Sae * memseg_hash[] was cleared, need to clear memseg_phash[] too.
355fedab560Sae */
356fedab560Sae void
hat_kpm_mseghash_clear(int nentries)357fedab560Sae hat_kpm_mseghash_clear(int nentries)
358fedab560Sae {
359fedab560Sae pgcnt_t i;
360fedab560Sae
361fedab560Sae if (kpm_enable == 0)
362fedab560Sae return;
363fedab560Sae
364fedab560Sae for (i = 0; i < nentries; i++)
365fedab560Sae memseg_phash[i] = MSEG_NULLPTR_PA;
366fedab560Sae }
367fedab560Sae
368fedab560Sae /*
369fedab560Sae * Update memseg_phash[inx] when memseg_hash[inx] was changed.
370fedab560Sae */
371fedab560Sae void
hat_kpm_mseghash_update(pgcnt_t inx,struct memseg * msp)372fedab560Sae hat_kpm_mseghash_update(pgcnt_t inx, struct memseg *msp)
373fedab560Sae {
374fedab560Sae if (kpm_enable == 0)
375fedab560Sae return;
376fedab560Sae
377fedab560Sae memseg_phash[inx] = (msp) ? va_to_pa(msp) : MSEG_NULLPTR_PA;
378fedab560Sae }
379fedab560Sae
380fedab560Sae /*
381fedab560Sae * Update kpm memseg members from basic memseg info.
382fedab560Sae */
383fedab560Sae void
hat_kpm_addmem_mseg_update(struct memseg * msp,pgcnt_t nkpmpgs,offset_t kpm_pages_off)384fedab560Sae hat_kpm_addmem_mseg_update(struct memseg *msp, pgcnt_t nkpmpgs,
385fedab560Sae offset_t kpm_pages_off)
386fedab560Sae {
387fedab560Sae if (kpm_enable == 0)
388fedab560Sae return;
389fedab560Sae
390fedab560Sae msp->kpm_pages = (kpm_page_t *)((caddr_t)msp->pages + kpm_pages_off);
391fedab560Sae msp->kpm_nkpmpgs = nkpmpgs;
392fedab560Sae msp->kpm_pbase = kpmptop(ptokpmp(msp->pages_base));
393fedab560Sae msp->pagespa = va_to_pa(msp->pages);
394fedab560Sae msp->epagespa = va_to_pa(msp->epages);
395fedab560Sae msp->kpm_pagespa = va_to_pa(msp->kpm_pages);
396fedab560Sae }
397fedab560Sae
398fedab560Sae /*
399fedab560Sae * Setup nextpa when a memseg is inserted.
400fedab560Sae * Assumes that the memsegslock is already held.
401fedab560Sae */
402fedab560Sae void
hat_kpm_addmem_mseg_insert(struct memseg * msp)403fedab560Sae hat_kpm_addmem_mseg_insert(struct memseg *msp)
404fedab560Sae {
405fedab560Sae if (kpm_enable == 0)
406fedab560Sae return;
407fedab560Sae
408ae115bc7Smrj ASSERT(memsegs_lock_held());
409fedab560Sae msp->nextpa = (memsegs) ? va_to_pa(memsegs) : MSEG_NULLPTR_PA;
410fedab560Sae }
411fedab560Sae
412fedab560Sae /*
413fedab560Sae * Setup memsegspa when a memseg is (head) inserted.
414fedab560Sae * Called before memsegs is updated to complete a
415fedab560Sae * memseg insert operation.
416fedab560Sae * Assumes that the memsegslock is already held.
417fedab560Sae */
418fedab560Sae void
hat_kpm_addmem_memsegs_update(struct memseg * msp)419fedab560Sae hat_kpm_addmem_memsegs_update(struct memseg *msp)
420fedab560Sae {
421fedab560Sae if (kpm_enable == 0)
422fedab560Sae return;
423fedab560Sae
424ae115bc7Smrj ASSERT(memsegs_lock_held());
425fedab560Sae ASSERT(memsegs);
426fedab560Sae memsegspa = va_to_pa(msp);
427fedab560Sae }
428fedab560Sae
429fedab560Sae /*
430fedab560Sae * Return end of metadata for an already setup memseg.
431fedab560Sae *
432fedab560Sae * Note: kpm_pages and kpm_spages are aliases and the underlying
433fedab560Sae * member of struct memseg is a union, therefore they always have
434fedab560Sae * the same address within a memseg. They must be differentiated
435fedab560Sae * when pointer arithmetic is used with them.
436fedab560Sae */
437fedab560Sae caddr_t
hat_kpm_mseg_reuse(struct memseg * msp)438fedab560Sae hat_kpm_mseg_reuse(struct memseg *msp)
439fedab560Sae {
440fedab560Sae caddr_t end;
441fedab560Sae
442fedab560Sae if (kpm_smallpages == 0)
443fedab560Sae end = (caddr_t)(msp->kpm_pages + msp->kpm_nkpmpgs);
444fedab560Sae else
445fedab560Sae end = (caddr_t)(msp->kpm_spages + msp->kpm_nkpmpgs);
446fedab560Sae
447fedab560Sae return (end);
448fedab560Sae }
449fedab560Sae
450fedab560Sae /*
451fedab560Sae * Update memsegspa (when first memseg in list
452fedab560Sae * is deleted) or nextpa when a memseg deleted.
453fedab560Sae * Assumes that the memsegslock is already held.
454fedab560Sae */
455fedab560Sae void
hat_kpm_delmem_mseg_update(struct memseg * msp,struct memseg ** mspp)456fedab560Sae hat_kpm_delmem_mseg_update(struct memseg *msp, struct memseg **mspp)
457fedab560Sae {
458fedab560Sae struct memseg *lmsp;
459fedab560Sae
460fedab560Sae if (kpm_enable == 0)
461fedab560Sae return;
462fedab560Sae
463ae115bc7Smrj ASSERT(memsegs_lock_held());
464fedab560Sae
465fedab560Sae if (mspp == &memsegs) {
466fedab560Sae memsegspa = (msp->next) ?
467444ce08eSDonghai Qiao va_to_pa(msp->next) : MSEG_NULLPTR_PA;
468fedab560Sae } else {
469fedab560Sae lmsp = (struct memseg *)
470444ce08eSDonghai Qiao ((uint64_t)mspp - offsetof(struct memseg, next));
471fedab560Sae lmsp->nextpa = (msp->next) ?
472444ce08eSDonghai Qiao va_to_pa(msp->next) : MSEG_NULLPTR_PA;
473fedab560Sae }
474fedab560Sae }
475fedab560Sae
476fedab560Sae /*
477fedab560Sae * Update kpm members for all memseg's involved in a split operation
478fedab560Sae * and do the atomic update of the physical memseg chain.
479fedab560Sae *
480fedab560Sae * Note: kpm_pages and kpm_spages are aliases and the underlying member
481fedab560Sae * of struct memseg is a union, therefore they always have the same
482fedab560Sae * address within a memseg. With that the direct assignments and
483fedab560Sae * va_to_pa conversions below don't have to be distinguished wrt. to
484fedab560Sae * kpm_smallpages. They must be differentiated when pointer arithmetic
485fedab560Sae * is used with them.
486fedab560Sae *
487fedab560Sae * Assumes that the memsegslock is already held.
488fedab560Sae */
489fedab560Sae void
hat_kpm_split_mseg_update(struct memseg * msp,struct memseg ** mspp,struct memseg * lo,struct memseg * mid,struct memseg * hi)490fedab560Sae hat_kpm_split_mseg_update(struct memseg *msp, struct memseg **mspp,
491fedab560Sae struct memseg *lo, struct memseg *mid, struct memseg *hi)
492fedab560Sae {
493fedab560Sae pgcnt_t start, end, kbase, kstart, num;
494fedab560Sae struct memseg *lmsp;
495fedab560Sae
496fedab560Sae if (kpm_enable == 0)
497fedab560Sae return;
498fedab560Sae
499ae115bc7Smrj ASSERT(memsegs_lock_held());
500fedab560Sae ASSERT(msp && mid && msp->kpm_pages);
501fedab560Sae
502fedab560Sae kbase = ptokpmp(msp->kpm_pbase);
503fedab560Sae
504fedab560Sae if (lo) {
505fedab560Sae num = lo->pages_end - lo->pages_base;
506fedab560Sae start = kpmptop(ptokpmp(lo->pages_base));
507fedab560Sae /* align end to kpm page size granularity */
508fedab560Sae end = kpmptop(ptokpmp(start + num - 1)) + kpmpnpgs;
509fedab560Sae lo->kpm_pbase = start;
510fedab560Sae lo->kpm_nkpmpgs = ptokpmp(end - start);
511fedab560Sae lo->kpm_pages = msp->kpm_pages;
512fedab560Sae lo->kpm_pagespa = va_to_pa(lo->kpm_pages);
513fedab560Sae lo->pagespa = va_to_pa(lo->pages);
514fedab560Sae lo->epagespa = va_to_pa(lo->epages);
515fedab560Sae lo->nextpa = va_to_pa(lo->next);
516fedab560Sae }
517fedab560Sae
518fedab560Sae /* mid */
519fedab560Sae num = mid->pages_end - mid->pages_base;
520fedab560Sae kstart = ptokpmp(mid->pages_base);
521fedab560Sae start = kpmptop(kstart);
522fedab560Sae /* align end to kpm page size granularity */
523fedab560Sae end = kpmptop(ptokpmp(start + num - 1)) + kpmpnpgs;
524fedab560Sae mid->kpm_pbase = start;
525fedab560Sae mid->kpm_nkpmpgs = ptokpmp(end - start);
526fedab560Sae if (kpm_smallpages == 0) {
527fedab560Sae mid->kpm_pages = msp->kpm_pages + (kstart - kbase);
528fedab560Sae } else {
529fedab560Sae mid->kpm_spages = msp->kpm_spages + (kstart - kbase);
530fedab560Sae }
531fedab560Sae mid->kpm_pagespa = va_to_pa(mid->kpm_pages);
532fedab560Sae mid->pagespa = va_to_pa(mid->pages);
533fedab560Sae mid->epagespa = va_to_pa(mid->epages);
534fedab560Sae mid->nextpa = (mid->next) ? va_to_pa(mid->next) : MSEG_NULLPTR_PA;
535fedab560Sae
536fedab560Sae if (hi) {
537fedab560Sae num = hi->pages_end - hi->pages_base;
538fedab560Sae kstart = ptokpmp(hi->pages_base);
539fedab560Sae start = kpmptop(kstart);
540fedab560Sae /* align end to kpm page size granularity */
541fedab560Sae end = kpmptop(ptokpmp(start + num - 1)) + kpmpnpgs;
542fedab560Sae hi->kpm_pbase = start;
543fedab560Sae hi->kpm_nkpmpgs = ptokpmp(end - start);
544fedab560Sae if (kpm_smallpages == 0) {
545fedab560Sae hi->kpm_pages = msp->kpm_pages + (kstart - kbase);
546fedab560Sae } else {
547fedab560Sae hi->kpm_spages = msp->kpm_spages + (kstart - kbase);
548fedab560Sae }
549fedab560Sae hi->kpm_pagespa = va_to_pa(hi->kpm_pages);
550fedab560Sae hi->pagespa = va_to_pa(hi->pages);
551fedab560Sae hi->epagespa = va_to_pa(hi->epages);
552fedab560Sae hi->nextpa = (hi->next) ? va_to_pa(hi->next) : MSEG_NULLPTR_PA;
553fedab560Sae }
554fedab560Sae
555fedab560Sae /*
556fedab560Sae * Atomic update of the physical memseg chain
557fedab560Sae */
558fedab560Sae if (mspp == &memsegs) {
559fedab560Sae memsegspa = (lo) ? va_to_pa(lo) : va_to_pa(mid);
560fedab560Sae } else {
561fedab560Sae lmsp = (struct memseg *)
562444ce08eSDonghai Qiao ((uint64_t)mspp - offsetof(struct memseg, next));
563fedab560Sae lmsp->nextpa = (lo) ? va_to_pa(lo) : va_to_pa(mid);
564fedab560Sae }
565fedab560Sae }
566fedab560Sae
567fedab560Sae /*
568fedab560Sae * Walk the memsegs chain, applying func to each memseg span and vcolor.
569fedab560Sae */
570fedab560Sae void
hat_kpm_walk(void (* func)(void *,void *,size_t),void * arg)571fedab560Sae hat_kpm_walk(void (*func)(void *, void *, size_t), void *arg)
572fedab560Sae {
573fedab560Sae pfn_t pbase, pend;
574fedab560Sae int vcolor;
575fedab560Sae void *base;
576fedab560Sae size_t size;
577fedab560Sae struct memseg *msp;
578fedab560Sae
579fedab560Sae for (msp = memsegs; msp; msp = msp->next) {
580fedab560Sae pbase = msp->pages_base;
581fedab560Sae pend = msp->pages_end;
582fedab560Sae for (vcolor = 0; vcolor < vac_colors; vcolor++) {
583fedab560Sae base = ptob(pbase) + kpm_vbase + kpm_size * vcolor;
584fedab560Sae size = ptob(pend - pbase);
585fedab560Sae func(arg, base, size);
586fedab560Sae }
587fedab560Sae }
588fedab560Sae }
589fedab560Sae
590fedab560Sae
591fedab560Sae /* -- sfmmu_kpm internal section -- */
592fedab560Sae
593fedab560Sae /*
594fedab560Sae * Return the page frame number if a valid segkpm mapping exists
595fedab560Sae * for vaddr, otherwise return PFN_INVALID. No locks are grabbed.
596fedab560Sae * Should only be used by other sfmmu routines.
597fedab560Sae */
598fedab560Sae pfn_t
sfmmu_kpm_vatopfn(caddr_t vaddr)599fedab560Sae sfmmu_kpm_vatopfn(caddr_t vaddr)
600fedab560Sae {
601fedab560Sae uintptr_t paddr;
602fedab560Sae pfn_t pfn;
603fedab560Sae page_t *pp;
604fedab560Sae
605fedab560Sae ASSERT(kpm_enable && IS_KPM_ADDR(vaddr));
606fedab560Sae
607fedab560Sae SFMMU_KPM_VTOP(vaddr, paddr);
608fedab560Sae pfn = (pfn_t)btop(paddr);
609fedab560Sae pp = page_numtopp_nolock(pfn);
610fedab560Sae if (pp && pp->p_kpmref)
611fedab560Sae return (pfn);
612fedab560Sae else
613fedab560Sae return ((pfn_t)PFN_INVALID);
614fedab560Sae }
615fedab560Sae
616fedab560Sae /*
617fedab560Sae * Lookup a kpme in the p_kpmelist.
618fedab560Sae */
619fedab560Sae static int
sfmmu_kpme_lookup(struct kpme * kpme,page_t * pp)620fedab560Sae sfmmu_kpme_lookup(struct kpme *kpme, page_t *pp)
621fedab560Sae {
622fedab560Sae struct kpme *p;
623fedab560Sae
624fedab560Sae for (p = pp->p_kpmelist; p; p = p->kpe_next) {
625fedab560Sae if (p == kpme)
626fedab560Sae return (1);
627fedab560Sae }
628fedab560Sae return (0);
629fedab560Sae }
630fedab560Sae
631fedab560Sae /*
632fedab560Sae * Insert a kpme into the p_kpmelist and increment
633fedab560Sae * the per page kpm reference count.
634fedab560Sae */
635fedab560Sae static void
sfmmu_kpme_add(struct kpme * kpme,page_t * pp)636fedab560Sae sfmmu_kpme_add(struct kpme *kpme, page_t *pp)
637fedab560Sae {
638fedab560Sae ASSERT(pp->p_kpmref >= 0);
639fedab560Sae
640fedab560Sae /* head insert */
641fedab560Sae kpme->kpe_prev = NULL;
642fedab560Sae kpme->kpe_next = pp->p_kpmelist;
643fedab560Sae
644fedab560Sae if (pp->p_kpmelist)
645fedab560Sae pp->p_kpmelist->kpe_prev = kpme;
646fedab560Sae
647fedab560Sae pp->p_kpmelist = kpme;
648fedab560Sae kpme->kpe_page = pp;
649fedab560Sae pp->p_kpmref++;
650fedab560Sae }
651fedab560Sae
652fedab560Sae /*
653fedab560Sae * Remove a kpme from the p_kpmelist and decrement
654fedab560Sae * the per page kpm reference count.
655fedab560Sae */
656fedab560Sae static void
sfmmu_kpme_sub(struct kpme * kpme,page_t * pp)657fedab560Sae sfmmu_kpme_sub(struct kpme *kpme, page_t *pp)
658fedab560Sae {
659fedab560Sae ASSERT(pp->p_kpmref > 0);
660fedab560Sae
661fedab560Sae if (kpme->kpe_prev) {
662fedab560Sae ASSERT(pp->p_kpmelist != kpme);
663fedab560Sae ASSERT(kpme->kpe_prev->kpe_page == pp);
664fedab560Sae kpme->kpe_prev->kpe_next = kpme->kpe_next;
665fedab560Sae } else {
666fedab560Sae ASSERT(pp->p_kpmelist == kpme);
667fedab560Sae pp->p_kpmelist = kpme->kpe_next;
668fedab560Sae }
669fedab560Sae
670fedab560Sae if (kpme->kpe_next) {
671fedab560Sae ASSERT(kpme->kpe_next->kpe_page == pp);
672fedab560Sae kpme->kpe_next->kpe_prev = kpme->kpe_prev;
673fedab560Sae }
674fedab560Sae
675fedab560Sae kpme->kpe_next = kpme->kpe_prev = NULL;
676fedab560Sae kpme->kpe_page = NULL;
677fedab560Sae pp->p_kpmref--;
678fedab560Sae }
679fedab560Sae
680fedab560Sae /*
681fedab560Sae * Mapin a single page, it is called every time a page changes it's state
682fedab560Sae * from kpm-unmapped to kpm-mapped. It may not be called, when only a new
683fedab560Sae * kpm instance does a mapin and wants to share the mapping.
684fedab560Sae * Assumes that the mlist mutex is already grabbed.
685fedab560Sae */
686fedab560Sae static caddr_t
sfmmu_kpm_mapin(page_t * pp)687fedab560Sae sfmmu_kpm_mapin(page_t *pp)
688fedab560Sae {
689fedab560Sae kpm_page_t *kp;
690fedab560Sae kpm_hlk_t *kpmp;
691fedab560Sae caddr_t vaddr;
692fedab560Sae int kpm_vac_range;
693fedab560Sae pfn_t pfn;
694fedab560Sae tte_t tte;
695fedab560Sae kmutex_t *pmtx;
696fedab560Sae int uncached;
697fedab560Sae kpm_spage_t *ksp;
698fedab560Sae kpm_shlk_t *kpmsp;
699fedab560Sae int oldval;
700fedab560Sae
701fedab560Sae ASSERT(sfmmu_mlist_held(pp));
702fedab560Sae ASSERT(pp->p_kpmref == 0);
703fedab560Sae
704fedab560Sae vaddr = sfmmu_kpm_getvaddr(pp, &kpm_vac_range);
705fedab560Sae
706fedab560Sae ASSERT(IS_KPM_ADDR(vaddr));
707fedab560Sae uncached = PP_ISNC(pp);
708fedab560Sae pfn = pp->p_pagenum;
709fedab560Sae
710fedab560Sae if (kpm_smallpages)
711fedab560Sae goto smallpages_mapin;
712fedab560Sae
713fedab560Sae PP2KPMPG(pp, kp);
714fedab560Sae
715fedab560Sae kpmp = KPMP_HASH(kp);
716fedab560Sae mutex_enter(&kpmp->khl_mutex);
717fedab560Sae
718fedab560Sae ASSERT(PP_ISKPMC(pp) == 0);
719fedab560Sae ASSERT(PP_ISKPMS(pp) == 0);
720fedab560Sae
721fedab560Sae if (uncached) {
722fedab560Sae /* ASSERT(pp->p_share); XXX use hat_page_getshare */
723fedab560Sae if (kpm_vac_range == 0) {
724fedab560Sae if (kp->kp_refcnts == 0) {
725fedab560Sae /*
726fedab560Sae * Must remove large page mapping if it exists.
727fedab560Sae * Pages in uncached state can only be mapped
728fedab560Sae * small (PAGESIZE) within the regular kpm
729fedab560Sae * range.
730fedab560Sae */
731fedab560Sae if (kp->kp_refcntc == -1) {
732fedab560Sae /* remove go indication */
733fedab560Sae sfmmu_kpm_tsbmtl(&kp->kp_refcntc,
734444ce08eSDonghai Qiao &kpmp->khl_lock, KPMTSBM_STOP);
735fedab560Sae }
736fedab560Sae if (kp->kp_refcnt > 0 && kp->kp_refcntc == 0)
737fedab560Sae sfmmu_kpm_demap_large(vaddr);
738fedab560Sae }
739fedab560Sae ASSERT(kp->kp_refcntc >= 0);
740fedab560Sae kp->kp_refcntc++;
741fedab560Sae }
742fedab560Sae pmtx = sfmmu_page_enter(pp);
743fedab560Sae PP_SETKPMC(pp);
744fedab560Sae sfmmu_page_exit(pmtx);
745fedab560Sae }
746fedab560Sae
747fedab560Sae if ((kp->kp_refcntc > 0 || kp->kp_refcnts > 0) && kpm_vac_range == 0) {
748fedab560Sae /*
749fedab560Sae * Have to do a small (PAGESIZE) mapin within this kpm_page
750fedab560Sae * range since it is marked to be in VAC conflict mode or
751fedab560Sae * when there are still other small mappings around.
752fedab560Sae */
753fedab560Sae
754fedab560Sae /* tte assembly */
755fedab560Sae if (uncached == 0)
756fedab560Sae KPM_TTE_VCACHED(tte.ll, pfn, TTE8K);
757fedab560Sae else
758fedab560Sae KPM_TTE_VUNCACHED(tte.ll, pfn, TTE8K);
759fedab560Sae
760fedab560Sae /* tsb dropin */
761fedab560Sae sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT);
762fedab560Sae
763fedab560Sae pmtx = sfmmu_page_enter(pp);
764fedab560Sae PP_SETKPMS(pp);
765fedab560Sae sfmmu_page_exit(pmtx);
766fedab560Sae
767fedab560Sae kp->kp_refcnts++;
768fedab560Sae ASSERT(kp->kp_refcnts > 0);
769fedab560Sae goto exit;
770fedab560Sae }
771fedab560Sae
772fedab560Sae if (kpm_vac_range == 0) {
773fedab560Sae /*
774fedab560Sae * Fast path / regular case, no VAC conflict handling
775fedab560Sae * in progress within this kpm_page range.
776fedab560Sae */
777fedab560Sae if (kp->kp_refcnt == 0) {
778fedab560Sae
779fedab560Sae /* tte assembly */
780fedab560Sae KPM_TTE_VCACHED(tte.ll, pfn, TTE4M);
781fedab560Sae
782fedab560Sae /* tsb dropin */
783fedab560Sae sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT4M);
784fedab560Sae
785fedab560Sae /* Set go flag for TL tsbmiss handler */
786fedab560Sae if (kp->kp_refcntc == 0)
787fedab560Sae sfmmu_kpm_tsbmtl(&kp->kp_refcntc,
788444ce08eSDonghai Qiao &kpmp->khl_lock, KPMTSBM_START);
789fedab560Sae
790fedab560Sae ASSERT(kp->kp_refcntc == -1);
791fedab560Sae }
792fedab560Sae kp->kp_refcnt++;
793fedab560Sae ASSERT(kp->kp_refcnt);
794fedab560Sae
795fedab560Sae } else {
796fedab560Sae /*
797fedab560Sae * The page is not setup according to the common VAC
798fedab560Sae * prevention rules for the regular and kpm mapping layer
799fedab560Sae * E.g. the page layer was not able to deliver a right
800fedab560Sae * vcolor'ed page for a given vaddr corresponding to
801fedab560Sae * the wanted p_offset. It has to be mapped in small in
802fedab560Sae * within the corresponding kpm vac range in order to
803fedab560Sae * prevent VAC alias conflicts.
804fedab560Sae */
805fedab560Sae
806fedab560Sae /* tte assembly */
807fedab560Sae if (uncached == 0) {
808fedab560Sae KPM_TTE_VCACHED(tte.ll, pfn, TTE8K);
809fedab560Sae } else {
810fedab560Sae KPM_TTE_VUNCACHED(tte.ll, pfn, TTE8K);
811fedab560Sae }
812fedab560Sae
813fedab560Sae /* tsb dropin */
814fedab560Sae sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT);
815fedab560Sae
816fedab560Sae kp->kp_refcnta++;
817fedab560Sae if (kp->kp_refcntc == -1) {
818fedab560Sae ASSERT(kp->kp_refcnt > 0);
819fedab560Sae
820fedab560Sae /* remove go indication */
821fedab560Sae sfmmu_kpm_tsbmtl(&kp->kp_refcntc, &kpmp->khl_lock,
822444ce08eSDonghai Qiao KPMTSBM_STOP);
823fedab560Sae }
824fedab560Sae ASSERT(kp->kp_refcntc >= 0);
825fedab560Sae }
826fedab560Sae exit:
827fedab560Sae mutex_exit(&kpmp->khl_mutex);
828fedab560Sae return (vaddr);
829fedab560Sae
830fedab560Sae smallpages_mapin:
831fedab560Sae if (uncached == 0) {
832fedab560Sae /* tte assembly */
833fedab560Sae KPM_TTE_VCACHED(tte.ll, pfn, TTE8K);
834fedab560Sae } else {
835444ce08eSDonghai Qiao /*
836444ce08eSDonghai Qiao * Just in case this same page was mapped cacheable prior to
837444ce08eSDonghai Qiao * this and the old tte remains in tlb.
838444ce08eSDonghai Qiao */
839444ce08eSDonghai Qiao sfmmu_kpm_demap_small(vaddr);
840444ce08eSDonghai Qiao
841fedab560Sae /* ASSERT(pp->p_share); XXX use hat_page_getshare */
842fedab560Sae pmtx = sfmmu_page_enter(pp);
843fedab560Sae PP_SETKPMC(pp);
844fedab560Sae sfmmu_page_exit(pmtx);
845fedab560Sae /* tte assembly */
846fedab560Sae KPM_TTE_VUNCACHED(tte.ll, pfn, TTE8K);
847fedab560Sae }
848fedab560Sae
849fedab560Sae /* tsb dropin */
850fedab560Sae sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT);
851fedab560Sae
852fedab560Sae PP2KPMSPG(pp, ksp);
853fedab560Sae kpmsp = KPMP_SHASH(ksp);
854fedab560Sae
855444ce08eSDonghai Qiao oldval = sfmmu_kpm_stsbmtl(&ksp->kp_mapped_flag, &kpmsp->kshl_lock,
856444ce08eSDonghai Qiao (uncached) ? (KPM_MAPPED_GO | KPM_MAPPEDSC) :
857444ce08eSDonghai Qiao (KPM_MAPPED_GO | KPM_MAPPEDS));
858fedab560Sae
859fedab560Sae if (oldval != 0)
860fedab560Sae panic("sfmmu_kpm_mapin: stale smallpages mapping");
861fedab560Sae
862fedab560Sae return (vaddr);
863fedab560Sae }
864fedab560Sae
865fedab560Sae /*
866fedab560Sae * Mapout a single page, it is called every time a page changes it's state
867fedab560Sae * from kpm-mapped to kpm-unmapped. It may not be called, when only a kpm
868fedab560Sae * instance calls mapout and there are still other instances mapping the
869fedab560Sae * page. Assumes that the mlist mutex is already grabbed.
870fedab560Sae *
871fedab560Sae * Note: In normal mode (no VAC conflict prevention pending) TLB's are
872fedab560Sae * not flushed. This is the core segkpm behavior to avoid xcalls. It is
873fedab560Sae * no problem because a translation from a segkpm virtual address to a
874fedab560Sae * physical address is always the same. The only downside is a slighty
875fedab560Sae * increased window of vulnerability for misbehaving _kernel_ modules.
876fedab560Sae */
877fedab560Sae static void
sfmmu_kpm_mapout(page_t * pp,caddr_t vaddr)878fedab560Sae sfmmu_kpm_mapout(page_t *pp, caddr_t vaddr)
879fedab560Sae {
880fedab560Sae kpm_page_t *kp;
881fedab560Sae kpm_hlk_t *kpmp;
882fedab560Sae int alias_range;
883fedab560Sae kmutex_t *pmtx;
884fedab560Sae kpm_spage_t *ksp;
885fedab560Sae kpm_shlk_t *kpmsp;
886fedab560Sae int oldval;
887fedab560Sae
888fedab560Sae ASSERT(sfmmu_mlist_held(pp));
889fedab560Sae ASSERT(pp->p_kpmref == 0);
890fedab560Sae
891fedab560Sae alias_range = IS_KPM_ALIAS_RANGE(vaddr);
892fedab560Sae
893fedab560Sae if (kpm_smallpages)
894fedab560Sae goto smallpages_mapout;
895fedab560Sae
896fedab560Sae PP2KPMPG(pp, kp);
897fedab560Sae kpmp = KPMP_HASH(kp);
898fedab560Sae mutex_enter(&kpmp->khl_mutex);
899fedab560Sae
900fedab560Sae if (alias_range) {
901fedab560Sae ASSERT(PP_ISKPMS(pp) == 0);
902fedab560Sae if (kp->kp_refcnta <= 0) {
903fedab560Sae panic("sfmmu_kpm_mapout: bad refcnta kp=%p",
904444ce08eSDonghai Qiao (void *)kp);
905fedab560Sae }
906fedab560Sae
907fedab560Sae if (PP_ISTNC(pp)) {
908fedab560Sae if (PP_ISKPMC(pp) == 0) {
909fedab560Sae /*
910fedab560Sae * Uncached kpm mappings must always have
911fedab560Sae * forced "small page" mode.
912fedab560Sae */
913fedab560Sae panic("sfmmu_kpm_mapout: uncached page not "
914444ce08eSDonghai Qiao "kpm marked");
915fedab560Sae }
916fedab560Sae sfmmu_kpm_demap_small(vaddr);
917fedab560Sae
918fedab560Sae pmtx = sfmmu_page_enter(pp);
919fedab560Sae PP_CLRKPMC(pp);
920fedab560Sae sfmmu_page_exit(pmtx);
921fedab560Sae
922fedab560Sae /*
923fedab560Sae * Check if we can resume cached mode. This might
924fedab560Sae * be the case if the kpm mapping was the only
925fedab560Sae * mapping in conflict with other non rule
926fedab560Sae * compliant mappings. The page is no more marked
927fedab560Sae * as kpm mapped, so the conv_tnc path will not
928fedab560Sae * change kpm state.
929fedab560Sae */
930fedab560Sae conv_tnc(pp, TTE8K);
931fedab560Sae
932fedab560Sae } else if (PP_ISKPMC(pp) == 0) {
933fedab560Sae /* remove TSB entry only */
934fedab560Sae sfmmu_kpm_unload_tsb(vaddr, MMU_PAGESHIFT);
935fedab560Sae
936fedab560Sae } else {
937fedab560Sae /* already demapped */
938fedab560Sae pmtx = sfmmu_page_enter(pp);
939fedab560Sae PP_CLRKPMC(pp);
940fedab560Sae sfmmu_page_exit(pmtx);
941fedab560Sae }
942fedab560Sae kp->kp_refcnta--;
943fedab560Sae goto exit;
944fedab560Sae }
945fedab560Sae
946fedab560Sae if (kp->kp_refcntc <= 0 && kp->kp_refcnts == 0) {
947fedab560Sae /*
948fedab560Sae * Fast path / regular case.
949fedab560Sae */
950fedab560Sae ASSERT(kp->kp_refcntc >= -1);
951fedab560Sae ASSERT(!(pp->p_nrm & (P_KPMC | P_KPMS | P_TNC | P_PNC)));
952fedab560Sae
953fedab560Sae if (kp->kp_refcnt <= 0)
954fedab560Sae panic("sfmmu_kpm_mapout: bad refcnt kp=%p", (void *)kp);
955fedab560Sae
956fedab560Sae if (--kp->kp_refcnt == 0) {
957fedab560Sae /* remove go indication */
958fedab560Sae if (kp->kp_refcntc == -1) {
959fedab560Sae sfmmu_kpm_tsbmtl(&kp->kp_refcntc,
960444ce08eSDonghai Qiao &kpmp->khl_lock, KPMTSBM_STOP);
961fedab560Sae }
962fedab560Sae ASSERT(kp->kp_refcntc == 0);
963fedab560Sae
964fedab560Sae /* remove TSB entry */
965fedab560Sae sfmmu_kpm_unload_tsb(vaddr, MMU_PAGESHIFT4M);
966fedab560Sae #ifdef DEBUG
967fedab560Sae if (kpm_tlb_flush)
968fedab560Sae sfmmu_kpm_demap_tlbs(vaddr);
969fedab560Sae #endif
970fedab560Sae }
971fedab560Sae
972fedab560Sae } else {
973fedab560Sae /*
974fedab560Sae * The VAC alias path.
975fedab560Sae * We come here if the kpm vaddr is not in any alias_range
976fedab560Sae * and we are unmapping a page within the regular kpm_page
977fedab560Sae * range. The kpm_page either holds conflict pages and/or
978fedab560Sae * is in "small page" mode. If the page is not marked
979fedab560Sae * P_KPMS it couldn't have a valid PAGESIZE sized TSB
980fedab560Sae * entry. Dcache flushing is done lazy and follows the
981fedab560Sae * rules of the regular virtual page coloring scheme.
982fedab560Sae *
983fedab560Sae * Per page states and required actions:
984fedab560Sae * P_KPMC: remove a kpm mapping that is conflicting.
985fedab560Sae * P_KPMS: remove a small kpm mapping within a kpm_page.
986fedab560Sae * P_TNC: check if we can re-cache the page.
987fedab560Sae * P_PNC: we cannot re-cache, sorry.
988fedab560Sae * Per kpm_page:
989fedab560Sae * kp_refcntc > 0: page is part of a kpm_page with conflicts.
990fedab560Sae * kp_refcnts > 0: rm a small mapped page within a kpm_page.
991fedab560Sae */
992fedab560Sae
993fedab560Sae if (PP_ISKPMS(pp)) {
994fedab560Sae if (kp->kp_refcnts < 1) {
995fedab560Sae panic("sfmmu_kpm_mapout: bad refcnts kp=%p",
996444ce08eSDonghai Qiao (void *)kp);
997fedab560Sae }
998fedab560Sae sfmmu_kpm_demap_small(vaddr);
999fedab560Sae
1000fedab560Sae /*
1001fedab560Sae * Check if we can resume cached mode. This might
1002fedab560Sae * be the case if the kpm mapping was the only
1003fedab560Sae * mapping in conflict with other non rule
1004fedab560Sae * compliant mappings. The page is no more marked
1005fedab560Sae * as kpm mapped, so the conv_tnc path will not
1006fedab560Sae * change kpm state.
1007fedab560Sae */
1008fedab560Sae if (PP_ISTNC(pp)) {
1009fedab560Sae if (!PP_ISKPMC(pp)) {
1010fedab560Sae /*
1011fedab560Sae * Uncached kpm mappings must always
1012fedab560Sae * have forced "small page" mode.
1013fedab560Sae */
1014fedab560Sae panic("sfmmu_kpm_mapout: uncached "
1015444ce08eSDonghai Qiao "page not kpm marked");
1016fedab560Sae }
1017fedab560Sae conv_tnc(pp, TTE8K);
1018fedab560Sae }
1019fedab560Sae kp->kp_refcnts--;
1020fedab560Sae kp->kp_refcnt++;
1021fedab560Sae pmtx = sfmmu_page_enter(pp);
1022fedab560Sae PP_CLRKPMS(pp);
1023fedab560Sae sfmmu_page_exit(pmtx);
1024fedab560Sae }
1025fedab560Sae
1026fedab560Sae if (PP_ISKPMC(pp)) {
1027fedab560Sae if (kp->kp_refcntc < 1) {
1028fedab560Sae panic("sfmmu_kpm_mapout: bad refcntc kp=%p",
1029444ce08eSDonghai Qiao (void *)kp);
1030fedab560Sae }
1031fedab560Sae pmtx = sfmmu_page_enter(pp);
1032fedab560Sae PP_CLRKPMC(pp);
1033fedab560Sae sfmmu_page_exit(pmtx);
1034fedab560Sae kp->kp_refcntc--;
1035fedab560Sae }
1036fedab560Sae
1037fedab560Sae if (kp->kp_refcnt-- < 1)
1038fedab560Sae panic("sfmmu_kpm_mapout: bad refcnt kp=%p", (void *)kp);
1039fedab560Sae }
1040fedab560Sae exit:
1041fedab560Sae mutex_exit(&kpmp->khl_mutex);
1042fedab560Sae return;
1043fedab560Sae
1044fedab560Sae smallpages_mapout:
1045fedab560Sae PP2KPMSPG(pp, ksp);
1046fedab560Sae kpmsp = KPMP_SHASH(ksp);
1047fedab560Sae
1048fedab560Sae if (PP_ISKPMC(pp) == 0) {
1049444ce08eSDonghai Qiao oldval = sfmmu_kpm_stsbmtl(&ksp->kp_mapped_flag,
1050444ce08eSDonghai Qiao &kpmsp->kshl_lock, 0);
1051fedab560Sae
1052fedab560Sae if (oldval != KPM_MAPPEDS) {
1053fedab560Sae /*
1054fedab560Sae * When we're called after sfmmu_kpm_hme_unload,
1055fedab560Sae * KPM_MAPPEDSC is valid too.
1056fedab560Sae */
1057fedab560Sae if (oldval != KPM_MAPPEDSC)
1058fedab560Sae panic("sfmmu_kpm_mapout: incorrect mapping");
1059fedab560Sae }
1060fedab560Sae
1061fedab560Sae /* remove TSB entry */
1062fedab560Sae sfmmu_kpm_unload_tsb(vaddr, MMU_PAGESHIFT);
1063fedab560Sae #ifdef DEBUG
1064fedab560Sae if (kpm_tlb_flush)
1065fedab560Sae sfmmu_kpm_demap_tlbs(vaddr);
1066fedab560Sae #endif
1067fedab560Sae
1068fedab560Sae } else if (PP_ISTNC(pp)) {
1069444ce08eSDonghai Qiao oldval = sfmmu_kpm_stsbmtl(&ksp->kp_mapped_flag,
1070444ce08eSDonghai Qiao &kpmsp->kshl_lock, 0);
1071fedab560Sae
1072fedab560Sae if (oldval != KPM_MAPPEDSC || PP_ISKPMC(pp) == 0)
1073fedab560Sae panic("sfmmu_kpm_mapout: inconsistent TNC mapping");
1074fedab560Sae
1075fedab560Sae sfmmu_kpm_demap_small(vaddr);
1076fedab560Sae
1077fedab560Sae pmtx = sfmmu_page_enter(pp);
1078fedab560Sae PP_CLRKPMC(pp);
1079fedab560Sae sfmmu_page_exit(pmtx);
1080fedab560Sae
1081fedab560Sae /*
1082fedab560Sae * Check if we can resume cached mode. This might be
1083fedab560Sae * the case if the kpm mapping was the only mapping
1084fedab560Sae * in conflict with other non rule compliant mappings.
1085fedab560Sae * The page is no more marked as kpm mapped, so the
1086fedab560Sae * conv_tnc path will not change the kpm state.
1087fedab560Sae */
1088fedab560Sae conv_tnc(pp, TTE8K);
1089fedab560Sae
1090fedab560Sae } else {
1091444ce08eSDonghai Qiao oldval = sfmmu_kpm_stsbmtl(&ksp->kp_mapped_flag,
1092444ce08eSDonghai Qiao &kpmsp->kshl_lock, 0);
1093fedab560Sae
1094fedab560Sae if (oldval != KPM_MAPPEDSC)
1095fedab560Sae panic("sfmmu_kpm_mapout: inconsistent mapping");
1096fedab560Sae
1097fedab560Sae pmtx = sfmmu_page_enter(pp);
1098fedab560Sae PP_CLRKPMC(pp);
1099fedab560Sae sfmmu_page_exit(pmtx);
1100fedab560Sae }
1101fedab560Sae }
1102fedab560Sae
1103fedab560Sae #define abs(x) ((x) < 0 ? -(x) : (x))
1104fedab560Sae
1105fedab560Sae /*
1106fedab560Sae * Determine appropriate kpm mapping address and handle any kpm/hme
1107fedab560Sae * conflicts. Page mapping list and its vcolor parts must be protected.
1108fedab560Sae */
1109fedab560Sae static caddr_t
sfmmu_kpm_getvaddr(page_t * pp,int * kpm_vac_rangep)1110fedab560Sae sfmmu_kpm_getvaddr(page_t *pp, int *kpm_vac_rangep)
1111fedab560Sae {
1112fedab560Sae int vcolor, vcolor_pa;
1113fedab560Sae caddr_t vaddr;
1114fedab560Sae uintptr_t paddr;
1115fedab560Sae
1116fedab560Sae
1117fedab560Sae ASSERT(sfmmu_mlist_held(pp));
1118fedab560Sae
1119fedab560Sae paddr = ptob(pp->p_pagenum);
1120fedab560Sae vcolor_pa = addr_to_vcolor(paddr);
1121fedab560Sae
1122fedab560Sae if (pp->p_vnode && IS_SWAPFSVP(pp->p_vnode)) {
1123fedab560Sae vcolor = (PP_NEWPAGE(pp) || PP_ISNC(pp)) ?
1124fedab560Sae vcolor_pa : PP_GET_VCOLOR(pp);
1125fedab560Sae } else {
1126fedab560Sae vcolor = addr_to_vcolor(pp->p_offset);
1127fedab560Sae }
1128fedab560Sae
1129fedab560Sae vaddr = kpm_vbase + paddr;
1130fedab560Sae *kpm_vac_rangep = 0;
1131fedab560Sae
1132fedab560Sae if (vcolor_pa != vcolor) {
1133fedab560Sae *kpm_vac_rangep = abs(vcolor - vcolor_pa);
1134fedab560Sae vaddr += ((uintptr_t)(vcolor - vcolor_pa) << MMU_PAGESHIFT);
1135fedab560Sae vaddr += (vcolor_pa > vcolor) ?
1136444ce08eSDonghai Qiao ((uintptr_t)vcolor_pa << kpm_size_shift) :
1137444ce08eSDonghai Qiao ((uintptr_t)(vcolor - vcolor_pa) << kpm_size_shift);
1138fedab560Sae
1139fedab560Sae ASSERT(!PP_ISMAPPED_LARGE(pp));
1140fedab560Sae }
1141fedab560Sae
1142fedab560Sae if (PP_ISNC(pp))
1143fedab560Sae return (vaddr);
1144fedab560Sae
1145fedab560Sae if (PP_NEWPAGE(pp)) {
1146fedab560Sae PP_SET_VCOLOR(pp, vcolor);
1147fedab560Sae return (vaddr);
1148fedab560Sae }
1149fedab560Sae
1150fedab560Sae if (PP_GET_VCOLOR(pp) == vcolor)
1151fedab560Sae return (vaddr);
1152fedab560Sae
1153fedab560Sae ASSERT(!PP_ISMAPPED_KPM(pp));
1154fedab560Sae sfmmu_kpm_vac_conflict(pp, vaddr);
1155fedab560Sae
1156fedab560Sae return (vaddr);
1157fedab560Sae }
1158fedab560Sae
1159fedab560Sae /*
1160fedab560Sae * VAC conflict state bit values.
1161fedab560Sae * The following defines are used to make the handling of the
1162fedab560Sae * various input states more concise. For that the kpm states
1163fedab560Sae * per kpm_page and per page are combined in a summary state.
1164fedab560Sae * Each single state has a corresponding bit value in the
1165fedab560Sae * summary state. These defines only apply for kpm large page
1166fedab560Sae * mappings. Within comments the abbreviations "kc, c, ks, s"
1167fedab560Sae * are used as short form of the actual state, e.g. "kc" for
1168fedab560Sae * "kp_refcntc > 0", etc.
1169fedab560Sae */
1170fedab560Sae #define KPM_KC 0x00000008 /* kpm_page: kp_refcntc > 0 */
1171fedab560Sae #define KPM_C 0x00000004 /* page: P_KPMC set */
1172fedab560Sae #define KPM_KS 0x00000002 /* kpm_page: kp_refcnts > 0 */
1173fedab560Sae #define KPM_S 0x00000001 /* page: P_KPMS set */
1174fedab560Sae
1175fedab560Sae /*
1176fedab560Sae * Summary states used in sfmmu_kpm_fault (KPM_TSBM_*).
1177fedab560Sae * See also more detailed comments within in the sfmmu_kpm_fault switch.
1178fedab560Sae * Abbreviations used:
1179fedab560Sae * CONFL: VAC conflict(s) within a kpm_page.
1180fedab560Sae * MAPS: Mapped small: Page mapped in using a regular page size kpm mapping.
1181fedab560Sae * RASM: Re-assembling of a large page mapping possible.
1182fedab560Sae * RPLS: Replace: TSB miss due to TSB replacement only.
1183fedab560Sae * BRKO: Breakup Other: A large kpm mapping has to be broken because another
1184fedab560Sae * page within the kpm_page is already involved in a VAC conflict.
1185fedab560Sae * BRKT: Breakup This: A large kpm mapping has to be broken, this page is
1186fedab560Sae * is involved in a VAC conflict.
1187fedab560Sae */
1188fedab560Sae #define KPM_TSBM_CONFL_GONE (0)
1189fedab560Sae #define KPM_TSBM_MAPS_RASM (KPM_KS)
1190fedab560Sae #define KPM_TSBM_RPLS_RASM (KPM_KS | KPM_S)
1191fedab560Sae #define KPM_TSBM_MAPS_BRKO (KPM_KC)
1192fedab560Sae #define KPM_TSBM_MAPS (KPM_KC | KPM_KS)
1193fedab560Sae #define KPM_TSBM_RPLS (KPM_KC | KPM_KS | KPM_S)
1194fedab560Sae #define KPM_TSBM_MAPS_BRKT (KPM_KC | KPM_C)
1195fedab560Sae #define KPM_TSBM_MAPS_CONFL (KPM_KC | KPM_C | KPM_KS)
1196fedab560Sae #define KPM_TSBM_RPLS_CONFL (KPM_KC | KPM_C | KPM_KS | KPM_S)
1197fedab560Sae
1198fedab560Sae /*
1199fedab560Sae * kpm fault handler for mappings with large page size.
1200fedab560Sae */
1201fedab560Sae int
sfmmu_kpm_fault(caddr_t vaddr,struct memseg * mseg,page_t * pp)1202fedab560Sae sfmmu_kpm_fault(caddr_t vaddr, struct memseg *mseg, page_t *pp)
1203fedab560Sae {
1204fedab560Sae int error;
1205fedab560Sae pgcnt_t inx;
1206fedab560Sae kpm_page_t *kp;
1207fedab560Sae tte_t tte;
1208fedab560Sae pfn_t pfn = pp->p_pagenum;
1209fedab560Sae kpm_hlk_t *kpmp;
1210fedab560Sae kmutex_t *pml;
1211fedab560Sae int alias_range;
1212fedab560Sae int uncached = 0;
1213fedab560Sae kmutex_t *pmtx;
1214fedab560Sae int badstate;
1215fedab560Sae uint_t tsbmcase;
1216fedab560Sae
1217fedab560Sae alias_range = IS_KPM_ALIAS_RANGE(vaddr);
1218fedab560Sae
1219fedab560Sae inx = ptokpmp(kpmptop(ptokpmp(pfn)) - mseg->kpm_pbase);
1220fedab560Sae if (inx >= mseg->kpm_nkpmpgs) {
1221fedab560Sae cmn_err(CE_PANIC, "sfmmu_kpm_fault: kpm overflow in memseg "
1222444ce08eSDonghai Qiao "0x%p pp 0x%p", (void *)mseg, (void *)pp);
1223fedab560Sae }
1224fedab560Sae
1225fedab560Sae kp = &mseg->kpm_pages[inx];
1226fedab560Sae kpmp = KPMP_HASH(kp);
1227fedab560Sae
1228fedab560Sae pml = sfmmu_mlist_enter(pp);
1229fedab560Sae
1230fedab560Sae if (!PP_ISMAPPED_KPM(pp)) {
1231fedab560Sae sfmmu_mlist_exit(pml);
1232fedab560Sae return (EFAULT);
1233fedab560Sae }
1234fedab560Sae
1235fedab560Sae mutex_enter(&kpmp->khl_mutex);
1236fedab560Sae
1237fedab560Sae if (alias_range) {
1238fedab560Sae ASSERT(!PP_ISMAPPED_LARGE(pp));
1239fedab560Sae if (kp->kp_refcnta > 0) {
1240fedab560Sae if (PP_ISKPMC(pp)) {
1241fedab560Sae pmtx = sfmmu_page_enter(pp);
1242fedab560Sae PP_CLRKPMC(pp);
1243fedab560Sae sfmmu_page_exit(pmtx);
1244fedab560Sae }
1245fedab560Sae /*
1246fedab560Sae * Check for vcolor conflicts. Return here
1247fedab560Sae * w/ either no conflict (fast path), removed hme
1248fedab560Sae * mapping chains (unload conflict) or uncached
1249fedab560Sae * (uncache conflict). VACaches are cleaned and
1250fedab560Sae * p_vcolor and PP_TNC are set accordingly for the
1251fedab560Sae * conflict cases. Drop kpmp for uncache conflict
1252fedab560Sae * cases since it will be grabbed within
1253fedab560Sae * sfmmu_kpm_page_cache in case of an uncache
1254fedab560Sae * conflict.
1255fedab560Sae */
1256fedab560Sae mutex_exit(&kpmp->khl_mutex);
1257fedab560Sae sfmmu_kpm_vac_conflict(pp, vaddr);
1258fedab560Sae mutex_enter(&kpmp->khl_mutex);
1259fedab560Sae
1260fedab560Sae if (PP_ISNC(pp)) {
1261fedab560Sae uncached = 1;
1262fedab560Sae pmtx = sfmmu_page_enter(pp);
1263fedab560Sae PP_SETKPMC(pp);
1264fedab560Sae sfmmu_page_exit(pmtx);
1265fedab560Sae }
1266fedab560Sae goto smallexit;
1267fedab560Sae
1268fedab560Sae } else {
1269fedab560Sae /*
1270fedab560Sae * We got a tsbmiss on a not active kpm_page range.
1271fedab560Sae * Let segkpm_fault decide how to panic.
1272fedab560Sae */
1273fedab560Sae error = EFAULT;
1274fedab560Sae }
1275fedab560Sae goto exit;
1276fedab560Sae }
1277fedab560Sae
1278fedab560Sae badstate = (kp->kp_refcnt < 0 || kp->kp_refcnts < 0);
1279fedab560Sae if (kp->kp_refcntc == -1) {
1280fedab560Sae /*
1281fedab560Sae * We should come here only if trap level tsb miss
1282fedab560Sae * handler is disabled.
1283fedab560Sae */
1284fedab560Sae badstate |= (kp->kp_refcnt == 0 || kp->kp_refcnts > 0 ||
1285444ce08eSDonghai Qiao PP_ISKPMC(pp) || PP_ISKPMS(pp) || PP_ISNC(pp));
1286fedab560Sae
1287fedab560Sae if (badstate == 0)
1288fedab560Sae goto largeexit;
1289fedab560Sae }
1290fedab560Sae
1291fedab560Sae if (badstate || kp->kp_refcntc < 0)
1292fedab560Sae goto badstate_exit;
1293fedab560Sae
1294fedab560Sae /*
1295fedab560Sae * Combine the per kpm_page and per page kpm VAC states to
1296fedab560Sae * a summary state in order to make the kpm fault handling
1297fedab560Sae * more concise.
1298fedab560Sae */
1299fedab560Sae tsbmcase = (((kp->kp_refcntc > 0) ? KPM_KC : 0) |
1300444ce08eSDonghai Qiao ((kp->kp_refcnts > 0) ? KPM_KS : 0) |
1301444ce08eSDonghai Qiao (PP_ISKPMC(pp) ? KPM_C : 0) |
1302444ce08eSDonghai Qiao (PP_ISKPMS(pp) ? KPM_S : 0));
1303fedab560Sae
1304fedab560Sae switch (tsbmcase) {
1305fedab560Sae case KPM_TSBM_CONFL_GONE: /* - - - - */
1306fedab560Sae /*
1307fedab560Sae * That's fine, we either have no more vac conflict in
1308fedab560Sae * this kpm page or someone raced in and has solved the
1309fedab560Sae * vac conflict for us -- call sfmmu_kpm_vac_conflict
1310fedab560Sae * to take care for correcting the vcolor and flushing
1311fedab560Sae * the dcache if required.
1312fedab560Sae */
1313fedab560Sae mutex_exit(&kpmp->khl_mutex);
1314fedab560Sae sfmmu_kpm_vac_conflict(pp, vaddr);
1315fedab560Sae mutex_enter(&kpmp->khl_mutex);
1316fedab560Sae
1317fedab560Sae if (PP_ISNC(pp) || kp->kp_refcnt <= 0 ||
1318fedab560Sae addr_to_vcolor(vaddr) != PP_GET_VCOLOR(pp)) {
1319fedab560Sae panic("sfmmu_kpm_fault: inconsistent CONFL_GONE "
1320444ce08eSDonghai Qiao "state, pp=%p", (void *)pp);
1321fedab560Sae }
1322fedab560Sae goto largeexit;
1323fedab560Sae
1324fedab560Sae case KPM_TSBM_MAPS_RASM: /* - - ks - */
1325fedab560Sae /*
1326fedab560Sae * All conflicts in this kpm page are gone but there are
1327fedab560Sae * already small mappings around, so we also map this
1328fedab560Sae * page small. This could be the trigger case for a
1329fedab560Sae * small mapping reaper, if this is really needed.
1330fedab560Sae * For now fall thru to the KPM_TSBM_MAPS handling.
1331fedab560Sae */
1332fedab560Sae
1333fedab560Sae case KPM_TSBM_MAPS: /* kc - ks - */
1334fedab560Sae /*
1335fedab560Sae * Large page mapping is already broken, this page is not
1336fedab560Sae * conflicting, so map it small. Call sfmmu_kpm_vac_conflict
1337fedab560Sae * to take care for correcting the vcolor and flushing
1338fedab560Sae * the dcache if required.
1339fedab560Sae */
1340fedab560Sae mutex_exit(&kpmp->khl_mutex);
1341fedab560Sae sfmmu_kpm_vac_conflict(pp, vaddr);
1342fedab560Sae mutex_enter(&kpmp->khl_mutex);
1343fedab560Sae
1344fedab560Sae if (PP_ISNC(pp) || kp->kp_refcnt <= 0 ||
1345fedab560Sae addr_to_vcolor(vaddr) != PP_GET_VCOLOR(pp)) {
1346fedab560Sae panic("sfmmu_kpm_fault: inconsistent MAPS state, "
1347444ce08eSDonghai Qiao "pp=%p", (void *)pp);
1348fedab560Sae }
1349fedab560Sae kp->kp_refcnt--;
1350fedab560Sae kp->kp_refcnts++;
1351fedab560Sae pmtx = sfmmu_page_enter(pp);
1352fedab560Sae PP_SETKPMS(pp);
1353fedab560Sae sfmmu_page_exit(pmtx);
1354fedab560Sae goto smallexit;
1355fedab560Sae
1356fedab560Sae case KPM_TSBM_RPLS_RASM: /* - - ks s */
1357fedab560Sae /*
1358fedab560Sae * All conflicts in this kpm page are gone but this page
1359fedab560Sae * is mapped small. This could be the trigger case for a
1360fedab560Sae * small mapping reaper, if this is really needed.
1361fedab560Sae * For now we drop it in small again. Fall thru to the
1362fedab560Sae * KPM_TSBM_RPLS handling.
1363fedab560Sae */
1364fedab560Sae
1365fedab560Sae case KPM_TSBM_RPLS: /* kc - ks s */
1366fedab560Sae /*
1367fedab560Sae * Large page mapping is already broken, this page is not
1368fedab560Sae * conflicting but already mapped small, so drop it in
1369fedab560Sae * small again.
1370fedab560Sae */
1371fedab560Sae if (PP_ISNC(pp) ||
1372fedab560Sae addr_to_vcolor(vaddr) != PP_GET_VCOLOR(pp)) {
1373fedab560Sae panic("sfmmu_kpm_fault: inconsistent RPLS state, "
1374444ce08eSDonghai Qiao "pp=%p", (void *)pp);
1375fedab560Sae }
1376fedab560Sae goto smallexit;
1377fedab560Sae
1378fedab560Sae case KPM_TSBM_MAPS_BRKO: /* kc - - - */
1379fedab560Sae /*
1380fedab560Sae * The kpm page where we live in is marked conflicting
1381fedab560Sae * but this page is not conflicting. So we have to map it
1382fedab560Sae * in small. Call sfmmu_kpm_vac_conflict to take care for
1383fedab560Sae * correcting the vcolor and flushing the dcache if required.
1384fedab560Sae */
1385fedab560Sae mutex_exit(&kpmp->khl_mutex);
1386fedab560Sae sfmmu_kpm_vac_conflict(pp, vaddr);
1387fedab560Sae mutex_enter(&kpmp->khl_mutex);
1388fedab560Sae
1389fedab560Sae if (PP_ISNC(pp) || kp->kp_refcnt <= 0 ||
1390fedab560Sae addr_to_vcolor(vaddr) != PP_GET_VCOLOR(pp)) {
1391fedab560Sae panic("sfmmu_kpm_fault: inconsistent MAPS_BRKO state, "
1392444ce08eSDonghai Qiao "pp=%p", (void *)pp);
1393fedab560Sae }
1394fedab560Sae kp->kp_refcnt--;
1395fedab560Sae kp->kp_refcnts++;
1396fedab560Sae pmtx = sfmmu_page_enter(pp);
1397fedab560Sae PP_SETKPMS(pp);
1398fedab560Sae sfmmu_page_exit(pmtx);
1399fedab560Sae goto smallexit;
1400fedab560Sae
1401fedab560Sae case KPM_TSBM_MAPS_BRKT: /* kc c - - */
1402fedab560Sae case KPM_TSBM_MAPS_CONFL: /* kc c ks - */
1403fedab560Sae if (!PP_ISMAPPED(pp)) {
1404fedab560Sae /*
1405fedab560Sae * We got a tsbmiss on kpm large page range that is
1406fedab560Sae * marked to contain vac conflicting pages introduced
1407fedab560Sae * by hme mappings. The hme mappings are all gone and
1408fedab560Sae * must have bypassed the kpm alias prevention logic.
1409fedab560Sae */
1410fedab560Sae panic("sfmmu_kpm_fault: stale VAC conflict, pp=%p",
1411444ce08eSDonghai Qiao (void *)pp);
1412fedab560Sae }
1413fedab560Sae
1414fedab560Sae /*
1415fedab560Sae * Check for vcolor conflicts. Return here w/ either no
1416fedab560Sae * conflict (fast path), removed hme mapping chains
1417fedab560Sae * (unload conflict) or uncached (uncache conflict).
1418fedab560Sae * Dcache is cleaned and p_vcolor and P_TNC are set
1419fedab560Sae * accordingly. Drop kpmp for uncache conflict cases
1420fedab560Sae * since it will be grabbed within sfmmu_kpm_page_cache
1421fedab560Sae * in case of an uncache conflict.
1422fedab560Sae */
1423fedab560Sae mutex_exit(&kpmp->khl_mutex);
1424fedab560Sae sfmmu_kpm_vac_conflict(pp, vaddr);
1425fedab560Sae mutex_enter(&kpmp->khl_mutex);
1426fedab560Sae
1427fedab560Sae if (kp->kp_refcnt <= 0)
1428fedab560Sae panic("sfmmu_kpm_fault: bad refcnt kp=%p", (void *)kp);
1429fedab560Sae
1430fedab560Sae if (PP_ISNC(pp)) {
1431fedab560Sae uncached = 1;
1432fedab560Sae } else {
1433fedab560Sae /*
1434fedab560Sae * When an unload conflict is solved and there are
1435fedab560Sae * no other small mappings around, we can resume
1436fedab560Sae * largepage mode. Otherwise we have to map or drop
1437fedab560Sae * in small. This could be a trigger for a small
1438fedab560Sae * mapping reaper when this was the last conflict
1439fedab560Sae * within the kpm page and when there are only
1440fedab560Sae * other small mappings around.
1441fedab560Sae */
1442fedab560Sae ASSERT(addr_to_vcolor(vaddr) == PP_GET_VCOLOR(pp));
1443fedab560Sae ASSERT(kp->kp_refcntc > 0);
1444fedab560Sae kp->kp_refcntc--;
1445fedab560Sae pmtx = sfmmu_page_enter(pp);
1446fedab560Sae PP_CLRKPMC(pp);
1447fedab560Sae sfmmu_page_exit(pmtx);
1448fedab560Sae ASSERT(PP_ISKPMS(pp) == 0);
1449fedab560Sae if (kp->kp_refcntc == 0 && kp->kp_refcnts == 0)
1450fedab560Sae goto largeexit;
1451fedab560Sae }
1452fedab560Sae
1453fedab560Sae kp->kp_refcnt--;
1454fedab560Sae kp->kp_refcnts++;
1455fedab560Sae pmtx = sfmmu_page_enter(pp);
1456fedab560Sae PP_SETKPMS(pp);
1457fedab560Sae sfmmu_page_exit(pmtx);
1458fedab560Sae goto smallexit;
1459fedab560Sae
1460fedab560Sae case KPM_TSBM_RPLS_CONFL: /* kc c ks s */
1461fedab560Sae if (!PP_ISMAPPED(pp)) {
1462fedab560Sae /*
1463fedab560Sae * We got a tsbmiss on kpm large page range that is
1464fedab560Sae * marked to contain vac conflicting pages introduced
1465fedab560Sae * by hme mappings. They are all gone and must have
1466fedab560Sae * somehow bypassed the kpm alias prevention logic.
1467fedab560Sae */
1468fedab560Sae panic("sfmmu_kpm_fault: stale VAC conflict, pp=%p",
1469444ce08eSDonghai Qiao (void *)pp);
1470fedab560Sae }
1471fedab560Sae
1472fedab560Sae /*
1473fedab560Sae * This state is only possible for an uncached mapping.
1474fedab560Sae */
1475fedab560Sae if (!PP_ISNC(pp)) {
1476fedab560Sae panic("sfmmu_kpm_fault: page not uncached, pp=%p",
1477444ce08eSDonghai Qiao (void *)pp);
1478fedab560Sae }
1479fedab560Sae uncached = 1;
1480fedab560Sae goto smallexit;
1481fedab560Sae
1482fedab560Sae default:
1483fedab560Sae badstate_exit:
1484fedab560Sae panic("sfmmu_kpm_fault: inconsistent VAC state, vaddr=%p kp=%p "
1485444ce08eSDonghai Qiao "pp=%p", (void *)vaddr, (void *)kp, (void *)pp);
1486fedab560Sae }
1487fedab560Sae
1488fedab560Sae smallexit:
1489fedab560Sae /* tte assembly */
1490fedab560Sae if (uncached == 0)
1491fedab560Sae KPM_TTE_VCACHED(tte.ll, pfn, TTE8K);
1492fedab560Sae else
1493fedab560Sae KPM_TTE_VUNCACHED(tte.ll, pfn, TTE8K);
1494fedab560Sae
1495fedab560Sae /* tsb dropin */
1496fedab560Sae sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT);
1497fedab560Sae
1498fedab560Sae error = 0;
1499fedab560Sae goto exit;
1500fedab560Sae
1501fedab560Sae largeexit:
1502fedab560Sae if (kp->kp_refcnt > 0) {
1503fedab560Sae
1504fedab560Sae /* tte assembly */
1505fedab560Sae KPM_TTE_VCACHED(tte.ll, pfn, TTE4M);
1506fedab560Sae
1507fedab560Sae /* tsb dropin */
1508fedab560Sae sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT4M);
1509fedab560Sae
1510fedab560Sae if (kp->kp_refcntc == 0) {
1511fedab560Sae /* Set "go" flag for TL tsbmiss handler */
1512fedab560Sae sfmmu_kpm_tsbmtl(&kp->kp_refcntc, &kpmp->khl_lock,
1513444ce08eSDonghai Qiao KPMTSBM_START);
1514fedab560Sae }
1515fedab560Sae ASSERT(kp->kp_refcntc == -1);
1516fedab560Sae error = 0;
1517fedab560Sae
1518fedab560Sae } else
1519fedab560Sae error = EFAULT;
1520fedab560Sae exit:
1521fedab560Sae mutex_exit(&kpmp->khl_mutex);
1522fedab560Sae sfmmu_mlist_exit(pml);
1523fedab560Sae return (error);
1524fedab560Sae }
1525fedab560Sae
1526fedab560Sae /*
1527fedab560Sae * kpm fault handler for mappings with small page size.
1528fedab560Sae */
1529fedab560Sae int
sfmmu_kpm_fault_small(caddr_t vaddr,struct memseg * mseg,page_t * pp)1530fedab560Sae sfmmu_kpm_fault_small(caddr_t vaddr, struct memseg *mseg, page_t *pp)
1531fedab560Sae {
1532fedab560Sae int error = 0;
1533fedab560Sae pgcnt_t inx;
1534fedab560Sae kpm_spage_t *ksp;
1535fedab560Sae kpm_shlk_t *kpmsp;
1536fedab560Sae kmutex_t *pml;
1537fedab560Sae pfn_t pfn = pp->p_pagenum;
1538fedab560Sae tte_t tte;
1539fedab560Sae kmutex_t *pmtx;
1540fedab560Sae int oldval;
1541fedab560Sae
1542fedab560Sae inx = pfn - mseg->kpm_pbase;
1543fedab560Sae ksp = &mseg->kpm_spages[inx];
1544fedab560Sae kpmsp = KPMP_SHASH(ksp);
1545fedab560Sae
1546fedab560Sae pml = sfmmu_mlist_enter(pp);
1547fedab560Sae
1548fedab560Sae if (!PP_ISMAPPED_KPM(pp)) {
1549fedab560Sae sfmmu_mlist_exit(pml);
1550fedab560Sae return (EFAULT);
1551fedab560Sae }
1552fedab560Sae
1553fedab560Sae /*
1554fedab560Sae * kp_mapped lookup protected by mlist mutex
1555fedab560Sae */
1556fedab560Sae if (ksp->kp_mapped == KPM_MAPPEDS) {
1557fedab560Sae /*
1558fedab560Sae * Fast path tsbmiss
1559fedab560Sae */
1560fedab560Sae ASSERT(!PP_ISKPMC(pp));
1561fedab560Sae ASSERT(!PP_ISNC(pp));
1562fedab560Sae
1563fedab560Sae /* tte assembly */
1564fedab560Sae KPM_TTE_VCACHED(tte.ll, pfn, TTE8K);
1565fedab560Sae
1566fedab560Sae /* tsb dropin */
1567fedab560Sae sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT);
1568fedab560Sae
1569fedab560Sae } else if (ksp->kp_mapped == KPM_MAPPEDSC) {
1570fedab560Sae /*
1571fedab560Sae * Got here due to existing or gone kpm/hme VAC conflict.
1572fedab560Sae * Recheck for vcolor conflicts. Return here w/ either
1573fedab560Sae * no conflict, removed hme mapping chain (unload
1574fedab560Sae * conflict) or uncached (uncache conflict). VACaches
1575fedab560Sae * are cleaned and p_vcolor and PP_TNC are set accordingly
1576fedab560Sae * for the conflict cases.
1577fedab560Sae */
1578fedab560Sae sfmmu_kpm_vac_conflict(pp, vaddr);
1579fedab560Sae
1580fedab560Sae if (PP_ISNC(pp)) {
1581fedab560Sae /* ASSERT(pp->p_share); XXX use hat_page_getshare */
1582fedab560Sae
1583fedab560Sae /* tte assembly */
1584fedab560Sae KPM_TTE_VUNCACHED(tte.ll, pfn, TTE8K);
1585fedab560Sae
1586fedab560Sae /* tsb dropin */
1587fedab560Sae sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT);
1588fedab560Sae
1589444ce08eSDonghai Qiao oldval = sfmmu_kpm_stsbmtl(&ksp->kp_mapped_flag,
1590444ce08eSDonghai Qiao &kpmsp->kshl_lock, (KPM_MAPPED_GO | KPM_MAPPEDSC));
1591444ce08eSDonghai Qiao
1592444ce08eSDonghai Qiao if (oldval != KPM_MAPPEDSC)
1593444ce08eSDonghai Qiao panic("sfmmu_kpm_fault_small: "
1594444ce08eSDonghai Qiao "stale smallpages mapping");
1595fedab560Sae } else {
1596fedab560Sae if (PP_ISKPMC(pp)) {
1597fedab560Sae pmtx = sfmmu_page_enter(pp);
1598fedab560Sae PP_CLRKPMC(pp);
1599fedab560Sae sfmmu_page_exit(pmtx);
1600fedab560Sae }
1601fedab560Sae
1602fedab560Sae /* tte assembly */
1603fedab560Sae KPM_TTE_VCACHED(tte.ll, pfn, TTE8K);
1604fedab560Sae
1605fedab560Sae /* tsb dropin */
1606fedab560Sae sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT);
1607fedab560Sae
1608444ce08eSDonghai Qiao oldval = sfmmu_kpm_stsbmtl(&ksp->kp_mapped_flag,
1609444ce08eSDonghai Qiao &kpmsp->kshl_lock, (KPM_MAPPED_GO | KPM_MAPPEDS));
1610fedab560Sae
1611fedab560Sae if (oldval != KPM_MAPPEDSC)
1612fedab560Sae panic("sfmmu_kpm_fault_small: "
1613444ce08eSDonghai Qiao "stale smallpages mapping");
1614fedab560Sae }
1615fedab560Sae
1616fedab560Sae } else {
1617fedab560Sae /*
1618fedab560Sae * We got a tsbmiss on a not active kpm_page range.
1619fedab560Sae * Let decide segkpm_fault how to panic.
1620fedab560Sae */
1621fedab560Sae error = EFAULT;
1622fedab560Sae }
1623fedab560Sae
1624fedab560Sae sfmmu_mlist_exit(pml);
1625fedab560Sae return (error);
1626fedab560Sae }
1627fedab560Sae
1628fedab560Sae /*
1629fedab560Sae * Check/handle potential hme/kpm mapping conflicts
1630fedab560Sae */
1631fedab560Sae static void
sfmmu_kpm_vac_conflict(page_t * pp,caddr_t vaddr)1632fedab560Sae sfmmu_kpm_vac_conflict(page_t *pp, caddr_t vaddr)
1633fedab560Sae {
1634fedab560Sae int vcolor;
1635fedab560Sae struct sf_hment *sfhmep;
1636fedab560Sae struct hat *tmphat;
1637fedab560Sae struct sf_hment *tmphme = NULL;
1638fedab560Sae struct hme_blk *hmeblkp;
1639fedab560Sae tte_t tte;
1640fedab560Sae
1641fedab560Sae ASSERT(sfmmu_mlist_held(pp));
1642fedab560Sae
1643fedab560Sae if (PP_ISNC(pp))
1644fedab560Sae return;
1645fedab560Sae
1646fedab560Sae vcolor = addr_to_vcolor(vaddr);
1647fedab560Sae if (PP_GET_VCOLOR(pp) == vcolor)
1648fedab560Sae return;
1649fedab560Sae
1650fedab560Sae /*
1651fedab560Sae * There could be no vcolor conflict between a large cached
1652fedab560Sae * hme page and a non alias range kpm page (neither large nor
1653fedab560Sae * small mapped). So if a hme conflict already exists between
1654fedab560Sae * a constituent page of a large hme mapping and a shared small
1655fedab560Sae * conflicting hme mapping, both mappings must be already
1656fedab560Sae * uncached at this point.
1657fedab560Sae */
1658fedab560Sae ASSERT(!PP_ISMAPPED_LARGE(pp));
1659fedab560Sae
1660fedab560Sae if (!PP_ISMAPPED(pp)) {
1661fedab560Sae /*
1662fedab560Sae * Previous hme user of page had a different color
1663fedab560Sae * but since there are no current users
1664fedab560Sae * we just flush the cache and change the color.
1665fedab560Sae */
1666fedab560Sae SFMMU_STAT(sf_pgcolor_conflict);
1667fedab560Sae sfmmu_cache_flush(pp->p_pagenum, PP_GET_VCOLOR(pp));
1668fedab560Sae PP_SET_VCOLOR(pp, vcolor);
1669fedab560Sae return;
1670fedab560Sae }
1671fedab560Sae
1672fedab560Sae /*
1673fedab560Sae * If we get here we have a vac conflict with a current hme
1674fedab560Sae * mapping. This must have been established by forcing a wrong
1675fedab560Sae * colored mapping, e.g. by using mmap(2) with MAP_FIXED.
1676fedab560Sae */
1677fedab560Sae
1678fedab560Sae /*
1679fedab560Sae * Check if any mapping is in same as or if it is locked
1680fedab560Sae * since in that case we need to uncache.
1681fedab560Sae */
1682fedab560Sae for (sfhmep = pp->p_mapping; sfhmep; sfhmep = tmphme) {
1683fedab560Sae tmphme = sfhmep->hme_next;
16847dacfc44Spaulsan if (IS_PAHME(sfhmep))
16857dacfc44Spaulsan continue;
1686fedab560Sae hmeblkp = sfmmu_hmetohblk(sfhmep);
1687fedab560Sae tmphat = hblktosfmmu(hmeblkp);
1688fedab560Sae sfmmu_copytte(&sfhmep->hme_tte, &tte);
1689fedab560Sae ASSERT(TTE_IS_VALID(&tte));
1690fedab560Sae if ((tmphat == ksfmmup) || hmeblkp->hblk_lckcnt) {
1691fedab560Sae /*
1692fedab560Sae * We have an uncache conflict
1693fedab560Sae */
1694fedab560Sae SFMMU_STAT(sf_uncache_conflict);
1695fedab560Sae sfmmu_page_cache_array(pp, HAT_TMPNC, CACHE_FLUSH, 1);
1696fedab560Sae return;
1697fedab560Sae }
1698fedab560Sae }
1699fedab560Sae
1700fedab560Sae /*
1701fedab560Sae * We have an unload conflict
1702fedab560Sae */
1703fedab560Sae SFMMU_STAT(sf_unload_conflict);
1704fedab560Sae
1705fedab560Sae for (sfhmep = pp->p_mapping; sfhmep; sfhmep = tmphme) {
1706fedab560Sae tmphme = sfhmep->hme_next;
17077dacfc44Spaulsan if (IS_PAHME(sfhmep))
17087dacfc44Spaulsan continue;
1709fedab560Sae hmeblkp = sfmmu_hmetohblk(sfhmep);
1710fedab560Sae (void) sfmmu_pageunload(pp, sfhmep, TTE8K);
1711fedab560Sae }
1712fedab560Sae
1713fedab560Sae /*
1714fedab560Sae * Unloads only does tlb flushes so we need to flush the
1715fedab560Sae * dcache vcolor here.
1716fedab560Sae */
1717fedab560Sae sfmmu_cache_flush(pp->p_pagenum, PP_GET_VCOLOR(pp));
1718fedab560Sae PP_SET_VCOLOR(pp, vcolor);
1719fedab560Sae }
1720fedab560Sae
1721fedab560Sae /*
1722fedab560Sae * Remove all kpm mappings using kpme's for pp and check that
1723fedab560Sae * all kpm mappings (w/ and w/o kpme's) are gone.
1724fedab560Sae */
1725fedab560Sae void
sfmmu_kpm_pageunload(page_t * pp)1726fedab560Sae sfmmu_kpm_pageunload(page_t *pp)
1727fedab560Sae {
1728fedab560Sae caddr_t vaddr;
1729fedab560Sae struct kpme *kpme, *nkpme;
1730fedab560Sae
1731fedab560Sae ASSERT(pp != NULL);
1732fedab560Sae ASSERT(pp->p_kpmref);
1733fedab560Sae ASSERT(sfmmu_mlist_held(pp));
1734fedab560Sae
1735fedab560Sae vaddr = hat_kpm_page2va(pp, 1);
1736fedab560Sae
1737fedab560Sae for (kpme = pp->p_kpmelist; kpme; kpme = nkpme) {
1738fedab560Sae ASSERT(kpme->kpe_page == pp);
1739fedab560Sae
1740fedab560Sae if (pp->p_kpmref == 0)
1741fedab560Sae panic("sfmmu_kpm_pageunload: stale p_kpmref pp=%p "
1742444ce08eSDonghai Qiao "kpme=%p", (void *)pp, (void *)kpme);
1743fedab560Sae
1744fedab560Sae nkpme = kpme->kpe_next;
1745fedab560Sae
1746fedab560Sae /* Add instance callback here here if needed later */
1747fedab560Sae sfmmu_kpme_sub(kpme, pp);
1748fedab560Sae }
1749fedab560Sae
1750fedab560Sae /*
1751fedab560Sae * Also correct after mixed kpme/nonkpme mappings. If nonkpme
1752fedab560Sae * segkpm clients have unlocked the page and forgot to mapout
1753fedab560Sae * we panic here.
1754fedab560Sae */
1755fedab560Sae if (pp->p_kpmref != 0)
1756fedab560Sae panic("sfmmu_kpm_pageunload: bad refcnt pp=%p", (void *)pp);
1757fedab560Sae
1758fedab560Sae sfmmu_kpm_mapout(pp, vaddr);
1759fedab560Sae }
1760fedab560Sae
1761fedab560Sae /*
1762fedab560Sae * Remove a large kpm mapping from kernel TSB and all TLB's.
1763fedab560Sae */
1764fedab560Sae static void
sfmmu_kpm_demap_large(caddr_t vaddr)1765fedab560Sae sfmmu_kpm_demap_large(caddr_t vaddr)
1766fedab560Sae {
1767fedab560Sae sfmmu_kpm_unload_tsb(vaddr, MMU_PAGESHIFT4M);
1768fedab560Sae sfmmu_kpm_demap_tlbs(vaddr);
1769fedab560Sae }
1770fedab560Sae
1771fedab560Sae /*
1772fedab560Sae * Remove a small kpm mapping from kernel TSB and all TLB's.
1773fedab560Sae */
1774fedab560Sae static void
sfmmu_kpm_demap_small(caddr_t vaddr)1775fedab560Sae sfmmu_kpm_demap_small(caddr_t vaddr)
1776fedab560Sae {
1777fedab560Sae sfmmu_kpm_unload_tsb(vaddr, MMU_PAGESHIFT);
1778fedab560Sae sfmmu_kpm_demap_tlbs(vaddr);
1779fedab560Sae }
1780fedab560Sae
1781fedab560Sae /*
1782fedab560Sae * Demap a kpm mapping in all TLB's.
1783fedab560Sae */
1784fedab560Sae static void
sfmmu_kpm_demap_tlbs(caddr_t vaddr)1785fedab560Sae sfmmu_kpm_demap_tlbs(caddr_t vaddr)
1786fedab560Sae {
1787fedab560Sae cpuset_t cpuset;
1788fedab560Sae
1789fedab560Sae kpreempt_disable();
1790fedab560Sae cpuset = ksfmmup->sfmmu_cpusran;
1791fedab560Sae CPUSET_AND(cpuset, cpu_ready_set);
1792fedab560Sae CPUSET_DEL(cpuset, CPU->cpu_id);
1793fedab560Sae SFMMU_XCALL_STATS(ksfmmup);
1794fedab560Sae
1795fedab560Sae xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)vaddr,
1796fedab560Sae (uint64_t)ksfmmup);
1797fedab560Sae vtag_flushpage(vaddr, (uint64_t)ksfmmup);
1798fedab560Sae
1799fedab560Sae kpreempt_enable();
1800fedab560Sae }
1801fedab560Sae
1802fedab560Sae /*
1803fedab560Sae * Summary states used in sfmmu_kpm_vac_unload (KPM_VUL__*).
1804fedab560Sae * See also more detailed comments within in the sfmmu_kpm_vac_unload switch.
1805fedab560Sae * Abbreviations used:
1806fedab560Sae * BIG: Large page kpm mapping in use.
1807fedab560Sae * CONFL: VAC conflict(s) within a kpm_page.
1808fedab560Sae * INCR: Count of conflicts within a kpm_page is going to be incremented.
1809fedab560Sae * DECR: Count of conflicts within a kpm_page is going to be decremented.
1810fedab560Sae * UNMAP_SMALL: A small (regular page size) mapping is going to be unmapped.
1811fedab560Sae * TNC: Temporary non cached: a kpm mapped page is mapped in TNC state.
1812fedab560Sae */
1813fedab560Sae #define KPM_VUL_BIG (0)
1814fedab560Sae #define KPM_VUL_CONFL_INCR1 (KPM_KS)
1815fedab560Sae #define KPM_VUL_UNMAP_SMALL1 (KPM_KS | KPM_S)
1816fedab560Sae #define KPM_VUL_CONFL_INCR2 (KPM_KC)
1817fedab560Sae #define KPM_VUL_CONFL_INCR3 (KPM_KC | KPM_KS)
1818fedab560Sae #define KPM_VUL_UNMAP_SMALL2 (KPM_KC | KPM_KS | KPM_S)
1819fedab560Sae #define KPM_VUL_CONFL_DECR1 (KPM_KC | KPM_C)
1820fedab560Sae #define KPM_VUL_CONFL_DECR2 (KPM_KC | KPM_C | KPM_KS)
1821fedab560Sae #define KPM_VUL_TNC (KPM_KC | KPM_C | KPM_KS | KPM_S)
1822fedab560Sae
1823fedab560Sae /*
1824fedab560Sae * Handle VAC unload conflicts introduced by hme mappings or vice
1825fedab560Sae * versa when a hme conflict mapping is replaced by a non conflict
1826fedab560Sae * one. Perform actions and state transitions according to the
1827fedab560Sae * various page and kpm_page entry states. VACache flushes are in
1828fedab560Sae * the responsibiliy of the caller. We still hold the mlist lock.
1829fedab560Sae */
1830fedab560Sae void
sfmmu_kpm_vac_unload(page_t * pp,caddr_t vaddr)1831fedab560Sae sfmmu_kpm_vac_unload(page_t *pp, caddr_t vaddr)
1832fedab560Sae {
1833fedab560Sae kpm_page_t *kp;
1834fedab560Sae kpm_hlk_t *kpmp;
1835fedab560Sae caddr_t kpmvaddr = hat_kpm_page2va(pp, 1);
1836fedab560Sae int newcolor;
1837fedab560Sae kmutex_t *pmtx;
1838fedab560Sae uint_t vacunlcase;
1839fedab560Sae int badstate = 0;
1840fedab560Sae kpm_spage_t *ksp;
1841fedab560Sae kpm_shlk_t *kpmsp;
1842fedab560Sae
1843fedab560Sae ASSERT(PAGE_LOCKED(pp));
1844fedab560Sae ASSERT(sfmmu_mlist_held(pp));
1845fedab560Sae ASSERT(!PP_ISNC(pp));
1846fedab560Sae
1847fedab560Sae newcolor = addr_to_vcolor(kpmvaddr) != addr_to_vcolor(vaddr);
1848fedab560Sae if (kpm_smallpages)
1849fedab560Sae goto smallpages_vac_unload;
1850fedab560Sae
1851fedab560Sae PP2KPMPG(pp, kp);
1852fedab560Sae kpmp = KPMP_HASH(kp);
1853fedab560Sae mutex_enter(&kpmp->khl_mutex);
1854fedab560Sae
1855fedab560Sae if (IS_KPM_ALIAS_RANGE(kpmvaddr)) {
1856fedab560Sae if (kp->kp_refcnta < 1) {
1857fedab560Sae panic("sfmmu_kpm_vac_unload: bad refcnta kpm_page=%p\n",
1858444ce08eSDonghai Qiao (void *)kp);
1859fedab560Sae }
1860fedab560Sae
1861fedab560Sae if (PP_ISKPMC(pp) == 0) {
1862fedab560Sae if (newcolor == 0)
1863fedab560Sae goto exit;
1864fedab560Sae sfmmu_kpm_demap_small(kpmvaddr);
1865fedab560Sae pmtx = sfmmu_page_enter(pp);
1866fedab560Sae PP_SETKPMC(pp);
1867fedab560Sae sfmmu_page_exit(pmtx);
1868fedab560Sae
1869fedab560Sae } else if (newcolor == 0) {
1870fedab560Sae pmtx = sfmmu_page_enter(pp);
1871fedab560Sae PP_CLRKPMC(pp);
1872fedab560Sae sfmmu_page_exit(pmtx);
1873fedab560Sae
1874fedab560Sae } else {
1875fedab560Sae badstate++;
1876fedab560Sae }
1877fedab560Sae
1878fedab560Sae goto exit;
1879fedab560Sae }
1880fedab560Sae
1881fedab560Sae badstate = (kp->kp_refcnt < 0 || kp->kp_refcnts < 0);
1882fedab560Sae if (kp->kp_refcntc == -1) {
1883fedab560Sae /*
1884fedab560Sae * We should come here only if trap level tsb miss
1885fedab560Sae * handler is disabled.
1886fedab560Sae */
1887fedab560Sae badstate |= (kp->kp_refcnt == 0 || kp->kp_refcnts > 0 ||
1888444ce08eSDonghai Qiao PP_ISKPMC(pp) || PP_ISKPMS(pp) || PP_ISNC(pp));
1889fedab560Sae } else {
1890fedab560Sae badstate |= (kp->kp_refcntc < 0);
1891fedab560Sae }
1892fedab560Sae
1893fedab560Sae if (badstate)
1894fedab560Sae goto exit;
1895fedab560Sae
1896fedab560Sae if (PP_ISKPMC(pp) == 0 && newcolor == 0) {
1897fedab560Sae ASSERT(PP_ISKPMS(pp) == 0);
1898fedab560Sae goto exit;
1899fedab560Sae }
1900fedab560Sae
1901fedab560Sae /*
1902fedab560Sae * Combine the per kpm_page and per page kpm VAC states
1903fedab560Sae * to a summary state in order to make the vac unload
1904fedab560Sae * handling more concise.
1905fedab560Sae */
1906fedab560Sae vacunlcase = (((kp->kp_refcntc > 0) ? KPM_KC : 0) |
1907444ce08eSDonghai Qiao ((kp->kp_refcnts > 0) ? KPM_KS : 0) |
1908444ce08eSDonghai Qiao (PP_ISKPMC(pp) ? KPM_C : 0) |
1909444ce08eSDonghai Qiao (PP_ISKPMS(pp) ? KPM_S : 0));
1910fedab560Sae
1911fedab560Sae switch (vacunlcase) {
1912fedab560Sae case KPM_VUL_BIG: /* - - - - */
1913fedab560Sae /*
1914fedab560Sae * Have to breakup the large page mapping to be
1915fedab560Sae * able to handle the conflicting hme vaddr.
1916fedab560Sae */
1917fedab560Sae if (kp->kp_refcntc == -1) {
1918fedab560Sae /* remove go indication */
1919fedab560Sae sfmmu_kpm_tsbmtl(&kp->kp_refcntc,
1920444ce08eSDonghai Qiao &kpmp->khl_lock, KPMTSBM_STOP);
1921fedab560Sae }
1922fedab560Sae sfmmu_kpm_demap_large(kpmvaddr);
1923fedab560Sae
1924fedab560Sae ASSERT(kp->kp_refcntc == 0);
1925fedab560Sae kp->kp_refcntc++;
1926fedab560Sae pmtx = sfmmu_page_enter(pp);
1927fedab560Sae PP_SETKPMC(pp);
1928fedab560Sae sfmmu_page_exit(pmtx);
1929fedab560Sae break;
1930fedab560Sae
1931fedab560Sae case KPM_VUL_UNMAP_SMALL1: /* - - ks s */
1932fedab560Sae case KPM_VUL_UNMAP_SMALL2: /* kc - ks s */
1933fedab560Sae /*
1934fedab560Sae * New conflict w/ an active kpm page, actually mapped
1935fedab560Sae * in by small TSB/TLB entries. Remove the mapping and
1936fedab560Sae * update states.
1937fedab560Sae */
1938fedab560Sae ASSERT(newcolor);
1939fedab560Sae sfmmu_kpm_demap_small(kpmvaddr);
1940fedab560Sae kp->kp_refcnts--;
1941fedab560Sae kp->kp_refcnt++;
1942fedab560Sae kp->kp_refcntc++;
1943fedab560Sae pmtx = sfmmu_page_enter(pp);
1944fedab560Sae PP_CLRKPMS(pp);
1945fedab560Sae PP_SETKPMC(pp);
1946fedab560Sae sfmmu_page_exit(pmtx);
1947fedab560Sae break;
1948fedab560Sae
1949fedab560Sae case KPM_VUL_CONFL_INCR1: /* - - ks - */
1950fedab560Sae case KPM_VUL_CONFL_INCR2: /* kc - - - */
1951fedab560Sae case KPM_VUL_CONFL_INCR3: /* kc - ks - */
1952fedab560Sae /*
1953fedab560Sae * New conflict on a active kpm mapped page not yet in
1954fedab560Sae * TSB/TLB. Mark page and increment the kpm_page conflict
1955fedab560Sae * count.
1956fedab560Sae */
1957fedab560Sae ASSERT(newcolor);
1958fedab560Sae kp->kp_refcntc++;
1959fedab560Sae pmtx = sfmmu_page_enter(pp);
1960fedab560Sae PP_SETKPMC(pp);
1961fedab560Sae sfmmu_page_exit(pmtx);
1962fedab560Sae break;
1963fedab560Sae
1964fedab560Sae case KPM_VUL_CONFL_DECR1: /* kc c - - */
1965fedab560Sae case KPM_VUL_CONFL_DECR2: /* kc c ks - */
1966fedab560Sae /*
1967fedab560Sae * A conflicting hme mapping is removed for an active
1968fedab560Sae * kpm page not yet in TSB/TLB. Unmark page and decrement
1969fedab560Sae * the kpm_page conflict count.
1970fedab560Sae */
1971fedab560Sae ASSERT(newcolor == 0);
1972fedab560Sae kp->kp_refcntc--;
1973fedab560Sae pmtx = sfmmu_page_enter(pp);
1974fedab560Sae PP_CLRKPMC(pp);
1975fedab560Sae sfmmu_page_exit(pmtx);
1976fedab560Sae break;
1977fedab560Sae
1978fedab560Sae case KPM_VUL_TNC: /* kc c ks s */
1979fedab560Sae cmn_err(CE_NOTE, "sfmmu_kpm_vac_unload: "
1980444ce08eSDonghai Qiao "page not in NC state");
1981fedab560Sae /* FALLTHRU */
1982fedab560Sae
1983fedab560Sae default:
1984fedab560Sae badstate++;
1985fedab560Sae }
1986fedab560Sae exit:
1987fedab560Sae if (badstate) {
1988fedab560Sae panic("sfmmu_kpm_vac_unload: inconsistent VAC state, "
1989444ce08eSDonghai Qiao "kpmvaddr=%p kp=%p pp=%p",
1990444ce08eSDonghai Qiao (void *)kpmvaddr, (void *)kp, (void *)pp);
1991fedab560Sae }
1992fedab560Sae mutex_exit(&kpmp->khl_mutex);
1993fedab560Sae
1994fedab560Sae return;
1995fedab560Sae
1996fedab560Sae smallpages_vac_unload:
1997fedab560Sae if (newcolor == 0)
1998fedab560Sae return;
1999fedab560Sae
2000fedab560Sae PP2KPMSPG(pp, ksp);
2001fedab560Sae kpmsp = KPMP_SHASH(ksp);
2002fedab560Sae
2003fedab560Sae if (PP_ISKPMC(pp) == 0) {
2004fedab560Sae if (ksp->kp_mapped == KPM_MAPPEDS) {
2005fedab560Sae /*
2006fedab560Sae * Stop TL tsbmiss handling
2007fedab560Sae */
2008444ce08eSDonghai Qiao (void) sfmmu_kpm_stsbmtl(&ksp->kp_mapped_flag,
2009444ce08eSDonghai Qiao &kpmsp->kshl_lock, KPM_MAPPEDSC);
2010fedab560Sae
2011fedab560Sae sfmmu_kpm_demap_small(kpmvaddr);
2012fedab560Sae
2013fedab560Sae } else if (ksp->kp_mapped != KPM_MAPPEDSC) {
2014fedab560Sae panic("sfmmu_kpm_vac_unload: inconsistent mapping");
2015fedab560Sae }
2016fedab560Sae
2017fedab560Sae pmtx = sfmmu_page_enter(pp);
2018fedab560Sae PP_SETKPMC(pp);
2019fedab560Sae sfmmu_page_exit(pmtx);
2020fedab560Sae
2021fedab560Sae } else {
2022fedab560Sae if (ksp->kp_mapped != KPM_MAPPEDSC)
2023fedab560Sae panic("sfmmu_kpm_vac_unload: inconsistent mapping");
2024fedab560Sae }
2025fedab560Sae }
2026fedab560Sae
2027fedab560Sae /*
2028fedab560Sae * Page is marked to be in VAC conflict to an existing kpm mapping
2029fedab560Sae * or is kpm mapped using only the regular pagesize. Called from
2030fedab560Sae * sfmmu_hblk_unload when a mlist is completely removed.
2031fedab560Sae */
2032fedab560Sae void
sfmmu_kpm_hme_unload(page_t * pp)2033fedab560Sae sfmmu_kpm_hme_unload(page_t *pp)
2034fedab560Sae {
2035fedab560Sae /* tte assembly */
2036fedab560Sae kpm_page_t *kp;
2037fedab560Sae kpm_hlk_t *kpmp;
2038fedab560Sae caddr_t vaddr;
2039fedab560Sae kmutex_t *pmtx;
2040fedab560Sae uint_t flags;
2041fedab560Sae kpm_spage_t *ksp;
2042fedab560Sae
2043fedab560Sae ASSERT(sfmmu_mlist_held(pp));
2044fedab560Sae ASSERT(PP_ISMAPPED_KPM(pp));
2045fedab560Sae
2046fedab560Sae flags = pp->p_nrm & (P_KPMC | P_KPMS);
2047fedab560Sae if (kpm_smallpages)
2048fedab560Sae goto smallpages_hme_unload;
2049fedab560Sae
2050fedab560Sae if (flags == (P_KPMC | P_KPMS)) {
2051fedab560Sae panic("sfmmu_kpm_hme_unload: page should be uncached");
2052fedab560Sae
2053fedab560Sae } else if (flags == P_KPMS) {
2054fedab560Sae /*
2055fedab560Sae * Page mapped small but not involved in VAC conflict
2056fedab560Sae */
2057fedab560Sae return;
2058fedab560Sae }
2059fedab560Sae
2060fedab560Sae vaddr = hat_kpm_page2va(pp, 1);
2061fedab560Sae
2062fedab560Sae PP2KPMPG(pp, kp);
2063fedab560Sae kpmp = KPMP_HASH(kp);
2064fedab560Sae mutex_enter(&kpmp->khl_mutex);
2065fedab560Sae
2066fedab560Sae if (IS_KPM_ALIAS_RANGE(vaddr)) {
2067fedab560Sae if (kp->kp_refcnta < 1) {
2068fedab560Sae panic("sfmmu_kpm_hme_unload: bad refcnta kpm_page=%p\n",
2069444ce08eSDonghai Qiao (void *)kp);
2070fedab560Sae }
2071fedab560Sae } else {
2072fedab560Sae if (kp->kp_refcntc < 1) {
2073fedab560Sae panic("sfmmu_kpm_hme_unload: bad refcntc kpm_page=%p\n",
2074444ce08eSDonghai Qiao (void *)kp);
2075fedab560Sae }
2076fedab560Sae kp->kp_refcntc--;
2077fedab560Sae }
2078fedab560Sae
2079fedab560Sae pmtx = sfmmu_page_enter(pp);
2080fedab560Sae PP_CLRKPMC(pp);
2081fedab560Sae sfmmu_page_exit(pmtx);
2082fedab560Sae
2083fedab560Sae mutex_exit(&kpmp->khl_mutex);
2084fedab560Sae return;
2085fedab560Sae
2086fedab560Sae smallpages_hme_unload:
2087fedab560Sae if (flags != P_KPMC)
2088fedab560Sae panic("sfmmu_kpm_hme_unload: page should be uncached");
2089fedab560Sae
2090fedab560Sae vaddr = hat_kpm_page2va(pp, 1);
2091fedab560Sae PP2KPMSPG(pp, ksp);
2092fedab560Sae
2093fedab560Sae if (ksp->kp_mapped != KPM_MAPPEDSC)
2094fedab560Sae panic("sfmmu_kpm_hme_unload: inconsistent mapping");
2095fedab560Sae
2096fedab560Sae /*
2097fedab560Sae * Keep KPM_MAPPEDSC until the next kpm tsbmiss where it
2098fedab560Sae * prevents TL tsbmiss handling and force a hat_kpm_fault.
2099fedab560Sae * There we can start over again.
2100fedab560Sae */
2101fedab560Sae
2102fedab560Sae pmtx = sfmmu_page_enter(pp);
2103fedab560Sae PP_CLRKPMC(pp);
2104fedab560Sae sfmmu_page_exit(pmtx);
2105fedab560Sae }
2106fedab560Sae
2107fedab560Sae /*
2108fedab560Sae * Special hooks for sfmmu_page_cache_array() when changing the
2109fedab560Sae * cacheability of a page. It is used to obey the hat_kpm lock
2110fedab560Sae * ordering (mlist -> kpmp -> spl, and back).
2111fedab560Sae */
2112fedab560Sae kpm_hlk_t *
sfmmu_kpm_kpmp_enter(page_t * pp,pgcnt_t npages)2113fedab560Sae sfmmu_kpm_kpmp_enter(page_t *pp, pgcnt_t npages)
2114fedab560Sae {
2115fedab560Sae kpm_page_t *kp;
2116fedab560Sae kpm_hlk_t *kpmp;
2117fedab560Sae
2118fedab560Sae ASSERT(sfmmu_mlist_held(pp));
2119fedab560Sae
2120fedab560Sae if (kpm_smallpages || PP_ISMAPPED_KPM(pp) == 0)
2121fedab560Sae return (NULL);
2122fedab560Sae
2123fedab560Sae ASSERT(npages <= kpmpnpgs);
2124fedab560Sae
2125fedab560Sae PP2KPMPG(pp, kp);
2126fedab560Sae kpmp = KPMP_HASH(kp);
2127fedab560Sae mutex_enter(&kpmp->khl_mutex);
2128fedab560Sae
2129fedab560Sae return (kpmp);
2130fedab560Sae }
2131fedab560Sae
2132fedab560Sae void
sfmmu_kpm_kpmp_exit(kpm_hlk_t * kpmp)2133fedab560Sae sfmmu_kpm_kpmp_exit(kpm_hlk_t *kpmp)
2134fedab560Sae {
2135fedab560Sae if (kpm_smallpages || kpmp == NULL)
2136fedab560Sae return;
2137fedab560Sae
2138fedab560Sae mutex_exit(&kpmp->khl_mutex);
2139fedab560Sae }
2140fedab560Sae
2141fedab560Sae /*
2142fedab560Sae * Summary states used in sfmmu_kpm_page_cache (KPM_*).
2143fedab560Sae * See also more detailed comments within in the sfmmu_kpm_page_cache switch.
2144fedab560Sae * Abbreviations used:
2145fedab560Sae * UNC: Input state for an uncache request.
2146fedab560Sae * BIG: Large page kpm mapping in use.
2147fedab560Sae * SMALL: Page has a small kpm mapping within a kpm_page range.
2148fedab560Sae * NODEMAP: No demap needed.
2149fedab560Sae * NOP: No operation needed on this input state.
2150fedab560Sae * CACHE: Input state for a re-cache request.
2151fedab560Sae * MAPS: Page is in TNC and kpm VAC conflict state and kpm mapped small.
2152fedab560Sae * NOMAP: Page is in TNC and kpm VAC conflict state, but not small kpm
2153fedab560Sae * mapped.
2154fedab560Sae * NOMAPO: Page is in TNC and kpm VAC conflict state, but not small kpm
2155fedab560Sae * mapped. There are also other small kpm mappings within this
2156fedab560Sae * kpm_page.
2157fedab560Sae */
2158fedab560Sae #define KPM_UNC_BIG (0)
2159fedab560Sae #define KPM_UNC_NODEMAP1 (KPM_KS)
2160fedab560Sae #define KPM_UNC_SMALL1 (KPM_KS | KPM_S)
2161fedab560Sae #define KPM_UNC_NODEMAP2 (KPM_KC)
2162fedab560Sae #define KPM_UNC_NODEMAP3 (KPM_KC | KPM_KS)
2163fedab560Sae #define KPM_UNC_SMALL2 (KPM_KC | KPM_KS | KPM_S)
2164fedab560Sae #define KPM_UNC_NOP1 (KPM_KC | KPM_C)
2165fedab560Sae #define KPM_UNC_NOP2 (KPM_KC | KPM_C | KPM_KS)
2166fedab560Sae #define KPM_CACHE_NOMAP (KPM_KC | KPM_C)
2167fedab560Sae #define KPM_CACHE_NOMAPO (KPM_KC | KPM_C | KPM_KS)
2168fedab560Sae #define KPM_CACHE_MAPS (KPM_KC | KPM_C | KPM_KS | KPM_S)
2169fedab560Sae
2170fedab560Sae /*
2171fedab560Sae * This function is called when the virtual cacheability of a page
2172fedab560Sae * is changed and the page has an actice kpm mapping. The mlist mutex,
2173fedab560Sae * the spl hash lock and the kpmp mutex (if needed) are already grabbed.
2174fedab560Sae */
2175fedab560Sae /*ARGSUSED2*/
2176fedab560Sae void
sfmmu_kpm_page_cache(page_t * pp,int flags,int cache_flush_tag)2177fedab560Sae sfmmu_kpm_page_cache(page_t *pp, int flags, int cache_flush_tag)
2178fedab560Sae {
2179fedab560Sae kpm_page_t *kp;
2180fedab560Sae kpm_hlk_t *kpmp;
2181fedab560Sae caddr_t kpmvaddr;
2182fedab560Sae int badstate = 0;
2183fedab560Sae uint_t pgcacase;
2184fedab560Sae kpm_spage_t *ksp;
2185fedab560Sae kpm_shlk_t *kpmsp;
2186fedab560Sae int oldval;
2187fedab560Sae
2188fedab560Sae ASSERT(PP_ISMAPPED_KPM(pp));
2189fedab560Sae ASSERT(sfmmu_mlist_held(pp));
2190fedab560Sae ASSERT(sfmmu_page_spl_held(pp));
2191fedab560Sae
2192fedab560Sae if (flags != HAT_TMPNC && flags != HAT_CACHE)
2193fedab560Sae panic("sfmmu_kpm_page_cache: bad flags");
2194fedab560Sae
2195fedab560Sae kpmvaddr = hat_kpm_page2va(pp, 1);
2196fedab560Sae
2197fedab560Sae if (flags == HAT_TMPNC && cache_flush_tag == CACHE_FLUSH) {
2198fedab560Sae pfn_t pfn = pp->p_pagenum;
2199fedab560Sae int vcolor = addr_to_vcolor(kpmvaddr);
2200fedab560Sae cpuset_t cpuset = cpu_ready_set;
2201fedab560Sae
2202fedab560Sae /* Flush vcolor in DCache */
2203fedab560Sae CPUSET_DEL(cpuset, CPU->cpu_id);
2204fedab560Sae SFMMU_XCALL_STATS(ksfmmup);
2205fedab560Sae xt_some(cpuset, vac_flushpage_tl1, pfn, vcolor);
2206fedab560Sae vac_flushpage(pfn, vcolor);
2207fedab560Sae }
2208fedab560Sae
2209fedab560Sae if (kpm_smallpages)
2210fedab560Sae goto smallpages_page_cache;
2211fedab560Sae
2212fedab560Sae PP2KPMPG(pp, kp);
2213fedab560Sae kpmp = KPMP_HASH(kp);
2214fedab560Sae ASSERT(MUTEX_HELD(&kpmp->khl_mutex));
2215fedab560Sae
2216fedab560Sae if (IS_KPM_ALIAS_RANGE(kpmvaddr)) {
2217fedab560Sae if (kp->kp_refcnta < 1) {
2218fedab560Sae panic("sfmmu_kpm_page_cache: bad refcnta "
2219444ce08eSDonghai Qiao "kpm_page=%p\n", (void *)kp);
2220fedab560Sae }
2221fedab560Sae sfmmu_kpm_demap_small(kpmvaddr);
2222fedab560Sae if (flags == HAT_TMPNC) {
2223fedab560Sae PP_SETKPMC(pp);
2224fedab560Sae ASSERT(!PP_ISKPMS(pp));
2225fedab560Sae } else {
2226fedab560Sae ASSERT(PP_ISKPMC(pp));
2227fedab560Sae PP_CLRKPMC(pp);
2228fedab560Sae }
2229fedab560Sae goto exit;
2230fedab560Sae }
2231fedab560Sae
2232fedab560Sae badstate = (kp->kp_refcnt < 0 || kp->kp_refcnts < 0);
2233fedab560Sae if (kp->kp_refcntc == -1) {
2234fedab560Sae /*
2235fedab560Sae * We should come here only if trap level tsb miss
2236fedab560Sae * handler is disabled.
2237fedab560Sae */
2238fedab560Sae badstate |= (kp->kp_refcnt == 0 || kp->kp_refcnts > 0 ||
2239444ce08eSDonghai Qiao PP_ISKPMC(pp) || PP_ISKPMS(pp) || PP_ISNC(pp));
2240fedab560Sae } else {
2241fedab560Sae badstate |= (kp->kp_refcntc < 0);
2242fedab560Sae }
2243fedab560Sae
2244fedab560Sae if (badstate)
2245fedab560Sae goto exit;
2246fedab560Sae
2247fedab560Sae /*
2248fedab560Sae * Combine the per kpm_page and per page kpm VAC states to
2249fedab560Sae * a summary state in order to make the VAC cache/uncache
2250fedab560Sae * handling more concise.
2251fedab560Sae */
2252fedab560Sae pgcacase = (((kp->kp_refcntc > 0) ? KPM_KC : 0) |
2253444ce08eSDonghai Qiao ((kp->kp_refcnts > 0) ? KPM_KS : 0) |
2254444ce08eSDonghai Qiao (PP_ISKPMC(pp) ? KPM_C : 0) |
2255444ce08eSDonghai Qiao (PP_ISKPMS(pp) ? KPM_S : 0));
2256fedab560Sae
2257fedab560Sae if (flags == HAT_CACHE) {
2258fedab560Sae switch (pgcacase) {
2259fedab560Sae case KPM_CACHE_MAPS: /* kc c ks s */
2260fedab560Sae sfmmu_kpm_demap_small(kpmvaddr);
2261fedab560Sae if (kp->kp_refcnts < 1) {
2262fedab560Sae panic("sfmmu_kpm_page_cache: bad refcnts "
2263fedab560Sae "kpm_page=%p\n", (void *)kp);
2264fedab560Sae }
2265fedab560Sae kp->kp_refcnts--;
2266fedab560Sae kp->kp_refcnt++;
2267fedab560Sae PP_CLRKPMS(pp);
2268fedab560Sae /* FALLTHRU */
2269fedab560Sae
2270fedab560Sae case KPM_CACHE_NOMAP: /* kc c - - */
2271fedab560Sae case KPM_CACHE_NOMAPO: /* kc c ks - */
2272fedab560Sae kp->kp_refcntc--;
2273fedab560Sae PP_CLRKPMC(pp);
2274fedab560Sae break;
2275fedab560Sae
2276fedab560Sae default:
2277fedab560Sae badstate++;
2278fedab560Sae }
2279fedab560Sae goto exit;
2280fedab560Sae }
2281fedab560Sae
2282fedab560Sae switch (pgcacase) {
2283fedab560Sae case KPM_UNC_BIG: /* - - - - */
2284fedab560Sae if (kp->kp_refcnt < 1) {
2285fedab560Sae panic("sfmmu_kpm_page_cache: bad refcnt "
2286444ce08eSDonghai Qiao "kpm_page=%p\n", (void *)kp);
2287fedab560Sae }
2288fedab560Sae
2289fedab560Sae /*
2290fedab560Sae * Have to breakup the large page mapping in preparation
2291fedab560Sae * to the upcoming TNC mode handled by small mappings.
2292fedab560Sae * The demap can already be done due to another conflict
2293fedab560Sae * within the kpm_page.
2294fedab560Sae */
2295fedab560Sae if (kp->kp_refcntc == -1) {
2296fedab560Sae /* remove go indication */
2297fedab560Sae sfmmu_kpm_tsbmtl(&kp->kp_refcntc,
2298444ce08eSDonghai Qiao &kpmp->khl_lock, KPMTSBM_STOP);
2299fedab560Sae }
2300fedab560Sae ASSERT(kp->kp_refcntc == 0);
2301fedab560Sae sfmmu_kpm_demap_large(kpmvaddr);
2302fedab560Sae kp->kp_refcntc++;
2303fedab560Sae PP_SETKPMC(pp);
2304fedab560Sae break;
2305fedab560Sae
2306fedab560Sae case KPM_UNC_SMALL1: /* - - ks s */
2307fedab560Sae case KPM_UNC_SMALL2: /* kc - ks s */
2308fedab560Sae /*
2309fedab560Sae * Have to demap an already small kpm mapping in preparation
2310fedab560Sae * to the upcoming TNC mode. The demap can already be done
2311fedab560Sae * due to another conflict within the kpm_page.
2312fedab560Sae */
2313fedab560Sae sfmmu_kpm_demap_small(kpmvaddr);
2314fedab560Sae kp->kp_refcntc++;
2315fedab560Sae kp->kp_refcnts--;
2316fedab560Sae kp->kp_refcnt++;
2317fedab560Sae PP_CLRKPMS(pp);
2318fedab560Sae PP_SETKPMC(pp);
2319fedab560Sae break;
2320fedab560Sae
2321fedab560Sae case KPM_UNC_NODEMAP1: /* - - ks - */
2322fedab560Sae /* fallthru */
2323fedab560Sae
2324fedab560Sae case KPM_UNC_NODEMAP2: /* kc - - - */
2325fedab560Sae case KPM_UNC_NODEMAP3: /* kc - ks - */
2326fedab560Sae kp->kp_refcntc++;
2327fedab560Sae PP_SETKPMC(pp);
2328fedab560Sae break;
2329fedab560Sae
2330fedab560Sae case KPM_UNC_NOP1: /* kc c - - */
2331fedab560Sae case KPM_UNC_NOP2: /* kc c ks - */
2332fedab560Sae break;
2333fedab560Sae
2334fedab560Sae default:
2335fedab560Sae badstate++;
2336fedab560Sae }
2337fedab560Sae exit:
2338fedab560Sae if (badstate) {
2339fedab560Sae panic("sfmmu_kpm_page_cache: inconsistent VAC state "
2340444ce08eSDonghai Qiao "kpmvaddr=%p kp=%p pp=%p", (void *)kpmvaddr,
2341444ce08eSDonghai Qiao (void *)kp, (void *)pp);
2342fedab560Sae }
2343fedab560Sae return;
2344fedab560Sae
2345fedab560Sae smallpages_page_cache:
2346fedab560Sae PP2KPMSPG(pp, ksp);
2347fedab560Sae kpmsp = KPMP_SHASH(ksp);
2348fedab560Sae
2349444ce08eSDonghai Qiao /*
2350444ce08eSDonghai Qiao * marked as nogo for we will fault in and resolve it
2351444ce08eSDonghai Qiao * through sfmmu_kpm_fault_small
2352444ce08eSDonghai Qiao */
2353444ce08eSDonghai Qiao oldval = sfmmu_kpm_stsbmtl(&ksp->kp_mapped_flag, &kpmsp->kshl_lock,
2354444ce08eSDonghai Qiao KPM_MAPPEDSC);
2355fedab560Sae
2356fedab560Sae if (!(oldval == KPM_MAPPEDS || oldval == KPM_MAPPEDSC))
2357fedab560Sae panic("smallpages_page_cache: inconsistent mapping");
2358fedab560Sae
2359fedab560Sae sfmmu_kpm_demap_small(kpmvaddr);
2360fedab560Sae
2361fedab560Sae if (flags == HAT_TMPNC) {
2362fedab560Sae PP_SETKPMC(pp);
2363fedab560Sae ASSERT(!PP_ISKPMS(pp));
2364fedab560Sae
2365fedab560Sae } else {
2366fedab560Sae ASSERT(PP_ISKPMC(pp));
2367fedab560Sae PP_CLRKPMC(pp);
2368fedab560Sae }
2369fedab560Sae
2370fedab560Sae /*
2371fedab560Sae * Keep KPM_MAPPEDSC until the next kpm tsbmiss where it
2372fedab560Sae * prevents TL tsbmiss handling and force a hat_kpm_fault.
2373fedab560Sae * There we can start over again.
2374fedab560Sae */
2375fedab560Sae }
2376