17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate * CDDL HEADER START
37c478bd9Sstevel@tonic-gate *
47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the
5a85a6733Sjosephb * Common Development and Distribution License (the "License").
6a85a6733Sjosephb * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate *
87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate * and limitations under the License.
127c478bd9Sstevel@tonic-gate *
137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate *
197c478bd9Sstevel@tonic-gate * CDDL HEADER END
207c478bd9Sstevel@tonic-gate */
21ae115bc7Smrj
227c478bd9Sstevel@tonic-gate /*
236c9930aeSJoe Bonasera * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
24a6a74e0eSMatthew Ahrens * Copyright (c) 2014 by Delphix. All rights reserved.
2574ecdb51SJohn Levon * Copyright 2018 Joyent, Inc.
267c478bd9Sstevel@tonic-gate */
277c478bd9Sstevel@tonic-gate
287c478bd9Sstevel@tonic-gate #include <sys/types.h>
297c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
307c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
317c478bd9Sstevel@tonic-gate #include <sys/atomic.h>
327c478bd9Sstevel@tonic-gate #include <sys/bitmap.h>
337c478bd9Sstevel@tonic-gate #include <sys/machparam.h>
347c478bd9Sstevel@tonic-gate #include <sys/machsystm.h>
357c478bd9Sstevel@tonic-gate #include <sys/mman.h>
367c478bd9Sstevel@tonic-gate #include <sys/systm.h>
377c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
387c478bd9Sstevel@tonic-gate #include <sys/thread.h>
397c478bd9Sstevel@tonic-gate #include <sys/proc.h>
407c478bd9Sstevel@tonic-gate #include <sys/cpu.h>
417c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
427c478bd9Sstevel@tonic-gate #include <sys/disp.h>
437c478bd9Sstevel@tonic-gate #include <sys/vmem.h>
447c478bd9Sstevel@tonic-gate #include <sys/vmsystm.h>
457c478bd9Sstevel@tonic-gate #include <sys/promif.h>
467c478bd9Sstevel@tonic-gate #include <sys/var.h>
477c478bd9Sstevel@tonic-gate #include <sys/x86_archext.h>
48ae115bc7Smrj #include <sys/archsystm.h>
497c478bd9Sstevel@tonic-gate #include <sys/bootconf.h>
507c478bd9Sstevel@tonic-gate #include <sys/dumphdr.h>
517c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
527c478bd9Sstevel@tonic-gate #include <vm/seg_kpm.h>
537c478bd9Sstevel@tonic-gate #include <vm/hat.h>
547c478bd9Sstevel@tonic-gate #include <vm/hat_i86.h>
557c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
56843e1988Sjohnlev #include <sys/panic.h>
57843e1988Sjohnlev
58843e1988Sjohnlev #ifdef __xpv
59843e1988Sjohnlev #include <sys/hypervisor.h>
60843e1988Sjohnlev #include <sys/xpv_panic.h>
61843e1988Sjohnlev #endif
627c478bd9Sstevel@tonic-gate
63ae115bc7Smrj #include <sys/bootinfo.h>
64ae115bc7Smrj #include <vm/kboot_mmu.h>
65ae115bc7Smrj
66ae115bc7Smrj static void x86pte_zero(htable_t *dest, uint_t entry, uint_t count);
67ae115bc7Smrj
687c478bd9Sstevel@tonic-gate kmem_cache_t *htable_cache;
697c478bd9Sstevel@tonic-gate
707c478bd9Sstevel@tonic-gate /*
717c478bd9Sstevel@tonic-gate * The variable htable_reserve_amount, rather than HTABLE_RESERVE_AMOUNT,
727c478bd9Sstevel@tonic-gate * is used in order to facilitate testing of the htable_steal() code.
737c478bd9Sstevel@tonic-gate * By resetting htable_reserve_amount to a lower value, we can force
747c478bd9Sstevel@tonic-gate * stealing to occur. The reserve amount is a guess to get us through boot.
757c478bd9Sstevel@tonic-gate */
767c478bd9Sstevel@tonic-gate #define HTABLE_RESERVE_AMOUNT (200)
777c478bd9Sstevel@tonic-gate uint_t htable_reserve_amount = HTABLE_RESERVE_AMOUNT;
787c478bd9Sstevel@tonic-gate kmutex_t htable_reserve_mutex;
797c478bd9Sstevel@tonic-gate uint_t htable_reserve_cnt;
807c478bd9Sstevel@tonic-gate htable_t *htable_reserve_pool;
817c478bd9Sstevel@tonic-gate
82a85a6733Sjosephb /*
83a85a6733Sjosephb * Used to hand test htable_steal().
84a85a6733Sjosephb */
85a85a6733Sjosephb #ifdef DEBUG
86a85a6733Sjosephb ulong_t force_steal = 0;
87a85a6733Sjosephb ulong_t ptable_cnt = 0;
88a85a6733Sjosephb #endif
89a85a6733Sjosephb
907c478bd9Sstevel@tonic-gate /*
917c478bd9Sstevel@tonic-gate * This variable is so that we can tune this via /etc/system
92a85a6733Sjosephb * Any value works, but a power of two <= mmu.ptes_per_table is best.
937c478bd9Sstevel@tonic-gate */
94a85a6733Sjosephb uint_t htable_steal_passes = 8;
957c478bd9Sstevel@tonic-gate
967c478bd9Sstevel@tonic-gate /*
977c478bd9Sstevel@tonic-gate * mutex stuff for access to htable hash
987c478bd9Sstevel@tonic-gate */
997c478bd9Sstevel@tonic-gate #define NUM_HTABLE_MUTEX 128
1007c478bd9Sstevel@tonic-gate kmutex_t htable_mutex[NUM_HTABLE_MUTEX];
1017c478bd9Sstevel@tonic-gate #define HTABLE_MUTEX_HASH(h) ((h) & (NUM_HTABLE_MUTEX - 1))
1027c478bd9Sstevel@tonic-gate
1037c478bd9Sstevel@tonic-gate #define HTABLE_ENTER(h) mutex_enter(&htable_mutex[HTABLE_MUTEX_HASH(h)]);
1047c478bd9Sstevel@tonic-gate #define HTABLE_EXIT(h) mutex_exit(&htable_mutex[HTABLE_MUTEX_HASH(h)]);
1057c478bd9Sstevel@tonic-gate
1067c478bd9Sstevel@tonic-gate /*
1077c478bd9Sstevel@tonic-gate * forward declarations
1087c478bd9Sstevel@tonic-gate */
1097c478bd9Sstevel@tonic-gate static void link_ptp(htable_t *higher, htable_t *new, uintptr_t vaddr);
1107c478bd9Sstevel@tonic-gate static void unlink_ptp(htable_t *higher, htable_t *old, uintptr_t vaddr);
1117c478bd9Sstevel@tonic-gate static void htable_free(htable_t *ht);
112ae115bc7Smrj static x86pte_t *x86pte_access_pagetable(htable_t *ht, uint_t index);
1137c478bd9Sstevel@tonic-gate static void x86pte_release_pagetable(htable_t *ht);
1147c478bd9Sstevel@tonic-gate static x86pte_t x86pte_cas(htable_t *ht, uint_t entry, x86pte_t old,
1157c478bd9Sstevel@tonic-gate x86pte_t new);
1167c478bd9Sstevel@tonic-gate
1177c478bd9Sstevel@tonic-gate /*
1187c478bd9Sstevel@tonic-gate * A counter to track if we are stealing or reaping htables. When non-zero
1197c478bd9Sstevel@tonic-gate * htable_free() will directly free htables (either to the reserve or kmem)
1207c478bd9Sstevel@tonic-gate * instead of putting them in a hat's htable cache.
1217c478bd9Sstevel@tonic-gate */
1227c478bd9Sstevel@tonic-gate uint32_t htable_dont_cache = 0;
1237c478bd9Sstevel@tonic-gate
1247c478bd9Sstevel@tonic-gate /*
1257c478bd9Sstevel@tonic-gate * Track the number of active pagetables, so we can know how many to reap
1267c478bd9Sstevel@tonic-gate */
1277c478bd9Sstevel@tonic-gate static uint32_t active_ptables = 0;
1287c478bd9Sstevel@tonic-gate
129843e1988Sjohnlev #ifdef __xpv
130843e1988Sjohnlev /*
131843e1988Sjohnlev * Deal with hypervisor complications.
132843e1988Sjohnlev */
133843e1988Sjohnlev void
xen_flush_va(caddr_t va)134843e1988Sjohnlev xen_flush_va(caddr_t va)
135843e1988Sjohnlev {
136843e1988Sjohnlev struct mmuext_op t;
137843e1988Sjohnlev uint_t count;
138843e1988Sjohnlev
139843e1988Sjohnlev if (IN_XPV_PANIC()) {
14074ecdb51SJohn Levon mmu_flush_tlb_page((uintptr_t)va);
141843e1988Sjohnlev } else {
142843e1988Sjohnlev t.cmd = MMUEXT_INVLPG_LOCAL;
143843e1988Sjohnlev t.arg1.linear_addr = (uintptr_t)va;
144843e1988Sjohnlev if (HYPERVISOR_mmuext_op(&t, 1, &count, DOMID_SELF) < 0)
145843e1988Sjohnlev panic("HYPERVISOR_mmuext_op() failed");
146843e1988Sjohnlev ASSERT(count == 1);
147843e1988Sjohnlev }
148843e1988Sjohnlev }
149843e1988Sjohnlev
150843e1988Sjohnlev void
xen_gflush_va(caddr_t va,cpuset_t cpus)151843e1988Sjohnlev xen_gflush_va(caddr_t va, cpuset_t cpus)
152843e1988Sjohnlev {
153843e1988Sjohnlev struct mmuext_op t;
154843e1988Sjohnlev uint_t count;
155843e1988Sjohnlev
156843e1988Sjohnlev if (IN_XPV_PANIC()) {
15774ecdb51SJohn Levon mmu_flush_tlb_page((uintptr_t)va);
158843e1988Sjohnlev return;
159843e1988Sjohnlev }
160843e1988Sjohnlev
161843e1988Sjohnlev t.cmd = MMUEXT_INVLPG_MULTI;
162843e1988Sjohnlev t.arg1.linear_addr = (uintptr_t)va;
163843e1988Sjohnlev /*LINTED: constant in conditional context*/
164843e1988Sjohnlev set_xen_guest_handle(t.arg2.vcpumask, &cpus);
165843e1988Sjohnlev if (HYPERVISOR_mmuext_op(&t, 1, &count, DOMID_SELF) < 0)
166843e1988Sjohnlev panic("HYPERVISOR_mmuext_op() failed");
167843e1988Sjohnlev ASSERT(count == 1);
168843e1988Sjohnlev }
169843e1988Sjohnlev
170843e1988Sjohnlev void
xen_flush_tlb()171843e1988Sjohnlev xen_flush_tlb()
172843e1988Sjohnlev {
173843e1988Sjohnlev struct mmuext_op t;
174843e1988Sjohnlev uint_t count;
175843e1988Sjohnlev
176843e1988Sjohnlev if (IN_XPV_PANIC()) {
177843e1988Sjohnlev xpv_panic_reload_cr3();
178843e1988Sjohnlev } else {
179843e1988Sjohnlev t.cmd = MMUEXT_TLB_FLUSH_LOCAL;
180843e1988Sjohnlev if (HYPERVISOR_mmuext_op(&t, 1, &count, DOMID_SELF) < 0)
181843e1988Sjohnlev panic("HYPERVISOR_mmuext_op() failed");
182843e1988Sjohnlev ASSERT(count == 1);
183843e1988Sjohnlev }
184843e1988Sjohnlev }
185843e1988Sjohnlev
186843e1988Sjohnlev void
xen_gflush_tlb(cpuset_t cpus)187843e1988Sjohnlev xen_gflush_tlb(cpuset_t cpus)
188843e1988Sjohnlev {
189843e1988Sjohnlev struct mmuext_op t;
190843e1988Sjohnlev uint_t count;
191843e1988Sjohnlev
192843e1988Sjohnlev ASSERT(!IN_XPV_PANIC());
193843e1988Sjohnlev t.cmd = MMUEXT_TLB_FLUSH_MULTI;
194843e1988Sjohnlev /*LINTED: constant in conditional context*/
195843e1988Sjohnlev set_xen_guest_handle(t.arg2.vcpumask, &cpus);
196843e1988Sjohnlev if (HYPERVISOR_mmuext_op(&t, 1, &count, DOMID_SELF) < 0)
197843e1988Sjohnlev panic("HYPERVISOR_mmuext_op() failed");
198843e1988Sjohnlev ASSERT(count == 1);
199843e1988Sjohnlev }
200843e1988Sjohnlev
201843e1988Sjohnlev /*
202843e1988Sjohnlev * Install/Adjust a kpm mapping under the hypervisor.
203843e1988Sjohnlev * Value of "how" should be:
204843e1988Sjohnlev * PT_WRITABLE | PT_VALID - regular kpm mapping
205843e1988Sjohnlev * PT_VALID - make mapping read-only
206843e1988Sjohnlev * 0 - remove mapping
207843e1988Sjohnlev *
208843e1988Sjohnlev * returns 0 on success. non-zero for failure.
209843e1988Sjohnlev */
210843e1988Sjohnlev int
xen_kpm_page(pfn_t pfn,uint_t how)211843e1988Sjohnlev xen_kpm_page(pfn_t pfn, uint_t how)
212843e1988Sjohnlev {
213843e1988Sjohnlev paddr_t pa = mmu_ptob((paddr_t)pfn);
214843e1988Sjohnlev x86pte_t pte = PT_NOCONSIST | PT_REF | PT_MOD;
215843e1988Sjohnlev
216843e1988Sjohnlev if (kpm_vbase == NULL)
217843e1988Sjohnlev return (0);
218843e1988Sjohnlev
219843e1988Sjohnlev if (how)
220843e1988Sjohnlev pte |= pa_to_ma(pa) | how;
221843e1988Sjohnlev else
222843e1988Sjohnlev pte = 0;
223843e1988Sjohnlev return (HYPERVISOR_update_va_mapping((uintptr_t)kpm_vbase + pa,
224843e1988Sjohnlev pte, UVMF_INVLPG | UVMF_ALL));
225843e1988Sjohnlev }
226843e1988Sjohnlev
227843e1988Sjohnlev void
xen_pin(pfn_t pfn,level_t lvl)228843e1988Sjohnlev xen_pin(pfn_t pfn, level_t lvl)
229843e1988Sjohnlev {
230843e1988Sjohnlev struct mmuext_op t;
231843e1988Sjohnlev uint_t count;
232843e1988Sjohnlev
233843e1988Sjohnlev t.cmd = MMUEXT_PIN_L1_TABLE + lvl;
234843e1988Sjohnlev t.arg1.mfn = pfn_to_mfn(pfn);
235843e1988Sjohnlev if (HYPERVISOR_mmuext_op(&t, 1, &count, DOMID_SELF) < 0)
236843e1988Sjohnlev panic("HYPERVISOR_mmuext_op() failed");
237843e1988Sjohnlev ASSERT(count == 1);
238843e1988Sjohnlev }
239843e1988Sjohnlev
240843e1988Sjohnlev void
xen_unpin(pfn_t pfn)241843e1988Sjohnlev xen_unpin(pfn_t pfn)
242843e1988Sjohnlev {
243843e1988Sjohnlev struct mmuext_op t;
244843e1988Sjohnlev uint_t count;
245843e1988Sjohnlev
246843e1988Sjohnlev t.cmd = MMUEXT_UNPIN_TABLE;
247843e1988Sjohnlev t.arg1.mfn = pfn_to_mfn(pfn);
248843e1988Sjohnlev if (HYPERVISOR_mmuext_op(&t, 1, &count, DOMID_SELF) < 0)
249843e1988Sjohnlev panic("HYPERVISOR_mmuext_op() failed");
250843e1988Sjohnlev ASSERT(count == 1);
251843e1988Sjohnlev }
252843e1988Sjohnlev
253843e1988Sjohnlev static void
xen_map(uint64_t pte,caddr_t va)254843e1988Sjohnlev xen_map(uint64_t pte, caddr_t va)
255843e1988Sjohnlev {
256843e1988Sjohnlev if (HYPERVISOR_update_va_mapping((uintptr_t)va, pte,
257843e1988Sjohnlev UVMF_INVLPG | UVMF_LOCAL))
258843e1988Sjohnlev panic("HYPERVISOR_update_va_mapping() failed");
259843e1988Sjohnlev }
260843e1988Sjohnlev #endif /* __xpv */
261843e1988Sjohnlev
2627c478bd9Sstevel@tonic-gate /*
2637c478bd9Sstevel@tonic-gate * Allocate a memory page for a hardware page table.
2647c478bd9Sstevel@tonic-gate *
265ae115bc7Smrj * A wrapper around page_get_physical(), with some extra checks.
2667c478bd9Sstevel@tonic-gate */
267ae115bc7Smrj static pfn_t
ptable_alloc(uintptr_t seed)268a77271f8SVikram Hegde ptable_alloc(uintptr_t seed)
2697c478bd9Sstevel@tonic-gate {
2707c478bd9Sstevel@tonic-gate pfn_t pfn;
2717c478bd9Sstevel@tonic-gate page_t *pp;
2727c478bd9Sstevel@tonic-gate
273ae115bc7Smrj pfn = PFN_INVALID;
2747c478bd9Sstevel@tonic-gate
275ae115bc7Smrj /*
276ae115bc7Smrj * The first check is to see if there is memory in the system. If we
277ae115bc7Smrj * drop to throttlefree, then fail the ptable_alloc() and let the
278ae115bc7Smrj * stealing code kick in. Note that we have to do this test here,
279ae115bc7Smrj * since the test in page_create_throttle() would let the NOSLEEP
280ae115bc7Smrj * allocation go through and deplete the page reserves.
281ae115bc7Smrj *
282ae115bc7Smrj * The !NOMEMWAIT() lets pageout, fsflush, etc. skip this check.
283ae115bc7Smrj */
284ae115bc7Smrj if (!NOMEMWAIT() && freemem <= throttlefree + 1)
285ae115bc7Smrj return (PFN_INVALID);
2867c478bd9Sstevel@tonic-gate
287a85a6733Sjosephb #ifdef DEBUG
288ae115bc7Smrj /*
289ae115bc7Smrj * This code makes htable_steal() easier to test. By setting
290ae115bc7Smrj * force_steal we force pagetable allocations to fall
291ae115bc7Smrj * into the stealing code. Roughly 1 in ever "force_steal"
292ae115bc7Smrj * page table allocations will fail.
293ae115bc7Smrj */
294ae115bc7Smrj if (proc_pageout != NULL && force_steal > 1 &&
295ae115bc7Smrj ++ptable_cnt > force_steal) {
296ae115bc7Smrj ptable_cnt = 0;
297ae115bc7Smrj return (PFN_INVALID);
298ae115bc7Smrj }
299a85a6733Sjosephb #endif /* DEBUG */
300a85a6733Sjosephb
301a77271f8SVikram Hegde pp = page_get_physical(seed);
302ae115bc7Smrj if (pp == NULL)
303ae115bc7Smrj return (PFN_INVALID);
3047c478bd9Sstevel@tonic-gate ASSERT(PAGE_SHARED(pp));
30586c1f4dcSVikram Hegde pfn = pp->p_pagenum;
3067c478bd9Sstevel@tonic-gate if (pfn == PFN_INVALID)
3077c478bd9Sstevel@tonic-gate panic("ptable_alloc(): Invalid PFN!!");
3081a5e258fSJosef 'Jeff' Sipek atomic_inc_32(&active_ptables);
309a85a6733Sjosephb HATSTAT_INC(hs_ptable_allocs);
310ae115bc7Smrj return (pfn);
3117c478bd9Sstevel@tonic-gate }
3127c478bd9Sstevel@tonic-gate
3137c478bd9Sstevel@tonic-gate /*
3147c478bd9Sstevel@tonic-gate * Free an htable's associated page table page. See the comments
3157c478bd9Sstevel@tonic-gate * for ptable_alloc().
3167c478bd9Sstevel@tonic-gate */
3177c478bd9Sstevel@tonic-gate static void
ptable_free(pfn_t pfn)318ae115bc7Smrj ptable_free(pfn_t pfn)
3197c478bd9Sstevel@tonic-gate {
320ae115bc7Smrj page_t *pp = page_numtopp_nolock(pfn);
3217c478bd9Sstevel@tonic-gate
3227c478bd9Sstevel@tonic-gate /*
3237c478bd9Sstevel@tonic-gate * need to destroy the page used for the pagetable
3247c478bd9Sstevel@tonic-gate */
3257c478bd9Sstevel@tonic-gate ASSERT(pfn != PFN_INVALID);
3267c478bd9Sstevel@tonic-gate HATSTAT_INC(hs_ptable_frees);
3271a5e258fSJosef 'Jeff' Sipek atomic_dec_32(&active_ptables);
3287c478bd9Sstevel@tonic-gate if (pp == NULL)
3297c478bd9Sstevel@tonic-gate panic("ptable_free(): no page for pfn!");
330a77271f8SVikram Hegde ASSERT(PAGE_SHARED(pp));
3317c478bd9Sstevel@tonic-gate ASSERT(pfn == pp->p_pagenum);
332843e1988Sjohnlev ASSERT(!IN_XPV_PANIC());
333a77271f8SVikram Hegde
334a77271f8SVikram Hegde /*
335a77271f8SVikram Hegde * Get an exclusive lock, might have to wait for a kmem reader.
336a77271f8SVikram Hegde */
337a77271f8SVikram Hegde if (!page_tryupgrade(pp)) {
3386c9930aeSJoe Bonasera u_offset_t off = pp->p_offset;
339a77271f8SVikram Hegde page_unlock(pp);
3406c9930aeSJoe Bonasera pp = page_lookup(&kvp, off, SE_EXCL);
3416c9930aeSJoe Bonasera if (pp == NULL)
3426c9930aeSJoe Bonasera panic("page not found");
343a77271f8SVikram Hegde }
344843e1988Sjohnlev #ifdef __xpv
345843e1988Sjohnlev if (kpm_vbase && xen_kpm_page(pfn, PT_VALID | PT_WRITABLE) < 0)
346843e1988Sjohnlev panic("failure making kpm r/w pfn=0x%lx", pfn);
347843e1988Sjohnlev #endif
3486c9930aeSJoe Bonasera page_hashout(pp, NULL);
349a77271f8SVikram Hegde page_free(pp, 1);
350a77271f8SVikram Hegde page_unresv(1);
3517c478bd9Sstevel@tonic-gate }
3527c478bd9Sstevel@tonic-gate
3537c478bd9Sstevel@tonic-gate /*
3547c478bd9Sstevel@tonic-gate * Put one htable on the reserve list.
3557c478bd9Sstevel@tonic-gate */
3567c478bd9Sstevel@tonic-gate static void
htable_put_reserve(htable_t * ht)3577c478bd9Sstevel@tonic-gate htable_put_reserve(htable_t *ht)
3587c478bd9Sstevel@tonic-gate {
3597c478bd9Sstevel@tonic-gate ht->ht_hat = NULL; /* no longer tied to a hat */
3607c478bd9Sstevel@tonic-gate ASSERT(ht->ht_pfn == PFN_INVALID);
3617c478bd9Sstevel@tonic-gate HATSTAT_INC(hs_htable_rputs);
3627c478bd9Sstevel@tonic-gate mutex_enter(&htable_reserve_mutex);
3637c478bd9Sstevel@tonic-gate ht->ht_next = htable_reserve_pool;
3647c478bd9Sstevel@tonic-gate htable_reserve_pool = ht;
3657c478bd9Sstevel@tonic-gate ++htable_reserve_cnt;
3667c478bd9Sstevel@tonic-gate mutex_exit(&htable_reserve_mutex);
3677c478bd9Sstevel@tonic-gate }
3687c478bd9Sstevel@tonic-gate
3697c478bd9Sstevel@tonic-gate /*
3707c478bd9Sstevel@tonic-gate * Take one htable from the reserve.
3717c478bd9Sstevel@tonic-gate */
3727c478bd9Sstevel@tonic-gate static htable_t *
htable_get_reserve(void)3737c478bd9Sstevel@tonic-gate htable_get_reserve(void)
3747c478bd9Sstevel@tonic-gate {
3757c478bd9Sstevel@tonic-gate htable_t *ht = NULL;
3767c478bd9Sstevel@tonic-gate
3777c478bd9Sstevel@tonic-gate mutex_enter(&htable_reserve_mutex);
3787c478bd9Sstevel@tonic-gate if (htable_reserve_cnt != 0) {
3797c478bd9Sstevel@tonic-gate ht = htable_reserve_pool;
3807c478bd9Sstevel@tonic-gate ASSERT(ht != NULL);
3817c478bd9Sstevel@tonic-gate ASSERT(ht->ht_pfn == PFN_INVALID);
3827c478bd9Sstevel@tonic-gate htable_reserve_pool = ht->ht_next;
3837c478bd9Sstevel@tonic-gate --htable_reserve_cnt;
3847c478bd9Sstevel@tonic-gate HATSTAT_INC(hs_htable_rgets);
3857c478bd9Sstevel@tonic-gate }
3867c478bd9Sstevel@tonic-gate mutex_exit(&htable_reserve_mutex);
3877c478bd9Sstevel@tonic-gate return (ht);
3887c478bd9Sstevel@tonic-gate }
3897c478bd9Sstevel@tonic-gate
3907c478bd9Sstevel@tonic-gate /*
391ae115bc7Smrj * Allocate initial htables and put them on the reserve list
3927c478bd9Sstevel@tonic-gate */
3937c478bd9Sstevel@tonic-gate void
htable_initial_reserve(uint_t count)3947c478bd9Sstevel@tonic-gate htable_initial_reserve(uint_t count)
3957c478bd9Sstevel@tonic-gate {
3967c478bd9Sstevel@tonic-gate htable_t *ht;
3977c478bd9Sstevel@tonic-gate
3987c478bd9Sstevel@tonic-gate count += HTABLE_RESERVE_AMOUNT;
3997c478bd9Sstevel@tonic-gate while (count > 0) {
4007c478bd9Sstevel@tonic-gate ht = kmem_cache_alloc(htable_cache, KM_NOSLEEP);
4017c478bd9Sstevel@tonic-gate ASSERT(ht != NULL);
4027c478bd9Sstevel@tonic-gate
4037c478bd9Sstevel@tonic-gate ASSERT(use_boot_reserve);
404ae115bc7Smrj ht->ht_pfn = PFN_INVALID;
405ae115bc7Smrj htable_put_reserve(ht);
4067c478bd9Sstevel@tonic-gate --count;
4077c478bd9Sstevel@tonic-gate }
4087c478bd9Sstevel@tonic-gate }
4097c478bd9Sstevel@tonic-gate
4107c478bd9Sstevel@tonic-gate /*
4117c478bd9Sstevel@tonic-gate * Readjust the reserves after a thread finishes using them.
4127c478bd9Sstevel@tonic-gate */
4137c478bd9Sstevel@tonic-gate void
htable_adjust_reserve()4147c478bd9Sstevel@tonic-gate htable_adjust_reserve()
4157c478bd9Sstevel@tonic-gate {
4167c478bd9Sstevel@tonic-gate htable_t *ht;
4177c478bd9Sstevel@tonic-gate
4187c478bd9Sstevel@tonic-gate /*
4197c478bd9Sstevel@tonic-gate * Free any excess htables in the reserve list
4207c478bd9Sstevel@tonic-gate */
421aac11643Sjosephb while (htable_reserve_cnt > htable_reserve_amount &&
422aac11643Sjosephb !USE_HAT_RESERVES()) {
4237c478bd9Sstevel@tonic-gate ht = htable_get_reserve();
4247c478bd9Sstevel@tonic-gate if (ht == NULL)
4257c478bd9Sstevel@tonic-gate return;
4267c478bd9Sstevel@tonic-gate ASSERT(ht->ht_pfn == PFN_INVALID);
4277c478bd9Sstevel@tonic-gate kmem_cache_free(htable_cache, ht);
4287c478bd9Sstevel@tonic-gate }
4297c478bd9Sstevel@tonic-gate }
4307c478bd9Sstevel@tonic-gate
431b59c4a48SBoris Protopopov /*
432b59c4a48SBoris Protopopov * Search the active htables for one to steal. Start at a different hash
433b59c4a48SBoris Protopopov * bucket every time to help spread the pain of stealing
434b59c4a48SBoris Protopopov */
435b59c4a48SBoris Protopopov static void
htable_steal_active(hat_t * hat,uint_t cnt,uint_t threshold,uint_t * stolen,htable_t ** list)436b59c4a48SBoris Protopopov htable_steal_active(hat_t *hat, uint_t cnt, uint_t threshold,
437b59c4a48SBoris Protopopov uint_t *stolen, htable_t **list)
438b59c4a48SBoris Protopopov {
439b59c4a48SBoris Protopopov static uint_t h_seed = 0;
440b59c4a48SBoris Protopopov htable_t *higher, *ht;
441b59c4a48SBoris Protopopov uint_t h, e, h_start;
442b59c4a48SBoris Protopopov uintptr_t va;
443b59c4a48SBoris Protopopov x86pte_t pte;
444b59c4a48SBoris Protopopov
445b59c4a48SBoris Protopopov h = h_start = h_seed++ % hat->hat_num_hash;
446b59c4a48SBoris Protopopov do {
447b59c4a48SBoris Protopopov higher = NULL;
448b59c4a48SBoris Protopopov HTABLE_ENTER(h);
449b59c4a48SBoris Protopopov for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) {
450b59c4a48SBoris Protopopov
451b59c4a48SBoris Protopopov /*
452b59c4a48SBoris Protopopov * Can we rule out reaping?
453b59c4a48SBoris Protopopov */
454b59c4a48SBoris Protopopov if (ht->ht_busy != 0 ||
455b59c4a48SBoris Protopopov (ht->ht_flags & HTABLE_SHARED_PFN) ||
456b59c4a48SBoris Protopopov ht->ht_level > 0 || ht->ht_valid_cnt > threshold ||
457b59c4a48SBoris Protopopov ht->ht_lock_cnt != 0)
458b59c4a48SBoris Protopopov continue;
459b59c4a48SBoris Protopopov
460b59c4a48SBoris Protopopov /*
461b59c4a48SBoris Protopopov * Increment busy so the htable can't disappear. We
462b59c4a48SBoris Protopopov * drop the htable mutex to avoid deadlocks with
463b59c4a48SBoris Protopopov * hat_pageunload() and the hment mutex while we
464b59c4a48SBoris Protopopov * call hat_pte_unmap()
465b59c4a48SBoris Protopopov */
466b59c4a48SBoris Protopopov ++ht->ht_busy;
467b59c4a48SBoris Protopopov HTABLE_EXIT(h);
468b59c4a48SBoris Protopopov
469b59c4a48SBoris Protopopov /*
470b59c4a48SBoris Protopopov * Try stealing.
471b59c4a48SBoris Protopopov * - unload and invalidate all PTEs
472b59c4a48SBoris Protopopov */
473b59c4a48SBoris Protopopov for (e = 0, va = ht->ht_vaddr;
474b59c4a48SBoris Protopopov e < HTABLE_NUM_PTES(ht) && ht->ht_valid_cnt > 0 &&
475b59c4a48SBoris Protopopov ht->ht_busy == 1 && ht->ht_lock_cnt == 0;
476b59c4a48SBoris Protopopov ++e, va += MMU_PAGESIZE) {
477b59c4a48SBoris Protopopov pte = x86pte_get(ht, e);
478b59c4a48SBoris Protopopov if (!PTE_ISVALID(pte))
479b59c4a48SBoris Protopopov continue;
480a6a74e0eSMatthew Ahrens hat_pte_unmap(ht, e, HAT_UNLOAD, pte, NULL,
481a6a74e0eSMatthew Ahrens B_TRUE);
482b59c4a48SBoris Protopopov }
483b59c4a48SBoris Protopopov
484b59c4a48SBoris Protopopov /*
485b59c4a48SBoris Protopopov * Reacquire htable lock. If we didn't remove all
486b59c4a48SBoris Protopopov * mappings in the table, or another thread added a new
487b59c4a48SBoris Protopopov * mapping behind us, give up on this table.
488b59c4a48SBoris Protopopov */
489b59c4a48SBoris Protopopov HTABLE_ENTER(h);
490b59c4a48SBoris Protopopov if (ht->ht_busy != 1 || ht->ht_valid_cnt != 0 ||
491b59c4a48SBoris Protopopov ht->ht_lock_cnt != 0) {
492b59c4a48SBoris Protopopov --ht->ht_busy;
493b59c4a48SBoris Protopopov continue;
494b59c4a48SBoris Protopopov }
495b59c4a48SBoris Protopopov
496b59c4a48SBoris Protopopov /*
497b59c4a48SBoris Protopopov * Steal it and unlink the page table.
498b59c4a48SBoris Protopopov */
499b59c4a48SBoris Protopopov higher = ht->ht_parent;
500b59c4a48SBoris Protopopov unlink_ptp(higher, ht, ht->ht_vaddr);
501b59c4a48SBoris Protopopov
502b59c4a48SBoris Protopopov /*
503b59c4a48SBoris Protopopov * remove from the hash list
504b59c4a48SBoris Protopopov */
505b59c4a48SBoris Protopopov if (ht->ht_next)
506b59c4a48SBoris Protopopov ht->ht_next->ht_prev = ht->ht_prev;
507b59c4a48SBoris Protopopov
508b59c4a48SBoris Protopopov if (ht->ht_prev) {
509b59c4a48SBoris Protopopov ht->ht_prev->ht_next = ht->ht_next;
510b59c4a48SBoris Protopopov } else {
511b59c4a48SBoris Protopopov ASSERT(hat->hat_ht_hash[h] == ht);
512b59c4a48SBoris Protopopov hat->hat_ht_hash[h] = ht->ht_next;
513b59c4a48SBoris Protopopov }
514b59c4a48SBoris Protopopov
515b59c4a48SBoris Protopopov /*
516b59c4a48SBoris Protopopov * Break to outer loop to release the
517b59c4a48SBoris Protopopov * higher (ht_parent) pagetable. This
518b59c4a48SBoris Protopopov * spreads out the pain caused by
519b59c4a48SBoris Protopopov * pagefaults.
520b59c4a48SBoris Protopopov */
521b59c4a48SBoris Protopopov ht->ht_next = *list;
522b59c4a48SBoris Protopopov *list = ht;
523b59c4a48SBoris Protopopov ++*stolen;
524b59c4a48SBoris Protopopov break;
525b59c4a48SBoris Protopopov }
526b59c4a48SBoris Protopopov HTABLE_EXIT(h);
527b59c4a48SBoris Protopopov if (higher != NULL)
528b59c4a48SBoris Protopopov htable_release(higher);
529b59c4a48SBoris Protopopov if (++h == hat->hat_num_hash)
530b59c4a48SBoris Protopopov h = 0;
531b59c4a48SBoris Protopopov } while (*stolen < cnt && h != h_start);
532b59c4a48SBoris Protopopov }
533b59c4a48SBoris Protopopov
534b59c4a48SBoris Protopopov /*
535b59c4a48SBoris Protopopov * Move hat to the end of the kas list
536b59c4a48SBoris Protopopov */
537b59c4a48SBoris Protopopov static void
move_victim(hat_t * hat)538b59c4a48SBoris Protopopov move_victim(hat_t *hat)
539b59c4a48SBoris Protopopov {
540b59c4a48SBoris Protopopov ASSERT(MUTEX_HELD(&hat_list_lock));
541b59c4a48SBoris Protopopov
542b59c4a48SBoris Protopopov /* unlink victim hat */
543b59c4a48SBoris Protopopov if (hat->hat_prev)
544b59c4a48SBoris Protopopov hat->hat_prev->hat_next = hat->hat_next;
545b59c4a48SBoris Protopopov else
546b59c4a48SBoris Protopopov kas.a_hat->hat_next = hat->hat_next;
547b59c4a48SBoris Protopopov
548b59c4a48SBoris Protopopov if (hat->hat_next)
549b59c4a48SBoris Protopopov hat->hat_next->hat_prev = hat->hat_prev;
550b59c4a48SBoris Protopopov else
551b59c4a48SBoris Protopopov kas.a_hat->hat_prev = hat->hat_prev;
552b59c4a48SBoris Protopopov /* relink at end of hat list */
553b59c4a48SBoris Protopopov hat->hat_next = NULL;
554b59c4a48SBoris Protopopov hat->hat_prev = kas.a_hat->hat_prev;
555b59c4a48SBoris Protopopov if (hat->hat_prev)
556b59c4a48SBoris Protopopov hat->hat_prev->hat_next = hat;
557b59c4a48SBoris Protopopov else
558b59c4a48SBoris Protopopov kas.a_hat->hat_next = hat;
559b59c4a48SBoris Protopopov
560b59c4a48SBoris Protopopov kas.a_hat->hat_prev = hat;
561b59c4a48SBoris Protopopov }
5627c478bd9Sstevel@tonic-gate
5637c478bd9Sstevel@tonic-gate /*
564b59c4a48SBoris Protopopov * This routine steals htables from user processes. Called by htable_reap
565b59c4a48SBoris Protopopov * (reap=TRUE) or htable_alloc (reap=FALSE).
5667c478bd9Sstevel@tonic-gate */
5677c478bd9Sstevel@tonic-gate static htable_t *
htable_steal(uint_t cnt,boolean_t reap)568b59c4a48SBoris Protopopov htable_steal(uint_t cnt, boolean_t reap)
5697c478bd9Sstevel@tonic-gate {
5707c478bd9Sstevel@tonic-gate hat_t *hat = kas.a_hat; /* list starts with khat */
5717c478bd9Sstevel@tonic-gate htable_t *list = NULL;
5727c478bd9Sstevel@tonic-gate htable_t *ht;
5737c478bd9Sstevel@tonic-gate uint_t stolen = 0;
574091194e6SBryan Cantrill uint_t pass, passes;
575a85a6733Sjosephb uint_t threshold;
5767c478bd9Sstevel@tonic-gate
5777c478bd9Sstevel@tonic-gate /*
5787c478bd9Sstevel@tonic-gate * Limit htable_steal_passes to something reasonable
5797c478bd9Sstevel@tonic-gate */
5807c478bd9Sstevel@tonic-gate if (htable_steal_passes == 0)
5817c478bd9Sstevel@tonic-gate htable_steal_passes = 1;
5827c478bd9Sstevel@tonic-gate if (htable_steal_passes > mmu.ptes_per_table)
5837c478bd9Sstevel@tonic-gate htable_steal_passes = mmu.ptes_per_table;
5847c478bd9Sstevel@tonic-gate
585091194e6SBryan Cantrill /*
586091194e6SBryan Cantrill * If we're stealing merely as part of kmem reaping (versus stealing
587091194e6SBryan Cantrill * to assure forward progress), we don't want to actually steal any
588091194e6SBryan Cantrill * active htables. (Stealing active htables merely to give memory
589091194e6SBryan Cantrill * back to the system can inadvertently kick off an htable crime wave
590091194e6SBryan Cantrill * as active processes repeatedly steal htables from one another,
591091194e6SBryan Cantrill * plummeting the system into a kind of HAT lawlessness that can
592091194e6SBryan Cantrill * become so violent as to impede the one thing that can end it: the
593091194e6SBryan Cantrill * freeing of memory via ARC reclaim and other means.) So if we're
594091194e6SBryan Cantrill * reaping, we limit ourselves to the first pass that steals cached
595091194e6SBryan Cantrill * htables that aren't in use -- which gives memory back, but averts
596091194e6SBryan Cantrill * the entire breakdown of social order.
597091194e6SBryan Cantrill */
598091194e6SBryan Cantrill passes = reap ? 0 : htable_steal_passes;
599091194e6SBryan Cantrill
6007c478bd9Sstevel@tonic-gate /*
601a85a6733Sjosephb * Loop through all user hats. The 1st pass takes cached htables that
6027c478bd9Sstevel@tonic-gate * aren't in use. The later passes steal by removing mappings, too.
6037c478bd9Sstevel@tonic-gate */
6041a5e258fSJosef 'Jeff' Sipek atomic_inc_32(&htable_dont_cache);
605091194e6SBryan Cantrill for (pass = 0; pass <= passes && stolen < cnt; ++pass) {
606a85a6733Sjosephb threshold = pass * mmu.ptes_per_table / htable_steal_passes;
6077c478bd9Sstevel@tonic-gate
608b59c4a48SBoris Protopopov mutex_enter(&hat_list_lock);
609a85a6733Sjosephb
610b59c4a48SBoris Protopopov /* skip the first hat (kernel) */
611b59c4a48SBoris Protopopov hat = kas.a_hat->hat_next;
612b59c4a48SBoris Protopopov for (;;) {
613a85a6733Sjosephb /*
614a85a6733Sjosephb * Skip any hat that is already being stolen from.
615a85a6733Sjosephb *
616a85a6733Sjosephb * We skip SHARED hats, as these are dummy
617a85a6733Sjosephb * hats that host ISM shared page tables.
618a85a6733Sjosephb *
619a85a6733Sjosephb * We also skip if HAT_FREEING because hat_pte_unmap()
620a85a6733Sjosephb * won't zero out the PTE's. That would lead to hitting
621a85a6733Sjosephb * stale PTEs either here or under hat_unload() when we
622a85a6733Sjosephb * steal and unload the same page table in competing
623a85a6733Sjosephb * threads.
62474ecdb51SJohn Levon *
62574ecdb51SJohn Levon * We skip HATs that belong to CPUs, to make our lives
62674ecdb51SJohn Levon * simpler.
627a85a6733Sjosephb */
62874ecdb51SJohn Levon while (hat != NULL && (hat->hat_flags &
62974ecdb51SJohn Levon (HAT_VICTIM | HAT_SHARED | HAT_FREEING |
63074ecdb51SJohn Levon HAT_PCP)) != 0) {
631a85a6733Sjosephb hat = hat->hat_next;
63274ecdb51SJohn Levon }
633a85a6733Sjosephb
634b59c4a48SBoris Protopopov if (hat == NULL)
635a85a6733Sjosephb break;
636a85a6733Sjosephb
637a85a6733Sjosephb /*
638b59c4a48SBoris Protopopov * Mark the HAT as a stealing victim so that it is
639b59c4a48SBoris Protopopov * not freed from under us, e.g. in as_free()
640a85a6733Sjosephb */
6417c478bd9Sstevel@tonic-gate hat->hat_flags |= HAT_VICTIM;
6427c478bd9Sstevel@tonic-gate mutex_exit(&hat_list_lock);
6437c478bd9Sstevel@tonic-gate
6447c478bd9Sstevel@tonic-gate /*
6457c478bd9Sstevel@tonic-gate * Take any htables from the hat's cached "free" list.
6467c478bd9Sstevel@tonic-gate */
6477c478bd9Sstevel@tonic-gate hat_enter(hat);
6487c478bd9Sstevel@tonic-gate while ((ht = hat->hat_ht_cached) != NULL &&
6497c478bd9Sstevel@tonic-gate stolen < cnt) {
6507c478bd9Sstevel@tonic-gate hat->hat_ht_cached = ht->ht_next;
6517c478bd9Sstevel@tonic-gate ht->ht_next = list;
6527c478bd9Sstevel@tonic-gate list = ht;
6537c478bd9Sstevel@tonic-gate ++stolen;
6547c478bd9Sstevel@tonic-gate }
6557c478bd9Sstevel@tonic-gate hat_exit(hat);
6567c478bd9Sstevel@tonic-gate
6577c478bd9Sstevel@tonic-gate /*
658b59c4a48SBoris Protopopov * Don't steal active htables on first pass.
6597c478bd9Sstevel@tonic-gate */
660b59c4a48SBoris Protopopov if (pass != 0 && (stolen < cnt))
661b59c4a48SBoris Protopopov htable_steal_active(hat, cnt, threshold,
662b59c4a48SBoris Protopopov &stolen, &list);
6637c478bd9Sstevel@tonic-gate
6647c478bd9Sstevel@tonic-gate /*
665b59c4a48SBoris Protopopov * do synchronous teardown for the reap case so that
666b59c4a48SBoris Protopopov * we can forget hat; at this time, hat is
667b59c4a48SBoris Protopopov * guaranteed to be around because HAT_VICTIM is set
668b59c4a48SBoris Protopopov * (see htable_free() for similar code)
6697c478bd9Sstevel@tonic-gate */
670b59c4a48SBoris Protopopov for (ht = list; (ht) && (reap); ht = ht->ht_next) {
671b59c4a48SBoris Protopopov if (ht->ht_hat == NULL)
672b59c4a48SBoris Protopopov continue;
673b59c4a48SBoris Protopopov ASSERT(ht->ht_hat == hat);
674*86ef0a63SRichard Lowe #if defined(__xpv)
67574ecdb51SJohn Levon ASSERT(!(ht->ht_flags & HTABLE_COPIED));
67674ecdb51SJohn Levon if (ht->ht_level == mmu.max_level) {
677b59c4a48SBoris Protopopov ptable_free(hat->hat_user_ptable);
678b59c4a48SBoris Protopopov hat->hat_user_ptable = PFN_INVALID;
6797c478bd9Sstevel@tonic-gate }
680b59c4a48SBoris Protopopov #endif
681b59c4a48SBoris Protopopov /*
682b59c4a48SBoris Protopopov * forget the hat
683b59c4a48SBoris Protopopov */
684b59c4a48SBoris Protopopov ht->ht_hat = NULL;
685b59c4a48SBoris Protopopov }
686b59c4a48SBoris Protopopov
687b59c4a48SBoris Protopopov mutex_enter(&hat_list_lock);
688b59c4a48SBoris Protopopov
689b59c4a48SBoris Protopopov /*
690b59c4a48SBoris Protopopov * Are we finished?
691b59c4a48SBoris Protopopov */
692b59c4a48SBoris Protopopov if (stolen == cnt) {
693b59c4a48SBoris Protopopov /*
694b59c4a48SBoris Protopopov * Try to spread the pain of stealing,
695b59c4a48SBoris Protopopov * move victim HAT to the end of the HAT list.
696b59c4a48SBoris Protopopov */
697b59c4a48SBoris Protopopov if (pass >= 1 && cnt == 1 &&
698b59c4a48SBoris Protopopov kas.a_hat->hat_prev != hat)
699b59c4a48SBoris Protopopov move_victim(hat);
700b59c4a48SBoris Protopopov /*
701b59c4a48SBoris Protopopov * We are finished
702b59c4a48SBoris Protopopov */
703b59c4a48SBoris Protopopov }
704b59c4a48SBoris Protopopov
705b59c4a48SBoris Protopopov /*
706b59c4a48SBoris Protopopov * Clear the victim flag, hat can go away now (once
707b59c4a48SBoris Protopopov * the lock is dropped)
708b59c4a48SBoris Protopopov */
709b59c4a48SBoris Protopopov if (hat->hat_flags & HAT_VICTIM) {
710b59c4a48SBoris Protopopov ASSERT(hat != kas.a_hat);
711b59c4a48SBoris Protopopov hat->hat_flags &= ~HAT_VICTIM;
712b59c4a48SBoris Protopopov cv_broadcast(&hat_list_cv);
713b59c4a48SBoris Protopopov }
714b59c4a48SBoris Protopopov
715b59c4a48SBoris Protopopov /* move on to the next hat */
716b59c4a48SBoris Protopopov hat = hat->hat_next;
7177c478bd9Sstevel@tonic-gate }
718b59c4a48SBoris Protopopov
719b59c4a48SBoris Protopopov mutex_exit(&hat_list_lock);
720b59c4a48SBoris Protopopov
7217c478bd9Sstevel@tonic-gate }
722b59c4a48SBoris Protopopov ASSERT(!MUTEX_HELD(&hat_list_lock));
723b59c4a48SBoris Protopopov
7241a5e258fSJosef 'Jeff' Sipek atomic_dec_32(&htable_dont_cache);
7257c478bd9Sstevel@tonic-gate return (list);
7267c478bd9Sstevel@tonic-gate }
7277c478bd9Sstevel@tonic-gate
7287c478bd9Sstevel@tonic-gate /*
7297c478bd9Sstevel@tonic-gate * This is invoked from kmem when the system is low on memory. We try
7307c478bd9Sstevel@tonic-gate * to free hments, htables, and ptables to improve the memory situation.
7317c478bd9Sstevel@tonic-gate */
7327c478bd9Sstevel@tonic-gate /*ARGSUSED*/
7337c478bd9Sstevel@tonic-gate static void
htable_reap(void * handle)7347c478bd9Sstevel@tonic-gate htable_reap(void *handle)
7357c478bd9Sstevel@tonic-gate {
7367c478bd9Sstevel@tonic-gate uint_t reap_cnt;
7377c478bd9Sstevel@tonic-gate htable_t *list;
7387c478bd9Sstevel@tonic-gate htable_t *ht;
7397c478bd9Sstevel@tonic-gate
7407c478bd9Sstevel@tonic-gate HATSTAT_INC(hs_reap_attempts);
7417c478bd9Sstevel@tonic-gate if (!can_steal_post_boot)
7427c478bd9Sstevel@tonic-gate return;
7437c478bd9Sstevel@tonic-gate
7447c478bd9Sstevel@tonic-gate /*
7457c478bd9Sstevel@tonic-gate * Try to reap 5% of the page tables bounded by a maximum of
7467c478bd9Sstevel@tonic-gate * 5% of physmem and a minimum of 10.
7477c478bd9Sstevel@tonic-gate */
7485f661bbcSJakub Jermar reap_cnt = MAX(MIN(physmem / 20, active_ptables / 20), 10);
7497c478bd9Sstevel@tonic-gate
750b59c4a48SBoris Protopopov /*
751b59c4a48SBoris Protopopov * Note: htable_dont_cache should be set at the time of
752b59c4a48SBoris Protopopov * invoking htable_free()
753b59c4a48SBoris Protopopov */
754b59c4a48SBoris Protopopov atomic_inc_32(&htable_dont_cache);
7557c478bd9Sstevel@tonic-gate /*
7567c478bd9Sstevel@tonic-gate * Let htable_steal() do the work, we just call htable_free()
7577c478bd9Sstevel@tonic-gate */
758843e1988Sjohnlev XPV_DISALLOW_MIGRATE();
759b59c4a48SBoris Protopopov list = htable_steal(reap_cnt, B_TRUE);
760843e1988Sjohnlev XPV_ALLOW_MIGRATE();
7617c478bd9Sstevel@tonic-gate while ((ht = list) != NULL) {
7627c478bd9Sstevel@tonic-gate list = ht->ht_next;
7637c478bd9Sstevel@tonic-gate HATSTAT_INC(hs_reaped);
7647c478bd9Sstevel@tonic-gate htable_free(ht);
7657c478bd9Sstevel@tonic-gate }
766b59c4a48SBoris Protopopov atomic_dec_32(&htable_dont_cache);
7677c478bd9Sstevel@tonic-gate
7687c478bd9Sstevel@tonic-gate /*
7697c478bd9Sstevel@tonic-gate * Free up excess reserves
7707c478bd9Sstevel@tonic-gate */
7717c478bd9Sstevel@tonic-gate htable_adjust_reserve();
7727c478bd9Sstevel@tonic-gate hment_adjust_reserve();
7737c478bd9Sstevel@tonic-gate }
7747c478bd9Sstevel@tonic-gate
7757c478bd9Sstevel@tonic-gate /*
776ae115bc7Smrj * Allocate an htable, stealing one or using the reserve if necessary
7777c478bd9Sstevel@tonic-gate */
7787c478bd9Sstevel@tonic-gate static htable_t *
htable_alloc(hat_t * hat,uintptr_t vaddr,level_t level,htable_t * shared)7797c478bd9Sstevel@tonic-gate htable_alloc(
7807c478bd9Sstevel@tonic-gate hat_t *hat,
7817c478bd9Sstevel@tonic-gate uintptr_t vaddr,
7827c478bd9Sstevel@tonic-gate level_t level,
7837c478bd9Sstevel@tonic-gate htable_t *shared)
7847c478bd9Sstevel@tonic-gate {
7857c478bd9Sstevel@tonic-gate htable_t *ht = NULL;
78674ecdb51SJohn Levon uint_t is_copied;
7877c478bd9Sstevel@tonic-gate uint_t is_bare = 0;
7887c478bd9Sstevel@tonic-gate uint_t need_to_zero = 1;
7897c478bd9Sstevel@tonic-gate int kmflags = (can_steal_post_boot ? KM_NOSLEEP : KM_SLEEP);
7907c478bd9Sstevel@tonic-gate
7917c478bd9Sstevel@tonic-gate if (level < 0 || level > TOP_LEVEL(hat))
7927c478bd9Sstevel@tonic-gate panic("htable_alloc(): level %d out of range\n", level);
7937c478bd9Sstevel@tonic-gate
79474ecdb51SJohn Levon is_copied = (hat->hat_flags & HAT_COPIED) &&
79574ecdb51SJohn Levon level == hat->hat_max_level;
79674ecdb51SJohn Levon if (is_copied || shared != NULL)
7977c478bd9Sstevel@tonic-gate is_bare = 1;
7987c478bd9Sstevel@tonic-gate
7997c478bd9Sstevel@tonic-gate /*
8007c478bd9Sstevel@tonic-gate * First reuse a cached htable from the hat_ht_cached field, this
801ae115bc7Smrj * avoids unnecessary trips through kmem/page allocators.
8027c478bd9Sstevel@tonic-gate */
8037c478bd9Sstevel@tonic-gate if (hat->hat_ht_cached != NULL && !is_bare) {
8047c478bd9Sstevel@tonic-gate hat_enter(hat);
8057c478bd9Sstevel@tonic-gate ht = hat->hat_ht_cached;
8067c478bd9Sstevel@tonic-gate if (ht != NULL) {
8077c478bd9Sstevel@tonic-gate hat->hat_ht_cached = ht->ht_next;
8087c478bd9Sstevel@tonic-gate need_to_zero = 0;
8097c478bd9Sstevel@tonic-gate /* XX64 ASSERT() they're all zero somehow */
8107c478bd9Sstevel@tonic-gate ASSERT(ht->ht_pfn != PFN_INVALID);
8117c478bd9Sstevel@tonic-gate }
8127c478bd9Sstevel@tonic-gate hat_exit(hat);
8137c478bd9Sstevel@tonic-gate }
8147c478bd9Sstevel@tonic-gate
8157c478bd9Sstevel@tonic-gate if (ht == NULL) {
8167c478bd9Sstevel@tonic-gate /*
81797704650Sjosephb * Allocate an htable, possibly refilling the reserves.
8187c478bd9Sstevel@tonic-gate */
81997704650Sjosephb if (USE_HAT_RESERVES()) {
8207c478bd9Sstevel@tonic-gate ht = htable_get_reserve();
8217c478bd9Sstevel@tonic-gate } else {
8227c478bd9Sstevel@tonic-gate /*
8237c478bd9Sstevel@tonic-gate * Donate successful htable allocations to the reserve.
8247c478bd9Sstevel@tonic-gate */
8257c478bd9Sstevel@tonic-gate for (;;) {
8267c478bd9Sstevel@tonic-gate ht = kmem_cache_alloc(htable_cache, kmflags);
8277c478bd9Sstevel@tonic-gate if (ht == NULL)
8287c478bd9Sstevel@tonic-gate break;
8297c478bd9Sstevel@tonic-gate ht->ht_pfn = PFN_INVALID;
83097704650Sjosephb if (USE_HAT_RESERVES() ||
8317c478bd9Sstevel@tonic-gate htable_reserve_cnt >= htable_reserve_amount)
8327c478bd9Sstevel@tonic-gate break;
8337c478bd9Sstevel@tonic-gate htable_put_reserve(ht);
8347c478bd9Sstevel@tonic-gate }
8357c478bd9Sstevel@tonic-gate }
8367c478bd9Sstevel@tonic-gate
8377c478bd9Sstevel@tonic-gate /*
8387c478bd9Sstevel@tonic-gate * allocate a page for the hardware page table if needed
8397c478bd9Sstevel@tonic-gate */
8407c478bd9Sstevel@tonic-gate if (ht != NULL && !is_bare) {
841a85a6733Sjosephb ht->ht_hat = hat;
842a77271f8SVikram Hegde ht->ht_pfn = ptable_alloc((uintptr_t)ht);
8437c478bd9Sstevel@tonic-gate if (ht->ht_pfn == PFN_INVALID) {
84497704650Sjosephb if (USE_HAT_RESERVES())
84597704650Sjosephb htable_put_reserve(ht);
84697704650Sjosephb else
84797704650Sjosephb kmem_cache_free(htable_cache, ht);
8487c478bd9Sstevel@tonic-gate ht = NULL;
8497c478bd9Sstevel@tonic-gate }
8507c478bd9Sstevel@tonic-gate }
8517c478bd9Sstevel@tonic-gate }
8527c478bd9Sstevel@tonic-gate
8537c478bd9Sstevel@tonic-gate /*
854a85a6733Sjosephb * If allocations failed, kick off a kmem_reap() and resort to
855a85a6733Sjosephb * htable steal(). We may spin here if the system is very low on
856a85a6733Sjosephb * memory. If the kernel itself has consumed all memory and kmem_reap()
857a85a6733Sjosephb * can't free up anything, then we'll really get stuck here.
858a85a6733Sjosephb * That should only happen in a system where the administrator has
859a85a6733Sjosephb * misconfigured VM parameters via /etc/system.
8607c478bd9Sstevel@tonic-gate */
861a85a6733Sjosephb while (ht == NULL && can_steal_post_boot) {
862a85a6733Sjosephb kmem_reap();
863b59c4a48SBoris Protopopov ht = htable_steal(1, B_FALSE);
8647c478bd9Sstevel@tonic-gate HATSTAT_INC(hs_steals);
8657c478bd9Sstevel@tonic-gate
8667c478bd9Sstevel@tonic-gate /*
867a85a6733Sjosephb * If we stole for a bare htable, release the pagetable page.
8687c478bd9Sstevel@tonic-gate */
869ae115bc7Smrj if (ht != NULL) {
870ae115bc7Smrj if (is_bare) {
871ae115bc7Smrj ptable_free(ht->ht_pfn);
872ae115bc7Smrj ht->ht_pfn = PFN_INVALID;
873*86ef0a63SRichard Lowe #if defined(__xpv)
874843e1988Sjohnlev /*
875843e1988Sjohnlev * make stolen page table writable again in kpm
876843e1988Sjohnlev */
877843e1988Sjohnlev } else if (kpm_vbase && xen_kpm_page(ht->ht_pfn,
878843e1988Sjohnlev PT_VALID | PT_WRITABLE) < 0) {
879843e1988Sjohnlev panic("failure making kpm r/w pfn=0x%lx",
880843e1988Sjohnlev ht->ht_pfn);
881843e1988Sjohnlev #endif
882ae115bc7Smrj }
883ae115bc7Smrj }
8847c478bd9Sstevel@tonic-gate }
8857c478bd9Sstevel@tonic-gate
8867c478bd9Sstevel@tonic-gate /*
887a85a6733Sjosephb * All attempts to allocate or steal failed. This should only happen
888a85a6733Sjosephb * if we run out of memory during boot, due perhaps to a huge
889a85a6733Sjosephb * boot_archive. At this point there's no way to continue.
8907c478bd9Sstevel@tonic-gate */
8917c478bd9Sstevel@tonic-gate if (ht == NULL)
8927c478bd9Sstevel@tonic-gate panic("htable_alloc(): couldn't steal\n");
8937c478bd9Sstevel@tonic-gate
894*86ef0a63SRichard Lowe #if defined(__xpv)
895843e1988Sjohnlev /*
896843e1988Sjohnlev * Under the 64-bit hypervisor, we have 2 top level page tables.
897843e1988Sjohnlev * If this allocation fails, we'll resort to stealing.
898843e1988Sjohnlev * We use the stolen page indirectly, by freeing the
899843e1988Sjohnlev * stolen htable first.
900843e1988Sjohnlev */
901843e1988Sjohnlev if (level == mmu.max_level) {
902843e1988Sjohnlev for (;;) {
903843e1988Sjohnlev htable_t *stolen;
904843e1988Sjohnlev
905a77271f8SVikram Hegde hat->hat_user_ptable = ptable_alloc((uintptr_t)ht + 1);
906843e1988Sjohnlev if (hat->hat_user_ptable != PFN_INVALID)
907843e1988Sjohnlev break;
908b59c4a48SBoris Protopopov stolen = htable_steal(1, B_FALSE);
909843e1988Sjohnlev if (stolen == NULL)
910843e1988Sjohnlev panic("2nd steal ptable failed\n");
911843e1988Sjohnlev htable_free(stolen);
912843e1988Sjohnlev }
913843e1988Sjohnlev block_zero_no_xmm(kpm_vbase + pfn_to_pa(hat->hat_user_ptable),
914843e1988Sjohnlev MMU_PAGESIZE);
915843e1988Sjohnlev }
916843e1988Sjohnlev #endif
917843e1988Sjohnlev
9187c478bd9Sstevel@tonic-gate /*
9197c478bd9Sstevel@tonic-gate * Shared page tables have all entries locked and entries may not
9207c478bd9Sstevel@tonic-gate * be added or deleted.
9217c478bd9Sstevel@tonic-gate */
9227c478bd9Sstevel@tonic-gate ht->ht_flags = 0;
9237c478bd9Sstevel@tonic-gate if (shared != NULL) {
9247c478bd9Sstevel@tonic-gate ASSERT(shared->ht_valid_cnt > 0);
9257c478bd9Sstevel@tonic-gate ht->ht_flags |= HTABLE_SHARED_PFN;
9267c478bd9Sstevel@tonic-gate ht->ht_pfn = shared->ht_pfn;
9277c478bd9Sstevel@tonic-gate ht->ht_lock_cnt = 0;
9287c478bd9Sstevel@tonic-gate ht->ht_valid_cnt = 0; /* updated in hat_share() */
9297c478bd9Sstevel@tonic-gate ht->ht_shares = shared;
9307c478bd9Sstevel@tonic-gate need_to_zero = 0;
9317c478bd9Sstevel@tonic-gate } else {
9327c478bd9Sstevel@tonic-gate ht->ht_shares = NULL;
9337c478bd9Sstevel@tonic-gate ht->ht_lock_cnt = 0;
9347c478bd9Sstevel@tonic-gate ht->ht_valid_cnt = 0;
9357c478bd9Sstevel@tonic-gate }
9367c478bd9Sstevel@tonic-gate
9377c478bd9Sstevel@tonic-gate /*
93874ecdb51SJohn Levon * setup flags, etc. for copied page tables.
9397c478bd9Sstevel@tonic-gate */
94074ecdb51SJohn Levon if (is_copied) {
94174ecdb51SJohn Levon ht->ht_flags |= HTABLE_COPIED;
9427c478bd9Sstevel@tonic-gate ASSERT(ht->ht_pfn == PFN_INVALID);
9437c478bd9Sstevel@tonic-gate need_to_zero = 0;
9447c478bd9Sstevel@tonic-gate }
9457c478bd9Sstevel@tonic-gate
9467c478bd9Sstevel@tonic-gate /*
9477c478bd9Sstevel@tonic-gate * fill in the htable
9487c478bd9Sstevel@tonic-gate */
9497c478bd9Sstevel@tonic-gate ht->ht_hat = hat;
9507c478bd9Sstevel@tonic-gate ht->ht_parent = NULL;
9517c478bd9Sstevel@tonic-gate ht->ht_vaddr = vaddr;
9527c478bd9Sstevel@tonic-gate ht->ht_level = level;
9537c478bd9Sstevel@tonic-gate ht->ht_busy = 1;
9547c478bd9Sstevel@tonic-gate ht->ht_next = NULL;
9557c478bd9Sstevel@tonic-gate ht->ht_prev = NULL;
9567c478bd9Sstevel@tonic-gate
9577c478bd9Sstevel@tonic-gate /*
9587c478bd9Sstevel@tonic-gate * Zero out any freshly allocated page table
9597c478bd9Sstevel@tonic-gate */
9607c478bd9Sstevel@tonic-gate if (need_to_zero)
9617c478bd9Sstevel@tonic-gate x86pte_zero(ht, 0, mmu.ptes_per_table);
962ae115bc7Smrj
963*86ef0a63SRichard Lowe #if defined(__xpv)
964843e1988Sjohnlev if (!is_bare && kpm_vbase) {
965843e1988Sjohnlev (void) xen_kpm_page(ht->ht_pfn, PT_VALID);
966843e1988Sjohnlev if (level == mmu.max_level)
967843e1988Sjohnlev (void) xen_kpm_page(hat->hat_user_ptable, PT_VALID);
968843e1988Sjohnlev }
969843e1988Sjohnlev #endif
970843e1988Sjohnlev
9717c478bd9Sstevel@tonic-gate return (ht);
9727c478bd9Sstevel@tonic-gate }
9737c478bd9Sstevel@tonic-gate
9747c478bd9Sstevel@tonic-gate /*
9757c478bd9Sstevel@tonic-gate * Free up an htable, either to a hat's cached list, the reserves or
9767c478bd9Sstevel@tonic-gate * back to kmem.
9777c478bd9Sstevel@tonic-gate */
9787c478bd9Sstevel@tonic-gate static void
htable_free(htable_t * ht)9797c478bd9Sstevel@tonic-gate htable_free(htable_t *ht)
9807c478bd9Sstevel@tonic-gate {
9817c478bd9Sstevel@tonic-gate hat_t *hat = ht->ht_hat;
9827c478bd9Sstevel@tonic-gate
9837c478bd9Sstevel@tonic-gate /*
9847c478bd9Sstevel@tonic-gate * If the process isn't exiting, cache the free htable in the hat
985843e1988Sjohnlev * structure. We always do this for the boot time reserve. We don't
9867c478bd9Sstevel@tonic-gate * do this if the hat is exiting or we are stealing/reaping htables.
9877c478bd9Sstevel@tonic-gate */
9887c478bd9Sstevel@tonic-gate if (hat != NULL &&
9897c478bd9Sstevel@tonic-gate !(ht->ht_flags & HTABLE_SHARED_PFN) &&
9907c478bd9Sstevel@tonic-gate (use_boot_reserve ||
9917c478bd9Sstevel@tonic-gate (!(hat->hat_flags & HAT_FREEING) && !htable_dont_cache))) {
99274ecdb51SJohn Levon ASSERT((ht->ht_flags & HTABLE_COPIED) == 0);
9937c478bd9Sstevel@tonic-gate ASSERT(ht->ht_pfn != PFN_INVALID);
9947c478bd9Sstevel@tonic-gate hat_enter(hat);
9957c478bd9Sstevel@tonic-gate ht->ht_next = hat->hat_ht_cached;
9967c478bd9Sstevel@tonic-gate hat->hat_ht_cached = ht;
9977c478bd9Sstevel@tonic-gate hat_exit(hat);
9987c478bd9Sstevel@tonic-gate return;
9997c478bd9Sstevel@tonic-gate }
10007c478bd9Sstevel@tonic-gate
10017c478bd9Sstevel@tonic-gate /*
10027c478bd9Sstevel@tonic-gate * If we have a hardware page table, free it.
1003ae115bc7Smrj * We don't free page tables that are accessed by sharing.
10047c478bd9Sstevel@tonic-gate */
10057c478bd9Sstevel@tonic-gate if (ht->ht_flags & HTABLE_SHARED_PFN) {
10067c478bd9Sstevel@tonic-gate ASSERT(ht->ht_pfn != PFN_INVALID);
100774ecdb51SJohn Levon } else if (!(ht->ht_flags & HTABLE_COPIED)) {
1008ae115bc7Smrj ptable_free(ht->ht_pfn);
1009*86ef0a63SRichard Lowe #if defined(__xpv)
1010b59c4a48SBoris Protopopov if (ht->ht_level == mmu.max_level && hat != NULL) {
1011843e1988Sjohnlev ptable_free(hat->hat_user_ptable);
1012843e1988Sjohnlev hat->hat_user_ptable = PFN_INVALID;
1013843e1988Sjohnlev }
1014843e1988Sjohnlev #endif
10157c478bd9Sstevel@tonic-gate }
1016ae115bc7Smrj ht->ht_pfn = PFN_INVALID;
10177c478bd9Sstevel@tonic-gate
10187c478bd9Sstevel@tonic-gate /*
1019843e1988Sjohnlev * Free it or put into reserves.
10207c478bd9Sstevel@tonic-gate */
1021aac11643Sjosephb if (USE_HAT_RESERVES() || htable_reserve_cnt < htable_reserve_amount) {
10227c478bd9Sstevel@tonic-gate htable_put_reserve(ht);
1023aac11643Sjosephb } else {
10247c478bd9Sstevel@tonic-gate kmem_cache_free(htable_cache, ht);
1025aac11643Sjosephb htable_adjust_reserve();
1026aac11643Sjosephb }
10277c478bd9Sstevel@tonic-gate }
10287c478bd9Sstevel@tonic-gate
10297c478bd9Sstevel@tonic-gate
10307c478bd9Sstevel@tonic-gate /*
10317c478bd9Sstevel@tonic-gate * This is called when a hat is being destroyed or swapped out. We reap all
10327c478bd9Sstevel@tonic-gate * the remaining htables in the hat cache. If destroying all left over
10337c478bd9Sstevel@tonic-gate * htables are also destroyed.
10347c478bd9Sstevel@tonic-gate *
10357c478bd9Sstevel@tonic-gate * We also don't need to invalidate any of the PTPs nor do any demapping.
10367c478bd9Sstevel@tonic-gate */
10377c478bd9Sstevel@tonic-gate void
htable_purge_hat(hat_t * hat)10387c478bd9Sstevel@tonic-gate htable_purge_hat(hat_t *hat)
10397c478bd9Sstevel@tonic-gate {
10407c478bd9Sstevel@tonic-gate htable_t *ht;
10417c478bd9Sstevel@tonic-gate int h;
10427c478bd9Sstevel@tonic-gate
10437c478bd9Sstevel@tonic-gate /*
10447c478bd9Sstevel@tonic-gate * Purge the htable cache if just reaping.
10457c478bd9Sstevel@tonic-gate */
10467c478bd9Sstevel@tonic-gate if (!(hat->hat_flags & HAT_FREEING)) {
10471a5e258fSJosef 'Jeff' Sipek atomic_inc_32(&htable_dont_cache);
10487c478bd9Sstevel@tonic-gate for (;;) {
10497c478bd9Sstevel@tonic-gate hat_enter(hat);
10507c478bd9Sstevel@tonic-gate ht = hat->hat_ht_cached;
10517c478bd9Sstevel@tonic-gate if (ht == NULL) {
10527c478bd9Sstevel@tonic-gate hat_exit(hat);
10537c478bd9Sstevel@tonic-gate break;
10547c478bd9Sstevel@tonic-gate }
10557c478bd9Sstevel@tonic-gate hat->hat_ht_cached = ht->ht_next;
10567c478bd9Sstevel@tonic-gate hat_exit(hat);
10577c478bd9Sstevel@tonic-gate htable_free(ht);
10587c478bd9Sstevel@tonic-gate }
10591a5e258fSJosef 'Jeff' Sipek atomic_dec_32(&htable_dont_cache);
10607c478bd9Sstevel@tonic-gate return;
10617c478bd9Sstevel@tonic-gate }
10627c478bd9Sstevel@tonic-gate
10637c478bd9Sstevel@tonic-gate /*
10647c478bd9Sstevel@tonic-gate * if freeing, no locking is needed
10657c478bd9Sstevel@tonic-gate */
10667c478bd9Sstevel@tonic-gate while ((ht = hat->hat_ht_cached) != NULL) {
10677c478bd9Sstevel@tonic-gate hat->hat_ht_cached = ht->ht_next;
10687c478bd9Sstevel@tonic-gate htable_free(ht);
10697c478bd9Sstevel@tonic-gate }
10707c478bd9Sstevel@tonic-gate
10717c478bd9Sstevel@tonic-gate /*
10727c478bd9Sstevel@tonic-gate * walk thru the htable hash table and free all the htables in it.
10737c478bd9Sstevel@tonic-gate */
10747c478bd9Sstevel@tonic-gate for (h = 0; h < hat->hat_num_hash; ++h) {
10757c478bd9Sstevel@tonic-gate while ((ht = hat->hat_ht_hash[h]) != NULL) {
10767c478bd9Sstevel@tonic-gate if (ht->ht_next)
10777c478bd9Sstevel@tonic-gate ht->ht_next->ht_prev = ht->ht_prev;
10787c478bd9Sstevel@tonic-gate
10797c478bd9Sstevel@tonic-gate if (ht->ht_prev) {
10807c478bd9Sstevel@tonic-gate ht->ht_prev->ht_next = ht->ht_next;
10817c478bd9Sstevel@tonic-gate } else {
10827c478bd9Sstevel@tonic-gate ASSERT(hat->hat_ht_hash[h] == ht);
10837c478bd9Sstevel@tonic-gate hat->hat_ht_hash[h] = ht->ht_next;
10847c478bd9Sstevel@tonic-gate }
10857c478bd9Sstevel@tonic-gate htable_free(ht);
10867c478bd9Sstevel@tonic-gate }
10877c478bd9Sstevel@tonic-gate }
10887c478bd9Sstevel@tonic-gate }
10897c478bd9Sstevel@tonic-gate
10907c478bd9Sstevel@tonic-gate /*
10917c478bd9Sstevel@tonic-gate * Unlink an entry for a table at vaddr and level out of the existing table
10927c478bd9Sstevel@tonic-gate * one level higher. We are always holding the HASH_ENTER() when doing this.
10937c478bd9Sstevel@tonic-gate */
10947c478bd9Sstevel@tonic-gate static void
unlink_ptp(htable_t * higher,htable_t * old,uintptr_t vaddr)10957c478bd9Sstevel@tonic-gate unlink_ptp(htable_t *higher, htable_t *old, uintptr_t vaddr)
10967c478bd9Sstevel@tonic-gate {
10977c478bd9Sstevel@tonic-gate uint_t entry = htable_va2entry(vaddr, higher);
10987c478bd9Sstevel@tonic-gate x86pte_t expect = MAKEPTP(old->ht_pfn, old->ht_level);
10997c478bd9Sstevel@tonic-gate x86pte_t found;
1100935f8dd0Sjosephb hat_t *hat = old->ht_hat;
11017c478bd9Sstevel@tonic-gate
11027c478bd9Sstevel@tonic-gate ASSERT(higher->ht_busy > 0);
11037c478bd9Sstevel@tonic-gate ASSERT(higher->ht_valid_cnt > 0);
11047c478bd9Sstevel@tonic-gate ASSERT(old->ht_valid_cnt == 0);
11057c478bd9Sstevel@tonic-gate found = x86pte_cas(higher, entry, expect, 0);
1106843e1988Sjohnlev #ifdef __xpv
1107843e1988Sjohnlev /*
1108843e1988Sjohnlev * This is weird, but Xen apparently automatically unlinks empty
1109843e1988Sjohnlev * pagetables from the upper page table. So allow PTP to be 0 already.
1110843e1988Sjohnlev */
1111843e1988Sjohnlev if (found != expect && found != 0)
1112843e1988Sjohnlev #else
11137c478bd9Sstevel@tonic-gate if (found != expect)
1114843e1988Sjohnlev #endif
11157c478bd9Sstevel@tonic-gate panic("Bad PTP found=" FMT_PTE ", expected=" FMT_PTE,
11167c478bd9Sstevel@tonic-gate found, expect);
1117935f8dd0Sjosephb
1118935f8dd0Sjosephb /*
111974ecdb51SJohn Levon * When a top level PTE changes for a copied htable, we must trigger a
112074ecdb51SJohn Levon * hat_pcp_update() on all HAT CPUs.
11217173d045Sjosephb *
112274ecdb51SJohn Levon * If we don't need do do that, then we still have to INVLPG against an
112374ecdb51SJohn Levon * address covered by the inner page table, as the latest processors
11247173d045Sjosephb * have TLB-like caches for non-leaf page table entries.
1125935f8dd0Sjosephb */
1126935f8dd0Sjosephb if (!(hat->hat_flags & HAT_FREEING)) {
112774ecdb51SJohn Levon hat_tlb_inval(hat, (higher->ht_flags & HTABLE_COPIED) ?
11287173d045Sjosephb DEMAP_ALL_ADDR : old->ht_vaddr);
1129935f8dd0Sjosephb }
1130935f8dd0Sjosephb
11317c478bd9Sstevel@tonic-gate HTABLE_DEC(higher->ht_valid_cnt);
11327c478bd9Sstevel@tonic-gate }
11337c478bd9Sstevel@tonic-gate
11347c478bd9Sstevel@tonic-gate /*
11357c478bd9Sstevel@tonic-gate * Link an entry for a new table at vaddr and level into the existing table
11367c478bd9Sstevel@tonic-gate * one level higher. We are always holding the HASH_ENTER() when doing this.
11377c478bd9Sstevel@tonic-gate */
11387c478bd9Sstevel@tonic-gate static void
link_ptp(htable_t * higher,htable_t * new,uintptr_t vaddr)11397c478bd9Sstevel@tonic-gate link_ptp(htable_t *higher, htable_t *new, uintptr_t vaddr)
11407c478bd9Sstevel@tonic-gate {
11417c478bd9Sstevel@tonic-gate uint_t entry = htable_va2entry(vaddr, higher);
11427c478bd9Sstevel@tonic-gate x86pte_t newptp = MAKEPTP(new->ht_pfn, new->ht_level);
11437c478bd9Sstevel@tonic-gate x86pte_t found;
11447c478bd9Sstevel@tonic-gate
11457c478bd9Sstevel@tonic-gate ASSERT(higher->ht_busy > 0);
11467c478bd9Sstevel@tonic-gate
11477c478bd9Sstevel@tonic-gate ASSERT(new->ht_level != mmu.max_level);
11487c478bd9Sstevel@tonic-gate
11497c478bd9Sstevel@tonic-gate HTABLE_INC(higher->ht_valid_cnt);
11507c478bd9Sstevel@tonic-gate
11517c478bd9Sstevel@tonic-gate found = x86pte_cas(higher, entry, 0, newptp);
1152b4b46911Skchow if ((found & ~PT_REF) != 0)
11537c478bd9Sstevel@tonic-gate panic("HAT: ptp not 0, found=" FMT_PTE, found);
1154935f8dd0Sjosephb
1155935f8dd0Sjosephb /*
115674ecdb51SJohn Levon * When a top level PTE changes for a copied htable, we must trigger a
115774ecdb51SJohn Levon * hat_pcp_update() on all HAT CPUs.
115874ecdb51SJohn Levon *
11596b60931cSjosephb * We also need to do this for the kernel hat on PAE 32 bit kernel.
1160935f8dd0Sjosephb */
1161*86ef0a63SRichard Lowe if ((higher->ht_flags & HTABLE_COPIED) != 0)
1162935f8dd0Sjosephb hat_tlb_inval(higher->ht_hat, DEMAP_ALL_ADDR);
11637c478bd9Sstevel@tonic-gate }
11647c478bd9Sstevel@tonic-gate
11657c478bd9Sstevel@tonic-gate /*
1166ae115bc7Smrj * Release of hold on an htable. If this is the last use and the pagetable
1167ae115bc7Smrj * is empty we may want to free it, then recursively look at the pagetable
1168ae115bc7Smrj * above it. The recursion is handled by the outer while() loop.
1169843e1988Sjohnlev *
1170843e1988Sjohnlev * On the metal, during process exit, we don't bother unlinking the tables from
1171843e1988Sjohnlev * upper level pagetables. They are instead handled in bulk by hat_free_end().
1172843e1988Sjohnlev * We can't do this on the hypervisor as we need the page table to be
1173843e1988Sjohnlev * implicitly unpinnned before it goes to the free page lists. This can't
1174843e1988Sjohnlev * happen unless we fully unlink it from the page table hierarchy.
11757c478bd9Sstevel@tonic-gate */
11767c478bd9Sstevel@tonic-gate void
htable_release(htable_t * ht)11777c478bd9Sstevel@tonic-gate htable_release(htable_t *ht)
11787c478bd9Sstevel@tonic-gate {
11797c478bd9Sstevel@tonic-gate uint_t hashval;
11807c478bd9Sstevel@tonic-gate htable_t *shared;
11817c478bd9Sstevel@tonic-gate htable_t *higher;
11827c478bd9Sstevel@tonic-gate hat_t *hat;
11837c478bd9Sstevel@tonic-gate uintptr_t va;
11847c478bd9Sstevel@tonic-gate level_t level;
11857c478bd9Sstevel@tonic-gate
11867c478bd9Sstevel@tonic-gate while (ht != NULL) {
11877c478bd9Sstevel@tonic-gate shared = NULL;
11887c478bd9Sstevel@tonic-gate for (;;) {
11897c478bd9Sstevel@tonic-gate hat = ht->ht_hat;
11907c478bd9Sstevel@tonic-gate va = ht->ht_vaddr;
11917c478bd9Sstevel@tonic-gate level = ht->ht_level;
11927c478bd9Sstevel@tonic-gate hashval = HTABLE_HASH(hat, va, level);
11937c478bd9Sstevel@tonic-gate
11947c478bd9Sstevel@tonic-gate /*
11957c478bd9Sstevel@tonic-gate * The common case is that this isn't the last use of
11967c478bd9Sstevel@tonic-gate * an htable so we don't want to free the htable.
11977c478bd9Sstevel@tonic-gate */
11987c478bd9Sstevel@tonic-gate HTABLE_ENTER(hashval);
11997c478bd9Sstevel@tonic-gate ASSERT(ht->ht_valid_cnt >= 0);
12007c478bd9Sstevel@tonic-gate ASSERT(ht->ht_busy > 0);
12017c478bd9Sstevel@tonic-gate if (ht->ht_valid_cnt > 0)
12027c478bd9Sstevel@tonic-gate break;
12037c478bd9Sstevel@tonic-gate if (ht->ht_busy > 1)
12047c478bd9Sstevel@tonic-gate break;
12052ba723d8Smec ASSERT(ht->ht_lock_cnt == 0);
12067c478bd9Sstevel@tonic-gate
1207843e1988Sjohnlev #if !defined(__xpv)
12087c478bd9Sstevel@tonic-gate /*
12097c478bd9Sstevel@tonic-gate * we always release empty shared htables
12107c478bd9Sstevel@tonic-gate */
12117c478bd9Sstevel@tonic-gate if (!(ht->ht_flags & HTABLE_SHARED_PFN)) {
12127c478bd9Sstevel@tonic-gate
12137c478bd9Sstevel@tonic-gate /*
12147c478bd9Sstevel@tonic-gate * don't release if in address space tear down
12157c478bd9Sstevel@tonic-gate */
12167c478bd9Sstevel@tonic-gate if (hat->hat_flags & HAT_FREEING)
12177c478bd9Sstevel@tonic-gate break;
12187c478bd9Sstevel@tonic-gate
12197c478bd9Sstevel@tonic-gate /*
12207c478bd9Sstevel@tonic-gate * At and above max_page_level, free if it's for
12217c478bd9Sstevel@tonic-gate * a boot-time kernel mapping below kernelbase.
12227c478bd9Sstevel@tonic-gate */
12237c478bd9Sstevel@tonic-gate if (level >= mmu.max_page_level &&
12247c478bd9Sstevel@tonic-gate (hat != kas.a_hat || va >= kernelbase))
12257c478bd9Sstevel@tonic-gate break;
12267c478bd9Sstevel@tonic-gate }
1227843e1988Sjohnlev #endif /* __xpv */
12287c478bd9Sstevel@tonic-gate
12297c478bd9Sstevel@tonic-gate /*
1230ae115bc7Smrj * Remember if we destroy an htable that shares its PFN
1231ae115bc7Smrj * from elsewhere.
12327c478bd9Sstevel@tonic-gate */
12337c478bd9Sstevel@tonic-gate if (ht->ht_flags & HTABLE_SHARED_PFN) {
12347c478bd9Sstevel@tonic-gate ASSERT(shared == NULL);
12357c478bd9Sstevel@tonic-gate shared = ht->ht_shares;
12367c478bd9Sstevel@tonic-gate HATSTAT_INC(hs_htable_unshared);
12377c478bd9Sstevel@tonic-gate }
12387c478bd9Sstevel@tonic-gate
12397c478bd9Sstevel@tonic-gate /*
12407c478bd9Sstevel@tonic-gate * Handle release of a table and freeing the htable_t.
12417c478bd9Sstevel@tonic-gate * Unlink it from the table higher (ie. ht_parent).
12427c478bd9Sstevel@tonic-gate */
12437c478bd9Sstevel@tonic-gate higher = ht->ht_parent;
12447c478bd9Sstevel@tonic-gate ASSERT(higher != NULL);
12457c478bd9Sstevel@tonic-gate
12467c478bd9Sstevel@tonic-gate /*
12477c478bd9Sstevel@tonic-gate * Unlink the pagetable.
12487c478bd9Sstevel@tonic-gate */
12497c478bd9Sstevel@tonic-gate unlink_ptp(higher, ht, va);
12507c478bd9Sstevel@tonic-gate
12517c478bd9Sstevel@tonic-gate /*
12527c478bd9Sstevel@tonic-gate * remove this htable from its hash list
12537c478bd9Sstevel@tonic-gate */
12547c478bd9Sstevel@tonic-gate if (ht->ht_next)
12557c478bd9Sstevel@tonic-gate ht->ht_next->ht_prev = ht->ht_prev;
12567c478bd9Sstevel@tonic-gate
12577c478bd9Sstevel@tonic-gate if (ht->ht_prev) {
12587c478bd9Sstevel@tonic-gate ht->ht_prev->ht_next = ht->ht_next;
12597c478bd9Sstevel@tonic-gate } else {
12607c478bd9Sstevel@tonic-gate ASSERT(hat->hat_ht_hash[hashval] == ht);
12617c478bd9Sstevel@tonic-gate hat->hat_ht_hash[hashval] = ht->ht_next;
12627c478bd9Sstevel@tonic-gate }
12637c478bd9Sstevel@tonic-gate HTABLE_EXIT(hashval);
12647c478bd9Sstevel@tonic-gate htable_free(ht);
12657c478bd9Sstevel@tonic-gate ht = higher;
12667c478bd9Sstevel@tonic-gate }
12677c478bd9Sstevel@tonic-gate
12687c478bd9Sstevel@tonic-gate ASSERT(ht->ht_busy >= 1);
12697c478bd9Sstevel@tonic-gate --ht->ht_busy;
12707c478bd9Sstevel@tonic-gate HTABLE_EXIT(hashval);
12717c478bd9Sstevel@tonic-gate
12727c478bd9Sstevel@tonic-gate /*
12737c478bd9Sstevel@tonic-gate * If we released a shared htable, do a release on the htable
12747c478bd9Sstevel@tonic-gate * from which it shared
12757c478bd9Sstevel@tonic-gate */
12767c478bd9Sstevel@tonic-gate ht = shared;
12777c478bd9Sstevel@tonic-gate }
12787c478bd9Sstevel@tonic-gate }
12797c478bd9Sstevel@tonic-gate
12807c478bd9Sstevel@tonic-gate /*
12817c478bd9Sstevel@tonic-gate * Find the htable for the pagetable at the given level for the given address.
12827c478bd9Sstevel@tonic-gate * If found acquires a hold that eventually needs to be htable_release()d
12837c478bd9Sstevel@tonic-gate */
12847c478bd9Sstevel@tonic-gate htable_t *
htable_lookup(hat_t * hat,uintptr_t vaddr,level_t level)12857c478bd9Sstevel@tonic-gate htable_lookup(hat_t *hat, uintptr_t vaddr, level_t level)
12867c478bd9Sstevel@tonic-gate {
12877c478bd9Sstevel@tonic-gate uintptr_t base;
12887c478bd9Sstevel@tonic-gate uint_t hashval;
12897c478bd9Sstevel@tonic-gate htable_t *ht = NULL;
12907c478bd9Sstevel@tonic-gate
12917c478bd9Sstevel@tonic-gate ASSERT(level >= 0);
12927c478bd9Sstevel@tonic-gate ASSERT(level <= TOP_LEVEL(hat));
12937c478bd9Sstevel@tonic-gate
12947173d045Sjosephb if (level == TOP_LEVEL(hat)) {
12957173d045Sjosephb /*
12967173d045Sjosephb * 32 bit address spaces on 64 bit kernels need to check
12977173d045Sjosephb * for overflow of the 32 bit address space
12987173d045Sjosephb */
129974ecdb51SJohn Levon if ((hat->hat_flags & HAT_COPIED_32) &&
130074ecdb51SJohn Levon vaddr >= ((uint64_t)1 << 32))
13017173d045Sjosephb return (NULL);
13027c478bd9Sstevel@tonic-gate base = 0;
13037173d045Sjosephb } else {
13047c478bd9Sstevel@tonic-gate base = vaddr & LEVEL_MASK(level + 1);
13057173d045Sjosephb }
13067c478bd9Sstevel@tonic-gate
13077c478bd9Sstevel@tonic-gate hashval = HTABLE_HASH(hat, base, level);
13087c478bd9Sstevel@tonic-gate HTABLE_ENTER(hashval);
13097c478bd9Sstevel@tonic-gate for (ht = hat->hat_ht_hash[hashval]; ht; ht = ht->ht_next) {
13107c478bd9Sstevel@tonic-gate if (ht->ht_hat == hat &&
13117c478bd9Sstevel@tonic-gate ht->ht_vaddr == base &&
13127c478bd9Sstevel@tonic-gate ht->ht_level == level)
13137c478bd9Sstevel@tonic-gate break;
13147c478bd9Sstevel@tonic-gate }
13157c478bd9Sstevel@tonic-gate if (ht)
13167c478bd9Sstevel@tonic-gate ++ht->ht_busy;
13177c478bd9Sstevel@tonic-gate
13187c478bd9Sstevel@tonic-gate HTABLE_EXIT(hashval);
13197c478bd9Sstevel@tonic-gate return (ht);
13207c478bd9Sstevel@tonic-gate }
13217c478bd9Sstevel@tonic-gate
13227c478bd9Sstevel@tonic-gate /*
13237c478bd9Sstevel@tonic-gate * Acquires a hold on a known htable (from a locked hment entry).
13247c478bd9Sstevel@tonic-gate */
13257c478bd9Sstevel@tonic-gate void
htable_acquire(htable_t * ht)13267c478bd9Sstevel@tonic-gate htable_acquire(htable_t *ht)
13277c478bd9Sstevel@tonic-gate {
13287c478bd9Sstevel@tonic-gate hat_t *hat = ht->ht_hat;
13297c478bd9Sstevel@tonic-gate level_t level = ht->ht_level;
13307c478bd9Sstevel@tonic-gate uintptr_t base = ht->ht_vaddr;
13317c478bd9Sstevel@tonic-gate uint_t hashval = HTABLE_HASH(hat, base, level);
13327c478bd9Sstevel@tonic-gate
13337c478bd9Sstevel@tonic-gate HTABLE_ENTER(hashval);
13347c478bd9Sstevel@tonic-gate #ifdef DEBUG
13357c478bd9Sstevel@tonic-gate /*
13367c478bd9Sstevel@tonic-gate * make sure the htable is there
13377c478bd9Sstevel@tonic-gate */
13387c478bd9Sstevel@tonic-gate {
13397c478bd9Sstevel@tonic-gate htable_t *h;
13407c478bd9Sstevel@tonic-gate
13417c478bd9Sstevel@tonic-gate for (h = hat->hat_ht_hash[hashval];
13427c478bd9Sstevel@tonic-gate h && h != ht;
13437c478bd9Sstevel@tonic-gate h = h->ht_next)
13447c478bd9Sstevel@tonic-gate ;
13457c478bd9Sstevel@tonic-gate ASSERT(h == ht);
13467c478bd9Sstevel@tonic-gate }
13477c478bd9Sstevel@tonic-gate #endif /* DEBUG */
13487c478bd9Sstevel@tonic-gate ++ht->ht_busy;
13497c478bd9Sstevel@tonic-gate HTABLE_EXIT(hashval);
13507c478bd9Sstevel@tonic-gate }
13517c478bd9Sstevel@tonic-gate
13527c478bd9Sstevel@tonic-gate /*
13537c478bd9Sstevel@tonic-gate * Find the htable for the pagetable at the given level for the given address.
13547c478bd9Sstevel@tonic-gate * If found acquires a hold that eventually needs to be htable_release()d
13557c478bd9Sstevel@tonic-gate * If not found the table is created.
13567c478bd9Sstevel@tonic-gate *
13577c478bd9Sstevel@tonic-gate * Since we can't hold a hash table mutex during allocation, we have to
13587c478bd9Sstevel@tonic-gate * drop it and redo the search on a create. Then we may have to free the newly
13597c478bd9Sstevel@tonic-gate * allocated htable if another thread raced in and created it ahead of us.
13607c478bd9Sstevel@tonic-gate */
13617c478bd9Sstevel@tonic-gate htable_t *
htable_create(hat_t * hat,uintptr_t vaddr,level_t level,htable_t * shared)13627c478bd9Sstevel@tonic-gate htable_create(
13637c478bd9Sstevel@tonic-gate hat_t *hat,
13647c478bd9Sstevel@tonic-gate uintptr_t vaddr,
13657c478bd9Sstevel@tonic-gate level_t level,
13667c478bd9Sstevel@tonic-gate htable_t *shared)
13677c478bd9Sstevel@tonic-gate {
13687c478bd9Sstevel@tonic-gate uint_t h;
13697c478bd9Sstevel@tonic-gate level_t l;
13707c478bd9Sstevel@tonic-gate uintptr_t base;
13717c478bd9Sstevel@tonic-gate htable_t *ht;
13727c478bd9Sstevel@tonic-gate htable_t *higher = NULL;
13737c478bd9Sstevel@tonic-gate htable_t *new = NULL;
13747c478bd9Sstevel@tonic-gate
13757c478bd9Sstevel@tonic-gate if (level < 0 || level > TOP_LEVEL(hat))
13767c478bd9Sstevel@tonic-gate panic("htable_create(): level %d out of range\n", level);
13777c478bd9Sstevel@tonic-gate
1378584b574aSToomas Soome ht = NULL;
13797c478bd9Sstevel@tonic-gate /*
13807c478bd9Sstevel@tonic-gate * Create the page tables in top down order.
13817c478bd9Sstevel@tonic-gate */
13827c478bd9Sstevel@tonic-gate for (l = TOP_LEVEL(hat); l >= level; --l) {
13837c478bd9Sstevel@tonic-gate new = NULL;
13847c478bd9Sstevel@tonic-gate if (l == TOP_LEVEL(hat))
13857c478bd9Sstevel@tonic-gate base = 0;
13867c478bd9Sstevel@tonic-gate else
13877c478bd9Sstevel@tonic-gate base = vaddr & LEVEL_MASK(l + 1);
13887c478bd9Sstevel@tonic-gate
13897c478bd9Sstevel@tonic-gate h = HTABLE_HASH(hat, base, l);
13907c478bd9Sstevel@tonic-gate try_again:
13917c478bd9Sstevel@tonic-gate /*
13927c478bd9Sstevel@tonic-gate * look up the htable at this level
13937c478bd9Sstevel@tonic-gate */
13947c478bd9Sstevel@tonic-gate HTABLE_ENTER(h);
13957c478bd9Sstevel@tonic-gate if (l == TOP_LEVEL(hat)) {
13967c478bd9Sstevel@tonic-gate ht = hat->hat_htable;
13977c478bd9Sstevel@tonic-gate } else {
13987c478bd9Sstevel@tonic-gate for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) {
13997c478bd9Sstevel@tonic-gate ASSERT(ht->ht_hat == hat);
14007c478bd9Sstevel@tonic-gate if (ht->ht_vaddr == base &&
14017c478bd9Sstevel@tonic-gate ht->ht_level == l)
14027c478bd9Sstevel@tonic-gate break;
14037c478bd9Sstevel@tonic-gate }
14047c478bd9Sstevel@tonic-gate }
14057c478bd9Sstevel@tonic-gate
14067c478bd9Sstevel@tonic-gate /*
14077c478bd9Sstevel@tonic-gate * if we found the htable, increment its busy cnt
14087c478bd9Sstevel@tonic-gate * and if we had allocated a new htable, free it.
14097c478bd9Sstevel@tonic-gate */
14107c478bd9Sstevel@tonic-gate if (ht != NULL) {
14117c478bd9Sstevel@tonic-gate /*
14127c478bd9Sstevel@tonic-gate * If we find a pre-existing shared table, it must
14137c478bd9Sstevel@tonic-gate * share from the same place.
14147c478bd9Sstevel@tonic-gate */
14157c478bd9Sstevel@tonic-gate if (l == level && shared && ht->ht_shares &&
14167c478bd9Sstevel@tonic-gate ht->ht_shares != shared) {
14177c478bd9Sstevel@tonic-gate panic("htable shared from wrong place "
1418903a11ebSrh "found htable=%p shared=%p",
1419903a11ebSrh (void *)ht, (void *)shared);
14207c478bd9Sstevel@tonic-gate }
14217c478bd9Sstevel@tonic-gate ++ht->ht_busy;
14227c478bd9Sstevel@tonic-gate HTABLE_EXIT(h);
14237c478bd9Sstevel@tonic-gate if (new)
14247c478bd9Sstevel@tonic-gate htable_free(new);
14257c478bd9Sstevel@tonic-gate if (higher != NULL)
14267c478bd9Sstevel@tonic-gate htable_release(higher);
14277c478bd9Sstevel@tonic-gate higher = ht;
14287c478bd9Sstevel@tonic-gate
14297c478bd9Sstevel@tonic-gate /*
14307c478bd9Sstevel@tonic-gate * if we didn't find it on the first search
14317c478bd9Sstevel@tonic-gate * allocate a new one and search again
14327c478bd9Sstevel@tonic-gate */
14337c478bd9Sstevel@tonic-gate } else if (new == NULL) {
14347c478bd9Sstevel@tonic-gate HTABLE_EXIT(h);
14357c478bd9Sstevel@tonic-gate new = htable_alloc(hat, base, l,
14367c478bd9Sstevel@tonic-gate l == level ? shared : NULL);
14377c478bd9Sstevel@tonic-gate goto try_again;
14387c478bd9Sstevel@tonic-gate
14397c478bd9Sstevel@tonic-gate /*
14407c478bd9Sstevel@tonic-gate * 2nd search and still not there, use "new" table
14417c478bd9Sstevel@tonic-gate * Link new table into higher, when not at top level.
14427c478bd9Sstevel@tonic-gate */
14437c478bd9Sstevel@tonic-gate } else {
14447c478bd9Sstevel@tonic-gate ht = new;
14457c478bd9Sstevel@tonic-gate if (higher != NULL) {
14467c478bd9Sstevel@tonic-gate link_ptp(higher, ht, base);
14477c478bd9Sstevel@tonic-gate ht->ht_parent = higher;
14487c478bd9Sstevel@tonic-gate }
14497c478bd9Sstevel@tonic-gate ht->ht_next = hat->hat_ht_hash[h];
14507c478bd9Sstevel@tonic-gate ASSERT(ht->ht_prev == NULL);
14517c478bd9Sstevel@tonic-gate if (hat->hat_ht_hash[h])
14527c478bd9Sstevel@tonic-gate hat->hat_ht_hash[h]->ht_prev = ht;
14537c478bd9Sstevel@tonic-gate hat->hat_ht_hash[h] = ht;
14547c478bd9Sstevel@tonic-gate HTABLE_EXIT(h);
14557c478bd9Sstevel@tonic-gate
14567c478bd9Sstevel@tonic-gate /*
14577c478bd9Sstevel@tonic-gate * Note we don't do htable_release(higher).
14587c478bd9Sstevel@tonic-gate * That happens recursively when "new" is removed by
14597c478bd9Sstevel@tonic-gate * htable_release() or htable_steal().
14607c478bd9Sstevel@tonic-gate */
14617c478bd9Sstevel@tonic-gate higher = ht;
14627c478bd9Sstevel@tonic-gate
14637c478bd9Sstevel@tonic-gate /*
14647c478bd9Sstevel@tonic-gate * If we just created a new shared page table we
14657c478bd9Sstevel@tonic-gate * increment the shared htable's busy count, so that
14667c478bd9Sstevel@tonic-gate * it can't be the victim of a steal even if it's empty.
14677c478bd9Sstevel@tonic-gate */
14687c478bd9Sstevel@tonic-gate if (l == level && shared) {
14697c478bd9Sstevel@tonic-gate (void) htable_lookup(shared->ht_hat,
14707c478bd9Sstevel@tonic-gate shared->ht_vaddr, shared->ht_level);
14717c478bd9Sstevel@tonic-gate HATSTAT_INC(hs_htable_shared);
14727c478bd9Sstevel@tonic-gate }
14737c478bd9Sstevel@tonic-gate }
14747c478bd9Sstevel@tonic-gate }
14757c478bd9Sstevel@tonic-gate
14767c478bd9Sstevel@tonic-gate return (ht);
14777c478bd9Sstevel@tonic-gate }
14787c478bd9Sstevel@tonic-gate
1479ae115bc7Smrj /*
1480843e1988Sjohnlev * Inherit initial pagetables from the boot program. On the 64-bit
1481843e1988Sjohnlev * hypervisor we also temporarily mark the p_index field of page table
1482843e1988Sjohnlev * pages, so we know not to try making them writable in seg_kpm.
1483ae115bc7Smrj */
1484ae115bc7Smrj void
htable_attach(hat_t * hat,uintptr_t base,level_t level,htable_t * parent,pfn_t pfn)1485ae115bc7Smrj htable_attach(
1486ae115bc7Smrj hat_t *hat,
1487ae115bc7Smrj uintptr_t base,
1488ae115bc7Smrj level_t level,
1489ae115bc7Smrj htable_t *parent,
1490ae115bc7Smrj pfn_t pfn)
1491ae115bc7Smrj {
1492ae115bc7Smrj htable_t *ht;
1493ae115bc7Smrj uint_t h;
1494ae115bc7Smrj uint_t i;
1495ae115bc7Smrj x86pte_t pte;
1496ae115bc7Smrj x86pte_t *ptep;
1497ae115bc7Smrj page_t *pp;
1498ae115bc7Smrj extern page_t *boot_claim_page(pfn_t);
1499ae115bc7Smrj
1500ae115bc7Smrj ht = htable_get_reserve();
1501ae115bc7Smrj if (level == mmu.max_level)
1502ae115bc7Smrj kas.a_hat->hat_htable = ht;
1503ae115bc7Smrj ht->ht_hat = hat;
1504ae115bc7Smrj ht->ht_parent = parent;
1505ae115bc7Smrj ht->ht_vaddr = base;
1506ae115bc7Smrj ht->ht_level = level;
1507ae115bc7Smrj ht->ht_busy = 1;
1508ae115bc7Smrj ht->ht_next = NULL;
1509ae115bc7Smrj ht->ht_prev = NULL;
1510ae115bc7Smrj ht->ht_flags = 0;
1511ae115bc7Smrj ht->ht_pfn = pfn;
1512ae115bc7Smrj ht->ht_lock_cnt = 0;
1513ae115bc7Smrj ht->ht_valid_cnt = 0;
1514ae115bc7Smrj if (parent != NULL)
1515ae115bc7Smrj ++parent->ht_busy;
1516ae115bc7Smrj
1517ae115bc7Smrj h = HTABLE_HASH(hat, base, level);
1518ae115bc7Smrj HTABLE_ENTER(h);
1519ae115bc7Smrj ht->ht_next = hat->hat_ht_hash[h];
1520ae115bc7Smrj ASSERT(ht->ht_prev == NULL);
1521ae115bc7Smrj if (hat->hat_ht_hash[h])
1522ae115bc7Smrj hat->hat_ht_hash[h]->ht_prev = ht;
1523ae115bc7Smrj hat->hat_ht_hash[h] = ht;
1524ae115bc7Smrj HTABLE_EXIT(h);
1525ae115bc7Smrj
1526ae115bc7Smrj /*
1527ae115bc7Smrj * make sure the page table physical page is not FREE
1528ae115bc7Smrj */
1529ae115bc7Smrj if (page_resv(1, KM_NOSLEEP) == 0)
1530ae115bc7Smrj panic("page_resv() failed in ptable alloc");
1531ae115bc7Smrj
1532ae115bc7Smrj pp = boot_claim_page(pfn);
1533ae115bc7Smrj ASSERT(pp != NULL);
15342d44e974SJoe Bonasera
15352d44e974SJoe Bonasera /*
15362d44e974SJoe Bonasera * Page table pages that were allocated by dboot or
15372d44e974SJoe Bonasera * in very early startup didn't go through boot_mapin()
15382d44e974SJoe Bonasera * and so won't have vnode/offsets. Fix that here.
15392d44e974SJoe Bonasera */
15402d44e974SJoe Bonasera if (pp->p_vnode == NULL) {
15412d44e974SJoe Bonasera /* match offset calculation in page_get_physical() */
15422d44e974SJoe Bonasera u_offset_t offset = (uintptr_t)ht;
15432d44e974SJoe Bonasera if (offset > kernelbase)
15442d44e974SJoe Bonasera offset -= kernelbase;
15452d44e974SJoe Bonasera offset <<= MMU_PAGESHIFT;
1546e4c16229SJoe Bonasera offset += mmu.hole_start; /* something in VA hole */
15472d44e974SJoe Bonasera ASSERT(page_exists(&kvp, offset) == NULL);
15482d44e974SJoe Bonasera (void) page_hashin(pp, &kvp, offset, NULL);
15492d44e974SJoe Bonasera }
1550ae115bc7Smrj page_downgrade(pp);
1551*86ef0a63SRichard Lowe #if defined(__xpv)
1552ae115bc7Smrj /*
1553ae115bc7Smrj * Record in the page_t that is a pagetable for segkpm setup.
1554ae115bc7Smrj */
1555ae115bc7Smrj if (kpm_vbase)
1556ae115bc7Smrj pp->p_index = 1;
1557843e1988Sjohnlev #endif
1558ae115bc7Smrj
1559ae115bc7Smrj /*
1560ae115bc7Smrj * Count valid mappings and recursively attach lower level pagetables.
1561ae115bc7Smrj */
1562ae115bc7Smrj ptep = kbm_remap_window(pfn_to_pa(pfn), 0);
1563ae115bc7Smrj for (i = 0; i < HTABLE_NUM_PTES(ht); ++i) {
1564ae115bc7Smrj if (mmu.pae_hat)
1565ae115bc7Smrj pte = ptep[i];
1566ae115bc7Smrj else
1567ae115bc7Smrj pte = ((x86pte32_t *)ptep)[i];
1568ae115bc7Smrj if (!IN_HYPERVISOR_VA(base) && PTE_ISVALID(pte)) {
1569ae115bc7Smrj ++ht->ht_valid_cnt;
1570ae115bc7Smrj if (!PTE_ISPAGE(pte, level)) {
1571ae115bc7Smrj htable_attach(hat, base, level - 1,
1572ae115bc7Smrj ht, PTE2PFN(pte, level));
1573ae115bc7Smrj ptep = kbm_remap_window(pfn_to_pa(pfn), 0);
1574ae115bc7Smrj }
1575ae115bc7Smrj }
1576ae115bc7Smrj base += LEVEL_SIZE(level);
1577ae115bc7Smrj if (base == mmu.hole_start)
1578ae115bc7Smrj base = (mmu.hole_end + MMU_PAGEOFFSET) & MMU_PAGEMASK;
1579ae115bc7Smrj }
1580ae115bc7Smrj
1581ae115bc7Smrj /*
1582ae115bc7Smrj * As long as all the mappings we had were below kernel base
1583ae115bc7Smrj * we can release the htable.
1584ae115bc7Smrj */
1585ae115bc7Smrj if (base < kernelbase)
1586ae115bc7Smrj htable_release(ht);
1587ae115bc7Smrj }
1588ae115bc7Smrj
15897c478bd9Sstevel@tonic-gate /*
15907c478bd9Sstevel@tonic-gate * Walk through a given htable looking for the first valid entry. This
15917c478bd9Sstevel@tonic-gate * routine takes both a starting and ending address. The starting address
15927c478bd9Sstevel@tonic-gate * is required to be within the htable provided by the caller, but there is
15937c478bd9Sstevel@tonic-gate * no such restriction on the ending address.
15947c478bd9Sstevel@tonic-gate *
15957c478bd9Sstevel@tonic-gate * If the routine finds a valid entry in the htable (at or beyond the
15967c478bd9Sstevel@tonic-gate * starting address), the PTE (and its address) will be returned.
15977c478bd9Sstevel@tonic-gate * This PTE may correspond to either a page or a pagetable - it is the
15987c478bd9Sstevel@tonic-gate * caller's responsibility to determine which. If no valid entry is
15997c478bd9Sstevel@tonic-gate * found, 0 (and invalid PTE) and the next unexamined address will be
16007c478bd9Sstevel@tonic-gate * returned.
16017c478bd9Sstevel@tonic-gate *
16027c478bd9Sstevel@tonic-gate * The loop has been carefully coded for optimization.
16037c478bd9Sstevel@tonic-gate */
16047c478bd9Sstevel@tonic-gate static x86pte_t
htable_scan(htable_t * ht,uintptr_t * vap,uintptr_t eaddr)16057c478bd9Sstevel@tonic-gate htable_scan(htable_t *ht, uintptr_t *vap, uintptr_t eaddr)
16067c478bd9Sstevel@tonic-gate {
16077c478bd9Sstevel@tonic-gate uint_t e;
16087c478bd9Sstevel@tonic-gate x86pte_t found_pte = (x86pte_t)0;
1609ae115bc7Smrj caddr_t pte_ptr;
1610ae115bc7Smrj caddr_t end_pte_ptr;
16117c478bd9Sstevel@tonic-gate int l = ht->ht_level;
16127c478bd9Sstevel@tonic-gate uintptr_t va = *vap & LEVEL_MASK(l);
16137c478bd9Sstevel@tonic-gate size_t pgsize = LEVEL_SIZE(l);
16147c478bd9Sstevel@tonic-gate
16157c478bd9Sstevel@tonic-gate ASSERT(va >= ht->ht_vaddr);
16167c478bd9Sstevel@tonic-gate ASSERT(va <= HTABLE_LAST_PAGE(ht));
16177c478bd9Sstevel@tonic-gate
16187c478bd9Sstevel@tonic-gate /*
16197c478bd9Sstevel@tonic-gate * Compute the starting index and ending virtual address
16207c478bd9Sstevel@tonic-gate */
16217c478bd9Sstevel@tonic-gate e = htable_va2entry(va, ht);
16227c478bd9Sstevel@tonic-gate
16237c478bd9Sstevel@tonic-gate /*
16247c478bd9Sstevel@tonic-gate * The following page table scan code knows that the valid
16257c478bd9Sstevel@tonic-gate * bit of a PTE is in the lowest byte AND that x86 is little endian!!
16267c478bd9Sstevel@tonic-gate */
1627ae115bc7Smrj pte_ptr = (caddr_t)x86pte_access_pagetable(ht, 0);
1628ae115bc7Smrj end_pte_ptr = (caddr_t)PT_INDEX_PTR(pte_ptr, HTABLE_NUM_PTES(ht));
1629ae115bc7Smrj pte_ptr = (caddr_t)PT_INDEX_PTR((x86pte_t *)pte_ptr, e);
163030f7a194Skchow while (!PTE_ISVALID(*pte_ptr)) {
16317c478bd9Sstevel@tonic-gate va += pgsize;
16327c478bd9Sstevel@tonic-gate if (va >= eaddr)
16337c478bd9Sstevel@tonic-gate break;
16347c478bd9Sstevel@tonic-gate pte_ptr += mmu.pte_size;
16357c478bd9Sstevel@tonic-gate ASSERT(pte_ptr <= end_pte_ptr);
16367c478bd9Sstevel@tonic-gate if (pte_ptr == end_pte_ptr)
16377c478bd9Sstevel@tonic-gate break;
16387c478bd9Sstevel@tonic-gate }
16397c478bd9Sstevel@tonic-gate
16407c478bd9Sstevel@tonic-gate /*
16417c478bd9Sstevel@tonic-gate * if we found a valid PTE, load the entire PTE
16427c478bd9Sstevel@tonic-gate */
1643ae115bc7Smrj if (va < eaddr && pte_ptr != end_pte_ptr)
1644ae115bc7Smrj found_pte = GET_PTE((x86pte_t *)pte_ptr);
16457c478bd9Sstevel@tonic-gate x86pte_release_pagetable(ht);
16467c478bd9Sstevel@tonic-gate
16477c478bd9Sstevel@tonic-gate /*
16487c478bd9Sstevel@tonic-gate * deal with VA hole on amd64
16497c478bd9Sstevel@tonic-gate */
16507c478bd9Sstevel@tonic-gate if (l == mmu.max_level && va >= mmu.hole_start && va <= mmu.hole_end)
16517c478bd9Sstevel@tonic-gate va = mmu.hole_end + va - mmu.hole_start;
16527c478bd9Sstevel@tonic-gate
16537c478bd9Sstevel@tonic-gate *vap = va;
16547c478bd9Sstevel@tonic-gate return (found_pte);
16557c478bd9Sstevel@tonic-gate }
16567c478bd9Sstevel@tonic-gate
16577c478bd9Sstevel@tonic-gate /*
16587c478bd9Sstevel@tonic-gate * Find the address and htable for the first populated translation at or
16597c478bd9Sstevel@tonic-gate * above the given virtual address. The caller may also specify an upper
16607c478bd9Sstevel@tonic-gate * limit to the address range to search. Uses level information to quickly
16617c478bd9Sstevel@tonic-gate * skip unpopulated sections of virtual address spaces.
16627c478bd9Sstevel@tonic-gate *
16637c478bd9Sstevel@tonic-gate * If not found returns NULL. When found, returns the htable and virt addr
16647c478bd9Sstevel@tonic-gate * and has a hold on the htable.
16657c478bd9Sstevel@tonic-gate */
16667c478bd9Sstevel@tonic-gate x86pte_t
htable_walk(struct hat * hat,htable_t ** htp,uintptr_t * vaddr,uintptr_t eaddr)16677c478bd9Sstevel@tonic-gate htable_walk(
16687c478bd9Sstevel@tonic-gate struct hat *hat,
16697c478bd9Sstevel@tonic-gate htable_t **htp,
16707c478bd9Sstevel@tonic-gate uintptr_t *vaddr,
16717c478bd9Sstevel@tonic-gate uintptr_t eaddr)
16727c478bd9Sstevel@tonic-gate {
16737c478bd9Sstevel@tonic-gate uintptr_t va = *vaddr;
16747c478bd9Sstevel@tonic-gate htable_t *ht;
16757c478bd9Sstevel@tonic-gate htable_t *prev = *htp;
16767c478bd9Sstevel@tonic-gate level_t l;
16777c478bd9Sstevel@tonic-gate level_t max_mapped_level;
16787c478bd9Sstevel@tonic-gate x86pte_t pte;
16797c478bd9Sstevel@tonic-gate
16807c478bd9Sstevel@tonic-gate ASSERT(eaddr > va);
16817c478bd9Sstevel@tonic-gate
16827c478bd9Sstevel@tonic-gate /*
16837c478bd9Sstevel@tonic-gate * If this is a user address, then we know we need not look beyond
16847c478bd9Sstevel@tonic-gate * kernelbase.
16857c478bd9Sstevel@tonic-gate */
16867c478bd9Sstevel@tonic-gate ASSERT(hat == kas.a_hat || eaddr <= kernelbase ||
16877c478bd9Sstevel@tonic-gate eaddr == HTABLE_WALK_TO_END);
16887c478bd9Sstevel@tonic-gate if (hat != kas.a_hat && eaddr == HTABLE_WALK_TO_END)
16897c478bd9Sstevel@tonic-gate eaddr = kernelbase;
16907c478bd9Sstevel@tonic-gate
16917c478bd9Sstevel@tonic-gate /*
16927c478bd9Sstevel@tonic-gate * If we're coming in with a previous page table, search it first
16937c478bd9Sstevel@tonic-gate * without doing an htable_lookup(), this should be frequent.
16947c478bd9Sstevel@tonic-gate */
16957c478bd9Sstevel@tonic-gate if (prev) {
16967c478bd9Sstevel@tonic-gate ASSERT(prev->ht_busy > 0);
16977c478bd9Sstevel@tonic-gate ASSERT(prev->ht_vaddr <= va);
16987c478bd9Sstevel@tonic-gate l = prev->ht_level;
16997c478bd9Sstevel@tonic-gate if (va <= HTABLE_LAST_PAGE(prev)) {
17007c478bd9Sstevel@tonic-gate pte = htable_scan(prev, &va, eaddr);
17017c478bd9Sstevel@tonic-gate
17027c478bd9Sstevel@tonic-gate if (PTE_ISPAGE(pte, l)) {
17037c478bd9Sstevel@tonic-gate *vaddr = va;
17047c478bd9Sstevel@tonic-gate *htp = prev;
17057c478bd9Sstevel@tonic-gate return (pte);
17067c478bd9Sstevel@tonic-gate }
17077c478bd9Sstevel@tonic-gate }
17087c478bd9Sstevel@tonic-gate
17097c478bd9Sstevel@tonic-gate /*
17107c478bd9Sstevel@tonic-gate * We found nothing in the htable provided by the caller,
17117c478bd9Sstevel@tonic-gate * so fall through and do the full search
17127c478bd9Sstevel@tonic-gate */
17137c478bd9Sstevel@tonic-gate htable_release(prev);
17147c478bd9Sstevel@tonic-gate }
17157c478bd9Sstevel@tonic-gate
17167c478bd9Sstevel@tonic-gate /*
17177c478bd9Sstevel@tonic-gate * Find the level of the largest pagesize used by this HAT.
17187c478bd9Sstevel@tonic-gate */
17197173d045Sjosephb if (hat->hat_ism_pgcnt > 0) {
172002bc52beSkchow max_mapped_level = mmu.umax_page_level;
17217173d045Sjosephb } else {
17227173d045Sjosephb max_mapped_level = 0;
17237173d045Sjosephb for (l = 1; l <= mmu.max_page_level; ++l)
17247173d045Sjosephb if (hat->hat_pages_mapped[l] != 0)
17257173d045Sjosephb max_mapped_level = l;
17267173d045Sjosephb }
17277c478bd9Sstevel@tonic-gate
17287c478bd9Sstevel@tonic-gate while (va < eaddr && va >= *vaddr) {
17297c478bd9Sstevel@tonic-gate /*
17307c478bd9Sstevel@tonic-gate * Find lowest table with any entry for given address.
17317c478bd9Sstevel@tonic-gate */
17327c478bd9Sstevel@tonic-gate for (l = 0; l <= TOP_LEVEL(hat); ++l) {
17337c478bd9Sstevel@tonic-gate ht = htable_lookup(hat, va, l);
17347c478bd9Sstevel@tonic-gate if (ht != NULL) {
17357c478bd9Sstevel@tonic-gate pte = htable_scan(ht, &va, eaddr);
17367c478bd9Sstevel@tonic-gate if (PTE_ISPAGE(pte, l)) {
17378c1d5be3SJoshua M. Clulow VERIFY(!IN_VA_HOLE(va));
17387c478bd9Sstevel@tonic-gate *vaddr = va;
17397c478bd9Sstevel@tonic-gate *htp = ht;
17407c478bd9Sstevel@tonic-gate return (pte);
17417c478bd9Sstevel@tonic-gate }
17427c478bd9Sstevel@tonic-gate htable_release(ht);
17437c478bd9Sstevel@tonic-gate break;
17447c478bd9Sstevel@tonic-gate }
17457c478bd9Sstevel@tonic-gate
17468b5842f9Sdm /*
17477173d045Sjosephb * No htable at this level for the address. If there
17487173d045Sjosephb * is no larger page size that could cover it, we can
17497173d045Sjosephb * skip right to the start of the next page table.
17508b5842f9Sdm */
17518b5842f9Sdm ASSERT(l < TOP_LEVEL(hat));
17528b5842f9Sdm if (l >= max_mapped_level) {
17537c478bd9Sstevel@tonic-gate va = NEXT_ENTRY_VA(va, l + 1);
17547173d045Sjosephb if (va >= eaddr)
17557173d045Sjosephb break;
17568b5842f9Sdm }
17577c478bd9Sstevel@tonic-gate }
17587c478bd9Sstevel@tonic-gate }
17597c478bd9Sstevel@tonic-gate
17607c478bd9Sstevel@tonic-gate *vaddr = 0;
17617c478bd9Sstevel@tonic-gate *htp = NULL;
17627c478bd9Sstevel@tonic-gate return (0);
17637c478bd9Sstevel@tonic-gate }
17647c478bd9Sstevel@tonic-gate
17657c478bd9Sstevel@tonic-gate /*
17667c478bd9Sstevel@tonic-gate * Find the htable and page table entry index of the given virtual address
17677c478bd9Sstevel@tonic-gate * with pagesize at or below given level.
17687c478bd9Sstevel@tonic-gate * If not found returns NULL. When found, returns the htable, sets
17697c478bd9Sstevel@tonic-gate * entry, and has a hold on the htable.
17707c478bd9Sstevel@tonic-gate */
17717c478bd9Sstevel@tonic-gate htable_t *
htable_getpte(struct hat * hat,uintptr_t vaddr,uint_t * entry,x86pte_t * pte,level_t level)17727c478bd9Sstevel@tonic-gate htable_getpte(
17737c478bd9Sstevel@tonic-gate struct hat *hat,
17747c478bd9Sstevel@tonic-gate uintptr_t vaddr,
17757c478bd9Sstevel@tonic-gate uint_t *entry,
17767c478bd9Sstevel@tonic-gate x86pte_t *pte,
17777c478bd9Sstevel@tonic-gate level_t level)
17787c478bd9Sstevel@tonic-gate {
17797c478bd9Sstevel@tonic-gate htable_t *ht;
17807c478bd9Sstevel@tonic-gate level_t l;
17817c478bd9Sstevel@tonic-gate uint_t e;
17827c478bd9Sstevel@tonic-gate
17837c478bd9Sstevel@tonic-gate ASSERT(level <= mmu.max_page_level);
17847c478bd9Sstevel@tonic-gate
17857c478bd9Sstevel@tonic-gate for (l = 0; l <= level; ++l) {
17867c478bd9Sstevel@tonic-gate ht = htable_lookup(hat, vaddr, l);
17877c478bd9Sstevel@tonic-gate if (ht == NULL)
17887c478bd9Sstevel@tonic-gate continue;
17897c478bd9Sstevel@tonic-gate e = htable_va2entry(vaddr, ht);
17907c478bd9Sstevel@tonic-gate if (entry != NULL)
17917c478bd9Sstevel@tonic-gate *entry = e;
17927c478bd9Sstevel@tonic-gate if (pte != NULL)
17937c478bd9Sstevel@tonic-gate *pte = x86pte_get(ht, e);
17947c478bd9Sstevel@tonic-gate return (ht);
17957c478bd9Sstevel@tonic-gate }
17967c478bd9Sstevel@tonic-gate return (NULL);
17977c478bd9Sstevel@tonic-gate }
17987c478bd9Sstevel@tonic-gate
17997c478bd9Sstevel@tonic-gate /*
18007c478bd9Sstevel@tonic-gate * Find the htable and page table entry index of the given virtual address.
18017c478bd9Sstevel@tonic-gate * There must be a valid page mapped at the given address.
18027c478bd9Sstevel@tonic-gate * If not found returns NULL. When found, returns the htable, sets
18037c478bd9Sstevel@tonic-gate * entry, and has a hold on the htable.
18047c478bd9Sstevel@tonic-gate */
18057c478bd9Sstevel@tonic-gate htable_t *
htable_getpage(struct hat * hat,uintptr_t vaddr,uint_t * entry)18067c478bd9Sstevel@tonic-gate htable_getpage(struct hat *hat, uintptr_t vaddr, uint_t *entry)
18077c478bd9Sstevel@tonic-gate {
18087c478bd9Sstevel@tonic-gate htable_t *ht;
18097c478bd9Sstevel@tonic-gate uint_t e;
18107c478bd9Sstevel@tonic-gate x86pte_t pte;
18117c478bd9Sstevel@tonic-gate
18127c478bd9Sstevel@tonic-gate ht = htable_getpte(hat, vaddr, &e, &pte, mmu.max_page_level);
18137c478bd9Sstevel@tonic-gate if (ht == NULL)
18147c478bd9Sstevel@tonic-gate return (NULL);
18157c478bd9Sstevel@tonic-gate
18167c478bd9Sstevel@tonic-gate if (entry)
18177c478bd9Sstevel@tonic-gate *entry = e;
18187c478bd9Sstevel@tonic-gate
18197c478bd9Sstevel@tonic-gate if (PTE_ISPAGE(pte, ht->ht_level))
18207c478bd9Sstevel@tonic-gate return (ht);
18217c478bd9Sstevel@tonic-gate htable_release(ht);
18227c478bd9Sstevel@tonic-gate return (NULL);
18237c478bd9Sstevel@tonic-gate }
18247c478bd9Sstevel@tonic-gate
18257c478bd9Sstevel@tonic-gate
18267c478bd9Sstevel@tonic-gate void
htable_init()18277c478bd9Sstevel@tonic-gate htable_init()
18287c478bd9Sstevel@tonic-gate {
18297c478bd9Sstevel@tonic-gate /*
18307c478bd9Sstevel@tonic-gate * To save on kernel VA usage, we avoid debug information in 32 bit
18317c478bd9Sstevel@tonic-gate * kernels.
18327c478bd9Sstevel@tonic-gate */
18337c478bd9Sstevel@tonic-gate int kmem_flags = KMC_NOHASH;
18347c478bd9Sstevel@tonic-gate
18357c478bd9Sstevel@tonic-gate /*
18367c478bd9Sstevel@tonic-gate * initialize kmem caches
18377c478bd9Sstevel@tonic-gate */
18387c478bd9Sstevel@tonic-gate htable_cache = kmem_cache_create("htable_t",
18397c478bd9Sstevel@tonic-gate sizeof (htable_t), 0, NULL, NULL,
18407c478bd9Sstevel@tonic-gate htable_reap, NULL, hat_memload_arena, kmem_flags);
18417c478bd9Sstevel@tonic-gate }
18427c478bd9Sstevel@tonic-gate
18437c478bd9Sstevel@tonic-gate /*
18447c478bd9Sstevel@tonic-gate * get the pte index for the virtual address in the given htable's pagetable
18457c478bd9Sstevel@tonic-gate */
18467c478bd9Sstevel@tonic-gate uint_t
htable_va2entry(uintptr_t va,htable_t * ht)18477c478bd9Sstevel@tonic-gate htable_va2entry(uintptr_t va, htable_t *ht)
18487c478bd9Sstevel@tonic-gate {
18497c478bd9Sstevel@tonic-gate level_t l = ht->ht_level;
18507c478bd9Sstevel@tonic-gate
18517c478bd9Sstevel@tonic-gate ASSERT(va >= ht->ht_vaddr);
18527c478bd9Sstevel@tonic-gate ASSERT(va <= HTABLE_LAST_PAGE(ht));
1853ae115bc7Smrj return ((va >> LEVEL_SHIFT(l)) & (HTABLE_NUM_PTES(ht) - 1));
18547c478bd9Sstevel@tonic-gate }
18557c478bd9Sstevel@tonic-gate
18567c478bd9Sstevel@tonic-gate /*
18577c478bd9Sstevel@tonic-gate * Given an htable and the index of a pte in it, return the virtual address
18587c478bd9Sstevel@tonic-gate * of the page.
18597c478bd9Sstevel@tonic-gate */
18607c478bd9Sstevel@tonic-gate uintptr_t
htable_e2va(htable_t * ht,uint_t entry)18617c478bd9Sstevel@tonic-gate htable_e2va(htable_t *ht, uint_t entry)
18627c478bd9Sstevel@tonic-gate {
18637c478bd9Sstevel@tonic-gate level_t l = ht->ht_level;
18647c478bd9Sstevel@tonic-gate uintptr_t va;
18657c478bd9Sstevel@tonic-gate
1866ae115bc7Smrj ASSERT(entry < HTABLE_NUM_PTES(ht));
18677c478bd9Sstevel@tonic-gate va = ht->ht_vaddr + ((uintptr_t)entry << LEVEL_SHIFT(l));
18687c478bd9Sstevel@tonic-gate
18697c478bd9Sstevel@tonic-gate /*
18707c478bd9Sstevel@tonic-gate * Need to skip over any VA hole in top level table
18717c478bd9Sstevel@tonic-gate */
18727c478bd9Sstevel@tonic-gate if (ht->ht_level == mmu.max_level && va >= mmu.hole_start)
18737c478bd9Sstevel@tonic-gate va += ((mmu.hole_end - mmu.hole_start) + 1);
18747c478bd9Sstevel@tonic-gate
18757c478bd9Sstevel@tonic-gate return (va);
18767c478bd9Sstevel@tonic-gate }
18777c478bd9Sstevel@tonic-gate
18787c478bd9Sstevel@tonic-gate /*
18797c478bd9Sstevel@tonic-gate * The code uses compare and swap instructions to read/write PTE's to
18807c478bd9Sstevel@tonic-gate * avoid atomicity problems, since PTEs can be 8 bytes on 32 bit systems.
18817c478bd9Sstevel@tonic-gate * will naturally be atomic.
18827c478bd9Sstevel@tonic-gate *
18837c478bd9Sstevel@tonic-gate * The combination of using kpreempt_disable()/_enable() and the hci_mutex
18847c478bd9Sstevel@tonic-gate * are used to ensure that an interrupt won't overwrite a temporary mapping
18857c478bd9Sstevel@tonic-gate * while it's in use. If an interrupt thread tries to access a PTE, it will
18867c478bd9Sstevel@tonic-gate * yield briefly back to the pinned thread which holds the cpu's hci_mutex.
18877c478bd9Sstevel@tonic-gate */
18887c478bd9Sstevel@tonic-gate void
x86pte_cpu_init(cpu_t * cpu)1889ae115bc7Smrj x86pte_cpu_init(cpu_t *cpu)
18907c478bd9Sstevel@tonic-gate {
18917c478bd9Sstevel@tonic-gate struct hat_cpu_info *hci;
18927c478bd9Sstevel@tonic-gate
1893ae115bc7Smrj hci = kmem_zalloc(sizeof (*hci), KM_SLEEP);
18947c478bd9Sstevel@tonic-gate mutex_init(&hci->hci_mutex, NULL, MUTEX_DEFAULT, NULL);
1895ae115bc7Smrj cpu->cpu_hat_info = hci;
1896ae115bc7Smrj }
18977c478bd9Sstevel@tonic-gate
1898ae115bc7Smrj void
x86pte_cpu_fini(cpu_t * cpu)1899ae115bc7Smrj x86pte_cpu_fini(cpu_t *cpu)
1900ae115bc7Smrj {
1901ae115bc7Smrj struct hat_cpu_info *hci = cpu->cpu_hat_info;
19027c478bd9Sstevel@tonic-gate
1903ae115bc7Smrj kmem_free(hci, sizeof (*hci));
1904ae115bc7Smrj cpu->cpu_hat_info = NULL;
19057c478bd9Sstevel@tonic-gate }
19067c478bd9Sstevel@tonic-gate
19077c478bd9Sstevel@tonic-gate /*
19087c478bd9Sstevel@tonic-gate * Disable preemption and establish a mapping to the pagetable with the
19097c478bd9Sstevel@tonic-gate * given pfn. This is optimized for there case where it's the same
19107c478bd9Sstevel@tonic-gate * pfn as we last used referenced from this CPU.
19117c478bd9Sstevel@tonic-gate */
19127c478bd9Sstevel@tonic-gate static x86pte_t *
x86pte_access_pagetable(htable_t * ht,uint_t index)1913ae115bc7Smrj x86pte_access_pagetable(htable_t *ht, uint_t index)
19147c478bd9Sstevel@tonic-gate {
19157c478bd9Sstevel@tonic-gate /*
191674ecdb51SJohn Levon * HTABLE_COPIED pagetables are contained in the hat_t
19177c478bd9Sstevel@tonic-gate */
191874ecdb51SJohn Levon if (ht->ht_flags & HTABLE_COPIED) {
191974ecdb51SJohn Levon ASSERT3U(index, <, ht->ht_hat->hat_num_copied);
192074ecdb51SJohn Levon return (PT_INDEX_PTR(ht->ht_hat->hat_copied_ptes, index));
192174ecdb51SJohn Levon }
1922ae115bc7Smrj return (x86pte_mapin(ht->ht_pfn, index, ht));
1923ae115bc7Smrj }
1924ae115bc7Smrj
1925ae115bc7Smrj /*
1926ae115bc7Smrj * map the given pfn into the page table window.
1927ae115bc7Smrj */
1928ae115bc7Smrj /*ARGSUSED*/
1929ae115bc7Smrj x86pte_t *
x86pte_mapin(pfn_t pfn,uint_t index,htable_t * ht)1930ae115bc7Smrj x86pte_mapin(pfn_t pfn, uint_t index, htable_t *ht)
1931ae115bc7Smrj {
1932ae115bc7Smrj x86pte_t *pteptr;
19338ea72728Sjosephb x86pte_t pte = 0;
1934ae115bc7Smrj x86pte_t newpte;
1935ae115bc7Smrj int x;
19367c478bd9Sstevel@tonic-gate
19377c478bd9Sstevel@tonic-gate ASSERT(pfn != PFN_INVALID);
19387c478bd9Sstevel@tonic-gate
19397c478bd9Sstevel@tonic-gate if (!khat_running) {
1940ae115bc7Smrj caddr_t va = kbm_remap_window(pfn_to_pa(pfn), 1);
1941ae115bc7Smrj return (PT_INDEX_PTR(va, index));
19427c478bd9Sstevel@tonic-gate }
19437c478bd9Sstevel@tonic-gate
19447c478bd9Sstevel@tonic-gate /*
1945ae115bc7Smrj * If kpm is available, use it.
1946ae115bc7Smrj */
1947ae115bc7Smrj if (kpm_vbase)
1948ae115bc7Smrj return (PT_INDEX_PTR(hat_kpm_pfn2va(pfn), index));
1949ae115bc7Smrj
1950ae115bc7Smrj /*
1951ae115bc7Smrj * Disable preemption and grab the CPU's hci_mutex
19527c478bd9Sstevel@tonic-gate */
19537c478bd9Sstevel@tonic-gate kpreempt_disable();
195474ecdb51SJohn Levon
1955ae115bc7Smrj ASSERT(CPU->cpu_hat_info != NULL);
195674ecdb51SJohn Levon ASSERT(!(getcr4() & CR4_PCIDE));
195774ecdb51SJohn Levon
1958ae115bc7Smrj mutex_enter(&CPU->cpu_hat_info->hci_mutex);
1959ae115bc7Smrj x = PWIN_TABLE(CPU->cpu_id);
1960ae115bc7Smrj pteptr = (x86pte_t *)PWIN_PTE_VA(x);
19618ea72728Sjosephb #ifndef __xpv
1962ae115bc7Smrj if (mmu.pae_hat)
1963ae115bc7Smrj pte = *pteptr;
1964ae115bc7Smrj else
1965ae115bc7Smrj pte = *(x86pte32_t *)pteptr;
19668ea72728Sjosephb #endif
1967ae115bc7Smrj
1968ae115bc7Smrj newpte = MAKEPTE(pfn, 0) | mmu.pt_global | mmu.pt_nx;
1969843e1988Sjohnlev
1970843e1988Sjohnlev /*
1971843e1988Sjohnlev * For hardware we can use a writable mapping.
1972843e1988Sjohnlev */
1973843e1988Sjohnlev #ifdef __xpv
1974843e1988Sjohnlev if (IN_XPV_PANIC())
1975843e1988Sjohnlev #endif
1976843e1988Sjohnlev newpte |= PT_WRITABLE;
1977ae115bc7Smrj
1978ae115bc7Smrj if (!PTE_EQUIV(newpte, pte)) {
1979843e1988Sjohnlev
1980843e1988Sjohnlev #ifdef __xpv
1981843e1988Sjohnlev if (!IN_XPV_PANIC()) {
1982843e1988Sjohnlev xen_map(newpte, PWIN_VA(x));
1983843e1988Sjohnlev } else
1984843e1988Sjohnlev #endif
1985843e1988Sjohnlev {
1986843e1988Sjohnlev XPV_ALLOW_PAGETABLE_UPDATES();
1987843e1988Sjohnlev if (mmu.pae_hat)
1988843e1988Sjohnlev *pteptr = newpte;
1989843e1988Sjohnlev else
1990843e1988Sjohnlev *(x86pte32_t *)pteptr = newpte;
1991843e1988Sjohnlev XPV_DISALLOW_PAGETABLE_UPDATES();
199274ecdb51SJohn Levon mmu_flush_tlb_kpage((uintptr_t)PWIN_VA(x));
1993843e1988Sjohnlev }
19947c478bd9Sstevel@tonic-gate }
1995ae115bc7Smrj return (PT_INDEX_PTR(PWIN_VA(x), index));
19967c478bd9Sstevel@tonic-gate }
19977c478bd9Sstevel@tonic-gate
19987c478bd9Sstevel@tonic-gate /*
19997c478bd9Sstevel@tonic-gate * Release access to a page table.
20007c478bd9Sstevel@tonic-gate */
20017c478bd9Sstevel@tonic-gate static void
x86pte_release_pagetable(htable_t * ht)20027c478bd9Sstevel@tonic-gate x86pte_release_pagetable(htable_t *ht)
20037c478bd9Sstevel@tonic-gate {
200474ecdb51SJohn Levon if (ht->ht_flags & HTABLE_COPIED)
20057c478bd9Sstevel@tonic-gate return;
20067c478bd9Sstevel@tonic-gate
2007ae115bc7Smrj x86pte_mapout();
2008ae115bc7Smrj }
2009ae115bc7Smrj
2010ae115bc7Smrj void
x86pte_mapout(void)2011ae115bc7Smrj x86pte_mapout(void)
2012ae115bc7Smrj {
2013843e1988Sjohnlev if (kpm_vbase != NULL || !khat_running)
20147c478bd9Sstevel@tonic-gate return;
20157c478bd9Sstevel@tonic-gate
20167c478bd9Sstevel@tonic-gate /*
2017ae115bc7Smrj * Drop the CPU's hci_mutex and restore preemption.
20187c478bd9Sstevel@tonic-gate */
20198ea72728Sjosephb #ifdef __xpv
20208ea72728Sjosephb if (!IN_XPV_PANIC()) {
20218ea72728Sjosephb uintptr_t va;
20228ea72728Sjosephb
20238ea72728Sjosephb /*
20248ea72728Sjosephb * We need to always clear the mapping in case a page
20258ea72728Sjosephb * that was once a page table page is ballooned out.
20268ea72728Sjosephb */
20278ea72728Sjosephb va = (uintptr_t)PWIN_VA(PWIN_TABLE(CPU->cpu_id));
20288ea72728Sjosephb (void) HYPERVISOR_update_va_mapping(va, 0,
20298ea72728Sjosephb UVMF_INVLPG | UVMF_LOCAL);
20308ea72728Sjosephb }
20318ea72728Sjosephb #endif
2032ae115bc7Smrj mutex_exit(&CPU->cpu_hat_info->hci_mutex);
20337c478bd9Sstevel@tonic-gate kpreempt_enable();
20347c478bd9Sstevel@tonic-gate }
20357c478bd9Sstevel@tonic-gate
20367c478bd9Sstevel@tonic-gate /*
20377c478bd9Sstevel@tonic-gate * Atomic retrieval of a pagetable entry
20387c478bd9Sstevel@tonic-gate */
20397c478bd9Sstevel@tonic-gate x86pte_t
x86pte_get(htable_t * ht,uint_t entry)20407c478bd9Sstevel@tonic-gate x86pte_get(htable_t *ht, uint_t entry)
20417c478bd9Sstevel@tonic-gate {
20427c478bd9Sstevel@tonic-gate x86pte_t pte;
2043aa2ed9e5Sjosephb x86pte_t *ptep;
20447c478bd9Sstevel@tonic-gate
20457c478bd9Sstevel@tonic-gate /*
2046aa2ed9e5Sjosephb * Be careful that loading PAE entries in 32 bit kernel is atomic.
20477c478bd9Sstevel@tonic-gate */
2048ae115bc7Smrj ASSERT(entry < mmu.ptes_per_table);
2049ae115bc7Smrj ptep = x86pte_access_pagetable(ht, entry);
2050ae115bc7Smrj pte = GET_PTE(ptep);
20517c478bd9Sstevel@tonic-gate x86pte_release_pagetable(ht);
20527c478bd9Sstevel@tonic-gate return (pte);
20537c478bd9Sstevel@tonic-gate }
20547c478bd9Sstevel@tonic-gate
20557c478bd9Sstevel@tonic-gate /*
20567c478bd9Sstevel@tonic-gate * Atomic unconditional set of a page table entry, it returns the previous
2057ae115bc7Smrj * value. For pre-existing mappings if the PFN changes, then we don't care
2058ae115bc7Smrj * about the old pte's REF / MOD bits. If the PFN remains the same, we leave
2059ae115bc7Smrj * the MOD/REF bits unchanged.
2060ae115bc7Smrj *
2061ae115bc7Smrj * If asked to overwrite a link to a lower page table with a large page
2062ae115bc7Smrj * mapping, this routine returns the special value of LPAGE_ERROR. This
2063ae115bc7Smrj * allows the upper HAT layers to retry with a smaller mapping size.
20647c478bd9Sstevel@tonic-gate */
20657c478bd9Sstevel@tonic-gate x86pte_t
x86pte_set(htable_t * ht,uint_t entry,x86pte_t new,void * ptr)20667c478bd9Sstevel@tonic-gate x86pte_set(htable_t *ht, uint_t entry, x86pte_t new, void *ptr)
20677c478bd9Sstevel@tonic-gate {
20687c478bd9Sstevel@tonic-gate x86pte_t old;
2069ae115bc7Smrj x86pte_t prev;
20707c478bd9Sstevel@tonic-gate x86pte_t *ptep;
2071ae115bc7Smrj level_t l = ht->ht_level;
2072ae115bc7Smrj x86pte_t pfn_mask = (l != 0) ? PT_PADDR_LGPG : PT_PADDR;
2073ae115bc7Smrj x86pte_t n;
2074ae115bc7Smrj uintptr_t addr = htable_e2va(ht, entry);
2075ae115bc7Smrj hat_t *hat = ht->ht_hat;
20767c478bd9Sstevel@tonic-gate
2077ae115bc7Smrj ASSERT(new != 0); /* don't use to invalidate a PTE, see x86pte_update */
20787c478bd9Sstevel@tonic-gate ASSERT(!(ht->ht_flags & HTABLE_SHARED_PFN));
2079ae115bc7Smrj if (ptr == NULL)
2080ae115bc7Smrj ptep = x86pte_access_pagetable(ht, entry);
2081ae115bc7Smrj else
20827c478bd9Sstevel@tonic-gate ptep = ptr;
20837c478bd9Sstevel@tonic-gate
2084ae115bc7Smrj /*
2085ae115bc7Smrj * Install the new PTE. If remapping the same PFN, then
2086ae115bc7Smrj * copy existing REF/MOD bits to new mapping.
2087ae115bc7Smrj */
2088ae115bc7Smrj do {
2089ae115bc7Smrj prev = GET_PTE(ptep);
2090ae115bc7Smrj n = new;
2091ae115bc7Smrj if (PTE_ISVALID(n) && (prev & pfn_mask) == (new & pfn_mask))
2092ae115bc7Smrj n |= prev & (PT_REF | PT_MOD);
2093ae115bc7Smrj
2094ae115bc7Smrj /*
2095ae115bc7Smrj * Another thread may have installed this mapping already,
2096ae115bc7Smrj * flush the local TLB and be done.
2097ae115bc7Smrj */
2098ae115bc7Smrj if (prev == n) {
2099ae115bc7Smrj old = new;
2100843e1988Sjohnlev #ifdef __xpv
2101843e1988Sjohnlev if (!IN_XPV_PANIC())
2102843e1988Sjohnlev xen_flush_va((caddr_t)addr);
2103843e1988Sjohnlev else
2104843e1988Sjohnlev #endif
210574ecdb51SJohn Levon mmu_flush_tlb_page(addr);
2106ae115bc7Smrj goto done;
21077c478bd9Sstevel@tonic-gate }
2108ae115bc7Smrj
2109ae115bc7Smrj /*
2110ae115bc7Smrj * Detect if we have a collision of installing a large
2111ae115bc7Smrj * page mapping where there already is a lower page table.
2112ae115bc7Smrj */
211397704650Sjosephb if (l > 0 && (prev & PT_VALID) && !(prev & PT_PAGESIZE)) {
211497704650Sjosephb old = LPAGE_ERROR;
211597704650Sjosephb goto done;
211697704650Sjosephb }
2117ae115bc7Smrj
2118843e1988Sjohnlev XPV_ALLOW_PAGETABLE_UPDATES();
2119ae115bc7Smrj old = CAS_PTE(ptep, prev, n);
2120843e1988Sjohnlev XPV_DISALLOW_PAGETABLE_UPDATES();
2121ae115bc7Smrj } while (old != prev);
2122ae115bc7Smrj
2123ae115bc7Smrj /*
2124ae115bc7Smrj * Do a TLB demap if needed, ie. the old pte was valid.
2125ae115bc7Smrj *
2126ae115bc7Smrj * Note that a stale TLB writeback to the PTE here either can't happen
2127ae115bc7Smrj * or doesn't matter. The PFN can only change for NOSYNC|NOCONSIST
2128ae115bc7Smrj * mappings, but they were created with REF and MOD already set, so
2129ae115bc7Smrj * no stale writeback will happen.
2130ae115bc7Smrj *
2131ae115bc7Smrj * Segmap is the only place where remaps happen on the same pfn and for
2132ae115bc7Smrj * that we want to preserve the stale REF/MOD bits.
2133ae115bc7Smrj */
2134ae115bc7Smrj if (old & PT_REF)
2135ae115bc7Smrj hat_tlb_inval(hat, addr);
2136ae115bc7Smrj
2137ae115bc7Smrj done:
21387c478bd9Sstevel@tonic-gate if (ptr == NULL)
21397c478bd9Sstevel@tonic-gate x86pte_release_pagetable(ht);
21407c478bd9Sstevel@tonic-gate return (old);
21417c478bd9Sstevel@tonic-gate }
21427c478bd9Sstevel@tonic-gate
21437c478bd9Sstevel@tonic-gate /*
2144ae115bc7Smrj * Atomic compare and swap of a page table entry. No TLB invalidates are done.
2145ae115bc7Smrj * This is used for links between pagetables of different levels.
2146ae115bc7Smrj * Note we always create these links with dirty/access set, so they should
2147ae115bc7Smrj * never change.
21487c478bd9Sstevel@tonic-gate */
2149ae115bc7Smrj x86pte_t
x86pte_cas(htable_t * ht,uint_t entry,x86pte_t old,x86pte_t new)21507c478bd9Sstevel@tonic-gate x86pte_cas(htable_t *ht, uint_t entry, x86pte_t old, x86pte_t new)
21517c478bd9Sstevel@tonic-gate {
21527c478bd9Sstevel@tonic-gate x86pte_t pte;
21537c478bd9Sstevel@tonic-gate x86pte_t *ptep;
2154843e1988Sjohnlev #ifdef __xpv
2155843e1988Sjohnlev /*
2156843e1988Sjohnlev * We can't use writable pagetables for upper level tables, so fake it.
2157843e1988Sjohnlev */
2158843e1988Sjohnlev mmu_update_t t[2];
2159843e1988Sjohnlev int cnt = 1;
2160843e1988Sjohnlev int count;
2161843e1988Sjohnlev maddr_t ma;
2162843e1988Sjohnlev
2163843e1988Sjohnlev if (!IN_XPV_PANIC()) {
216474ecdb51SJohn Levon ASSERT(!(ht->ht_flags & HTABLE_COPIED));
2165843e1988Sjohnlev ma = pa_to_ma(PT_INDEX_PHYSADDR(pfn_to_pa(ht->ht_pfn), entry));
2166843e1988Sjohnlev t[0].ptr = ma | MMU_NORMAL_PT_UPDATE;
2167843e1988Sjohnlev t[0].val = new;
2168843e1988Sjohnlev
2169843e1988Sjohnlev /*
2170843e1988Sjohnlev * On the 64-bit hypervisor we need to maintain the user mode
2171843e1988Sjohnlev * top page table too.
2172843e1988Sjohnlev */
2173843e1988Sjohnlev if (ht->ht_level == mmu.max_level && ht->ht_hat != kas.a_hat) {
2174843e1988Sjohnlev ma = pa_to_ma(PT_INDEX_PHYSADDR(pfn_to_pa(
2175843e1988Sjohnlev ht->ht_hat->hat_user_ptable), entry));
2176843e1988Sjohnlev t[1].ptr = ma | MMU_NORMAL_PT_UPDATE;
2177843e1988Sjohnlev t[1].val = new;
2178843e1988Sjohnlev ++cnt;
2179843e1988Sjohnlev }
21807c478bd9Sstevel@tonic-gate
2181843e1988Sjohnlev if (HYPERVISOR_mmu_update(t, cnt, &count, DOMID_SELF))
2182843e1988Sjohnlev panic("HYPERVISOR_mmu_update() failed");
2183843e1988Sjohnlev ASSERT(count == cnt);
2184843e1988Sjohnlev return (old);
2185843e1988Sjohnlev }
2186843e1988Sjohnlev #endif
2187ae115bc7Smrj ptep = x86pte_access_pagetable(ht, entry);
2188843e1988Sjohnlev XPV_ALLOW_PAGETABLE_UPDATES();
2189ae115bc7Smrj pte = CAS_PTE(ptep, old, new);
2190843e1988Sjohnlev XPV_DISALLOW_PAGETABLE_UPDATES();
21917c478bd9Sstevel@tonic-gate x86pte_release_pagetable(ht);
21927c478bd9Sstevel@tonic-gate return (pte);
21937c478bd9Sstevel@tonic-gate }
21947c478bd9Sstevel@tonic-gate
21957c478bd9Sstevel@tonic-gate /*
2196ae115bc7Smrj * Invalidate a page table entry as long as it currently maps something that
2197ae115bc7Smrj * matches the value determined by expect.
2198ae115bc7Smrj *
2199a6a74e0eSMatthew Ahrens * If tlb is set, also invalidates any TLB entries.
2200a6a74e0eSMatthew Ahrens *
2201a6a74e0eSMatthew Ahrens * Returns the previous value of the PTE.
22027c478bd9Sstevel@tonic-gate */
22037c478bd9Sstevel@tonic-gate x86pte_t
x86pte_inval(htable_t * ht,uint_t entry,x86pte_t expect,x86pte_t * pte_ptr,boolean_t tlb)2204ae115bc7Smrj x86pte_inval(
2205ae115bc7Smrj htable_t *ht,
2206ae115bc7Smrj uint_t entry,
2207ae115bc7Smrj x86pte_t expect,
2208a6a74e0eSMatthew Ahrens x86pte_t *pte_ptr,
2209a6a74e0eSMatthew Ahrens boolean_t tlb)
22107c478bd9Sstevel@tonic-gate {
22117c478bd9Sstevel@tonic-gate x86pte_t *ptep;
221295c0a3c8Sjosephb x86pte_t oldpte;
221395c0a3c8Sjosephb x86pte_t found;
22147c478bd9Sstevel@tonic-gate
22157c478bd9Sstevel@tonic-gate ASSERT(!(ht->ht_flags & HTABLE_SHARED_PFN));
221602bc52beSkchow ASSERT(ht->ht_level <= mmu.max_page_level);
221797704650Sjosephb
2218ae115bc7Smrj if (pte_ptr != NULL)
22197c478bd9Sstevel@tonic-gate ptep = pte_ptr;
2220ae115bc7Smrj else
2221ae115bc7Smrj ptep = x86pte_access_pagetable(ht, entry);
22227c478bd9Sstevel@tonic-gate
2223843e1988Sjohnlev #if defined(__xpv)
2224843e1988Sjohnlev /*
2225843e1988Sjohnlev * If exit()ing just use HYPERVISOR_mmu_update(), as we can't be racing
2226843e1988Sjohnlev * with anything else.
2227843e1988Sjohnlev */
2228843e1988Sjohnlev if ((ht->ht_hat->hat_flags & HAT_FREEING) && !IN_XPV_PANIC()) {
2229843e1988Sjohnlev int count;
2230843e1988Sjohnlev mmu_update_t t[1];
2231843e1988Sjohnlev maddr_t ma;
2232843e1988Sjohnlev
2233843e1988Sjohnlev oldpte = GET_PTE(ptep);
2234843e1988Sjohnlev if (expect != 0 && (oldpte & PT_PADDR) != (expect & PT_PADDR))
2235843e1988Sjohnlev goto done;
2236843e1988Sjohnlev ma = pa_to_ma(PT_INDEX_PHYSADDR(pfn_to_pa(ht->ht_pfn), entry));
2237843e1988Sjohnlev t[0].ptr = ma | MMU_NORMAL_PT_UPDATE;
2238843e1988Sjohnlev t[0].val = 0;
2239843e1988Sjohnlev if (HYPERVISOR_mmu_update(t, 1, &count, DOMID_SELF))
2240843e1988Sjohnlev panic("HYPERVISOR_mmu_update() failed");
2241843e1988Sjohnlev ASSERT(count == 1);
2242843e1988Sjohnlev goto done;
2243843e1988Sjohnlev }
2244843e1988Sjohnlev #endif /* __xpv */
2245843e1988Sjohnlev
22467c478bd9Sstevel@tonic-gate /*
224797704650Sjosephb * Note that the loop is needed to handle changes due to h/w updating
224897704650Sjosephb * of PT_MOD/PT_REF.
22497c478bd9Sstevel@tonic-gate */
2250ae115bc7Smrj do {
225195c0a3c8Sjosephb oldpte = GET_PTE(ptep);
225295c0a3c8Sjosephb if (expect != 0 && (oldpte & PT_PADDR) != (expect & PT_PADDR))
225395c0a3c8Sjosephb goto done;
2254843e1988Sjohnlev XPV_ALLOW_PAGETABLE_UPDATES();
225595c0a3c8Sjosephb found = CAS_PTE(ptep, oldpte, 0);
2256843e1988Sjohnlev XPV_DISALLOW_PAGETABLE_UPDATES();
225795c0a3c8Sjosephb } while (found != oldpte);
2258a6a74e0eSMatthew Ahrens if (tlb && (oldpte & (PT_REF | PT_MOD)))
225995c0a3c8Sjosephb hat_tlb_inval(ht->ht_hat, htable_e2va(ht, entry));
226097704650Sjosephb
226195c0a3c8Sjosephb done:
22627c478bd9Sstevel@tonic-gate if (pte_ptr == NULL)
22637c478bd9Sstevel@tonic-gate x86pte_release_pagetable(ht);
226495c0a3c8Sjosephb return (oldpte);
22657c478bd9Sstevel@tonic-gate }
22667c478bd9Sstevel@tonic-gate
22677c478bd9Sstevel@tonic-gate /*
2268ae115bc7Smrj * Change a page table entry af it currently matches the value in expect.
22697c478bd9Sstevel@tonic-gate */
22707c478bd9Sstevel@tonic-gate x86pte_t
x86pte_update(htable_t * ht,uint_t entry,x86pte_t expect,x86pte_t new)2271ae115bc7Smrj x86pte_update(
2272ae115bc7Smrj htable_t *ht,
2273ae115bc7Smrj uint_t entry,
2274ae115bc7Smrj x86pte_t expect,
2275ae115bc7Smrj x86pte_t new)
22767c478bd9Sstevel@tonic-gate {
22777c478bd9Sstevel@tonic-gate x86pte_t *ptep;
2278ae115bc7Smrj x86pte_t found;
22797c478bd9Sstevel@tonic-gate
2280ae115bc7Smrj ASSERT(new != 0);
22817c478bd9Sstevel@tonic-gate ASSERT(!(ht->ht_flags & HTABLE_SHARED_PFN));
228202bc52beSkchow ASSERT(ht->ht_level <= mmu.max_page_level);
22837c478bd9Sstevel@tonic-gate
2284ae115bc7Smrj ptep = x86pte_access_pagetable(ht, entry);
2285843e1988Sjohnlev XPV_ALLOW_PAGETABLE_UPDATES();
2286ae115bc7Smrj found = CAS_PTE(ptep, expect, new);
2287843e1988Sjohnlev XPV_DISALLOW_PAGETABLE_UPDATES();
2288ae115bc7Smrj if (found == expect) {
2289ae115bc7Smrj hat_tlb_inval(ht->ht_hat, htable_e2va(ht, entry));
22907c478bd9Sstevel@tonic-gate
2291ae115bc7Smrj /*
2292ae115bc7Smrj * When removing write permission *and* clearing the
2293ae115bc7Smrj * MOD bit, check if a write happened via a stale
2294ae115bc7Smrj * TLB entry before the TLB shootdown finished.
2295ae115bc7Smrj *
2296ae115bc7Smrj * If it did happen, simply re-enable write permission and
2297ae115bc7Smrj * act like the original CAS failed.
2298ae115bc7Smrj */
2299ae115bc7Smrj if ((expect & (PT_WRITABLE | PT_MOD)) == PT_WRITABLE &&
2300ae115bc7Smrj (new & (PT_WRITABLE | PT_MOD)) == 0 &&
2301ae115bc7Smrj (GET_PTE(ptep) & PT_MOD) != 0) {
2302ae115bc7Smrj do {
2303ae115bc7Smrj found = GET_PTE(ptep);
2304843e1988Sjohnlev XPV_ALLOW_PAGETABLE_UPDATES();
2305ae115bc7Smrj found =
2306ae115bc7Smrj CAS_PTE(ptep, found, found | PT_WRITABLE);
2307843e1988Sjohnlev XPV_DISALLOW_PAGETABLE_UPDATES();
2308ae115bc7Smrj } while ((found & PT_WRITABLE) == 0);
2309ae115bc7Smrj }
2310ae115bc7Smrj }
23117c478bd9Sstevel@tonic-gate x86pte_release_pagetable(ht);
2312ae115bc7Smrj return (found);
23137c478bd9Sstevel@tonic-gate }
23147c478bd9Sstevel@tonic-gate
2315843e1988Sjohnlev #ifndef __xpv
23167c478bd9Sstevel@tonic-gate /*
23177c478bd9Sstevel@tonic-gate * Copy page tables - this is just a little more complicated than the
23187c478bd9Sstevel@tonic-gate * previous routines. Note that it's also not atomic! It also is never
231974ecdb51SJohn Levon * used for HTABLE_COPIED pagetables.
23207c478bd9Sstevel@tonic-gate */
23217c478bd9Sstevel@tonic-gate void
x86pte_copy(htable_t * src,htable_t * dest,uint_t entry,uint_t count)23227c478bd9Sstevel@tonic-gate x86pte_copy(htable_t *src, htable_t *dest, uint_t entry, uint_t count)
23237c478bd9Sstevel@tonic-gate {
23247c478bd9Sstevel@tonic-gate caddr_t src_va;
23257c478bd9Sstevel@tonic-gate caddr_t dst_va;
23267c478bd9Sstevel@tonic-gate size_t size;
2327ae115bc7Smrj x86pte_t *pteptr;
2328ae115bc7Smrj x86pte_t pte;
23297c478bd9Sstevel@tonic-gate
23307c478bd9Sstevel@tonic-gate ASSERT(khat_running);
233174ecdb51SJohn Levon ASSERT(!(dest->ht_flags & HTABLE_COPIED));
233274ecdb51SJohn Levon ASSERT(!(src->ht_flags & HTABLE_COPIED));
23337c478bd9Sstevel@tonic-gate ASSERT(!(src->ht_flags & HTABLE_SHARED_PFN));
23347c478bd9Sstevel@tonic-gate ASSERT(!(dest->ht_flags & HTABLE_SHARED_PFN));
23357c478bd9Sstevel@tonic-gate
23367c478bd9Sstevel@tonic-gate /*
2337ae115bc7Smrj * Acquire access to the CPU pagetable windows for the dest and source.
23387c478bd9Sstevel@tonic-gate */
2339ae115bc7Smrj dst_va = (caddr_t)x86pte_access_pagetable(dest, entry);
2340ae115bc7Smrj if (kpm_vbase) {
2341ae115bc7Smrj src_va = (caddr_t)
2342ae115bc7Smrj PT_INDEX_PTR(hat_kpm_pfn2va(src->ht_pfn), entry);
23437c478bd9Sstevel@tonic-gate } else {
2344ae115bc7Smrj uint_t x = PWIN_SRC(CPU->cpu_id);
23457c478bd9Sstevel@tonic-gate
234674ecdb51SJohn Levon ASSERT(!(getcr4() & CR4_PCIDE));
234774ecdb51SJohn Levon
23487c478bd9Sstevel@tonic-gate /*
23497c478bd9Sstevel@tonic-gate * Finish defining the src pagetable mapping
23507c478bd9Sstevel@tonic-gate */
2351ae115bc7Smrj src_va = (caddr_t)PT_INDEX_PTR(PWIN_VA(x), entry);
2352ae115bc7Smrj pte = MAKEPTE(src->ht_pfn, 0) | mmu.pt_global | mmu.pt_nx;
2353ae115bc7Smrj pteptr = (x86pte_t *)PWIN_PTE_VA(x);
2354ae115bc7Smrj if (mmu.pae_hat)
2355ae115bc7Smrj *pteptr = pte;
2356ae115bc7Smrj else
2357ae115bc7Smrj *(x86pte32_t *)pteptr = pte;
235874ecdb51SJohn Levon mmu_flush_tlb_kpage((uintptr_t)PWIN_VA(x));
23597c478bd9Sstevel@tonic-gate }
23607c478bd9Sstevel@tonic-gate
23617c478bd9Sstevel@tonic-gate /*
23627c478bd9Sstevel@tonic-gate * now do the copy
23637c478bd9Sstevel@tonic-gate */
23647c478bd9Sstevel@tonic-gate size = count << mmu.pte_size_shift;
23657c478bd9Sstevel@tonic-gate bcopy(src_va, dst_va, size);
23667c478bd9Sstevel@tonic-gate
23677c478bd9Sstevel@tonic-gate x86pte_release_pagetable(dest);
23687c478bd9Sstevel@tonic-gate }
23697c478bd9Sstevel@tonic-gate
2370843e1988Sjohnlev #else /* __xpv */
2371843e1988Sjohnlev
2372843e1988Sjohnlev /*
2373843e1988Sjohnlev * The hypervisor only supports writable pagetables at level 0, so we have
2374843e1988Sjohnlev * to install these 1 by 1 the slow way.
2375843e1988Sjohnlev */
2376843e1988Sjohnlev void
x86pte_copy(htable_t * src,htable_t * dest,uint_t entry,uint_t count)2377843e1988Sjohnlev x86pte_copy(htable_t *src, htable_t *dest, uint_t entry, uint_t count)
2378843e1988Sjohnlev {
2379843e1988Sjohnlev caddr_t src_va;
2380843e1988Sjohnlev x86pte_t pte;
2381843e1988Sjohnlev
2382843e1988Sjohnlev ASSERT(!IN_XPV_PANIC());
2383843e1988Sjohnlev src_va = (caddr_t)x86pte_access_pagetable(src, entry);
2384843e1988Sjohnlev while (count) {
2385843e1988Sjohnlev if (mmu.pae_hat)
2386843e1988Sjohnlev pte = *(x86pte_t *)src_va;
2387843e1988Sjohnlev else
2388843e1988Sjohnlev pte = *(x86pte32_t *)src_va;
2389843e1988Sjohnlev if (pte != 0) {
2390843e1988Sjohnlev set_pteval(pfn_to_pa(dest->ht_pfn), entry,
2391843e1988Sjohnlev dest->ht_level, pte);
2392843e1988Sjohnlev if (dest->ht_level == mmu.max_level &&
2393843e1988Sjohnlev htable_e2va(dest, entry) < HYPERVISOR_VIRT_END)
2394843e1988Sjohnlev set_pteval(
2395843e1988Sjohnlev pfn_to_pa(dest->ht_hat->hat_user_ptable),
2396843e1988Sjohnlev entry, dest->ht_level, pte);
2397843e1988Sjohnlev }
2398843e1988Sjohnlev --count;
2399843e1988Sjohnlev ++entry;
2400843e1988Sjohnlev src_va += mmu.pte_size;
2401843e1988Sjohnlev }
2402843e1988Sjohnlev x86pte_release_pagetable(src);
2403843e1988Sjohnlev }
2404843e1988Sjohnlev #endif /* __xpv */
2405843e1988Sjohnlev
24067c478bd9Sstevel@tonic-gate /*
24077c478bd9Sstevel@tonic-gate * Zero page table entries - Note this doesn't use atomic stores!
24087c478bd9Sstevel@tonic-gate */
2409ae115bc7Smrj static void
x86pte_zero(htable_t * dest,uint_t entry,uint_t count)24107c478bd9Sstevel@tonic-gate x86pte_zero(htable_t *dest, uint_t entry, uint_t count)
24117c478bd9Sstevel@tonic-gate {
24127c478bd9Sstevel@tonic-gate caddr_t dst_va;
24137c478bd9Sstevel@tonic-gate size_t size;
2414843e1988Sjohnlev #ifdef __xpv
24152a9992ecSToomas Soome int x = 0;
2416843e1988Sjohnlev x86pte_t newpte;
2417843e1988Sjohnlev #endif
24187c478bd9Sstevel@tonic-gate
24197c478bd9Sstevel@tonic-gate /*
24207c478bd9Sstevel@tonic-gate * Map in the page table to be zeroed.
24217c478bd9Sstevel@tonic-gate */
24227c478bd9Sstevel@tonic-gate ASSERT(!(dest->ht_flags & HTABLE_SHARED_PFN));
242374ecdb51SJohn Levon ASSERT(!(dest->ht_flags & HTABLE_COPIED));
2424ae115bc7Smrj
2425843e1988Sjohnlev /*
2426843e1988Sjohnlev * On the hypervisor we don't use x86pte_access_pagetable() since
2427843e1988Sjohnlev * in this case the page is not pinned yet.
2428843e1988Sjohnlev */
2429843e1988Sjohnlev #ifdef __xpv
2430843e1988Sjohnlev if (kpm_vbase == NULL) {
2431843e1988Sjohnlev kpreempt_disable();
2432843e1988Sjohnlev ASSERT(CPU->cpu_hat_info != NULL);
2433843e1988Sjohnlev mutex_enter(&CPU->cpu_hat_info->hci_mutex);
2434843e1988Sjohnlev x = PWIN_TABLE(CPU->cpu_id);
2435843e1988Sjohnlev newpte = MAKEPTE(dest->ht_pfn, 0) | PT_WRITABLE;
2436843e1988Sjohnlev xen_map(newpte, PWIN_VA(x));
2437843e1988Sjohnlev dst_va = (caddr_t)PT_INDEX_PTR(PWIN_VA(x), entry);
2438843e1988Sjohnlev } else
2439843e1988Sjohnlev #endif
2440843e1988Sjohnlev dst_va = (caddr_t)x86pte_access_pagetable(dest, entry);
2441ae115bc7Smrj
24427c478bd9Sstevel@tonic-gate size = count << mmu.pte_size_shift;
2443ae115bc7Smrj ASSERT(size > BLOCKZEROALIGN);
2444*86ef0a63SRichard Lowe block_zero_no_xmm(dst_va, size);
2445ae115bc7Smrj
2446843e1988Sjohnlev #ifdef __xpv
2447843e1988Sjohnlev if (kpm_vbase == NULL) {
2448843e1988Sjohnlev xen_map(0, PWIN_VA(x));
2449843e1988Sjohnlev mutex_exit(&CPU->cpu_hat_info->hci_mutex);
2450843e1988Sjohnlev kpreempt_enable();
2451843e1988Sjohnlev } else
2452843e1988Sjohnlev #endif
2453843e1988Sjohnlev x86pte_release_pagetable(dest);
24547c478bd9Sstevel@tonic-gate }
24557c478bd9Sstevel@tonic-gate
24567c478bd9Sstevel@tonic-gate /*
24577c478bd9Sstevel@tonic-gate * Called to ensure that all pagetables are in the system dump
24587c478bd9Sstevel@tonic-gate */
24597c478bd9Sstevel@tonic-gate void
hat_dump(void)24607c478bd9Sstevel@tonic-gate hat_dump(void)
24617c478bd9Sstevel@tonic-gate {
24627c478bd9Sstevel@tonic-gate hat_t *hat;
24637c478bd9Sstevel@tonic-gate uint_t h;
24647c478bd9Sstevel@tonic-gate htable_t *ht;
24657c478bd9Sstevel@tonic-gate
24667c478bd9Sstevel@tonic-gate /*
2467a85a6733Sjosephb * Dump all page tables
24687c478bd9Sstevel@tonic-gate */
2469a85a6733Sjosephb for (hat = kas.a_hat; hat != NULL; hat = hat->hat_next) {
24707c478bd9Sstevel@tonic-gate for (h = 0; h < hat->hat_num_hash; ++h) {
24717c478bd9Sstevel@tonic-gate for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) {
247274ecdb51SJohn Levon if ((ht->ht_flags & HTABLE_COPIED) == 0)
24737c478bd9Sstevel@tonic-gate dump_page(ht->ht_pfn);
24747c478bd9Sstevel@tonic-gate }
24757c478bd9Sstevel@tonic-gate }
24767c478bd9Sstevel@tonic-gate }
24777c478bd9Sstevel@tonic-gate }
2478