xref: /illumos-gate/usr/src/uts/i86pc/vm/htable.c (revision 86ef0a63)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright (c) 2014 by Delphix. All rights reserved.
25  * Copyright 2018 Joyent, Inc.
26  */
27 
28 #include <sys/types.h>
29 #include <sys/sysmacros.h>
30 #include <sys/kmem.h>
31 #include <sys/atomic.h>
32 #include <sys/bitmap.h>
33 #include <sys/machparam.h>
34 #include <sys/machsystm.h>
35 #include <sys/mman.h>
36 #include <sys/systm.h>
37 #include <sys/cpuvar.h>
38 #include <sys/thread.h>
39 #include <sys/proc.h>
40 #include <sys/cpu.h>
41 #include <sys/kmem.h>
42 #include <sys/disp.h>
43 #include <sys/vmem.h>
44 #include <sys/vmsystm.h>
45 #include <sys/promif.h>
46 #include <sys/var.h>
47 #include <sys/x86_archext.h>
48 #include <sys/archsystm.h>
49 #include <sys/bootconf.h>
50 #include <sys/dumphdr.h>
51 #include <vm/seg_kmem.h>
52 #include <vm/seg_kpm.h>
53 #include <vm/hat.h>
54 #include <vm/hat_i86.h>
55 #include <sys/cmn_err.h>
56 #include <sys/panic.h>
57 
58 #ifdef __xpv
59 #include <sys/hypervisor.h>
60 #include <sys/xpv_panic.h>
61 #endif
62 
63 #include <sys/bootinfo.h>
64 #include <vm/kboot_mmu.h>
65 
66 static void x86pte_zero(htable_t *dest, uint_t entry, uint_t count);
67 
68 kmem_cache_t *htable_cache;
69 
70 /*
71  * The variable htable_reserve_amount, rather than HTABLE_RESERVE_AMOUNT,
72  * is used in order to facilitate testing of the htable_steal() code.
73  * By resetting htable_reserve_amount to a lower value, we can force
74  * stealing to occur.  The reserve amount is a guess to get us through boot.
75  */
76 #define	HTABLE_RESERVE_AMOUNT	(200)
77 uint_t htable_reserve_amount = HTABLE_RESERVE_AMOUNT;
78 kmutex_t htable_reserve_mutex;
79 uint_t htable_reserve_cnt;
80 htable_t *htable_reserve_pool;
81 
82 /*
83  * Used to hand test htable_steal().
84  */
85 #ifdef DEBUG
86 ulong_t force_steal = 0;
87 ulong_t ptable_cnt = 0;
88 #endif
89 
90 /*
91  * This variable is so that we can tune this via /etc/system
92  * Any value works, but a power of two <= mmu.ptes_per_table is best.
93  */
94 uint_t htable_steal_passes = 8;
95 
96 /*
97  * mutex stuff for access to htable hash
98  */
99 #define	NUM_HTABLE_MUTEX 128
100 kmutex_t htable_mutex[NUM_HTABLE_MUTEX];
101 #define	HTABLE_MUTEX_HASH(h) ((h) & (NUM_HTABLE_MUTEX - 1))
102 
103 #define	HTABLE_ENTER(h)	mutex_enter(&htable_mutex[HTABLE_MUTEX_HASH(h)]);
104 #define	HTABLE_EXIT(h)	mutex_exit(&htable_mutex[HTABLE_MUTEX_HASH(h)]);
105 
106 /*
107  * forward declarations
108  */
109 static void link_ptp(htable_t *higher, htable_t *new, uintptr_t vaddr);
110 static void unlink_ptp(htable_t *higher, htable_t *old, uintptr_t vaddr);
111 static void htable_free(htable_t *ht);
112 static x86pte_t *x86pte_access_pagetable(htable_t *ht, uint_t index);
113 static void x86pte_release_pagetable(htable_t *ht);
114 static x86pte_t x86pte_cas(htable_t *ht, uint_t entry, x86pte_t old,
115 	x86pte_t new);
116 
117 /*
118  * A counter to track if we are stealing or reaping htables. When non-zero
119  * htable_free() will directly free htables (either to the reserve or kmem)
120  * instead of putting them in a hat's htable cache.
121  */
122 uint32_t htable_dont_cache = 0;
123 
124 /*
125  * Track the number of active pagetables, so we can know how many to reap
126  */
127 static uint32_t active_ptables = 0;
128 
129 #ifdef __xpv
130 /*
131  * Deal with hypervisor complications.
132  */
133 void
xen_flush_va(caddr_t va)134 xen_flush_va(caddr_t va)
135 {
136 	struct mmuext_op t;
137 	uint_t count;
138 
139 	if (IN_XPV_PANIC()) {
140 		mmu_flush_tlb_page((uintptr_t)va);
141 	} else {
142 		t.cmd = MMUEXT_INVLPG_LOCAL;
143 		t.arg1.linear_addr = (uintptr_t)va;
144 		if (HYPERVISOR_mmuext_op(&t, 1, &count, DOMID_SELF) < 0)
145 			panic("HYPERVISOR_mmuext_op() failed");
146 		ASSERT(count == 1);
147 	}
148 }
149 
150 void
xen_gflush_va(caddr_t va,cpuset_t cpus)151 xen_gflush_va(caddr_t va, cpuset_t cpus)
152 {
153 	struct mmuext_op t;
154 	uint_t count;
155 
156 	if (IN_XPV_PANIC()) {
157 		mmu_flush_tlb_page((uintptr_t)va);
158 		return;
159 	}
160 
161 	t.cmd = MMUEXT_INVLPG_MULTI;
162 	t.arg1.linear_addr = (uintptr_t)va;
163 	/*LINTED: constant in conditional context*/
164 	set_xen_guest_handle(t.arg2.vcpumask, &cpus);
165 	if (HYPERVISOR_mmuext_op(&t, 1, &count, DOMID_SELF) < 0)
166 		panic("HYPERVISOR_mmuext_op() failed");
167 	ASSERT(count == 1);
168 }
169 
170 void
xen_flush_tlb()171 xen_flush_tlb()
172 {
173 	struct mmuext_op t;
174 	uint_t count;
175 
176 	if (IN_XPV_PANIC()) {
177 		xpv_panic_reload_cr3();
178 	} else {
179 		t.cmd = MMUEXT_TLB_FLUSH_LOCAL;
180 		if (HYPERVISOR_mmuext_op(&t, 1, &count, DOMID_SELF) < 0)
181 			panic("HYPERVISOR_mmuext_op() failed");
182 		ASSERT(count == 1);
183 	}
184 }
185 
186 void
xen_gflush_tlb(cpuset_t cpus)187 xen_gflush_tlb(cpuset_t cpus)
188 {
189 	struct mmuext_op t;
190 	uint_t count;
191 
192 	ASSERT(!IN_XPV_PANIC());
193 	t.cmd = MMUEXT_TLB_FLUSH_MULTI;
194 	/*LINTED: constant in conditional context*/
195 	set_xen_guest_handle(t.arg2.vcpumask, &cpus);
196 	if (HYPERVISOR_mmuext_op(&t, 1, &count, DOMID_SELF) < 0)
197 		panic("HYPERVISOR_mmuext_op() failed");
198 	ASSERT(count == 1);
199 }
200 
201 /*
202  * Install/Adjust a kpm mapping under the hypervisor.
203  * Value of "how" should be:
204  *	PT_WRITABLE | PT_VALID - regular kpm mapping
205  *	PT_VALID - make mapping read-only
206  *	0	- remove mapping
207  *
208  * returns 0 on success. non-zero for failure.
209  */
210 int
xen_kpm_page(pfn_t pfn,uint_t how)211 xen_kpm_page(pfn_t pfn, uint_t how)
212 {
213 	paddr_t pa = mmu_ptob((paddr_t)pfn);
214 	x86pte_t pte = PT_NOCONSIST | PT_REF | PT_MOD;
215 
216 	if (kpm_vbase == NULL)
217 		return (0);
218 
219 	if (how)
220 		pte |= pa_to_ma(pa) | how;
221 	else
222 		pte = 0;
223 	return (HYPERVISOR_update_va_mapping((uintptr_t)kpm_vbase + pa,
224 	    pte, UVMF_INVLPG | UVMF_ALL));
225 }
226 
227 void
xen_pin(pfn_t pfn,level_t lvl)228 xen_pin(pfn_t pfn, level_t lvl)
229 {
230 	struct mmuext_op t;
231 	uint_t count;
232 
233 	t.cmd = MMUEXT_PIN_L1_TABLE + lvl;
234 	t.arg1.mfn = pfn_to_mfn(pfn);
235 	if (HYPERVISOR_mmuext_op(&t, 1, &count, DOMID_SELF) < 0)
236 		panic("HYPERVISOR_mmuext_op() failed");
237 	ASSERT(count == 1);
238 }
239 
240 void
xen_unpin(pfn_t pfn)241 xen_unpin(pfn_t pfn)
242 {
243 	struct mmuext_op t;
244 	uint_t count;
245 
246 	t.cmd = MMUEXT_UNPIN_TABLE;
247 	t.arg1.mfn = pfn_to_mfn(pfn);
248 	if (HYPERVISOR_mmuext_op(&t, 1, &count, DOMID_SELF) < 0)
249 		panic("HYPERVISOR_mmuext_op() failed");
250 	ASSERT(count == 1);
251 }
252 
253 static void
xen_map(uint64_t pte,caddr_t va)254 xen_map(uint64_t pte, caddr_t va)
255 {
256 	if (HYPERVISOR_update_va_mapping((uintptr_t)va, pte,
257 	    UVMF_INVLPG | UVMF_LOCAL))
258 		panic("HYPERVISOR_update_va_mapping() failed");
259 }
260 #endif /* __xpv */
261 
262 /*
263  * Allocate a memory page for a hardware page table.
264  *
265  * A wrapper around page_get_physical(), with some extra checks.
266  */
267 static pfn_t
ptable_alloc(uintptr_t seed)268 ptable_alloc(uintptr_t seed)
269 {
270 	pfn_t pfn;
271 	page_t *pp;
272 
273 	pfn = PFN_INVALID;
274 
275 	/*
276 	 * The first check is to see if there is memory in the system. If we
277 	 * drop to throttlefree, then fail the ptable_alloc() and let the
278 	 * stealing code kick in. Note that we have to do this test here,
279 	 * since the test in page_create_throttle() would let the NOSLEEP
280 	 * allocation go through and deplete the page reserves.
281 	 *
282 	 * The !NOMEMWAIT() lets pageout, fsflush, etc. skip this check.
283 	 */
284 	if (!NOMEMWAIT() && freemem <= throttlefree + 1)
285 		return (PFN_INVALID);
286 
287 #ifdef DEBUG
288 	/*
289 	 * This code makes htable_steal() easier to test. By setting
290 	 * force_steal we force pagetable allocations to fall
291 	 * into the stealing code. Roughly 1 in ever "force_steal"
292 	 * page table allocations will fail.
293 	 */
294 	if (proc_pageout != NULL && force_steal > 1 &&
295 	    ++ptable_cnt > force_steal) {
296 		ptable_cnt = 0;
297 		return (PFN_INVALID);
298 	}
299 #endif /* DEBUG */
300 
301 	pp = page_get_physical(seed);
302 	if (pp == NULL)
303 		return (PFN_INVALID);
304 	ASSERT(PAGE_SHARED(pp));
305 	pfn = pp->p_pagenum;
306 	if (pfn == PFN_INVALID)
307 		panic("ptable_alloc(): Invalid PFN!!");
308 	atomic_inc_32(&active_ptables);
309 	HATSTAT_INC(hs_ptable_allocs);
310 	return (pfn);
311 }
312 
313 /*
314  * Free an htable's associated page table page.  See the comments
315  * for ptable_alloc().
316  */
317 static void
ptable_free(pfn_t pfn)318 ptable_free(pfn_t pfn)
319 {
320 	page_t *pp = page_numtopp_nolock(pfn);
321 
322 	/*
323 	 * need to destroy the page used for the pagetable
324 	 */
325 	ASSERT(pfn != PFN_INVALID);
326 	HATSTAT_INC(hs_ptable_frees);
327 	atomic_dec_32(&active_ptables);
328 	if (pp == NULL)
329 		panic("ptable_free(): no page for pfn!");
330 	ASSERT(PAGE_SHARED(pp));
331 	ASSERT(pfn == pp->p_pagenum);
332 	ASSERT(!IN_XPV_PANIC());
333 
334 	/*
335 	 * Get an exclusive lock, might have to wait for a kmem reader.
336 	 */
337 	if (!page_tryupgrade(pp)) {
338 		u_offset_t off = pp->p_offset;
339 		page_unlock(pp);
340 		pp = page_lookup(&kvp, off, SE_EXCL);
341 		if (pp == NULL)
342 			panic("page not found");
343 	}
344 #ifdef __xpv
345 	if (kpm_vbase && xen_kpm_page(pfn, PT_VALID | PT_WRITABLE) < 0)
346 		panic("failure making kpm r/w pfn=0x%lx", pfn);
347 #endif
348 	page_hashout(pp, NULL);
349 	page_free(pp, 1);
350 	page_unresv(1);
351 }
352 
353 /*
354  * Put one htable on the reserve list.
355  */
356 static void
htable_put_reserve(htable_t * ht)357 htable_put_reserve(htable_t *ht)
358 {
359 	ht->ht_hat = NULL;		/* no longer tied to a hat */
360 	ASSERT(ht->ht_pfn == PFN_INVALID);
361 	HATSTAT_INC(hs_htable_rputs);
362 	mutex_enter(&htable_reserve_mutex);
363 	ht->ht_next = htable_reserve_pool;
364 	htable_reserve_pool = ht;
365 	++htable_reserve_cnt;
366 	mutex_exit(&htable_reserve_mutex);
367 }
368 
369 /*
370  * Take one htable from the reserve.
371  */
372 static htable_t *
htable_get_reserve(void)373 htable_get_reserve(void)
374 {
375 	htable_t *ht = NULL;
376 
377 	mutex_enter(&htable_reserve_mutex);
378 	if (htable_reserve_cnt != 0) {
379 		ht = htable_reserve_pool;
380 		ASSERT(ht != NULL);
381 		ASSERT(ht->ht_pfn == PFN_INVALID);
382 		htable_reserve_pool = ht->ht_next;
383 		--htable_reserve_cnt;
384 		HATSTAT_INC(hs_htable_rgets);
385 	}
386 	mutex_exit(&htable_reserve_mutex);
387 	return (ht);
388 }
389 
390 /*
391  * Allocate initial htables and put them on the reserve list
392  */
393 void
htable_initial_reserve(uint_t count)394 htable_initial_reserve(uint_t count)
395 {
396 	htable_t *ht;
397 
398 	count += HTABLE_RESERVE_AMOUNT;
399 	while (count > 0) {
400 		ht = kmem_cache_alloc(htable_cache, KM_NOSLEEP);
401 		ASSERT(ht != NULL);
402 
403 		ASSERT(use_boot_reserve);
404 		ht->ht_pfn = PFN_INVALID;
405 		htable_put_reserve(ht);
406 		--count;
407 	}
408 }
409 
410 /*
411  * Readjust the reserves after a thread finishes using them.
412  */
413 void
htable_adjust_reserve()414 htable_adjust_reserve()
415 {
416 	htable_t *ht;
417 
418 	/*
419 	 * Free any excess htables in the reserve list
420 	 */
421 	while (htable_reserve_cnt > htable_reserve_amount &&
422 	    !USE_HAT_RESERVES()) {
423 		ht = htable_get_reserve();
424 		if (ht == NULL)
425 			return;
426 		ASSERT(ht->ht_pfn == PFN_INVALID);
427 		kmem_cache_free(htable_cache, ht);
428 	}
429 }
430 
431 /*
432  * Search the active htables for one to steal. Start at a different hash
433  * bucket every time to help spread the pain of stealing
434  */
435 static void
htable_steal_active(hat_t * hat,uint_t cnt,uint_t threshold,uint_t * stolen,htable_t ** list)436 htable_steal_active(hat_t *hat, uint_t cnt, uint_t threshold,
437     uint_t *stolen, htable_t **list)
438 {
439 	static uint_t	h_seed = 0;
440 	htable_t	*higher, *ht;
441 	uint_t		h, e, h_start;
442 	uintptr_t	va;
443 	x86pte_t	pte;
444 
445 	h = h_start = h_seed++ % hat->hat_num_hash;
446 	do {
447 		higher = NULL;
448 		HTABLE_ENTER(h);
449 		for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) {
450 
451 			/*
452 			 * Can we rule out reaping?
453 			 */
454 			if (ht->ht_busy != 0 ||
455 			    (ht->ht_flags & HTABLE_SHARED_PFN) ||
456 			    ht->ht_level > 0 || ht->ht_valid_cnt > threshold ||
457 			    ht->ht_lock_cnt != 0)
458 				continue;
459 
460 			/*
461 			 * Increment busy so the htable can't disappear. We
462 			 * drop the htable mutex to avoid deadlocks with
463 			 * hat_pageunload() and the hment mutex while we
464 			 * call hat_pte_unmap()
465 			 */
466 			++ht->ht_busy;
467 			HTABLE_EXIT(h);
468 
469 			/*
470 			 * Try stealing.
471 			 * - unload and invalidate all PTEs
472 			 */
473 			for (e = 0, va = ht->ht_vaddr;
474 			    e < HTABLE_NUM_PTES(ht) && ht->ht_valid_cnt > 0 &&
475 			    ht->ht_busy == 1 && ht->ht_lock_cnt == 0;
476 			    ++e, va += MMU_PAGESIZE) {
477 				pte = x86pte_get(ht, e);
478 				if (!PTE_ISVALID(pte))
479 					continue;
480 				hat_pte_unmap(ht, e, HAT_UNLOAD, pte, NULL,
481 				    B_TRUE);
482 			}
483 
484 			/*
485 			 * Reacquire htable lock. If we didn't remove all
486 			 * mappings in the table, or another thread added a new
487 			 * mapping behind us, give up on this table.
488 			 */
489 			HTABLE_ENTER(h);
490 			if (ht->ht_busy != 1 || ht->ht_valid_cnt != 0 ||
491 			    ht->ht_lock_cnt != 0) {
492 				--ht->ht_busy;
493 				continue;
494 			}
495 
496 			/*
497 			 * Steal it and unlink the page table.
498 			 */
499 			higher = ht->ht_parent;
500 			unlink_ptp(higher, ht, ht->ht_vaddr);
501 
502 			/*
503 			 * remove from the hash list
504 			 */
505 			if (ht->ht_next)
506 				ht->ht_next->ht_prev = ht->ht_prev;
507 
508 			if (ht->ht_prev) {
509 				ht->ht_prev->ht_next = ht->ht_next;
510 			} else {
511 				ASSERT(hat->hat_ht_hash[h] == ht);
512 				hat->hat_ht_hash[h] = ht->ht_next;
513 			}
514 
515 			/*
516 			 * Break to outer loop to release the
517 			 * higher (ht_parent) pagetable. This
518 			 * spreads out the pain caused by
519 			 * pagefaults.
520 			 */
521 			ht->ht_next = *list;
522 			*list = ht;
523 			++*stolen;
524 			break;
525 		}
526 		HTABLE_EXIT(h);
527 		if (higher != NULL)
528 			htable_release(higher);
529 		if (++h == hat->hat_num_hash)
530 			h = 0;
531 	} while (*stolen < cnt && h != h_start);
532 }
533 
534 /*
535  * Move hat to the end of the kas list
536  */
537 static void
move_victim(hat_t * hat)538 move_victim(hat_t *hat)
539 {
540 	ASSERT(MUTEX_HELD(&hat_list_lock));
541 
542 	/* unlink victim hat */
543 	if (hat->hat_prev)
544 		hat->hat_prev->hat_next = hat->hat_next;
545 	else
546 		kas.a_hat->hat_next = hat->hat_next;
547 
548 	if (hat->hat_next)
549 		hat->hat_next->hat_prev = hat->hat_prev;
550 	else
551 		kas.a_hat->hat_prev = hat->hat_prev;
552 	/* relink at end of hat list */
553 	hat->hat_next = NULL;
554 	hat->hat_prev = kas.a_hat->hat_prev;
555 	if (hat->hat_prev)
556 		hat->hat_prev->hat_next = hat;
557 	else
558 		kas.a_hat->hat_next = hat;
559 
560 	kas.a_hat->hat_prev = hat;
561 }
562 
563 /*
564  * This routine steals htables from user processes.  Called by htable_reap
565  * (reap=TRUE) or htable_alloc (reap=FALSE).
566  */
567 static htable_t *
htable_steal(uint_t cnt,boolean_t reap)568 htable_steal(uint_t cnt, boolean_t reap)
569 {
570 	hat_t		*hat = kas.a_hat;	/* list starts with khat */
571 	htable_t	*list = NULL;
572 	htable_t	*ht;
573 	uint_t		stolen = 0;
574 	uint_t		pass, passes;
575 	uint_t		threshold;
576 
577 	/*
578 	 * Limit htable_steal_passes to something reasonable
579 	 */
580 	if (htable_steal_passes == 0)
581 		htable_steal_passes = 1;
582 	if (htable_steal_passes > mmu.ptes_per_table)
583 		htable_steal_passes = mmu.ptes_per_table;
584 
585 	/*
586 	 * If we're stealing merely as part of kmem reaping (versus stealing
587 	 * to assure forward progress), we don't want to actually steal any
588 	 * active htables.  (Stealing active htables merely to give memory
589 	 * back to the system can inadvertently kick off an htable crime wave
590 	 * as active processes repeatedly steal htables from one another,
591 	 * plummeting the system into a kind of HAT lawlessness that can
592 	 * become so violent as to impede the one thing that can end it:  the
593 	 * freeing of memory via ARC reclaim and other means.)  So if we're
594 	 * reaping, we limit ourselves to the first pass that steals cached
595 	 * htables that aren't in use -- which gives memory back, but averts
596 	 * the entire breakdown of social order.
597 	 */
598 	passes = reap ? 0 : htable_steal_passes;
599 
600 	/*
601 	 * Loop through all user hats. The 1st pass takes cached htables that
602 	 * aren't in use. The later passes steal by removing mappings, too.
603 	 */
604 	atomic_inc_32(&htable_dont_cache);
605 	for (pass = 0; pass <= passes && stolen < cnt; ++pass) {
606 		threshold = pass * mmu.ptes_per_table / htable_steal_passes;
607 
608 		mutex_enter(&hat_list_lock);
609 
610 		/* skip the first hat (kernel) */
611 		hat = kas.a_hat->hat_next;
612 		for (;;) {
613 			/*
614 			 * Skip any hat that is already being stolen from.
615 			 *
616 			 * We skip SHARED hats, as these are dummy
617 			 * hats that host ISM shared page tables.
618 			 *
619 			 * We also skip if HAT_FREEING because hat_pte_unmap()
620 			 * won't zero out the PTE's. That would lead to hitting
621 			 * stale PTEs either here or under hat_unload() when we
622 			 * steal and unload the same page table in competing
623 			 * threads.
624 			 *
625 			 * We skip HATs that belong to CPUs, to make our lives
626 			 * simpler.
627 			 */
628 			while (hat != NULL && (hat->hat_flags &
629 			    (HAT_VICTIM | HAT_SHARED | HAT_FREEING |
630 			    HAT_PCP)) != 0) {
631 				hat = hat->hat_next;
632 			}
633 
634 			if (hat == NULL)
635 				break;
636 
637 			/*
638 			 * Mark the HAT as a stealing victim so that it is
639 			 * not freed from under us, e.g. in as_free()
640 			 */
641 			hat->hat_flags |= HAT_VICTIM;
642 			mutex_exit(&hat_list_lock);
643 
644 			/*
645 			 * Take any htables from the hat's cached "free" list.
646 			 */
647 			hat_enter(hat);
648 			while ((ht = hat->hat_ht_cached) != NULL &&
649 			    stolen < cnt) {
650 				hat->hat_ht_cached = ht->ht_next;
651 				ht->ht_next = list;
652 				list = ht;
653 				++stolen;
654 			}
655 			hat_exit(hat);
656 
657 			/*
658 			 * Don't steal active htables on first pass.
659 			 */
660 			if (pass != 0 && (stolen < cnt))
661 				htable_steal_active(hat, cnt, threshold,
662 				    &stolen, &list);
663 
664 			/*
665 			 * do synchronous teardown for the reap case so that
666 			 * we can forget hat; at this time, hat is
667 			 * guaranteed to be around because HAT_VICTIM is set
668 			 * (see htable_free() for similar code)
669 			 */
670 			for (ht = list; (ht) && (reap); ht = ht->ht_next) {
671 				if (ht->ht_hat == NULL)
672 					continue;
673 				ASSERT(ht->ht_hat == hat);
674 #if defined(__xpv)
675 				ASSERT(!(ht->ht_flags & HTABLE_COPIED));
676 				if (ht->ht_level == mmu.max_level) {
677 					ptable_free(hat->hat_user_ptable);
678 					hat->hat_user_ptable = PFN_INVALID;
679 				}
680 #endif
681 				/*
682 				 * forget the hat
683 				 */
684 				ht->ht_hat = NULL;
685 			}
686 
687 			mutex_enter(&hat_list_lock);
688 
689 			/*
690 			 * Are we finished?
691 			 */
692 			if (stolen == cnt) {
693 				/*
694 				 * Try to spread the pain of stealing,
695 				 * move victim HAT to the end of the HAT list.
696 				 */
697 				if (pass >= 1 && cnt == 1 &&
698 				    kas.a_hat->hat_prev != hat)
699 					move_victim(hat);
700 				/*
701 				 * We are finished
702 				 */
703 			}
704 
705 			/*
706 			 * Clear the victim flag, hat can go away now (once
707 			 * the lock is dropped)
708 			 */
709 			if (hat->hat_flags & HAT_VICTIM) {
710 				ASSERT(hat != kas.a_hat);
711 				hat->hat_flags &= ~HAT_VICTIM;
712 				cv_broadcast(&hat_list_cv);
713 			}
714 
715 			/* move on to the next hat */
716 			hat = hat->hat_next;
717 		}
718 
719 		mutex_exit(&hat_list_lock);
720 
721 	}
722 	ASSERT(!MUTEX_HELD(&hat_list_lock));
723 
724 	atomic_dec_32(&htable_dont_cache);
725 	return (list);
726 }
727 
728 /*
729  * This is invoked from kmem when the system is low on memory.  We try
730  * to free hments, htables, and ptables to improve the memory situation.
731  */
732 /*ARGSUSED*/
733 static void
htable_reap(void * handle)734 htable_reap(void *handle)
735 {
736 	uint_t		reap_cnt;
737 	htable_t	*list;
738 	htable_t	*ht;
739 
740 	HATSTAT_INC(hs_reap_attempts);
741 	if (!can_steal_post_boot)
742 		return;
743 
744 	/*
745 	 * Try to reap 5% of the page tables bounded by a maximum of
746 	 * 5% of physmem and a minimum of 10.
747 	 */
748 	reap_cnt = MAX(MIN(physmem / 20, active_ptables / 20), 10);
749 
750 	/*
751 	 * Note: htable_dont_cache should be set at the time of
752 	 * invoking htable_free()
753 	 */
754 	atomic_inc_32(&htable_dont_cache);
755 	/*
756 	 * Let htable_steal() do the work, we just call htable_free()
757 	 */
758 	XPV_DISALLOW_MIGRATE();
759 	list = htable_steal(reap_cnt, B_TRUE);
760 	XPV_ALLOW_MIGRATE();
761 	while ((ht = list) != NULL) {
762 		list = ht->ht_next;
763 		HATSTAT_INC(hs_reaped);
764 		htable_free(ht);
765 	}
766 	atomic_dec_32(&htable_dont_cache);
767 
768 	/*
769 	 * Free up excess reserves
770 	 */
771 	htable_adjust_reserve();
772 	hment_adjust_reserve();
773 }
774 
775 /*
776  * Allocate an htable, stealing one or using the reserve if necessary
777  */
778 static htable_t *
htable_alloc(hat_t * hat,uintptr_t vaddr,level_t level,htable_t * shared)779 htable_alloc(
780 	hat_t		*hat,
781 	uintptr_t	vaddr,
782 	level_t		level,
783 	htable_t	*shared)
784 {
785 	htable_t	*ht = NULL;
786 	uint_t		is_copied;
787 	uint_t		is_bare = 0;
788 	uint_t		need_to_zero = 1;
789 	int		kmflags = (can_steal_post_boot ? KM_NOSLEEP : KM_SLEEP);
790 
791 	if (level < 0 || level > TOP_LEVEL(hat))
792 		panic("htable_alloc(): level %d out of range\n", level);
793 
794 	is_copied = (hat->hat_flags & HAT_COPIED) &&
795 	    level == hat->hat_max_level;
796 	if (is_copied || shared != NULL)
797 		is_bare = 1;
798 
799 	/*
800 	 * First reuse a cached htable from the hat_ht_cached field, this
801 	 * avoids unnecessary trips through kmem/page allocators.
802 	 */
803 	if (hat->hat_ht_cached != NULL && !is_bare) {
804 		hat_enter(hat);
805 		ht = hat->hat_ht_cached;
806 		if (ht != NULL) {
807 			hat->hat_ht_cached = ht->ht_next;
808 			need_to_zero = 0;
809 			/* XX64 ASSERT() they're all zero somehow */
810 			ASSERT(ht->ht_pfn != PFN_INVALID);
811 		}
812 		hat_exit(hat);
813 	}
814 
815 	if (ht == NULL) {
816 		/*
817 		 * Allocate an htable, possibly refilling the reserves.
818 		 */
819 		if (USE_HAT_RESERVES()) {
820 			ht = htable_get_reserve();
821 		} else {
822 			/*
823 			 * Donate successful htable allocations to the reserve.
824 			 */
825 			for (;;) {
826 				ht = kmem_cache_alloc(htable_cache, kmflags);
827 				if (ht == NULL)
828 					break;
829 				ht->ht_pfn = PFN_INVALID;
830 				if (USE_HAT_RESERVES() ||
831 				    htable_reserve_cnt >= htable_reserve_amount)
832 					break;
833 				htable_put_reserve(ht);
834 			}
835 		}
836 
837 		/*
838 		 * allocate a page for the hardware page table if needed
839 		 */
840 		if (ht != NULL && !is_bare) {
841 			ht->ht_hat = hat;
842 			ht->ht_pfn = ptable_alloc((uintptr_t)ht);
843 			if (ht->ht_pfn == PFN_INVALID) {
844 				if (USE_HAT_RESERVES())
845 					htable_put_reserve(ht);
846 				else
847 					kmem_cache_free(htable_cache, ht);
848 				ht = NULL;
849 			}
850 		}
851 	}
852 
853 	/*
854 	 * If allocations failed, kick off a kmem_reap() and resort to
855 	 * htable steal(). We may spin here if the system is very low on
856 	 * memory. If the kernel itself has consumed all memory and kmem_reap()
857 	 * can't free up anything, then we'll really get stuck here.
858 	 * That should only happen in a system where the administrator has
859 	 * misconfigured VM parameters via /etc/system.
860 	 */
861 	while (ht == NULL && can_steal_post_boot) {
862 		kmem_reap();
863 		ht = htable_steal(1, B_FALSE);
864 		HATSTAT_INC(hs_steals);
865 
866 		/*
867 		 * If we stole for a bare htable, release the pagetable page.
868 		 */
869 		if (ht != NULL) {
870 			if (is_bare) {
871 				ptable_free(ht->ht_pfn);
872 				ht->ht_pfn = PFN_INVALID;
873 #if defined(__xpv)
874 			/*
875 			 * make stolen page table writable again in kpm
876 			 */
877 			} else if (kpm_vbase && xen_kpm_page(ht->ht_pfn,
878 			    PT_VALID | PT_WRITABLE) < 0) {
879 				panic("failure making kpm r/w pfn=0x%lx",
880 				    ht->ht_pfn);
881 #endif
882 			}
883 		}
884 	}
885 
886 	/*
887 	 * All attempts to allocate or steal failed. This should only happen
888 	 * if we run out of memory during boot, due perhaps to a huge
889 	 * boot_archive. At this point there's no way to continue.
890 	 */
891 	if (ht == NULL)
892 		panic("htable_alloc(): couldn't steal\n");
893 
894 #if defined(__xpv)
895 	/*
896 	 * Under the 64-bit hypervisor, we have 2 top level page tables.
897 	 * If this allocation fails, we'll resort to stealing.
898 	 * We use the stolen page indirectly, by freeing the
899 	 * stolen htable first.
900 	 */
901 	if (level == mmu.max_level) {
902 		for (;;) {
903 			htable_t *stolen;
904 
905 			hat->hat_user_ptable = ptable_alloc((uintptr_t)ht + 1);
906 			if (hat->hat_user_ptable != PFN_INVALID)
907 				break;
908 			stolen = htable_steal(1, B_FALSE);
909 			if (stolen == NULL)
910 				panic("2nd steal ptable failed\n");
911 			htable_free(stolen);
912 		}
913 		block_zero_no_xmm(kpm_vbase + pfn_to_pa(hat->hat_user_ptable),
914 		    MMU_PAGESIZE);
915 	}
916 #endif
917 
918 	/*
919 	 * Shared page tables have all entries locked and entries may not
920 	 * be added or deleted.
921 	 */
922 	ht->ht_flags = 0;
923 	if (shared != NULL) {
924 		ASSERT(shared->ht_valid_cnt > 0);
925 		ht->ht_flags |= HTABLE_SHARED_PFN;
926 		ht->ht_pfn = shared->ht_pfn;
927 		ht->ht_lock_cnt = 0;
928 		ht->ht_valid_cnt = 0;		/* updated in hat_share() */
929 		ht->ht_shares = shared;
930 		need_to_zero = 0;
931 	} else {
932 		ht->ht_shares = NULL;
933 		ht->ht_lock_cnt = 0;
934 		ht->ht_valid_cnt = 0;
935 	}
936 
937 	/*
938 	 * setup flags, etc. for copied page tables.
939 	 */
940 	if (is_copied) {
941 		ht->ht_flags |= HTABLE_COPIED;
942 		ASSERT(ht->ht_pfn == PFN_INVALID);
943 		need_to_zero = 0;
944 	}
945 
946 	/*
947 	 * fill in the htable
948 	 */
949 	ht->ht_hat = hat;
950 	ht->ht_parent = NULL;
951 	ht->ht_vaddr = vaddr;
952 	ht->ht_level = level;
953 	ht->ht_busy = 1;
954 	ht->ht_next = NULL;
955 	ht->ht_prev = NULL;
956 
957 	/*
958 	 * Zero out any freshly allocated page table
959 	 */
960 	if (need_to_zero)
961 		x86pte_zero(ht, 0, mmu.ptes_per_table);
962 
963 #if defined(__xpv)
964 	if (!is_bare && kpm_vbase) {
965 		(void) xen_kpm_page(ht->ht_pfn, PT_VALID);
966 		if (level == mmu.max_level)
967 			(void) xen_kpm_page(hat->hat_user_ptable, PT_VALID);
968 	}
969 #endif
970 
971 	return (ht);
972 }
973 
974 /*
975  * Free up an htable, either to a hat's cached list, the reserves or
976  * back to kmem.
977  */
978 static void
htable_free(htable_t * ht)979 htable_free(htable_t *ht)
980 {
981 	hat_t *hat = ht->ht_hat;
982 
983 	/*
984 	 * If the process isn't exiting, cache the free htable in the hat
985 	 * structure. We always do this for the boot time reserve. We don't
986 	 * do this if the hat is exiting or we are stealing/reaping htables.
987 	 */
988 	if (hat != NULL &&
989 	    !(ht->ht_flags & HTABLE_SHARED_PFN) &&
990 	    (use_boot_reserve ||
991 	    (!(hat->hat_flags & HAT_FREEING) && !htable_dont_cache))) {
992 		ASSERT((ht->ht_flags & HTABLE_COPIED) == 0);
993 		ASSERT(ht->ht_pfn != PFN_INVALID);
994 		hat_enter(hat);
995 		ht->ht_next = hat->hat_ht_cached;
996 		hat->hat_ht_cached = ht;
997 		hat_exit(hat);
998 		return;
999 	}
1000 
1001 	/*
1002 	 * If we have a hardware page table, free it.
1003 	 * We don't free page tables that are accessed by sharing.
1004 	 */
1005 	if (ht->ht_flags & HTABLE_SHARED_PFN) {
1006 		ASSERT(ht->ht_pfn != PFN_INVALID);
1007 	} else if (!(ht->ht_flags & HTABLE_COPIED)) {
1008 		ptable_free(ht->ht_pfn);
1009 #if defined(__xpv)
1010 		if (ht->ht_level == mmu.max_level && hat != NULL) {
1011 			ptable_free(hat->hat_user_ptable);
1012 			hat->hat_user_ptable = PFN_INVALID;
1013 		}
1014 #endif
1015 	}
1016 	ht->ht_pfn = PFN_INVALID;
1017 
1018 	/*
1019 	 * Free it or put into reserves.
1020 	 */
1021 	if (USE_HAT_RESERVES() || htable_reserve_cnt < htable_reserve_amount) {
1022 		htable_put_reserve(ht);
1023 	} else {
1024 		kmem_cache_free(htable_cache, ht);
1025 		htable_adjust_reserve();
1026 	}
1027 }
1028 
1029 
1030 /*
1031  * This is called when a hat is being destroyed or swapped out. We reap all
1032  * the remaining htables in the hat cache. If destroying all left over
1033  * htables are also destroyed.
1034  *
1035  * We also don't need to invalidate any of the PTPs nor do any demapping.
1036  */
1037 void
htable_purge_hat(hat_t * hat)1038 htable_purge_hat(hat_t *hat)
1039 {
1040 	htable_t *ht;
1041 	int h;
1042 
1043 	/*
1044 	 * Purge the htable cache if just reaping.
1045 	 */
1046 	if (!(hat->hat_flags & HAT_FREEING)) {
1047 		atomic_inc_32(&htable_dont_cache);
1048 		for (;;) {
1049 			hat_enter(hat);
1050 			ht = hat->hat_ht_cached;
1051 			if (ht == NULL) {
1052 				hat_exit(hat);
1053 				break;
1054 			}
1055 			hat->hat_ht_cached = ht->ht_next;
1056 			hat_exit(hat);
1057 			htable_free(ht);
1058 		}
1059 		atomic_dec_32(&htable_dont_cache);
1060 		return;
1061 	}
1062 
1063 	/*
1064 	 * if freeing, no locking is needed
1065 	 */
1066 	while ((ht = hat->hat_ht_cached) != NULL) {
1067 		hat->hat_ht_cached = ht->ht_next;
1068 		htable_free(ht);
1069 	}
1070 
1071 	/*
1072 	 * walk thru the htable hash table and free all the htables in it.
1073 	 */
1074 	for (h = 0; h < hat->hat_num_hash; ++h) {
1075 		while ((ht = hat->hat_ht_hash[h]) != NULL) {
1076 			if (ht->ht_next)
1077 				ht->ht_next->ht_prev = ht->ht_prev;
1078 
1079 			if (ht->ht_prev) {
1080 				ht->ht_prev->ht_next = ht->ht_next;
1081 			} else {
1082 				ASSERT(hat->hat_ht_hash[h] == ht);
1083 				hat->hat_ht_hash[h] = ht->ht_next;
1084 			}
1085 			htable_free(ht);
1086 		}
1087 	}
1088 }
1089 
1090 /*
1091  * Unlink an entry for a table at vaddr and level out of the existing table
1092  * one level higher. We are always holding the HASH_ENTER() when doing this.
1093  */
1094 static void
unlink_ptp(htable_t * higher,htable_t * old,uintptr_t vaddr)1095 unlink_ptp(htable_t *higher, htable_t *old, uintptr_t vaddr)
1096 {
1097 	uint_t		entry = htable_va2entry(vaddr, higher);
1098 	x86pte_t	expect = MAKEPTP(old->ht_pfn, old->ht_level);
1099 	x86pte_t	found;
1100 	hat_t		*hat = old->ht_hat;
1101 
1102 	ASSERT(higher->ht_busy > 0);
1103 	ASSERT(higher->ht_valid_cnt > 0);
1104 	ASSERT(old->ht_valid_cnt == 0);
1105 	found = x86pte_cas(higher, entry, expect, 0);
1106 #ifdef __xpv
1107 	/*
1108 	 * This is weird, but Xen apparently automatically unlinks empty
1109 	 * pagetables from the upper page table. So allow PTP to be 0 already.
1110 	 */
1111 	if (found != expect && found != 0)
1112 #else
1113 	if (found != expect)
1114 #endif
1115 		panic("Bad PTP found=" FMT_PTE ", expected=" FMT_PTE,
1116 		    found, expect);
1117 
1118 	/*
1119 	 * When a top level PTE changes for a copied htable, we must trigger a
1120 	 * hat_pcp_update() on all HAT CPUs.
1121 	 *
1122 	 * If we don't need do do that, then we still have to INVLPG against an
1123 	 * address covered by the inner page table, as the latest processors
1124 	 * have TLB-like caches for non-leaf page table entries.
1125 	 */
1126 	if (!(hat->hat_flags & HAT_FREEING)) {
1127 		hat_tlb_inval(hat, (higher->ht_flags & HTABLE_COPIED) ?
1128 		    DEMAP_ALL_ADDR : old->ht_vaddr);
1129 	}
1130 
1131 	HTABLE_DEC(higher->ht_valid_cnt);
1132 }
1133 
1134 /*
1135  * Link an entry for a new table at vaddr and level into the existing table
1136  * one level higher. We are always holding the HASH_ENTER() when doing this.
1137  */
1138 static void
link_ptp(htable_t * higher,htable_t * new,uintptr_t vaddr)1139 link_ptp(htable_t *higher, htable_t *new, uintptr_t vaddr)
1140 {
1141 	uint_t		entry = htable_va2entry(vaddr, higher);
1142 	x86pte_t	newptp = MAKEPTP(new->ht_pfn, new->ht_level);
1143 	x86pte_t	found;
1144 
1145 	ASSERT(higher->ht_busy > 0);
1146 
1147 	ASSERT(new->ht_level != mmu.max_level);
1148 
1149 	HTABLE_INC(higher->ht_valid_cnt);
1150 
1151 	found = x86pte_cas(higher, entry, 0, newptp);
1152 	if ((found & ~PT_REF) != 0)
1153 		panic("HAT: ptp not 0, found=" FMT_PTE, found);
1154 
1155 	/*
1156 	 * When a top level PTE changes for a copied htable, we must trigger a
1157 	 * hat_pcp_update() on all HAT CPUs.
1158 	 *
1159 	 * We also need to do this for the kernel hat on PAE 32 bit kernel.
1160 	 */
1161 	if ((higher->ht_flags & HTABLE_COPIED) != 0)
1162 		hat_tlb_inval(higher->ht_hat, DEMAP_ALL_ADDR);
1163 }
1164 
1165 /*
1166  * Release of hold on an htable. If this is the last use and the pagetable
1167  * is empty we may want to free it, then recursively look at the pagetable
1168  * above it. The recursion is handled by the outer while() loop.
1169  *
1170  * On the metal, during process exit, we don't bother unlinking the tables from
1171  * upper level pagetables. They are instead handled in bulk by hat_free_end().
1172  * We can't do this on the hypervisor as we need the page table to be
1173  * implicitly unpinnned before it goes to the free page lists. This can't
1174  * happen unless we fully unlink it from the page table hierarchy.
1175  */
1176 void
htable_release(htable_t * ht)1177 htable_release(htable_t *ht)
1178 {
1179 	uint_t		hashval;
1180 	htable_t	*shared;
1181 	htable_t	*higher;
1182 	hat_t		*hat;
1183 	uintptr_t	va;
1184 	level_t		level;
1185 
1186 	while (ht != NULL) {
1187 		shared = NULL;
1188 		for (;;) {
1189 			hat = ht->ht_hat;
1190 			va = ht->ht_vaddr;
1191 			level = ht->ht_level;
1192 			hashval = HTABLE_HASH(hat, va, level);
1193 
1194 			/*
1195 			 * The common case is that this isn't the last use of
1196 			 * an htable so we don't want to free the htable.
1197 			 */
1198 			HTABLE_ENTER(hashval);
1199 			ASSERT(ht->ht_valid_cnt >= 0);
1200 			ASSERT(ht->ht_busy > 0);
1201 			if (ht->ht_valid_cnt > 0)
1202 				break;
1203 			if (ht->ht_busy > 1)
1204 				break;
1205 			ASSERT(ht->ht_lock_cnt == 0);
1206 
1207 #if !defined(__xpv)
1208 			/*
1209 			 * we always release empty shared htables
1210 			 */
1211 			if (!(ht->ht_flags & HTABLE_SHARED_PFN)) {
1212 
1213 				/*
1214 				 * don't release if in address space tear down
1215 				 */
1216 				if (hat->hat_flags & HAT_FREEING)
1217 					break;
1218 
1219 				/*
1220 				 * At and above max_page_level, free if it's for
1221 				 * a boot-time kernel mapping below kernelbase.
1222 				 */
1223 				if (level >= mmu.max_page_level &&
1224 				    (hat != kas.a_hat || va >= kernelbase))
1225 					break;
1226 			}
1227 #endif /* __xpv */
1228 
1229 			/*
1230 			 * Remember if we destroy an htable that shares its PFN
1231 			 * from elsewhere.
1232 			 */
1233 			if (ht->ht_flags & HTABLE_SHARED_PFN) {
1234 				ASSERT(shared == NULL);
1235 				shared = ht->ht_shares;
1236 				HATSTAT_INC(hs_htable_unshared);
1237 			}
1238 
1239 			/*
1240 			 * Handle release of a table and freeing the htable_t.
1241 			 * Unlink it from the table higher (ie. ht_parent).
1242 			 */
1243 			higher = ht->ht_parent;
1244 			ASSERT(higher != NULL);
1245 
1246 			/*
1247 			 * Unlink the pagetable.
1248 			 */
1249 			unlink_ptp(higher, ht, va);
1250 
1251 			/*
1252 			 * remove this htable from its hash list
1253 			 */
1254 			if (ht->ht_next)
1255 				ht->ht_next->ht_prev = ht->ht_prev;
1256 
1257 			if (ht->ht_prev) {
1258 				ht->ht_prev->ht_next = ht->ht_next;
1259 			} else {
1260 				ASSERT(hat->hat_ht_hash[hashval] == ht);
1261 				hat->hat_ht_hash[hashval] = ht->ht_next;
1262 			}
1263 			HTABLE_EXIT(hashval);
1264 			htable_free(ht);
1265 			ht = higher;
1266 		}
1267 
1268 		ASSERT(ht->ht_busy >= 1);
1269 		--ht->ht_busy;
1270 		HTABLE_EXIT(hashval);
1271 
1272 		/*
1273 		 * If we released a shared htable, do a release on the htable
1274 		 * from which it shared
1275 		 */
1276 		ht = shared;
1277 	}
1278 }
1279 
1280 /*
1281  * Find the htable for the pagetable at the given level for the given address.
1282  * If found acquires a hold that eventually needs to be htable_release()d
1283  */
1284 htable_t *
htable_lookup(hat_t * hat,uintptr_t vaddr,level_t level)1285 htable_lookup(hat_t *hat, uintptr_t vaddr, level_t level)
1286 {
1287 	uintptr_t	base;
1288 	uint_t		hashval;
1289 	htable_t	*ht = NULL;
1290 
1291 	ASSERT(level >= 0);
1292 	ASSERT(level <= TOP_LEVEL(hat));
1293 
1294 	if (level == TOP_LEVEL(hat)) {
1295 		/*
1296 		 * 32 bit address spaces on 64 bit kernels need to check
1297 		 * for overflow of the 32 bit address space
1298 		 */
1299 		if ((hat->hat_flags & HAT_COPIED_32) &&
1300 		    vaddr >= ((uint64_t)1 << 32))
1301 			return (NULL);
1302 		base = 0;
1303 	} else {
1304 		base = vaddr & LEVEL_MASK(level + 1);
1305 	}
1306 
1307 	hashval = HTABLE_HASH(hat, base, level);
1308 	HTABLE_ENTER(hashval);
1309 	for (ht = hat->hat_ht_hash[hashval]; ht; ht = ht->ht_next) {
1310 		if (ht->ht_hat == hat &&
1311 		    ht->ht_vaddr == base &&
1312 		    ht->ht_level == level)
1313 			break;
1314 	}
1315 	if (ht)
1316 		++ht->ht_busy;
1317 
1318 	HTABLE_EXIT(hashval);
1319 	return (ht);
1320 }
1321 
1322 /*
1323  * Acquires a hold on a known htable (from a locked hment entry).
1324  */
1325 void
htable_acquire(htable_t * ht)1326 htable_acquire(htable_t *ht)
1327 {
1328 	hat_t		*hat = ht->ht_hat;
1329 	level_t		level = ht->ht_level;
1330 	uintptr_t	base = ht->ht_vaddr;
1331 	uint_t		hashval = HTABLE_HASH(hat, base, level);
1332 
1333 	HTABLE_ENTER(hashval);
1334 #ifdef DEBUG
1335 	/*
1336 	 * make sure the htable is there
1337 	 */
1338 	{
1339 		htable_t	*h;
1340 
1341 		for (h = hat->hat_ht_hash[hashval];
1342 		    h && h != ht;
1343 		    h = h->ht_next)
1344 			;
1345 		ASSERT(h == ht);
1346 	}
1347 #endif /* DEBUG */
1348 	++ht->ht_busy;
1349 	HTABLE_EXIT(hashval);
1350 }
1351 
1352 /*
1353  * Find the htable for the pagetable at the given level for the given address.
1354  * If found acquires a hold that eventually needs to be htable_release()d
1355  * If not found the table is created.
1356  *
1357  * Since we can't hold a hash table mutex during allocation, we have to
1358  * drop it and redo the search on a create. Then we may have to free the newly
1359  * allocated htable if another thread raced in and created it ahead of us.
1360  */
1361 htable_t *
htable_create(hat_t * hat,uintptr_t vaddr,level_t level,htable_t * shared)1362 htable_create(
1363 	hat_t		*hat,
1364 	uintptr_t	vaddr,
1365 	level_t		level,
1366 	htable_t	*shared)
1367 {
1368 	uint_t		h;
1369 	level_t		l;
1370 	uintptr_t	base;
1371 	htable_t	*ht;
1372 	htable_t	*higher = NULL;
1373 	htable_t	*new = NULL;
1374 
1375 	if (level < 0 || level > TOP_LEVEL(hat))
1376 		panic("htable_create(): level %d out of range\n", level);
1377 
1378 	ht = NULL;
1379 	/*
1380 	 * Create the page tables in top down order.
1381 	 */
1382 	for (l = TOP_LEVEL(hat); l >= level; --l) {
1383 		new = NULL;
1384 		if (l == TOP_LEVEL(hat))
1385 			base = 0;
1386 		else
1387 			base = vaddr & LEVEL_MASK(l + 1);
1388 
1389 		h = HTABLE_HASH(hat, base, l);
1390 try_again:
1391 		/*
1392 		 * look up the htable at this level
1393 		 */
1394 		HTABLE_ENTER(h);
1395 		if (l == TOP_LEVEL(hat)) {
1396 			ht = hat->hat_htable;
1397 		} else {
1398 			for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) {
1399 				ASSERT(ht->ht_hat == hat);
1400 				if (ht->ht_vaddr == base &&
1401 				    ht->ht_level == l)
1402 					break;
1403 			}
1404 		}
1405 
1406 		/*
1407 		 * if we found the htable, increment its busy cnt
1408 		 * and if we had allocated a new htable, free it.
1409 		 */
1410 		if (ht != NULL) {
1411 			/*
1412 			 * If we find a pre-existing shared table, it must
1413 			 * share from the same place.
1414 			 */
1415 			if (l == level && shared && ht->ht_shares &&
1416 			    ht->ht_shares != shared) {
1417 				panic("htable shared from wrong place "
1418 				    "found htable=%p shared=%p",
1419 				    (void *)ht, (void *)shared);
1420 			}
1421 			++ht->ht_busy;
1422 			HTABLE_EXIT(h);
1423 			if (new)
1424 				htable_free(new);
1425 			if (higher != NULL)
1426 				htable_release(higher);
1427 			higher = ht;
1428 
1429 		/*
1430 		 * if we didn't find it on the first search
1431 		 * allocate a new one and search again
1432 		 */
1433 		} else if (new == NULL) {
1434 			HTABLE_EXIT(h);
1435 			new = htable_alloc(hat, base, l,
1436 			    l == level ? shared : NULL);
1437 			goto try_again;
1438 
1439 		/*
1440 		 * 2nd search and still not there, use "new" table
1441 		 * Link new table into higher, when not at top level.
1442 		 */
1443 		} else {
1444 			ht = new;
1445 			if (higher != NULL) {
1446 				link_ptp(higher, ht, base);
1447 				ht->ht_parent = higher;
1448 			}
1449 			ht->ht_next = hat->hat_ht_hash[h];
1450 			ASSERT(ht->ht_prev == NULL);
1451 			if (hat->hat_ht_hash[h])
1452 				hat->hat_ht_hash[h]->ht_prev = ht;
1453 			hat->hat_ht_hash[h] = ht;
1454 			HTABLE_EXIT(h);
1455 
1456 			/*
1457 			 * Note we don't do htable_release(higher).
1458 			 * That happens recursively when "new" is removed by
1459 			 * htable_release() or htable_steal().
1460 			 */
1461 			higher = ht;
1462 
1463 			/*
1464 			 * If we just created a new shared page table we
1465 			 * increment the shared htable's busy count, so that
1466 			 * it can't be the victim of a steal even if it's empty.
1467 			 */
1468 			if (l == level && shared) {
1469 				(void) htable_lookup(shared->ht_hat,
1470 				    shared->ht_vaddr, shared->ht_level);
1471 				HATSTAT_INC(hs_htable_shared);
1472 			}
1473 		}
1474 	}
1475 
1476 	return (ht);
1477 }
1478 
1479 /*
1480  * Inherit initial pagetables from the boot program. On the 64-bit
1481  * hypervisor we also temporarily mark the p_index field of page table
1482  * pages, so we know not to try making them writable in seg_kpm.
1483  */
1484 void
htable_attach(hat_t * hat,uintptr_t base,level_t level,htable_t * parent,pfn_t pfn)1485 htable_attach(
1486 	hat_t *hat,
1487 	uintptr_t base,
1488 	level_t level,
1489 	htable_t *parent,
1490 	pfn_t pfn)
1491 {
1492 	htable_t	*ht;
1493 	uint_t		h;
1494 	uint_t		i;
1495 	x86pte_t	pte;
1496 	x86pte_t	*ptep;
1497 	page_t		*pp;
1498 	extern page_t	*boot_claim_page(pfn_t);
1499 
1500 	ht = htable_get_reserve();
1501 	if (level == mmu.max_level)
1502 		kas.a_hat->hat_htable = ht;
1503 	ht->ht_hat = hat;
1504 	ht->ht_parent = parent;
1505 	ht->ht_vaddr = base;
1506 	ht->ht_level = level;
1507 	ht->ht_busy = 1;
1508 	ht->ht_next = NULL;
1509 	ht->ht_prev = NULL;
1510 	ht->ht_flags = 0;
1511 	ht->ht_pfn = pfn;
1512 	ht->ht_lock_cnt = 0;
1513 	ht->ht_valid_cnt = 0;
1514 	if (parent != NULL)
1515 		++parent->ht_busy;
1516 
1517 	h = HTABLE_HASH(hat, base, level);
1518 	HTABLE_ENTER(h);
1519 	ht->ht_next = hat->hat_ht_hash[h];
1520 	ASSERT(ht->ht_prev == NULL);
1521 	if (hat->hat_ht_hash[h])
1522 		hat->hat_ht_hash[h]->ht_prev = ht;
1523 	hat->hat_ht_hash[h] = ht;
1524 	HTABLE_EXIT(h);
1525 
1526 	/*
1527 	 * make sure the page table physical page is not FREE
1528 	 */
1529 	if (page_resv(1, KM_NOSLEEP) == 0)
1530 		panic("page_resv() failed in ptable alloc");
1531 
1532 	pp = boot_claim_page(pfn);
1533 	ASSERT(pp != NULL);
1534 
1535 	/*
1536 	 * Page table pages that were allocated by dboot or
1537 	 * in very early startup didn't go through boot_mapin()
1538 	 * and so won't have vnode/offsets. Fix that here.
1539 	 */
1540 	if (pp->p_vnode == NULL) {
1541 		/* match offset calculation in page_get_physical() */
1542 		u_offset_t offset = (uintptr_t)ht;
1543 		if (offset > kernelbase)
1544 			offset -= kernelbase;
1545 		offset <<= MMU_PAGESHIFT;
1546 		offset += mmu.hole_start;	/* something in VA hole */
1547 		ASSERT(page_exists(&kvp, offset) == NULL);
1548 		(void) page_hashin(pp, &kvp, offset, NULL);
1549 	}
1550 	page_downgrade(pp);
1551 #if defined(__xpv)
1552 	/*
1553 	 * Record in the page_t that is a pagetable for segkpm setup.
1554 	 */
1555 	if (kpm_vbase)
1556 		pp->p_index = 1;
1557 #endif
1558 
1559 	/*
1560 	 * Count valid mappings and recursively attach lower level pagetables.
1561 	 */
1562 	ptep = kbm_remap_window(pfn_to_pa(pfn), 0);
1563 	for (i = 0; i < HTABLE_NUM_PTES(ht); ++i) {
1564 		if (mmu.pae_hat)
1565 			pte = ptep[i];
1566 		else
1567 			pte = ((x86pte32_t *)ptep)[i];
1568 		if (!IN_HYPERVISOR_VA(base) && PTE_ISVALID(pte)) {
1569 			++ht->ht_valid_cnt;
1570 			if (!PTE_ISPAGE(pte, level)) {
1571 				htable_attach(hat, base, level - 1,
1572 				    ht, PTE2PFN(pte, level));
1573 				ptep = kbm_remap_window(pfn_to_pa(pfn), 0);
1574 			}
1575 		}
1576 		base += LEVEL_SIZE(level);
1577 		if (base == mmu.hole_start)
1578 			base = (mmu.hole_end + MMU_PAGEOFFSET) & MMU_PAGEMASK;
1579 	}
1580 
1581 	/*
1582 	 * As long as all the mappings we had were below kernel base
1583 	 * we can release the htable.
1584 	 */
1585 	if (base < kernelbase)
1586 		htable_release(ht);
1587 }
1588 
1589 /*
1590  * Walk through a given htable looking for the first valid entry.  This
1591  * routine takes both a starting and ending address.  The starting address
1592  * is required to be within the htable provided by the caller, but there is
1593  * no such restriction on the ending address.
1594  *
1595  * If the routine finds a valid entry in the htable (at or beyond the
1596  * starting address), the PTE (and its address) will be returned.
1597  * This PTE may correspond to either a page or a pagetable - it is the
1598  * caller's responsibility to determine which.  If no valid entry is
1599  * found, 0 (and invalid PTE) and the next unexamined address will be
1600  * returned.
1601  *
1602  * The loop has been carefully coded for optimization.
1603  */
1604 static x86pte_t
htable_scan(htable_t * ht,uintptr_t * vap,uintptr_t eaddr)1605 htable_scan(htable_t *ht, uintptr_t *vap, uintptr_t eaddr)
1606 {
1607 	uint_t e;
1608 	x86pte_t found_pte = (x86pte_t)0;
1609 	caddr_t pte_ptr;
1610 	caddr_t end_pte_ptr;
1611 	int l = ht->ht_level;
1612 	uintptr_t va = *vap & LEVEL_MASK(l);
1613 	size_t pgsize = LEVEL_SIZE(l);
1614 
1615 	ASSERT(va >= ht->ht_vaddr);
1616 	ASSERT(va <= HTABLE_LAST_PAGE(ht));
1617 
1618 	/*
1619 	 * Compute the starting index and ending virtual address
1620 	 */
1621 	e = htable_va2entry(va, ht);
1622 
1623 	/*
1624 	 * The following page table scan code knows that the valid
1625 	 * bit of a PTE is in the lowest byte AND that x86 is little endian!!
1626 	 */
1627 	pte_ptr = (caddr_t)x86pte_access_pagetable(ht, 0);
1628 	end_pte_ptr = (caddr_t)PT_INDEX_PTR(pte_ptr, HTABLE_NUM_PTES(ht));
1629 	pte_ptr = (caddr_t)PT_INDEX_PTR((x86pte_t *)pte_ptr, e);
1630 	while (!PTE_ISVALID(*pte_ptr)) {
1631 		va += pgsize;
1632 		if (va >= eaddr)
1633 			break;
1634 		pte_ptr += mmu.pte_size;
1635 		ASSERT(pte_ptr <= end_pte_ptr);
1636 		if (pte_ptr == end_pte_ptr)
1637 			break;
1638 	}
1639 
1640 	/*
1641 	 * if we found a valid PTE, load the entire PTE
1642 	 */
1643 	if (va < eaddr && pte_ptr != end_pte_ptr)
1644 		found_pte = GET_PTE((x86pte_t *)pte_ptr);
1645 	x86pte_release_pagetable(ht);
1646 
1647 	/*
1648 	 * deal with VA hole on amd64
1649 	 */
1650 	if (l == mmu.max_level && va >= mmu.hole_start && va <= mmu.hole_end)
1651 		va = mmu.hole_end + va - mmu.hole_start;
1652 
1653 	*vap = va;
1654 	return (found_pte);
1655 }
1656 
1657 /*
1658  * Find the address and htable for the first populated translation at or
1659  * above the given virtual address.  The caller may also specify an upper
1660  * limit to the address range to search.  Uses level information to quickly
1661  * skip unpopulated sections of virtual address spaces.
1662  *
1663  * If not found returns NULL. When found, returns the htable and virt addr
1664  * and has a hold on the htable.
1665  */
1666 x86pte_t
htable_walk(struct hat * hat,htable_t ** htp,uintptr_t * vaddr,uintptr_t eaddr)1667 htable_walk(
1668 	struct hat *hat,
1669 	htable_t **htp,
1670 	uintptr_t *vaddr,
1671 	uintptr_t eaddr)
1672 {
1673 	uintptr_t va = *vaddr;
1674 	htable_t *ht;
1675 	htable_t *prev = *htp;
1676 	level_t l;
1677 	level_t max_mapped_level;
1678 	x86pte_t pte;
1679 
1680 	ASSERT(eaddr > va);
1681 
1682 	/*
1683 	 * If this is a user address, then we know we need not look beyond
1684 	 * kernelbase.
1685 	 */
1686 	ASSERT(hat == kas.a_hat || eaddr <= kernelbase ||
1687 	    eaddr == HTABLE_WALK_TO_END);
1688 	if (hat != kas.a_hat && eaddr == HTABLE_WALK_TO_END)
1689 		eaddr = kernelbase;
1690 
1691 	/*
1692 	 * If we're coming in with a previous page table, search it first
1693 	 * without doing an htable_lookup(), this should be frequent.
1694 	 */
1695 	if (prev) {
1696 		ASSERT(prev->ht_busy > 0);
1697 		ASSERT(prev->ht_vaddr <= va);
1698 		l = prev->ht_level;
1699 		if (va <= HTABLE_LAST_PAGE(prev)) {
1700 			pte = htable_scan(prev, &va, eaddr);
1701 
1702 			if (PTE_ISPAGE(pte, l)) {
1703 				*vaddr = va;
1704 				*htp = prev;
1705 				return (pte);
1706 			}
1707 		}
1708 
1709 		/*
1710 		 * We found nothing in the htable provided by the caller,
1711 		 * so fall through and do the full search
1712 		 */
1713 		htable_release(prev);
1714 	}
1715 
1716 	/*
1717 	 * Find the level of the largest pagesize used by this HAT.
1718 	 */
1719 	if (hat->hat_ism_pgcnt > 0) {
1720 		max_mapped_level = mmu.umax_page_level;
1721 	} else {
1722 		max_mapped_level = 0;
1723 		for (l = 1; l <= mmu.max_page_level; ++l)
1724 			if (hat->hat_pages_mapped[l] != 0)
1725 				max_mapped_level = l;
1726 	}
1727 
1728 	while (va < eaddr && va >= *vaddr) {
1729 		/*
1730 		 *  Find lowest table with any entry for given address.
1731 		 */
1732 		for (l = 0; l <= TOP_LEVEL(hat); ++l) {
1733 			ht = htable_lookup(hat, va, l);
1734 			if (ht != NULL) {
1735 				pte = htable_scan(ht, &va, eaddr);
1736 				if (PTE_ISPAGE(pte, l)) {
1737 					VERIFY(!IN_VA_HOLE(va));
1738 					*vaddr = va;
1739 					*htp = ht;
1740 					return (pte);
1741 				}
1742 				htable_release(ht);
1743 				break;
1744 			}
1745 
1746 			/*
1747 			 * No htable at this level for the address. If there
1748 			 * is no larger page size that could cover it, we can
1749 			 * skip right to the start of the next page table.
1750 			 */
1751 			ASSERT(l < TOP_LEVEL(hat));
1752 			if (l >= max_mapped_level) {
1753 				va = NEXT_ENTRY_VA(va, l + 1);
1754 				if (va >= eaddr)
1755 					break;
1756 			}
1757 		}
1758 	}
1759 
1760 	*vaddr = 0;
1761 	*htp = NULL;
1762 	return (0);
1763 }
1764 
1765 /*
1766  * Find the htable and page table entry index of the given virtual address
1767  * with pagesize at or below given level.
1768  * If not found returns NULL. When found, returns the htable, sets
1769  * entry, and has a hold on the htable.
1770  */
1771 htable_t *
htable_getpte(struct hat * hat,uintptr_t vaddr,uint_t * entry,x86pte_t * pte,level_t level)1772 htable_getpte(
1773 	struct hat *hat,
1774 	uintptr_t vaddr,
1775 	uint_t *entry,
1776 	x86pte_t *pte,
1777 	level_t level)
1778 {
1779 	htable_t	*ht;
1780 	level_t		l;
1781 	uint_t		e;
1782 
1783 	ASSERT(level <= mmu.max_page_level);
1784 
1785 	for (l = 0; l <= level; ++l) {
1786 		ht = htable_lookup(hat, vaddr, l);
1787 		if (ht == NULL)
1788 			continue;
1789 		e = htable_va2entry(vaddr, ht);
1790 		if (entry != NULL)
1791 			*entry = e;
1792 		if (pte != NULL)
1793 			*pte = x86pte_get(ht, e);
1794 		return (ht);
1795 	}
1796 	return (NULL);
1797 }
1798 
1799 /*
1800  * Find the htable and page table entry index of the given virtual address.
1801  * There must be a valid page mapped at the given address.
1802  * If not found returns NULL. When found, returns the htable, sets
1803  * entry, and has a hold on the htable.
1804  */
1805 htable_t *
htable_getpage(struct hat * hat,uintptr_t vaddr,uint_t * entry)1806 htable_getpage(struct hat *hat, uintptr_t vaddr, uint_t *entry)
1807 {
1808 	htable_t	*ht;
1809 	uint_t		e;
1810 	x86pte_t	pte;
1811 
1812 	ht = htable_getpte(hat, vaddr, &e, &pte, mmu.max_page_level);
1813 	if (ht == NULL)
1814 		return (NULL);
1815 
1816 	if (entry)
1817 		*entry = e;
1818 
1819 	if (PTE_ISPAGE(pte, ht->ht_level))
1820 		return (ht);
1821 	htable_release(ht);
1822 	return (NULL);
1823 }
1824 
1825 
1826 void
htable_init()1827 htable_init()
1828 {
1829 	/*
1830 	 * To save on kernel VA usage, we avoid debug information in 32 bit
1831 	 * kernels.
1832 	 */
1833 	int	kmem_flags = KMC_NOHASH;
1834 
1835 	/*
1836 	 * initialize kmem caches
1837 	 */
1838 	htable_cache = kmem_cache_create("htable_t",
1839 	    sizeof (htable_t), 0, NULL, NULL,
1840 	    htable_reap, NULL, hat_memload_arena, kmem_flags);
1841 }
1842 
1843 /*
1844  * get the pte index for the virtual address in the given htable's pagetable
1845  */
1846 uint_t
htable_va2entry(uintptr_t va,htable_t * ht)1847 htable_va2entry(uintptr_t va, htable_t *ht)
1848 {
1849 	level_t	l = ht->ht_level;
1850 
1851 	ASSERT(va >= ht->ht_vaddr);
1852 	ASSERT(va <= HTABLE_LAST_PAGE(ht));
1853 	return ((va >> LEVEL_SHIFT(l)) & (HTABLE_NUM_PTES(ht) - 1));
1854 }
1855 
1856 /*
1857  * Given an htable and the index of a pte in it, return the virtual address
1858  * of the page.
1859  */
1860 uintptr_t
htable_e2va(htable_t * ht,uint_t entry)1861 htable_e2va(htable_t *ht, uint_t entry)
1862 {
1863 	level_t	l = ht->ht_level;
1864 	uintptr_t va;
1865 
1866 	ASSERT(entry < HTABLE_NUM_PTES(ht));
1867 	va = ht->ht_vaddr + ((uintptr_t)entry << LEVEL_SHIFT(l));
1868 
1869 	/*
1870 	 * Need to skip over any VA hole in top level table
1871 	 */
1872 	if (ht->ht_level == mmu.max_level && va >= mmu.hole_start)
1873 		va += ((mmu.hole_end - mmu.hole_start) + 1);
1874 
1875 	return (va);
1876 }
1877 
1878 /*
1879  * The code uses compare and swap instructions to read/write PTE's to
1880  * avoid atomicity problems, since PTEs can be 8 bytes on 32 bit systems.
1881  * will naturally be atomic.
1882  *
1883  * The combination of using kpreempt_disable()/_enable() and the hci_mutex
1884  * are used to ensure that an interrupt won't overwrite a temporary mapping
1885  * while it's in use. If an interrupt thread tries to access a PTE, it will
1886  * yield briefly back to the pinned thread which holds the cpu's hci_mutex.
1887  */
1888 void
x86pte_cpu_init(cpu_t * cpu)1889 x86pte_cpu_init(cpu_t *cpu)
1890 {
1891 	struct hat_cpu_info *hci;
1892 
1893 	hci = kmem_zalloc(sizeof (*hci), KM_SLEEP);
1894 	mutex_init(&hci->hci_mutex, NULL, MUTEX_DEFAULT, NULL);
1895 	cpu->cpu_hat_info = hci;
1896 }
1897 
1898 void
x86pte_cpu_fini(cpu_t * cpu)1899 x86pte_cpu_fini(cpu_t *cpu)
1900 {
1901 	struct hat_cpu_info *hci = cpu->cpu_hat_info;
1902 
1903 	kmem_free(hci, sizeof (*hci));
1904 	cpu->cpu_hat_info = NULL;
1905 }
1906 
1907 /*
1908  * Disable preemption and establish a mapping to the pagetable with the
1909  * given pfn. This is optimized for there case where it's the same
1910  * pfn as we last used referenced from this CPU.
1911  */
1912 static x86pte_t *
x86pte_access_pagetable(htable_t * ht,uint_t index)1913 x86pte_access_pagetable(htable_t *ht, uint_t index)
1914 {
1915 	/*
1916 	 * HTABLE_COPIED pagetables are contained in the hat_t
1917 	 */
1918 	if (ht->ht_flags & HTABLE_COPIED) {
1919 		ASSERT3U(index, <, ht->ht_hat->hat_num_copied);
1920 		return (PT_INDEX_PTR(ht->ht_hat->hat_copied_ptes, index));
1921 	}
1922 	return (x86pte_mapin(ht->ht_pfn, index, ht));
1923 }
1924 
1925 /*
1926  * map the given pfn into the page table window.
1927  */
1928 /*ARGSUSED*/
1929 x86pte_t *
x86pte_mapin(pfn_t pfn,uint_t index,htable_t * ht)1930 x86pte_mapin(pfn_t pfn, uint_t index, htable_t *ht)
1931 {
1932 	x86pte_t *pteptr;
1933 	x86pte_t pte = 0;
1934 	x86pte_t newpte;
1935 	int x;
1936 
1937 	ASSERT(pfn != PFN_INVALID);
1938 
1939 	if (!khat_running) {
1940 		caddr_t va = kbm_remap_window(pfn_to_pa(pfn), 1);
1941 		return (PT_INDEX_PTR(va, index));
1942 	}
1943 
1944 	/*
1945 	 * If kpm is available, use it.
1946 	 */
1947 	if (kpm_vbase)
1948 		return (PT_INDEX_PTR(hat_kpm_pfn2va(pfn), index));
1949 
1950 	/*
1951 	 * Disable preemption and grab the CPU's hci_mutex
1952 	 */
1953 	kpreempt_disable();
1954 
1955 	ASSERT(CPU->cpu_hat_info != NULL);
1956 	ASSERT(!(getcr4() & CR4_PCIDE));
1957 
1958 	mutex_enter(&CPU->cpu_hat_info->hci_mutex);
1959 	x = PWIN_TABLE(CPU->cpu_id);
1960 	pteptr = (x86pte_t *)PWIN_PTE_VA(x);
1961 #ifndef __xpv
1962 	if (mmu.pae_hat)
1963 		pte = *pteptr;
1964 	else
1965 		pte = *(x86pte32_t *)pteptr;
1966 #endif
1967 
1968 	newpte = MAKEPTE(pfn, 0) | mmu.pt_global | mmu.pt_nx;
1969 
1970 	/*
1971 	 * For hardware we can use a writable mapping.
1972 	 */
1973 #ifdef __xpv
1974 	if (IN_XPV_PANIC())
1975 #endif
1976 		newpte |= PT_WRITABLE;
1977 
1978 	if (!PTE_EQUIV(newpte, pte)) {
1979 
1980 #ifdef __xpv
1981 		if (!IN_XPV_PANIC()) {
1982 			xen_map(newpte, PWIN_VA(x));
1983 		} else
1984 #endif
1985 		{
1986 			XPV_ALLOW_PAGETABLE_UPDATES();
1987 			if (mmu.pae_hat)
1988 				*pteptr = newpte;
1989 			else
1990 				*(x86pte32_t *)pteptr = newpte;
1991 			XPV_DISALLOW_PAGETABLE_UPDATES();
1992 			mmu_flush_tlb_kpage((uintptr_t)PWIN_VA(x));
1993 		}
1994 	}
1995 	return (PT_INDEX_PTR(PWIN_VA(x), index));
1996 }
1997 
1998 /*
1999  * Release access to a page table.
2000  */
2001 static void
x86pte_release_pagetable(htable_t * ht)2002 x86pte_release_pagetable(htable_t *ht)
2003 {
2004 	if (ht->ht_flags & HTABLE_COPIED)
2005 		return;
2006 
2007 	x86pte_mapout();
2008 }
2009 
2010 void
x86pte_mapout(void)2011 x86pte_mapout(void)
2012 {
2013 	if (kpm_vbase != NULL || !khat_running)
2014 		return;
2015 
2016 	/*
2017 	 * Drop the CPU's hci_mutex and restore preemption.
2018 	 */
2019 #ifdef __xpv
2020 	if (!IN_XPV_PANIC()) {
2021 		uintptr_t va;
2022 
2023 		/*
2024 		 * We need to always clear the mapping in case a page
2025 		 * that was once a page table page is ballooned out.
2026 		 */
2027 		va = (uintptr_t)PWIN_VA(PWIN_TABLE(CPU->cpu_id));
2028 		(void) HYPERVISOR_update_va_mapping(va, 0,
2029 		    UVMF_INVLPG | UVMF_LOCAL);
2030 	}
2031 #endif
2032 	mutex_exit(&CPU->cpu_hat_info->hci_mutex);
2033 	kpreempt_enable();
2034 }
2035 
2036 /*
2037  * Atomic retrieval of a pagetable entry
2038  */
2039 x86pte_t
x86pte_get(htable_t * ht,uint_t entry)2040 x86pte_get(htable_t *ht, uint_t entry)
2041 {
2042 	x86pte_t	pte;
2043 	x86pte_t	*ptep;
2044 
2045 	/*
2046 	 * Be careful that loading PAE entries in 32 bit kernel is atomic.
2047 	 */
2048 	ASSERT(entry < mmu.ptes_per_table);
2049 	ptep = x86pte_access_pagetable(ht, entry);
2050 	pte = GET_PTE(ptep);
2051 	x86pte_release_pagetable(ht);
2052 	return (pte);
2053 }
2054 
2055 /*
2056  * Atomic unconditional set of a page table entry, it returns the previous
2057  * value. For pre-existing mappings if the PFN changes, then we don't care
2058  * about the old pte's REF / MOD bits. If the PFN remains the same, we leave
2059  * the MOD/REF bits unchanged.
2060  *
2061  * If asked to overwrite a link to a lower page table with a large page
2062  * mapping, this routine returns the special value of LPAGE_ERROR. This
2063  * allows the upper HAT layers to retry with a smaller mapping size.
2064  */
2065 x86pte_t
x86pte_set(htable_t * ht,uint_t entry,x86pte_t new,void * ptr)2066 x86pte_set(htable_t *ht, uint_t entry, x86pte_t new, void *ptr)
2067 {
2068 	x86pte_t	old;
2069 	x86pte_t	prev;
2070 	x86pte_t	*ptep;
2071 	level_t		l = ht->ht_level;
2072 	x86pte_t	pfn_mask = (l != 0) ? PT_PADDR_LGPG : PT_PADDR;
2073 	x86pte_t	n;
2074 	uintptr_t	addr = htable_e2va(ht, entry);
2075 	hat_t		*hat = ht->ht_hat;
2076 
2077 	ASSERT(new != 0); /* don't use to invalidate a PTE, see x86pte_update */
2078 	ASSERT(!(ht->ht_flags & HTABLE_SHARED_PFN));
2079 	if (ptr == NULL)
2080 		ptep = x86pte_access_pagetable(ht, entry);
2081 	else
2082 		ptep = ptr;
2083 
2084 	/*
2085 	 * Install the new PTE. If remapping the same PFN, then
2086 	 * copy existing REF/MOD bits to new mapping.
2087 	 */
2088 	do {
2089 		prev = GET_PTE(ptep);
2090 		n = new;
2091 		if (PTE_ISVALID(n) && (prev & pfn_mask) == (new & pfn_mask))
2092 			n |= prev & (PT_REF | PT_MOD);
2093 
2094 		/*
2095 		 * Another thread may have installed this mapping already,
2096 		 * flush the local TLB and be done.
2097 		 */
2098 		if (prev == n) {
2099 			old = new;
2100 #ifdef __xpv
2101 			if (!IN_XPV_PANIC())
2102 				xen_flush_va((caddr_t)addr);
2103 			else
2104 #endif
2105 				mmu_flush_tlb_page(addr);
2106 			goto done;
2107 		}
2108 
2109 		/*
2110 		 * Detect if we have a collision of installing a large
2111 		 * page mapping where there already is a lower page table.
2112 		 */
2113 		if (l > 0 && (prev & PT_VALID) && !(prev & PT_PAGESIZE)) {
2114 			old = LPAGE_ERROR;
2115 			goto done;
2116 		}
2117 
2118 		XPV_ALLOW_PAGETABLE_UPDATES();
2119 		old = CAS_PTE(ptep, prev, n);
2120 		XPV_DISALLOW_PAGETABLE_UPDATES();
2121 	} while (old != prev);
2122 
2123 	/*
2124 	 * Do a TLB demap if needed, ie. the old pte was valid.
2125 	 *
2126 	 * Note that a stale TLB writeback to the PTE here either can't happen
2127 	 * or doesn't matter. The PFN can only change for NOSYNC|NOCONSIST
2128 	 * mappings, but they were created with REF and MOD already set, so
2129 	 * no stale writeback will happen.
2130 	 *
2131 	 * Segmap is the only place where remaps happen on the same pfn and for
2132 	 * that we want to preserve the stale REF/MOD bits.
2133 	 */
2134 	if (old & PT_REF)
2135 		hat_tlb_inval(hat, addr);
2136 
2137 done:
2138 	if (ptr == NULL)
2139 		x86pte_release_pagetable(ht);
2140 	return (old);
2141 }
2142 
2143 /*
2144  * Atomic compare and swap of a page table entry. No TLB invalidates are done.
2145  * This is used for links between pagetables of different levels.
2146  * Note we always create these links with dirty/access set, so they should
2147  * never change.
2148  */
2149 x86pte_t
x86pte_cas(htable_t * ht,uint_t entry,x86pte_t old,x86pte_t new)2150 x86pte_cas(htable_t *ht, uint_t entry, x86pte_t old, x86pte_t new)
2151 {
2152 	x86pte_t	pte;
2153 	x86pte_t	*ptep;
2154 #ifdef __xpv
2155 	/*
2156 	 * We can't use writable pagetables for upper level tables, so fake it.
2157 	 */
2158 	mmu_update_t t[2];
2159 	int cnt = 1;
2160 	int count;
2161 	maddr_t ma;
2162 
2163 	if (!IN_XPV_PANIC()) {
2164 		ASSERT(!(ht->ht_flags & HTABLE_COPIED));
2165 		ma = pa_to_ma(PT_INDEX_PHYSADDR(pfn_to_pa(ht->ht_pfn), entry));
2166 		t[0].ptr = ma | MMU_NORMAL_PT_UPDATE;
2167 		t[0].val = new;
2168 
2169 		/*
2170 		 * On the 64-bit hypervisor we need to maintain the user mode
2171 		 * top page table too.
2172 		 */
2173 		if (ht->ht_level == mmu.max_level && ht->ht_hat != kas.a_hat) {
2174 			ma = pa_to_ma(PT_INDEX_PHYSADDR(pfn_to_pa(
2175 			    ht->ht_hat->hat_user_ptable), entry));
2176 			t[1].ptr = ma | MMU_NORMAL_PT_UPDATE;
2177 			t[1].val = new;
2178 			++cnt;
2179 		}
2180 
2181 		if (HYPERVISOR_mmu_update(t, cnt, &count, DOMID_SELF))
2182 			panic("HYPERVISOR_mmu_update() failed");
2183 		ASSERT(count == cnt);
2184 		return (old);
2185 	}
2186 #endif
2187 	ptep = x86pte_access_pagetable(ht, entry);
2188 	XPV_ALLOW_PAGETABLE_UPDATES();
2189 	pte = CAS_PTE(ptep, old, new);
2190 	XPV_DISALLOW_PAGETABLE_UPDATES();
2191 	x86pte_release_pagetable(ht);
2192 	return (pte);
2193 }
2194 
2195 /*
2196  * Invalidate a page table entry as long as it currently maps something that
2197  * matches the value determined by expect.
2198  *
2199  * If tlb is set, also invalidates any TLB entries.
2200  *
2201  * Returns the previous value of the PTE.
2202  */
2203 x86pte_t
x86pte_inval(htable_t * ht,uint_t entry,x86pte_t expect,x86pte_t * pte_ptr,boolean_t tlb)2204 x86pte_inval(
2205 	htable_t *ht,
2206 	uint_t entry,
2207 	x86pte_t expect,
2208 	x86pte_t *pte_ptr,
2209 	boolean_t tlb)
2210 {
2211 	x86pte_t	*ptep;
2212 	x86pte_t	oldpte;
2213 	x86pte_t	found;
2214 
2215 	ASSERT(!(ht->ht_flags & HTABLE_SHARED_PFN));
2216 	ASSERT(ht->ht_level <= mmu.max_page_level);
2217 
2218 	if (pte_ptr != NULL)
2219 		ptep = pte_ptr;
2220 	else
2221 		ptep = x86pte_access_pagetable(ht, entry);
2222 
2223 #if defined(__xpv)
2224 	/*
2225 	 * If exit()ing just use HYPERVISOR_mmu_update(), as we can't be racing
2226 	 * with anything else.
2227 	 */
2228 	if ((ht->ht_hat->hat_flags & HAT_FREEING) && !IN_XPV_PANIC()) {
2229 		int count;
2230 		mmu_update_t t[1];
2231 		maddr_t ma;
2232 
2233 		oldpte = GET_PTE(ptep);
2234 		if (expect != 0 && (oldpte & PT_PADDR) != (expect & PT_PADDR))
2235 			goto done;
2236 		ma = pa_to_ma(PT_INDEX_PHYSADDR(pfn_to_pa(ht->ht_pfn), entry));
2237 		t[0].ptr = ma | MMU_NORMAL_PT_UPDATE;
2238 		t[0].val = 0;
2239 		if (HYPERVISOR_mmu_update(t, 1, &count, DOMID_SELF))
2240 			panic("HYPERVISOR_mmu_update() failed");
2241 		ASSERT(count == 1);
2242 		goto done;
2243 	}
2244 #endif /* __xpv */
2245 
2246 	/*
2247 	 * Note that the loop is needed to handle changes due to h/w updating
2248 	 * of PT_MOD/PT_REF.
2249 	 */
2250 	do {
2251 		oldpte = GET_PTE(ptep);
2252 		if (expect != 0 && (oldpte & PT_PADDR) != (expect & PT_PADDR))
2253 			goto done;
2254 		XPV_ALLOW_PAGETABLE_UPDATES();
2255 		found = CAS_PTE(ptep, oldpte, 0);
2256 		XPV_DISALLOW_PAGETABLE_UPDATES();
2257 	} while (found != oldpte);
2258 	if (tlb && (oldpte & (PT_REF | PT_MOD)))
2259 		hat_tlb_inval(ht->ht_hat, htable_e2va(ht, entry));
2260 
2261 done:
2262 	if (pte_ptr == NULL)
2263 		x86pte_release_pagetable(ht);
2264 	return (oldpte);
2265 }
2266 
2267 /*
2268  * Change a page table entry af it currently matches the value in expect.
2269  */
2270 x86pte_t
x86pte_update(htable_t * ht,uint_t entry,x86pte_t expect,x86pte_t new)2271 x86pte_update(
2272 	htable_t *ht,
2273 	uint_t entry,
2274 	x86pte_t expect,
2275 	x86pte_t new)
2276 {
2277 	x86pte_t	*ptep;
2278 	x86pte_t	found;
2279 
2280 	ASSERT(new != 0);
2281 	ASSERT(!(ht->ht_flags & HTABLE_SHARED_PFN));
2282 	ASSERT(ht->ht_level <= mmu.max_page_level);
2283 
2284 	ptep = x86pte_access_pagetable(ht, entry);
2285 	XPV_ALLOW_PAGETABLE_UPDATES();
2286 	found = CAS_PTE(ptep, expect, new);
2287 	XPV_DISALLOW_PAGETABLE_UPDATES();
2288 	if (found == expect) {
2289 		hat_tlb_inval(ht->ht_hat, htable_e2va(ht, entry));
2290 
2291 		/*
2292 		 * When removing write permission *and* clearing the
2293 		 * MOD bit, check if a write happened via a stale
2294 		 * TLB entry before the TLB shootdown finished.
2295 		 *
2296 		 * If it did happen, simply re-enable write permission and
2297 		 * act like the original CAS failed.
2298 		 */
2299 		if ((expect & (PT_WRITABLE | PT_MOD)) == PT_WRITABLE &&
2300 		    (new & (PT_WRITABLE | PT_MOD)) == 0 &&
2301 		    (GET_PTE(ptep) & PT_MOD) != 0) {
2302 			do {
2303 				found = GET_PTE(ptep);
2304 				XPV_ALLOW_PAGETABLE_UPDATES();
2305 				found =
2306 				    CAS_PTE(ptep, found, found | PT_WRITABLE);
2307 				XPV_DISALLOW_PAGETABLE_UPDATES();
2308 			} while ((found & PT_WRITABLE) == 0);
2309 		}
2310 	}
2311 	x86pte_release_pagetable(ht);
2312 	return (found);
2313 }
2314 
2315 #ifndef __xpv
2316 /*
2317  * Copy page tables - this is just a little more complicated than the
2318  * previous routines. Note that it's also not atomic! It also is never
2319  * used for HTABLE_COPIED pagetables.
2320  */
2321 void
x86pte_copy(htable_t * src,htable_t * dest,uint_t entry,uint_t count)2322 x86pte_copy(htable_t *src, htable_t *dest, uint_t entry, uint_t count)
2323 {
2324 	caddr_t	src_va;
2325 	caddr_t dst_va;
2326 	size_t size;
2327 	x86pte_t *pteptr;
2328 	x86pte_t pte;
2329 
2330 	ASSERT(khat_running);
2331 	ASSERT(!(dest->ht_flags & HTABLE_COPIED));
2332 	ASSERT(!(src->ht_flags & HTABLE_COPIED));
2333 	ASSERT(!(src->ht_flags & HTABLE_SHARED_PFN));
2334 	ASSERT(!(dest->ht_flags & HTABLE_SHARED_PFN));
2335 
2336 	/*
2337 	 * Acquire access to the CPU pagetable windows for the dest and source.
2338 	 */
2339 	dst_va = (caddr_t)x86pte_access_pagetable(dest, entry);
2340 	if (kpm_vbase) {
2341 		src_va = (caddr_t)
2342 		    PT_INDEX_PTR(hat_kpm_pfn2va(src->ht_pfn), entry);
2343 	} else {
2344 		uint_t x = PWIN_SRC(CPU->cpu_id);
2345 
2346 		ASSERT(!(getcr4() & CR4_PCIDE));
2347 
2348 		/*
2349 		 * Finish defining the src pagetable mapping
2350 		 */
2351 		src_va = (caddr_t)PT_INDEX_PTR(PWIN_VA(x), entry);
2352 		pte = MAKEPTE(src->ht_pfn, 0) | mmu.pt_global | mmu.pt_nx;
2353 		pteptr = (x86pte_t *)PWIN_PTE_VA(x);
2354 		if (mmu.pae_hat)
2355 			*pteptr = pte;
2356 		else
2357 			*(x86pte32_t *)pteptr = pte;
2358 		mmu_flush_tlb_kpage((uintptr_t)PWIN_VA(x));
2359 	}
2360 
2361 	/*
2362 	 * now do the copy
2363 	 */
2364 	size = count << mmu.pte_size_shift;
2365 	bcopy(src_va, dst_va, size);
2366 
2367 	x86pte_release_pagetable(dest);
2368 }
2369 
2370 #else /* __xpv */
2371 
2372 /*
2373  * The hypervisor only supports writable pagetables at level 0, so we have
2374  * to install these 1 by 1 the slow way.
2375  */
2376 void
x86pte_copy(htable_t * src,htable_t * dest,uint_t entry,uint_t count)2377 x86pte_copy(htable_t *src, htable_t *dest, uint_t entry, uint_t count)
2378 {
2379 	caddr_t	src_va;
2380 	x86pte_t pte;
2381 
2382 	ASSERT(!IN_XPV_PANIC());
2383 	src_va = (caddr_t)x86pte_access_pagetable(src, entry);
2384 	while (count) {
2385 		if (mmu.pae_hat)
2386 			pte = *(x86pte_t *)src_va;
2387 		else
2388 			pte = *(x86pte32_t *)src_va;
2389 		if (pte != 0) {
2390 			set_pteval(pfn_to_pa(dest->ht_pfn), entry,
2391 			    dest->ht_level, pte);
2392 			if (dest->ht_level == mmu.max_level &&
2393 			    htable_e2va(dest, entry) < HYPERVISOR_VIRT_END)
2394 				set_pteval(
2395 				    pfn_to_pa(dest->ht_hat->hat_user_ptable),
2396 				    entry, dest->ht_level, pte);
2397 		}
2398 		--count;
2399 		++entry;
2400 		src_va += mmu.pte_size;
2401 	}
2402 	x86pte_release_pagetable(src);
2403 }
2404 #endif /* __xpv */
2405 
2406 /*
2407  * Zero page table entries - Note this doesn't use atomic stores!
2408  */
2409 static void
x86pte_zero(htable_t * dest,uint_t entry,uint_t count)2410 x86pte_zero(htable_t *dest, uint_t entry, uint_t count)
2411 {
2412 	caddr_t dst_va;
2413 	size_t size;
2414 #ifdef __xpv
2415 	int x = 0;
2416 	x86pte_t newpte;
2417 #endif
2418 
2419 	/*
2420 	 * Map in the page table to be zeroed.
2421 	 */
2422 	ASSERT(!(dest->ht_flags & HTABLE_SHARED_PFN));
2423 	ASSERT(!(dest->ht_flags & HTABLE_COPIED));
2424 
2425 	/*
2426 	 * On the hypervisor we don't use x86pte_access_pagetable() since
2427 	 * in this case the page is not pinned yet.
2428 	 */
2429 #ifdef __xpv
2430 	if (kpm_vbase == NULL) {
2431 		kpreempt_disable();
2432 		ASSERT(CPU->cpu_hat_info != NULL);
2433 		mutex_enter(&CPU->cpu_hat_info->hci_mutex);
2434 		x = PWIN_TABLE(CPU->cpu_id);
2435 		newpte = MAKEPTE(dest->ht_pfn, 0) | PT_WRITABLE;
2436 		xen_map(newpte, PWIN_VA(x));
2437 		dst_va = (caddr_t)PT_INDEX_PTR(PWIN_VA(x), entry);
2438 	} else
2439 #endif
2440 		dst_va = (caddr_t)x86pte_access_pagetable(dest, entry);
2441 
2442 	size = count << mmu.pte_size_shift;
2443 	ASSERT(size > BLOCKZEROALIGN);
2444 	block_zero_no_xmm(dst_va, size);
2445 
2446 #ifdef __xpv
2447 	if (kpm_vbase == NULL) {
2448 		xen_map(0, PWIN_VA(x));
2449 		mutex_exit(&CPU->cpu_hat_info->hci_mutex);
2450 		kpreempt_enable();
2451 	} else
2452 #endif
2453 		x86pte_release_pagetable(dest);
2454 }
2455 
2456 /*
2457  * Called to ensure that all pagetables are in the system dump
2458  */
2459 void
hat_dump(void)2460 hat_dump(void)
2461 {
2462 	hat_t *hat;
2463 	uint_t h;
2464 	htable_t *ht;
2465 
2466 	/*
2467 	 * Dump all page tables
2468 	 */
2469 	for (hat = kas.a_hat; hat != NULL; hat = hat->hat_next) {
2470 		for (h = 0; h < hat->hat_num_hash; ++h) {
2471 			for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) {
2472 				if ((ht->ht_flags & HTABLE_COPIED) == 0)
2473 					dump_page(ht->ht_pfn);
2474 			}
2475 		}
2476 	}
2477 }
2478