xref: /illumos-gate/usr/src/uts/i86pc/vm/hat_i86.c (revision aa2ed9e57a76431fbe2ba230496f4fcd22b2d41d)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 /*
30  * VM - Hardware Address Translation management for i386 and amd64
31  *
32  * Implementation of the interfaces described in <common/vm/hat.h>
33  *
34  * Nearly all the details of how the hardware is managed should not be
35  * visible outside this layer except for misc. machine specific functions
36  * that work in conjunction with this code.
37  *
38  * Routines used only inside of i86pc/vm start with hati_ for HAT Internal.
39  */
40 
41 #include <sys/machparam.h>
42 #include <sys/machsystm.h>
43 #include <sys/mman.h>
44 #include <sys/types.h>
45 #include <sys/systm.h>
46 #include <sys/cpuvar.h>
47 #include <sys/thread.h>
48 #include <sys/proc.h>
49 #include <sys/cpu.h>
50 #include <sys/kmem.h>
51 #include <sys/disp.h>
52 #include <sys/shm.h>
53 #include <sys/sysmacros.h>
54 #include <sys/machparam.h>
55 #include <sys/vmem.h>
56 #include <sys/vmsystm.h>
57 #include <sys/promif.h>
58 #include <sys/var.h>
59 #include <sys/x86_archext.h>
60 #include <sys/atomic.h>
61 #include <sys/bitmap.h>
62 
63 #include <vm/seg_kmem.h>
64 #include <vm/hat_i86.h>
65 #include <vm/as.h>
66 #include <vm/seg.h>
67 #include <vm/page.h>
68 #include <vm/seg_kp.h>
69 #include <vm/seg_kpm.h>
70 #include <vm/vm_dep.h>
71 
72 #include <sys/cmn_err.h>
73 
74 
75 /*
76  * Basic parameters for hat operation.
77  */
78 struct hat_mmu_info mmu;
79 uint_t force_pae_off = 0;	/* for testing, change with kernel debugger */
80 uint_t force_pae_on = 0;	/* for testing, change with kernel debugger */
81 
82 /*
83  * The page that is the kernel's top level pagetable.
84  *
85  * For 32 bit VLP support, the kernel hat will use the 1st 4 entries
86  * on this 4K page for its top level page table. The remaining groups of
87  * 4 entries are used for per processor copies of user VLP pagetables for
88  * running threads.  See hat_switch() and reload_pae32() for details.
89  *
90  * vlp_page[0] - 0th level==2 PTE for kernel HAT (will be zero)
91  * vlp_page[1] - 1st level==2 PTE for kernel HAT (will be zero)
92  * vlp_page[2] - 2nd level==2 PTE for kernel HAT (zero for small memory)
93  * vlp_page[3] - 3rd level==2 PTE for kernel
94  *
95  * vlp_page[4] - 0th level==2 PTE for user thread on cpu 0
96  * vlp_page[5] - 1st level==2 PTE for user thread on cpu 0
97  * vlp_page[6] - 2nd level==2 PTE for user thread on cpu 0
98  * vlp_page[7] - probably copy of kernel PTE
99  *
100  * vlp_page[8]  - 0th level==2 PTE for user thread on cpu 1
101  * vlp_page[9]  - 1st level==2 PTE for user thread on cpu 1
102  * vlp_page[10] - 2nd level==2 PTE for user thread on cpu 1
103  * vlp_page[11] - probably copy of kernel PTE
104  * ...
105  *
106  * when / where the kernel PTE's are (entry 2 or 3 or none) depends
107  * on kernelbase.
108  */
109 static x86pte_t *vlp_page;
110 
111 /*
112  * forward declaration of internal utility routines
113  */
114 static x86pte_t hati_update_pte(htable_t *ht, uint_t entry, x86pte_t expected,
115 	x86pte_t new);
116 
117 /*
118  * The kernel address space exists in all HATs. To implement this the
119  * kernel reserves a fixed number of entries in every topmost level page
120  * table. The values are setup in hat_init() and then copied to every hat
121  * created by hat_alloc(). This means that kernelbase must be:
122  *
123  *	  4Meg aligned for 32 bit kernels
124  *	512Gig aligned for x86_64 64 bit kernel
125  *
126  * The PAE 32 bit hat is handled as a special case. Otherwise requiring 1Gig
127  * alignment would use too much VA for the kernel.
128  *
129  */
130 static uint_t	khat_start;	/* index of 1st entry in kernel's top ptable */
131 static uint_t	khat_entries;	/* number of entries in kernel's top ptable */
132 
133 #if defined(__i386)
134 
135 static htable_t	*khat_pae32_htable = NULL;
136 static uint_t	khat_pae32_start;
137 static uint_t	khat_pae32_entries;
138 
139 #endif
140 
141 /*
142  * Locks, etc. to control use of the hat reserves when recursively
143  * allocating pagetables for the hat data structures.
144  */
145 static kmutex_t hat_reserves_lock;
146 static kcondvar_t hat_reserves_cv;
147 kthread_t *hat_reserves_thread;
148 uint_t use_boot_reserve = 1;	/* cleared after early boot process */
149 uint_t can_steal_post_boot = 0;	/* set late in boot to enable stealing */
150 
151 /*
152  * A cpuset for all cpus. This is used for kernel address cross calls, since
153  * the kernel addresses apply to all cpus.
154  */
155 cpuset_t khat_cpuset;
156 
157 /*
158  * management stuff for hat structures
159  */
160 kmutex_t	hat_list_lock;
161 kcondvar_t	hat_list_cv;
162 kmem_cache_t	*hat_cache;
163 kmem_cache_t	*hat_hash_cache;
164 kmem_cache_t	*vlp_hash_cache;
165 
166 /*
167  * Simple statistics
168  */
169 struct hatstats hatstat;
170 
171 /*
172  * macros to detect addresses in use by kernel only during boot
173  */
174 #if defined(__amd64)
175 
176 #define	BOOT_VA(va) ((va) < kernelbase ||			\
177 	((va) >= BOOT_DOUBLEMAP_BASE &&				\
178 	(va) < BOOT_DOUBLEMAP_BASE + BOOT_DOUBLEMAP_SIZE))
179 
180 #elif defined(__i386)
181 
182 #define	BOOT_VA(va) ((va) < kernelbase)
183 
184 #endif	/* __i386 */
185 
186 /*
187  * useful stuff for atomic access/clearing/setting REF/MOD/RO bits in page_t's.
188  */
189 extern void atomic_orb(uchar_t *addr, uchar_t val);
190 extern void atomic_andb(uchar_t *addr, uchar_t val);
191 
192 #define	PP_GETRM(pp, rmmask)    (pp->p_nrm & rmmask)
193 #define	PP_ISMOD(pp)		PP_GETRM(pp, P_MOD)
194 #define	PP_ISREF(pp)		PP_GETRM(pp, P_REF)
195 #define	PP_ISRO(pp)		PP_GETRM(pp, P_RO)
196 
197 #define	PP_SETRM(pp, rm)	atomic_orb(&(pp->p_nrm), rm)
198 #define	PP_SETMOD(pp)		PP_SETRM(pp, P_MOD)
199 #define	PP_SETREF(pp)		PP_SETRM(pp, P_REF)
200 #define	PP_SETRO(pp)		PP_SETRM(pp, P_RO)
201 
202 #define	PP_CLRRM(pp, rm)	atomic_andb(&(pp->p_nrm), ~(rm))
203 #define	PP_CLRMOD(pp)   	PP_CLRRM(pp, P_MOD)
204 #define	PP_CLRREF(pp)   	PP_CLRRM(pp, P_REF)
205 #define	PP_CLRRO(pp)    	PP_CLRRM(pp, P_RO)
206 #define	PP_CLRALL(pp)		PP_CLRRM(pp, P_MOD | P_REF | P_RO)
207 
208 /*
209  * some useful tracing macros
210  */
211 
212 int hattrace = 0;
213 #ifdef DEBUG
214 
215 #define	HATIN(r, h, a, l)	\
216 	if (hattrace) prom_printf("->%s hat=%p, adr=%p, len=%lx\n", #r, h, a, l)
217 
218 #define	HATOUT(r, h, a)		\
219 	if (hattrace) prom_printf("<-%s hat=%p, adr=%p\n", #r, h, a)
220 #else
221 
222 #define	HATIN(r, h, a, l)
223 #define	HATOUT(r, h, a)
224 
225 #endif
226 
227 
228 /*
229  * kmem cache constructor for struct hat
230  */
231 /*ARGSUSED*/
232 static int
233 hati_constructor(void *buf, void *handle, int kmflags)
234 {
235 	hat_t	*hat = buf;
236 
237 	mutex_init(&hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL);
238 	bzero(hat->hat_pages_mapped,
239 	    sizeof (pgcnt_t) * (mmu.max_page_level + 1));
240 	hat->hat_stats = 0;
241 	hat->hat_flags = 0;
242 	mutex_init(&hat->hat_switch_mutex, NULL, MUTEX_DRIVER,
243 	    (void *)ipltospl(DISP_LEVEL));
244 	CPUSET_ZERO(hat->hat_cpus);
245 	hat->hat_htable = NULL;
246 	hat->hat_ht_hash = NULL;
247 	return (0);
248 }
249 
250 /*
251  * Allocate a hat structure for as. We also create the top level
252  * htable and initialize it to contain the kernel hat entries.
253  */
254 hat_t *
255 hat_alloc(struct as *as)
256 {
257 	hat_t		*hat;
258 	htable_t	*ht;	/* top level htable */
259 	uint_t		use_vlp;
260 
261 	/*
262 	 * Once we start creating user process HATs we can enable
263 	 * the htable_steal() code.
264 	 */
265 	if (can_steal_post_boot == 0)
266 		can_steal_post_boot = 1;
267 
268 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
269 	hat = kmem_cache_alloc(hat_cache, KM_SLEEP);
270 	hat->hat_as = as;
271 	mutex_init(&hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL);
272 	ASSERT(hat->hat_flags == 0);
273 
274 	/*
275 	 * a 32 bit process uses a VLP style hat when using PAE
276 	 */
277 #if defined(__amd64)
278 	use_vlp = (ttoproc(curthread)->p_model == DATAMODEL_ILP32);
279 #elif defined(__i386)
280 	use_vlp = mmu.pae_hat;
281 #endif
282 	if (use_vlp) {
283 		hat->hat_flags = HAT_VLP;
284 		bzero(hat->hat_vlp_ptes, VLP_SIZE);
285 	}
286 
287 	/*
288 	 * Allocate the htable hash
289 	 */
290 	if ((hat->hat_flags & HAT_VLP)) {
291 		hat->hat_num_hash = mmu.vlp_hash_cnt;
292 		hat->hat_ht_hash = kmem_cache_alloc(vlp_hash_cache, KM_SLEEP);
293 	} else {
294 		hat->hat_num_hash = mmu.hash_cnt;
295 		hat->hat_ht_hash = kmem_cache_alloc(hat_hash_cache, KM_SLEEP);
296 	}
297 	bzero(hat->hat_ht_hash, hat->hat_num_hash * sizeof (htable_t *));
298 
299 	/*
300 	 * Initialize Kernel HAT entries at the top of the top level page
301 	 * table for the new hat.
302 	 *
303 	 * Note that we don't call htable_release() for the top level, that
304 	 * happens when the hat is destroyed in hat_free_end()
305 	 */
306 	hat->hat_htable = NULL;
307 	hat->hat_ht_cached = NULL;
308 	ht = htable_create(hat, (uintptr_t)0, TOP_LEVEL(hat), NULL);
309 	if (!(hat->hat_flags & HAT_VLP))
310 		x86pte_copy(kas.a_hat->hat_htable, ht, khat_start,
311 		    khat_entries);
312 #if defined(__i386)
313 	else if (khat_entries > 0)
314 		bcopy(vlp_page + khat_start, hat->hat_vlp_ptes + khat_start,
315 		    khat_entries * sizeof (x86pte_t));
316 #endif
317 	hat->hat_htable = ht;
318 
319 #if defined(__i386)
320 	/*
321 	 * PAE32 HAT alignment is less restrictive than the others to keep
322 	 * the kernel from using too much VA. Because of this we may need
323 	 * one layer further down when kernelbase isn't 1Gig aligned.
324 	 * See hat_free_end() for the htable_release() that goes with this
325 	 * htable_create()
326 	 */
327 	if (khat_pae32_htable != NULL) {
328 		ht = htable_create(hat, kernelbase,
329 		    khat_pae32_htable->ht_level, NULL);
330 		x86pte_copy(khat_pae32_htable, ht, khat_pae32_start,
331 		    khat_pae32_entries);
332 		ht->ht_valid_cnt = khat_pae32_entries;
333 	}
334 #endif
335 
336 	/*
337 	 * Put it in the global list of all hats (used by stealing, etc.)
338 	 */
339 	mutex_enter(&hat_list_lock);
340 	if (kas.a_hat->hat_next != NULL) {
341 		hat->hat_next = kas.a_hat->hat_next;
342 		hat->hat_prev = kas.a_hat->hat_next->hat_prev;
343 		kas.a_hat->hat_next->hat_prev->hat_next = hat;
344 		kas.a_hat->hat_next->hat_prev = hat;
345 	} else {
346 		hat->hat_next = hat;
347 		hat->hat_prev = hat;
348 	}
349 	kas.a_hat->hat_next = hat;
350 	mutex_exit(&hat_list_lock);
351 
352 
353 	return (hat);
354 }
355 
356 /*
357  * process has finished executing but as has not been cleaned up yet.
358  */
359 /*ARGSUSED*/
360 void
361 hat_free_start(hat_t *hat)
362 {
363 	ASSERT(AS_WRITE_HELD(hat->hat_as, &hat->hat_as->a_lock));
364 	mutex_enter(&hat_list_lock);
365 	hat->hat_flags |= HAT_FREEING;
366 	mutex_exit(&hat_list_lock);
367 }
368 
369 /*
370  * An address space is being destroyed, so we destroy the associated hat.
371  */
372 void
373 hat_free_end(hat_t *hat)
374 {
375 	int i;
376 	kmem_cache_t *cache;
377 
378 #ifdef DEBUG
379 	for (i = 0; i <= mmu.max_page_level; i++)
380 		ASSERT(hat->hat_pages_mapped[i] == 0);
381 #endif
382 	ASSERT(hat->hat_flags & HAT_FREEING);
383 
384 	/*
385 	 * must not be running on the given hat
386 	 */
387 	ASSERT(CPU->cpu_current_hat != hat);
388 
389 	/*
390 	 * If the hat is currently a stealing victim, wait for the stealing
391 	 * to finish.  Once we've removed it from the list, nobody can
392 	 * find these htables anymore.
393 	 */
394 	mutex_enter(&hat_list_lock);
395 	while (hat->hat_flags & HAT_VICTIM)
396 		cv_wait(&hat_list_cv, &hat_list_lock);
397 	hat->hat_next->hat_prev = hat->hat_prev;
398 	hat->hat_prev->hat_next = hat->hat_next;
399 	if (kas.a_hat->hat_next == hat) {
400 		kas.a_hat->hat_next = hat->hat_next;
401 		if (kas.a_hat->hat_next == hat)
402 			kas.a_hat->hat_next = NULL;
403 	}
404 	mutex_exit(&hat_list_lock);
405 
406 	/*
407 	 * Make a pass through the htables freeing them all up.
408 	 */
409 	htable_purge_hat(hat);
410 
411 	/*
412 	 * Decide which kmem cache the hash table came from, then free it.
413 	 */
414 	if (hat->hat_flags & HAT_VLP)
415 		cache = vlp_hash_cache;
416 	else
417 		cache = hat_hash_cache;
418 	kmem_cache_free(cache, hat->hat_ht_hash);
419 	hat->hat_ht_hash = NULL;
420 
421 	hat->hat_flags = 0;
422 	kmem_cache_free(hat_cache, hat);
423 }
424 
425 /*
426  * round kernelbase down to a supported value to use for _userlimit
427  *
428  * userlimit must be aligned down to an entry in the top level htable.
429  * The one exception is for 32 bit HAT's running PAE.
430  */
431 uintptr_t
432 hat_kernelbase(uintptr_t va)
433 {
434 #if defined(__i386)
435 	va &= LEVEL_MASK(1);
436 #endif
437 	if (IN_VA_HOLE(va))
438 		panic("_userlimit %p will fall in VA hole\n", (void *)va);
439 	return (va);
440 }
441 
442 /*
443  * Initialize hat data structures based on processor MMU information.
444  */
445 void
446 mmu_init(void)
447 {
448 	uint_t max_htables;
449 	uint_t pa_bits;
450 	uint_t va_bits;
451 	int i;
452 
453 	/*
454 	 * if CPU enabled the page table global bit, use it for the kernel
455 	 * This is bit 7 in CR4 (PGE - Page Global Enable)
456 	 */
457 	if ((x86_feature & X86_PGE) != 0 && (getcr4() & 0x80) != 0)
458 		mmu.pt_global = PT_GLOBAL;
459 
460 	/*
461 	 * We use PAE except when we aren't on an AMD64 and this is
462 	 * a 32 bit kernel with all physical addresses less than 4 Gig.
463 	 */
464 	mmu.pae_hat = 1;
465 	if (x86_feature & X86_NX) {
466 		mmu.pt_nx = PT_NX;
467 	} else {
468 		mmu.pt_nx = 0;
469 #if defined(__i386)
470 		if (!PFN_ABOVE4G(physmax))
471 			mmu.pae_hat = 0;
472 #endif
473 	}
474 
475 #if defined(__i386)
476 	/*
477 	 * Setting one of these two lets you force testing of the different
478 	 * hat modes for 32 bit, regardless of the hardware setup.
479 	 */
480 	if (force_pae_on) {
481 		mmu.pae_hat = 1;
482 	} else if (force_pae_off) {
483 		mmu.pae_hat = 0;
484 		mmu.pt_nx = 0;
485 	}
486 #endif
487 
488 	/*
489 	 * Use CPU info to set various MMU parameters
490 	 */
491 	cpuid_get_addrsize(CPU, &pa_bits, &va_bits);
492 
493 	if (va_bits < sizeof (void *) * NBBY) {
494 		mmu.hole_start = (1ul << (va_bits - 1));
495 		mmu.hole_end = 0ul - mmu.hole_start - 1;
496 	} else {
497 		mmu.hole_end = 0;
498 		mmu.hole_start = mmu.hole_end - 1;
499 	}
500 #if defined(OPTERON_ERRATUM_121)
501 	/*
502 	 * If erratum 121 has already been detected at this time, hole_start
503 	 * contains the value to be subtracted from mmu.hole_start.
504 	 */
505 	ASSERT(hole_start == 0 || opteron_erratum_121 != 0);
506 	hole_start = mmu.hole_start - hole_start;
507 #else
508 	hole_start = mmu.hole_start;
509 #endif
510 	hole_end = mmu.hole_end;
511 
512 	mmu.highest_pfn = mmu_btop((1ull << pa_bits) - 1);
513 	if (mmu.pae_hat == 0 && pa_bits > 32)
514 		mmu.highest_pfn = PFN_4G - 1;
515 
516 	if (mmu.pae_hat) {
517 		mmu.pte_size = 8;	/* 8 byte PTEs */
518 		mmu.pte_size_shift = 3;
519 	} else {
520 		mmu.pte_size = 4;	/* 4 byte PTEs */
521 		mmu.pte_size_shift = 2;
522 	}
523 
524 	if (mmu.pae_hat && (x86_feature & X86_PAE) == 0)
525 		panic("Processor does not support PAE");
526 
527 	if ((x86_feature & X86_CX8) == 0)
528 		panic("Processor does not support cmpxchg8b instruction");
529 
530 	/*
531 	 * Initialize parameters based on the 64 or 32 bit kernels and
532 	 * for the 32 bit kernel decide if we should use PAE.
533 	 */
534 	if (x86_feature & X86_LARGEPAGE)
535 		mmu.max_page_level = 1;
536 	else
537 		mmu.max_page_level = 0;
538 	mmu_page_sizes = mmu.max_page_level + 1;
539 	mmu_exported_page_sizes = mmu_page_sizes;
540 
541 #if defined(__amd64)
542 
543 	mmu.num_level = 4;
544 	mmu.max_level = 3;
545 	mmu.ptes_per_table = 512;
546 	mmu.top_level_count = 512;
547 
548 	mmu.level_shift[0] = 12;
549 	mmu.level_shift[1] = 21;
550 	mmu.level_shift[2] = 30;
551 	mmu.level_shift[3] = 39;
552 
553 #elif defined(__i386)
554 
555 	if (mmu.pae_hat) {
556 		mmu.num_level = 3;
557 		mmu.max_level = 2;
558 		mmu.ptes_per_table = 512;
559 		mmu.top_level_count = 4;
560 
561 		mmu.level_shift[0] = 12;
562 		mmu.level_shift[1] = 21;
563 		mmu.level_shift[2] = 30;
564 
565 	} else {
566 		mmu.num_level = 2;
567 		mmu.max_level = 1;
568 		mmu.ptes_per_table = 1024;
569 		mmu.top_level_count = 1024;
570 
571 		mmu.level_shift[0] = 12;
572 		mmu.level_shift[1] = 22;
573 	}
574 
575 #endif	/* __i386 */
576 
577 	for (i = 0; i < mmu.num_level; ++i) {
578 		mmu.level_size[i] = 1UL << mmu.level_shift[i];
579 		mmu.level_offset[i] = mmu.level_size[i] - 1;
580 		mmu.level_mask[i] = ~mmu.level_offset[i];
581 	}
582 
583 	mmu.pte_bits[0] = PT_VALID;
584 	for (i = 1; i <= mmu.max_page_level; ++i)
585 		mmu.pte_bits[i] = PT_VALID | PT_PAGESIZE;
586 
587 	/*
588 	 * NOTE Legacy 32 bit PAE mode only has the P_VALID bit at top level.
589 	 */
590 	for (i = 1; i < mmu.num_level; ++i)
591 		mmu.ptp_bits[i] = PT_PTPBITS;
592 #if defined(__i386)
593 	mmu.ptp_bits[2] = PT_VALID;
594 #endif
595 
596 	/*
597 	 * Compute how many hash table entries to have per process for htables.
598 	 * We start with 1 page's worth of entries.
599 	 *
600 	 * If physical memory is small, reduce the amount need to cover it.
601 	 */
602 	max_htables = physmax / mmu.ptes_per_table;
603 	mmu.hash_cnt = MMU_PAGESIZE / sizeof (htable_t *);
604 	while (mmu.hash_cnt > 16 && mmu.hash_cnt >= max_htables)
605 		mmu.hash_cnt >>= 1;
606 	mmu.vlp_hash_cnt = mmu.hash_cnt;
607 
608 #if defined(__amd64)
609 	/*
610 	 * If running in 64 bits and physical memory is large,
611 	 * increase the size of the cache to cover all of memory for
612 	 * a 64 bit process.
613 	 */
614 #define	HASH_MAX_LENGTH 4
615 	while (mmu.hash_cnt * HASH_MAX_LENGTH < max_htables)
616 		mmu.hash_cnt <<= 1;
617 #endif
618 
619 	/*
620 	 * This code knows that there are only 2 pagesizes.
621 	 * We ignore 4MB (non-PAE) for now. The value is only used
622 	 * for optimizing demaps across large ranges.
623 	 * These return zero if no information is known.
624 	 */
625 	mmu.tlb_entries[0] = cpuid_get_dtlb_nent(NULL, MMU_PAGESIZE);
626 	mmu.tlb_entries[1] = cpuid_get_dtlb_nent(NULL, 2 * 1024 * 1024);
627 }
628 
629 
630 /*
631  * initialize hat data structures
632  */
633 void
634 hat_init()
635 {
636 #if defined(__i386)
637 	/*
638 	 * _userlimit must be aligned correctly
639 	 */
640 	if ((_userlimit & LEVEL_MASK(1)) != _userlimit) {
641 		prom_printf("hat_init(): _userlimit=%p, not aligned at %p\n",
642 		    (void *)_userlimit, (void *)LEVEL_SIZE(1));
643 		halt("hat_init(): Unable to continue");
644 	}
645 #endif
646 
647 	cv_init(&hat_list_cv, NULL, CV_DEFAULT, NULL);
648 
649 	/*
650 	 * initialize kmem caches
651 	 */
652 	htable_init();
653 	hment_init();
654 
655 	hat_cache = kmem_cache_create("hat_t",
656 	    sizeof (hat_t), 0, hati_constructor, NULL, NULL,
657 	    NULL, 0, 0);
658 
659 	hat_hash_cache = kmem_cache_create("HatHash",
660 	    mmu.hash_cnt * sizeof (htable_t *), 0, NULL, NULL, NULL,
661 	    NULL, 0, 0);
662 
663 	/*
664 	 * VLP hats can use a smaller hash table size on large memroy machines
665 	 */
666 	if (mmu.hash_cnt == mmu.vlp_hash_cnt) {
667 		vlp_hash_cache = hat_hash_cache;
668 	} else {
669 		vlp_hash_cache = kmem_cache_create("HatVlpHash",
670 		    mmu.vlp_hash_cnt * sizeof (htable_t *), 0, NULL, NULL, NULL,
671 		    NULL, 0, 0);
672 	}
673 
674 	/*
675 	 * Set up the kernel's hat
676 	 */
677 	AS_LOCK_ENTER(&kas, &kas.a_lock, RW_WRITER);
678 	kas.a_hat = kmem_cache_alloc(hat_cache, KM_NOSLEEP);
679 	mutex_init(&kas.a_hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL);
680 	kas.a_hat->hat_as = &kas;
681 	kas.a_hat->hat_flags = 0;
682 	AS_LOCK_EXIT(&kas, &kas.a_lock);
683 
684 	CPUSET_ZERO(khat_cpuset);
685 	CPUSET_ADD(khat_cpuset, CPU->cpu_id);
686 
687 	/*
688 	 * The kernel hat's next pointer serves as the head of the hat list .
689 	 */
690 	kas.a_hat->hat_next = NULL;
691 
692 	/*
693 	 * Allocate an htable hash bucket for the kernel
694 	 * XX64 - tune for 64 bit procs
695 	 */
696 	kas.a_hat->hat_num_hash = mmu.hash_cnt;
697 	kas.a_hat->hat_ht_hash = kmem_cache_alloc(hat_hash_cache, KM_NOSLEEP);
698 	bzero(kas.a_hat->hat_ht_hash, mmu.hash_cnt * sizeof (htable_t *));
699 
700 	/*
701 	 * zero out the top level and cached htable pointers
702 	 */
703 	kas.a_hat->hat_ht_cached = NULL;
704 	kas.a_hat->hat_htable = NULL;
705 }
706 
707 /*
708  * Prepare CPU specific pagetables for VLP processes on 64 bit kernels.
709  *
710  * Each CPU has a set of 2 pagetables that are reused for any 32 bit
711  * process it runs. They are the top level pagetable, hci_vlp_l3ptes, and
712  * the next to top level table for the bottom 512 Gig, hci_vlp_l2ptes.
713  */
714 /*ARGSUSED*/
715 static void
716 hat_vlp_setup(struct cpu *cpu)
717 {
718 #if defined(__amd64)
719 	struct hat_cpu_info *hci = cpu->cpu_hat_info;
720 	pfn_t pfn;
721 
722 	/*
723 	 * allocate the level==2 page table for the bottom most
724 	 * 512Gig of address space (this is where 32 bit apps live)
725 	 */
726 	ASSERT(hci != NULL);
727 	hci->hci_vlp_l2ptes = kmem_zalloc(MMU_PAGESIZE, KM_SLEEP);
728 
729 	/*
730 	 * Allocate a top level pagetable and copy the kernel's
731 	 * entries into it. Then link in hci_vlp_l2ptes in the 1st entry.
732 	 */
733 	hci->hci_vlp_l3ptes = kmem_zalloc(MMU_PAGESIZE, KM_SLEEP);
734 	hci->hci_vlp_pfn =
735 	    hat_getpfnum(kas.a_hat, (caddr_t)hci->hci_vlp_l3ptes);
736 	ASSERT(hci->hci_vlp_pfn != PFN_INVALID);
737 	bcopy(vlp_page + khat_start, hci->hci_vlp_l3ptes + khat_start,
738 	    khat_entries * sizeof (x86pte_t));
739 
740 	pfn = hat_getpfnum(kas.a_hat, (caddr_t)hci->hci_vlp_l2ptes);
741 	ASSERT(pfn != PFN_INVALID);
742 	hci->hci_vlp_l3ptes[0] = MAKEPTP(pfn, 2);
743 #endif /* __amd64 */
744 }
745 
746 /*
747  * Finish filling in the kernel hat.
748  * Pre fill in all top level kernel page table entries for the kernel's
749  * part of the address range.  From this point on we can't use any new
750  * kernel large pages if they need PTE's at max_level
751  */
752 void
753 hat_init_finish(void)
754 {
755 	htable_t	*top = kas.a_hat->hat_htable;
756 	htable_t	*ht;
757 	uint_t		e;
758 	x86pte_t	pte;
759 	uintptr_t	va = kernelbase;
760 
761 
762 #if defined(__i386)
763 	ASSERT((va & LEVEL_MASK(1)) == va);
764 
765 	/*
766 	 * Deal with kernelbase not 1Gig aligned for 32 bit PAE hats.
767 	 */
768 	if (!mmu.pae_hat || (va & LEVEL_OFFSET(mmu.max_level)) == 0) {
769 		khat_pae32_htable = NULL;
770 	} else {
771 		ASSERT(mmu.max_level == 2);
772 		ASSERT((va & LEVEL_OFFSET(mmu.max_level - 1)) == 0);
773 		khat_pae32_htable =
774 		    htable_create(kas.a_hat, va, mmu.max_level - 1, NULL);
775 		khat_pae32_start = htable_va2entry(va, khat_pae32_htable);
776 		khat_pae32_entries = mmu.ptes_per_table - khat_pae32_start;
777 		for (e = khat_pae32_start; e < mmu.ptes_per_table;
778 		    ++e, va += LEVEL_SIZE(mmu.max_level - 1)) {
779 			pte = x86pte_get(khat_pae32_htable, e);
780 			if (PTE_ISVALID(pte))
781 				continue;
782 			ht = htable_create(kas.a_hat, va, mmu.max_level - 2,
783 			    NULL);
784 			ASSERT(ht != NULL);
785 		}
786 	}
787 #endif
788 
789 	/*
790 	 * The kernel hat will need fixed values in the highest level
791 	 * ptable for copying to all other hat's. This implies
792 	 * alignment restrictions on _userlimit.
793 	 *
794 	 * Note we don't htable_release() these htables. This keeps them
795 	 * from ever being stolen or free'd.
796 	 *
797 	 * top_level_count is used instead of ptes_per_table, since
798 	 * on 32-bit PAE we only have 4 usable entries at the top level ptable.
799 	 */
800 	if (va == 0)
801 		khat_start = mmu.top_level_count;
802 	else
803 		khat_start = htable_va2entry(va, kas.a_hat->hat_htable);
804 	khat_entries = mmu.top_level_count - khat_start;
805 	for (e = khat_start; e < mmu.top_level_count;
806 	    ++e, va += LEVEL_SIZE(mmu.max_level)) {
807 		pte = x86pte_get(top, e);
808 		if (PTE_ISVALID(pte))
809 			continue;
810 		ht = htable_create(kas.a_hat, va, mmu.max_level - 1, NULL);
811 		ASSERT(ht != NULL);
812 	}
813 
814 	/*
815 	 * We are now effectively running on the kernel hat.
816 	 * Clearing use_boot_reserve shuts off using the pre-allocated boot
817 	 * reserve for all HAT allocations.  From here on, the reserves are
818 	 * only used when mapping in memory for the hat's own allocations.
819 	 */
820 	use_boot_reserve = 0;
821 	htable_adjust_reserve();
822 
823 	/*
824 	 * 32 bit kernels use only 4 of the 512 entries in its top level
825 	 * pagetable. We'll use the remainder for the "per CPU" page tables
826 	 * for VLP processes.
827 	 *
828 	 * We map the top level kernel pagetable into the kernel's AS to make
829 	 * it easy to use bcopy for kernel entry PTEs.
830 	 *
831 	 * We were guaranteed to get a physical address < 4Gig, since the 32 bit
832 	 * boot loader uses non-PAE page tables.
833 	 */
834 	if (mmu.pae_hat) {
835 		vlp_page = vmem_alloc(heap_arena, MMU_PAGESIZE, VM_SLEEP);
836 		hat_devload(kas.a_hat, (caddr_t)vlp_page, MMU_PAGESIZE,
837 		    kas.a_hat->hat_htable->ht_pfn,
838 		    PROT_READ | PROT_WRITE | HAT_NOSYNC | HAT_UNORDERED_OK,
839 		    HAT_LOAD | HAT_LOAD_NOCONSIST);
840 	}
841 	hat_vlp_setup(CPU);
842 }
843 
844 /*
845  * On 32 bit PAE mode, PTE's are 64 bits, but ordinary atomic memory references
846  * are 32 bit, so for safety we must use cas64() to install these.
847  */
848 #ifdef __i386
849 static void
850 reload_pae32(hat_t *hat, cpu_t *cpu)
851 {
852 	x86pte_t *src;
853 	x86pte_t *dest;
854 	x86pte_t pte;
855 	int i;
856 
857 	/*
858 	 * Load the 4 entries of the level 2 page table into this
859 	 * cpu's range of the vlp_page and point cr3 at them.
860 	 */
861 	ASSERT(mmu.pae_hat);
862 	src = hat->hat_vlp_ptes;
863 	dest = vlp_page + (cpu->cpu_id + 1) * VLP_NUM_PTES;
864 	for (i = 0; i < VLP_NUM_PTES; ++i) {
865 		for (;;) {
866 			pte = dest[i];
867 			if (pte == src[i])
868 				break;
869 			if (cas64(dest + i, pte, src[i]) != src[i])
870 				break;
871 		}
872 	}
873 }
874 #endif
875 
876 /*
877  * Switch to a new active hat, maintaining bit masks to track active CPUs.
878  */
879 void
880 hat_switch(hat_t *hat)
881 {
882 	uintptr_t	newcr3;
883 	cpu_t		*cpu = CPU;
884 	hat_t		*old = cpu->cpu_current_hat;
885 
886 	/*
887 	 * set up this information first, so we don't miss any cross calls
888 	 */
889 	if (old != NULL) {
890 		if (old == hat)
891 			return;
892 		if (old != kas.a_hat)
893 			CPUSET_ATOMIC_DEL(old->hat_cpus, cpu->cpu_id);
894 	}
895 
896 	/*
897 	 * Wait for any in flight pagetable invalidates on this hat to finish.
898 	 * This is a spin lock at DISP_LEVEL
899 	 */
900 	if (hat != kas.a_hat) {
901 		mutex_enter(&hat->hat_switch_mutex);
902 		CPUSET_ATOMIC_ADD(hat->hat_cpus, cpu->cpu_id);
903 		mutex_exit(&hat->hat_switch_mutex);
904 	}
905 	cpu->cpu_current_hat = hat;
906 
907 	/*
908 	 * now go ahead and load cr3
909 	 */
910 	if (hat->hat_flags & HAT_VLP) {
911 #if defined(__amd64)
912 		x86pte_t *vlpptep = cpu->cpu_hat_info->hci_vlp_l2ptes;
913 
914 		VLP_COPY(hat->hat_vlp_ptes, vlpptep);
915 		newcr3 = MAKECR3(cpu->cpu_hat_info->hci_vlp_pfn);
916 #elif defined(__i386)
917 		reload_pae32(hat, cpu);
918 		newcr3 = MAKECR3(kas.a_hat->hat_htable->ht_pfn) +
919 		    (cpu->cpu_id + 1) * VLP_SIZE;
920 #endif
921 	} else {
922 		newcr3 = MAKECR3(hat->hat_htable->ht_pfn);
923 	}
924 	setcr3(newcr3);
925 	ASSERT(cpu == CPU);
926 }
927 
928 /*
929  * Utility to return a valid x86pte_t from protections, pfn, and level number
930  */
931 static x86pte_t
932 hati_mkpte(pfn_t pfn, uint_t attr, level_t level, uint_t flags)
933 {
934 	x86pte_t	pte;
935 	uint_t		cache_attr = attr & HAT_ORDER_MASK;
936 
937 	pte = MAKEPTE(pfn, level);
938 
939 	if (attr & PROT_WRITE)
940 		PTE_SET(pte, PT_WRITABLE);
941 
942 	if (attr & PROT_USER)
943 		PTE_SET(pte, PT_USER);
944 
945 	if (!(attr & PROT_EXEC))
946 		PTE_SET(pte, mmu.pt_nx);
947 
948 	/*
949 	 * set the software bits used track ref/mod sync's and hments
950 	 */
951 	if (attr & HAT_NOSYNC)
952 		PTE_SET(pte, PT_NOSYNC);
953 	if (flags & HAT_LOAD_NOCONSIST)
954 		PTE_SET(pte, PT_NOCONSIST | PT_NOSYNC);
955 
956 	/*
957 	 * Set the caching attributes in the PTE. The combination
958 	 * of attributes are poorly defined, so we pay attention
959 	 * to them in the given order.
960 	 *
961 	 * The test for HAT_STRICTORDER is different because it's defined
962 	 * as "0" - which was a stupid thing to do, but is too late to change!
963 	 */
964 	if (cache_attr == HAT_STRICTORDER) {
965 		PTE_SET(pte, PT_NOCACHE);
966 	/*LINTED [Lint hates empty ifs, but it's the obvious way to do this] */
967 	} else if (cache_attr & (HAT_UNORDERED_OK | HAT_STORECACHING_OK)) {
968 		/* nothing to set */;
969 	} else if (cache_attr & (HAT_MERGING_OK | HAT_LOADCACHING_OK)) {
970 		PTE_SET(pte, PT_NOCACHE);
971 		if (x86_feature & X86_PAT)
972 			PTE_SET(pte, (level == 0) ? PT_PAT_4K : PT_PAT_LARGE);
973 		else
974 			PTE_SET(pte, PT_WRITETHRU);
975 	} else {
976 		panic("hati_mkpte(): bad caching attributes: %x\n", cache_attr);
977 	}
978 
979 	return (pte);
980 }
981 
982 /*
983  * Duplicate address translations of the parent to the child.
984  * This function really isn't used anymore.
985  */
986 /*ARGSUSED*/
987 int
988 hat_dup(hat_t *old, hat_t *new, caddr_t addr, size_t len, uint_t flag)
989 {
990 	ASSERT((uintptr_t)addr < kernelbase);
991 	ASSERT(new != kas.a_hat);
992 	ASSERT(old != kas.a_hat);
993 	return (0);
994 }
995 
996 /*
997  * Allocate any hat resources required for a process being swapped in.
998  */
999 /*ARGSUSED*/
1000 void
1001 hat_swapin(hat_t *hat)
1002 {
1003 	/* do nothing - we let everything fault back in */
1004 }
1005 
1006 /*
1007  * Unload all translations associated with an address space of a process
1008  * that is being swapped out.
1009  */
1010 void
1011 hat_swapout(hat_t *hat)
1012 {
1013 	uintptr_t	vaddr = (uintptr_t)0;
1014 	uintptr_t	eaddr = _userlimit;
1015 	htable_t	*ht = NULL;
1016 	level_t		l;
1017 
1018 	/*
1019 	 * We can't just call hat_unload(hat, 0, _userlimit...)  here, because
1020 	 * seg_spt and shared pagetables can't be swapped out.
1021 	 * Take a look at segspt_shmswapout() - it's a big no-op.
1022 	 *
1023 	 * Instead we'll walk through all the address space and unload
1024 	 * any mappings which we are sure are not shared, not locked.
1025 	 */
1026 	ASSERT(IS_PAGEALIGNED(vaddr));
1027 	ASSERT(IS_PAGEALIGNED(eaddr));
1028 	ASSERT(AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1029 	if ((uintptr_t)hat->hat_as->a_userlimit < eaddr)
1030 		eaddr = (uintptr_t)hat->hat_as->a_userlimit;
1031 
1032 	while (vaddr < eaddr) {
1033 		(void) htable_walk(hat, &ht, &vaddr, eaddr);
1034 		if (ht == NULL)
1035 			break;
1036 
1037 		ASSERT(!IN_VA_HOLE(vaddr));
1038 
1039 		/*
1040 		 * If the page table is shared skip its entire range.
1041 		 * This code knows that only level 0 page tables are shared
1042 		 */
1043 		l = ht->ht_level;
1044 		if (ht->ht_flags & HTABLE_SHARED_PFN) {
1045 			ASSERT(l == 0);
1046 			vaddr = ht->ht_vaddr + LEVEL_SIZE(1);
1047 			htable_release(ht);
1048 			ht = NULL;
1049 			continue;
1050 		}
1051 
1052 		/*
1053 		 * If the page table has no locked entries, unload this one.
1054 		 */
1055 		if (ht->ht_lock_cnt == 0)
1056 			hat_unload(hat, (caddr_t)vaddr, LEVEL_SIZE(l),
1057 			    HAT_UNLOAD_UNMAP);
1058 
1059 		/*
1060 		 * If we have a level 0 page table with locked entries,
1061 		 * skip the entire page table, otherwise skip just one entry.
1062 		 */
1063 		if (ht->ht_lock_cnt > 0 && l == 0)
1064 			vaddr = ht->ht_vaddr + LEVEL_SIZE(1);
1065 		else
1066 			vaddr += LEVEL_SIZE(l);
1067 	}
1068 	if (ht)
1069 		htable_release(ht);
1070 
1071 	/*
1072 	 * We're in swapout because the system is low on memory, so
1073 	 * go back and flush all the htables off the cached list.
1074 	 */
1075 	htable_purge_hat(hat);
1076 }
1077 
1078 /*
1079  * returns number of bytes that have valid mappings in hat.
1080  */
1081 size_t
1082 hat_get_mapped_size(hat_t *hat)
1083 {
1084 	size_t total = 0;
1085 	int l;
1086 
1087 	for (l = 0; l <= mmu.max_page_level; l++)
1088 		total += (hat->hat_pages_mapped[l] << LEVEL_SHIFT(l));
1089 
1090 	return (total);
1091 }
1092 
1093 /*
1094  * enable/disable collection of stats for hat.
1095  */
1096 int
1097 hat_stats_enable(hat_t *hat)
1098 {
1099 	atomic_add_32(&hat->hat_stats, 1);
1100 	return (1);
1101 }
1102 
1103 void
1104 hat_stats_disable(hat_t *hat)
1105 {
1106 	atomic_add_32(&hat->hat_stats, -1);
1107 }
1108 
1109 /*
1110  * Utility to sync the ref/mod bits from a page table entry to the page_t
1111  * We must be holding the mapping list lock when this is called.
1112  */
1113 static void
1114 hati_sync_pte_to_page(page_t *pp, x86pte_t pte, level_t level)
1115 {
1116 	uint_t	rm = 0;
1117 	pgcnt_t	pgcnt;
1118 
1119 	if (PTE_GET(pte, PT_NOSYNC))
1120 		return;
1121 
1122 	if (PTE_GET(pte, PT_REF))
1123 		rm |= P_REF;
1124 
1125 	if (PTE_GET(pte, PT_MOD))
1126 		rm |= P_MOD;
1127 
1128 	if (rm == 0)
1129 		return;
1130 
1131 	/*
1132 	 * sync to all constituent pages of a large page
1133 	 */
1134 	ASSERT(x86_hm_held(pp));
1135 	pgcnt = page_get_pagecnt(level);
1136 	ASSERT(IS_P2ALIGNED(pp->p_pagenum, pgcnt));
1137 	for (; pgcnt > 0; --pgcnt) {
1138 		/*
1139 		 * hat_page_demote() can't decrease
1140 		 * pszc below this mapping size
1141 		 * since this large mapping existed after we
1142 		 * took mlist lock.
1143 		 */
1144 		ASSERT(pp->p_szc >= level);
1145 		hat_page_setattr(pp, rm);
1146 		++pp;
1147 	}
1148 }
1149 
1150 /*
1151  * This the set of PTE bits for PFN, permissions and caching
1152  * that require a TLB flush (hat_demap) if changed on a HAT_LOAD_REMAP
1153  */
1154 #define	PT_REMAP_BITS							\
1155 	(PT_PADDR | PT_NX | PT_WRITABLE | PT_WRITETHRU |		\
1156 	PT_NOCACHE | PT_PAT_4K | PT_PAT_LARGE)
1157 
1158 /*
1159  * Do the low-level work to get a mapping entered into a HAT's pagetables
1160  * and in the mapping list of the associated page_t.
1161  */
1162 static void
1163 hati_pte_map(
1164 	htable_t	*ht,
1165 	uint_t		entry,
1166 	page_t		*pp,
1167 	x86pte_t	pte,
1168 	int		flags,
1169 	void		*pte_ptr)
1170 {
1171 	hat_t		*hat = ht->ht_hat;
1172 	x86pte_t	old_pte;
1173 	level_t		l = ht->ht_level;
1174 	hment_t		*hm;
1175 	uint_t		is_consist;
1176 
1177 	/*
1178 	 * Is this a consistant (ie. need mapping list lock) mapping?
1179 	 */
1180 	is_consist = (pp != NULL && (flags & HAT_LOAD_NOCONSIST) == 0);
1181 
1182 	/*
1183 	 * Track locked mapping count in the htable.  Do this first,
1184 	 * as we track locking even if there already is a mapping present.
1185 	 */
1186 	if ((flags & HAT_LOAD_LOCK) != 0 && hat != kas.a_hat)
1187 		HTABLE_LOCK_INC(ht);
1188 
1189 	/*
1190 	 * Acquire the page's mapping list lock and get an hment to use.
1191 	 * Note that hment_prepare() might return NULL.
1192 	 */
1193 	if (is_consist) {
1194 		x86_hm_enter(pp);
1195 		hm = hment_prepare(ht, entry, pp);
1196 	}
1197 
1198 	/*
1199 	 * Set the new pte, retrieving the old one at the same time.
1200 	 */
1201 	old_pte = x86pte_set(ht, entry, pte, pte_ptr);
1202 
1203 	/*
1204 	 * If the mapping didn't change there is nothing more to do.
1205 	 */
1206 	if (PTE_EQUIV(pte, old_pte)) {
1207 		if (is_consist) {
1208 			x86_hm_exit(pp);
1209 			if (hm != NULL)
1210 				hment_free(hm);
1211 		}
1212 		return;
1213 	}
1214 
1215 	/*
1216 	 * Install a new mapping in the page's mapping list
1217 	 */
1218 	if (!PTE_ISVALID(old_pte)) {
1219 		if (is_consist) {
1220 			hment_assign(ht, entry, pp, hm);
1221 			x86_hm_exit(pp);
1222 		} else {
1223 			ASSERT(flags & HAT_LOAD_NOCONSIST);
1224 		}
1225 		HTABLE_INC(ht->ht_valid_cnt);
1226 		PGCNT_INC(hat, l);
1227 		return;
1228 	}
1229 
1230 	/*
1231 	 * Remap's are more complicated:
1232 	 *  - HAT_LOAD_REMAP must be specified if changing the pfn.
1233 	 *    We also require that NOCONSIST be specified.
1234 	 *  - Otherwise only permission or caching bits may change.
1235 	 */
1236 	if (!PTE_ISPAGE(old_pte, l))
1237 		panic("non-null/page mapping pte=" FMT_PTE, old_pte);
1238 
1239 	if (PTE2PFN(old_pte, l) != PTE2PFN(pte, l)) {
1240 		ASSERT(flags & HAT_LOAD_REMAP);
1241 		ASSERT(flags & HAT_LOAD_NOCONSIST);
1242 		ASSERT(PTE_GET(old_pte, PT_NOCONSIST));
1243 		ASSERT(pf_is_memory(PTE2PFN(old_pte, l)) ==
1244 		    pf_is_memory(PTE2PFN(pte, l)));
1245 		ASSERT(!is_consist);
1246 	}
1247 
1248 	/*
1249 	 * We only let remaps change the bits for PFNs, permissions
1250 	 * or caching type.
1251 	 */
1252 	ASSERT(PTE_GET(old_pte, ~(PT_REMAP_BITS | PT_REF | PT_MOD)) ==
1253 	    PTE_GET(pte, ~PT_REMAP_BITS));
1254 
1255 	/*
1256 	 * A remap requires invalidating the TLBs, since remapping the
1257 	 * same PFN requires NOCONSIST, we don't have to sync R/M bits.
1258 	 */
1259 	hat_demap(hat, htable_e2va(ht, entry));
1260 
1261 	/*
1262 	 * We don't create any mapping list entries on a remap, so release
1263 	 * any allocated hment after we drop the mapping list lock.
1264 	 */
1265 	if (is_consist) {
1266 		x86_hm_exit(pp);
1267 		if (hm != NULL)
1268 			hment_free(hm);
1269 	}
1270 }
1271 
1272 /*
1273  * The t_hatdepth field is an 8-bit counter.  We use the lower seven bits
1274  * to track exactly how deep we are in the memload->kmem_alloc recursion.
1275  * If the depth is greater than 1, that indicates that we are performing a
1276  * hat operation to satisfy another hat operation.  To prevent infinite
1277  * recursion, we switch over to using pre-allocated "reserves" of htables
1278  * and hments.
1279  *
1280  * The uppermost bit is used to indicate that we are transitioning away
1281  * from being the reserves thread.  See hati_reserves_exit() for the
1282  * details.
1283  */
1284 #define	EXITING_FLAG		(1 << 7)
1285 #define	DEPTH_MASK		(~EXITING_FLAG)
1286 #define	HAT_DEPTH(t)		((t)->t_hatdepth & DEPTH_MASK)
1287 #define	EXITING_RESERVES(t)	((t)->t_hatdepth & EXITING_FLAG)
1288 
1289 /*
1290  * Access to reserves for HAT_NO_KALLOC is single threaded.
1291  * If someone else is in the reserves, we'll politely wait for them
1292  * to finish. This keeps normal hat_memload()s from eating up
1293  * the mappings needed to replenish the reserve.
1294  */
1295 static void
1296 hati_reserves_enter(uint_t kmem_for_hat)
1297 {
1298 	/*
1299 	 * 64 is an arbitrary number to catch serious problems.  I'm not
1300 	 * sure what the absolute maximum depth is, but it should be
1301 	 * substantially less than this.
1302 	 */
1303 	ASSERT(HAT_DEPTH(curthread) < 64);
1304 
1305 	/*
1306 	 * If we are doing a memload to satisfy a kmem operation, we enter
1307 	 * the reserves immediately; we don't wait to recurse to a second
1308 	 * level of memload.
1309 	 */
1310 	ASSERT(kmem_for_hat < 2);
1311 	curthread->t_hatdepth += (1 + kmem_for_hat);
1312 
1313 	if (hat_reserves_thread == curthread || use_boot_reserve)
1314 		return;
1315 
1316 	if (HAT_DEPTH(curthread) > 1 || hat_reserves_thread != NULL) {
1317 		mutex_enter(&hat_reserves_lock);
1318 		while (hat_reserves_thread != NULL)
1319 			cv_wait(&hat_reserves_cv, &hat_reserves_lock);
1320 
1321 		if (HAT_DEPTH(curthread) > 1)
1322 			hat_reserves_thread = curthread;
1323 
1324 		mutex_exit(&hat_reserves_lock);
1325 	}
1326 }
1327 
1328 /*
1329  * If we are the reserves_thread and we've finally finished with all our
1330  * memloads (ie. no longer doing hat slabs), we can release our use of the
1331  * reserve.
1332  */
1333 static void
1334 hati_reserves_exit(uint_t kmem_for_hat)
1335 {
1336 	ASSERT(kmem_for_hat < 2);
1337 	curthread->t_hatdepth -= (1 + kmem_for_hat);
1338 
1339 	/*
1340 	 * Simple case: either we are not the reserves thread, or we are
1341 	 * the reserves thread and we are nested deeply enough that we
1342 	 * should still be the reserves thread.
1343 	 *
1344 	 * Note: we may not become the reserves thread after we recursively
1345 	 * enter our second HAT routine, but we don't stop being the
1346 	 * reserves thread until we exit the toplevel HAT routine.  This is
1347 	 * to work around vmem's inability to determine when an allocation
1348 	 * should be satisfied from the hat_memload arena, which can lead
1349 	 * to an infinite loop of memload->vmem_populate->memload->.
1350 	 */
1351 	if (curthread != hat_reserves_thread || HAT_DEPTH(curthread) > 0 ||
1352 	    use_boot_reserve)
1353 		return;
1354 
1355 	mutex_enter(&hat_reserves_lock);
1356 	ASSERT(hat_reserves_thread == curthread);
1357 	hat_reserves_thread = NULL;
1358 	cv_broadcast(&hat_reserves_cv);
1359 	mutex_exit(&hat_reserves_lock);
1360 
1361 	/*
1362 	 * As we leave the reserves, we want to be sure the reserve lists
1363 	 * aren't overstocked.  Freeing excess reserves requires that we
1364 	 * call kmem_free(), which may require additional allocations,
1365 	 * causing us to re-enter the reserves.  To avoid infinite
1366 	 * recursion, we only try to adjust reserves at the very top level.
1367 	 */
1368 	if (!kmem_for_hat && !EXITING_RESERVES(curthread)) {
1369 		curthread->t_hatdepth |= EXITING_FLAG;
1370 		htable_adjust_reserve();
1371 		hment_adjust_reserve();
1372 		curthread->t_hatdepth &= (~EXITING_FLAG);
1373 	}
1374 
1375 	/*
1376 	 * just in case something went wrong in doing adjust reserves
1377 	 */
1378 	ASSERT(hat_reserves_thread != curthread);
1379 }
1380 
1381 /*
1382  * Internal routine to load a single page table entry.
1383  */
1384 static void
1385 hati_load_common(
1386 	hat_t		*hat,
1387 	uintptr_t	va,
1388 	page_t		*pp,
1389 	uint_t		attr,
1390 	uint_t		flags,
1391 	level_t		level,
1392 	pfn_t		pfn)
1393 {
1394 	htable_t	*ht;
1395 	uint_t		entry;
1396 	x86pte_t	pte;
1397 	uint_t		kmem_for_hat = (flags & HAT_NO_KALLOC) ? 1 : 0;
1398 
1399 	ASSERT(hat == kas.a_hat ||
1400 	    AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1401 
1402 	if (flags & HAT_LOAD_SHARE)
1403 		hat->hat_flags |= HAT_SHARED;
1404 
1405 	/*
1406 	 * Find the page table that maps this page if it already exists.
1407 	 */
1408 	ht = htable_lookup(hat, va, level);
1409 
1410 	/*
1411 	 * All threads go through hati_reserves_enter() to at least wait
1412 	 * for any existing reserves user to finish. This helps reduce
1413 	 * pressure on the reserves. In addition, if this thread needs
1414 	 * to become the new reserve user it will.
1415 	 */
1416 	hati_reserves_enter(kmem_for_hat);
1417 
1418 	ASSERT(HAT_DEPTH(curthread) == 1 || va >= kernelbase);
1419 
1420 	/*
1421 	 * Kernel memloads for HAT data should never use hments!
1422 	 * If it did that would seriously complicate the reserves system, since
1423 	 * hment_alloc() would need to know about HAT_NO_KALLOC.
1424 	 *
1425 	 * We also must have HAT_LOAD_NOCONSIST if page_t is NULL.
1426 	 */
1427 	if (HAT_DEPTH(curthread) > 1 || pp == NULL)
1428 		flags |= HAT_LOAD_NOCONSIST;
1429 
1430 	if (ht == NULL) {
1431 		ht = htable_create(hat, va, level, NULL);
1432 		ASSERT(ht != NULL);
1433 	}
1434 	entry = htable_va2entry(va, ht);
1435 
1436 	/*
1437 	 * a bunch of paranoid error checking
1438 	 */
1439 	ASSERT(ht->ht_busy > 0);
1440 	if (ht->ht_vaddr > va || va > HTABLE_LAST_PAGE(ht))
1441 		panic("hati_load_common: bad htable %p, va %p", ht, (void *)va);
1442 	ASSERT(ht->ht_level == level);
1443 
1444 	/*
1445 	 * construct the new PTE
1446 	 */
1447 	if (hat == kas.a_hat)
1448 		attr &= ~PROT_USER;
1449 	pte = hati_mkpte(pfn, attr, level, flags);
1450 	if (hat == kas.a_hat && va >= kernelbase)
1451 		PTE_SET(pte, mmu.pt_global);
1452 
1453 	/*
1454 	 * establish the mapping
1455 	 */
1456 	hati_pte_map(ht, entry, pp, pte, flags, NULL);
1457 
1458 	/*
1459 	 * release the htable and any reserves
1460 	 */
1461 	htable_release(ht);
1462 	hati_reserves_exit(kmem_for_hat);
1463 }
1464 
1465 /*
1466  * special case of hat_memload to deal with some kernel addrs for performance
1467  */
1468 static void
1469 hat_kmap_load(
1470 	caddr_t		addr,
1471 	page_t		*pp,
1472 	uint_t		attr,
1473 	uint_t		flags)
1474 {
1475 	uintptr_t	va = (uintptr_t)addr;
1476 	x86pte_t	pte;
1477 	pfn_t		pfn = page_pptonum(pp);
1478 	pgcnt_t		pg_off = mmu_btop(va - mmu.kmap_addr);
1479 	htable_t	*ht;
1480 	uint_t		entry;
1481 	void		*pte_ptr;
1482 
1483 	/*
1484 	 * construct the requested PTE
1485 	 */
1486 	attr &= ~PROT_USER;
1487 	attr |= HAT_STORECACHING_OK;
1488 	pte = hati_mkpte(pfn, attr, 0, flags);
1489 	PTE_SET(pte, mmu.pt_global);
1490 
1491 	/*
1492 	 * Figure out the pte_ptr and htable and use common code to finish up
1493 	 */
1494 	if (mmu.pae_hat)
1495 		pte_ptr = mmu.kmap_ptes + pg_off;
1496 	else
1497 		pte_ptr = (x86pte32_t *)mmu.kmap_ptes + pg_off;
1498 	ht = mmu.kmap_htables[(va - mmu.kmap_htables[0]->ht_vaddr) >>
1499 	    LEVEL_SHIFT(1)];
1500 	entry = htable_va2entry(va, ht);
1501 	hati_pte_map(ht, entry, pp, pte, flags, pte_ptr);
1502 }
1503 
1504 /*
1505  * hat_memload() - load a translation to the given page struct
1506  *
1507  * Flags for hat_memload/hat_devload/hat_*attr.
1508  *
1509  * 	HAT_LOAD	Default flags to load a translation to the page.
1510  *
1511  * 	HAT_LOAD_LOCK	Lock down mapping resources; hat_map(), hat_memload(),
1512  *			and hat_devload().
1513  *
1514  *	HAT_LOAD_NOCONSIST Do not add mapping to page_t mapping list.
1515  *			sets PT_NOCONSIST (soft bit)
1516  *
1517  *	HAT_LOAD_SHARE	A flag to hat_memload() to indicate h/w page tables
1518  *			that map some user pages (not kas) is shared by more
1519  *			than one process (eg. ISM).
1520  *
1521  *	HAT_LOAD_REMAP	Reload a valid pte with a different page frame.
1522  *
1523  *	HAT_NO_KALLOC	Do not kmem_alloc while creating the mapping; at this
1524  *			point, it's setting up mapping to allocate internal
1525  *			hat layer data structures.  This flag forces hat layer
1526  *			to tap its reserves in order to prevent infinite
1527  *			recursion.
1528  *
1529  * The following is a protection attribute (like PROT_READ, etc.)
1530  *
1531  *	HAT_NOSYNC	set PT_NOSYNC (soft bit) - this mapping's ref/mod bits
1532  *			are never cleared.
1533  *
1534  * Installing new valid PTE's and creation of the mapping list
1535  * entry are controlled under the same lock. It's derived from the
1536  * page_t being mapped.
1537  */
1538 static uint_t supported_memload_flags =
1539 	HAT_LOAD | HAT_LOAD_LOCK | HAT_LOAD_ADV | HAT_LOAD_NOCONSIST |
1540 	HAT_LOAD_SHARE | HAT_NO_KALLOC | HAT_LOAD_REMAP | HAT_LOAD_TEXT;
1541 
1542 void
1543 hat_memload(
1544 	hat_t		*hat,
1545 	caddr_t		addr,
1546 	page_t		*pp,
1547 	uint_t		attr,
1548 	uint_t		flags)
1549 {
1550 	uintptr_t	va = (uintptr_t)addr;
1551 	level_t		level = 0;
1552 	pfn_t		pfn = page_pptonum(pp);
1553 
1554 	HATIN(hat_memload, hat, addr, (size_t)MMU_PAGESIZE);
1555 	ASSERT(IS_PAGEALIGNED(va));
1556 	ASSERT(hat == kas.a_hat || va <= kernelbase);
1557 	ASSERT(hat == kas.a_hat ||
1558 	    AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1559 	ASSERT((flags & supported_memload_flags) == flags);
1560 
1561 	ASSERT(!IN_VA_HOLE(va));
1562 	ASSERT(!PP_ISFREE(pp));
1563 
1564 	/*
1565 	 * kernel address special case for performance.
1566 	 */
1567 	if (mmu.kmap_addr <= va && va < mmu.kmap_eaddr) {
1568 		ASSERT(hat == kas.a_hat);
1569 		hat_kmap_load(addr, pp, attr, flags);
1570 		return;
1571 	}
1572 
1573 	/*
1574 	 * This is used for memory with normal caching enabled, so
1575 	 * always set HAT_STORECACHING_OK.
1576 	 */
1577 	attr |= HAT_STORECACHING_OK;
1578 	hati_load_common(hat, va, pp, attr, flags, level, pfn);
1579 	HATOUT(hat_memload, hat, addr);
1580 }
1581 
1582 /*
1583  * Load the given array of page structs using large pages when possible
1584  */
1585 void
1586 hat_memload_array(
1587 	hat_t		*hat,
1588 	caddr_t		addr,
1589 	size_t		len,
1590 	page_t		**pages,
1591 	uint_t		attr,
1592 	uint_t		flags)
1593 {
1594 	uintptr_t	va = (uintptr_t)addr;
1595 	uintptr_t	eaddr = va + len;
1596 	level_t		level;
1597 	size_t		pgsize;
1598 	pgcnt_t		pgindx = 0;
1599 	pfn_t		pfn;
1600 	pgcnt_t		i;
1601 
1602 	HATIN(hat_memload_array, hat, addr, len);
1603 	ASSERT(IS_PAGEALIGNED(va));
1604 	ASSERT(hat == kas.a_hat || va + len <= kernelbase);
1605 	ASSERT(hat == kas.a_hat ||
1606 	    AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1607 	ASSERT((flags & supported_memload_flags) == flags);
1608 
1609 	/*
1610 	 * memload is used for memory with full caching enabled, so
1611 	 * set HAT_STORECACHING_OK.
1612 	 */
1613 	attr |= HAT_STORECACHING_OK;
1614 
1615 	/*
1616 	 * handle all pages using largest possible pagesize
1617 	 */
1618 	while (va < eaddr) {
1619 		/*
1620 		 * decide what level mapping to use (ie. pagesize)
1621 		 */
1622 		pfn = page_pptonum(pages[pgindx]);
1623 		for (level = mmu.max_page_level; ; --level) {
1624 			pgsize = LEVEL_SIZE(level);
1625 			if (level == 0)
1626 				break;
1627 			if (!IS_P2ALIGNED(va, pgsize) ||
1628 			    (eaddr - va) < pgsize ||
1629 			    !IS_P2ALIGNED(pfn << MMU_PAGESHIFT, pgsize))
1630 				continue;
1631 
1632 			/*
1633 			 * To use a large mapping of this size, all the
1634 			 * pages we are passed must be sequential subpages
1635 			 * of the large page.
1636 			 * hat_page_demote() can't change p_szc because
1637 			 * all pages are locked.
1638 			 */
1639 			if (pages[pgindx]->p_szc >= level) {
1640 				for (i = 0; i < mmu_btop(pgsize); ++i) {
1641 					if (pfn + i !=
1642 					    page_pptonum(pages[pgindx + i]))
1643 						break;
1644 					ASSERT(pages[pgindx + i]->p_szc >=
1645 					    level);
1646 					ASSERT(pages[pgindx] + i ==
1647 					    pages[pgindx + i]);
1648 				}
1649 				if (i == mmu_btop(pgsize))
1650 					break;
1651 			}
1652 		}
1653 
1654 		/*
1655 		 * Shared page tables for DISM might have a pre-existing
1656 		 * level 0 page table that wasn't unlinked from all the
1657 		 * sharing hats. If we hit this for a large page, back off
1658 		 * to using level 0 pages.
1659 		 *
1660 		 * This can't be made better (ie. use large pages) until we
1661 		 * track all the htable's sharing and rewrite hat_pageunload().
1662 		 * Note that would cost a pointer in htable_t for a rare case.
1663 		 *
1664 		 * Since the 32 bit kernel caches empty page tables, check
1665 		 * the kernel too.
1666 		 */
1667 		if ((hat == kas.a_hat || (hat->hat_flags & HAT_SHARED)) &&
1668 		    level > 0) {
1669 			htable_t *lower;
1670 
1671 			lower = htable_getpte(hat, va, NULL, NULL, level - 1);
1672 			if (lower != NULL) {
1673 				level = 0;
1674 				pgsize = LEVEL_SIZE(0);
1675 				htable_release(lower);
1676 			}
1677 		}
1678 
1679 		/*
1680 		 * load this page mapping
1681 		 */
1682 		ASSERT(!IN_VA_HOLE(va));
1683 		hati_load_common(hat, va, pages[pgindx], attr, flags,
1684 		    level, pfn);
1685 
1686 		/*
1687 		 * move to next page
1688 		 */
1689 		va += pgsize;
1690 		pgindx += mmu_btop(pgsize);
1691 	}
1692 	HATOUT(hat_memload_array, hat, addr);
1693 }
1694 
1695 /*
1696  * void hat_devload(hat, addr, len, pf, attr, flags)
1697  *	load/lock the given page frame number
1698  *
1699  * Advisory ordering attributes. Apply only to device mappings.
1700  *
1701  * HAT_STRICTORDER: the CPU must issue the references in order, as the
1702  *	programmer specified.  This is the default.
1703  * HAT_UNORDERED_OK: the CPU may reorder the references (this is all kinds
1704  *	of reordering; store or load with store or load).
1705  * HAT_MERGING_OK: merging and batching: the CPU may merge individual stores
1706  *	to consecutive locations (for example, turn two consecutive byte
1707  *	stores into one halfword store), and it may batch individual loads
1708  *	(for example, turn two consecutive byte loads into one halfword load).
1709  *	This also implies re-ordering.
1710  * HAT_LOADCACHING_OK: the CPU may cache the data it fetches and reuse it
1711  *	until another store occurs.  The default is to fetch new data
1712  *	on every load.  This also implies merging.
1713  * HAT_STORECACHING_OK: the CPU may keep the data in the cache and push it to
1714  *	the device (perhaps with other data) at a later time.  The default is
1715  *	to push the data right away.  This also implies load caching.
1716  *
1717  * Equivalent of hat_memload(), but can be used for device memory where
1718  * there are no page_t's and we support additional flags (write merging, etc).
1719  * Note that we can have large page mappings with this interface.
1720  */
1721 int supported_devload_flags = HAT_LOAD | HAT_LOAD_LOCK |
1722 	HAT_LOAD_NOCONSIST | HAT_STRICTORDER | HAT_UNORDERED_OK |
1723 	HAT_MERGING_OK | HAT_LOADCACHING_OK | HAT_STORECACHING_OK;
1724 
1725 void
1726 hat_devload(
1727 	hat_t		*hat,
1728 	caddr_t		addr,
1729 	size_t		len,
1730 	pfn_t		pfn,
1731 	uint_t		attr,
1732 	int		flags)
1733 {
1734 	uintptr_t	va = ALIGN2PAGE(addr);
1735 	uintptr_t	eva = va + len;
1736 	level_t		level;
1737 	size_t		pgsize;
1738 	page_t		*pp;
1739 	int		f;	/* per PTE copy of flags  - maybe modified */
1740 	uint_t		a;	/* per PTE copy of attr */
1741 
1742 	HATIN(hat_devload, hat, addr, len);
1743 	ASSERT(IS_PAGEALIGNED(va));
1744 	ASSERT(hat == kas.a_hat || eva <= kernelbase);
1745 	ASSERT(hat == kas.a_hat ||
1746 	    AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1747 	ASSERT((flags & supported_devload_flags) == flags);
1748 
1749 	/*
1750 	 * handle all pages
1751 	 */
1752 	while (va < eva) {
1753 
1754 		/*
1755 		 * decide what level mapping to use (ie. pagesize)
1756 		 */
1757 		for (level = mmu.max_page_level; ; --level) {
1758 			pgsize = LEVEL_SIZE(level);
1759 			if (level == 0)
1760 				break;
1761 			if (IS_P2ALIGNED(va, pgsize) &&
1762 			    (eva - va) >= pgsize &&
1763 			    IS_P2ALIGNED(pfn, mmu_btop(pgsize)))
1764 				break;
1765 		}
1766 
1767 		/*
1768 		 * Some kernel addresses have permanently existing page tables,
1769 		 * so be sure to use a compatible pagesize.
1770 		 */
1771 		if (hat == kas.a_hat && level > 0) {
1772 			htable_t *lower;
1773 
1774 			lower = htable_getpte(hat, va, NULL, NULL, level - 1);
1775 			if (lower != NULL) {
1776 				level = 0;
1777 				pgsize = LEVEL_SIZE(0);
1778 				htable_release(lower);
1779 			}
1780 		}
1781 
1782 		/*
1783 		 * If it is memory get page_t and allow caching (this happens
1784 		 * for the nucleus pages) - though HAT_PLAT_NOCACHE can be used
1785 		 * to override that. If we don't have a page_t, make sure
1786 		 * NOCONSIST is set.
1787 		 */
1788 		a = attr;
1789 		f = flags;
1790 		if (pf_is_memory(pfn)) {
1791 			if (!(a & HAT_PLAT_NOCACHE))
1792 				a |= HAT_STORECACHING_OK;
1793 
1794 			if (f & HAT_LOAD_NOCONSIST)
1795 				pp = NULL;
1796 			else
1797 				pp = page_numtopp_nolock(pfn);
1798 		} else {
1799 			pp = NULL;
1800 			f |= HAT_LOAD_NOCONSIST;
1801 		}
1802 
1803 		/*
1804 		 * load this page mapping
1805 		 */
1806 		ASSERT(!IN_VA_HOLE(va));
1807 		hati_load_common(hat, va, pp, a, f, level, pfn);
1808 
1809 		/*
1810 		 * move to next page
1811 		 */
1812 		va += pgsize;
1813 		pfn += mmu_btop(pgsize);
1814 	}
1815 	HATOUT(hat_devload, hat, addr);
1816 }
1817 
1818 /*
1819  * void hat_unlock(hat, addr, len)
1820  *	unlock the mappings to a given range of addresses
1821  *
1822  * Locks are tracked by ht_lock_cnt in the htable.
1823  */
1824 void
1825 hat_unlock(hat_t *hat, caddr_t addr, size_t len)
1826 {
1827 	uintptr_t	vaddr = (uintptr_t)addr;
1828 	uintptr_t	eaddr = vaddr + len;
1829 	htable_t	*ht = NULL;
1830 
1831 	/*
1832 	 * kernel entries are always locked, we don't track lock counts
1833 	 */
1834 	ASSERT(hat == kas.a_hat || eaddr <= kernelbase);
1835 	ASSERT(IS_PAGEALIGNED(vaddr));
1836 	ASSERT(IS_PAGEALIGNED(eaddr));
1837 	if (hat == kas.a_hat)
1838 		return;
1839 	if (eaddr > _userlimit)
1840 		panic("hat_unlock() address out of range - above _userlimit");
1841 
1842 	ASSERT(AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1843 	while (vaddr < eaddr) {
1844 		(void) htable_walk(hat, &ht, &vaddr, eaddr);
1845 		if (ht == NULL)
1846 			break;
1847 
1848 		ASSERT(!IN_VA_HOLE(vaddr));
1849 
1850 		if (ht->ht_lock_cnt < 1)
1851 			panic("hat_unlock(): lock_cnt < 1, "
1852 			    "htable=%p, vaddr=%p\n", ht, (caddr_t)vaddr);
1853 		HTABLE_LOCK_DEC(ht);
1854 
1855 		vaddr += LEVEL_SIZE(ht->ht_level);
1856 	}
1857 	if (ht)
1858 		htable_release(ht);
1859 }
1860 
1861 /*
1862  * Cross call service routine to demap a virtual page on
1863  * the current CPU or flush all mappings in TLB.
1864  */
1865 /*ARGSUSED*/
1866 static int
1867 hati_demap_func(xc_arg_t a1, xc_arg_t a2, xc_arg_t a3)
1868 {
1869 	hat_t	*hat = (hat_t *)a1;
1870 	caddr_t	addr = (caddr_t)a2;
1871 
1872 	/*
1873 	 * If the target hat isn't the kernel and this CPU isn't operating
1874 	 * in the target hat, we can ignore the cross call.
1875 	 */
1876 	if (hat != kas.a_hat && hat != CPU->cpu_current_hat)
1877 		return (0);
1878 
1879 	/*
1880 	 * For a normal address, we just flush one page mapping
1881 	 */
1882 	if ((uintptr_t)addr != DEMAP_ALL_ADDR) {
1883 		mmu_tlbflush_entry((caddr_t)addr);
1884 		return (0);
1885 	}
1886 
1887 	/*
1888 	 * Otherwise we reload cr3 to effect a complete TLB flush.
1889 	 *
1890 	 * A reload of cr3 on a VLP process also means we must also recopy in
1891 	 * the pte values from the struct hat
1892 	 */
1893 	if (hat->hat_flags & HAT_VLP) {
1894 #if defined(__amd64)
1895 		x86pte_t *vlpptep = CPU->cpu_hat_info->hci_vlp_l2ptes;
1896 
1897 		VLP_COPY(hat->hat_vlp_ptes, vlpptep);
1898 #elif defined(__i386)
1899 		reload_pae32(hat, CPU);
1900 #endif
1901 	}
1902 	reload_cr3();
1903 	return (0);
1904 }
1905 
1906 /*
1907  * Internal routine to do cross calls to invalidate a range of pages on
1908  * all CPUs using a given hat.
1909  */
1910 void
1911 hat_demap(hat_t *hat, uintptr_t va)
1912 {
1913 	extern int	flushes_require_xcalls;	/* from mp_startup.c */
1914 	cpuset_t	justme;
1915 
1916 	/*
1917 	 * If the hat is being destroyed, there are no more users, so
1918 	 * demap need not do anything.
1919 	 */
1920 	if (hat->hat_flags & HAT_FREEING)
1921 		return;
1922 
1923 	/*
1924 	 * If demapping from a shared pagetable, we best demap the
1925 	 * entire set of user TLBs, since we don't know what addresses
1926 	 * these were shared at.
1927 	 */
1928 	if (hat->hat_flags & HAT_SHARED) {
1929 		hat = kas.a_hat;
1930 		va = DEMAP_ALL_ADDR;
1931 	}
1932 
1933 	/*
1934 	 * if not running with multiple CPUs, don't use cross calls
1935 	 */
1936 	if (panicstr || !flushes_require_xcalls) {
1937 		(void) hati_demap_func((xc_arg_t)hat, (xc_arg_t)va, NULL);
1938 		return;
1939 	}
1940 
1941 
1942 	/*
1943 	 * All CPUs must see kernel hat changes.
1944 	 */
1945 	if (hat == kas.a_hat) {
1946 		kpreempt_disable();
1947 		xc_call((xc_arg_t)hat, (xc_arg_t)va, NULL,
1948 		    X_CALL_HIPRI, khat_cpuset, hati_demap_func);
1949 		kpreempt_enable();
1950 		return;
1951 	}
1952 
1953 	/*
1954 	 * Otherwise we notify CPUs currently running in this HAT
1955 	 */
1956 	hat_enter(hat);
1957 	kpreempt_disable();
1958 	CPUSET_ONLY(justme, CPU->cpu_id);
1959 	if (CPUSET_ISEQUAL(hat->hat_cpus, justme))
1960 		(void) hati_demap_func((xc_arg_t)hat, (xc_arg_t)va, NULL);
1961 	else
1962 		xc_call((xc_arg_t)hat, (xc_arg_t)va, NULL,
1963 		    X_CALL_HIPRI, hat->hat_cpus, hati_demap_func);
1964 	kpreempt_enable();
1965 	hat_exit(hat);
1966 }
1967 
1968 /*
1969  * Interior routine for HAT_UNLOADs from hat_unload_callback(),
1970  * hat_kmap_unload() OR from hat_steal() code.  This routine doesn't
1971  * handle releasing of the htables.
1972  */
1973 void
1974 hat_pte_unmap(
1975 	htable_t	*ht,
1976 	uint_t		entry,
1977 	uint_t		flags,
1978 	x86pte_t	old_pte,
1979 	void		*pte_ptr)
1980 {
1981 	hat_t		*hat = ht->ht_hat;
1982 	hment_t		*hm = NULL;
1983 	page_t		*pp = NULL;
1984 	level_t		l = ht->ht_level;
1985 	pfn_t		pfn;
1986 
1987 	/*
1988 	 * We always track the locking counts, even if nothing is unmapped
1989 	 */
1990 	if ((flags & HAT_UNLOAD_UNLOCK) != 0 && hat != kas.a_hat) {
1991 		ASSERT(ht->ht_lock_cnt > 0);
1992 		HTABLE_LOCK_DEC(ht);
1993 	}
1994 
1995 	/*
1996 	 * Figure out which page's mapping list lock to acquire using the PFN
1997 	 * passed in "old" PTE. We then attempt to invalidate the PTE.
1998 	 * If another thread, probably a hat_pageunload, has asynchronously
1999 	 * unmapped/remapped this address we'll loop here.
2000 	 */
2001 	ASSERT(ht->ht_busy > 0);
2002 	while (PTE_ISVALID(old_pte)) {
2003 		pfn = PTE2PFN(old_pte, l);
2004 		if (PTE_GET(old_pte, PT_NOCONSIST)) {
2005 			pp = NULL;
2006 		} else {
2007 			pp = page_numtopp_nolock(pfn);
2008 			if (pp == NULL) {
2009 				panic("no page_t, not NOCONSIST: old_pte="
2010 				    FMT_PTE " ht=%lx entry=0x%x pte_ptr=%lx",
2011 				    old_pte, (uintptr_t)ht, entry,
2012 				    (uintptr_t)pte_ptr);
2013 			}
2014 			x86_hm_enter(pp);
2015 		}
2016 
2017 		/*
2018 		 * If freeing the address space, check that the PTE
2019 		 * hasn't changed, as the mappings are no longer in use by
2020 		 * any thread, invalidation is unnecessary.
2021 		 * If not freeing, do a full invalidate.
2022 		 */
2023 		if (hat->hat_flags & HAT_FREEING)
2024 			old_pte = x86pte_get(ht, entry);
2025 		else
2026 			old_pte =
2027 			    x86pte_invalidate_pfn(ht, entry, pfn, pte_ptr);
2028 
2029 		/*
2030 		 * If the page hadn't changed we've unmapped it and can proceed
2031 		 */
2032 		if (PTE_ISVALID(old_pte) && PTE2PFN(old_pte, l) == pfn)
2033 			break;
2034 
2035 		/*
2036 		 * Otherwise, we'll have to retry with the current old_pte.
2037 		 * Drop the hment lock, since the pfn may have changed.
2038 		 */
2039 		if (pp != NULL) {
2040 			x86_hm_exit(pp);
2041 			pp = NULL;
2042 		} else {
2043 			ASSERT(PTE_GET(old_pte, PT_NOCONSIST));
2044 		}
2045 	}
2046 
2047 	/*
2048 	 * If the old mapping wasn't valid, there's nothing more to do
2049 	 */
2050 	if (!PTE_ISVALID(old_pte)) {
2051 		if (pp != NULL)
2052 			x86_hm_exit(pp);
2053 		return;
2054 	}
2055 
2056 	/*
2057 	 * Take care of syncing any MOD/REF bits and removing the hment.
2058 	 */
2059 	if (pp != NULL) {
2060 		if (!(flags & HAT_UNLOAD_NOSYNC))
2061 			hati_sync_pte_to_page(pp, old_pte, l);
2062 		hm = hment_remove(pp, ht, entry);
2063 		x86_hm_exit(pp);
2064 		if (hm != NULL)
2065 			hment_free(hm);
2066 	}
2067 
2068 	/*
2069 	 * Handle book keeping in the htable and hat
2070 	 */
2071 	ASSERT(ht->ht_valid_cnt > 0);
2072 	HTABLE_DEC(ht->ht_valid_cnt);
2073 	PGCNT_DEC(hat, l);
2074 }
2075 
2076 /*
2077  * very cheap unload implementation to special case some kernel addresses
2078  */
2079 static void
2080 hat_kmap_unload(caddr_t addr, size_t len, uint_t flags)
2081 {
2082 	uintptr_t	va = (uintptr_t)addr;
2083 	uintptr_t	eva = va + len;
2084 	pgcnt_t		pg_off;
2085 	htable_t	*ht;
2086 	uint_t		entry;
2087 	void		*pte_ptr;
2088 	x86pte_t	old_pte;
2089 
2090 	for (; va < eva; va += MMU_PAGESIZE) {
2091 		/*
2092 		 * Get the PTE
2093 		 */
2094 		pg_off = mmu_btop(va - mmu.kmap_addr);
2095 		if (mmu.pae_hat) {
2096 			pte_ptr = mmu.kmap_ptes + pg_off;
2097 			ATOMIC_LOAD64((x86pte_t *)pte_ptr, old_pte);
2098 		} else {
2099 			pte_ptr = (x86pte32_t *)mmu.kmap_ptes + pg_off;
2100 			old_pte = *(x86pte32_t *)pte_ptr;
2101 		}
2102 
2103 		/*
2104 		 * get the htable / entry
2105 		 */
2106 		ht = mmu.kmap_htables[(va - mmu.kmap_htables[0]->ht_vaddr)
2107 		    >> LEVEL_SHIFT(1)];
2108 		entry = htable_va2entry(va, ht);
2109 
2110 		/*
2111 		 * use mostly common code to unmap it.
2112 		 */
2113 		hat_pte_unmap(ht, entry, flags, old_pte, pte_ptr);
2114 	}
2115 }
2116 
2117 
2118 /*
2119  * unload a range of virtual address space (no callback)
2120  */
2121 void
2122 hat_unload(hat_t *hat, caddr_t addr, size_t len, uint_t flags)
2123 {
2124 	uintptr_t va = (uintptr_t)addr;
2125 	ASSERT(hat == kas.a_hat || va + len <= kernelbase);
2126 
2127 	/*
2128 	 * special case for performance.
2129 	 */
2130 	if (mmu.kmap_addr <= va && va < mmu.kmap_eaddr) {
2131 		ASSERT(hat == kas.a_hat);
2132 		hat_kmap_unload(addr, len, flags);
2133 		return;
2134 	}
2135 	hat_unload_callback(hat, addr, len, flags, NULL);
2136 }
2137 
2138 /*
2139  * Do the callbacks for ranges being unloaded.
2140  */
2141 typedef struct range_info {
2142 	uintptr_t	rng_va;
2143 	ulong_t		rng_cnt;
2144 	level_t		rng_level;
2145 } range_info_t;
2146 
2147 static void
2148 handle_ranges(hat_callback_t *cb, uint_t cnt, range_info_t *range)
2149 {
2150 	/*
2151 	 * do callbacks to upper level VM system
2152 	 */
2153 	while (cb != NULL && cnt > 0) {
2154 		--cnt;
2155 		cb->hcb_start_addr = (caddr_t)range[cnt].rng_va;
2156 		cb->hcb_end_addr = cb->hcb_start_addr;
2157 		cb->hcb_end_addr +=
2158 		    range[cnt].rng_cnt << LEVEL_SIZE(range[cnt].rng_level);
2159 		cb->hcb_function(cb);
2160 	}
2161 }
2162 
2163 /*
2164  * Unload a given range of addresses (has optional callback)
2165  *
2166  * Flags:
2167  * define	HAT_UNLOAD		0x00
2168  * define	HAT_UNLOAD_NOSYNC	0x02
2169  * define	HAT_UNLOAD_UNLOCK	0x04
2170  * define	HAT_UNLOAD_OTHER	0x08 - not used
2171  * define	HAT_UNLOAD_UNMAP	0x10 - same as HAT_UNLOAD
2172  */
2173 #define	MAX_UNLOAD_CNT (8)
2174 void
2175 hat_unload_callback(
2176 	hat_t		*hat,
2177 	caddr_t		addr,
2178 	size_t		len,
2179 	uint_t		flags,
2180 	hat_callback_t	*cb)
2181 {
2182 	uintptr_t	vaddr = (uintptr_t)addr;
2183 	uintptr_t	eaddr = vaddr + len;
2184 	htable_t	*ht = NULL;
2185 	uint_t		entry;
2186 	uintptr_t	contig_va = (uintptr_t)-1L;
2187 	range_info_t	r[MAX_UNLOAD_CNT];
2188 	uint_t		r_cnt = 0;
2189 	x86pte_t	old_pte;
2190 
2191 	HATIN(hat_unload_callback, hat, addr, len);
2192 	ASSERT(hat == kas.a_hat || eaddr <= kernelbase);
2193 	ASSERT(IS_PAGEALIGNED(vaddr));
2194 	ASSERT(IS_PAGEALIGNED(eaddr));
2195 
2196 	while (vaddr < eaddr) {
2197 		old_pte = htable_walk(hat, &ht, &vaddr, eaddr);
2198 		if (ht == NULL)
2199 			break;
2200 
2201 		ASSERT(!IN_VA_HOLE(vaddr));
2202 
2203 		if (vaddr < (uintptr_t)addr)
2204 			panic("hat_unload_callback(): unmap inside large page");
2205 
2206 		/*
2207 		 * We'll do the call backs for contiguous ranges
2208 		 */
2209 		if (vaddr != contig_va ||
2210 		    (r_cnt > 0 && r[r_cnt - 1].rng_level != ht->ht_level)) {
2211 			if (r_cnt == MAX_UNLOAD_CNT) {
2212 				handle_ranges(cb, r_cnt, r);
2213 				r_cnt = 0;
2214 			}
2215 			r[r_cnt].rng_va = vaddr;
2216 			r[r_cnt].rng_cnt = 0;
2217 			r[r_cnt].rng_level = ht->ht_level;
2218 			++r_cnt;
2219 		}
2220 
2221 		/*
2222 		 * Unload one mapping from the page tables.
2223 		 */
2224 		entry = htable_va2entry(vaddr, ht);
2225 		hat_pte_unmap(ht, entry, flags, old_pte, NULL);
2226 
2227 		ASSERT(ht->ht_level <= mmu.max_page_level);
2228 		vaddr += LEVEL_SIZE(ht->ht_level);
2229 		contig_va = vaddr;
2230 		++r[r_cnt - 1].rng_cnt;
2231 	}
2232 	if (ht)
2233 		htable_release(ht);
2234 
2235 	/*
2236 	 * handle last range for callbacks
2237 	 */
2238 	if (r_cnt > 0)
2239 		handle_ranges(cb, r_cnt, r);
2240 
2241 	HATOUT(hat_unload_callback, hat, addr);
2242 }
2243 
2244 /*
2245  * synchronize mapping with software data structures
2246  *
2247  * This interface is currently only used by the working set monitor
2248  * driver.
2249  */
2250 /*ARGSUSED*/
2251 void
2252 hat_sync(hat_t *hat, caddr_t addr, size_t len, uint_t flags)
2253 {
2254 	uintptr_t	vaddr = (uintptr_t)addr;
2255 	uintptr_t	eaddr = vaddr + len;
2256 	htable_t	*ht = NULL;
2257 	uint_t		entry;
2258 	x86pte_t	pte;
2259 	x86pte_t	save_pte;
2260 	x86pte_t	new;
2261 	page_t		*pp;
2262 
2263 	ASSERT(!IN_VA_HOLE(vaddr));
2264 	ASSERT(IS_PAGEALIGNED(vaddr));
2265 	ASSERT(IS_PAGEALIGNED(eaddr));
2266 	ASSERT(hat == kas.a_hat || eaddr <= kernelbase);
2267 
2268 	for (; vaddr < eaddr; vaddr += LEVEL_SIZE(ht->ht_level)) {
2269 try_again:
2270 		pte = htable_walk(hat, &ht, &vaddr, eaddr);
2271 		if (ht == NULL)
2272 			break;
2273 		entry = htable_va2entry(vaddr, ht);
2274 
2275 		if (PTE_GET(pte, PT_NOSYNC) ||
2276 		    PTE_GET(pte, PT_REF | PT_MOD) == 0)
2277 			continue;
2278 
2279 		/*
2280 		 * We need to acquire the mapping list lock to protect
2281 		 * against hat_pageunload(), hat_unload(), etc.
2282 		 */
2283 		pp = page_numtopp_nolock(PTE2PFN(pte, ht->ht_level));
2284 		if (pp == NULL)
2285 			break;
2286 		x86_hm_enter(pp);
2287 		save_pte = pte;
2288 		pte = x86pte_get(ht, entry);
2289 		if (pte != save_pte) {
2290 			x86_hm_exit(pp);
2291 			goto try_again;
2292 		}
2293 		if (PTE_GET(pte, PT_NOSYNC) ||
2294 		    PTE_GET(pte, PT_REF | PT_MOD) == 0) {
2295 			x86_hm_exit(pp);
2296 			continue;
2297 		}
2298 
2299 		/*
2300 		 * Need to clear ref or mod bits. We may compete with
2301 		 * hardware updating the R/M bits and have to try again.
2302 		 */
2303 		if (flags == HAT_SYNC_ZERORM) {
2304 			new = pte;
2305 			PTE_CLR(new, PT_REF | PT_MOD);
2306 			pte = hati_update_pte(ht, entry, pte, new);
2307 			if (pte != 0) {
2308 				x86_hm_exit(pp);
2309 				goto try_again;
2310 			}
2311 		} else {
2312 			/*
2313 			 * sync the PTE to the page_t
2314 			 */
2315 			hati_sync_pte_to_page(pp, save_pte, ht->ht_level);
2316 		}
2317 		x86_hm_exit(pp);
2318 	}
2319 	if (ht)
2320 		htable_release(ht);
2321 }
2322 
2323 /*
2324  * void	hat_map(hat, addr, len, flags)
2325  */
2326 /*ARGSUSED*/
2327 void
2328 hat_map(hat_t *hat, caddr_t addr, size_t len, uint_t flags)
2329 {
2330 	/* does nothing */
2331 }
2332 
2333 /*
2334  * uint_t hat_getattr(hat, addr, *attr)
2335  *	returns attr for <hat,addr> in *attr.  returns 0 if there was a
2336  *	mapping and *attr is valid, nonzero if there was no mapping and
2337  *	*attr is not valid.
2338  */
2339 uint_t
2340 hat_getattr(hat_t *hat, caddr_t addr, uint_t *attr)
2341 {
2342 	uintptr_t	vaddr = ALIGN2PAGE(addr);
2343 	htable_t	*ht = NULL;
2344 	x86pte_t	pte;
2345 
2346 	ASSERT(hat == kas.a_hat || vaddr < kernelbase);
2347 
2348 	if (IN_VA_HOLE(vaddr))
2349 		return ((uint_t)-1);
2350 
2351 	ht = htable_getpte(hat, vaddr, NULL, &pte, MAX_PAGE_LEVEL);
2352 	if (ht == NULL)
2353 		return ((uint_t)-1);
2354 
2355 	if (!PTE_ISVALID(pte) || !PTE_ISPAGE(pte, ht->ht_level)) {
2356 		htable_release(ht);
2357 		return ((uint_t)-1);
2358 	}
2359 
2360 	*attr = PROT_READ;
2361 	if (PTE_GET(pte, PT_WRITABLE))
2362 		*attr |= PROT_WRITE;
2363 	if (PTE_GET(pte, PT_USER))
2364 		*attr |= PROT_USER;
2365 	if (!PTE_GET(pte, mmu.pt_nx))
2366 		*attr |= PROT_EXEC;
2367 	if (PTE_GET(pte, PT_NOSYNC))
2368 		*attr |= HAT_NOSYNC;
2369 	htable_release(ht);
2370 	return (0);
2371 }
2372 
2373 /*
2374  * hat_updateattr() applies the given attribute change to an existing mapping
2375  */
2376 #define	HAT_LOAD_ATTR		1
2377 #define	HAT_SET_ATTR		2
2378 #define	HAT_CLR_ATTR		3
2379 
2380 static void
2381 hat_updateattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr, int what)
2382 {
2383 	uintptr_t	vaddr = (uintptr_t)addr;
2384 	uintptr_t	eaddr = (uintptr_t)addr + len;
2385 	htable_t	*ht = NULL;
2386 	uint_t		entry;
2387 	x86pte_t	oldpte, newpte;
2388 	page_t		*pp;
2389 
2390 	ASSERT(IS_PAGEALIGNED(vaddr));
2391 	ASSERT(IS_PAGEALIGNED(eaddr));
2392 	ASSERT(hat == kas.a_hat ||
2393 	    AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
2394 	for (; vaddr < eaddr; vaddr += LEVEL_SIZE(ht->ht_level)) {
2395 try_again:
2396 		oldpte = htable_walk(hat, &ht, &vaddr, eaddr);
2397 		if (ht == NULL)
2398 			break;
2399 		if (PTE_GET(oldpte, PT_NOCONSIST))
2400 			continue;
2401 
2402 		pp = page_numtopp_nolock(PTE2PFN(oldpte, ht->ht_level));
2403 		if (pp == NULL)
2404 			continue;
2405 		x86_hm_enter(pp);
2406 
2407 		newpte = oldpte;
2408 		/*
2409 		 * We found a page table entry in the desired range,
2410 		 * figure out the new attributes.
2411 		 */
2412 		if (what == HAT_SET_ATTR || what == HAT_LOAD_ATTR) {
2413 			if ((attr & PROT_WRITE) &&
2414 			    !PTE_GET(oldpte, PT_WRITABLE))
2415 				newpte |= PT_WRITABLE;
2416 
2417 			if ((attr & HAT_NOSYNC) && !PTE_GET(oldpte, PT_NOSYNC))
2418 				newpte |= PT_NOSYNC;
2419 
2420 			if ((attr & PROT_EXEC) && PTE_GET(oldpte, mmu.pt_nx))
2421 				newpte &= ~mmu.pt_nx;
2422 		}
2423 
2424 		if (what == HAT_LOAD_ATTR) {
2425 			if (!(attr & PROT_WRITE) &&
2426 			    PTE_GET(oldpte, PT_WRITABLE))
2427 				newpte &= ~PT_WRITABLE;
2428 
2429 			if (!(attr & HAT_NOSYNC) && PTE_GET(oldpte, PT_NOSYNC))
2430 				newpte &= ~PT_NOSYNC;
2431 
2432 			if (!(attr & PROT_EXEC) && !PTE_GET(oldpte, mmu.pt_nx))
2433 				newpte |= mmu.pt_nx;
2434 		}
2435 
2436 		if (what == HAT_CLR_ATTR) {
2437 			if ((attr & PROT_WRITE) && PTE_GET(oldpte, PT_WRITABLE))
2438 				newpte &= ~PT_WRITABLE;
2439 
2440 			if ((attr & HAT_NOSYNC) && PTE_GET(oldpte, PT_NOSYNC))
2441 				newpte &= ~PT_NOSYNC;
2442 
2443 			if ((attr & PROT_EXEC) && !PTE_GET(oldpte, mmu.pt_nx))
2444 				newpte |= mmu.pt_nx;
2445 		}
2446 
2447 		/*
2448 		 * what about PROT_READ or others? this code only handles:
2449 		 * EXEC, WRITE, NOSYNC
2450 		 */
2451 
2452 		/*
2453 		 * If new PTE really changed, update the table.
2454 		 */
2455 		if (newpte != oldpte) {
2456 			entry = htable_va2entry(vaddr, ht);
2457 			oldpte = hati_update_pte(ht, entry, oldpte, newpte);
2458 			if (oldpte != 0) {
2459 				x86_hm_exit(pp);
2460 				goto try_again;
2461 			}
2462 		}
2463 		x86_hm_exit(pp);
2464 	}
2465 	if (ht)
2466 		htable_release(ht);
2467 }
2468 
2469 /*
2470  * Various wrappers for hat_updateattr()
2471  */
2472 void
2473 hat_setattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr)
2474 {
2475 	ASSERT(hat == kas.a_hat || (uintptr_t)addr + len <= kernelbase);
2476 	hat_updateattr(hat, addr, len, attr, HAT_SET_ATTR);
2477 }
2478 
2479 void
2480 hat_clrattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr)
2481 {
2482 	ASSERT(hat == kas.a_hat || (uintptr_t)addr + len <= kernelbase);
2483 	hat_updateattr(hat, addr, len, attr, HAT_CLR_ATTR);
2484 }
2485 
2486 void
2487 hat_chgattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr)
2488 {
2489 	ASSERT(hat == kas.a_hat || (uintptr_t)addr + len <= kernelbase);
2490 	hat_updateattr(hat, addr, len, attr, HAT_LOAD_ATTR);
2491 }
2492 
2493 void
2494 hat_chgprot(hat_t *hat, caddr_t addr, size_t len, uint_t vprot)
2495 {
2496 	ASSERT(hat == kas.a_hat || (uintptr_t)addr + len <= kernelbase);
2497 	hat_updateattr(hat, addr, len, vprot & HAT_PROT_MASK, HAT_LOAD_ATTR);
2498 }
2499 
2500 /*ARGSUSED*/
2501 void
2502 hat_chgattr_pagedir(hat_t *hat, caddr_t addr, size_t len, uint_t attr)
2503 {
2504 	panic("hat_chgattr_pgdir() not supported - used by 80387 emulation");
2505 }
2506 
2507 /*
2508  * size_t hat_getpagesize(hat, addr)
2509  *	returns pagesize in bytes for <hat, addr>. returns -1 of there is
2510  *	no mapping. This is an advisory call.
2511  */
2512 ssize_t
2513 hat_getpagesize(hat_t *hat, caddr_t addr)
2514 {
2515 	uintptr_t	vaddr = ALIGN2PAGE(addr);
2516 	htable_t	*ht;
2517 	size_t		pagesize;
2518 
2519 	ASSERT(hat == kas.a_hat || vaddr < kernelbase);
2520 	if (IN_VA_HOLE(vaddr))
2521 		return (-1);
2522 	ht = htable_getpage(hat, vaddr, NULL);
2523 	if (ht == NULL)
2524 		return (-1);
2525 	pagesize = LEVEL_SIZE(ht->ht_level);
2526 	htable_release(ht);
2527 	return (pagesize);
2528 }
2529 
2530 
2531 
2532 /*
2533  * pfn_t hat_getpfnum(hat, addr)
2534  *	returns pfn for <hat, addr> or PFN_INVALID if mapping is invalid.
2535  */
2536 pfn_t
2537 hat_getpfnum(hat_t *hat, caddr_t addr)
2538 {
2539 	uintptr_t	vaddr = ALIGN2PAGE(addr);
2540 	htable_t	*ht;
2541 	uint_t		entry;
2542 	pfn_t		pfn = PFN_INVALID;
2543 
2544 	ASSERT(hat == kas.a_hat || vaddr < kernelbase);
2545 	if (khat_running == 0)
2546 		panic("hat_getpfnum(): called too early\n");
2547 
2548 	if (IN_VA_HOLE(vaddr))
2549 		return (PFN_INVALID);
2550 
2551 	/*
2552 	 * A very common use of hat_getpfnum() is from the DDI for kernel pages.
2553 	 * Use the kmap_ptes (which also covers the 32 bit heap) to speed
2554 	 * this up.
2555 	 */
2556 	if (mmu.kmap_addr <= vaddr && vaddr < mmu.kmap_eaddr) {
2557 		x86pte_t pte;
2558 		pgcnt_t pg_off;
2559 
2560 		pg_off = mmu_btop(vaddr - mmu.kmap_addr);
2561 		if (mmu.pae_hat) {
2562 			ATOMIC_LOAD64(mmu.kmap_ptes + pg_off, pte);
2563 		} else {
2564 			pte = ((x86pte32_t *)mmu.kmap_ptes)[pg_off];
2565 		}
2566 		if (!PTE_ISVALID(pte))
2567 			return (PFN_INVALID);
2568 		/*LINTED [use of constant 0 causes a silly lint warning] */
2569 		return (PTE2PFN(pte, 0));
2570 	}
2571 
2572 	ht = htable_getpage(hat, vaddr, &entry);
2573 	if (ht == NULL)
2574 		return (PFN_INVALID);
2575 	ASSERT(vaddr >= ht->ht_vaddr);
2576 	ASSERT(vaddr <= HTABLE_LAST_PAGE(ht));
2577 	pfn = PTE2PFN(x86pte_get(ht, entry), ht->ht_level);
2578 	if (ht->ht_level > 0)
2579 		pfn += mmu_btop(vaddr & LEVEL_OFFSET(ht->ht_level));
2580 	htable_release(ht);
2581 	return (pfn);
2582 }
2583 
2584 /*
2585  * hat_getkpfnum() is an obsolete DDI routine, and its use is discouraged.
2586  * Use hat_getpfnum(kas.a_hat, ...) instead.
2587  *
2588  * We'd like to return PFN_INVALID if the mappings have underlying page_t's
2589  * but can't right now due to the fact that some software has grown to use
2590  * this interface incorrectly. So for now when the interface is misused,
2591  * return a warning to the user that in the future it won't work in the
2592  * way they're abusing it, and carry on.
2593  *
2594  * Note that hat_getkpfnum() is never supported on amd64.
2595  */
2596 #if !defined(__amd64)
2597 pfn_t
2598 hat_getkpfnum(caddr_t addr)
2599 {
2600 	pfn_t	pfn;
2601 	int badcaller = 0;
2602 
2603 
2604 	if (khat_running == 0)
2605 		panic("hat_getkpfnum(): called too early\n");
2606 	if ((uintptr_t)addr < kernelbase)
2607 		return (PFN_INVALID);
2608 
2609 
2610 	if (segkpm && IS_KPM_ADDR(addr)) {
2611 		badcaller = 1;
2612 		pfn = hat_kpm_va2pfn(addr);
2613 	} else {
2614 		pfn = hat_getpfnum(kas.a_hat, addr);
2615 		badcaller = pf_is_memory(pfn);
2616 	}
2617 
2618 	if (badcaller)
2619 		hat_getkpfnum_badcall(caller());
2620 	return (pfn);
2621 }
2622 #endif /* __amd64 */
2623 
2624 /*
2625  * int hat_probe(hat, addr)
2626  *	return 0 if no valid mapping is present.  Faster version
2627  *	of hat_getattr in certain architectures.
2628  */
2629 int
2630 hat_probe(hat_t *hat, caddr_t addr)
2631 {
2632 	uintptr_t	vaddr = ALIGN2PAGE(addr);
2633 	uint_t		entry;
2634 	htable_t	*ht;
2635 	pgcnt_t		pg_off;
2636 
2637 	ASSERT(hat == kas.a_hat || vaddr < kernelbase);
2638 	ASSERT(hat == kas.a_hat ||
2639 	    AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
2640 	if (IN_VA_HOLE(vaddr))
2641 		return (0);
2642 
2643 	/*
2644 	 * Most common use of hat_probe is from segmap. We special case it
2645 	 * for performance.
2646 	 */
2647 	if (mmu.kmap_addr <= vaddr && vaddr < mmu.kmap_eaddr) {
2648 		pg_off = mmu_btop(vaddr - mmu.kmap_addr);
2649 		if (mmu.pae_hat)
2650 			return (PTE_ISVALID(mmu.kmap_ptes[pg_off]));
2651 		else
2652 			return (PTE_ISVALID(
2653 			    ((x86pte32_t *)mmu.kmap_ptes)[pg_off]));
2654 	}
2655 
2656 	ht = htable_getpage(hat, vaddr, &entry);
2657 	if (ht == NULL)
2658 		return (0);
2659 	htable_release(ht);
2660 	return (1);
2661 }
2662 
2663 /*
2664  * Simple implementation of ISM. hat_share() is just like hat_memload_array(),
2665  * except that we use the ism_hat's existing mappings to determine the pages
2666  * and protections to use for this hat. In case we find a properly aligned
2667  * and sized pagetable of 4K mappings, we will attempt to share the pagetable
2668  * itself.
2669  */
2670 /*ARGSUSED*/
2671 int
2672 hat_share(
2673 	hat_t		*hat,
2674 	caddr_t		addr,
2675 	hat_t		*ism_hat,
2676 	caddr_t		src_addr,
2677 	size_t		len,	/* almost useless value, see below.. */
2678 	uint_t		ismszc)
2679 {
2680 	uintptr_t	vaddr_start = (uintptr_t)addr;
2681 	uintptr_t	vaddr;
2682 	uintptr_t	pt_vaddr;
2683 	uintptr_t	eaddr = vaddr_start + len;
2684 	uintptr_t	ism_addr_start = (uintptr_t)src_addr;
2685 	uintptr_t	ism_addr = ism_addr_start;
2686 	uintptr_t	e_ism_addr = ism_addr + len;
2687 	htable_t	*ism_ht = NULL;
2688 	htable_t	*ht;
2689 	x86pte_t	pte;
2690 	page_t		*pp;
2691 	pfn_t		pfn;
2692 	level_t		l;
2693 	pgcnt_t		pgcnt;
2694 	uint_t		prot;
2695 	uint_t		valid_cnt;
2696 
2697 	/*
2698 	 * We might be asked to share an empty DISM hat by as_dup()
2699 	 */
2700 	ASSERT(hat != kas.a_hat);
2701 	ASSERT(eaddr <= kernelbase);
2702 	if (!(ism_hat->hat_flags & HAT_SHARED)) {
2703 		ASSERT(hat_get_mapped_size(ism_hat) == 0);
2704 		return (0);
2705 	}
2706 
2707 	/*
2708 	 * The SPT segment driver often passes us a size larger than there are
2709 	 * valid mappings. That's because it rounds the segment size up to a
2710 	 * large pagesize, even if the actual memory mapped by ism_hat is less.
2711 	 */
2712 	HATIN(hat_share, hat, addr, len);
2713 	ASSERT(IS_PAGEALIGNED(vaddr_start));
2714 	ASSERT(IS_PAGEALIGNED(ism_addr_start));
2715 	ASSERT(ism_hat->hat_flags & HAT_SHARED);
2716 	while (ism_addr < e_ism_addr) {
2717 		/*
2718 		 * use htable_walk to get the next valid ISM mapping
2719 		 */
2720 		pte = htable_walk(ism_hat, &ism_ht, &ism_addr, e_ism_addr);
2721 		if (ism_ht == NULL)
2722 			break;
2723 
2724 		/*
2725 		 * Find the largest page size we can use, based on the
2726 		 * ISM mapping size, our address alignment and the remaining
2727 		 * map length.
2728 		 */
2729 		vaddr = vaddr_start + (ism_addr - ism_addr_start);
2730 		for (l = ism_ht->ht_level; l > 0; --l) {
2731 			if (LEVEL_SIZE(l) <= eaddr - vaddr &&
2732 			    (vaddr & LEVEL_OFFSET(l)) == 0)
2733 				break;
2734 		}
2735 
2736 		/*
2737 		 * attempt to share the pagetable
2738 		 *
2739 		 * - only 4K pagetables are shared (ie. level == 0)
2740 		 * - the hat_share() length must cover the whole pagetable
2741 		 * - the shared address must align at level 1
2742 		 * - a shared PTE for this address already exists OR
2743 		 * - no page table for this address exists yet
2744 		 */
2745 		pt_vaddr =
2746 		    vaddr_start + (ism_ht->ht_vaddr - ism_addr_start);
2747 		if (ism_ht->ht_level == 0 &&
2748 		    ism_ht->ht_vaddr + LEVEL_SIZE(1) <= e_ism_addr &&
2749 		    (pt_vaddr & LEVEL_OFFSET(1)) == 0) {
2750 
2751 			ht = htable_lookup(hat, pt_vaddr, 0);
2752 			if (ht == NULL)
2753 				ht = htable_create(hat, pt_vaddr, 0, ism_ht);
2754 
2755 			if (ht->ht_level > 0 ||
2756 			    !(ht->ht_flags & HTABLE_SHARED_PFN)) {
2757 
2758 				htable_release(ht);
2759 
2760 			} else {
2761 
2762 				/*
2763 				 * share the page table
2764 				 */
2765 				ASSERT(ht->ht_level == 0);
2766 				ASSERT(ht->ht_shares == ism_ht);
2767 				valid_cnt = ism_ht->ht_valid_cnt;
2768 				atomic_add_long(&hat->hat_pages_mapped[0],
2769 				    valid_cnt - ht->ht_valid_cnt);
2770 				ht->ht_valid_cnt = valid_cnt;
2771 				htable_release(ht);
2772 				ism_addr = ism_ht->ht_vaddr + LEVEL_SIZE(1);
2773 				htable_release(ism_ht);
2774 				ism_ht = NULL;
2775 				continue;
2776 			}
2777 		}
2778 
2779 		/*
2780 		 * Unable to share the page table. Instead we will
2781 		 * create new mappings from the values in the ISM mappings.
2782 		 *
2783 		 * The ISM mapping might be larger than the share area,
2784 		 * be careful to trunctate it if needed.
2785 		 */
2786 		if (eaddr - vaddr >= LEVEL_SIZE(ism_ht->ht_level)) {
2787 			pgcnt = mmu_btop(LEVEL_SIZE(ism_ht->ht_level));
2788 		} else {
2789 			pgcnt = mmu_btop(eaddr - vaddr);
2790 			l = 0;
2791 		}
2792 
2793 		pfn = PTE2PFN(pte, ism_ht->ht_level);
2794 		ASSERT(pfn != PFN_INVALID);
2795 		while (pgcnt > 0) {
2796 			/*
2797 			 * Make a new pte for the PFN for this level.
2798 			 * Copy protections for the pte from the ISM pte.
2799 			 */
2800 			pp = page_numtopp_nolock(pfn);
2801 			ASSERT(pp != NULL);
2802 
2803 			prot = PROT_USER | PROT_READ | HAT_UNORDERED_OK;
2804 			if (PTE_GET(pte, PT_WRITABLE))
2805 				prot |= PROT_WRITE;
2806 			if (!PTE_GET(pte, PT_NX))
2807 				prot |= PROT_EXEC;
2808 
2809 			/*
2810 			 * XX64 -- can shm ever be written to swap?
2811 			 * if not we could use HAT_NOSYNC here.
2812 			 */
2813 			hati_load_common(hat, vaddr, pp, prot,
2814 			    HAT_LOAD, l, pfn);
2815 
2816 			vaddr += LEVEL_SIZE(l);
2817 			ism_addr += LEVEL_SIZE(l);
2818 			pfn += mmu_btop(LEVEL_SIZE(l));
2819 			pgcnt -= mmu_btop(LEVEL_SIZE(l));
2820 		}
2821 	}
2822 	if (ism_ht != NULL)
2823 		htable_release(ism_ht);
2824 
2825 	HATOUT(hat_share, hat, addr);
2826 	return (0);
2827 }
2828 
2829 
2830 /*
2831  * hat_unshare() is similar to hat_unload_callback(), but
2832  * we have to look for empty shared pagetables. Note that
2833  * hat_unshare() is always invoked against an entire segment.
2834  */
2835 /*ARGSUSED*/
2836 void
2837 hat_unshare(hat_t *hat, caddr_t addr, size_t len, uint_t ismszc)
2838 {
2839 	uintptr_t	vaddr = (uintptr_t)addr;
2840 	uintptr_t	eaddr = vaddr + len;
2841 	htable_t	*ht = NULL;
2842 	uint_t		need_demaps = 0;
2843 
2844 	ASSERT(hat != kas.a_hat);
2845 	ASSERT(eaddr <= kernelbase);
2846 	HATIN(hat_unshare, hat, addr, len);
2847 	ASSERT(IS_PAGEALIGNED(vaddr));
2848 	ASSERT(IS_PAGEALIGNED(eaddr));
2849 
2850 	/*
2851 	 * First go through and remove any shared pagetables.
2852 	 *
2853 	 * Note that it's ok to delay the demap until the entire range is
2854 	 * finished, because if hat_pageunload() were to unload a shared
2855 	 * pagetable page, its hat_demap() will do a global user TLB invalidate.
2856 	 */
2857 	while (vaddr < eaddr) {
2858 		ASSERT(!IN_VA_HOLE(vaddr));
2859 		/*
2860 		 * find the pagetable that would map the current address
2861 		 */
2862 		ht = htable_lookup(hat, vaddr, 0);
2863 		if (ht != NULL) {
2864 			if (ht->ht_flags & HTABLE_SHARED_PFN) {
2865 				/*
2866 				 * clear mapped pages count, set valid_cnt to 0
2867 				 * and let htable_release() finish the job
2868 				 */
2869 				atomic_add_long(&hat->hat_pages_mapped[0],
2870 				    -ht->ht_valid_cnt);
2871 				ht->ht_valid_cnt = 0;
2872 				need_demaps = 1;
2873 			}
2874 			htable_release(ht);
2875 		}
2876 		vaddr = (vaddr & LEVEL_MASK(1)) + LEVEL_SIZE(1);
2877 	}
2878 
2879 	/*
2880 	 * flush the TLBs - since we're probably dealing with MANY mappings
2881 	 * we do just one CR3 reload.
2882 	 */
2883 	if (!(hat->hat_flags & HAT_FREEING) && need_demaps)
2884 		hat_demap(hat, DEMAP_ALL_ADDR);
2885 
2886 	/*
2887 	 * Now go back and clean up any unaligned mappings that
2888 	 * couldn't share pagetables.
2889 	 */
2890 	hat_unload(hat, addr, len, HAT_UNLOAD_UNMAP);
2891 
2892 	HATOUT(hat_unshare, hat, addr);
2893 }
2894 
2895 
2896 /*
2897  * hat_reserve() does nothing
2898  */
2899 /*ARGSUSED*/
2900 void
2901 hat_reserve(struct as *as, caddr_t addr, size_t len)
2902 {
2903 }
2904 
2905 
2906 /*
2907  * Called when all mappings to a page should have write permission removed.
2908  * Mostly stolem from hat_pagesync()
2909  */
2910 static void
2911 hati_page_clrwrt(struct page *pp)
2912 {
2913 	hment_t		*hm = NULL;
2914 	htable_t	*ht;
2915 	uint_t		entry;
2916 	x86pte_t	old;
2917 	x86pte_t	new;
2918 	uint_t		pszc = 0;
2919 
2920 next_size:
2921 	/*
2922 	 * walk thru the mapping list clearing write permission
2923 	 */
2924 	x86_hm_enter(pp);
2925 	while ((hm = hment_walk(pp, &ht, &entry, hm)) != NULL) {
2926 		if (ht->ht_level < pszc)
2927 			continue;
2928 		old = x86pte_get(ht, entry);
2929 
2930 		for (;;) {
2931 			/*
2932 			 * Is this mapping of interest?
2933 			 */
2934 			if (PTE2PFN(old, ht->ht_level) != pp->p_pagenum ||
2935 			    PTE_GET(old, PT_WRITABLE) == 0)
2936 				break;
2937 
2938 			/*
2939 			 * Clear ref/mod writable bits. This requires cross
2940 			 * calls to ensure any executing TLBs see cleared bits.
2941 			 */
2942 			new = old;
2943 			PTE_CLR(new, PT_REF | PT_MOD | PT_WRITABLE);
2944 			old = hati_update_pte(ht, entry, old, new);
2945 			if (old != 0)
2946 				continue;
2947 
2948 			break;
2949 		}
2950 	}
2951 	x86_hm_exit(pp);
2952 	while (pszc < pp->p_szc) {
2953 		page_t *tpp;
2954 		pszc++;
2955 		tpp = PP_GROUPLEADER(pp, pszc);
2956 		if (pp != tpp) {
2957 			pp = tpp;
2958 			goto next_size;
2959 		}
2960 	}
2961 }
2962 
2963 /*
2964  * void hat_page_setattr(pp, flag)
2965  * void hat_page_clrattr(pp, flag)
2966  *	used to set/clr ref/mod bits.
2967  */
2968 void
2969 hat_page_setattr(struct page *pp, uint_t flag)
2970 {
2971 	vnode_t		*vp = pp->p_vnode;
2972 	kmutex_t	*vphm = NULL;
2973 	page_t		**listp;
2974 
2975 	if (PP_GETRM(pp, flag) == flag)
2976 		return;
2977 
2978 	if ((flag & P_MOD) != 0 && vp != NULL && IS_VMODSORT(vp)) {
2979 		vphm = page_vnode_mutex(vp);
2980 		mutex_enter(vphm);
2981 	}
2982 
2983 	PP_SETRM(pp, flag);
2984 
2985 	if (vphm != NULL) {
2986 
2987 		/*
2988 		 * Some File Systems examine v_pages for NULL w/o
2989 		 * grabbing the vphm mutex. Must not let it become NULL when
2990 		 * pp is the only page on the list.
2991 		 */
2992 		if (pp->p_vpnext != pp) {
2993 			page_vpsub(&vp->v_pages, pp);
2994 			if (vp->v_pages != NULL)
2995 				listp = &vp->v_pages->p_vpprev->p_vpnext;
2996 			else
2997 				listp = &vp->v_pages;
2998 			page_vpadd(listp, pp);
2999 		}
3000 		mutex_exit(vphm);
3001 	}
3002 }
3003 
3004 void
3005 hat_page_clrattr(struct page *pp, uint_t flag)
3006 {
3007 	vnode_t		*vp = pp->p_vnode;
3008 	kmutex_t	*vphm = NULL;
3009 	ASSERT(!(flag & ~(P_MOD | P_REF | P_RO)));
3010 
3011 	/*
3012 	 * for vnode with a sorted v_pages list, we need to change
3013 	 * the attributes and the v_pages list together under page_vnode_mutex.
3014 	 */
3015 	if ((flag & P_MOD) != 0 && vp != NULL && IS_VMODSORT(vp)) {
3016 		vphm = page_vnode_mutex(vp);
3017 		mutex_enter(vphm);
3018 	}
3019 
3020 	PP_CLRRM(pp, flag);
3021 
3022 	if (vphm != NULL) {
3023 
3024 		/*
3025 		 * Some File Systems examine v_pages for NULL w/o
3026 		 * grabbing the vphm mutex. Must not let it become NULL when
3027 		 * pp is the only page on the list.
3028 		 */
3029 		if (pp->p_vpnext != pp) {
3030 			page_vpsub(&vp->v_pages, pp);
3031 			page_vpadd(&vp->v_pages, pp);
3032 		}
3033 		mutex_exit(vphm);
3034 
3035 		/*
3036 		 * VMODSORT works by removing write permissions and getting
3037 		 * a fault when a page is made dirty. At this point
3038 		 * we need to remove write permission from all mappings
3039 		 * to this page.
3040 		 */
3041 		hati_page_clrwrt(pp);
3042 	}
3043 }
3044 
3045 /*
3046  *	If flag is specified, returns 0 if attribute is disabled
3047  *	and non zero if enabled.  If flag specifes multiple attributs
3048  *	then returns 0 if ALL atriibutes are disabled.  This is an advisory
3049  *	call.
3050  */
3051 uint_t
3052 hat_page_getattr(struct page *pp, uint_t flag)
3053 {
3054 	return (PP_GETRM(pp, flag));
3055 }
3056 
3057 
3058 /*
3059  * common code used by hat_pageunload() and hment_steal()
3060  */
3061 hment_t *
3062 hati_page_unmap(page_t *pp, htable_t *ht, uint_t entry)
3063 {
3064 	x86pte_t old_pte;
3065 	pfn_t pfn = pp->p_pagenum;
3066 	hment_t *hm;
3067 
3068 	/*
3069 	 * We need to acquire a hold on the htable in order to
3070 	 * do the invalidate. We know the htable must exist, since
3071 	 * unmap's don't release the htable until after removing any
3072 	 * hment. Having x86_hm_enter() keeps that from proceeding.
3073 	 */
3074 	htable_acquire(ht);
3075 
3076 	/*
3077 	 * Invalidate the PTE and remove the hment.
3078 	 */
3079 	old_pte = x86pte_invalidate_pfn(ht, entry, pfn, NULL);
3080 	if (PTE2PFN(old_pte, ht->ht_level) != pfn) {
3081 		panic("x86pte_invalidate_pfn() failure found PTE = " FMT_PTE
3082 		    " pfn being unmapped is %lx ht=0x%lx entry=0x%x",
3083 		    old_pte, pfn, (uintptr_t)ht, entry);
3084 	}
3085 
3086 	/*
3087 	 * Clean up all the htable information for this mapping
3088 	 */
3089 	ASSERT(ht->ht_valid_cnt > 0);
3090 	HTABLE_DEC(ht->ht_valid_cnt);
3091 	PGCNT_DEC(ht->ht_hat, ht->ht_level);
3092 
3093 	/*
3094 	 * sync ref/mod bits to the page_t
3095 	 */
3096 	if (PTE_GET(old_pte, PT_NOSYNC) == 0)
3097 		hati_sync_pte_to_page(pp, old_pte, ht->ht_level);
3098 
3099 	/*
3100 	 * Remove the mapping list entry for this page.
3101 	 */
3102 	hm = hment_remove(pp, ht, entry);
3103 
3104 	/*
3105 	 * drop the mapping list lock so that we might free the
3106 	 * hment and htable.
3107 	 */
3108 	x86_hm_exit(pp);
3109 	htable_release(ht);
3110 	return (hm);
3111 }
3112 
3113 /*
3114  * Unload all translations to a page. If the page is a subpage of a large
3115  * page, the large page mappings are also removed.
3116  *
3117  * The forceflags are unused.
3118  */
3119 
3120 /*ARGSUSED*/
3121 static int
3122 hati_pageunload(struct page *pp, uint_t pg_szcd, uint_t forceflag)
3123 {
3124 	page_t		*cur_pp = pp;
3125 	hment_t		*hm;
3126 	hment_t		*prev;
3127 	htable_t	*ht;
3128 	uint_t		entry;
3129 	level_t		level;
3130 
3131 	/*
3132 	 * The loop with next_size handles pages with multiple pagesize mappings
3133 	 */
3134 next_size:
3135 	for (;;) {
3136 
3137 		/*
3138 		 * Get a mapping list entry
3139 		 */
3140 		x86_hm_enter(cur_pp);
3141 		for (prev = NULL; ; prev = hm) {
3142 			hm = hment_walk(cur_pp, &ht, &entry, prev);
3143 			if (hm == NULL) {
3144 				x86_hm_exit(cur_pp);
3145 
3146 				/*
3147 				 * If not part of a larger page, we're done.
3148 				 */
3149 				if (cur_pp->p_szc <= pg_szcd)
3150 					return (0);
3151 
3152 				/*
3153 				 * Else check the next larger page size.
3154 				 * hat_page_demote() may decrease p_szc
3155 				 * but that's ok we'll just take an extra
3156 				 * trip discover there're no larger mappings
3157 				 * and return.
3158 				 */
3159 				++pg_szcd;
3160 				cur_pp = PP_GROUPLEADER(cur_pp, pg_szcd);
3161 				goto next_size;
3162 			}
3163 
3164 			/*
3165 			 * If this mapping size matches, remove it.
3166 			 */
3167 			level = ht->ht_level;
3168 			if (level == pg_szcd)
3169 				break;
3170 		}
3171 
3172 		/*
3173 		 * Remove the mapping list entry for this page.
3174 		 * Note this does the x86_hm_exit() for us.
3175 		 */
3176 		hm = hati_page_unmap(cur_pp, ht, entry);
3177 		if (hm != NULL)
3178 			hment_free(hm);
3179 	}
3180 }
3181 
3182 int
3183 hat_pageunload(struct page *pp, uint_t forceflag)
3184 {
3185 	ASSERT(PAGE_EXCL(pp));
3186 	return (hati_pageunload(pp, 0, forceflag));
3187 }
3188 
3189 /*
3190  * Unload all large mappings to pp and reduce by 1 p_szc field of every large
3191  * page level that included pp.
3192  *
3193  * pp must be locked EXCL. Even though no other constituent pages are locked
3194  * it's legal to unload large mappings to pp because all constituent pages of
3195  * large locked mappings have to be locked SHARED.  therefore if we have EXCL
3196  * lock on one of constituent pages none of the large mappings to pp are
3197  * locked.
3198  *
3199  * Change (always decrease) p_szc field starting from the last constituent
3200  * page and ending with root constituent page so that root's pszc always shows
3201  * the area where hat_page_demote() may be active.
3202  *
3203  * This mechanism is only used for file system pages where it's not always
3204  * possible to get EXCL locks on all constituent pages to demote the size code
3205  * (as is done for anonymous or kernel large pages).
3206  */
3207 void
3208 hat_page_demote(page_t *pp)
3209 {
3210 	uint_t		pszc;
3211 	uint_t		rszc;
3212 	uint_t		szc;
3213 	page_t		*rootpp;
3214 	page_t		*firstpp;
3215 	page_t		*lastpp;
3216 	pgcnt_t		pgcnt;
3217 
3218 	ASSERT(PAGE_EXCL(pp));
3219 	ASSERT(!PP_ISFREE(pp));
3220 	ASSERT(page_szc_lock_assert(pp));
3221 
3222 	if (pp->p_szc == 0)
3223 		return;
3224 
3225 	rootpp = PP_GROUPLEADER(pp, 1);
3226 	(void) hati_pageunload(rootpp, 1, HAT_FORCE_PGUNLOAD);
3227 
3228 	/*
3229 	 * all large mappings to pp are gone
3230 	 * and no new can be setup since pp is locked exclusively.
3231 	 *
3232 	 * Lock the root to make sure there's only one hat_page_demote()
3233 	 * outstanding within the area of this root's pszc.
3234 	 *
3235 	 * Second potential hat_page_demote() is already eliminated by upper
3236 	 * VM layer via page_szc_lock() but we don't rely on it and use our
3237 	 * own locking (so that upper layer locking can be changed without
3238 	 * assumptions that hat depends on upper layer VM to prevent multiple
3239 	 * hat_page_demote() to be issued simultaneously to the same large
3240 	 * page).
3241 	 */
3242 again:
3243 	pszc = pp->p_szc;
3244 	if (pszc == 0)
3245 		return;
3246 	rootpp = PP_GROUPLEADER(pp, pszc);
3247 	x86_hm_enter(rootpp);
3248 	/*
3249 	 * If root's p_szc is different from pszc we raced with another
3250 	 * hat_page_demote().  Drop the lock and try to find the root again.
3251 	 * If root's p_szc is greater than pszc previous hat_page_demote() is
3252 	 * not done yet.  Take and release mlist lock of root's root to wait
3253 	 * for previous hat_page_demote() to complete.
3254 	 */
3255 	if ((rszc = rootpp->p_szc) != pszc) {
3256 		x86_hm_exit(rootpp);
3257 		if (rszc > pszc) {
3258 			/* p_szc of a locked non free page can't increase */
3259 			ASSERT(pp != rootpp);
3260 
3261 			rootpp = PP_GROUPLEADER(rootpp, rszc);
3262 			x86_hm_enter(rootpp);
3263 			x86_hm_exit(rootpp);
3264 		}
3265 		goto again;
3266 	}
3267 	ASSERT(pp->p_szc == pszc);
3268 
3269 	/*
3270 	 * Decrement by 1 p_szc of every constituent page of a region that
3271 	 * covered pp. For example if original szc is 3 it gets changed to 2
3272 	 * everywhere except in region 2 that covered pp. Region 2 that
3273 	 * covered pp gets demoted to 1 everywhere except in region 1 that
3274 	 * covered pp. The region 1 that covered pp is demoted to region
3275 	 * 0. It's done this way because from region 3 we removed level 3
3276 	 * mappings, from region 2 that covered pp we removed level 2 mappings
3277 	 * and from region 1 that covered pp we removed level 1 mappings.  All
3278 	 * changes are done from from high pfn's to low pfn's so that roots
3279 	 * are changed last allowing one to know the largest region where
3280 	 * hat_page_demote() is stil active by only looking at the root page.
3281 	 *
3282 	 * This algorithm is implemented in 2 while loops. First loop changes
3283 	 * p_szc of pages to the right of pp's level 1 region and second
3284 	 * loop changes p_szc of pages of level 1 region that covers pp
3285 	 * and all pages to the left of level 1 region that covers pp.
3286 	 * In the first loop p_szc keeps dropping with every iteration
3287 	 * and in the second loop it keeps increasing with every iteration.
3288 	 *
3289 	 * First loop description: Demote pages to the right of pp outside of
3290 	 * level 1 region that covers pp.  In every iteration of the while
3291 	 * loop below find the last page of szc region and the first page of
3292 	 * (szc - 1) region that is immediately to the right of (szc - 1)
3293 	 * region that covers pp.  From last such page to first such page
3294 	 * change every page's szc to szc - 1. Decrement szc and continue
3295 	 * looping until szc is 1. If pp belongs to the last (szc - 1) region
3296 	 * of szc region skip to the next iteration.
3297 	 */
3298 	szc = pszc;
3299 	while (szc > 1) {
3300 		lastpp = PP_GROUPLEADER(pp, szc);
3301 		pgcnt = page_get_pagecnt(szc);
3302 		lastpp += pgcnt - 1;
3303 		firstpp = PP_GROUPLEADER(pp, (szc - 1));
3304 		pgcnt = page_get_pagecnt(szc - 1);
3305 		if (lastpp - firstpp < pgcnt) {
3306 			szc--;
3307 			continue;
3308 		}
3309 		firstpp += pgcnt;
3310 		while (lastpp != firstpp) {
3311 			ASSERT(lastpp->p_szc == pszc);
3312 			lastpp->p_szc = szc - 1;
3313 			lastpp--;
3314 		}
3315 		firstpp->p_szc = szc - 1;
3316 		szc--;
3317 	}
3318 
3319 	/*
3320 	 * Second loop description:
3321 	 * First iteration changes p_szc to 0 of every
3322 	 * page of level 1 region that covers pp.
3323 	 * Subsequent iterations find last page of szc region
3324 	 * immediately to the left of szc region that covered pp
3325 	 * and first page of (szc + 1) region that covers pp.
3326 	 * From last to first page change p_szc of every page to szc.
3327 	 * Increment szc and continue looping until szc is pszc.
3328 	 * If pp belongs to the fist szc region of (szc + 1) region
3329 	 * skip to the next iteration.
3330 	 *
3331 	 */
3332 	szc = 0;
3333 	while (szc < pszc) {
3334 		firstpp = PP_GROUPLEADER(pp, (szc + 1));
3335 		if (szc == 0) {
3336 			pgcnt = page_get_pagecnt(1);
3337 			lastpp = firstpp + (pgcnt - 1);
3338 		} else {
3339 			lastpp = PP_GROUPLEADER(pp, szc);
3340 			if (firstpp == lastpp) {
3341 				szc++;
3342 				continue;
3343 			}
3344 			lastpp--;
3345 			pgcnt = page_get_pagecnt(szc);
3346 		}
3347 		while (lastpp != firstpp) {
3348 			ASSERT(lastpp->p_szc == pszc);
3349 			lastpp->p_szc = szc;
3350 			lastpp--;
3351 		}
3352 		firstpp->p_szc = szc;
3353 		if (firstpp == rootpp)
3354 			break;
3355 		szc++;
3356 	}
3357 	x86_hm_exit(rootpp);
3358 }
3359 
3360 /*
3361  * get hw stats from hardware into page struct and reset hw stats
3362  * returns attributes of page
3363  * Flags for hat_pagesync, hat_getstat, hat_sync
3364  *
3365  * define	HAT_SYNC_ZERORM		0x01
3366  *
3367  * Additional flags for hat_pagesync
3368  *
3369  * define	HAT_SYNC_STOPON_REF	0x02
3370  * define	HAT_SYNC_STOPON_MOD	0x04
3371  * define	HAT_SYNC_STOPON_RM	0x06
3372  * define	HAT_SYNC_STOPON_SHARED	0x08
3373  */
3374 uint_t
3375 hat_pagesync(struct page *pp, uint_t flags)
3376 {
3377 	hment_t		*hm = NULL;
3378 	htable_t	*ht;
3379 	uint_t		entry;
3380 	x86pte_t	old, save_old;
3381 	x86pte_t	new;
3382 	uchar_t		nrmbits = P_REF|P_MOD|P_RO;
3383 	extern ulong_t	po_share;
3384 	page_t		*save_pp = pp;
3385 	uint_t		pszc = 0;
3386 
3387 	ASSERT(PAGE_LOCKED(pp) || panicstr);
3388 
3389 	if (PP_ISRO(pp) && (flags & HAT_SYNC_STOPON_MOD))
3390 		return (pp->p_nrm & nrmbits);
3391 
3392 	if ((flags & HAT_SYNC_ZERORM) == 0) {
3393 
3394 		if ((flags & HAT_SYNC_STOPON_REF) != 0 && PP_ISREF(pp))
3395 			return (pp->p_nrm & nrmbits);
3396 
3397 		if ((flags & HAT_SYNC_STOPON_MOD) != 0 && PP_ISMOD(pp))
3398 			return (pp->p_nrm & nrmbits);
3399 
3400 		if ((flags & HAT_SYNC_STOPON_SHARED) != 0 &&
3401 		    hat_page_getshare(pp) > po_share) {
3402 			if (PP_ISRO(pp))
3403 				PP_SETREF(pp);
3404 			return (pp->p_nrm & nrmbits);
3405 		}
3406 	}
3407 
3408 next_size:
3409 	/*
3410 	 * walk thru the mapping list syncing (and clearing) ref/mod bits.
3411 	 */
3412 	x86_hm_enter(pp);
3413 	while ((hm = hment_walk(pp, &ht, &entry, hm)) != NULL) {
3414 		if (ht->ht_level < pszc)
3415 			continue;
3416 		old = x86pte_get(ht, entry);
3417 try_again:
3418 
3419 		ASSERT(PTE2PFN(old, ht->ht_level) == pp->p_pagenum);
3420 
3421 		if (PTE_GET(old, PT_REF | PT_MOD) == 0)
3422 			continue;
3423 
3424 		save_old = old;
3425 		if ((flags & HAT_SYNC_ZERORM) != 0) {
3426 
3427 			/*
3428 			 * Need to clear ref or mod bits. Need to demap
3429 			 * to make sure any executing TLBs see cleared bits.
3430 			 */
3431 			new = old;
3432 			PTE_CLR(new, PT_REF | PT_MOD);
3433 			old = hati_update_pte(ht, entry, old, new);
3434 			if (old != 0)
3435 				goto try_again;
3436 
3437 			old = save_old;
3438 		}
3439 
3440 		/*
3441 		 * Sync the PTE
3442 		 */
3443 		if (!(flags & HAT_SYNC_ZERORM) && PTE_GET(old, PT_NOSYNC) == 0)
3444 			hati_sync_pte_to_page(pp, old, ht->ht_level);
3445 
3446 		/*
3447 		 * can stop short if we found a ref'd or mod'd page
3448 		 */
3449 		if ((flags & HAT_SYNC_STOPON_MOD) && PP_ISMOD(save_pp) ||
3450 		    (flags & HAT_SYNC_STOPON_REF) && PP_ISREF(save_pp)) {
3451 			x86_hm_exit(pp);
3452 			return (save_pp->p_nrm & nrmbits);
3453 		}
3454 	}
3455 	x86_hm_exit(pp);
3456 	while (pszc < pp->p_szc) {
3457 		page_t *tpp;
3458 		pszc++;
3459 		tpp = PP_GROUPLEADER(pp, pszc);
3460 		if (pp != tpp) {
3461 			pp = tpp;
3462 			goto next_size;
3463 		}
3464 	}
3465 	return (save_pp->p_nrm & nrmbits);
3466 }
3467 
3468 /*
3469  * returns approx number of mappings to this pp.  A return of 0 implies
3470  * there are no mappings to the page.
3471  */
3472 ulong_t
3473 hat_page_getshare(page_t *pp)
3474 {
3475 	uint_t cnt;
3476 	cnt = hment_mapcnt(pp);
3477 	return (cnt);
3478 }
3479 
3480 /*
3481  * hat_softlock isn't supported anymore
3482  */
3483 /*ARGSUSED*/
3484 faultcode_t
3485 hat_softlock(
3486 	hat_t *hat,
3487 	caddr_t addr,
3488 	size_t *len,
3489 	struct page **page_array,
3490 	uint_t flags)
3491 {
3492 	return (FC_NOSUPPORT);
3493 }
3494 
3495 
3496 
3497 /*
3498  * Routine to expose supported HAT features to platform independent code.
3499  */
3500 /*ARGSUSED*/
3501 int
3502 hat_supported(enum hat_features feature, void *arg)
3503 {
3504 	switch (feature) {
3505 
3506 	case HAT_SHARED_PT:	/* this is really ISM */
3507 		return (1);
3508 
3509 	case HAT_DYNAMIC_ISM_UNMAP:
3510 		return (0);
3511 
3512 	case HAT_VMODSORT:
3513 		return (1);
3514 
3515 	default:
3516 		panic("hat_supported() - unknown feature");
3517 	}
3518 	return (0);
3519 }
3520 
3521 /*
3522  * Called when a thread is exiting and has been switched to the kernel AS
3523  */
3524 void
3525 hat_thread_exit(kthread_t *thd)
3526 {
3527 	ASSERT(thd->t_procp->p_as == &kas);
3528 	hat_switch(thd->t_procp->p_as->a_hat);
3529 }
3530 
3531 /*
3532  * Setup the given brand new hat structure as the new HAT on this cpu's mmu.
3533  */
3534 /*ARGSUSED*/
3535 void
3536 hat_setup(hat_t *hat, int flags)
3537 {
3538 	kpreempt_disable();
3539 
3540 	hat_switch(hat);
3541 
3542 	kpreempt_enable();
3543 }
3544 
3545 /*
3546  * Prepare for a CPU private mapping for the given address.
3547  *
3548  * The address can only be used from a single CPU and can be remapped
3549  * using hat_mempte_remap().  Return the address of the PTE.
3550  *
3551  * We do the htable_create() if necessary and increment the valid count so
3552  * the htable can't disappear.  We also hat_devload() the page table into
3553  * kernel so that the PTE is quickly accessed.
3554  */
3555 void *
3556 hat_mempte_kern_setup(caddr_t addr, void *pt)
3557 {
3558 	uintptr_t	va = (uintptr_t)addr;
3559 	htable_t	*ht;
3560 	uint_t		entry;
3561 	x86pte_t	oldpte;
3562 	caddr_t		p = (caddr_t)pt;
3563 
3564 	ASSERT(IS_PAGEALIGNED(va));
3565 	ASSERT(!IN_VA_HOLE(va));
3566 	ht = htable_getpte(kas.a_hat, va, &entry, &oldpte, 0);
3567 	if (ht == NULL) {
3568 		/*
3569 		 * Note that we don't need a hat_reserves_exit() check
3570 		 * for this htable_create(), since that'll be done by the
3571 		 * hat_devload() just below.
3572 		 */
3573 		ht = htable_create(kas.a_hat, va, 0, NULL);
3574 		entry = htable_va2entry(va, ht);
3575 		ASSERT(ht->ht_level == 0);
3576 		oldpte = x86pte_get(ht, entry);
3577 	}
3578 	if (PTE_ISVALID(oldpte))
3579 		panic("hat_mempte_setup(): address already mapped"
3580 		    "ht=%p, entry=%d, pte=" FMT_PTE, ht, entry, oldpte);
3581 
3582 	/*
3583 	 * increment ht_valid_cnt so that the pagetable can't disappear
3584 	 */
3585 	HTABLE_INC(ht->ht_valid_cnt);
3586 
3587 	/*
3588 	 * now we need to map the page holding the pagetable for va into
3589 	 * the kernel's address space.
3590 	 */
3591 	hat_devload(kas.a_hat, p, MMU_PAGESIZE, ht->ht_pfn,
3592 	    PROT_READ | PROT_WRITE | HAT_NOSYNC | HAT_UNORDERED_OK,
3593 	    HAT_LOAD | HAT_LOAD_NOCONSIST);
3594 
3595 	/*
3596 	 * return the PTE address to the caller.
3597 	 */
3598 	htable_release(ht);
3599 	p += entry << mmu.pte_size_shift;
3600 	return ((void *)p);
3601 }
3602 
3603 /*
3604  * Prepare for a CPU private mapping for the given address.
3605  */
3606 void *
3607 hat_mempte_setup(caddr_t addr)
3608 {
3609 	x86pte_t	*p;
3610 
3611 	p = vmem_alloc(heap_arena, MMU_PAGESIZE, VM_SLEEP);
3612 	return (hat_mempte_kern_setup(addr, p));
3613 }
3614 
3615 /*
3616  * Release a CPU private mapping for the given address.
3617  * We decrement the htable valid count so it might be destroyed.
3618  */
3619 void
3620 hat_mempte_release(caddr_t addr, void *pteptr)
3621 {
3622 	htable_t	*ht;
3623 	uintptr_t	va = ALIGN2PAGE(pteptr);
3624 
3625 	/*
3626 	 * first invalidate any left over mapping and decrement the
3627 	 * htable's mapping count
3628 	 */
3629 	if (mmu.pae_hat)
3630 		*(x86pte_t *)pteptr = 0;
3631 	else
3632 		*(x86pte32_t *)pteptr = 0;
3633 	mmu_tlbflush_entry(addr);
3634 	ht = htable_getpte(kas.a_hat, ALIGN2PAGE(addr), NULL, NULL, 0);
3635 	if (ht == NULL)
3636 		panic("hat_mempte_release(): invalid address");
3637 	ASSERT(ht->ht_level == 0);
3638 	HTABLE_DEC(ht->ht_valid_cnt);
3639 	htable_release(ht);
3640 
3641 	/*
3642 	 * now blow away the kernel mapping to the page table page
3643 	 * XX64 -- see comment in hat_mempte_setup()
3644 	 */
3645 	hat_unload_callback(kas.a_hat, (caddr_t)va, MMU_PAGESIZE,
3646 	    HAT_UNLOAD, NULL);
3647 }
3648 
3649 /*
3650  * Apply a temporary CPU private mapping to a page. We flush the TLB only
3651  * on this CPU, so this ought to have been called with preemption disabled.
3652  */
3653 void
3654 hat_mempte_remap(
3655 	pfn_t pfn,
3656 	caddr_t addr,
3657 	void *pteptr,
3658 	uint_t attr,
3659 	uint_t flags)
3660 {
3661 	uintptr_t	va = (uintptr_t)addr;
3662 	x86pte_t	pte;
3663 
3664 	/*
3665 	 * Remap the given PTE to the new page's PFN. Invalidate only
3666 	 * on this CPU.
3667 	 */
3668 #ifdef DEBUG
3669 	htable_t	*ht;
3670 	uint_t		entry;
3671 
3672 	ASSERT(IS_PAGEALIGNED(va));
3673 	ASSERT(!IN_VA_HOLE(va));
3674 	ht = htable_getpte(kas.a_hat, va, &entry, NULL, 0);
3675 	ASSERT(ht != NULL);
3676 	ASSERT(ht->ht_level == 0);
3677 	ASSERT(ht->ht_valid_cnt > 0);
3678 	htable_release(ht);
3679 #endif
3680 	pte = hati_mkpte(pfn, attr, 0, flags);
3681 	if (mmu.pae_hat)
3682 		*(x86pte_t *)pteptr = pte;
3683 	else
3684 		*(x86pte32_t *)pteptr = (x86pte32_t)pte;
3685 	mmu_tlbflush_entry(addr);
3686 }
3687 
3688 
3689 
3690 /*
3691  * Hat locking functions
3692  * XXX - these two functions are currently being used by hatstats
3693  * 	they can be removed by using a per-as mutex for hatstats.
3694  */
3695 void
3696 hat_enter(hat_t *hat)
3697 {
3698 	mutex_enter(&hat->hat_mutex);
3699 }
3700 
3701 void
3702 hat_exit(hat_t *hat)
3703 {
3704 	mutex_exit(&hat->hat_mutex);
3705 }
3706 
3707 
3708 /*
3709  * Used by hat_kern_setup() to create initial kernel HAT mappings from
3710  * the boot loader's mappings.
3711  *
3712  * - size is either PAGESIZE or some multiple of a level one pagesize
3713  * - there may not be page_t's for every pfn. (ie. the nucleus pages)
3714  * - pfn's are continguous for the given va range (va to va + size * cnt)
3715  */
3716 void
3717 hati_kern_setup_load(
3718 	uintptr_t va,	/* starting va of range to map */
3719 	size_t size,	/* either PAGESIZE or multiple of large page size */
3720 	pfn_t pfn,	/* starting PFN */
3721 	pgcnt_t cnt,	/* number of mappings, (cnt * size) == total size */
3722 	uint_t prot)	/* protections (PROT_READ, PROT_WRITE, PROT_EXEC) */
3723 {
3724 	level_t level = (size == MMU_PAGESIZE ? 0 : 1);
3725 	size_t bytes = size * cnt;
3726 	size_t pgsize = LEVEL_SIZE(level);
3727 	page_t *pp;
3728 	uint_t flags = HAT_LOAD;
3729 
3730 	/*
3731 	 * We're only going to throw away mappings below kernelbase or in
3732 	 * boot's special double-mapping region, so set noconsist to avoid
3733 	 * using hments
3734 	 */
3735 	if (BOOT_VA(va))
3736 		flags |= HAT_LOAD_NOCONSIST;
3737 
3738 	prot |= HAT_STORECACHING_OK;
3739 	while (bytes != 0) {
3740 		ASSERT(bytes >= pgsize);
3741 
3742 		pp = NULL;
3743 		if (pf_is_memory(pfn) && !BOOT_VA(va) && level == 0)
3744 			pp = page_numtopp_nolock(pfn);
3745 
3746 		hati_load_common(kas.a_hat, va, pp, prot, flags, level, pfn);
3747 
3748 		va += pgsize;
3749 		pfn += mmu_btop(pgsize);
3750 		bytes -= pgsize;
3751 	}
3752 }
3753 
3754 /*
3755  * HAT part of cpu intialization.
3756  */
3757 void
3758 hat_cpu_online(struct cpu *cpup)
3759 {
3760 	if (cpup != CPU) {
3761 		x86pte_cpu_init(cpup, NULL);
3762 		hat_vlp_setup(cpup);
3763 	}
3764 	CPUSET_ATOMIC_ADD(khat_cpuset, cpup->cpu_id);
3765 }
3766 
3767 /*
3768  * Function called after all CPUs are brought online.
3769  * Used to remove low address boot mappings.
3770  */
3771 void
3772 clear_boot_mappings(uintptr_t low, uintptr_t high)
3773 {
3774 	uintptr_t vaddr = low;
3775 	htable_t *ht = NULL;
3776 	level_t level;
3777 	uint_t entry;
3778 	x86pte_t pte;
3779 
3780 	/*
3781 	 * On 1st CPU we can unload the prom mappings, basically we blow away
3782 	 * all virtual mappings under kernelbase.
3783 	 */
3784 	while (vaddr < high) {
3785 		pte = htable_walk(kas.a_hat, &ht, &vaddr, high);
3786 		if (ht == NULL)
3787 			break;
3788 
3789 		level = ht->ht_level;
3790 		entry = htable_va2entry(vaddr, ht);
3791 		ASSERT(level <= mmu.max_page_level);
3792 		ASSERT(PTE_ISPAGE(pte, level));
3793 
3794 		/*
3795 		 * Unload the mapping from the page tables.
3796 		 */
3797 		(void) x86pte_set(ht, entry, 0, NULL);
3798 		ASSERT(ht->ht_valid_cnt > 0);
3799 		HTABLE_DEC(ht->ht_valid_cnt);
3800 		PGCNT_DEC(ht->ht_hat, ht->ht_level);
3801 
3802 		vaddr += LEVEL_SIZE(ht->ht_level);
3803 	}
3804 	if (ht)
3805 		htable_release(ht);
3806 
3807 	/*
3808 	 * cross call for a complete invalidate.
3809 	 */
3810 	hat_demap(kas.a_hat, DEMAP_ALL_ADDR);
3811 }
3812 
3813 /*
3814  * Initialize a special area in the kernel that always holds some PTEs for
3815  * faster performance. This always holds segmap's PTEs.
3816  * In the 32 bit kernel this maps the kernel heap too.
3817  */
3818 void
3819 hat_kmap_init(uintptr_t base, size_t len)
3820 {
3821 	uintptr_t map_addr;	/* base rounded down to large page size */
3822 	uintptr_t map_eaddr;	/* base + len rounded up */
3823 	size_t map_len;
3824 	caddr_t ptes;		/* mapping area in kernel as for ptes */
3825 	size_t window_size;	/* size of mapping area for ptes */
3826 	ulong_t htable_cnt;	/* # of page tables to cover map_len */
3827 	ulong_t i;
3828 	htable_t *ht;
3829 
3830 	/*
3831 	 * we have to map in an area that matches an entire page table
3832 	 */
3833 	map_addr = base & LEVEL_MASK(1);
3834 	map_eaddr = (base + len + LEVEL_SIZE(1) - 1) & LEVEL_MASK(1);
3835 	map_len = map_eaddr - map_addr;
3836 	window_size = mmu_btop(map_len) * mmu.pte_size;
3837 	htable_cnt = mmu_btop(map_len) / mmu.ptes_per_table;
3838 
3839 	/*
3840 	 * allocate vmem for the kmap_ptes
3841 	 */
3842 	ptes = vmem_xalloc(heap_arena, window_size, MMU_PAGESIZE, 0,
3843 	    0, NULL, NULL, VM_SLEEP);
3844 	mmu.kmap_htables =
3845 	    kmem_alloc(htable_cnt * sizeof (htable_t *), KM_SLEEP);
3846 
3847 	/*
3848 	 * Map the page tables that cover kmap into the allocated range.
3849 	 * Note we don't ever htable_release() the kmap page tables - they
3850 	 * can't ever be stolen, freed, etc.
3851 	 */
3852 	for (i = 0; i < htable_cnt; ++i) {
3853 		ht = htable_create(kas.a_hat, map_addr + i * LEVEL_SIZE(1),
3854 		    0, NULL);
3855 		mmu.kmap_htables[i] = ht;
3856 
3857 		hat_devload(kas.a_hat, ptes + i * MMU_PAGESIZE, MMU_PAGESIZE,
3858 		    ht->ht_pfn,
3859 		    PROT_READ | PROT_WRITE | HAT_NOSYNC | HAT_UNORDERED_OK,
3860 		    HAT_LOAD | HAT_LOAD_NOCONSIST);
3861 
3862 	}
3863 
3864 	/*
3865 	 * set information in mmu to activate handling of kmap
3866 	 */
3867 	mmu.kmap_addr = base;
3868 	mmu.kmap_eaddr = base + len;
3869 	mmu.kmap_ptes =
3870 	    (x86pte_t *)(ptes + mmu.pte_size * mmu_btop(base - map_addr));
3871 }
3872 
3873 /*
3874  * Atomically update a new translation for a single page.  If the
3875  * currently installed PTE doesn't match the value we expect to find,
3876  * it's not updated and we return the PTE we found.
3877  *
3878  * If activating nosync or NOWRITE and the page was modified we need to sync
3879  * with the page_t. Also sync with page_t if clearing ref/mod bits.
3880  */
3881 static x86pte_t
3882 hati_update_pte(htable_t *ht, uint_t entry, x86pte_t expected, x86pte_t new)
3883 {
3884 	page_t		*pp;
3885 	uint_t		rm = 0;
3886 	x86pte_t	replaced;
3887 
3888 	if (!PTE_GET(expected, PT_NOSYNC | PT_NOCONSIST) &&
3889 	    PTE_GET(expected, PT_MOD | PT_REF) &&
3890 	    (PTE_GET(new, PT_NOSYNC) || !PTE_GET(new, PT_WRITABLE) ||
3891 		!PTE_GET(new, PT_MOD | PT_REF))) {
3892 
3893 		pp = page_numtopp_nolock(PTE2PFN(expected, ht->ht_level));
3894 		ASSERT(pp != NULL);
3895 		if (PTE_GET(expected, PT_MOD))
3896 			rm |= P_MOD;
3897 		if (PTE_GET(expected, PT_REF))
3898 			rm |= P_REF;
3899 		PTE_CLR(new, PT_MOD | PT_REF);
3900 	}
3901 
3902 	replaced = x86pte_update(ht, entry, expected, new);
3903 	if (replaced != expected)
3904 		return (replaced);
3905 
3906 	if (rm) {
3907 		/*
3908 		 * sync to all constituent pages of a large page
3909 		 */
3910 		pgcnt_t pgcnt = page_get_pagecnt(ht->ht_level);
3911 		ASSERT(IS_P2ALIGNED(pp->p_pagenum, pgcnt));
3912 		while (pgcnt-- > 0) {
3913 			/*
3914 			 * hat_page_demote() can't decrease
3915 			 * pszc below this mapping size
3916 			 * since large mapping existed after we
3917 			 * took mlist lock.
3918 			 */
3919 			ASSERT(pp->p_szc >= ht->ht_level);
3920 			hat_page_setattr(pp, rm);
3921 			++pp;
3922 		}
3923 	}
3924 
3925 	return (0);
3926 }
3927 
3928 /*
3929  * Kernel Physical Mapping (kpm) facility
3930  *
3931  * Most of the routines needed to support segkpm are almost no-ops on the
3932  * x86 platform.  We map in the entire segment when it is created and leave
3933  * it mapped in, so there is no additional work required to set up and tear
3934  * down individual mappings.  All of these routines were created to support
3935  * SPARC platforms that have to avoid aliasing in their virtually indexed
3936  * caches.
3937  *
3938  * Most of the routines have sanity checks in them (e.g. verifying that the
3939  * passed-in page is locked).  We don't actually care about most of these
3940  * checks on x86, but we leave them in place to identify problems in the
3941  * upper levels.
3942  */
3943 
3944 /*
3945  * Map in a locked page and return the vaddr.
3946  */
3947 /*ARGSUSED*/
3948 caddr_t
3949 hat_kpm_mapin(struct page *pp, struct kpme *kpme)
3950 {
3951 	caddr_t		vaddr;
3952 
3953 #ifdef DEBUG
3954 	if (kpm_enable == 0) {
3955 		cmn_err(CE_WARN, "hat_kpm_mapin: kpm_enable not set\n");
3956 		return ((caddr_t)NULL);
3957 	}
3958 
3959 	if (pp == NULL || PAGE_LOCKED(pp) == 0) {
3960 		cmn_err(CE_WARN, "hat_kpm_mapin: pp zero or not locked\n");
3961 		return ((caddr_t)NULL);
3962 	}
3963 #endif
3964 
3965 	vaddr = hat_kpm_page2va(pp, 1);
3966 
3967 	return (vaddr);
3968 }
3969 
3970 /*
3971  * Mapout a locked page.
3972  */
3973 /*ARGSUSED*/
3974 void
3975 hat_kpm_mapout(struct page *pp, struct kpme *kpme, caddr_t vaddr)
3976 {
3977 #ifdef DEBUG
3978 	if (kpm_enable == 0) {
3979 		cmn_err(CE_WARN, "hat_kpm_mapout: kpm_enable not set\n");
3980 		return;
3981 	}
3982 
3983 	if (IS_KPM_ADDR(vaddr) == 0) {
3984 		cmn_err(CE_WARN, "hat_kpm_mapout: no kpm address\n");
3985 		return;
3986 	}
3987 
3988 	if (pp == NULL || PAGE_LOCKED(pp) == 0) {
3989 		cmn_err(CE_WARN, "hat_kpm_mapout: page zero or not locked\n");
3990 		return;
3991 	}
3992 #endif
3993 }
3994 
3995 /*
3996  * Return the kpm virtual address for a specific pfn
3997  */
3998 caddr_t
3999 hat_kpm_pfn2va(pfn_t pfn)
4000 {
4001 	uintptr_t vaddr;
4002 
4003 	ASSERT(kpm_enable);
4004 
4005 	vaddr = (uintptr_t)kpm_vbase + mmu_ptob(pfn);
4006 
4007 	return ((caddr_t)vaddr);
4008 }
4009 
4010 /*
4011  * Return the kpm virtual address for the page at pp.
4012  */
4013 /*ARGSUSED*/
4014 caddr_t
4015 hat_kpm_page2va(struct page *pp, int checkswap)
4016 {
4017 	return (hat_kpm_pfn2va(pp->p_pagenum));
4018 }
4019 
4020 /*
4021  * Return the page frame number for the kpm virtual address vaddr.
4022  */
4023 pfn_t
4024 hat_kpm_va2pfn(caddr_t vaddr)
4025 {
4026 	pfn_t		pfn;
4027 
4028 	ASSERT(IS_KPM_ADDR(vaddr));
4029 
4030 	pfn = (pfn_t)btop(vaddr - kpm_vbase);
4031 
4032 	return (pfn);
4033 }
4034 
4035 
4036 /*
4037  * Return the page for the kpm virtual address vaddr.
4038  */
4039 page_t *
4040 hat_kpm_vaddr2page(caddr_t vaddr)
4041 {
4042 	pfn_t		pfn;
4043 
4044 	ASSERT(IS_KPM_ADDR(vaddr));
4045 
4046 	pfn = hat_kpm_va2pfn(vaddr);
4047 
4048 	return (page_numtopp_nolock(pfn));
4049 }
4050 
4051 /*
4052  * hat_kpm_fault is called from segkpm_fault when we take a page fault on a
4053  * KPM page.  This should never happen on x86
4054  */
4055 int
4056 hat_kpm_fault(hat_t *hat, caddr_t vaddr)
4057 {
4058 	panic("pagefault in seg_kpm.  hat: 0x%p  vaddr: 0x%p", hat, vaddr);
4059 
4060 	return (0);
4061 }
4062 
4063 /*ARGSUSED*/
4064 void
4065 hat_kpm_mseghash_clear(int nentries)
4066 {}
4067 
4068 /*ARGSUSED*/
4069 void
4070 hat_kpm_mseghash_update(pgcnt_t inx, struct memseg *msp)
4071 {}
4072