xref: /illumos-gate/usr/src/uts/i86pc/vm/vm_machdep.c (revision 2e8a1712c9ef0c384682b03b664b5c2b6c8622fe)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
27 /*	All Rights Reserved   */
28 
29 /*
30  * Portions of this source code were derived from Berkeley 4.3 BSD
31  * under license from the Regents of the University of California.
32  */
33 
34 #pragma ident	"%Z%%M%	%I%	%E% SMI"
35 
36 /*
37  * UNIX machine dependent virtual memory support.
38  */
39 
40 #include <sys/types.h>
41 #include <sys/param.h>
42 #include <sys/systm.h>
43 #include <sys/user.h>
44 #include <sys/proc.h>
45 #include <sys/kmem.h>
46 #include <sys/vmem.h>
47 #include <sys/buf.h>
48 #include <sys/cpuvar.h>
49 #include <sys/lgrp.h>
50 #include <sys/disp.h>
51 #include <sys/vm.h>
52 #include <sys/mman.h>
53 #include <sys/vnode.h>
54 #include <sys/cred.h>
55 #include <sys/exec.h>
56 #include <sys/exechdr.h>
57 #include <sys/debug.h>
58 #include <sys/vmsystm.h>
59 
60 #include <vm/hat.h>
61 #include <vm/as.h>
62 #include <vm/seg.h>
63 #include <vm/seg_kp.h>
64 #include <vm/seg_vn.h>
65 #include <vm/page.h>
66 #include <vm/seg_kmem.h>
67 #include <vm/seg_kpm.h>
68 #include <vm/vm_dep.h>
69 
70 #include <sys/cpu.h>
71 #include <sys/vm_machparam.h>
72 #include <sys/memlist.h>
73 #include <sys/bootconf.h> /* XXX the memlist stuff belongs in memlist_plat.h */
74 #include <vm/hat_i86.h>
75 #include <sys/x86_archext.h>
76 #include <sys/elf_386.h>
77 #include <sys/cmn_err.h>
78 #include <sys/archsystm.h>
79 #include <sys/machsystm.h>
80 
81 #include <sys/vtrace.h>
82 #include <sys/ddidmareq.h>
83 #include <sys/promif.h>
84 #include <sys/memnode.h>
85 #include <sys/stack.h>
86 #include <util/qsort.h>
87 #include <sys/taskq.h>
88 
89 #ifdef __xpv
90 
91 #include <sys/hypervisor.h>
92 #include <sys/xen_mmu.h>
93 #include <sys/balloon_impl.h>
94 
95 /*
96  * domain 0 pages usable for DMA are kept pre-allocated and kept in
97  * distinct lists, ordered by increasing mfn.
98  */
99 static kmutex_t io_pool_lock;
100 static kmutex_t contig_list_lock;
101 static page_t *io_pool_4g;	/* pool for 32 bit dma limited devices */
102 static page_t *io_pool_16m;	/* pool for 24 bit dma limited legacy devices */
103 static long io_pool_cnt;
104 static long io_pool_cnt_max = 0;
105 #define	DEFAULT_IO_POOL_MIN	128
106 static long io_pool_cnt_min = DEFAULT_IO_POOL_MIN;
107 static long io_pool_cnt_lowater = 0;
108 static long io_pool_shrink_attempts; /* how many times did we try to shrink */
109 static long io_pool_shrinks;	/* how many times did we really shrink */
110 static long io_pool_grows;	/* how many times did we grow */
111 static mfn_t start_mfn = 1;
112 static caddr_t io_pool_kva;	/* use to alloc pages when needed */
113 
114 static int create_contig_pfnlist(uint_t);
115 
116 /*
117  * percentage of phys mem to hold in the i/o pool
118  */
119 #define	DEFAULT_IO_POOL_PCT	2
120 static long io_pool_physmem_pct = DEFAULT_IO_POOL_PCT;
121 static void page_io_pool_sub(page_t **, page_t *, page_t *);
122 int ioalloc_dbg = 0;
123 
124 #endif /* __xpv */
125 
126 uint_t vac_colors = 1;
127 
128 int largepagesupport = 0;
129 extern uint_t page_create_new;
130 extern uint_t page_create_exists;
131 extern uint_t page_create_putbacks;
132 extern uint_t page_create_putbacks;
133 /*
134  * Allow users to disable the kernel's use of SSE.
135  */
136 extern int use_sse_pagecopy, use_sse_pagezero;
137 
138 /*
139  * combined memory ranges from mnode and memranges[] to manage single
140  * mnode/mtype dimension in the page lists.
141  */
142 typedef struct {
143 	pfn_t	mnr_pfnlo;
144 	pfn_t	mnr_pfnhi;
145 	int	mnr_mnode;
146 	int	mnr_memrange;		/* index into memranges[] */
147 	/* maintain page list stats */
148 	pgcnt_t	mnr_mt_clpgcnt;		/* cache list cnt */
149 	pgcnt_t	mnr_mt_flpgcnt[MMU_PAGE_SIZES];	/* free list cnt per szc */
150 	pgcnt_t	mnr_mt_totcnt;		/* sum of cache and free lists */
151 #ifdef DEBUG
152 	struct mnr_mts {		/* mnode/mtype szc stats */
153 		pgcnt_t	mnr_mts_pgcnt;
154 		int	mnr_mts_colors;
155 		pgcnt_t *mnr_mtsc_pgcnt;
156 	} 	*mnr_mts;
157 #endif
158 } mnoderange_t;
159 
160 #define	MEMRANGEHI(mtype)						\
161 	((mtype > 0) ? memranges[mtype - 1] - 1: physmax)
162 #define	MEMRANGELO(mtype)	(memranges[mtype])
163 
164 #define	MTYPE_FREEMEM(mt)	(mnoderanges[mt].mnr_mt_totcnt)
165 
166 /*
167  * As the PC architecture evolved memory up was clumped into several
168  * ranges for various historical I/O devices to do DMA.
169  * < 16Meg - ISA bus
170  * < 2Gig - ???
171  * < 4Gig - PCI bus or drivers that don't understand PAE mode
172  *
173  * These are listed in reverse order, so that we can skip over unused
174  * ranges on machines with small memories.
175  *
176  * For now under the Hypervisor, we'll only ever have one memrange.
177  */
178 #define	PFN_4GIG	0x100000
179 #define	PFN_16MEG	0x1000
180 static pfn_t arch_memranges[NUM_MEM_RANGES] = {
181     PFN_4GIG,	/* pfn range for 4G and above */
182     0x80000,	/* pfn range for 2G-4G */
183     PFN_16MEG,	/* pfn range for 16M-2G */
184     0x00000,	/* pfn range for 0-16M */
185 };
186 pfn_t *memranges = &arch_memranges[0];
187 int nranges = NUM_MEM_RANGES;
188 
189 /*
190  * This combines mem_node_config and memranges into one data
191  * structure to be used for page list management.
192  */
193 mnoderange_t	*mnoderanges;
194 int		mnoderangecnt;
195 int		mtype4g;
196 
197 /*
198  * 4g memory management variables for systems with more than 4g of memory:
199  *
200  * physical memory below 4g is required for 32bit dma devices and, currently,
201  * for kmem memory. On systems with more than 4g of memory, the pool of memory
202  * below 4g can be depleted without any paging activity given that there is
203  * likely to be sufficient memory above 4g.
204  *
205  * physmax4g is set true if the largest pfn is over 4g. The rest of the
206  * 4g memory management code is enabled only when physmax4g is true.
207  *
208  * maxmem4g is the count of the maximum number of pages on the page lists
209  * with physical addresses below 4g. It can be a lot less then 4g given that
210  * BIOS may reserve large chunks of space below 4g for hot plug pci devices,
211  * agp aperture etc.
212  *
213  * freemem4g maintains the count of the number of available pages on the
214  * page lists with physical addresses below 4g.
215  *
216  * DESFREE4G specifies the desired amount of below 4g memory. It defaults to
217  * 6% (desfree4gshift = 4) of maxmem4g.
218  *
219  * RESTRICT4G_ALLOC returns true if freemem4g falls below DESFREE4G
220  * and the amount of physical memory above 4g is greater than freemem4g.
221  * In this case, page_get_* routines will restrict below 4g allocations
222  * for requests that don't specifically require it.
223  */
224 
225 #define	LOTSFREE4G	(maxmem4g >> lotsfree4gshift)
226 #define	DESFREE4G	(maxmem4g >> desfree4gshift)
227 
228 #define	RESTRICT4G_ALLOC					\
229 	(physmax4g && (freemem4g < DESFREE4G) && ((freemem4g << 1) < freemem))
230 
231 static pgcnt_t	maxmem4g;
232 static pgcnt_t	freemem4g;
233 static int	physmax4g;
234 static int	desfree4gshift = 4;	/* maxmem4g shift to derive DESFREE4G */
235 static int	lotsfree4gshift = 3;
236 
237 /*
238  * 16m memory management:
239  *
240  * reserve some amount of physical memory below 16m for legacy devices.
241  *
242  * RESTRICT16M_ALLOC returns true if an there are sufficient free pages above
243  * 16m or if the 16m pool drops below DESFREE16M.
244  *
245  * In this case, general page allocations via page_get_{free,cache}list
246  * routines will be restricted from allocating from the 16m pool. Allocations
247  * that require specific pfn ranges (page_get_anylist) and PG_PANIC allocations
248  * are not restricted.
249  */
250 
251 #define	FREEMEM16M	MTYPE_FREEMEM(0)
252 #define	DESFREE16M	desfree16m
253 #define	RESTRICT16M_ALLOC(freemem, pgcnt, flags)		\
254 	((freemem != 0) && ((flags & PG_PANIC) == 0) &&		\
255 	    ((freemem >= (FREEMEM16M)) ||			\
256 	    (FREEMEM16M  < (DESFREE16M + pgcnt))))
257 
258 static pgcnt_t	desfree16m = 0x380;
259 
260 /*
261  * This can be patched via /etc/system to allow old non-PAE aware device
262  * drivers to use kmem_alloc'd memory on 32 bit systems with > 4Gig RAM.
263  */
264 int restricted_kmemalloc = 0;
265 
266 #ifdef VM_STATS
267 struct {
268 	ulong_t	pga_alloc;
269 	ulong_t	pga_notfullrange;
270 	ulong_t	pga_nulldmaattr;
271 	ulong_t	pga_allocok;
272 	ulong_t	pga_allocfailed;
273 	ulong_t	pgma_alloc;
274 	ulong_t	pgma_allocok;
275 	ulong_t	pgma_allocfailed;
276 	ulong_t	pgma_allocempty;
277 } pga_vmstats;
278 #endif
279 
280 uint_t mmu_page_sizes;
281 
282 /* How many page sizes the users can see */
283 uint_t mmu_exported_page_sizes;
284 
285 /* page sizes that legacy applications can see */
286 uint_t mmu_legacy_page_sizes;
287 
288 /*
289  * Number of pages in 1 GB.  Don't enable automatic large pages if we have
290  * fewer than this many pages.
291  */
292 pgcnt_t shm_lpg_min_physmem = 1 << (30 - MMU_PAGESHIFT);
293 pgcnt_t privm_lpg_min_physmem = 1 << (30 - MMU_PAGESHIFT);
294 
295 /*
296  * Maximum and default segment size tunables for user private
297  * and shared anon memory, and user text and initialized data.
298  * These can be patched via /etc/system to allow large pages
299  * to be used for mapping application private and shared anon memory.
300  */
301 size_t mcntl0_lpsize = MMU_PAGESIZE;
302 size_t max_uheap_lpsize = MMU_PAGESIZE;
303 size_t default_uheap_lpsize = MMU_PAGESIZE;
304 size_t max_ustack_lpsize = MMU_PAGESIZE;
305 size_t default_ustack_lpsize = MMU_PAGESIZE;
306 size_t max_privmap_lpsize = MMU_PAGESIZE;
307 size_t max_uidata_lpsize = MMU_PAGESIZE;
308 size_t max_utext_lpsize = MMU_PAGESIZE;
309 size_t max_shm_lpsize = MMU_PAGESIZE;
310 
311 
312 /*
313  * initialized by page_coloring_init().
314  */
315 uint_t	page_colors;
316 uint_t	page_colors_mask;
317 uint_t	page_coloring_shift;
318 int	cpu_page_colors;
319 static uint_t	l2_colors;
320 
321 /*
322  * Page freelists and cachelists are dynamically allocated once mnoderangecnt
323  * and page_colors are calculated from the l2 cache n-way set size.  Within a
324  * mnode range, the page freelist and cachelist are hashed into bins based on
325  * color. This makes it easier to search for a page within a specific memory
326  * range.
327  */
328 #define	PAGE_COLORS_MIN	16
329 
330 page_t ****page_freelists;
331 page_t ***page_cachelists;
332 
333 
334 /*
335  * Used by page layer to know about page sizes
336  */
337 hw_pagesize_t hw_page_array[MAX_NUM_LEVEL + 1];
338 
339 kmutex_t	*fpc_mutex[NPC_MUTEX];
340 kmutex_t	*cpc_mutex[NPC_MUTEX];
341 
342 /*
343  * Only let one thread at a time try to coalesce large pages, to
344  * prevent them from working against each other.
345  */
346 static kmutex_t	contig_lock;
347 #define	CONTIG_LOCK()	mutex_enter(&contig_lock);
348 #define	CONTIG_UNLOCK()	mutex_exit(&contig_lock);
349 
350 #define	PFN_16M		(mmu_btop((uint64_t)0x1000000))
351 
352 /*
353  * Return the optimum page size for a given mapping
354  */
355 /*ARGSUSED*/
356 size_t
357 map_pgsz(int maptype, struct proc *p, caddr_t addr, size_t len, int memcntl)
358 {
359 	level_t l = 0;
360 	size_t pgsz = MMU_PAGESIZE;
361 	size_t max_lpsize;
362 	uint_t mszc;
363 
364 	ASSERT(maptype != MAPPGSZ_VA);
365 
366 	if (maptype != MAPPGSZ_ISM && physmem < privm_lpg_min_physmem) {
367 		return (MMU_PAGESIZE);
368 	}
369 
370 	switch (maptype) {
371 	case MAPPGSZ_HEAP:
372 	case MAPPGSZ_STK:
373 		max_lpsize = memcntl ? mcntl0_lpsize : (maptype ==
374 		    MAPPGSZ_HEAP ? max_uheap_lpsize : max_ustack_lpsize);
375 		if (max_lpsize == MMU_PAGESIZE) {
376 			return (MMU_PAGESIZE);
377 		}
378 		if (len == 0) {
379 			len = (maptype == MAPPGSZ_HEAP) ? p->p_brkbase +
380 			    p->p_brksize - p->p_bssbase : p->p_stksize;
381 		}
382 		len = (maptype == MAPPGSZ_HEAP) ? MAX(len,
383 		    default_uheap_lpsize) : MAX(len, default_ustack_lpsize);
384 
385 		/*
386 		 * use the pages size that best fits len
387 		 */
388 		for (l = mmu.umax_page_level; l > 0; --l) {
389 			if (LEVEL_SIZE(l) > max_lpsize || len < LEVEL_SIZE(l)) {
390 				continue;
391 			} else {
392 				pgsz = LEVEL_SIZE(l);
393 			}
394 			break;
395 		}
396 
397 		mszc = (maptype == MAPPGSZ_HEAP ? p->p_brkpageszc :
398 		    p->p_stkpageszc);
399 		if (addr == 0 && (pgsz < hw_page_array[mszc].hp_size)) {
400 			pgsz = hw_page_array[mszc].hp_size;
401 		}
402 		return (pgsz);
403 
404 	case MAPPGSZ_ISM:
405 		for (l = mmu.umax_page_level; l > 0; --l) {
406 			if (len >= LEVEL_SIZE(l))
407 				return (LEVEL_SIZE(l));
408 		}
409 		return (LEVEL_SIZE(0));
410 	}
411 	return (pgsz);
412 }
413 
414 static uint_t
415 map_szcvec(caddr_t addr, size_t size, uintptr_t off, size_t max_lpsize,
416     size_t min_physmem)
417 {
418 	caddr_t eaddr = addr + size;
419 	uint_t szcvec = 0;
420 	caddr_t raddr;
421 	caddr_t readdr;
422 	size_t	pgsz;
423 	int i;
424 
425 	if (physmem < min_physmem || max_lpsize <= MMU_PAGESIZE) {
426 		return (0);
427 	}
428 
429 	for (i = mmu_exported_page_sizes - 1; i > 0; i--) {
430 		pgsz = page_get_pagesize(i);
431 		if (pgsz > max_lpsize) {
432 			continue;
433 		}
434 		raddr = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz);
435 		readdr = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz);
436 		if (raddr < addr || raddr >= readdr) {
437 			continue;
438 		}
439 		if (P2PHASE((uintptr_t)addr ^ off, pgsz)) {
440 			continue;
441 		}
442 		/*
443 		 * Set szcvec to the remaining page sizes.
444 		 */
445 		szcvec = ((1 << (i + 1)) - 1) & ~1;
446 		break;
447 	}
448 	return (szcvec);
449 }
450 
451 /*
452  * Return a bit vector of large page size codes that
453  * can be used to map [addr, addr + len) region.
454  */
455 /*ARGSUSED*/
456 uint_t
457 map_pgszcvec(caddr_t addr, size_t size, uintptr_t off, int flags, int type,
458     int memcntl)
459 {
460 	size_t max_lpsize = mcntl0_lpsize;
461 
462 	if (mmu.max_page_level == 0)
463 		return (0);
464 
465 	if (flags & MAP_TEXT) {
466 		if (!memcntl)
467 			max_lpsize = max_utext_lpsize;
468 		return (map_szcvec(addr, size, off, max_lpsize,
469 		    shm_lpg_min_physmem));
470 
471 	} else if (flags & MAP_INITDATA) {
472 		if (!memcntl)
473 			max_lpsize = max_uidata_lpsize;
474 		return (map_szcvec(addr, size, off, max_lpsize,
475 		    privm_lpg_min_physmem));
476 
477 	} else if (type == MAPPGSZC_SHM) {
478 		if (!memcntl)
479 			max_lpsize = max_shm_lpsize;
480 		return (map_szcvec(addr, size, off, max_lpsize,
481 		    shm_lpg_min_physmem));
482 
483 	} else if (type == MAPPGSZC_HEAP) {
484 		if (!memcntl)
485 			max_lpsize = max_uheap_lpsize;
486 		return (map_szcvec(addr, size, off, max_lpsize,
487 		    privm_lpg_min_physmem));
488 
489 	} else if (type == MAPPGSZC_STACK) {
490 		if (!memcntl)
491 			max_lpsize = max_ustack_lpsize;
492 		return (map_szcvec(addr, size, off, max_lpsize,
493 		    privm_lpg_min_physmem));
494 
495 	} else {
496 		if (!memcntl)
497 			max_lpsize = max_privmap_lpsize;
498 		return (map_szcvec(addr, size, off, max_lpsize,
499 		    privm_lpg_min_physmem));
500 	}
501 }
502 
503 /*
504  * Handle a pagefault.
505  */
506 faultcode_t
507 pagefault(
508 	caddr_t addr,
509 	enum fault_type type,
510 	enum seg_rw rw,
511 	int iskernel)
512 {
513 	struct as *as;
514 	struct hat *hat;
515 	struct proc *p;
516 	kthread_t *t;
517 	faultcode_t res;
518 	caddr_t base;
519 	size_t len;
520 	int err;
521 	int mapped_red;
522 	uintptr_t ea;
523 
524 	ASSERT_STACK_ALIGNED();
525 
526 	if (INVALID_VADDR(addr))
527 		return (FC_NOMAP);
528 
529 	mapped_red = segkp_map_red();
530 
531 	if (iskernel) {
532 		as = &kas;
533 		hat = as->a_hat;
534 	} else {
535 		t = curthread;
536 		p = ttoproc(t);
537 		as = p->p_as;
538 		hat = as->a_hat;
539 	}
540 
541 	/*
542 	 * Dispatch pagefault.
543 	 */
544 	res = as_fault(hat, as, addr, 1, type, rw);
545 
546 	/*
547 	 * If this isn't a potential unmapped hole in the user's
548 	 * UNIX data or stack segments, just return status info.
549 	 */
550 	if (res != FC_NOMAP || iskernel)
551 		goto out;
552 
553 	/*
554 	 * Check to see if we happened to faulted on a currently unmapped
555 	 * part of the UNIX data or stack segments.  If so, create a zfod
556 	 * mapping there and then try calling the fault routine again.
557 	 */
558 	base = p->p_brkbase;
559 	len = p->p_brksize;
560 
561 	if (addr < base || addr >= base + len) {		/* data seg? */
562 		base = (caddr_t)p->p_usrstack - p->p_stksize;
563 		len = p->p_stksize;
564 		if (addr < base || addr >= p->p_usrstack) {	/* stack seg? */
565 			/* not in either UNIX data or stack segments */
566 			res = FC_NOMAP;
567 			goto out;
568 		}
569 	}
570 
571 	/*
572 	 * the rest of this function implements a 3.X 4.X 5.X compatibility
573 	 * This code is probably not needed anymore
574 	 */
575 	if (p->p_model == DATAMODEL_ILP32) {
576 
577 		/* expand the gap to the page boundaries on each side */
578 		ea = P2ROUNDUP((uintptr_t)base + len, MMU_PAGESIZE);
579 		base = (caddr_t)P2ALIGN((uintptr_t)base, MMU_PAGESIZE);
580 		len = ea - (uintptr_t)base;
581 
582 		as_rangelock(as);
583 		if (as_gap(as, MMU_PAGESIZE, &base, &len, AH_CONTAIN, addr) ==
584 		    0) {
585 			err = as_map(as, base, len, segvn_create, zfod_argsp);
586 			as_rangeunlock(as);
587 			if (err) {
588 				res = FC_MAKE_ERR(err);
589 				goto out;
590 			}
591 		} else {
592 			/*
593 			 * This page is already mapped by another thread after
594 			 * we returned from as_fault() above.  We just fall
595 			 * through as_fault() below.
596 			 */
597 			as_rangeunlock(as);
598 		}
599 
600 		res = as_fault(hat, as, addr, 1, F_INVAL, rw);
601 	}
602 
603 out:
604 	if (mapped_red)
605 		segkp_unmap_red();
606 
607 	return (res);
608 }
609 
610 void
611 map_addr(caddr_t *addrp, size_t len, offset_t off, int vacalign, uint_t flags)
612 {
613 	struct proc *p = curproc;
614 	caddr_t userlimit = (flags & _MAP_LOW32) ?
615 	    (caddr_t)_userlimit32 : p->p_as->a_userlimit;
616 
617 	map_addr_proc(addrp, len, off, vacalign, userlimit, curproc, flags);
618 }
619 
620 /*ARGSUSED*/
621 int
622 map_addr_vacalign_check(caddr_t addr, u_offset_t off)
623 {
624 	return (0);
625 }
626 
627 /*
628  * map_addr_proc() is the routine called when the system is to
629  * choose an address for the user.  We will pick an address
630  * range which is the highest available below userlimit.
631  *
632  * Every mapping will have a redzone of a single page on either side of
633  * the request. This is done to leave one page unmapped between segments.
634  * This is not required, but it's useful for the user because if their
635  * program strays across a segment boundary, it will catch a fault
636  * immediately making debugging a little easier.  Currently the redzone
637  * is mandatory.
638  *
639  * addrp is a value/result parameter.
640  *	On input it is a hint from the user to be used in a completely
641  *	machine dependent fashion.  We decide to completely ignore this hint.
642  *	If MAP_ALIGN was specified, addrp contains the minimal alignment, which
643  *	must be some "power of two" multiple of pagesize.
644  *
645  *	On output it is NULL if no address can be found in the current
646  *	processes address space or else an address that is currently
647  *	not mapped for len bytes with a page of red zone on either side.
648  *
649  *	vacalign is not needed on x86 (it's for viturally addressed caches)
650  */
651 /*ARGSUSED*/
652 void
653 map_addr_proc(
654 	caddr_t *addrp,
655 	size_t len,
656 	offset_t off,
657 	int vacalign,
658 	caddr_t userlimit,
659 	struct proc *p,
660 	uint_t flags)
661 {
662 	struct as *as = p->p_as;
663 	caddr_t addr;
664 	caddr_t base;
665 	size_t slen;
666 	size_t align_amount;
667 
668 	ASSERT32(userlimit == as->a_userlimit);
669 
670 	base = p->p_brkbase;
671 #if defined(__amd64)
672 	/*
673 	 * XX64 Yes, this needs more work.
674 	 */
675 	if (p->p_model == DATAMODEL_NATIVE) {
676 		if (userlimit < as->a_userlimit) {
677 			/*
678 			 * This happens when a program wants to map
679 			 * something in a range that's accessible to a
680 			 * program in a smaller address space.  For example,
681 			 * a 64-bit program calling mmap32(2) to guarantee
682 			 * that the returned address is below 4Gbytes.
683 			 */
684 			ASSERT((uintptr_t)userlimit < ADDRESS_C(0xffffffff));
685 
686 			if (userlimit > base)
687 				slen = userlimit - base;
688 			else {
689 				*addrp = NULL;
690 				return;
691 			}
692 		} else {
693 			/*
694 			 * XX64 This layout is probably wrong .. but in
695 			 * the event we make the amd64 address space look
696 			 * like sparcv9 i.e. with the stack -above- the
697 			 * heap, this bit of code might even be correct.
698 			 */
699 			slen = p->p_usrstack - base -
700 			    (((size_t)rctl_enforced_value(
701 			    rctlproc_legacy[RLIMIT_STACK],
702 			    p->p_rctls, p) + PAGEOFFSET) & PAGEMASK);
703 		}
704 	} else
705 #endif
706 		slen = userlimit - base;
707 
708 	/* Make len be a multiple of PAGESIZE */
709 	len = (len + PAGEOFFSET) & PAGEMASK;
710 
711 	/*
712 	 * figure out what the alignment should be
713 	 *
714 	 * XX64 -- is there an ELF_AMD64_MAXPGSZ or is it the same????
715 	 */
716 	if (len <= ELF_386_MAXPGSZ) {
717 		/*
718 		 * Align virtual addresses to ensure that ELF shared libraries
719 		 * are mapped with the appropriate alignment constraints by
720 		 * the run-time linker.
721 		 */
722 		align_amount = ELF_386_MAXPGSZ;
723 	} else {
724 		int l = mmu.umax_page_level;
725 
726 		while (l && len < LEVEL_SIZE(l))
727 			--l;
728 
729 		align_amount = LEVEL_SIZE(l);
730 	}
731 
732 	if ((flags & MAP_ALIGN) && ((uintptr_t)*addrp > align_amount))
733 		align_amount = (uintptr_t)*addrp;
734 
735 	ASSERT(ISP2(align_amount));
736 	ASSERT(align_amount == 0 || align_amount >= PAGESIZE);
737 
738 	off = off & (align_amount - 1);
739 	/*
740 	 * Look for a large enough hole starting below userlimit.
741 	 * After finding it, use the upper part.
742 	 */
743 	if (as_gap_aligned(as, len, &base, &slen, AH_HI, NULL, align_amount,
744 	    PAGESIZE, off) == 0) {
745 		caddr_t as_addr;
746 
747 		/*
748 		 * addr is the highest possible address to use since we have
749 		 * a PAGESIZE redzone at the beginning and end.
750 		 */
751 		addr = base + slen - (PAGESIZE + len);
752 		as_addr = addr;
753 		/*
754 		 * Round address DOWN to the alignment amount and
755 		 * add the offset in.
756 		 * If addr is greater than as_addr, len would not be large
757 		 * enough to include the redzone, so we must adjust down
758 		 * by the alignment amount.
759 		 */
760 		addr = (caddr_t)((uintptr_t)addr & (~(align_amount - 1)));
761 		addr += (uintptr_t)off;
762 		if (addr > as_addr) {
763 			addr -= align_amount;
764 		}
765 
766 		ASSERT(addr > base);
767 		ASSERT(addr + len < base + slen);
768 		ASSERT(((uintptr_t)addr & (align_amount - 1)) ==
769 		    ((uintptr_t)(off)));
770 		*addrp = addr;
771 	} else {
772 		*addrp = NULL;	/* no more virtual space */
773 	}
774 }
775 
776 int valid_va_range_aligned_wraparound;
777 
778 /*
779  * Determine whether [*basep, *basep + *lenp) contains a mappable range of
780  * addresses at least "minlen" long, where the base of the range is at "off"
781  * phase from an "align" boundary and there is space for a "redzone"-sized
782  * redzone on either side of the range.  On success, 1 is returned and *basep
783  * and *lenp are adjusted to describe the acceptable range (including
784  * the redzone).  On failure, 0 is returned.
785  */
786 /*ARGSUSED3*/
787 int
788 valid_va_range_aligned(caddr_t *basep, size_t *lenp, size_t minlen, int dir,
789     size_t align, size_t redzone, size_t off)
790 {
791 	uintptr_t hi, lo;
792 	size_t tot_len;
793 
794 	ASSERT(align == 0 ? off == 0 : off < align);
795 	ASSERT(ISP2(align));
796 	ASSERT(align == 0 || align >= PAGESIZE);
797 
798 	lo = (uintptr_t)*basep;
799 	hi = lo + *lenp;
800 	tot_len = minlen + 2 * redzone; /* need at least this much space */
801 
802 	/*
803 	 * If hi rolled over the top, try cutting back.
804 	 */
805 	if (hi < lo) {
806 		*lenp = 0UL - lo - 1UL;
807 		/* See if this really happens. If so, then we figure out why */
808 		valid_va_range_aligned_wraparound++;
809 		hi = lo + *lenp;
810 	}
811 	if (*lenp < tot_len) {
812 		return (0);
813 	}
814 
815 #if defined(__amd64)
816 	/*
817 	 * Deal with a possible hole in the address range between
818 	 * hole_start and hole_end that should never be mapped.
819 	 */
820 	if (lo < hole_start) {
821 		if (hi > hole_start) {
822 			if (hi < hole_end) {
823 				hi = hole_start;
824 			} else {
825 				/* lo < hole_start && hi >= hole_end */
826 				if (dir == AH_LO) {
827 					/*
828 					 * prefer lowest range
829 					 */
830 					if (hole_start - lo >= tot_len)
831 						hi = hole_start;
832 					else if (hi - hole_end >= tot_len)
833 						lo = hole_end;
834 					else
835 						return (0);
836 				} else {
837 					/*
838 					 * prefer highest range
839 					 */
840 					if (hi - hole_end >= tot_len)
841 						lo = hole_end;
842 					else if (hole_start - lo >= tot_len)
843 						hi = hole_start;
844 					else
845 						return (0);
846 				}
847 			}
848 		}
849 	} else {
850 		/* lo >= hole_start */
851 		if (hi < hole_end)
852 			return (0);
853 		if (lo < hole_end)
854 			lo = hole_end;
855 	}
856 #endif
857 
858 	if (hi - lo < tot_len)
859 		return (0);
860 
861 	if (align > 1) {
862 		uintptr_t tlo = lo + redzone;
863 		uintptr_t thi = hi - redzone;
864 		tlo = (uintptr_t)P2PHASEUP(tlo, align, off);
865 		if (tlo < lo + redzone) {
866 			return (0);
867 		}
868 		if (thi < tlo || thi - tlo < minlen) {
869 			return (0);
870 		}
871 	}
872 
873 	*basep = (caddr_t)lo;
874 	*lenp = hi - lo;
875 	return (1);
876 }
877 
878 /*
879  * Determine whether [*basep, *basep + *lenp) contains a mappable range of
880  * addresses at least "minlen" long.  On success, 1 is returned and *basep
881  * and *lenp are adjusted to describe the acceptable range.  On failure, 0
882  * is returned.
883  */
884 int
885 valid_va_range(caddr_t *basep, size_t *lenp, size_t minlen, int dir)
886 {
887 	return (valid_va_range_aligned(basep, lenp, minlen, dir, 0, 0, 0));
888 }
889 
890 /*
891  * Determine whether [addr, addr+len] are valid user addresses.
892  */
893 /*ARGSUSED*/
894 int
895 valid_usr_range(caddr_t addr, size_t len, uint_t prot, struct as *as,
896     caddr_t userlimit)
897 {
898 	caddr_t eaddr = addr + len;
899 
900 	if (eaddr <= addr || addr >= userlimit || eaddr > userlimit)
901 		return (RANGE_BADADDR);
902 
903 #if defined(__amd64)
904 	/*
905 	 * Check for the VA hole
906 	 */
907 	if (eaddr > (caddr_t)hole_start && addr < (caddr_t)hole_end)
908 		return (RANGE_BADADDR);
909 #endif
910 
911 	return (RANGE_OKAY);
912 }
913 
914 /*
915  * Return 1 if the page frame is onboard memory, else 0.
916  */
917 int
918 pf_is_memory(pfn_t pf)
919 {
920 	if (pfn_is_foreign(pf))
921 		return (0);
922 	return (address_in_memlist(phys_install, pfn_to_pa(pf), 1));
923 }
924 
925 /*
926  * return the memrange containing pfn
927  */
928 int
929 memrange_num(pfn_t pfn)
930 {
931 	int n;
932 
933 	for (n = 0; n < nranges - 1; ++n) {
934 		if (pfn >= memranges[n])
935 			break;
936 	}
937 	return (n);
938 }
939 
940 /*
941  * return the mnoderange containing pfn
942  */
943 /*ARGSUSED*/
944 int
945 pfn_2_mtype(pfn_t pfn)
946 {
947 #if defined(__xpv)
948 	return (0);
949 #else
950 	int	n;
951 
952 	for (n = mnoderangecnt - 1; n >= 0; n--) {
953 		if (pfn >= mnoderanges[n].mnr_pfnlo) {
954 			break;
955 		}
956 	}
957 	return (n);
958 #endif
959 }
960 
961 #if !defined(__xpv)
962 /*
963  * is_contigpage_free:
964  *	returns a page list of contiguous pages. It minimally has to return
965  *	minctg pages. Caller determines minctg based on the scatter-gather
966  *	list length.
967  *
968  *	pfnp is set to the next page frame to search on return.
969  */
970 static page_t *
971 is_contigpage_free(
972 	pfn_t *pfnp,
973 	pgcnt_t *pgcnt,
974 	pgcnt_t minctg,
975 	uint64_t pfnseg,
976 	int iolock)
977 {
978 	int	i = 0;
979 	pfn_t	pfn = *pfnp;
980 	page_t	*pp;
981 	page_t	*plist = NULL;
982 
983 	/*
984 	 * fail if pfn + minctg crosses a segment boundary.
985 	 * Adjust for next starting pfn to begin at segment boundary.
986 	 */
987 
988 	if (((*pfnp + minctg - 1) & pfnseg) < (*pfnp & pfnseg)) {
989 		*pfnp = roundup(*pfnp, pfnseg + 1);
990 		return (NULL);
991 	}
992 
993 	do {
994 retry:
995 		pp = page_numtopp_nolock(pfn + i);
996 		if ((pp == NULL) ||
997 		    (page_trylock(pp, SE_EXCL) == 0)) {
998 			(*pfnp)++;
999 			break;
1000 		}
1001 		if (page_pptonum(pp) != pfn + i) {
1002 			page_unlock(pp);
1003 			goto retry;
1004 		}
1005 
1006 		if (!(PP_ISFREE(pp))) {
1007 			page_unlock(pp);
1008 			(*pfnp)++;
1009 			break;
1010 		}
1011 
1012 		if (!PP_ISAGED(pp)) {
1013 			page_list_sub(pp, PG_CACHE_LIST);
1014 			page_hashout(pp, (kmutex_t *)NULL);
1015 		} else {
1016 			page_list_sub(pp, PG_FREE_LIST);
1017 		}
1018 
1019 		if (iolock)
1020 			page_io_lock(pp);
1021 		page_list_concat(&plist, &pp);
1022 
1023 		/*
1024 		 * exit loop when pgcnt satisfied or segment boundary reached.
1025 		 */
1026 
1027 	} while ((++i < *pgcnt) && ((pfn + i) & pfnseg));
1028 
1029 	*pfnp += i;		/* set to next pfn to search */
1030 
1031 	if (i >= minctg) {
1032 		*pgcnt -= i;
1033 		return (plist);
1034 	}
1035 
1036 	/*
1037 	 * failure: minctg not satisfied.
1038 	 *
1039 	 * if next request crosses segment boundary, set next pfn
1040 	 * to search from the segment boundary.
1041 	 */
1042 	if (((*pfnp + minctg - 1) & pfnseg) < (*pfnp & pfnseg))
1043 		*pfnp = roundup(*pfnp, pfnseg + 1);
1044 
1045 	/* clean up any pages already allocated */
1046 
1047 	while (plist) {
1048 		pp = plist;
1049 		page_sub(&plist, pp);
1050 		page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
1051 		if (iolock)
1052 			page_io_unlock(pp);
1053 		page_unlock(pp);
1054 	}
1055 
1056 	return (NULL);
1057 }
1058 #endif	/* !__xpv */
1059 
1060 /*
1061  * verify that pages being returned from allocator have correct DMA attribute
1062  */
1063 #ifndef DEBUG
1064 #define	check_dma(a, b, c) (0)
1065 #else
1066 static void
1067 check_dma(ddi_dma_attr_t *dma_attr, page_t *pp, int cnt)
1068 {
1069 	if (dma_attr == NULL)
1070 		return;
1071 
1072 	while (cnt-- > 0) {
1073 		if (pa_to_ma(pfn_to_pa(pp->p_pagenum)) <
1074 		    dma_attr->dma_attr_addr_lo)
1075 			panic("PFN (pp=%p) below dma_attr_addr_lo", pp);
1076 		if (pa_to_ma(pfn_to_pa(pp->p_pagenum)) >=
1077 		    dma_attr->dma_attr_addr_hi)
1078 			panic("PFN (pp=%p) above dma_attr_addr_hi", pp);
1079 		pp = pp->p_next;
1080 	}
1081 }
1082 #endif
1083 
1084 #if !defined(__xpv)
1085 static page_t *
1086 page_get_contigpage(pgcnt_t *pgcnt, ddi_dma_attr_t *mattr, int iolock)
1087 {
1088 	pfn_t		pfn;
1089 	int		sgllen;
1090 	uint64_t	pfnseg;
1091 	pgcnt_t		minctg;
1092 	page_t		*pplist = NULL, *plist;
1093 	uint64_t	lo, hi;
1094 	pgcnt_t		pfnalign = 0;
1095 	static pfn_t	startpfn;
1096 	static pgcnt_t	lastctgcnt;
1097 	uintptr_t	align;
1098 
1099 	CONTIG_LOCK();
1100 
1101 	if (mattr) {
1102 		lo = mmu_btop((mattr->dma_attr_addr_lo + MMU_PAGEOFFSET));
1103 		hi = mmu_btop(mattr->dma_attr_addr_hi);
1104 		if (hi >= physmax)
1105 			hi = physmax - 1;
1106 		sgllen = mattr->dma_attr_sgllen;
1107 		pfnseg = mmu_btop(mattr->dma_attr_seg);
1108 
1109 		align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
1110 		if (align > MMU_PAGESIZE)
1111 			pfnalign = mmu_btop(align);
1112 
1113 		/*
1114 		 * in order to satisfy the request, must minimally
1115 		 * acquire minctg contiguous pages
1116 		 */
1117 		minctg = howmany(*pgcnt, sgllen);
1118 
1119 		ASSERT(hi >= lo);
1120 
1121 		/*
1122 		 * start from where last searched if the minctg >= lastctgcnt
1123 		 */
1124 		if (minctg < lastctgcnt || startpfn < lo || startpfn > hi)
1125 			startpfn = lo;
1126 	} else {
1127 		hi = physmax - 1;
1128 		lo = 0;
1129 		sgllen = 1;
1130 		pfnseg = mmu.highest_pfn;
1131 		minctg = *pgcnt;
1132 
1133 		if (minctg < lastctgcnt)
1134 			startpfn = lo;
1135 	}
1136 	lastctgcnt = minctg;
1137 
1138 	ASSERT(pfnseg + 1 >= (uint64_t)minctg);
1139 
1140 	/* conserve 16m memory - start search above 16m when possible */
1141 	if (hi > PFN_16M && startpfn < PFN_16M)
1142 		startpfn = PFN_16M;
1143 
1144 	pfn = startpfn;
1145 	if (pfnalign)
1146 		pfn = P2ROUNDUP(pfn, pfnalign);
1147 
1148 	while (pfn + minctg - 1 <= hi) {
1149 
1150 		plist = is_contigpage_free(&pfn, pgcnt, minctg, pfnseg, iolock);
1151 		if (plist) {
1152 			page_list_concat(&pplist, &plist);
1153 			sgllen--;
1154 			/*
1155 			 * return when contig pages no longer needed
1156 			 */
1157 			if (!*pgcnt || ((*pgcnt <= sgllen) && !pfnalign)) {
1158 				startpfn = pfn;
1159 				CONTIG_UNLOCK();
1160 				check_dma(mattr, pplist, *pgcnt);
1161 				return (pplist);
1162 			}
1163 			minctg = howmany(*pgcnt, sgllen);
1164 		}
1165 		if (pfnalign)
1166 			pfn = P2ROUNDUP(pfn, pfnalign);
1167 	}
1168 
1169 	/* cannot find contig pages in specified range */
1170 	if (startpfn == lo) {
1171 		CONTIG_UNLOCK();
1172 		return (NULL);
1173 	}
1174 
1175 	/* did not start with lo previously */
1176 	pfn = lo;
1177 	if (pfnalign)
1178 		pfn = P2ROUNDUP(pfn, pfnalign);
1179 
1180 	/* allow search to go above startpfn */
1181 	while (pfn < startpfn) {
1182 
1183 		plist = is_contigpage_free(&pfn, pgcnt, minctg, pfnseg, iolock);
1184 		if (plist != NULL) {
1185 
1186 			page_list_concat(&pplist, &plist);
1187 			sgllen--;
1188 
1189 			/*
1190 			 * return when contig pages no longer needed
1191 			 */
1192 			if (!*pgcnt || ((*pgcnt <= sgllen) && !pfnalign)) {
1193 				startpfn = pfn;
1194 				CONTIG_UNLOCK();
1195 				check_dma(mattr, pplist, *pgcnt);
1196 				return (pplist);
1197 			}
1198 			minctg = howmany(*pgcnt, sgllen);
1199 		}
1200 		if (pfnalign)
1201 			pfn = P2ROUNDUP(pfn, pfnalign);
1202 	}
1203 	CONTIG_UNLOCK();
1204 	return (NULL);
1205 }
1206 #endif	/* !__xpv */
1207 
1208 /*
1209  * mnode_range_cnt() calculates the number of memory ranges for mnode and
1210  * memranges[]. Used to determine the size of page lists and mnoderanges.
1211  */
1212 int
1213 mnode_range_cnt(int mnode)
1214 {
1215 #if defined(__xpv)
1216 	ASSERT(mnode == 0);
1217 	return (1);
1218 #else	/* __xpv */
1219 	int	mri;
1220 	int	mnrcnt = 0;
1221 
1222 	if (mem_node_config[mnode].exists != 0) {
1223 		mri = nranges - 1;
1224 
1225 		/* find the memranges index below contained in mnode range */
1226 
1227 		while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
1228 			mri--;
1229 
1230 		/*
1231 		 * increment mnode range counter when memranges or mnode
1232 		 * boundary is reached.
1233 		 */
1234 		while (mri >= 0 &&
1235 		    mem_node_config[mnode].physmax >= MEMRANGELO(mri)) {
1236 			mnrcnt++;
1237 			if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
1238 				mri--;
1239 			else
1240 				break;
1241 		}
1242 	}
1243 	ASSERT(mnrcnt <= MAX_MNODE_MRANGES);
1244 	return (mnrcnt);
1245 #endif	/* __xpv */
1246 }
1247 
1248 /*
1249  * mnode_range_setup() initializes mnoderanges.
1250  */
1251 void
1252 mnode_range_setup(mnoderange_t *mnoderanges)
1253 {
1254 	int	mnode, mri;
1255 
1256 	for (mnode = 0; mnode < max_mem_nodes; mnode++) {
1257 		if (mem_node_config[mnode].exists == 0)
1258 			continue;
1259 
1260 		mri = nranges - 1;
1261 
1262 		while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
1263 			mri--;
1264 
1265 		while (mri >= 0 && mem_node_config[mnode].physmax >=
1266 		    MEMRANGELO(mri)) {
1267 			mnoderanges->mnr_pfnlo = MAX(MEMRANGELO(mri),
1268 			    mem_node_config[mnode].physbase);
1269 			mnoderanges->mnr_pfnhi = MIN(MEMRANGEHI(mri),
1270 			    mem_node_config[mnode].physmax);
1271 			mnoderanges->mnr_mnode = mnode;
1272 			mnoderanges->mnr_memrange = mri;
1273 			mnoderanges++;
1274 			if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
1275 				mri--;
1276 			else
1277 				break;
1278 		}
1279 	}
1280 }
1281 
1282 /*ARGSUSED*/
1283 int
1284 mtype_init(vnode_t *vp, caddr_t vaddr, uint_t *flags, size_t pgsz)
1285 {
1286 	int mtype = mnoderangecnt - 1;
1287 
1288 #if !defined(__xpv)
1289 #if defined(__i386)
1290 	/*
1291 	 * set the mtype range
1292 	 * - kmem requests needs to be below 4g if restricted_kmemalloc is set.
1293 	 * - for non kmem requests, set range to above 4g if memory below 4g
1294 	 * runs low.
1295 	 */
1296 	if (restricted_kmemalloc && VN_ISKAS(vp) &&
1297 	    (caddr_t)(vaddr) >= kernelheap &&
1298 	    (caddr_t)(vaddr) < ekernelheap) {
1299 		ASSERT(physmax4g);
1300 		mtype = mtype4g;
1301 		if (RESTRICT16M_ALLOC(freemem4g - btop(pgsz),
1302 		    btop(pgsz), *flags)) {
1303 			*flags |= PGI_MT_RANGE16M;
1304 		} else {
1305 			VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1306 			VM_STAT_COND_ADD((*flags & PG_PANIC),
1307 			    vmm_vmstats.pgpanicalloc);
1308 			*flags |= PGI_MT_RANGE0;
1309 		}
1310 		return (mtype);
1311 	}
1312 #endif	/* __i386 */
1313 
1314 	if (RESTRICT4G_ALLOC) {
1315 		VM_STAT_ADD(vmm_vmstats.restrict4gcnt);
1316 		/* here only for > 4g systems */
1317 		*flags |= PGI_MT_RANGE4G;
1318 	} else if (RESTRICT16M_ALLOC(freemem, btop(pgsz), *flags)) {
1319 		*flags |= PGI_MT_RANGE16M;
1320 	} else {
1321 		VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1322 		VM_STAT_COND_ADD((*flags & PG_PANIC), vmm_vmstats.pgpanicalloc);
1323 		*flags |= PGI_MT_RANGE0;
1324 	}
1325 #endif /* !__xpv */
1326 	return (mtype);
1327 }
1328 
1329 
1330 /* mtype init for page_get_replacement_page */
1331 /*ARGSUSED*/
1332 int
1333 mtype_pgr_init(int *flags, page_t *pp, int mnode, pgcnt_t pgcnt)
1334 {
1335 	int mtype = mnoderangecnt - 1;
1336 #if !defined(__ixpv)
1337 	if (RESTRICT16M_ALLOC(freemem, pgcnt, *flags)) {
1338 		*flags |= PGI_MT_RANGE16M;
1339 	} else {
1340 		VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1341 		*flags |= PGI_MT_RANGE0;
1342 	}
1343 #endif
1344 	return (mtype);
1345 }
1346 
1347 /*
1348  * Determine if the mnode range specified in mtype contains memory belonging
1349  * to memory node mnode.  If flags & PGI_MT_RANGE is set then mtype contains
1350  * the range of indices from high pfn to 0, 16m or 4g.
1351  *
1352  * Return first mnode range type index found otherwise return -1 if none found.
1353  */
1354 int
1355 mtype_func(int mnode, int mtype, uint_t flags)
1356 {
1357 	if (flags & PGI_MT_RANGE) {
1358 		int	mtlim = 0;
1359 
1360 		if (flags & PGI_MT_NEXT)
1361 			mtype--;
1362 		if (flags & PGI_MT_RANGE4G)
1363 			mtlim = mtype4g + 1;	/* exclude 0-4g range */
1364 		else if (flags & PGI_MT_RANGE16M)
1365 			mtlim = 1;		/* exclude 0-16m range */
1366 		while (mtype >= mtlim) {
1367 			if (mnoderanges[mtype].mnr_mnode == mnode)
1368 				return (mtype);
1369 			mtype--;
1370 		}
1371 	} else if (mnoderanges[mtype].mnr_mnode == mnode) {
1372 		return (mtype);
1373 	}
1374 	return (-1);
1375 }
1376 
1377 /*
1378  * Update the page list max counts with the pfn range specified by the
1379  * input parameters.  Called from add_physmem() when physical memory with
1380  * page_t's are initially added to the page lists.
1381  */
1382 void
1383 mtype_modify_max(pfn_t startpfn, long cnt)
1384 {
1385 	int	mtype = 0;
1386 	pfn_t	endpfn = startpfn + cnt, pfn;
1387 	pgcnt_t	inc;
1388 
1389 	ASSERT(cnt > 0);
1390 
1391 	if (!physmax4g)
1392 		return;
1393 
1394 	for (pfn = startpfn; pfn < endpfn; ) {
1395 		if (pfn <= mnoderanges[mtype].mnr_pfnhi) {
1396 			if (endpfn < mnoderanges[mtype].mnr_pfnhi) {
1397 				inc = endpfn - pfn;
1398 			} else {
1399 				inc = mnoderanges[mtype].mnr_pfnhi - pfn + 1;
1400 			}
1401 			if (mtype <= mtype4g)
1402 				maxmem4g += inc;
1403 			pfn += inc;
1404 		}
1405 		mtype++;
1406 		ASSERT(mtype < mnoderangecnt || pfn >= endpfn);
1407 	}
1408 }
1409 
1410 int
1411 mtype_2_mrange(int mtype)
1412 {
1413 	return (mnoderanges[mtype].mnr_memrange);
1414 }
1415 
1416 void
1417 mnodetype_2_pfn(int mnode, int mtype, pfn_t *pfnlo, pfn_t *pfnhi)
1418 {
1419 	ASSERT(mnoderanges[mtype].mnr_mnode == mnode);
1420 	*pfnlo = mnoderanges[mtype].mnr_pfnlo;
1421 	*pfnhi = mnoderanges[mtype].mnr_pfnhi;
1422 }
1423 
1424 size_t
1425 plcnt_sz(size_t ctrs_sz)
1426 {
1427 #ifdef DEBUG
1428 	int	szc, colors;
1429 
1430 	ctrs_sz += mnoderangecnt * sizeof (struct mnr_mts) * mmu_page_sizes;
1431 	for (szc = 0; szc < mmu_page_sizes; szc++) {
1432 		colors = page_get_pagecolors(szc);
1433 		ctrs_sz += mnoderangecnt * sizeof (pgcnt_t) * colors;
1434 	}
1435 #endif
1436 	return (ctrs_sz);
1437 }
1438 
1439 caddr_t
1440 plcnt_init(caddr_t addr)
1441 {
1442 #ifdef DEBUG
1443 	int	mt, szc, colors;
1444 
1445 	for (mt = 0; mt < mnoderangecnt; mt++) {
1446 		mnoderanges[mt].mnr_mts = (struct mnr_mts *)addr;
1447 		addr += (sizeof (struct mnr_mts) * mmu_page_sizes);
1448 		for (szc = 0; szc < mmu_page_sizes; szc++) {
1449 			colors = page_get_pagecolors(szc);
1450 			mnoderanges[mt].mnr_mts[szc].mnr_mts_colors = colors;
1451 			mnoderanges[mt].mnr_mts[szc].mnr_mtsc_pgcnt =
1452 			    (pgcnt_t *)addr;
1453 			addr += (sizeof (pgcnt_t) * colors);
1454 		}
1455 	}
1456 #endif
1457 	return (addr);
1458 }
1459 
1460 void
1461 plcnt_inc_dec(page_t *pp, int mtype, int szc, long cnt, int flags)
1462 {
1463 #ifdef DEBUG
1464 	int	bin = PP_2_BIN(pp);
1465 
1466 	atomic_add_long(&mnoderanges[mtype].mnr_mts[szc].mnr_mts_pgcnt, cnt);
1467 	atomic_add_long(&mnoderanges[mtype].mnr_mts[szc].mnr_mtsc_pgcnt[bin],
1468 	    cnt);
1469 #endif
1470 	ASSERT(mtype == PP_2_MTYPE(pp));
1471 	if (physmax4g && mtype <= mtype4g)
1472 		atomic_add_long(&freemem4g, cnt);
1473 	if (flags & PG_CACHE_LIST)
1474 		atomic_add_long(&mnoderanges[mtype].mnr_mt_clpgcnt, cnt);
1475 	else
1476 		atomic_add_long(&mnoderanges[mtype].mnr_mt_flpgcnt[szc], cnt);
1477 	atomic_add_long(&mnoderanges[mtype].mnr_mt_totcnt, cnt);
1478 }
1479 
1480 /*
1481  * Returns the free page count for mnode
1482  */
1483 int
1484 mnode_pgcnt(int mnode)
1485 {
1486 	int	mtype = mnoderangecnt - 1;
1487 	int	flags = PGI_MT_RANGE0;
1488 	pgcnt_t	pgcnt = 0;
1489 
1490 	mtype = mtype_func(mnode, mtype, flags);
1491 
1492 	while (mtype != -1) {
1493 		pgcnt += MTYPE_FREEMEM(mtype);
1494 		mtype = mtype_func(mnode, mtype, flags | PGI_MT_NEXT);
1495 	}
1496 	return (pgcnt);
1497 }
1498 
1499 /*
1500  * Initialize page coloring variables based on the l2 cache parameters.
1501  * Calculate and return memory needed for page coloring data structures.
1502  */
1503 size_t
1504 page_coloring_init(uint_t l2_sz, int l2_linesz, int l2_assoc)
1505 {
1506 	size_t	colorsz = 0;
1507 	int	i;
1508 	int	colors;
1509 
1510 #if defined(__xpv)
1511 	/*
1512 	 * Hypervisor domains currently don't have any concept of NUMA.
1513 	 * Hence we'll act like there is only 1 memrange.
1514 	 */
1515 	i = memrange_num(1);
1516 #else /* !__xpv */
1517 	/*
1518 	 * Reduce the memory ranges lists if we don't have large amounts
1519 	 * of memory. This avoids searching known empty free lists.
1520 	 */
1521 	i = memrange_num(physmax);
1522 #if defined(__i386)
1523 	if (i > 0)
1524 		restricted_kmemalloc = 0;
1525 #endif
1526 	/* physmax greater than 4g */
1527 	if (i == 0)
1528 		physmax4g = 1;
1529 #endif /* !__xpv */
1530 	memranges += i;
1531 	nranges -= i;
1532 
1533 	ASSERT(mmu_page_sizes <= MMU_PAGE_SIZES);
1534 
1535 	ASSERT(ISP2(l2_sz));
1536 	ASSERT(ISP2(l2_linesz));
1537 	ASSERT(l2_sz > MMU_PAGESIZE);
1538 
1539 	/* l2_assoc is 0 for fully associative l2 cache */
1540 	if (l2_assoc)
1541 		l2_colors = MAX(1, l2_sz / (l2_assoc * MMU_PAGESIZE));
1542 	else
1543 		l2_colors = 1;
1544 
1545 	/* for scalability, configure at least PAGE_COLORS_MIN color bins */
1546 	page_colors = MAX(l2_colors, PAGE_COLORS_MIN);
1547 
1548 	/*
1549 	 * cpu_page_colors is non-zero when a page color may be spread across
1550 	 * multiple bins.
1551 	 */
1552 	if (l2_colors < page_colors)
1553 		cpu_page_colors = l2_colors;
1554 
1555 	ASSERT(ISP2(page_colors));
1556 
1557 	page_colors_mask = page_colors - 1;
1558 
1559 	ASSERT(ISP2(CPUSETSIZE()));
1560 	page_coloring_shift = lowbit(CPUSETSIZE());
1561 
1562 	/* initialize number of colors per page size */
1563 	for (i = 0; i <= mmu.max_page_level; i++) {
1564 		hw_page_array[i].hp_size = LEVEL_SIZE(i);
1565 		hw_page_array[i].hp_shift = LEVEL_SHIFT(i);
1566 		hw_page_array[i].hp_pgcnt = LEVEL_SIZE(i) >> LEVEL_SHIFT(0);
1567 		hw_page_array[i].hp_colors = (page_colors_mask >>
1568 		    (hw_page_array[i].hp_shift - hw_page_array[0].hp_shift))
1569 		    + 1;
1570 		colorequivszc[i] = 0;
1571 	}
1572 
1573 	/*
1574 	 * The value of cpu_page_colors determines if additional color bins
1575 	 * need to be checked for a particular color in the page_get routines.
1576 	 */
1577 	if (cpu_page_colors != 0) {
1578 
1579 		int a = lowbit(page_colors) - lowbit(cpu_page_colors);
1580 		ASSERT(a > 0);
1581 		ASSERT(a < 16);
1582 
1583 		for (i = 0; i <= mmu.max_page_level; i++) {
1584 			if ((colors = hw_page_array[i].hp_colors) <= 1) {
1585 				colorequivszc[i] = 0;
1586 				continue;
1587 			}
1588 			while ((colors >> a) == 0)
1589 				a--;
1590 			ASSERT(a >= 0);
1591 
1592 			/* higher 4 bits encodes color equiv mask */
1593 			colorequivszc[i] = (a << 4);
1594 		}
1595 	}
1596 
1597 	/* factor in colorequiv to check additional 'equivalent' bins. */
1598 	if (colorequiv > 1) {
1599 
1600 		int a = lowbit(colorequiv) - 1;
1601 		if (a > 15)
1602 			a = 15;
1603 
1604 		for (i = 0; i <= mmu.max_page_level; i++) {
1605 			if ((colors = hw_page_array[i].hp_colors) <= 1) {
1606 				continue;
1607 			}
1608 			while ((colors >> a) == 0)
1609 				a--;
1610 			if ((a << 4) > colorequivszc[i]) {
1611 				colorequivszc[i] = (a << 4);
1612 			}
1613 		}
1614 	}
1615 
1616 	/* size for mnoderanges */
1617 	for (mnoderangecnt = 0, i = 0; i < max_mem_nodes; i++)
1618 		mnoderangecnt += mnode_range_cnt(i);
1619 	colorsz = mnoderangecnt * sizeof (mnoderange_t);
1620 
1621 	/* size for fpc_mutex and cpc_mutex */
1622 	colorsz += (2 * max_mem_nodes * sizeof (kmutex_t) * NPC_MUTEX);
1623 
1624 	/* size of page_freelists */
1625 	colorsz += mnoderangecnt * sizeof (page_t ***);
1626 	colorsz += mnoderangecnt * mmu_page_sizes * sizeof (page_t **);
1627 
1628 	for (i = 0; i < mmu_page_sizes; i++) {
1629 		colors = page_get_pagecolors(i);
1630 		colorsz += mnoderangecnt * colors * sizeof (page_t *);
1631 	}
1632 
1633 	/* size of page_cachelists */
1634 	colorsz += mnoderangecnt * sizeof (page_t **);
1635 	colorsz += mnoderangecnt * page_colors * sizeof (page_t *);
1636 
1637 	return (colorsz);
1638 }
1639 
1640 /*
1641  * Called once at startup to configure page_coloring data structures and
1642  * does the 1st page_free()/page_freelist_add().
1643  */
1644 void
1645 page_coloring_setup(caddr_t pcmemaddr)
1646 {
1647 	int	i;
1648 	int	j;
1649 	int	k;
1650 	caddr_t	addr;
1651 	int	colors;
1652 
1653 	/*
1654 	 * do page coloring setup
1655 	 */
1656 	addr = pcmemaddr;
1657 
1658 	mnoderanges = (mnoderange_t *)addr;
1659 	addr += (mnoderangecnt * sizeof (mnoderange_t));
1660 
1661 	mnode_range_setup(mnoderanges);
1662 
1663 	if (physmax4g)
1664 		mtype4g = pfn_2_mtype(0xfffff);
1665 
1666 	for (k = 0; k < NPC_MUTEX; k++) {
1667 		fpc_mutex[k] = (kmutex_t *)addr;
1668 		addr += (max_mem_nodes * sizeof (kmutex_t));
1669 	}
1670 	for (k = 0; k < NPC_MUTEX; k++) {
1671 		cpc_mutex[k] = (kmutex_t *)addr;
1672 		addr += (max_mem_nodes * sizeof (kmutex_t));
1673 	}
1674 	page_freelists = (page_t ****)addr;
1675 	addr += (mnoderangecnt * sizeof (page_t ***));
1676 
1677 	page_cachelists = (page_t ***)addr;
1678 	addr += (mnoderangecnt * sizeof (page_t **));
1679 
1680 	for (i = 0; i < mnoderangecnt; i++) {
1681 		page_freelists[i] = (page_t ***)addr;
1682 		addr += (mmu_page_sizes * sizeof (page_t **));
1683 
1684 		for (j = 0; j < mmu_page_sizes; j++) {
1685 			colors = page_get_pagecolors(j);
1686 			page_freelists[i][j] = (page_t **)addr;
1687 			addr += (colors * sizeof (page_t *));
1688 		}
1689 		page_cachelists[i] = (page_t **)addr;
1690 		addr += (page_colors * sizeof (page_t *));
1691 	}
1692 }
1693 
1694 #if defined(__xpv)
1695 /*
1696  * Give back 10% of the io_pool pages to the free list.
1697  * Don't shrink the pool below some absolute minimum.
1698  */
1699 static void
1700 page_io_pool_shrink()
1701 {
1702 	int retcnt;
1703 	page_t *pp, *pp_first, *pp_last, **curpool;
1704 	mfn_t mfn;
1705 	int bothpools = 0;
1706 
1707 	mutex_enter(&io_pool_lock);
1708 	io_pool_shrink_attempts++;	/* should be a kstat? */
1709 	retcnt = io_pool_cnt / 10;
1710 	if (io_pool_cnt - retcnt < io_pool_cnt_min)
1711 		retcnt = io_pool_cnt - io_pool_cnt_min;
1712 	if (retcnt <= 0)
1713 		goto done;
1714 	io_pool_shrinks++;	/* should be a kstat? */
1715 	curpool = &io_pool_4g;
1716 domore:
1717 	/*
1718 	 * Loop through taking pages from the end of the list
1719 	 * (highest mfns) till amount to return reached.
1720 	 */
1721 	for (pp = *curpool; pp && retcnt > 0; ) {
1722 		pp_first = pp_last = pp->p_prev;
1723 		if (pp_first == *curpool)
1724 			break;
1725 		retcnt--;
1726 		io_pool_cnt--;
1727 		page_io_pool_sub(curpool, pp_first, pp_last);
1728 		if ((mfn = pfn_to_mfn(pp->p_pagenum)) < start_mfn)
1729 			start_mfn = mfn;
1730 		page_free(pp_first, 1);
1731 		pp = *curpool;
1732 	}
1733 	if (retcnt != 0 && !bothpools) {
1734 		/*
1735 		 * If not enough found in less constrained pool try the
1736 		 * more constrained one.
1737 		 */
1738 		curpool = &io_pool_16m;
1739 		bothpools = 1;
1740 		goto domore;
1741 	}
1742 done:
1743 	mutex_exit(&io_pool_lock);
1744 }
1745 
1746 #endif	/* __xpv */
1747 
1748 uint_t
1749 page_create_update_flags_x86(uint_t flags)
1750 {
1751 #if defined(__xpv)
1752 	/*
1753 	 * Check this is an urgent allocation and free pages are depleted.
1754 	 */
1755 	if (!(flags & PG_WAIT) && freemem < desfree)
1756 		page_io_pool_shrink();
1757 #else /* !__xpv */
1758 	/*
1759 	 * page_create_get_something may call this because 4g memory may be
1760 	 * depleted. Set flags to allow for relocation of base page below
1761 	 * 4g if necessary.
1762 	 */
1763 	if (physmax4g)
1764 		flags |= (PGI_PGCPSZC0 | PGI_PGCPHIPRI);
1765 #endif /* __xpv */
1766 	return (flags);
1767 }
1768 
1769 /*ARGSUSED*/
1770 int
1771 bp_color(struct buf *bp)
1772 {
1773 	return (0);
1774 }
1775 
1776 #if defined(__xpv)
1777 
1778 /*
1779  * Take pages out of an io_pool
1780  */
1781 static void
1782 page_io_pool_sub(page_t **poolp, page_t *pp_first, page_t *pp_last)
1783 {
1784 	if (*poolp == pp_first) {
1785 		*poolp = pp_last->p_next;
1786 		if (*poolp == pp_first)
1787 			*poolp = NULL;
1788 	}
1789 	pp_first->p_prev->p_next = pp_last->p_next;
1790 	pp_last->p_next->p_prev = pp_first->p_prev;
1791 	pp_first->p_prev = pp_last;
1792 	pp_last->p_next = pp_first;
1793 }
1794 
1795 /*
1796  * Put a page on the io_pool list. The list is ordered by increasing MFN.
1797  */
1798 static void
1799 page_io_pool_add(page_t **poolp, page_t *pp)
1800 {
1801 	page_t	*look;
1802 	mfn_t	mfn = mfn_list[pp->p_pagenum];
1803 
1804 	if (*poolp == NULL) {
1805 		*poolp = pp;
1806 		pp->p_next = pp;
1807 		pp->p_prev = pp;
1808 		return;
1809 	}
1810 
1811 	/*
1812 	 * Since we try to take pages from the high end of the pool
1813 	 * chances are good that the pages to be put on the list will
1814 	 * go at or near the end of the list. so start at the end and
1815 	 * work backwards.
1816 	 */
1817 	look = (*poolp)->p_prev;
1818 	while (mfn < mfn_list[look->p_pagenum]) {
1819 		look = look->p_prev;
1820 		if (look == (*poolp)->p_prev)
1821 			break; /* backed all the way to front of list */
1822 	}
1823 
1824 	/* insert after look */
1825 	pp->p_prev = look;
1826 	pp->p_next = look->p_next;
1827 	pp->p_next->p_prev = pp;
1828 	look->p_next = pp;
1829 	if (mfn < mfn_list[(*poolp)->p_pagenum]) {
1830 		/*
1831 		 * we inserted a new first list element
1832 		 * adjust pool pointer to newly inserted element
1833 		 */
1834 		*poolp = pp;
1835 	}
1836 }
1837 
1838 /*
1839  * Add a page to the io_pool.  Setting the force flag will force the page
1840  * into the io_pool no matter what.
1841  */
1842 static void
1843 add_page_to_pool(page_t *pp, int force)
1844 {
1845 	page_t *highest;
1846 	page_t *freep = NULL;
1847 
1848 	mutex_enter(&io_pool_lock);
1849 	/*
1850 	 * Always keep the scarce low memory pages
1851 	 */
1852 	if (mfn_list[pp->p_pagenum] < PFN_16MEG) {
1853 		++io_pool_cnt;
1854 		page_io_pool_add(&io_pool_16m, pp);
1855 		goto done;
1856 	}
1857 	if (io_pool_cnt < io_pool_cnt_max || force || io_pool_4g == NULL) {
1858 		++io_pool_cnt;
1859 		page_io_pool_add(&io_pool_4g, pp);
1860 	} else {
1861 		highest = io_pool_4g->p_prev;
1862 		if (mfn_list[pp->p_pagenum] < mfn_list[highest->p_pagenum]) {
1863 			page_io_pool_sub(&io_pool_4g, highest, highest);
1864 			page_io_pool_add(&io_pool_4g, pp);
1865 			freep = highest;
1866 		} else {
1867 			freep = pp;
1868 		}
1869 	}
1870 done:
1871 	mutex_exit(&io_pool_lock);
1872 	if (freep)
1873 		page_free(freep, 1);
1874 }
1875 
1876 
1877 int contig_pfn_cnt;	/* no of pfns in the contig pfn list */
1878 int contig_pfn_max;	/* capacity of the contig pfn list */
1879 int next_alloc_pfn;	/* next position in list to start a contig search */
1880 int contig_pfnlist_updates;	/* pfn list update count */
1881 int contig_pfnlist_builds;	/* how many times have we (re)built list */
1882 int contig_pfnlist_buildfailed;	/* how many times has list build failed */
1883 int create_contig_pending;	/* nonzero means taskq creating contig list */
1884 pfn_t *contig_pfn_list = NULL;	/* list of contig pfns in ascending mfn order */
1885 
1886 /*
1887  * Function to use in sorting a list of pfns by their underlying mfns.
1888  */
1889 static int
1890 mfn_compare(const void *pfnp1, const void *pfnp2)
1891 {
1892 	mfn_t mfn1 = mfn_list[*(pfn_t *)pfnp1];
1893 	mfn_t mfn2 = mfn_list[*(pfn_t *)pfnp2];
1894 
1895 	if (mfn1 > mfn2)
1896 		return (1);
1897 	if (mfn1 < mfn2)
1898 		return (-1);
1899 	return (0);
1900 }
1901 
1902 /*
1903  * Compact the contig_pfn_list by tossing all the non-contiguous
1904  * elements from the list.
1905  */
1906 static void
1907 compact_contig_pfn_list(void)
1908 {
1909 	pfn_t pfn, lapfn, prev_lapfn;
1910 	mfn_t mfn;
1911 	int i, newcnt = 0;
1912 
1913 	prev_lapfn = 0;
1914 	for (i = 0; i < contig_pfn_cnt - 1; i++) {
1915 		pfn = contig_pfn_list[i];
1916 		lapfn = contig_pfn_list[i + 1];
1917 		mfn = mfn_list[pfn];
1918 		/*
1919 		 * See if next pfn is for a contig mfn
1920 		 */
1921 		if (mfn_list[lapfn] != mfn + 1)
1922 			continue;
1923 		/*
1924 		 * pfn and lookahead are both put in list
1925 		 * unless pfn is the previous lookahead.
1926 		 */
1927 		if (pfn != prev_lapfn)
1928 			contig_pfn_list[newcnt++] = pfn;
1929 		contig_pfn_list[newcnt++] = lapfn;
1930 		prev_lapfn = lapfn;
1931 	}
1932 	for (i = newcnt; i < contig_pfn_cnt; i++)
1933 		contig_pfn_list[i] = 0;
1934 	contig_pfn_cnt = newcnt;
1935 }
1936 
1937 /*ARGSUSED*/
1938 static void
1939 call_create_contiglist(void *arg)
1940 {
1941 	(void) create_contig_pfnlist(PG_WAIT);
1942 }
1943 
1944 /*
1945  * Create list of freelist pfns that have underlying
1946  * contiguous mfns.  The list is kept in ascending mfn order.
1947  * returns 1 if list created else 0.
1948  */
1949 static int
1950 create_contig_pfnlist(uint_t flags)
1951 {
1952 	pfn_t pfn;
1953 	page_t *pp;
1954 	int ret = 1;
1955 
1956 	mutex_enter(&contig_list_lock);
1957 	if (contig_pfn_list != NULL)
1958 		goto out;
1959 	contig_pfn_max = freemem + (freemem / 10);
1960 	contig_pfn_list = kmem_zalloc(contig_pfn_max * sizeof (pfn_t),
1961 	    (flags & PG_WAIT) ? KM_SLEEP : KM_NOSLEEP);
1962 	if (contig_pfn_list == NULL) {
1963 		/*
1964 		 * If we could not create the contig list (because
1965 		 * we could not sleep for memory).  Dispatch a taskq that can
1966 		 * sleep to get the memory.
1967 		 */
1968 		if (!create_contig_pending) {
1969 			if (taskq_dispatch(system_taskq, call_create_contiglist,
1970 			    NULL, TQ_NOSLEEP) != NULL)
1971 				create_contig_pending = 1;
1972 		}
1973 		contig_pfnlist_buildfailed++;	/* count list build failures */
1974 		ret = 0;
1975 		goto out;
1976 	}
1977 	create_contig_pending = 0;
1978 	ASSERT(contig_pfn_cnt == 0);
1979 	for (pfn = 0; pfn < mfn_count; pfn++) {
1980 		pp = page_numtopp_nolock(pfn);
1981 		if (pp == NULL || !PP_ISFREE(pp))
1982 			continue;
1983 		contig_pfn_list[contig_pfn_cnt] = pfn;
1984 		if (++contig_pfn_cnt == contig_pfn_max)
1985 			break;
1986 	}
1987 	qsort(contig_pfn_list, contig_pfn_cnt, sizeof (pfn_t), mfn_compare);
1988 	compact_contig_pfn_list();
1989 	/*
1990 	 * Make sure next search of the newly created contiguous pfn
1991 	 * list starts at the beginning of the list.
1992 	 */
1993 	next_alloc_pfn = 0;
1994 	contig_pfnlist_builds++;	/* count list builds */
1995 out:
1996 	mutex_exit(&contig_list_lock);
1997 	return (ret);
1998 }
1999 
2000 
2001 /*
2002  * Toss the current contig pfnlist.  Someone is about to do a massive
2003  * update to pfn<->mfn mappings.  So we have them destroy the list and lock
2004  * it till they are done with their update.
2005  */
2006 void
2007 clear_and_lock_contig_pfnlist()
2008 {
2009 	pfn_t *listp = NULL;
2010 	size_t listsize;
2011 
2012 	mutex_enter(&contig_list_lock);
2013 	if (contig_pfn_list != NULL) {
2014 		listp = contig_pfn_list;
2015 		listsize = contig_pfn_max * sizeof (pfn_t);
2016 		contig_pfn_list = NULL;
2017 		contig_pfn_max = contig_pfn_cnt = 0;
2018 	}
2019 	if (listp != NULL)
2020 		kmem_free(listp, listsize);
2021 }
2022 
2023 /*
2024  * Unlock the contig_pfn_list.  The next attempted use of it will cause
2025  * it to be re-created.
2026  */
2027 void
2028 unlock_contig_pfnlist()
2029 {
2030 	mutex_exit(&contig_list_lock);
2031 }
2032 
2033 /*
2034  * Update the contiguous pfn list in response to a pfn <-> mfn reassignment
2035  */
2036 void
2037 update_contig_pfnlist(pfn_t pfn, mfn_t oldmfn, mfn_t newmfn)
2038 {
2039 	int probe_hi, probe_lo, probe_pos, insert_after, insert_point;
2040 	pfn_t probe_pfn;
2041 	mfn_t probe_mfn;
2042 	int drop_lock = 0;
2043 
2044 	if (mutex_owner(&contig_list_lock) != curthread) {
2045 		drop_lock = 1;
2046 		mutex_enter(&contig_list_lock);
2047 	}
2048 	if (contig_pfn_list == NULL)
2049 		goto done;
2050 	contig_pfnlist_updates++;
2051 	/*
2052 	 * Find the pfn in the current list.  Use a binary chop to locate it.
2053 	 */
2054 	probe_hi = contig_pfn_cnt - 1;
2055 	probe_lo = 0;
2056 	probe_pos = (probe_hi + probe_lo) / 2;
2057 	while ((probe_pfn = contig_pfn_list[probe_pos]) != pfn) {
2058 		if (probe_pos == probe_lo) { /* pfn not in list */
2059 			probe_pos = -1;
2060 			break;
2061 		}
2062 		if (pfn_to_mfn(probe_pfn) <= oldmfn)
2063 			probe_lo = probe_pos;
2064 		else
2065 			probe_hi = probe_pos;
2066 		probe_pos = (probe_hi + probe_lo) / 2;
2067 	}
2068 	if (probe_pos >= 0)  { /* remove pfn fom list */
2069 		contig_pfn_cnt--;
2070 		ovbcopy(&contig_pfn_list[probe_pos + 1],
2071 		    &contig_pfn_list[probe_pos],
2072 		    (contig_pfn_cnt - probe_pos) * sizeof (pfn_t));
2073 	}
2074 	if (newmfn == MFN_INVALID)
2075 		goto done;
2076 	/*
2077 	 * Check if new mfn has adjacent mfns in the list
2078 	 */
2079 	probe_hi = contig_pfn_cnt - 1;
2080 	probe_lo = 0;
2081 	insert_after = -2;
2082 	do {
2083 		probe_pos = (probe_hi + probe_lo) / 2;
2084 		probe_mfn = pfn_to_mfn(contig_pfn_list[probe_pos]);
2085 		if (newmfn == probe_mfn + 1)
2086 			insert_after = probe_pos;
2087 		else if (newmfn == probe_mfn - 1)
2088 			insert_after = probe_pos - 1;
2089 		if (probe_pos == probe_lo)
2090 			break;
2091 		if (probe_mfn <= newmfn)
2092 			probe_lo = probe_pos;
2093 		else
2094 			probe_hi = probe_pos;
2095 	} while (insert_after == -2);
2096 	/*
2097 	 * If there is space in the list and there are adjacent mfns
2098 	 * insert the pfn in to its proper place in the list.
2099 	 */
2100 	if (insert_after != -2 && contig_pfn_cnt + 1 <= contig_pfn_max) {
2101 		insert_point = insert_after + 1;
2102 		ovbcopy(&contig_pfn_list[insert_point],
2103 		    &contig_pfn_list[insert_point + 1],
2104 		    (contig_pfn_cnt - insert_point) * sizeof (pfn_t));
2105 		contig_pfn_list[insert_point] = pfn;
2106 		contig_pfn_cnt++;
2107 	}
2108 done:
2109 	if (drop_lock)
2110 		mutex_exit(&contig_list_lock);
2111 }
2112 
2113 /*
2114  * Called to (re-)populate the io_pool from the free page lists.
2115  */
2116 long
2117 populate_io_pool(void)
2118 {
2119 	pfn_t pfn;
2120 	mfn_t mfn, max_mfn;
2121 	page_t *pp;
2122 
2123 	/*
2124 	 * Figure out the bounds of the pool on first invocation.
2125 	 * We use a percentage of memory for the io pool size.
2126 	 * we allow that to shrink, but not to less than a fixed minimum
2127 	 */
2128 	if (io_pool_cnt_max == 0) {
2129 		io_pool_cnt_max = physmem / (100 / io_pool_physmem_pct);
2130 		io_pool_cnt_lowater = io_pool_cnt_max;
2131 		/*
2132 		 * This is the first time in populate_io_pool, grab a va to use
2133 		 * when we need to allocate pages.
2134 		 */
2135 		io_pool_kva = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP);
2136 	}
2137 	/*
2138 	 * If we are out of pages in the pool, then grow the size of the pool
2139 	 */
2140 	if (io_pool_cnt == 0) {
2141 		/*
2142 		 * Grow the max size of the io pool by 5%, but never more than
2143 		 * 25% of physical memory.
2144 		 */
2145 		if (io_pool_cnt_max < physmem / 4)
2146 			io_pool_cnt_max += io_pool_cnt_max / 20;
2147 	}
2148 	io_pool_grows++;	/* should be a kstat? */
2149 
2150 	/*
2151 	 * Get highest mfn on this platform, but limit to the 32 bit DMA max.
2152 	 */
2153 	(void) mfn_to_pfn(start_mfn);
2154 	max_mfn = MIN(cached_max_mfn, PFN_4GIG);
2155 	for (mfn = start_mfn; mfn < max_mfn; start_mfn = ++mfn) {
2156 		pfn = mfn_to_pfn(mfn);
2157 		if (pfn & PFN_IS_FOREIGN_MFN)
2158 			continue;
2159 		/*
2160 		 * try to allocate it from free pages
2161 		 */
2162 		pp = page_numtopp_alloc(pfn);
2163 		if (pp == NULL)
2164 			continue;
2165 		PP_CLRFREE(pp);
2166 		add_page_to_pool(pp, 1);
2167 		if (io_pool_cnt >= io_pool_cnt_max)
2168 			break;
2169 	}
2170 
2171 	return (io_pool_cnt);
2172 }
2173 
2174 /*
2175  * Destroy a page that was being used for DMA I/O. It may or
2176  * may not actually go back to the io_pool.
2177  */
2178 void
2179 page_destroy_io(page_t *pp)
2180 {
2181 	mfn_t mfn = mfn_list[pp->p_pagenum];
2182 
2183 	/*
2184 	 * When the page was alloc'd a reservation was made, release it now
2185 	 */
2186 	page_unresv(1);
2187 	/*
2188 	 * Unload translations, if any, then hash out the
2189 	 * page to erase its identity.
2190 	 */
2191 	(void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
2192 	page_hashout(pp, NULL);
2193 
2194 	/*
2195 	 * If the page came from the free lists, just put it back to them.
2196 	 * DomU pages always go on the free lists as well.
2197 	 */
2198 	if (!DOMAIN_IS_INITDOMAIN(xen_info) || mfn >= PFN_4GIG) {
2199 		page_free(pp, 1);
2200 		return;
2201 	}
2202 
2203 	add_page_to_pool(pp, 0);
2204 }
2205 
2206 
2207 long contig_searches;		/* count of times contig pages requested */
2208 long contig_search_restarts;	/* count of contig ranges tried */
2209 long contig_search_failed;	/* count of contig alloc failures */
2210 
2211 /*
2212  * Look thru the contiguous pfns that are not part of the io_pool for
2213  * contiguous free pages.  Return a list of the found pages or NULL.
2214  */
2215 page_t *
2216 find_contig_free(uint_t npages, uint_t flags, uint64_t pfnseg)
2217 {
2218 	page_t *pp, *plist = NULL;
2219 	mfn_t mfn, prev_mfn, start_mfn;
2220 	pfn_t pfn;
2221 	int pages_needed, pages_requested;
2222 	int search_start;
2223 
2224 	/*
2225 	 * create the contig pfn list if not already done
2226 	 */
2227 retry:
2228 	mutex_enter(&contig_list_lock);
2229 	if (contig_pfn_list == NULL) {
2230 		mutex_exit(&contig_list_lock);
2231 		if (!create_contig_pfnlist(flags)) {
2232 			return (NULL);
2233 		}
2234 		goto retry;
2235 	}
2236 	contig_searches++;
2237 	/*
2238 	 * Search contiguous pfn list for physically contiguous pages not in
2239 	 * the io_pool.  Start the search where the last search left off.
2240 	 */
2241 	pages_requested = pages_needed = npages;
2242 	search_start = next_alloc_pfn;
2243 	start_mfn = prev_mfn = 0;
2244 	while (pages_needed) {
2245 		pfn = contig_pfn_list[next_alloc_pfn];
2246 		mfn = pfn_to_mfn(pfn);
2247 		/*
2248 		 * Check if mfn is first one or contig to previous one and
2249 		 * if page corresponding to mfn is free and that mfn
2250 		 * range is not crossing a segment boundary.
2251 		 */
2252 		if ((prev_mfn == 0 || mfn == prev_mfn + 1) &&
2253 		    (pp = page_numtopp_alloc(pfn)) != NULL &&
2254 		    !((mfn & pfnseg) < (start_mfn & pfnseg))) {
2255 			PP_CLRFREE(pp);
2256 			page_io_pool_add(&plist, pp);
2257 			pages_needed--;
2258 			if (prev_mfn == 0)
2259 				start_mfn = mfn;
2260 			prev_mfn = mfn;
2261 		} else {
2262 			contig_search_restarts++;
2263 			/*
2264 			 * free partial page list
2265 			 */
2266 			while (plist != NULL) {
2267 				pp = plist;
2268 				page_io_pool_sub(&plist, pp, pp);
2269 				page_free(pp, 1);
2270 			}
2271 			pages_needed = pages_requested;
2272 			start_mfn = prev_mfn = 0;
2273 		}
2274 		if (++next_alloc_pfn == contig_pfn_cnt)
2275 			next_alloc_pfn = 0;
2276 		if (next_alloc_pfn == search_start)
2277 			break; /* all pfns searched */
2278 	}
2279 	mutex_exit(&contig_list_lock);
2280 	if (pages_needed) {
2281 		contig_search_failed++;
2282 		/*
2283 		 * Failed to find enough contig pages.
2284 		 * free partial page list
2285 		 */
2286 		while (plist != NULL) {
2287 			pp = plist;
2288 			page_io_pool_sub(&plist, pp, pp);
2289 			page_free(pp, 1);
2290 		}
2291 	}
2292 	return (plist);
2293 }
2294 
2295 /*
2296  * Search the reserved io pool pages for a page range with the
2297  * desired characteristics.
2298  */
2299 page_t *
2300 page_io_pool_alloc(ddi_dma_attr_t *mattr, int contig, pgcnt_t minctg)
2301 {
2302 	page_t *pp_first, *pp_last;
2303 	page_t *pp, **poolp;
2304 	pgcnt_t nwanted, pfnalign;
2305 	uint64_t pfnseg;
2306 	mfn_t mfn, tmfn, hi_mfn, lo_mfn;
2307 	int align, attempt = 0;
2308 
2309 	if (minctg == 1)
2310 		contig = 0;
2311 	lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
2312 	hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
2313 	pfnseg = mmu_btop(mattr->dma_attr_seg);
2314 	align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
2315 	if (align > MMU_PAGESIZE)
2316 		pfnalign = mmu_btop(align);
2317 	else
2318 		pfnalign = 0;
2319 
2320 try_again:
2321 	/*
2322 	 * See if we want pages for a legacy device
2323 	 */
2324 	if (hi_mfn < PFN_16MEG)
2325 		poolp = &io_pool_16m;
2326 	else
2327 		poolp = &io_pool_4g;
2328 try_smaller:
2329 	/*
2330 	 * Take pages from I/O pool. We'll use pages from the highest
2331 	 * MFN range possible.
2332 	 */
2333 	pp_first = pp_last = NULL;
2334 	mutex_enter(&io_pool_lock);
2335 	nwanted = minctg;
2336 	for (pp = *poolp; pp && nwanted > 0; ) {
2337 		pp = pp->p_prev;
2338 
2339 		/*
2340 		 * skip pages above allowable range
2341 		 */
2342 		mfn = mfn_list[pp->p_pagenum];
2343 		if (hi_mfn < mfn)
2344 			goto skip;
2345 
2346 		/*
2347 		 * stop at pages below allowable range
2348 		 */
2349 		if (lo_mfn > mfn)
2350 			break;
2351 restart:
2352 		if (pp_last == NULL) {
2353 			/*
2354 			 * Check alignment
2355 			 */
2356 			tmfn = mfn - (minctg - 1);
2357 			if (pfnalign && tmfn != P2ROUNDUP(tmfn, pfnalign))
2358 				goto skip; /* not properly aligned */
2359 			/*
2360 			 * Check segment
2361 			 */
2362 			if ((mfn & pfnseg) < (tmfn & pfnseg))
2363 				goto skip; /* crosses seg boundary */
2364 			/*
2365 			 * Start building page list
2366 			 */
2367 			pp_first = pp_last = pp;
2368 			nwanted--;
2369 		} else {
2370 			/*
2371 			 * check physical contiguity if required
2372 			 */
2373 			if (contig &&
2374 			    mfn_list[pp_first->p_pagenum] != mfn + 1) {
2375 				/*
2376 				 * not a contiguous page, restart list.
2377 				 */
2378 				pp_last = NULL;
2379 				nwanted = minctg;
2380 				goto restart;
2381 			} else { /* add page to list */
2382 				pp_first = pp;
2383 				nwanted--;
2384 			}
2385 		}
2386 skip:
2387 		if (pp == *poolp)
2388 			break;
2389 	}
2390 
2391 	/*
2392 	 * If we didn't find memory. Try the more constrained pool, then
2393 	 * sweep free pages into the DMA pool and try again.
2394 	 */
2395 	if (nwanted != 0) {
2396 		mutex_exit(&io_pool_lock);
2397 		/*
2398 		 * If we were looking in the less constrained pool and
2399 		 * didn't find pages, try the more constrained pool.
2400 		 */
2401 		if (poolp == &io_pool_4g) {
2402 			poolp = &io_pool_16m;
2403 			goto try_smaller;
2404 		}
2405 		kmem_reap();
2406 		if (++attempt < 4) {
2407 			/*
2408 			 * Grab some more io_pool pages
2409 			 */
2410 			(void) populate_io_pool();
2411 			goto try_again; /* go around and retry */
2412 		}
2413 		return (NULL);
2414 	}
2415 	/*
2416 	 * Found the pages, now snip them from the list
2417 	 */
2418 	page_io_pool_sub(poolp, pp_first, pp_last);
2419 	io_pool_cnt -= minctg;
2420 	/*
2421 	 * reset low water mark
2422 	 */
2423 	if (io_pool_cnt < io_pool_cnt_lowater)
2424 		io_pool_cnt_lowater = io_pool_cnt;
2425 	mutex_exit(&io_pool_lock);
2426 	return (pp_first);
2427 }
2428 
2429 page_t *
2430 page_swap_with_hypervisor(struct vnode *vp, u_offset_t off, caddr_t vaddr,
2431     ddi_dma_attr_t *mattr, uint_t flags, pgcnt_t minctg)
2432 {
2433 	uint_t kflags;
2434 	int order, extra, extpages, i, contig, nbits, extents;
2435 	page_t *pp, *expp, *pp_first, **pplist = NULL;
2436 	mfn_t *mfnlist = NULL;
2437 
2438 	contig = flags & PG_PHYSCONTIG;
2439 	if (minctg == 1)
2440 		contig = 0;
2441 	flags &= ~PG_PHYSCONTIG;
2442 	kflags = flags & PG_WAIT ? KM_SLEEP : KM_NOSLEEP;
2443 	/*
2444 	 * Hypervisor will allocate extents, if we want contig
2445 	 * pages extent must be >= minctg
2446 	 */
2447 	if (contig) {
2448 		order = highbit(minctg) - 1;
2449 		if (minctg & ((1 << order) - 1))
2450 			order++;
2451 		extpages = 1 << order;
2452 	} else {
2453 		order = 0;
2454 		extpages = minctg;
2455 	}
2456 	if (extpages > minctg) {
2457 		extra = extpages - minctg;
2458 		if (!page_resv(extra, kflags))
2459 			return (NULL);
2460 	}
2461 	pp_first = NULL;
2462 	pplist = kmem_alloc(extpages * sizeof (page_t *), kflags);
2463 	if (pplist == NULL)
2464 		goto balloon_fail;
2465 	mfnlist = kmem_alloc(extpages * sizeof (mfn_t), kflags);
2466 	if (mfnlist == NULL)
2467 		goto balloon_fail;
2468 	pp = page_create_va(vp, off, minctg * PAGESIZE, flags, &kvseg, vaddr);
2469 	if (pp == NULL)
2470 		goto balloon_fail;
2471 	pp_first = pp;
2472 	if (extpages > minctg) {
2473 		/*
2474 		 * fill out the rest of extent pages to swap
2475 		 * with the hypervisor
2476 		 */
2477 		for (i = 0; i < extra; i++) {
2478 			expp = page_create_va(vp,
2479 			    (u_offset_t)(uintptr_t)io_pool_kva,
2480 			    PAGESIZE, flags, &kvseg, io_pool_kva);
2481 			if (expp == NULL)
2482 				goto balloon_fail;
2483 			(void) hat_pageunload(expp, HAT_FORCE_PGUNLOAD);
2484 			page_io_unlock(expp);
2485 			page_hashout(expp, NULL);
2486 			page_io_lock(expp);
2487 			/*
2488 			 * add page to end of list
2489 			 */
2490 			expp->p_prev = pp_first->p_prev;
2491 			expp->p_next = pp_first;
2492 			expp->p_prev->p_next = expp;
2493 			pp_first->p_prev = expp;
2494 		}
2495 
2496 	}
2497 	for (i = 0; i < extpages; i++) {
2498 		pplist[i] = pp;
2499 		pp = pp->p_next;
2500 	}
2501 	nbits = highbit(mattr->dma_attr_addr_hi);
2502 	extents = contig ? 1 : minctg;
2503 	if (balloon_replace_pages(extents, pplist, nbits, order,
2504 	    mfnlist) != extents) {
2505 		if (ioalloc_dbg)
2506 			cmn_err(CE_NOTE, "request to hypervisor"
2507 			    " for %d pages, maxaddr %" PRIx64 " failed",
2508 			    extpages, mattr->dma_attr_addr_hi);
2509 		goto balloon_fail;
2510 	}
2511 
2512 	kmem_free(pplist, extpages * sizeof (page_t *));
2513 	kmem_free(mfnlist, extpages * sizeof (mfn_t));
2514 	/*
2515 	 * Return any excess pages to free list
2516 	 */
2517 	if (extpages > minctg) {
2518 		for (i = 0; i < extra; i++) {
2519 			pp = pp_first->p_prev;
2520 			page_sub(&pp_first, pp);
2521 			page_io_unlock(pp);
2522 			page_unresv(1);
2523 			page_free(pp, 1);
2524 		}
2525 	}
2526 	return (pp_first);
2527 balloon_fail:
2528 	/*
2529 	 * Return pages to free list and return failure
2530 	 */
2531 	while (pp_first != NULL) {
2532 		pp = pp_first;
2533 		page_sub(&pp_first, pp);
2534 		page_io_unlock(pp);
2535 		if (pp->p_vnode != NULL)
2536 			page_hashout(pp, NULL);
2537 		page_free(pp, 1);
2538 	}
2539 	if (pplist)
2540 		kmem_free(pplist, extpages * sizeof (page_t *));
2541 	if (mfnlist)
2542 		kmem_free(mfnlist, extpages * sizeof (mfn_t));
2543 	page_unresv(extpages - minctg);
2544 	return (NULL);
2545 }
2546 
2547 static void
2548 return_partial_alloc(page_t *plist)
2549 {
2550 	page_t *pp;
2551 
2552 	while (plist != NULL) {
2553 		pp = plist;
2554 		page_sub(&plist, pp);
2555 		page_destroy_io(pp);
2556 	}
2557 }
2558 
2559 static page_t *
2560 page_get_contigpages(
2561 	struct vnode	*vp,
2562 	u_offset_t	off,
2563 	int		*npagesp,
2564 	uint_t		flags,
2565 	caddr_t		vaddr,
2566 	ddi_dma_attr_t	*mattr)
2567 {
2568 	mfn_t	max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL);
2569 	page_t	*plist;	/* list to return */
2570 	page_t	*pp, *mcpl;
2571 	int	contig, anyaddr, npages, getone = 0;
2572 	mfn_t	lo_mfn;
2573 	mfn_t	hi_mfn;
2574 	pgcnt_t	pfnalign = 0;
2575 	int	align, sgllen;
2576 	uint64_t pfnseg;
2577 	pgcnt_t	minctg;
2578 
2579 	npages = *npagesp;
2580 	ASSERT(mattr != NULL);
2581 	lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
2582 	hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
2583 	sgllen = mattr->dma_attr_sgllen;
2584 	pfnseg = mmu_btop(mattr->dma_attr_seg);
2585 	align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
2586 	if (align > MMU_PAGESIZE)
2587 		pfnalign = mmu_btop(align);
2588 
2589 	/*
2590 	 * Clear the contig flag if only one page is needed.
2591 	 */
2592 	contig = flags & PG_PHYSCONTIG;
2593 	if (npages == 1) {
2594 		getone = 1;
2595 		contig = 0;
2596 	}
2597 
2598 	/*
2599 	 * Check if any page in the system is fine.
2600 	 */
2601 	anyaddr = lo_mfn == 0 && hi_mfn >= max_mfn && !pfnalign;
2602 	if (!contig && anyaddr) {
2603 		flags &= ~PG_PHYSCONTIG;
2604 		plist = page_create_va(vp, off, npages * MMU_PAGESIZE,
2605 		    flags, &kvseg, vaddr);
2606 		if (plist != NULL) {
2607 			*npagesp = 0;
2608 			return (plist);
2609 		}
2610 	}
2611 	plist = NULL;
2612 	minctg = howmany(npages, sgllen);
2613 	while (npages > sgllen || getone) {
2614 		if (minctg > npages)
2615 			minctg = npages;
2616 		mcpl = NULL;
2617 		/*
2618 		 * We could just want unconstrained but contig pages.
2619 		 */
2620 		if (anyaddr && contig) {
2621 			/*
2622 			 * Look for free contig pages to satisfy the request.
2623 			 */
2624 			mcpl = find_contig_free(minctg, flags, pfnseg);
2625 		}
2626 		/*
2627 		 * Try the reserved io pools next
2628 		 */
2629 		if (mcpl == NULL)
2630 			mcpl = page_io_pool_alloc(mattr, contig, minctg);
2631 		if (mcpl != NULL) {
2632 			pp = mcpl;
2633 			do {
2634 				if (!page_hashin(pp, vp, off, NULL)) {
2635 					panic("page_get_contigpages:"
2636 					    " hashin failed"
2637 					    " pp %p, vp %p, off %llx",
2638 					    (void *)pp, (void *)vp, off);
2639 				}
2640 				off += MMU_PAGESIZE;
2641 				PP_CLRFREE(pp);
2642 				PP_CLRAGED(pp);
2643 				page_set_props(pp, P_REF);
2644 				page_io_lock(pp);
2645 				pp = pp->p_next;
2646 			} while (pp != mcpl);
2647 		} else {
2648 			/*
2649 			 * Hypervisor exchange doesn't handle segment or
2650 			 * alignment constraints
2651 			 */
2652 			if (mattr->dma_attr_seg < mattr->dma_attr_addr_hi ||
2653 			    pfnalign)
2654 				goto fail;
2655 			/*
2656 			 * Try exchanging pages with the hypervisor
2657 			 */
2658 			mcpl = page_swap_with_hypervisor(vp, off, vaddr, mattr,
2659 			    flags, minctg);
2660 			if (mcpl == NULL)
2661 				goto fail;
2662 			off += minctg * MMU_PAGESIZE;
2663 		}
2664 		check_dma(mattr, mcpl, minctg);
2665 		/*
2666 		 * Here with a minctg run of contiguous pages, add them to the
2667 		 * list we will return for this request.
2668 		 */
2669 		page_list_concat(&plist, &mcpl);
2670 		npages -= minctg;
2671 		*npagesp = npages;
2672 		sgllen--;
2673 		if (getone)
2674 			break;
2675 	}
2676 	return (plist);
2677 fail:
2678 	return_partial_alloc(plist);
2679 	return (NULL);
2680 }
2681 
2682 /*
2683  * Allocator for domain 0 I/O pages. We match the required
2684  * DMA attributes and contiguity constraints.
2685  */
2686 /*ARGSUSED*/
2687 page_t *
2688 page_create_io(
2689 	struct vnode	*vp,
2690 	u_offset_t	off,
2691 	uint_t		bytes,
2692 	uint_t		flags,
2693 	struct as	*as,
2694 	caddr_t		vaddr,
2695 	ddi_dma_attr_t	*mattr)
2696 {
2697 	page_t	*plist = NULL, *pp;
2698 	int	npages = 0, contig, anyaddr, pages_req;
2699 	mfn_t	lo_mfn;
2700 	mfn_t	hi_mfn;
2701 	pgcnt_t	pfnalign = 0;
2702 	int	align;
2703 	int	is_domu = 0;
2704 	int	dummy, bytes_got;
2705 	mfn_t	max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL);
2706 
2707 	ASSERT(mattr != NULL);
2708 	lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
2709 	hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
2710 	align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
2711 	if (align > MMU_PAGESIZE)
2712 		pfnalign = mmu_btop(align);
2713 
2714 	/*
2715 	 * Clear the contig flag if only one page is needed or the scatter
2716 	 * gather list length is >= npages.
2717 	 */
2718 	pages_req = npages = mmu_btopr(bytes);
2719 	contig = (flags & PG_PHYSCONTIG);
2720 	bytes = P2ROUNDUP(bytes, MMU_PAGESIZE);
2721 	if (bytes == MMU_PAGESIZE || mattr->dma_attr_sgllen >= npages)
2722 		contig = 0;
2723 
2724 	/*
2725 	 * Check if any old page in the system is fine.
2726 	 * DomU should always go down this path.
2727 	 */
2728 	is_domu = !DOMAIN_IS_INITDOMAIN(xen_info);
2729 	anyaddr = lo_mfn == 0 && hi_mfn >= max_mfn && !pfnalign;
2730 	if ((!contig && anyaddr) || is_domu) {
2731 		flags &= ~PG_PHYSCONTIG;
2732 		plist = page_create_va(vp, off, bytes, flags, &kvseg, vaddr);
2733 		if (plist != NULL)
2734 			return (plist);
2735 		else if (is_domu)
2736 			return (NULL); /* no memory available */
2737 	}
2738 	/*
2739 	 * DomU should never reach here
2740 	 */
2741 	if (contig) {
2742 		plist = page_get_contigpages(vp, off, &npages, flags, vaddr,
2743 		    mattr);
2744 		if (plist == NULL)
2745 			goto fail;
2746 		bytes_got = (pages_req - npages) << MMU_PAGESHIFT;
2747 		vaddr += bytes_got;
2748 		off += bytes_got;
2749 		/*
2750 		 * We now have all the contiguous pages we need, but
2751 		 * we may still need additional non-contiguous pages.
2752 		 */
2753 	}
2754 	/*
2755 	 * now loop collecting the requested number of pages, these do
2756 	 * not have to be contiguous pages but we will use the contig
2757 	 * page alloc code to get the pages since it will honor any
2758 	 * other constraints the pages may have.
2759 	 */
2760 	while (npages--) {
2761 		dummy = 1;
2762 		pp = page_get_contigpages(vp, off, &dummy, flags, vaddr, mattr);
2763 		if (pp == NULL)
2764 			goto fail;
2765 		page_add(&plist, pp);
2766 		vaddr += MMU_PAGESIZE;
2767 		off += MMU_PAGESIZE;
2768 	}
2769 	return (plist);
2770 fail:
2771 	/*
2772 	 * Failed to get enough pages, return ones we did get
2773 	 */
2774 	return_partial_alloc(plist);
2775 	return (NULL);
2776 }
2777 
2778 /*
2779  * Lock and return the page with the highest mfn that we can find.  last_mfn
2780  * holds the last one found, so the next search can start from there.  We
2781  * also keep a counter so that we don't loop forever if the machine has no
2782  * free pages.
2783  *
2784  * This is called from the balloon thread to find pages to give away.  new_high
2785  * is used when new mfn's have been added to the system - we will reset our
2786  * search if the new mfn's are higher than our current search position.
2787  */
2788 page_t *
2789 page_get_high_mfn(mfn_t new_high)
2790 {
2791 	static mfn_t last_mfn = 0;
2792 	pfn_t pfn;
2793 	page_t *pp;
2794 	ulong_t loop_count = 0;
2795 
2796 	if (new_high > last_mfn)
2797 		last_mfn = new_high;
2798 
2799 	for (; loop_count < mfn_count; loop_count++, last_mfn--) {
2800 		if (last_mfn == 0) {
2801 			last_mfn = cached_max_mfn;
2802 		}
2803 
2804 		pfn = mfn_to_pfn(last_mfn);
2805 		if (pfn & PFN_IS_FOREIGN_MFN)
2806 			continue;
2807 
2808 		/* See if the page is free.  If so, lock it. */
2809 		pp = page_numtopp_alloc(pfn);
2810 		if (pp == NULL)
2811 			continue;
2812 		PP_CLRFREE(pp);
2813 
2814 		ASSERT(PAGE_EXCL(pp));
2815 		ASSERT(pp->p_vnode == NULL);
2816 		ASSERT(!hat_page_is_mapped(pp));
2817 		last_mfn--;
2818 		return (pp);
2819 	}
2820 	return (NULL);
2821 }
2822 
2823 #else /* !__xpv */
2824 
2825 /*
2826  * get a page from any list with the given mnode
2827  */
2828 static page_t *
2829 page_get_mnode_anylist(ulong_t origbin, uchar_t szc, uint_t flags,
2830     int mnode, int mtype, ddi_dma_attr_t *dma_attr)
2831 {
2832 	kmutex_t		*pcm;
2833 	int			i;
2834 	page_t			*pp;
2835 	page_t			*first_pp;
2836 	uint64_t		pgaddr;
2837 	ulong_t			bin;
2838 	int			mtypestart;
2839 	int			plw_initialized;
2840 	page_list_walker_t	plw;
2841 
2842 	VM_STAT_ADD(pga_vmstats.pgma_alloc);
2843 
2844 	ASSERT((flags & PG_MATCH_COLOR) == 0);
2845 	ASSERT(szc == 0);
2846 	ASSERT(dma_attr != NULL);
2847 
2848 	MTYPE_START(mnode, mtype, flags);
2849 	if (mtype < 0) {
2850 		VM_STAT_ADD(pga_vmstats.pgma_allocempty);
2851 		return (NULL);
2852 	}
2853 
2854 	mtypestart = mtype;
2855 
2856 	bin = origbin;
2857 
2858 	/*
2859 	 * check up to page_colors + 1 bins - origbin may be checked twice
2860 	 * because of BIN_STEP skip
2861 	 */
2862 	do {
2863 		plw_initialized = 0;
2864 
2865 		for (plw.plw_count = 0;
2866 		    plw.plw_count < page_colors; plw.plw_count++) {
2867 
2868 			if (PAGE_FREELISTS(mnode, szc, bin, mtype) == NULL)
2869 				goto nextfreebin;
2870 
2871 			pcm = PC_BIN_MUTEX(mnode, bin, PG_FREE_LIST);
2872 			mutex_enter(pcm);
2873 			pp = PAGE_FREELISTS(mnode, szc, bin, mtype);
2874 			first_pp = pp;
2875 			while (pp != NULL) {
2876 				if (page_trylock(pp, SE_EXCL) == 0) {
2877 					pp = pp->p_next;
2878 					if (pp == first_pp) {
2879 						pp = NULL;
2880 					}
2881 					continue;
2882 				}
2883 
2884 				ASSERT(PP_ISFREE(pp));
2885 				ASSERT(PP_ISAGED(pp));
2886 				ASSERT(pp->p_vnode == NULL);
2887 				ASSERT(pp->p_hash == NULL);
2888 				ASSERT(pp->p_offset == (u_offset_t)-1);
2889 				ASSERT(pp->p_szc == szc);
2890 				ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode);
2891 				/* check if page within DMA attributes */
2892 				pgaddr = pa_to_ma(pfn_to_pa(pp->p_pagenum));
2893 				if ((pgaddr >= dma_attr->dma_attr_addr_lo) &&
2894 				    (pgaddr + MMU_PAGESIZE - 1 <=
2895 				    dma_attr->dma_attr_addr_hi)) {
2896 					break;
2897 				}
2898 
2899 				/* continue looking */
2900 				page_unlock(pp);
2901 				pp = pp->p_next;
2902 				if (pp == first_pp)
2903 					pp = NULL;
2904 
2905 			}
2906 			if (pp != NULL) {
2907 				ASSERT(mtype == PP_2_MTYPE(pp));
2908 				ASSERT(pp->p_szc == 0);
2909 
2910 				/* found a page with specified DMA attributes */
2911 				page_sub(&PAGE_FREELISTS(mnode, szc, bin,
2912 				    mtype), pp);
2913 				page_ctr_sub(mnode, mtype, pp, PG_FREE_LIST);
2914 
2915 				if ((PP_ISFREE(pp) == 0) ||
2916 				    (PP_ISAGED(pp) == 0)) {
2917 					cmn_err(CE_PANIC, "page %p is not free",
2918 					    (void *)pp);
2919 				}
2920 
2921 				mutex_exit(pcm);
2922 				check_dma(dma_attr, pp, 1);
2923 				VM_STAT_ADD(pga_vmstats.pgma_allocok);
2924 				return (pp);
2925 			}
2926 			mutex_exit(pcm);
2927 nextfreebin:
2928 			if (plw_initialized == 0) {
2929 				page_list_walk_init(szc, 0, bin, 1, 0, &plw);
2930 				ASSERT(plw.plw_ceq_dif == page_colors);
2931 				plw_initialized = 1;
2932 			}
2933 
2934 			if (plw.plw_do_split) {
2935 				pp = page_freelist_split(szc, bin, mnode,
2936 				    mtype,
2937 				    mmu_btop(dma_attr->dma_attr_addr_hi + 1),
2938 				    &plw);
2939 				if (pp != NULL)
2940 					return (pp);
2941 			}
2942 
2943 			bin = page_list_walk_next_bin(szc, bin, &plw);
2944 		}
2945 
2946 		MTYPE_NEXT(mnode, mtype, flags);
2947 	} while (mtype >= 0);
2948 
2949 	/* failed to find a page in the freelist; try it in the cachelist */
2950 
2951 	/* reset mtype start for cachelist search */
2952 	mtype = mtypestart;
2953 	ASSERT(mtype >= 0);
2954 
2955 	/* start with the bin of matching color */
2956 	bin = origbin;
2957 
2958 	do {
2959 		for (i = 0; i <= page_colors; i++) {
2960 			if (PAGE_CACHELISTS(mnode, bin, mtype) == NULL)
2961 				goto nextcachebin;
2962 			pcm = PC_BIN_MUTEX(mnode, bin, PG_CACHE_LIST);
2963 			mutex_enter(pcm);
2964 			pp = PAGE_CACHELISTS(mnode, bin, mtype);
2965 			first_pp = pp;
2966 			while (pp != NULL) {
2967 				if (page_trylock(pp, SE_EXCL) == 0) {
2968 					pp = pp->p_next;
2969 					if (pp == first_pp)
2970 						break;
2971 					continue;
2972 				}
2973 				ASSERT(pp->p_vnode);
2974 				ASSERT(PP_ISAGED(pp) == 0);
2975 				ASSERT(pp->p_szc == 0);
2976 				ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode);
2977 
2978 				/* check if page within DMA attributes */
2979 
2980 				pgaddr = pa_to_ma(pfn_to_pa(pp->p_pagenum));
2981 				if ((pgaddr >= dma_attr->dma_attr_addr_lo) &&
2982 				    (pgaddr + MMU_PAGESIZE - 1 <=
2983 				    dma_attr->dma_attr_addr_hi)) {
2984 					break;
2985 				}
2986 
2987 				/* continue looking */
2988 				page_unlock(pp);
2989 				pp = pp->p_next;
2990 				if (pp == first_pp)
2991 					pp = NULL;
2992 			}
2993 
2994 			if (pp != NULL) {
2995 				ASSERT(mtype == PP_2_MTYPE(pp));
2996 				ASSERT(pp->p_szc == 0);
2997 
2998 				/* found a page with specified DMA attributes */
2999 				page_sub(&PAGE_CACHELISTS(mnode, bin,
3000 				    mtype), pp);
3001 				page_ctr_sub(mnode, mtype, pp, PG_CACHE_LIST);
3002 
3003 				mutex_exit(pcm);
3004 				ASSERT(pp->p_vnode);
3005 				ASSERT(PP_ISAGED(pp) == 0);
3006 				check_dma(dma_attr, pp, 1);
3007 				VM_STAT_ADD(pga_vmstats.pgma_allocok);
3008 				return (pp);
3009 			}
3010 			mutex_exit(pcm);
3011 nextcachebin:
3012 			bin += (i == 0) ? BIN_STEP : 1;
3013 			bin &= page_colors_mask;
3014 		}
3015 		MTYPE_NEXT(mnode, mtype, flags);
3016 	} while (mtype >= 0);
3017 
3018 	VM_STAT_ADD(pga_vmstats.pgma_allocfailed);
3019 	return (NULL);
3020 }
3021 
3022 /*
3023  * This function is similar to page_get_freelist()/page_get_cachelist()
3024  * but it searches both the lists to find a page with the specified
3025  * color (or no color) and DMA attributes. The search is done in the
3026  * freelist first and then in the cache list within the highest memory
3027  * range (based on DMA attributes) before searching in the lower
3028  * memory ranges.
3029  *
3030  * Note: This function is called only by page_create_io().
3031  */
3032 /*ARGSUSED*/
3033 static page_t *
3034 page_get_anylist(struct vnode *vp, u_offset_t off, struct as *as, caddr_t vaddr,
3035     size_t size, uint_t flags, ddi_dma_attr_t *dma_attr, lgrp_t	*lgrp)
3036 {
3037 	uint_t		bin;
3038 	int		mtype;
3039 	page_t		*pp;
3040 	int		n;
3041 	int		m;
3042 	int		szc;
3043 	int		fullrange;
3044 	int		mnode;
3045 	int		local_failed_stat = 0;
3046 	lgrp_mnode_cookie_t	lgrp_cookie;
3047 
3048 	VM_STAT_ADD(pga_vmstats.pga_alloc);
3049 
3050 	/* only base pagesize currently supported */
3051 	if (size != MMU_PAGESIZE)
3052 		return (NULL);
3053 
3054 	/*
3055 	 * If we're passed a specific lgroup, we use it.  Otherwise,
3056 	 * assume first-touch placement is desired.
3057 	 */
3058 	if (!LGRP_EXISTS(lgrp))
3059 		lgrp = lgrp_home_lgrp();
3060 
3061 	/* LINTED */
3062 	AS_2_BIN(as, seg, vp, vaddr, bin, 0);
3063 
3064 	/*
3065 	 * Only hold one freelist or cachelist lock at a time, that way we
3066 	 * can start anywhere and not have to worry about lock
3067 	 * ordering.
3068 	 */
3069 	if (dma_attr == NULL) {
3070 		n = 0;
3071 		m = mnoderangecnt - 1;
3072 		fullrange = 1;
3073 		VM_STAT_ADD(pga_vmstats.pga_nulldmaattr);
3074 	} else {
3075 		pfn_t pfnlo = mmu_btop(dma_attr->dma_attr_addr_lo);
3076 		pfn_t pfnhi = mmu_btop(dma_attr->dma_attr_addr_hi);
3077 
3078 		/*
3079 		 * We can guarantee alignment only for page boundary.
3080 		 */
3081 		if (dma_attr->dma_attr_align > MMU_PAGESIZE)
3082 			return (NULL);
3083 
3084 		n = pfn_2_mtype(pfnlo);
3085 		m = pfn_2_mtype(pfnhi);
3086 
3087 		fullrange = ((pfnlo == mnoderanges[n].mnr_pfnlo) &&
3088 		    (pfnhi >= mnoderanges[m].mnr_pfnhi));
3089 	}
3090 	VM_STAT_COND_ADD(fullrange == 0, pga_vmstats.pga_notfullrange);
3091 
3092 	if (n > m)
3093 		return (NULL);
3094 
3095 	szc = 0;
3096 
3097 	/* cylcing thru mtype handled by RANGE0 if n == 0 */
3098 	if (n == 0) {
3099 		flags |= PGI_MT_RANGE0;
3100 		n = m;
3101 	}
3102 
3103 	/*
3104 	 * Try local memory node first, but try remote if we can't
3105 	 * get a page of the right color.
3106 	 */
3107 	LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, LGRP_SRCH_HIER);
3108 	while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) {
3109 		/*
3110 		 * allocate pages from high pfn to low.
3111 		 */
3112 		for (mtype = m; mtype >= n; mtype--) {
3113 			if (fullrange != 0) {
3114 				pp = page_get_mnode_freelist(mnode,
3115 				    bin, mtype, szc, flags);
3116 				if (pp == NULL) {
3117 					pp = page_get_mnode_cachelist(
3118 					    bin, flags, mnode, mtype);
3119 				}
3120 			} else {
3121 				pp = page_get_mnode_anylist(bin, szc,
3122 				    flags, mnode, mtype, dma_attr);
3123 			}
3124 			if (pp != NULL) {
3125 				VM_STAT_ADD(pga_vmstats.pga_allocok);
3126 				check_dma(dma_attr, pp, 1);
3127 				return (pp);
3128 			}
3129 		}
3130 		if (!local_failed_stat) {
3131 			lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_ALLOC_FAIL, 1);
3132 			local_failed_stat = 1;
3133 		}
3134 	}
3135 	VM_STAT_ADD(pga_vmstats.pga_allocfailed);
3136 
3137 	return (NULL);
3138 }
3139 
3140 /*
3141  * page_create_io()
3142  *
3143  * This function is a copy of page_create_va() with an additional
3144  * argument 'mattr' that specifies DMA memory requirements to
3145  * the page list functions. This function is used by the segkmem
3146  * allocator so it is only to create new pages (i.e PG_EXCL is
3147  * set).
3148  *
3149  * Note: This interface is currently used by x86 PSM only and is
3150  *	 not fully specified so the commitment level is only for
3151  *	 private interface specific to x86. This interface uses PSM
3152  *	 specific page_get_anylist() interface.
3153  */
3154 
3155 #define	PAGE_HASH_SEARCH(index, pp, vp, off) { \
3156 	for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash) { \
3157 		if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \
3158 			break; \
3159 	} \
3160 }
3161 
3162 
3163 page_t *
3164 page_create_io(
3165 	struct vnode	*vp,
3166 	u_offset_t	off,
3167 	uint_t		bytes,
3168 	uint_t		flags,
3169 	struct as	*as,
3170 	caddr_t		vaddr,
3171 	ddi_dma_attr_t	*mattr)	/* DMA memory attributes if any */
3172 {
3173 	page_t		*plist = NULL;
3174 	uint_t		plist_len = 0;
3175 	pgcnt_t		npages;
3176 	page_t		*npp = NULL;
3177 	uint_t		pages_req;
3178 	page_t		*pp;
3179 	kmutex_t	*phm = NULL;
3180 	uint_t		index;
3181 
3182 	TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START,
3183 	    "page_create_start:vp %p off %llx bytes %u flags %x",
3184 	    vp, off, bytes, flags);
3185 
3186 	ASSERT((flags & ~(PG_EXCL | PG_WAIT | PG_PHYSCONTIG)) == 0);
3187 
3188 	pages_req = npages = mmu_btopr(bytes);
3189 
3190 	/*
3191 	 * Do the freemem and pcf accounting.
3192 	 */
3193 	if (!page_create_wait(npages, flags)) {
3194 		return (NULL);
3195 	}
3196 
3197 	TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS,
3198 	    "page_create_success:vp %p off %llx", vp, off);
3199 
3200 	/*
3201 	 * If satisfying this request has left us with too little
3202 	 * memory, start the wheels turning to get some back.  The
3203 	 * first clause of the test prevents waking up the pageout
3204 	 * daemon in situations where it would decide that there's
3205 	 * nothing to do.
3206 	 */
3207 	if (nscan < desscan && freemem < minfree) {
3208 		TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL,
3209 		    "pageout_cv_signal:freemem %ld", freemem);
3210 		cv_signal(&proc_pageout->p_cv);
3211 	}
3212 
3213 	if (flags & PG_PHYSCONTIG) {
3214 
3215 		plist = page_get_contigpage(&npages, mattr, 1);
3216 		if (plist == NULL) {
3217 			page_create_putback(npages);
3218 			return (NULL);
3219 		}
3220 
3221 		pp = plist;
3222 
3223 		do {
3224 			if (!page_hashin(pp, vp, off, NULL)) {
3225 				panic("pg_creat_io: hashin failed %p %p %llx",
3226 				    (void *)pp, (void *)vp, off);
3227 			}
3228 			VM_STAT_ADD(page_create_new);
3229 			off += MMU_PAGESIZE;
3230 			PP_CLRFREE(pp);
3231 			PP_CLRAGED(pp);
3232 			page_set_props(pp, P_REF);
3233 			pp = pp->p_next;
3234 		} while (pp != plist);
3235 
3236 		if (!npages) {
3237 			check_dma(mattr, plist, pages_req);
3238 			return (plist);
3239 		} else {
3240 			vaddr += (pages_req - npages) << MMU_PAGESHIFT;
3241 		}
3242 
3243 		/*
3244 		 * fall-thru:
3245 		 *
3246 		 * page_get_contigpage returns when npages <= sgllen.
3247 		 * Grab the rest of the non-contig pages below from anylist.
3248 		 */
3249 	}
3250 
3251 	/*
3252 	 * Loop around collecting the requested number of pages.
3253 	 * Most of the time, we have to `create' a new page. With
3254 	 * this in mind, pull the page off the free list before
3255 	 * getting the hash lock.  This will minimize the hash
3256 	 * lock hold time, nesting, and the like.  If it turns
3257 	 * out we don't need the page, we put it back at the end.
3258 	 */
3259 	while (npages--) {
3260 		phm = NULL;
3261 
3262 		index = PAGE_HASH_FUNC(vp, off);
3263 top:
3264 		ASSERT(phm == NULL);
3265 		ASSERT(index == PAGE_HASH_FUNC(vp, off));
3266 		ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
3267 
3268 		if (npp == NULL) {
3269 			/*
3270 			 * Try to get the page of any color either from
3271 			 * the freelist or from the cache list.
3272 			 */
3273 			npp = page_get_anylist(vp, off, as, vaddr, MMU_PAGESIZE,
3274 			    flags & ~PG_MATCH_COLOR, mattr, NULL);
3275 			if (npp == NULL) {
3276 				if (mattr == NULL) {
3277 					/*
3278 					 * Not looking for a special page;
3279 					 * panic!
3280 					 */
3281 					panic("no page found %d", (int)npages);
3282 				}
3283 				/*
3284 				 * No page found! This can happen
3285 				 * if we are looking for a page
3286 				 * within a specific memory range
3287 				 * for DMA purposes. If PG_WAIT is
3288 				 * specified then we wait for a
3289 				 * while and then try again. The
3290 				 * wait could be forever if we
3291 				 * don't get the page(s) we need.
3292 				 *
3293 				 * Note: XXX We really need a mechanism
3294 				 * to wait for pages in the desired
3295 				 * range. For now, we wait for any
3296 				 * pages and see if we can use it.
3297 				 */
3298 
3299 				if ((mattr != NULL) && (flags & PG_WAIT)) {
3300 					delay(10);
3301 					goto top;
3302 				}
3303 				goto fail; /* undo accounting stuff */
3304 			}
3305 
3306 			if (PP_ISAGED(npp) == 0) {
3307 				/*
3308 				 * Since this page came from the
3309 				 * cachelist, we must destroy the
3310 				 * old vnode association.
3311 				 */
3312 				page_hashout(npp, (kmutex_t *)NULL);
3313 			}
3314 		}
3315 
3316 		/*
3317 		 * We own this page!
3318 		 */
3319 		ASSERT(PAGE_EXCL(npp));
3320 		ASSERT(npp->p_vnode == NULL);
3321 		ASSERT(!hat_page_is_mapped(npp));
3322 		PP_CLRFREE(npp);
3323 		PP_CLRAGED(npp);
3324 
3325 		/*
3326 		 * Here we have a page in our hot little mits and are
3327 		 * just waiting to stuff it on the appropriate lists.
3328 		 * Get the mutex and check to see if it really does
3329 		 * not exist.
3330 		 */
3331 		phm = PAGE_HASH_MUTEX(index);
3332 		mutex_enter(phm);
3333 		PAGE_HASH_SEARCH(index, pp, vp, off);
3334 		if (pp == NULL) {
3335 			VM_STAT_ADD(page_create_new);
3336 			pp = npp;
3337 			npp = NULL;
3338 			if (!page_hashin(pp, vp, off, phm)) {
3339 				/*
3340 				 * Since we hold the page hash mutex and
3341 				 * just searched for this page, page_hashin
3342 				 * had better not fail.  If it does, that
3343 				 * means somethread did not follow the
3344 				 * page hash mutex rules.  Panic now and
3345 				 * get it over with.  As usual, go down
3346 				 * holding all the locks.
3347 				 */
3348 				ASSERT(MUTEX_HELD(phm));
3349 				panic("page_create: hashin fail %p %p %llx %p",
3350 				    (void *)pp, (void *)vp, off, (void *)phm);
3351 
3352 			}
3353 			ASSERT(MUTEX_HELD(phm));
3354 			mutex_exit(phm);
3355 			phm = NULL;
3356 
3357 			/*
3358 			 * Hat layer locking need not be done to set
3359 			 * the following bits since the page is not hashed
3360 			 * and was on the free list (i.e., had no mappings).
3361 			 *
3362 			 * Set the reference bit to protect
3363 			 * against immediate pageout
3364 			 *
3365 			 * XXXmh modify freelist code to set reference
3366 			 * bit so we don't have to do it here.
3367 			 */
3368 			page_set_props(pp, P_REF);
3369 		} else {
3370 			ASSERT(MUTEX_HELD(phm));
3371 			mutex_exit(phm);
3372 			phm = NULL;
3373 			/*
3374 			 * NOTE: This should not happen for pages associated
3375 			 *	 with kernel vnode 'kvp'.
3376 			 */
3377 			/* XX64 - to debug why this happens! */
3378 			ASSERT(!VN_ISKAS(vp));
3379 			if (VN_ISKAS(vp))
3380 				cmn_err(CE_NOTE,
3381 				    "page_create: page not expected "
3382 				    "in hash list for kernel vnode - pp 0x%p",
3383 				    (void *)pp);
3384 			VM_STAT_ADD(page_create_exists);
3385 			goto fail;
3386 		}
3387 
3388 		/*
3389 		 * Got a page!  It is locked.  Acquire the i/o
3390 		 * lock since we are going to use the p_next and
3391 		 * p_prev fields to link the requested pages together.
3392 		 */
3393 		page_io_lock(pp);
3394 		page_add(&plist, pp);
3395 		plist = plist->p_next;
3396 		off += MMU_PAGESIZE;
3397 		vaddr += MMU_PAGESIZE;
3398 	}
3399 
3400 	check_dma(mattr, plist, pages_req);
3401 	return (plist);
3402 
3403 fail:
3404 	if (npp != NULL) {
3405 		/*
3406 		 * Did not need this page after all.
3407 		 * Put it back on the free list.
3408 		 */
3409 		VM_STAT_ADD(page_create_putbacks);
3410 		PP_SETFREE(npp);
3411 		PP_SETAGED(npp);
3412 		npp->p_offset = (u_offset_t)-1;
3413 		page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL);
3414 		page_unlock(npp);
3415 	}
3416 
3417 	/*
3418 	 * Give up the pages we already got.
3419 	 */
3420 	while (plist != NULL) {
3421 		pp = plist;
3422 		page_sub(&plist, pp);
3423 		page_io_unlock(pp);
3424 		plist_len++;
3425 		/*LINTED: constant in conditional ctx*/
3426 		VN_DISPOSE(pp, B_INVAL, 0, kcred);
3427 	}
3428 
3429 	/*
3430 	 * VN_DISPOSE does freemem accounting for the pages in plist
3431 	 * by calling page_free. So, we need to undo the pcf accounting
3432 	 * for only the remaining pages.
3433 	 */
3434 	VM_STAT_ADD(page_create_putbacks);
3435 	page_create_putback(pages_req - plist_len);
3436 
3437 	return (NULL);
3438 }
3439 #endif /* !__xpv */
3440 
3441 
3442 /*
3443  * Copy the data from the physical page represented by "frompp" to
3444  * that represented by "topp". ppcopy uses CPU->cpu_caddr1 and
3445  * CPU->cpu_caddr2.  It assumes that no one uses either map at interrupt
3446  * level and no one sleeps with an active mapping there.
3447  *
3448  * Note that the ref/mod bits in the page_t's are not affected by
3449  * this operation, hence it is up to the caller to update them appropriately.
3450  */
3451 int
3452 ppcopy(page_t *frompp, page_t *topp)
3453 {
3454 	caddr_t		pp_addr1;
3455 	caddr_t		pp_addr2;
3456 	hat_mempte_t	pte1;
3457 	hat_mempte_t	pte2;
3458 	kmutex_t	*ppaddr_mutex;
3459 	label_t		ljb;
3460 	int		ret = 1;
3461 
3462 	ASSERT_STACK_ALIGNED();
3463 	ASSERT(PAGE_LOCKED(frompp));
3464 	ASSERT(PAGE_LOCKED(topp));
3465 
3466 	if (kpm_enable) {
3467 		pp_addr1 = hat_kpm_page2va(frompp, 0);
3468 		pp_addr2 = hat_kpm_page2va(topp, 0);
3469 		kpreempt_disable();
3470 	} else {
3471 		/*
3472 		 * disable pre-emption so that CPU can't change
3473 		 */
3474 		kpreempt_disable();
3475 
3476 		pp_addr1 = CPU->cpu_caddr1;
3477 		pp_addr2 = CPU->cpu_caddr2;
3478 		pte1 = CPU->cpu_caddr1pte;
3479 		pte2 = CPU->cpu_caddr2pte;
3480 
3481 		ppaddr_mutex = &CPU->cpu_ppaddr_mutex;
3482 		mutex_enter(ppaddr_mutex);
3483 
3484 		hat_mempte_remap(page_pptonum(frompp), pp_addr1, pte1,
3485 		    PROT_READ | HAT_STORECACHING_OK, HAT_LOAD_NOCONSIST);
3486 		hat_mempte_remap(page_pptonum(topp), pp_addr2, pte2,
3487 		    PROT_READ | PROT_WRITE | HAT_STORECACHING_OK,
3488 		    HAT_LOAD_NOCONSIST);
3489 	}
3490 
3491 	if (on_fault(&ljb)) {
3492 		ret = 0;
3493 		goto faulted;
3494 	}
3495 	if (use_sse_pagecopy)
3496 #ifdef __xpv
3497 		page_copy_no_xmm(pp_addr2, pp_addr1);
3498 #else
3499 		hwblkpagecopy(pp_addr1, pp_addr2);
3500 #endif
3501 	else
3502 		bcopy(pp_addr1, pp_addr2, PAGESIZE);
3503 
3504 	no_fault();
3505 faulted:
3506 	if (!kpm_enable) {
3507 #ifdef __xpv
3508 		/*
3509 		 * We can't leave unused mappings laying about under the
3510 		 * hypervisor, so blow them away.
3511 		 */
3512 		if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr1, 0,
3513 		    UVMF_INVLPG | UVMF_LOCAL) < 0)
3514 			panic("HYPERVISOR_update_va_mapping() failed");
3515 		if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr2, 0,
3516 		    UVMF_INVLPG | UVMF_LOCAL) < 0)
3517 			panic("HYPERVISOR_update_va_mapping() failed");
3518 #endif
3519 		mutex_exit(ppaddr_mutex);
3520 	}
3521 	kpreempt_enable();
3522 	return (ret);
3523 }
3524 
3525 void
3526 pagezero(page_t *pp, uint_t off, uint_t len)
3527 {
3528 	ASSERT(PAGE_LOCKED(pp));
3529 	pfnzero(page_pptonum(pp), off, len);
3530 }
3531 
3532 /*
3533  * Zero the physical page from off to off + len given by pfn
3534  * without changing the reference and modified bits of page.
3535  *
3536  * We use this using CPU private page address #2, see ppcopy() for more info.
3537  * pfnzero() must not be called at interrupt level.
3538  */
3539 void
3540 pfnzero(pfn_t pfn, uint_t off, uint_t len)
3541 {
3542 	caddr_t		pp_addr2;
3543 	hat_mempte_t	pte2;
3544 	kmutex_t	*ppaddr_mutex = NULL;
3545 
3546 	ASSERT_STACK_ALIGNED();
3547 	ASSERT(len <= MMU_PAGESIZE);
3548 	ASSERT(off <= MMU_PAGESIZE);
3549 	ASSERT(off + len <= MMU_PAGESIZE);
3550 
3551 	if (kpm_enable && !pfn_is_foreign(pfn)) {
3552 		pp_addr2 = hat_kpm_pfn2va(pfn);
3553 		kpreempt_disable();
3554 	} else {
3555 		kpreempt_disable();
3556 
3557 		pp_addr2 = CPU->cpu_caddr2;
3558 		pte2 = CPU->cpu_caddr2pte;
3559 
3560 		ppaddr_mutex = &CPU->cpu_ppaddr_mutex;
3561 		mutex_enter(ppaddr_mutex);
3562 
3563 		hat_mempte_remap(pfn, pp_addr2, pte2,
3564 		    PROT_READ | PROT_WRITE | HAT_STORECACHING_OK,
3565 		    HAT_LOAD_NOCONSIST);
3566 	}
3567 
3568 	if (use_sse_pagezero) {
3569 #ifdef __xpv
3570 		uint_t rem;
3571 
3572 		/*
3573 		 * zero a byte at a time until properly aligned for
3574 		 * block_zero_no_xmm().
3575 		 */
3576 		while (!P2NPHASE(off, ((uint_t)BLOCKZEROALIGN)) && len-- > 0)
3577 			pp_addr2[off++] = 0;
3578 
3579 		/*
3580 		 * Now use faster block_zero_no_xmm() for any range
3581 		 * that is properly aligned and sized.
3582 		 */
3583 		rem = P2PHASE(len, ((uint_t)BLOCKZEROALIGN));
3584 		len -= rem;
3585 		if (len != 0) {
3586 			block_zero_no_xmm(pp_addr2 + off, len);
3587 			off += len;
3588 		}
3589 
3590 		/*
3591 		 * zero remainder with byte stores.
3592 		 */
3593 		while (rem-- > 0)
3594 			pp_addr2[off++] = 0;
3595 #else
3596 		hwblkclr(pp_addr2 + off, len);
3597 #endif
3598 	} else {
3599 		bzero(pp_addr2 + off, len);
3600 	}
3601 
3602 	if (!kpm_enable || pfn_is_foreign(pfn)) {
3603 #ifdef __xpv
3604 		/*
3605 		 * On the hypervisor this page might get used for a page
3606 		 * table before any intervening change to this mapping,
3607 		 * so blow it away.
3608 		 */
3609 		if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr2, 0,
3610 		    UVMF_INVLPG) < 0)
3611 			panic("HYPERVISOR_update_va_mapping() failed");
3612 #endif
3613 		mutex_exit(ppaddr_mutex);
3614 	}
3615 
3616 	kpreempt_enable();
3617 }
3618 
3619 /*
3620  * Platform-dependent page scrub call.
3621  */
3622 void
3623 pagescrub(page_t *pp, uint_t off, uint_t len)
3624 {
3625 	/*
3626 	 * For now, we rely on the fact that pagezero() will
3627 	 * always clear UEs.
3628 	 */
3629 	pagezero(pp, off, len);
3630 }
3631 
3632 /*
3633  * set up two private addresses for use on a given CPU for use in ppcopy()
3634  */
3635 void
3636 setup_vaddr_for_ppcopy(struct cpu *cpup)
3637 {
3638 	void *addr;
3639 	hat_mempte_t pte_pa;
3640 
3641 	addr = vmem_alloc(heap_arena, mmu_ptob(1), VM_SLEEP);
3642 	pte_pa = hat_mempte_setup(addr);
3643 	cpup->cpu_caddr1 = addr;
3644 	cpup->cpu_caddr1pte = pte_pa;
3645 
3646 	addr = vmem_alloc(heap_arena, mmu_ptob(1), VM_SLEEP);
3647 	pte_pa = hat_mempte_setup(addr);
3648 	cpup->cpu_caddr2 = addr;
3649 	cpup->cpu_caddr2pte = pte_pa;
3650 
3651 	mutex_init(&cpup->cpu_ppaddr_mutex, NULL, MUTEX_DEFAULT, NULL);
3652 }
3653 
3654 /*
3655  * Undo setup_vaddr_for_ppcopy
3656  */
3657 void
3658 teardown_vaddr_for_ppcopy(struct cpu *cpup)
3659 {
3660 	mutex_destroy(&cpup->cpu_ppaddr_mutex);
3661 
3662 	hat_mempte_release(cpup->cpu_caddr2, cpup->cpu_caddr2pte);
3663 	cpup->cpu_caddr2pte = 0;
3664 	vmem_free(heap_arena, cpup->cpu_caddr2, mmu_ptob(1));
3665 	cpup->cpu_caddr2 = 0;
3666 
3667 	hat_mempte_release(cpup->cpu_caddr1, cpup->cpu_caddr1pte);
3668 	cpup->cpu_caddr1pte = 0;
3669 	vmem_free(heap_arena, cpup->cpu_caddr1, mmu_ptob(1));
3670 	cpup->cpu_caddr1 = 0;
3671 }
3672 
3673 /*
3674  * Create the pageout scanner thread. The thread has to
3675  * start at procedure with process pp and priority pri.
3676  */
3677 void
3678 pageout_init(void (*procedure)(), proc_t *pp, pri_t pri)
3679 {
3680 	(void) thread_create(NULL, 0, procedure, NULL, 0, pp, TS_RUN, pri);
3681 }
3682 
3683 /*
3684  * Function for flushing D-cache when performing module relocations
3685  * to an alternate mapping.  Unnecessary on Intel / AMD platforms.
3686  */
3687 void
3688 dcache_flushall()
3689 {}
3690 
3691 size_t
3692 exec_get_spslew(void)
3693 {
3694 	return (0);
3695 }
3696 
3697 /*
3698  * Allocate a memory page.  The argument 'seed' can be any pseudo-random
3699  * number to vary where the pages come from.  This is quite a hacked up
3700  * method -- it works for now, but really needs to be fixed up a bit.
3701  *
3702  * We currently use page_create_va() on the kvp with fake offsets,
3703  * segments and virt address.  This is pretty bogus, but was copied from the
3704  * old hat_i86.c code.  A better approach would be to specify either mnode
3705  * random or mnode local and takes a page from whatever color has the MOST
3706  * available - this would have a minimal impact on page coloring.
3707  */
3708 page_t *
3709 page_get_physical(uintptr_t seed)
3710 {
3711 	page_t *pp;
3712 	u_offset_t offset;
3713 	static struct seg tmpseg;
3714 	static uintptr_t ctr = 0;
3715 
3716 	/*
3717 	 * This code is gross, we really need a simpler page allocator.
3718 	 *
3719 	 * We need assign an offset for the page to call page_create_va().
3720 	 * To avoid conflicts with other pages, we get creative with the offset.
3721 	 * For 32 bits, we pick an offset > 4Gig
3722 	 * For 64 bits, pick an offset somewhere in the VA hole.
3723 	 */
3724 	offset = seed;
3725 	if (offset > kernelbase)
3726 		offset -= kernelbase;
3727 	offset <<= MMU_PAGESHIFT;
3728 #if defined(__amd64)
3729 	offset += mmu.hole_start;	/* something in VA hole */
3730 #else
3731 	offset += 1ULL << 40;		/* something > 4 Gig */
3732 #endif
3733 
3734 	if (page_resv(1, KM_NOSLEEP) == 0)
3735 		return (NULL);
3736 
3737 #ifdef	DEBUG
3738 	pp = page_exists(&kvp, offset);
3739 	if (pp != NULL)
3740 		panic("page already exists %p", pp);
3741 #endif
3742 
3743 	pp = page_create_va(&kvp, offset, MMU_PAGESIZE, PG_EXCL,
3744 	    &tmpseg, (caddr_t)(ctr += MMU_PAGESIZE));	/* changing VA usage */
3745 	if (pp == NULL)
3746 		return (NULL);
3747 	page_io_unlock(pp);
3748 	page_hashout(pp, NULL);
3749 	return (pp);
3750 }
3751