xref: /illumos-gate/usr/src/uts/i86pc/vm/vm_machdep.c (revision 193974072f41a843678abf5f61979c748687e66b)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
27 /*	All Rights Reserved   */
28 
29 /*
30  * Portions of this source code were derived from Berkeley 4.3 BSD
31  * under license from the Regents of the University of California.
32  */
33 
34 /*
35  * UNIX machine dependent virtual memory support.
36  */
37 
38 #include <sys/types.h>
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/user.h>
42 #include <sys/proc.h>
43 #include <sys/kmem.h>
44 #include <sys/vmem.h>
45 #include <sys/buf.h>
46 #include <sys/cpuvar.h>
47 #include <sys/lgrp.h>
48 #include <sys/disp.h>
49 #include <sys/vm.h>
50 #include <sys/mman.h>
51 #include <sys/vnode.h>
52 #include <sys/cred.h>
53 #include <sys/exec.h>
54 #include <sys/exechdr.h>
55 #include <sys/debug.h>
56 #include <sys/vmsystm.h>
57 
58 #include <vm/hat.h>
59 #include <vm/as.h>
60 #include <vm/seg.h>
61 #include <vm/seg_kp.h>
62 #include <vm/seg_vn.h>
63 #include <vm/page.h>
64 #include <vm/seg_kmem.h>
65 #include <vm/seg_kpm.h>
66 #include <vm/vm_dep.h>
67 
68 #include <sys/cpu.h>
69 #include <sys/vm_machparam.h>
70 #include <sys/memlist.h>
71 #include <sys/bootconf.h> /* XXX the memlist stuff belongs in memlist_plat.h */
72 #include <vm/hat_i86.h>
73 #include <sys/x86_archext.h>
74 #include <sys/elf_386.h>
75 #include <sys/cmn_err.h>
76 #include <sys/archsystm.h>
77 #include <sys/machsystm.h>
78 
79 #include <sys/vtrace.h>
80 #include <sys/ddidmareq.h>
81 #include <sys/promif.h>
82 #include <sys/memnode.h>
83 #include <sys/stack.h>
84 #include <util/qsort.h>
85 #include <sys/taskq.h>
86 
87 #ifdef __xpv
88 
89 #include <sys/hypervisor.h>
90 #include <sys/xen_mmu.h>
91 #include <sys/balloon_impl.h>
92 
93 /*
94  * domain 0 pages usable for DMA are kept pre-allocated and kept in
95  * distinct lists, ordered by increasing mfn.
96  */
97 static kmutex_t io_pool_lock;
98 static kmutex_t contig_list_lock;
99 static page_t *io_pool_4g;	/* pool for 32 bit dma limited devices */
100 static page_t *io_pool_16m;	/* pool for 24 bit dma limited legacy devices */
101 static long io_pool_cnt;
102 static long io_pool_cnt_max = 0;
103 #define	DEFAULT_IO_POOL_MIN	128
104 static long io_pool_cnt_min = DEFAULT_IO_POOL_MIN;
105 static long io_pool_cnt_lowater = 0;
106 static long io_pool_shrink_attempts; /* how many times did we try to shrink */
107 static long io_pool_shrinks;	/* how many times did we really shrink */
108 static long io_pool_grows;	/* how many times did we grow */
109 static mfn_t start_mfn = 1;
110 static caddr_t io_pool_kva;	/* use to alloc pages when needed */
111 
112 static int create_contig_pfnlist(uint_t);
113 
114 /*
115  * percentage of phys mem to hold in the i/o pool
116  */
117 #define	DEFAULT_IO_POOL_PCT	2
118 static long io_pool_physmem_pct = DEFAULT_IO_POOL_PCT;
119 static void page_io_pool_sub(page_t **, page_t *, page_t *);
120 int ioalloc_dbg = 0;
121 
122 #endif /* __xpv */
123 
124 uint_t vac_colors = 1;
125 
126 int largepagesupport = 0;
127 extern uint_t page_create_new;
128 extern uint_t page_create_exists;
129 extern uint_t page_create_putbacks;
130 extern uint_t page_create_putbacks;
131 /*
132  * Allow users to disable the kernel's use of SSE.
133  */
134 extern int use_sse_pagecopy, use_sse_pagezero;
135 
136 /*
137  * combined memory ranges from mnode and memranges[] to manage single
138  * mnode/mtype dimension in the page lists.
139  */
140 typedef struct {
141 	pfn_t	mnr_pfnlo;
142 	pfn_t	mnr_pfnhi;
143 	int	mnr_mnode;
144 	int	mnr_memrange;		/* index into memranges[] */
145 	/* maintain page list stats */
146 	pgcnt_t	mnr_mt_clpgcnt;		/* cache list cnt */
147 	pgcnt_t	mnr_mt_flpgcnt[MMU_PAGE_SIZES];	/* free list cnt per szc */
148 	pgcnt_t	mnr_mt_totcnt;		/* sum of cache and free lists */
149 #ifdef DEBUG
150 	struct mnr_mts {		/* mnode/mtype szc stats */
151 		pgcnt_t	mnr_mts_pgcnt;
152 		int	mnr_mts_colors;
153 		pgcnt_t *mnr_mtsc_pgcnt;
154 	} 	*mnr_mts;
155 #endif
156 } mnoderange_t;
157 
158 #define	MEMRANGEHI(mtype)						\
159 	((mtype > 0) ? memranges[mtype - 1] - 1: physmax)
160 #define	MEMRANGELO(mtype)	(memranges[mtype])
161 
162 #define	MTYPE_FREEMEM(mt)	(mnoderanges[mt].mnr_mt_totcnt)
163 
164 /*
165  * As the PC architecture evolved memory up was clumped into several
166  * ranges for various historical I/O devices to do DMA.
167  * < 16Meg - ISA bus
168  * < 2Gig - ???
169  * < 4Gig - PCI bus or drivers that don't understand PAE mode
170  *
171  * These are listed in reverse order, so that we can skip over unused
172  * ranges on machines with small memories.
173  *
174  * For now under the Hypervisor, we'll only ever have one memrange.
175  */
176 #define	PFN_4GIG	0x100000
177 #define	PFN_16MEG	0x1000
178 static pfn_t arch_memranges[NUM_MEM_RANGES] = {
179     PFN_4GIG,	/* pfn range for 4G and above */
180     0x80000,	/* pfn range for 2G-4G */
181     PFN_16MEG,	/* pfn range for 16M-2G */
182     0x00000,	/* pfn range for 0-16M */
183 };
184 pfn_t *memranges = &arch_memranges[0];
185 int nranges = NUM_MEM_RANGES;
186 
187 /*
188  * This combines mem_node_config and memranges into one data
189  * structure to be used for page list management.
190  */
191 mnoderange_t	*mnoderanges;
192 int		mnoderangecnt;
193 int		mtype4g;
194 
195 /*
196  * 4g memory management variables for systems with more than 4g of memory:
197  *
198  * physical memory below 4g is required for 32bit dma devices and, currently,
199  * for kmem memory. On systems with more than 4g of memory, the pool of memory
200  * below 4g can be depleted without any paging activity given that there is
201  * likely to be sufficient memory above 4g.
202  *
203  * physmax4g is set true if the largest pfn is over 4g. The rest of the
204  * 4g memory management code is enabled only when physmax4g is true.
205  *
206  * maxmem4g is the count of the maximum number of pages on the page lists
207  * with physical addresses below 4g. It can be a lot less then 4g given that
208  * BIOS may reserve large chunks of space below 4g for hot plug pci devices,
209  * agp aperture etc.
210  *
211  * freemem4g maintains the count of the number of available pages on the
212  * page lists with physical addresses below 4g.
213  *
214  * DESFREE4G specifies the desired amount of below 4g memory. It defaults to
215  * 6% (desfree4gshift = 4) of maxmem4g.
216  *
217  * RESTRICT4G_ALLOC returns true if freemem4g falls below DESFREE4G
218  * and the amount of physical memory above 4g is greater than freemem4g.
219  * In this case, page_get_* routines will restrict below 4g allocations
220  * for requests that don't specifically require it.
221  */
222 
223 #define	LOTSFREE4G	(maxmem4g >> lotsfree4gshift)
224 #define	DESFREE4G	(maxmem4g >> desfree4gshift)
225 
226 #define	RESTRICT4G_ALLOC					\
227 	(physmax4g && (freemem4g < DESFREE4G) && ((freemem4g << 1) < freemem))
228 
229 static pgcnt_t	maxmem4g;
230 static pgcnt_t	freemem4g;
231 static int	physmax4g;
232 static int	desfree4gshift = 4;	/* maxmem4g shift to derive DESFREE4G */
233 static int	lotsfree4gshift = 3;
234 
235 /*
236  * 16m memory management:
237  *
238  * reserve some amount of physical memory below 16m for legacy devices.
239  *
240  * RESTRICT16M_ALLOC returns true if an there are sufficient free pages above
241  * 16m or if the 16m pool drops below DESFREE16M.
242  *
243  * In this case, general page allocations via page_get_{free,cache}list
244  * routines will be restricted from allocating from the 16m pool. Allocations
245  * that require specific pfn ranges (page_get_anylist) and PG_PANIC allocations
246  * are not restricted.
247  */
248 
249 #define	FREEMEM16M	MTYPE_FREEMEM(0)
250 #define	DESFREE16M	desfree16m
251 #define	RESTRICT16M_ALLOC(freemem, pgcnt, flags)		\
252 	((freemem != 0) && ((flags & PG_PANIC) == 0) &&		\
253 	    ((freemem >= (FREEMEM16M)) ||			\
254 	    (FREEMEM16M  < (DESFREE16M + pgcnt))))
255 
256 static pgcnt_t	desfree16m = 0x380;
257 
258 /*
259  * This can be patched via /etc/system to allow old non-PAE aware device
260  * drivers to use kmem_alloc'd memory on 32 bit systems with > 4Gig RAM.
261  */
262 int restricted_kmemalloc = 0;
263 
264 #ifdef VM_STATS
265 struct {
266 	ulong_t	pga_alloc;
267 	ulong_t	pga_notfullrange;
268 	ulong_t	pga_nulldmaattr;
269 	ulong_t	pga_allocok;
270 	ulong_t	pga_allocfailed;
271 	ulong_t	pgma_alloc;
272 	ulong_t	pgma_allocok;
273 	ulong_t	pgma_allocfailed;
274 	ulong_t	pgma_allocempty;
275 } pga_vmstats;
276 #endif
277 
278 uint_t mmu_page_sizes;
279 
280 /* How many page sizes the users can see */
281 uint_t mmu_exported_page_sizes;
282 
283 /* page sizes that legacy applications can see */
284 uint_t mmu_legacy_page_sizes;
285 
286 /*
287  * Number of pages in 1 GB.  Don't enable automatic large pages if we have
288  * fewer than this many pages.
289  */
290 pgcnt_t shm_lpg_min_physmem = 1 << (30 - MMU_PAGESHIFT);
291 pgcnt_t privm_lpg_min_physmem = 1 << (30 - MMU_PAGESHIFT);
292 
293 /*
294  * Maximum and default segment size tunables for user private
295  * and shared anon memory, and user text and initialized data.
296  * These can be patched via /etc/system to allow large pages
297  * to be used for mapping application private and shared anon memory.
298  */
299 size_t mcntl0_lpsize = MMU_PAGESIZE;
300 size_t max_uheap_lpsize = MMU_PAGESIZE;
301 size_t default_uheap_lpsize = MMU_PAGESIZE;
302 size_t max_ustack_lpsize = MMU_PAGESIZE;
303 size_t default_ustack_lpsize = MMU_PAGESIZE;
304 size_t max_privmap_lpsize = MMU_PAGESIZE;
305 size_t max_uidata_lpsize = MMU_PAGESIZE;
306 size_t max_utext_lpsize = MMU_PAGESIZE;
307 size_t max_shm_lpsize = MMU_PAGESIZE;
308 
309 
310 /*
311  * initialized by page_coloring_init().
312  */
313 uint_t	page_colors;
314 uint_t	page_colors_mask;
315 uint_t	page_coloring_shift;
316 int	cpu_page_colors;
317 static uint_t	l2_colors;
318 
319 /*
320  * Page freelists and cachelists are dynamically allocated once mnoderangecnt
321  * and page_colors are calculated from the l2 cache n-way set size.  Within a
322  * mnode range, the page freelist and cachelist are hashed into bins based on
323  * color. This makes it easier to search for a page within a specific memory
324  * range.
325  */
326 #define	PAGE_COLORS_MIN	16
327 
328 page_t ****page_freelists;
329 page_t ***page_cachelists;
330 
331 
332 /*
333  * Used by page layer to know about page sizes
334  */
335 hw_pagesize_t hw_page_array[MAX_NUM_LEVEL + 1];
336 
337 kmutex_t	*fpc_mutex[NPC_MUTEX];
338 kmutex_t	*cpc_mutex[NPC_MUTEX];
339 
340 /*
341  * Only let one thread at a time try to coalesce large pages, to
342  * prevent them from working against each other.
343  */
344 static kmutex_t	contig_lock;
345 #define	CONTIG_LOCK()	mutex_enter(&contig_lock);
346 #define	CONTIG_UNLOCK()	mutex_exit(&contig_lock);
347 
348 #define	PFN_16M		(mmu_btop((uint64_t)0x1000000))
349 
350 /*
351  * Return the optimum page size for a given mapping
352  */
353 /*ARGSUSED*/
354 size_t
355 map_pgsz(int maptype, struct proc *p, caddr_t addr, size_t len, int memcntl)
356 {
357 	level_t l = 0;
358 	size_t pgsz = MMU_PAGESIZE;
359 	size_t max_lpsize;
360 	uint_t mszc;
361 
362 	ASSERT(maptype != MAPPGSZ_VA);
363 
364 	if (maptype != MAPPGSZ_ISM && physmem < privm_lpg_min_physmem) {
365 		return (MMU_PAGESIZE);
366 	}
367 
368 	switch (maptype) {
369 	case MAPPGSZ_HEAP:
370 	case MAPPGSZ_STK:
371 		max_lpsize = memcntl ? mcntl0_lpsize : (maptype ==
372 		    MAPPGSZ_HEAP ? max_uheap_lpsize : max_ustack_lpsize);
373 		if (max_lpsize == MMU_PAGESIZE) {
374 			return (MMU_PAGESIZE);
375 		}
376 		if (len == 0) {
377 			len = (maptype == MAPPGSZ_HEAP) ? p->p_brkbase +
378 			    p->p_brksize - p->p_bssbase : p->p_stksize;
379 		}
380 		len = (maptype == MAPPGSZ_HEAP) ? MAX(len,
381 		    default_uheap_lpsize) : MAX(len, default_ustack_lpsize);
382 
383 		/*
384 		 * use the pages size that best fits len
385 		 */
386 		for (l = mmu.umax_page_level; l > 0; --l) {
387 			if (LEVEL_SIZE(l) > max_lpsize || len < LEVEL_SIZE(l)) {
388 				continue;
389 			} else {
390 				pgsz = LEVEL_SIZE(l);
391 			}
392 			break;
393 		}
394 
395 		mszc = (maptype == MAPPGSZ_HEAP ? p->p_brkpageszc :
396 		    p->p_stkpageszc);
397 		if (addr == 0 && (pgsz < hw_page_array[mszc].hp_size)) {
398 			pgsz = hw_page_array[mszc].hp_size;
399 		}
400 		return (pgsz);
401 
402 	case MAPPGSZ_ISM:
403 		for (l = mmu.umax_page_level; l > 0; --l) {
404 			if (len >= LEVEL_SIZE(l))
405 				return (LEVEL_SIZE(l));
406 		}
407 		return (LEVEL_SIZE(0));
408 	}
409 	return (pgsz);
410 }
411 
412 static uint_t
413 map_szcvec(caddr_t addr, size_t size, uintptr_t off, size_t max_lpsize,
414     size_t min_physmem)
415 {
416 	caddr_t eaddr = addr + size;
417 	uint_t szcvec = 0;
418 	caddr_t raddr;
419 	caddr_t readdr;
420 	size_t	pgsz;
421 	int i;
422 
423 	if (physmem < min_physmem || max_lpsize <= MMU_PAGESIZE) {
424 		return (0);
425 	}
426 
427 	for (i = mmu_exported_page_sizes - 1; i > 0; i--) {
428 		pgsz = page_get_pagesize(i);
429 		if (pgsz > max_lpsize) {
430 			continue;
431 		}
432 		raddr = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz);
433 		readdr = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz);
434 		if (raddr < addr || raddr >= readdr) {
435 			continue;
436 		}
437 		if (P2PHASE((uintptr_t)addr ^ off, pgsz)) {
438 			continue;
439 		}
440 		/*
441 		 * Set szcvec to the remaining page sizes.
442 		 */
443 		szcvec = ((1 << (i + 1)) - 1) & ~1;
444 		break;
445 	}
446 	return (szcvec);
447 }
448 
449 /*
450  * Return a bit vector of large page size codes that
451  * can be used to map [addr, addr + len) region.
452  */
453 /*ARGSUSED*/
454 uint_t
455 map_pgszcvec(caddr_t addr, size_t size, uintptr_t off, int flags, int type,
456     int memcntl)
457 {
458 	size_t max_lpsize = mcntl0_lpsize;
459 
460 	if (mmu.max_page_level == 0)
461 		return (0);
462 
463 	if (flags & MAP_TEXT) {
464 		if (!memcntl)
465 			max_lpsize = max_utext_lpsize;
466 		return (map_szcvec(addr, size, off, max_lpsize,
467 		    shm_lpg_min_physmem));
468 
469 	} else if (flags & MAP_INITDATA) {
470 		if (!memcntl)
471 			max_lpsize = max_uidata_lpsize;
472 		return (map_szcvec(addr, size, off, max_lpsize,
473 		    privm_lpg_min_physmem));
474 
475 	} else if (type == MAPPGSZC_SHM) {
476 		if (!memcntl)
477 			max_lpsize = max_shm_lpsize;
478 		return (map_szcvec(addr, size, off, max_lpsize,
479 		    shm_lpg_min_physmem));
480 
481 	} else if (type == MAPPGSZC_HEAP) {
482 		if (!memcntl)
483 			max_lpsize = max_uheap_lpsize;
484 		return (map_szcvec(addr, size, off, max_lpsize,
485 		    privm_lpg_min_physmem));
486 
487 	} else if (type == MAPPGSZC_STACK) {
488 		if (!memcntl)
489 			max_lpsize = max_ustack_lpsize;
490 		return (map_szcvec(addr, size, off, max_lpsize,
491 		    privm_lpg_min_physmem));
492 
493 	} else {
494 		if (!memcntl)
495 			max_lpsize = max_privmap_lpsize;
496 		return (map_szcvec(addr, size, off, max_lpsize,
497 		    privm_lpg_min_physmem));
498 	}
499 }
500 
501 /*
502  * Handle a pagefault.
503  */
504 faultcode_t
505 pagefault(
506 	caddr_t addr,
507 	enum fault_type type,
508 	enum seg_rw rw,
509 	int iskernel)
510 {
511 	struct as *as;
512 	struct hat *hat;
513 	struct proc *p;
514 	kthread_t *t;
515 	faultcode_t res;
516 	caddr_t base;
517 	size_t len;
518 	int err;
519 	int mapped_red;
520 	uintptr_t ea;
521 
522 	ASSERT_STACK_ALIGNED();
523 
524 	if (INVALID_VADDR(addr))
525 		return (FC_NOMAP);
526 
527 	mapped_red = segkp_map_red();
528 
529 	if (iskernel) {
530 		as = &kas;
531 		hat = as->a_hat;
532 	} else {
533 		t = curthread;
534 		p = ttoproc(t);
535 		as = p->p_as;
536 		hat = as->a_hat;
537 	}
538 
539 	/*
540 	 * Dispatch pagefault.
541 	 */
542 	res = as_fault(hat, as, addr, 1, type, rw);
543 
544 	/*
545 	 * If this isn't a potential unmapped hole in the user's
546 	 * UNIX data or stack segments, just return status info.
547 	 */
548 	if (res != FC_NOMAP || iskernel)
549 		goto out;
550 
551 	/*
552 	 * Check to see if we happened to faulted on a currently unmapped
553 	 * part of the UNIX data or stack segments.  If so, create a zfod
554 	 * mapping there and then try calling the fault routine again.
555 	 */
556 	base = p->p_brkbase;
557 	len = p->p_brksize;
558 
559 	if (addr < base || addr >= base + len) {		/* data seg? */
560 		base = (caddr_t)p->p_usrstack - p->p_stksize;
561 		len = p->p_stksize;
562 		if (addr < base || addr >= p->p_usrstack) {	/* stack seg? */
563 			/* not in either UNIX data or stack segments */
564 			res = FC_NOMAP;
565 			goto out;
566 		}
567 	}
568 
569 	/*
570 	 * the rest of this function implements a 3.X 4.X 5.X compatibility
571 	 * This code is probably not needed anymore
572 	 */
573 	if (p->p_model == DATAMODEL_ILP32) {
574 
575 		/* expand the gap to the page boundaries on each side */
576 		ea = P2ROUNDUP((uintptr_t)base + len, MMU_PAGESIZE);
577 		base = (caddr_t)P2ALIGN((uintptr_t)base, MMU_PAGESIZE);
578 		len = ea - (uintptr_t)base;
579 
580 		as_rangelock(as);
581 		if (as_gap(as, MMU_PAGESIZE, &base, &len, AH_CONTAIN, addr) ==
582 		    0) {
583 			err = as_map(as, base, len, segvn_create, zfod_argsp);
584 			as_rangeunlock(as);
585 			if (err) {
586 				res = FC_MAKE_ERR(err);
587 				goto out;
588 			}
589 		} else {
590 			/*
591 			 * This page is already mapped by another thread after
592 			 * we returned from as_fault() above.  We just fall
593 			 * through as_fault() below.
594 			 */
595 			as_rangeunlock(as);
596 		}
597 
598 		res = as_fault(hat, as, addr, 1, F_INVAL, rw);
599 	}
600 
601 out:
602 	if (mapped_red)
603 		segkp_unmap_red();
604 
605 	return (res);
606 }
607 
608 void
609 map_addr(caddr_t *addrp, size_t len, offset_t off, int vacalign, uint_t flags)
610 {
611 	struct proc *p = curproc;
612 	caddr_t userlimit = (flags & _MAP_LOW32) ?
613 	    (caddr_t)_userlimit32 : p->p_as->a_userlimit;
614 
615 	map_addr_proc(addrp, len, off, vacalign, userlimit, curproc, flags);
616 }
617 
618 /*ARGSUSED*/
619 int
620 map_addr_vacalign_check(caddr_t addr, u_offset_t off)
621 {
622 	return (0);
623 }
624 
625 /*
626  * map_addr_proc() is the routine called when the system is to
627  * choose an address for the user.  We will pick an address
628  * range which is the highest available below userlimit.
629  *
630  * Every mapping will have a redzone of a single page on either side of
631  * the request. This is done to leave one page unmapped between segments.
632  * This is not required, but it's useful for the user because if their
633  * program strays across a segment boundary, it will catch a fault
634  * immediately making debugging a little easier.  Currently the redzone
635  * is mandatory.
636  *
637  * addrp is a value/result parameter.
638  *	On input it is a hint from the user to be used in a completely
639  *	machine dependent fashion.  We decide to completely ignore this hint.
640  *	If MAP_ALIGN was specified, addrp contains the minimal alignment, which
641  *	must be some "power of two" multiple of pagesize.
642  *
643  *	On output it is NULL if no address can be found in the current
644  *	processes address space or else an address that is currently
645  *	not mapped for len bytes with a page of red zone on either side.
646  *
647  *	vacalign is not needed on x86 (it's for viturally addressed caches)
648  */
649 /*ARGSUSED*/
650 void
651 map_addr_proc(
652 	caddr_t *addrp,
653 	size_t len,
654 	offset_t off,
655 	int vacalign,
656 	caddr_t userlimit,
657 	struct proc *p,
658 	uint_t flags)
659 {
660 	struct as *as = p->p_as;
661 	caddr_t addr;
662 	caddr_t base;
663 	size_t slen;
664 	size_t align_amount;
665 
666 	ASSERT32(userlimit == as->a_userlimit);
667 
668 	base = p->p_brkbase;
669 #if defined(__amd64)
670 	/*
671 	 * XX64 Yes, this needs more work.
672 	 */
673 	if (p->p_model == DATAMODEL_NATIVE) {
674 		if (userlimit < as->a_userlimit) {
675 			/*
676 			 * This happens when a program wants to map
677 			 * something in a range that's accessible to a
678 			 * program in a smaller address space.  For example,
679 			 * a 64-bit program calling mmap32(2) to guarantee
680 			 * that the returned address is below 4Gbytes.
681 			 */
682 			ASSERT((uintptr_t)userlimit < ADDRESS_C(0xffffffff));
683 
684 			if (userlimit > base)
685 				slen = userlimit - base;
686 			else {
687 				*addrp = NULL;
688 				return;
689 			}
690 		} else {
691 			/*
692 			 * XX64 This layout is probably wrong .. but in
693 			 * the event we make the amd64 address space look
694 			 * like sparcv9 i.e. with the stack -above- the
695 			 * heap, this bit of code might even be correct.
696 			 */
697 			slen = p->p_usrstack - base -
698 			    (((size_t)rctl_enforced_value(
699 			    rctlproc_legacy[RLIMIT_STACK],
700 			    p->p_rctls, p) + PAGEOFFSET) & PAGEMASK);
701 		}
702 	} else
703 #endif
704 		slen = userlimit - base;
705 
706 	/* Make len be a multiple of PAGESIZE */
707 	len = (len + PAGEOFFSET) & PAGEMASK;
708 
709 	/*
710 	 * figure out what the alignment should be
711 	 *
712 	 * XX64 -- is there an ELF_AMD64_MAXPGSZ or is it the same????
713 	 */
714 	if (len <= ELF_386_MAXPGSZ) {
715 		/*
716 		 * Align virtual addresses to ensure that ELF shared libraries
717 		 * are mapped with the appropriate alignment constraints by
718 		 * the run-time linker.
719 		 */
720 		align_amount = ELF_386_MAXPGSZ;
721 	} else {
722 		int l = mmu.umax_page_level;
723 
724 		while (l && len < LEVEL_SIZE(l))
725 			--l;
726 
727 		align_amount = LEVEL_SIZE(l);
728 	}
729 
730 	if ((flags & MAP_ALIGN) && ((uintptr_t)*addrp > align_amount))
731 		align_amount = (uintptr_t)*addrp;
732 
733 	ASSERT(ISP2(align_amount));
734 	ASSERT(align_amount == 0 || align_amount >= PAGESIZE);
735 
736 	off = off & (align_amount - 1);
737 	/*
738 	 * Look for a large enough hole starting below userlimit.
739 	 * After finding it, use the upper part.
740 	 */
741 	if (as_gap_aligned(as, len, &base, &slen, AH_HI, NULL, align_amount,
742 	    PAGESIZE, off) == 0) {
743 		caddr_t as_addr;
744 
745 		/*
746 		 * addr is the highest possible address to use since we have
747 		 * a PAGESIZE redzone at the beginning and end.
748 		 */
749 		addr = base + slen - (PAGESIZE + len);
750 		as_addr = addr;
751 		/*
752 		 * Round address DOWN to the alignment amount and
753 		 * add the offset in.
754 		 * If addr is greater than as_addr, len would not be large
755 		 * enough to include the redzone, so we must adjust down
756 		 * by the alignment amount.
757 		 */
758 		addr = (caddr_t)((uintptr_t)addr & (~(align_amount - 1)));
759 		addr += (uintptr_t)off;
760 		if (addr > as_addr) {
761 			addr -= align_amount;
762 		}
763 
764 		ASSERT(addr > base);
765 		ASSERT(addr + len < base + slen);
766 		ASSERT(((uintptr_t)addr & (align_amount - 1)) ==
767 		    ((uintptr_t)(off)));
768 		*addrp = addr;
769 	} else {
770 		*addrp = NULL;	/* no more virtual space */
771 	}
772 }
773 
774 int valid_va_range_aligned_wraparound;
775 
776 /*
777  * Determine whether [*basep, *basep + *lenp) contains a mappable range of
778  * addresses at least "minlen" long, where the base of the range is at "off"
779  * phase from an "align" boundary and there is space for a "redzone"-sized
780  * redzone on either side of the range.  On success, 1 is returned and *basep
781  * and *lenp are adjusted to describe the acceptable range (including
782  * the redzone).  On failure, 0 is returned.
783  */
784 /*ARGSUSED3*/
785 int
786 valid_va_range_aligned(caddr_t *basep, size_t *lenp, size_t minlen, int dir,
787     size_t align, size_t redzone, size_t off)
788 {
789 	uintptr_t hi, lo;
790 	size_t tot_len;
791 
792 	ASSERT(align == 0 ? off == 0 : off < align);
793 	ASSERT(ISP2(align));
794 	ASSERT(align == 0 || align >= PAGESIZE);
795 
796 	lo = (uintptr_t)*basep;
797 	hi = lo + *lenp;
798 	tot_len = minlen + 2 * redzone; /* need at least this much space */
799 
800 	/*
801 	 * If hi rolled over the top, try cutting back.
802 	 */
803 	if (hi < lo) {
804 		*lenp = 0UL - lo - 1UL;
805 		/* See if this really happens. If so, then we figure out why */
806 		valid_va_range_aligned_wraparound++;
807 		hi = lo + *lenp;
808 	}
809 	if (*lenp < tot_len) {
810 		return (0);
811 	}
812 
813 #if defined(__amd64)
814 	/*
815 	 * Deal with a possible hole in the address range between
816 	 * hole_start and hole_end that should never be mapped.
817 	 */
818 	if (lo < hole_start) {
819 		if (hi > hole_start) {
820 			if (hi < hole_end) {
821 				hi = hole_start;
822 			} else {
823 				/* lo < hole_start && hi >= hole_end */
824 				if (dir == AH_LO) {
825 					/*
826 					 * prefer lowest range
827 					 */
828 					if (hole_start - lo >= tot_len)
829 						hi = hole_start;
830 					else if (hi - hole_end >= tot_len)
831 						lo = hole_end;
832 					else
833 						return (0);
834 				} else {
835 					/*
836 					 * prefer highest range
837 					 */
838 					if (hi - hole_end >= tot_len)
839 						lo = hole_end;
840 					else if (hole_start - lo >= tot_len)
841 						hi = hole_start;
842 					else
843 						return (0);
844 				}
845 			}
846 		}
847 	} else {
848 		/* lo >= hole_start */
849 		if (hi < hole_end)
850 			return (0);
851 		if (lo < hole_end)
852 			lo = hole_end;
853 	}
854 #endif
855 
856 	if (hi - lo < tot_len)
857 		return (0);
858 
859 	if (align > 1) {
860 		uintptr_t tlo = lo + redzone;
861 		uintptr_t thi = hi - redzone;
862 		tlo = (uintptr_t)P2PHASEUP(tlo, align, off);
863 		if (tlo < lo + redzone) {
864 			return (0);
865 		}
866 		if (thi < tlo || thi - tlo < minlen) {
867 			return (0);
868 		}
869 	}
870 
871 	*basep = (caddr_t)lo;
872 	*lenp = hi - lo;
873 	return (1);
874 }
875 
876 /*
877  * Determine whether [*basep, *basep + *lenp) contains a mappable range of
878  * addresses at least "minlen" long.  On success, 1 is returned and *basep
879  * and *lenp are adjusted to describe the acceptable range.  On failure, 0
880  * is returned.
881  */
882 int
883 valid_va_range(caddr_t *basep, size_t *lenp, size_t minlen, int dir)
884 {
885 	return (valid_va_range_aligned(basep, lenp, minlen, dir, 0, 0, 0));
886 }
887 
888 /*
889  * Determine whether [addr, addr+len] are valid user addresses.
890  */
891 /*ARGSUSED*/
892 int
893 valid_usr_range(caddr_t addr, size_t len, uint_t prot, struct as *as,
894     caddr_t userlimit)
895 {
896 	caddr_t eaddr = addr + len;
897 
898 	if (eaddr <= addr || addr >= userlimit || eaddr > userlimit)
899 		return (RANGE_BADADDR);
900 
901 #if defined(__amd64)
902 	/*
903 	 * Check for the VA hole
904 	 */
905 	if (eaddr > (caddr_t)hole_start && addr < (caddr_t)hole_end)
906 		return (RANGE_BADADDR);
907 #endif
908 
909 	return (RANGE_OKAY);
910 }
911 
912 /*
913  * Return 1 if the page frame is onboard memory, else 0.
914  */
915 int
916 pf_is_memory(pfn_t pf)
917 {
918 	if (pfn_is_foreign(pf))
919 		return (0);
920 	return (address_in_memlist(phys_install, pfn_to_pa(pf), 1));
921 }
922 
923 /*
924  * return the memrange containing pfn
925  */
926 int
927 memrange_num(pfn_t pfn)
928 {
929 	int n;
930 
931 	for (n = 0; n < nranges - 1; ++n) {
932 		if (pfn >= memranges[n])
933 			break;
934 	}
935 	return (n);
936 }
937 
938 /*
939  * return the mnoderange containing pfn
940  */
941 /*ARGSUSED*/
942 int
943 pfn_2_mtype(pfn_t pfn)
944 {
945 #if defined(__xpv)
946 	return (0);
947 #else
948 	int	n;
949 
950 	for (n = mnoderangecnt - 1; n >= 0; n--) {
951 		if (pfn >= mnoderanges[n].mnr_pfnlo) {
952 			break;
953 		}
954 	}
955 	return (n);
956 #endif
957 }
958 
959 #if !defined(__xpv)
960 /*
961  * is_contigpage_free:
962  *	returns a page list of contiguous pages. It minimally has to return
963  *	minctg pages. Caller determines minctg based on the scatter-gather
964  *	list length.
965  *
966  *	pfnp is set to the next page frame to search on return.
967  */
968 static page_t *
969 is_contigpage_free(
970 	pfn_t *pfnp,
971 	pgcnt_t *pgcnt,
972 	pgcnt_t minctg,
973 	uint64_t pfnseg,
974 	int iolock)
975 {
976 	int	i = 0;
977 	pfn_t	pfn = *pfnp;
978 	page_t	*pp;
979 	page_t	*plist = NULL;
980 
981 	/*
982 	 * fail if pfn + minctg crosses a segment boundary.
983 	 * Adjust for next starting pfn to begin at segment boundary.
984 	 */
985 
986 	if (((*pfnp + minctg - 1) & pfnseg) < (*pfnp & pfnseg)) {
987 		*pfnp = roundup(*pfnp, pfnseg + 1);
988 		return (NULL);
989 	}
990 
991 	do {
992 retry:
993 		pp = page_numtopp_nolock(pfn + i);
994 		if ((pp == NULL) ||
995 		    (page_trylock(pp, SE_EXCL) == 0)) {
996 			(*pfnp)++;
997 			break;
998 		}
999 		if (page_pptonum(pp) != pfn + i) {
1000 			page_unlock(pp);
1001 			goto retry;
1002 		}
1003 
1004 		if (!(PP_ISFREE(pp))) {
1005 			page_unlock(pp);
1006 			(*pfnp)++;
1007 			break;
1008 		}
1009 
1010 		if (!PP_ISAGED(pp)) {
1011 			page_list_sub(pp, PG_CACHE_LIST);
1012 			page_hashout(pp, (kmutex_t *)NULL);
1013 		} else {
1014 			page_list_sub(pp, PG_FREE_LIST);
1015 		}
1016 
1017 		if (iolock)
1018 			page_io_lock(pp);
1019 		page_list_concat(&plist, &pp);
1020 
1021 		/*
1022 		 * exit loop when pgcnt satisfied or segment boundary reached.
1023 		 */
1024 
1025 	} while ((++i < *pgcnt) && ((pfn + i) & pfnseg));
1026 
1027 	*pfnp += i;		/* set to next pfn to search */
1028 
1029 	if (i >= minctg) {
1030 		*pgcnt -= i;
1031 		return (plist);
1032 	}
1033 
1034 	/*
1035 	 * failure: minctg not satisfied.
1036 	 *
1037 	 * if next request crosses segment boundary, set next pfn
1038 	 * to search from the segment boundary.
1039 	 */
1040 	if (((*pfnp + minctg - 1) & pfnseg) < (*pfnp & pfnseg))
1041 		*pfnp = roundup(*pfnp, pfnseg + 1);
1042 
1043 	/* clean up any pages already allocated */
1044 
1045 	while (plist) {
1046 		pp = plist;
1047 		page_sub(&plist, pp);
1048 		page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
1049 		if (iolock)
1050 			page_io_unlock(pp);
1051 		page_unlock(pp);
1052 	}
1053 
1054 	return (NULL);
1055 }
1056 #endif	/* !__xpv */
1057 
1058 /*
1059  * verify that pages being returned from allocator have correct DMA attribute
1060  */
1061 #ifndef DEBUG
1062 #define	check_dma(a, b, c) (0)
1063 #else
1064 static void
1065 check_dma(ddi_dma_attr_t *dma_attr, page_t *pp, int cnt)
1066 {
1067 	if (dma_attr == NULL)
1068 		return;
1069 
1070 	while (cnt-- > 0) {
1071 		if (pa_to_ma(pfn_to_pa(pp->p_pagenum)) <
1072 		    dma_attr->dma_attr_addr_lo)
1073 			panic("PFN (pp=%p) below dma_attr_addr_lo", (void *)pp);
1074 		if (pa_to_ma(pfn_to_pa(pp->p_pagenum)) >=
1075 		    dma_attr->dma_attr_addr_hi)
1076 			panic("PFN (pp=%p) above dma_attr_addr_hi", (void *)pp);
1077 		pp = pp->p_next;
1078 	}
1079 }
1080 #endif
1081 
1082 #if !defined(__xpv)
1083 static page_t *
1084 page_get_contigpage(pgcnt_t *pgcnt, ddi_dma_attr_t *mattr, int iolock)
1085 {
1086 	pfn_t		pfn;
1087 	int		sgllen;
1088 	uint64_t	pfnseg;
1089 	pgcnt_t		minctg;
1090 	page_t		*pplist = NULL, *plist;
1091 	uint64_t	lo, hi;
1092 	pgcnt_t		pfnalign = 0;
1093 	static pfn_t	startpfn;
1094 	static pgcnt_t	lastctgcnt;
1095 	uintptr_t	align;
1096 
1097 	CONTIG_LOCK();
1098 
1099 	if (mattr) {
1100 		lo = mmu_btop((mattr->dma_attr_addr_lo + MMU_PAGEOFFSET));
1101 		hi = mmu_btop(mattr->dma_attr_addr_hi);
1102 		if (hi >= physmax)
1103 			hi = physmax - 1;
1104 		sgllen = mattr->dma_attr_sgllen;
1105 		pfnseg = mmu_btop(mattr->dma_attr_seg);
1106 
1107 		align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
1108 		if (align > MMU_PAGESIZE)
1109 			pfnalign = mmu_btop(align);
1110 
1111 		/*
1112 		 * in order to satisfy the request, must minimally
1113 		 * acquire minctg contiguous pages
1114 		 */
1115 		minctg = howmany(*pgcnt, sgllen);
1116 
1117 		ASSERT(hi >= lo);
1118 
1119 		/*
1120 		 * start from where last searched if the minctg >= lastctgcnt
1121 		 */
1122 		if (minctg < lastctgcnt || startpfn < lo || startpfn > hi)
1123 			startpfn = lo;
1124 	} else {
1125 		hi = physmax - 1;
1126 		lo = 0;
1127 		sgllen = 1;
1128 		pfnseg = mmu.highest_pfn;
1129 		minctg = *pgcnt;
1130 
1131 		if (minctg < lastctgcnt)
1132 			startpfn = lo;
1133 	}
1134 	lastctgcnt = minctg;
1135 
1136 	ASSERT(pfnseg + 1 >= (uint64_t)minctg);
1137 
1138 	/* conserve 16m memory - start search above 16m when possible */
1139 	if (hi > PFN_16M && startpfn < PFN_16M)
1140 		startpfn = PFN_16M;
1141 
1142 	pfn = startpfn;
1143 	if (pfnalign)
1144 		pfn = P2ROUNDUP(pfn, pfnalign);
1145 
1146 	while (pfn + minctg - 1 <= hi) {
1147 
1148 		plist = is_contigpage_free(&pfn, pgcnt, minctg, pfnseg, iolock);
1149 		if (plist) {
1150 			page_list_concat(&pplist, &plist);
1151 			sgllen--;
1152 			/*
1153 			 * return when contig pages no longer needed
1154 			 */
1155 			if (!*pgcnt || ((*pgcnt <= sgllen) && !pfnalign)) {
1156 				startpfn = pfn;
1157 				CONTIG_UNLOCK();
1158 				check_dma(mattr, pplist, *pgcnt);
1159 				return (pplist);
1160 			}
1161 			minctg = howmany(*pgcnt, sgllen);
1162 		}
1163 		if (pfnalign)
1164 			pfn = P2ROUNDUP(pfn, pfnalign);
1165 	}
1166 
1167 	/* cannot find contig pages in specified range */
1168 	if (startpfn == lo) {
1169 		CONTIG_UNLOCK();
1170 		return (NULL);
1171 	}
1172 
1173 	/* did not start with lo previously */
1174 	pfn = lo;
1175 	if (pfnalign)
1176 		pfn = P2ROUNDUP(pfn, pfnalign);
1177 
1178 	/* allow search to go above startpfn */
1179 	while (pfn < startpfn) {
1180 
1181 		plist = is_contigpage_free(&pfn, pgcnt, minctg, pfnseg, iolock);
1182 		if (plist != NULL) {
1183 
1184 			page_list_concat(&pplist, &plist);
1185 			sgllen--;
1186 
1187 			/*
1188 			 * return when contig pages no longer needed
1189 			 */
1190 			if (!*pgcnt || ((*pgcnt <= sgllen) && !pfnalign)) {
1191 				startpfn = pfn;
1192 				CONTIG_UNLOCK();
1193 				check_dma(mattr, pplist, *pgcnt);
1194 				return (pplist);
1195 			}
1196 			minctg = howmany(*pgcnt, sgllen);
1197 		}
1198 		if (pfnalign)
1199 			pfn = P2ROUNDUP(pfn, pfnalign);
1200 	}
1201 	CONTIG_UNLOCK();
1202 	return (NULL);
1203 }
1204 #endif	/* !__xpv */
1205 
1206 /*
1207  * mnode_range_cnt() calculates the number of memory ranges for mnode and
1208  * memranges[]. Used to determine the size of page lists and mnoderanges.
1209  */
1210 int
1211 mnode_range_cnt(int mnode)
1212 {
1213 #if defined(__xpv)
1214 	ASSERT(mnode == 0);
1215 	return (1);
1216 #else	/* __xpv */
1217 	int	mri;
1218 	int	mnrcnt = 0;
1219 
1220 	if (mem_node_config[mnode].exists != 0) {
1221 		mri = nranges - 1;
1222 
1223 		/* find the memranges index below contained in mnode range */
1224 
1225 		while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
1226 			mri--;
1227 
1228 		/*
1229 		 * increment mnode range counter when memranges or mnode
1230 		 * boundary is reached.
1231 		 */
1232 		while (mri >= 0 &&
1233 		    mem_node_config[mnode].physmax >= MEMRANGELO(mri)) {
1234 			mnrcnt++;
1235 			if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
1236 				mri--;
1237 			else
1238 				break;
1239 		}
1240 	}
1241 	ASSERT(mnrcnt <= MAX_MNODE_MRANGES);
1242 	return (mnrcnt);
1243 #endif	/* __xpv */
1244 }
1245 
1246 /*
1247  * mnode_range_setup() initializes mnoderanges.
1248  */
1249 void
1250 mnode_range_setup(mnoderange_t *mnoderanges)
1251 {
1252 	int	mnode, mri;
1253 
1254 	for (mnode = 0; mnode < max_mem_nodes; mnode++) {
1255 		if (mem_node_config[mnode].exists == 0)
1256 			continue;
1257 
1258 		mri = nranges - 1;
1259 
1260 		while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
1261 			mri--;
1262 
1263 		while (mri >= 0 && mem_node_config[mnode].physmax >=
1264 		    MEMRANGELO(mri)) {
1265 			mnoderanges->mnr_pfnlo = MAX(MEMRANGELO(mri),
1266 			    mem_node_config[mnode].physbase);
1267 			mnoderanges->mnr_pfnhi = MIN(MEMRANGEHI(mri),
1268 			    mem_node_config[mnode].physmax);
1269 			mnoderanges->mnr_mnode = mnode;
1270 			mnoderanges->mnr_memrange = mri;
1271 			mnoderanges++;
1272 			if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
1273 				mri--;
1274 			else
1275 				break;
1276 		}
1277 	}
1278 }
1279 
1280 /*ARGSUSED*/
1281 int
1282 mtype_init(vnode_t *vp, caddr_t vaddr, uint_t *flags, size_t pgsz)
1283 {
1284 	int mtype = mnoderangecnt - 1;
1285 
1286 #if !defined(__xpv)
1287 #if defined(__i386)
1288 	/*
1289 	 * set the mtype range
1290 	 * - kmem requests needs to be below 4g if restricted_kmemalloc is set.
1291 	 * - for non kmem requests, set range to above 4g if memory below 4g
1292 	 * runs low.
1293 	 */
1294 	if (restricted_kmemalloc && VN_ISKAS(vp) &&
1295 	    (caddr_t)(vaddr) >= kernelheap &&
1296 	    (caddr_t)(vaddr) < ekernelheap) {
1297 		ASSERT(physmax4g);
1298 		mtype = mtype4g;
1299 		if (RESTRICT16M_ALLOC(freemem4g - btop(pgsz),
1300 		    btop(pgsz), *flags)) {
1301 			*flags |= PGI_MT_RANGE16M;
1302 		} else {
1303 			VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1304 			VM_STAT_COND_ADD((*flags & PG_PANIC),
1305 			    vmm_vmstats.pgpanicalloc);
1306 			*flags |= PGI_MT_RANGE0;
1307 		}
1308 		return (mtype);
1309 	}
1310 #endif	/* __i386 */
1311 
1312 	if (RESTRICT4G_ALLOC) {
1313 		VM_STAT_ADD(vmm_vmstats.restrict4gcnt);
1314 		/* here only for > 4g systems */
1315 		*flags |= PGI_MT_RANGE4G;
1316 	} else if (RESTRICT16M_ALLOC(freemem, btop(pgsz), *flags)) {
1317 		*flags |= PGI_MT_RANGE16M;
1318 	} else {
1319 		VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1320 		VM_STAT_COND_ADD((*flags & PG_PANIC), vmm_vmstats.pgpanicalloc);
1321 		*flags |= PGI_MT_RANGE0;
1322 	}
1323 #endif /* !__xpv */
1324 	return (mtype);
1325 }
1326 
1327 
1328 /* mtype init for page_get_replacement_page */
1329 /*ARGSUSED*/
1330 int
1331 mtype_pgr_init(int *flags, page_t *pp, int mnode, pgcnt_t pgcnt)
1332 {
1333 	int mtype = mnoderangecnt - 1;
1334 #if !defined(__ixpv)
1335 	if (RESTRICT16M_ALLOC(freemem, pgcnt, *flags)) {
1336 		*flags |= PGI_MT_RANGE16M;
1337 	} else {
1338 		VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1339 		*flags |= PGI_MT_RANGE0;
1340 	}
1341 #endif
1342 	return (mtype);
1343 }
1344 
1345 /*
1346  * Determine if the mnode range specified in mtype contains memory belonging
1347  * to memory node mnode.  If flags & PGI_MT_RANGE is set then mtype contains
1348  * the range of indices from high pfn to 0, 16m or 4g.
1349  *
1350  * Return first mnode range type index found otherwise return -1 if none found.
1351  */
1352 int
1353 mtype_func(int mnode, int mtype, uint_t flags)
1354 {
1355 	if (flags & PGI_MT_RANGE) {
1356 		int	mtlim = 0;
1357 
1358 		if (flags & PGI_MT_NEXT)
1359 			mtype--;
1360 		if (flags & PGI_MT_RANGE4G)
1361 			mtlim = mtype4g + 1;	/* exclude 0-4g range */
1362 		else if (flags & PGI_MT_RANGE16M)
1363 			mtlim = 1;		/* exclude 0-16m range */
1364 		while (mtype >= mtlim) {
1365 			if (mnoderanges[mtype].mnr_mnode == mnode)
1366 				return (mtype);
1367 			mtype--;
1368 		}
1369 	} else if (mnoderanges[mtype].mnr_mnode == mnode) {
1370 		return (mtype);
1371 	}
1372 	return (-1);
1373 }
1374 
1375 /*
1376  * Update the page list max counts with the pfn range specified by the
1377  * input parameters.  Called from add_physmem() when physical memory with
1378  * page_t's are initially added to the page lists.
1379  */
1380 void
1381 mtype_modify_max(pfn_t startpfn, long cnt)
1382 {
1383 	int	mtype = 0;
1384 	pfn_t	endpfn = startpfn + cnt, pfn;
1385 	pgcnt_t	inc;
1386 
1387 	ASSERT(cnt > 0);
1388 
1389 	if (!physmax4g)
1390 		return;
1391 
1392 	for (pfn = startpfn; pfn < endpfn; ) {
1393 		if (pfn <= mnoderanges[mtype].mnr_pfnhi) {
1394 			if (endpfn < mnoderanges[mtype].mnr_pfnhi) {
1395 				inc = endpfn - pfn;
1396 			} else {
1397 				inc = mnoderanges[mtype].mnr_pfnhi - pfn + 1;
1398 			}
1399 			if (mtype <= mtype4g)
1400 				maxmem4g += inc;
1401 			pfn += inc;
1402 		}
1403 		mtype++;
1404 		ASSERT(mtype < mnoderangecnt || pfn >= endpfn);
1405 	}
1406 }
1407 
1408 int
1409 mtype_2_mrange(int mtype)
1410 {
1411 	return (mnoderanges[mtype].mnr_memrange);
1412 }
1413 
1414 void
1415 mnodetype_2_pfn(int mnode, int mtype, pfn_t *pfnlo, pfn_t *pfnhi)
1416 {
1417 	ASSERT(mnoderanges[mtype].mnr_mnode == mnode);
1418 	*pfnlo = mnoderanges[mtype].mnr_pfnlo;
1419 	*pfnhi = mnoderanges[mtype].mnr_pfnhi;
1420 }
1421 
1422 size_t
1423 plcnt_sz(size_t ctrs_sz)
1424 {
1425 #ifdef DEBUG
1426 	int	szc, colors;
1427 
1428 	ctrs_sz += mnoderangecnt * sizeof (struct mnr_mts) * mmu_page_sizes;
1429 	for (szc = 0; szc < mmu_page_sizes; szc++) {
1430 		colors = page_get_pagecolors(szc);
1431 		ctrs_sz += mnoderangecnt * sizeof (pgcnt_t) * colors;
1432 	}
1433 #endif
1434 	return (ctrs_sz);
1435 }
1436 
1437 caddr_t
1438 plcnt_init(caddr_t addr)
1439 {
1440 #ifdef DEBUG
1441 	int	mt, szc, colors;
1442 
1443 	for (mt = 0; mt < mnoderangecnt; mt++) {
1444 		mnoderanges[mt].mnr_mts = (struct mnr_mts *)addr;
1445 		addr += (sizeof (struct mnr_mts) * mmu_page_sizes);
1446 		for (szc = 0; szc < mmu_page_sizes; szc++) {
1447 			colors = page_get_pagecolors(szc);
1448 			mnoderanges[mt].mnr_mts[szc].mnr_mts_colors = colors;
1449 			mnoderanges[mt].mnr_mts[szc].mnr_mtsc_pgcnt =
1450 			    (pgcnt_t *)addr;
1451 			addr += (sizeof (pgcnt_t) * colors);
1452 		}
1453 	}
1454 #endif
1455 	return (addr);
1456 }
1457 
1458 void
1459 plcnt_inc_dec(page_t *pp, int mtype, int szc, long cnt, int flags)
1460 {
1461 #ifdef DEBUG
1462 	int	bin = PP_2_BIN(pp);
1463 
1464 	atomic_add_long(&mnoderanges[mtype].mnr_mts[szc].mnr_mts_pgcnt, cnt);
1465 	atomic_add_long(&mnoderanges[mtype].mnr_mts[szc].mnr_mtsc_pgcnt[bin],
1466 	    cnt);
1467 #endif
1468 	ASSERT(mtype == PP_2_MTYPE(pp));
1469 	if (physmax4g && mtype <= mtype4g)
1470 		atomic_add_long(&freemem4g, cnt);
1471 	if (flags & PG_CACHE_LIST)
1472 		atomic_add_long(&mnoderanges[mtype].mnr_mt_clpgcnt, cnt);
1473 	else
1474 		atomic_add_long(&mnoderanges[mtype].mnr_mt_flpgcnt[szc], cnt);
1475 	atomic_add_long(&mnoderanges[mtype].mnr_mt_totcnt, cnt);
1476 }
1477 
1478 /*
1479  * Returns the free page count for mnode
1480  */
1481 int
1482 mnode_pgcnt(int mnode)
1483 {
1484 	int	mtype = mnoderangecnt - 1;
1485 	int	flags = PGI_MT_RANGE0;
1486 	pgcnt_t	pgcnt = 0;
1487 
1488 	mtype = mtype_func(mnode, mtype, flags);
1489 
1490 	while (mtype != -1) {
1491 		pgcnt += MTYPE_FREEMEM(mtype);
1492 		mtype = mtype_func(mnode, mtype, flags | PGI_MT_NEXT);
1493 	}
1494 	return (pgcnt);
1495 }
1496 
1497 /*
1498  * Initialize page coloring variables based on the l2 cache parameters.
1499  * Calculate and return memory needed for page coloring data structures.
1500  */
1501 size_t
1502 page_coloring_init(uint_t l2_sz, int l2_linesz, int l2_assoc)
1503 {
1504 	size_t	colorsz = 0;
1505 	int	i;
1506 	int	colors;
1507 
1508 #if defined(__xpv)
1509 	/*
1510 	 * Hypervisor domains currently don't have any concept of NUMA.
1511 	 * Hence we'll act like there is only 1 memrange.
1512 	 */
1513 	i = memrange_num(1);
1514 #else /* !__xpv */
1515 	/*
1516 	 * Reduce the memory ranges lists if we don't have large amounts
1517 	 * of memory. This avoids searching known empty free lists.
1518 	 */
1519 	i = memrange_num(physmax);
1520 #if defined(__i386)
1521 	if (i > 0)
1522 		restricted_kmemalloc = 0;
1523 #endif
1524 	/* physmax greater than 4g */
1525 	if (i == 0)
1526 		physmax4g = 1;
1527 #endif /* !__xpv */
1528 	memranges += i;
1529 	nranges -= i;
1530 
1531 	ASSERT(mmu_page_sizes <= MMU_PAGE_SIZES);
1532 
1533 	ASSERT(ISP2(l2_linesz));
1534 	ASSERT(l2_sz > MMU_PAGESIZE);
1535 
1536 	/* l2_assoc is 0 for fully associative l2 cache */
1537 	if (l2_assoc)
1538 		l2_colors = MAX(1, l2_sz / (l2_assoc * MMU_PAGESIZE));
1539 	else
1540 		l2_colors = 1;
1541 
1542 	ASSERT(ISP2(l2_colors));
1543 
1544 	/* for scalability, configure at least PAGE_COLORS_MIN color bins */
1545 	page_colors = MAX(l2_colors, PAGE_COLORS_MIN);
1546 
1547 	/*
1548 	 * cpu_page_colors is non-zero when a page color may be spread across
1549 	 * multiple bins.
1550 	 */
1551 	if (l2_colors < page_colors)
1552 		cpu_page_colors = l2_colors;
1553 
1554 	ASSERT(ISP2(page_colors));
1555 
1556 	page_colors_mask = page_colors - 1;
1557 
1558 	ASSERT(ISP2(CPUSETSIZE()));
1559 	page_coloring_shift = lowbit(CPUSETSIZE());
1560 
1561 	/* initialize number of colors per page size */
1562 	for (i = 0; i <= mmu.max_page_level; i++) {
1563 		hw_page_array[i].hp_size = LEVEL_SIZE(i);
1564 		hw_page_array[i].hp_shift = LEVEL_SHIFT(i);
1565 		hw_page_array[i].hp_pgcnt = LEVEL_SIZE(i) >> LEVEL_SHIFT(0);
1566 		hw_page_array[i].hp_colors = (page_colors_mask >>
1567 		    (hw_page_array[i].hp_shift - hw_page_array[0].hp_shift))
1568 		    + 1;
1569 		colorequivszc[i] = 0;
1570 	}
1571 
1572 	/*
1573 	 * The value of cpu_page_colors determines if additional color bins
1574 	 * need to be checked for a particular color in the page_get routines.
1575 	 */
1576 	if (cpu_page_colors != 0) {
1577 
1578 		int a = lowbit(page_colors) - lowbit(cpu_page_colors);
1579 		ASSERT(a > 0);
1580 		ASSERT(a < 16);
1581 
1582 		for (i = 0; i <= mmu.max_page_level; i++) {
1583 			if ((colors = hw_page_array[i].hp_colors) <= 1) {
1584 				colorequivszc[i] = 0;
1585 				continue;
1586 			}
1587 			while ((colors >> a) == 0)
1588 				a--;
1589 			ASSERT(a >= 0);
1590 
1591 			/* higher 4 bits encodes color equiv mask */
1592 			colorequivszc[i] = (a << 4);
1593 		}
1594 	}
1595 
1596 	/* factor in colorequiv to check additional 'equivalent' bins. */
1597 	if (colorequiv > 1) {
1598 
1599 		int a = lowbit(colorequiv) - 1;
1600 		if (a > 15)
1601 			a = 15;
1602 
1603 		for (i = 0; i <= mmu.max_page_level; i++) {
1604 			if ((colors = hw_page_array[i].hp_colors) <= 1) {
1605 				continue;
1606 			}
1607 			while ((colors >> a) == 0)
1608 				a--;
1609 			if ((a << 4) > colorequivszc[i]) {
1610 				colorequivszc[i] = (a << 4);
1611 			}
1612 		}
1613 	}
1614 
1615 	/* size for mnoderanges */
1616 	for (mnoderangecnt = 0, i = 0; i < max_mem_nodes; i++)
1617 		mnoderangecnt += mnode_range_cnt(i);
1618 	colorsz = mnoderangecnt * sizeof (mnoderange_t);
1619 
1620 	/* size for fpc_mutex and cpc_mutex */
1621 	colorsz += (2 * max_mem_nodes * sizeof (kmutex_t) * NPC_MUTEX);
1622 
1623 	/* size of page_freelists */
1624 	colorsz += mnoderangecnt * sizeof (page_t ***);
1625 	colorsz += mnoderangecnt * mmu_page_sizes * sizeof (page_t **);
1626 
1627 	for (i = 0; i < mmu_page_sizes; i++) {
1628 		colors = page_get_pagecolors(i);
1629 		colorsz += mnoderangecnt * colors * sizeof (page_t *);
1630 	}
1631 
1632 	/* size of page_cachelists */
1633 	colorsz += mnoderangecnt * sizeof (page_t **);
1634 	colorsz += mnoderangecnt * page_colors * sizeof (page_t *);
1635 
1636 	return (colorsz);
1637 }
1638 
1639 /*
1640  * Called once at startup to configure page_coloring data structures and
1641  * does the 1st page_free()/page_freelist_add().
1642  */
1643 void
1644 page_coloring_setup(caddr_t pcmemaddr)
1645 {
1646 	int	i;
1647 	int	j;
1648 	int	k;
1649 	caddr_t	addr;
1650 	int	colors;
1651 
1652 	/*
1653 	 * do page coloring setup
1654 	 */
1655 	addr = pcmemaddr;
1656 
1657 	mnoderanges = (mnoderange_t *)addr;
1658 	addr += (mnoderangecnt * sizeof (mnoderange_t));
1659 
1660 	mnode_range_setup(mnoderanges);
1661 
1662 	if (physmax4g)
1663 		mtype4g = pfn_2_mtype(0xfffff);
1664 
1665 	for (k = 0; k < NPC_MUTEX; k++) {
1666 		fpc_mutex[k] = (kmutex_t *)addr;
1667 		addr += (max_mem_nodes * sizeof (kmutex_t));
1668 	}
1669 	for (k = 0; k < NPC_MUTEX; k++) {
1670 		cpc_mutex[k] = (kmutex_t *)addr;
1671 		addr += (max_mem_nodes * sizeof (kmutex_t));
1672 	}
1673 	page_freelists = (page_t ****)addr;
1674 	addr += (mnoderangecnt * sizeof (page_t ***));
1675 
1676 	page_cachelists = (page_t ***)addr;
1677 	addr += (mnoderangecnt * sizeof (page_t **));
1678 
1679 	for (i = 0; i < mnoderangecnt; i++) {
1680 		page_freelists[i] = (page_t ***)addr;
1681 		addr += (mmu_page_sizes * sizeof (page_t **));
1682 
1683 		for (j = 0; j < mmu_page_sizes; j++) {
1684 			colors = page_get_pagecolors(j);
1685 			page_freelists[i][j] = (page_t **)addr;
1686 			addr += (colors * sizeof (page_t *));
1687 		}
1688 		page_cachelists[i] = (page_t **)addr;
1689 		addr += (page_colors * sizeof (page_t *));
1690 	}
1691 }
1692 
1693 #if defined(__xpv)
1694 /*
1695  * Give back 10% of the io_pool pages to the free list.
1696  * Don't shrink the pool below some absolute minimum.
1697  */
1698 static void
1699 page_io_pool_shrink()
1700 {
1701 	int retcnt;
1702 	page_t *pp, *pp_first, *pp_last, **curpool;
1703 	mfn_t mfn;
1704 	int bothpools = 0;
1705 
1706 	mutex_enter(&io_pool_lock);
1707 	io_pool_shrink_attempts++;	/* should be a kstat? */
1708 	retcnt = io_pool_cnt / 10;
1709 	if (io_pool_cnt - retcnt < io_pool_cnt_min)
1710 		retcnt = io_pool_cnt - io_pool_cnt_min;
1711 	if (retcnt <= 0)
1712 		goto done;
1713 	io_pool_shrinks++;	/* should be a kstat? */
1714 	curpool = &io_pool_4g;
1715 domore:
1716 	/*
1717 	 * Loop through taking pages from the end of the list
1718 	 * (highest mfns) till amount to return reached.
1719 	 */
1720 	for (pp = *curpool; pp && retcnt > 0; ) {
1721 		pp_first = pp_last = pp->p_prev;
1722 		if (pp_first == *curpool)
1723 			break;
1724 		retcnt--;
1725 		io_pool_cnt--;
1726 		page_io_pool_sub(curpool, pp_first, pp_last);
1727 		if ((mfn = pfn_to_mfn(pp->p_pagenum)) < start_mfn)
1728 			start_mfn = mfn;
1729 		page_free(pp_first, 1);
1730 		pp = *curpool;
1731 	}
1732 	if (retcnt != 0 && !bothpools) {
1733 		/*
1734 		 * If not enough found in less constrained pool try the
1735 		 * more constrained one.
1736 		 */
1737 		curpool = &io_pool_16m;
1738 		bothpools = 1;
1739 		goto domore;
1740 	}
1741 done:
1742 	mutex_exit(&io_pool_lock);
1743 }
1744 
1745 #endif	/* __xpv */
1746 
1747 uint_t
1748 page_create_update_flags_x86(uint_t flags)
1749 {
1750 #if defined(__xpv)
1751 	/*
1752 	 * Check this is an urgent allocation and free pages are depleted.
1753 	 */
1754 	if (!(flags & PG_WAIT) && freemem < desfree)
1755 		page_io_pool_shrink();
1756 #else /* !__xpv */
1757 	/*
1758 	 * page_create_get_something may call this because 4g memory may be
1759 	 * depleted. Set flags to allow for relocation of base page below
1760 	 * 4g if necessary.
1761 	 */
1762 	if (physmax4g)
1763 		flags |= (PGI_PGCPSZC0 | PGI_PGCPHIPRI);
1764 #endif /* __xpv */
1765 	return (flags);
1766 }
1767 
1768 /*ARGSUSED*/
1769 int
1770 bp_color(struct buf *bp)
1771 {
1772 	return (0);
1773 }
1774 
1775 #if defined(__xpv)
1776 
1777 /*
1778  * Take pages out of an io_pool
1779  */
1780 static void
1781 page_io_pool_sub(page_t **poolp, page_t *pp_first, page_t *pp_last)
1782 {
1783 	if (*poolp == pp_first) {
1784 		*poolp = pp_last->p_next;
1785 		if (*poolp == pp_first)
1786 			*poolp = NULL;
1787 	}
1788 	pp_first->p_prev->p_next = pp_last->p_next;
1789 	pp_last->p_next->p_prev = pp_first->p_prev;
1790 	pp_first->p_prev = pp_last;
1791 	pp_last->p_next = pp_first;
1792 }
1793 
1794 /*
1795  * Put a page on the io_pool list. The list is ordered by increasing MFN.
1796  */
1797 static void
1798 page_io_pool_add(page_t **poolp, page_t *pp)
1799 {
1800 	page_t	*look;
1801 	mfn_t	mfn = mfn_list[pp->p_pagenum];
1802 
1803 	if (*poolp == NULL) {
1804 		*poolp = pp;
1805 		pp->p_next = pp;
1806 		pp->p_prev = pp;
1807 		return;
1808 	}
1809 
1810 	/*
1811 	 * Since we try to take pages from the high end of the pool
1812 	 * chances are good that the pages to be put on the list will
1813 	 * go at or near the end of the list. so start at the end and
1814 	 * work backwards.
1815 	 */
1816 	look = (*poolp)->p_prev;
1817 	while (mfn < mfn_list[look->p_pagenum]) {
1818 		look = look->p_prev;
1819 		if (look == (*poolp)->p_prev)
1820 			break; /* backed all the way to front of list */
1821 	}
1822 
1823 	/* insert after look */
1824 	pp->p_prev = look;
1825 	pp->p_next = look->p_next;
1826 	pp->p_next->p_prev = pp;
1827 	look->p_next = pp;
1828 	if (mfn < mfn_list[(*poolp)->p_pagenum]) {
1829 		/*
1830 		 * we inserted a new first list element
1831 		 * adjust pool pointer to newly inserted element
1832 		 */
1833 		*poolp = pp;
1834 	}
1835 }
1836 
1837 /*
1838  * Add a page to the io_pool.  Setting the force flag will force the page
1839  * into the io_pool no matter what.
1840  */
1841 static void
1842 add_page_to_pool(page_t *pp, int force)
1843 {
1844 	page_t *highest;
1845 	page_t *freep = NULL;
1846 
1847 	mutex_enter(&io_pool_lock);
1848 	/*
1849 	 * Always keep the scarce low memory pages
1850 	 */
1851 	if (mfn_list[pp->p_pagenum] < PFN_16MEG) {
1852 		++io_pool_cnt;
1853 		page_io_pool_add(&io_pool_16m, pp);
1854 		goto done;
1855 	}
1856 	if (io_pool_cnt < io_pool_cnt_max || force || io_pool_4g == NULL) {
1857 		++io_pool_cnt;
1858 		page_io_pool_add(&io_pool_4g, pp);
1859 	} else {
1860 		highest = io_pool_4g->p_prev;
1861 		if (mfn_list[pp->p_pagenum] < mfn_list[highest->p_pagenum]) {
1862 			page_io_pool_sub(&io_pool_4g, highest, highest);
1863 			page_io_pool_add(&io_pool_4g, pp);
1864 			freep = highest;
1865 		} else {
1866 			freep = pp;
1867 		}
1868 	}
1869 done:
1870 	mutex_exit(&io_pool_lock);
1871 	if (freep)
1872 		page_free(freep, 1);
1873 }
1874 
1875 
1876 int contig_pfn_cnt;	/* no of pfns in the contig pfn list */
1877 int contig_pfn_max;	/* capacity of the contig pfn list */
1878 int next_alloc_pfn;	/* next position in list to start a contig search */
1879 int contig_pfnlist_updates;	/* pfn list update count */
1880 int contig_pfnlist_builds;	/* how many times have we (re)built list */
1881 int contig_pfnlist_buildfailed;	/* how many times has list build failed */
1882 int create_contig_pending;	/* nonzero means taskq creating contig list */
1883 pfn_t *contig_pfn_list = NULL;	/* list of contig pfns in ascending mfn order */
1884 
1885 /*
1886  * Function to use in sorting a list of pfns by their underlying mfns.
1887  */
1888 static int
1889 mfn_compare(const void *pfnp1, const void *pfnp2)
1890 {
1891 	mfn_t mfn1 = mfn_list[*(pfn_t *)pfnp1];
1892 	mfn_t mfn2 = mfn_list[*(pfn_t *)pfnp2];
1893 
1894 	if (mfn1 > mfn2)
1895 		return (1);
1896 	if (mfn1 < mfn2)
1897 		return (-1);
1898 	return (0);
1899 }
1900 
1901 /*
1902  * Compact the contig_pfn_list by tossing all the non-contiguous
1903  * elements from the list.
1904  */
1905 static void
1906 compact_contig_pfn_list(void)
1907 {
1908 	pfn_t pfn, lapfn, prev_lapfn;
1909 	mfn_t mfn;
1910 	int i, newcnt = 0;
1911 
1912 	prev_lapfn = 0;
1913 	for (i = 0; i < contig_pfn_cnt - 1; i++) {
1914 		pfn = contig_pfn_list[i];
1915 		lapfn = contig_pfn_list[i + 1];
1916 		mfn = mfn_list[pfn];
1917 		/*
1918 		 * See if next pfn is for a contig mfn
1919 		 */
1920 		if (mfn_list[lapfn] != mfn + 1)
1921 			continue;
1922 		/*
1923 		 * pfn and lookahead are both put in list
1924 		 * unless pfn is the previous lookahead.
1925 		 */
1926 		if (pfn != prev_lapfn)
1927 			contig_pfn_list[newcnt++] = pfn;
1928 		contig_pfn_list[newcnt++] = lapfn;
1929 		prev_lapfn = lapfn;
1930 	}
1931 	for (i = newcnt; i < contig_pfn_cnt; i++)
1932 		contig_pfn_list[i] = 0;
1933 	contig_pfn_cnt = newcnt;
1934 }
1935 
1936 /*ARGSUSED*/
1937 static void
1938 call_create_contiglist(void *arg)
1939 {
1940 	(void) create_contig_pfnlist(PG_WAIT);
1941 }
1942 
1943 /*
1944  * Create list of freelist pfns that have underlying
1945  * contiguous mfns.  The list is kept in ascending mfn order.
1946  * returns 1 if list created else 0.
1947  */
1948 static int
1949 create_contig_pfnlist(uint_t flags)
1950 {
1951 	pfn_t pfn;
1952 	page_t *pp;
1953 	int ret = 1;
1954 
1955 	mutex_enter(&contig_list_lock);
1956 	if (contig_pfn_list != NULL)
1957 		goto out;
1958 	contig_pfn_max = freemem + (freemem / 10);
1959 	contig_pfn_list = kmem_zalloc(contig_pfn_max * sizeof (pfn_t),
1960 	    (flags & PG_WAIT) ? KM_SLEEP : KM_NOSLEEP);
1961 	if (contig_pfn_list == NULL) {
1962 		/*
1963 		 * If we could not create the contig list (because
1964 		 * we could not sleep for memory).  Dispatch a taskq that can
1965 		 * sleep to get the memory.
1966 		 */
1967 		if (!create_contig_pending) {
1968 			if (taskq_dispatch(system_taskq, call_create_contiglist,
1969 			    NULL, TQ_NOSLEEP) != NULL)
1970 				create_contig_pending = 1;
1971 		}
1972 		contig_pfnlist_buildfailed++;	/* count list build failures */
1973 		ret = 0;
1974 		goto out;
1975 	}
1976 	create_contig_pending = 0;
1977 	ASSERT(contig_pfn_cnt == 0);
1978 	for (pfn = 0; pfn < mfn_count; pfn++) {
1979 		pp = page_numtopp_nolock(pfn);
1980 		if (pp == NULL || !PP_ISFREE(pp))
1981 			continue;
1982 		contig_pfn_list[contig_pfn_cnt] = pfn;
1983 		if (++contig_pfn_cnt == contig_pfn_max)
1984 			break;
1985 	}
1986 	qsort(contig_pfn_list, contig_pfn_cnt, sizeof (pfn_t), mfn_compare);
1987 	compact_contig_pfn_list();
1988 	/*
1989 	 * Make sure next search of the newly created contiguous pfn
1990 	 * list starts at the beginning of the list.
1991 	 */
1992 	next_alloc_pfn = 0;
1993 	contig_pfnlist_builds++;	/* count list builds */
1994 out:
1995 	mutex_exit(&contig_list_lock);
1996 	return (ret);
1997 }
1998 
1999 
2000 /*
2001  * Toss the current contig pfnlist.  Someone is about to do a massive
2002  * update to pfn<->mfn mappings.  So we have them destroy the list and lock
2003  * it till they are done with their update.
2004  */
2005 void
2006 clear_and_lock_contig_pfnlist()
2007 {
2008 	pfn_t *listp = NULL;
2009 	size_t listsize;
2010 
2011 	mutex_enter(&contig_list_lock);
2012 	if (contig_pfn_list != NULL) {
2013 		listp = contig_pfn_list;
2014 		listsize = contig_pfn_max * sizeof (pfn_t);
2015 		contig_pfn_list = NULL;
2016 		contig_pfn_max = contig_pfn_cnt = 0;
2017 	}
2018 	if (listp != NULL)
2019 		kmem_free(listp, listsize);
2020 }
2021 
2022 /*
2023  * Unlock the contig_pfn_list.  The next attempted use of it will cause
2024  * it to be re-created.
2025  */
2026 void
2027 unlock_contig_pfnlist()
2028 {
2029 	mutex_exit(&contig_list_lock);
2030 }
2031 
2032 /*
2033  * Update the contiguous pfn list in response to a pfn <-> mfn reassignment
2034  */
2035 void
2036 update_contig_pfnlist(pfn_t pfn, mfn_t oldmfn, mfn_t newmfn)
2037 {
2038 	int probe_hi, probe_lo, probe_pos, insert_after, insert_point;
2039 	pfn_t probe_pfn;
2040 	mfn_t probe_mfn;
2041 	int drop_lock = 0;
2042 
2043 	if (mutex_owner(&contig_list_lock) != curthread) {
2044 		drop_lock = 1;
2045 		mutex_enter(&contig_list_lock);
2046 	}
2047 	if (contig_pfn_list == NULL)
2048 		goto done;
2049 	contig_pfnlist_updates++;
2050 	/*
2051 	 * Find the pfn in the current list.  Use a binary chop to locate it.
2052 	 */
2053 	probe_hi = contig_pfn_cnt - 1;
2054 	probe_lo = 0;
2055 	probe_pos = (probe_hi + probe_lo) / 2;
2056 	while ((probe_pfn = contig_pfn_list[probe_pos]) != pfn) {
2057 		if (probe_pos == probe_lo) { /* pfn not in list */
2058 			probe_pos = -1;
2059 			break;
2060 		}
2061 		if (pfn_to_mfn(probe_pfn) <= oldmfn)
2062 			probe_lo = probe_pos;
2063 		else
2064 			probe_hi = probe_pos;
2065 		probe_pos = (probe_hi + probe_lo) / 2;
2066 	}
2067 	if (probe_pos >= 0)  { /* remove pfn fom list */
2068 		contig_pfn_cnt--;
2069 		ovbcopy(&contig_pfn_list[probe_pos + 1],
2070 		    &contig_pfn_list[probe_pos],
2071 		    (contig_pfn_cnt - probe_pos) * sizeof (pfn_t));
2072 	}
2073 	if (newmfn == MFN_INVALID)
2074 		goto done;
2075 	/*
2076 	 * Check if new mfn has adjacent mfns in the list
2077 	 */
2078 	probe_hi = contig_pfn_cnt - 1;
2079 	probe_lo = 0;
2080 	insert_after = -2;
2081 	do {
2082 		probe_pos = (probe_hi + probe_lo) / 2;
2083 		probe_mfn = pfn_to_mfn(contig_pfn_list[probe_pos]);
2084 		if (newmfn == probe_mfn + 1)
2085 			insert_after = probe_pos;
2086 		else if (newmfn == probe_mfn - 1)
2087 			insert_after = probe_pos - 1;
2088 		if (probe_pos == probe_lo)
2089 			break;
2090 		if (probe_mfn <= newmfn)
2091 			probe_lo = probe_pos;
2092 		else
2093 			probe_hi = probe_pos;
2094 	} while (insert_after == -2);
2095 	/*
2096 	 * If there is space in the list and there are adjacent mfns
2097 	 * insert the pfn in to its proper place in the list.
2098 	 */
2099 	if (insert_after != -2 && contig_pfn_cnt + 1 <= contig_pfn_max) {
2100 		insert_point = insert_after + 1;
2101 		ovbcopy(&contig_pfn_list[insert_point],
2102 		    &contig_pfn_list[insert_point + 1],
2103 		    (contig_pfn_cnt - insert_point) * sizeof (pfn_t));
2104 		contig_pfn_list[insert_point] = pfn;
2105 		contig_pfn_cnt++;
2106 	}
2107 done:
2108 	if (drop_lock)
2109 		mutex_exit(&contig_list_lock);
2110 }
2111 
2112 /*
2113  * Called to (re-)populate the io_pool from the free page lists.
2114  */
2115 long
2116 populate_io_pool(void)
2117 {
2118 	pfn_t pfn;
2119 	mfn_t mfn, max_mfn;
2120 	page_t *pp;
2121 
2122 	/*
2123 	 * Figure out the bounds of the pool on first invocation.
2124 	 * We use a percentage of memory for the io pool size.
2125 	 * we allow that to shrink, but not to less than a fixed minimum
2126 	 */
2127 	if (io_pool_cnt_max == 0) {
2128 		io_pool_cnt_max = physmem / (100 / io_pool_physmem_pct);
2129 		io_pool_cnt_lowater = io_pool_cnt_max;
2130 		/*
2131 		 * This is the first time in populate_io_pool, grab a va to use
2132 		 * when we need to allocate pages.
2133 		 */
2134 		io_pool_kva = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP);
2135 	}
2136 	/*
2137 	 * If we are out of pages in the pool, then grow the size of the pool
2138 	 */
2139 	if (io_pool_cnt == 0) {
2140 		/*
2141 		 * Grow the max size of the io pool by 5%, but never more than
2142 		 * 25% of physical memory.
2143 		 */
2144 		if (io_pool_cnt_max < physmem / 4)
2145 			io_pool_cnt_max += io_pool_cnt_max / 20;
2146 	}
2147 	io_pool_grows++;	/* should be a kstat? */
2148 
2149 	/*
2150 	 * Get highest mfn on this platform, but limit to the 32 bit DMA max.
2151 	 */
2152 	(void) mfn_to_pfn(start_mfn);
2153 	max_mfn = MIN(cached_max_mfn, PFN_4GIG);
2154 	for (mfn = start_mfn; mfn < max_mfn; start_mfn = ++mfn) {
2155 		pfn = mfn_to_pfn(mfn);
2156 		if (pfn & PFN_IS_FOREIGN_MFN)
2157 			continue;
2158 		/*
2159 		 * try to allocate it from free pages
2160 		 */
2161 		pp = page_numtopp_alloc(pfn);
2162 		if (pp == NULL)
2163 			continue;
2164 		PP_CLRFREE(pp);
2165 		add_page_to_pool(pp, 1);
2166 		if (io_pool_cnt >= io_pool_cnt_max)
2167 			break;
2168 	}
2169 
2170 	return (io_pool_cnt);
2171 }
2172 
2173 /*
2174  * Destroy a page that was being used for DMA I/O. It may or
2175  * may not actually go back to the io_pool.
2176  */
2177 void
2178 page_destroy_io(page_t *pp)
2179 {
2180 	mfn_t mfn = mfn_list[pp->p_pagenum];
2181 
2182 	/*
2183 	 * When the page was alloc'd a reservation was made, release it now
2184 	 */
2185 	page_unresv(1);
2186 	/*
2187 	 * Unload translations, if any, then hash out the
2188 	 * page to erase its identity.
2189 	 */
2190 	(void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
2191 	page_hashout(pp, NULL);
2192 
2193 	/*
2194 	 * If the page came from the free lists, just put it back to them.
2195 	 * DomU pages always go on the free lists as well.
2196 	 */
2197 	if (!DOMAIN_IS_INITDOMAIN(xen_info) || mfn >= PFN_4GIG) {
2198 		page_free(pp, 1);
2199 		return;
2200 	}
2201 
2202 	add_page_to_pool(pp, 0);
2203 }
2204 
2205 
2206 long contig_searches;		/* count of times contig pages requested */
2207 long contig_search_restarts;	/* count of contig ranges tried */
2208 long contig_search_failed;	/* count of contig alloc failures */
2209 
2210 /*
2211  * Look thru the contiguous pfns that are not part of the io_pool for
2212  * contiguous free pages.  Return a list of the found pages or NULL.
2213  */
2214 page_t *
2215 find_contig_free(uint_t npages, uint_t flags, uint64_t pfnseg)
2216 {
2217 	page_t *pp, *plist = NULL;
2218 	mfn_t mfn, prev_mfn, start_mfn;
2219 	pfn_t pfn;
2220 	int pages_needed, pages_requested;
2221 	int search_start;
2222 
2223 	/*
2224 	 * create the contig pfn list if not already done
2225 	 */
2226 retry:
2227 	mutex_enter(&contig_list_lock);
2228 	if (contig_pfn_list == NULL) {
2229 		mutex_exit(&contig_list_lock);
2230 		if (!create_contig_pfnlist(flags)) {
2231 			return (NULL);
2232 		}
2233 		goto retry;
2234 	}
2235 	contig_searches++;
2236 	/*
2237 	 * Search contiguous pfn list for physically contiguous pages not in
2238 	 * the io_pool.  Start the search where the last search left off.
2239 	 */
2240 	pages_requested = pages_needed = npages;
2241 	search_start = next_alloc_pfn;
2242 	start_mfn = prev_mfn = 0;
2243 	while (pages_needed) {
2244 		pfn = contig_pfn_list[next_alloc_pfn];
2245 		mfn = pfn_to_mfn(pfn);
2246 		/*
2247 		 * Check if mfn is first one or contig to previous one and
2248 		 * if page corresponding to mfn is free and that mfn
2249 		 * range is not crossing a segment boundary.
2250 		 */
2251 		if ((prev_mfn == 0 || mfn == prev_mfn + 1) &&
2252 		    (pp = page_numtopp_alloc(pfn)) != NULL &&
2253 		    !((mfn & pfnseg) < (start_mfn & pfnseg))) {
2254 			PP_CLRFREE(pp);
2255 			page_io_pool_add(&plist, pp);
2256 			pages_needed--;
2257 			if (prev_mfn == 0)
2258 				start_mfn = mfn;
2259 			prev_mfn = mfn;
2260 		} else {
2261 			contig_search_restarts++;
2262 			/*
2263 			 * free partial page list
2264 			 */
2265 			while (plist != NULL) {
2266 				pp = plist;
2267 				page_io_pool_sub(&plist, pp, pp);
2268 				page_free(pp, 1);
2269 			}
2270 			pages_needed = pages_requested;
2271 			start_mfn = prev_mfn = 0;
2272 		}
2273 		if (++next_alloc_pfn == contig_pfn_cnt)
2274 			next_alloc_pfn = 0;
2275 		if (next_alloc_pfn == search_start)
2276 			break; /* all pfns searched */
2277 	}
2278 	mutex_exit(&contig_list_lock);
2279 	if (pages_needed) {
2280 		contig_search_failed++;
2281 		/*
2282 		 * Failed to find enough contig pages.
2283 		 * free partial page list
2284 		 */
2285 		while (plist != NULL) {
2286 			pp = plist;
2287 			page_io_pool_sub(&plist, pp, pp);
2288 			page_free(pp, 1);
2289 		}
2290 	}
2291 	return (plist);
2292 }
2293 
2294 /*
2295  * Search the reserved io pool pages for a page range with the
2296  * desired characteristics.
2297  */
2298 page_t *
2299 page_io_pool_alloc(ddi_dma_attr_t *mattr, int contig, pgcnt_t minctg)
2300 {
2301 	page_t *pp_first, *pp_last;
2302 	page_t *pp, **poolp;
2303 	pgcnt_t nwanted, pfnalign;
2304 	uint64_t pfnseg;
2305 	mfn_t mfn, tmfn, hi_mfn, lo_mfn;
2306 	int align, attempt = 0;
2307 
2308 	if (minctg == 1)
2309 		contig = 0;
2310 	lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
2311 	hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
2312 	pfnseg = mmu_btop(mattr->dma_attr_seg);
2313 	align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
2314 	if (align > MMU_PAGESIZE)
2315 		pfnalign = mmu_btop(align);
2316 	else
2317 		pfnalign = 0;
2318 
2319 try_again:
2320 	/*
2321 	 * See if we want pages for a legacy device
2322 	 */
2323 	if (hi_mfn < PFN_16MEG)
2324 		poolp = &io_pool_16m;
2325 	else
2326 		poolp = &io_pool_4g;
2327 try_smaller:
2328 	/*
2329 	 * Take pages from I/O pool. We'll use pages from the highest
2330 	 * MFN range possible.
2331 	 */
2332 	pp_first = pp_last = NULL;
2333 	mutex_enter(&io_pool_lock);
2334 	nwanted = minctg;
2335 	for (pp = *poolp; pp && nwanted > 0; ) {
2336 		pp = pp->p_prev;
2337 
2338 		/*
2339 		 * skip pages above allowable range
2340 		 */
2341 		mfn = mfn_list[pp->p_pagenum];
2342 		if (hi_mfn < mfn)
2343 			goto skip;
2344 
2345 		/*
2346 		 * stop at pages below allowable range
2347 		 */
2348 		if (lo_mfn > mfn)
2349 			break;
2350 restart:
2351 		if (pp_last == NULL) {
2352 			/*
2353 			 * Check alignment
2354 			 */
2355 			tmfn = mfn - (minctg - 1);
2356 			if (pfnalign && tmfn != P2ROUNDUP(tmfn, pfnalign))
2357 				goto skip; /* not properly aligned */
2358 			/*
2359 			 * Check segment
2360 			 */
2361 			if ((mfn & pfnseg) < (tmfn & pfnseg))
2362 				goto skip; /* crosses seg boundary */
2363 			/*
2364 			 * Start building page list
2365 			 */
2366 			pp_first = pp_last = pp;
2367 			nwanted--;
2368 		} else {
2369 			/*
2370 			 * check physical contiguity if required
2371 			 */
2372 			if (contig &&
2373 			    mfn_list[pp_first->p_pagenum] != mfn + 1) {
2374 				/*
2375 				 * not a contiguous page, restart list.
2376 				 */
2377 				pp_last = NULL;
2378 				nwanted = minctg;
2379 				goto restart;
2380 			} else { /* add page to list */
2381 				pp_first = pp;
2382 				nwanted--;
2383 			}
2384 		}
2385 skip:
2386 		if (pp == *poolp)
2387 			break;
2388 	}
2389 
2390 	/*
2391 	 * If we didn't find memory. Try the more constrained pool, then
2392 	 * sweep free pages into the DMA pool and try again.
2393 	 */
2394 	if (nwanted != 0) {
2395 		mutex_exit(&io_pool_lock);
2396 		/*
2397 		 * If we were looking in the less constrained pool and
2398 		 * didn't find pages, try the more constrained pool.
2399 		 */
2400 		if (poolp == &io_pool_4g) {
2401 			poolp = &io_pool_16m;
2402 			goto try_smaller;
2403 		}
2404 		kmem_reap();
2405 		if (++attempt < 4) {
2406 			/*
2407 			 * Grab some more io_pool pages
2408 			 */
2409 			(void) populate_io_pool();
2410 			goto try_again; /* go around and retry */
2411 		}
2412 		return (NULL);
2413 	}
2414 	/*
2415 	 * Found the pages, now snip them from the list
2416 	 */
2417 	page_io_pool_sub(poolp, pp_first, pp_last);
2418 	io_pool_cnt -= minctg;
2419 	/*
2420 	 * reset low water mark
2421 	 */
2422 	if (io_pool_cnt < io_pool_cnt_lowater)
2423 		io_pool_cnt_lowater = io_pool_cnt;
2424 	mutex_exit(&io_pool_lock);
2425 	return (pp_first);
2426 }
2427 
2428 page_t *
2429 page_swap_with_hypervisor(struct vnode *vp, u_offset_t off, caddr_t vaddr,
2430     ddi_dma_attr_t *mattr, uint_t flags, pgcnt_t minctg)
2431 {
2432 	uint_t kflags;
2433 	int order, extra, extpages, i, contig, nbits, extents;
2434 	page_t *pp, *expp, *pp_first, **pplist = NULL;
2435 	mfn_t *mfnlist = NULL;
2436 
2437 	contig = flags & PG_PHYSCONTIG;
2438 	if (minctg == 1)
2439 		contig = 0;
2440 	flags &= ~PG_PHYSCONTIG;
2441 	kflags = flags & PG_WAIT ? KM_SLEEP : KM_NOSLEEP;
2442 	/*
2443 	 * Hypervisor will allocate extents, if we want contig
2444 	 * pages extent must be >= minctg
2445 	 */
2446 	if (contig) {
2447 		order = highbit(minctg) - 1;
2448 		if (minctg & ((1 << order) - 1))
2449 			order++;
2450 		extpages = 1 << order;
2451 	} else {
2452 		order = 0;
2453 		extpages = minctg;
2454 	}
2455 	if (extpages > minctg) {
2456 		extra = extpages - minctg;
2457 		if (!page_resv(extra, kflags))
2458 			return (NULL);
2459 	}
2460 	pp_first = NULL;
2461 	pplist = kmem_alloc(extpages * sizeof (page_t *), kflags);
2462 	if (pplist == NULL)
2463 		goto balloon_fail;
2464 	mfnlist = kmem_alloc(extpages * sizeof (mfn_t), kflags);
2465 	if (mfnlist == NULL)
2466 		goto balloon_fail;
2467 	pp = page_create_va(vp, off, minctg * PAGESIZE, flags, &kvseg, vaddr);
2468 	if (pp == NULL)
2469 		goto balloon_fail;
2470 	pp_first = pp;
2471 	if (extpages > minctg) {
2472 		/*
2473 		 * fill out the rest of extent pages to swap
2474 		 * with the hypervisor
2475 		 */
2476 		for (i = 0; i < extra; i++) {
2477 			expp = page_create_va(vp,
2478 			    (u_offset_t)(uintptr_t)io_pool_kva,
2479 			    PAGESIZE, flags, &kvseg, io_pool_kva);
2480 			if (expp == NULL)
2481 				goto balloon_fail;
2482 			(void) hat_pageunload(expp, HAT_FORCE_PGUNLOAD);
2483 			page_io_unlock(expp);
2484 			page_hashout(expp, NULL);
2485 			page_io_lock(expp);
2486 			/*
2487 			 * add page to end of list
2488 			 */
2489 			expp->p_prev = pp_first->p_prev;
2490 			expp->p_next = pp_first;
2491 			expp->p_prev->p_next = expp;
2492 			pp_first->p_prev = expp;
2493 		}
2494 
2495 	}
2496 	for (i = 0; i < extpages; i++) {
2497 		pplist[i] = pp;
2498 		pp = pp->p_next;
2499 	}
2500 	nbits = highbit(mattr->dma_attr_addr_hi);
2501 	extents = contig ? 1 : minctg;
2502 	if (balloon_replace_pages(extents, pplist, nbits, order,
2503 	    mfnlist) != extents) {
2504 		if (ioalloc_dbg)
2505 			cmn_err(CE_NOTE, "request to hypervisor"
2506 			    " for %d pages, maxaddr %" PRIx64 " failed",
2507 			    extpages, mattr->dma_attr_addr_hi);
2508 		goto balloon_fail;
2509 	}
2510 
2511 	kmem_free(pplist, extpages * sizeof (page_t *));
2512 	kmem_free(mfnlist, extpages * sizeof (mfn_t));
2513 	/*
2514 	 * Return any excess pages to free list
2515 	 */
2516 	if (extpages > minctg) {
2517 		for (i = 0; i < extra; i++) {
2518 			pp = pp_first->p_prev;
2519 			page_sub(&pp_first, pp);
2520 			page_io_unlock(pp);
2521 			page_unresv(1);
2522 			page_free(pp, 1);
2523 		}
2524 	}
2525 	return (pp_first);
2526 balloon_fail:
2527 	/*
2528 	 * Return pages to free list and return failure
2529 	 */
2530 	while (pp_first != NULL) {
2531 		pp = pp_first;
2532 		page_sub(&pp_first, pp);
2533 		page_io_unlock(pp);
2534 		if (pp->p_vnode != NULL)
2535 			page_hashout(pp, NULL);
2536 		page_free(pp, 1);
2537 	}
2538 	if (pplist)
2539 		kmem_free(pplist, extpages * sizeof (page_t *));
2540 	if (mfnlist)
2541 		kmem_free(mfnlist, extpages * sizeof (mfn_t));
2542 	page_unresv(extpages - minctg);
2543 	return (NULL);
2544 }
2545 
2546 static void
2547 return_partial_alloc(page_t *plist)
2548 {
2549 	page_t *pp;
2550 
2551 	while (plist != NULL) {
2552 		pp = plist;
2553 		page_sub(&plist, pp);
2554 		page_io_unlock(pp);
2555 		page_destroy_io(pp);
2556 	}
2557 }
2558 
2559 static page_t *
2560 page_get_contigpages(
2561 	struct vnode	*vp,
2562 	u_offset_t	off,
2563 	int		*npagesp,
2564 	uint_t		flags,
2565 	caddr_t		vaddr,
2566 	ddi_dma_attr_t	*mattr)
2567 {
2568 	mfn_t	max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL);
2569 	page_t	*plist;	/* list to return */
2570 	page_t	*pp, *mcpl;
2571 	int	contig, anyaddr, npages, getone = 0;
2572 	mfn_t	lo_mfn;
2573 	mfn_t	hi_mfn;
2574 	pgcnt_t	pfnalign = 0;
2575 	int	align, sgllen;
2576 	uint64_t pfnseg;
2577 	pgcnt_t	minctg;
2578 
2579 	npages = *npagesp;
2580 	ASSERT(mattr != NULL);
2581 	lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
2582 	hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
2583 	sgllen = mattr->dma_attr_sgllen;
2584 	pfnseg = mmu_btop(mattr->dma_attr_seg);
2585 	align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
2586 	if (align > MMU_PAGESIZE)
2587 		pfnalign = mmu_btop(align);
2588 
2589 	/*
2590 	 * Clear the contig flag if only one page is needed.
2591 	 */
2592 	contig = flags & PG_PHYSCONTIG;
2593 	if (npages == 1) {
2594 		getone = 1;
2595 		contig = 0;
2596 	}
2597 
2598 	/*
2599 	 * Check if any page in the system is fine.
2600 	 */
2601 	anyaddr = lo_mfn == 0 && hi_mfn >= max_mfn && !pfnalign;
2602 	if (!contig && anyaddr) {
2603 		flags &= ~PG_PHYSCONTIG;
2604 		plist = page_create_va(vp, off, npages * MMU_PAGESIZE,
2605 		    flags, &kvseg, vaddr);
2606 		if (plist != NULL) {
2607 			*npagesp = 0;
2608 			return (plist);
2609 		}
2610 	}
2611 	plist = NULL;
2612 	minctg = howmany(npages, sgllen);
2613 	while (npages > sgllen || getone) {
2614 		if (minctg > npages)
2615 			minctg = npages;
2616 		mcpl = NULL;
2617 		/*
2618 		 * We could just want unconstrained but contig pages.
2619 		 */
2620 		if (anyaddr && contig) {
2621 			/*
2622 			 * Look for free contig pages to satisfy the request.
2623 			 */
2624 			mcpl = find_contig_free(minctg, flags, pfnseg);
2625 		}
2626 		/*
2627 		 * Try the reserved io pools next
2628 		 */
2629 		if (mcpl == NULL)
2630 			mcpl = page_io_pool_alloc(mattr, contig, minctg);
2631 		if (mcpl != NULL) {
2632 			pp = mcpl;
2633 			do {
2634 				if (!page_hashin(pp, vp, off, NULL)) {
2635 					panic("page_get_contigpages:"
2636 					    " hashin failed"
2637 					    " pp %p, vp %p, off %llx",
2638 					    (void *)pp, (void *)vp, off);
2639 				}
2640 				off += MMU_PAGESIZE;
2641 				PP_CLRFREE(pp);
2642 				PP_CLRAGED(pp);
2643 				page_set_props(pp, P_REF);
2644 				page_io_lock(pp);
2645 				pp = pp->p_next;
2646 			} while (pp != mcpl);
2647 		} else {
2648 			/*
2649 			 * Hypervisor exchange doesn't handle segment or
2650 			 * alignment constraints
2651 			 */
2652 			if (mattr->dma_attr_seg < mattr->dma_attr_addr_hi ||
2653 			    pfnalign)
2654 				goto fail;
2655 			/*
2656 			 * Try exchanging pages with the hypervisor
2657 			 */
2658 			mcpl = page_swap_with_hypervisor(vp, off, vaddr, mattr,
2659 			    flags, minctg);
2660 			if (mcpl == NULL)
2661 				goto fail;
2662 			off += minctg * MMU_PAGESIZE;
2663 		}
2664 		check_dma(mattr, mcpl, minctg);
2665 		/*
2666 		 * Here with a minctg run of contiguous pages, add them to the
2667 		 * list we will return for this request.
2668 		 */
2669 		page_list_concat(&plist, &mcpl);
2670 		npages -= minctg;
2671 		*npagesp = npages;
2672 		sgllen--;
2673 		if (getone)
2674 			break;
2675 	}
2676 	return (plist);
2677 fail:
2678 	return_partial_alloc(plist);
2679 	return (NULL);
2680 }
2681 
2682 /*
2683  * Allocator for domain 0 I/O pages. We match the required
2684  * DMA attributes and contiguity constraints.
2685  */
2686 /*ARGSUSED*/
2687 page_t *
2688 page_create_io(
2689 	struct vnode	*vp,
2690 	u_offset_t	off,
2691 	uint_t		bytes,
2692 	uint_t		flags,
2693 	struct as	*as,
2694 	caddr_t		vaddr,
2695 	ddi_dma_attr_t	*mattr)
2696 {
2697 	page_t	*plist = NULL, *pp;
2698 	int	npages = 0, contig, anyaddr, pages_req;
2699 	mfn_t	lo_mfn;
2700 	mfn_t	hi_mfn;
2701 	pgcnt_t	pfnalign = 0;
2702 	int	align;
2703 	int	is_domu = 0;
2704 	int	dummy, bytes_got;
2705 	mfn_t	max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL);
2706 
2707 	ASSERT(mattr != NULL);
2708 	lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
2709 	hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
2710 	align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
2711 	if (align > MMU_PAGESIZE)
2712 		pfnalign = mmu_btop(align);
2713 
2714 	/*
2715 	 * Clear the contig flag if only one page is needed or the scatter
2716 	 * gather list length is >= npages.
2717 	 */
2718 	pages_req = npages = mmu_btopr(bytes);
2719 	contig = (flags & PG_PHYSCONTIG);
2720 	bytes = P2ROUNDUP(bytes, MMU_PAGESIZE);
2721 	if (bytes == MMU_PAGESIZE || mattr->dma_attr_sgllen >= npages)
2722 		contig = 0;
2723 
2724 	/*
2725 	 * Check if any old page in the system is fine.
2726 	 * DomU should always go down this path.
2727 	 */
2728 	is_domu = !DOMAIN_IS_INITDOMAIN(xen_info);
2729 	anyaddr = lo_mfn == 0 && hi_mfn >= max_mfn && !pfnalign;
2730 	if ((!contig && anyaddr) || is_domu) {
2731 		flags &= ~PG_PHYSCONTIG;
2732 		plist = page_create_va(vp, off, bytes, flags, &kvseg, vaddr);
2733 		if (plist != NULL)
2734 			return (plist);
2735 		else if (is_domu)
2736 			return (NULL); /* no memory available */
2737 	}
2738 	/*
2739 	 * DomU should never reach here
2740 	 */
2741 	if (contig) {
2742 		plist = page_get_contigpages(vp, off, &npages, flags, vaddr,
2743 		    mattr);
2744 		if (plist == NULL)
2745 			goto fail;
2746 		bytes_got = (pages_req - npages) << MMU_PAGESHIFT;
2747 		vaddr += bytes_got;
2748 		off += bytes_got;
2749 		/*
2750 		 * We now have all the contiguous pages we need, but
2751 		 * we may still need additional non-contiguous pages.
2752 		 */
2753 	}
2754 	/*
2755 	 * now loop collecting the requested number of pages, these do
2756 	 * not have to be contiguous pages but we will use the contig
2757 	 * page alloc code to get the pages since it will honor any
2758 	 * other constraints the pages may have.
2759 	 */
2760 	while (npages--) {
2761 		dummy = 1;
2762 		pp = page_get_contigpages(vp, off, &dummy, flags, vaddr, mattr);
2763 		if (pp == NULL)
2764 			goto fail;
2765 		page_add(&plist, pp);
2766 		vaddr += MMU_PAGESIZE;
2767 		off += MMU_PAGESIZE;
2768 	}
2769 	return (plist);
2770 fail:
2771 	/*
2772 	 * Failed to get enough pages, return ones we did get
2773 	 */
2774 	return_partial_alloc(plist);
2775 	return (NULL);
2776 }
2777 
2778 /*
2779  * Lock and return the page with the highest mfn that we can find.  last_mfn
2780  * holds the last one found, so the next search can start from there.  We
2781  * also keep a counter so that we don't loop forever if the machine has no
2782  * free pages.
2783  *
2784  * This is called from the balloon thread to find pages to give away.  new_high
2785  * is used when new mfn's have been added to the system - we will reset our
2786  * search if the new mfn's are higher than our current search position.
2787  */
2788 page_t *
2789 page_get_high_mfn(mfn_t new_high)
2790 {
2791 	static mfn_t last_mfn = 0;
2792 	pfn_t pfn;
2793 	page_t *pp;
2794 	ulong_t loop_count = 0;
2795 
2796 	if (new_high > last_mfn)
2797 		last_mfn = new_high;
2798 
2799 	for (; loop_count < mfn_count; loop_count++, last_mfn--) {
2800 		if (last_mfn == 0) {
2801 			last_mfn = cached_max_mfn;
2802 		}
2803 
2804 		pfn = mfn_to_pfn(last_mfn);
2805 		if (pfn & PFN_IS_FOREIGN_MFN)
2806 			continue;
2807 
2808 		/* See if the page is free.  If so, lock it. */
2809 		pp = page_numtopp_alloc(pfn);
2810 		if (pp == NULL)
2811 			continue;
2812 		PP_CLRFREE(pp);
2813 
2814 		ASSERT(PAGE_EXCL(pp));
2815 		ASSERT(pp->p_vnode == NULL);
2816 		ASSERT(!hat_page_is_mapped(pp));
2817 		last_mfn--;
2818 		return (pp);
2819 	}
2820 	return (NULL);
2821 }
2822 
2823 #else /* !__xpv */
2824 
2825 /*
2826  * get a page from any list with the given mnode
2827  */
2828 static page_t *
2829 page_get_mnode_anylist(ulong_t origbin, uchar_t szc, uint_t flags,
2830     int mnode, int mtype, ddi_dma_attr_t *dma_attr)
2831 {
2832 	kmutex_t		*pcm;
2833 	int			i;
2834 	page_t			*pp;
2835 	page_t			*first_pp;
2836 	uint64_t		pgaddr;
2837 	ulong_t			bin;
2838 	int			mtypestart;
2839 	int			plw_initialized;
2840 	page_list_walker_t	plw;
2841 
2842 	VM_STAT_ADD(pga_vmstats.pgma_alloc);
2843 
2844 	ASSERT((flags & PG_MATCH_COLOR) == 0);
2845 	ASSERT(szc == 0);
2846 	ASSERT(dma_attr != NULL);
2847 
2848 	MTYPE_START(mnode, mtype, flags);
2849 	if (mtype < 0) {
2850 		VM_STAT_ADD(pga_vmstats.pgma_allocempty);
2851 		return (NULL);
2852 	}
2853 
2854 	mtypestart = mtype;
2855 
2856 	bin = origbin;
2857 
2858 	/*
2859 	 * check up to page_colors + 1 bins - origbin may be checked twice
2860 	 * because of BIN_STEP skip
2861 	 */
2862 	do {
2863 		plw_initialized = 0;
2864 
2865 		for (plw.plw_count = 0;
2866 		    plw.plw_count < page_colors; plw.plw_count++) {
2867 
2868 			if (PAGE_FREELISTS(mnode, szc, bin, mtype) == NULL)
2869 				goto nextfreebin;
2870 
2871 			pcm = PC_BIN_MUTEX(mnode, bin, PG_FREE_LIST);
2872 			mutex_enter(pcm);
2873 			pp = PAGE_FREELISTS(mnode, szc, bin, mtype);
2874 			first_pp = pp;
2875 			while (pp != NULL) {
2876 				if (page_trylock(pp, SE_EXCL) == 0) {
2877 					pp = pp->p_next;
2878 					if (pp == first_pp) {
2879 						pp = NULL;
2880 					}
2881 					continue;
2882 				}
2883 
2884 				ASSERT(PP_ISFREE(pp));
2885 				ASSERT(PP_ISAGED(pp));
2886 				ASSERT(pp->p_vnode == NULL);
2887 				ASSERT(pp->p_hash == NULL);
2888 				ASSERT(pp->p_offset == (u_offset_t)-1);
2889 				ASSERT(pp->p_szc == szc);
2890 				ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode);
2891 				/* check if page within DMA attributes */
2892 				pgaddr = pa_to_ma(pfn_to_pa(pp->p_pagenum));
2893 				if ((pgaddr >= dma_attr->dma_attr_addr_lo) &&
2894 				    (pgaddr + MMU_PAGESIZE - 1 <=
2895 				    dma_attr->dma_attr_addr_hi)) {
2896 					break;
2897 				}
2898 
2899 				/* continue looking */
2900 				page_unlock(pp);
2901 				pp = pp->p_next;
2902 				if (pp == first_pp)
2903 					pp = NULL;
2904 
2905 			}
2906 			if (pp != NULL) {
2907 				ASSERT(mtype == PP_2_MTYPE(pp));
2908 				ASSERT(pp->p_szc == 0);
2909 
2910 				/* found a page with specified DMA attributes */
2911 				page_sub(&PAGE_FREELISTS(mnode, szc, bin,
2912 				    mtype), pp);
2913 				page_ctr_sub(mnode, mtype, pp, PG_FREE_LIST);
2914 
2915 				if ((PP_ISFREE(pp) == 0) ||
2916 				    (PP_ISAGED(pp) == 0)) {
2917 					cmn_err(CE_PANIC, "page %p is not free",
2918 					    (void *)pp);
2919 				}
2920 
2921 				mutex_exit(pcm);
2922 				check_dma(dma_attr, pp, 1);
2923 				VM_STAT_ADD(pga_vmstats.pgma_allocok);
2924 				return (pp);
2925 			}
2926 			mutex_exit(pcm);
2927 nextfreebin:
2928 			if (plw_initialized == 0) {
2929 				page_list_walk_init(szc, 0, bin, 1, 0, &plw);
2930 				ASSERT(plw.plw_ceq_dif == page_colors);
2931 				plw_initialized = 1;
2932 			}
2933 
2934 			if (plw.plw_do_split) {
2935 				pp = page_freelist_split(szc, bin, mnode,
2936 				    mtype,
2937 				    mmu_btop(dma_attr->dma_attr_addr_lo),
2938 				    mmu_btop(dma_attr->dma_attr_addr_hi + 1),
2939 				    &plw);
2940 				if (pp != NULL) {
2941 					check_dma(dma_attr, pp, 1);
2942 					return (pp);
2943 				}
2944 			}
2945 
2946 			bin = page_list_walk_next_bin(szc, bin, &plw);
2947 		}
2948 
2949 		MTYPE_NEXT(mnode, mtype, flags);
2950 	} while (mtype >= 0);
2951 
2952 	/* failed to find a page in the freelist; try it in the cachelist */
2953 
2954 	/* reset mtype start for cachelist search */
2955 	mtype = mtypestart;
2956 	ASSERT(mtype >= 0);
2957 
2958 	/* start with the bin of matching color */
2959 	bin = origbin;
2960 
2961 	do {
2962 		for (i = 0; i <= page_colors; i++) {
2963 			if (PAGE_CACHELISTS(mnode, bin, mtype) == NULL)
2964 				goto nextcachebin;
2965 			pcm = PC_BIN_MUTEX(mnode, bin, PG_CACHE_LIST);
2966 			mutex_enter(pcm);
2967 			pp = PAGE_CACHELISTS(mnode, bin, mtype);
2968 			first_pp = pp;
2969 			while (pp != NULL) {
2970 				if (page_trylock(pp, SE_EXCL) == 0) {
2971 					pp = pp->p_next;
2972 					if (pp == first_pp)
2973 						break;
2974 					continue;
2975 				}
2976 				ASSERT(pp->p_vnode);
2977 				ASSERT(PP_ISAGED(pp) == 0);
2978 				ASSERT(pp->p_szc == 0);
2979 				ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode);
2980 
2981 				/* check if page within DMA attributes */
2982 
2983 				pgaddr = pa_to_ma(pfn_to_pa(pp->p_pagenum));
2984 				if ((pgaddr >= dma_attr->dma_attr_addr_lo) &&
2985 				    (pgaddr + MMU_PAGESIZE - 1 <=
2986 				    dma_attr->dma_attr_addr_hi)) {
2987 					break;
2988 				}
2989 
2990 				/* continue looking */
2991 				page_unlock(pp);
2992 				pp = pp->p_next;
2993 				if (pp == first_pp)
2994 					pp = NULL;
2995 			}
2996 
2997 			if (pp != NULL) {
2998 				ASSERT(mtype == PP_2_MTYPE(pp));
2999 				ASSERT(pp->p_szc == 0);
3000 
3001 				/* found a page with specified DMA attributes */
3002 				page_sub(&PAGE_CACHELISTS(mnode, bin,
3003 				    mtype), pp);
3004 				page_ctr_sub(mnode, mtype, pp, PG_CACHE_LIST);
3005 
3006 				mutex_exit(pcm);
3007 				ASSERT(pp->p_vnode);
3008 				ASSERT(PP_ISAGED(pp) == 0);
3009 				check_dma(dma_attr, pp, 1);
3010 				VM_STAT_ADD(pga_vmstats.pgma_allocok);
3011 				return (pp);
3012 			}
3013 			mutex_exit(pcm);
3014 nextcachebin:
3015 			bin += (i == 0) ? BIN_STEP : 1;
3016 			bin &= page_colors_mask;
3017 		}
3018 		MTYPE_NEXT(mnode, mtype, flags);
3019 	} while (mtype >= 0);
3020 
3021 	VM_STAT_ADD(pga_vmstats.pgma_allocfailed);
3022 	return (NULL);
3023 }
3024 
3025 /*
3026  * This function is similar to page_get_freelist()/page_get_cachelist()
3027  * but it searches both the lists to find a page with the specified
3028  * color (or no color) and DMA attributes. The search is done in the
3029  * freelist first and then in the cache list within the highest memory
3030  * range (based on DMA attributes) before searching in the lower
3031  * memory ranges.
3032  *
3033  * Note: This function is called only by page_create_io().
3034  */
3035 /*ARGSUSED*/
3036 static page_t *
3037 page_get_anylist(struct vnode *vp, u_offset_t off, struct as *as, caddr_t vaddr,
3038     size_t size, uint_t flags, ddi_dma_attr_t *dma_attr, lgrp_t	*lgrp)
3039 {
3040 	uint_t		bin;
3041 	int		mtype;
3042 	page_t		*pp;
3043 	int		n;
3044 	int		m;
3045 	int		szc;
3046 	int		fullrange;
3047 	int		mnode;
3048 	int		local_failed_stat = 0;
3049 	lgrp_mnode_cookie_t	lgrp_cookie;
3050 
3051 	VM_STAT_ADD(pga_vmstats.pga_alloc);
3052 
3053 	/* only base pagesize currently supported */
3054 	if (size != MMU_PAGESIZE)
3055 		return (NULL);
3056 
3057 	/*
3058 	 * If we're passed a specific lgroup, we use it.  Otherwise,
3059 	 * assume first-touch placement is desired.
3060 	 */
3061 	if (!LGRP_EXISTS(lgrp))
3062 		lgrp = lgrp_home_lgrp();
3063 
3064 	/* LINTED */
3065 	AS_2_BIN(as, seg, vp, vaddr, bin, 0);
3066 
3067 	/*
3068 	 * Only hold one freelist or cachelist lock at a time, that way we
3069 	 * can start anywhere and not have to worry about lock
3070 	 * ordering.
3071 	 */
3072 	if (dma_attr == NULL) {
3073 		n = 0;
3074 		m = mnoderangecnt - 1;
3075 		fullrange = 1;
3076 		VM_STAT_ADD(pga_vmstats.pga_nulldmaattr);
3077 	} else {
3078 		pfn_t pfnlo = mmu_btop(dma_attr->dma_attr_addr_lo);
3079 		pfn_t pfnhi = mmu_btop(dma_attr->dma_attr_addr_hi);
3080 
3081 		/*
3082 		 * We can guarantee alignment only for page boundary.
3083 		 */
3084 		if (dma_attr->dma_attr_align > MMU_PAGESIZE)
3085 			return (NULL);
3086 
3087 		n = pfn_2_mtype(pfnlo);
3088 		m = pfn_2_mtype(pfnhi);
3089 
3090 		fullrange = ((pfnlo == mnoderanges[n].mnr_pfnlo) &&
3091 		    (pfnhi >= mnoderanges[m].mnr_pfnhi));
3092 	}
3093 	VM_STAT_COND_ADD(fullrange == 0, pga_vmstats.pga_notfullrange);
3094 
3095 	if (n > m)
3096 		return (NULL);
3097 
3098 	szc = 0;
3099 
3100 	/* cylcing thru mtype handled by RANGE0 if n == 0 */
3101 	if (n == 0) {
3102 		flags |= PGI_MT_RANGE0;
3103 		n = m;
3104 	}
3105 
3106 	/*
3107 	 * Try local memory node first, but try remote if we can't
3108 	 * get a page of the right color.
3109 	 */
3110 	LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, LGRP_SRCH_HIER);
3111 	while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) {
3112 		/*
3113 		 * allocate pages from high pfn to low.
3114 		 */
3115 		for (mtype = m; mtype >= n; mtype--) {
3116 			if (fullrange != 0) {
3117 				pp = page_get_mnode_freelist(mnode,
3118 				    bin, mtype, szc, flags);
3119 				if (pp == NULL) {
3120 					pp = page_get_mnode_cachelist(
3121 					    bin, flags, mnode, mtype);
3122 				}
3123 			} else {
3124 				pp = page_get_mnode_anylist(bin, szc,
3125 				    flags, mnode, mtype, dma_attr);
3126 			}
3127 			if (pp != NULL) {
3128 				VM_STAT_ADD(pga_vmstats.pga_allocok);
3129 				check_dma(dma_attr, pp, 1);
3130 				return (pp);
3131 			}
3132 		}
3133 		if (!local_failed_stat) {
3134 			lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_ALLOC_FAIL, 1);
3135 			local_failed_stat = 1;
3136 		}
3137 	}
3138 	VM_STAT_ADD(pga_vmstats.pga_allocfailed);
3139 
3140 	return (NULL);
3141 }
3142 
3143 /*
3144  * page_create_io()
3145  *
3146  * This function is a copy of page_create_va() with an additional
3147  * argument 'mattr' that specifies DMA memory requirements to
3148  * the page list functions. This function is used by the segkmem
3149  * allocator so it is only to create new pages (i.e PG_EXCL is
3150  * set).
3151  *
3152  * Note: This interface is currently used by x86 PSM only and is
3153  *	 not fully specified so the commitment level is only for
3154  *	 private interface specific to x86. This interface uses PSM
3155  *	 specific page_get_anylist() interface.
3156  */
3157 
3158 #define	PAGE_HASH_SEARCH(index, pp, vp, off) { \
3159 	for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash) { \
3160 		if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \
3161 			break; \
3162 	} \
3163 }
3164 
3165 
3166 page_t *
3167 page_create_io(
3168 	struct vnode	*vp,
3169 	u_offset_t	off,
3170 	uint_t		bytes,
3171 	uint_t		flags,
3172 	struct as	*as,
3173 	caddr_t		vaddr,
3174 	ddi_dma_attr_t	*mattr)	/* DMA memory attributes if any */
3175 {
3176 	page_t		*plist = NULL;
3177 	uint_t		plist_len = 0;
3178 	pgcnt_t		npages;
3179 	page_t		*npp = NULL;
3180 	uint_t		pages_req;
3181 	page_t		*pp;
3182 	kmutex_t	*phm = NULL;
3183 	uint_t		index;
3184 
3185 	TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START,
3186 	    "page_create_start:vp %p off %llx bytes %u flags %x",
3187 	    vp, off, bytes, flags);
3188 
3189 	ASSERT((flags & ~(PG_EXCL | PG_WAIT | PG_PHYSCONTIG)) == 0);
3190 
3191 	pages_req = npages = mmu_btopr(bytes);
3192 
3193 	/*
3194 	 * Do the freemem and pcf accounting.
3195 	 */
3196 	if (!page_create_wait(npages, flags)) {
3197 		return (NULL);
3198 	}
3199 
3200 	TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS,
3201 	    "page_create_success:vp %p off %llx", vp, off);
3202 
3203 	/*
3204 	 * If satisfying this request has left us with too little
3205 	 * memory, start the wheels turning to get some back.  The
3206 	 * first clause of the test prevents waking up the pageout
3207 	 * daemon in situations where it would decide that there's
3208 	 * nothing to do.
3209 	 */
3210 	if (nscan < desscan && freemem < minfree) {
3211 		TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL,
3212 		    "pageout_cv_signal:freemem %ld", freemem);
3213 		cv_signal(&proc_pageout->p_cv);
3214 	}
3215 
3216 	if (flags & PG_PHYSCONTIG) {
3217 
3218 		plist = page_get_contigpage(&npages, mattr, 1);
3219 		if (plist == NULL) {
3220 			page_create_putback(npages);
3221 			return (NULL);
3222 		}
3223 
3224 		pp = plist;
3225 
3226 		do {
3227 			if (!page_hashin(pp, vp, off, NULL)) {
3228 				panic("pg_creat_io: hashin failed %p %p %llx",
3229 				    (void *)pp, (void *)vp, off);
3230 			}
3231 			VM_STAT_ADD(page_create_new);
3232 			off += MMU_PAGESIZE;
3233 			PP_CLRFREE(pp);
3234 			PP_CLRAGED(pp);
3235 			page_set_props(pp, P_REF);
3236 			pp = pp->p_next;
3237 		} while (pp != plist);
3238 
3239 		if (!npages) {
3240 			check_dma(mattr, plist, pages_req);
3241 			return (plist);
3242 		} else {
3243 			vaddr += (pages_req - npages) << MMU_PAGESHIFT;
3244 		}
3245 
3246 		/*
3247 		 * fall-thru:
3248 		 *
3249 		 * page_get_contigpage returns when npages <= sgllen.
3250 		 * Grab the rest of the non-contig pages below from anylist.
3251 		 */
3252 	}
3253 
3254 	/*
3255 	 * Loop around collecting the requested number of pages.
3256 	 * Most of the time, we have to `create' a new page. With
3257 	 * this in mind, pull the page off the free list before
3258 	 * getting the hash lock.  This will minimize the hash
3259 	 * lock hold time, nesting, and the like.  If it turns
3260 	 * out we don't need the page, we put it back at the end.
3261 	 */
3262 	while (npages--) {
3263 		phm = NULL;
3264 
3265 		index = PAGE_HASH_FUNC(vp, off);
3266 top:
3267 		ASSERT(phm == NULL);
3268 		ASSERT(index == PAGE_HASH_FUNC(vp, off));
3269 		ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
3270 
3271 		if (npp == NULL) {
3272 			/*
3273 			 * Try to get the page of any color either from
3274 			 * the freelist or from the cache list.
3275 			 */
3276 			npp = page_get_anylist(vp, off, as, vaddr, MMU_PAGESIZE,
3277 			    flags & ~PG_MATCH_COLOR, mattr, NULL);
3278 			if (npp == NULL) {
3279 				if (mattr == NULL) {
3280 					/*
3281 					 * Not looking for a special page;
3282 					 * panic!
3283 					 */
3284 					panic("no page found %d", (int)npages);
3285 				}
3286 				/*
3287 				 * No page found! This can happen
3288 				 * if we are looking for a page
3289 				 * within a specific memory range
3290 				 * for DMA purposes. If PG_WAIT is
3291 				 * specified then we wait for a
3292 				 * while and then try again. The
3293 				 * wait could be forever if we
3294 				 * don't get the page(s) we need.
3295 				 *
3296 				 * Note: XXX We really need a mechanism
3297 				 * to wait for pages in the desired
3298 				 * range. For now, we wait for any
3299 				 * pages and see if we can use it.
3300 				 */
3301 
3302 				if ((mattr != NULL) && (flags & PG_WAIT)) {
3303 					delay(10);
3304 					goto top;
3305 				}
3306 				goto fail; /* undo accounting stuff */
3307 			}
3308 
3309 			if (PP_ISAGED(npp) == 0) {
3310 				/*
3311 				 * Since this page came from the
3312 				 * cachelist, we must destroy the
3313 				 * old vnode association.
3314 				 */
3315 				page_hashout(npp, (kmutex_t *)NULL);
3316 			}
3317 		}
3318 
3319 		/*
3320 		 * We own this page!
3321 		 */
3322 		ASSERT(PAGE_EXCL(npp));
3323 		ASSERT(npp->p_vnode == NULL);
3324 		ASSERT(!hat_page_is_mapped(npp));
3325 		PP_CLRFREE(npp);
3326 		PP_CLRAGED(npp);
3327 
3328 		/*
3329 		 * Here we have a page in our hot little mits and are
3330 		 * just waiting to stuff it on the appropriate lists.
3331 		 * Get the mutex and check to see if it really does
3332 		 * not exist.
3333 		 */
3334 		phm = PAGE_HASH_MUTEX(index);
3335 		mutex_enter(phm);
3336 		PAGE_HASH_SEARCH(index, pp, vp, off);
3337 		if (pp == NULL) {
3338 			VM_STAT_ADD(page_create_new);
3339 			pp = npp;
3340 			npp = NULL;
3341 			if (!page_hashin(pp, vp, off, phm)) {
3342 				/*
3343 				 * Since we hold the page hash mutex and
3344 				 * just searched for this page, page_hashin
3345 				 * had better not fail.  If it does, that
3346 				 * means somethread did not follow the
3347 				 * page hash mutex rules.  Panic now and
3348 				 * get it over with.  As usual, go down
3349 				 * holding all the locks.
3350 				 */
3351 				ASSERT(MUTEX_HELD(phm));
3352 				panic("page_create: hashin fail %p %p %llx %p",
3353 				    (void *)pp, (void *)vp, off, (void *)phm);
3354 
3355 			}
3356 			ASSERT(MUTEX_HELD(phm));
3357 			mutex_exit(phm);
3358 			phm = NULL;
3359 
3360 			/*
3361 			 * Hat layer locking need not be done to set
3362 			 * the following bits since the page is not hashed
3363 			 * and was on the free list (i.e., had no mappings).
3364 			 *
3365 			 * Set the reference bit to protect
3366 			 * against immediate pageout
3367 			 *
3368 			 * XXXmh modify freelist code to set reference
3369 			 * bit so we don't have to do it here.
3370 			 */
3371 			page_set_props(pp, P_REF);
3372 		} else {
3373 			ASSERT(MUTEX_HELD(phm));
3374 			mutex_exit(phm);
3375 			phm = NULL;
3376 			/*
3377 			 * NOTE: This should not happen for pages associated
3378 			 *	 with kernel vnode 'kvp'.
3379 			 */
3380 			/* XX64 - to debug why this happens! */
3381 			ASSERT(!VN_ISKAS(vp));
3382 			if (VN_ISKAS(vp))
3383 				cmn_err(CE_NOTE,
3384 				    "page_create: page not expected "
3385 				    "in hash list for kernel vnode - pp 0x%p",
3386 				    (void *)pp);
3387 			VM_STAT_ADD(page_create_exists);
3388 			goto fail;
3389 		}
3390 
3391 		/*
3392 		 * Got a page!  It is locked.  Acquire the i/o
3393 		 * lock since we are going to use the p_next and
3394 		 * p_prev fields to link the requested pages together.
3395 		 */
3396 		page_io_lock(pp);
3397 		page_add(&plist, pp);
3398 		plist = plist->p_next;
3399 		off += MMU_PAGESIZE;
3400 		vaddr += MMU_PAGESIZE;
3401 	}
3402 
3403 	check_dma(mattr, plist, pages_req);
3404 	return (plist);
3405 
3406 fail:
3407 	if (npp != NULL) {
3408 		/*
3409 		 * Did not need this page after all.
3410 		 * Put it back on the free list.
3411 		 */
3412 		VM_STAT_ADD(page_create_putbacks);
3413 		PP_SETFREE(npp);
3414 		PP_SETAGED(npp);
3415 		npp->p_offset = (u_offset_t)-1;
3416 		page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL);
3417 		page_unlock(npp);
3418 	}
3419 
3420 	/*
3421 	 * Give up the pages we already got.
3422 	 */
3423 	while (plist != NULL) {
3424 		pp = plist;
3425 		page_sub(&plist, pp);
3426 		page_io_unlock(pp);
3427 		plist_len++;
3428 		/*LINTED: constant in conditional ctx*/
3429 		VN_DISPOSE(pp, B_INVAL, 0, kcred);
3430 	}
3431 
3432 	/*
3433 	 * VN_DISPOSE does freemem accounting for the pages in plist
3434 	 * by calling page_free. So, we need to undo the pcf accounting
3435 	 * for only the remaining pages.
3436 	 */
3437 	VM_STAT_ADD(page_create_putbacks);
3438 	page_create_putback(pages_req - plist_len);
3439 
3440 	return (NULL);
3441 }
3442 #endif /* !__xpv */
3443 
3444 
3445 /*
3446  * Copy the data from the physical page represented by "frompp" to
3447  * that represented by "topp". ppcopy uses CPU->cpu_caddr1 and
3448  * CPU->cpu_caddr2.  It assumes that no one uses either map at interrupt
3449  * level and no one sleeps with an active mapping there.
3450  *
3451  * Note that the ref/mod bits in the page_t's are not affected by
3452  * this operation, hence it is up to the caller to update them appropriately.
3453  */
3454 int
3455 ppcopy(page_t *frompp, page_t *topp)
3456 {
3457 	caddr_t		pp_addr1;
3458 	caddr_t		pp_addr2;
3459 	hat_mempte_t	pte1;
3460 	hat_mempte_t	pte2;
3461 	kmutex_t	*ppaddr_mutex;
3462 	label_t		ljb;
3463 	int		ret = 1;
3464 
3465 	ASSERT_STACK_ALIGNED();
3466 	ASSERT(PAGE_LOCKED(frompp));
3467 	ASSERT(PAGE_LOCKED(topp));
3468 
3469 	if (kpm_enable) {
3470 		pp_addr1 = hat_kpm_page2va(frompp, 0);
3471 		pp_addr2 = hat_kpm_page2va(topp, 0);
3472 		kpreempt_disable();
3473 	} else {
3474 		/*
3475 		 * disable pre-emption so that CPU can't change
3476 		 */
3477 		kpreempt_disable();
3478 
3479 		pp_addr1 = CPU->cpu_caddr1;
3480 		pp_addr2 = CPU->cpu_caddr2;
3481 		pte1 = CPU->cpu_caddr1pte;
3482 		pte2 = CPU->cpu_caddr2pte;
3483 
3484 		ppaddr_mutex = &CPU->cpu_ppaddr_mutex;
3485 		mutex_enter(ppaddr_mutex);
3486 
3487 		hat_mempte_remap(page_pptonum(frompp), pp_addr1, pte1,
3488 		    PROT_READ | HAT_STORECACHING_OK, HAT_LOAD_NOCONSIST);
3489 		hat_mempte_remap(page_pptonum(topp), pp_addr2, pte2,
3490 		    PROT_READ | PROT_WRITE | HAT_STORECACHING_OK,
3491 		    HAT_LOAD_NOCONSIST);
3492 	}
3493 
3494 	if (on_fault(&ljb)) {
3495 		ret = 0;
3496 		goto faulted;
3497 	}
3498 	if (use_sse_pagecopy)
3499 #ifdef __xpv
3500 		page_copy_no_xmm(pp_addr2, pp_addr1);
3501 #else
3502 		hwblkpagecopy(pp_addr1, pp_addr2);
3503 #endif
3504 	else
3505 		bcopy(pp_addr1, pp_addr2, PAGESIZE);
3506 
3507 	no_fault();
3508 faulted:
3509 	if (!kpm_enable) {
3510 #ifdef __xpv
3511 		/*
3512 		 * We can't leave unused mappings laying about under the
3513 		 * hypervisor, so blow them away.
3514 		 */
3515 		if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr1, 0,
3516 		    UVMF_INVLPG | UVMF_LOCAL) < 0)
3517 			panic("HYPERVISOR_update_va_mapping() failed");
3518 		if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr2, 0,
3519 		    UVMF_INVLPG | UVMF_LOCAL) < 0)
3520 			panic("HYPERVISOR_update_va_mapping() failed");
3521 #endif
3522 		mutex_exit(ppaddr_mutex);
3523 	}
3524 	kpreempt_enable();
3525 	return (ret);
3526 }
3527 
3528 void
3529 pagezero(page_t *pp, uint_t off, uint_t len)
3530 {
3531 	ASSERT(PAGE_LOCKED(pp));
3532 	pfnzero(page_pptonum(pp), off, len);
3533 }
3534 
3535 /*
3536  * Zero the physical page from off to off + len given by pfn
3537  * without changing the reference and modified bits of page.
3538  *
3539  * We use this using CPU private page address #2, see ppcopy() for more info.
3540  * pfnzero() must not be called at interrupt level.
3541  */
3542 void
3543 pfnzero(pfn_t pfn, uint_t off, uint_t len)
3544 {
3545 	caddr_t		pp_addr2;
3546 	hat_mempte_t	pte2;
3547 	kmutex_t	*ppaddr_mutex = NULL;
3548 
3549 	ASSERT_STACK_ALIGNED();
3550 	ASSERT(len <= MMU_PAGESIZE);
3551 	ASSERT(off <= MMU_PAGESIZE);
3552 	ASSERT(off + len <= MMU_PAGESIZE);
3553 
3554 	if (kpm_enable && !pfn_is_foreign(pfn)) {
3555 		pp_addr2 = hat_kpm_pfn2va(pfn);
3556 		kpreempt_disable();
3557 	} else {
3558 		kpreempt_disable();
3559 
3560 		pp_addr2 = CPU->cpu_caddr2;
3561 		pte2 = CPU->cpu_caddr2pte;
3562 
3563 		ppaddr_mutex = &CPU->cpu_ppaddr_mutex;
3564 		mutex_enter(ppaddr_mutex);
3565 
3566 		hat_mempte_remap(pfn, pp_addr2, pte2,
3567 		    PROT_READ | PROT_WRITE | HAT_STORECACHING_OK,
3568 		    HAT_LOAD_NOCONSIST);
3569 	}
3570 
3571 	if (use_sse_pagezero) {
3572 #ifdef __xpv
3573 		uint_t rem;
3574 
3575 		/*
3576 		 * zero a byte at a time until properly aligned for
3577 		 * block_zero_no_xmm().
3578 		 */
3579 		while (!P2NPHASE(off, ((uint_t)BLOCKZEROALIGN)) && len-- > 0)
3580 			pp_addr2[off++] = 0;
3581 
3582 		/*
3583 		 * Now use faster block_zero_no_xmm() for any range
3584 		 * that is properly aligned and sized.
3585 		 */
3586 		rem = P2PHASE(len, ((uint_t)BLOCKZEROALIGN));
3587 		len -= rem;
3588 		if (len != 0) {
3589 			block_zero_no_xmm(pp_addr2 + off, len);
3590 			off += len;
3591 		}
3592 
3593 		/*
3594 		 * zero remainder with byte stores.
3595 		 */
3596 		while (rem-- > 0)
3597 			pp_addr2[off++] = 0;
3598 #else
3599 		hwblkclr(pp_addr2 + off, len);
3600 #endif
3601 	} else {
3602 		bzero(pp_addr2 + off, len);
3603 	}
3604 
3605 	if (!kpm_enable || pfn_is_foreign(pfn)) {
3606 #ifdef __xpv
3607 		/*
3608 		 * On the hypervisor this page might get used for a page
3609 		 * table before any intervening change to this mapping,
3610 		 * so blow it away.
3611 		 */
3612 		if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr2, 0,
3613 		    UVMF_INVLPG) < 0)
3614 			panic("HYPERVISOR_update_va_mapping() failed");
3615 #endif
3616 		mutex_exit(ppaddr_mutex);
3617 	}
3618 
3619 	kpreempt_enable();
3620 }
3621 
3622 /*
3623  * Platform-dependent page scrub call.
3624  */
3625 void
3626 pagescrub(page_t *pp, uint_t off, uint_t len)
3627 {
3628 	/*
3629 	 * For now, we rely on the fact that pagezero() will
3630 	 * always clear UEs.
3631 	 */
3632 	pagezero(pp, off, len);
3633 }
3634 
3635 /*
3636  * set up two private addresses for use on a given CPU for use in ppcopy()
3637  */
3638 void
3639 setup_vaddr_for_ppcopy(struct cpu *cpup)
3640 {
3641 	void *addr;
3642 	hat_mempte_t pte_pa;
3643 
3644 	addr = vmem_alloc(heap_arena, mmu_ptob(1), VM_SLEEP);
3645 	pte_pa = hat_mempte_setup(addr);
3646 	cpup->cpu_caddr1 = addr;
3647 	cpup->cpu_caddr1pte = pte_pa;
3648 
3649 	addr = vmem_alloc(heap_arena, mmu_ptob(1), VM_SLEEP);
3650 	pte_pa = hat_mempte_setup(addr);
3651 	cpup->cpu_caddr2 = addr;
3652 	cpup->cpu_caddr2pte = pte_pa;
3653 
3654 	mutex_init(&cpup->cpu_ppaddr_mutex, NULL, MUTEX_DEFAULT, NULL);
3655 }
3656 
3657 /*
3658  * Undo setup_vaddr_for_ppcopy
3659  */
3660 void
3661 teardown_vaddr_for_ppcopy(struct cpu *cpup)
3662 {
3663 	mutex_destroy(&cpup->cpu_ppaddr_mutex);
3664 
3665 	hat_mempte_release(cpup->cpu_caddr2, cpup->cpu_caddr2pte);
3666 	cpup->cpu_caddr2pte = 0;
3667 	vmem_free(heap_arena, cpup->cpu_caddr2, mmu_ptob(1));
3668 	cpup->cpu_caddr2 = 0;
3669 
3670 	hat_mempte_release(cpup->cpu_caddr1, cpup->cpu_caddr1pte);
3671 	cpup->cpu_caddr1pte = 0;
3672 	vmem_free(heap_arena, cpup->cpu_caddr1, mmu_ptob(1));
3673 	cpup->cpu_caddr1 = 0;
3674 }
3675 
3676 /*
3677  * Create the pageout scanner thread. The thread has to
3678  * start at procedure with process pp and priority pri.
3679  */
3680 void
3681 pageout_init(void (*procedure)(), proc_t *pp, pri_t pri)
3682 {
3683 	(void) thread_create(NULL, 0, procedure, NULL, 0, pp, TS_RUN, pri);
3684 }
3685 
3686 /*
3687  * Function for flushing D-cache when performing module relocations
3688  * to an alternate mapping.  Unnecessary on Intel / AMD platforms.
3689  */
3690 void
3691 dcache_flushall()
3692 {}
3693 
3694 size_t
3695 exec_get_spslew(void)
3696 {
3697 	return (0);
3698 }
3699 
3700 /*
3701  * Allocate a memory page.  The argument 'seed' can be any pseudo-random
3702  * number to vary where the pages come from.  This is quite a hacked up
3703  * method -- it works for now, but really needs to be fixed up a bit.
3704  *
3705  * We currently use page_create_va() on the kvp with fake offsets,
3706  * segments and virt address.  This is pretty bogus, but was copied from the
3707  * old hat_i86.c code.  A better approach would be to specify either mnode
3708  * random or mnode local and takes a page from whatever color has the MOST
3709  * available - this would have a minimal impact on page coloring.
3710  */
3711 page_t *
3712 page_get_physical(int flags)
3713 {
3714 	page_t *pp;
3715 	u_offset_t offset = (u_offset_t)1 << 41;	/* in VA hole */
3716 	static struct seg tmpseg;
3717 	static uintptr_t ctr = 0;
3718 	static kmutex_t pgp_mutex;
3719 
3720 	/*
3721 	 * This code is gross, we really need a simpler page allocator.
3722 	 *
3723 	 * To avoid conflicts with other pages, we get creative with the offset.
3724 	 * For 32 bits, we need an offset > 4Gig
3725 	 * For 64 bits, need an offset somewhere in the VA hole.
3726 	 */
3727 	if (page_resv(1, flags & KM_NOSLEEP) == 0)
3728 		return (NULL);
3729 
3730 	mutex_enter(&pgp_mutex);
3731 #ifdef	DEBUG
3732 	pp = page_exists(&kvp, offset);
3733 	if (pp != NULL)
3734 		panic("page already exists %p", (void *)pp);
3735 #endif
3736 
3737 	pp = page_create_va(&kvp, offset, MMU_PAGESIZE, PG_EXCL,
3738 	    &tmpseg, (caddr_t)(ctr += MMU_PAGESIZE));	/* changing VA usage */
3739 	if (pp != NULL) {
3740 		page_io_unlock(pp);
3741 		page_hashout(pp, NULL);
3742 	}
3743 	mutex_exit(&pgp_mutex);
3744 	page_downgrade(pp);
3745 	return (pp);
3746 }
3747 
3748 void
3749 page_free_physical(page_t *pp)
3750 {
3751 	/*
3752 	 * Get an exclusive lock, might have to wait for a kmem reader.
3753 	 */
3754 	ASSERT(PAGE_SHARED(pp));
3755 	if (!page_tryupgrade(pp)) {
3756 		page_unlock(pp);
3757 		/*
3758 		 * RFE: we could change this to not loop forever
3759 		 * George Cameron had some idea on how to do that.
3760 		 * For now looping works - it's just like sfmmu.
3761 		 */
3762 		while (!page_lock(pp, SE_EXCL, (kmutex_t *)NULL, P_RECLAIM))
3763 			continue;
3764 	}
3765 	page_free(pp, 1);
3766 	page_unresv(1);
3767 }
3768