xref: /illumos-gate/usr/src/uts/i86pc/vm/vm_machdep.c (revision 6c9930ae88d144c4338a277ae8b1005272198328)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 /*
25  * Copyright (c) 2010, Intel Corporation.
26  * All rights reserved.
27  */
28 
29 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
30 /*	All Rights Reserved   */
31 
32 /*
33  * Portions of this source code were derived from Berkeley 4.3 BSD
34  * under license from the Regents of the University of California.
35  */
36 
37 /*
38  * UNIX machine dependent virtual memory support.
39  */
40 
41 #include <sys/types.h>
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/user.h>
45 #include <sys/proc.h>
46 #include <sys/kmem.h>
47 #include <sys/vmem.h>
48 #include <sys/buf.h>
49 #include <sys/cpuvar.h>
50 #include <sys/lgrp.h>
51 #include <sys/disp.h>
52 #include <sys/vm.h>
53 #include <sys/mman.h>
54 #include <sys/vnode.h>
55 #include <sys/cred.h>
56 #include <sys/exec.h>
57 #include <sys/exechdr.h>
58 #include <sys/debug.h>
59 #include <sys/vmsystm.h>
60 #include <sys/swap.h>
61 #include <sys/dumphdr.h>
62 
63 #include <vm/hat.h>
64 #include <vm/as.h>
65 #include <vm/seg.h>
66 #include <vm/seg_kp.h>
67 #include <vm/seg_vn.h>
68 #include <vm/page.h>
69 #include <vm/seg_kmem.h>
70 #include <vm/seg_kpm.h>
71 #include <vm/vm_dep.h>
72 
73 #include <sys/cpu.h>
74 #include <sys/vm_machparam.h>
75 #include <sys/memlist.h>
76 #include <sys/bootconf.h> /* XXX the memlist stuff belongs in memlist_plat.h */
77 #include <vm/hat_i86.h>
78 #include <sys/x86_archext.h>
79 #include <sys/elf_386.h>
80 #include <sys/cmn_err.h>
81 #include <sys/archsystm.h>
82 #include <sys/machsystm.h>
83 
84 #include <sys/vtrace.h>
85 #include <sys/ddidmareq.h>
86 #include <sys/promif.h>
87 #include <sys/memnode.h>
88 #include <sys/stack.h>
89 #include <util/qsort.h>
90 #include <sys/taskq.h>
91 
92 #ifdef __xpv
93 
94 #include <sys/hypervisor.h>
95 #include <sys/xen_mmu.h>
96 #include <sys/balloon_impl.h>
97 
98 /*
99  * domain 0 pages usable for DMA are kept pre-allocated and kept in
100  * distinct lists, ordered by increasing mfn.
101  */
102 static kmutex_t io_pool_lock;
103 static kmutex_t contig_list_lock;
104 static page_t *io_pool_4g;	/* pool for 32 bit dma limited devices */
105 static page_t *io_pool_16m;	/* pool for 24 bit dma limited legacy devices */
106 static long io_pool_cnt;
107 static long io_pool_cnt_max = 0;
108 #define	DEFAULT_IO_POOL_MIN	128
109 static long io_pool_cnt_min = DEFAULT_IO_POOL_MIN;
110 static long io_pool_cnt_lowater = 0;
111 static long io_pool_shrink_attempts; /* how many times did we try to shrink */
112 static long io_pool_shrinks;	/* how many times did we really shrink */
113 static long io_pool_grows;	/* how many times did we grow */
114 static mfn_t start_mfn = 1;
115 static caddr_t io_pool_kva;	/* use to alloc pages when needed */
116 
117 static int create_contig_pfnlist(uint_t);
118 
119 /*
120  * percentage of phys mem to hold in the i/o pool
121  */
122 #define	DEFAULT_IO_POOL_PCT	2
123 static long io_pool_physmem_pct = DEFAULT_IO_POOL_PCT;
124 static void page_io_pool_sub(page_t **, page_t *, page_t *);
125 int ioalloc_dbg = 0;
126 
127 #endif /* __xpv */
128 
129 uint_t vac_colors = 1;
130 
131 int largepagesupport = 0;
132 extern uint_t page_create_new;
133 extern uint_t page_create_exists;
134 extern uint_t page_create_putbacks;
135 /*
136  * Allow users to disable the kernel's use of SSE.
137  */
138 extern int use_sse_pagecopy, use_sse_pagezero;
139 
140 /*
141  * combined memory ranges from mnode and memranges[] to manage single
142  * mnode/mtype dimension in the page lists.
143  */
144 typedef struct {
145 	pfn_t	mnr_pfnlo;
146 	pfn_t	mnr_pfnhi;
147 	int	mnr_mnode;
148 	int	mnr_memrange;		/* index into memranges[] */
149 	int	mnr_next;		/* next lower PA mnoderange */
150 	int	mnr_exists;
151 	/* maintain page list stats */
152 	pgcnt_t	mnr_mt_clpgcnt;		/* cache list cnt */
153 	pgcnt_t	mnr_mt_flpgcnt[MMU_PAGE_SIZES];	/* free list cnt per szc */
154 	pgcnt_t	mnr_mt_totcnt;		/* sum of cache and free lists */
155 #ifdef DEBUG
156 	struct mnr_mts {		/* mnode/mtype szc stats */
157 		pgcnt_t	mnr_mts_pgcnt;
158 		int	mnr_mts_colors;
159 		pgcnt_t *mnr_mtsc_pgcnt;
160 	} 	*mnr_mts;
161 #endif
162 } mnoderange_t;
163 
164 #define	MEMRANGEHI(mtype)						\
165 	((mtype > 0) ? memranges[mtype - 1] - 1: physmax)
166 #define	MEMRANGELO(mtype)	(memranges[mtype])
167 
168 #define	MTYPE_FREEMEM(mt)	(mnoderanges[mt].mnr_mt_totcnt)
169 
170 /*
171  * As the PC architecture evolved memory up was clumped into several
172  * ranges for various historical I/O devices to do DMA.
173  * < 16Meg - ISA bus
174  * < 2Gig - ???
175  * < 4Gig - PCI bus or drivers that don't understand PAE mode
176  *
177  * These are listed in reverse order, so that we can skip over unused
178  * ranges on machines with small memories.
179  *
180  * For now under the Hypervisor, we'll only ever have one memrange.
181  */
182 #define	PFN_4GIG	0x100000
183 #define	PFN_16MEG	0x1000
184 /* Indices into the memory range (arch_memranges) array. */
185 #define	MRI_4G		0
186 #define	MRI_2G		1
187 #define	MRI_16M		2
188 #define	MRI_0		3
189 static pfn_t arch_memranges[NUM_MEM_RANGES] = {
190     PFN_4GIG,	/* pfn range for 4G and above */
191     0x80000,	/* pfn range for 2G-4G */
192     PFN_16MEG,	/* pfn range for 16M-2G */
193     0x00000,	/* pfn range for 0-16M */
194 };
195 pfn_t *memranges = &arch_memranges[0];
196 int nranges = NUM_MEM_RANGES;
197 
198 /*
199  * This combines mem_node_config and memranges into one data
200  * structure to be used for page list management.
201  */
202 mnoderange_t	*mnoderanges;
203 int		mnoderangecnt;
204 int		mtype4g;
205 int		mtype16m;
206 int		mtypetop;	/* index of highest pfn'ed mnoderange */
207 
208 /*
209  * 4g memory management variables for systems with more than 4g of memory:
210  *
211  * physical memory below 4g is required for 32bit dma devices and, currently,
212  * for kmem memory. On systems with more than 4g of memory, the pool of memory
213  * below 4g can be depleted without any paging activity given that there is
214  * likely to be sufficient memory above 4g.
215  *
216  * physmax4g is set true if the largest pfn is over 4g. The rest of the
217  * 4g memory management code is enabled only when physmax4g is true.
218  *
219  * maxmem4g is the count of the maximum number of pages on the page lists
220  * with physical addresses below 4g. It can be a lot less then 4g given that
221  * BIOS may reserve large chunks of space below 4g for hot plug pci devices,
222  * agp aperture etc.
223  *
224  * freemem4g maintains the count of the number of available pages on the
225  * page lists with physical addresses below 4g.
226  *
227  * DESFREE4G specifies the desired amount of below 4g memory. It defaults to
228  * 6% (desfree4gshift = 4) of maxmem4g.
229  *
230  * RESTRICT4G_ALLOC returns true if freemem4g falls below DESFREE4G
231  * and the amount of physical memory above 4g is greater than freemem4g.
232  * In this case, page_get_* routines will restrict below 4g allocations
233  * for requests that don't specifically require it.
234  */
235 
236 #define	DESFREE4G	(maxmem4g >> desfree4gshift)
237 
238 #define	RESTRICT4G_ALLOC					\
239 	(physmax4g && (freemem4g < DESFREE4G) && ((freemem4g << 1) < freemem))
240 
241 static pgcnt_t	maxmem4g;
242 static pgcnt_t	freemem4g;
243 static int	physmax4g;
244 static int	desfree4gshift = 4;	/* maxmem4g shift to derive DESFREE4G */
245 
246 /*
247  * 16m memory management:
248  *
249  * reserve some amount of physical memory below 16m for legacy devices.
250  *
251  * RESTRICT16M_ALLOC returns true if an there are sufficient free pages above
252  * 16m or if the 16m pool drops below DESFREE16M.
253  *
254  * In this case, general page allocations via page_get_{free,cache}list
255  * routines will be restricted from allocating from the 16m pool. Allocations
256  * that require specific pfn ranges (page_get_anylist) and PG_PANIC allocations
257  * are not restricted.
258  */
259 
260 #define	FREEMEM16M	MTYPE_FREEMEM(mtype16m)
261 #define	DESFREE16M	desfree16m
262 #define	RESTRICT16M_ALLOC(freemem, pgcnt, flags)		\
263 	((freemem != 0) && ((flags & PG_PANIC) == 0) &&		\
264 	    ((freemem >= (FREEMEM16M)) ||			\
265 	    (FREEMEM16M  < (DESFREE16M + pgcnt))))
266 
267 static pgcnt_t	desfree16m = 0x380;
268 
269 /*
270  * This can be patched via /etc/system to allow old non-PAE aware device
271  * drivers to use kmem_alloc'd memory on 32 bit systems with > 4Gig RAM.
272  */
273 int restricted_kmemalloc = 0;
274 
275 #ifdef VM_STATS
276 struct {
277 	ulong_t	pga_alloc;
278 	ulong_t	pga_notfullrange;
279 	ulong_t	pga_nulldmaattr;
280 	ulong_t	pga_allocok;
281 	ulong_t	pga_allocfailed;
282 	ulong_t	pgma_alloc;
283 	ulong_t	pgma_allocok;
284 	ulong_t	pgma_allocfailed;
285 	ulong_t	pgma_allocempty;
286 } pga_vmstats;
287 #endif
288 
289 uint_t mmu_page_sizes;
290 
291 /* How many page sizes the users can see */
292 uint_t mmu_exported_page_sizes;
293 
294 /* page sizes that legacy applications can see */
295 uint_t mmu_legacy_page_sizes;
296 
297 /*
298  * Number of pages in 1 GB.  Don't enable automatic large pages if we have
299  * fewer than this many pages.
300  */
301 pgcnt_t shm_lpg_min_physmem = 1 << (30 - MMU_PAGESHIFT);
302 pgcnt_t privm_lpg_min_physmem = 1 << (30 - MMU_PAGESHIFT);
303 
304 /*
305  * Maximum and default segment size tunables for user private
306  * and shared anon memory, and user text and initialized data.
307  * These can be patched via /etc/system to allow large pages
308  * to be used for mapping application private and shared anon memory.
309  */
310 size_t mcntl0_lpsize = MMU_PAGESIZE;
311 size_t max_uheap_lpsize = MMU_PAGESIZE;
312 size_t default_uheap_lpsize = MMU_PAGESIZE;
313 size_t max_ustack_lpsize = MMU_PAGESIZE;
314 size_t default_ustack_lpsize = MMU_PAGESIZE;
315 size_t max_privmap_lpsize = MMU_PAGESIZE;
316 size_t max_uidata_lpsize = MMU_PAGESIZE;
317 size_t max_utext_lpsize = MMU_PAGESIZE;
318 size_t max_shm_lpsize = MMU_PAGESIZE;
319 
320 
321 /*
322  * initialized by page_coloring_init().
323  */
324 uint_t	page_colors;
325 uint_t	page_colors_mask;
326 uint_t	page_coloring_shift;
327 int	cpu_page_colors;
328 static uint_t	l2_colors;
329 
330 /*
331  * Page freelists and cachelists are dynamically allocated once mnoderangecnt
332  * and page_colors are calculated from the l2 cache n-way set size.  Within a
333  * mnode range, the page freelist and cachelist are hashed into bins based on
334  * color. This makes it easier to search for a page within a specific memory
335  * range.
336  */
337 #define	PAGE_COLORS_MIN	16
338 
339 page_t ****page_freelists;
340 page_t ***page_cachelists;
341 
342 
343 /*
344  * Used by page layer to know about page sizes
345  */
346 hw_pagesize_t hw_page_array[MAX_NUM_LEVEL + 1];
347 
348 kmutex_t	*fpc_mutex[NPC_MUTEX];
349 kmutex_t	*cpc_mutex[NPC_MUTEX];
350 
351 /* Lock to protect mnoderanges array for memory DR operations. */
352 static kmutex_t mnoderange_lock;
353 
354 /*
355  * Only let one thread at a time try to coalesce large pages, to
356  * prevent them from working against each other.
357  */
358 static kmutex_t	contig_lock;
359 #define	CONTIG_LOCK()	mutex_enter(&contig_lock);
360 #define	CONTIG_UNLOCK()	mutex_exit(&contig_lock);
361 
362 #define	PFN_16M		(mmu_btop((uint64_t)0x1000000))
363 
364 /*
365  * Return the optimum page size for a given mapping
366  */
367 /*ARGSUSED*/
368 size_t
369 map_pgsz(int maptype, struct proc *p, caddr_t addr, size_t len, int memcntl)
370 {
371 	level_t l = 0;
372 	size_t pgsz = MMU_PAGESIZE;
373 	size_t max_lpsize;
374 	uint_t mszc;
375 
376 	ASSERT(maptype != MAPPGSZ_VA);
377 
378 	if (maptype != MAPPGSZ_ISM && physmem < privm_lpg_min_physmem) {
379 		return (MMU_PAGESIZE);
380 	}
381 
382 	switch (maptype) {
383 	case MAPPGSZ_HEAP:
384 	case MAPPGSZ_STK:
385 		max_lpsize = memcntl ? mcntl0_lpsize : (maptype ==
386 		    MAPPGSZ_HEAP ? max_uheap_lpsize : max_ustack_lpsize);
387 		if (max_lpsize == MMU_PAGESIZE) {
388 			return (MMU_PAGESIZE);
389 		}
390 		if (len == 0) {
391 			len = (maptype == MAPPGSZ_HEAP) ? p->p_brkbase +
392 			    p->p_brksize - p->p_bssbase : p->p_stksize;
393 		}
394 		len = (maptype == MAPPGSZ_HEAP) ? MAX(len,
395 		    default_uheap_lpsize) : MAX(len, default_ustack_lpsize);
396 
397 		/*
398 		 * use the pages size that best fits len
399 		 */
400 		for (l = mmu.umax_page_level; l > 0; --l) {
401 			if (LEVEL_SIZE(l) > max_lpsize || len < LEVEL_SIZE(l)) {
402 				continue;
403 			} else {
404 				pgsz = LEVEL_SIZE(l);
405 			}
406 			break;
407 		}
408 
409 		mszc = (maptype == MAPPGSZ_HEAP ? p->p_brkpageszc :
410 		    p->p_stkpageszc);
411 		if (addr == 0 && (pgsz < hw_page_array[mszc].hp_size)) {
412 			pgsz = hw_page_array[mszc].hp_size;
413 		}
414 		return (pgsz);
415 
416 	case MAPPGSZ_ISM:
417 		for (l = mmu.umax_page_level; l > 0; --l) {
418 			if (len >= LEVEL_SIZE(l))
419 				return (LEVEL_SIZE(l));
420 		}
421 		return (LEVEL_SIZE(0));
422 	}
423 	return (pgsz);
424 }
425 
426 static uint_t
427 map_szcvec(caddr_t addr, size_t size, uintptr_t off, size_t max_lpsize,
428     size_t min_physmem)
429 {
430 	caddr_t eaddr = addr + size;
431 	uint_t szcvec = 0;
432 	caddr_t raddr;
433 	caddr_t readdr;
434 	size_t	pgsz;
435 	int i;
436 
437 	if (physmem < min_physmem || max_lpsize <= MMU_PAGESIZE) {
438 		return (0);
439 	}
440 
441 	for (i = mmu_exported_page_sizes - 1; i > 0; i--) {
442 		pgsz = page_get_pagesize(i);
443 		if (pgsz > max_lpsize) {
444 			continue;
445 		}
446 		raddr = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz);
447 		readdr = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz);
448 		if (raddr < addr || raddr >= readdr) {
449 			continue;
450 		}
451 		if (P2PHASE((uintptr_t)addr ^ off, pgsz)) {
452 			continue;
453 		}
454 		/*
455 		 * Set szcvec to the remaining page sizes.
456 		 */
457 		szcvec = ((1 << (i + 1)) - 1) & ~1;
458 		break;
459 	}
460 	return (szcvec);
461 }
462 
463 /*
464  * Return a bit vector of large page size codes that
465  * can be used to map [addr, addr + len) region.
466  */
467 /*ARGSUSED*/
468 uint_t
469 map_pgszcvec(caddr_t addr, size_t size, uintptr_t off, int flags, int type,
470     int memcntl)
471 {
472 	size_t max_lpsize = mcntl0_lpsize;
473 
474 	if (mmu.max_page_level == 0)
475 		return (0);
476 
477 	if (flags & MAP_TEXT) {
478 		if (!memcntl)
479 			max_lpsize = max_utext_lpsize;
480 		return (map_szcvec(addr, size, off, max_lpsize,
481 		    shm_lpg_min_physmem));
482 
483 	} else if (flags & MAP_INITDATA) {
484 		if (!memcntl)
485 			max_lpsize = max_uidata_lpsize;
486 		return (map_szcvec(addr, size, off, max_lpsize,
487 		    privm_lpg_min_physmem));
488 
489 	} else if (type == MAPPGSZC_SHM) {
490 		if (!memcntl)
491 			max_lpsize = max_shm_lpsize;
492 		return (map_szcvec(addr, size, off, max_lpsize,
493 		    shm_lpg_min_physmem));
494 
495 	} else if (type == MAPPGSZC_HEAP) {
496 		if (!memcntl)
497 			max_lpsize = max_uheap_lpsize;
498 		return (map_szcvec(addr, size, off, max_lpsize,
499 		    privm_lpg_min_physmem));
500 
501 	} else if (type == MAPPGSZC_STACK) {
502 		if (!memcntl)
503 			max_lpsize = max_ustack_lpsize;
504 		return (map_szcvec(addr, size, off, max_lpsize,
505 		    privm_lpg_min_physmem));
506 
507 	} else {
508 		if (!memcntl)
509 			max_lpsize = max_privmap_lpsize;
510 		return (map_szcvec(addr, size, off, max_lpsize,
511 		    privm_lpg_min_physmem));
512 	}
513 }
514 
515 /*
516  * Handle a pagefault.
517  */
518 faultcode_t
519 pagefault(
520 	caddr_t addr,
521 	enum fault_type type,
522 	enum seg_rw rw,
523 	int iskernel)
524 {
525 	struct as *as;
526 	struct hat *hat;
527 	struct proc *p;
528 	kthread_t *t;
529 	faultcode_t res;
530 	caddr_t base;
531 	size_t len;
532 	int err;
533 	int mapped_red;
534 	uintptr_t ea;
535 
536 	ASSERT_STACK_ALIGNED();
537 
538 	if (INVALID_VADDR(addr))
539 		return (FC_NOMAP);
540 
541 	mapped_red = segkp_map_red();
542 
543 	if (iskernel) {
544 		as = &kas;
545 		hat = as->a_hat;
546 	} else {
547 		t = curthread;
548 		p = ttoproc(t);
549 		as = p->p_as;
550 		hat = as->a_hat;
551 	}
552 
553 	/*
554 	 * Dispatch pagefault.
555 	 */
556 	res = as_fault(hat, as, addr, 1, type, rw);
557 
558 	/*
559 	 * If this isn't a potential unmapped hole in the user's
560 	 * UNIX data or stack segments, just return status info.
561 	 */
562 	if (res != FC_NOMAP || iskernel)
563 		goto out;
564 
565 	/*
566 	 * Check to see if we happened to faulted on a currently unmapped
567 	 * part of the UNIX data or stack segments.  If so, create a zfod
568 	 * mapping there and then try calling the fault routine again.
569 	 */
570 	base = p->p_brkbase;
571 	len = p->p_brksize;
572 
573 	if (addr < base || addr >= base + len) {		/* data seg? */
574 		base = (caddr_t)p->p_usrstack - p->p_stksize;
575 		len = p->p_stksize;
576 		if (addr < base || addr >= p->p_usrstack) {	/* stack seg? */
577 			/* not in either UNIX data or stack segments */
578 			res = FC_NOMAP;
579 			goto out;
580 		}
581 	}
582 
583 	/*
584 	 * the rest of this function implements a 3.X 4.X 5.X compatibility
585 	 * This code is probably not needed anymore
586 	 */
587 	if (p->p_model == DATAMODEL_ILP32) {
588 
589 		/* expand the gap to the page boundaries on each side */
590 		ea = P2ROUNDUP((uintptr_t)base + len, MMU_PAGESIZE);
591 		base = (caddr_t)P2ALIGN((uintptr_t)base, MMU_PAGESIZE);
592 		len = ea - (uintptr_t)base;
593 
594 		as_rangelock(as);
595 		if (as_gap(as, MMU_PAGESIZE, &base, &len, AH_CONTAIN, addr) ==
596 		    0) {
597 			err = as_map(as, base, len, segvn_create, zfod_argsp);
598 			as_rangeunlock(as);
599 			if (err) {
600 				res = FC_MAKE_ERR(err);
601 				goto out;
602 			}
603 		} else {
604 			/*
605 			 * This page is already mapped by another thread after
606 			 * we returned from as_fault() above.  We just fall
607 			 * through as_fault() below.
608 			 */
609 			as_rangeunlock(as);
610 		}
611 
612 		res = as_fault(hat, as, addr, 1, F_INVAL, rw);
613 	}
614 
615 out:
616 	if (mapped_red)
617 		segkp_unmap_red();
618 
619 	return (res);
620 }
621 
622 void
623 map_addr(caddr_t *addrp, size_t len, offset_t off, int vacalign, uint_t flags)
624 {
625 	struct proc *p = curproc;
626 	caddr_t userlimit = (flags & _MAP_LOW32) ?
627 	    (caddr_t)_userlimit32 : p->p_as->a_userlimit;
628 
629 	map_addr_proc(addrp, len, off, vacalign, userlimit, curproc, flags);
630 }
631 
632 /*ARGSUSED*/
633 int
634 map_addr_vacalign_check(caddr_t addr, u_offset_t off)
635 {
636 	return (0);
637 }
638 
639 /*
640  * map_addr_proc() is the routine called when the system is to
641  * choose an address for the user.  We will pick an address
642  * range which is the highest available below userlimit.
643  *
644  * Every mapping will have a redzone of a single page on either side of
645  * the request. This is done to leave one page unmapped between segments.
646  * This is not required, but it's useful for the user because if their
647  * program strays across a segment boundary, it will catch a fault
648  * immediately making debugging a little easier.  Currently the redzone
649  * is mandatory.
650  *
651  * addrp is a value/result parameter.
652  *	On input it is a hint from the user to be used in a completely
653  *	machine dependent fashion.  We decide to completely ignore this hint.
654  *	If MAP_ALIGN was specified, addrp contains the minimal alignment, which
655  *	must be some "power of two" multiple of pagesize.
656  *
657  *	On output it is NULL if no address can be found in the current
658  *	processes address space or else an address that is currently
659  *	not mapped for len bytes with a page of red zone on either side.
660  *
661  *	vacalign is not needed on x86 (it's for viturally addressed caches)
662  */
663 /*ARGSUSED*/
664 void
665 map_addr_proc(
666 	caddr_t *addrp,
667 	size_t len,
668 	offset_t off,
669 	int vacalign,
670 	caddr_t userlimit,
671 	struct proc *p,
672 	uint_t flags)
673 {
674 	struct as *as = p->p_as;
675 	caddr_t addr;
676 	caddr_t base;
677 	size_t slen;
678 	size_t align_amount;
679 
680 	ASSERT32(userlimit == as->a_userlimit);
681 
682 	base = p->p_brkbase;
683 #if defined(__amd64)
684 	/*
685 	 * XX64 Yes, this needs more work.
686 	 */
687 	if (p->p_model == DATAMODEL_NATIVE) {
688 		if (userlimit < as->a_userlimit) {
689 			/*
690 			 * This happens when a program wants to map
691 			 * something in a range that's accessible to a
692 			 * program in a smaller address space.  For example,
693 			 * a 64-bit program calling mmap32(2) to guarantee
694 			 * that the returned address is below 4Gbytes.
695 			 */
696 			ASSERT((uintptr_t)userlimit < ADDRESS_C(0xffffffff));
697 
698 			if (userlimit > base)
699 				slen = userlimit - base;
700 			else {
701 				*addrp = NULL;
702 				return;
703 			}
704 		} else {
705 			/*
706 			 * XX64 This layout is probably wrong .. but in
707 			 * the event we make the amd64 address space look
708 			 * like sparcv9 i.e. with the stack -above- the
709 			 * heap, this bit of code might even be correct.
710 			 */
711 			slen = p->p_usrstack - base -
712 			    ((p->p_stk_ctl + PAGEOFFSET) & PAGEMASK);
713 		}
714 	} else
715 #endif
716 		slen = userlimit - base;
717 
718 	/* Make len be a multiple of PAGESIZE */
719 	len = (len + PAGEOFFSET) & PAGEMASK;
720 
721 	/*
722 	 * figure out what the alignment should be
723 	 *
724 	 * XX64 -- is there an ELF_AMD64_MAXPGSZ or is it the same????
725 	 */
726 	if (len <= ELF_386_MAXPGSZ) {
727 		/*
728 		 * Align virtual addresses to ensure that ELF shared libraries
729 		 * are mapped with the appropriate alignment constraints by
730 		 * the run-time linker.
731 		 */
732 		align_amount = ELF_386_MAXPGSZ;
733 	} else {
734 		/*
735 		 * For 32-bit processes, only those which have specified
736 		 * MAP_ALIGN and an addr will be aligned on a larger page size.
737 		 * Not doing so can potentially waste up to 1G of process
738 		 * address space.
739 		 */
740 		int lvl = (p->p_model == DATAMODEL_ILP32) ? 1 :
741 		    mmu.umax_page_level;
742 
743 		while (lvl && len < LEVEL_SIZE(lvl))
744 			--lvl;
745 
746 		align_amount = LEVEL_SIZE(lvl);
747 	}
748 	if ((flags & MAP_ALIGN) && ((uintptr_t)*addrp > align_amount))
749 		align_amount = (uintptr_t)*addrp;
750 
751 	ASSERT(ISP2(align_amount));
752 	ASSERT(align_amount == 0 || align_amount >= PAGESIZE);
753 
754 	off = off & (align_amount - 1);
755 	/*
756 	 * Look for a large enough hole starting below userlimit.
757 	 * After finding it, use the upper part.
758 	 */
759 	if (as_gap_aligned(as, len, &base, &slen, AH_HI, NULL, align_amount,
760 	    PAGESIZE, off) == 0) {
761 		caddr_t as_addr;
762 
763 		/*
764 		 * addr is the highest possible address to use since we have
765 		 * a PAGESIZE redzone at the beginning and end.
766 		 */
767 		addr = base + slen - (PAGESIZE + len);
768 		as_addr = addr;
769 		/*
770 		 * Round address DOWN to the alignment amount and
771 		 * add the offset in.
772 		 * If addr is greater than as_addr, len would not be large
773 		 * enough to include the redzone, so we must adjust down
774 		 * by the alignment amount.
775 		 */
776 		addr = (caddr_t)((uintptr_t)addr & (~(align_amount - 1)));
777 		addr += (uintptr_t)off;
778 		if (addr > as_addr) {
779 			addr -= align_amount;
780 		}
781 
782 		ASSERT(addr > base);
783 		ASSERT(addr + len < base + slen);
784 		ASSERT(((uintptr_t)addr & (align_amount - 1)) ==
785 		    ((uintptr_t)(off)));
786 		*addrp = addr;
787 	} else {
788 		*addrp = NULL;	/* no more virtual space */
789 	}
790 }
791 
792 int valid_va_range_aligned_wraparound;
793 
794 /*
795  * Determine whether [*basep, *basep + *lenp) contains a mappable range of
796  * addresses at least "minlen" long, where the base of the range is at "off"
797  * phase from an "align" boundary and there is space for a "redzone"-sized
798  * redzone on either side of the range.  On success, 1 is returned and *basep
799  * and *lenp are adjusted to describe the acceptable range (including
800  * the redzone).  On failure, 0 is returned.
801  */
802 /*ARGSUSED3*/
803 int
804 valid_va_range_aligned(caddr_t *basep, size_t *lenp, size_t minlen, int dir,
805     size_t align, size_t redzone, size_t off)
806 {
807 	uintptr_t hi, lo;
808 	size_t tot_len;
809 
810 	ASSERT(align == 0 ? off == 0 : off < align);
811 	ASSERT(ISP2(align));
812 	ASSERT(align == 0 || align >= PAGESIZE);
813 
814 	lo = (uintptr_t)*basep;
815 	hi = lo + *lenp;
816 	tot_len = minlen + 2 * redzone; /* need at least this much space */
817 
818 	/*
819 	 * If hi rolled over the top, try cutting back.
820 	 */
821 	if (hi < lo) {
822 		*lenp = 0UL - lo - 1UL;
823 		/* See if this really happens. If so, then we figure out why */
824 		valid_va_range_aligned_wraparound++;
825 		hi = lo + *lenp;
826 	}
827 	if (*lenp < tot_len) {
828 		return (0);
829 	}
830 
831 #if defined(__amd64)
832 	/*
833 	 * Deal with a possible hole in the address range between
834 	 * hole_start and hole_end that should never be mapped.
835 	 */
836 	if (lo < hole_start) {
837 		if (hi > hole_start) {
838 			if (hi < hole_end) {
839 				hi = hole_start;
840 			} else {
841 				/* lo < hole_start && hi >= hole_end */
842 				if (dir == AH_LO) {
843 					/*
844 					 * prefer lowest range
845 					 */
846 					if (hole_start - lo >= tot_len)
847 						hi = hole_start;
848 					else if (hi - hole_end >= tot_len)
849 						lo = hole_end;
850 					else
851 						return (0);
852 				} else {
853 					/*
854 					 * prefer highest range
855 					 */
856 					if (hi - hole_end >= tot_len)
857 						lo = hole_end;
858 					else if (hole_start - lo >= tot_len)
859 						hi = hole_start;
860 					else
861 						return (0);
862 				}
863 			}
864 		}
865 	} else {
866 		/* lo >= hole_start */
867 		if (hi < hole_end)
868 			return (0);
869 		if (lo < hole_end)
870 			lo = hole_end;
871 	}
872 #endif
873 
874 	if (hi - lo < tot_len)
875 		return (0);
876 
877 	if (align > 1) {
878 		uintptr_t tlo = lo + redzone;
879 		uintptr_t thi = hi - redzone;
880 		tlo = (uintptr_t)P2PHASEUP(tlo, align, off);
881 		if (tlo < lo + redzone) {
882 			return (0);
883 		}
884 		if (thi < tlo || thi - tlo < minlen) {
885 			return (0);
886 		}
887 	}
888 
889 	*basep = (caddr_t)lo;
890 	*lenp = hi - lo;
891 	return (1);
892 }
893 
894 /*
895  * Determine whether [*basep, *basep + *lenp) contains a mappable range of
896  * addresses at least "minlen" long.  On success, 1 is returned and *basep
897  * and *lenp are adjusted to describe the acceptable range.  On failure, 0
898  * is returned.
899  */
900 int
901 valid_va_range(caddr_t *basep, size_t *lenp, size_t minlen, int dir)
902 {
903 	return (valid_va_range_aligned(basep, lenp, minlen, dir, 0, 0, 0));
904 }
905 
906 /*
907  * Determine whether [addr, addr+len] are valid user addresses.
908  */
909 /*ARGSUSED*/
910 int
911 valid_usr_range(caddr_t addr, size_t len, uint_t prot, struct as *as,
912     caddr_t userlimit)
913 {
914 	caddr_t eaddr = addr + len;
915 
916 	if (eaddr <= addr || addr >= userlimit || eaddr > userlimit)
917 		return (RANGE_BADADDR);
918 
919 #if defined(__amd64)
920 	/*
921 	 * Check for the VA hole
922 	 */
923 	if (eaddr > (caddr_t)hole_start && addr < (caddr_t)hole_end)
924 		return (RANGE_BADADDR);
925 #endif
926 
927 	return (RANGE_OKAY);
928 }
929 
930 /*
931  * Return 1 if the page frame is onboard memory, else 0.
932  */
933 int
934 pf_is_memory(pfn_t pf)
935 {
936 	if (pfn_is_foreign(pf))
937 		return (0);
938 	return (address_in_memlist(phys_install, pfn_to_pa(pf), 1));
939 }
940 
941 /*
942  * return the memrange containing pfn
943  */
944 int
945 memrange_num(pfn_t pfn)
946 {
947 	int n;
948 
949 	for (n = 0; n < nranges - 1; ++n) {
950 		if (pfn >= memranges[n])
951 			break;
952 	}
953 	return (n);
954 }
955 
956 /*
957  * return the mnoderange containing pfn
958  */
959 /*ARGSUSED*/
960 int
961 pfn_2_mtype(pfn_t pfn)
962 {
963 #if defined(__xpv)
964 	return (0);
965 #else
966 	int	n;
967 
968 	/* Always start from highest pfn and work our way down */
969 	for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) {
970 		if (pfn >= mnoderanges[n].mnr_pfnlo) {
971 			break;
972 		}
973 	}
974 	return (n);
975 #endif
976 }
977 
978 #if !defined(__xpv)
979 /*
980  * is_contigpage_free:
981  *	returns a page list of contiguous pages. It minimally has to return
982  *	minctg pages. Caller determines minctg based on the scatter-gather
983  *	list length.
984  *
985  *	pfnp is set to the next page frame to search on return.
986  */
987 static page_t *
988 is_contigpage_free(
989 	pfn_t *pfnp,
990 	pgcnt_t *pgcnt,
991 	pgcnt_t minctg,
992 	uint64_t pfnseg,
993 	int iolock)
994 {
995 	int	i = 0;
996 	pfn_t	pfn = *pfnp;
997 	page_t	*pp;
998 	page_t	*plist = NULL;
999 
1000 	/*
1001 	 * fail if pfn + minctg crosses a segment boundary.
1002 	 * Adjust for next starting pfn to begin at segment boundary.
1003 	 */
1004 
1005 	if (((*pfnp + minctg - 1) & pfnseg) < (*pfnp & pfnseg)) {
1006 		*pfnp = roundup(*pfnp, pfnseg + 1);
1007 		return (NULL);
1008 	}
1009 
1010 	do {
1011 retry:
1012 		pp = page_numtopp_nolock(pfn + i);
1013 		if ((pp == NULL) || IS_DUMP_PAGE(pp) ||
1014 		    (page_trylock(pp, SE_EXCL) == 0)) {
1015 			(*pfnp)++;
1016 			break;
1017 		}
1018 		if (page_pptonum(pp) != pfn + i) {
1019 			page_unlock(pp);
1020 			goto retry;
1021 		}
1022 
1023 		if (!(PP_ISFREE(pp))) {
1024 			page_unlock(pp);
1025 			(*pfnp)++;
1026 			break;
1027 		}
1028 
1029 		if (!PP_ISAGED(pp)) {
1030 			page_list_sub(pp, PG_CACHE_LIST);
1031 			page_hashout(pp, (kmutex_t *)NULL);
1032 		} else {
1033 			page_list_sub(pp, PG_FREE_LIST);
1034 		}
1035 
1036 		if (iolock)
1037 			page_io_lock(pp);
1038 		page_list_concat(&plist, &pp);
1039 
1040 		/*
1041 		 * exit loop when pgcnt satisfied or segment boundary reached.
1042 		 */
1043 
1044 	} while ((++i < *pgcnt) && ((pfn + i) & pfnseg));
1045 
1046 	*pfnp += i;		/* set to next pfn to search */
1047 
1048 	if (i >= minctg) {
1049 		*pgcnt -= i;
1050 		return (plist);
1051 	}
1052 
1053 	/*
1054 	 * failure: minctg not satisfied.
1055 	 *
1056 	 * if next request crosses segment boundary, set next pfn
1057 	 * to search from the segment boundary.
1058 	 */
1059 	if (((*pfnp + minctg - 1) & pfnseg) < (*pfnp & pfnseg))
1060 		*pfnp = roundup(*pfnp, pfnseg + 1);
1061 
1062 	/* clean up any pages already allocated */
1063 
1064 	while (plist) {
1065 		pp = plist;
1066 		page_sub(&plist, pp);
1067 		page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
1068 		if (iolock)
1069 			page_io_unlock(pp);
1070 		page_unlock(pp);
1071 	}
1072 
1073 	return (NULL);
1074 }
1075 #endif	/* !__xpv */
1076 
1077 /*
1078  * verify that pages being returned from allocator have correct DMA attribute
1079  */
1080 #ifndef DEBUG
1081 #define	check_dma(a, b, c) (void)(0)
1082 #else
1083 static void
1084 check_dma(ddi_dma_attr_t *dma_attr, page_t *pp, int cnt)
1085 {
1086 	if (dma_attr == NULL)
1087 		return;
1088 
1089 	while (cnt-- > 0) {
1090 		if (pa_to_ma(pfn_to_pa(pp->p_pagenum)) <
1091 		    dma_attr->dma_attr_addr_lo)
1092 			panic("PFN (pp=%p) below dma_attr_addr_lo", (void *)pp);
1093 		if (pa_to_ma(pfn_to_pa(pp->p_pagenum)) >=
1094 		    dma_attr->dma_attr_addr_hi)
1095 			panic("PFN (pp=%p) above dma_attr_addr_hi", (void *)pp);
1096 		pp = pp->p_next;
1097 	}
1098 }
1099 #endif
1100 
1101 #if !defined(__xpv)
1102 static page_t *
1103 page_get_contigpage(pgcnt_t *pgcnt, ddi_dma_attr_t *mattr, int iolock)
1104 {
1105 	pfn_t		pfn;
1106 	int		sgllen;
1107 	uint64_t	pfnseg;
1108 	pgcnt_t		minctg;
1109 	page_t		*pplist = NULL, *plist;
1110 	uint64_t	lo, hi;
1111 	pgcnt_t		pfnalign = 0;
1112 	static pfn_t	startpfn;
1113 	static pgcnt_t	lastctgcnt;
1114 	uintptr_t	align;
1115 
1116 	CONTIG_LOCK();
1117 
1118 	if (mattr) {
1119 		lo = mmu_btop((mattr->dma_attr_addr_lo + MMU_PAGEOFFSET));
1120 		hi = mmu_btop(mattr->dma_attr_addr_hi);
1121 		if (hi >= physmax)
1122 			hi = physmax - 1;
1123 		sgllen = mattr->dma_attr_sgllen;
1124 		pfnseg = mmu_btop(mattr->dma_attr_seg);
1125 
1126 		align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
1127 		if (align > MMU_PAGESIZE)
1128 			pfnalign = mmu_btop(align);
1129 
1130 		/*
1131 		 * in order to satisfy the request, must minimally
1132 		 * acquire minctg contiguous pages
1133 		 */
1134 		minctg = howmany(*pgcnt, sgllen);
1135 
1136 		ASSERT(hi >= lo);
1137 
1138 		/*
1139 		 * start from where last searched if the minctg >= lastctgcnt
1140 		 */
1141 		if (minctg < lastctgcnt || startpfn < lo || startpfn > hi)
1142 			startpfn = lo;
1143 	} else {
1144 		hi = physmax - 1;
1145 		lo = 0;
1146 		sgllen = 1;
1147 		pfnseg = mmu.highest_pfn;
1148 		minctg = *pgcnt;
1149 
1150 		if (minctg < lastctgcnt)
1151 			startpfn = lo;
1152 	}
1153 	lastctgcnt = minctg;
1154 
1155 	ASSERT(pfnseg + 1 >= (uint64_t)minctg);
1156 
1157 	/* conserve 16m memory - start search above 16m when possible */
1158 	if (hi > PFN_16M && startpfn < PFN_16M)
1159 		startpfn = PFN_16M;
1160 
1161 	pfn = startpfn;
1162 	if (pfnalign)
1163 		pfn = P2ROUNDUP(pfn, pfnalign);
1164 
1165 	while (pfn + minctg - 1 <= hi) {
1166 
1167 		plist = is_contigpage_free(&pfn, pgcnt, minctg, pfnseg, iolock);
1168 		if (plist) {
1169 			page_list_concat(&pplist, &plist);
1170 			sgllen--;
1171 			/*
1172 			 * return when contig pages no longer needed
1173 			 */
1174 			if (!*pgcnt || ((*pgcnt <= sgllen) && !pfnalign)) {
1175 				startpfn = pfn;
1176 				CONTIG_UNLOCK();
1177 				check_dma(mattr, pplist, *pgcnt);
1178 				return (pplist);
1179 			}
1180 			minctg = howmany(*pgcnt, sgllen);
1181 		}
1182 		if (pfnalign)
1183 			pfn = P2ROUNDUP(pfn, pfnalign);
1184 	}
1185 
1186 	/* cannot find contig pages in specified range */
1187 	if (startpfn == lo) {
1188 		CONTIG_UNLOCK();
1189 		return (NULL);
1190 	}
1191 
1192 	/* did not start with lo previously */
1193 	pfn = lo;
1194 	if (pfnalign)
1195 		pfn = P2ROUNDUP(pfn, pfnalign);
1196 
1197 	/* allow search to go above startpfn */
1198 	while (pfn < startpfn) {
1199 
1200 		plist = is_contigpage_free(&pfn, pgcnt, minctg, pfnseg, iolock);
1201 		if (plist != NULL) {
1202 
1203 			page_list_concat(&pplist, &plist);
1204 			sgllen--;
1205 
1206 			/*
1207 			 * return when contig pages no longer needed
1208 			 */
1209 			if (!*pgcnt || ((*pgcnt <= sgllen) && !pfnalign)) {
1210 				startpfn = pfn;
1211 				CONTIG_UNLOCK();
1212 				check_dma(mattr, pplist, *pgcnt);
1213 				return (pplist);
1214 			}
1215 			minctg = howmany(*pgcnt, sgllen);
1216 		}
1217 		if (pfnalign)
1218 			pfn = P2ROUNDUP(pfn, pfnalign);
1219 	}
1220 	CONTIG_UNLOCK();
1221 	return (NULL);
1222 }
1223 #endif	/* !__xpv */
1224 
1225 /*
1226  * mnode_range_cnt() calculates the number of memory ranges for mnode and
1227  * memranges[]. Used to determine the size of page lists and mnoderanges.
1228  */
1229 int
1230 mnode_range_cnt(int mnode)
1231 {
1232 #if defined(__xpv)
1233 	ASSERT(mnode == 0);
1234 	return (1);
1235 #else	/* __xpv */
1236 	int	mri;
1237 	int	mnrcnt = 0;
1238 
1239 	if (mem_node_config[mnode].exists != 0) {
1240 		mri = nranges - 1;
1241 
1242 		/* find the memranges index below contained in mnode range */
1243 
1244 		while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
1245 			mri--;
1246 
1247 		/*
1248 		 * increment mnode range counter when memranges or mnode
1249 		 * boundary is reached.
1250 		 */
1251 		while (mri >= 0 &&
1252 		    mem_node_config[mnode].physmax >= MEMRANGELO(mri)) {
1253 			mnrcnt++;
1254 			if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
1255 				mri--;
1256 			else
1257 				break;
1258 		}
1259 	}
1260 	ASSERT(mnrcnt <= MAX_MNODE_MRANGES);
1261 	return (mnrcnt);
1262 #endif	/* __xpv */
1263 }
1264 
1265 /*
1266  * mnode_range_setup() initializes mnoderanges.
1267  */
1268 void
1269 mnode_range_setup(mnoderange_t *mnoderanges)
1270 {
1271 	mnoderange_t *mp = mnoderanges;
1272 	int	mnode, mri;
1273 	int	mindex = 0;	/* current index into mnoderanges array */
1274 	int	i, j;
1275 	pfn_t	hipfn;
1276 	int	last, hi;
1277 
1278 	for (mnode = 0; mnode < max_mem_nodes; mnode++) {
1279 		if (mem_node_config[mnode].exists == 0)
1280 			continue;
1281 
1282 		mri = nranges - 1;
1283 
1284 		while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
1285 			mri--;
1286 
1287 		while (mri >= 0 && mem_node_config[mnode].physmax >=
1288 		    MEMRANGELO(mri)) {
1289 			mnoderanges->mnr_pfnlo = MAX(MEMRANGELO(mri),
1290 			    mem_node_config[mnode].physbase);
1291 			mnoderanges->mnr_pfnhi = MIN(MEMRANGEHI(mri),
1292 			    mem_node_config[mnode].physmax);
1293 			mnoderanges->mnr_mnode = mnode;
1294 			mnoderanges->mnr_memrange = mri;
1295 			mnoderanges->mnr_exists = 1;
1296 			mnoderanges++;
1297 			mindex++;
1298 			if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
1299 				mri--;
1300 			else
1301 				break;
1302 		}
1303 	}
1304 
1305 	/*
1306 	 * For now do a simple sort of the mnoderanges array to fill in
1307 	 * the mnr_next fields.  Since mindex is expected to be relatively
1308 	 * small, using a simple O(N^2) algorithm.
1309 	 */
1310 	for (i = 0; i < mindex; i++) {
1311 		if (mp[i].mnr_pfnlo == 0)	/* find lowest */
1312 			break;
1313 	}
1314 	ASSERT(i < mindex);
1315 	last = i;
1316 	mtype16m = last;
1317 	mp[last].mnr_next = -1;
1318 	for (i = 0; i < mindex - 1; i++) {
1319 		hipfn = (pfn_t)(-1);
1320 		hi = -1;
1321 		/* find next highest mnode range */
1322 		for (j = 0; j < mindex; j++) {
1323 			if (mp[j].mnr_pfnlo > mp[last].mnr_pfnlo &&
1324 			    mp[j].mnr_pfnlo < hipfn) {
1325 				hipfn = mp[j].mnr_pfnlo;
1326 				hi = j;
1327 			}
1328 		}
1329 		mp[hi].mnr_next = last;
1330 		last = hi;
1331 	}
1332 	mtypetop = last;
1333 }
1334 
1335 #ifndef	__xpv
1336 /*
1337  * Update mnoderanges for memory hot-add DR operations.
1338  */
1339 static void
1340 mnode_range_add(int mnode)
1341 {
1342 	int	*prev;
1343 	int	n, mri;
1344 	pfn_t	start, end;
1345 	extern	void membar_sync(void);
1346 
1347 	ASSERT(0 <= mnode && mnode < max_mem_nodes);
1348 	ASSERT(mem_node_config[mnode].exists);
1349 	start = mem_node_config[mnode].physbase;
1350 	end = mem_node_config[mnode].physmax;
1351 	ASSERT(start <= end);
1352 	mutex_enter(&mnoderange_lock);
1353 
1354 #ifdef	DEBUG
1355 	/* Check whether it interleaves with other memory nodes. */
1356 	for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) {
1357 		ASSERT(mnoderanges[n].mnr_exists);
1358 		if (mnoderanges[n].mnr_mnode == mnode)
1359 			continue;
1360 		ASSERT(start > mnoderanges[n].mnr_pfnhi ||
1361 		    end < mnoderanges[n].mnr_pfnlo);
1362 	}
1363 #endif	/* DEBUG */
1364 
1365 	mri = nranges - 1;
1366 	while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
1367 		mri--;
1368 	while (mri >= 0 && mem_node_config[mnode].physmax >= MEMRANGELO(mri)) {
1369 		/* Check whether mtype already exists. */
1370 		for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) {
1371 			if (mnoderanges[n].mnr_mnode == mnode &&
1372 			    mnoderanges[n].mnr_memrange == mri) {
1373 				mnoderanges[n].mnr_pfnlo = MAX(MEMRANGELO(mri),
1374 				    start);
1375 				mnoderanges[n].mnr_pfnhi = MIN(MEMRANGEHI(mri),
1376 				    end);
1377 				break;
1378 			}
1379 		}
1380 
1381 		/* Add a new entry if it doesn't exist yet. */
1382 		if (n == -1) {
1383 			/* Try to find an unused entry in mnoderanges array. */
1384 			for (n = 0; n < mnoderangecnt; n++) {
1385 				if (mnoderanges[n].mnr_exists == 0)
1386 					break;
1387 			}
1388 			ASSERT(n < mnoderangecnt);
1389 			mnoderanges[n].mnr_pfnlo = MAX(MEMRANGELO(mri), start);
1390 			mnoderanges[n].mnr_pfnhi = MIN(MEMRANGEHI(mri), end);
1391 			mnoderanges[n].mnr_mnode = mnode;
1392 			mnoderanges[n].mnr_memrange = mri;
1393 			mnoderanges[n].mnr_exists = 1;
1394 			/* Page 0 should always be present. */
1395 			for (prev = &mtypetop;
1396 			    mnoderanges[*prev].mnr_pfnlo > start;
1397 			    prev = &mnoderanges[*prev].mnr_next) {
1398 				ASSERT(mnoderanges[*prev].mnr_next >= 0);
1399 				ASSERT(mnoderanges[*prev].mnr_pfnlo > end);
1400 			}
1401 			mnoderanges[n].mnr_next = *prev;
1402 			membar_sync();
1403 			*prev = n;
1404 		}
1405 
1406 		if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
1407 			mri--;
1408 		else
1409 			break;
1410 	}
1411 
1412 	mutex_exit(&mnoderange_lock);
1413 }
1414 
1415 /*
1416  * Update mnoderanges for memory hot-removal DR operations.
1417  */
1418 static void
1419 mnode_range_del(int mnode)
1420 {
1421 	_NOTE(ARGUNUSED(mnode));
1422 	ASSERT(0 <= mnode && mnode < max_mem_nodes);
1423 	/* TODO: support deletion operation. */
1424 	ASSERT(0);
1425 }
1426 
1427 void
1428 plat_slice_add(pfn_t start, pfn_t end)
1429 {
1430 	mem_node_add_slice(start, end);
1431 	if (plat_dr_enabled()) {
1432 		mnode_range_add(PFN_2_MEM_NODE(start));
1433 	}
1434 }
1435 
1436 void
1437 plat_slice_del(pfn_t start, pfn_t end)
1438 {
1439 	ASSERT(PFN_2_MEM_NODE(start) == PFN_2_MEM_NODE(end));
1440 	ASSERT(plat_dr_enabled());
1441 	mnode_range_del(PFN_2_MEM_NODE(start));
1442 	mem_node_del_slice(start, end);
1443 }
1444 #endif	/* __xpv */
1445 
1446 /*ARGSUSED*/
1447 int
1448 mtype_init(vnode_t *vp, caddr_t vaddr, uint_t *flags, size_t pgsz)
1449 {
1450 	int mtype = mtypetop;
1451 
1452 #if !defined(__xpv)
1453 #if defined(__i386)
1454 	/*
1455 	 * set the mtype range
1456 	 * - kmem requests need to be below 4g if restricted_kmemalloc is set.
1457 	 * - for non kmem requests, set range to above 4g if memory below 4g
1458 	 * runs low.
1459 	 */
1460 	if (restricted_kmemalloc && VN_ISKAS(vp) &&
1461 	    (caddr_t)(vaddr) >= kernelheap &&
1462 	    (caddr_t)(vaddr) < ekernelheap) {
1463 		ASSERT(physmax4g);
1464 		mtype = mtype4g;
1465 		if (RESTRICT16M_ALLOC(freemem4g - btop(pgsz),
1466 		    btop(pgsz), *flags)) {
1467 			*flags |= PGI_MT_RANGE16M;
1468 		} else {
1469 			VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1470 			VM_STAT_COND_ADD((*flags & PG_PANIC),
1471 			    vmm_vmstats.pgpanicalloc);
1472 			*flags |= PGI_MT_RANGE0;
1473 		}
1474 		return (mtype);
1475 	}
1476 #endif	/* __i386 */
1477 
1478 	if (RESTRICT4G_ALLOC) {
1479 		VM_STAT_ADD(vmm_vmstats.restrict4gcnt);
1480 		/* here only for > 4g systems */
1481 		*flags |= PGI_MT_RANGE4G;
1482 	} else if (RESTRICT16M_ALLOC(freemem, btop(pgsz), *flags)) {
1483 		*flags |= PGI_MT_RANGE16M;
1484 	} else {
1485 		VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1486 		VM_STAT_COND_ADD((*flags & PG_PANIC), vmm_vmstats.pgpanicalloc);
1487 		*flags |= PGI_MT_RANGE0;
1488 	}
1489 #endif /* !__xpv */
1490 	return (mtype);
1491 }
1492 
1493 
1494 /* mtype init for page_get_replacement_page */
1495 /*ARGSUSED*/
1496 int
1497 mtype_pgr_init(int *flags, page_t *pp, int mnode, pgcnt_t pgcnt)
1498 {
1499 	int mtype = mtypetop;
1500 #if !defined(__xpv)
1501 	if (RESTRICT16M_ALLOC(freemem, pgcnt, *flags)) {
1502 		*flags |= PGI_MT_RANGE16M;
1503 	} else {
1504 		VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1505 		*flags |= PGI_MT_RANGE0;
1506 	}
1507 #endif
1508 	return (mtype);
1509 }
1510 
1511 /*
1512  * Determine if the mnode range specified in mtype contains memory belonging
1513  * to memory node mnode.  If flags & PGI_MT_RANGE is set then mtype contains
1514  * the range from high pfn to 0, 16m or 4g.
1515  *
1516  * Return first mnode range type index found otherwise return -1 if none found.
1517  */
1518 int
1519 mtype_func(int mnode, int mtype, uint_t flags)
1520 {
1521 	if (flags & PGI_MT_RANGE) {
1522 		int	mnr_lim = MRI_0;
1523 
1524 		if (flags & PGI_MT_NEXT) {
1525 			mtype = mnoderanges[mtype].mnr_next;
1526 		}
1527 		if (flags & PGI_MT_RANGE4G)
1528 			mnr_lim = MRI_4G;	/* exclude 0-4g range */
1529 		else if (flags & PGI_MT_RANGE16M)
1530 			mnr_lim = MRI_16M;	/* exclude 0-16m range */
1531 		while (mtype != -1 &&
1532 		    mnoderanges[mtype].mnr_memrange <= mnr_lim) {
1533 			if (mnoderanges[mtype].mnr_mnode == mnode)
1534 				return (mtype);
1535 			mtype = mnoderanges[mtype].mnr_next;
1536 		}
1537 	} else if (mnoderanges[mtype].mnr_mnode == mnode) {
1538 		return (mtype);
1539 	}
1540 	return (-1);
1541 }
1542 
1543 /*
1544  * Update the page list max counts with the pfn range specified by the
1545  * input parameters.
1546  */
1547 void
1548 mtype_modify_max(pfn_t startpfn, long cnt)
1549 {
1550 	int		mtype;
1551 	pgcnt_t		inc;
1552 	spgcnt_t	scnt = (spgcnt_t)(cnt);
1553 	pgcnt_t		acnt = ABS(scnt);
1554 	pfn_t		endpfn = startpfn + acnt;
1555 	pfn_t		pfn, lo;
1556 
1557 	if (!physmax4g)
1558 		return;
1559 
1560 	mtype = mtypetop;
1561 	for (pfn = endpfn; pfn > startpfn; ) {
1562 		ASSERT(mtype != -1);
1563 		lo = mnoderanges[mtype].mnr_pfnlo;
1564 		if (pfn > lo) {
1565 			if (startpfn >= lo) {
1566 				inc = pfn - startpfn;
1567 			} else {
1568 				inc = pfn - lo;
1569 			}
1570 			if (mnoderanges[mtype].mnr_memrange != MRI_4G) {
1571 				if (scnt > 0)
1572 					maxmem4g += inc;
1573 				else
1574 					maxmem4g -= inc;
1575 			}
1576 			pfn -= inc;
1577 		}
1578 		mtype = mnoderanges[mtype].mnr_next;
1579 	}
1580 }
1581 
1582 int
1583 mtype_2_mrange(int mtype)
1584 {
1585 	return (mnoderanges[mtype].mnr_memrange);
1586 }
1587 
1588 void
1589 mnodetype_2_pfn(int mnode, int mtype, pfn_t *pfnlo, pfn_t *pfnhi)
1590 {
1591 	_NOTE(ARGUNUSED(mnode));
1592 	ASSERT(mnoderanges[mtype].mnr_mnode == mnode);
1593 	*pfnlo = mnoderanges[mtype].mnr_pfnlo;
1594 	*pfnhi = mnoderanges[mtype].mnr_pfnhi;
1595 }
1596 
1597 size_t
1598 plcnt_sz(size_t ctrs_sz)
1599 {
1600 #ifdef DEBUG
1601 	int	szc, colors;
1602 
1603 	ctrs_sz += mnoderangecnt * sizeof (struct mnr_mts) * mmu_page_sizes;
1604 	for (szc = 0; szc < mmu_page_sizes; szc++) {
1605 		colors = page_get_pagecolors(szc);
1606 		ctrs_sz += mnoderangecnt * sizeof (pgcnt_t) * colors;
1607 	}
1608 #endif
1609 	return (ctrs_sz);
1610 }
1611 
1612 caddr_t
1613 plcnt_init(caddr_t addr)
1614 {
1615 #ifdef DEBUG
1616 	int	mt, szc, colors;
1617 
1618 	for (mt = 0; mt < mnoderangecnt; mt++) {
1619 		mnoderanges[mt].mnr_mts = (struct mnr_mts *)addr;
1620 		addr += (sizeof (struct mnr_mts) * mmu_page_sizes);
1621 		for (szc = 0; szc < mmu_page_sizes; szc++) {
1622 			colors = page_get_pagecolors(szc);
1623 			mnoderanges[mt].mnr_mts[szc].mnr_mts_colors = colors;
1624 			mnoderanges[mt].mnr_mts[szc].mnr_mtsc_pgcnt =
1625 			    (pgcnt_t *)addr;
1626 			addr += (sizeof (pgcnt_t) * colors);
1627 		}
1628 	}
1629 #endif
1630 	return (addr);
1631 }
1632 
1633 void
1634 plcnt_inc_dec(page_t *pp, int mtype, int szc, long cnt, int flags)
1635 {
1636 	_NOTE(ARGUNUSED(pp));
1637 #ifdef DEBUG
1638 	int	bin = PP_2_BIN(pp);
1639 
1640 	atomic_add_long(&mnoderanges[mtype].mnr_mts[szc].mnr_mts_pgcnt, cnt);
1641 	atomic_add_long(&mnoderanges[mtype].mnr_mts[szc].mnr_mtsc_pgcnt[bin],
1642 	    cnt);
1643 #endif
1644 	ASSERT(mtype == PP_2_MTYPE(pp));
1645 	if (physmax4g && mnoderanges[mtype].mnr_memrange != MRI_4G)
1646 		atomic_add_long(&freemem4g, cnt);
1647 	if (flags & PG_CACHE_LIST)
1648 		atomic_add_long(&mnoderanges[mtype].mnr_mt_clpgcnt, cnt);
1649 	else
1650 		atomic_add_long(&mnoderanges[mtype].mnr_mt_flpgcnt[szc], cnt);
1651 	atomic_add_long(&mnoderanges[mtype].mnr_mt_totcnt, cnt);
1652 }
1653 
1654 /*
1655  * Returns the free page count for mnode
1656  */
1657 int
1658 mnode_pgcnt(int mnode)
1659 {
1660 	int	mtype = mtypetop;
1661 	int	flags = PGI_MT_RANGE0;
1662 	pgcnt_t	pgcnt = 0;
1663 
1664 	mtype = mtype_func(mnode, mtype, flags);
1665 
1666 	while (mtype != -1) {
1667 		pgcnt += MTYPE_FREEMEM(mtype);
1668 		mtype = mtype_func(mnode, mtype, flags | PGI_MT_NEXT);
1669 	}
1670 	return (pgcnt);
1671 }
1672 
1673 /*
1674  * Initialize page coloring variables based on the l2 cache parameters.
1675  * Calculate and return memory needed for page coloring data structures.
1676  */
1677 size_t
1678 page_coloring_init(uint_t l2_sz, int l2_linesz, int l2_assoc)
1679 {
1680 	_NOTE(ARGUNUSED(l2_linesz));
1681 	size_t	colorsz = 0;
1682 	int	i;
1683 	int	colors;
1684 
1685 #if defined(__xpv)
1686 	/*
1687 	 * Hypervisor domains currently don't have any concept of NUMA.
1688 	 * Hence we'll act like there is only 1 memrange.
1689 	 */
1690 	i = memrange_num(1);
1691 #else /* !__xpv */
1692 	/*
1693 	 * Reduce the memory ranges lists if we don't have large amounts
1694 	 * of memory. This avoids searching known empty free lists.
1695 	 * To support memory DR operations, we need to keep memory ranges
1696 	 * for possible memory hot-add operations.
1697 	 */
1698 	if (plat_dr_physmax > physmax)
1699 		i = memrange_num(plat_dr_physmax);
1700 	else
1701 		i = memrange_num(physmax);
1702 #if defined(__i386)
1703 	if (i > MRI_4G)
1704 		restricted_kmemalloc = 0;
1705 #endif
1706 	/* physmax greater than 4g */
1707 	if (i == MRI_4G)
1708 		physmax4g = 1;
1709 #endif /* !__xpv */
1710 	memranges += i;
1711 	nranges -= i;
1712 
1713 	ASSERT(mmu_page_sizes <= MMU_PAGE_SIZES);
1714 
1715 	ASSERT(ISP2(l2_linesz));
1716 	ASSERT(l2_sz > MMU_PAGESIZE);
1717 
1718 	/* l2_assoc is 0 for fully associative l2 cache */
1719 	if (l2_assoc)
1720 		l2_colors = MAX(1, l2_sz / (l2_assoc * MMU_PAGESIZE));
1721 	else
1722 		l2_colors = 1;
1723 
1724 	ASSERT(ISP2(l2_colors));
1725 
1726 	/* for scalability, configure at least PAGE_COLORS_MIN color bins */
1727 	page_colors = MAX(l2_colors, PAGE_COLORS_MIN);
1728 
1729 	/*
1730 	 * cpu_page_colors is non-zero when a page color may be spread across
1731 	 * multiple bins.
1732 	 */
1733 	if (l2_colors < page_colors)
1734 		cpu_page_colors = l2_colors;
1735 
1736 	ASSERT(ISP2(page_colors));
1737 
1738 	page_colors_mask = page_colors - 1;
1739 
1740 	ASSERT(ISP2(CPUSETSIZE()));
1741 	page_coloring_shift = lowbit(CPUSETSIZE());
1742 
1743 	/* initialize number of colors per page size */
1744 	for (i = 0; i <= mmu.max_page_level; i++) {
1745 		hw_page_array[i].hp_size = LEVEL_SIZE(i);
1746 		hw_page_array[i].hp_shift = LEVEL_SHIFT(i);
1747 		hw_page_array[i].hp_pgcnt = LEVEL_SIZE(i) >> LEVEL_SHIFT(0);
1748 		hw_page_array[i].hp_colors = (page_colors_mask >>
1749 		    (hw_page_array[i].hp_shift - hw_page_array[0].hp_shift))
1750 		    + 1;
1751 		colorequivszc[i] = 0;
1752 	}
1753 
1754 	/*
1755 	 * The value of cpu_page_colors determines if additional color bins
1756 	 * need to be checked for a particular color in the page_get routines.
1757 	 */
1758 	if (cpu_page_colors != 0) {
1759 
1760 		int a = lowbit(page_colors) - lowbit(cpu_page_colors);
1761 		ASSERT(a > 0);
1762 		ASSERT(a < 16);
1763 
1764 		for (i = 0; i <= mmu.max_page_level; i++) {
1765 			if ((colors = hw_page_array[i].hp_colors) <= 1) {
1766 				colorequivszc[i] = 0;
1767 				continue;
1768 			}
1769 			while ((colors >> a) == 0)
1770 				a--;
1771 			ASSERT(a >= 0);
1772 
1773 			/* higher 4 bits encodes color equiv mask */
1774 			colorequivszc[i] = (a << 4);
1775 		}
1776 	}
1777 
1778 	/* factor in colorequiv to check additional 'equivalent' bins. */
1779 	if (colorequiv > 1) {
1780 
1781 		int a = lowbit(colorequiv) - 1;
1782 		if (a > 15)
1783 			a = 15;
1784 
1785 		for (i = 0; i <= mmu.max_page_level; i++) {
1786 			if ((colors = hw_page_array[i].hp_colors) <= 1) {
1787 				continue;
1788 			}
1789 			while ((colors >> a) == 0)
1790 				a--;
1791 			if ((a << 4) > colorequivszc[i]) {
1792 				colorequivszc[i] = (a << 4);
1793 			}
1794 		}
1795 	}
1796 
1797 	/* size for mnoderanges */
1798 	for (mnoderangecnt = 0, i = 0; i < max_mem_nodes; i++)
1799 		mnoderangecnt += mnode_range_cnt(i);
1800 	if (plat_dr_support_memory()) {
1801 		/*
1802 		 * Reserve enough space for memory DR operations.
1803 		 * Two extra mnoderanges for possbile fragmentations,
1804 		 * one for the 2G boundary and the other for the 4G boundary.
1805 		 * We don't expect a memory board crossing the 16M boundary
1806 		 * for memory hot-add operations on x86 platforms.
1807 		 */
1808 		mnoderangecnt += 2 + max_mem_nodes - lgrp_plat_node_cnt;
1809 	}
1810 	colorsz = mnoderangecnt * sizeof (mnoderange_t);
1811 
1812 	/* size for fpc_mutex and cpc_mutex */
1813 	colorsz += (2 * max_mem_nodes * sizeof (kmutex_t) * NPC_MUTEX);
1814 
1815 	/* size of page_freelists */
1816 	colorsz += mnoderangecnt * sizeof (page_t ***);
1817 	colorsz += mnoderangecnt * mmu_page_sizes * sizeof (page_t **);
1818 
1819 	for (i = 0; i < mmu_page_sizes; i++) {
1820 		colors = page_get_pagecolors(i);
1821 		colorsz += mnoderangecnt * colors * sizeof (page_t *);
1822 	}
1823 
1824 	/* size of page_cachelists */
1825 	colorsz += mnoderangecnt * sizeof (page_t **);
1826 	colorsz += mnoderangecnt * page_colors * sizeof (page_t *);
1827 
1828 	return (colorsz);
1829 }
1830 
1831 /*
1832  * Called once at startup to configure page_coloring data structures and
1833  * does the 1st page_free()/page_freelist_add().
1834  */
1835 void
1836 page_coloring_setup(caddr_t pcmemaddr)
1837 {
1838 	int	i;
1839 	int	j;
1840 	int	k;
1841 	caddr_t	addr;
1842 	int	colors;
1843 
1844 	/*
1845 	 * do page coloring setup
1846 	 */
1847 	addr = pcmemaddr;
1848 
1849 	mnoderanges = (mnoderange_t *)addr;
1850 	addr += (mnoderangecnt * sizeof (mnoderange_t));
1851 
1852 	mnode_range_setup(mnoderanges);
1853 
1854 	if (physmax4g)
1855 		mtype4g = pfn_2_mtype(0xfffff);
1856 
1857 	for (k = 0; k < NPC_MUTEX; k++) {
1858 		fpc_mutex[k] = (kmutex_t *)addr;
1859 		addr += (max_mem_nodes * sizeof (kmutex_t));
1860 	}
1861 	for (k = 0; k < NPC_MUTEX; k++) {
1862 		cpc_mutex[k] = (kmutex_t *)addr;
1863 		addr += (max_mem_nodes * sizeof (kmutex_t));
1864 	}
1865 	page_freelists = (page_t ****)addr;
1866 	addr += (mnoderangecnt * sizeof (page_t ***));
1867 
1868 	page_cachelists = (page_t ***)addr;
1869 	addr += (mnoderangecnt * sizeof (page_t **));
1870 
1871 	for (i = 0; i < mnoderangecnt; i++) {
1872 		page_freelists[i] = (page_t ***)addr;
1873 		addr += (mmu_page_sizes * sizeof (page_t **));
1874 
1875 		for (j = 0; j < mmu_page_sizes; j++) {
1876 			colors = page_get_pagecolors(j);
1877 			page_freelists[i][j] = (page_t **)addr;
1878 			addr += (colors * sizeof (page_t *));
1879 		}
1880 		page_cachelists[i] = (page_t **)addr;
1881 		addr += (page_colors * sizeof (page_t *));
1882 	}
1883 }
1884 
1885 #if defined(__xpv)
1886 /*
1887  * Give back 10% of the io_pool pages to the free list.
1888  * Don't shrink the pool below some absolute minimum.
1889  */
1890 static void
1891 page_io_pool_shrink()
1892 {
1893 	int retcnt;
1894 	page_t *pp, *pp_first, *pp_last, **curpool;
1895 	mfn_t mfn;
1896 	int bothpools = 0;
1897 
1898 	mutex_enter(&io_pool_lock);
1899 	io_pool_shrink_attempts++;	/* should be a kstat? */
1900 	retcnt = io_pool_cnt / 10;
1901 	if (io_pool_cnt - retcnt < io_pool_cnt_min)
1902 		retcnt = io_pool_cnt - io_pool_cnt_min;
1903 	if (retcnt <= 0)
1904 		goto done;
1905 	io_pool_shrinks++;	/* should be a kstat? */
1906 	curpool = &io_pool_4g;
1907 domore:
1908 	/*
1909 	 * Loop through taking pages from the end of the list
1910 	 * (highest mfns) till amount to return reached.
1911 	 */
1912 	for (pp = *curpool; pp && retcnt > 0; ) {
1913 		pp_first = pp_last = pp->p_prev;
1914 		if (pp_first == *curpool)
1915 			break;
1916 		retcnt--;
1917 		io_pool_cnt--;
1918 		page_io_pool_sub(curpool, pp_first, pp_last);
1919 		if ((mfn = pfn_to_mfn(pp->p_pagenum)) < start_mfn)
1920 			start_mfn = mfn;
1921 		page_free(pp_first, 1);
1922 		pp = *curpool;
1923 	}
1924 	if (retcnt != 0 && !bothpools) {
1925 		/*
1926 		 * If not enough found in less constrained pool try the
1927 		 * more constrained one.
1928 		 */
1929 		curpool = &io_pool_16m;
1930 		bothpools = 1;
1931 		goto domore;
1932 	}
1933 done:
1934 	mutex_exit(&io_pool_lock);
1935 }
1936 
1937 #endif	/* __xpv */
1938 
1939 uint_t
1940 page_create_update_flags_x86(uint_t flags)
1941 {
1942 #if defined(__xpv)
1943 	/*
1944 	 * Check this is an urgent allocation and free pages are depleted.
1945 	 */
1946 	if (!(flags & PG_WAIT) && freemem < desfree)
1947 		page_io_pool_shrink();
1948 #else /* !__xpv */
1949 	/*
1950 	 * page_create_get_something may call this because 4g memory may be
1951 	 * depleted. Set flags to allow for relocation of base page below
1952 	 * 4g if necessary.
1953 	 */
1954 	if (physmax4g)
1955 		flags |= (PGI_PGCPSZC0 | PGI_PGCPHIPRI);
1956 #endif /* __xpv */
1957 	return (flags);
1958 }
1959 
1960 /*ARGSUSED*/
1961 int
1962 bp_color(struct buf *bp)
1963 {
1964 	return (0);
1965 }
1966 
1967 #if defined(__xpv)
1968 
1969 /*
1970  * Take pages out of an io_pool
1971  */
1972 static void
1973 page_io_pool_sub(page_t **poolp, page_t *pp_first, page_t *pp_last)
1974 {
1975 	if (*poolp == pp_first) {
1976 		*poolp = pp_last->p_next;
1977 		if (*poolp == pp_first)
1978 			*poolp = NULL;
1979 	}
1980 	pp_first->p_prev->p_next = pp_last->p_next;
1981 	pp_last->p_next->p_prev = pp_first->p_prev;
1982 	pp_first->p_prev = pp_last;
1983 	pp_last->p_next = pp_first;
1984 }
1985 
1986 /*
1987  * Put a page on the io_pool list. The list is ordered by increasing MFN.
1988  */
1989 static void
1990 page_io_pool_add(page_t **poolp, page_t *pp)
1991 {
1992 	page_t	*look;
1993 	mfn_t	mfn = mfn_list[pp->p_pagenum];
1994 
1995 	if (*poolp == NULL) {
1996 		*poolp = pp;
1997 		pp->p_next = pp;
1998 		pp->p_prev = pp;
1999 		return;
2000 	}
2001 
2002 	/*
2003 	 * Since we try to take pages from the high end of the pool
2004 	 * chances are good that the pages to be put on the list will
2005 	 * go at or near the end of the list. so start at the end and
2006 	 * work backwards.
2007 	 */
2008 	look = (*poolp)->p_prev;
2009 	while (mfn < mfn_list[look->p_pagenum]) {
2010 		look = look->p_prev;
2011 		if (look == (*poolp)->p_prev)
2012 			break; /* backed all the way to front of list */
2013 	}
2014 
2015 	/* insert after look */
2016 	pp->p_prev = look;
2017 	pp->p_next = look->p_next;
2018 	pp->p_next->p_prev = pp;
2019 	look->p_next = pp;
2020 	if (mfn < mfn_list[(*poolp)->p_pagenum]) {
2021 		/*
2022 		 * we inserted a new first list element
2023 		 * adjust pool pointer to newly inserted element
2024 		 */
2025 		*poolp = pp;
2026 	}
2027 }
2028 
2029 /*
2030  * Add a page to the io_pool.  Setting the force flag will force the page
2031  * into the io_pool no matter what.
2032  */
2033 static void
2034 add_page_to_pool(page_t *pp, int force)
2035 {
2036 	page_t *highest;
2037 	page_t *freep = NULL;
2038 
2039 	mutex_enter(&io_pool_lock);
2040 	/*
2041 	 * Always keep the scarce low memory pages
2042 	 */
2043 	if (mfn_list[pp->p_pagenum] < PFN_16MEG) {
2044 		++io_pool_cnt;
2045 		page_io_pool_add(&io_pool_16m, pp);
2046 		goto done;
2047 	}
2048 	if (io_pool_cnt < io_pool_cnt_max || force || io_pool_4g == NULL) {
2049 		++io_pool_cnt;
2050 		page_io_pool_add(&io_pool_4g, pp);
2051 	} else {
2052 		highest = io_pool_4g->p_prev;
2053 		if (mfn_list[pp->p_pagenum] < mfn_list[highest->p_pagenum]) {
2054 			page_io_pool_sub(&io_pool_4g, highest, highest);
2055 			page_io_pool_add(&io_pool_4g, pp);
2056 			freep = highest;
2057 		} else {
2058 			freep = pp;
2059 		}
2060 	}
2061 done:
2062 	mutex_exit(&io_pool_lock);
2063 	if (freep)
2064 		page_free(freep, 1);
2065 }
2066 
2067 
2068 int contig_pfn_cnt;	/* no of pfns in the contig pfn list */
2069 int contig_pfn_max;	/* capacity of the contig pfn list */
2070 int next_alloc_pfn;	/* next position in list to start a contig search */
2071 int contig_pfnlist_updates;	/* pfn list update count */
2072 int contig_pfnlist_builds;	/* how many times have we (re)built list */
2073 int contig_pfnlist_buildfailed;	/* how many times has list build failed */
2074 int create_contig_pending;	/* nonzero means taskq creating contig list */
2075 pfn_t *contig_pfn_list = NULL;	/* list of contig pfns in ascending mfn order */
2076 
2077 /*
2078  * Function to use in sorting a list of pfns by their underlying mfns.
2079  */
2080 static int
2081 mfn_compare(const void *pfnp1, const void *pfnp2)
2082 {
2083 	mfn_t mfn1 = mfn_list[*(pfn_t *)pfnp1];
2084 	mfn_t mfn2 = mfn_list[*(pfn_t *)pfnp2];
2085 
2086 	if (mfn1 > mfn2)
2087 		return (1);
2088 	if (mfn1 < mfn2)
2089 		return (-1);
2090 	return (0);
2091 }
2092 
2093 /*
2094  * Compact the contig_pfn_list by tossing all the non-contiguous
2095  * elements from the list.
2096  */
2097 static void
2098 compact_contig_pfn_list(void)
2099 {
2100 	pfn_t pfn, lapfn, prev_lapfn;
2101 	mfn_t mfn;
2102 	int i, newcnt = 0;
2103 
2104 	prev_lapfn = 0;
2105 	for (i = 0; i < contig_pfn_cnt - 1; i++) {
2106 		pfn = contig_pfn_list[i];
2107 		lapfn = contig_pfn_list[i + 1];
2108 		mfn = mfn_list[pfn];
2109 		/*
2110 		 * See if next pfn is for a contig mfn
2111 		 */
2112 		if (mfn_list[lapfn] != mfn + 1)
2113 			continue;
2114 		/*
2115 		 * pfn and lookahead are both put in list
2116 		 * unless pfn is the previous lookahead.
2117 		 */
2118 		if (pfn != prev_lapfn)
2119 			contig_pfn_list[newcnt++] = pfn;
2120 		contig_pfn_list[newcnt++] = lapfn;
2121 		prev_lapfn = lapfn;
2122 	}
2123 	for (i = newcnt; i < contig_pfn_cnt; i++)
2124 		contig_pfn_list[i] = 0;
2125 	contig_pfn_cnt = newcnt;
2126 }
2127 
2128 /*ARGSUSED*/
2129 static void
2130 call_create_contiglist(void *arg)
2131 {
2132 	(void) create_contig_pfnlist(PG_WAIT);
2133 }
2134 
2135 /*
2136  * Create list of freelist pfns that have underlying
2137  * contiguous mfns.  The list is kept in ascending mfn order.
2138  * returns 1 if list created else 0.
2139  */
2140 static int
2141 create_contig_pfnlist(uint_t flags)
2142 {
2143 	pfn_t pfn;
2144 	page_t *pp;
2145 	int ret = 1;
2146 
2147 	mutex_enter(&contig_list_lock);
2148 	if (contig_pfn_list != NULL)
2149 		goto out;
2150 	contig_pfn_max = freemem + (freemem / 10);
2151 	contig_pfn_list = kmem_zalloc(contig_pfn_max * sizeof (pfn_t),
2152 	    (flags & PG_WAIT) ? KM_SLEEP : KM_NOSLEEP);
2153 	if (contig_pfn_list == NULL) {
2154 		/*
2155 		 * If we could not create the contig list (because
2156 		 * we could not sleep for memory).  Dispatch a taskq that can
2157 		 * sleep to get the memory.
2158 		 */
2159 		if (!create_contig_pending) {
2160 			if (taskq_dispatch(system_taskq, call_create_contiglist,
2161 			    NULL, TQ_NOSLEEP) != NULL)
2162 				create_contig_pending = 1;
2163 		}
2164 		contig_pfnlist_buildfailed++;	/* count list build failures */
2165 		ret = 0;
2166 		goto out;
2167 	}
2168 	create_contig_pending = 0;
2169 	ASSERT(contig_pfn_cnt == 0);
2170 	for (pfn = 0; pfn < mfn_count; pfn++) {
2171 		pp = page_numtopp_nolock(pfn);
2172 		if (pp == NULL || !PP_ISFREE(pp))
2173 			continue;
2174 		contig_pfn_list[contig_pfn_cnt] = pfn;
2175 		if (++contig_pfn_cnt == contig_pfn_max)
2176 			break;
2177 	}
2178 	/*
2179 	 * Sanity check the new list.
2180 	 */
2181 	if (contig_pfn_cnt < 2) { /* no contig pfns */
2182 		contig_pfn_cnt = 0;
2183 		contig_pfnlist_buildfailed++;
2184 		kmem_free(contig_pfn_list, contig_pfn_max * sizeof (pfn_t));
2185 		contig_pfn_list = NULL;
2186 		contig_pfn_max = 0;
2187 		ret = 0;
2188 		goto out;
2189 	}
2190 	qsort(contig_pfn_list, contig_pfn_cnt, sizeof (pfn_t), mfn_compare);
2191 	compact_contig_pfn_list();
2192 	/*
2193 	 * Make sure next search of the newly created contiguous pfn
2194 	 * list starts at the beginning of the list.
2195 	 */
2196 	next_alloc_pfn = 0;
2197 	contig_pfnlist_builds++;	/* count list builds */
2198 out:
2199 	mutex_exit(&contig_list_lock);
2200 	return (ret);
2201 }
2202 
2203 
2204 /*
2205  * Toss the current contig pfnlist.  Someone is about to do a massive
2206  * update to pfn<->mfn mappings.  So we have them destroy the list and lock
2207  * it till they are done with their update.
2208  */
2209 void
2210 clear_and_lock_contig_pfnlist()
2211 {
2212 	pfn_t *listp = NULL;
2213 	size_t listsize;
2214 
2215 	mutex_enter(&contig_list_lock);
2216 	if (contig_pfn_list != NULL) {
2217 		listp = contig_pfn_list;
2218 		listsize = contig_pfn_max * sizeof (pfn_t);
2219 		contig_pfn_list = NULL;
2220 		contig_pfn_max = contig_pfn_cnt = 0;
2221 	}
2222 	if (listp != NULL)
2223 		kmem_free(listp, listsize);
2224 }
2225 
2226 /*
2227  * Unlock the contig_pfn_list.  The next attempted use of it will cause
2228  * it to be re-created.
2229  */
2230 void
2231 unlock_contig_pfnlist()
2232 {
2233 	mutex_exit(&contig_list_lock);
2234 }
2235 
2236 /*
2237  * Update the contiguous pfn list in response to a pfn <-> mfn reassignment
2238  */
2239 void
2240 update_contig_pfnlist(pfn_t pfn, mfn_t oldmfn, mfn_t newmfn)
2241 {
2242 	int probe_hi, probe_lo, probe_pos, insert_after, insert_point;
2243 	pfn_t probe_pfn;
2244 	mfn_t probe_mfn;
2245 	int drop_lock = 0;
2246 
2247 	if (mutex_owner(&contig_list_lock) != curthread) {
2248 		drop_lock = 1;
2249 		mutex_enter(&contig_list_lock);
2250 	}
2251 	if (contig_pfn_list == NULL)
2252 		goto done;
2253 	contig_pfnlist_updates++;
2254 	/*
2255 	 * Find the pfn in the current list.  Use a binary chop to locate it.
2256 	 */
2257 	probe_hi = contig_pfn_cnt - 1;
2258 	probe_lo = 0;
2259 	probe_pos = (probe_hi + probe_lo) / 2;
2260 	while ((probe_pfn = contig_pfn_list[probe_pos]) != pfn) {
2261 		if (probe_pos == probe_lo) { /* pfn not in list */
2262 			probe_pos = -1;
2263 			break;
2264 		}
2265 		if (pfn_to_mfn(probe_pfn) <= oldmfn)
2266 			probe_lo = probe_pos;
2267 		else
2268 			probe_hi = probe_pos;
2269 		probe_pos = (probe_hi + probe_lo) / 2;
2270 	}
2271 	if (probe_pos >= 0) {
2272 		/*
2273 		 * Remove pfn from list and ensure next alloc
2274 		 * position stays in bounds.
2275 		 */
2276 		if (--contig_pfn_cnt <= next_alloc_pfn)
2277 			next_alloc_pfn = 0;
2278 		if (contig_pfn_cnt < 2) { /* no contig pfns */
2279 			contig_pfn_cnt = 0;
2280 			kmem_free(contig_pfn_list,
2281 			    contig_pfn_max * sizeof (pfn_t));
2282 			contig_pfn_list = NULL;
2283 			contig_pfn_max = 0;
2284 			goto done;
2285 		}
2286 		ovbcopy(&contig_pfn_list[probe_pos + 1],
2287 		    &contig_pfn_list[probe_pos],
2288 		    (contig_pfn_cnt - probe_pos) * sizeof (pfn_t));
2289 	}
2290 	if (newmfn == MFN_INVALID)
2291 		goto done;
2292 	/*
2293 	 * Check if new mfn has adjacent mfns in the list
2294 	 */
2295 	probe_hi = contig_pfn_cnt - 1;
2296 	probe_lo = 0;
2297 	insert_after = -2;
2298 	do {
2299 		probe_pos = (probe_hi + probe_lo) / 2;
2300 		probe_mfn = pfn_to_mfn(contig_pfn_list[probe_pos]);
2301 		if (newmfn == probe_mfn + 1)
2302 			insert_after = probe_pos;
2303 		else if (newmfn == probe_mfn - 1)
2304 			insert_after = probe_pos - 1;
2305 		if (probe_pos == probe_lo)
2306 			break;
2307 		if (probe_mfn <= newmfn)
2308 			probe_lo = probe_pos;
2309 		else
2310 			probe_hi = probe_pos;
2311 	} while (insert_after == -2);
2312 	/*
2313 	 * If there is space in the list and there are adjacent mfns
2314 	 * insert the pfn in to its proper place in the list.
2315 	 */
2316 	if (insert_after != -2 && contig_pfn_cnt + 1 <= contig_pfn_max) {
2317 		insert_point = insert_after + 1;
2318 		ovbcopy(&contig_pfn_list[insert_point],
2319 		    &contig_pfn_list[insert_point + 1],
2320 		    (contig_pfn_cnt - insert_point) * sizeof (pfn_t));
2321 		contig_pfn_list[insert_point] = pfn;
2322 		contig_pfn_cnt++;
2323 	}
2324 done:
2325 	if (drop_lock)
2326 		mutex_exit(&contig_list_lock);
2327 }
2328 
2329 /*
2330  * Called to (re-)populate the io_pool from the free page lists.
2331  */
2332 long
2333 populate_io_pool(void)
2334 {
2335 	pfn_t pfn;
2336 	mfn_t mfn, max_mfn;
2337 	page_t *pp;
2338 
2339 	/*
2340 	 * Figure out the bounds of the pool on first invocation.
2341 	 * We use a percentage of memory for the io pool size.
2342 	 * we allow that to shrink, but not to less than a fixed minimum
2343 	 */
2344 	if (io_pool_cnt_max == 0) {
2345 		io_pool_cnt_max = physmem / (100 / io_pool_physmem_pct);
2346 		io_pool_cnt_lowater = io_pool_cnt_max;
2347 		/*
2348 		 * This is the first time in populate_io_pool, grab a va to use
2349 		 * when we need to allocate pages.
2350 		 */
2351 		io_pool_kva = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP);
2352 	}
2353 	/*
2354 	 * If we are out of pages in the pool, then grow the size of the pool
2355 	 */
2356 	if (io_pool_cnt == 0) {
2357 		/*
2358 		 * Grow the max size of the io pool by 5%, but never more than
2359 		 * 25% of physical memory.
2360 		 */
2361 		if (io_pool_cnt_max < physmem / 4)
2362 			io_pool_cnt_max += io_pool_cnt_max / 20;
2363 	}
2364 	io_pool_grows++;	/* should be a kstat? */
2365 
2366 	/*
2367 	 * Get highest mfn on this platform, but limit to the 32 bit DMA max.
2368 	 */
2369 	(void) mfn_to_pfn(start_mfn);
2370 	max_mfn = MIN(cached_max_mfn, PFN_4GIG);
2371 	for (mfn = start_mfn; mfn < max_mfn; start_mfn = ++mfn) {
2372 		pfn = mfn_to_pfn(mfn);
2373 		if (pfn & PFN_IS_FOREIGN_MFN)
2374 			continue;
2375 		/*
2376 		 * try to allocate it from free pages
2377 		 */
2378 		pp = page_numtopp_alloc(pfn);
2379 		if (pp == NULL)
2380 			continue;
2381 		PP_CLRFREE(pp);
2382 		add_page_to_pool(pp, 1);
2383 		if (io_pool_cnt >= io_pool_cnt_max)
2384 			break;
2385 	}
2386 
2387 	return (io_pool_cnt);
2388 }
2389 
2390 /*
2391  * Destroy a page that was being used for DMA I/O. It may or
2392  * may not actually go back to the io_pool.
2393  */
2394 void
2395 page_destroy_io(page_t *pp)
2396 {
2397 	mfn_t mfn = mfn_list[pp->p_pagenum];
2398 
2399 	/*
2400 	 * When the page was alloc'd a reservation was made, release it now
2401 	 */
2402 	page_unresv(1);
2403 	/*
2404 	 * Unload translations, if any, then hash out the
2405 	 * page to erase its identity.
2406 	 */
2407 	(void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
2408 	page_hashout(pp, NULL);
2409 
2410 	/*
2411 	 * If the page came from the free lists, just put it back to them.
2412 	 * DomU pages always go on the free lists as well.
2413 	 */
2414 	if (!DOMAIN_IS_INITDOMAIN(xen_info) || mfn >= PFN_4GIG) {
2415 		page_free(pp, 1);
2416 		return;
2417 	}
2418 
2419 	add_page_to_pool(pp, 0);
2420 }
2421 
2422 
2423 long contig_searches;		/* count of times contig pages requested */
2424 long contig_search_restarts;	/* count of contig ranges tried */
2425 long contig_search_failed;	/* count of contig alloc failures */
2426 
2427 /*
2428  * Free partial page list
2429  */
2430 static void
2431 free_partial_list(page_t **pplist)
2432 {
2433 	page_t *pp;
2434 
2435 	while (*pplist != NULL) {
2436 		pp = *pplist;
2437 		page_io_pool_sub(pplist, pp, pp);
2438 		page_free(pp, 1);
2439 	}
2440 }
2441 
2442 /*
2443  * Look thru the contiguous pfns that are not part of the io_pool for
2444  * contiguous free pages.  Return a list of the found pages or NULL.
2445  */
2446 page_t *
2447 find_contig_free(uint_t npages, uint_t flags, uint64_t pfnseg,
2448     pgcnt_t pfnalign)
2449 {
2450 	page_t *pp, *plist = NULL;
2451 	mfn_t mfn, prev_mfn, start_mfn;
2452 	pfn_t pfn;
2453 	int pages_needed, pages_requested;
2454 	int search_start;
2455 
2456 	/*
2457 	 * create the contig pfn list if not already done
2458 	 */
2459 retry:
2460 	mutex_enter(&contig_list_lock);
2461 	if (contig_pfn_list == NULL) {
2462 		mutex_exit(&contig_list_lock);
2463 		if (!create_contig_pfnlist(flags)) {
2464 			return (NULL);
2465 		}
2466 		goto retry;
2467 	}
2468 	contig_searches++;
2469 	/*
2470 	 * Search contiguous pfn list for physically contiguous pages not in
2471 	 * the io_pool.  Start the search where the last search left off.
2472 	 */
2473 	pages_requested = pages_needed = npages;
2474 	search_start = next_alloc_pfn;
2475 	start_mfn = prev_mfn = 0;
2476 	while (pages_needed) {
2477 		pfn = contig_pfn_list[next_alloc_pfn];
2478 		mfn = pfn_to_mfn(pfn);
2479 		/*
2480 		 * Check if mfn is first one or contig to previous one and
2481 		 * if page corresponding to mfn is free and that mfn
2482 		 * range is not crossing a segment boundary.
2483 		 */
2484 		if ((prev_mfn == 0 || mfn == prev_mfn + 1) &&
2485 		    (pp = page_numtopp_alloc(pfn)) != NULL &&
2486 		    !((mfn & pfnseg) < (start_mfn & pfnseg))) {
2487 			PP_CLRFREE(pp);
2488 			page_io_pool_add(&plist, pp);
2489 			pages_needed--;
2490 			if (prev_mfn == 0) {
2491 				if (pfnalign &&
2492 				    mfn != P2ROUNDUP(mfn, pfnalign)) {
2493 					/*
2494 					 * not properly aligned
2495 					 */
2496 					contig_search_restarts++;
2497 					free_partial_list(&plist);
2498 					pages_needed = pages_requested;
2499 					start_mfn = prev_mfn = 0;
2500 					goto skip;
2501 				}
2502 				start_mfn = mfn;
2503 			}
2504 			prev_mfn = mfn;
2505 		} else {
2506 			contig_search_restarts++;
2507 			free_partial_list(&plist);
2508 			pages_needed = pages_requested;
2509 			start_mfn = prev_mfn = 0;
2510 		}
2511 skip:
2512 		if (++next_alloc_pfn == contig_pfn_cnt)
2513 			next_alloc_pfn = 0;
2514 		if (next_alloc_pfn == search_start)
2515 			break; /* all pfns searched */
2516 	}
2517 	mutex_exit(&contig_list_lock);
2518 	if (pages_needed) {
2519 		contig_search_failed++;
2520 		/*
2521 		 * Failed to find enough contig pages.
2522 		 * free partial page list
2523 		 */
2524 		free_partial_list(&plist);
2525 	}
2526 	return (plist);
2527 }
2528 
2529 /*
2530  * Search the reserved io pool pages for a page range with the
2531  * desired characteristics.
2532  */
2533 page_t *
2534 page_io_pool_alloc(ddi_dma_attr_t *mattr, int contig, pgcnt_t minctg)
2535 {
2536 	page_t *pp_first, *pp_last;
2537 	page_t *pp, **poolp;
2538 	pgcnt_t nwanted, pfnalign;
2539 	uint64_t pfnseg;
2540 	mfn_t mfn, tmfn, hi_mfn, lo_mfn;
2541 	int align, attempt = 0;
2542 
2543 	if (minctg == 1)
2544 		contig = 0;
2545 	lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
2546 	hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
2547 	pfnseg = mmu_btop(mattr->dma_attr_seg);
2548 	align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
2549 	if (align > MMU_PAGESIZE)
2550 		pfnalign = mmu_btop(align);
2551 	else
2552 		pfnalign = 0;
2553 
2554 try_again:
2555 	/*
2556 	 * See if we want pages for a legacy device
2557 	 */
2558 	if (hi_mfn < PFN_16MEG)
2559 		poolp = &io_pool_16m;
2560 	else
2561 		poolp = &io_pool_4g;
2562 try_smaller:
2563 	/*
2564 	 * Take pages from I/O pool. We'll use pages from the highest
2565 	 * MFN range possible.
2566 	 */
2567 	pp_first = pp_last = NULL;
2568 	mutex_enter(&io_pool_lock);
2569 	nwanted = minctg;
2570 	for (pp = *poolp; pp && nwanted > 0; ) {
2571 		pp = pp->p_prev;
2572 
2573 		/*
2574 		 * skip pages above allowable range
2575 		 */
2576 		mfn = mfn_list[pp->p_pagenum];
2577 		if (hi_mfn < mfn)
2578 			goto skip;
2579 
2580 		/*
2581 		 * stop at pages below allowable range
2582 		 */
2583 		if (lo_mfn > mfn)
2584 			break;
2585 restart:
2586 		if (pp_last == NULL) {
2587 			/*
2588 			 * Check alignment
2589 			 */
2590 			tmfn = mfn - (minctg - 1);
2591 			if (pfnalign && tmfn != P2ROUNDUP(tmfn, pfnalign))
2592 				goto skip; /* not properly aligned */
2593 			/*
2594 			 * Check segment
2595 			 */
2596 			if ((mfn & pfnseg) < (tmfn & pfnseg))
2597 				goto skip; /* crosses seg boundary */
2598 			/*
2599 			 * Start building page list
2600 			 */
2601 			pp_first = pp_last = pp;
2602 			nwanted--;
2603 		} else {
2604 			/*
2605 			 * check physical contiguity if required
2606 			 */
2607 			if (contig &&
2608 			    mfn_list[pp_first->p_pagenum] != mfn + 1) {
2609 				/*
2610 				 * not a contiguous page, restart list.
2611 				 */
2612 				pp_last = NULL;
2613 				nwanted = minctg;
2614 				goto restart;
2615 			} else { /* add page to list */
2616 				pp_first = pp;
2617 				nwanted--;
2618 			}
2619 		}
2620 skip:
2621 		if (pp == *poolp)
2622 			break;
2623 	}
2624 
2625 	/*
2626 	 * If we didn't find memory. Try the more constrained pool, then
2627 	 * sweep free pages into the DMA pool and try again.
2628 	 */
2629 	if (nwanted != 0) {
2630 		mutex_exit(&io_pool_lock);
2631 		/*
2632 		 * If we were looking in the less constrained pool and
2633 		 * didn't find pages, try the more constrained pool.
2634 		 */
2635 		if (poolp == &io_pool_4g) {
2636 			poolp = &io_pool_16m;
2637 			goto try_smaller;
2638 		}
2639 		kmem_reap();
2640 		if (++attempt < 4) {
2641 			/*
2642 			 * Grab some more io_pool pages
2643 			 */
2644 			(void) populate_io_pool();
2645 			goto try_again; /* go around and retry */
2646 		}
2647 		return (NULL);
2648 	}
2649 	/*
2650 	 * Found the pages, now snip them from the list
2651 	 */
2652 	page_io_pool_sub(poolp, pp_first, pp_last);
2653 	io_pool_cnt -= minctg;
2654 	/*
2655 	 * reset low water mark
2656 	 */
2657 	if (io_pool_cnt < io_pool_cnt_lowater)
2658 		io_pool_cnt_lowater = io_pool_cnt;
2659 	mutex_exit(&io_pool_lock);
2660 	return (pp_first);
2661 }
2662 
2663 page_t *
2664 page_swap_with_hypervisor(struct vnode *vp, u_offset_t off, caddr_t vaddr,
2665     ddi_dma_attr_t *mattr, uint_t flags, pgcnt_t minctg)
2666 {
2667 	uint_t kflags;
2668 	int order, extra, extpages, i, contig, nbits, extents;
2669 	page_t *pp, *expp, *pp_first, **pplist = NULL;
2670 	mfn_t *mfnlist = NULL;
2671 
2672 	contig = flags & PG_PHYSCONTIG;
2673 	if (minctg == 1)
2674 		contig = 0;
2675 	flags &= ~PG_PHYSCONTIG;
2676 	kflags = flags & PG_WAIT ? KM_SLEEP : KM_NOSLEEP;
2677 	/*
2678 	 * Hypervisor will allocate extents, if we want contig
2679 	 * pages extent must be >= minctg
2680 	 */
2681 	if (contig) {
2682 		order = highbit(minctg) - 1;
2683 		if (minctg & ((1 << order) - 1))
2684 			order++;
2685 		extpages = 1 << order;
2686 	} else {
2687 		order = 0;
2688 		extpages = minctg;
2689 	}
2690 	if (extpages > minctg) {
2691 		extra = extpages - minctg;
2692 		if (!page_resv(extra, kflags))
2693 			return (NULL);
2694 	}
2695 	pp_first = NULL;
2696 	pplist = kmem_alloc(extpages * sizeof (page_t *), kflags);
2697 	if (pplist == NULL)
2698 		goto balloon_fail;
2699 	mfnlist = kmem_alloc(extpages * sizeof (mfn_t), kflags);
2700 	if (mfnlist == NULL)
2701 		goto balloon_fail;
2702 	pp = page_create_va(vp, off, minctg * PAGESIZE, flags, &kvseg, vaddr);
2703 	if (pp == NULL)
2704 		goto balloon_fail;
2705 	pp_first = pp;
2706 	if (extpages > minctg) {
2707 		/*
2708 		 * fill out the rest of extent pages to swap
2709 		 * with the hypervisor
2710 		 */
2711 		for (i = 0; i < extra; i++) {
2712 			expp = page_create_va(vp,
2713 			    (u_offset_t)(uintptr_t)io_pool_kva,
2714 			    PAGESIZE, flags, &kvseg, io_pool_kva);
2715 			if (expp == NULL)
2716 				goto balloon_fail;
2717 			(void) hat_pageunload(expp, HAT_FORCE_PGUNLOAD);
2718 			page_io_unlock(expp);
2719 			page_hashout(expp, NULL);
2720 			page_io_lock(expp);
2721 			/*
2722 			 * add page to end of list
2723 			 */
2724 			expp->p_prev = pp_first->p_prev;
2725 			expp->p_next = pp_first;
2726 			expp->p_prev->p_next = expp;
2727 			pp_first->p_prev = expp;
2728 		}
2729 
2730 	}
2731 	for (i = 0; i < extpages; i++) {
2732 		pplist[i] = pp;
2733 		pp = pp->p_next;
2734 	}
2735 	nbits = highbit(mattr->dma_attr_addr_hi);
2736 	extents = contig ? 1 : minctg;
2737 	if (balloon_replace_pages(extents, pplist, nbits, order,
2738 	    mfnlist) != extents) {
2739 		if (ioalloc_dbg)
2740 			cmn_err(CE_NOTE, "request to hypervisor"
2741 			    " for %d pages, maxaddr %" PRIx64 " failed",
2742 			    extpages, mattr->dma_attr_addr_hi);
2743 		goto balloon_fail;
2744 	}
2745 
2746 	kmem_free(pplist, extpages * sizeof (page_t *));
2747 	kmem_free(mfnlist, extpages * sizeof (mfn_t));
2748 	/*
2749 	 * Return any excess pages to free list
2750 	 */
2751 	if (extpages > minctg) {
2752 		for (i = 0; i < extra; i++) {
2753 			pp = pp_first->p_prev;
2754 			page_sub(&pp_first, pp);
2755 			page_io_unlock(pp);
2756 			page_unresv(1);
2757 			page_free(pp, 1);
2758 		}
2759 	}
2760 	return (pp_first);
2761 balloon_fail:
2762 	/*
2763 	 * Return pages to free list and return failure
2764 	 */
2765 	while (pp_first != NULL) {
2766 		pp = pp_first;
2767 		page_sub(&pp_first, pp);
2768 		page_io_unlock(pp);
2769 		if (pp->p_vnode != NULL)
2770 			page_hashout(pp, NULL);
2771 		page_free(pp, 1);
2772 	}
2773 	if (pplist)
2774 		kmem_free(pplist, extpages * sizeof (page_t *));
2775 	if (mfnlist)
2776 		kmem_free(mfnlist, extpages * sizeof (mfn_t));
2777 	page_unresv(extpages - minctg);
2778 	return (NULL);
2779 }
2780 
2781 static void
2782 return_partial_alloc(page_t *plist)
2783 {
2784 	page_t *pp;
2785 
2786 	while (plist != NULL) {
2787 		pp = plist;
2788 		page_sub(&plist, pp);
2789 		page_io_unlock(pp);
2790 		page_destroy_io(pp);
2791 	}
2792 }
2793 
2794 static page_t *
2795 page_get_contigpages(
2796 	struct vnode	*vp,
2797 	u_offset_t	off,
2798 	int		*npagesp,
2799 	uint_t		flags,
2800 	caddr_t		vaddr,
2801 	ddi_dma_attr_t	*mattr)
2802 {
2803 	mfn_t	max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL);
2804 	page_t	*plist;	/* list to return */
2805 	page_t	*pp, *mcpl;
2806 	int	contig, anyaddr, npages, getone = 0;
2807 	mfn_t	lo_mfn;
2808 	mfn_t	hi_mfn;
2809 	pgcnt_t	pfnalign = 0;
2810 	int	align, sgllen;
2811 	uint64_t pfnseg;
2812 	pgcnt_t	minctg;
2813 
2814 	npages = *npagesp;
2815 	ASSERT(mattr != NULL);
2816 	lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
2817 	hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
2818 	sgllen = mattr->dma_attr_sgllen;
2819 	pfnseg = mmu_btop(mattr->dma_attr_seg);
2820 	align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
2821 	if (align > MMU_PAGESIZE)
2822 		pfnalign = mmu_btop(align);
2823 
2824 	contig = flags & PG_PHYSCONTIG;
2825 	if (npages == -1) {
2826 		npages = 1;
2827 		pfnalign = 0;
2828 	}
2829 	/*
2830 	 * Clear the contig flag if only one page is needed.
2831 	 */
2832 	if (npages == 1) {
2833 		getone = 1;
2834 		contig = 0;
2835 	}
2836 
2837 	/*
2838 	 * Check if any page in the system is fine.
2839 	 */
2840 	anyaddr = lo_mfn == 0 && hi_mfn >= max_mfn;
2841 	if (!contig && anyaddr && !pfnalign) {
2842 		flags &= ~PG_PHYSCONTIG;
2843 		plist = page_create_va(vp, off, npages * MMU_PAGESIZE,
2844 		    flags, &kvseg, vaddr);
2845 		if (plist != NULL) {
2846 			*npagesp = 0;
2847 			return (plist);
2848 		}
2849 	}
2850 	plist = NULL;
2851 	minctg = howmany(npages, sgllen);
2852 	while (npages > sgllen || getone) {
2853 		if (minctg > npages)
2854 			minctg = npages;
2855 		mcpl = NULL;
2856 		/*
2857 		 * We could want contig pages with no address range limits.
2858 		 */
2859 		if (anyaddr && contig) {
2860 			/*
2861 			 * Look for free contig pages to satisfy the request.
2862 			 */
2863 			mcpl = find_contig_free(minctg, flags, pfnseg,
2864 			    pfnalign);
2865 		}
2866 		/*
2867 		 * Try the reserved io pools next
2868 		 */
2869 		if (mcpl == NULL)
2870 			mcpl = page_io_pool_alloc(mattr, contig, minctg);
2871 		if (mcpl != NULL) {
2872 			pp = mcpl;
2873 			do {
2874 				if (!page_hashin(pp, vp, off, NULL)) {
2875 					panic("page_get_contigpages:"
2876 					    " hashin failed"
2877 					    " pp %p, vp %p, off %llx",
2878 					    (void *)pp, (void *)vp, off);
2879 				}
2880 				off += MMU_PAGESIZE;
2881 				PP_CLRFREE(pp);
2882 				PP_CLRAGED(pp);
2883 				page_set_props(pp, P_REF);
2884 				page_io_lock(pp);
2885 				pp = pp->p_next;
2886 			} while (pp != mcpl);
2887 		} else {
2888 			/*
2889 			 * Hypervisor exchange doesn't handle segment or
2890 			 * alignment constraints
2891 			 */
2892 			if (mattr->dma_attr_seg < mattr->dma_attr_addr_hi ||
2893 			    pfnalign)
2894 				goto fail;
2895 			/*
2896 			 * Try exchanging pages with the hypervisor
2897 			 */
2898 			mcpl = page_swap_with_hypervisor(vp, off, vaddr, mattr,
2899 			    flags, minctg);
2900 			if (mcpl == NULL)
2901 				goto fail;
2902 			off += minctg * MMU_PAGESIZE;
2903 		}
2904 		check_dma(mattr, mcpl, minctg);
2905 		/*
2906 		 * Here with a minctg run of contiguous pages, add them to the
2907 		 * list we will return for this request.
2908 		 */
2909 		page_list_concat(&plist, &mcpl);
2910 		npages -= minctg;
2911 		*npagesp = npages;
2912 		sgllen--;
2913 		if (getone)
2914 			break;
2915 	}
2916 	return (plist);
2917 fail:
2918 	return_partial_alloc(plist);
2919 	return (NULL);
2920 }
2921 
2922 /*
2923  * Allocator for domain 0 I/O pages. We match the required
2924  * DMA attributes and contiguity constraints.
2925  */
2926 /*ARGSUSED*/
2927 page_t *
2928 page_create_io(
2929 	struct vnode	*vp,
2930 	u_offset_t	off,
2931 	uint_t		bytes,
2932 	uint_t		flags,
2933 	struct as	*as,
2934 	caddr_t		vaddr,
2935 	ddi_dma_attr_t	*mattr)
2936 {
2937 	page_t	*plist = NULL, *pp;
2938 	int	npages = 0, contig, anyaddr, pages_req;
2939 	mfn_t	lo_mfn;
2940 	mfn_t	hi_mfn;
2941 	pgcnt_t	pfnalign = 0;
2942 	int	align;
2943 	int	is_domu = 0;
2944 	int	dummy, bytes_got;
2945 	mfn_t	max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL);
2946 
2947 	ASSERT(mattr != NULL);
2948 	lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
2949 	hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
2950 	align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
2951 	if (align > MMU_PAGESIZE)
2952 		pfnalign = mmu_btop(align);
2953 
2954 	/*
2955 	 * Clear the contig flag if only one page is needed or the scatter
2956 	 * gather list length is >= npages.
2957 	 */
2958 	pages_req = npages = mmu_btopr(bytes);
2959 	contig = (flags & PG_PHYSCONTIG);
2960 	bytes = P2ROUNDUP(bytes, MMU_PAGESIZE);
2961 	if (bytes == MMU_PAGESIZE || mattr->dma_attr_sgllen >= npages)
2962 		contig = 0;
2963 
2964 	/*
2965 	 * Check if any old page in the system is fine.
2966 	 * DomU should always go down this path.
2967 	 */
2968 	is_domu = !DOMAIN_IS_INITDOMAIN(xen_info);
2969 	anyaddr = lo_mfn == 0 && hi_mfn >= max_mfn && !pfnalign;
2970 	if ((!contig && anyaddr) || is_domu) {
2971 		flags &= ~PG_PHYSCONTIG;
2972 		plist = page_create_va(vp, off, bytes, flags, &kvseg, vaddr);
2973 		if (plist != NULL)
2974 			return (plist);
2975 		else if (is_domu)
2976 			return (NULL); /* no memory available */
2977 	}
2978 	/*
2979 	 * DomU should never reach here
2980 	 */
2981 	if (contig) {
2982 		plist = page_get_contigpages(vp, off, &npages, flags, vaddr,
2983 		    mattr);
2984 		if (plist == NULL)
2985 			goto fail;
2986 		bytes_got = (pages_req - npages) << MMU_PAGESHIFT;
2987 		vaddr += bytes_got;
2988 		off += bytes_got;
2989 		/*
2990 		 * We now have all the contiguous pages we need, but
2991 		 * we may still need additional non-contiguous pages.
2992 		 */
2993 	}
2994 	/*
2995 	 * now loop collecting the requested number of pages, these do
2996 	 * not have to be contiguous pages but we will use the contig
2997 	 * page alloc code to get the pages since it will honor any
2998 	 * other constraints the pages may have.
2999 	 */
3000 	while (npages--) {
3001 		dummy = -1;
3002 		pp = page_get_contigpages(vp, off, &dummy, flags, vaddr, mattr);
3003 		if (pp == NULL)
3004 			goto fail;
3005 		page_add(&plist, pp);
3006 		vaddr += MMU_PAGESIZE;
3007 		off += MMU_PAGESIZE;
3008 	}
3009 	return (plist);
3010 fail:
3011 	/*
3012 	 * Failed to get enough pages, return ones we did get
3013 	 */
3014 	return_partial_alloc(plist);
3015 	return (NULL);
3016 }
3017 
3018 /*
3019  * Lock and return the page with the highest mfn that we can find.  last_mfn
3020  * holds the last one found, so the next search can start from there.  We
3021  * also keep a counter so that we don't loop forever if the machine has no
3022  * free pages.
3023  *
3024  * This is called from the balloon thread to find pages to give away.  new_high
3025  * is used when new mfn's have been added to the system - we will reset our
3026  * search if the new mfn's are higher than our current search position.
3027  */
3028 page_t *
3029 page_get_high_mfn(mfn_t new_high)
3030 {
3031 	static mfn_t last_mfn = 0;
3032 	pfn_t pfn;
3033 	page_t *pp;
3034 	ulong_t loop_count = 0;
3035 
3036 	if (new_high > last_mfn)
3037 		last_mfn = new_high;
3038 
3039 	for (; loop_count < mfn_count; loop_count++, last_mfn--) {
3040 		if (last_mfn == 0) {
3041 			last_mfn = cached_max_mfn;
3042 		}
3043 
3044 		pfn = mfn_to_pfn(last_mfn);
3045 		if (pfn & PFN_IS_FOREIGN_MFN)
3046 			continue;
3047 
3048 		/* See if the page is free.  If so, lock it. */
3049 		pp = page_numtopp_alloc(pfn);
3050 		if (pp == NULL)
3051 			continue;
3052 		PP_CLRFREE(pp);
3053 
3054 		ASSERT(PAGE_EXCL(pp));
3055 		ASSERT(pp->p_vnode == NULL);
3056 		ASSERT(!hat_page_is_mapped(pp));
3057 		last_mfn--;
3058 		return (pp);
3059 	}
3060 	return (NULL);
3061 }
3062 
3063 #else /* !__xpv */
3064 
3065 /*
3066  * get a page from any list with the given mnode
3067  */
3068 static page_t *
3069 page_get_mnode_anylist(ulong_t origbin, uchar_t szc, uint_t flags,
3070     int mnode, int mtype, ddi_dma_attr_t *dma_attr)
3071 {
3072 	kmutex_t		*pcm;
3073 	int			i;
3074 	page_t			*pp;
3075 	page_t			*first_pp;
3076 	uint64_t		pgaddr;
3077 	ulong_t			bin;
3078 	int			mtypestart;
3079 	int			plw_initialized;
3080 	page_list_walker_t	plw;
3081 
3082 	VM_STAT_ADD(pga_vmstats.pgma_alloc);
3083 
3084 	ASSERT((flags & PG_MATCH_COLOR) == 0);
3085 	ASSERT(szc == 0);
3086 	ASSERT(dma_attr != NULL);
3087 
3088 	MTYPE_START(mnode, mtype, flags);
3089 	if (mtype < 0) {
3090 		VM_STAT_ADD(pga_vmstats.pgma_allocempty);
3091 		return (NULL);
3092 	}
3093 
3094 	mtypestart = mtype;
3095 
3096 	bin = origbin;
3097 
3098 	/*
3099 	 * check up to page_colors + 1 bins - origbin may be checked twice
3100 	 * because of BIN_STEP skip
3101 	 */
3102 	do {
3103 		plw_initialized = 0;
3104 
3105 		for (plw.plw_count = 0;
3106 		    plw.plw_count < page_colors; plw.plw_count++) {
3107 
3108 			if (PAGE_FREELISTS(mnode, szc, bin, mtype) == NULL)
3109 				goto nextfreebin;
3110 
3111 			pcm = PC_BIN_MUTEX(mnode, bin, PG_FREE_LIST);
3112 			mutex_enter(pcm);
3113 			pp = PAGE_FREELISTS(mnode, szc, bin, mtype);
3114 			first_pp = pp;
3115 			while (pp != NULL) {
3116 				if (IS_DUMP_PAGE(pp) || page_trylock(pp,
3117 				    SE_EXCL) == 0) {
3118 					pp = pp->p_next;
3119 					if (pp == first_pp) {
3120 						pp = NULL;
3121 					}
3122 					continue;
3123 				}
3124 
3125 				ASSERT(PP_ISFREE(pp));
3126 				ASSERT(PP_ISAGED(pp));
3127 				ASSERT(pp->p_vnode == NULL);
3128 				ASSERT(pp->p_hash == NULL);
3129 				ASSERT(pp->p_offset == (u_offset_t)-1);
3130 				ASSERT(pp->p_szc == szc);
3131 				ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode);
3132 				/* check if page within DMA attributes */
3133 				pgaddr = pa_to_ma(pfn_to_pa(pp->p_pagenum));
3134 				if ((pgaddr >= dma_attr->dma_attr_addr_lo) &&
3135 				    (pgaddr + MMU_PAGESIZE - 1 <=
3136 				    dma_attr->dma_attr_addr_hi)) {
3137 					break;
3138 				}
3139 
3140 				/* continue looking */
3141 				page_unlock(pp);
3142 				pp = pp->p_next;
3143 				if (pp == first_pp)
3144 					pp = NULL;
3145 
3146 			}
3147 			if (pp != NULL) {
3148 				ASSERT(mtype == PP_2_MTYPE(pp));
3149 				ASSERT(pp->p_szc == 0);
3150 
3151 				/* found a page with specified DMA attributes */
3152 				page_sub(&PAGE_FREELISTS(mnode, szc, bin,
3153 				    mtype), pp);
3154 				page_ctr_sub(mnode, mtype, pp, PG_FREE_LIST);
3155 
3156 				if ((PP_ISFREE(pp) == 0) ||
3157 				    (PP_ISAGED(pp) == 0)) {
3158 					cmn_err(CE_PANIC, "page %p is not free",
3159 					    (void *)pp);
3160 				}
3161 
3162 				mutex_exit(pcm);
3163 				check_dma(dma_attr, pp, 1);
3164 				VM_STAT_ADD(pga_vmstats.pgma_allocok);
3165 				return (pp);
3166 			}
3167 			mutex_exit(pcm);
3168 nextfreebin:
3169 			if (plw_initialized == 0) {
3170 				page_list_walk_init(szc, 0, bin, 1, 0, &plw);
3171 				ASSERT(plw.plw_ceq_dif == page_colors);
3172 				plw_initialized = 1;
3173 			}
3174 
3175 			if (plw.plw_do_split) {
3176 				pp = page_freelist_split(szc, bin, mnode,
3177 				    mtype,
3178 				    mmu_btop(dma_attr->dma_attr_addr_lo),
3179 				    mmu_btop(dma_attr->dma_attr_addr_hi + 1),
3180 				    &plw);
3181 				if (pp != NULL) {
3182 					check_dma(dma_attr, pp, 1);
3183 					return (pp);
3184 				}
3185 			}
3186 
3187 			bin = page_list_walk_next_bin(szc, bin, &plw);
3188 		}
3189 
3190 		MTYPE_NEXT(mnode, mtype, flags);
3191 	} while (mtype >= 0);
3192 
3193 	/* failed to find a page in the freelist; try it in the cachelist */
3194 
3195 	/* reset mtype start for cachelist search */
3196 	mtype = mtypestart;
3197 	ASSERT(mtype >= 0);
3198 
3199 	/* start with the bin of matching color */
3200 	bin = origbin;
3201 
3202 	do {
3203 		for (i = 0; i <= page_colors; i++) {
3204 			if (PAGE_CACHELISTS(mnode, bin, mtype) == NULL)
3205 				goto nextcachebin;
3206 			pcm = PC_BIN_MUTEX(mnode, bin, PG_CACHE_LIST);
3207 			mutex_enter(pcm);
3208 			pp = PAGE_CACHELISTS(mnode, bin, mtype);
3209 			first_pp = pp;
3210 			while (pp != NULL) {
3211 				if (IS_DUMP_PAGE(pp) || page_trylock(pp,
3212 				    SE_EXCL) == 0) {
3213 					pp = pp->p_next;
3214 					if (pp == first_pp)
3215 						pp = NULL;
3216 					continue;
3217 				}
3218 				ASSERT(pp->p_vnode);
3219 				ASSERT(PP_ISAGED(pp) == 0);
3220 				ASSERT(pp->p_szc == 0);
3221 				ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode);
3222 
3223 				/* check if page within DMA attributes */
3224 
3225 				pgaddr = pa_to_ma(pfn_to_pa(pp->p_pagenum));
3226 				if ((pgaddr >= dma_attr->dma_attr_addr_lo) &&
3227 				    (pgaddr + MMU_PAGESIZE - 1 <=
3228 				    dma_attr->dma_attr_addr_hi)) {
3229 					break;
3230 				}
3231 
3232 				/* continue looking */
3233 				page_unlock(pp);
3234 				pp = pp->p_next;
3235 				if (pp == first_pp)
3236 					pp = NULL;
3237 			}
3238 
3239 			if (pp != NULL) {
3240 				ASSERT(mtype == PP_2_MTYPE(pp));
3241 				ASSERT(pp->p_szc == 0);
3242 
3243 				/* found a page with specified DMA attributes */
3244 				page_sub(&PAGE_CACHELISTS(mnode, bin,
3245 				    mtype), pp);
3246 				page_ctr_sub(mnode, mtype, pp, PG_CACHE_LIST);
3247 
3248 				mutex_exit(pcm);
3249 				ASSERT(pp->p_vnode);
3250 				ASSERT(PP_ISAGED(pp) == 0);
3251 				check_dma(dma_attr, pp, 1);
3252 				VM_STAT_ADD(pga_vmstats.pgma_allocok);
3253 				return (pp);
3254 			}
3255 			mutex_exit(pcm);
3256 nextcachebin:
3257 			bin += (i == 0) ? BIN_STEP : 1;
3258 			bin &= page_colors_mask;
3259 		}
3260 		MTYPE_NEXT(mnode, mtype, flags);
3261 	} while (mtype >= 0);
3262 
3263 	VM_STAT_ADD(pga_vmstats.pgma_allocfailed);
3264 	return (NULL);
3265 }
3266 
3267 /*
3268  * This function is similar to page_get_freelist()/page_get_cachelist()
3269  * but it searches both the lists to find a page with the specified
3270  * color (or no color) and DMA attributes. The search is done in the
3271  * freelist first and then in the cache list within the highest memory
3272  * range (based on DMA attributes) before searching in the lower
3273  * memory ranges.
3274  *
3275  * Note: This function is called only by page_create_io().
3276  */
3277 /*ARGSUSED*/
3278 static page_t *
3279 page_get_anylist(struct vnode *vp, u_offset_t off, struct as *as, caddr_t vaddr,
3280     size_t size, uint_t flags, ddi_dma_attr_t *dma_attr, lgrp_t	*lgrp)
3281 {
3282 	uint_t		bin;
3283 	int		mtype;
3284 	page_t		*pp;
3285 	int		n;
3286 	int		m;
3287 	int		szc;
3288 	int		fullrange;
3289 	int		mnode;
3290 	int		local_failed_stat = 0;
3291 	lgrp_mnode_cookie_t	lgrp_cookie;
3292 
3293 	VM_STAT_ADD(pga_vmstats.pga_alloc);
3294 
3295 	/* only base pagesize currently supported */
3296 	if (size != MMU_PAGESIZE)
3297 		return (NULL);
3298 
3299 	/*
3300 	 * If we're passed a specific lgroup, we use it.  Otherwise,
3301 	 * assume first-touch placement is desired.
3302 	 */
3303 	if (!LGRP_EXISTS(lgrp))
3304 		lgrp = lgrp_home_lgrp();
3305 
3306 	/* LINTED */
3307 	AS_2_BIN(as, seg, vp, vaddr, bin, 0);
3308 
3309 	/*
3310 	 * Only hold one freelist or cachelist lock at a time, that way we
3311 	 * can start anywhere and not have to worry about lock
3312 	 * ordering.
3313 	 */
3314 	if (dma_attr == NULL) {
3315 		n = mtype16m;
3316 		m = mtypetop;
3317 		fullrange = 1;
3318 		VM_STAT_ADD(pga_vmstats.pga_nulldmaattr);
3319 	} else {
3320 		pfn_t pfnlo = mmu_btop(dma_attr->dma_attr_addr_lo);
3321 		pfn_t pfnhi = mmu_btop(dma_attr->dma_attr_addr_hi);
3322 
3323 		/*
3324 		 * We can guarantee alignment only for page boundary.
3325 		 */
3326 		if (dma_attr->dma_attr_align > MMU_PAGESIZE)
3327 			return (NULL);
3328 
3329 		/* Sanity check the dma_attr */
3330 		if (pfnlo > pfnhi)
3331 			return (NULL);
3332 
3333 		n = pfn_2_mtype(pfnlo);
3334 		m = pfn_2_mtype(pfnhi);
3335 
3336 		fullrange = ((pfnlo == mnoderanges[n].mnr_pfnlo) &&
3337 		    (pfnhi >= mnoderanges[m].mnr_pfnhi));
3338 	}
3339 	VM_STAT_COND_ADD(fullrange == 0, pga_vmstats.pga_notfullrange);
3340 
3341 	szc = 0;
3342 
3343 	/* cylcing thru mtype handled by RANGE0 if n == mtype16m */
3344 	if (n == mtype16m) {
3345 		flags |= PGI_MT_RANGE0;
3346 		n = m;
3347 	}
3348 
3349 	/*
3350 	 * Try local memory node first, but try remote if we can't
3351 	 * get a page of the right color.
3352 	 */
3353 	LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, LGRP_SRCH_HIER);
3354 	while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) {
3355 		/*
3356 		 * allocate pages from high pfn to low.
3357 		 */
3358 		mtype = m;
3359 		do {
3360 			if (fullrange != 0) {
3361 				pp = page_get_mnode_freelist(mnode,
3362 				    bin, mtype, szc, flags);
3363 				if (pp == NULL) {
3364 					pp = page_get_mnode_cachelist(
3365 					    bin, flags, mnode, mtype);
3366 				}
3367 			} else {
3368 				pp = page_get_mnode_anylist(bin, szc,
3369 				    flags, mnode, mtype, dma_attr);
3370 			}
3371 			if (pp != NULL) {
3372 				VM_STAT_ADD(pga_vmstats.pga_allocok);
3373 				check_dma(dma_attr, pp, 1);
3374 				return (pp);
3375 			}
3376 		} while (mtype != n &&
3377 		    (mtype = mnoderanges[mtype].mnr_next) != -1);
3378 		if (!local_failed_stat) {
3379 			lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_ALLOC_FAIL, 1);
3380 			local_failed_stat = 1;
3381 		}
3382 	}
3383 	VM_STAT_ADD(pga_vmstats.pga_allocfailed);
3384 
3385 	return (NULL);
3386 }
3387 
3388 /*
3389  * page_create_io()
3390  *
3391  * This function is a copy of page_create_va() with an additional
3392  * argument 'mattr' that specifies DMA memory requirements to
3393  * the page list functions. This function is used by the segkmem
3394  * allocator so it is only to create new pages (i.e PG_EXCL is
3395  * set).
3396  *
3397  * Note: This interface is currently used by x86 PSM only and is
3398  *	 not fully specified so the commitment level is only for
3399  *	 private interface specific to x86. This interface uses PSM
3400  *	 specific page_get_anylist() interface.
3401  */
3402 
3403 #define	PAGE_HASH_SEARCH(index, pp, vp, off) { \
3404 	for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash) { \
3405 		if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \
3406 			break; \
3407 	} \
3408 }
3409 
3410 
3411 page_t *
3412 page_create_io(
3413 	struct vnode	*vp,
3414 	u_offset_t	off,
3415 	uint_t		bytes,
3416 	uint_t		flags,
3417 	struct as	*as,
3418 	caddr_t		vaddr,
3419 	ddi_dma_attr_t	*mattr)	/* DMA memory attributes if any */
3420 {
3421 	page_t		*plist = NULL;
3422 	uint_t		plist_len = 0;
3423 	pgcnt_t		npages;
3424 	page_t		*npp = NULL;
3425 	uint_t		pages_req;
3426 	page_t		*pp;
3427 	kmutex_t	*phm = NULL;
3428 	uint_t		index;
3429 
3430 	TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START,
3431 	    "page_create_start:vp %p off %llx bytes %u flags %x",
3432 	    vp, off, bytes, flags);
3433 
3434 	ASSERT((flags & ~(PG_EXCL | PG_WAIT | PG_PHYSCONTIG)) == 0);
3435 
3436 	pages_req = npages = mmu_btopr(bytes);
3437 
3438 	/*
3439 	 * Do the freemem and pcf accounting.
3440 	 */
3441 	if (!page_create_wait(npages, flags)) {
3442 		return (NULL);
3443 	}
3444 
3445 	TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS,
3446 	    "page_create_success:vp %p off %llx", vp, off);
3447 
3448 	/*
3449 	 * If satisfying this request has left us with too little
3450 	 * memory, start the wheels turning to get some back.  The
3451 	 * first clause of the test prevents waking up the pageout
3452 	 * daemon in situations where it would decide that there's
3453 	 * nothing to do.
3454 	 */
3455 	if (nscan < desscan && freemem < minfree) {
3456 		TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL,
3457 		    "pageout_cv_signal:freemem %ld", freemem);
3458 		cv_signal(&proc_pageout->p_cv);
3459 	}
3460 
3461 	if (flags & PG_PHYSCONTIG) {
3462 
3463 		plist = page_get_contigpage(&npages, mattr, 1);
3464 		if (plist == NULL) {
3465 			page_create_putback(npages);
3466 			return (NULL);
3467 		}
3468 
3469 		pp = plist;
3470 
3471 		do {
3472 			if (!page_hashin(pp, vp, off, NULL)) {
3473 				panic("pg_creat_io: hashin failed %p %p %llx",
3474 				    (void *)pp, (void *)vp, off);
3475 			}
3476 			VM_STAT_ADD(page_create_new);
3477 			off += MMU_PAGESIZE;
3478 			PP_CLRFREE(pp);
3479 			PP_CLRAGED(pp);
3480 			page_set_props(pp, P_REF);
3481 			pp = pp->p_next;
3482 		} while (pp != plist);
3483 
3484 		if (!npages) {
3485 			check_dma(mattr, plist, pages_req);
3486 			return (plist);
3487 		} else {
3488 			vaddr += (pages_req - npages) << MMU_PAGESHIFT;
3489 		}
3490 
3491 		/*
3492 		 * fall-thru:
3493 		 *
3494 		 * page_get_contigpage returns when npages <= sgllen.
3495 		 * Grab the rest of the non-contig pages below from anylist.
3496 		 */
3497 	}
3498 
3499 	/*
3500 	 * Loop around collecting the requested number of pages.
3501 	 * Most of the time, we have to `create' a new page. With
3502 	 * this in mind, pull the page off the free list before
3503 	 * getting the hash lock.  This will minimize the hash
3504 	 * lock hold time, nesting, and the like.  If it turns
3505 	 * out we don't need the page, we put it back at the end.
3506 	 */
3507 	while (npages--) {
3508 		phm = NULL;
3509 
3510 		index = PAGE_HASH_FUNC(vp, off);
3511 top:
3512 		ASSERT(phm == NULL);
3513 		ASSERT(index == PAGE_HASH_FUNC(vp, off));
3514 		ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
3515 
3516 		if (npp == NULL) {
3517 			/*
3518 			 * Try to get the page of any color either from
3519 			 * the freelist or from the cache list.
3520 			 */
3521 			npp = page_get_anylist(vp, off, as, vaddr, MMU_PAGESIZE,
3522 			    flags & ~PG_MATCH_COLOR, mattr, NULL);
3523 			if (npp == NULL) {
3524 				if (mattr == NULL) {
3525 					/*
3526 					 * Not looking for a special page;
3527 					 * panic!
3528 					 */
3529 					panic("no page found %d", (int)npages);
3530 				}
3531 				/*
3532 				 * No page found! This can happen
3533 				 * if we are looking for a page
3534 				 * within a specific memory range
3535 				 * for DMA purposes. If PG_WAIT is
3536 				 * specified then we wait for a
3537 				 * while and then try again. The
3538 				 * wait could be forever if we
3539 				 * don't get the page(s) we need.
3540 				 *
3541 				 * Note: XXX We really need a mechanism
3542 				 * to wait for pages in the desired
3543 				 * range. For now, we wait for any
3544 				 * pages and see if we can use it.
3545 				 */
3546 
3547 				if ((mattr != NULL) && (flags & PG_WAIT)) {
3548 					delay(10);
3549 					goto top;
3550 				}
3551 				goto fail; /* undo accounting stuff */
3552 			}
3553 
3554 			if (PP_ISAGED(npp) == 0) {
3555 				/*
3556 				 * Since this page came from the
3557 				 * cachelist, we must destroy the
3558 				 * old vnode association.
3559 				 */
3560 				page_hashout(npp, (kmutex_t *)NULL);
3561 			}
3562 		}
3563 
3564 		/*
3565 		 * We own this page!
3566 		 */
3567 		ASSERT(PAGE_EXCL(npp));
3568 		ASSERT(npp->p_vnode == NULL);
3569 		ASSERT(!hat_page_is_mapped(npp));
3570 		PP_CLRFREE(npp);
3571 		PP_CLRAGED(npp);
3572 
3573 		/*
3574 		 * Here we have a page in our hot little mits and are
3575 		 * just waiting to stuff it on the appropriate lists.
3576 		 * Get the mutex and check to see if it really does
3577 		 * not exist.
3578 		 */
3579 		phm = PAGE_HASH_MUTEX(index);
3580 		mutex_enter(phm);
3581 		PAGE_HASH_SEARCH(index, pp, vp, off);
3582 		if (pp == NULL) {
3583 			VM_STAT_ADD(page_create_new);
3584 			pp = npp;
3585 			npp = NULL;
3586 			if (!page_hashin(pp, vp, off, phm)) {
3587 				/*
3588 				 * Since we hold the page hash mutex and
3589 				 * just searched for this page, page_hashin
3590 				 * had better not fail.  If it does, that
3591 				 * means somethread did not follow the
3592 				 * page hash mutex rules.  Panic now and
3593 				 * get it over with.  As usual, go down
3594 				 * holding all the locks.
3595 				 */
3596 				ASSERT(MUTEX_HELD(phm));
3597 				panic("page_create: hashin fail %p %p %llx %p",
3598 				    (void *)pp, (void *)vp, off, (void *)phm);
3599 
3600 			}
3601 			ASSERT(MUTEX_HELD(phm));
3602 			mutex_exit(phm);
3603 			phm = NULL;
3604 
3605 			/*
3606 			 * Hat layer locking need not be done to set
3607 			 * the following bits since the page is not hashed
3608 			 * and was on the free list (i.e., had no mappings).
3609 			 *
3610 			 * Set the reference bit to protect
3611 			 * against immediate pageout
3612 			 *
3613 			 * XXXmh modify freelist code to set reference
3614 			 * bit so we don't have to do it here.
3615 			 */
3616 			page_set_props(pp, P_REF);
3617 		} else {
3618 			ASSERT(MUTEX_HELD(phm));
3619 			mutex_exit(phm);
3620 			phm = NULL;
3621 			/*
3622 			 * NOTE: This should not happen for pages associated
3623 			 *	 with kernel vnode 'kvp'.
3624 			 */
3625 			/* XX64 - to debug why this happens! */
3626 			ASSERT(!VN_ISKAS(vp));
3627 			if (VN_ISKAS(vp))
3628 				cmn_err(CE_NOTE,
3629 				    "page_create: page not expected "
3630 				    "in hash list for kernel vnode - pp 0x%p",
3631 				    (void *)pp);
3632 			VM_STAT_ADD(page_create_exists);
3633 			goto fail;
3634 		}
3635 
3636 		/*
3637 		 * Got a page!  It is locked.  Acquire the i/o
3638 		 * lock since we are going to use the p_next and
3639 		 * p_prev fields to link the requested pages together.
3640 		 */
3641 		page_io_lock(pp);
3642 		page_add(&plist, pp);
3643 		plist = plist->p_next;
3644 		off += MMU_PAGESIZE;
3645 		vaddr += MMU_PAGESIZE;
3646 	}
3647 
3648 	check_dma(mattr, plist, pages_req);
3649 	return (plist);
3650 
3651 fail:
3652 	if (npp != NULL) {
3653 		/*
3654 		 * Did not need this page after all.
3655 		 * Put it back on the free list.
3656 		 */
3657 		VM_STAT_ADD(page_create_putbacks);
3658 		PP_SETFREE(npp);
3659 		PP_SETAGED(npp);
3660 		npp->p_offset = (u_offset_t)-1;
3661 		page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL);
3662 		page_unlock(npp);
3663 	}
3664 
3665 	/*
3666 	 * Give up the pages we already got.
3667 	 */
3668 	while (plist != NULL) {
3669 		pp = plist;
3670 		page_sub(&plist, pp);
3671 		page_io_unlock(pp);
3672 		plist_len++;
3673 		/*LINTED: constant in conditional ctx*/
3674 		VN_DISPOSE(pp, B_INVAL, 0, kcred);
3675 	}
3676 
3677 	/*
3678 	 * VN_DISPOSE does freemem accounting for the pages in plist
3679 	 * by calling page_free. So, we need to undo the pcf accounting
3680 	 * for only the remaining pages.
3681 	 */
3682 	VM_STAT_ADD(page_create_putbacks);
3683 	page_create_putback(pages_req - plist_len);
3684 
3685 	return (NULL);
3686 }
3687 #endif /* !__xpv */
3688 
3689 
3690 /*
3691  * Copy the data from the physical page represented by "frompp" to
3692  * that represented by "topp". ppcopy uses CPU->cpu_caddr1 and
3693  * CPU->cpu_caddr2.  It assumes that no one uses either map at interrupt
3694  * level and no one sleeps with an active mapping there.
3695  *
3696  * Note that the ref/mod bits in the page_t's are not affected by
3697  * this operation, hence it is up to the caller to update them appropriately.
3698  */
3699 int
3700 ppcopy(page_t *frompp, page_t *topp)
3701 {
3702 	caddr_t		pp_addr1;
3703 	caddr_t		pp_addr2;
3704 	hat_mempte_t	pte1;
3705 	hat_mempte_t	pte2;
3706 	kmutex_t	*ppaddr_mutex;
3707 	label_t		ljb;
3708 	int		ret = 1;
3709 
3710 	ASSERT_STACK_ALIGNED();
3711 	ASSERT(PAGE_LOCKED(frompp));
3712 	ASSERT(PAGE_LOCKED(topp));
3713 
3714 	if (kpm_enable) {
3715 		pp_addr1 = hat_kpm_page2va(frompp, 0);
3716 		pp_addr2 = hat_kpm_page2va(topp, 0);
3717 		kpreempt_disable();
3718 	} else {
3719 		/*
3720 		 * disable pre-emption so that CPU can't change
3721 		 */
3722 		kpreempt_disable();
3723 
3724 		pp_addr1 = CPU->cpu_caddr1;
3725 		pp_addr2 = CPU->cpu_caddr2;
3726 		pte1 = CPU->cpu_caddr1pte;
3727 		pte2 = CPU->cpu_caddr2pte;
3728 
3729 		ppaddr_mutex = &CPU->cpu_ppaddr_mutex;
3730 		mutex_enter(ppaddr_mutex);
3731 
3732 		hat_mempte_remap(page_pptonum(frompp), pp_addr1, pte1,
3733 		    PROT_READ | HAT_STORECACHING_OK, HAT_LOAD_NOCONSIST);
3734 		hat_mempte_remap(page_pptonum(topp), pp_addr2, pte2,
3735 		    PROT_READ | PROT_WRITE | HAT_STORECACHING_OK,
3736 		    HAT_LOAD_NOCONSIST);
3737 	}
3738 
3739 	if (on_fault(&ljb)) {
3740 		ret = 0;
3741 		goto faulted;
3742 	}
3743 	if (use_sse_pagecopy)
3744 #ifdef __xpv
3745 		page_copy_no_xmm(pp_addr2, pp_addr1);
3746 #else
3747 		hwblkpagecopy(pp_addr1, pp_addr2);
3748 #endif
3749 	else
3750 		bcopy(pp_addr1, pp_addr2, PAGESIZE);
3751 
3752 	no_fault();
3753 faulted:
3754 	if (!kpm_enable) {
3755 #ifdef __xpv
3756 		/*
3757 		 * We can't leave unused mappings laying about under the
3758 		 * hypervisor, so blow them away.
3759 		 */
3760 		if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr1, 0,
3761 		    UVMF_INVLPG | UVMF_LOCAL) < 0)
3762 			panic("HYPERVISOR_update_va_mapping() failed");
3763 		if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr2, 0,
3764 		    UVMF_INVLPG | UVMF_LOCAL) < 0)
3765 			panic("HYPERVISOR_update_va_mapping() failed");
3766 #endif
3767 		mutex_exit(ppaddr_mutex);
3768 	}
3769 	kpreempt_enable();
3770 	return (ret);
3771 }
3772 
3773 void
3774 pagezero(page_t *pp, uint_t off, uint_t len)
3775 {
3776 	ASSERT(PAGE_LOCKED(pp));
3777 	pfnzero(page_pptonum(pp), off, len);
3778 }
3779 
3780 /*
3781  * Zero the physical page from off to off + len given by pfn
3782  * without changing the reference and modified bits of page.
3783  *
3784  * We use this using CPU private page address #2, see ppcopy() for more info.
3785  * pfnzero() must not be called at interrupt level.
3786  */
3787 void
3788 pfnzero(pfn_t pfn, uint_t off, uint_t len)
3789 {
3790 	caddr_t		pp_addr2;
3791 	hat_mempte_t	pte2;
3792 	kmutex_t	*ppaddr_mutex = NULL;
3793 
3794 	ASSERT_STACK_ALIGNED();
3795 	ASSERT(len <= MMU_PAGESIZE);
3796 	ASSERT(off <= MMU_PAGESIZE);
3797 	ASSERT(off + len <= MMU_PAGESIZE);
3798 
3799 	if (kpm_enable && !pfn_is_foreign(pfn)) {
3800 		pp_addr2 = hat_kpm_pfn2va(pfn);
3801 		kpreempt_disable();
3802 	} else {
3803 		kpreempt_disable();
3804 
3805 		pp_addr2 = CPU->cpu_caddr2;
3806 		pte2 = CPU->cpu_caddr2pte;
3807 
3808 		ppaddr_mutex = &CPU->cpu_ppaddr_mutex;
3809 		mutex_enter(ppaddr_mutex);
3810 
3811 		hat_mempte_remap(pfn, pp_addr2, pte2,
3812 		    PROT_READ | PROT_WRITE | HAT_STORECACHING_OK,
3813 		    HAT_LOAD_NOCONSIST);
3814 	}
3815 
3816 	if (use_sse_pagezero) {
3817 #ifdef __xpv
3818 		uint_t rem;
3819 
3820 		/*
3821 		 * zero a byte at a time until properly aligned for
3822 		 * block_zero_no_xmm().
3823 		 */
3824 		while (!P2NPHASE(off, ((uint_t)BLOCKZEROALIGN)) && len-- > 0)
3825 			pp_addr2[off++] = 0;
3826 
3827 		/*
3828 		 * Now use faster block_zero_no_xmm() for any range
3829 		 * that is properly aligned and sized.
3830 		 */
3831 		rem = P2PHASE(len, ((uint_t)BLOCKZEROALIGN));
3832 		len -= rem;
3833 		if (len != 0) {
3834 			block_zero_no_xmm(pp_addr2 + off, len);
3835 			off += len;
3836 		}
3837 
3838 		/*
3839 		 * zero remainder with byte stores.
3840 		 */
3841 		while (rem-- > 0)
3842 			pp_addr2[off++] = 0;
3843 #else
3844 		hwblkclr(pp_addr2 + off, len);
3845 #endif
3846 	} else {
3847 		bzero(pp_addr2 + off, len);
3848 	}
3849 
3850 	if (!kpm_enable || pfn_is_foreign(pfn)) {
3851 #ifdef __xpv
3852 		/*
3853 		 * On the hypervisor this page might get used for a page
3854 		 * table before any intervening change to this mapping,
3855 		 * so blow it away.
3856 		 */
3857 		if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr2, 0,
3858 		    UVMF_INVLPG) < 0)
3859 			panic("HYPERVISOR_update_va_mapping() failed");
3860 #endif
3861 		mutex_exit(ppaddr_mutex);
3862 	}
3863 
3864 	kpreempt_enable();
3865 }
3866 
3867 /*
3868  * Platform-dependent page scrub call.
3869  */
3870 void
3871 pagescrub(page_t *pp, uint_t off, uint_t len)
3872 {
3873 	/*
3874 	 * For now, we rely on the fact that pagezero() will
3875 	 * always clear UEs.
3876 	 */
3877 	pagezero(pp, off, len);
3878 }
3879 
3880 /*
3881  * set up two private addresses for use on a given CPU for use in ppcopy()
3882  */
3883 void
3884 setup_vaddr_for_ppcopy(struct cpu *cpup)
3885 {
3886 	void *addr;
3887 	hat_mempte_t pte_pa;
3888 
3889 	addr = vmem_alloc(heap_arena, mmu_ptob(1), VM_SLEEP);
3890 	pte_pa = hat_mempte_setup(addr);
3891 	cpup->cpu_caddr1 = addr;
3892 	cpup->cpu_caddr1pte = pte_pa;
3893 
3894 	addr = vmem_alloc(heap_arena, mmu_ptob(1), VM_SLEEP);
3895 	pte_pa = hat_mempte_setup(addr);
3896 	cpup->cpu_caddr2 = addr;
3897 	cpup->cpu_caddr2pte = pte_pa;
3898 
3899 	mutex_init(&cpup->cpu_ppaddr_mutex, NULL, MUTEX_DEFAULT, NULL);
3900 }
3901 
3902 /*
3903  * Undo setup_vaddr_for_ppcopy
3904  */
3905 void
3906 teardown_vaddr_for_ppcopy(struct cpu *cpup)
3907 {
3908 	mutex_destroy(&cpup->cpu_ppaddr_mutex);
3909 
3910 	hat_mempte_release(cpup->cpu_caddr2, cpup->cpu_caddr2pte);
3911 	cpup->cpu_caddr2pte = 0;
3912 	vmem_free(heap_arena, cpup->cpu_caddr2, mmu_ptob(1));
3913 	cpup->cpu_caddr2 = 0;
3914 
3915 	hat_mempte_release(cpup->cpu_caddr1, cpup->cpu_caddr1pte);
3916 	cpup->cpu_caddr1pte = 0;
3917 	vmem_free(heap_arena, cpup->cpu_caddr1, mmu_ptob(1));
3918 	cpup->cpu_caddr1 = 0;
3919 }
3920 
3921 /*
3922  * Function for flushing D-cache when performing module relocations
3923  * to an alternate mapping.  Unnecessary on Intel / AMD platforms.
3924  */
3925 void
3926 dcache_flushall()
3927 {}
3928 
3929 size_t
3930 exec_get_spslew(void)
3931 {
3932 	return (0);
3933 }
3934 
3935 /*
3936  * Allocate a memory page.  The argument 'seed' can be any pseudo-random
3937  * number to vary where the pages come from.  This is quite a hacked up
3938  * method -- it works for now, but really needs to be fixed up a bit.
3939  *
3940  * We currently use page_create_va() on the kvp with fake offsets,
3941  * segments and virt address.  This is pretty bogus, but was copied from the
3942  * old hat_i86.c code.  A better approach would be to specify either mnode
3943  * random or mnode local and takes a page from whatever color has the MOST
3944  * available - this would have a minimal impact on page coloring.
3945  */
3946 page_t *
3947 page_get_physical(uintptr_t seed)
3948 {
3949 	page_t *pp;
3950 	u_offset_t offset;
3951 	static struct seg tmpseg;
3952 	static uintptr_t ctr = 0;
3953 
3954 	/*
3955 	 * This code is gross, we really need a simpler page allocator.
3956 	 *
3957 	 * We need to assign an offset for the page to call page_create_va()
3958 	 * To avoid conflicts with other pages, we get creative with the offset.
3959 	 * For 32 bits, we need an offset > 4Gig
3960 	 * For 64 bits, need an offset somewhere in the VA hole.
3961 	 */
3962 	offset = seed;
3963 	if (offset > kernelbase)
3964 		offset -= kernelbase;
3965 	offset <<= MMU_PAGESHIFT;
3966 #if defined(__amd64)
3967 	offset += mmu.hole_start;	/* something in VA hole */
3968 #else
3969 	offset += 1ULL << 40;	/* something > 4 Gig */
3970 #endif
3971 
3972 	if (page_resv(1, KM_NOSLEEP) == 0)
3973 		return (NULL);
3974 
3975 #ifdef	DEBUG
3976 	pp = page_exists(&kvp, offset);
3977 	if (pp != NULL)
3978 		panic("page already exists %p", (void *)pp);
3979 #endif
3980 
3981 	pp = page_create_va(&kvp, offset, MMU_PAGESIZE, PG_EXCL,
3982 	    &tmpseg, (caddr_t)(ctr += MMU_PAGESIZE));	/* changing VA usage */
3983 	if (pp != NULL) {
3984 		page_io_unlock(pp);
3985 		page_downgrade(pp);
3986 	}
3987 	return (pp);
3988 }
3989