17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate * CDDL HEADER START
37c478bd9Sstevel@tonic-gate *
47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the
5aa042c4bSkchow * Common Development and Distribution License (the "License").
6aa042c4bSkchow * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate *
87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate * and limitations under the License.
127c478bd9Sstevel@tonic-gate *
137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate *
197c478bd9Sstevel@tonic-gate * CDDL HEADER END
207c478bd9Sstevel@tonic-gate */
217c478bd9Sstevel@tonic-gate /*
22cb15d5d9SPeter Rival * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
237c478bd9Sstevel@tonic-gate */
24a3114836SGerry Liu /*
25a3114836SGerry Liu * Copyright (c) 2010, Intel Corporation.
26a3114836SGerry Liu * All rights reserved.
27dddac438SJohn Levon * Copyright 2019, Joyent, Inc.
28a3114836SGerry Liu */
297c478bd9Sstevel@tonic-gate
307c478bd9Sstevel@tonic-gate /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
317c478bd9Sstevel@tonic-gate /* All Rights Reserved */
327c478bd9Sstevel@tonic-gate
337c478bd9Sstevel@tonic-gate /*
347c478bd9Sstevel@tonic-gate * Portions of this source code were derived from Berkeley 4.3 BSD
357c478bd9Sstevel@tonic-gate * under license from the Regents of the University of California.
367c478bd9Sstevel@tonic-gate */
377c478bd9Sstevel@tonic-gate
387c478bd9Sstevel@tonic-gate /*
397c478bd9Sstevel@tonic-gate * UNIX machine dependent virtual memory support.
407c478bd9Sstevel@tonic-gate */
417c478bd9Sstevel@tonic-gate
427c478bd9Sstevel@tonic-gate #include <sys/types.h>
437c478bd9Sstevel@tonic-gate #include <sys/param.h>
447c478bd9Sstevel@tonic-gate #include <sys/systm.h>
457c478bd9Sstevel@tonic-gate #include <sys/user.h>
467c478bd9Sstevel@tonic-gate #include <sys/proc.h>
477c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
487c478bd9Sstevel@tonic-gate #include <sys/vmem.h>
497c478bd9Sstevel@tonic-gate #include <sys/buf.h>
507c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
517c478bd9Sstevel@tonic-gate #include <sys/lgrp.h>
527c478bd9Sstevel@tonic-gate #include <sys/disp.h>
537c478bd9Sstevel@tonic-gate #include <sys/vm.h>
547c478bd9Sstevel@tonic-gate #include <sys/mman.h>
557c478bd9Sstevel@tonic-gate #include <sys/vnode.h>
567c478bd9Sstevel@tonic-gate #include <sys/cred.h>
577c478bd9Sstevel@tonic-gate #include <sys/exec.h>
587c478bd9Sstevel@tonic-gate #include <sys/exechdr.h>
597c478bd9Sstevel@tonic-gate #include <sys/debug.h>
60ec25b48fSsusans #include <sys/vmsystm.h>
61cb15d5d9SPeter Rival #include <sys/swap.h>
621f84c0d7SDave Plauger #include <sys/dumphdr.h>
63d2a70789SRichard Lowe #include <sys/random.h>
647c478bd9Sstevel@tonic-gate
657c478bd9Sstevel@tonic-gate #include <vm/hat.h>
667c478bd9Sstevel@tonic-gate #include <vm/as.h>
677c478bd9Sstevel@tonic-gate #include <vm/seg.h>
687c478bd9Sstevel@tonic-gate #include <vm/seg_kp.h>
697c478bd9Sstevel@tonic-gate #include <vm/seg_vn.h>
707c478bd9Sstevel@tonic-gate #include <vm/page.h>
717c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
727c478bd9Sstevel@tonic-gate #include <vm/seg_kpm.h>
737c478bd9Sstevel@tonic-gate #include <vm/vm_dep.h>
747c478bd9Sstevel@tonic-gate
757c478bd9Sstevel@tonic-gate #include <sys/cpu.h>
767c478bd9Sstevel@tonic-gate #include <sys/vm_machparam.h>
777c478bd9Sstevel@tonic-gate #include <sys/memlist.h>
787c478bd9Sstevel@tonic-gate #include <sys/bootconf.h> /* XXX the memlist stuff belongs in memlist_plat.h */
797c478bd9Sstevel@tonic-gate #include <vm/hat_i86.h>
807c478bd9Sstevel@tonic-gate #include <sys/x86_archext.h>
817c478bd9Sstevel@tonic-gate #include <sys/elf_386.h>
827c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
837c478bd9Sstevel@tonic-gate #include <sys/archsystm.h>
847c478bd9Sstevel@tonic-gate #include <sys/machsystm.h>
85d2a70789SRichard Lowe #include <sys/secflags.h>
867c478bd9Sstevel@tonic-gate
877c478bd9Sstevel@tonic-gate #include <sys/vtrace.h>
887c478bd9Sstevel@tonic-gate #include <sys/ddidmareq.h>
897c478bd9Sstevel@tonic-gate #include <sys/promif.h>
907c478bd9Sstevel@tonic-gate #include <sys/memnode.h>
917c478bd9Sstevel@tonic-gate #include <sys/stack.h>
92843e1988Sjohnlev #include <util/qsort.h>
93843e1988Sjohnlev #include <sys/taskq.h>
94843e1988Sjohnlev
95843e1988Sjohnlev #ifdef __xpv
96843e1988Sjohnlev
97843e1988Sjohnlev #include <sys/hypervisor.h>
98843e1988Sjohnlev #include <sys/xen_mmu.h>
99843e1988Sjohnlev #include <sys/balloon_impl.h>
100843e1988Sjohnlev
101843e1988Sjohnlev /*
102843e1988Sjohnlev * domain 0 pages usable for DMA are kept pre-allocated and kept in
103843e1988Sjohnlev * distinct lists, ordered by increasing mfn.
104843e1988Sjohnlev */
105843e1988Sjohnlev static kmutex_t io_pool_lock;
106b9bc7f78Ssmaybe static kmutex_t contig_list_lock;
107843e1988Sjohnlev static page_t *io_pool_4g; /* pool for 32 bit dma limited devices */
108843e1988Sjohnlev static page_t *io_pool_16m; /* pool for 24 bit dma limited legacy devices */
109843e1988Sjohnlev static long io_pool_cnt;
110843e1988Sjohnlev static long io_pool_cnt_max = 0;
111843e1988Sjohnlev #define DEFAULT_IO_POOL_MIN 128
112843e1988Sjohnlev static long io_pool_cnt_min = DEFAULT_IO_POOL_MIN;
113843e1988Sjohnlev static long io_pool_cnt_lowater = 0;
114843e1988Sjohnlev static long io_pool_shrink_attempts; /* how many times did we try to shrink */
115843e1988Sjohnlev static long io_pool_shrinks; /* how many times did we really shrink */
116843e1988Sjohnlev static long io_pool_grows; /* how many times did we grow */
117843e1988Sjohnlev static mfn_t start_mfn = 1;
118843e1988Sjohnlev static caddr_t io_pool_kva; /* use to alloc pages when needed */
119843e1988Sjohnlev
120843e1988Sjohnlev static int create_contig_pfnlist(uint_t);
121843e1988Sjohnlev
122843e1988Sjohnlev /*
123843e1988Sjohnlev * percentage of phys mem to hold in the i/o pool
124843e1988Sjohnlev */
125843e1988Sjohnlev #define DEFAULT_IO_POOL_PCT 2
126843e1988Sjohnlev static long io_pool_physmem_pct = DEFAULT_IO_POOL_PCT;
127843e1988Sjohnlev static void page_io_pool_sub(page_t **, page_t *, page_t *);
128b9bc7f78Ssmaybe int ioalloc_dbg = 0;
129843e1988Sjohnlev
130843e1988Sjohnlev #endif /* __xpv */
1317c478bd9Sstevel@tonic-gate
1325d07b933Sdp uint_t vac_colors = 1;
1337c478bd9Sstevel@tonic-gate
1347c478bd9Sstevel@tonic-gate int largepagesupport = 0;
1357c478bd9Sstevel@tonic-gate extern uint_t page_create_new;
1367c478bd9Sstevel@tonic-gate extern uint_t page_create_exists;
1377c478bd9Sstevel@tonic-gate extern uint_t page_create_putbacks;
138ae115bc7Smrj /*
139ae115bc7Smrj * Allow users to disable the kernel's use of SSE.
140ae115bc7Smrj */
141ae115bc7Smrj extern int use_sse_pagecopy, use_sse_pagezero;
1427c478bd9Sstevel@tonic-gate
143d94ffb28Sjmcp /*
144d94ffb28Sjmcp * combined memory ranges from mnode and memranges[] to manage single
145d94ffb28Sjmcp * mnode/mtype dimension in the page lists.
146d94ffb28Sjmcp */
147d94ffb28Sjmcp typedef struct {
148d94ffb28Sjmcp pfn_t mnr_pfnlo;
149d94ffb28Sjmcp pfn_t mnr_pfnhi;
150d94ffb28Sjmcp int mnr_mnode;
151d94ffb28Sjmcp int mnr_memrange; /* index into memranges[] */
152d94ffb28Sjmcp int mnr_next; /* next lower PA mnoderange */
153d94ffb28Sjmcp int mnr_exists;
154d94ffb28Sjmcp /* maintain page list stats */
155d94ffb28Sjmcp pgcnt_t mnr_mt_clpgcnt; /* cache list cnt */
156d94ffb28Sjmcp pgcnt_t mnr_mt_flpgcnt[MMU_PAGE_SIZES]; /* free list cnt per szc */
157d94ffb28Sjmcp pgcnt_t mnr_mt_totcnt; /* sum of cache and free lists */
158d94ffb28Sjmcp #ifdef DEBUG
159d94ffb28Sjmcp struct mnr_mts { /* mnode/mtype szc stats */
160d94ffb28Sjmcp pgcnt_t mnr_mts_pgcnt;
161d94ffb28Sjmcp int mnr_mts_colors;
162d94ffb28Sjmcp pgcnt_t *mnr_mtsc_pgcnt;
163dddac438SJohn Levon } *mnr_mts;
164d94ffb28Sjmcp #endif
165d94ffb28Sjmcp } mnoderange_t;
166d94ffb28Sjmcp
167d94ffb28Sjmcp #define MEMRANGEHI(mtype) \
168d94ffb28Sjmcp ((mtype > 0) ? memranges[mtype - 1] - 1: physmax)
169d94ffb28Sjmcp #define MEMRANGELO(mtype) (memranges[mtype])
170d94ffb28Sjmcp
171d94ffb28Sjmcp #define MTYPE_FREEMEM(mt) (mnoderanges[mt].mnr_mt_totcnt)
172d94ffb28Sjmcp
173843e1988Sjohnlev /*
174843e1988Sjohnlev * As the PC architecture evolved memory up was clumped into several
175843e1988Sjohnlev * ranges for various historical I/O devices to do DMA.
176843e1988Sjohnlev * < 16Meg - ISA bus
177843e1988Sjohnlev * < 2Gig - ???
178843e1988Sjohnlev * < 4Gig - PCI bus or drivers that don't understand PAE mode
179843e1988Sjohnlev *
180843e1988Sjohnlev * These are listed in reverse order, so that we can skip over unused
181843e1988Sjohnlev * ranges on machines with small memories.
182843e1988Sjohnlev *
183843e1988Sjohnlev * For now under the Hypervisor, we'll only ever have one memrange.
184843e1988Sjohnlev */
185843e1988Sjohnlev #define PFN_4GIG 0x100000
186843e1988Sjohnlev #define PFN_16MEG 0x1000
187a3114836SGerry Liu /* Indices into the memory range (arch_memranges) array. */
188a3114836SGerry Liu #define MRI_4G 0
189a3114836SGerry Liu #define MRI_2G 1
190a3114836SGerry Liu #define MRI_16M 2
191a3114836SGerry Liu #define MRI_0 3
192843e1988Sjohnlev static pfn_t arch_memranges[NUM_MEM_RANGES] = {
193843e1988Sjohnlev PFN_4GIG, /* pfn range for 4G and above */
194843e1988Sjohnlev 0x80000, /* pfn range for 2G-4G */
195843e1988Sjohnlev PFN_16MEG, /* pfn range for 16M-2G */
196843e1988Sjohnlev 0x00000, /* pfn range for 0-16M */
197843e1988Sjohnlev };
198843e1988Sjohnlev pfn_t *memranges = &arch_memranges[0];
199843e1988Sjohnlev int nranges = NUM_MEM_RANGES;
200843e1988Sjohnlev
201843e1988Sjohnlev /*
202843e1988Sjohnlev * This combines mem_node_config and memranges into one data
203843e1988Sjohnlev * structure to be used for page list management.
204843e1988Sjohnlev */
205dddac438SJohn Levon static mnoderange_t *mnoderanges;
206dddac438SJohn Levon static int mnoderangecnt;
207dddac438SJohn Levon static int mtype4g;
208dddac438SJohn Levon static int mtype16m;
209dddac438SJohn Levon static int mtypetop;
210843e1988Sjohnlev
211843e1988Sjohnlev /*
212843e1988Sjohnlev * 4g memory management variables for systems with more than 4g of memory:
213843e1988Sjohnlev *
214843e1988Sjohnlev * physical memory below 4g is required for 32bit dma devices and, currently,
215843e1988Sjohnlev * for kmem memory. On systems with more than 4g of memory, the pool of memory
216843e1988Sjohnlev * below 4g can be depleted without any paging activity given that there is
217843e1988Sjohnlev * likely to be sufficient memory above 4g.
218843e1988Sjohnlev *
219843e1988Sjohnlev * physmax4g is set true if the largest pfn is over 4g. The rest of the
220843e1988Sjohnlev * 4g memory management code is enabled only when physmax4g is true.
221843e1988Sjohnlev *
222843e1988Sjohnlev * maxmem4g is the count of the maximum number of pages on the page lists
223843e1988Sjohnlev * with physical addresses below 4g. It can be a lot less then 4g given that
224843e1988Sjohnlev * BIOS may reserve large chunks of space below 4g for hot plug pci devices,
225843e1988Sjohnlev * agp aperture etc.
226843e1988Sjohnlev *
227843e1988Sjohnlev * freemem4g maintains the count of the number of available pages on the
228843e1988Sjohnlev * page lists with physical addresses below 4g.
229843e1988Sjohnlev *
230843e1988Sjohnlev * DESFREE4G specifies the desired amount of below 4g memory. It defaults to
231843e1988Sjohnlev * 6% (desfree4gshift = 4) of maxmem4g.
232843e1988Sjohnlev *
233843e1988Sjohnlev * RESTRICT4G_ALLOC returns true if freemem4g falls below DESFREE4G
234843e1988Sjohnlev * and the amount of physical memory above 4g is greater than freemem4g.
235843e1988Sjohnlev * In this case, page_get_* routines will restrict below 4g allocations
236843e1988Sjohnlev * for requests that don't specifically require it.
237843e1988Sjohnlev */
238843e1988Sjohnlev
239843e1988Sjohnlev #define DESFREE4G (maxmem4g >> desfree4gshift)
240843e1988Sjohnlev
241843e1988Sjohnlev #define RESTRICT4G_ALLOC \
242843e1988Sjohnlev (physmax4g && (freemem4g < DESFREE4G) && ((freemem4g << 1) < freemem))
2437c478bd9Sstevel@tonic-gate
244843e1988Sjohnlev static pgcnt_t maxmem4g;
245843e1988Sjohnlev static pgcnt_t freemem4g;
246843e1988Sjohnlev static int physmax4g;
247843e1988Sjohnlev static int desfree4gshift = 4; /* maxmem4g shift to derive DESFREE4G */
248843e1988Sjohnlev
249843e1988Sjohnlev /*
250843e1988Sjohnlev * 16m memory management:
251843e1988Sjohnlev *
252843e1988Sjohnlev * reserve some amount of physical memory below 16m for legacy devices.
253843e1988Sjohnlev *
254843e1988Sjohnlev * RESTRICT16M_ALLOC returns true if an there are sufficient free pages above
255843e1988Sjohnlev * 16m or if the 16m pool drops below DESFREE16M.
256843e1988Sjohnlev *
257843e1988Sjohnlev * In this case, general page allocations via page_get_{free,cache}list
258843e1988Sjohnlev * routines will be restricted from allocating from the 16m pool. Allocations
259843e1988Sjohnlev * that require specific pfn ranges (page_get_anylist) and PG_PANIC allocations
260843e1988Sjohnlev * are not restricted.
261843e1988Sjohnlev */
262843e1988Sjohnlev
263a3114836SGerry Liu #define FREEMEM16M MTYPE_FREEMEM(mtype16m)
264843e1988Sjohnlev #define DESFREE16M desfree16m
265dddac438SJohn Levon #define RESTRICT16M_ALLOC(freemem, pgcnt, flags) \
266dddac438SJohn Levon (mtype16m != -1 && (freemem != 0) && ((flags & PG_PANIC) == 0) && \
267dddac438SJohn Levon ((freemem >= (FREEMEM16M)) || \
268843e1988Sjohnlev (FREEMEM16M < (DESFREE16M + pgcnt))))
269843e1988Sjohnlev
270843e1988Sjohnlev static pgcnt_t desfree16m = 0x380;
271843e1988Sjohnlev
272843e1988Sjohnlev /*
273843e1988Sjohnlev * This can be patched via /etc/system to allow old non-PAE aware device
274843e1988Sjohnlev * drivers to use kmem_alloc'd memory on 32 bit systems with > 4Gig RAM.
275843e1988Sjohnlev */
276843e1988Sjohnlev int restricted_kmemalloc = 0;
27707ad560dSkchow
2787c478bd9Sstevel@tonic-gate #ifdef VM_STATS
2797c478bd9Sstevel@tonic-gate struct {
2807c478bd9Sstevel@tonic-gate ulong_t pga_alloc;
2817c478bd9Sstevel@tonic-gate ulong_t pga_notfullrange;
2827c478bd9Sstevel@tonic-gate ulong_t pga_nulldmaattr;
2837c478bd9Sstevel@tonic-gate ulong_t pga_allocok;
2847c478bd9Sstevel@tonic-gate ulong_t pga_allocfailed;
2857c478bd9Sstevel@tonic-gate ulong_t pgma_alloc;
2867c478bd9Sstevel@tonic-gate ulong_t pgma_allocok;
2877c478bd9Sstevel@tonic-gate ulong_t pgma_allocfailed;
2887c478bd9Sstevel@tonic-gate ulong_t pgma_allocempty;
2897c478bd9Sstevel@tonic-gate } pga_vmstats;
2907c478bd9Sstevel@tonic-gate #endif
2917c478bd9Sstevel@tonic-gate
2927c478bd9Sstevel@tonic-gate uint_t mmu_page_sizes;
2937c478bd9Sstevel@tonic-gate
2947c478bd9Sstevel@tonic-gate /* How many page sizes the users can see */
2957c478bd9Sstevel@tonic-gate uint_t mmu_exported_page_sizes;
2967c478bd9Sstevel@tonic-gate
29702bc52beSkchow /* page sizes that legacy applications can see */
29802bc52beSkchow uint_t mmu_legacy_page_sizes;
29902bc52beSkchow
300beb1bda0Sdavemq /*
301beb1bda0Sdavemq * Number of pages in 1 GB. Don't enable automatic large pages if we have
302beb1bda0Sdavemq * fewer than this many pages.
303beb1bda0Sdavemq */
304ec25b48fSsusans pgcnt_t shm_lpg_min_physmem = 1 << (30 - MMU_PAGESHIFT);
305ec25b48fSsusans pgcnt_t privm_lpg_min_physmem = 1 << (30 - MMU_PAGESHIFT);
306ec25b48fSsusans
307ec25b48fSsusans /*
308ec25b48fSsusans * Maximum and default segment size tunables for user private
309ec25b48fSsusans * and shared anon memory, and user text and initialized data.
310ec25b48fSsusans * These can be patched via /etc/system to allow large pages
311ec25b48fSsusans * to be used for mapping application private and shared anon memory.
312ec25b48fSsusans */
313ec25b48fSsusans size_t mcntl0_lpsize = MMU_PAGESIZE;
314ec25b48fSsusans size_t max_uheap_lpsize = MMU_PAGESIZE;
315ec25b48fSsusans size_t default_uheap_lpsize = MMU_PAGESIZE;
316ec25b48fSsusans size_t max_ustack_lpsize = MMU_PAGESIZE;
317ec25b48fSsusans size_t default_ustack_lpsize = MMU_PAGESIZE;
318ec25b48fSsusans size_t max_privmap_lpsize = MMU_PAGESIZE;
319ec25b48fSsusans size_t max_uidata_lpsize = MMU_PAGESIZE;
320ec25b48fSsusans size_t max_utext_lpsize = MMU_PAGESIZE;
321ec25b48fSsusans size_t max_shm_lpsize = MMU_PAGESIZE;
3227c478bd9Sstevel@tonic-gate
323843e1988Sjohnlev
324843e1988Sjohnlev /*
325843e1988Sjohnlev * initialized by page_coloring_init().
326843e1988Sjohnlev */
327843e1988Sjohnlev uint_t page_colors;
328843e1988Sjohnlev uint_t page_colors_mask;
329843e1988Sjohnlev uint_t page_coloring_shift;
330843e1988Sjohnlev int cpu_page_colors;
331843e1988Sjohnlev static uint_t l2_colors;
332843e1988Sjohnlev
333843e1988Sjohnlev /*
334843e1988Sjohnlev * Page freelists and cachelists are dynamically allocated once mnoderangecnt
335843e1988Sjohnlev * and page_colors are calculated from the l2 cache n-way set size. Within a
336843e1988Sjohnlev * mnode range, the page freelist and cachelist are hashed into bins based on
337843e1988Sjohnlev * color. This makes it easier to search for a page within a specific memory
338843e1988Sjohnlev * range.
339843e1988Sjohnlev */
340843e1988Sjohnlev #define PAGE_COLORS_MIN 16
341843e1988Sjohnlev
342843e1988Sjohnlev page_t ****page_freelists;
343843e1988Sjohnlev page_t ***page_cachelists;
344843e1988Sjohnlev
345843e1988Sjohnlev
346843e1988Sjohnlev /*
347843e1988Sjohnlev * Used by page layer to know about page sizes
348843e1988Sjohnlev */
349843e1988Sjohnlev hw_pagesize_t hw_page_array[MAX_NUM_LEVEL + 1];
350843e1988Sjohnlev
351d94ffb28Sjmcp kmutex_t *fpc_mutex[NPC_MUTEX];
352d94ffb28Sjmcp kmutex_t *cpc_mutex[NPC_MUTEX];
353843e1988Sjohnlev
354a3114836SGerry Liu /* Lock to protect mnoderanges array for memory DR operations. */
355a3114836SGerry Liu static kmutex_t mnoderange_lock;
356a3114836SGerry Liu
357843e1988Sjohnlev /*
358843e1988Sjohnlev * Only let one thread at a time try to coalesce large pages, to
359843e1988Sjohnlev * prevent them from working against each other.
360843e1988Sjohnlev */
361843e1988Sjohnlev static kmutex_t contig_lock;
362843e1988Sjohnlev #define CONTIG_LOCK() mutex_enter(&contig_lock);
363843e1988Sjohnlev #define CONTIG_UNLOCK() mutex_exit(&contig_lock);
364843e1988Sjohnlev
365843e1988Sjohnlev #define PFN_16M (mmu_btop((uint64_t)0x1000000))
366843e1988Sjohnlev
36774ecdb51SJohn Levon caddr_t
i86devmap(pfn_t pf,pgcnt_t pgcnt,uint_t prot)36874ecdb51SJohn Levon i86devmap(pfn_t pf, pgcnt_t pgcnt, uint_t prot)
36974ecdb51SJohn Levon {
37074ecdb51SJohn Levon caddr_t addr;
37174ecdb51SJohn Levon caddr_t addr1;
37274ecdb51SJohn Levon page_t *pp;
37374ecdb51SJohn Levon
37474ecdb51SJohn Levon addr1 = addr = vmem_alloc(heap_arena, mmu_ptob(pgcnt), VM_SLEEP);
37574ecdb51SJohn Levon
37674ecdb51SJohn Levon for (; pgcnt != 0; addr += MMU_PAGESIZE, ++pf, --pgcnt) {
37774ecdb51SJohn Levon pp = page_numtopp_nolock(pf);
37874ecdb51SJohn Levon if (pp == NULL) {
37974ecdb51SJohn Levon hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pf,
38074ecdb51SJohn Levon prot | HAT_NOSYNC, HAT_LOAD_LOCK);
38174ecdb51SJohn Levon } else {
38274ecdb51SJohn Levon hat_memload(kas.a_hat, addr, pp,
38374ecdb51SJohn Levon prot | HAT_NOSYNC, HAT_LOAD_LOCK);
38474ecdb51SJohn Levon }
38574ecdb51SJohn Levon }
38674ecdb51SJohn Levon
38774ecdb51SJohn Levon return (addr1);
38874ecdb51SJohn Levon }
38974ecdb51SJohn Levon
39074ecdb51SJohn Levon /*
39174ecdb51SJohn Levon * This routine is like page_numtopp, but accepts only free pages, which
39274ecdb51SJohn Levon * it allocates (unfrees) and returns with the exclusive lock held.
39374ecdb51SJohn Levon * It is used by machdep.c/dma_init() to find contiguous free pages.
39474ecdb51SJohn Levon */
39574ecdb51SJohn Levon page_t *
page_numtopp_alloc(pfn_t pfnum)39674ecdb51SJohn Levon page_numtopp_alloc(pfn_t pfnum)
39774ecdb51SJohn Levon {
39874ecdb51SJohn Levon page_t *pp;
39974ecdb51SJohn Levon
40074ecdb51SJohn Levon retry:
40174ecdb51SJohn Levon pp = page_numtopp_nolock(pfnum);
40274ecdb51SJohn Levon if (pp == NULL) {
40374ecdb51SJohn Levon return (NULL);
40474ecdb51SJohn Levon }
40574ecdb51SJohn Levon
40674ecdb51SJohn Levon if (!page_trylock(pp, SE_EXCL)) {
40774ecdb51SJohn Levon return (NULL);
40874ecdb51SJohn Levon }
40974ecdb51SJohn Levon
41074ecdb51SJohn Levon if (page_pptonum(pp) != pfnum) {
41174ecdb51SJohn Levon page_unlock(pp);
41274ecdb51SJohn Levon goto retry;
41374ecdb51SJohn Levon }
41474ecdb51SJohn Levon
41574ecdb51SJohn Levon if (!PP_ISFREE(pp)) {
41674ecdb51SJohn Levon page_unlock(pp);
41774ecdb51SJohn Levon return (NULL);
41874ecdb51SJohn Levon }
41974ecdb51SJohn Levon if (pp->p_szc) {
42074ecdb51SJohn Levon page_demote_free_pages(pp);
42174ecdb51SJohn Levon page_unlock(pp);
42274ecdb51SJohn Levon goto retry;
42374ecdb51SJohn Levon }
42474ecdb51SJohn Levon
42574ecdb51SJohn Levon /* If associated with a vnode, destroy mappings */
42674ecdb51SJohn Levon
42774ecdb51SJohn Levon if (pp->p_vnode) {
42874ecdb51SJohn Levon
42974ecdb51SJohn Levon page_destroy_free(pp);
43074ecdb51SJohn Levon
43174ecdb51SJohn Levon if (!page_lock(pp, SE_EXCL, (kmutex_t *)NULL, P_NO_RECLAIM)) {
43274ecdb51SJohn Levon return (NULL);
43374ecdb51SJohn Levon }
43474ecdb51SJohn Levon
43574ecdb51SJohn Levon if (page_pptonum(pp) != pfnum) {
43674ecdb51SJohn Levon page_unlock(pp);
43774ecdb51SJohn Levon goto retry;
43874ecdb51SJohn Levon }
43974ecdb51SJohn Levon }
44074ecdb51SJohn Levon
44174ecdb51SJohn Levon if (!PP_ISFREE(pp)) {
44274ecdb51SJohn Levon page_unlock(pp);
44374ecdb51SJohn Levon return (NULL);
44474ecdb51SJohn Levon }
44574ecdb51SJohn Levon
44674ecdb51SJohn Levon if (!page_reclaim(pp, (kmutex_t *)NULL))
44774ecdb51SJohn Levon return (NULL);
44874ecdb51SJohn Levon
44974ecdb51SJohn Levon return (pp);
45074ecdb51SJohn Levon }
45174ecdb51SJohn Levon
4527c478bd9Sstevel@tonic-gate /*
4537c478bd9Sstevel@tonic-gate * Return the optimum page size for a given mapping
4547c478bd9Sstevel@tonic-gate */
4557c478bd9Sstevel@tonic-gate /*ARGSUSED*/
4567c478bd9Sstevel@tonic-gate size_t
map_pgsz(int maptype,struct proc * p,caddr_t addr,size_t len,int memcntl)457ec25b48fSsusans map_pgsz(int maptype, struct proc *p, caddr_t addr, size_t len, int memcntl)
4587c478bd9Sstevel@tonic-gate {
459ec25b48fSsusans level_t l = 0;
460ec25b48fSsusans size_t pgsz = MMU_PAGESIZE;
461ec25b48fSsusans size_t max_lpsize;
462ec25b48fSsusans uint_t mszc;
4637c478bd9Sstevel@tonic-gate
464ec25b48fSsusans ASSERT(maptype != MAPPGSZ_VA);
4657c478bd9Sstevel@tonic-gate
466ec25b48fSsusans if (maptype != MAPPGSZ_ISM && physmem < privm_lpg_min_physmem) {
467ec25b48fSsusans return (MMU_PAGESIZE);
468ec25b48fSsusans }
4697c478bd9Sstevel@tonic-gate
470ec25b48fSsusans switch (maptype) {
4717c478bd9Sstevel@tonic-gate case MAPPGSZ_HEAP:
472ec25b48fSsusans case MAPPGSZ_STK:
473ec25b48fSsusans max_lpsize = memcntl ? mcntl0_lpsize : (maptype ==
474ec25b48fSsusans MAPPGSZ_HEAP ? max_uheap_lpsize : max_ustack_lpsize);
475ec25b48fSsusans if (max_lpsize == MMU_PAGESIZE) {
476ec25b48fSsusans return (MMU_PAGESIZE);
477ec25b48fSsusans }
478ec25b48fSsusans if (len == 0) {
479ec25b48fSsusans len = (maptype == MAPPGSZ_HEAP) ? p->p_brkbase +
480ec25b48fSsusans p->p_brksize - p->p_bssbase : p->p_stksize;
481ec25b48fSsusans }
482ec25b48fSsusans len = (maptype == MAPPGSZ_HEAP) ? MAX(len,
483ec25b48fSsusans default_uheap_lpsize) : MAX(len, default_ustack_lpsize);
484ec25b48fSsusans
4857c478bd9Sstevel@tonic-gate /*
4867c478bd9Sstevel@tonic-gate * use the pages size that best fits len
4877c478bd9Sstevel@tonic-gate */
48802bc52beSkchow for (l = mmu.umax_page_level; l > 0; --l) {
489ec25b48fSsusans if (LEVEL_SIZE(l) > max_lpsize || len < LEVEL_SIZE(l)) {
4907c478bd9Sstevel@tonic-gate continue;
491ec25b48fSsusans } else {
492ec25b48fSsusans pgsz = LEVEL_SIZE(l);
493ec25b48fSsusans }
4947c478bd9Sstevel@tonic-gate break;
4957c478bd9Sstevel@tonic-gate }
496ec25b48fSsusans
497ec25b48fSsusans mszc = (maptype == MAPPGSZ_HEAP ? p->p_brkpageszc :
498ec25b48fSsusans p->p_stkpageszc);
499ec25b48fSsusans if (addr == 0 && (pgsz < hw_page_array[mszc].hp_size)) {
500ec25b48fSsusans pgsz = hw_page_array[mszc].hp_size;
501ec25b48fSsusans }
502ec25b48fSsusans return (pgsz);
5037c478bd9Sstevel@tonic-gate
5047c478bd9Sstevel@tonic-gate case MAPPGSZ_ISM:
50502bc52beSkchow for (l = mmu.umax_page_level; l > 0; --l) {
50602bc52beSkchow if (len >= LEVEL_SIZE(l))
50702bc52beSkchow return (LEVEL_SIZE(l));
50802bc52beSkchow }
50902bc52beSkchow return (LEVEL_SIZE(0));
5107c478bd9Sstevel@tonic-gate }
511ec25b48fSsusans return (pgsz);
5127c478bd9Sstevel@tonic-gate }
5137c478bd9Sstevel@tonic-gate
514ec25b48fSsusans static uint_t
map_szcvec(caddr_t addr,size_t size,uintptr_t off,size_t max_lpsize,size_t min_physmem)515ec25b48fSsusans map_szcvec(caddr_t addr, size_t size, uintptr_t off, size_t max_lpsize,
516ec25b48fSsusans size_t min_physmem)
5177c478bd9Sstevel@tonic-gate {
518ec25b48fSsusans caddr_t eaddr = addr + size;
519ec25b48fSsusans uint_t szcvec = 0;
520ec25b48fSsusans caddr_t raddr;
521ec25b48fSsusans caddr_t readdr;
5227c478bd9Sstevel@tonic-gate size_t pgsz;
523ec25b48fSsusans int i;
5247c478bd9Sstevel@tonic-gate
525ec25b48fSsusans if (physmem < min_physmem || max_lpsize <= MMU_PAGESIZE) {
5267c478bd9Sstevel@tonic-gate return (0);
5277c478bd9Sstevel@tonic-gate }
528ec25b48fSsusans
52902bc52beSkchow for (i = mmu_exported_page_sizes - 1; i > 0; i--) {
530ec25b48fSsusans pgsz = page_get_pagesize(i);
531ec25b48fSsusans if (pgsz > max_lpsize) {
532ec25b48fSsusans continue;
533ec25b48fSsusans }
534ec25b48fSsusans raddr = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz);
535ec25b48fSsusans readdr = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz);
536ec25b48fSsusans if (raddr < addr || raddr >= readdr) {
537ec25b48fSsusans continue;
538ec25b48fSsusans }
539ec25b48fSsusans if (P2PHASE((uintptr_t)addr ^ off, pgsz)) {
540ec25b48fSsusans continue;
541ec25b48fSsusans }
542ec25b48fSsusans /*
543ec25b48fSsusans * Set szcvec to the remaining page sizes.
544ec25b48fSsusans */
545ec25b48fSsusans szcvec = ((1 << (i + 1)) - 1) & ~1;
546ec25b48fSsusans break;
5477c478bd9Sstevel@tonic-gate }
548ec25b48fSsusans return (szcvec);
5497c478bd9Sstevel@tonic-gate }
5507c478bd9Sstevel@tonic-gate
551ec25b48fSsusans /*
552ec25b48fSsusans * Return a bit vector of large page size codes that
553ec25b48fSsusans * can be used to map [addr, addr + len) region.
554ec25b48fSsusans */
555ec25b48fSsusans /*ARGSUSED*/
55607b65a64Saguzovsk uint_t
map_pgszcvec(caddr_t addr,size_t size,uintptr_t off,int flags,int type,int memcntl)557ec25b48fSsusans map_pgszcvec(caddr_t addr, size_t size, uintptr_t off, int flags, int type,
558ec25b48fSsusans int memcntl)
55907b65a64Saguzovsk {
560ec25b48fSsusans size_t max_lpsize = mcntl0_lpsize;
56107b65a64Saguzovsk
562ec25b48fSsusans if (mmu.max_page_level == 0)
56307b65a64Saguzovsk return (0);
56407b65a64Saguzovsk
565ec25b48fSsusans if (flags & MAP_TEXT) {
566843e1988Sjohnlev if (!memcntl)
567843e1988Sjohnlev max_lpsize = max_utext_lpsize;
568843e1988Sjohnlev return (map_szcvec(addr, size, off, max_lpsize,
569ec25b48fSsusans shm_lpg_min_physmem));
570ec25b48fSsusans
571ec25b48fSsusans } else if (flags & MAP_INITDATA) {
572843e1988Sjohnlev if (!memcntl)
573843e1988Sjohnlev max_lpsize = max_uidata_lpsize;
574843e1988Sjohnlev return (map_szcvec(addr, size, off, max_lpsize,
575ec25b48fSsusans privm_lpg_min_physmem));
576ec25b48fSsusans
577ec25b48fSsusans } else if (type == MAPPGSZC_SHM) {
578843e1988Sjohnlev if (!memcntl)
579843e1988Sjohnlev max_lpsize = max_shm_lpsize;
580843e1988Sjohnlev return (map_szcvec(addr, size, off, max_lpsize,
581ec25b48fSsusans shm_lpg_min_physmem));
582ec25b48fSsusans
583ec25b48fSsusans } else if (type == MAPPGSZC_HEAP) {
584843e1988Sjohnlev if (!memcntl)
585843e1988Sjohnlev max_lpsize = max_uheap_lpsize;
586843e1988Sjohnlev return (map_szcvec(addr, size, off, max_lpsize,
587ec25b48fSsusans privm_lpg_min_physmem));
588ec25b48fSsusans
589ec25b48fSsusans } else if (type == MAPPGSZC_STACK) {
590843e1988Sjohnlev if (!memcntl)
591843e1988Sjohnlev max_lpsize = max_ustack_lpsize;
592843e1988Sjohnlev return (map_szcvec(addr, size, off, max_lpsize,
593ec25b48fSsusans privm_lpg_min_physmem));
594ec25b48fSsusans
595ec25b48fSsusans } else {
596843e1988Sjohnlev if (!memcntl)
597843e1988Sjohnlev max_lpsize = max_privmap_lpsize;
598843e1988Sjohnlev return (map_szcvec(addr, size, off, max_lpsize,
599ec25b48fSsusans privm_lpg_min_physmem));
60007b65a64Saguzovsk }
60107b65a64Saguzovsk }
60207b65a64Saguzovsk
6037c478bd9Sstevel@tonic-gate /*
6047c478bd9Sstevel@tonic-gate * Handle a pagefault.
6057c478bd9Sstevel@tonic-gate */
6067c478bd9Sstevel@tonic-gate faultcode_t
pagefault(caddr_t addr,enum fault_type type,enum seg_rw rw,int iskernel)6077c478bd9Sstevel@tonic-gate pagefault(
6087c478bd9Sstevel@tonic-gate caddr_t addr,
6097c478bd9Sstevel@tonic-gate enum fault_type type,
6107c478bd9Sstevel@tonic-gate enum seg_rw rw,
6117c478bd9Sstevel@tonic-gate int iskernel)
6127c478bd9Sstevel@tonic-gate {
6137c478bd9Sstevel@tonic-gate struct as *as;
6147c478bd9Sstevel@tonic-gate struct hat *hat;
6157c478bd9Sstevel@tonic-gate struct proc *p;
6167c478bd9Sstevel@tonic-gate kthread_t *t;
6177c478bd9Sstevel@tonic-gate faultcode_t res;
6187c478bd9Sstevel@tonic-gate caddr_t base;
6197c478bd9Sstevel@tonic-gate size_t len;
6207c478bd9Sstevel@tonic-gate int err;
6217c478bd9Sstevel@tonic-gate int mapped_red;
6227c478bd9Sstevel@tonic-gate uintptr_t ea;
6237c478bd9Sstevel@tonic-gate
6247c478bd9Sstevel@tonic-gate ASSERT_STACK_ALIGNED();
6257c478bd9Sstevel@tonic-gate
6267c478bd9Sstevel@tonic-gate if (INVALID_VADDR(addr))
6277c478bd9Sstevel@tonic-gate return (FC_NOMAP);
6287c478bd9Sstevel@tonic-gate
6297c478bd9Sstevel@tonic-gate mapped_red = segkp_map_red();
6307c478bd9Sstevel@tonic-gate
6317c478bd9Sstevel@tonic-gate if (iskernel) {
6327c478bd9Sstevel@tonic-gate as = &kas;
6337c478bd9Sstevel@tonic-gate hat = as->a_hat;
6347c478bd9Sstevel@tonic-gate } else {
6357c478bd9Sstevel@tonic-gate t = curthread;
6367c478bd9Sstevel@tonic-gate p = ttoproc(t);
6377c478bd9Sstevel@tonic-gate as = p->p_as;
6387c478bd9Sstevel@tonic-gate hat = as->a_hat;
6397c478bd9Sstevel@tonic-gate }
6407c478bd9Sstevel@tonic-gate
6417c478bd9Sstevel@tonic-gate /*
6427c478bd9Sstevel@tonic-gate * Dispatch pagefault.
6437c478bd9Sstevel@tonic-gate */
6447c478bd9Sstevel@tonic-gate res = as_fault(hat, as, addr, 1, type, rw);
6457c478bd9Sstevel@tonic-gate
6467c478bd9Sstevel@tonic-gate /*
6477c478bd9Sstevel@tonic-gate * If this isn't a potential unmapped hole in the user's
6487c478bd9Sstevel@tonic-gate * UNIX data or stack segments, just return status info.
6497c478bd9Sstevel@tonic-gate */
6507c478bd9Sstevel@tonic-gate if (res != FC_NOMAP || iskernel)
6517c478bd9Sstevel@tonic-gate goto out;
6527c478bd9Sstevel@tonic-gate
6537c478bd9Sstevel@tonic-gate /*
6547c478bd9Sstevel@tonic-gate * Check to see if we happened to faulted on a currently unmapped
6557c478bd9Sstevel@tonic-gate * part of the UNIX data or stack segments. If so, create a zfod
6567c478bd9Sstevel@tonic-gate * mapping there and then try calling the fault routine again.
6577c478bd9Sstevel@tonic-gate */
6587c478bd9Sstevel@tonic-gate base = p->p_brkbase;
6597c478bd9Sstevel@tonic-gate len = p->p_brksize;
6607c478bd9Sstevel@tonic-gate
6617c478bd9Sstevel@tonic-gate if (addr < base || addr >= base + len) { /* data seg? */
6627c478bd9Sstevel@tonic-gate base = (caddr_t)p->p_usrstack - p->p_stksize;
6637c478bd9Sstevel@tonic-gate len = p->p_stksize;
6647c478bd9Sstevel@tonic-gate if (addr < base || addr >= p->p_usrstack) { /* stack seg? */
6657c478bd9Sstevel@tonic-gate /* not in either UNIX data or stack segments */
6667c478bd9Sstevel@tonic-gate res = FC_NOMAP;
6677c478bd9Sstevel@tonic-gate goto out;
6687c478bd9Sstevel@tonic-gate }
6697c478bd9Sstevel@tonic-gate }
6707c478bd9Sstevel@tonic-gate
6717c478bd9Sstevel@tonic-gate /*
6727c478bd9Sstevel@tonic-gate * the rest of this function implements a 3.X 4.X 5.X compatibility
6737c478bd9Sstevel@tonic-gate * This code is probably not needed anymore
6747c478bd9Sstevel@tonic-gate */
6757c478bd9Sstevel@tonic-gate if (p->p_model == DATAMODEL_ILP32) {
6767c478bd9Sstevel@tonic-gate
6777c478bd9Sstevel@tonic-gate /* expand the gap to the page boundaries on each side */
6787c478bd9Sstevel@tonic-gate ea = P2ROUNDUP((uintptr_t)base + len, MMU_PAGESIZE);
6797c478bd9Sstevel@tonic-gate base = (caddr_t)P2ALIGN((uintptr_t)base, MMU_PAGESIZE);
6807c478bd9Sstevel@tonic-gate len = ea - (uintptr_t)base;
6817c478bd9Sstevel@tonic-gate
6827c478bd9Sstevel@tonic-gate as_rangelock(as);
6837c478bd9Sstevel@tonic-gate if (as_gap(as, MMU_PAGESIZE, &base, &len, AH_CONTAIN, addr) ==
6847c478bd9Sstevel@tonic-gate 0) {
6857c478bd9Sstevel@tonic-gate err = as_map(as, base, len, segvn_create, zfod_argsp);
6867c478bd9Sstevel@tonic-gate as_rangeunlock(as);
6877c478bd9Sstevel@tonic-gate if (err) {
6887c478bd9Sstevel@tonic-gate res = FC_MAKE_ERR(err);
6897c478bd9Sstevel@tonic-gate goto out;
6907c478bd9Sstevel@tonic-gate }
6917c478bd9Sstevel@tonic-gate } else {
6927c478bd9Sstevel@tonic-gate /*
6937c478bd9Sstevel@tonic-gate * This page is already mapped by another thread after
6947c478bd9Sstevel@tonic-gate * we returned from as_fault() above. We just fall
6957c478bd9Sstevel@tonic-gate * through as_fault() below.
6967c478bd9Sstevel@tonic-gate */
6977c478bd9Sstevel@tonic-gate as_rangeunlock(as);
6987c478bd9Sstevel@tonic-gate }
6997c478bd9Sstevel@tonic-gate
7007c478bd9Sstevel@tonic-gate res = as_fault(hat, as, addr, 1, F_INVAL, rw);
7017c478bd9Sstevel@tonic-gate }
7027c478bd9Sstevel@tonic-gate
7037c478bd9Sstevel@tonic-gate out:
7047c478bd9Sstevel@tonic-gate if (mapped_red)
7057c478bd9Sstevel@tonic-gate segkp_unmap_red();
7067c478bd9Sstevel@tonic-gate
7077c478bd9Sstevel@tonic-gate return (res);
7087c478bd9Sstevel@tonic-gate }
7097c478bd9Sstevel@tonic-gate
7107c478bd9Sstevel@tonic-gate void
map_addr(caddr_t * addrp,size_t len,offset_t off,int vacalign,uint_t flags)7117c478bd9Sstevel@tonic-gate map_addr(caddr_t *addrp, size_t len, offset_t off, int vacalign, uint_t flags)
7127c478bd9Sstevel@tonic-gate {
7137c478bd9Sstevel@tonic-gate struct proc *p = curproc;
7147c478bd9Sstevel@tonic-gate caddr_t userlimit = (flags & _MAP_LOW32) ?
7157c478bd9Sstevel@tonic-gate (caddr_t)_userlimit32 : p->p_as->a_userlimit;
7167c478bd9Sstevel@tonic-gate
7177c478bd9Sstevel@tonic-gate map_addr_proc(addrp, len, off, vacalign, userlimit, curproc, flags);
7187c478bd9Sstevel@tonic-gate }
7197c478bd9Sstevel@tonic-gate
7207c478bd9Sstevel@tonic-gate /*ARGSUSED*/
7217c478bd9Sstevel@tonic-gate int
map_addr_vacalign_check(caddr_t addr,u_offset_t off)7227c478bd9Sstevel@tonic-gate map_addr_vacalign_check(caddr_t addr, u_offset_t off)
7237c478bd9Sstevel@tonic-gate {
7247c478bd9Sstevel@tonic-gate return (0);
7257c478bd9Sstevel@tonic-gate }
7267c478bd9Sstevel@tonic-gate
727d2a70789SRichard Lowe /*
728d2a70789SRichard Lowe * The maximum amount a randomized mapping will be slewed. We should perhaps
729d2a70789SRichard Lowe * arrange things so these tunables can be separate for mmap, mmapobj, and
730d2a70789SRichard Lowe * ld.so
731d2a70789SRichard Lowe */
732d2a70789SRichard Lowe size_t aslr_max_map_skew = 256 * 1024 * 1024; /* 256MB */
733d2a70789SRichard Lowe
7347c478bd9Sstevel@tonic-gate /*
7357c478bd9Sstevel@tonic-gate * map_addr_proc() is the routine called when the system is to
7367c478bd9Sstevel@tonic-gate * choose an address for the user. We will pick an address
737ae115bc7Smrj * range which is the highest available below userlimit.
7387c478bd9Sstevel@tonic-gate *
73946ab9534Smec * Every mapping will have a redzone of a single page on either side of
74046ab9534Smec * the request. This is done to leave one page unmapped between segments.
74146ab9534Smec * This is not required, but it's useful for the user because if their
74246ab9534Smec * program strays across a segment boundary, it will catch a fault
74346ab9534Smec * immediately making debugging a little easier. Currently the redzone
74446ab9534Smec * is mandatory.
74546ab9534Smec *
7467c478bd9Sstevel@tonic-gate * addrp is a value/result parameter.
7477c478bd9Sstevel@tonic-gate * On input it is a hint from the user to be used in a completely
7487c478bd9Sstevel@tonic-gate * machine dependent fashion. We decide to completely ignore this hint.
74946ab9534Smec * If MAP_ALIGN was specified, addrp contains the minimal alignment, which
75046ab9534Smec * must be some "power of two" multiple of pagesize.
7517c478bd9Sstevel@tonic-gate *
7527c478bd9Sstevel@tonic-gate * On output it is NULL if no address can be found in the current
7537c478bd9Sstevel@tonic-gate * processes address space or else an address that is currently
7547c478bd9Sstevel@tonic-gate * not mapped for len bytes with a page of red zone on either side.
7557c478bd9Sstevel@tonic-gate *
75646ab9534Smec * vacalign is not needed on x86 (it's for viturally addressed caches)
7577c478bd9Sstevel@tonic-gate */
7587c478bd9Sstevel@tonic-gate /*ARGSUSED*/
7597c478bd9Sstevel@tonic-gate void
map_addr_proc(caddr_t * addrp,size_t len,offset_t off,int vacalign,caddr_t userlimit,struct proc * p,uint_t flags)7607c478bd9Sstevel@tonic-gate map_addr_proc(
7617c478bd9Sstevel@tonic-gate caddr_t *addrp,
7627c478bd9Sstevel@tonic-gate size_t len,
7637c478bd9Sstevel@tonic-gate offset_t off,
7647c478bd9Sstevel@tonic-gate int vacalign,
7657c478bd9Sstevel@tonic-gate caddr_t userlimit,
7667c478bd9Sstevel@tonic-gate struct proc *p,
7677c478bd9Sstevel@tonic-gate uint_t flags)
7687c478bd9Sstevel@tonic-gate {
7697c478bd9Sstevel@tonic-gate struct as *as = p->p_as;
7707c478bd9Sstevel@tonic-gate caddr_t addr;
7717c478bd9Sstevel@tonic-gate caddr_t base;
7727c478bd9Sstevel@tonic-gate size_t slen;
7737c478bd9Sstevel@tonic-gate size_t align_amount;
7747c478bd9Sstevel@tonic-gate
7757c478bd9Sstevel@tonic-gate ASSERT32(userlimit == as->a_userlimit);
7767c478bd9Sstevel@tonic-gate
7777c478bd9Sstevel@tonic-gate base = p->p_brkbase;
7787c478bd9Sstevel@tonic-gate if (p->p_model == DATAMODEL_NATIVE) {
7797c478bd9Sstevel@tonic-gate if (userlimit < as->a_userlimit) {
7807c478bd9Sstevel@tonic-gate /*
7817c478bd9Sstevel@tonic-gate * This happens when a program wants to map
7827c478bd9Sstevel@tonic-gate * something in a range that's accessible to a
7837c478bd9Sstevel@tonic-gate * program in a smaller address space. For example,
7847c478bd9Sstevel@tonic-gate * a 64-bit program calling mmap32(2) to guarantee
7857c478bd9Sstevel@tonic-gate * that the returned address is below 4Gbytes.
7867c478bd9Sstevel@tonic-gate */
7877c478bd9Sstevel@tonic-gate ASSERT((uintptr_t)userlimit < ADDRESS_C(0xffffffff));
7887c478bd9Sstevel@tonic-gate
7897c478bd9Sstevel@tonic-gate if (userlimit > base)
7907c478bd9Sstevel@tonic-gate slen = userlimit - base;
7917c478bd9Sstevel@tonic-gate else {
7927c478bd9Sstevel@tonic-gate *addrp = NULL;
7937c478bd9Sstevel@tonic-gate return;
7947c478bd9Sstevel@tonic-gate }
7957c478bd9Sstevel@tonic-gate } else {
7967c478bd9Sstevel@tonic-gate /*
797284ce987SPatrick Mooney * With the stack positioned at a higher address than
798284ce987SPatrick Mooney * the heap for 64-bit processes, it is necessary to be
799284ce987SPatrick Mooney * mindful of its location and potential size.
800284ce987SPatrick Mooney *
801284ce987SPatrick Mooney * Unallocated space above the top of the stack (that
802284ce987SPatrick Mooney * is, at a lower address) but still within the bounds
803284ce987SPatrick Mooney * of the stack limit should be considered unavailable.
804284ce987SPatrick Mooney *
805284ce987SPatrick Mooney * As the 64-bit stack guard is mapped in immediately
806284ce987SPatrick Mooney * adjacent to the stack limit boundary, this prevents
807284ce987SPatrick Mooney * new mappings from having accidentally dangerous
808284ce987SPatrick Mooney * proximity to the stack.
8097c478bd9Sstevel@tonic-gate */
8107c478bd9Sstevel@tonic-gate slen = p->p_usrstack - base -
8111e1e1eecSMichael Corcoran ((p->p_stk_ctl + PAGEOFFSET) & PAGEMASK);
8127c478bd9Sstevel@tonic-gate }
81386ef0a63SRichard Lowe } else {
8147c478bd9Sstevel@tonic-gate slen = userlimit - base;
81586ef0a63SRichard Lowe }
8167c478bd9Sstevel@tonic-gate
81746ab9534Smec /* Make len be a multiple of PAGESIZE */
8187c478bd9Sstevel@tonic-gate len = (len + PAGEOFFSET) & PAGEMASK;
8197c478bd9Sstevel@tonic-gate
8207c478bd9Sstevel@tonic-gate /*
8217c478bd9Sstevel@tonic-gate * figure out what the alignment should be
8227c478bd9Sstevel@tonic-gate *
8237c478bd9Sstevel@tonic-gate * XX64 -- is there an ELF_AMD64_MAXPGSZ or is it the same????
8247c478bd9Sstevel@tonic-gate */
8257c478bd9Sstevel@tonic-gate if (len <= ELF_386_MAXPGSZ) {
8267c478bd9Sstevel@tonic-gate /*
8277c478bd9Sstevel@tonic-gate * Align virtual addresses to ensure that ELF shared libraries
8287c478bd9Sstevel@tonic-gate * are mapped with the appropriate alignment constraints by
8297c478bd9Sstevel@tonic-gate * the run-time linker.
8307c478bd9Sstevel@tonic-gate */
8317c478bd9Sstevel@tonic-gate align_amount = ELF_386_MAXPGSZ;
8327c478bd9Sstevel@tonic-gate } else {
833534f2768SSudheer A /*
834534f2768SSudheer A * For 32-bit processes, only those which have specified
835534f2768SSudheer A * MAP_ALIGN and an addr will be aligned on a larger page size.
836534f2768SSudheer A * Not doing so can potentially waste up to 1G of process
837534f2768SSudheer A * address space.
838534f2768SSudheer A */
839534f2768SSudheer A int lvl = (p->p_model == DATAMODEL_ILP32) ? 1 :
840534f2768SSudheer A mmu.umax_page_level;
8417c478bd9Sstevel@tonic-gate
842534f2768SSudheer A while (lvl && len < LEVEL_SIZE(lvl))
843534f2768SSudheer A --lvl;
8447c478bd9Sstevel@tonic-gate
845534f2768SSudheer A align_amount = LEVEL_SIZE(lvl);
8467c478bd9Sstevel@tonic-gate }
8477c478bd9Sstevel@tonic-gate if ((flags & MAP_ALIGN) && ((uintptr_t)*addrp > align_amount))
8487c478bd9Sstevel@tonic-gate align_amount = (uintptr_t)*addrp;
8497c478bd9Sstevel@tonic-gate
85046ab9534Smec ASSERT(ISP2(align_amount));
85146ab9534Smec ASSERT(align_amount == 0 || align_amount >= PAGESIZE);
8527c478bd9Sstevel@tonic-gate
85346ab9534Smec off = off & (align_amount - 1);
854d2a70789SRichard Lowe
8557c478bd9Sstevel@tonic-gate /*
8567c478bd9Sstevel@tonic-gate * Look for a large enough hole starting below userlimit.
85746ab9534Smec * After finding it, use the upper part.
8587c478bd9Sstevel@tonic-gate */
85946ab9534Smec if (as_gap_aligned(as, len, &base, &slen, AH_HI, NULL, align_amount,
86046ab9534Smec PAGESIZE, off) == 0) {
8617c478bd9Sstevel@tonic-gate caddr_t as_addr;
8627c478bd9Sstevel@tonic-gate
86346ab9534Smec /*
86446ab9534Smec * addr is the highest possible address to use since we have
86546ab9534Smec * a PAGESIZE redzone at the beginning and end.
86646ab9534Smec */
86746ab9534Smec addr = base + slen - (PAGESIZE + len);
8687c478bd9Sstevel@tonic-gate as_addr = addr;
8697c478bd9Sstevel@tonic-gate /*
87046ab9534Smec * Round address DOWN to the alignment amount and
87146ab9534Smec * add the offset in.
87246ab9534Smec * If addr is greater than as_addr, len would not be large
87346ab9534Smec * enough to include the redzone, so we must adjust down
87446ab9534Smec * by the alignment amount.
8757c478bd9Sstevel@tonic-gate */
8767c478bd9Sstevel@tonic-gate addr = (caddr_t)((uintptr_t)addr & (~(align_amount - 1)));
87746ab9534Smec addr += (uintptr_t)off;
87846ab9534Smec if (addr > as_addr) {
87946ab9534Smec addr -= align_amount;
88046ab9534Smec }
8817c478bd9Sstevel@tonic-gate
882d2a70789SRichard Lowe /*
883d2a70789SRichard Lowe * If randomization is requested, slew the allocation
884d2a70789SRichard Lowe * backwards, within the same gap, by a random amount.
885d2a70789SRichard Lowe */
886d2a70789SRichard Lowe if (flags & _MAP_RANDOMIZE) {
887d2a70789SRichard Lowe uint32_t slew;
888d2a70789SRichard Lowe
889d2a70789SRichard Lowe (void) random_get_pseudo_bytes((uint8_t *)&slew,
890d2a70789SRichard Lowe sizeof (slew));
891d2a70789SRichard Lowe
892d2a70789SRichard Lowe slew = slew % MIN(aslr_max_map_skew, (addr - base));
893d2a70789SRichard Lowe addr -= P2ALIGN(slew, align_amount);
894d2a70789SRichard Lowe }
895d2a70789SRichard Lowe
89646ab9534Smec ASSERT(addr > base);
89746ab9534Smec ASSERT(addr + len < base + slen);
8987c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)addr & (align_amount - 1)) ==
89946ab9534Smec ((uintptr_t)(off)));
9007c478bd9Sstevel@tonic-gate *addrp = addr;
9017c478bd9Sstevel@tonic-gate } else {
9027c478bd9Sstevel@tonic-gate *addrp = NULL; /* no more virtual space */
9037c478bd9Sstevel@tonic-gate }
9047c478bd9Sstevel@tonic-gate }
9057c478bd9Sstevel@tonic-gate
90646ab9534Smec int valid_va_range_aligned_wraparound;
90746ab9534Smec
9087c478bd9Sstevel@tonic-gate /*
90946ab9534Smec * Determine whether [*basep, *basep + *lenp) contains a mappable range of
91046ab9534Smec * addresses at least "minlen" long, where the base of the range is at "off"
91146ab9534Smec * phase from an "align" boundary and there is space for a "redzone"-sized
91246ab9534Smec * redzone on either side of the range. On success, 1 is returned and *basep
91346ab9534Smec * and *lenp are adjusted to describe the acceptable range (including
91446ab9534Smec * the redzone). On failure, 0 is returned.
9157c478bd9Sstevel@tonic-gate */
9167c478bd9Sstevel@tonic-gate /*ARGSUSED3*/
9177c478bd9Sstevel@tonic-gate int
valid_va_range_aligned(caddr_t * basep,size_t * lenp,size_t minlen,int dir,size_t align,size_t redzone,size_t off)91846ab9534Smec valid_va_range_aligned(caddr_t *basep, size_t *lenp, size_t minlen, int dir,
91946ab9534Smec size_t align, size_t redzone, size_t off)
9207c478bd9Sstevel@tonic-gate {
9217c478bd9Sstevel@tonic-gate uintptr_t hi, lo;
92246ab9534Smec size_t tot_len;
92346ab9534Smec
92446ab9534Smec ASSERT(align == 0 ? off == 0 : off < align);
92546ab9534Smec ASSERT(ISP2(align));
92646ab9534Smec ASSERT(align == 0 || align >= PAGESIZE);
9277c478bd9Sstevel@tonic-gate
9287c478bd9Sstevel@tonic-gate lo = (uintptr_t)*basep;
9297c478bd9Sstevel@tonic-gate hi = lo + *lenp;
93046ab9534Smec tot_len = minlen + 2 * redzone; /* need at least this much space */
9317c478bd9Sstevel@tonic-gate
9327c478bd9Sstevel@tonic-gate /*
9337c478bd9Sstevel@tonic-gate * If hi rolled over the top, try cutting back.
9347c478bd9Sstevel@tonic-gate */
9357c478bd9Sstevel@tonic-gate if (hi < lo) {
93646ab9534Smec *lenp = 0UL - lo - 1UL;
93746ab9534Smec /* See if this really happens. If so, then we figure out why */
93846ab9534Smec valid_va_range_aligned_wraparound++;
93946ab9534Smec hi = lo + *lenp;
94046ab9534Smec }
94146ab9534Smec if (*lenp < tot_len) {
9427c478bd9Sstevel@tonic-gate return (0);
9437c478bd9Sstevel@tonic-gate }
94446ab9534Smec
9457c478bd9Sstevel@tonic-gate /*
9467c478bd9Sstevel@tonic-gate * Deal with a possible hole in the address range between
9477c478bd9Sstevel@tonic-gate * hole_start and hole_end that should never be mapped.
9487c478bd9Sstevel@tonic-gate */
9497c478bd9Sstevel@tonic-gate if (lo < hole_start) {
9507c478bd9Sstevel@tonic-gate if (hi > hole_start) {
9517c478bd9Sstevel@tonic-gate if (hi < hole_end) {
9527c478bd9Sstevel@tonic-gate hi = hole_start;
9537c478bd9Sstevel@tonic-gate } else {
9547c478bd9Sstevel@tonic-gate /* lo < hole_start && hi >= hole_end */
9557c478bd9Sstevel@tonic-gate if (dir == AH_LO) {
9567c478bd9Sstevel@tonic-gate /*
9577c478bd9Sstevel@tonic-gate * prefer lowest range
9587c478bd9Sstevel@tonic-gate */
95946ab9534Smec if (hole_start - lo >= tot_len)
9607c478bd9Sstevel@tonic-gate hi = hole_start;
96146ab9534Smec else if (hi - hole_end >= tot_len)
9627c478bd9Sstevel@tonic-gate lo = hole_end;
9637c478bd9Sstevel@tonic-gate else
9647c478bd9Sstevel@tonic-gate return (0);
9657c478bd9Sstevel@tonic-gate } else {
9667c478bd9Sstevel@tonic-gate /*
9677c478bd9Sstevel@tonic-gate * prefer highest range
9687c478bd9Sstevel@tonic-gate */
96946ab9534Smec if (hi - hole_end >= tot_len)
9707c478bd9Sstevel@tonic-gate lo = hole_end;
97146ab9534Smec else if (hole_start - lo >= tot_len)
9727c478bd9Sstevel@tonic-gate hi = hole_start;
9737c478bd9Sstevel@tonic-gate else
9747c478bd9Sstevel@tonic-gate return (0);
9757c478bd9Sstevel@tonic-gate }
9767c478bd9Sstevel@tonic-gate }
9777c478bd9Sstevel@tonic-gate }
9787c478bd9Sstevel@tonic-gate } else {
9797c478bd9Sstevel@tonic-gate /* lo >= hole_start */
9807c478bd9Sstevel@tonic-gate if (hi < hole_end)
9817c478bd9Sstevel@tonic-gate return (0);
9827c478bd9Sstevel@tonic-gate if (lo < hole_end)
9837c478bd9Sstevel@tonic-gate lo = hole_end;
9847c478bd9Sstevel@tonic-gate }
9857c478bd9Sstevel@tonic-gate
98646ab9534Smec if (hi - lo < tot_len)
9877c478bd9Sstevel@tonic-gate return (0);
9887c478bd9Sstevel@tonic-gate
98946ab9534Smec if (align > 1) {
99046ab9534Smec uintptr_t tlo = lo + redzone;
99146ab9534Smec uintptr_t thi = hi - redzone;
99246ab9534Smec tlo = (uintptr_t)P2PHASEUP(tlo, align, off);
99346ab9534Smec if (tlo < lo + redzone) {
99446ab9534Smec return (0);
99546ab9534Smec }
99646ab9534Smec if (thi < tlo || thi - tlo < minlen) {
99746ab9534Smec return (0);
99846ab9534Smec }
99946ab9534Smec }
100046ab9534Smec
10017c478bd9Sstevel@tonic-gate *basep = (caddr_t)lo;
10027c478bd9Sstevel@tonic-gate *lenp = hi - lo;
10037c478bd9Sstevel@tonic-gate return (1);
10047c478bd9Sstevel@tonic-gate }
10057c478bd9Sstevel@tonic-gate
100646ab9534Smec /*
100746ab9534Smec * Determine whether [*basep, *basep + *lenp) contains a mappable range of
100846ab9534Smec * addresses at least "minlen" long. On success, 1 is returned and *basep
100946ab9534Smec * and *lenp are adjusted to describe the acceptable range. On failure, 0
101046ab9534Smec * is returned.
101146ab9534Smec */
101246ab9534Smec int
valid_va_range(caddr_t * basep,size_t * lenp,size_t minlen,int dir)101346ab9534Smec valid_va_range(caddr_t *basep, size_t *lenp, size_t minlen, int dir)
101446ab9534Smec {
101546ab9534Smec return (valid_va_range_aligned(basep, lenp, minlen, dir, 0, 0, 0));
101646ab9534Smec }
101746ab9534Smec
1018d2a70789SRichard Lowe /*
1019d2a70789SRichard Lowe * Default to forbidding the first 64k of address space. This protects most
1020d2a70789SRichard Lowe * reasonably sized structures from dereferences through NULL:
1021d2a70789SRichard Lowe * ((foo_t *)0)->bar
1022d2a70789SRichard Lowe */
1023d2a70789SRichard Lowe uintptr_t forbidden_null_mapping_sz = 0x10000;
1024d2a70789SRichard Lowe
10257c478bd9Sstevel@tonic-gate /*
10267c478bd9Sstevel@tonic-gate * Determine whether [addr, addr+len] are valid user addresses.
10277c478bd9Sstevel@tonic-gate */
10287c478bd9Sstevel@tonic-gate /*ARGSUSED*/
10297c478bd9Sstevel@tonic-gate int
valid_usr_range(caddr_t addr,size_t len,uint_t prot,struct as * as,caddr_t userlimit)10307c478bd9Sstevel@tonic-gate valid_usr_range(caddr_t addr, size_t len, uint_t prot, struct as *as,
10317c478bd9Sstevel@tonic-gate caddr_t userlimit)
10327c478bd9Sstevel@tonic-gate {
10337c478bd9Sstevel@tonic-gate caddr_t eaddr = addr + len;
10347c478bd9Sstevel@tonic-gate
10357c478bd9Sstevel@tonic-gate if (eaddr <= addr || addr >= userlimit || eaddr > userlimit)
10367c478bd9Sstevel@tonic-gate return (RANGE_BADADDR);
10377c478bd9Sstevel@tonic-gate
1038d2a70789SRichard Lowe if ((addr <= (caddr_t)forbidden_null_mapping_sz) &&
1039a02406b9SPatrick Mooney as->a_proc != NULL &&
1040d2a70789SRichard Lowe secflag_enabled(as->a_proc, PROC_SEC_FORBIDNULLMAP))
1041d2a70789SRichard Lowe return (RANGE_BADADDR);
1042d2a70789SRichard Lowe
10437c478bd9Sstevel@tonic-gate /*
10447c478bd9Sstevel@tonic-gate * Check for the VA hole
10457c478bd9Sstevel@tonic-gate */
10467c478bd9Sstevel@tonic-gate if (eaddr > (caddr_t)hole_start && addr < (caddr_t)hole_end)
10477c478bd9Sstevel@tonic-gate return (RANGE_BADADDR);
10487c478bd9Sstevel@tonic-gate
10497c478bd9Sstevel@tonic-gate return (RANGE_OKAY);
10507c478bd9Sstevel@tonic-gate }
10517c478bd9Sstevel@tonic-gate
10527c478bd9Sstevel@tonic-gate /*
10537c478bd9Sstevel@tonic-gate * Return 1 if the page frame is onboard memory, else 0.
10547c478bd9Sstevel@tonic-gate */
10557c478bd9Sstevel@tonic-gate int
pf_is_memory(pfn_t pf)10567c478bd9Sstevel@tonic-gate pf_is_memory(pfn_t pf)
10577c478bd9Sstevel@tonic-gate {
1058ae115bc7Smrj if (pfn_is_foreign(pf))
1059ae115bc7Smrj return (0);
1060ae115bc7Smrj return (address_in_memlist(phys_install, pfn_to_pa(pf), 1));
10617c478bd9Sstevel@tonic-gate }
10627c478bd9Sstevel@tonic-gate
10637c478bd9Sstevel@tonic-gate /*
10647c478bd9Sstevel@tonic-gate * return the memrange containing pfn
10657c478bd9Sstevel@tonic-gate */
10667c478bd9Sstevel@tonic-gate int
memrange_num(pfn_t pfn)10677c478bd9Sstevel@tonic-gate memrange_num(pfn_t pfn)
10687c478bd9Sstevel@tonic-gate {
10697c478bd9Sstevel@tonic-gate int n;
10707c478bd9Sstevel@tonic-gate
10717c478bd9Sstevel@tonic-gate for (n = 0; n < nranges - 1; ++n) {
10727c478bd9Sstevel@tonic-gate if (pfn >= memranges[n])
10737c478bd9Sstevel@tonic-gate break;
10747c478bd9Sstevel@tonic-gate }
10757c478bd9Sstevel@tonic-gate return (n);
10767c478bd9Sstevel@tonic-gate }
10777c478bd9Sstevel@tonic-gate
10787c478bd9Sstevel@tonic-gate /*
10797c478bd9Sstevel@tonic-gate * return the mnoderange containing pfn
10807c478bd9Sstevel@tonic-gate */
1081843e1988Sjohnlev /*ARGSUSED*/
10827c478bd9Sstevel@tonic-gate int
pfn_2_mtype(pfn_t pfn)10837c478bd9Sstevel@tonic-gate pfn_2_mtype(pfn_t pfn)
10847c478bd9Sstevel@tonic-gate {
1085843e1988Sjohnlev #if defined(__xpv)
1086843e1988Sjohnlev return (0);
1087843e1988Sjohnlev #else
10887c478bd9Sstevel@tonic-gate int n;
10897c478bd9Sstevel@tonic-gate
1090a3114836SGerry Liu /* Always start from highest pfn and work our way down */
1091a3114836SGerry Liu for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) {
10927c478bd9Sstevel@tonic-gate if (pfn >= mnoderanges[n].mnr_pfnlo) {
10937c478bd9Sstevel@tonic-gate break;
10947c478bd9Sstevel@tonic-gate }
10957c478bd9Sstevel@tonic-gate }
10967c478bd9Sstevel@tonic-gate return (n);
1097843e1988Sjohnlev #endif
10987c478bd9Sstevel@tonic-gate }
10997c478bd9Sstevel@tonic-gate
1100843e1988Sjohnlev #if !defined(__xpv)
11017c478bd9Sstevel@tonic-gate /*
11027c478bd9Sstevel@tonic-gate * is_contigpage_free:
11037c478bd9Sstevel@tonic-gate * returns a page list of contiguous pages. It minimally has to return
11047c478bd9Sstevel@tonic-gate * minctg pages. Caller determines minctg based on the scatter-gather
11057c478bd9Sstevel@tonic-gate * list length.
11067c478bd9Sstevel@tonic-gate *
11077c478bd9Sstevel@tonic-gate * pfnp is set to the next page frame to search on return.
11087c478bd9Sstevel@tonic-gate */
11097c478bd9Sstevel@tonic-gate static page_t *
is_contigpage_free(pfn_t * pfnp,pgcnt_t * pgcnt,pgcnt_t minctg,uint64_t pfnseg,int iolock)11107c478bd9Sstevel@tonic-gate is_contigpage_free(
11117c478bd9Sstevel@tonic-gate pfn_t *pfnp,
11127c478bd9Sstevel@tonic-gate pgcnt_t *pgcnt,
11137c478bd9Sstevel@tonic-gate pgcnt_t minctg,
11147c478bd9Sstevel@tonic-gate uint64_t pfnseg,
11157c478bd9Sstevel@tonic-gate int iolock)
11167c478bd9Sstevel@tonic-gate {
11177c478bd9Sstevel@tonic-gate int i = 0;
11187c478bd9Sstevel@tonic-gate pfn_t pfn = *pfnp;
11197c478bd9Sstevel@tonic-gate page_t *pp;
11207c478bd9Sstevel@tonic-gate page_t *plist = NULL;
11217c478bd9Sstevel@tonic-gate
11227c478bd9Sstevel@tonic-gate /*
11237c478bd9Sstevel@tonic-gate * fail if pfn + minctg crosses a segment boundary.
11247c478bd9Sstevel@tonic-gate * Adjust for next starting pfn to begin at segment boundary.
11257c478bd9Sstevel@tonic-gate */
11267c478bd9Sstevel@tonic-gate
11277c478bd9Sstevel@tonic-gate if (((*pfnp + minctg - 1) & pfnseg) < (*pfnp & pfnseg)) {
11287c478bd9Sstevel@tonic-gate *pfnp = roundup(*pfnp, pfnseg + 1);
11297c478bd9Sstevel@tonic-gate return (NULL);
11307c478bd9Sstevel@tonic-gate }
11317c478bd9Sstevel@tonic-gate
11327c478bd9Sstevel@tonic-gate do {
11337c478bd9Sstevel@tonic-gate retry:
11347c478bd9Sstevel@tonic-gate pp = page_numtopp_nolock(pfn + i);
11351f84c0d7SDave Plauger if ((pp == NULL) || IS_DUMP_PAGE(pp) ||
11367c478bd9Sstevel@tonic-gate (page_trylock(pp, SE_EXCL) == 0)) {
11377c478bd9Sstevel@tonic-gate (*pfnp)++;
11387c478bd9Sstevel@tonic-gate break;
11397c478bd9Sstevel@tonic-gate }
11407c478bd9Sstevel@tonic-gate if (page_pptonum(pp) != pfn + i) {
11417c478bd9Sstevel@tonic-gate page_unlock(pp);
11427c478bd9Sstevel@tonic-gate goto retry;
11437c478bd9Sstevel@tonic-gate }
11447c478bd9Sstevel@tonic-gate
11457c478bd9Sstevel@tonic-gate if (!(PP_ISFREE(pp))) {
11467c478bd9Sstevel@tonic-gate page_unlock(pp);
11477c478bd9Sstevel@tonic-gate (*pfnp)++;
11487c478bd9Sstevel@tonic-gate break;
11497c478bd9Sstevel@tonic-gate }
11507c478bd9Sstevel@tonic-gate
11517c478bd9Sstevel@tonic-gate if (!PP_ISAGED(pp)) {
11527c478bd9Sstevel@tonic-gate page_list_sub(pp, PG_CACHE_LIST);
11537c478bd9Sstevel@tonic-gate page_hashout(pp, (kmutex_t *)NULL);
11547c478bd9Sstevel@tonic-gate } else {
11557c478bd9Sstevel@tonic-gate page_list_sub(pp, PG_FREE_LIST);
11567c478bd9Sstevel@tonic-gate }
11577c478bd9Sstevel@tonic-gate
11587c478bd9Sstevel@tonic-gate if (iolock)
11597c478bd9Sstevel@tonic-gate page_io_lock(pp);
11607c478bd9Sstevel@tonic-gate page_list_concat(&plist, &pp);
11617c478bd9Sstevel@tonic-gate
11627c478bd9Sstevel@tonic-gate /*
11637c478bd9Sstevel@tonic-gate * exit loop when pgcnt satisfied or segment boundary reached.
11647c478bd9Sstevel@tonic-gate */
11657c478bd9Sstevel@tonic-gate
11667c478bd9Sstevel@tonic-gate } while ((++i < *pgcnt) && ((pfn + i) & pfnseg));
11677c478bd9Sstevel@tonic-gate
11687c478bd9Sstevel@tonic-gate *pfnp += i; /* set to next pfn to search */
11697c478bd9Sstevel@tonic-gate
11707c478bd9Sstevel@tonic-gate if (i >= minctg) {
11717c478bd9Sstevel@tonic-gate *pgcnt -= i;
11727c478bd9Sstevel@tonic-gate return (plist);
11737c478bd9Sstevel@tonic-gate }
11747c478bd9Sstevel@tonic-gate
11757c478bd9Sstevel@tonic-gate /*
11767c478bd9Sstevel@tonic-gate * failure: minctg not satisfied.
11777c478bd9Sstevel@tonic-gate *
11787c478bd9Sstevel@tonic-gate * if next request crosses segment boundary, set next pfn
11797c478bd9Sstevel@tonic-gate * to search from the segment boundary.
11807c478bd9Sstevel@tonic-gate */
11817c478bd9Sstevel@tonic-gate if (((*pfnp + minctg - 1) & pfnseg) < (*pfnp & pfnseg))
11827c478bd9Sstevel@tonic-gate *pfnp = roundup(*pfnp, pfnseg + 1);
11837c478bd9Sstevel@tonic-gate
11847c478bd9Sstevel@tonic-gate /* clean up any pages already allocated */
11857c478bd9Sstevel@tonic-gate
11867c478bd9Sstevel@tonic-gate while (plist) {
11877c478bd9Sstevel@tonic-gate pp = plist;
11887c478bd9Sstevel@tonic-gate page_sub(&plist, pp);
11897c478bd9Sstevel@tonic-gate page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
11907c478bd9Sstevel@tonic-gate if (iolock)
11917c478bd9Sstevel@tonic-gate page_io_unlock(pp);
11927c478bd9Sstevel@tonic-gate page_unlock(pp);
11937c478bd9Sstevel@tonic-gate }
11947c478bd9Sstevel@tonic-gate
11957c478bd9Sstevel@tonic-gate return (NULL);
11967c478bd9Sstevel@tonic-gate }
1197843e1988Sjohnlev #endif /* !__xpv */
11987c478bd9Sstevel@tonic-gate
11997c478bd9Sstevel@tonic-gate /*
12007c478bd9Sstevel@tonic-gate * verify that pages being returned from allocator have correct DMA attribute
12017c478bd9Sstevel@tonic-gate */
12027c478bd9Sstevel@tonic-gate #ifndef DEBUG
1203a3114836SGerry Liu #define check_dma(a, b, c) (void)(0)
12047c478bd9Sstevel@tonic-gate #else
12057c478bd9Sstevel@tonic-gate static void
check_dma(ddi_dma_attr_t * dma_attr,page_t * pp,int cnt)12067c478bd9Sstevel@tonic-gate check_dma(ddi_dma_attr_t *dma_attr, page_t *pp, int cnt)
12077c478bd9Sstevel@tonic-gate {
12087c478bd9Sstevel@tonic-gate if (dma_attr == NULL)
12097c478bd9Sstevel@tonic-gate return;
12107c478bd9Sstevel@tonic-gate
12117c478bd9Sstevel@tonic-gate while (cnt-- > 0) {
1212ae115bc7Smrj if (pa_to_ma(pfn_to_pa(pp->p_pagenum)) <
12137c478bd9Sstevel@tonic-gate dma_attr->dma_attr_addr_lo)
1214903a11ebSrh panic("PFN (pp=%p) below dma_attr_addr_lo", (void *)pp);
1215ae115bc7Smrj if (pa_to_ma(pfn_to_pa(pp->p_pagenum)) >=
12167c478bd9Sstevel@tonic-gate dma_attr->dma_attr_addr_hi)
1217903a11ebSrh panic("PFN (pp=%p) above dma_attr_addr_hi", (void *)pp);
12187c478bd9Sstevel@tonic-gate pp = pp->p_next;
12197c478bd9Sstevel@tonic-gate }
12207c478bd9Sstevel@tonic-gate }
12217c478bd9Sstevel@tonic-gate #endif
12227c478bd9Sstevel@tonic-gate
1223843e1988Sjohnlev #if !defined(__xpv)
12247c478bd9Sstevel@tonic-gate static page_t *
page_get_contigpage(pgcnt_t * pgcnt,ddi_dma_attr_t * mattr,int iolock)12257c478bd9Sstevel@tonic-gate page_get_contigpage(pgcnt_t *pgcnt, ddi_dma_attr_t *mattr, int iolock)
12267c478bd9Sstevel@tonic-gate {
12277c478bd9Sstevel@tonic-gate pfn_t pfn;
12287c478bd9Sstevel@tonic-gate int sgllen;
12297c478bd9Sstevel@tonic-gate uint64_t pfnseg;
12307c478bd9Sstevel@tonic-gate pgcnt_t minctg;
12317c478bd9Sstevel@tonic-gate page_t *pplist = NULL, *plist;
12327c478bd9Sstevel@tonic-gate uint64_t lo, hi;
12337c478bd9Sstevel@tonic-gate pgcnt_t pfnalign = 0;
12347c478bd9Sstevel@tonic-gate static pfn_t startpfn;
12357c478bd9Sstevel@tonic-gate static pgcnt_t lastctgcnt;
12367c478bd9Sstevel@tonic-gate uintptr_t align;
12377c478bd9Sstevel@tonic-gate
12387c478bd9Sstevel@tonic-gate CONTIG_LOCK();
12397c478bd9Sstevel@tonic-gate
12407c478bd9Sstevel@tonic-gate if (mattr) {
12417c478bd9Sstevel@tonic-gate lo = mmu_btop((mattr->dma_attr_addr_lo + MMU_PAGEOFFSET));
12427c478bd9Sstevel@tonic-gate hi = mmu_btop(mattr->dma_attr_addr_hi);
12437c478bd9Sstevel@tonic-gate if (hi >= physmax)
12447c478bd9Sstevel@tonic-gate hi = physmax - 1;
12457c478bd9Sstevel@tonic-gate sgllen = mattr->dma_attr_sgllen;
12467c478bd9Sstevel@tonic-gate pfnseg = mmu_btop(mattr->dma_attr_seg);
12477c478bd9Sstevel@tonic-gate
12487c478bd9Sstevel@tonic-gate align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
12497c478bd9Sstevel@tonic-gate if (align > MMU_PAGESIZE)
12507c478bd9Sstevel@tonic-gate pfnalign = mmu_btop(align);
12517c478bd9Sstevel@tonic-gate
12527c478bd9Sstevel@tonic-gate /*
12537c478bd9Sstevel@tonic-gate * in order to satisfy the request, must minimally
12547c478bd9Sstevel@tonic-gate * acquire minctg contiguous pages
12557c478bd9Sstevel@tonic-gate */
12567c478bd9Sstevel@tonic-gate minctg = howmany(*pgcnt, sgllen);
12577c478bd9Sstevel@tonic-gate
12587c478bd9Sstevel@tonic-gate ASSERT(hi >= lo);
12597c478bd9Sstevel@tonic-gate
12607c478bd9Sstevel@tonic-gate /*
12617c478bd9Sstevel@tonic-gate * start from where last searched if the minctg >= lastctgcnt
12627c478bd9Sstevel@tonic-gate */
12637c478bd9Sstevel@tonic-gate if (minctg < lastctgcnt || startpfn < lo || startpfn > hi)
12647c478bd9Sstevel@tonic-gate startpfn = lo;
12657c478bd9Sstevel@tonic-gate } else {
12667c478bd9Sstevel@tonic-gate hi = physmax - 1;
12677c478bd9Sstevel@tonic-gate lo = 0;
12687c478bd9Sstevel@tonic-gate sgllen = 1;
12697c478bd9Sstevel@tonic-gate pfnseg = mmu.highest_pfn;
12707c478bd9Sstevel@tonic-gate minctg = *pgcnt;
12717c478bd9Sstevel@tonic-gate
12727c478bd9Sstevel@tonic-gate if (minctg < lastctgcnt)
12737c478bd9Sstevel@tonic-gate startpfn = lo;
12747c478bd9Sstevel@tonic-gate }
12757c478bd9Sstevel@tonic-gate lastctgcnt = minctg;
12767c478bd9Sstevel@tonic-gate
12777c478bd9Sstevel@tonic-gate ASSERT(pfnseg + 1 >= (uint64_t)minctg);
12787c478bd9Sstevel@tonic-gate
12797c478bd9Sstevel@tonic-gate /* conserve 16m memory - start search above 16m when possible */
12807c478bd9Sstevel@tonic-gate if (hi > PFN_16M && startpfn < PFN_16M)
12817c478bd9Sstevel@tonic-gate startpfn = PFN_16M;
12827c478bd9Sstevel@tonic-gate
12837c478bd9Sstevel@tonic-gate pfn = startpfn;
12847c478bd9Sstevel@tonic-gate if (pfnalign)
12857c478bd9Sstevel@tonic-gate pfn = P2ROUNDUP(pfn, pfnalign);
12867c478bd9Sstevel@tonic-gate
12877c478bd9Sstevel@tonic-gate while (pfn + minctg - 1 <= hi) {
12887c478bd9Sstevel@tonic-gate
12897c478bd9Sstevel@tonic-gate plist = is_contigpage_free(&pfn, pgcnt, minctg, pfnseg, iolock);
12907c478bd9Sstevel@tonic-gate if (plist) {
12917c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &plist);
12927c478bd9Sstevel@tonic-gate sgllen--;
12937c478bd9Sstevel@tonic-gate /*
12947c478bd9Sstevel@tonic-gate * return when contig pages no longer needed
12957c478bd9Sstevel@tonic-gate */
12967c478bd9Sstevel@tonic-gate if (!*pgcnt || ((*pgcnt <= sgllen) && !pfnalign)) {
12977c478bd9Sstevel@tonic-gate startpfn = pfn;
12987c478bd9Sstevel@tonic-gate CONTIG_UNLOCK();
12997c478bd9Sstevel@tonic-gate check_dma(mattr, pplist, *pgcnt);
13007c478bd9Sstevel@tonic-gate return (pplist);
13017c478bd9Sstevel@tonic-gate }
13027c478bd9Sstevel@tonic-gate minctg = howmany(*pgcnt, sgllen);
13037c478bd9Sstevel@tonic-gate }
13047c478bd9Sstevel@tonic-gate if (pfnalign)
13057c478bd9Sstevel@tonic-gate pfn = P2ROUNDUP(pfn, pfnalign);
13067c478bd9Sstevel@tonic-gate }
13077c478bd9Sstevel@tonic-gate
13087c478bd9Sstevel@tonic-gate /* cannot find contig pages in specified range */
13097c478bd9Sstevel@tonic-gate if (startpfn == lo) {
13107c478bd9Sstevel@tonic-gate CONTIG_UNLOCK();
13117c478bd9Sstevel@tonic-gate return (NULL);
13127c478bd9Sstevel@tonic-gate }
13137c478bd9Sstevel@tonic-gate
13147c478bd9Sstevel@tonic-gate /* did not start with lo previously */
13157c478bd9Sstevel@tonic-gate pfn = lo;
13167c478bd9Sstevel@tonic-gate if (pfnalign)
13177c478bd9Sstevel@tonic-gate pfn = P2ROUNDUP(pfn, pfnalign);
13187c478bd9Sstevel@tonic-gate
13197c478bd9Sstevel@tonic-gate /* allow search to go above startpfn */
13207c478bd9Sstevel@tonic-gate while (pfn < startpfn) {
13217c478bd9Sstevel@tonic-gate
13227c478bd9Sstevel@tonic-gate plist = is_contigpage_free(&pfn, pgcnt, minctg, pfnseg, iolock);
13237c478bd9Sstevel@tonic-gate if (plist != NULL) {
13247c478bd9Sstevel@tonic-gate
13257c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &plist);
13267c478bd9Sstevel@tonic-gate sgllen--;
13277c478bd9Sstevel@tonic-gate
13287c478bd9Sstevel@tonic-gate /*
13297c478bd9Sstevel@tonic-gate * return when contig pages no longer needed
13307c478bd9Sstevel@tonic-gate */
13317c478bd9Sstevel@tonic-gate if (!*pgcnt || ((*pgcnt <= sgllen) && !pfnalign)) {
13327c478bd9Sstevel@tonic-gate startpfn = pfn;
13337c478bd9Sstevel@tonic-gate CONTIG_UNLOCK();
13347c478bd9Sstevel@tonic-gate check_dma(mattr, pplist, *pgcnt);
13357c478bd9Sstevel@tonic-gate return (pplist);
13367c478bd9Sstevel@tonic-gate }
13377c478bd9Sstevel@tonic-gate minctg = howmany(*pgcnt, sgllen);
13387c478bd9Sstevel@tonic-gate }
13397c478bd9Sstevel@tonic-gate if (pfnalign)
13407c478bd9Sstevel@tonic-gate pfn = P2ROUNDUP(pfn, pfnalign);
13417c478bd9Sstevel@tonic-gate }
13427c478bd9Sstevel@tonic-gate CONTIG_UNLOCK();
13437c478bd9Sstevel@tonic-gate return (NULL);
13447c478bd9Sstevel@tonic-gate }
1345843e1988Sjohnlev #endif /* !__xpv */
13467c478bd9Sstevel@tonic-gate
13477c478bd9Sstevel@tonic-gate /*
13487c478bd9Sstevel@tonic-gate * mnode_range_cnt() calculates the number of memory ranges for mnode and
13497c478bd9Sstevel@tonic-gate * memranges[]. Used to determine the size of page lists and mnoderanges.
13507c478bd9Sstevel@tonic-gate */
13517c478bd9Sstevel@tonic-gate int
mnode_range_cnt(int mnode)13525d07b933Sdp mnode_range_cnt(int mnode)
13537c478bd9Sstevel@tonic-gate {
1354843e1988Sjohnlev #if defined(__xpv)
1355843e1988Sjohnlev ASSERT(mnode == 0);
1356843e1988Sjohnlev return (1);
1357843e1988Sjohnlev #else /* __xpv */
13587c478bd9Sstevel@tonic-gate int mri;
13597c478bd9Sstevel@tonic-gate int mnrcnt = 0;
13607c478bd9Sstevel@tonic-gate
13615d07b933Sdp if (mem_node_config[mnode].exists != 0) {
13627c478bd9Sstevel@tonic-gate mri = nranges - 1;
13637c478bd9Sstevel@tonic-gate
13647c478bd9Sstevel@tonic-gate /* find the memranges index below contained in mnode range */
13657c478bd9Sstevel@tonic-gate
13667c478bd9Sstevel@tonic-gate while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
13677c478bd9Sstevel@tonic-gate mri--;
13687c478bd9Sstevel@tonic-gate
13697c478bd9Sstevel@tonic-gate /*
13707c478bd9Sstevel@tonic-gate * increment mnode range counter when memranges or mnode
13717c478bd9Sstevel@tonic-gate * boundary is reached.
13727c478bd9Sstevel@tonic-gate */
13737c478bd9Sstevel@tonic-gate while (mri >= 0 &&
13747c478bd9Sstevel@tonic-gate mem_node_config[mnode].physmax >= MEMRANGELO(mri)) {
13757c478bd9Sstevel@tonic-gate mnrcnt++;
13767c478bd9Sstevel@tonic-gate if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
13777c478bd9Sstevel@tonic-gate mri--;
13787c478bd9Sstevel@tonic-gate else
13797c478bd9Sstevel@tonic-gate break;
13807c478bd9Sstevel@tonic-gate }
13817c478bd9Sstevel@tonic-gate }
13825d07b933Sdp ASSERT(mnrcnt <= MAX_MNODE_MRANGES);
13837c478bd9Sstevel@tonic-gate return (mnrcnt);
1384843e1988Sjohnlev #endif /* __xpv */
13857c478bd9Sstevel@tonic-gate }
13867c478bd9Sstevel@tonic-gate
1387dddac438SJohn Levon static int
mnoderange_cmp(const void * v1,const void * v2)1388dddac438SJohn Levon mnoderange_cmp(const void *v1, const void *v2)
1389dddac438SJohn Levon {
1390dddac438SJohn Levon const mnoderange_t *m1 = v1;
1391dddac438SJohn Levon const mnoderange_t *m2 = v2;
1392dddac438SJohn Levon
1393dddac438SJohn Levon if (m1->mnr_pfnlo < m2->mnr_pfnlo)
1394dddac438SJohn Levon return (-1);
1395dddac438SJohn Levon return (m1->mnr_pfnlo > m2->mnr_pfnlo);
1396dddac438SJohn Levon }
1397dddac438SJohn Levon
13987c478bd9Sstevel@tonic-gate void
mnode_range_setup(mnoderange_t * mnoderanges)13997c478bd9Sstevel@tonic-gate mnode_range_setup(mnoderange_t *mnoderanges)
14007c478bd9Sstevel@tonic-gate {
1401dddac438SJohn Levon mnoderange_t *mp;
14023df2e8b2SRobert Mustacchi ssize_t nr_ranges;
1403dddac438SJohn Levon size_t mnode;
1404dddac438SJohn Levon
1405dddac438SJohn Levon for (mnode = 0, nr_ranges = 0, mp = mnoderanges;
1406dddac438SJohn Levon mnode < max_mem_nodes; mnode++) {
14073df2e8b2SRobert Mustacchi ssize_t mri = nranges - 1;
1408dddac438SJohn Levon
14097c478bd9Sstevel@tonic-gate if (mem_node_config[mnode].exists == 0)
14107c478bd9Sstevel@tonic-gate continue;
14117c478bd9Sstevel@tonic-gate
14127c478bd9Sstevel@tonic-gate while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
14137c478bd9Sstevel@tonic-gate mri--;
14147c478bd9Sstevel@tonic-gate
14157c478bd9Sstevel@tonic-gate while (mri >= 0 && mem_node_config[mnode].physmax >=
14167c478bd9Sstevel@tonic-gate MEMRANGELO(mri)) {
1417dddac438SJohn Levon mp->mnr_pfnlo = MAX(MEMRANGELO(mri),
1418843e1988Sjohnlev mem_node_config[mnode].physbase);
1419dddac438SJohn Levon mp->mnr_pfnhi = MIN(MEMRANGEHI(mri),
1420843e1988Sjohnlev mem_node_config[mnode].physmax);
1421dddac438SJohn Levon mp->mnr_mnode = mnode;
1422dddac438SJohn Levon mp->mnr_memrange = mri;
1423dddac438SJohn Levon mp->mnr_next = -1;
1424dddac438SJohn Levon mp->mnr_exists = 1;
1425dddac438SJohn Levon mp++;
1426dddac438SJohn Levon nr_ranges++;
14277c478bd9Sstevel@tonic-gate if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
14287c478bd9Sstevel@tonic-gate mri--;
14297c478bd9Sstevel@tonic-gate else
14307c478bd9Sstevel@tonic-gate break;
14317c478bd9Sstevel@tonic-gate }
14327c478bd9Sstevel@tonic-gate }
1433a3114836SGerry Liu
1434a3114836SGerry Liu /*
1435dddac438SJohn Levon * mnoderangecnt can be larger than nr_ranges when memory DR is
1436dddac438SJohn Levon * supposedly supported.
1437a3114836SGerry Liu */
1438dddac438SJohn Levon VERIFY3U(nr_ranges, <=, mnoderangecnt);
1439dddac438SJohn Levon
1440dddac438SJohn Levon qsort(mnoderanges, nr_ranges, sizeof (mnoderange_t), mnoderange_cmp);
1441dddac438SJohn Levon
1442dddac438SJohn Levon /*
1443dddac438SJohn Levon * If some intrepid soul takes the axe to the memory DR code, we can
1444dddac438SJohn Levon * remove ->mnr_next altogether, as we just sorted by ->mnr_pfnlo order.
1445dddac438SJohn Levon *
1446dddac438SJohn Levon * The VERIFY3U() above can be "==" then too.
1447dddac438SJohn Levon */
1448dddac438SJohn Levon for (size_t i = 1; i < nr_ranges; i++)
1449dddac438SJohn Levon mnoderanges[i].mnr_next = i - 1;
1450dddac438SJohn Levon
1451dddac438SJohn Levon mtypetop = nr_ranges - 1;
1452dddac438SJohn Levon mtype16m = pfn_2_mtype(PFN_16MEG - 1); /* Can be -1 ... */
1453dddac438SJohn Levon if (physmax4g)
1454dddac438SJohn Levon mtype4g = pfn_2_mtype(0xfffff);
14557c478bd9Sstevel@tonic-gate }
14567c478bd9Sstevel@tonic-gate
1457a3114836SGerry Liu #ifndef __xpv
1458a3114836SGerry Liu /*
1459a3114836SGerry Liu * Update mnoderanges for memory hot-add DR operations.
1460a3114836SGerry Liu */
1461a3114836SGerry Liu static void
mnode_range_add(int mnode)1462a3114836SGerry Liu mnode_range_add(int mnode)
1463a3114836SGerry Liu {
1464a3114836SGerry Liu int *prev;
1465a3114836SGerry Liu int n, mri;
1466a3114836SGerry Liu pfn_t start, end;
1467a3114836SGerry Liu extern void membar_sync(void);
1468a3114836SGerry Liu
1469a3114836SGerry Liu ASSERT(0 <= mnode && mnode < max_mem_nodes);
1470a3114836SGerry Liu ASSERT(mem_node_config[mnode].exists);
1471a3114836SGerry Liu start = mem_node_config[mnode].physbase;
1472a3114836SGerry Liu end = mem_node_config[mnode].physmax;
1473a3114836SGerry Liu ASSERT(start <= end);
1474a3114836SGerry Liu mutex_enter(&mnoderange_lock);
1475a3114836SGerry Liu
1476a3114836SGerry Liu #ifdef DEBUG
1477a3114836SGerry Liu /* Check whether it interleaves with other memory nodes. */
1478a3114836SGerry Liu for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) {
1479a3114836SGerry Liu ASSERT(mnoderanges[n].mnr_exists);
1480a3114836SGerry Liu if (mnoderanges[n].mnr_mnode == mnode)
1481a3114836SGerry Liu continue;
1482a3114836SGerry Liu ASSERT(start > mnoderanges[n].mnr_pfnhi ||
1483a3114836SGerry Liu end < mnoderanges[n].mnr_pfnlo);
1484a3114836SGerry Liu }
1485a3114836SGerry Liu #endif /* DEBUG */
1486a3114836SGerry Liu
1487a3114836SGerry Liu mri = nranges - 1;
1488a3114836SGerry Liu while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
1489a3114836SGerry Liu mri--;
1490a3114836SGerry Liu while (mri >= 0 && mem_node_config[mnode].physmax >= MEMRANGELO(mri)) {
1491a3114836SGerry Liu /* Check whether mtype already exists. */
1492a3114836SGerry Liu for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) {
1493a3114836SGerry Liu if (mnoderanges[n].mnr_mnode == mnode &&
1494a3114836SGerry Liu mnoderanges[n].mnr_memrange == mri) {
1495a3114836SGerry Liu mnoderanges[n].mnr_pfnlo = MAX(MEMRANGELO(mri),
1496a3114836SGerry Liu start);
1497a3114836SGerry Liu mnoderanges[n].mnr_pfnhi = MIN(MEMRANGEHI(mri),
1498a3114836SGerry Liu end);
1499a3114836SGerry Liu break;
1500a3114836SGerry Liu }
1501a3114836SGerry Liu }
1502a3114836SGerry Liu
1503a3114836SGerry Liu /* Add a new entry if it doesn't exist yet. */
1504a3114836SGerry Liu if (n == -1) {
1505a3114836SGerry Liu /* Try to find an unused entry in mnoderanges array. */
1506a3114836SGerry Liu for (n = 0; n < mnoderangecnt; n++) {
1507a3114836SGerry Liu if (mnoderanges[n].mnr_exists == 0)
1508a3114836SGerry Liu break;
1509a3114836SGerry Liu }
1510a3114836SGerry Liu ASSERT(n < mnoderangecnt);
1511a3114836SGerry Liu mnoderanges[n].mnr_pfnlo = MAX(MEMRANGELO(mri), start);
1512a3114836SGerry Liu mnoderanges[n].mnr_pfnhi = MIN(MEMRANGEHI(mri), end);
1513a3114836SGerry Liu mnoderanges[n].mnr_mnode = mnode;
1514a3114836SGerry Liu mnoderanges[n].mnr_memrange = mri;
1515a3114836SGerry Liu mnoderanges[n].mnr_exists = 1;
1516a3114836SGerry Liu /* Page 0 should always be present. */
1517a3114836SGerry Liu for (prev = &mtypetop;
1518a3114836SGerry Liu mnoderanges[*prev].mnr_pfnlo > start;
1519a3114836SGerry Liu prev = &mnoderanges[*prev].mnr_next) {
1520a3114836SGerry Liu ASSERT(mnoderanges[*prev].mnr_next >= 0);
1521a3114836SGerry Liu ASSERT(mnoderanges[*prev].mnr_pfnlo > end);
1522a3114836SGerry Liu }
1523a3114836SGerry Liu mnoderanges[n].mnr_next = *prev;
1524a3114836SGerry Liu membar_sync();
1525a3114836SGerry Liu *prev = n;
1526a3114836SGerry Liu }
1527a3114836SGerry Liu
1528a3114836SGerry Liu if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
1529a3114836SGerry Liu mri--;
1530a3114836SGerry Liu else
1531a3114836SGerry Liu break;
1532a3114836SGerry Liu }
1533a3114836SGerry Liu
1534a3114836SGerry Liu mutex_exit(&mnoderange_lock);
1535a3114836SGerry Liu }
1536a3114836SGerry Liu
1537a3114836SGerry Liu /*
1538a3114836SGerry Liu * Update mnoderanges for memory hot-removal DR operations.
1539a3114836SGerry Liu */
1540a3114836SGerry Liu static void
mnode_range_del(int mnode)1541a3114836SGerry Liu mnode_range_del(int mnode)
1542a3114836SGerry Liu {
1543a3114836SGerry Liu _NOTE(ARGUNUSED(mnode));
1544a3114836SGerry Liu ASSERT(0 <= mnode && mnode < max_mem_nodes);
1545a3114836SGerry Liu /* TODO: support deletion operation. */
1546a3114836SGerry Liu ASSERT(0);
1547a3114836SGerry Liu }
1548a3114836SGerry Liu
1549a3114836SGerry Liu void
plat_slice_add(pfn_t start,pfn_t end)1550a3114836SGerry Liu plat_slice_add(pfn_t start, pfn_t end)
1551a3114836SGerry Liu {
1552a3114836SGerry Liu mem_node_add_slice(start, end);
1553a3114836SGerry Liu if (plat_dr_enabled()) {
1554a3114836SGerry Liu mnode_range_add(PFN_2_MEM_NODE(start));
1555a3114836SGerry Liu }
1556a3114836SGerry Liu }
1557a3114836SGerry Liu
1558a3114836SGerry Liu void
plat_slice_del(pfn_t start,pfn_t end)1559a3114836SGerry Liu plat_slice_del(pfn_t start, pfn_t end)
1560a3114836SGerry Liu {
1561a3114836SGerry Liu ASSERT(PFN_2_MEM_NODE(start) == PFN_2_MEM_NODE(end));
1562a3114836SGerry Liu ASSERT(plat_dr_enabled());
1563a3114836SGerry Liu mnode_range_del(PFN_2_MEM_NODE(start));
1564a3114836SGerry Liu mem_node_del_slice(start, end);
1565a3114836SGerry Liu }
1566a3114836SGerry Liu #endif /* __xpv */
1567a3114836SGerry Liu
1568843e1988Sjohnlev /*ARGSUSED*/
1569843e1988Sjohnlev int
mtype_init(vnode_t * vp,caddr_t vaddr,uint_t * flags,size_t pgsz)1570843e1988Sjohnlev mtype_init(vnode_t *vp, caddr_t vaddr, uint_t *flags, size_t pgsz)
1571843e1988Sjohnlev {
1572a3114836SGerry Liu int mtype = mtypetop;
1573843e1988Sjohnlev
1574843e1988Sjohnlev #if !defined(__xpv)
1575843e1988Sjohnlev if (RESTRICT4G_ALLOC) {
1576843e1988Sjohnlev VM_STAT_ADD(vmm_vmstats.restrict4gcnt);
1577843e1988Sjohnlev /* here only for > 4g systems */
1578843e1988Sjohnlev *flags |= PGI_MT_RANGE4G;
1579843e1988Sjohnlev } else if (RESTRICT16M_ALLOC(freemem, btop(pgsz), *flags)) {
1580843e1988Sjohnlev *flags |= PGI_MT_RANGE16M;
1581843e1988Sjohnlev } else {
1582843e1988Sjohnlev VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1583843e1988Sjohnlev VM_STAT_COND_ADD((*flags & PG_PANIC), vmm_vmstats.pgpanicalloc);
1584843e1988Sjohnlev *flags |= PGI_MT_RANGE0;
1585843e1988Sjohnlev }
1586843e1988Sjohnlev #endif /* !__xpv */
1587843e1988Sjohnlev return (mtype);
1588843e1988Sjohnlev }
1589843e1988Sjohnlev
1590843e1988Sjohnlev
1591843e1988Sjohnlev /* mtype init for page_get_replacement_page */
1592843e1988Sjohnlev /*ARGSUSED*/
1593843e1988Sjohnlev int
mtype_pgr_init(int * flags,page_t * pp,pgcnt_t pgcnt)1594d3b5f563SJohn Levon mtype_pgr_init(int *flags, page_t *pp, pgcnt_t pgcnt)
1595843e1988Sjohnlev {
1596a3114836SGerry Liu int mtype = mtypetop;
1597a3114836SGerry Liu #if !defined(__xpv)
1598843e1988Sjohnlev if (RESTRICT16M_ALLOC(freemem, pgcnt, *flags)) {
1599843e1988Sjohnlev *flags |= PGI_MT_RANGE16M;
1600843e1988Sjohnlev } else {
1601843e1988Sjohnlev VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1602843e1988Sjohnlev *flags |= PGI_MT_RANGE0;
1603843e1988Sjohnlev }
1604843e1988Sjohnlev #endif
1605843e1988Sjohnlev return (mtype);
1606843e1988Sjohnlev }
1607843e1988Sjohnlev
16087c478bd9Sstevel@tonic-gate /*
16097c478bd9Sstevel@tonic-gate * Determine if the mnode range specified in mtype contains memory belonging
16107c478bd9Sstevel@tonic-gate * to memory node mnode. If flags & PGI_MT_RANGE is set then mtype contains
1611a3114836SGerry Liu * the range from high pfn to 0, 16m or 4g.
16127c478bd9Sstevel@tonic-gate *
16137c478bd9Sstevel@tonic-gate * Return first mnode range type index found otherwise return -1 if none found.
16147c478bd9Sstevel@tonic-gate */
16157c478bd9Sstevel@tonic-gate int
mtype_func(int mnode,int mtype,uint_t flags)16167c478bd9Sstevel@tonic-gate mtype_func(int mnode, int mtype, uint_t flags)
16177c478bd9Sstevel@tonic-gate {
16187c478bd9Sstevel@tonic-gate if (flags & PGI_MT_RANGE) {
1619a3114836SGerry Liu int mnr_lim = MRI_0;
16207c478bd9Sstevel@tonic-gate
1621a3114836SGerry Liu if (flags & PGI_MT_NEXT) {
1622a3114836SGerry Liu mtype = mnoderanges[mtype].mnr_next;
1623a3114836SGerry Liu }
1624843e1988Sjohnlev if (flags & PGI_MT_RANGE4G)
1625a3114836SGerry Liu mnr_lim = MRI_4G; /* exclude 0-4g range */
162607ad560dSkchow else if (flags & PGI_MT_RANGE16M)
1627a3114836SGerry Liu mnr_lim = MRI_16M; /* exclude 0-16m range */
1628a3114836SGerry Liu while (mtype != -1 &&
1629a3114836SGerry Liu mnoderanges[mtype].mnr_memrange <= mnr_lim) {
16307c478bd9Sstevel@tonic-gate if (mnoderanges[mtype].mnr_mnode == mnode)
16317c478bd9Sstevel@tonic-gate return (mtype);
1632a3114836SGerry Liu mtype = mnoderanges[mtype].mnr_next;
16337c478bd9Sstevel@tonic-gate }
1634843e1988Sjohnlev } else if (mnoderanges[mtype].mnr_mnode == mnode) {
1635843e1988Sjohnlev return (mtype);
16367c478bd9Sstevel@tonic-gate }
16377c478bd9Sstevel@tonic-gate return (-1);
16387c478bd9Sstevel@tonic-gate }
16397c478bd9Sstevel@tonic-gate
1640e21bae1bSkchow /*
1641e21bae1bSkchow * Update the page list max counts with the pfn range specified by the
1642a3114836SGerry Liu * input parameters.
1643e21bae1bSkchow */
1644e21bae1bSkchow void
mtype_modify_max(pfn_t startpfn,long cnt)1645e21bae1bSkchow mtype_modify_max(pfn_t startpfn, long cnt)
1646e21bae1bSkchow {
1647a3114836SGerry Liu int mtype;
1648a3114836SGerry Liu pgcnt_t inc;
1649a3114836SGerry Liu spgcnt_t scnt = (spgcnt_t)(cnt);
1650a3114836SGerry Liu pgcnt_t acnt = ABS(scnt);
1651a3114836SGerry Liu pfn_t endpfn = startpfn + acnt;
1652a3114836SGerry Liu pfn_t pfn, lo;
1653e21bae1bSkchow
1654843e1988Sjohnlev if (!physmax4g)
1655843e1988Sjohnlev return;
1656843e1988Sjohnlev
1657a3114836SGerry Liu mtype = mtypetop;
1658a3114836SGerry Liu for (pfn = endpfn; pfn > startpfn; ) {
1659a3114836SGerry Liu ASSERT(mtype != -1);
1660a3114836SGerry Liu lo = mnoderanges[mtype].mnr_pfnlo;
1661a3114836SGerry Liu if (pfn > lo) {
1662a3114836SGerry Liu if (startpfn >= lo) {
1663a3114836SGerry Liu inc = pfn - startpfn;
1664e21bae1bSkchow } else {
1665a3114836SGerry Liu inc = pfn - lo;
1666a3114836SGerry Liu }
1667a3114836SGerry Liu if (mnoderanges[mtype].mnr_memrange != MRI_4G) {
1668a3114836SGerry Liu if (scnt > 0)
1669a3114836SGerry Liu maxmem4g += inc;
1670a3114836SGerry Liu else
1671a3114836SGerry Liu maxmem4g -= inc;
1672e21bae1bSkchow }
1673a3114836SGerry Liu pfn -= inc;
1674e21bae1bSkchow }
1675a3114836SGerry Liu mtype = mnoderanges[mtype].mnr_next;
1676e21bae1bSkchow }
1677e21bae1bSkchow }
1678e21bae1bSkchow
1679843e1988Sjohnlev int
mtype_2_mrange(int mtype)1680843e1988Sjohnlev mtype_2_mrange(int mtype)
1681843e1988Sjohnlev {
1682843e1988Sjohnlev return (mnoderanges[mtype].mnr_memrange);
1683843e1988Sjohnlev }
1684843e1988Sjohnlev
1685843e1988Sjohnlev void
mnodetype_2_pfn(int mnode,int mtype,pfn_t * pfnlo,pfn_t * pfnhi)1686843e1988Sjohnlev mnodetype_2_pfn(int mnode, int mtype, pfn_t *pfnlo, pfn_t *pfnhi)
1687843e1988Sjohnlev {
1688a3114836SGerry Liu _NOTE(ARGUNUSED(mnode));
1689843e1988Sjohnlev ASSERT(mnoderanges[mtype].mnr_mnode == mnode);
1690843e1988Sjohnlev *pfnlo = mnoderanges[mtype].mnr_pfnlo;
1691843e1988Sjohnlev *pfnhi = mnoderanges[mtype].mnr_pfnhi;
1692843e1988Sjohnlev }
1693843e1988Sjohnlev
1694843e1988Sjohnlev size_t
plcnt_sz(size_t ctrs_sz)1695843e1988Sjohnlev plcnt_sz(size_t ctrs_sz)
1696843e1988Sjohnlev {
1697843e1988Sjohnlev #ifdef DEBUG
1698843e1988Sjohnlev int szc, colors;
1699843e1988Sjohnlev
1700843e1988Sjohnlev ctrs_sz += mnoderangecnt * sizeof (struct mnr_mts) * mmu_page_sizes;
1701843e1988Sjohnlev for (szc = 0; szc < mmu_page_sizes; szc++) {
1702843e1988Sjohnlev colors = page_get_pagecolors(szc);
1703843e1988Sjohnlev ctrs_sz += mnoderangecnt * sizeof (pgcnt_t) * colors;
1704843e1988Sjohnlev }
1705843e1988Sjohnlev #endif
1706843e1988Sjohnlev return (ctrs_sz);
1707843e1988Sjohnlev }
1708843e1988Sjohnlev
1709843e1988Sjohnlev caddr_t
plcnt_init(caddr_t addr)1710843e1988Sjohnlev plcnt_init(caddr_t addr)
1711843e1988Sjohnlev {
1712843e1988Sjohnlev #ifdef DEBUG
1713843e1988Sjohnlev int mt, szc, colors;
1714843e1988Sjohnlev
1715843e1988Sjohnlev for (mt = 0; mt < mnoderangecnt; mt++) {
1716843e1988Sjohnlev mnoderanges[mt].mnr_mts = (struct mnr_mts *)addr;
1717843e1988Sjohnlev addr += (sizeof (struct mnr_mts) * mmu_page_sizes);
1718843e1988Sjohnlev for (szc = 0; szc < mmu_page_sizes; szc++) {
1719843e1988Sjohnlev colors = page_get_pagecolors(szc);
1720843e1988Sjohnlev mnoderanges[mt].mnr_mts[szc].mnr_mts_colors = colors;
1721843e1988Sjohnlev mnoderanges[mt].mnr_mts[szc].mnr_mtsc_pgcnt =
1722843e1988Sjohnlev (pgcnt_t *)addr;
1723843e1988Sjohnlev addr += (sizeof (pgcnt_t) * colors);
1724843e1988Sjohnlev }
1725843e1988Sjohnlev }
1726843e1988Sjohnlev #endif
1727843e1988Sjohnlev return (addr);
1728843e1988Sjohnlev }
1729843e1988Sjohnlev
1730843e1988Sjohnlev void
plcnt_inc_dec(page_t * pp,int mtype,int szc,long cnt,int flags)1731843e1988Sjohnlev plcnt_inc_dec(page_t *pp, int mtype, int szc, long cnt, int flags)
1732843e1988Sjohnlev {
1733a3114836SGerry Liu _NOTE(ARGUNUSED(pp));
1734843e1988Sjohnlev #ifdef DEBUG
1735843e1988Sjohnlev int bin = PP_2_BIN(pp);
1736843e1988Sjohnlev
1737843e1988Sjohnlev atomic_add_long(&mnoderanges[mtype].mnr_mts[szc].mnr_mts_pgcnt, cnt);
1738843e1988Sjohnlev atomic_add_long(&mnoderanges[mtype].mnr_mts[szc].mnr_mtsc_pgcnt[bin],
1739843e1988Sjohnlev cnt);
1740843e1988Sjohnlev #endif
1741843e1988Sjohnlev ASSERT(mtype == PP_2_MTYPE(pp));
1742a3114836SGerry Liu if (physmax4g && mnoderanges[mtype].mnr_memrange != MRI_4G)
1743843e1988Sjohnlev atomic_add_long(&freemem4g, cnt);
1744843e1988Sjohnlev if (flags & PG_CACHE_LIST)
1745843e1988Sjohnlev atomic_add_long(&mnoderanges[mtype].mnr_mt_clpgcnt, cnt);
1746843e1988Sjohnlev else
174778b03d3aSkchow atomic_add_long(&mnoderanges[mtype].mnr_mt_flpgcnt[szc], cnt);
174878b03d3aSkchow atomic_add_long(&mnoderanges[mtype].mnr_mt_totcnt, cnt);
1749843e1988Sjohnlev }
1750843e1988Sjohnlev
1751affbd3ccSkchow /*
1752affbd3ccSkchow * Returns the free page count for mnode
1753affbd3ccSkchow */
1754affbd3ccSkchow int
mnode_pgcnt(int mnode)1755affbd3ccSkchow mnode_pgcnt(int mnode)
1756affbd3ccSkchow {
1757a3114836SGerry Liu int mtype = mtypetop;
1758affbd3ccSkchow int flags = PGI_MT_RANGE0;
1759affbd3ccSkchow pgcnt_t pgcnt = 0;
1760affbd3ccSkchow
1761affbd3ccSkchow mtype = mtype_func(mnode, mtype, flags);
1762affbd3ccSkchow
1763affbd3ccSkchow while (mtype != -1) {
176407ad560dSkchow pgcnt += MTYPE_FREEMEM(mtype);
1765affbd3ccSkchow mtype = mtype_func(mnode, mtype, flags | PGI_MT_NEXT);
1766affbd3ccSkchow }
1767affbd3ccSkchow return (pgcnt);
1768affbd3ccSkchow }
1769affbd3ccSkchow
17707c478bd9Sstevel@tonic-gate /*
17717c478bd9Sstevel@tonic-gate * Initialize page coloring variables based on the l2 cache parameters.
17727c478bd9Sstevel@tonic-gate * Calculate and return memory needed for page coloring data structures.
17737c478bd9Sstevel@tonic-gate */
17747c478bd9Sstevel@tonic-gate size_t
page_coloring_init(uint_t l2_sz,int l2_linesz,int l2_assoc)17757c478bd9Sstevel@tonic-gate page_coloring_init(uint_t l2_sz, int l2_linesz, int l2_assoc)
17767c478bd9Sstevel@tonic-gate {
1777a3114836SGerry Liu _NOTE(ARGUNUSED(l2_linesz));
17787c478bd9Sstevel@tonic-gate size_t colorsz = 0;
17797c478bd9Sstevel@tonic-gate int i;
17807c478bd9Sstevel@tonic-gate int colors;
17817c478bd9Sstevel@tonic-gate
1782843e1988Sjohnlev #if defined(__xpv)
1783843e1988Sjohnlev /*
1784843e1988Sjohnlev * Hypervisor domains currently don't have any concept of NUMA.
1785843e1988Sjohnlev * Hence we'll act like there is only 1 memrange.
1786843e1988Sjohnlev */
1787843e1988Sjohnlev i = memrange_num(1);
1788843e1988Sjohnlev #else /* !__xpv */
17897c478bd9Sstevel@tonic-gate /*
17907c478bd9Sstevel@tonic-gate * Reduce the memory ranges lists if we don't have large amounts
17917c478bd9Sstevel@tonic-gate * of memory. This avoids searching known empty free lists.
1792a3114836SGerry Liu * To support memory DR operations, we need to keep memory ranges
1793a3114836SGerry Liu * for possible memory hot-add operations.
17947c478bd9Sstevel@tonic-gate */
1795a3114836SGerry Liu if (plat_dr_physmax > physmax)
1796a3114836SGerry Liu i = memrange_num(plat_dr_physmax);
1797a3114836SGerry Liu else
1798a3114836SGerry Liu i = memrange_num(physmax);
17997c478bd9Sstevel@tonic-gate /* physmax greater than 4g */
1800a3114836SGerry Liu if (i == MRI_4G)
18017c478bd9Sstevel@tonic-gate physmax4g = 1;
1802843e1988Sjohnlev #endif /* !__xpv */
1803843e1988Sjohnlev memranges += i;
1804843e1988Sjohnlev nranges -= i;
18057c478bd9Sstevel@tonic-gate
180602bc52beSkchow ASSERT(mmu_page_sizes <= MMU_PAGE_SIZES);
180702bc52beSkchow
18087c478bd9Sstevel@tonic-gate ASSERT(ISP2(l2_linesz));
18097c478bd9Sstevel@tonic-gate ASSERT(l2_sz > MMU_PAGESIZE);
18107c478bd9Sstevel@tonic-gate
18117c478bd9Sstevel@tonic-gate /* l2_assoc is 0 for fully associative l2 cache */
18127c478bd9Sstevel@tonic-gate if (l2_assoc)
18137c478bd9Sstevel@tonic-gate l2_colors = MAX(1, l2_sz / (l2_assoc * MMU_PAGESIZE));
18147c478bd9Sstevel@tonic-gate else
18157c478bd9Sstevel@tonic-gate l2_colors = 1;
18167c478bd9Sstevel@tonic-gate
1817e4ab3d6dSvd ASSERT(ISP2(l2_colors));
1818e4ab3d6dSvd
18197c478bd9Sstevel@tonic-gate /* for scalability, configure at least PAGE_COLORS_MIN color bins */
18207c478bd9Sstevel@tonic-gate page_colors = MAX(l2_colors, PAGE_COLORS_MIN);
18217c478bd9Sstevel@tonic-gate
18227c478bd9Sstevel@tonic-gate /*
18237c478bd9Sstevel@tonic-gate * cpu_page_colors is non-zero when a page color may be spread across
18247c478bd9Sstevel@tonic-gate * multiple bins.
18257c478bd9Sstevel@tonic-gate */
18267c478bd9Sstevel@tonic-gate if (l2_colors < page_colors)
18277c478bd9Sstevel@tonic-gate cpu_page_colors = l2_colors;
18287c478bd9Sstevel@tonic-gate
18297c478bd9Sstevel@tonic-gate ASSERT(ISP2(page_colors));
18307c478bd9Sstevel@tonic-gate
18317c478bd9Sstevel@tonic-gate page_colors_mask = page_colors - 1;
18327c478bd9Sstevel@tonic-gate
18337c478bd9Sstevel@tonic-gate ASSERT(ISP2(CPUSETSIZE()));
18347c478bd9Sstevel@tonic-gate page_coloring_shift = lowbit(CPUSETSIZE());
18357c478bd9Sstevel@tonic-gate
18365d07b933Sdp /* initialize number of colors per page size */
18375d07b933Sdp for (i = 0; i <= mmu.max_page_level; i++) {
18385d07b933Sdp hw_page_array[i].hp_size = LEVEL_SIZE(i);
18395d07b933Sdp hw_page_array[i].hp_shift = LEVEL_SHIFT(i);
18405d07b933Sdp hw_page_array[i].hp_pgcnt = LEVEL_SIZE(i) >> LEVEL_SHIFT(0);
18415d07b933Sdp hw_page_array[i].hp_colors = (page_colors_mask >>
18425d07b933Sdp (hw_page_array[i].hp_shift - hw_page_array[0].hp_shift))
18435d07b933Sdp + 1;
1844932dc8e5Sdp colorequivszc[i] = 0;
18455d07b933Sdp }
18465d07b933Sdp
18475d07b933Sdp /*
18485d07b933Sdp * The value of cpu_page_colors determines if additional color bins
18495d07b933Sdp * need to be checked for a particular color in the page_get routines.
18505d07b933Sdp */
18515d07b933Sdp if (cpu_page_colors != 0) {
18525d07b933Sdp
18535d07b933Sdp int a = lowbit(page_colors) - lowbit(cpu_page_colors);
18545d07b933Sdp ASSERT(a > 0);
18555d07b933Sdp ASSERT(a < 16);
18565d07b933Sdp
18575d07b933Sdp for (i = 0; i <= mmu.max_page_level; i++) {
18585d07b933Sdp if ((colors = hw_page_array[i].hp_colors) <= 1) {
18595d07b933Sdp colorequivszc[i] = 0;
18605d07b933Sdp continue;
18615d07b933Sdp }
18625d07b933Sdp while ((colors >> a) == 0)
18635d07b933Sdp a--;
18645d07b933Sdp ASSERT(a >= 0);
18655d07b933Sdp
18665d07b933Sdp /* higher 4 bits encodes color equiv mask */
18675d07b933Sdp colorequivszc[i] = (a << 4);
18685d07b933Sdp }
18695d07b933Sdp }
18705d07b933Sdp
1871843e1988Sjohnlev /* factor in colorequiv to check additional 'equivalent' bins. */
1872843e1988Sjohnlev if (colorequiv > 1) {
1873843e1988Sjohnlev
1874843e1988Sjohnlev int a = lowbit(colorequiv) - 1;
1875843e1988Sjohnlev if (a > 15)
1876843e1988Sjohnlev a = 15;
1877843e1988Sjohnlev
1878843e1988Sjohnlev for (i = 0; i <= mmu.max_page_level; i++) {
1879843e1988Sjohnlev if ((colors = hw_page_array[i].hp_colors) <= 1) {
1880843e1988Sjohnlev continue;
1881843e1988Sjohnlev }
1882843e1988Sjohnlev while ((colors >> a) == 0)
1883843e1988Sjohnlev a--;
1884843e1988Sjohnlev if ((a << 4) > colorequivszc[i]) {
1885843e1988Sjohnlev colorequivszc[i] = (a << 4);
1886843e1988Sjohnlev }
1887843e1988Sjohnlev }
1888843e1988Sjohnlev }
1889843e1988Sjohnlev
18907c478bd9Sstevel@tonic-gate /* size for mnoderanges */
18915d07b933Sdp for (mnoderangecnt = 0, i = 0; i < max_mem_nodes; i++)
18925d07b933Sdp mnoderangecnt += mnode_range_cnt(i);
1893a3114836SGerry Liu if (plat_dr_support_memory()) {
1894a3114836SGerry Liu /*
1895a3114836SGerry Liu * Reserve enough space for memory DR operations.
1896a3114836SGerry Liu * Two extra mnoderanges for possbile fragmentations,
1897a3114836SGerry Liu * one for the 2G boundary and the other for the 4G boundary.
1898a3114836SGerry Liu * We don't expect a memory board crossing the 16M boundary
1899a3114836SGerry Liu * for memory hot-add operations on x86 platforms.
1900a3114836SGerry Liu */
1901a3114836SGerry Liu mnoderangecnt += 2 + max_mem_nodes - lgrp_plat_node_cnt;
1902a3114836SGerry Liu }
19037c478bd9Sstevel@tonic-gate colorsz = mnoderangecnt * sizeof (mnoderange_t);
19047c478bd9Sstevel@tonic-gate
19057c478bd9Sstevel@tonic-gate /* size for fpc_mutex and cpc_mutex */
19067c478bd9Sstevel@tonic-gate colorsz += (2 * max_mem_nodes * sizeof (kmutex_t) * NPC_MUTEX);
19077c478bd9Sstevel@tonic-gate
19087c478bd9Sstevel@tonic-gate /* size of page_freelists */
19097c478bd9Sstevel@tonic-gate colorsz += mnoderangecnt * sizeof (page_t ***);
19107c478bd9Sstevel@tonic-gate colorsz += mnoderangecnt * mmu_page_sizes * sizeof (page_t **);
19117c478bd9Sstevel@tonic-gate
19127c478bd9Sstevel@tonic-gate for (i = 0; i < mmu_page_sizes; i++) {
19137c478bd9Sstevel@tonic-gate colors = page_get_pagecolors(i);
19147c478bd9Sstevel@tonic-gate colorsz += mnoderangecnt * colors * sizeof (page_t *);
19157c478bd9Sstevel@tonic-gate }
19167c478bd9Sstevel@tonic-gate
19177c478bd9Sstevel@tonic-gate /* size of page_cachelists */
19187c478bd9Sstevel@tonic-gate colorsz += mnoderangecnt * sizeof (page_t **);
19197c478bd9Sstevel@tonic-gate colorsz += mnoderangecnt * page_colors * sizeof (page_t *);
19207c478bd9Sstevel@tonic-gate
19217c478bd9Sstevel@tonic-gate return (colorsz);
19227c478bd9Sstevel@tonic-gate }
19237c478bd9Sstevel@tonic-gate
19247c478bd9Sstevel@tonic-gate /*
19257c478bd9Sstevel@tonic-gate * Called once at startup to configure page_coloring data structures and
19267c478bd9Sstevel@tonic-gate * does the 1st page_free()/page_freelist_add().
19277c478bd9Sstevel@tonic-gate */
19287c478bd9Sstevel@tonic-gate void
page_coloring_setup(caddr_t pcmemaddr)19297c478bd9Sstevel@tonic-gate page_coloring_setup(caddr_t pcmemaddr)
19307c478bd9Sstevel@tonic-gate {
19317c478bd9Sstevel@tonic-gate int i;
19327c478bd9Sstevel@tonic-gate int j;
19337c478bd9Sstevel@tonic-gate int k;
19347c478bd9Sstevel@tonic-gate caddr_t addr;
19357c478bd9Sstevel@tonic-gate int colors;
19367c478bd9Sstevel@tonic-gate
19377c478bd9Sstevel@tonic-gate /*
19387c478bd9Sstevel@tonic-gate * do page coloring setup
19397c478bd9Sstevel@tonic-gate */
19407c478bd9Sstevel@tonic-gate addr = pcmemaddr;
19417c478bd9Sstevel@tonic-gate
19427c478bd9Sstevel@tonic-gate mnoderanges = (mnoderange_t *)addr;
19437c478bd9Sstevel@tonic-gate addr += (mnoderangecnt * sizeof (mnoderange_t));
19447c478bd9Sstevel@tonic-gate
19457c478bd9Sstevel@tonic-gate mnode_range_setup(mnoderanges);
19467c478bd9Sstevel@tonic-gate
19477c478bd9Sstevel@tonic-gate for (k = 0; k < NPC_MUTEX; k++) {
19487c478bd9Sstevel@tonic-gate fpc_mutex[k] = (kmutex_t *)addr;
19497c478bd9Sstevel@tonic-gate addr += (max_mem_nodes * sizeof (kmutex_t));
19507c478bd9Sstevel@tonic-gate }
19517c478bd9Sstevel@tonic-gate for (k = 0; k < NPC_MUTEX; k++) {
19527c478bd9Sstevel@tonic-gate cpc_mutex[k] = (kmutex_t *)addr;
19537c478bd9Sstevel@tonic-gate addr += (max_mem_nodes * sizeof (kmutex_t));
19547c478bd9Sstevel@tonic-gate }
1955d94ffb28Sjmcp page_freelists = (page_t ****)addr;
19567c478bd9Sstevel@tonic-gate addr += (mnoderangecnt * sizeof (page_t ***));
19577c478bd9Sstevel@tonic-gate
19587c478bd9Sstevel@tonic-gate page_cachelists = (page_t ***)addr;
19597c478bd9Sstevel@tonic-gate addr += (mnoderangecnt * sizeof (page_t **));
19607c478bd9Sstevel@tonic-gate
19617c478bd9Sstevel@tonic-gate for (i = 0; i < mnoderangecnt; i++) {
1962d94ffb28Sjmcp page_freelists[i] = (page_t ***)addr;
19637c478bd9Sstevel@tonic-gate addr += (mmu_page_sizes * sizeof (page_t **));
19647c478bd9Sstevel@tonic-gate
19657c478bd9Sstevel@tonic-gate for (j = 0; j < mmu_page_sizes; j++) {
19667c478bd9Sstevel@tonic-gate colors = page_get_pagecolors(j);
1967d94ffb28Sjmcp page_freelists[i][j] = (page_t **)addr;
19687c478bd9Sstevel@tonic-gate addr += (colors * sizeof (page_t *));
19697c478bd9Sstevel@tonic-gate }
19707c478bd9Sstevel@tonic-gate page_cachelists[i] = (page_t **)addr;
19717c478bd9Sstevel@tonic-gate addr += (page_colors * sizeof (page_t *));
19727c478bd9Sstevel@tonic-gate }
19737c478bd9Sstevel@tonic-gate }
19747c478bd9Sstevel@tonic-gate
1975843e1988Sjohnlev #if defined(__xpv)
1976843e1988Sjohnlev /*
1977843e1988Sjohnlev * Give back 10% of the io_pool pages to the free list.
1978843e1988Sjohnlev * Don't shrink the pool below some absolute minimum.
1979843e1988Sjohnlev */
1980843e1988Sjohnlev static void
page_io_pool_shrink()1981843e1988Sjohnlev page_io_pool_shrink()
1982843e1988Sjohnlev {
1983843e1988Sjohnlev int retcnt;
1984843e1988Sjohnlev page_t *pp, *pp_first, *pp_last, **curpool;
1985843e1988Sjohnlev mfn_t mfn;
1986843e1988Sjohnlev int bothpools = 0;
1987843e1988Sjohnlev
1988843e1988Sjohnlev mutex_enter(&io_pool_lock);
1989843e1988Sjohnlev io_pool_shrink_attempts++; /* should be a kstat? */
1990843e1988Sjohnlev retcnt = io_pool_cnt / 10;
1991843e1988Sjohnlev if (io_pool_cnt - retcnt < io_pool_cnt_min)
1992843e1988Sjohnlev retcnt = io_pool_cnt - io_pool_cnt_min;
1993843e1988Sjohnlev if (retcnt <= 0)
1994843e1988Sjohnlev goto done;
1995843e1988Sjohnlev io_pool_shrinks++; /* should be a kstat? */
1996843e1988Sjohnlev curpool = &io_pool_4g;
1997843e1988Sjohnlev domore:
1998843e1988Sjohnlev /*
1999843e1988Sjohnlev * Loop through taking pages from the end of the list
2000843e1988Sjohnlev * (highest mfns) till amount to return reached.
2001843e1988Sjohnlev */
2002843e1988Sjohnlev for (pp = *curpool; pp && retcnt > 0; ) {
2003843e1988Sjohnlev pp_first = pp_last = pp->p_prev;
2004843e1988Sjohnlev if (pp_first == *curpool)
2005843e1988Sjohnlev break;
2006843e1988Sjohnlev retcnt--;
2007843e1988Sjohnlev io_pool_cnt--;
2008843e1988Sjohnlev page_io_pool_sub(curpool, pp_first, pp_last);
2009843e1988Sjohnlev if ((mfn = pfn_to_mfn(pp->p_pagenum)) < start_mfn)
2010843e1988Sjohnlev start_mfn = mfn;
2011843e1988Sjohnlev page_free(pp_first, 1);
2012843e1988Sjohnlev pp = *curpool;
2013843e1988Sjohnlev }
2014843e1988Sjohnlev if (retcnt != 0 && !bothpools) {
2015843e1988Sjohnlev /*
2016843e1988Sjohnlev * If not enough found in less constrained pool try the
2017843e1988Sjohnlev * more constrained one.
2018843e1988Sjohnlev */
2019843e1988Sjohnlev curpool = &io_pool_16m;
2020843e1988Sjohnlev bothpools = 1;
2021843e1988Sjohnlev goto domore;
2022843e1988Sjohnlev }
2023843e1988Sjohnlev done:
2024843e1988Sjohnlev mutex_exit(&io_pool_lock);
2025843e1988Sjohnlev }
2026843e1988Sjohnlev
2027843e1988Sjohnlev #endif /* __xpv */
2028843e1988Sjohnlev
2029843e1988Sjohnlev uint_t
page_create_update_flags_x86(uint_t flags)2030843e1988Sjohnlev page_create_update_flags_x86(uint_t flags)
2031843e1988Sjohnlev {
2032843e1988Sjohnlev #if defined(__xpv)
2033843e1988Sjohnlev /*
2034843e1988Sjohnlev * Check this is an urgent allocation and free pages are depleted.
2035843e1988Sjohnlev */
2036843e1988Sjohnlev if (!(flags & PG_WAIT) && freemem < desfree)
2037843e1988Sjohnlev page_io_pool_shrink();
2038843e1988Sjohnlev #else /* !__xpv */
2039843e1988Sjohnlev /*
2040843e1988Sjohnlev * page_create_get_something may call this because 4g memory may be
2041843e1988Sjohnlev * depleted. Set flags to allow for relocation of base page below
2042843e1988Sjohnlev * 4g if necessary.
2043843e1988Sjohnlev */
2044843e1988Sjohnlev if (physmax4g)
2045843e1988Sjohnlev flags |= (PGI_PGCPSZC0 | PGI_PGCPHIPRI);
2046843e1988Sjohnlev #endif /* __xpv */
2047843e1988Sjohnlev return (flags);
2048843e1988Sjohnlev }
2049843e1988Sjohnlev
2050843e1988Sjohnlev /*ARGSUSED*/
2051843e1988Sjohnlev int
bp_color(struct buf * bp)2052843e1988Sjohnlev bp_color(struct buf *bp)
20537c478bd9Sstevel@tonic-gate {
20547c478bd9Sstevel@tonic-gate return (0);
20557c478bd9Sstevel@tonic-gate }
20567c478bd9Sstevel@tonic-gate
2057843e1988Sjohnlev #if defined(__xpv)
2058843e1988Sjohnlev
20597c478bd9Sstevel@tonic-gate /*
2060843e1988Sjohnlev * Take pages out of an io_pool
2061843e1988Sjohnlev */
2062843e1988Sjohnlev static void
page_io_pool_sub(page_t ** poolp,page_t * pp_first,page_t * pp_last)2063843e1988Sjohnlev page_io_pool_sub(page_t **poolp, page_t *pp_first, page_t *pp_last)
2064843e1988Sjohnlev {
2065843e1988Sjohnlev if (*poolp == pp_first) {
2066843e1988Sjohnlev *poolp = pp_last->p_next;
2067843e1988Sjohnlev if (*poolp == pp_first)
2068843e1988Sjohnlev *poolp = NULL;
2069843e1988Sjohnlev }
2070843e1988Sjohnlev pp_first->p_prev->p_next = pp_last->p_next;
2071843e1988Sjohnlev pp_last->p_next->p_prev = pp_first->p_prev;
2072843e1988Sjohnlev pp_first->p_prev = pp_last;
2073843e1988Sjohnlev pp_last->p_next = pp_first;
2074843e1988Sjohnlev }
2075843e1988Sjohnlev
2076843e1988Sjohnlev /*
2077843e1988Sjohnlev * Put a page on the io_pool list. The list is ordered by increasing MFN.
2078843e1988Sjohnlev */
2079843e1988Sjohnlev static void
page_io_pool_add(page_t ** poolp,page_t * pp)2080843e1988Sjohnlev page_io_pool_add(page_t **poolp, page_t *pp)
2081843e1988Sjohnlev {
2082843e1988Sjohnlev page_t *look;
2083843e1988Sjohnlev mfn_t mfn = mfn_list[pp->p_pagenum];
2084843e1988Sjohnlev
2085843e1988Sjohnlev if (*poolp == NULL) {
2086843e1988Sjohnlev *poolp = pp;
2087843e1988Sjohnlev pp->p_next = pp;
2088843e1988Sjohnlev pp->p_prev = pp;
2089843e1988Sjohnlev return;
2090843e1988Sjohnlev }
2091843e1988Sjohnlev
2092843e1988Sjohnlev /*
2093843e1988Sjohnlev * Since we try to take pages from the high end of the pool
2094843e1988Sjohnlev * chances are good that the pages to be put on the list will
2095843e1988Sjohnlev * go at or near the end of the list. so start at the end and
2096843e1988Sjohnlev * work backwards.
2097843e1988Sjohnlev */
2098843e1988Sjohnlev look = (*poolp)->p_prev;
2099843e1988Sjohnlev while (mfn < mfn_list[look->p_pagenum]) {
2100843e1988Sjohnlev look = look->p_prev;
2101843e1988Sjohnlev if (look == (*poolp)->p_prev)
2102843e1988Sjohnlev break; /* backed all the way to front of list */
2103843e1988Sjohnlev }
2104843e1988Sjohnlev
2105843e1988Sjohnlev /* insert after look */
2106843e1988Sjohnlev pp->p_prev = look;
2107843e1988Sjohnlev pp->p_next = look->p_next;
2108843e1988Sjohnlev pp->p_next->p_prev = pp;
2109843e1988Sjohnlev look->p_next = pp;
2110843e1988Sjohnlev if (mfn < mfn_list[(*poolp)->p_pagenum]) {
2111843e1988Sjohnlev /*
2112843e1988Sjohnlev * we inserted a new first list element
2113843e1988Sjohnlev * adjust pool pointer to newly inserted element
2114843e1988Sjohnlev */
2115843e1988Sjohnlev *poolp = pp;
2116843e1988Sjohnlev }
2117843e1988Sjohnlev }
2118843e1988Sjohnlev
2119843e1988Sjohnlev /*
2120843e1988Sjohnlev * Add a page to the io_pool. Setting the force flag will force the page
2121843e1988Sjohnlev * into the io_pool no matter what.
2122843e1988Sjohnlev */
2123843e1988Sjohnlev static void
add_page_to_pool(page_t * pp,int force)2124843e1988Sjohnlev add_page_to_pool(page_t *pp, int force)
2125843e1988Sjohnlev {
2126843e1988Sjohnlev page_t *highest;
2127843e1988Sjohnlev page_t *freep = NULL;
2128843e1988Sjohnlev
2129843e1988Sjohnlev mutex_enter(&io_pool_lock);
2130843e1988Sjohnlev /*
2131843e1988Sjohnlev * Always keep the scarce low memory pages
2132843e1988Sjohnlev */
2133843e1988Sjohnlev if (mfn_list[pp->p_pagenum] < PFN_16MEG) {
2134843e1988Sjohnlev ++io_pool_cnt;
2135843e1988Sjohnlev page_io_pool_add(&io_pool_16m, pp);
2136843e1988Sjohnlev goto done;
2137843e1988Sjohnlev }
2138cf902cd2Ssmaybe if (io_pool_cnt < io_pool_cnt_max || force || io_pool_4g == NULL) {
2139843e1988Sjohnlev ++io_pool_cnt;
2140843e1988Sjohnlev page_io_pool_add(&io_pool_4g, pp);
2141843e1988Sjohnlev } else {
2142843e1988Sjohnlev highest = io_pool_4g->p_prev;
2143843e1988Sjohnlev if (mfn_list[pp->p_pagenum] < mfn_list[highest->p_pagenum]) {
2144843e1988Sjohnlev page_io_pool_sub(&io_pool_4g, highest, highest);
2145843e1988Sjohnlev page_io_pool_add(&io_pool_4g, pp);
2146843e1988Sjohnlev freep = highest;
2147843e1988Sjohnlev } else {
2148843e1988Sjohnlev freep = pp;
2149843e1988Sjohnlev }
2150843e1988Sjohnlev }
2151843e1988Sjohnlev done:
2152843e1988Sjohnlev mutex_exit(&io_pool_lock);
2153843e1988Sjohnlev if (freep)
2154843e1988Sjohnlev page_free(freep, 1);
2155843e1988Sjohnlev }
2156843e1988Sjohnlev
2157843e1988Sjohnlev
2158843e1988Sjohnlev int contig_pfn_cnt; /* no of pfns in the contig pfn list */
2159843e1988Sjohnlev int contig_pfn_max; /* capacity of the contig pfn list */
2160843e1988Sjohnlev int next_alloc_pfn; /* next position in list to start a contig search */
2161843e1988Sjohnlev int contig_pfnlist_updates; /* pfn list update count */
2162843e1988Sjohnlev int contig_pfnlist_builds; /* how many times have we (re)built list */
2163843e1988Sjohnlev int contig_pfnlist_buildfailed; /* how many times has list build failed */
2164843e1988Sjohnlev int create_contig_pending; /* nonzero means taskq creating contig list */
2165843e1988Sjohnlev pfn_t *contig_pfn_list = NULL; /* list of contig pfns in ascending mfn order */
2166843e1988Sjohnlev
2167843e1988Sjohnlev /*
2168843e1988Sjohnlev * Function to use in sorting a list of pfns by their underlying mfns.
2169843e1988Sjohnlev */
2170843e1988Sjohnlev static int
mfn_compare(const void * pfnp1,const void * pfnp2)2171843e1988Sjohnlev mfn_compare(const void *pfnp1, const void *pfnp2)
2172843e1988Sjohnlev {
2173843e1988Sjohnlev mfn_t mfn1 = mfn_list[*(pfn_t *)pfnp1];
2174843e1988Sjohnlev mfn_t mfn2 = mfn_list[*(pfn_t *)pfnp2];
2175843e1988Sjohnlev
2176843e1988Sjohnlev if (mfn1 > mfn2)
2177843e1988Sjohnlev return (1);
2178843e1988Sjohnlev if (mfn1 < mfn2)
2179843e1988Sjohnlev return (-1);
2180843e1988Sjohnlev return (0);
2181843e1988Sjohnlev }
2182843e1988Sjohnlev
2183843e1988Sjohnlev /*
2184843e1988Sjohnlev * Compact the contig_pfn_list by tossing all the non-contiguous
2185843e1988Sjohnlev * elements from the list.
2186843e1988Sjohnlev */
2187843e1988Sjohnlev static void
compact_contig_pfn_list(void)2188843e1988Sjohnlev compact_contig_pfn_list(void)
2189843e1988Sjohnlev {
2190843e1988Sjohnlev pfn_t pfn, lapfn, prev_lapfn;
2191843e1988Sjohnlev mfn_t mfn;
2192843e1988Sjohnlev int i, newcnt = 0;
2193843e1988Sjohnlev
2194843e1988Sjohnlev prev_lapfn = 0;
2195843e1988Sjohnlev for (i = 0; i < contig_pfn_cnt - 1; i++) {
2196843e1988Sjohnlev pfn = contig_pfn_list[i];
2197843e1988Sjohnlev lapfn = contig_pfn_list[i + 1];
2198843e1988Sjohnlev mfn = mfn_list[pfn];
2199843e1988Sjohnlev /*
2200843e1988Sjohnlev * See if next pfn is for a contig mfn
2201843e1988Sjohnlev */
2202843e1988Sjohnlev if (mfn_list[lapfn] != mfn + 1)
2203843e1988Sjohnlev continue;
2204843e1988Sjohnlev /*
2205843e1988Sjohnlev * pfn and lookahead are both put in list
2206843e1988Sjohnlev * unless pfn is the previous lookahead.
2207843e1988Sjohnlev */
2208843e1988Sjohnlev if (pfn != prev_lapfn)
2209843e1988Sjohnlev contig_pfn_list[newcnt++] = pfn;
2210843e1988Sjohnlev contig_pfn_list[newcnt++] = lapfn;
2211843e1988Sjohnlev prev_lapfn = lapfn;
2212843e1988Sjohnlev }
2213843e1988Sjohnlev for (i = newcnt; i < contig_pfn_cnt; i++)
2214843e1988Sjohnlev contig_pfn_list[i] = 0;
2215843e1988Sjohnlev contig_pfn_cnt = newcnt;
2216843e1988Sjohnlev }
2217843e1988Sjohnlev
2218843e1988Sjohnlev /*ARGSUSED*/
2219843e1988Sjohnlev static void
call_create_contiglist(void * arg)2220843e1988Sjohnlev call_create_contiglist(void *arg)
2221843e1988Sjohnlev {
2222843e1988Sjohnlev (void) create_contig_pfnlist(PG_WAIT);
2223843e1988Sjohnlev }
2224843e1988Sjohnlev
2225843e1988Sjohnlev /*
2226843e1988Sjohnlev * Create list of freelist pfns that have underlying
2227843e1988Sjohnlev * contiguous mfns. The list is kept in ascending mfn order.
2228843e1988Sjohnlev * returns 1 if list created else 0.
2229843e1988Sjohnlev */
2230843e1988Sjohnlev static int
create_contig_pfnlist(uint_t flags)2231843e1988Sjohnlev create_contig_pfnlist(uint_t flags)
2232843e1988Sjohnlev {
2233843e1988Sjohnlev pfn_t pfn;
2234843e1988Sjohnlev page_t *pp;
2235b9bc7f78Ssmaybe int ret = 1;
2236843e1988Sjohnlev
2237b9bc7f78Ssmaybe mutex_enter(&contig_list_lock);
2238843e1988Sjohnlev if (contig_pfn_list != NULL)
2239b9bc7f78Ssmaybe goto out;
2240843e1988Sjohnlev contig_pfn_max = freemem + (freemem / 10);
2241843e1988Sjohnlev contig_pfn_list = kmem_zalloc(contig_pfn_max * sizeof (pfn_t),
2242843e1988Sjohnlev (flags & PG_WAIT) ? KM_SLEEP : KM_NOSLEEP);
2243843e1988Sjohnlev if (contig_pfn_list == NULL) {
2244843e1988Sjohnlev /*
2245843e1988Sjohnlev * If we could not create the contig list (because
2246843e1988Sjohnlev * we could not sleep for memory). Dispatch a taskq that can
2247843e1988Sjohnlev * sleep to get the memory.
2248843e1988Sjohnlev */
2249843e1988Sjohnlev if (!create_contig_pending) {
2250843e1988Sjohnlev if (taskq_dispatch(system_taskq, call_create_contiglist,
2251fc8ae2ecSToomas Soome NULL, TQ_NOSLEEP) != TASKQID_INVALID)
2252843e1988Sjohnlev create_contig_pending = 1;
2253843e1988Sjohnlev }
2254843e1988Sjohnlev contig_pfnlist_buildfailed++; /* count list build failures */
2255b9bc7f78Ssmaybe ret = 0;
2256b9bc7f78Ssmaybe goto out;
2257843e1988Sjohnlev }
2258b9bc7f78Ssmaybe create_contig_pending = 0;
2259843e1988Sjohnlev ASSERT(contig_pfn_cnt == 0);
2260843e1988Sjohnlev for (pfn = 0; pfn < mfn_count; pfn++) {
2261843e1988Sjohnlev pp = page_numtopp_nolock(pfn);
2262843e1988Sjohnlev if (pp == NULL || !PP_ISFREE(pp))
2263843e1988Sjohnlev continue;
2264843e1988Sjohnlev contig_pfn_list[contig_pfn_cnt] = pfn;
2265843e1988Sjohnlev if (++contig_pfn_cnt == contig_pfn_max)
2266843e1988Sjohnlev break;
2267843e1988Sjohnlev }
22686358f641SStuart Maybee /*
22696358f641SStuart Maybee * Sanity check the new list.
22706358f641SStuart Maybee */
22716358f641SStuart Maybee if (contig_pfn_cnt < 2) { /* no contig pfns */
22726358f641SStuart Maybee contig_pfn_cnt = 0;
22736358f641SStuart Maybee contig_pfnlist_buildfailed++;
22746358f641SStuart Maybee kmem_free(contig_pfn_list, contig_pfn_max * sizeof (pfn_t));
22756358f641SStuart Maybee contig_pfn_list = NULL;
22766358f641SStuart Maybee contig_pfn_max = 0;
22776358f641SStuart Maybee ret = 0;
22786358f641SStuart Maybee goto out;
22796358f641SStuart Maybee }
2280843e1988Sjohnlev qsort(contig_pfn_list, contig_pfn_cnt, sizeof (pfn_t), mfn_compare);
2281843e1988Sjohnlev compact_contig_pfn_list();
2282843e1988Sjohnlev /*
2283843e1988Sjohnlev * Make sure next search of the newly created contiguous pfn
2284843e1988Sjohnlev * list starts at the beginning of the list.
2285843e1988Sjohnlev */
2286843e1988Sjohnlev next_alloc_pfn = 0;
2287843e1988Sjohnlev contig_pfnlist_builds++; /* count list builds */
2288b9bc7f78Ssmaybe out:
2289b9bc7f78Ssmaybe mutex_exit(&contig_list_lock);
2290b9bc7f78Ssmaybe return (ret);
2291843e1988Sjohnlev }
2292843e1988Sjohnlev
2293843e1988Sjohnlev
2294843e1988Sjohnlev /*
2295843e1988Sjohnlev * Toss the current contig pfnlist. Someone is about to do a massive
2296843e1988Sjohnlev * update to pfn<->mfn mappings. So we have them destroy the list and lock
2297843e1988Sjohnlev * it till they are done with their update.
2298843e1988Sjohnlev */
2299843e1988Sjohnlev void
clear_and_lock_contig_pfnlist()2300843e1988Sjohnlev clear_and_lock_contig_pfnlist()
2301843e1988Sjohnlev {
2302843e1988Sjohnlev pfn_t *listp = NULL;
2303843e1988Sjohnlev size_t listsize;
2304843e1988Sjohnlev
2305b9bc7f78Ssmaybe mutex_enter(&contig_list_lock);
2306843e1988Sjohnlev if (contig_pfn_list != NULL) {
2307843e1988Sjohnlev listp = contig_pfn_list;
2308843e1988Sjohnlev listsize = contig_pfn_max * sizeof (pfn_t);
2309843e1988Sjohnlev contig_pfn_list = NULL;
2310843e1988Sjohnlev contig_pfn_max = contig_pfn_cnt = 0;
2311843e1988Sjohnlev }
2312843e1988Sjohnlev if (listp != NULL)
2313843e1988Sjohnlev kmem_free(listp, listsize);
2314843e1988Sjohnlev }
2315843e1988Sjohnlev
2316843e1988Sjohnlev /*
2317843e1988Sjohnlev * Unlock the contig_pfn_list. The next attempted use of it will cause
2318843e1988Sjohnlev * it to be re-created.
2319843e1988Sjohnlev */
2320843e1988Sjohnlev void
unlock_contig_pfnlist()2321843e1988Sjohnlev unlock_contig_pfnlist()
2322843e1988Sjohnlev {
2323b9bc7f78Ssmaybe mutex_exit(&contig_list_lock);
2324843e1988Sjohnlev }
2325843e1988Sjohnlev
2326843e1988Sjohnlev /*
2327843e1988Sjohnlev * Update the contiguous pfn list in response to a pfn <-> mfn reassignment
2328843e1988Sjohnlev */
2329843e1988Sjohnlev void
update_contig_pfnlist(pfn_t pfn,mfn_t oldmfn,mfn_t newmfn)2330843e1988Sjohnlev update_contig_pfnlist(pfn_t pfn, mfn_t oldmfn, mfn_t newmfn)
2331843e1988Sjohnlev {
2332843e1988Sjohnlev int probe_hi, probe_lo, probe_pos, insert_after, insert_point;
2333843e1988Sjohnlev pfn_t probe_pfn;
2334843e1988Sjohnlev mfn_t probe_mfn;
2335b9bc7f78Ssmaybe int drop_lock = 0;
2336843e1988Sjohnlev
2337b9bc7f78Ssmaybe if (mutex_owner(&contig_list_lock) != curthread) {
2338b9bc7f78Ssmaybe drop_lock = 1;
2339b9bc7f78Ssmaybe mutex_enter(&contig_list_lock);
2340b9bc7f78Ssmaybe }
2341843e1988Sjohnlev if (contig_pfn_list == NULL)
2342b9bc7f78Ssmaybe goto done;
2343843e1988Sjohnlev contig_pfnlist_updates++;
2344843e1988Sjohnlev /*
2345843e1988Sjohnlev * Find the pfn in the current list. Use a binary chop to locate it.
2346843e1988Sjohnlev */
2347843e1988Sjohnlev probe_hi = contig_pfn_cnt - 1;
2348843e1988Sjohnlev probe_lo = 0;
2349843e1988Sjohnlev probe_pos = (probe_hi + probe_lo) / 2;
2350843e1988Sjohnlev while ((probe_pfn = contig_pfn_list[probe_pos]) != pfn) {
2351843e1988Sjohnlev if (probe_pos == probe_lo) { /* pfn not in list */
2352843e1988Sjohnlev probe_pos = -1;
2353843e1988Sjohnlev break;
2354843e1988Sjohnlev }
2355843e1988Sjohnlev if (pfn_to_mfn(probe_pfn) <= oldmfn)
2356843e1988Sjohnlev probe_lo = probe_pos;
2357843e1988Sjohnlev else
2358843e1988Sjohnlev probe_hi = probe_pos;
2359843e1988Sjohnlev probe_pos = (probe_hi + probe_lo) / 2;
2360843e1988Sjohnlev }
23616358f641SStuart Maybee if (probe_pos >= 0) {
23626358f641SStuart Maybee /*
23636358f641SStuart Maybee * Remove pfn from list and ensure next alloc
23646358f641SStuart Maybee * position stays in bounds.
23656358f641SStuart Maybee */
23666358f641SStuart Maybee if (--contig_pfn_cnt <= next_alloc_pfn)
23676358f641SStuart Maybee next_alloc_pfn = 0;
2368349b53ddSStuart Maybee if (contig_pfn_cnt < 2) { /* no contig pfns */
2369349b53ddSStuart Maybee contig_pfn_cnt = 0;
2370349b53ddSStuart Maybee kmem_free(contig_pfn_list,
2371349b53ddSStuart Maybee contig_pfn_max * sizeof (pfn_t));
2372349b53ddSStuart Maybee contig_pfn_list = NULL;
2373349b53ddSStuart Maybee contig_pfn_max = 0;
2374349b53ddSStuart Maybee goto done;
2375349b53ddSStuart Maybee }
2376843e1988Sjohnlev ovbcopy(&contig_pfn_list[probe_pos + 1],
2377843e1988Sjohnlev &contig_pfn_list[probe_pos],
2378843e1988Sjohnlev (contig_pfn_cnt - probe_pos) * sizeof (pfn_t));
2379843e1988Sjohnlev }
2380843e1988Sjohnlev if (newmfn == MFN_INVALID)
2381843e1988Sjohnlev goto done;
2382843e1988Sjohnlev /*
2383843e1988Sjohnlev * Check if new mfn has adjacent mfns in the list
2384843e1988Sjohnlev */
2385843e1988Sjohnlev probe_hi = contig_pfn_cnt - 1;
2386843e1988Sjohnlev probe_lo = 0;
2387843e1988Sjohnlev insert_after = -2;
2388843e1988Sjohnlev do {
2389843e1988Sjohnlev probe_pos = (probe_hi + probe_lo) / 2;
2390843e1988Sjohnlev probe_mfn = pfn_to_mfn(contig_pfn_list[probe_pos]);
2391843e1988Sjohnlev if (newmfn == probe_mfn + 1)
2392843e1988Sjohnlev insert_after = probe_pos;
2393843e1988Sjohnlev else if (newmfn == probe_mfn - 1)
2394843e1988Sjohnlev insert_after = probe_pos - 1;
2395843e1988Sjohnlev if (probe_pos == probe_lo)
2396843e1988Sjohnlev break;
2397843e1988Sjohnlev if (probe_mfn <= newmfn)
2398843e1988Sjohnlev probe_lo = probe_pos;
2399843e1988Sjohnlev else
2400843e1988Sjohnlev probe_hi = probe_pos;
2401843e1988Sjohnlev } while (insert_after == -2);
2402843e1988Sjohnlev /*
2403843e1988Sjohnlev * If there is space in the list and there are adjacent mfns
2404843e1988Sjohnlev * insert the pfn in to its proper place in the list.
2405843e1988Sjohnlev */
2406843e1988Sjohnlev if (insert_after != -2 && contig_pfn_cnt + 1 <= contig_pfn_max) {
2407843e1988Sjohnlev insert_point = insert_after + 1;
2408843e1988Sjohnlev ovbcopy(&contig_pfn_list[insert_point],
2409843e1988Sjohnlev &contig_pfn_list[insert_point + 1],
2410843e1988Sjohnlev (contig_pfn_cnt - insert_point) * sizeof (pfn_t));
2411843e1988Sjohnlev contig_pfn_list[insert_point] = pfn;
2412843e1988Sjohnlev contig_pfn_cnt++;
2413843e1988Sjohnlev }
2414843e1988Sjohnlev done:
2415b9bc7f78Ssmaybe if (drop_lock)
2416b9bc7f78Ssmaybe mutex_exit(&contig_list_lock);
2417843e1988Sjohnlev }
2418843e1988Sjohnlev
2419843e1988Sjohnlev /*
2420843e1988Sjohnlev * Called to (re-)populate the io_pool from the free page lists.
2421843e1988Sjohnlev */
2422843e1988Sjohnlev long
populate_io_pool(void)2423843e1988Sjohnlev populate_io_pool(void)
2424843e1988Sjohnlev {
2425843e1988Sjohnlev pfn_t pfn;
2426843e1988Sjohnlev mfn_t mfn, max_mfn;
2427843e1988Sjohnlev page_t *pp;
2428843e1988Sjohnlev
2429843e1988Sjohnlev /*
2430843e1988Sjohnlev * Figure out the bounds of the pool on first invocation.
2431843e1988Sjohnlev * We use a percentage of memory for the io pool size.
2432843e1988Sjohnlev * we allow that to shrink, but not to less than a fixed minimum
2433843e1988Sjohnlev */
2434843e1988Sjohnlev if (io_pool_cnt_max == 0) {
2435843e1988Sjohnlev io_pool_cnt_max = physmem / (100 / io_pool_physmem_pct);
2436843e1988Sjohnlev io_pool_cnt_lowater = io_pool_cnt_max;
2437843e1988Sjohnlev /*
2438843e1988Sjohnlev * This is the first time in populate_io_pool, grab a va to use
2439843e1988Sjohnlev * when we need to allocate pages.
2440843e1988Sjohnlev */
2441843e1988Sjohnlev io_pool_kva = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP);
2442843e1988Sjohnlev }
2443843e1988Sjohnlev /*
2444843e1988Sjohnlev * If we are out of pages in the pool, then grow the size of the pool
2445843e1988Sjohnlev */
2446cf902cd2Ssmaybe if (io_pool_cnt == 0) {
2447cf902cd2Ssmaybe /*
2448cf902cd2Ssmaybe * Grow the max size of the io pool by 5%, but never more than
2449cf902cd2Ssmaybe * 25% of physical memory.
2450cf902cd2Ssmaybe */
2451cf902cd2Ssmaybe if (io_pool_cnt_max < physmem / 4)
2452cf902cd2Ssmaybe io_pool_cnt_max += io_pool_cnt_max / 20;
2453cf902cd2Ssmaybe }
2454843e1988Sjohnlev io_pool_grows++; /* should be a kstat? */
2455843e1988Sjohnlev
2456843e1988Sjohnlev /*
2457843e1988Sjohnlev * Get highest mfn on this platform, but limit to the 32 bit DMA max.
2458843e1988Sjohnlev */
2459843e1988Sjohnlev (void) mfn_to_pfn(start_mfn);
2460843e1988Sjohnlev max_mfn = MIN(cached_max_mfn, PFN_4GIG);
2461843e1988Sjohnlev for (mfn = start_mfn; mfn < max_mfn; start_mfn = ++mfn) {
2462843e1988Sjohnlev pfn = mfn_to_pfn(mfn);
2463843e1988Sjohnlev if (pfn & PFN_IS_FOREIGN_MFN)
2464843e1988Sjohnlev continue;
2465843e1988Sjohnlev /*
2466843e1988Sjohnlev * try to allocate it from free pages
2467843e1988Sjohnlev */
2468843e1988Sjohnlev pp = page_numtopp_alloc(pfn);
2469843e1988Sjohnlev if (pp == NULL)
2470843e1988Sjohnlev continue;
2471843e1988Sjohnlev PP_CLRFREE(pp);
2472843e1988Sjohnlev add_page_to_pool(pp, 1);
2473843e1988Sjohnlev if (io_pool_cnt >= io_pool_cnt_max)
2474843e1988Sjohnlev break;
2475843e1988Sjohnlev }
2476843e1988Sjohnlev
2477843e1988Sjohnlev return (io_pool_cnt);
2478843e1988Sjohnlev }
2479843e1988Sjohnlev
2480843e1988Sjohnlev /*
2481843e1988Sjohnlev * Destroy a page that was being used for DMA I/O. It may or
2482843e1988Sjohnlev * may not actually go back to the io_pool.
2483843e1988Sjohnlev */
2484843e1988Sjohnlev void
page_destroy_io(page_t * pp)2485843e1988Sjohnlev page_destroy_io(page_t *pp)
2486843e1988Sjohnlev {
2487843e1988Sjohnlev mfn_t mfn = mfn_list[pp->p_pagenum];
2488843e1988Sjohnlev
2489843e1988Sjohnlev /*
2490843e1988Sjohnlev * When the page was alloc'd a reservation was made, release it now
2491843e1988Sjohnlev */
2492843e1988Sjohnlev page_unresv(1);
2493843e1988Sjohnlev /*
2494843e1988Sjohnlev * Unload translations, if any, then hash out the
2495843e1988Sjohnlev * page to erase its identity.
2496843e1988Sjohnlev */
2497843e1988Sjohnlev (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
2498843e1988Sjohnlev page_hashout(pp, NULL);
2499843e1988Sjohnlev
2500843e1988Sjohnlev /*
2501843e1988Sjohnlev * If the page came from the free lists, just put it back to them.
2502843e1988Sjohnlev * DomU pages always go on the free lists as well.
2503843e1988Sjohnlev */
2504843e1988Sjohnlev if (!DOMAIN_IS_INITDOMAIN(xen_info) || mfn >= PFN_4GIG) {
2505843e1988Sjohnlev page_free(pp, 1);
2506843e1988Sjohnlev return;
2507843e1988Sjohnlev }
2508843e1988Sjohnlev
2509843e1988Sjohnlev add_page_to_pool(pp, 0);
2510843e1988Sjohnlev }
2511843e1988Sjohnlev
2512843e1988Sjohnlev
2513843e1988Sjohnlev long contig_searches; /* count of times contig pages requested */
2514843e1988Sjohnlev long contig_search_restarts; /* count of contig ranges tried */
2515843e1988Sjohnlev long contig_search_failed; /* count of contig alloc failures */
2516843e1988Sjohnlev
2517349b53ddSStuart Maybee /*
2518349b53ddSStuart Maybee * Free partial page list
2519349b53ddSStuart Maybee */
2520349b53ddSStuart Maybee static void
free_partial_list(page_t ** pplist)2521349b53ddSStuart Maybee free_partial_list(page_t **pplist)
2522349b53ddSStuart Maybee {
2523349b53ddSStuart Maybee page_t *pp;
2524349b53ddSStuart Maybee
2525349b53ddSStuart Maybee while (*pplist != NULL) {
2526349b53ddSStuart Maybee pp = *pplist;
2527349b53ddSStuart Maybee page_io_pool_sub(pplist, pp, pp);
2528349b53ddSStuart Maybee page_free(pp, 1);
2529349b53ddSStuart Maybee }
2530349b53ddSStuart Maybee }
2531349b53ddSStuart Maybee
2532843e1988Sjohnlev /*
2533843e1988Sjohnlev * Look thru the contiguous pfns that are not part of the io_pool for
2534843e1988Sjohnlev * contiguous free pages. Return a list of the found pages or NULL.
2535843e1988Sjohnlev */
2536843e1988Sjohnlev page_t *
find_contig_free(uint_t npages,uint_t flags,uint64_t pfnseg,pgcnt_t pfnalign)2537349b53ddSStuart Maybee find_contig_free(uint_t npages, uint_t flags, uint64_t pfnseg,
2538349b53ddSStuart Maybee pgcnt_t pfnalign)
2539843e1988Sjohnlev {
2540843e1988Sjohnlev page_t *pp, *plist = NULL;
25412e8a1712Ssmaybe mfn_t mfn, prev_mfn, start_mfn;
2542843e1988Sjohnlev pfn_t pfn;
2543843e1988Sjohnlev int pages_needed, pages_requested;
2544843e1988Sjohnlev int search_start;
2545843e1988Sjohnlev
2546843e1988Sjohnlev /*
2547843e1988Sjohnlev * create the contig pfn list if not already done
2548843e1988Sjohnlev */
2549b9bc7f78Ssmaybe retry:
2550b9bc7f78Ssmaybe mutex_enter(&contig_list_lock);
2551843e1988Sjohnlev if (contig_pfn_list == NULL) {
2552b9bc7f78Ssmaybe mutex_exit(&contig_list_lock);
2553b9bc7f78Ssmaybe if (!create_contig_pfnlist(flags)) {
2554843e1988Sjohnlev return (NULL);
2555843e1988Sjohnlev }
2556b9bc7f78Ssmaybe goto retry;
2557843e1988Sjohnlev }
2558843e1988Sjohnlev contig_searches++;
2559843e1988Sjohnlev /*
2560843e1988Sjohnlev * Search contiguous pfn list for physically contiguous pages not in
2561843e1988Sjohnlev * the io_pool. Start the search where the last search left off.
2562843e1988Sjohnlev */
25636f235fc0Ssmaybe pages_requested = pages_needed = npages;
2564843e1988Sjohnlev search_start = next_alloc_pfn;
25652e8a1712Ssmaybe start_mfn = prev_mfn = 0;
2566843e1988Sjohnlev while (pages_needed) {
2567843e1988Sjohnlev pfn = contig_pfn_list[next_alloc_pfn];
2568843e1988Sjohnlev mfn = pfn_to_mfn(pfn);
25692e8a1712Ssmaybe /*
25702e8a1712Ssmaybe * Check if mfn is first one or contig to previous one and
25712e8a1712Ssmaybe * if page corresponding to mfn is free and that mfn
25722e8a1712Ssmaybe * range is not crossing a segment boundary.
25732e8a1712Ssmaybe */
2574843e1988Sjohnlev if ((prev_mfn == 0 || mfn == prev_mfn + 1) &&
25752e8a1712Ssmaybe (pp = page_numtopp_alloc(pfn)) != NULL &&
25762e8a1712Ssmaybe !((mfn & pfnseg) < (start_mfn & pfnseg))) {
2577843e1988Sjohnlev PP_CLRFREE(pp);
2578843e1988Sjohnlev page_io_pool_add(&plist, pp);
2579843e1988Sjohnlev pages_needed--;
2580349b53ddSStuart Maybee if (prev_mfn == 0) {
2581349b53ddSStuart Maybee if (pfnalign &&
2582349b53ddSStuart Maybee mfn != P2ROUNDUP(mfn, pfnalign)) {
2583349b53ddSStuart Maybee /*
2584349b53ddSStuart Maybee * not properly aligned
2585349b53ddSStuart Maybee */
2586349b53ddSStuart Maybee contig_search_restarts++;
2587349b53ddSStuart Maybee free_partial_list(&plist);
2588349b53ddSStuart Maybee pages_needed = pages_requested;
2589349b53ddSStuart Maybee start_mfn = prev_mfn = 0;
2590349b53ddSStuart Maybee goto skip;
2591349b53ddSStuart Maybee }
25922e8a1712Ssmaybe start_mfn = mfn;
2593349b53ddSStuart Maybee }
2594843e1988Sjohnlev prev_mfn = mfn;
2595843e1988Sjohnlev } else {
2596843e1988Sjohnlev contig_search_restarts++;
2597349b53ddSStuart Maybee free_partial_list(&plist);
2598843e1988Sjohnlev pages_needed = pages_requested;
25992e8a1712Ssmaybe start_mfn = prev_mfn = 0;
2600843e1988Sjohnlev }
2601349b53ddSStuart Maybee skip:
2602843e1988Sjohnlev if (++next_alloc_pfn == contig_pfn_cnt)
2603843e1988Sjohnlev next_alloc_pfn = 0;
2604843e1988Sjohnlev if (next_alloc_pfn == search_start)
2605843e1988Sjohnlev break; /* all pfns searched */
2606843e1988Sjohnlev }
2607b9bc7f78Ssmaybe mutex_exit(&contig_list_lock);
2608843e1988Sjohnlev if (pages_needed) {
2609843e1988Sjohnlev contig_search_failed++;
2610843e1988Sjohnlev /*
2611843e1988Sjohnlev * Failed to find enough contig pages.
2612843e1988Sjohnlev * free partial page list
2613843e1988Sjohnlev */
2614349b53ddSStuart Maybee free_partial_list(&plist);
2615843e1988Sjohnlev }
2616843e1988Sjohnlev return (plist);
2617843e1988Sjohnlev }
2618843e1988Sjohnlev
2619843e1988Sjohnlev /*
26206f235fc0Ssmaybe * Search the reserved io pool pages for a page range with the
26216f235fc0Ssmaybe * desired characteristics.
26227c478bd9Sstevel@tonic-gate */
26237c478bd9Sstevel@tonic-gate page_t *
page_io_pool_alloc(ddi_dma_attr_t * mattr,int contig,pgcnt_t minctg)26246f235fc0Ssmaybe page_io_pool_alloc(ddi_dma_attr_t *mattr, int contig, pgcnt_t minctg)
2625843e1988Sjohnlev {
26266f235fc0Ssmaybe page_t *pp_first, *pp_last;
26276f235fc0Ssmaybe page_t *pp, **poolp;
26286f235fc0Ssmaybe pgcnt_t nwanted, pfnalign;
2629843e1988Sjohnlev uint64_t pfnseg;
26306f235fc0Ssmaybe mfn_t mfn, tmfn, hi_mfn, lo_mfn;
26316f235fc0Ssmaybe int align, attempt = 0;
2632843e1988Sjohnlev
26336f235fc0Ssmaybe if (minctg == 1)
26346f235fc0Ssmaybe contig = 0;
2635843e1988Sjohnlev lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
2636843e1988Sjohnlev hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
26376f235fc0Ssmaybe pfnseg = mmu_btop(mattr->dma_attr_seg);
2638843e1988Sjohnlev align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
2639843e1988Sjohnlev if (align > MMU_PAGESIZE)
2640843e1988Sjohnlev pfnalign = mmu_btop(align);
26416f235fc0Ssmaybe else
26426f235fc0Ssmaybe pfnalign = 0;
2643843e1988Sjohnlev
2644843e1988Sjohnlev try_again:
2645843e1988Sjohnlev /*
2646843e1988Sjohnlev * See if we want pages for a legacy device
2647843e1988Sjohnlev */
2648843e1988Sjohnlev if (hi_mfn < PFN_16MEG)
2649843e1988Sjohnlev poolp = &io_pool_16m;
2650843e1988Sjohnlev else
2651843e1988Sjohnlev poolp = &io_pool_4g;
2652843e1988Sjohnlev try_smaller:
2653843e1988Sjohnlev /*
26546f235fc0Ssmaybe * Take pages from I/O pool. We'll use pages from the highest
26556f235fc0Ssmaybe * MFN range possible.
2656843e1988Sjohnlev */
2657843e1988Sjohnlev pp_first = pp_last = NULL;
2658843e1988Sjohnlev mutex_enter(&io_pool_lock);
26596f235fc0Ssmaybe nwanted = minctg;
26606f235fc0Ssmaybe for (pp = *poolp; pp && nwanted > 0; ) {
2661843e1988Sjohnlev pp = pp->p_prev;
2662843e1988Sjohnlev
2663843e1988Sjohnlev /*
2664843e1988Sjohnlev * skip pages above allowable range
2665843e1988Sjohnlev */
2666843e1988Sjohnlev mfn = mfn_list[pp->p_pagenum];
2667843e1988Sjohnlev if (hi_mfn < mfn)
2668843e1988Sjohnlev goto skip;
2669843e1988Sjohnlev
2670843e1988Sjohnlev /*
2671843e1988Sjohnlev * stop at pages below allowable range
2672843e1988Sjohnlev */
2673843e1988Sjohnlev if (lo_mfn > mfn)
2674843e1988Sjohnlev break;
2675843e1988Sjohnlev restart:
2676843e1988Sjohnlev if (pp_last == NULL) {
2677843e1988Sjohnlev /*
2678843e1988Sjohnlev * Check alignment
2679843e1988Sjohnlev */
26806f235fc0Ssmaybe tmfn = mfn - (minctg - 1);
26816f235fc0Ssmaybe if (pfnalign && tmfn != P2ROUNDUP(tmfn, pfnalign))
26826f235fc0Ssmaybe goto skip; /* not properly aligned */
2683843e1988Sjohnlev /*
2684843e1988Sjohnlev * Check segment
2685843e1988Sjohnlev */
2686843e1988Sjohnlev if ((mfn & pfnseg) < (tmfn & pfnseg))
26876f235fc0Ssmaybe goto skip; /* crosses seg boundary */
2688843e1988Sjohnlev /*
2689843e1988Sjohnlev * Start building page list
2690843e1988Sjohnlev */
2691843e1988Sjohnlev pp_first = pp_last = pp;
26926f235fc0Ssmaybe nwanted--;
2693843e1988Sjohnlev } else {
2694843e1988Sjohnlev /*
2695843e1988Sjohnlev * check physical contiguity if required
2696843e1988Sjohnlev */
2697843e1988Sjohnlev if (contig &&
2698843e1988Sjohnlev mfn_list[pp_first->p_pagenum] != mfn + 1) {
2699843e1988Sjohnlev /*
2700843e1988Sjohnlev * not a contiguous page, restart list.
2701843e1988Sjohnlev */
2702843e1988Sjohnlev pp_last = NULL;
27036f235fc0Ssmaybe nwanted = minctg;
2704843e1988Sjohnlev goto restart;
2705843e1988Sjohnlev } else { /* add page to list */
2706843e1988Sjohnlev pp_first = pp;
27076f235fc0Ssmaybe nwanted--;
2708843e1988Sjohnlev }
2709843e1988Sjohnlev }
2710843e1988Sjohnlev skip:
2711843e1988Sjohnlev if (pp == *poolp)
2712843e1988Sjohnlev break;
2713843e1988Sjohnlev }
2714843e1988Sjohnlev
2715843e1988Sjohnlev /*
2716843e1988Sjohnlev * If we didn't find memory. Try the more constrained pool, then
27176f235fc0Ssmaybe * sweep free pages into the DMA pool and try again.
2718843e1988Sjohnlev */
27196f235fc0Ssmaybe if (nwanted != 0) {
2720843e1988Sjohnlev mutex_exit(&io_pool_lock);
2721843e1988Sjohnlev /*
27226f235fc0Ssmaybe * If we were looking in the less constrained pool and
27236f235fc0Ssmaybe * didn't find pages, try the more constrained pool.
2724843e1988Sjohnlev */
2725843e1988Sjohnlev if (poolp == &io_pool_4g) {
2726843e1988Sjohnlev poolp = &io_pool_16m;
2727843e1988Sjohnlev goto try_smaller;
2728843e1988Sjohnlev }
2729843e1988Sjohnlev kmem_reap();
2730843e1988Sjohnlev if (++attempt < 4) {
2731843e1988Sjohnlev /*
2732843e1988Sjohnlev * Grab some more io_pool pages
2733843e1988Sjohnlev */
2734843e1988Sjohnlev (void) populate_io_pool();
27356f235fc0Ssmaybe goto try_again; /* go around and retry */
2736843e1988Sjohnlev }
27376f235fc0Ssmaybe return (NULL);
27386f235fc0Ssmaybe }
27396f235fc0Ssmaybe /*
27406f235fc0Ssmaybe * Found the pages, now snip them from the list
27416f235fc0Ssmaybe */
27426f235fc0Ssmaybe page_io_pool_sub(poolp, pp_first, pp_last);
27436f235fc0Ssmaybe io_pool_cnt -= minctg;
27446f235fc0Ssmaybe /*
27456f235fc0Ssmaybe * reset low water mark
27466f235fc0Ssmaybe */
27476f235fc0Ssmaybe if (io_pool_cnt < io_pool_cnt_lowater)
27486f235fc0Ssmaybe io_pool_cnt_lowater = io_pool_cnt;
27496f235fc0Ssmaybe mutex_exit(&io_pool_lock);
27506f235fc0Ssmaybe return (pp_first);
27516f235fc0Ssmaybe }
2752843e1988Sjohnlev
27536f235fc0Ssmaybe page_t *
page_swap_with_hypervisor(struct vnode * vp,u_offset_t off,caddr_t vaddr,ddi_dma_attr_t * mattr,uint_t flags,pgcnt_t minctg)27546f235fc0Ssmaybe page_swap_with_hypervisor(struct vnode *vp, u_offset_t off, caddr_t vaddr,
27556f235fc0Ssmaybe ddi_dma_attr_t *mattr, uint_t flags, pgcnt_t minctg)
27566f235fc0Ssmaybe {
27576f235fc0Ssmaybe uint_t kflags;
27586f235fc0Ssmaybe int order, extra, extpages, i, contig, nbits, extents;
27596f235fc0Ssmaybe page_t *pp, *expp, *pp_first, **pplist = NULL;
27606f235fc0Ssmaybe mfn_t *mfnlist = NULL;
27616f235fc0Ssmaybe
27622a9992ecSToomas Soome extra = 0;
27636f235fc0Ssmaybe contig = flags & PG_PHYSCONTIG;
27646f235fc0Ssmaybe if (minctg == 1)
27656f235fc0Ssmaybe contig = 0;
27666f235fc0Ssmaybe flags &= ~PG_PHYSCONTIG;
27676f235fc0Ssmaybe kflags = flags & PG_WAIT ? KM_SLEEP : KM_NOSLEEP;
27686f235fc0Ssmaybe /*
27696f235fc0Ssmaybe * Hypervisor will allocate extents, if we want contig
27706f235fc0Ssmaybe * pages extent must be >= minctg
27716f235fc0Ssmaybe */
27726f235fc0Ssmaybe if (contig) {
27736f235fc0Ssmaybe order = highbit(minctg) - 1;
27746f235fc0Ssmaybe if (minctg & ((1 << order) - 1))
27756f235fc0Ssmaybe order++;
27766f235fc0Ssmaybe extpages = 1 << order;
27776f235fc0Ssmaybe } else {
27786f235fc0Ssmaybe order = 0;
27796f235fc0Ssmaybe extpages = minctg;
27806f235fc0Ssmaybe }
27816f235fc0Ssmaybe if (extpages > minctg) {
27826f235fc0Ssmaybe extra = extpages - minctg;
27836f235fc0Ssmaybe if (!page_resv(extra, kflags))
2784843e1988Sjohnlev return (NULL);
27856f235fc0Ssmaybe }
27866f235fc0Ssmaybe pp_first = NULL;
27876f235fc0Ssmaybe pplist = kmem_alloc(extpages * sizeof (page_t *), kflags);
27886f235fc0Ssmaybe if (pplist == NULL)
27896f235fc0Ssmaybe goto balloon_fail;
27906f235fc0Ssmaybe mfnlist = kmem_alloc(extpages * sizeof (mfn_t), kflags);
27916f235fc0Ssmaybe if (mfnlist == NULL)
27926f235fc0Ssmaybe goto balloon_fail;
27936f235fc0Ssmaybe pp = page_create_va(vp, off, minctg * PAGESIZE, flags, &kvseg, vaddr);
27946f235fc0Ssmaybe if (pp == NULL)
27956f235fc0Ssmaybe goto balloon_fail;
27966f235fc0Ssmaybe pp_first = pp;
27976f235fc0Ssmaybe if (extpages > minctg) {
2798843e1988Sjohnlev /*
27996f235fc0Ssmaybe * fill out the rest of extent pages to swap
28006f235fc0Ssmaybe * with the hypervisor
2801843e1988Sjohnlev */
28026f235fc0Ssmaybe for (i = 0; i < extra; i++) {
28036f235fc0Ssmaybe expp = page_create_va(vp,
28046f235fc0Ssmaybe (u_offset_t)(uintptr_t)io_pool_kva,
28056f235fc0Ssmaybe PAGESIZE, flags, &kvseg, io_pool_kva);
28066f235fc0Ssmaybe if (expp == NULL)
28076f235fc0Ssmaybe goto balloon_fail;
28086f235fc0Ssmaybe (void) hat_pageunload(expp, HAT_FORCE_PGUNLOAD);
28096f235fc0Ssmaybe page_io_unlock(expp);
28106f235fc0Ssmaybe page_hashout(expp, NULL);
28116f235fc0Ssmaybe page_io_lock(expp);
2812843e1988Sjohnlev /*
28136f235fc0Ssmaybe * add page to end of list
2814843e1988Sjohnlev */
28156f235fc0Ssmaybe expp->p_prev = pp_first->p_prev;
28166f235fc0Ssmaybe expp->p_next = pp_first;
28176f235fc0Ssmaybe expp->p_prev->p_next = expp;
28186f235fc0Ssmaybe pp_first->p_prev = expp;
2819b9bc7f78Ssmaybe }
2820843e1988Sjohnlev
28216f235fc0Ssmaybe }
28226f235fc0Ssmaybe for (i = 0; i < extpages; i++) {
28236f235fc0Ssmaybe pplist[i] = pp;
28246f235fc0Ssmaybe pp = pp->p_next;
28256f235fc0Ssmaybe }
28266f235fc0Ssmaybe nbits = highbit(mattr->dma_attr_addr_hi);
28276f235fc0Ssmaybe extents = contig ? 1 : minctg;
28286f235fc0Ssmaybe if (balloon_replace_pages(extents, pplist, nbits, order,
28296f235fc0Ssmaybe mfnlist) != extents) {
28306f235fc0Ssmaybe if (ioalloc_dbg)
28316f235fc0Ssmaybe cmn_err(CE_NOTE, "request to hypervisor"
28326f235fc0Ssmaybe " for %d pages, maxaddr %" PRIx64 " failed",
28336f235fc0Ssmaybe extpages, mattr->dma_attr_addr_hi);
28346f235fc0Ssmaybe goto balloon_fail;
2835843e1988Sjohnlev }
2836843e1988Sjohnlev
28376f235fc0Ssmaybe kmem_free(pplist, extpages * sizeof (page_t *));
28386f235fc0Ssmaybe kmem_free(mfnlist, extpages * sizeof (mfn_t));
2839843e1988Sjohnlev /*
28406f235fc0Ssmaybe * Return any excess pages to free list
2841843e1988Sjohnlev */
28426f235fc0Ssmaybe if (extpages > minctg) {
28436f235fc0Ssmaybe for (i = 0; i < extra; i++) {
28446f235fc0Ssmaybe pp = pp_first->p_prev;
28456f235fc0Ssmaybe page_sub(&pp_first, pp);
28466f235fc0Ssmaybe page_io_unlock(pp);
28476f235fc0Ssmaybe page_unresv(1);
28486f235fc0Ssmaybe page_free(pp, 1);
2849843e1988Sjohnlev }
28506f235fc0Ssmaybe }
2851843e1988Sjohnlev return (pp_first);
2852843e1988Sjohnlev balloon_fail:
2853843e1988Sjohnlev /*
2854843e1988Sjohnlev * Return pages to free list and return failure
2855843e1988Sjohnlev */
2856843e1988Sjohnlev while (pp_first != NULL) {
2857843e1988Sjohnlev pp = pp_first;
2858843e1988Sjohnlev page_sub(&pp_first, pp);
2859843e1988Sjohnlev page_io_unlock(pp);
2860843e1988Sjohnlev if (pp->p_vnode != NULL)
2861843e1988Sjohnlev page_hashout(pp, NULL);
2862843e1988Sjohnlev page_free(pp, 1);
2863843e1988Sjohnlev }
2864843e1988Sjohnlev if (pplist)
2865843e1988Sjohnlev kmem_free(pplist, extpages * sizeof (page_t *));
2866843e1988Sjohnlev if (mfnlist)
2867843e1988Sjohnlev kmem_free(mfnlist, extpages * sizeof (mfn_t));
28686f235fc0Ssmaybe page_unresv(extpages - minctg);
28696f235fc0Ssmaybe return (NULL);
28706f235fc0Ssmaybe }
28716f235fc0Ssmaybe
28726f235fc0Ssmaybe static void
return_partial_alloc(page_t * plist)28736f235fc0Ssmaybe return_partial_alloc(page_t *plist)
28746f235fc0Ssmaybe {
28756f235fc0Ssmaybe page_t *pp;
28766f235fc0Ssmaybe
28776f235fc0Ssmaybe while (plist != NULL) {
28786f235fc0Ssmaybe pp = plist;
28796f235fc0Ssmaybe page_sub(&plist, pp);
2880d21b39ddSmrj page_io_unlock(pp);
28816f235fc0Ssmaybe page_destroy_io(pp);
28826f235fc0Ssmaybe }
28836f235fc0Ssmaybe }
28846f235fc0Ssmaybe
28856f235fc0Ssmaybe static page_t *
page_get_contigpages(struct vnode * vp,u_offset_t off,int * npagesp,uint_t flags,caddr_t vaddr,ddi_dma_attr_t * mattr)28866f235fc0Ssmaybe page_get_contigpages(
28876f235fc0Ssmaybe struct vnode *vp,
28886f235fc0Ssmaybe u_offset_t off,
28896f235fc0Ssmaybe int *npagesp,
28906f235fc0Ssmaybe uint_t flags,
28916f235fc0Ssmaybe caddr_t vaddr,
28926f235fc0Ssmaybe ddi_dma_attr_t *mattr)
28936f235fc0Ssmaybe {
28946f235fc0Ssmaybe mfn_t max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL);
28956f235fc0Ssmaybe page_t *plist; /* list to return */
28966f235fc0Ssmaybe page_t *pp, *mcpl;
28976f235fc0Ssmaybe int contig, anyaddr, npages, getone = 0;
28986f235fc0Ssmaybe mfn_t lo_mfn;
28996f235fc0Ssmaybe mfn_t hi_mfn;
29006f235fc0Ssmaybe pgcnt_t pfnalign = 0;
29016f235fc0Ssmaybe int align, sgllen;
29026f235fc0Ssmaybe uint64_t pfnseg;
29036f235fc0Ssmaybe pgcnt_t minctg;
29046f235fc0Ssmaybe
29056f235fc0Ssmaybe npages = *npagesp;
29066f235fc0Ssmaybe ASSERT(mattr != NULL);
29076f235fc0Ssmaybe lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
29086f235fc0Ssmaybe hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
29096f235fc0Ssmaybe sgllen = mattr->dma_attr_sgllen;
29106f235fc0Ssmaybe pfnseg = mmu_btop(mattr->dma_attr_seg);
29116f235fc0Ssmaybe align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
29126f235fc0Ssmaybe if (align > MMU_PAGESIZE)
29136f235fc0Ssmaybe pfnalign = mmu_btop(align);
29146f235fc0Ssmaybe
2915349b53ddSStuart Maybee contig = flags & PG_PHYSCONTIG;
2916349b53ddSStuart Maybee if (npages == -1) {
2917349b53ddSStuart Maybee npages = 1;
2918349b53ddSStuart Maybee pfnalign = 0;
2919349b53ddSStuart Maybee }
29206f235fc0Ssmaybe /*
29216f235fc0Ssmaybe * Clear the contig flag if only one page is needed.
29226f235fc0Ssmaybe */
29236f235fc0Ssmaybe if (npages == 1) {
29246f235fc0Ssmaybe getone = 1;
29256f235fc0Ssmaybe contig = 0;
29266f235fc0Ssmaybe }
29276f235fc0Ssmaybe
29286f235fc0Ssmaybe /*
29296f235fc0Ssmaybe * Check if any page in the system is fine.
29306f235fc0Ssmaybe */
2931349b53ddSStuart Maybee anyaddr = lo_mfn == 0 && hi_mfn >= max_mfn;
2932349b53ddSStuart Maybee if (!contig && anyaddr && !pfnalign) {
29336f235fc0Ssmaybe flags &= ~PG_PHYSCONTIG;
29346f235fc0Ssmaybe plist = page_create_va(vp, off, npages * MMU_PAGESIZE,
29356f235fc0Ssmaybe flags, &kvseg, vaddr);
29366f235fc0Ssmaybe if (plist != NULL) {
29376f235fc0Ssmaybe *npagesp = 0;
29386f235fc0Ssmaybe return (plist);
29396f235fc0Ssmaybe }
29406f235fc0Ssmaybe }
29416f235fc0Ssmaybe plist = NULL;
29426f235fc0Ssmaybe minctg = howmany(npages, sgllen);
29436f235fc0Ssmaybe while (npages > sgllen || getone) {
2944804cf79fSsmaybe if (minctg > npages)
2945804cf79fSsmaybe minctg = npages;
2946804cf79fSsmaybe mcpl = NULL;
29476f235fc0Ssmaybe /*
2948349b53ddSStuart Maybee * We could want contig pages with no address range limits.
29496f235fc0Ssmaybe */
29502e8a1712Ssmaybe if (anyaddr && contig) {
29516f235fc0Ssmaybe /*
29526f235fc0Ssmaybe * Look for free contig pages to satisfy the request.
29536f235fc0Ssmaybe */
2954349b53ddSStuart Maybee mcpl = find_contig_free(minctg, flags, pfnseg,
2955349b53ddSStuart Maybee pfnalign);
29566f235fc0Ssmaybe }
29576f235fc0Ssmaybe /*
29586f235fc0Ssmaybe * Try the reserved io pools next
29596f235fc0Ssmaybe */
29606f235fc0Ssmaybe if (mcpl == NULL)
29616f235fc0Ssmaybe mcpl = page_io_pool_alloc(mattr, contig, minctg);
29626f235fc0Ssmaybe if (mcpl != NULL) {
29636f235fc0Ssmaybe pp = mcpl;
29646f235fc0Ssmaybe do {
29656f235fc0Ssmaybe if (!page_hashin(pp, vp, off, NULL)) {
29666f235fc0Ssmaybe panic("page_get_contigpages:"
29676f235fc0Ssmaybe " hashin failed"
29686f235fc0Ssmaybe " pp %p, vp %p, off %llx",
29696f235fc0Ssmaybe (void *)pp, (void *)vp, off);
29706f235fc0Ssmaybe }
29716f235fc0Ssmaybe off += MMU_PAGESIZE;
29726f235fc0Ssmaybe PP_CLRFREE(pp);
29736f235fc0Ssmaybe PP_CLRAGED(pp);
29746f235fc0Ssmaybe page_set_props(pp, P_REF);
29756f235fc0Ssmaybe page_io_lock(pp);
29766f235fc0Ssmaybe pp = pp->p_next;
29776f235fc0Ssmaybe } while (pp != mcpl);
29786f235fc0Ssmaybe } else {
29796f235fc0Ssmaybe /*
29806f235fc0Ssmaybe * Hypervisor exchange doesn't handle segment or
29816f235fc0Ssmaybe * alignment constraints
29826f235fc0Ssmaybe */
29836f235fc0Ssmaybe if (mattr->dma_attr_seg < mattr->dma_attr_addr_hi ||
29846f235fc0Ssmaybe pfnalign)
29856f235fc0Ssmaybe goto fail;
29866f235fc0Ssmaybe /*
29876f235fc0Ssmaybe * Try exchanging pages with the hypervisor
29886f235fc0Ssmaybe */
29896f235fc0Ssmaybe mcpl = page_swap_with_hypervisor(vp, off, vaddr, mattr,
29906f235fc0Ssmaybe flags, minctg);
29916f235fc0Ssmaybe if (mcpl == NULL)
29926f235fc0Ssmaybe goto fail;
29936f235fc0Ssmaybe off += minctg * MMU_PAGESIZE;
29946f235fc0Ssmaybe }
29956f235fc0Ssmaybe check_dma(mattr, mcpl, minctg);
29966f235fc0Ssmaybe /*
29976f235fc0Ssmaybe * Here with a minctg run of contiguous pages, add them to the
29986f235fc0Ssmaybe * list we will return for this request.
29996f235fc0Ssmaybe */
30006f235fc0Ssmaybe page_list_concat(&plist, &mcpl);
30016f235fc0Ssmaybe npages -= minctg;
30026f235fc0Ssmaybe *npagesp = npages;
30036f235fc0Ssmaybe sgllen--;
3004804cf79fSsmaybe if (getone)
3005804cf79fSsmaybe break;
30066f235fc0Ssmaybe }
30076f235fc0Ssmaybe return (plist);
30086f235fc0Ssmaybe fail:
30096f235fc0Ssmaybe return_partial_alloc(plist);
30106f235fc0Ssmaybe return (NULL);
30116f235fc0Ssmaybe }
30126f235fc0Ssmaybe
30136f235fc0Ssmaybe /*
30146f235fc0Ssmaybe * Allocator for domain 0 I/O pages. We match the required
30156f235fc0Ssmaybe * DMA attributes and contiguity constraints.
30166f235fc0Ssmaybe */
30176f235fc0Ssmaybe /*ARGSUSED*/
30186f235fc0Ssmaybe page_t *
page_create_io(struct vnode * vp,u_offset_t off,uint_t bytes,uint_t flags,struct as * as,caddr_t vaddr,ddi_dma_attr_t * mattr)30196f235fc0Ssmaybe page_create_io(
30206f235fc0Ssmaybe struct vnode *vp,
30216f235fc0Ssmaybe u_offset_t off,
30226f235fc0Ssmaybe uint_t bytes,
30236f235fc0Ssmaybe uint_t flags,
30246f235fc0Ssmaybe struct as *as,
30256f235fc0Ssmaybe caddr_t vaddr,
30266f235fc0Ssmaybe ddi_dma_attr_t *mattr)
30276f235fc0Ssmaybe {
30286f235fc0Ssmaybe page_t *plist = NULL, *pp;
30296f235fc0Ssmaybe int npages = 0, contig, anyaddr, pages_req;
30306f235fc0Ssmaybe mfn_t lo_mfn;
30316f235fc0Ssmaybe mfn_t hi_mfn;
30326f235fc0Ssmaybe pgcnt_t pfnalign = 0;
30336f235fc0Ssmaybe int align;
30346f235fc0Ssmaybe int is_domu = 0;
30356f235fc0Ssmaybe int dummy, bytes_got;
30366f235fc0Ssmaybe mfn_t max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL);
30376f235fc0Ssmaybe
30386f235fc0Ssmaybe ASSERT(mattr != NULL);
30396f235fc0Ssmaybe lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
30406f235fc0Ssmaybe hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
30416f235fc0Ssmaybe align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
30426f235fc0Ssmaybe if (align > MMU_PAGESIZE)
30436f235fc0Ssmaybe pfnalign = mmu_btop(align);
30446f235fc0Ssmaybe
30456f235fc0Ssmaybe /*
30466f235fc0Ssmaybe * Clear the contig flag if only one page is needed or the scatter
30476f235fc0Ssmaybe * gather list length is >= npages.
30486f235fc0Ssmaybe */
30496f235fc0Ssmaybe pages_req = npages = mmu_btopr(bytes);
30506f235fc0Ssmaybe contig = (flags & PG_PHYSCONTIG);
30516f235fc0Ssmaybe bytes = P2ROUNDUP(bytes, MMU_PAGESIZE);
30526f235fc0Ssmaybe if (bytes == MMU_PAGESIZE || mattr->dma_attr_sgllen >= npages)
30536f235fc0Ssmaybe contig = 0;
30546f235fc0Ssmaybe
30556f235fc0Ssmaybe /*
30566f235fc0Ssmaybe * Check if any old page in the system is fine.
30576f235fc0Ssmaybe * DomU should always go down this path.
30586f235fc0Ssmaybe */
30596f235fc0Ssmaybe is_domu = !DOMAIN_IS_INITDOMAIN(xen_info);
30606f235fc0Ssmaybe anyaddr = lo_mfn == 0 && hi_mfn >= max_mfn && !pfnalign;
30616f235fc0Ssmaybe if ((!contig && anyaddr) || is_domu) {
30626f235fc0Ssmaybe flags &= ~PG_PHYSCONTIG;
30636f235fc0Ssmaybe plist = page_create_va(vp, off, bytes, flags, &kvseg, vaddr);
30646f235fc0Ssmaybe if (plist != NULL)
30656f235fc0Ssmaybe return (plist);
30666f235fc0Ssmaybe else if (is_domu)
30676f235fc0Ssmaybe return (NULL); /* no memory available */
30686f235fc0Ssmaybe }
30696f235fc0Ssmaybe /*
30706f235fc0Ssmaybe * DomU should never reach here
30716f235fc0Ssmaybe */
30726f235fc0Ssmaybe if (contig) {
30736f235fc0Ssmaybe plist = page_get_contigpages(vp, off, &npages, flags, vaddr,
30746f235fc0Ssmaybe mattr);
30756f235fc0Ssmaybe if (plist == NULL)
30766f235fc0Ssmaybe goto fail;
30776f235fc0Ssmaybe bytes_got = (pages_req - npages) << MMU_PAGESHIFT;
30786f235fc0Ssmaybe vaddr += bytes_got;
30796f235fc0Ssmaybe off += bytes_got;
30806f235fc0Ssmaybe /*
30816f235fc0Ssmaybe * We now have all the contiguous pages we need, but
30826f235fc0Ssmaybe * we may still need additional non-contiguous pages.
30836f235fc0Ssmaybe */
30846f235fc0Ssmaybe }
30856f235fc0Ssmaybe /*
30866f235fc0Ssmaybe * now loop collecting the requested number of pages, these do
30876f235fc0Ssmaybe * not have to be contiguous pages but we will use the contig
30886f235fc0Ssmaybe * page alloc code to get the pages since it will honor any
30896f235fc0Ssmaybe * other constraints the pages may have.
30906f235fc0Ssmaybe */
30916f235fc0Ssmaybe while (npages--) {
3092349b53ddSStuart Maybee dummy = -1;
30936f235fc0Ssmaybe pp = page_get_contigpages(vp, off, &dummy, flags, vaddr, mattr);
30946f235fc0Ssmaybe if (pp == NULL)
30956f235fc0Ssmaybe goto fail;
30966f235fc0Ssmaybe page_add(&plist, pp);
30976f235fc0Ssmaybe vaddr += MMU_PAGESIZE;
30986f235fc0Ssmaybe off += MMU_PAGESIZE;
30996f235fc0Ssmaybe }
31006f235fc0Ssmaybe return (plist);
31016f235fc0Ssmaybe fail:
31026f235fc0Ssmaybe /*
31036f235fc0Ssmaybe * Failed to get enough pages, return ones we did get
31046f235fc0Ssmaybe */
31056f235fc0Ssmaybe return_partial_alloc(plist);
3106843e1988Sjohnlev return (NULL);
3107843e1988Sjohnlev }
3108843e1988Sjohnlev
3109843e1988Sjohnlev /*
3110843e1988Sjohnlev * Lock and return the page with the highest mfn that we can find. last_mfn
3111843e1988Sjohnlev * holds the last one found, so the next search can start from there. We
3112843e1988Sjohnlev * also keep a counter so that we don't loop forever if the machine has no
3113843e1988Sjohnlev * free pages.
3114843e1988Sjohnlev *
3115843e1988Sjohnlev * This is called from the balloon thread to find pages to give away. new_high
3116843e1988Sjohnlev * is used when new mfn's have been added to the system - we will reset our
3117843e1988Sjohnlev * search if the new mfn's are higher than our current search position.
3118843e1988Sjohnlev */
3119843e1988Sjohnlev page_t *
page_get_high_mfn(mfn_t new_high)3120843e1988Sjohnlev page_get_high_mfn(mfn_t new_high)
3121843e1988Sjohnlev {
3122843e1988Sjohnlev static mfn_t last_mfn = 0;
3123843e1988Sjohnlev pfn_t pfn;
3124843e1988Sjohnlev page_t *pp;
3125843e1988Sjohnlev ulong_t loop_count = 0;
3126843e1988Sjohnlev
3127843e1988Sjohnlev if (new_high > last_mfn)
3128843e1988Sjohnlev last_mfn = new_high;
3129843e1988Sjohnlev
3130843e1988Sjohnlev for (; loop_count < mfn_count; loop_count++, last_mfn--) {
3131843e1988Sjohnlev if (last_mfn == 0) {
3132843e1988Sjohnlev last_mfn = cached_max_mfn;
3133843e1988Sjohnlev }
3134843e1988Sjohnlev
3135843e1988Sjohnlev pfn = mfn_to_pfn(last_mfn);
3136843e1988Sjohnlev if (pfn & PFN_IS_FOREIGN_MFN)
3137843e1988Sjohnlev continue;
3138843e1988Sjohnlev
3139843e1988Sjohnlev /* See if the page is free. If so, lock it. */
3140843e1988Sjohnlev pp = page_numtopp_alloc(pfn);
3141843e1988Sjohnlev if (pp == NULL)
3142843e1988Sjohnlev continue;
3143843e1988Sjohnlev PP_CLRFREE(pp);
3144843e1988Sjohnlev
3145843e1988Sjohnlev ASSERT(PAGE_EXCL(pp));
3146843e1988Sjohnlev ASSERT(pp->p_vnode == NULL);
3147843e1988Sjohnlev ASSERT(!hat_page_is_mapped(pp));
3148843e1988Sjohnlev last_mfn--;
3149843e1988Sjohnlev return (pp);
3150843e1988Sjohnlev }
3151843e1988Sjohnlev return (NULL);
3152843e1988Sjohnlev }
3153843e1988Sjohnlev
3154843e1988Sjohnlev #else /* !__xpv */
3155843e1988Sjohnlev
3156843e1988Sjohnlev /*
3157843e1988Sjohnlev * get a page from any list with the given mnode
3158843e1988Sjohnlev */
3159843e1988Sjohnlev static page_t *
page_get_mnode_anylist(ulong_t origbin,uchar_t szc,uint_t flags,int mnode,int mtype,ddi_dma_attr_t * dma_attr)31607c478bd9Sstevel@tonic-gate page_get_mnode_anylist(ulong_t origbin, uchar_t szc, uint_t flags,
31617c478bd9Sstevel@tonic-gate int mnode, int mtype, ddi_dma_attr_t *dma_attr)
31627c478bd9Sstevel@tonic-gate {
31635d07b933Sdp kmutex_t *pcm;
31645d07b933Sdp int i;
31655d07b933Sdp page_t *pp;
31665d07b933Sdp page_t *first_pp;
31675d07b933Sdp uint64_t pgaddr;
31685d07b933Sdp ulong_t bin;
31695d07b933Sdp int mtypestart;
31705d07b933Sdp int plw_initialized;
31715d07b933Sdp page_list_walker_t plw;
31727c478bd9Sstevel@tonic-gate
31737c478bd9Sstevel@tonic-gate VM_STAT_ADD(pga_vmstats.pgma_alloc);
31747c478bd9Sstevel@tonic-gate
31757c478bd9Sstevel@tonic-gate ASSERT((flags & PG_MATCH_COLOR) == 0);
31767c478bd9Sstevel@tonic-gate ASSERT(szc == 0);
31777c478bd9Sstevel@tonic-gate ASSERT(dma_attr != NULL);
31787c478bd9Sstevel@tonic-gate
31797c478bd9Sstevel@tonic-gate MTYPE_START(mnode, mtype, flags);
31807c478bd9Sstevel@tonic-gate if (mtype < 0) {
31817c478bd9Sstevel@tonic-gate VM_STAT_ADD(pga_vmstats.pgma_allocempty);
31827c478bd9Sstevel@tonic-gate return (NULL);
31837c478bd9Sstevel@tonic-gate }
31847c478bd9Sstevel@tonic-gate
31857c478bd9Sstevel@tonic-gate mtypestart = mtype;
31867c478bd9Sstevel@tonic-gate
31877c478bd9Sstevel@tonic-gate bin = origbin;
31887c478bd9Sstevel@tonic-gate
31897c478bd9Sstevel@tonic-gate /*
31907c478bd9Sstevel@tonic-gate * check up to page_colors + 1 bins - origbin may be checked twice
31917c478bd9Sstevel@tonic-gate * because of BIN_STEP skip
31927c478bd9Sstevel@tonic-gate */
31937c478bd9Sstevel@tonic-gate do {
31945d07b933Sdp plw_initialized = 0;
31955d07b933Sdp
31965d07b933Sdp for (plw.plw_count = 0;
31975d07b933Sdp plw.plw_count < page_colors; plw.plw_count++) {
31985d07b933Sdp
3199d94ffb28Sjmcp if (PAGE_FREELISTS(mnode, szc, bin, mtype) == NULL)
32007c478bd9Sstevel@tonic-gate goto nextfreebin;
32017c478bd9Sstevel@tonic-gate
3202d94ffb28Sjmcp pcm = PC_BIN_MUTEX(mnode, bin, PG_FREE_LIST);
32037c478bd9Sstevel@tonic-gate mutex_enter(pcm);
3204d94ffb28Sjmcp pp = PAGE_FREELISTS(mnode, szc, bin, mtype);
32057c478bd9Sstevel@tonic-gate first_pp = pp;
32067c478bd9Sstevel@tonic-gate while (pp != NULL) {
32071f84c0d7SDave Plauger if (IS_DUMP_PAGE(pp) || page_trylock(pp,
32081f84c0d7SDave Plauger SE_EXCL) == 0) {
32097c478bd9Sstevel@tonic-gate pp = pp->p_next;
32107c478bd9Sstevel@tonic-gate if (pp == first_pp) {
32117c478bd9Sstevel@tonic-gate pp = NULL;
32127c478bd9Sstevel@tonic-gate }
32137c478bd9Sstevel@tonic-gate continue;
32147c478bd9Sstevel@tonic-gate }
32157c478bd9Sstevel@tonic-gate
32167c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp));
32177c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp));
32187c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode == NULL);
32197c478bd9Sstevel@tonic-gate ASSERT(pp->p_hash == NULL);
32207c478bd9Sstevel@tonic-gate ASSERT(pp->p_offset == (u_offset_t)-1);
32217c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == szc);
32227c478bd9Sstevel@tonic-gate ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode);
32237c478bd9Sstevel@tonic-gate /* check if page within DMA attributes */
3224ae115bc7Smrj pgaddr = pa_to_ma(pfn_to_pa(pp->p_pagenum));
32257c478bd9Sstevel@tonic-gate if ((pgaddr >= dma_attr->dma_attr_addr_lo) &&
32267c478bd9Sstevel@tonic-gate (pgaddr + MMU_PAGESIZE - 1 <=
32277c478bd9Sstevel@tonic-gate dma_attr->dma_attr_addr_hi)) {
32287c478bd9Sstevel@tonic-gate break;
32297c478bd9Sstevel@tonic-gate }
32307c478bd9Sstevel@tonic-gate
32317c478bd9Sstevel@tonic-gate /* continue looking */
32327c478bd9Sstevel@tonic-gate page_unlock(pp);
32337c478bd9Sstevel@tonic-gate pp = pp->p_next;
32347c478bd9Sstevel@tonic-gate if (pp == first_pp)
32357c478bd9Sstevel@tonic-gate pp = NULL;
32367c478bd9Sstevel@tonic-gate
32377c478bd9Sstevel@tonic-gate }
32387c478bd9Sstevel@tonic-gate if (pp != NULL) {
32397c478bd9Sstevel@tonic-gate ASSERT(mtype == PP_2_MTYPE(pp));
32407c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0);
32417c478bd9Sstevel@tonic-gate
32427c478bd9Sstevel@tonic-gate /* found a page with specified DMA attributes */
3243d94ffb28Sjmcp page_sub(&PAGE_FREELISTS(mnode, szc, bin,
3244d94ffb28Sjmcp mtype), pp);
3245affbd3ccSkchow page_ctr_sub(mnode, mtype, pp, PG_FREE_LIST);
32467c478bd9Sstevel@tonic-gate
32477c478bd9Sstevel@tonic-gate if ((PP_ISFREE(pp) == 0) ||
32487c478bd9Sstevel@tonic-gate (PP_ISAGED(pp) == 0)) {
32497c478bd9Sstevel@tonic-gate cmn_err(CE_PANIC, "page %p is not free",
32507c478bd9Sstevel@tonic-gate (void *)pp);
32517c478bd9Sstevel@tonic-gate }
32527c478bd9Sstevel@tonic-gate
32537c478bd9Sstevel@tonic-gate mutex_exit(pcm);
32547c478bd9Sstevel@tonic-gate check_dma(dma_attr, pp, 1);
32557c478bd9Sstevel@tonic-gate VM_STAT_ADD(pga_vmstats.pgma_allocok);
32567c478bd9Sstevel@tonic-gate return (pp);
32577c478bd9Sstevel@tonic-gate }
32587c478bd9Sstevel@tonic-gate mutex_exit(pcm);
32597c478bd9Sstevel@tonic-gate nextfreebin:
32605d07b933Sdp if (plw_initialized == 0) {
32615d07b933Sdp page_list_walk_init(szc, 0, bin, 1, 0, &plw);
32625d07b933Sdp ASSERT(plw.plw_ceq_dif == page_colors);
32635d07b933Sdp plw_initialized = 1;
32645d07b933Sdp }
32657c478bd9Sstevel@tonic-gate
32665d07b933Sdp if (plw.plw_do_split) {
32675d07b933Sdp pp = page_freelist_split(szc, bin, mnode,
32685d07b933Sdp mtype,
326919397407SSherry Moore mmu_btop(dma_attr->dma_attr_addr_lo),
32705d07b933Sdp mmu_btop(dma_attr->dma_attr_addr_hi + 1),
32715d07b933Sdp &plw);
327219397407SSherry Moore if (pp != NULL) {
327319397407SSherry Moore check_dma(dma_attr, pp, 1);
32745d07b933Sdp return (pp);
327519397407SSherry Moore }
32765d07b933Sdp }
32775d07b933Sdp
32785d07b933Sdp bin = page_list_walk_next_bin(szc, bin, &plw);
32797c478bd9Sstevel@tonic-gate }
32805d07b933Sdp
3281affbd3ccSkchow MTYPE_NEXT(mnode, mtype, flags);
3282affbd3ccSkchow } while (mtype >= 0);
32837c478bd9Sstevel@tonic-gate
32847c478bd9Sstevel@tonic-gate /* failed to find a page in the freelist; try it in the cachelist */
32857c478bd9Sstevel@tonic-gate
32867c478bd9Sstevel@tonic-gate /* reset mtype start for cachelist search */
32877c478bd9Sstevel@tonic-gate mtype = mtypestart;
32887c478bd9Sstevel@tonic-gate ASSERT(mtype >= 0);
32897c478bd9Sstevel@tonic-gate
32907c478bd9Sstevel@tonic-gate /* start with the bin of matching color */
32917c478bd9Sstevel@tonic-gate bin = origbin;
32927c478bd9Sstevel@tonic-gate
32937c478bd9Sstevel@tonic-gate do {
32947c478bd9Sstevel@tonic-gate for (i = 0; i <= page_colors; i++) {
32957c478bd9Sstevel@tonic-gate if (PAGE_CACHELISTS(mnode, bin, mtype) == NULL)
32967c478bd9Sstevel@tonic-gate goto nextcachebin;
3297d94ffb28Sjmcp pcm = PC_BIN_MUTEX(mnode, bin, PG_CACHE_LIST);
32987c478bd9Sstevel@tonic-gate mutex_enter(pcm);
32997c478bd9Sstevel@tonic-gate pp = PAGE_CACHELISTS(mnode, bin, mtype);
33007c478bd9Sstevel@tonic-gate first_pp = pp;
33017c478bd9Sstevel@tonic-gate while (pp != NULL) {
33021f84c0d7SDave Plauger if (IS_DUMP_PAGE(pp) || page_trylock(pp,
33031f84c0d7SDave Plauger SE_EXCL) == 0) {
33047c478bd9Sstevel@tonic-gate pp = pp->p_next;
33057c478bd9Sstevel@tonic-gate if (pp == first_pp)
3306e172a44eSSherry Moore pp = NULL;
33077c478bd9Sstevel@tonic-gate continue;
33087c478bd9Sstevel@tonic-gate }
33097c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode);
33107c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp) == 0);
33117c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0);
33127c478bd9Sstevel@tonic-gate ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode);
33137c478bd9Sstevel@tonic-gate
33147c478bd9Sstevel@tonic-gate /* check if page within DMA attributes */
33157c478bd9Sstevel@tonic-gate
3316ae115bc7Smrj pgaddr = pa_to_ma(pfn_to_pa(pp->p_pagenum));
33177c478bd9Sstevel@tonic-gate if ((pgaddr >= dma_attr->dma_attr_addr_lo) &&
33187c478bd9Sstevel@tonic-gate (pgaddr + MMU_PAGESIZE - 1 <=
33197c478bd9Sstevel@tonic-gate dma_attr->dma_attr_addr_hi)) {
33207c478bd9Sstevel@tonic-gate break;
33217c478bd9Sstevel@tonic-gate }
33227c478bd9Sstevel@tonic-gate
33237c478bd9Sstevel@tonic-gate /* continue looking */
33247c478bd9Sstevel@tonic-gate page_unlock(pp);
33257c478bd9Sstevel@tonic-gate pp = pp->p_next;
33267c478bd9Sstevel@tonic-gate if (pp == first_pp)
33277c478bd9Sstevel@tonic-gate pp = NULL;
33287c478bd9Sstevel@tonic-gate }
33297c478bd9Sstevel@tonic-gate
33307c478bd9Sstevel@tonic-gate if (pp != NULL) {
33317c478bd9Sstevel@tonic-gate ASSERT(mtype == PP_2_MTYPE(pp));
33327c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0);
33337c478bd9Sstevel@tonic-gate
33347c478bd9Sstevel@tonic-gate /* found a page with specified DMA attributes */
33357c478bd9Sstevel@tonic-gate page_sub(&PAGE_CACHELISTS(mnode, bin,
33367c478bd9Sstevel@tonic-gate mtype), pp);
3337affbd3ccSkchow page_ctr_sub(mnode, mtype, pp, PG_CACHE_LIST);
33387c478bd9Sstevel@tonic-gate
33397c478bd9Sstevel@tonic-gate mutex_exit(pcm);
33407c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode);
33417c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp) == 0);
33427c478bd9Sstevel@tonic-gate check_dma(dma_attr, pp, 1);
33437c478bd9Sstevel@tonic-gate VM_STAT_ADD(pga_vmstats.pgma_allocok);
33447c478bd9Sstevel@tonic-gate return (pp);
33457c478bd9Sstevel@tonic-gate }
33467c478bd9Sstevel@tonic-gate mutex_exit(pcm);
33477c478bd9Sstevel@tonic-gate nextcachebin:
33487c478bd9Sstevel@tonic-gate bin += (i == 0) ? BIN_STEP : 1;
33497c478bd9Sstevel@tonic-gate bin &= page_colors_mask;
33507c478bd9Sstevel@tonic-gate }
3351affbd3ccSkchow MTYPE_NEXT(mnode, mtype, flags);
3352affbd3ccSkchow } while (mtype >= 0);
33537c478bd9Sstevel@tonic-gate
33547c478bd9Sstevel@tonic-gate VM_STAT_ADD(pga_vmstats.pgma_allocfailed);
33557c478bd9Sstevel@tonic-gate return (NULL);
33567c478bd9Sstevel@tonic-gate }
33577c478bd9Sstevel@tonic-gate
33587c478bd9Sstevel@tonic-gate /*
33597c478bd9Sstevel@tonic-gate * This function is similar to page_get_freelist()/page_get_cachelist()
33607c478bd9Sstevel@tonic-gate * but it searches both the lists to find a page with the specified
33617c478bd9Sstevel@tonic-gate * color (or no color) and DMA attributes. The search is done in the
33627c478bd9Sstevel@tonic-gate * freelist first and then in the cache list within the highest memory
33637c478bd9Sstevel@tonic-gate * range (based on DMA attributes) before searching in the lower
33647c478bd9Sstevel@tonic-gate * memory ranges.
33657c478bd9Sstevel@tonic-gate *
33667c478bd9Sstevel@tonic-gate * Note: This function is called only by page_create_io().
33677c478bd9Sstevel@tonic-gate */
33687c478bd9Sstevel@tonic-gate /*ARGSUSED*/
3369843e1988Sjohnlev static page_t *
page_get_anylist(struct vnode * vp,u_offset_t off,struct as * as,caddr_t vaddr,size_t size,uint_t flags,ddi_dma_attr_t * dma_attr,lgrp_t * lgrp)33707c478bd9Sstevel@tonic-gate page_get_anylist(struct vnode *vp, u_offset_t off, struct as *as, caddr_t vaddr,
33717c478bd9Sstevel@tonic-gate size_t size, uint_t flags, ddi_dma_attr_t *dma_attr, lgrp_t *lgrp)
33727c478bd9Sstevel@tonic-gate {
33737c478bd9Sstevel@tonic-gate uint_t bin;
33747c478bd9Sstevel@tonic-gate int mtype;
33757c478bd9Sstevel@tonic-gate page_t *pp;
33767c478bd9Sstevel@tonic-gate int n;
33777c478bd9Sstevel@tonic-gate int m;
33787c478bd9Sstevel@tonic-gate int szc;
33797c478bd9Sstevel@tonic-gate int fullrange;
33807c478bd9Sstevel@tonic-gate int mnode;
33817c478bd9Sstevel@tonic-gate int local_failed_stat = 0;
33827c478bd9Sstevel@tonic-gate lgrp_mnode_cookie_t lgrp_cookie;
33837c478bd9Sstevel@tonic-gate
33847c478bd9Sstevel@tonic-gate VM_STAT_ADD(pga_vmstats.pga_alloc);
33857c478bd9Sstevel@tonic-gate
33867c478bd9Sstevel@tonic-gate /* only base pagesize currently supported */
33877c478bd9Sstevel@tonic-gate if (size != MMU_PAGESIZE)
33887c478bd9Sstevel@tonic-gate return (NULL);
33897c478bd9Sstevel@tonic-gate
33907c478bd9Sstevel@tonic-gate /*
33917c478bd9Sstevel@tonic-gate * If we're passed a specific lgroup, we use it. Otherwise,
33927c478bd9Sstevel@tonic-gate * assume first-touch placement is desired.
33937c478bd9Sstevel@tonic-gate */
33947c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp))
33957c478bd9Sstevel@tonic-gate lgrp = lgrp_home_lgrp();
33967c478bd9Sstevel@tonic-gate
33977c478bd9Sstevel@tonic-gate /* LINTED */
3398d94ffb28Sjmcp AS_2_BIN(as, seg, vp, vaddr, bin, 0);
33997c478bd9Sstevel@tonic-gate
34007c478bd9Sstevel@tonic-gate /*
34017c478bd9Sstevel@tonic-gate * Only hold one freelist or cachelist lock at a time, that way we
34027c478bd9Sstevel@tonic-gate * can start anywhere and not have to worry about lock
34037c478bd9Sstevel@tonic-gate * ordering.
34047c478bd9Sstevel@tonic-gate */
34057c478bd9Sstevel@tonic-gate if (dma_attr == NULL) {
3406a3114836SGerry Liu n = mtype16m;
3407a3114836SGerry Liu m = mtypetop;
34087c478bd9Sstevel@tonic-gate fullrange = 1;
34097c478bd9Sstevel@tonic-gate VM_STAT_ADD(pga_vmstats.pga_nulldmaattr);
34107c478bd9Sstevel@tonic-gate } else {
34117c478bd9Sstevel@tonic-gate pfn_t pfnlo = mmu_btop(dma_attr->dma_attr_addr_lo);
34127c478bd9Sstevel@tonic-gate pfn_t pfnhi = mmu_btop(dma_attr->dma_attr_addr_hi);
34137c478bd9Sstevel@tonic-gate
34147c478bd9Sstevel@tonic-gate /*
34157c478bd9Sstevel@tonic-gate * We can guarantee alignment only for page boundary.
34167c478bd9Sstevel@tonic-gate */
34177c478bd9Sstevel@tonic-gate if (dma_attr->dma_attr_align > MMU_PAGESIZE)
34187c478bd9Sstevel@tonic-gate return (NULL);
34197c478bd9Sstevel@tonic-gate
3420a3114836SGerry Liu /* Sanity check the dma_attr */
3421a3114836SGerry Liu if (pfnlo > pfnhi)
3422a3114836SGerry Liu return (NULL);
3423a3114836SGerry Liu
34247c478bd9Sstevel@tonic-gate n = pfn_2_mtype(pfnlo);
34257c478bd9Sstevel@tonic-gate m = pfn_2_mtype(pfnhi);
34267c478bd9Sstevel@tonic-gate
34277c478bd9Sstevel@tonic-gate fullrange = ((pfnlo == mnoderanges[n].mnr_pfnlo) &&
34287c478bd9Sstevel@tonic-gate (pfnhi >= mnoderanges[m].mnr_pfnhi));
34297c478bd9Sstevel@tonic-gate }
34307c478bd9Sstevel@tonic-gate VM_STAT_COND_ADD(fullrange == 0, pga_vmstats.pga_notfullrange);
34317c478bd9Sstevel@tonic-gate
34327c478bd9Sstevel@tonic-gate szc = 0;
34337c478bd9Sstevel@tonic-gate
3434a3114836SGerry Liu /* cylcing thru mtype handled by RANGE0 if n == mtype16m */
3435a3114836SGerry Liu if (n == mtype16m) {
34367c478bd9Sstevel@tonic-gate flags |= PGI_MT_RANGE0;
34377c478bd9Sstevel@tonic-gate n = m;
34387c478bd9Sstevel@tonic-gate }
34397c478bd9Sstevel@tonic-gate
34407c478bd9Sstevel@tonic-gate /*
34417c478bd9Sstevel@tonic-gate * Try local memory node first, but try remote if we can't
34427c478bd9Sstevel@tonic-gate * get a page of the right color.
34437c478bd9Sstevel@tonic-gate */
34447c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, LGRP_SRCH_HIER);
34457c478bd9Sstevel@tonic-gate while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) {
34467c478bd9Sstevel@tonic-gate /*
34477c478bd9Sstevel@tonic-gate * allocate pages from high pfn to low.
34487c478bd9Sstevel@tonic-gate */
3449a3114836SGerry Liu mtype = m;
3450a3114836SGerry Liu do {
34517c478bd9Sstevel@tonic-gate if (fullrange != 0) {
3452d94ffb28Sjmcp pp = page_get_mnode_freelist(mnode,
34537c478bd9Sstevel@tonic-gate bin, mtype, szc, flags);
34547c478bd9Sstevel@tonic-gate if (pp == NULL) {
34557c478bd9Sstevel@tonic-gate pp = page_get_mnode_cachelist(
3456843e1988Sjohnlev bin, flags, mnode, mtype);
34577c478bd9Sstevel@tonic-gate }
34587c478bd9Sstevel@tonic-gate } else {
34597c478bd9Sstevel@tonic-gate pp = page_get_mnode_anylist(bin, szc,
34607c478bd9Sstevel@tonic-gate flags, mnode, mtype, dma_attr);
34617c478bd9Sstevel@tonic-gate }
34627c478bd9Sstevel@tonic-gate if (pp != NULL) {
34637c478bd9Sstevel@tonic-gate VM_STAT_ADD(pga_vmstats.pga_allocok);
34647c478bd9Sstevel@tonic-gate check_dma(dma_attr, pp, 1);
34657c478bd9Sstevel@tonic-gate return (pp);
34667c478bd9Sstevel@tonic-gate }
3467a3114836SGerry Liu } while (mtype != n &&
3468a3114836SGerry Liu (mtype = mnoderanges[mtype].mnr_next) != -1);
34697c478bd9Sstevel@tonic-gate if (!local_failed_stat) {
34707c478bd9Sstevel@tonic-gate lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_ALLOC_FAIL, 1);
34717c478bd9Sstevel@tonic-gate local_failed_stat = 1;
34727c478bd9Sstevel@tonic-gate }
34737c478bd9Sstevel@tonic-gate }
34747c478bd9Sstevel@tonic-gate VM_STAT_ADD(pga_vmstats.pga_allocfailed);
34757c478bd9Sstevel@tonic-gate
34767c478bd9Sstevel@tonic-gate return (NULL);
34777c478bd9Sstevel@tonic-gate }
34787c478bd9Sstevel@tonic-gate
34797c478bd9Sstevel@tonic-gate /*
34807c478bd9Sstevel@tonic-gate * page_create_io()
34817c478bd9Sstevel@tonic-gate *
34827c478bd9Sstevel@tonic-gate * This function is a copy of page_create_va() with an additional
34837c478bd9Sstevel@tonic-gate * argument 'mattr' that specifies DMA memory requirements to
34847c478bd9Sstevel@tonic-gate * the page list functions. This function is used by the segkmem
34857c478bd9Sstevel@tonic-gate * allocator so it is only to create new pages (i.e PG_EXCL is
34867c478bd9Sstevel@tonic-gate * set).
34877c478bd9Sstevel@tonic-gate *
34887c478bd9Sstevel@tonic-gate * Note: This interface is currently used by x86 PSM only and is
34897c478bd9Sstevel@tonic-gate * not fully specified so the commitment level is only for
34907c478bd9Sstevel@tonic-gate * private interface specific to x86. This interface uses PSM
34917c478bd9Sstevel@tonic-gate * specific page_get_anylist() interface.
34927c478bd9Sstevel@tonic-gate */
34937c478bd9Sstevel@tonic-gate
34947c478bd9Sstevel@tonic-gate #define PAGE_HASH_SEARCH(index, pp, vp, off) { \
34957c478bd9Sstevel@tonic-gate for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash) { \
34967c478bd9Sstevel@tonic-gate if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \
34977c478bd9Sstevel@tonic-gate break; \
34987c478bd9Sstevel@tonic-gate } \
34997c478bd9Sstevel@tonic-gate }
35007c478bd9Sstevel@tonic-gate
35017c478bd9Sstevel@tonic-gate
35027c478bd9Sstevel@tonic-gate page_t *
page_create_io(struct vnode * vp,u_offset_t off,uint_t bytes,uint_t flags,struct as * as,caddr_t vaddr,ddi_dma_attr_t * mattr)35037c478bd9Sstevel@tonic-gate page_create_io(
35047c478bd9Sstevel@tonic-gate struct vnode *vp,
35057c478bd9Sstevel@tonic-gate u_offset_t off,
35067c478bd9Sstevel@tonic-gate uint_t bytes,
35077c478bd9Sstevel@tonic-gate uint_t flags,
35087c478bd9Sstevel@tonic-gate struct as *as,
35097c478bd9Sstevel@tonic-gate caddr_t vaddr,
35107c478bd9Sstevel@tonic-gate ddi_dma_attr_t *mattr) /* DMA memory attributes if any */
35117c478bd9Sstevel@tonic-gate {
35127c478bd9Sstevel@tonic-gate page_t *plist = NULL;
35137c478bd9Sstevel@tonic-gate uint_t plist_len = 0;
35147c478bd9Sstevel@tonic-gate pgcnt_t npages;
35157c478bd9Sstevel@tonic-gate page_t *npp = NULL;
35167c478bd9Sstevel@tonic-gate uint_t pages_req;
35177c478bd9Sstevel@tonic-gate page_t *pp;
35187c478bd9Sstevel@tonic-gate kmutex_t *phm = NULL;
35197c478bd9Sstevel@tonic-gate uint_t index;
35207c478bd9Sstevel@tonic-gate
35217c478bd9Sstevel@tonic-gate TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START,
3522843e1988Sjohnlev "page_create_start:vp %p off %llx bytes %u flags %x",
3523843e1988Sjohnlev vp, off, bytes, flags);
35247c478bd9Sstevel@tonic-gate
35257c478bd9Sstevel@tonic-gate ASSERT((flags & ~(PG_EXCL | PG_WAIT | PG_PHYSCONTIG)) == 0);
35267c478bd9Sstevel@tonic-gate
35277c478bd9Sstevel@tonic-gate pages_req = npages = mmu_btopr(bytes);
35287c478bd9Sstevel@tonic-gate
35297c478bd9Sstevel@tonic-gate /*
35307c478bd9Sstevel@tonic-gate * Do the freemem and pcf accounting.
35317c478bd9Sstevel@tonic-gate */
35327c478bd9Sstevel@tonic-gate if (!page_create_wait(npages, flags)) {
35337c478bd9Sstevel@tonic-gate return (NULL);
35347c478bd9Sstevel@tonic-gate }
35357c478bd9Sstevel@tonic-gate
35367c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS,
3537843e1988Sjohnlev "page_create_success:vp %p off %llx", vp, off);
35387c478bd9Sstevel@tonic-gate
35397c478bd9Sstevel@tonic-gate /*
35407c478bd9Sstevel@tonic-gate * If satisfying this request has left us with too little
35417c478bd9Sstevel@tonic-gate * memory, start the wheels turning to get some back. The
35427c478bd9Sstevel@tonic-gate * first clause of the test prevents waking up the pageout
35437c478bd9Sstevel@tonic-gate * daemon in situations where it would decide that there's
35447c478bd9Sstevel@tonic-gate * nothing to do.
35457c478bd9Sstevel@tonic-gate */
35467c478bd9Sstevel@tonic-gate if (nscan < desscan && freemem < minfree) {
35477c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL,
3548843e1988Sjohnlev "pageout_cv_signal:freemem %ld", freemem);
3549*338664dfSAndy Fiddaman WAKE_PAGEOUT_SCANNER(page__create__io);
35507c478bd9Sstevel@tonic-gate }
35517c478bd9Sstevel@tonic-gate
35527c478bd9Sstevel@tonic-gate if (flags & PG_PHYSCONTIG) {
35537c478bd9Sstevel@tonic-gate
35547c478bd9Sstevel@tonic-gate plist = page_get_contigpage(&npages, mattr, 1);
35557c478bd9Sstevel@tonic-gate if (plist == NULL) {
35567c478bd9Sstevel@tonic-gate page_create_putback(npages);
35577c478bd9Sstevel@tonic-gate return (NULL);
35587c478bd9Sstevel@tonic-gate }
35597c478bd9Sstevel@tonic-gate
35607c478bd9Sstevel@tonic-gate pp = plist;
35617c478bd9Sstevel@tonic-gate
35627c478bd9Sstevel@tonic-gate do {
35637c478bd9Sstevel@tonic-gate if (!page_hashin(pp, vp, off, NULL)) {
35647c478bd9Sstevel@tonic-gate panic("pg_creat_io: hashin failed %p %p %llx",
35657c478bd9Sstevel@tonic-gate (void *)pp, (void *)vp, off);
35667c478bd9Sstevel@tonic-gate }
35677c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_new);
35687c478bd9Sstevel@tonic-gate off += MMU_PAGESIZE;
35697c478bd9Sstevel@tonic-gate PP_CLRFREE(pp);
35707c478bd9Sstevel@tonic-gate PP_CLRAGED(pp);
35717c478bd9Sstevel@tonic-gate page_set_props(pp, P_REF);
35727c478bd9Sstevel@tonic-gate pp = pp->p_next;
35737c478bd9Sstevel@tonic-gate } while (pp != plist);
35747c478bd9Sstevel@tonic-gate
35757c478bd9Sstevel@tonic-gate if (!npages) {
35767c478bd9Sstevel@tonic-gate check_dma(mattr, plist, pages_req);
35777c478bd9Sstevel@tonic-gate return (plist);
35787c478bd9Sstevel@tonic-gate } else {
35797c478bd9Sstevel@tonic-gate vaddr += (pages_req - npages) << MMU_PAGESHIFT;
35807c478bd9Sstevel@tonic-gate }
35817c478bd9Sstevel@tonic-gate
35827c478bd9Sstevel@tonic-gate /*
35837c478bd9Sstevel@tonic-gate * fall-thru:
35847c478bd9Sstevel@tonic-gate *
35857c478bd9Sstevel@tonic-gate * page_get_contigpage returns when npages <= sgllen.
35867c478bd9Sstevel@tonic-gate * Grab the rest of the non-contig pages below from anylist.
35877c478bd9Sstevel@tonic-gate */
35887c478bd9Sstevel@tonic-gate }
35897c478bd9Sstevel@tonic-gate
35907c478bd9Sstevel@tonic-gate /*
35917c478bd9Sstevel@tonic-gate * Loop around collecting the requested number of pages.
35927c478bd9Sstevel@tonic-gate * Most of the time, we have to `create' a new page. With
35937c478bd9Sstevel@tonic-gate * this in mind, pull the page off the free list before
35947c478bd9Sstevel@tonic-gate * getting the hash lock. This will minimize the hash
35957c478bd9Sstevel@tonic-gate * lock hold time, nesting, and the like. If it turns
35967c478bd9Sstevel@tonic-gate * out we don't need the page, we put it back at the end.
35977c478bd9Sstevel@tonic-gate */
35987c478bd9Sstevel@tonic-gate while (npages--) {
35997c478bd9Sstevel@tonic-gate phm = NULL;
36007c478bd9Sstevel@tonic-gate
36017c478bd9Sstevel@tonic-gate index = PAGE_HASH_FUNC(vp, off);
36027c478bd9Sstevel@tonic-gate top:
36037c478bd9Sstevel@tonic-gate ASSERT(phm == NULL);
36047c478bd9Sstevel@tonic-gate ASSERT(index == PAGE_HASH_FUNC(vp, off));
36057c478bd9Sstevel@tonic-gate ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
36067c478bd9Sstevel@tonic-gate
36077c478bd9Sstevel@tonic-gate if (npp == NULL) {
36087c478bd9Sstevel@tonic-gate /*
36097c478bd9Sstevel@tonic-gate * Try to get the page of any color either from
36107c478bd9Sstevel@tonic-gate * the freelist or from the cache list.
36117c478bd9Sstevel@tonic-gate */
36127c478bd9Sstevel@tonic-gate npp = page_get_anylist(vp, off, as, vaddr, MMU_PAGESIZE,
36137c478bd9Sstevel@tonic-gate flags & ~PG_MATCH_COLOR, mattr, NULL);
36147c478bd9Sstevel@tonic-gate if (npp == NULL) {
36157c478bd9Sstevel@tonic-gate if (mattr == NULL) {
36167c478bd9Sstevel@tonic-gate /*
36177c478bd9Sstevel@tonic-gate * Not looking for a special page;
36187c478bd9Sstevel@tonic-gate * panic!
36197c478bd9Sstevel@tonic-gate */
36207c478bd9Sstevel@tonic-gate panic("no page found %d", (int)npages);
36217c478bd9Sstevel@tonic-gate }
36227c478bd9Sstevel@tonic-gate /*
36237c478bd9Sstevel@tonic-gate * No page found! This can happen
36247c478bd9Sstevel@tonic-gate * if we are looking for a page
36257c478bd9Sstevel@tonic-gate * within a specific memory range
36267c478bd9Sstevel@tonic-gate * for DMA purposes. If PG_WAIT is
36277c478bd9Sstevel@tonic-gate * specified then we wait for a
36287c478bd9Sstevel@tonic-gate * while and then try again. The
36297c478bd9Sstevel@tonic-gate * wait could be forever if we
36307c478bd9Sstevel@tonic-gate * don't get the page(s) we need.
36317c478bd9Sstevel@tonic-gate *
36327c478bd9Sstevel@tonic-gate * Note: XXX We really need a mechanism
36337c478bd9Sstevel@tonic-gate * to wait for pages in the desired
36347c478bd9Sstevel@tonic-gate * range. For now, we wait for any
36357c478bd9Sstevel@tonic-gate * pages and see if we can use it.
36367c478bd9Sstevel@tonic-gate */
36377c478bd9Sstevel@tonic-gate
36387c478bd9Sstevel@tonic-gate if ((mattr != NULL) && (flags & PG_WAIT)) {
36397c478bd9Sstevel@tonic-gate delay(10);
36407c478bd9Sstevel@tonic-gate goto top;
36417c478bd9Sstevel@tonic-gate }
36427c478bd9Sstevel@tonic-gate goto fail; /* undo accounting stuff */
36437c478bd9Sstevel@tonic-gate }
36447c478bd9Sstevel@tonic-gate
36457c478bd9Sstevel@tonic-gate if (PP_ISAGED(npp) == 0) {
36467c478bd9Sstevel@tonic-gate /*
36477c478bd9Sstevel@tonic-gate * Since this page came from the
36487c478bd9Sstevel@tonic-gate * cachelist, we must destroy the
36497c478bd9Sstevel@tonic-gate * old vnode association.
36507c478bd9Sstevel@tonic-gate */
36517c478bd9Sstevel@tonic-gate page_hashout(npp, (kmutex_t *)NULL);
36527c478bd9Sstevel@tonic-gate }
36537c478bd9Sstevel@tonic-gate }
36547c478bd9Sstevel@tonic-gate
36557c478bd9Sstevel@tonic-gate /*
36567c478bd9Sstevel@tonic-gate * We own this page!
36577c478bd9Sstevel@tonic-gate */
36587c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(npp));
36597c478bd9Sstevel@tonic-gate ASSERT(npp->p_vnode == NULL);
36607c478bd9Sstevel@tonic-gate ASSERT(!hat_page_is_mapped(npp));
36617c478bd9Sstevel@tonic-gate PP_CLRFREE(npp);
36627c478bd9Sstevel@tonic-gate PP_CLRAGED(npp);
36637c478bd9Sstevel@tonic-gate
36647c478bd9Sstevel@tonic-gate /*
36657c478bd9Sstevel@tonic-gate * Here we have a page in our hot little mits and are
36667c478bd9Sstevel@tonic-gate * just waiting to stuff it on the appropriate lists.
36677c478bd9Sstevel@tonic-gate * Get the mutex and check to see if it really does
36687c478bd9Sstevel@tonic-gate * not exist.
36697c478bd9Sstevel@tonic-gate */
36707c478bd9Sstevel@tonic-gate phm = PAGE_HASH_MUTEX(index);
36717c478bd9Sstevel@tonic-gate mutex_enter(phm);
36727c478bd9Sstevel@tonic-gate PAGE_HASH_SEARCH(index, pp, vp, off);
36737c478bd9Sstevel@tonic-gate if (pp == NULL) {
36747c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_new);
36757c478bd9Sstevel@tonic-gate pp = npp;
36767c478bd9Sstevel@tonic-gate npp = NULL;
36777c478bd9Sstevel@tonic-gate if (!page_hashin(pp, vp, off, phm)) {
36787c478bd9Sstevel@tonic-gate /*
36797c478bd9Sstevel@tonic-gate * Since we hold the page hash mutex and
36807c478bd9Sstevel@tonic-gate * just searched for this page, page_hashin
36817c478bd9Sstevel@tonic-gate * had better not fail. If it does, that
36827c478bd9Sstevel@tonic-gate * means somethread did not follow the
36837c478bd9Sstevel@tonic-gate * page hash mutex rules. Panic now and
36847c478bd9Sstevel@tonic-gate * get it over with. As usual, go down
36857c478bd9Sstevel@tonic-gate * holding all the locks.
36867c478bd9Sstevel@tonic-gate */
36877c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(phm));
36887c478bd9Sstevel@tonic-gate panic("page_create: hashin fail %p %p %llx %p",
36897c478bd9Sstevel@tonic-gate (void *)pp, (void *)vp, off, (void *)phm);
36907c478bd9Sstevel@tonic-gate
36917c478bd9Sstevel@tonic-gate }
36927c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(phm));
36937c478bd9Sstevel@tonic-gate mutex_exit(phm);
36947c478bd9Sstevel@tonic-gate phm = NULL;
36957c478bd9Sstevel@tonic-gate
36967c478bd9Sstevel@tonic-gate /*
36977c478bd9Sstevel@tonic-gate * Hat layer locking need not be done to set
36987c478bd9Sstevel@tonic-gate * the following bits since the page is not hashed
36997c478bd9Sstevel@tonic-gate * and was on the free list (i.e., had no mappings).
37007c478bd9Sstevel@tonic-gate *
37017c478bd9Sstevel@tonic-gate * Set the reference bit to protect
37027c478bd9Sstevel@tonic-gate * against immediate pageout
37037c478bd9Sstevel@tonic-gate *
37047c478bd9Sstevel@tonic-gate * XXXmh modify freelist code to set reference
37057c478bd9Sstevel@tonic-gate * bit so we don't have to do it here.
37067c478bd9Sstevel@tonic-gate */
37077c478bd9Sstevel@tonic-gate page_set_props(pp, P_REF);
37087c478bd9Sstevel@tonic-gate } else {
37097c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(phm));
37107c478bd9Sstevel@tonic-gate mutex_exit(phm);
37117c478bd9Sstevel@tonic-gate phm = NULL;
37127c478bd9Sstevel@tonic-gate /*
37137c478bd9Sstevel@tonic-gate * NOTE: This should not happen for pages associated
37147c478bd9Sstevel@tonic-gate * with kernel vnode 'kvp'.
37157c478bd9Sstevel@tonic-gate */
37167c478bd9Sstevel@tonic-gate /* XX64 - to debug why this happens! */
3717ad23a2dbSjohansen ASSERT(!VN_ISKAS(vp));
3718ad23a2dbSjohansen if (VN_ISKAS(vp))
37197c478bd9Sstevel@tonic-gate cmn_err(CE_NOTE,
37207c478bd9Sstevel@tonic-gate "page_create: page not expected "
37217c478bd9Sstevel@tonic-gate "in hash list for kernel vnode - pp 0x%p",
37227c478bd9Sstevel@tonic-gate (void *)pp);
37237c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_exists);
37247c478bd9Sstevel@tonic-gate goto fail;
37257c478bd9Sstevel@tonic-gate }
37267c478bd9Sstevel@tonic-gate
37277c478bd9Sstevel@tonic-gate /*
37287c478bd9Sstevel@tonic-gate * Got a page! It is locked. Acquire the i/o
37297c478bd9Sstevel@tonic-gate * lock since we are going to use the p_next and
37307c478bd9Sstevel@tonic-gate * p_prev fields to link the requested pages together.
37317c478bd9Sstevel@tonic-gate */
37327c478bd9Sstevel@tonic-gate page_io_lock(pp);
37337c478bd9Sstevel@tonic-gate page_add(&plist, pp);
37347c478bd9Sstevel@tonic-gate plist = plist->p_next;
37357c478bd9Sstevel@tonic-gate off += MMU_PAGESIZE;
37367c478bd9Sstevel@tonic-gate vaddr += MMU_PAGESIZE;
37377c478bd9Sstevel@tonic-gate }
37387c478bd9Sstevel@tonic-gate
37397c478bd9Sstevel@tonic-gate check_dma(mattr, plist, pages_req);
37407c478bd9Sstevel@tonic-gate return (plist);
37417c478bd9Sstevel@tonic-gate
37427c478bd9Sstevel@tonic-gate fail:
37437c478bd9Sstevel@tonic-gate if (npp != NULL) {
37447c478bd9Sstevel@tonic-gate /*
37457c478bd9Sstevel@tonic-gate * Did not need this page after all.
37467c478bd9Sstevel@tonic-gate * Put it back on the free list.
37477c478bd9Sstevel@tonic-gate */
37487c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_putbacks);
37497c478bd9Sstevel@tonic-gate PP_SETFREE(npp);
37507c478bd9Sstevel@tonic-gate PP_SETAGED(npp);
37517c478bd9Sstevel@tonic-gate npp->p_offset = (u_offset_t)-1;
37527c478bd9Sstevel@tonic-gate page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL);
37537c478bd9Sstevel@tonic-gate page_unlock(npp);
37547c478bd9Sstevel@tonic-gate }
37557c478bd9Sstevel@tonic-gate
37567c478bd9Sstevel@tonic-gate /*
37577c478bd9Sstevel@tonic-gate * Give up the pages we already got.
37587c478bd9Sstevel@tonic-gate */
37597c478bd9Sstevel@tonic-gate while (plist != NULL) {
37607c478bd9Sstevel@tonic-gate pp = plist;
37617c478bd9Sstevel@tonic-gate page_sub(&plist, pp);
37627c478bd9Sstevel@tonic-gate page_io_unlock(pp);
37637c478bd9Sstevel@tonic-gate plist_len++;
37647c478bd9Sstevel@tonic-gate /*LINTED: constant in conditional ctx*/
37657c478bd9Sstevel@tonic-gate VN_DISPOSE(pp, B_INVAL, 0, kcred);
37667c478bd9Sstevel@tonic-gate }
37677c478bd9Sstevel@tonic-gate
37687c478bd9Sstevel@tonic-gate /*
37697c478bd9Sstevel@tonic-gate * VN_DISPOSE does freemem accounting for the pages in plist
37707c478bd9Sstevel@tonic-gate * by calling page_free. So, we need to undo the pcf accounting
37717c478bd9Sstevel@tonic-gate * for only the remaining pages.
37727c478bd9Sstevel@tonic-gate */
37737c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_putbacks);
37747c478bd9Sstevel@tonic-gate page_create_putback(pages_req - plist_len);
37757c478bd9Sstevel@tonic-gate
37767c478bd9Sstevel@tonic-gate return (NULL);
37777c478bd9Sstevel@tonic-gate }
3778843e1988Sjohnlev #endif /* !__xpv */
37797c478bd9Sstevel@tonic-gate
37807c478bd9Sstevel@tonic-gate
37817c478bd9Sstevel@tonic-gate /*
37827c478bd9Sstevel@tonic-gate * Copy the data from the physical page represented by "frompp" to
37837c478bd9Sstevel@tonic-gate * that represented by "topp". ppcopy uses CPU->cpu_caddr1 and
37847c478bd9Sstevel@tonic-gate * CPU->cpu_caddr2. It assumes that no one uses either map at interrupt
37857c478bd9Sstevel@tonic-gate * level and no one sleeps with an active mapping there.
37867c478bd9Sstevel@tonic-gate *
37877c478bd9Sstevel@tonic-gate * Note that the ref/mod bits in the page_t's are not affected by
37887c478bd9Sstevel@tonic-gate * this operation, hence it is up to the caller to update them appropriately.
37897c478bd9Sstevel@tonic-gate */
37908b464eb8Smec int
ppcopy(page_t * frompp,page_t * topp)37917c478bd9Sstevel@tonic-gate ppcopy(page_t *frompp, page_t *topp)
37927c478bd9Sstevel@tonic-gate {
37937c478bd9Sstevel@tonic-gate caddr_t pp_addr1;
37947c478bd9Sstevel@tonic-gate caddr_t pp_addr2;
3795ae115bc7Smrj hat_mempte_t pte1;
3796ae115bc7Smrj hat_mempte_t pte2;
37978b464eb8Smec label_t ljb;
37983df2e8b2SRobert Mustacchi int ret;
37997c478bd9Sstevel@tonic-gate
38007c478bd9Sstevel@tonic-gate ASSERT_STACK_ALIGNED();
38017c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(frompp));
38027c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(topp));
38037c478bd9Sstevel@tonic-gate
38047c478bd9Sstevel@tonic-gate if (kpm_enable) {
38057c478bd9Sstevel@tonic-gate pp_addr1 = hat_kpm_page2va(frompp, 0);
38067c478bd9Sstevel@tonic-gate pp_addr2 = hat_kpm_page2va(topp, 0);
38077c478bd9Sstevel@tonic-gate kpreempt_disable();
38087c478bd9Sstevel@tonic-gate } else {
38097c478bd9Sstevel@tonic-gate /*
38107c478bd9Sstevel@tonic-gate * disable pre-emption so that CPU can't change
38117c478bd9Sstevel@tonic-gate */
38127c478bd9Sstevel@tonic-gate kpreempt_disable();
38137c478bd9Sstevel@tonic-gate
38147c478bd9Sstevel@tonic-gate pp_addr1 = CPU->cpu_caddr1;
38157c478bd9Sstevel@tonic-gate pp_addr2 = CPU->cpu_caddr2;
3816ae115bc7Smrj pte1 = CPU->cpu_caddr1pte;
3817ae115bc7Smrj pte2 = CPU->cpu_caddr2pte;
38187c478bd9Sstevel@tonic-gate
38193df2e8b2SRobert Mustacchi mutex_enter(&CPU->cpu_ppaddr_mutex);
38207c478bd9Sstevel@tonic-gate
38217c478bd9Sstevel@tonic-gate hat_mempte_remap(page_pptonum(frompp), pp_addr1, pte1,
38227c478bd9Sstevel@tonic-gate PROT_READ | HAT_STORECACHING_OK, HAT_LOAD_NOCONSIST);
38237c478bd9Sstevel@tonic-gate hat_mempte_remap(page_pptonum(topp), pp_addr2, pte2,
38247c478bd9Sstevel@tonic-gate PROT_READ | PROT_WRITE | HAT_STORECACHING_OK,
38257c478bd9Sstevel@tonic-gate HAT_LOAD_NOCONSIST);
38267c478bd9Sstevel@tonic-gate }
38277c478bd9Sstevel@tonic-gate
38288b464eb8Smec if (on_fault(&ljb)) {
38298b464eb8Smec ret = 0;
38308b464eb8Smec goto faulted;
38313df2e8b2SRobert Mustacchi } else {
38323df2e8b2SRobert Mustacchi ret = 1;
38338b464eb8Smec }
38347c478bd9Sstevel@tonic-gate if (use_sse_pagecopy)
3835843e1988Sjohnlev #ifdef __xpv
3836843e1988Sjohnlev page_copy_no_xmm(pp_addr2, pp_addr1);
3837843e1988Sjohnlev #else
38387c478bd9Sstevel@tonic-gate hwblkpagecopy(pp_addr1, pp_addr2);
3839843e1988Sjohnlev #endif
38407c478bd9Sstevel@tonic-gate else
38417c478bd9Sstevel@tonic-gate bcopy(pp_addr1, pp_addr2, PAGESIZE);
38427c478bd9Sstevel@tonic-gate
38438b464eb8Smec no_fault();
38448b464eb8Smec faulted:
3845ae115bc7Smrj if (!kpm_enable) {
3846843e1988Sjohnlev #ifdef __xpv
3847843e1988Sjohnlev /*
38488ea72728Sjosephb * We can't leave unused mappings laying about under the
38498ea72728Sjosephb * hypervisor, so blow them away.
3850843e1988Sjohnlev */
38518ea72728Sjosephb if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr1, 0,
38528ea72728Sjosephb UVMF_INVLPG | UVMF_LOCAL) < 0)
38538ea72728Sjosephb panic("HYPERVISOR_update_va_mapping() failed");
3854843e1988Sjohnlev if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr2, 0,
3855843e1988Sjohnlev UVMF_INVLPG | UVMF_LOCAL) < 0)
3856843e1988Sjohnlev panic("HYPERVISOR_update_va_mapping() failed");
3857843e1988Sjohnlev #endif
38583df2e8b2SRobert Mustacchi mutex_exit(&CPU->cpu_ppaddr_mutex);
3859ae115bc7Smrj }
38607c478bd9Sstevel@tonic-gate kpreempt_enable();
38618b464eb8Smec return (ret);
38627c478bd9Sstevel@tonic-gate }
38637c478bd9Sstevel@tonic-gate
3864d2b85481Srscott void
pagezero(page_t * pp,uint_t off,uint_t len)3865d2b85481Srscott pagezero(page_t *pp, uint_t off, uint_t len)
3866d2b85481Srscott {
3867d2b85481Srscott ASSERT(PAGE_LOCKED(pp));
3868d2b85481Srscott pfnzero(page_pptonum(pp), off, len);
3869d2b85481Srscott }
3870d2b85481Srscott
38717c478bd9Sstevel@tonic-gate /*
3872d2b85481Srscott * Zero the physical page from off to off + len given by pfn
38737c478bd9Sstevel@tonic-gate * without changing the reference and modified bits of page.
38747c478bd9Sstevel@tonic-gate *
38757c478bd9Sstevel@tonic-gate * We use this using CPU private page address #2, see ppcopy() for more info.
3876d2b85481Srscott * pfnzero() must not be called at interrupt level.
38777c478bd9Sstevel@tonic-gate */
38787c478bd9Sstevel@tonic-gate void
pfnzero(pfn_t pfn,uint_t off,uint_t len)3879d2b85481Srscott pfnzero(pfn_t pfn, uint_t off, uint_t len)
38807c478bd9Sstevel@tonic-gate {
38817c478bd9Sstevel@tonic-gate caddr_t pp_addr2;
3882ae115bc7Smrj hat_mempte_t pte2;
3883d2b85481Srscott kmutex_t *ppaddr_mutex = NULL;
38847c478bd9Sstevel@tonic-gate
38857c478bd9Sstevel@tonic-gate ASSERT_STACK_ALIGNED();
38867c478bd9Sstevel@tonic-gate ASSERT(len <= MMU_PAGESIZE);
38877c478bd9Sstevel@tonic-gate ASSERT(off <= MMU_PAGESIZE);
38887c478bd9Sstevel@tonic-gate ASSERT(off + len <= MMU_PAGESIZE);
38897c478bd9Sstevel@tonic-gate
3890d2b85481Srscott if (kpm_enable && !pfn_is_foreign(pfn)) {
3891d2b85481Srscott pp_addr2 = hat_kpm_pfn2va(pfn);
38927c478bd9Sstevel@tonic-gate kpreempt_disable();
38937c478bd9Sstevel@tonic-gate } else {
38947c478bd9Sstevel@tonic-gate kpreempt_disable();
38957c478bd9Sstevel@tonic-gate
38967c478bd9Sstevel@tonic-gate pp_addr2 = CPU->cpu_caddr2;
3897ae115bc7Smrj pte2 = CPU->cpu_caddr2pte;
38987c478bd9Sstevel@tonic-gate
38997c478bd9Sstevel@tonic-gate ppaddr_mutex = &CPU->cpu_ppaddr_mutex;
39007c478bd9Sstevel@tonic-gate mutex_enter(ppaddr_mutex);
39017c478bd9Sstevel@tonic-gate
3902d2b85481Srscott hat_mempte_remap(pfn, pp_addr2, pte2,
39037c478bd9Sstevel@tonic-gate PROT_READ | PROT_WRITE | HAT_STORECACHING_OK,
39047c478bd9Sstevel@tonic-gate HAT_LOAD_NOCONSIST);
39057c478bd9Sstevel@tonic-gate }
39067c478bd9Sstevel@tonic-gate
3907ae115bc7Smrj if (use_sse_pagezero) {
3908843e1988Sjohnlev #ifdef __xpv
3909843e1988Sjohnlev uint_t rem;
3910843e1988Sjohnlev
3911843e1988Sjohnlev /*
3912843e1988Sjohnlev * zero a byte at a time until properly aligned for
3913843e1988Sjohnlev * block_zero_no_xmm().
3914843e1988Sjohnlev */
3915843e1988Sjohnlev while (!P2NPHASE(off, ((uint_t)BLOCKZEROALIGN)) && len-- > 0)
3916843e1988Sjohnlev pp_addr2[off++] = 0;
3917843e1988Sjohnlev
3918843e1988Sjohnlev /*
3919843e1988Sjohnlev * Now use faster block_zero_no_xmm() for any range
3920843e1988Sjohnlev * that is properly aligned and sized.
3921843e1988Sjohnlev */
3922843e1988Sjohnlev rem = P2PHASE(len, ((uint_t)BLOCKZEROALIGN));
3923843e1988Sjohnlev len -= rem;
3924843e1988Sjohnlev if (len != 0) {
3925843e1988Sjohnlev block_zero_no_xmm(pp_addr2 + off, len);
3926843e1988Sjohnlev off += len;
3927843e1988Sjohnlev }
3928843e1988Sjohnlev
3929843e1988Sjohnlev /*
3930843e1988Sjohnlev * zero remainder with byte stores.
3931843e1988Sjohnlev */
3932843e1988Sjohnlev while (rem-- > 0)
3933843e1988Sjohnlev pp_addr2[off++] = 0;
3934843e1988Sjohnlev #else
39357c478bd9Sstevel@tonic-gate hwblkclr(pp_addr2 + off, len);
3936843e1988Sjohnlev #endif
3937ae115bc7Smrj } else {
39387c478bd9Sstevel@tonic-gate bzero(pp_addr2 + off, len);
3939ae115bc7Smrj }
39407c478bd9Sstevel@tonic-gate
3941d2b85481Srscott if (!kpm_enable || pfn_is_foreign(pfn)) {
3942843e1988Sjohnlev #ifdef __xpv
3943d2b85481Srscott /*
3944d2b85481Srscott * On the hypervisor this page might get used for a page
3945d2b85481Srscott * table before any intervening change to this mapping,
3946d2b85481Srscott * so blow it away.
3947d2b85481Srscott */
3948d2b85481Srscott if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr2, 0,
3949d2b85481Srscott UVMF_INVLPG) < 0)
3950d2b85481Srscott panic("HYPERVISOR_update_va_mapping() failed");
3951843e1988Sjohnlev #endif
39527c478bd9Sstevel@tonic-gate mutex_exit(ppaddr_mutex);
3953d2b85481Srscott }
3954d2b85481Srscott
39557c478bd9Sstevel@tonic-gate kpreempt_enable();
39567c478bd9Sstevel@tonic-gate }
39577c478bd9Sstevel@tonic-gate
39587c478bd9Sstevel@tonic-gate /*
39597c478bd9Sstevel@tonic-gate * Platform-dependent page scrub call.
39607c478bd9Sstevel@tonic-gate */
39617c478bd9Sstevel@tonic-gate void
pagescrub(page_t * pp,uint_t off,uint_t len)39627c478bd9Sstevel@tonic-gate pagescrub(page_t *pp, uint_t off, uint_t len)
39637c478bd9Sstevel@tonic-gate {
39647c478bd9Sstevel@tonic-gate /*
39657c478bd9Sstevel@tonic-gate * For now, we rely on the fact that pagezero() will
39667c478bd9Sstevel@tonic-gate * always clear UEs.
39677c478bd9Sstevel@tonic-gate */
39687c478bd9Sstevel@tonic-gate pagezero(pp, off, len);
39697c478bd9Sstevel@tonic-gate }
39707c478bd9Sstevel@tonic-gate
39717c478bd9Sstevel@tonic-gate /*
39727c478bd9Sstevel@tonic-gate * set up two private addresses for use on a given CPU for use in ppcopy()
39737c478bd9Sstevel@tonic-gate */
39747c478bd9Sstevel@tonic-gate void
setup_vaddr_for_ppcopy(struct cpu * cpup)39757c478bd9Sstevel@tonic-gate setup_vaddr_for_ppcopy(struct cpu *cpup)
39767c478bd9Sstevel@tonic-gate {
39777c478bd9Sstevel@tonic-gate void *addr;
3978ae115bc7Smrj hat_mempte_t pte_pa;
39797c478bd9Sstevel@tonic-gate
39807c478bd9Sstevel@tonic-gate addr = vmem_alloc(heap_arena, mmu_ptob(1), VM_SLEEP);
3981ae115bc7Smrj pte_pa = hat_mempte_setup(addr);
39827c478bd9Sstevel@tonic-gate cpup->cpu_caddr1 = addr;
3983ae115bc7Smrj cpup->cpu_caddr1pte = pte_pa;
39847c478bd9Sstevel@tonic-gate
39857c478bd9Sstevel@tonic-gate addr = vmem_alloc(heap_arena, mmu_ptob(1), VM_SLEEP);
3986ae115bc7Smrj pte_pa = hat_mempte_setup(addr);
39877c478bd9Sstevel@tonic-gate cpup->cpu_caddr2 = addr;
3988ae115bc7Smrj cpup->cpu_caddr2pte = pte_pa;
39897c478bd9Sstevel@tonic-gate
39907c478bd9Sstevel@tonic-gate mutex_init(&cpup->cpu_ppaddr_mutex, NULL, MUTEX_DEFAULT, NULL);
39917c478bd9Sstevel@tonic-gate }
39927c478bd9Sstevel@tonic-gate
3993ae115bc7Smrj /*
3994ae115bc7Smrj * Undo setup_vaddr_for_ppcopy
3995ae115bc7Smrj */
3996ae115bc7Smrj void
teardown_vaddr_for_ppcopy(struct cpu * cpup)3997ae115bc7Smrj teardown_vaddr_for_ppcopy(struct cpu *cpup)
3998ae115bc7Smrj {
3999ae115bc7Smrj mutex_destroy(&cpup->cpu_ppaddr_mutex);
4000ae115bc7Smrj
4001ae115bc7Smrj hat_mempte_release(cpup->cpu_caddr2, cpup->cpu_caddr2pte);
4002ae115bc7Smrj cpup->cpu_caddr2pte = 0;
4003ae115bc7Smrj vmem_free(heap_arena, cpup->cpu_caddr2, mmu_ptob(1));
4004ae115bc7Smrj cpup->cpu_caddr2 = 0;
4005ae115bc7Smrj
4006ae115bc7Smrj hat_mempte_release(cpup->cpu_caddr1, cpup->cpu_caddr1pte);
4007ae115bc7Smrj cpup->cpu_caddr1pte = 0;
4008ae115bc7Smrj vmem_free(heap_arena, cpup->cpu_caddr1, mmu_ptob(1));
4009ae115bc7Smrj cpup->cpu_caddr1 = 0;
4010ae115bc7Smrj }
40117c478bd9Sstevel@tonic-gate
40127c478bd9Sstevel@tonic-gate /*
40137c478bd9Sstevel@tonic-gate * Function for flushing D-cache when performing module relocations
40147c478bd9Sstevel@tonic-gate * to an alternate mapping. Unnecessary on Intel / AMD platforms.
40157c478bd9Sstevel@tonic-gate */
40167c478bd9Sstevel@tonic-gate void
dcache_flushall()40177c478bd9Sstevel@tonic-gate dcache_flushall()
40187c478bd9Sstevel@tonic-gate {}
4019102033aaSdp
4020ae115bc7Smrj /*
4021ae115bc7Smrj * Allocate a memory page. The argument 'seed' can be any pseudo-random
4022ae115bc7Smrj * number to vary where the pages come from. This is quite a hacked up
4023ae115bc7Smrj * method -- it works for now, but really needs to be fixed up a bit.
4024ae115bc7Smrj *
4025ae115bc7Smrj * We currently use page_create_va() on the kvp with fake offsets,
4026ae115bc7Smrj * segments and virt address. This is pretty bogus, but was copied from the
4027ae115bc7Smrj * old hat_i86.c code. A better approach would be to specify either mnode
4028ae115bc7Smrj * random or mnode local and takes a page from whatever color has the MOST
4029ae115bc7Smrj * available - this would have a minimal impact on page coloring.
4030ae115bc7Smrj */
4031ae115bc7Smrj page_t *
page_get_physical(uintptr_t seed)4032a77271f8SVikram Hegde page_get_physical(uintptr_t seed)
4033ae115bc7Smrj {
4034ae115bc7Smrj page_t *pp;
4035a77271f8SVikram Hegde u_offset_t offset;
4036ae115bc7Smrj static struct seg tmpseg;
4037ae115bc7Smrj static uintptr_t ctr = 0;
4038ae115bc7Smrj
4039ae115bc7Smrj /*
4040ae115bc7Smrj * This code is gross, we really need a simpler page allocator.
4041ae115bc7Smrj *
4042a77271f8SVikram Hegde * We need to assign an offset for the page to call page_create_va()
4043ae115bc7Smrj * To avoid conflicts with other pages, we get creative with the offset.
404486c1f4dcSVikram Hegde * For 32 bits, we need an offset > 4Gig
404586c1f4dcSVikram Hegde * For 64 bits, need an offset somewhere in the VA hole.
4046ae115bc7Smrj */
4047a77271f8SVikram Hegde offset = seed;
4048a77271f8SVikram Hegde if (offset > kernelbase)
4049a77271f8SVikram Hegde offset -= kernelbase;
4050a77271f8SVikram Hegde offset <<= MMU_PAGESHIFT;
4051a77271f8SVikram Hegde offset += mmu.hole_start; /* something in VA hole */
4052a77271f8SVikram Hegde
4053a77271f8SVikram Hegde if (page_resv(1, KM_NOSLEEP) == 0)
4054ae115bc7Smrj return (NULL);
4055ae115bc7Smrj
4056ae115bc7Smrj #ifdef DEBUG
4057ae115bc7Smrj pp = page_exists(&kvp, offset);
4058ae115bc7Smrj if (pp != NULL)
4059903a11ebSrh panic("page already exists %p", (void *)pp);
4060ae115bc7Smrj #endif
4061ae115bc7Smrj
4062843e1988Sjohnlev pp = page_create_va(&kvp, offset, MMU_PAGESIZE, PG_EXCL,
4063ae115bc7Smrj &tmpseg, (caddr_t)(ctr += MMU_PAGESIZE)); /* changing VA usage */
406486c1f4dcSVikram Hegde if (pp != NULL) {
406586c1f4dcSVikram Hegde page_io_unlock(pp);
4066408a1f8eSVikram Hegde page_downgrade(pp);
406786c1f4dcSVikram Hegde }
4068ae115bc7Smrj return (pp);
4069ae115bc7Smrj }
4070