xref: /illumos-gate/usr/src/uts/common/vm/seg_kmem.c (revision 04909c8c)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright 2019 Joyent, Inc.
24  */
25 
26 #include <sys/types.h>
27 #include <sys/t_lock.h>
28 #include <sys/param.h>
29 #include <sys/sysmacros.h>
30 #include <sys/tuneable.h>
31 #include <sys/systm.h>
32 #include <sys/vm.h>
33 #include <sys/kmem.h>
34 #include <sys/vmem.h>
35 #include <sys/mman.h>
36 #include <sys/cmn_err.h>
37 #include <sys/debug.h>
38 #include <sys/dumphdr.h>
39 #include <sys/bootconf.h>
40 #include <sys/lgrp.h>
41 #include <vm/seg_kmem.h>
42 #include <vm/hat.h>
43 #include <vm/page.h>
44 #include <vm/vm_dep.h>
45 #include <vm/faultcode.h>
46 #include <sys/promif.h>
47 #include <vm/seg_kp.h>
48 #include <sys/bitmap.h>
49 #include <sys/mem_cage.h>
50 
51 #ifdef __sparc
52 #include <sys/ivintr.h>
53 #include <sys/panic.h>
54 #endif
55 
56 /*
57  * seg_kmem is the primary kernel memory segment driver.  It
58  * maps the kernel heap [kernelheap, ekernelheap), module text,
59  * and all memory which was allocated before the VM was initialized
60  * into kas.
61  *
62  * Pages which belong to seg_kmem are hashed into &kvp vnode at
63  * an offset equal to (u_offset_t)virt_addr, and have p_lckcnt >= 1.
64  * They must never be paged out since segkmem_fault() is a no-op to
65  * prevent recursive faults.
66  *
67  * Currently, seg_kmem pages are sharelocked (p_sharelock == 1) on
68  * __x86 and are unlocked (p_sharelock == 0) on __sparc.  Once __x86
69  * supports relocation the #ifdef kludges can be removed.
70  *
71  * seg_kmem pages may be subject to relocation by page_relocate(),
72  * provided that the HAT supports it; if this is so, segkmem_reloc
73  * will be set to a nonzero value. All boot time allocated memory as
74  * well as static memory is considered off limits to relocation.
75  * Pages are "relocatable" if p_state does not have P_NORELOC set, so
76  * we request P_NORELOC pages for memory that isn't safe to relocate.
77  *
78  * The kernel heap is logically divided up into four pieces:
79  *
80  *   heap32_arena is for allocations that require 32-bit absolute
81  *   virtual addresses (e.g. code that uses 32-bit pointers/offsets).
82  *
83  *   heap_core is for allocations that require 2GB *relative*
84  *   offsets; in other words all memory from heap_core is within
85  *   2GB of all other memory from the same arena. This is a requirement
86  *   of the addressing modes of some processors in supervisor code.
87  *
88  *   heap_arena is the general heap arena.
89  *
90  *   static_arena is the static memory arena.  Allocations from it
91  *   are not subject to relocation so it is safe to use the memory
92  *   physical address as well as the virtual address (e.g. the VA to
93  *   PA translations are static).  Caches may import from static_arena;
94  *   all other static memory allocations should use static_alloc_arena.
95  *
96  * On some platforms which have limited virtual address space, seg_kmem
97  * may share [kernelheap, ekernelheap) with seg_kp; if this is so,
98  * segkp_bitmap is non-NULL, and each bit represents a page of virtual
99  * address space which is actually seg_kp mapped.
100  */
101 
102 extern ulong_t *segkp_bitmap;   /* Is set if segkp is from the kernel heap */
103 
104 char *kernelheap;		/* start of primary kernel heap */
105 char *ekernelheap;		/* end of primary kernel heap */
106 struct seg kvseg;		/* primary kernel heap segment */
107 struct seg kvseg_core;		/* "core" kernel heap segment */
108 struct seg kzioseg;		/* Segment for zio mappings */
109 vmem_t *heap_arena;		/* primary kernel heap arena */
110 vmem_t *heap_core_arena;	/* core kernel heap arena */
111 char *heap_core_base;		/* start of core kernel heap arena */
112 char *heap_lp_base;		/* start of kernel large page heap arena */
113 char *heap_lp_end;		/* end of kernel large page heap arena */
114 vmem_t *hat_memload_arena;	/* HAT translation data */
115 struct seg kvseg32;		/* 32-bit kernel heap segment */
116 vmem_t *heap32_arena;		/* 32-bit kernel heap arena */
117 vmem_t *heaptext_arena;		/* heaptext arena */
118 struct as kas;			/* kernel address space */
119 int segkmem_reloc;		/* enable/disable relocatable segkmem pages */
120 vmem_t *static_arena;		/* arena for caches to import static memory */
121 vmem_t *static_alloc_arena;	/* arena for allocating static memory */
122 vmem_t *zio_arena = NULL;	/* arena for allocating zio memory */
123 vmem_t *zio_alloc_arena = NULL;	/* arena for allocating zio memory */
124 
125 #if defined(__amd64)
126 vmem_t *kvmm_arena;		/* arena for vmm VA */
127 struct seg kvmmseg;		/* Segment for vmm memory */
128 #endif
129 
130 /*
131  * seg_kmem driver can map part of the kernel heap with large pages.
132  * Currently this functionality is implemented for sparc platforms only.
133  *
134  * The large page size "segkmem_lpsize" for kernel heap is selected in the
135  * platform specific code. It can also be modified via /etc/system file.
136  * Setting segkmem_lpsize to PAGESIZE in /etc/system disables usage of large
137  * pages for kernel heap. "segkmem_lpshift" is adjusted appropriately to
138  * match segkmem_lpsize.
139  *
140  * At boot time we carve from kernel heap arena a range of virtual addresses
141  * that will be used for large page mappings. This range [heap_lp_base,
142  * heap_lp_end) is set up as a separate vmem arena - "heap_lp_arena". We also
143  * create "kmem_lp_arena" that caches memory already backed up by large
144  * pages. kmem_lp_arena imports virtual segments from heap_lp_arena.
145  */
146 
147 size_t	segkmem_lpsize;
148 static  uint_t	segkmem_lpshift = PAGESHIFT;
149 int	segkmem_lpszc = 0;
150 
151 size_t  segkmem_kmemlp_quantum = 0x400000;	/* 4MB */
152 size_t  segkmem_heaplp_quantum;
153 vmem_t *heap_lp_arena;
154 static  vmem_t *kmem_lp_arena;
155 static  vmem_t *segkmem_ppa_arena;
156 static	segkmem_lpcb_t segkmem_lpcb;
157 
158 /*
159  * We use "segkmem_kmemlp_max" to limit the total amount of physical memory
160  * consumed by the large page heap. By default this parameter is set to 1/8 of
161  * physmem but can be adjusted through /etc/system either directly or
162  * indirectly by setting "segkmem_kmemlp_pcnt" to the percent of physmem
163  * we allow for large page heap.
164  */
165 size_t  segkmem_kmemlp_max;
166 static  uint_t  segkmem_kmemlp_pcnt;
167 
168 /*
169  * Getting large pages for kernel heap could be problematic due to
170  * physical memory fragmentation. That's why we allow to preallocate
171  * "segkmem_kmemlp_min" bytes at boot time.
172  */
173 static  size_t	segkmem_kmemlp_min;
174 
175 /*
176  * Throttling is used to avoid expensive tries to allocate large pages
177  * for kernel heap when a lot of succesive attempts to do so fail.
178  */
179 static  ulong_t segkmem_lpthrottle_max = 0x400000;
180 static  ulong_t segkmem_lpthrottle_start = 0x40;
181 static  ulong_t segkmem_use_lpthrottle = 1;
182 
183 /*
184  * Freed pages accumulate on a garbage list until segkmem is ready,
185  * at which point we call segkmem_gc() to free it all.
186  */
187 typedef struct segkmem_gc_list {
188 	struct segkmem_gc_list	*gc_next;
189 	vmem_t			*gc_arena;
190 	size_t			gc_size;
191 } segkmem_gc_list_t;
192 
193 static segkmem_gc_list_t *segkmem_gc_list;
194 
195 /*
196  * Allocations from the hat_memload arena add VM_MEMLOAD to their
197  * vmflags so that segkmem_xalloc() can inform the hat layer that it needs
198  * to take steps to prevent infinite recursion.  HAT allocations also
199  * must be non-relocatable to prevent recursive page faults.
200  */
201 static void *
hat_memload_alloc(vmem_t * vmp,size_t size,int flags)202 hat_memload_alloc(vmem_t *vmp, size_t size, int flags)
203 {
204 	flags |= (VM_MEMLOAD | VM_NORELOC);
205 	return (segkmem_alloc(vmp, size, flags));
206 }
207 
208 /*
209  * Allocations from static_arena arena (or any other arena that uses
210  * segkmem_alloc_permanent()) require non-relocatable (permanently
211  * wired) memory pages, since these pages are referenced by physical
212  * as well as virtual address.
213  */
214 void *
segkmem_alloc_permanent(vmem_t * vmp,size_t size,int flags)215 segkmem_alloc_permanent(vmem_t *vmp, size_t size, int flags)
216 {
217 	return (segkmem_alloc(vmp, size, flags | VM_NORELOC));
218 }
219 
220 /*
221  * Initialize kernel heap boundaries.
222  */
223 void
kernelheap_init(void * heap_start,void * heap_end,char * first_avail,void * core_start,void * core_end)224 kernelheap_init(
225 	void *heap_start,
226 	void *heap_end,
227 	char *first_avail,
228 	void *core_start,
229 	void *core_end)
230 {
231 	uintptr_t textbase;
232 	size_t core_size;
233 	size_t heap_size;
234 	vmem_t *heaptext_parent;
235 	size_t	heap_lp_size = 0;
236 #ifdef __sparc
237 	size_t kmem64_sz = kmem64_aligned_end - kmem64_base;
238 #endif	/* __sparc */
239 
240 	kernelheap = heap_start;
241 	ekernelheap = heap_end;
242 
243 #ifdef __sparc
244 	heap_lp_size = (((uintptr_t)heap_end - (uintptr_t)heap_start) / 4);
245 	/*
246 	 * Bias heap_lp start address by kmem64_sz to reduce collisions
247 	 * in 4M kernel TSB between kmem64 area and heap_lp
248 	 */
249 	kmem64_sz = P2ROUNDUP(kmem64_sz, MMU_PAGESIZE256M);
250 	if (kmem64_sz <= heap_lp_size / 2)
251 		heap_lp_size -= kmem64_sz;
252 	heap_lp_base = ekernelheap - heap_lp_size;
253 	heap_lp_end = heap_lp_base + heap_lp_size;
254 #endif	/* __sparc */
255 
256 	/*
257 	 * If this platform has a 'core' heap area, then the space for
258 	 * overflow module text should be carved out of the end of that
259 	 * heap.  Otherwise, it gets carved out of the general purpose
260 	 * heap.
261 	 */
262 	core_size = (uintptr_t)core_end - (uintptr_t)core_start;
263 	if (core_size > 0) {
264 		ASSERT(core_size >= HEAPTEXT_SIZE);
265 		textbase = (uintptr_t)core_end - HEAPTEXT_SIZE;
266 		core_size -= HEAPTEXT_SIZE;
267 	}
268 #ifndef __sparc
269 	else {
270 		ekernelheap -= HEAPTEXT_SIZE;
271 		textbase = (uintptr_t)ekernelheap;
272 	}
273 #endif
274 
275 	heap_size = (uintptr_t)ekernelheap - (uintptr_t)kernelheap;
276 	heap_arena = vmem_init("heap", kernelheap, heap_size, PAGESIZE,
277 	    segkmem_alloc, segkmem_free);
278 
279 	if (core_size > 0) {
280 		heap_core_arena = vmem_create("heap_core", core_start,
281 		    core_size, PAGESIZE, NULL, NULL, NULL, 0, VM_SLEEP);
282 		heap_core_base = core_start;
283 	} else {
284 		heap_core_arena = heap_arena;
285 		heap_core_base = kernelheap;
286 	}
287 
288 	/*
289 	 * reserve space for the large page heap. If large pages for kernel
290 	 * heap is enabled large page heap arean will be created later in the
291 	 * boot sequence in segkmem_heap_lp_init(). Otherwise the allocated
292 	 * range will be returned back to the heap_arena.
293 	 */
294 	if (heap_lp_size) {
295 		(void) vmem_xalloc(heap_arena, heap_lp_size, PAGESIZE, 0, 0,
296 		    heap_lp_base, heap_lp_end,
297 		    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
298 	}
299 
300 	/*
301 	 * Remove the already-spoken-for memory range [kernelheap, first_avail).
302 	 */
303 	(void) vmem_xalloc(heap_arena, first_avail - kernelheap, PAGESIZE,
304 	    0, 0, kernelheap, first_avail, VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
305 
306 #ifdef __sparc
307 	heap32_arena = vmem_create("heap32", (void *)SYSBASE32,
308 	    SYSLIMIT32 - SYSBASE32 - HEAPTEXT_SIZE, PAGESIZE, NULL,
309 	    NULL, NULL, 0, VM_SLEEP);
310 	/*
311 	 * Prom claims the physical and virtual resources used by panicbuf
312 	 * and inter_vec_table. So reserve space for panicbuf, intr_vec_table,
313 	 * reserved interrupt vector data structures from 32-bit heap.
314 	 */
315 	(void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0,
316 	    panicbuf, panicbuf + PANICBUFSIZE,
317 	    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
318 
319 	(void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0,
320 	    intr_vec_table, (caddr_t)intr_vec_table + IVSIZE,
321 	    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
322 
323 	textbase = SYSLIMIT32 - HEAPTEXT_SIZE;
324 	heaptext_parent = NULL;
325 #else	/* __sparc */
326 	heap32_arena = heap_core_arena;
327 	heaptext_parent = heap_core_arena;
328 #endif	/* __sparc */
329 
330 	heaptext_arena = vmem_create("heaptext", (void *)textbase,
331 	    HEAPTEXT_SIZE, PAGESIZE, NULL, NULL, heaptext_parent, 0, VM_SLEEP);
332 
333 	/*
334 	 * Create a set of arenas for memory with static translations
335 	 * (e.g. VA -> PA translations cannot change).  Since using
336 	 * kernel pages by physical address implies it isn't safe to
337 	 * walk across page boundaries, the static_arena quantum must
338 	 * be PAGESIZE.  Any kmem caches that require static memory
339 	 * should source from static_arena, while direct allocations
340 	 * should only use static_alloc_arena.
341 	 */
342 	static_arena = vmem_create("static", NULL, 0, PAGESIZE,
343 	    segkmem_alloc_permanent, segkmem_free, heap_arena, 0, VM_SLEEP);
344 	static_alloc_arena = vmem_create("static_alloc", NULL, 0,
345 	    sizeof (uint64_t), vmem_alloc, vmem_free, static_arena,
346 	    0, VM_SLEEP);
347 
348 	/*
349 	 * Create an arena for translation data (ptes, hmes, or hblks).
350 	 * We need an arena for this because hat_memload() is essential
351 	 * to vmem_populate() (see comments in common/os/vmem.c).
352 	 *
353 	 * Note: any kmem cache that allocates from hat_memload_arena
354 	 * must be created as a KMC_NOHASH cache (i.e. no external slab
355 	 * and bufctl structures to allocate) so that slab creation doesn't
356 	 * require anything more than a single vmem_alloc().
357 	 */
358 	hat_memload_arena = vmem_create("hat_memload", NULL, 0, PAGESIZE,
359 	    hat_memload_alloc, segkmem_free, heap_arena, 0,
360 	    VM_SLEEP | VMC_POPULATOR | VMC_DUMPSAFE);
361 }
362 
363 void
boot_mapin(caddr_t addr,size_t size)364 boot_mapin(caddr_t addr, size_t size)
365 {
366 	caddr_t	 eaddr;
367 	page_t	*pp;
368 	pfn_t	 pfnum;
369 
370 	if (page_resv(btop(size), KM_NOSLEEP) == 0)
371 		panic("boot_mapin: page_resv failed");
372 
373 	for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
374 		pfnum = va_to_pfn(addr);
375 		if (pfnum == PFN_INVALID)
376 			continue;
377 		if ((pp = page_numtopp_nolock(pfnum)) == NULL)
378 			panic("boot_mapin(): No pp for pfnum = %lx", pfnum);
379 
380 		/*
381 		 * must break up any large pages that may have constituent
382 		 * pages being utilized for BOP_ALLOC()'s before calling
383 		 * page_numtopp().The locking code (ie. page_reclaim())
384 		 * can't handle them
385 		 */
386 		if (pp->p_szc != 0)
387 			page_boot_demote(pp);
388 
389 		pp = page_numtopp(pfnum, SE_EXCL);
390 		if (pp == NULL || PP_ISFREE(pp))
391 			panic("boot_alloc: pp is NULL or free");
392 
393 		/*
394 		 * If the cage is on but doesn't yet contain this page,
395 		 * mark it as non-relocatable.
396 		 */
397 		if (kcage_on && !PP_ISNORELOC(pp)) {
398 			PP_SETNORELOC(pp);
399 			PLCNT_XFER_NORELOC(pp);
400 		}
401 
402 		(void) page_hashin(pp, &kvp, (u_offset_t)(uintptr_t)addr, NULL);
403 		pp->p_lckcnt = 1;
404 #if defined(__x86)
405 		page_downgrade(pp);
406 #else
407 		page_unlock(pp);
408 #endif
409 	}
410 }
411 
412 /*
413  * Get pages from boot and hash them into the kernel's vp.
414  * Used after page structs have been allocated, but before segkmem is ready.
415  */
416 void *
boot_alloc(void * inaddr,size_t size,uint_t align)417 boot_alloc(void *inaddr, size_t size, uint_t align)
418 {
419 	caddr_t addr = inaddr;
420 
421 	if (bootops == NULL)
422 		prom_panic("boot_alloc: attempt to allocate memory after "
423 		    "BOP_GONE");
424 
425 	size = ptob(btopr(size));
426 #ifdef __sparc
427 	if (bop_alloc_chunk(addr, size, align) != (caddr_t)addr)
428 		panic("boot_alloc: bop_alloc_chunk failed");
429 #else
430 	if (BOP_ALLOC(bootops, addr, size, align) != addr)
431 		panic("boot_alloc: BOP_ALLOC failed");
432 #endif
433 	boot_mapin((caddr_t)addr, size);
434 	return (addr);
435 }
436 
437 static void
segkmem_badop()438 segkmem_badop()
439 {
440 	panic("segkmem_badop");
441 }
442 
443 #define	SEGKMEM_BADOP(t)	(t(*)())(uintptr_t)segkmem_badop
444 
445 /*ARGSUSED*/
446 static faultcode_t
segkmem_fault(struct hat * hat,struct seg * seg,caddr_t addr,size_t size,enum fault_type type,enum seg_rw rw)447 segkmem_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t size,
448     enum fault_type type, enum seg_rw rw)
449 {
450 	pgcnt_t npages;
451 	spgcnt_t pg;
452 	page_t *pp;
453 	struct vnode *vp = seg->s_data;
454 
455 	ASSERT(RW_READ_HELD(&seg->s_as->a_lock));
456 
457 	if (seg->s_as != &kas || size > seg->s_size ||
458 	    addr < seg->s_base || addr + size > seg->s_base + seg->s_size)
459 		panic("segkmem_fault: bad args");
460 
461 	/*
462 	 * If it is one of segkp pages, call segkp_fault.
463 	 */
464 	if (segkp_bitmap && seg == &kvseg &&
465 	    BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
466 		return (SEGOP_FAULT(hat, segkp, addr, size, type, rw));
467 
468 	if (rw != S_READ && rw != S_WRITE && rw != S_OTHER)
469 		return (FC_NOSUPPORT);
470 
471 	npages = btopr(size);
472 
473 	switch (type) {
474 	case F_SOFTLOCK:	/* lock down already-loaded translations */
475 		for (pg = 0; pg < npages; pg++) {
476 			pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr,
477 			    SE_SHARED);
478 			if (pp == NULL) {
479 				/*
480 				 * Hmm, no page. Does a kernel mapping
481 				 * exist for it?
482 				 */
483 				if (!hat_probe(kas.a_hat, addr)) {
484 					addr -= PAGESIZE;
485 					while (--pg >= 0) {
486 						pp = page_find(vp, (u_offset_t)
487 						    (uintptr_t)addr);
488 						if (pp)
489 							page_unlock(pp);
490 						addr -= PAGESIZE;
491 					}
492 					return (FC_NOMAP);
493 				}
494 			}
495 			addr += PAGESIZE;
496 		}
497 		if (rw == S_OTHER)
498 			hat_reserve(seg->s_as, addr, size);
499 		return (0);
500 	case F_SOFTUNLOCK:
501 		while (npages--) {
502 			pp = page_find(vp, (u_offset_t)(uintptr_t)addr);
503 			if (pp)
504 				page_unlock(pp);
505 			addr += PAGESIZE;
506 		}
507 		return (0);
508 	default:
509 		return (FC_NOSUPPORT);
510 	}
511 	/*NOTREACHED*/
512 }
513 
514 static int
segkmem_setprot(struct seg * seg,caddr_t addr,size_t size,uint_t prot)515 segkmem_setprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
516 {
517 	ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
518 
519 	if (seg->s_as != &kas || size > seg->s_size ||
520 	    addr < seg->s_base || addr + size > seg->s_base + seg->s_size)
521 		panic("segkmem_setprot: bad args");
522 
523 	/*
524 	 * If it is one of segkp pages, call segkp.
525 	 */
526 	if (segkp_bitmap && seg == &kvseg &&
527 	    BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
528 		return (SEGOP_SETPROT(segkp, addr, size, prot));
529 
530 	if (prot == 0)
531 		hat_unload(kas.a_hat, addr, size, HAT_UNLOAD);
532 	else
533 		hat_chgprot(kas.a_hat, addr, size, prot);
534 	return (0);
535 }
536 
537 /*
538  * This is a dummy segkmem function overloaded to call segkp
539  * when segkp is under the heap.
540  */
541 /* ARGSUSED */
542 static int
segkmem_checkprot(struct seg * seg,caddr_t addr,size_t size,uint_t prot)543 segkmem_checkprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
544 {
545 	ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
546 
547 	if (seg->s_as != &kas)
548 		segkmem_badop();
549 
550 	/*
551 	 * If it is one of segkp pages, call into segkp.
552 	 */
553 	if (segkp_bitmap && seg == &kvseg &&
554 	    BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
555 		return (SEGOP_CHECKPROT(segkp, addr, size, prot));
556 
557 	segkmem_badop();
558 	return (0);
559 }
560 
561 /*
562  * This is a dummy segkmem function overloaded to call segkp
563  * when segkp is under the heap.
564  */
565 /* ARGSUSED */
566 static int
segkmem_kluster(struct seg * seg,caddr_t addr,ssize_t delta)567 segkmem_kluster(struct seg *seg, caddr_t addr, ssize_t delta)
568 {
569 	ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
570 
571 	if (seg->s_as != &kas)
572 		segkmem_badop();
573 
574 	/*
575 	 * If it is one of segkp pages, call into segkp.
576 	 */
577 	if (segkp_bitmap && seg == &kvseg &&
578 	    BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
579 		return (SEGOP_KLUSTER(segkp, addr, delta));
580 
581 	segkmem_badop();
582 	return (0);
583 }
584 
585 static void
segkmem_xdump_range(void * arg,void * start,size_t size)586 segkmem_xdump_range(void *arg, void *start, size_t size)
587 {
588 	struct as *as = arg;
589 	caddr_t addr = start;
590 	caddr_t addr_end = addr + size;
591 
592 	while (addr < addr_end) {
593 		pfn_t pfn = hat_getpfnum(kas.a_hat, addr);
594 		if (pfn != PFN_INVALID && pfn <= physmax && pf_is_memory(pfn))
595 			dump_addpage(as, addr, pfn);
596 		addr += PAGESIZE;
597 		dump_timeleft = dump_timeout;
598 	}
599 }
600 
601 static void
segkmem_dump_range(void * arg,void * start,size_t size)602 segkmem_dump_range(void *arg, void *start, size_t size)
603 {
604 	caddr_t addr = start;
605 	caddr_t addr_end = addr + size;
606 
607 	/*
608 	 * If we are about to start dumping the range of addresses we
609 	 * carved out of the kernel heap for the large page heap walk
610 	 * heap_lp_arena to find what segments are actually populated
611 	 */
612 	if (SEGKMEM_USE_LARGEPAGES &&
613 	    addr == heap_lp_base && addr_end == heap_lp_end &&
614 	    vmem_size(heap_lp_arena, VMEM_ALLOC) < size) {
615 		vmem_walk(heap_lp_arena, VMEM_ALLOC | VMEM_REENTRANT,
616 		    segkmem_xdump_range, arg);
617 	} else {
618 		segkmem_xdump_range(arg, start, size);
619 	}
620 }
621 
622 static void
segkmem_dump(struct seg * seg)623 segkmem_dump(struct seg *seg)
624 {
625 	/*
626 	 * The kernel's heap_arena (represented by kvseg) is a very large
627 	 * VA space, most of which is typically unused.  To speed up dumping
628 	 * we use vmem_walk() to quickly find the pieces of heap_arena that
629 	 * are actually in use.  We do the same for heap32_arena and
630 	 * heap_core.
631 	 *
632 	 * We specify VMEM_REENTRANT to vmem_walk() because dump_addpage()
633 	 * may ultimately need to allocate memory.  Reentrant walks are
634 	 * necessarily imperfect snapshots.  The kernel heap continues
635 	 * to change during a live crash dump, for example.  For a normal
636 	 * crash dump, however, we know that there won't be any other threads
637 	 * messing with the heap.  Therefore, at worst, we may fail to dump
638 	 * the pages that get allocated by the act of dumping; but we will
639 	 * always dump every page that was allocated when the walk began.
640 	 *
641 	 * The other segkmem segments are dense (fully populated), so there's
642 	 * no need to use this technique when dumping them.
643 	 *
644 	 * Note: when adding special dump handling for any new sparsely-
645 	 * populated segments, be sure to add similar handling to the ::kgrep
646 	 * code in mdb.
647 	 */
648 	if (seg == &kvseg) {
649 		vmem_walk(heap_arena, VMEM_ALLOC | VMEM_REENTRANT,
650 		    segkmem_dump_range, seg->s_as);
651 #ifndef __sparc
652 		vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT,
653 		    segkmem_dump_range, seg->s_as);
654 #endif
655 	} else if (seg == &kvseg_core) {
656 		vmem_walk(heap_core_arena, VMEM_ALLOC | VMEM_REENTRANT,
657 		    segkmem_dump_range, seg->s_as);
658 	} else if (seg == &kvseg32) {
659 		vmem_walk(heap32_arena, VMEM_ALLOC | VMEM_REENTRANT,
660 		    segkmem_dump_range, seg->s_as);
661 		vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT,
662 		    segkmem_dump_range, seg->s_as);
663 	/*
664 	 * We don't want to dump pages attached to kzioseg since they
665 	 * contain file data from ZFS.  If this page's segment is
666 	 * kzioseg return instead of writing it to the dump device.
667 	 *
668 	 * Same applies to VM memory allocations.
669 	 */
670 	} else if (seg == &kzioseg) {
671 		return;
672 #if defined(__amd64)
673 	} else if (seg == &kvmmseg) {
674 		return;
675 #endif
676 	} else {
677 		segkmem_dump_range(seg->s_as, seg->s_base, seg->s_size);
678 	}
679 }
680 
681 /*
682  * lock/unlock kmem pages over a given range [addr, addr+len).
683  * Returns a shadow list of pages in ppp. If there are holes
684  * in the range (e.g. some of the kernel mappings do not have
685  * underlying page_ts) returns ENOTSUP so that as_pagelock()
686  * will handle the range via as_fault(F_SOFTLOCK).
687  */
688 /*ARGSUSED*/
689 static int
segkmem_pagelock(struct seg * seg,caddr_t addr,size_t len,page_t *** ppp,enum lock_type type,enum seg_rw rw)690 segkmem_pagelock(struct seg *seg, caddr_t addr, size_t len,
691     page_t ***ppp, enum lock_type type, enum seg_rw rw)
692 {
693 	page_t **pplist, *pp;
694 	pgcnt_t npages;
695 	spgcnt_t pg;
696 	size_t nb;
697 	struct vnode *vp = seg->s_data;
698 
699 	ASSERT(ppp != NULL);
700 
701 	/*
702 	 * If it is one of segkp pages, call into segkp.
703 	 */
704 	if (segkp_bitmap && seg == &kvseg &&
705 	    BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
706 		return (SEGOP_PAGELOCK(segkp, addr, len, ppp, type, rw));
707 
708 	npages = btopr(len);
709 	nb = sizeof (page_t *) * npages;
710 
711 	if (type == L_PAGEUNLOCK) {
712 		pplist = *ppp;
713 		ASSERT(pplist != NULL);
714 
715 		for (pg = 0; pg < npages; pg++) {
716 			pp = pplist[pg];
717 			page_unlock(pp);
718 		}
719 		kmem_free(pplist, nb);
720 		return (0);
721 	}
722 
723 	ASSERT(type == L_PAGELOCK);
724 
725 	pplist = kmem_alloc(nb, KM_NOSLEEP);
726 	if (pplist == NULL) {
727 		*ppp = NULL;
728 		return (ENOTSUP);	/* take the slow path */
729 	}
730 
731 	for (pg = 0; pg < npages; pg++) {
732 		pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr, SE_SHARED);
733 		if (pp == NULL) {
734 			while (--pg >= 0)
735 				page_unlock(pplist[pg]);
736 			kmem_free(pplist, nb);
737 			*ppp = NULL;
738 			return (ENOTSUP);
739 		}
740 		pplist[pg] = pp;
741 		addr += PAGESIZE;
742 	}
743 
744 	*ppp = pplist;
745 	return (0);
746 }
747 
748 /*
749  * This is a dummy segkmem function overloaded to call segkp
750  * when segkp is under the heap.
751  */
752 /* ARGSUSED */
753 static int
segkmem_getmemid(struct seg * seg,caddr_t addr,memid_t * memidp)754 segkmem_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
755 {
756 	ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
757 
758 	if (seg->s_as != &kas)
759 		segkmem_badop();
760 
761 	/*
762 	 * If it is one of segkp pages, call into segkp.
763 	 */
764 	if (segkp_bitmap && seg == &kvseg &&
765 	    BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
766 		return (SEGOP_GETMEMID(segkp, addr, memidp));
767 
768 	segkmem_badop();
769 	return (0);
770 }
771 
772 /*ARGSUSED*/
773 static lgrp_mem_policy_info_t *
segkmem_getpolicy(struct seg * seg,caddr_t addr)774 segkmem_getpolicy(struct seg *seg, caddr_t addr)
775 {
776 	return (NULL);
777 }
778 
779 /*ARGSUSED*/
780 static int
segkmem_capable(struct seg * seg,segcapability_t capability)781 segkmem_capable(struct seg *seg, segcapability_t capability)
782 {
783 	if (capability == S_CAPABILITY_NOMINFLT)
784 		return (1);
785 	return (0);
786 }
787 
788 struct seg_ops segkmem_ops = {
789 	SEGKMEM_BADOP(int),		/* dup */
790 	SEGKMEM_BADOP(int),		/* unmap */
791 	SEGKMEM_BADOP(void),		/* free */
792 	segkmem_fault,
793 	SEGKMEM_BADOP(faultcode_t),	/* faulta */
794 	segkmem_setprot,
795 	segkmem_checkprot,
796 	segkmem_kluster,
797 	SEGKMEM_BADOP(size_t),		/* swapout */
798 	SEGKMEM_BADOP(int),		/* sync */
799 	SEGKMEM_BADOP(size_t),		/* incore */
800 	SEGKMEM_BADOP(int),		/* lockop */
801 	SEGKMEM_BADOP(int),		/* getprot */
802 	SEGKMEM_BADOP(u_offset_t),	/* getoffset */
803 	SEGKMEM_BADOP(int),		/* gettype */
804 	SEGKMEM_BADOP(int),		/* getvp */
805 	SEGKMEM_BADOP(int),		/* advise */
806 	segkmem_dump,
807 	segkmem_pagelock,
808 	SEGKMEM_BADOP(int),		/* setpgsz */
809 	segkmem_getmemid,
810 	segkmem_getpolicy,		/* getpolicy */
811 	segkmem_capable,		/* capable */
812 	seg_inherit_notsup		/* inherit */
813 };
814 
815 int
segkmem_create(struct seg * seg)816 segkmem_create(struct seg *seg)
817 {
818 	ASSERT(seg->s_as == &kas && RW_WRITE_HELD(&kas.a_lock));
819 	seg->s_ops = &segkmem_ops;
820 	if (seg == &kzioseg)
821 		seg->s_data = &kvps[KV_ZVP];
822 #if defined(__amd64)
823 	else if (seg == &kvmmseg)
824 		seg->s_data = &kvps[KV_VVP];
825 #endif
826 	else
827 		seg->s_data = &kvps[KV_KVP];
828 	kas.a_size += seg->s_size;
829 	return (0);
830 }
831 
832 /*ARGSUSED*/
833 page_t *
segkmem_page_create(void * addr,size_t size,int vmflag,void * arg)834 segkmem_page_create(void *addr, size_t size, int vmflag, void *arg)
835 {
836 	struct seg kseg = { 0 };
837 	int pgflags = PG_EXCL;
838 	struct vnode *vp = arg;
839 
840 	if (vp == NULL)
841 		vp = &kvp;
842 
843 	kseg.s_as = &kas;
844 
845 	if (segkmem_reloc == 0 || (vmflag & VM_NORELOC))
846 		pgflags |= PG_NORELOC;
847 	if ((vmflag & VM_NOSLEEP) == 0)
848 		pgflags |= PG_WAIT;
849 	if (vmflag & VM_PANIC)
850 		pgflags |= PG_PANIC;
851 	if (vmflag & VM_PUSHPAGE)
852 		pgflags |= PG_PUSHPAGE;
853 	if (vmflag & VM_NORMALPRI) {
854 		ASSERT(vmflag & VM_NOSLEEP);
855 		pgflags |= PG_NORMALPRI;
856 	}
857 
858 	return (page_create_va(vp, (u_offset_t)(uintptr_t)addr, size,
859 	    pgflags, &kseg, addr));
860 }
861 
862 /*
863  * Allocate pages to back the virtual address range [addr, addr + size).
864  * If addr is NULL, allocate the virtual address space as well.
865  */
866 void *
segkmem_xalloc(vmem_t * vmp,void * inaddr,size_t size,int vmflag,uint_t attr,page_t * (* page_create_func)(void *,size_t,int,void *),void * pcarg)867 segkmem_xalloc(vmem_t *vmp, void *inaddr, size_t size, int vmflag, uint_t attr,
868     page_t *(*page_create_func)(void *, size_t, int, void *), void *pcarg)
869 {
870 	page_t *ppl;
871 	caddr_t addr = inaddr;
872 	pgcnt_t npages = btopr(size);
873 	int allocflag;
874 
875 	if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL)
876 		return (NULL);
877 
878 	ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
879 
880 	if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) {
881 		if (inaddr == NULL)
882 			vmem_free(vmp, addr, size);
883 		return (NULL);
884 	}
885 
886 	ppl = page_create_func(addr, size, vmflag, pcarg);
887 	if (ppl == NULL) {
888 		if (inaddr == NULL)
889 			vmem_free(vmp, addr, size);
890 		page_unresv(npages);
891 		return (NULL);
892 	}
893 
894 	/*
895 	 * Under certain conditions, we need to let the HAT layer know
896 	 * that it cannot safely allocate memory.  Allocations from
897 	 * the hat_memload vmem arena always need this, to prevent
898 	 * infinite recursion.
899 	 *
900 	 * In addition, the x86 hat cannot safely do memory
901 	 * allocations while in vmem_populate(), because there
902 	 * is no simple bound on its usage.
903 	 */
904 	if (vmflag & VM_MEMLOAD)
905 		allocflag = HAT_NO_KALLOC;
906 #if defined(__x86)
907 	else if (vmem_is_populator())
908 		allocflag = HAT_NO_KALLOC;
909 #endif
910 	else
911 		allocflag = 0;
912 
913 	while (ppl != NULL) {
914 		page_t *pp = ppl;
915 		page_sub(&ppl, pp);
916 		ASSERT(page_iolock_assert(pp));
917 		ASSERT(PAGE_EXCL(pp));
918 		page_io_unlock(pp);
919 		hat_memload(kas.a_hat, (caddr_t)(uintptr_t)pp->p_offset, pp,
920 		    (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr,
921 		    HAT_LOAD_LOCK | allocflag);
922 		pp->p_lckcnt = 1;
923 #if defined(__x86)
924 		page_downgrade(pp);
925 #else
926 		if (vmflag & SEGKMEM_SHARELOCKED)
927 			page_downgrade(pp);
928 		else
929 			page_unlock(pp);
930 #endif
931 	}
932 
933 	return (addr);
934 }
935 
936 static void *
segkmem_alloc_vn(vmem_t * vmp,size_t size,int vmflag,struct vnode * vp)937 segkmem_alloc_vn(vmem_t *vmp, size_t size, int vmflag, struct vnode *vp)
938 {
939 	void *addr;
940 	segkmem_gc_list_t *gcp, **prev_gcpp;
941 
942 	ASSERT(vp != NULL);
943 
944 	if (kvseg.s_base == NULL) {
945 #ifndef __sparc
946 		if (bootops->bsys_alloc == NULL)
947 			halt("Memory allocation between bop_alloc() and "
948 			    "kmem_alloc().\n");
949 #endif
950 
951 		/*
952 		 * There's not a lot of memory to go around during boot,
953 		 * so recycle it if we can.
954 		 */
955 		for (prev_gcpp = &segkmem_gc_list; (gcp = *prev_gcpp) != NULL;
956 		    prev_gcpp = &gcp->gc_next) {
957 			if (gcp->gc_arena == vmp && gcp->gc_size == size) {
958 				*prev_gcpp = gcp->gc_next;
959 				return (gcp);
960 			}
961 		}
962 
963 		addr = vmem_alloc(vmp, size, vmflag | VM_PANIC);
964 		if (boot_alloc(addr, size, BO_NO_ALIGN) != addr)
965 			panic("segkmem_alloc: boot_alloc failed");
966 		return (addr);
967 	}
968 	return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
969 	    segkmem_page_create, vp));
970 }
971 
972 void *
segkmem_alloc(vmem_t * vmp,size_t size,int vmflag)973 segkmem_alloc(vmem_t *vmp, size_t size, int vmflag)
974 {
975 	return (segkmem_alloc_vn(vmp, size, vmflag, &kvp));
976 }
977 
978 static void *
segkmem_zio_alloc(vmem_t * vmp,size_t size,int vmflag)979 segkmem_zio_alloc(vmem_t *vmp, size_t size, int vmflag)
980 {
981 	return (segkmem_alloc_vn(vmp, size, vmflag, &kvps[KV_ZVP]));
982 }
983 
984 /*
985  * Any changes to this routine must also be carried over to
986  * devmap_free_pages() in the seg_dev driver. This is because
987  * we currently don't have a special kernel segment for non-paged
988  * kernel memory that is exported by drivers to user space.
989  */
990 void
segkmem_xfree(vmem_t * vmp,void * inaddr,size_t size,struct vnode * vp,void (* func)(page_t *))991 segkmem_xfree(vmem_t *vmp, void *inaddr, size_t size, struct vnode *vp,
992     void (*func)(page_t *))
993 {
994 	page_t *pp;
995 	caddr_t addr = inaddr;
996 	caddr_t eaddr;
997 	pgcnt_t npages = btopr(size);
998 
999 	ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
1000 	ASSERT(vp != NULL);
1001 
1002 	if (kvseg.s_base == NULL) {
1003 		segkmem_gc_list_t *gc = inaddr;
1004 		gc->gc_arena = vmp;
1005 		gc->gc_size = size;
1006 		gc->gc_next = segkmem_gc_list;
1007 		segkmem_gc_list = gc;
1008 		return;
1009 	}
1010 
1011 	hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1012 
1013 	for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
1014 #if defined(__x86)
1015 		pp = page_find(vp, (u_offset_t)(uintptr_t)addr);
1016 		if (pp == NULL)
1017 			panic("segkmem_free: page not found");
1018 		if (!page_tryupgrade(pp)) {
1019 			/*
1020 			 * Some other thread has a sharelock. Wait for
1021 			 * it to drop the lock so we can free this page.
1022 			 */
1023 			page_unlock(pp);
1024 			pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr,
1025 			    SE_EXCL);
1026 		}
1027 #else
1028 		pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr, SE_EXCL);
1029 #endif
1030 		if (pp == NULL)
1031 			panic("segkmem_free: page not found");
1032 		/* Clear p_lckcnt so page_destroy() doesn't update availrmem */
1033 		pp->p_lckcnt = 0;
1034 		if (func)
1035 			func(pp);
1036 		else
1037 			page_destroy(pp, 0);
1038 	}
1039 	if (func == NULL)
1040 		page_unresv(npages);
1041 
1042 	if (vmp != NULL)
1043 		vmem_free(vmp, inaddr, size);
1044 
1045 }
1046 
1047 void
segkmem_free(vmem_t * vmp,void * inaddr,size_t size)1048 segkmem_free(vmem_t *vmp, void *inaddr, size_t size)
1049 {
1050 	segkmem_xfree(vmp, inaddr, size, &kvp, NULL);
1051 }
1052 
1053 static void
segkmem_zio_free(vmem_t * vmp,void * inaddr,size_t size)1054 segkmem_zio_free(vmem_t *vmp, void *inaddr, size_t size)
1055 {
1056 	segkmem_xfree(vmp, inaddr, size, &kvps[KV_ZVP], NULL);
1057 }
1058 
1059 void
segkmem_gc(void)1060 segkmem_gc(void)
1061 {
1062 	ASSERT(kvseg.s_base != NULL);
1063 	while (segkmem_gc_list != NULL) {
1064 		segkmem_gc_list_t *gc = segkmem_gc_list;
1065 		segkmem_gc_list = gc->gc_next;
1066 		segkmem_free(gc->gc_arena, gc, gc->gc_size);
1067 	}
1068 }
1069 
1070 /*
1071  * Legacy entry points from here to end of file.
1072  */
1073 void
segkmem_mapin(struct seg * seg,void * addr,size_t size,uint_t vprot,pfn_t pfn,uint_t flags)1074 segkmem_mapin(struct seg *seg, void *addr, size_t size, uint_t vprot,
1075     pfn_t pfn, uint_t flags)
1076 {
1077 	hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1078 	hat_devload(seg->s_as->a_hat, addr, size, pfn, vprot,
1079 	    flags | HAT_LOAD_LOCK);
1080 }
1081 
1082 void
segkmem_mapout(struct seg * seg,void * addr,size_t size)1083 segkmem_mapout(struct seg *seg, void *addr, size_t size)
1084 {
1085 	hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1086 }
1087 
1088 void *
kmem_getpages(pgcnt_t npages,int kmflag)1089 kmem_getpages(pgcnt_t npages, int kmflag)
1090 {
1091 	return (kmem_alloc(ptob(npages), kmflag));
1092 }
1093 
1094 void
kmem_freepages(void * addr,pgcnt_t npages)1095 kmem_freepages(void *addr, pgcnt_t npages)
1096 {
1097 	kmem_free(addr, ptob(npages));
1098 }
1099 
1100 /*
1101  * segkmem_page_create_large() allocates a large page to be used for the kmem
1102  * caches. If kpr is enabled we ask for a relocatable page unless requested
1103  * otherwise. If kpr is disabled we have to ask for a non-reloc page
1104  */
1105 static page_t *
segkmem_page_create_large(void * addr,size_t size,int vmflag,void * arg)1106 segkmem_page_create_large(void *addr, size_t size, int vmflag, void *arg)
1107 {
1108 	int pgflags;
1109 
1110 	pgflags = PG_EXCL;
1111 
1112 	if (segkmem_reloc == 0 || (vmflag & VM_NORELOC))
1113 		pgflags |= PG_NORELOC;
1114 	if (!(vmflag & VM_NOSLEEP))
1115 		pgflags |= PG_WAIT;
1116 	if (vmflag & VM_PUSHPAGE)
1117 		pgflags |= PG_PUSHPAGE;
1118 	if (vmflag & VM_NORMALPRI)
1119 		pgflags |= PG_NORMALPRI;
1120 
1121 	return (page_create_va_large(&kvp, (u_offset_t)(uintptr_t)addr, size,
1122 	    pgflags, &kvseg, addr, arg));
1123 }
1124 
1125 /*
1126  * Allocate a large page to back the virtual address range
1127  * [addr, addr + size).  If addr is NULL, allocate the virtual address
1128  * space as well.
1129  */
1130 static void *
segkmem_xalloc_lp(vmem_t * vmp,void * inaddr,size_t size,int vmflag,uint_t attr,page_t * (* page_create_func)(void *,size_t,int,void *),void * pcarg)1131 segkmem_xalloc_lp(vmem_t *vmp, void *inaddr, size_t size, int vmflag,
1132     uint_t attr, page_t *(*page_create_func)(void *, size_t, int, void *),
1133     void *pcarg)
1134 {
1135 	caddr_t addr = inaddr, pa;
1136 	size_t  lpsize = segkmem_lpsize;
1137 	pgcnt_t npages = btopr(size);
1138 	pgcnt_t nbpages = btop(lpsize);
1139 	pgcnt_t nlpages = size >> segkmem_lpshift;
1140 	size_t  ppasize = nbpages * sizeof (page_t *);
1141 	page_t *pp, *rootpp, **ppa, *pplist = NULL;
1142 	int i;
1143 
1144 	vmflag |= VM_NOSLEEP;
1145 
1146 	if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) {
1147 		return (NULL);
1148 	}
1149 
1150 	/*
1151 	 * allocate an array we need for hat_memload_array.
1152 	 * we use a separate arena to avoid recursion.
1153 	 * we will not need this array when hat_memload_array learns pp++
1154 	 */
1155 	if ((ppa = vmem_alloc(segkmem_ppa_arena, ppasize, vmflag)) == NULL) {
1156 		goto fail_array_alloc;
1157 	}
1158 
1159 	if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL)
1160 		goto fail_vmem_alloc;
1161 
1162 	ASSERT(((uintptr_t)addr & (lpsize - 1)) == 0);
1163 
1164 	/* create all the pages */
1165 	for (pa = addr, i = 0; i < nlpages; i++, pa += lpsize) {
1166 		if ((pp = page_create_func(pa, lpsize, vmflag, pcarg)) == NULL)
1167 			goto fail_page_create;
1168 		page_list_concat(&pplist, &pp);
1169 	}
1170 
1171 	/* at this point we have all the resource to complete the request */
1172 	while ((rootpp = pplist) != NULL) {
1173 		for (i = 0; i < nbpages; i++) {
1174 			ASSERT(pplist != NULL);
1175 			pp = pplist;
1176 			page_sub(&pplist, pp);
1177 			ASSERT(page_iolock_assert(pp));
1178 			page_io_unlock(pp);
1179 			ppa[i] = pp;
1180 		}
1181 		/*
1182 		 * Load the locked entry. It's OK to preload the entry into the
1183 		 * TSB since we now support large mappings in the kernel TSB.
1184 		 */
1185 		hat_memload_array(kas.a_hat,
1186 		    (caddr_t)(uintptr_t)rootpp->p_offset, lpsize,
1187 		    ppa, (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr,
1188 		    HAT_LOAD_LOCK);
1189 
1190 		for (--i; i >= 0; --i) {
1191 			ppa[i]->p_lckcnt = 1;
1192 			page_unlock(ppa[i]);
1193 		}
1194 	}
1195 
1196 	vmem_free(segkmem_ppa_arena, ppa, ppasize);
1197 	return (addr);
1198 
1199 fail_page_create:
1200 	while ((rootpp = pplist) != NULL) {
1201 		for (i = 0, pp = pplist; i < nbpages; i++, pp = pplist) {
1202 			ASSERT(pp != NULL);
1203 			page_sub(&pplist, pp);
1204 			ASSERT(page_iolock_assert(pp));
1205 			page_io_unlock(pp);
1206 		}
1207 		page_destroy_pages(rootpp);
1208 	}
1209 
1210 	if (inaddr == NULL)
1211 		vmem_free(vmp, addr, size);
1212 
1213 fail_vmem_alloc:
1214 	vmem_free(segkmem_ppa_arena, ppa, ppasize);
1215 
1216 fail_array_alloc:
1217 	page_unresv(npages);
1218 
1219 	return (NULL);
1220 }
1221 
1222 static void
segkmem_free_one_lp(caddr_t addr,size_t size)1223 segkmem_free_one_lp(caddr_t addr, size_t size)
1224 {
1225 	page_t		*pp, *rootpp = NULL;
1226 	pgcnt_t		pgs_left = btopr(size);
1227 
1228 	ASSERT(size == segkmem_lpsize);
1229 
1230 	hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1231 
1232 	for (; pgs_left > 0; addr += PAGESIZE, pgs_left--) {
1233 		pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, SE_EXCL);
1234 		if (pp == NULL)
1235 			panic("segkmem_free_one_lp: page not found");
1236 		ASSERT(PAGE_EXCL(pp));
1237 		pp->p_lckcnt = 0;
1238 		if (rootpp == NULL)
1239 			rootpp = pp;
1240 	}
1241 	ASSERT(rootpp != NULL);
1242 	page_destroy_pages(rootpp);
1243 
1244 	/* page_unresv() is done by the caller */
1245 }
1246 
1247 /*
1248  * This function is called to import new spans into the vmem arenas like
1249  * kmem_default_arena and kmem_oversize_arena. It first tries to import
1250  * spans from large page arena - kmem_lp_arena. In order to do this it might
1251  * have to "upgrade the requested size" to kmem_lp_arena quantum. If
1252  * it was not able to satisfy the upgraded request it then calls regular
1253  * segkmem_alloc() that satisfies the request by importing from "*vmp" arena
1254  */
1255 /*ARGSUSED*/
1256 void *
segkmem_alloc_lp(vmem_t * vmp,size_t * sizep,size_t align,int vmflag)1257 segkmem_alloc_lp(vmem_t *vmp, size_t *sizep, size_t align, int vmflag)
1258 {
1259 	size_t size;
1260 	kthread_t *t = curthread;
1261 	segkmem_lpcb_t *lpcb = &segkmem_lpcb;
1262 
1263 	ASSERT(sizep != NULL);
1264 
1265 	size = *sizep;
1266 
1267 	if (lpcb->lp_uselp && !(t->t_flag & T_PANIC) &&
1268 	    !(vmflag & SEGKMEM_SHARELOCKED)) {
1269 
1270 		size_t kmemlp_qnt = segkmem_kmemlp_quantum;
1271 		size_t asize = P2ROUNDUP(size, kmemlp_qnt);
1272 		void  *addr = NULL;
1273 		ulong_t *lpthrtp = &lpcb->lp_throttle;
1274 		ulong_t lpthrt = *lpthrtp;
1275 		int	dowakeup = 0;
1276 		int	doalloc = 1;
1277 
1278 		ASSERT(kmem_lp_arena != NULL);
1279 		ASSERT(asize >= size);
1280 
1281 		if (lpthrt != 0) {
1282 			/* try to update the throttle value */
1283 			lpthrt = atomic_inc_ulong_nv(lpthrtp);
1284 			if (lpthrt >= segkmem_lpthrottle_max) {
1285 				lpthrt = atomic_cas_ulong(lpthrtp, lpthrt,
1286 				    segkmem_lpthrottle_max / 4);
1287 			}
1288 
1289 			/*
1290 			 * when we get above throttle start do an exponential
1291 			 * backoff at trying large pages and reaping
1292 			 */
1293 			if (lpthrt > segkmem_lpthrottle_start &&
1294 			    !ISP2(lpthrt)) {
1295 				lpcb->allocs_throttled++;
1296 				lpthrt--;
1297 				if (ISP2(lpthrt))
1298 					kmem_reap();
1299 				return (segkmem_alloc(vmp, size, vmflag));
1300 			}
1301 		}
1302 
1303 		if (!(vmflag & VM_NOSLEEP) &&
1304 		    segkmem_heaplp_quantum >= (8 * kmemlp_qnt) &&
1305 		    vmem_size(kmem_lp_arena, VMEM_FREE) <= kmemlp_qnt &&
1306 		    asize < (segkmem_heaplp_quantum - kmemlp_qnt)) {
1307 
1308 			/*
1309 			 * we are low on free memory in kmem_lp_arena
1310 			 * we let only one guy to allocate heap_lp
1311 			 * quantum size chunk that everybody is going to
1312 			 * share
1313 			 */
1314 			mutex_enter(&lpcb->lp_lock);
1315 
1316 			if (lpcb->lp_wait) {
1317 
1318 				/* we are not the first one - wait */
1319 				cv_wait(&lpcb->lp_cv, &lpcb->lp_lock);
1320 				if (vmem_size(kmem_lp_arena, VMEM_FREE) <
1321 				    kmemlp_qnt)  {
1322 					doalloc = 0;
1323 				}
1324 			} else if (vmem_size(kmem_lp_arena, VMEM_FREE) <=
1325 			    kmemlp_qnt) {
1326 
1327 				/*
1328 				 * we are the first one, make sure we import
1329 				 * a large page
1330 				 */
1331 				if (asize == kmemlp_qnt)
1332 					asize += kmemlp_qnt;
1333 				dowakeup = 1;
1334 				lpcb->lp_wait = 1;
1335 			}
1336 
1337 			mutex_exit(&lpcb->lp_lock);
1338 		}
1339 
1340 		/*
1341 		 * VM_ABORT flag prevents sleeps in vmem_xalloc when
1342 		 * large pages are not available. In that case this allocation
1343 		 * attempt will fail and we will retry allocation with small
1344 		 * pages. We also do not want to panic if this allocation fails
1345 		 * because we are going to retry.
1346 		 */
1347 		if (doalloc) {
1348 			addr = vmem_alloc(kmem_lp_arena, asize,
1349 			    (vmflag | VM_ABORT) & ~VM_PANIC);
1350 
1351 			if (dowakeup) {
1352 				mutex_enter(&lpcb->lp_lock);
1353 				ASSERT(lpcb->lp_wait != 0);
1354 				lpcb->lp_wait = 0;
1355 				cv_broadcast(&lpcb->lp_cv);
1356 				mutex_exit(&lpcb->lp_lock);
1357 			}
1358 		}
1359 
1360 		if (addr != NULL) {
1361 			*sizep = asize;
1362 			*lpthrtp = 0;
1363 			return (addr);
1364 		}
1365 
1366 		if (vmflag & VM_NOSLEEP)
1367 			lpcb->nosleep_allocs_failed++;
1368 		else
1369 			lpcb->sleep_allocs_failed++;
1370 		lpcb->alloc_bytes_failed += size;
1371 
1372 		/* if large page throttling is not started yet do it */
1373 		if (segkmem_use_lpthrottle && lpthrt == 0) {
1374 			lpthrt = atomic_cas_ulong(lpthrtp, lpthrt, 1);
1375 		}
1376 	}
1377 	return (segkmem_alloc(vmp, size, vmflag));
1378 }
1379 
1380 void
segkmem_free_lp(vmem_t * vmp,void * inaddr,size_t size)1381 segkmem_free_lp(vmem_t *vmp, void *inaddr, size_t size)
1382 {
1383 	if (kmem_lp_arena == NULL || !IS_KMEM_VA_LARGEPAGE((caddr_t)inaddr)) {
1384 		segkmem_free(vmp, inaddr, size);
1385 	} else {
1386 		vmem_free(kmem_lp_arena, inaddr, size);
1387 	}
1388 }
1389 
1390 /*
1391  * segkmem_alloc_lpi() imports virtual memory from large page heap arena
1392  * into kmem_lp arena. In the process it maps the imported segment with
1393  * large pages
1394  */
1395 static void *
segkmem_alloc_lpi(vmem_t * vmp,size_t size,int vmflag)1396 segkmem_alloc_lpi(vmem_t *vmp, size_t size, int vmflag)
1397 {
1398 	segkmem_lpcb_t *lpcb = &segkmem_lpcb;
1399 	void  *addr;
1400 
1401 	ASSERT(size != 0);
1402 	ASSERT(vmp == heap_lp_arena);
1403 
1404 	/* do not allow large page heap grow beyound limits */
1405 	if (vmem_size(vmp, VMEM_ALLOC) >= segkmem_kmemlp_max) {
1406 		lpcb->allocs_limited++;
1407 		return (NULL);
1408 	}
1409 
1410 	addr = segkmem_xalloc_lp(vmp, NULL, size, vmflag, 0,
1411 	    segkmem_page_create_large, NULL);
1412 	return (addr);
1413 }
1414 
1415 /*
1416  * segkmem_free_lpi() returns virtual memory back into large page heap arena
1417  * from kmem_lp arena. Beore doing this it unmaps the segment and frees
1418  * large pages used to map it.
1419  */
1420 static void
segkmem_free_lpi(vmem_t * vmp,void * inaddr,size_t size)1421 segkmem_free_lpi(vmem_t *vmp, void *inaddr, size_t size)
1422 {
1423 	pgcnt_t		nlpages = size >> segkmem_lpshift;
1424 	size_t		lpsize = segkmem_lpsize;
1425 	caddr_t		addr = inaddr;
1426 	pgcnt_t		npages = btopr(size);
1427 	int		i;
1428 
1429 	ASSERT(vmp == heap_lp_arena);
1430 	ASSERT(IS_KMEM_VA_LARGEPAGE(addr));
1431 	ASSERT(((uintptr_t)inaddr & (lpsize - 1)) == 0);
1432 
1433 	for (i = 0; i < nlpages; i++) {
1434 		segkmem_free_one_lp(addr, lpsize);
1435 		addr += lpsize;
1436 	}
1437 
1438 	page_unresv(npages);
1439 
1440 	vmem_free(vmp, inaddr, size);
1441 }
1442 
1443 /*
1444  * This function is called at system boot time by kmem_init right after
1445  * /etc/system file has been read. It checks based on hardware configuration
1446  * and /etc/system settings if system is going to use large pages. The
1447  * initialiazation necessary to actually start using large pages
1448  * happens later in the process after segkmem_heap_lp_init() is called.
1449  */
1450 int
segkmem_lpsetup()1451 segkmem_lpsetup()
1452 {
1453 	int use_large_pages = 0;
1454 
1455 #ifdef __sparc
1456 
1457 	size_t memtotal = physmem * PAGESIZE;
1458 
1459 	if (heap_lp_base == NULL) {
1460 		segkmem_lpsize = PAGESIZE;
1461 		return (0);
1462 	}
1463 
1464 	/* get a platform dependent value of large page size for kernel heap */
1465 	segkmem_lpsize = get_segkmem_lpsize(segkmem_lpsize);
1466 
1467 	if (segkmem_lpsize <= PAGESIZE) {
1468 		/*
1469 		 * put virtual space reserved for the large page kernel
1470 		 * back to the regular heap
1471 		 */
1472 		vmem_xfree(heap_arena, heap_lp_base,
1473 		    heap_lp_end - heap_lp_base);
1474 		heap_lp_base = NULL;
1475 		heap_lp_end = NULL;
1476 		segkmem_lpsize = PAGESIZE;
1477 		return (0);
1478 	}
1479 
1480 	/* set heap_lp quantum if necessary */
1481 	if (segkmem_heaplp_quantum == 0 || !ISP2(segkmem_heaplp_quantum) ||
1482 	    P2PHASE(segkmem_heaplp_quantum, segkmem_lpsize)) {
1483 		segkmem_heaplp_quantum = segkmem_lpsize;
1484 	}
1485 
1486 	/* set kmem_lp quantum if necessary */
1487 	if (segkmem_kmemlp_quantum == 0 || !ISP2(segkmem_kmemlp_quantum) ||
1488 	    segkmem_kmemlp_quantum > segkmem_heaplp_quantum) {
1489 		segkmem_kmemlp_quantum = segkmem_heaplp_quantum;
1490 	}
1491 
1492 	/* set total amount of memory allowed for large page kernel heap */
1493 	if (segkmem_kmemlp_max == 0) {
1494 		if (segkmem_kmemlp_pcnt == 0 || segkmem_kmemlp_pcnt > 100)
1495 			segkmem_kmemlp_pcnt = 12;
1496 		segkmem_kmemlp_max = (memtotal * segkmem_kmemlp_pcnt) / 100;
1497 	}
1498 	segkmem_kmemlp_max = P2ROUNDUP(segkmem_kmemlp_max,
1499 	    segkmem_heaplp_quantum);
1500 
1501 	/* fix lp kmem preallocation request if necesssary */
1502 	if (segkmem_kmemlp_min) {
1503 		segkmem_kmemlp_min = P2ROUNDUP(segkmem_kmemlp_min,
1504 		    segkmem_heaplp_quantum);
1505 		if (segkmem_kmemlp_min > segkmem_kmemlp_max)
1506 			segkmem_kmemlp_min = segkmem_kmemlp_max;
1507 	}
1508 
1509 	use_large_pages = 1;
1510 	segkmem_lpszc = page_szc(segkmem_lpsize);
1511 	segkmem_lpshift = page_get_shift(segkmem_lpszc);
1512 
1513 #endif
1514 	return (use_large_pages);
1515 }
1516 
1517 void
segkmem_zio_init(void * zio_mem_base,size_t zio_mem_size)1518 segkmem_zio_init(void *zio_mem_base, size_t zio_mem_size)
1519 {
1520 	ASSERT(zio_mem_base != NULL);
1521 	ASSERT(zio_mem_size != 0);
1522 
1523 	/*
1524 	 * To reduce VA space fragmentation, we set up quantum caches for the
1525 	 * smaller sizes;  we chose 32k because that translates to 128k VA
1526 	 * slabs, which matches nicely with the common 128k zio_data bufs.
1527 	 */
1528 	zio_arena = vmem_create("zfs_file_data", zio_mem_base, zio_mem_size,
1529 	    PAGESIZE, NULL, NULL, NULL, 32 * 1024, VM_SLEEP);
1530 
1531 	zio_alloc_arena = vmem_create("zfs_file_data_buf", NULL, 0, PAGESIZE,
1532 	    segkmem_zio_alloc, segkmem_zio_free, zio_arena, 0, VM_SLEEP);
1533 
1534 	ASSERT(zio_arena != NULL);
1535 	ASSERT(zio_alloc_arena != NULL);
1536 }
1537 
1538 #if defined(__amd64)
1539 
1540 void
segkmem_kvmm_init(void * base,size_t size)1541 segkmem_kvmm_init(void *base, size_t size)
1542 {
1543 	ASSERT(base != NULL);
1544 	ASSERT(size != 0);
1545 
1546 	kvmm_arena = vmem_create("kvmm_arena", base, size, 1024 * 1024,
1547 	    NULL, NULL, NULL, 0, VM_SLEEP);
1548 
1549 	ASSERT(kvmm_arena != NULL);
1550 }
1551 
1552 #elif defined(__sparc)
1553 
1554 static void *
segkmem_alloc_ppa(vmem_t * vmp,size_t size,int vmflag)1555 segkmem_alloc_ppa(vmem_t *vmp, size_t size, int vmflag)
1556 {
1557 	size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *);
1558 	void   *addr;
1559 
1560 	if (ppaquantum <= PAGESIZE)
1561 		return (segkmem_alloc(vmp, size, vmflag));
1562 
1563 	ASSERT((size & (ppaquantum - 1)) == 0);
1564 
1565 	addr = vmem_xalloc(vmp, size, ppaquantum, 0, 0, NULL, NULL, vmflag);
1566 	if (addr != NULL && segkmem_xalloc(vmp, addr, size, vmflag, 0,
1567 	    segkmem_page_create, NULL) == NULL) {
1568 		vmem_xfree(vmp, addr, size);
1569 		addr = NULL;
1570 	}
1571 
1572 	return (addr);
1573 }
1574 
1575 static void
segkmem_free_ppa(vmem_t * vmp,void * addr,size_t size)1576 segkmem_free_ppa(vmem_t *vmp, void *addr, size_t size)
1577 {
1578 	size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *);
1579 
1580 	ASSERT(addr != NULL);
1581 
1582 	if (ppaquantum <= PAGESIZE) {
1583 		segkmem_free(vmp, addr, size);
1584 	} else {
1585 		segkmem_free(NULL, addr, size);
1586 		vmem_xfree(vmp, addr, size);
1587 	}
1588 }
1589 
1590 void
segkmem_heap_lp_init()1591 segkmem_heap_lp_init()
1592 {
1593 	segkmem_lpcb_t *lpcb = &segkmem_lpcb;
1594 	size_t heap_lp_size = heap_lp_end - heap_lp_base;
1595 	size_t lpsize = segkmem_lpsize;
1596 	size_t ppaquantum;
1597 	void   *addr;
1598 
1599 	if (segkmem_lpsize <= PAGESIZE) {
1600 		ASSERT(heap_lp_base == NULL);
1601 		ASSERT(heap_lp_end == NULL);
1602 		return;
1603 	}
1604 
1605 	ASSERT(segkmem_heaplp_quantum >= lpsize);
1606 	ASSERT((segkmem_heaplp_quantum & (lpsize - 1)) == 0);
1607 	ASSERT(lpcb->lp_uselp == 0);
1608 	ASSERT(heap_lp_base != NULL);
1609 	ASSERT(heap_lp_end != NULL);
1610 	ASSERT(heap_lp_base < heap_lp_end);
1611 	ASSERT(heap_lp_arena == NULL);
1612 	ASSERT(((uintptr_t)heap_lp_base & (lpsize - 1)) == 0);
1613 	ASSERT(((uintptr_t)heap_lp_end & (lpsize - 1)) == 0);
1614 
1615 	/* create large page heap arena */
1616 	heap_lp_arena = vmem_create("heap_lp", heap_lp_base, heap_lp_size,
1617 	    segkmem_heaplp_quantum, NULL, NULL, NULL, 0, VM_SLEEP);
1618 
1619 	ASSERT(heap_lp_arena != NULL);
1620 
1621 	/* This arena caches memory already mapped by large pages */
1622 	kmem_lp_arena = vmem_create("kmem_lp", NULL, 0, segkmem_kmemlp_quantum,
1623 	    segkmem_alloc_lpi, segkmem_free_lpi, heap_lp_arena, 0, VM_SLEEP);
1624 
1625 	ASSERT(kmem_lp_arena != NULL);
1626 
1627 	mutex_init(&lpcb->lp_lock, NULL, MUTEX_DEFAULT, NULL);
1628 	cv_init(&lpcb->lp_cv, NULL, CV_DEFAULT, NULL);
1629 
1630 	/*
1631 	 * this arena is used for the array of page_t pointers necessary
1632 	 * to call hat_mem_load_array
1633 	 */
1634 	ppaquantum = btopr(lpsize) * sizeof (page_t *);
1635 	segkmem_ppa_arena = vmem_create("segkmem_ppa", NULL, 0, ppaquantum,
1636 	    segkmem_alloc_ppa, segkmem_free_ppa, heap_arena, ppaquantum,
1637 	    VM_SLEEP);
1638 
1639 	ASSERT(segkmem_ppa_arena != NULL);
1640 
1641 	/* prealloacate some memory for the lp kernel heap */
1642 	if (segkmem_kmemlp_min) {
1643 
1644 		ASSERT(P2PHASE(segkmem_kmemlp_min,
1645 		    segkmem_heaplp_quantum) == 0);
1646 
1647 		if ((addr = segkmem_alloc_lpi(heap_lp_arena,
1648 		    segkmem_kmemlp_min, VM_SLEEP)) != NULL) {
1649 
1650 			addr = vmem_add(kmem_lp_arena, addr,
1651 			    segkmem_kmemlp_min, VM_SLEEP);
1652 			ASSERT(addr != NULL);
1653 		}
1654 	}
1655 
1656 	lpcb->lp_uselp = 1;
1657 }
1658 
1659 #endif
1660