xref: /illumos-gate/usr/src/uts/common/vm/seg_kp.c (revision 11494be0)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 
25 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
26 /*	All Rights Reserved   */
27 
28 /*
29  * Portions of this source code were derived from Berkeley 4.3 BSD
30  * under license from the Regents of the University of California.
31  */
32 
33 /*
34  * segkp is a segment driver that administers the allocation and deallocation
35  * of pageable variable size chunks of kernel virtual address space. Each
36  * allocated resource is page-aligned.
37  *
38  * The user may specify whether the resource should be initialized to 0,
39  * include a redzone, or locked in memory.
40  */
41 
42 #include <sys/types.h>
43 #include <sys/t_lock.h>
44 #include <sys/thread.h>
45 #include <sys/param.h>
46 #include <sys/errno.h>
47 #include <sys/sysmacros.h>
48 #include <sys/systm.h>
49 #include <sys/buf.h>
50 #include <sys/mman.h>
51 #include <sys/vnode.h>
52 #include <sys/cmn_err.h>
53 #include <sys/swap.h>
54 #include <sys/tuneable.h>
55 #include <sys/kmem.h>
56 #include <sys/vmem.h>
57 #include <sys/cred.h>
58 #include <sys/dumphdr.h>
59 #include <sys/debug.h>
60 #include <sys/vtrace.h>
61 #include <sys/stack.h>
62 #include <sys/atomic.h>
63 #include <sys/archsystm.h>
64 #include <sys/lgrp.h>
65 
66 #include <vm/as.h>
67 #include <vm/seg.h>
68 #include <vm/seg_kp.h>
69 #include <vm/seg_kmem.h>
70 #include <vm/anon.h>
71 #include <vm/page.h>
72 #include <vm/hat.h>
73 #include <sys/bitmap.h>
74 
75 /*
76  * Private seg op routines
77  */
78 static void	segkp_badop(void);
79 static void	segkp_dump(struct seg *seg);
80 static int	segkp_checkprot(struct seg *seg, caddr_t addr, size_t len,
81 			uint_t prot);
82 static int	segkp_kluster(struct seg *seg, caddr_t addr, ssize_t delta);
83 static int	segkp_pagelock(struct seg *seg, caddr_t addr, size_t len,
84 			struct page ***page, enum lock_type type,
85 			enum seg_rw rw);
86 static void	segkp_insert(struct seg *seg, struct segkp_data *kpd);
87 static void	segkp_delete(struct seg *seg, struct segkp_data *kpd);
88 static caddr_t	segkp_get_internal(struct seg *seg, size_t len, uint_t flags,
89 			struct segkp_data **tkpd, struct anon_map *amp);
90 static void	segkp_release_internal(struct seg *seg,
91 			struct segkp_data *kpd, size_t len);
92 static int	segkp_unlock(struct hat *hat, struct seg *seg, caddr_t vaddr,
93 			size_t len, struct segkp_data *kpd, uint_t flags);
94 static int	segkp_load(struct hat *hat, struct seg *seg, caddr_t vaddr,
95 			size_t len, struct segkp_data *kpd, uint_t flags);
96 static struct	segkp_data *segkp_find(struct seg *seg, caddr_t vaddr);
97 static int	segkp_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp);
98 static lgrp_mem_policy_info_t	*segkp_getpolicy(struct seg *seg,
99     caddr_t addr);
100 static int	segkp_capable(struct seg *seg, segcapability_t capability);
101 
102 /*
103  * Lock used to protect the hash table(s) and caches.
104  */
105 static kmutex_t	segkp_lock;
106 
107 /*
108  * The segkp caches
109  */
110 static struct segkp_cache segkp_cache[SEGKP_MAX_CACHE];
111 
112 #define	SEGKP_BADOP(t)	(t(*)())segkp_badop
113 
114 /*
115  * When there are fewer than red_minavail bytes left on the stack,
116  * segkp_map_red() will map in the redzone (if called).  5000 seems
117  * to work reasonably well...
118  */
119 long		red_minavail = 5000;
120 
121 /*
122  * will be set to 1 for 32 bit x86 systems only, in startup.c
123  */
124 int	segkp_fromheap = 0;
125 ulong_t *segkp_bitmap;
126 
127 /*
128  * If segkp_map_red() is called with the redzone already mapped and
129  * with less than RED_DEEP_THRESHOLD bytes available on the stack,
130  * then the stack situation has become quite serious;  if much more stack
131  * is consumed, we have the potential of scrogging the next thread/LWP
132  * structure.  To help debug the "can't happen" panics which may
133  * result from this condition, we record hrestime and the calling thread
134  * in red_deep_hires and red_deep_thread respectively.
135  */
136 #define	RED_DEEP_THRESHOLD	2000
137 
138 hrtime_t	red_deep_hires;
139 kthread_t	*red_deep_thread;
140 
141 uint32_t	red_nmapped;
142 uint32_t	red_closest = UINT_MAX;
143 uint32_t	red_ndoubles;
144 
145 pgcnt_t anon_segkp_pages_locked;	/* See vm/anon.h */
146 pgcnt_t anon_segkp_pages_resv;		/* anon reserved by seg_kp */
147 
148 static struct	seg_ops segkp_ops = {
149 	SEGKP_BADOP(int),		/* dup */
150 	SEGKP_BADOP(int),		/* unmap */
151 	SEGKP_BADOP(void),		/* free */
152 	segkp_fault,
153 	SEGKP_BADOP(faultcode_t),	/* faulta */
154 	SEGKP_BADOP(int),		/* setprot */
155 	segkp_checkprot,
156 	segkp_kluster,
157 	SEGKP_BADOP(size_t),		/* swapout */
158 	SEGKP_BADOP(int),		/* sync */
159 	SEGKP_BADOP(size_t),		/* incore */
160 	SEGKP_BADOP(int),		/* lockop */
161 	SEGKP_BADOP(int),		/* getprot */
162 	SEGKP_BADOP(u_offset_t),		/* getoffset */
163 	SEGKP_BADOP(int),		/* gettype */
164 	SEGKP_BADOP(int),		/* getvp */
165 	SEGKP_BADOP(int),		/* advise */
166 	segkp_dump,			/* dump */
167 	segkp_pagelock,			/* pagelock */
168 	SEGKP_BADOP(int),		/* setpgsz */
169 	segkp_getmemid,			/* getmemid */
170 	segkp_getpolicy,		/* getpolicy */
171 	segkp_capable,			/* capable */
172 };
173 
174 
175 static void
176 segkp_badop(void)
177 {
178 	panic("segkp_badop");
179 	/*NOTREACHED*/
180 }
181 
182 static void segkpinit_mem_config(struct seg *);
183 
184 static uint32_t segkp_indel;
185 
186 /*
187  * Allocate the segment specific private data struct and fill it in
188  * with the per kp segment mutex, anon ptr. array and hash table.
189  */
190 int
191 segkp_create(struct seg *seg)
192 {
193 	struct segkp_segdata *kpsd;
194 	size_t	np;
195 
196 	ASSERT(seg != NULL && seg->s_as == &kas);
197 	ASSERT(RW_WRITE_HELD(&seg->s_as->a_lock));
198 
199 	if (seg->s_size & PAGEOFFSET) {
200 		panic("Bad segkp size");
201 		/*NOTREACHED*/
202 	}
203 
204 	kpsd = kmem_zalloc(sizeof (struct segkp_segdata), KM_SLEEP);
205 
206 	/*
207 	 * Allocate the virtual memory for segkp and initialize it
208 	 */
209 	if (segkp_fromheap) {
210 		np = btop(kvseg.s_size);
211 		segkp_bitmap = kmem_zalloc(BT_SIZEOFMAP(np), KM_SLEEP);
212 		kpsd->kpsd_arena = vmem_create("segkp", NULL, 0, PAGESIZE,
213 		    vmem_alloc, vmem_free, heap_arena, 5 * PAGESIZE, VM_SLEEP);
214 	} else {
215 		segkp_bitmap = NULL;
216 		np = btop(seg->s_size);
217 		kpsd->kpsd_arena = vmem_create("segkp", seg->s_base,
218 		    seg->s_size, PAGESIZE, NULL, NULL, NULL, 5 * PAGESIZE,
219 		    VM_SLEEP);
220 	}
221 
222 	kpsd->kpsd_anon = anon_create(np, ANON_SLEEP | ANON_ALLOC_FORCE);
223 
224 	kpsd->kpsd_hash = kmem_zalloc(SEGKP_HASHSZ * sizeof (struct segkp *),
225 	    KM_SLEEP);
226 	seg->s_data = (void *)kpsd;
227 	seg->s_ops = &segkp_ops;
228 	segkpinit_mem_config(seg);
229 	return (0);
230 }
231 
232 
233 /*
234  * Find a free 'freelist' and initialize it with the appropriate attributes
235  */
236 void *
237 segkp_cache_init(struct seg *seg, int maxsize, size_t len, uint_t flags)
238 {
239 	int i;
240 
241 	if ((flags & KPD_NO_ANON) && !(flags & KPD_LOCKED))
242 		return ((void *)-1);
243 
244 	mutex_enter(&segkp_lock);
245 	for (i = 0; i < SEGKP_MAX_CACHE; i++) {
246 		if (segkp_cache[i].kpf_inuse)
247 			continue;
248 		segkp_cache[i].kpf_inuse = 1;
249 		segkp_cache[i].kpf_max = maxsize;
250 		segkp_cache[i].kpf_flags = flags;
251 		segkp_cache[i].kpf_seg = seg;
252 		segkp_cache[i].kpf_len = len;
253 		mutex_exit(&segkp_lock);
254 		return ((void *)(uintptr_t)i);
255 	}
256 	mutex_exit(&segkp_lock);
257 	return ((void *)-1);
258 }
259 
260 /*
261  * Free all the cache resources.
262  */
263 void
264 segkp_cache_free(void)
265 {
266 	struct segkp_data *kpd;
267 	struct seg *seg;
268 	int i;
269 
270 	mutex_enter(&segkp_lock);
271 	for (i = 0; i < SEGKP_MAX_CACHE; i++) {
272 		if (!segkp_cache[i].kpf_inuse)
273 			continue;
274 		/*
275 		 * Disconnect the freelist and process each element
276 		 */
277 		kpd = segkp_cache[i].kpf_list;
278 		seg = segkp_cache[i].kpf_seg;
279 		segkp_cache[i].kpf_list = NULL;
280 		segkp_cache[i].kpf_count = 0;
281 		mutex_exit(&segkp_lock);
282 
283 		while (kpd != NULL) {
284 			struct segkp_data *next;
285 
286 			next = kpd->kp_next;
287 			segkp_release_internal(seg, kpd, kpd->kp_len);
288 			kpd = next;
289 		}
290 		mutex_enter(&segkp_lock);
291 	}
292 	mutex_exit(&segkp_lock);
293 }
294 
295 /*
296  * There are 2 entries into segkp_get_internal. The first includes a cookie
297  * used to access a pool of cached segkp resources. The second does not
298  * use the cache.
299  */
300 caddr_t
301 segkp_get(struct seg *seg, size_t len, uint_t flags)
302 {
303 	struct segkp_data *kpd = NULL;
304 
305 	if (segkp_get_internal(seg, len, flags, &kpd, NULL) != NULL) {
306 		kpd->kp_cookie = -1;
307 		return (stom(kpd->kp_base, flags));
308 	}
309 	return (NULL);
310 }
311 
312 /*
313  * Return a 'cached' segkp address
314  */
315 caddr_t
316 segkp_cache_get(void *cookie)
317 {
318 	struct segkp_cache *freelist = NULL;
319 	struct segkp_data *kpd = NULL;
320 	int index = (int)(uintptr_t)cookie;
321 	struct seg *seg;
322 	size_t len;
323 	uint_t flags;
324 
325 	if (index < 0 || index >= SEGKP_MAX_CACHE)
326 		return (NULL);
327 	freelist = &segkp_cache[index];
328 
329 	mutex_enter(&segkp_lock);
330 	seg = freelist->kpf_seg;
331 	flags = freelist->kpf_flags;
332 	if (freelist->kpf_list != NULL) {
333 		kpd = freelist->kpf_list;
334 		freelist->kpf_list = kpd->kp_next;
335 		freelist->kpf_count--;
336 		mutex_exit(&segkp_lock);
337 		kpd->kp_next = NULL;
338 		segkp_insert(seg, kpd);
339 		return (stom(kpd->kp_base, flags));
340 	}
341 	len = freelist->kpf_len;
342 	mutex_exit(&segkp_lock);
343 	if (segkp_get_internal(seg, len, flags, &kpd, NULL) != NULL) {
344 		kpd->kp_cookie = index;
345 		return (stom(kpd->kp_base, flags));
346 	}
347 	return (NULL);
348 }
349 
350 caddr_t
351 segkp_get_withanonmap(
352 	struct seg *seg,
353 	size_t len,
354 	uint_t flags,
355 	struct anon_map *amp)
356 {
357 	struct segkp_data *kpd = NULL;
358 
359 	ASSERT(amp != NULL);
360 	flags |= KPD_HASAMP;
361 	if (segkp_get_internal(seg, len, flags, &kpd, amp) != NULL) {
362 		kpd->kp_cookie = -1;
363 		return (stom(kpd->kp_base, flags));
364 	}
365 	return (NULL);
366 }
367 
368 /*
369  * This does the real work of segkp allocation.
370  * Return to client base addr. len must be page-aligned. A null value is
371  * returned if there are no more vm resources (e.g. pages, swap). The len
372  * and base recorded in the private data structure include the redzone
373  * and the redzone length (if applicable). If the user requests a redzone
374  * either the first or last page is left unmapped depending whether stacks
375  * grow to low or high memory.
376  *
377  * The client may also specify a no-wait flag. If that is set then the
378  * request will choose a non-blocking path when requesting resources.
379  * The default is make the client wait.
380  */
381 static caddr_t
382 segkp_get_internal(
383 	struct seg *seg,
384 	size_t len,
385 	uint_t flags,
386 	struct segkp_data **tkpd,
387 	struct anon_map *amp)
388 {
389 	struct segkp_segdata	*kpsd = (struct segkp_segdata *)seg->s_data;
390 	struct segkp_data	*kpd;
391 	caddr_t vbase = NULL;	/* always first virtual, may not be mapped */
392 	pgcnt_t np = 0;		/* number of pages in the resource */
393 	pgcnt_t segkpindex;
394 	long i;
395 	caddr_t va;
396 	pgcnt_t pages = 0;
397 	ulong_t anon_idx = 0;
398 	int kmflag = (flags & KPD_NOWAIT) ? KM_NOSLEEP : KM_SLEEP;
399 	caddr_t s_base = (segkp_fromheap) ? kvseg.s_base : seg->s_base;
400 
401 	if (len & PAGEOFFSET) {
402 		panic("segkp_get: len is not page-aligned");
403 		/*NOTREACHED*/
404 	}
405 
406 	ASSERT(((flags & KPD_HASAMP) == 0) == (amp == NULL));
407 
408 	/* Only allow KPD_NO_ANON if we are going to lock it down */
409 	if ((flags & (KPD_LOCKED|KPD_NO_ANON)) == KPD_NO_ANON)
410 		return (NULL);
411 
412 	if ((kpd = kmem_zalloc(sizeof (struct segkp_data), kmflag)) == NULL)
413 		return (NULL);
414 	/*
415 	 * Fix up the len to reflect the REDZONE if applicable
416 	 */
417 	if (flags & KPD_HASREDZONE)
418 		len += PAGESIZE;
419 	np = btop(len);
420 
421 	vbase = vmem_alloc(SEGKP_VMEM(seg), len, kmflag | VM_BESTFIT);
422 	if (vbase == NULL) {
423 		kmem_free(kpd, sizeof (struct segkp_data));
424 		return (NULL);
425 	}
426 
427 	/* If locking, reserve physical memory */
428 	if (flags & KPD_LOCKED) {
429 		pages = btop(SEGKP_MAPLEN(len, flags));
430 		if (page_resv(pages, kmflag) == 0) {
431 			vmem_free(SEGKP_VMEM(seg), vbase, len);
432 			kmem_free(kpd, sizeof (struct segkp_data));
433 			return (NULL);
434 		}
435 		if ((flags & KPD_NO_ANON) == 0)
436 			atomic_add_long(&anon_segkp_pages_locked, pages);
437 	}
438 
439 	/*
440 	 * Reserve sufficient swap space for this vm resource.  We'll
441 	 * actually allocate it in the loop below, but reserving it
442 	 * here allows us to back out more gracefully than if we
443 	 * had an allocation failure in the body of the loop.
444 	 *
445 	 * Note that we don't need swap space for the red zone page.
446 	 */
447 	if (amp != NULL) {
448 		/*
449 		 * The swap reservation has been done, if required, and the
450 		 * anon_hdr is separate.
451 		 */
452 		anon_idx = 0;
453 		kpd->kp_anon_idx = anon_idx;
454 		kpd->kp_anon = amp->ahp;
455 
456 		TRACE_5(TR_FAC_VM, TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u",
457 		    kpd, vbase, len, flags, 1);
458 
459 	} else if ((flags & KPD_NO_ANON) == 0) {
460 		if (anon_resv_zone(SEGKP_MAPLEN(len, flags), NULL) == 0) {
461 			if (flags & KPD_LOCKED) {
462 				atomic_add_long(&anon_segkp_pages_locked,
463 				    -pages);
464 				page_unresv(pages);
465 			}
466 			vmem_free(SEGKP_VMEM(seg), vbase, len);
467 			kmem_free(kpd, sizeof (struct segkp_data));
468 			return (NULL);
469 		}
470 		atomic_add_long(&anon_segkp_pages_resv,
471 		    btop(SEGKP_MAPLEN(len, flags)));
472 		anon_idx = ((uintptr_t)(vbase - s_base)) >> PAGESHIFT;
473 		kpd->kp_anon_idx = anon_idx;
474 		kpd->kp_anon = kpsd->kpsd_anon;
475 
476 		TRACE_5(TR_FAC_VM, TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u",
477 		    kpd, vbase, len, flags, 1);
478 	} else {
479 		kpd->kp_anon = NULL;
480 		kpd->kp_anon_idx = 0;
481 	}
482 
483 	/*
484 	 * Allocate page and anon resources for the virtual address range
485 	 * except the redzone
486 	 */
487 	if (segkp_fromheap)
488 		segkpindex = btop((uintptr_t)(vbase - kvseg.s_base));
489 	for (i = 0, va = vbase; i < np; i++, va += PAGESIZE) {
490 		page_t		*pl[2];
491 		struct vnode	*vp;
492 		anoff_t		off;
493 		int		err;
494 		page_t		*pp = NULL;
495 
496 		/*
497 		 * Mark this page to be a segkp page in the bitmap.
498 		 */
499 		if (segkp_fromheap) {
500 			BT_ATOMIC_SET(segkp_bitmap, segkpindex);
501 			segkpindex++;
502 		}
503 
504 		/*
505 		 * If this page is the red zone page, we don't need swap
506 		 * space for it.  Note that we skip over the code that
507 		 * establishes MMU mappings, so that the page remains
508 		 * invalid.
509 		 */
510 		if ((flags & KPD_HASREDZONE) && KPD_REDZONE(kpd) == i)
511 			continue;
512 
513 		if (kpd->kp_anon != NULL) {
514 			struct anon *ap;
515 
516 			ASSERT(anon_get_ptr(kpd->kp_anon, anon_idx + i)
517 			    == NULL);
518 			/*
519 			 * Determine the "vp" and "off" of the anon slot.
520 			 */
521 			ap = anon_alloc(NULL, 0);
522 			if (amp != NULL)
523 				ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
524 			(void) anon_set_ptr(kpd->kp_anon, anon_idx + i,
525 			    ap, ANON_SLEEP);
526 			if (amp != NULL)
527 				ANON_LOCK_EXIT(&amp->a_rwlock);
528 			swap_xlate(ap, &vp, &off);
529 
530 			/*
531 			 * Create a page with the specified identity.  The
532 			 * page is returned with the "shared" lock held.
533 			 */
534 			err = VOP_GETPAGE(vp, (offset_t)off, PAGESIZE,
535 			    NULL, pl, PAGESIZE, seg, va, S_CREATE,
536 			    kcred, NULL);
537 			if (err) {
538 				/*
539 				 * XXX - This should not fail.
540 				 */
541 				panic("segkp_get: no pages");
542 				/*NOTREACHED*/
543 			}
544 			pp = pl[0];
545 		} else {
546 			ASSERT(page_exists(&kvp,
547 			    (u_offset_t)(uintptr_t)va) == NULL);
548 
549 			if ((pp = page_create_va(&kvp,
550 			    (u_offset_t)(uintptr_t)va, PAGESIZE,
551 			    (flags & KPD_NOWAIT ? 0 : PG_WAIT) | PG_EXCL |
552 			    PG_NORELOC, seg, va)) == NULL) {
553 				/*
554 				 * Legitimize resource; then destroy it.
555 				 * Easier than trying to unwind here.
556 				 */
557 				kpd->kp_flags = flags;
558 				kpd->kp_base = vbase;
559 				kpd->kp_len = len;
560 				segkp_release_internal(seg, kpd, va - vbase);
561 				return (NULL);
562 			}
563 			page_io_unlock(pp);
564 		}
565 
566 		if (flags & KPD_ZERO)
567 			pagezero(pp, 0, PAGESIZE);
568 
569 		/*
570 		 * Load and lock an MMU translation for the page.
571 		 */
572 		hat_memload(seg->s_as->a_hat, va, pp, (PROT_READ|PROT_WRITE),
573 		    ((flags & KPD_LOCKED) ? HAT_LOAD_LOCK : HAT_LOAD));
574 
575 		/*
576 		 * Now, release lock on the page.
577 		 */
578 		if (flags & KPD_LOCKED) {
579 			/*
580 			 * Indicate to page_retire framework that this
581 			 * page can only be retired when it is freed.
582 			 */
583 			PP_SETRAF(pp);
584 			page_downgrade(pp);
585 		} else
586 			page_unlock(pp);
587 	}
588 
589 	kpd->kp_flags = flags;
590 	kpd->kp_base = vbase;
591 	kpd->kp_len = len;
592 	segkp_insert(seg, kpd);
593 	*tkpd = kpd;
594 	return (stom(kpd->kp_base, flags));
595 }
596 
597 /*
598  * Release the resource to cache if the pool(designate by the cookie)
599  * has less than the maximum allowable. If inserted in cache,
600  * segkp_delete insures element is taken off of active list.
601  */
602 void
603 segkp_release(struct seg *seg, caddr_t vaddr)
604 {
605 	struct segkp_cache *freelist;
606 	struct segkp_data *kpd = NULL;
607 
608 	if ((kpd = segkp_find(seg, vaddr)) == NULL) {
609 		panic("segkp_release: null kpd");
610 		/*NOTREACHED*/
611 	}
612 
613 	if (kpd->kp_cookie != -1) {
614 		freelist = &segkp_cache[kpd->kp_cookie];
615 		mutex_enter(&segkp_lock);
616 		if (!segkp_indel && freelist->kpf_count < freelist->kpf_max) {
617 			segkp_delete(seg, kpd);
618 			kpd->kp_next = freelist->kpf_list;
619 			freelist->kpf_list = kpd;
620 			freelist->kpf_count++;
621 			mutex_exit(&segkp_lock);
622 			return;
623 		} else {
624 			mutex_exit(&segkp_lock);
625 			kpd->kp_cookie = -1;
626 		}
627 	}
628 	segkp_release_internal(seg, kpd, kpd->kp_len);
629 }
630 
631 /*
632  * Free the entire resource. segkp_unlock gets called with the start of the
633  * mapped portion of the resource. The length is the size of the mapped
634  * portion
635  */
636 static void
637 segkp_release_internal(struct seg *seg, struct segkp_data *kpd, size_t len)
638 {
639 	caddr_t		va;
640 	long		i;
641 	long		redzone;
642 	size_t		np;
643 	page_t		*pp;
644 	struct vnode 	*vp;
645 	anoff_t		off;
646 	struct anon	*ap;
647 	pgcnt_t		segkpindex;
648 
649 	ASSERT(kpd != NULL);
650 	ASSERT((kpd->kp_flags & KPD_HASAMP) == 0 || kpd->kp_cookie == -1);
651 	np = btop(len);
652 
653 	/* Remove from active hash list */
654 	if (kpd->kp_cookie == -1) {
655 		mutex_enter(&segkp_lock);
656 		segkp_delete(seg, kpd);
657 		mutex_exit(&segkp_lock);
658 	}
659 
660 	/*
661 	 * Precompute redzone page index.
662 	 */
663 	redzone = -1;
664 	if (kpd->kp_flags & KPD_HASREDZONE)
665 		redzone = KPD_REDZONE(kpd);
666 
667 
668 	va = kpd->kp_base;
669 
670 	hat_unload(seg->s_as->a_hat, va, (np << PAGESHIFT),
671 	    ((kpd->kp_flags & KPD_LOCKED) ? HAT_UNLOAD_UNLOCK : HAT_UNLOAD));
672 	/*
673 	 * Free up those anon resources that are quiescent.
674 	 */
675 	if (segkp_fromheap)
676 		segkpindex = btop((uintptr_t)(va - kvseg.s_base));
677 	for (i = 0; i < np; i++, va += PAGESIZE) {
678 
679 		/*
680 		 * Clear the bit for this page from the bitmap.
681 		 */
682 		if (segkp_fromheap) {
683 			BT_ATOMIC_CLEAR(segkp_bitmap, segkpindex);
684 			segkpindex++;
685 		}
686 
687 		if (i == redzone)
688 			continue;
689 		if (kpd->kp_anon) {
690 			/*
691 			 * Free up anon resources and destroy the
692 			 * associated pages.
693 			 *
694 			 * Release the lock if there is one. Have to get the
695 			 * page to do this, unfortunately.
696 			 */
697 			if (kpd->kp_flags & KPD_LOCKED) {
698 				ap = anon_get_ptr(kpd->kp_anon,
699 				    kpd->kp_anon_idx + i);
700 				swap_xlate(ap, &vp, &off);
701 				/* Find the shared-locked page. */
702 				pp = page_find(vp, (u_offset_t)off);
703 				if (pp == NULL) {
704 					panic("segkp_release: "
705 					    "kp_anon: no page to unlock ");
706 					/*NOTREACHED*/
707 				}
708 				if (PP_ISRAF(pp))
709 					PP_CLRRAF(pp);
710 
711 				page_unlock(pp);
712 			}
713 			if ((kpd->kp_flags & KPD_HASAMP) == 0) {
714 				anon_free(kpd->kp_anon, kpd->kp_anon_idx + i,
715 				    PAGESIZE);
716 				anon_unresv_zone(PAGESIZE, NULL);
717 				atomic_add_long(&anon_segkp_pages_resv,
718 				    -1);
719 			}
720 			TRACE_5(TR_FAC_VM,
721 			    TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u",
722 			    kpd, va, PAGESIZE, 0, 0);
723 		} else {
724 			if (kpd->kp_flags & KPD_LOCKED) {
725 				pp = page_find(&kvp, (u_offset_t)(uintptr_t)va);
726 				if (pp == NULL) {
727 					panic("segkp_release: "
728 					    "no page to unlock");
729 					/*NOTREACHED*/
730 				}
731 				if (PP_ISRAF(pp))
732 					PP_CLRRAF(pp);
733 				/*
734 				 * We should just upgrade the lock here
735 				 * but there is no upgrade that waits.
736 				 */
737 				page_unlock(pp);
738 			}
739 			pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)va,
740 			    SE_EXCL);
741 			if (pp != NULL)
742 				page_destroy(pp, 0);
743 		}
744 	}
745 
746 	/* If locked, release physical memory reservation */
747 	if (kpd->kp_flags & KPD_LOCKED) {
748 		pgcnt_t pages = btop(SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags));
749 		if ((kpd->kp_flags & KPD_NO_ANON) == 0)
750 			atomic_add_long(&anon_segkp_pages_locked, -pages);
751 		page_unresv(pages);
752 	}
753 
754 	vmem_free(SEGKP_VMEM(seg), kpd->kp_base, kpd->kp_len);
755 	kmem_free(kpd, sizeof (struct segkp_data));
756 }
757 
758 /*
759  * segkp_map_red() will check the current frame pointer against the
760  * stack base.  If the amount of stack remaining is questionable
761  * (less than red_minavail), then segkp_map_red() will map in the redzone
762  * and return 1.  Otherwise, it will return 0.  segkp_map_red() can
763  * _only_ be called when:
764  *
765  *   - it is safe to sleep on page_create_va().
766  *   - the caller is non-swappable.
767  *
768  * It is up to the caller to remember whether segkp_map_red() successfully
769  * mapped the redzone, and, if so, to call segkp_unmap_red() at a later
770  * time.  Note that the caller must _remain_ non-swappable until after
771  * calling segkp_unmap_red().
772  *
773  * Currently, this routine is only called from pagefault() (which necessarily
774  * satisfies the above conditions).
775  */
776 #if defined(STACK_GROWTH_DOWN)
777 int
778 segkp_map_red(void)
779 {
780 	uintptr_t fp = STACK_BIAS + (uintptr_t)getfp();
781 #ifndef _LP64
782 	caddr_t stkbase;
783 #endif
784 
785 	ASSERT(curthread->t_schedflag & TS_DONT_SWAP);
786 
787 	/*
788 	 * Optimize for the common case where we simply return.
789 	 */
790 	if ((curthread->t_red_pp == NULL) &&
791 	    (fp - (uintptr_t)curthread->t_stkbase >= red_minavail))
792 		return (0);
793 
794 #if defined(_LP64)
795 	/*
796 	 * XXX	We probably need something better than this.
797 	 */
798 	panic("kernel stack overflow");
799 	/*NOTREACHED*/
800 #else /* _LP64 */
801 	if (curthread->t_red_pp == NULL) {
802 		page_t *red_pp;
803 		struct seg kseg;
804 
805 		caddr_t red_va = (caddr_t)
806 		    (((uintptr_t)curthread->t_stkbase & (uintptr_t)PAGEMASK) -
807 		    PAGESIZE);
808 
809 		ASSERT(page_exists(&kvp, (u_offset_t)(uintptr_t)red_va) ==
810 		    NULL);
811 
812 		/*
813 		 * Allocate the physical for the red page.
814 		 */
815 		/*
816 		 * No PG_NORELOC here to avoid waits. Unlikely to get
817 		 * a relocate happening in the short time the page exists
818 		 * and it will be OK anyway.
819 		 */
820 
821 		kseg.s_as = &kas;
822 		red_pp = page_create_va(&kvp, (u_offset_t)(uintptr_t)red_va,
823 		    PAGESIZE, PG_WAIT | PG_EXCL, &kseg, red_va);
824 		ASSERT(red_pp != NULL);
825 
826 		/*
827 		 * So we now have a page to jam into the redzone...
828 		 */
829 		page_io_unlock(red_pp);
830 
831 		hat_memload(kas.a_hat, red_va, red_pp,
832 		    (PROT_READ|PROT_WRITE), HAT_LOAD_LOCK);
833 		page_downgrade(red_pp);
834 
835 		/*
836 		 * The page is left SE_SHARED locked so we can hold on to
837 		 * the page_t pointer.
838 		 */
839 		curthread->t_red_pp = red_pp;
840 
841 		atomic_add_32(&red_nmapped, 1);
842 		while (fp - (uintptr_t)curthread->t_stkbase < red_closest) {
843 			(void) cas32(&red_closest, red_closest,
844 			    (uint32_t)(fp - (uintptr_t)curthread->t_stkbase));
845 		}
846 		return (1);
847 	}
848 
849 	stkbase = (caddr_t)(((uintptr_t)curthread->t_stkbase &
850 	    (uintptr_t)PAGEMASK) - PAGESIZE);
851 
852 	atomic_add_32(&red_ndoubles, 1);
853 
854 	if (fp - (uintptr_t)stkbase < RED_DEEP_THRESHOLD) {
855 		/*
856 		 * Oh boy.  We're already deep within the mapped-in
857 		 * redzone page, and the caller is trying to prepare
858 		 * for a deep stack run.  We're running without a
859 		 * redzone right now:  if the caller plows off the
860 		 * end of the stack, it'll plow another thread or
861 		 * LWP structure.  That situation could result in
862 		 * a very hard-to-debug panic, so, in the spirit of
863 		 * recording the name of one's killer in one's own
864 		 * blood, we're going to record hrestime and the calling
865 		 * thread.
866 		 */
867 		red_deep_hires = hrestime.tv_nsec;
868 		red_deep_thread = curthread;
869 	}
870 
871 	/*
872 	 * If this is a DEBUG kernel, and we've run too deep for comfort, toss.
873 	 */
874 	ASSERT(fp - (uintptr_t)stkbase >= RED_DEEP_THRESHOLD);
875 	return (0);
876 #endif /* _LP64 */
877 }
878 
879 void
880 segkp_unmap_red(void)
881 {
882 	page_t *pp;
883 	caddr_t red_va = (caddr_t)(((uintptr_t)curthread->t_stkbase &
884 	    (uintptr_t)PAGEMASK) - PAGESIZE);
885 
886 	ASSERT(curthread->t_red_pp != NULL);
887 	ASSERT(curthread->t_schedflag & TS_DONT_SWAP);
888 
889 	/*
890 	 * Because we locked the mapping down, we can't simply rely
891 	 * on page_destroy() to clean everything up;  we need to call
892 	 * hat_unload() to explicitly unlock the mapping resources.
893 	 */
894 	hat_unload(kas.a_hat, red_va, PAGESIZE, HAT_UNLOAD_UNLOCK);
895 
896 	pp = curthread->t_red_pp;
897 
898 	ASSERT(pp == page_find(&kvp, (u_offset_t)(uintptr_t)red_va));
899 
900 	/*
901 	 * Need to upgrade the SE_SHARED lock to SE_EXCL.
902 	 */
903 	if (!page_tryupgrade(pp)) {
904 		/*
905 		 * As there is now wait for upgrade, release the
906 		 * SE_SHARED lock and wait for SE_EXCL.
907 		 */
908 		page_unlock(pp);
909 		pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)red_va, SE_EXCL);
910 		/* pp may be NULL here, hence the test below */
911 	}
912 
913 	/*
914 	 * Destroy the page, with dontfree set to zero (i.e. free it).
915 	 */
916 	if (pp != NULL)
917 		page_destroy(pp, 0);
918 	curthread->t_red_pp = NULL;
919 }
920 #else
921 #error Red stacks only supported with downwards stack growth.
922 #endif
923 
924 /*
925  * Handle a fault on an address corresponding to one of the
926  * resources in the segkp segment.
927  */
928 faultcode_t
929 segkp_fault(
930 	struct hat	*hat,
931 	struct seg	*seg,
932 	caddr_t		vaddr,
933 	size_t		len,
934 	enum fault_type	type,
935 	enum seg_rw rw)
936 {
937 	struct segkp_data	*kpd = NULL;
938 	int			err;
939 
940 	ASSERT(seg->s_as == &kas && RW_READ_HELD(&seg->s_as->a_lock));
941 
942 	/*
943 	 * Sanity checks.
944 	 */
945 	if (type == F_PROT) {
946 		panic("segkp_fault: unexpected F_PROT fault");
947 		/*NOTREACHED*/
948 	}
949 
950 	if ((kpd = segkp_find(seg, vaddr)) == NULL)
951 		return (FC_NOMAP);
952 
953 	mutex_enter(&kpd->kp_lock);
954 
955 	if (type == F_SOFTLOCK) {
956 		ASSERT(!(kpd->kp_flags & KPD_LOCKED));
957 		/*
958 		 * The F_SOFTLOCK case has more stringent
959 		 * range requirements: the given range must exactly coincide
960 		 * with the resource's mapped portion. Note reference to
961 		 * redzone is handled since vaddr would not equal base
962 		 */
963 		if (vaddr != stom(kpd->kp_base, kpd->kp_flags) ||
964 		    len != SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)) {
965 			mutex_exit(&kpd->kp_lock);
966 			return (FC_MAKE_ERR(EFAULT));
967 		}
968 
969 		if ((err = segkp_load(hat, seg, vaddr, len, kpd, KPD_LOCKED))) {
970 			mutex_exit(&kpd->kp_lock);
971 			return (FC_MAKE_ERR(err));
972 		}
973 		kpd->kp_flags |= KPD_LOCKED;
974 		mutex_exit(&kpd->kp_lock);
975 		return (0);
976 	}
977 
978 	if (type == F_INVAL) {
979 		ASSERT(!(kpd->kp_flags & KPD_NO_ANON));
980 
981 		/*
982 		 * Check if we touched the redzone. Somewhat optimistic
983 		 * here if we are touching the redzone of our own stack
984 		 * since we wouldn't have a stack to get this far...
985 		 */
986 		if ((kpd->kp_flags & KPD_HASREDZONE) &&
987 		    btop((uintptr_t)(vaddr - kpd->kp_base)) == KPD_REDZONE(kpd))
988 			panic("segkp_fault: accessing redzone");
989 
990 		/*
991 		 * This fault may occur while the page is being F_SOFTLOCK'ed.
992 		 * Return since a 2nd segkp_load is unnecessary and also would
993 		 * result in the page being locked twice and eventually
994 		 * hang the thread_reaper thread.
995 		 */
996 		if (kpd->kp_flags & KPD_LOCKED) {
997 			mutex_exit(&kpd->kp_lock);
998 			return (0);
999 		}
1000 
1001 		err = segkp_load(hat, seg, vaddr, len, kpd, kpd->kp_flags);
1002 		mutex_exit(&kpd->kp_lock);
1003 		return (err ? FC_MAKE_ERR(err) : 0);
1004 	}
1005 
1006 	if (type == F_SOFTUNLOCK) {
1007 		uint_t	flags;
1008 
1009 		/*
1010 		 * Make sure the addr is LOCKED and it has anon backing
1011 		 * before unlocking
1012 		 */
1013 		if ((kpd->kp_flags & (KPD_LOCKED|KPD_NO_ANON)) != KPD_LOCKED) {
1014 			panic("segkp_fault: bad unlock");
1015 			/*NOTREACHED*/
1016 		}
1017 
1018 		if (vaddr != stom(kpd->kp_base, kpd->kp_flags) ||
1019 		    len != SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)) {
1020 			panic("segkp_fault: bad range");
1021 			/*NOTREACHED*/
1022 		}
1023 
1024 		if (rw == S_WRITE)
1025 			flags = kpd->kp_flags | KPD_WRITEDIRTY;
1026 		else
1027 			flags = kpd->kp_flags;
1028 		err = segkp_unlock(hat, seg, vaddr, len, kpd, flags);
1029 		kpd->kp_flags &= ~KPD_LOCKED;
1030 		mutex_exit(&kpd->kp_lock);
1031 		return (err ? FC_MAKE_ERR(err) : 0);
1032 	}
1033 	mutex_exit(&kpd->kp_lock);
1034 	panic("segkp_fault: bogus fault type: %d\n", type);
1035 	/*NOTREACHED*/
1036 }
1037 
1038 /*
1039  * Check that the given protections suffice over the range specified by
1040  * vaddr and len.  For this segment type, the only issue is whether or
1041  * not the range lies completely within the mapped part of an allocated
1042  * resource.
1043  */
1044 /* ARGSUSED */
1045 static int
1046 segkp_checkprot(struct seg *seg, caddr_t vaddr, size_t len, uint_t prot)
1047 {
1048 	struct segkp_data *kpd = NULL;
1049 	caddr_t mbase;
1050 	size_t mlen;
1051 
1052 	if ((kpd = segkp_find(seg, vaddr)) == NULL)
1053 		return (EACCES);
1054 
1055 	mutex_enter(&kpd->kp_lock);
1056 	mbase = stom(kpd->kp_base, kpd->kp_flags);
1057 	mlen = SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags);
1058 	if (len > mlen || vaddr < mbase ||
1059 	    ((vaddr + len) > (mbase + mlen))) {
1060 		mutex_exit(&kpd->kp_lock);
1061 		return (EACCES);
1062 	}
1063 	mutex_exit(&kpd->kp_lock);
1064 	return (0);
1065 }
1066 
1067 
1068 /*
1069  * Check to see if it makes sense to do kluster/read ahead to
1070  * addr + delta relative to the mapping at addr.  We assume here
1071  * that delta is a signed PAGESIZE'd multiple (which can be negative).
1072  *
1073  * For seg_u we always "approve" of this action from our standpoint.
1074  */
1075 /*ARGSUSED*/
1076 static int
1077 segkp_kluster(struct seg *seg, caddr_t addr, ssize_t delta)
1078 {
1079 	return (0);
1080 }
1081 
1082 /*
1083  * Load and possibly lock intra-slot resources in the range given by
1084  * vaddr and len.
1085  */
1086 static int
1087 segkp_load(
1088 	struct hat *hat,
1089 	struct seg *seg,
1090 	caddr_t vaddr,
1091 	size_t len,
1092 	struct segkp_data *kpd,
1093 	uint_t flags)
1094 {
1095 	caddr_t va;
1096 	caddr_t vlim;
1097 	ulong_t i;
1098 	uint_t lock;
1099 
1100 	ASSERT(MUTEX_HELD(&kpd->kp_lock));
1101 
1102 	len = P2ROUNDUP(len, PAGESIZE);
1103 
1104 	/* If locking, reserve physical memory */
1105 	if (flags & KPD_LOCKED) {
1106 		pgcnt_t pages = btop(len);
1107 		if ((kpd->kp_flags & KPD_NO_ANON) == 0)
1108 			atomic_add_long(&anon_segkp_pages_locked, pages);
1109 		(void) page_resv(pages, KM_SLEEP);
1110 	}
1111 
1112 	/*
1113 	 * Loop through the pages in the given range.
1114 	 */
1115 	va = (caddr_t)((uintptr_t)vaddr & (uintptr_t)PAGEMASK);
1116 	vaddr = va;
1117 	vlim = va + len;
1118 	lock = flags & KPD_LOCKED;
1119 	i = ((uintptr_t)(va - kpd->kp_base)) >> PAGESHIFT;
1120 	for (; va < vlim; va += PAGESIZE, i++) {
1121 		page_t		*pl[2];	/* second element NULL terminator */
1122 		struct vnode    *vp;
1123 		anoff_t		off;
1124 		int		err;
1125 		struct anon	*ap;
1126 
1127 		/*
1128 		 * Summon the page.  If it's not resident, arrange
1129 		 * for synchronous i/o to pull it in.
1130 		 */
1131 		ap = anon_get_ptr(kpd->kp_anon, kpd->kp_anon_idx + i);
1132 		swap_xlate(ap, &vp, &off);
1133 
1134 		/*
1135 		 * The returned page list will have exactly one entry,
1136 		 * which is returned to us already kept.
1137 		 */
1138 		err = VOP_GETPAGE(vp, (offset_t)off, PAGESIZE, NULL,
1139 		    pl, PAGESIZE, seg, va, S_READ, kcred, NULL);
1140 
1141 		if (err) {
1142 			/*
1143 			 * Back out of what we've done so far.
1144 			 */
1145 			(void) segkp_unlock(hat, seg, vaddr,
1146 			    (va - vaddr), kpd, flags);
1147 			return (err);
1148 		}
1149 
1150 		/*
1151 		 * Load an MMU translation for the page.
1152 		 */
1153 		hat_memload(hat, va, pl[0], (PROT_READ|PROT_WRITE),
1154 		    lock ? HAT_LOAD_LOCK : HAT_LOAD);
1155 
1156 		if (!lock) {
1157 			/*
1158 			 * Now, release "shared" lock on the page.
1159 			 */
1160 			page_unlock(pl[0]);
1161 		}
1162 	}
1163 	return (0);
1164 }
1165 
1166 /*
1167  * At the very least unload the mmu-translations and unlock the range if locked
1168  * Can be called with the following flag value KPD_WRITEDIRTY which specifies
1169  * any dirty pages should be written to disk.
1170  */
1171 static int
1172 segkp_unlock(
1173 	struct hat *hat,
1174 	struct seg *seg,
1175 	caddr_t vaddr,
1176 	size_t len,
1177 	struct segkp_data *kpd,
1178 	uint_t flags)
1179 {
1180 	caddr_t va;
1181 	caddr_t vlim;
1182 	ulong_t i;
1183 	struct page *pp;
1184 	struct vnode *vp;
1185 	anoff_t off;
1186 	struct anon *ap;
1187 
1188 #ifdef lint
1189 	seg = seg;
1190 #endif /* lint */
1191 
1192 	ASSERT(MUTEX_HELD(&kpd->kp_lock));
1193 
1194 	/*
1195 	 * Loop through the pages in the given range. It is assumed
1196 	 * segkp_unlock is called with page aligned base
1197 	 */
1198 	va = vaddr;
1199 	vlim = va + len;
1200 	i = ((uintptr_t)(va - kpd->kp_base)) >> PAGESHIFT;
1201 	hat_unload(hat, va, len,
1202 	    ((flags & KPD_LOCKED) ? HAT_UNLOAD_UNLOCK : HAT_UNLOAD));
1203 	for (; va < vlim; va += PAGESIZE, i++) {
1204 		/*
1205 		 * Find the page associated with this part of the
1206 		 * slot, tracking it down through its associated swap
1207 		 * space.
1208 		 */
1209 		ap = anon_get_ptr(kpd->kp_anon, kpd->kp_anon_idx + i);
1210 		swap_xlate(ap, &vp, &off);
1211 
1212 		if (flags & KPD_LOCKED) {
1213 			if ((pp = page_find(vp, off)) == NULL) {
1214 				if (flags & KPD_LOCKED) {
1215 					panic("segkp_softunlock: missing page");
1216 					/*NOTREACHED*/
1217 				}
1218 			}
1219 		} else {
1220 			/*
1221 			 * Nothing to do if the slot is not locked and the
1222 			 * page doesn't exist.
1223 			 */
1224 			if ((pp = page_lookup(vp, off, SE_SHARED)) == NULL)
1225 				continue;
1226 		}
1227 
1228 		/*
1229 		 * If the page doesn't have any translations, is
1230 		 * dirty and not being shared, then push it out
1231 		 * asynchronously and avoid waiting for the
1232 		 * pageout daemon to do it for us.
1233 		 *
1234 		 * XXX - Do we really need to get the "exclusive"
1235 		 * lock via an upgrade?
1236 		 */
1237 		if ((flags & KPD_WRITEDIRTY) && !hat_page_is_mapped(pp) &&
1238 		    hat_ismod(pp) && page_tryupgrade(pp)) {
1239 			/*
1240 			 * Hold the vnode before releasing the page lock to
1241 			 * prevent it from being freed and re-used by some
1242 			 * other thread.
1243 			 */
1244 			VN_HOLD(vp);
1245 			page_unlock(pp);
1246 
1247 			/*
1248 			 * Want most powerful credentials we can get so
1249 			 * use kcred.
1250 			 */
1251 			(void) VOP_PUTPAGE(vp, (offset_t)off, PAGESIZE,
1252 			    B_ASYNC | B_FREE, kcred, NULL);
1253 			VN_RELE(vp);
1254 		} else {
1255 			page_unlock(pp);
1256 		}
1257 	}
1258 
1259 	/* If unlocking, release physical memory */
1260 	if (flags & KPD_LOCKED) {
1261 		pgcnt_t pages = btopr(len);
1262 		if ((kpd->kp_flags & KPD_NO_ANON) == 0)
1263 			atomic_add_long(&anon_segkp_pages_locked, -pages);
1264 		page_unresv(pages);
1265 	}
1266 	return (0);
1267 }
1268 
1269 /*
1270  * Insert the kpd in the hash table.
1271  */
1272 static void
1273 segkp_insert(struct seg *seg, struct segkp_data *kpd)
1274 {
1275 	struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
1276 	int index;
1277 
1278 	/*
1279 	 * Insert the kpd based on the address that will be returned
1280 	 * via segkp_release.
1281 	 */
1282 	index = SEGKP_HASH(stom(kpd->kp_base, kpd->kp_flags));
1283 	mutex_enter(&segkp_lock);
1284 	kpd->kp_next = kpsd->kpsd_hash[index];
1285 	kpsd->kpsd_hash[index] = kpd;
1286 	mutex_exit(&segkp_lock);
1287 }
1288 
1289 /*
1290  * Remove kpd from the hash table.
1291  */
1292 static void
1293 segkp_delete(struct seg *seg, struct segkp_data *kpd)
1294 {
1295 	struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
1296 	struct segkp_data **kpp;
1297 	int index;
1298 
1299 	ASSERT(MUTEX_HELD(&segkp_lock));
1300 
1301 	index = SEGKP_HASH(stom(kpd->kp_base, kpd->kp_flags));
1302 	for (kpp = &kpsd->kpsd_hash[index];
1303 	    *kpp != NULL; kpp = &((*kpp)->kp_next)) {
1304 		if (*kpp == kpd) {
1305 			*kpp = kpd->kp_next;
1306 			return;
1307 		}
1308 	}
1309 	panic("segkp_delete: unable to find element to delete");
1310 	/*NOTREACHED*/
1311 }
1312 
1313 /*
1314  * Find the kpd associated with a vaddr.
1315  *
1316  * Most of the callers of segkp_find will pass the vaddr that
1317  * hashes to the desired index, but there are cases where
1318  * this is not true in which case we have to (potentially) scan
1319  * the whole table looking for it. This should be very rare
1320  * (e.g. a segkp_fault(F_INVAL) on an address somewhere in the
1321  * middle of the segkp_data region).
1322  */
1323 static struct segkp_data *
1324 segkp_find(struct seg *seg, caddr_t vaddr)
1325 {
1326 	struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
1327 	struct segkp_data *kpd;
1328 	int	i;
1329 	int	stop;
1330 
1331 	i = stop = SEGKP_HASH(vaddr);
1332 	mutex_enter(&segkp_lock);
1333 	do {
1334 		for (kpd = kpsd->kpsd_hash[i]; kpd != NULL;
1335 		    kpd = kpd->kp_next) {
1336 			if (vaddr >= kpd->kp_base &&
1337 			    vaddr < kpd->kp_base + kpd->kp_len) {
1338 				mutex_exit(&segkp_lock);
1339 				return (kpd);
1340 			}
1341 		}
1342 		if (--i < 0)
1343 			i = SEGKP_HASHSZ - 1;	/* Wrap */
1344 	} while (i != stop);
1345 	mutex_exit(&segkp_lock);
1346 	return (NULL);		/* Not found */
1347 }
1348 
1349 /*
1350  * returns size of swappable area.
1351  */
1352 size_t
1353 swapsize(caddr_t v)
1354 {
1355 	struct segkp_data *kpd;
1356 
1357 	if ((kpd = segkp_find(segkp, v)) != NULL)
1358 		return (SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags));
1359 	else
1360 		return (NULL);
1361 }
1362 
1363 /*
1364  * Dump out all the active segkp pages
1365  */
1366 static void
1367 segkp_dump(struct seg *seg)
1368 {
1369 	int i;
1370 	struct segkp_data *kpd;
1371 	struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
1372 
1373 	for (i = 0; i < SEGKP_HASHSZ; i++) {
1374 		for (kpd = kpsd->kpsd_hash[i];
1375 		    kpd != NULL; kpd = kpd->kp_next) {
1376 			pfn_t pfn;
1377 			caddr_t addr;
1378 			caddr_t eaddr;
1379 
1380 			addr = kpd->kp_base;
1381 			eaddr = addr + kpd->kp_len;
1382 			while (addr < eaddr) {
1383 				ASSERT(seg->s_as == &kas);
1384 				pfn = hat_getpfnum(seg->s_as->a_hat, addr);
1385 				if (pfn != PFN_INVALID)
1386 					dump_addpage(seg->s_as, addr, pfn);
1387 				addr += PAGESIZE;
1388 				dump_timeleft = dump_timeout;
1389 			}
1390 		}
1391 	}
1392 }
1393 
1394 /*ARGSUSED*/
1395 static int
1396 segkp_pagelock(struct seg *seg, caddr_t addr, size_t len,
1397     struct page ***ppp, enum lock_type type, enum seg_rw rw)
1398 {
1399 	return (ENOTSUP);
1400 }
1401 
1402 /*ARGSUSED*/
1403 static int
1404 segkp_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
1405 {
1406 	return (ENODEV);
1407 }
1408 
1409 /*ARGSUSED*/
1410 static lgrp_mem_policy_info_t	*
1411 segkp_getpolicy(struct seg *seg, caddr_t addr)
1412 {
1413 	return (NULL);
1414 }
1415 
1416 /*ARGSUSED*/
1417 static int
1418 segkp_capable(struct seg *seg, segcapability_t capability)
1419 {
1420 	return (0);
1421 }
1422 
1423 #include <sys/mem_config.h>
1424 
1425 /*ARGSUSED*/
1426 static void
1427 segkp_mem_config_post_add(void *arg, pgcnt_t delta_pages)
1428 {}
1429 
1430 /*
1431  * During memory delete, turn off caches so that pages are not held.
1432  * A better solution may be to unlock the pages while they are
1433  * in the cache so that they may be collected naturally.
1434  */
1435 
1436 /*ARGSUSED*/
1437 static int
1438 segkp_mem_config_pre_del(void *arg, pgcnt_t delta_pages)
1439 {
1440 	atomic_add_32(&segkp_indel, 1);
1441 	segkp_cache_free();
1442 	return (0);
1443 }
1444 
1445 /*ARGSUSED*/
1446 static void
1447 segkp_mem_config_post_del(void *arg, pgcnt_t delta_pages, int cancelled)
1448 {
1449 	atomic_add_32(&segkp_indel, -1);
1450 }
1451 
1452 static kphysm_setup_vector_t segkp_mem_config_vec = {
1453 	KPHYSM_SETUP_VECTOR_VERSION,
1454 	segkp_mem_config_post_add,
1455 	segkp_mem_config_pre_del,
1456 	segkp_mem_config_post_del,
1457 };
1458 
1459 static void
1460 segkpinit_mem_config(struct seg *seg)
1461 {
1462 	int ret;
1463 
1464 	ret = kphysm_setup_func_register(&segkp_mem_config_vec, (void *)seg);
1465 	ASSERT(ret == 0);
1466 }
1467