xref: /illumos-gate/usr/src/uts/common/vm/seg_kp.c (revision da6c28aa)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
50209230bSgjelinek  * Common Development and Distribution License (the "License").
60209230bSgjelinek  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
22*da6c28aaSamw  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
237c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate  */
257c478bd9Sstevel@tonic-gate 
267c478bd9Sstevel@tonic-gate /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
277c478bd9Sstevel@tonic-gate /*	All Rights Reserved   */
287c478bd9Sstevel@tonic-gate 
297c478bd9Sstevel@tonic-gate /*
307c478bd9Sstevel@tonic-gate  * Portions of this source code were derived from Berkeley 4.3 BSD
317c478bd9Sstevel@tonic-gate  * under license from the Regents of the University of California.
327c478bd9Sstevel@tonic-gate  */
337c478bd9Sstevel@tonic-gate 
347c478bd9Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
357c478bd9Sstevel@tonic-gate 
367c478bd9Sstevel@tonic-gate /*
377c478bd9Sstevel@tonic-gate  * segkp is a segment driver that administers the allocation and deallocation
387c478bd9Sstevel@tonic-gate  * of pageable variable size chunks of kernel virtual address space. Each
397c478bd9Sstevel@tonic-gate  * allocated resource is page-aligned.
407c478bd9Sstevel@tonic-gate  *
417c478bd9Sstevel@tonic-gate  * The user may specify whether the resource should be initialized to 0,
427c478bd9Sstevel@tonic-gate  * include a redzone, or locked in memory.
437c478bd9Sstevel@tonic-gate  */
447c478bd9Sstevel@tonic-gate 
457c478bd9Sstevel@tonic-gate #include <sys/types.h>
467c478bd9Sstevel@tonic-gate #include <sys/t_lock.h>
477c478bd9Sstevel@tonic-gate #include <sys/thread.h>
487c478bd9Sstevel@tonic-gate #include <sys/param.h>
497c478bd9Sstevel@tonic-gate #include <sys/errno.h>
507c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
517c478bd9Sstevel@tonic-gate #include <sys/systm.h>
527c478bd9Sstevel@tonic-gate #include <sys/buf.h>
537c478bd9Sstevel@tonic-gate #include <sys/mman.h>
547c478bd9Sstevel@tonic-gate #include <sys/vnode.h>
557c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
567c478bd9Sstevel@tonic-gate #include <sys/swap.h>
577c478bd9Sstevel@tonic-gate #include <sys/tuneable.h>
587c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
597c478bd9Sstevel@tonic-gate #include <sys/vmem.h>
607c478bd9Sstevel@tonic-gate #include <sys/cred.h>
617c478bd9Sstevel@tonic-gate #include <sys/dumphdr.h>
627c478bd9Sstevel@tonic-gate #include <sys/debug.h>
637c478bd9Sstevel@tonic-gate #include <sys/vtrace.h>
647c478bd9Sstevel@tonic-gate #include <sys/stack.h>
657c478bd9Sstevel@tonic-gate #include <sys/atomic.h>
667c478bd9Sstevel@tonic-gate #include <sys/archsystm.h>
677c478bd9Sstevel@tonic-gate #include <sys/lgrp.h>
687c478bd9Sstevel@tonic-gate 
697c478bd9Sstevel@tonic-gate #include <vm/as.h>
707c478bd9Sstevel@tonic-gate #include <vm/seg.h>
717c478bd9Sstevel@tonic-gate #include <vm/seg_kp.h>
727c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
737c478bd9Sstevel@tonic-gate #include <vm/anon.h>
747c478bd9Sstevel@tonic-gate #include <vm/page.h>
757c478bd9Sstevel@tonic-gate #include <vm/hat.h>
767c478bd9Sstevel@tonic-gate #include <sys/bitmap.h>
777c478bd9Sstevel@tonic-gate 
787c478bd9Sstevel@tonic-gate /*
797c478bd9Sstevel@tonic-gate  * Private seg op routines
807c478bd9Sstevel@tonic-gate  */
817c478bd9Sstevel@tonic-gate static void	segkp_badop(void);
827c478bd9Sstevel@tonic-gate static void	segkp_dump(struct seg *seg);
837c478bd9Sstevel@tonic-gate static int	segkp_checkprot(struct seg *seg, caddr_t addr, size_t len,
847c478bd9Sstevel@tonic-gate 			uint_t prot);
857c478bd9Sstevel@tonic-gate static int	segkp_kluster(struct seg *seg, caddr_t addr, ssize_t delta);
867c478bd9Sstevel@tonic-gate static int	segkp_pagelock(struct seg *seg, caddr_t addr, size_t len,
877c478bd9Sstevel@tonic-gate 			struct page ***page, enum lock_type type,
887c478bd9Sstevel@tonic-gate 			enum seg_rw rw);
897c478bd9Sstevel@tonic-gate static void	segkp_insert(struct seg *seg, struct segkp_data *kpd);
907c478bd9Sstevel@tonic-gate static void	segkp_delete(struct seg *seg, struct segkp_data *kpd);
917c478bd9Sstevel@tonic-gate static caddr_t	segkp_get_internal(struct seg *seg, size_t len, uint_t flags,
927c478bd9Sstevel@tonic-gate 			struct segkp_data **tkpd, struct anon_map *amp);
937c478bd9Sstevel@tonic-gate static void	segkp_release_internal(struct seg *seg,
947c478bd9Sstevel@tonic-gate 			struct segkp_data *kpd, size_t len);
957c478bd9Sstevel@tonic-gate static int	segkp_unlock(struct hat *hat, struct seg *seg, caddr_t vaddr,
967c478bd9Sstevel@tonic-gate 			size_t len, struct segkp_data *kpd, uint_t flags);
977c478bd9Sstevel@tonic-gate static int	segkp_load(struct hat *hat, struct seg *seg, caddr_t vaddr,
987c478bd9Sstevel@tonic-gate 			size_t len, struct segkp_data *kpd, uint_t flags);
997c478bd9Sstevel@tonic-gate static struct	segkp_data *segkp_find(struct seg *seg, caddr_t vaddr);
1007c478bd9Sstevel@tonic-gate static int	segkp_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp);
1017c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t	*segkp_getpolicy(struct seg *seg,
1027c478bd9Sstevel@tonic-gate     caddr_t addr);
1031bd5c35fSelowe static int	segkp_capable(struct seg *seg, segcapability_t capability);
1047c478bd9Sstevel@tonic-gate 
1057c478bd9Sstevel@tonic-gate /*
1067c478bd9Sstevel@tonic-gate  * Lock used to protect the hash table(s) and caches.
1077c478bd9Sstevel@tonic-gate  */
1087c478bd9Sstevel@tonic-gate static kmutex_t	segkp_lock;
1097c478bd9Sstevel@tonic-gate 
1107c478bd9Sstevel@tonic-gate /*
1117c478bd9Sstevel@tonic-gate  * The segkp caches
1127c478bd9Sstevel@tonic-gate  */
1137c478bd9Sstevel@tonic-gate static struct segkp_cache segkp_cache[SEGKP_MAX_CACHE];
1147c478bd9Sstevel@tonic-gate 
1157c478bd9Sstevel@tonic-gate #define	SEGKP_BADOP(t)	(t(*)())segkp_badop
1167c478bd9Sstevel@tonic-gate 
1177c478bd9Sstevel@tonic-gate /*
1187c478bd9Sstevel@tonic-gate  * When there are fewer than red_minavail bytes left on the stack,
1197c478bd9Sstevel@tonic-gate  * segkp_map_red() will map in the redzone (if called).  5000 seems
1207c478bd9Sstevel@tonic-gate  * to work reasonably well...
1217c478bd9Sstevel@tonic-gate  */
1227c478bd9Sstevel@tonic-gate long		red_minavail = 5000;
1237c478bd9Sstevel@tonic-gate 
1247c478bd9Sstevel@tonic-gate /*
1257c478bd9Sstevel@tonic-gate  * will be set to 1 for 32 bit x86 systems only, in startup.c
1267c478bd9Sstevel@tonic-gate  */
1277c478bd9Sstevel@tonic-gate int	segkp_fromheap = 0;
1287c478bd9Sstevel@tonic-gate ulong_t *segkp_bitmap;
1297c478bd9Sstevel@tonic-gate 
1307c478bd9Sstevel@tonic-gate /*
1317c478bd9Sstevel@tonic-gate  * If segkp_map_red() is called with the redzone already mapped and
1327c478bd9Sstevel@tonic-gate  * with less than RED_DEEP_THRESHOLD bytes available on the stack,
1337c478bd9Sstevel@tonic-gate  * then the stack situation has become quite serious;  if much more stack
1347c478bd9Sstevel@tonic-gate  * is consumed, we have the potential of scrogging the next thread/LWP
1357c478bd9Sstevel@tonic-gate  * structure.  To help debug the "can't happen" panics which may
1367c478bd9Sstevel@tonic-gate  * result from this condition, we record lbolt and the calling thread
1377c478bd9Sstevel@tonic-gate  * in red_deep_lbolt and red_deep_thread respectively.
1387c478bd9Sstevel@tonic-gate  */
1397c478bd9Sstevel@tonic-gate #define	RED_DEEP_THRESHOLD	2000
1407c478bd9Sstevel@tonic-gate 
1417c478bd9Sstevel@tonic-gate clock_t		red_deep_lbolt;
1427c478bd9Sstevel@tonic-gate kthread_t	*red_deep_thread;
1437c478bd9Sstevel@tonic-gate 
1447c478bd9Sstevel@tonic-gate uint32_t	red_nmapped;
1457c478bd9Sstevel@tonic-gate uint32_t	red_closest = UINT_MAX;
1467c478bd9Sstevel@tonic-gate uint32_t	red_ndoubles;
1477c478bd9Sstevel@tonic-gate 
1487c478bd9Sstevel@tonic-gate pgcnt_t anon_segkp_pages_locked;	/* See vm/anon.h */
1490209230bSgjelinek pgcnt_t anon_segkp_pages_resv;		/* anon reserved by seg_kp */
1507c478bd9Sstevel@tonic-gate 
1517c478bd9Sstevel@tonic-gate static struct	seg_ops segkp_ops = {
1527c478bd9Sstevel@tonic-gate 	SEGKP_BADOP(int),		/* dup */
1537c478bd9Sstevel@tonic-gate 	SEGKP_BADOP(int),		/* unmap */
1547c478bd9Sstevel@tonic-gate 	SEGKP_BADOP(void),		/* free */
1557c478bd9Sstevel@tonic-gate 	segkp_fault,
1567c478bd9Sstevel@tonic-gate 	SEGKP_BADOP(faultcode_t),	/* faulta */
1577c478bd9Sstevel@tonic-gate 	SEGKP_BADOP(int),		/* setprot */
1587c478bd9Sstevel@tonic-gate 	segkp_checkprot,
1597c478bd9Sstevel@tonic-gate 	segkp_kluster,
1607c478bd9Sstevel@tonic-gate 	SEGKP_BADOP(size_t),		/* swapout */
1617c478bd9Sstevel@tonic-gate 	SEGKP_BADOP(int),		/* sync */
1627c478bd9Sstevel@tonic-gate 	SEGKP_BADOP(size_t),		/* incore */
1637c478bd9Sstevel@tonic-gate 	SEGKP_BADOP(int),		/* lockop */
1647c478bd9Sstevel@tonic-gate 	SEGKP_BADOP(int),		/* getprot */
1657c478bd9Sstevel@tonic-gate 	SEGKP_BADOP(u_offset_t),		/* getoffset */
1667c478bd9Sstevel@tonic-gate 	SEGKP_BADOP(int),		/* gettype */
1677c478bd9Sstevel@tonic-gate 	SEGKP_BADOP(int),		/* getvp */
1687c478bd9Sstevel@tonic-gate 	SEGKP_BADOP(int),		/* advise */
1697c478bd9Sstevel@tonic-gate 	segkp_dump,			/* dump */
1707c478bd9Sstevel@tonic-gate 	segkp_pagelock,			/* pagelock */
1717c478bd9Sstevel@tonic-gate 	SEGKP_BADOP(int),		/* setpgsz */
1727c478bd9Sstevel@tonic-gate 	segkp_getmemid,			/* getmemid */
1737c478bd9Sstevel@tonic-gate 	segkp_getpolicy,		/* getpolicy */
1741bd5c35fSelowe 	segkp_capable,			/* capable */
1757c478bd9Sstevel@tonic-gate };
1767c478bd9Sstevel@tonic-gate 
1777c478bd9Sstevel@tonic-gate 
1787c478bd9Sstevel@tonic-gate static void
1797c478bd9Sstevel@tonic-gate segkp_badop(void)
1807c478bd9Sstevel@tonic-gate {
1817c478bd9Sstevel@tonic-gate 	panic("segkp_badop");
1827c478bd9Sstevel@tonic-gate 	/*NOTREACHED*/
1837c478bd9Sstevel@tonic-gate }
1847c478bd9Sstevel@tonic-gate 
1857c478bd9Sstevel@tonic-gate static void segkpinit_mem_config(struct seg *);
1867c478bd9Sstevel@tonic-gate 
1877c478bd9Sstevel@tonic-gate static uint32_t segkp_indel;
1887c478bd9Sstevel@tonic-gate 
1897c478bd9Sstevel@tonic-gate /*
1907c478bd9Sstevel@tonic-gate  * Allocate the segment specific private data struct and fill it in
1917c478bd9Sstevel@tonic-gate  * with the per kp segment mutex, anon ptr. array and hash table.
1927c478bd9Sstevel@tonic-gate  */
1937c478bd9Sstevel@tonic-gate int
1947c478bd9Sstevel@tonic-gate segkp_create(struct seg *seg)
1957c478bd9Sstevel@tonic-gate {
1967c478bd9Sstevel@tonic-gate 	struct segkp_segdata *kpsd;
1977c478bd9Sstevel@tonic-gate 	size_t	np;
1987c478bd9Sstevel@tonic-gate 
1997c478bd9Sstevel@tonic-gate 	ASSERT(seg != NULL && seg->s_as == &kas);
2007c478bd9Sstevel@tonic-gate 	ASSERT(RW_WRITE_HELD(&seg->s_as->a_lock));
2017c478bd9Sstevel@tonic-gate 
2027c478bd9Sstevel@tonic-gate 	if (seg->s_size & PAGEOFFSET) {
2037c478bd9Sstevel@tonic-gate 		panic("Bad segkp size");
2047c478bd9Sstevel@tonic-gate 		/*NOTREACHED*/
2057c478bd9Sstevel@tonic-gate 	}
2067c478bd9Sstevel@tonic-gate 
2077c478bd9Sstevel@tonic-gate 	kpsd = kmem_zalloc(sizeof (struct segkp_segdata), KM_SLEEP);
2087c478bd9Sstevel@tonic-gate 
2097c478bd9Sstevel@tonic-gate 	/*
2107c478bd9Sstevel@tonic-gate 	 * Allocate the virtual memory for segkp and initialize it
2117c478bd9Sstevel@tonic-gate 	 */
2127c478bd9Sstevel@tonic-gate 	if (segkp_fromheap) {
2137c478bd9Sstevel@tonic-gate 		np = btop(kvseg.s_size);
2147c478bd9Sstevel@tonic-gate 		segkp_bitmap = kmem_zalloc(BT_SIZEOFMAP(np), KM_SLEEP);
2157c478bd9Sstevel@tonic-gate 		kpsd->kpsd_arena = vmem_create("segkp", NULL, 0, PAGESIZE,
2167c478bd9Sstevel@tonic-gate 		    vmem_alloc, vmem_free, heap_arena, 5 * PAGESIZE, VM_SLEEP);
2177c478bd9Sstevel@tonic-gate 	} else {
2187c478bd9Sstevel@tonic-gate 		segkp_bitmap = NULL;
2197c478bd9Sstevel@tonic-gate 		np = btop(seg->s_size);
2207c478bd9Sstevel@tonic-gate 		kpsd->kpsd_arena = vmem_create("segkp", seg->s_base,
2217c478bd9Sstevel@tonic-gate 		    seg->s_size, PAGESIZE, NULL, NULL, NULL, 5 * PAGESIZE,
2227c478bd9Sstevel@tonic-gate 		    VM_SLEEP);
2237c478bd9Sstevel@tonic-gate 	}
2247c478bd9Sstevel@tonic-gate 
2257c478bd9Sstevel@tonic-gate 	kpsd->kpsd_anon = anon_create(np, ANON_SLEEP | ANON_ALLOC_FORCE);
2267c478bd9Sstevel@tonic-gate 
2277c478bd9Sstevel@tonic-gate 	kpsd->kpsd_hash = kmem_zalloc(SEGKP_HASHSZ * sizeof (struct segkp *),
2287c478bd9Sstevel@tonic-gate 	    KM_SLEEP);
2297c478bd9Sstevel@tonic-gate 	seg->s_data = (void *)kpsd;
2307c478bd9Sstevel@tonic-gate 	seg->s_ops = &segkp_ops;
2317c478bd9Sstevel@tonic-gate 	segkpinit_mem_config(seg);
2327c478bd9Sstevel@tonic-gate 	return (0);
2337c478bd9Sstevel@tonic-gate }
2347c478bd9Sstevel@tonic-gate 
2357c478bd9Sstevel@tonic-gate 
2367c478bd9Sstevel@tonic-gate /*
2377c478bd9Sstevel@tonic-gate  * Find a free 'freelist' and initialize it with the appropriate attributes
2387c478bd9Sstevel@tonic-gate  */
2397c478bd9Sstevel@tonic-gate void *
2407c478bd9Sstevel@tonic-gate segkp_cache_init(struct seg *seg, int maxsize, size_t len, uint_t flags)
2417c478bd9Sstevel@tonic-gate {
2427c478bd9Sstevel@tonic-gate 	int i;
2437c478bd9Sstevel@tonic-gate 
2447c478bd9Sstevel@tonic-gate 	if ((flags & KPD_NO_ANON) && !(flags & KPD_LOCKED))
2457c478bd9Sstevel@tonic-gate 		return ((void *)-1);
2467c478bd9Sstevel@tonic-gate 
2477c478bd9Sstevel@tonic-gate 	mutex_enter(&segkp_lock);
2487c478bd9Sstevel@tonic-gate 	for (i = 0; i < SEGKP_MAX_CACHE; i++) {
2497c478bd9Sstevel@tonic-gate 		if (segkp_cache[i].kpf_inuse)
2507c478bd9Sstevel@tonic-gate 			continue;
2517c478bd9Sstevel@tonic-gate 		segkp_cache[i].kpf_inuse = 1;
2527c478bd9Sstevel@tonic-gate 		segkp_cache[i].kpf_max = maxsize;
2537c478bd9Sstevel@tonic-gate 		segkp_cache[i].kpf_flags = flags;
2547c478bd9Sstevel@tonic-gate 		segkp_cache[i].kpf_seg = seg;
2557c478bd9Sstevel@tonic-gate 		segkp_cache[i].kpf_len = len;
2567c478bd9Sstevel@tonic-gate 		mutex_exit(&segkp_lock);
2577c478bd9Sstevel@tonic-gate 		return ((void *)(uintptr_t)i);
2587c478bd9Sstevel@tonic-gate 	}
2597c478bd9Sstevel@tonic-gate 	mutex_exit(&segkp_lock);
2607c478bd9Sstevel@tonic-gate 	return ((void *)-1);
2617c478bd9Sstevel@tonic-gate }
2627c478bd9Sstevel@tonic-gate 
2637c478bd9Sstevel@tonic-gate /*
2647c478bd9Sstevel@tonic-gate  * Free all the cache resources.
2657c478bd9Sstevel@tonic-gate  */
2667c478bd9Sstevel@tonic-gate void
2677c478bd9Sstevel@tonic-gate segkp_cache_free(void)
2687c478bd9Sstevel@tonic-gate {
2697c478bd9Sstevel@tonic-gate 	struct segkp_data *kpd;
2707c478bd9Sstevel@tonic-gate 	struct seg *seg;
2717c478bd9Sstevel@tonic-gate 	int i;
2727c478bd9Sstevel@tonic-gate 
2737c478bd9Sstevel@tonic-gate 	mutex_enter(&segkp_lock);
2747c478bd9Sstevel@tonic-gate 	for (i = 0; i < SEGKP_MAX_CACHE; i++) {
2757c478bd9Sstevel@tonic-gate 		if (!segkp_cache[i].kpf_inuse)
2767c478bd9Sstevel@tonic-gate 			continue;
2777c478bd9Sstevel@tonic-gate 		/*
2787c478bd9Sstevel@tonic-gate 		 * Disconnect the freelist and process each element
2797c478bd9Sstevel@tonic-gate 		 */
2807c478bd9Sstevel@tonic-gate 		kpd = segkp_cache[i].kpf_list;
2817c478bd9Sstevel@tonic-gate 		seg = segkp_cache[i].kpf_seg;
2827c478bd9Sstevel@tonic-gate 		segkp_cache[i].kpf_list = NULL;
2837c478bd9Sstevel@tonic-gate 		segkp_cache[i].kpf_count = 0;
2847c478bd9Sstevel@tonic-gate 		mutex_exit(&segkp_lock);
2857c478bd9Sstevel@tonic-gate 
2867c478bd9Sstevel@tonic-gate 		while (kpd != NULL) {
2877c478bd9Sstevel@tonic-gate 			struct segkp_data *next;
2887c478bd9Sstevel@tonic-gate 
2897c478bd9Sstevel@tonic-gate 			next = kpd->kp_next;
2907c478bd9Sstevel@tonic-gate 			segkp_release_internal(seg, kpd, kpd->kp_len);
2917c478bd9Sstevel@tonic-gate 			kpd = next;
2927c478bd9Sstevel@tonic-gate 		}
2937c478bd9Sstevel@tonic-gate 		mutex_enter(&segkp_lock);
2947c478bd9Sstevel@tonic-gate 	}
2957c478bd9Sstevel@tonic-gate 	mutex_exit(&segkp_lock);
2967c478bd9Sstevel@tonic-gate }
2977c478bd9Sstevel@tonic-gate 
2987c478bd9Sstevel@tonic-gate /*
2997c478bd9Sstevel@tonic-gate  * There are 2 entries into segkp_get_internal. The first includes a cookie
3007c478bd9Sstevel@tonic-gate  * used to access a pool of cached segkp resources. The second does not
3017c478bd9Sstevel@tonic-gate  * use the cache.
3027c478bd9Sstevel@tonic-gate  */
3037c478bd9Sstevel@tonic-gate caddr_t
3047c478bd9Sstevel@tonic-gate segkp_get(struct seg *seg, size_t len, uint_t flags)
3057c478bd9Sstevel@tonic-gate {
3067c478bd9Sstevel@tonic-gate 	struct segkp_data *kpd = NULL;
3077c478bd9Sstevel@tonic-gate 
3087c478bd9Sstevel@tonic-gate 	if (segkp_get_internal(seg, len, flags, &kpd, NULL) != NULL) {
3097c478bd9Sstevel@tonic-gate 		kpd->kp_cookie = -1;
3107c478bd9Sstevel@tonic-gate 		return (stom(kpd->kp_base, flags));
3117c478bd9Sstevel@tonic-gate 	}
3127c478bd9Sstevel@tonic-gate 	return (NULL);
3137c478bd9Sstevel@tonic-gate }
3147c478bd9Sstevel@tonic-gate 
3157c478bd9Sstevel@tonic-gate /*
3167c478bd9Sstevel@tonic-gate  * Return a 'cached' segkp address
3177c478bd9Sstevel@tonic-gate  */
3187c478bd9Sstevel@tonic-gate caddr_t
3197c478bd9Sstevel@tonic-gate segkp_cache_get(void *cookie)
3207c478bd9Sstevel@tonic-gate {
3217c478bd9Sstevel@tonic-gate 	struct segkp_cache *freelist = NULL;
3227c478bd9Sstevel@tonic-gate 	struct segkp_data *kpd = NULL;
3237c478bd9Sstevel@tonic-gate 	int index = (int)(uintptr_t)cookie;
3247c478bd9Sstevel@tonic-gate 	struct seg *seg;
3257c478bd9Sstevel@tonic-gate 	size_t len;
3267c478bd9Sstevel@tonic-gate 	uint_t flags;
3277c478bd9Sstevel@tonic-gate 
3287c478bd9Sstevel@tonic-gate 	if (index < 0 || index >= SEGKP_MAX_CACHE)
3297c478bd9Sstevel@tonic-gate 		return (NULL);
3307c478bd9Sstevel@tonic-gate 	freelist = &segkp_cache[index];
3317c478bd9Sstevel@tonic-gate 
3327c478bd9Sstevel@tonic-gate 	mutex_enter(&segkp_lock);
3337c478bd9Sstevel@tonic-gate 	seg = freelist->kpf_seg;
3347c478bd9Sstevel@tonic-gate 	flags = freelist->kpf_flags;
3357c478bd9Sstevel@tonic-gate 	if (freelist->kpf_list != NULL) {
3367c478bd9Sstevel@tonic-gate 		kpd = freelist->kpf_list;
3377c478bd9Sstevel@tonic-gate 		freelist->kpf_list = kpd->kp_next;
3387c478bd9Sstevel@tonic-gate 		freelist->kpf_count--;
3397c478bd9Sstevel@tonic-gate 		mutex_exit(&segkp_lock);
3407c478bd9Sstevel@tonic-gate 		kpd->kp_next = NULL;
3417c478bd9Sstevel@tonic-gate 		segkp_insert(seg, kpd);
3427c478bd9Sstevel@tonic-gate 		return (stom(kpd->kp_base, flags));
3437c478bd9Sstevel@tonic-gate 	}
3447c478bd9Sstevel@tonic-gate 	len = freelist->kpf_len;
3457c478bd9Sstevel@tonic-gate 	mutex_exit(&segkp_lock);
3467c478bd9Sstevel@tonic-gate 	if (segkp_get_internal(seg, len, flags, &kpd, NULL) != NULL) {
3477c478bd9Sstevel@tonic-gate 		kpd->kp_cookie = index;
3487c478bd9Sstevel@tonic-gate 		return (stom(kpd->kp_base, flags));
3497c478bd9Sstevel@tonic-gate 	}
3507c478bd9Sstevel@tonic-gate 	return (NULL);
3517c478bd9Sstevel@tonic-gate }
3527c478bd9Sstevel@tonic-gate 
3537c478bd9Sstevel@tonic-gate caddr_t
3547c478bd9Sstevel@tonic-gate segkp_get_withanonmap(
3557c478bd9Sstevel@tonic-gate 	struct seg *seg,
3567c478bd9Sstevel@tonic-gate 	size_t len,
3577c478bd9Sstevel@tonic-gate 	uint_t flags,
3587c478bd9Sstevel@tonic-gate 	struct anon_map *amp)
3597c478bd9Sstevel@tonic-gate {
3607c478bd9Sstevel@tonic-gate 	struct segkp_data *kpd = NULL;
3617c478bd9Sstevel@tonic-gate 
3627c478bd9Sstevel@tonic-gate 	ASSERT(amp != NULL);
3637c478bd9Sstevel@tonic-gate 	flags |= KPD_HASAMP;
3647c478bd9Sstevel@tonic-gate 	if (segkp_get_internal(seg, len, flags, &kpd, amp) != NULL) {
3657c478bd9Sstevel@tonic-gate 		kpd->kp_cookie = -1;
3667c478bd9Sstevel@tonic-gate 		return (stom(kpd->kp_base, flags));
3677c478bd9Sstevel@tonic-gate 	}
3687c478bd9Sstevel@tonic-gate 	return (NULL);
3697c478bd9Sstevel@tonic-gate }
3707c478bd9Sstevel@tonic-gate 
3717c478bd9Sstevel@tonic-gate /*
3727c478bd9Sstevel@tonic-gate  * This does the real work of segkp allocation.
3737c478bd9Sstevel@tonic-gate  * Return to client base addr. len must be page-aligned. A null value is
3747c478bd9Sstevel@tonic-gate  * returned if there are no more vm resources (e.g. pages, swap). The len
3757c478bd9Sstevel@tonic-gate  * and base recorded in the private data structure include the redzone
3767c478bd9Sstevel@tonic-gate  * and the redzone length (if applicable). If the user requests a redzone
3777c478bd9Sstevel@tonic-gate  * either the first or last page is left unmapped depending whether stacks
3787c478bd9Sstevel@tonic-gate  * grow to low or high memory.
3797c478bd9Sstevel@tonic-gate  *
3807c478bd9Sstevel@tonic-gate  * The client may also specify a no-wait flag. If that is set then the
3817c478bd9Sstevel@tonic-gate  * request will choose a non-blocking path when requesting resources.
3827c478bd9Sstevel@tonic-gate  * The default is make the client wait.
3837c478bd9Sstevel@tonic-gate  */
3847c478bd9Sstevel@tonic-gate static caddr_t
3857c478bd9Sstevel@tonic-gate segkp_get_internal(
3867c478bd9Sstevel@tonic-gate 	struct seg *seg,
3877c478bd9Sstevel@tonic-gate 	size_t len,
3887c478bd9Sstevel@tonic-gate 	uint_t flags,
3897c478bd9Sstevel@tonic-gate 	struct segkp_data **tkpd,
3907c478bd9Sstevel@tonic-gate 	struct anon_map *amp)
3917c478bd9Sstevel@tonic-gate {
3927c478bd9Sstevel@tonic-gate 	struct segkp_segdata	*kpsd = (struct segkp_segdata *)seg->s_data;
3937c478bd9Sstevel@tonic-gate 	struct segkp_data	*kpd;
3947c478bd9Sstevel@tonic-gate 	caddr_t vbase = NULL;	/* always first virtual, may not be mapped */
3957c478bd9Sstevel@tonic-gate 	pgcnt_t np = 0;		/* number of pages in the resource */
3967c478bd9Sstevel@tonic-gate 	pgcnt_t segkpindex;
3977c478bd9Sstevel@tonic-gate 	long i;
3987c478bd9Sstevel@tonic-gate 	caddr_t va;
3997c478bd9Sstevel@tonic-gate 	pgcnt_t pages = 0;
4007c478bd9Sstevel@tonic-gate 	ulong_t anon_idx = 0;
4017c478bd9Sstevel@tonic-gate 	int kmflag = (flags & KPD_NOWAIT) ? KM_NOSLEEP : KM_SLEEP;
4027c478bd9Sstevel@tonic-gate 	caddr_t s_base = (segkp_fromheap) ? kvseg.s_base : seg->s_base;
4037c478bd9Sstevel@tonic-gate 
4047c478bd9Sstevel@tonic-gate 	if (len & PAGEOFFSET) {
4057c478bd9Sstevel@tonic-gate 		panic("segkp_get: len is not page-aligned");
4067c478bd9Sstevel@tonic-gate 		/*NOTREACHED*/
4077c478bd9Sstevel@tonic-gate 	}
4087c478bd9Sstevel@tonic-gate 
4097c478bd9Sstevel@tonic-gate 	ASSERT(((flags & KPD_HASAMP) == 0) == (amp == NULL));
4107c478bd9Sstevel@tonic-gate 
4117c478bd9Sstevel@tonic-gate 	/* Only allow KPD_NO_ANON if we are going to lock it down */
4127c478bd9Sstevel@tonic-gate 	if ((flags & (KPD_LOCKED|KPD_NO_ANON)) == KPD_NO_ANON)
4137c478bd9Sstevel@tonic-gate 		return (NULL);
4147c478bd9Sstevel@tonic-gate 
4157c478bd9Sstevel@tonic-gate 	if ((kpd = kmem_zalloc(sizeof (struct segkp_data), kmflag)) == NULL)
4167c478bd9Sstevel@tonic-gate 		return (NULL);
4177c478bd9Sstevel@tonic-gate 	/*
4187c478bd9Sstevel@tonic-gate 	 * Fix up the len to reflect the REDZONE if applicable
4197c478bd9Sstevel@tonic-gate 	 */
4207c478bd9Sstevel@tonic-gate 	if (flags & KPD_HASREDZONE)
4217c478bd9Sstevel@tonic-gate 		len += PAGESIZE;
4227c478bd9Sstevel@tonic-gate 	np = btop(len);
4237c478bd9Sstevel@tonic-gate 
4247c478bd9Sstevel@tonic-gate 	vbase = vmem_alloc(SEGKP_VMEM(seg), len, kmflag | VM_BESTFIT);
4257c478bd9Sstevel@tonic-gate 	if (vbase == NULL) {
4267c478bd9Sstevel@tonic-gate 		kmem_free(kpd, sizeof (struct segkp_data));
4277c478bd9Sstevel@tonic-gate 		return (NULL);
4287c478bd9Sstevel@tonic-gate 	}
4297c478bd9Sstevel@tonic-gate 
4307c478bd9Sstevel@tonic-gate 	/* If locking, reserve physical memory */
4317c478bd9Sstevel@tonic-gate 	if (flags & KPD_LOCKED) {
4327c478bd9Sstevel@tonic-gate 		pages = btop(SEGKP_MAPLEN(len, flags));
4337c478bd9Sstevel@tonic-gate 		if (page_resv(pages, kmflag) == 0) {
4347c478bd9Sstevel@tonic-gate 			vmem_free(SEGKP_VMEM(seg), vbase, len);
4357c478bd9Sstevel@tonic-gate 			kmem_free(kpd, sizeof (struct segkp_data));
4367c478bd9Sstevel@tonic-gate 			return (NULL);
4377c478bd9Sstevel@tonic-gate 		}
4387c478bd9Sstevel@tonic-gate 		if ((flags & KPD_NO_ANON) == 0)
4397c478bd9Sstevel@tonic-gate 			atomic_add_long(&anon_segkp_pages_locked, pages);
4407c478bd9Sstevel@tonic-gate 	}
4417c478bd9Sstevel@tonic-gate 
4427c478bd9Sstevel@tonic-gate 	/*
4437c478bd9Sstevel@tonic-gate 	 * Reserve sufficient swap space for this vm resource.  We'll
4447c478bd9Sstevel@tonic-gate 	 * actually allocate it in the loop below, but reserving it
4457c478bd9Sstevel@tonic-gate 	 * here allows us to back out more gracefully than if we
4467c478bd9Sstevel@tonic-gate 	 * had an allocation failure in the body of the loop.
4477c478bd9Sstevel@tonic-gate 	 *
4487c478bd9Sstevel@tonic-gate 	 * Note that we don't need swap space for the red zone page.
4497c478bd9Sstevel@tonic-gate 	 */
4507c478bd9Sstevel@tonic-gate 	if (amp != NULL) {
4510209230bSgjelinek 		/*
4520209230bSgjelinek 		 * The swap reservation has been done, if required, and the
4530209230bSgjelinek 		 * anon_hdr is separate.
4540209230bSgjelinek 		 */
4557c478bd9Sstevel@tonic-gate 		anon_idx = 0;
4567c478bd9Sstevel@tonic-gate 		kpd->kp_anon_idx = anon_idx;
4577c478bd9Sstevel@tonic-gate 		kpd->kp_anon = amp->ahp;
4587c478bd9Sstevel@tonic-gate 
4597c478bd9Sstevel@tonic-gate 		TRACE_5(TR_FAC_VM, TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u",
4607c478bd9Sstevel@tonic-gate 		    kpd, vbase, len, flags, 1);
4617c478bd9Sstevel@tonic-gate 
4627c478bd9Sstevel@tonic-gate 	} else if ((flags & KPD_NO_ANON) == 0) {
4630209230bSgjelinek 		if (anon_resv_zone(SEGKP_MAPLEN(len, flags), NULL) == 0) {
4647c478bd9Sstevel@tonic-gate 			if (flags & KPD_LOCKED) {
4657c478bd9Sstevel@tonic-gate 				atomic_add_long(&anon_segkp_pages_locked,
4667c478bd9Sstevel@tonic-gate 				    -pages);
4677c478bd9Sstevel@tonic-gate 				page_unresv(pages);
4687c478bd9Sstevel@tonic-gate 			}
4697c478bd9Sstevel@tonic-gate 			vmem_free(SEGKP_VMEM(seg), vbase, len);
4707c478bd9Sstevel@tonic-gate 			kmem_free(kpd, sizeof (struct segkp_data));
4717c478bd9Sstevel@tonic-gate 			return (NULL);
4727c478bd9Sstevel@tonic-gate 		}
4730209230bSgjelinek 		atomic_add_long(&anon_segkp_pages_resv,
4740209230bSgjelinek 		    btop(SEGKP_MAPLEN(len, flags)));
4757c478bd9Sstevel@tonic-gate 		anon_idx = ((uintptr_t)(vbase - s_base)) >> PAGESHIFT;
4767c478bd9Sstevel@tonic-gate 		kpd->kp_anon_idx = anon_idx;
4777c478bd9Sstevel@tonic-gate 		kpd->kp_anon = kpsd->kpsd_anon;
4787c478bd9Sstevel@tonic-gate 
4797c478bd9Sstevel@tonic-gate 		TRACE_5(TR_FAC_VM, TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u",
4807c478bd9Sstevel@tonic-gate 		    kpd, vbase, len, flags, 1);
4817c478bd9Sstevel@tonic-gate 	} else {
4827c478bd9Sstevel@tonic-gate 		kpd->kp_anon = NULL;
4837c478bd9Sstevel@tonic-gate 		kpd->kp_anon_idx = 0;
4847c478bd9Sstevel@tonic-gate 	}
4857c478bd9Sstevel@tonic-gate 
4867c478bd9Sstevel@tonic-gate 	/*
4877c478bd9Sstevel@tonic-gate 	 * Allocate page and anon resources for the virtual address range
4887c478bd9Sstevel@tonic-gate 	 * except the redzone
4897c478bd9Sstevel@tonic-gate 	 */
4907c478bd9Sstevel@tonic-gate 	if (segkp_fromheap)
4917c478bd9Sstevel@tonic-gate 		segkpindex = btop((uintptr_t)(vbase - kvseg.s_base));
4927c478bd9Sstevel@tonic-gate 	for (i = 0, va = vbase; i < np; i++, va += PAGESIZE) {
4937c478bd9Sstevel@tonic-gate 		page_t		*pl[2];
4947c478bd9Sstevel@tonic-gate 		struct vnode	*vp;
4957c478bd9Sstevel@tonic-gate 		anoff_t		off;
4967c478bd9Sstevel@tonic-gate 		int		err;
4977c478bd9Sstevel@tonic-gate 		page_t		*pp = NULL;
4987c478bd9Sstevel@tonic-gate 
4997c478bd9Sstevel@tonic-gate 		/*
5007c478bd9Sstevel@tonic-gate 		 * Mark this page to be a segkp page in the bitmap.
5017c478bd9Sstevel@tonic-gate 		 */
5027c478bd9Sstevel@tonic-gate 		if (segkp_fromheap) {
5037c478bd9Sstevel@tonic-gate 			BT_ATOMIC_SET(segkp_bitmap, segkpindex);
5047c478bd9Sstevel@tonic-gate 			segkpindex++;
5057c478bd9Sstevel@tonic-gate 		}
5067c478bd9Sstevel@tonic-gate 
5077c478bd9Sstevel@tonic-gate 		/*
5087c478bd9Sstevel@tonic-gate 		 * If this page is the red zone page, we don't need swap
5097c478bd9Sstevel@tonic-gate 		 * space for it.  Note that we skip over the code that
5107c478bd9Sstevel@tonic-gate 		 * establishes MMU mappings, so that the page remains
5117c478bd9Sstevel@tonic-gate 		 * invalid.
5127c478bd9Sstevel@tonic-gate 		 */
5137c478bd9Sstevel@tonic-gate 		if ((flags & KPD_HASREDZONE) && KPD_REDZONE(kpd) == i)
5147c478bd9Sstevel@tonic-gate 			continue;
5157c478bd9Sstevel@tonic-gate 
5167c478bd9Sstevel@tonic-gate 		if (kpd->kp_anon != NULL) {
5177c478bd9Sstevel@tonic-gate 			struct anon *ap;
5187c478bd9Sstevel@tonic-gate 
5197c478bd9Sstevel@tonic-gate 			ASSERT(anon_get_ptr(kpd->kp_anon, anon_idx + i)
5207c478bd9Sstevel@tonic-gate 			    == NULL);
5217c478bd9Sstevel@tonic-gate 			/*
5227c478bd9Sstevel@tonic-gate 			 * Determine the "vp" and "off" of the anon slot.
5237c478bd9Sstevel@tonic-gate 			 */
5247c478bd9Sstevel@tonic-gate 			ap = anon_alloc(NULL, 0);
5257c478bd9Sstevel@tonic-gate 			if (amp != NULL)
5267c478bd9Sstevel@tonic-gate 				ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
5277c478bd9Sstevel@tonic-gate 			(void) anon_set_ptr(kpd->kp_anon, anon_idx + i,
5287c478bd9Sstevel@tonic-gate 			    ap, ANON_SLEEP);
5297c478bd9Sstevel@tonic-gate 			if (amp != NULL)
5307c478bd9Sstevel@tonic-gate 				ANON_LOCK_EXIT(&amp->a_rwlock);
5317c478bd9Sstevel@tonic-gate 			swap_xlate(ap, &vp, &off);
5327c478bd9Sstevel@tonic-gate 
5337c478bd9Sstevel@tonic-gate 			/*
5347c478bd9Sstevel@tonic-gate 			 * Create a page with the specified identity.  The
5357c478bd9Sstevel@tonic-gate 			 * page is returned with the "shared" lock held.
5367c478bd9Sstevel@tonic-gate 			 */
5377c478bd9Sstevel@tonic-gate 			err = VOP_GETPAGE(vp, (offset_t)off, PAGESIZE,
5387c478bd9Sstevel@tonic-gate 			    NULL, pl, PAGESIZE, seg, va, S_CREATE,
539*da6c28aaSamw 			    kcred, NULL);
5407c478bd9Sstevel@tonic-gate 			if (err) {
5417c478bd9Sstevel@tonic-gate 				/*
5427c478bd9Sstevel@tonic-gate 				 * XXX - This should not fail.
5437c478bd9Sstevel@tonic-gate 				 */
5447c478bd9Sstevel@tonic-gate 				panic("segkp_get: no pages");
5457c478bd9Sstevel@tonic-gate 				/*NOTREACHED*/
5467c478bd9Sstevel@tonic-gate 			}
5477c478bd9Sstevel@tonic-gate 			pp = pl[0];
5487c478bd9Sstevel@tonic-gate 		} else {
5497c478bd9Sstevel@tonic-gate 			ASSERT(page_exists(&kvp,
5507c478bd9Sstevel@tonic-gate 			    (u_offset_t)(uintptr_t)va) == NULL);
5517c478bd9Sstevel@tonic-gate 
5527c478bd9Sstevel@tonic-gate 			if ((pp = page_create_va(&kvp,
5537c478bd9Sstevel@tonic-gate 			    (u_offset_t)(uintptr_t)va, PAGESIZE,
5547c478bd9Sstevel@tonic-gate 			    (flags & KPD_NOWAIT ? 0 : PG_WAIT) | PG_EXCL |
5557c478bd9Sstevel@tonic-gate 			    PG_NORELOC, seg, va)) == NULL) {
5567c478bd9Sstevel@tonic-gate 				/*
5577c478bd9Sstevel@tonic-gate 				 * Legitimize resource; then destroy it.
5587c478bd9Sstevel@tonic-gate 				 * Easier than trying to unwind here.
5597c478bd9Sstevel@tonic-gate 				 */
5607c478bd9Sstevel@tonic-gate 				kpd->kp_flags = flags;
5617c478bd9Sstevel@tonic-gate 				kpd->kp_base = vbase;
5627c478bd9Sstevel@tonic-gate 				kpd->kp_len = len;
5637c478bd9Sstevel@tonic-gate 				segkp_release_internal(seg, kpd, va - vbase);
5647c478bd9Sstevel@tonic-gate 				return (NULL);
5657c478bd9Sstevel@tonic-gate 			}
5667c478bd9Sstevel@tonic-gate 			page_io_unlock(pp);
5677c478bd9Sstevel@tonic-gate 		}
5687c478bd9Sstevel@tonic-gate 
5697c478bd9Sstevel@tonic-gate 		if (flags & KPD_ZERO)
5707c478bd9Sstevel@tonic-gate 			pagezero(pp, 0, PAGESIZE);
5717c478bd9Sstevel@tonic-gate 
5727c478bd9Sstevel@tonic-gate 		/*
5737c478bd9Sstevel@tonic-gate 		 * Load and lock an MMU translation for the page.
5747c478bd9Sstevel@tonic-gate 		 */
5757c478bd9Sstevel@tonic-gate 		hat_memload(seg->s_as->a_hat, va, pp, (PROT_READ|PROT_WRITE),
5767c478bd9Sstevel@tonic-gate 		    ((flags & KPD_LOCKED) ? HAT_LOAD_LOCK : HAT_LOAD));
5777c478bd9Sstevel@tonic-gate 
5787c478bd9Sstevel@tonic-gate 		/*
5797c478bd9Sstevel@tonic-gate 		 * Now, release lock on the page.
5807c478bd9Sstevel@tonic-gate 		 */
5817c478bd9Sstevel@tonic-gate 		if (flags & KPD_LOCKED)
5827c478bd9Sstevel@tonic-gate 			page_downgrade(pp);
5837c478bd9Sstevel@tonic-gate 		else
5847c478bd9Sstevel@tonic-gate 			page_unlock(pp);
5857c478bd9Sstevel@tonic-gate 	}
5867c478bd9Sstevel@tonic-gate 
5877c478bd9Sstevel@tonic-gate 	kpd->kp_flags = flags;
5887c478bd9Sstevel@tonic-gate 	kpd->kp_base = vbase;
5897c478bd9Sstevel@tonic-gate 	kpd->kp_len = len;
5907c478bd9Sstevel@tonic-gate 	segkp_insert(seg, kpd);
5917c478bd9Sstevel@tonic-gate 	*tkpd = kpd;
5927c478bd9Sstevel@tonic-gate 	return (stom(kpd->kp_base, flags));
5937c478bd9Sstevel@tonic-gate }
5947c478bd9Sstevel@tonic-gate 
5957c478bd9Sstevel@tonic-gate /*
5967c478bd9Sstevel@tonic-gate  * Release the resource to cache if the pool(designate by the cookie)
5977c478bd9Sstevel@tonic-gate  * has less than the maximum allowable. If inserted in cache,
5987c478bd9Sstevel@tonic-gate  * segkp_delete insures element is taken off of active list.
5997c478bd9Sstevel@tonic-gate  */
6007c478bd9Sstevel@tonic-gate void
6017c478bd9Sstevel@tonic-gate segkp_release(struct seg *seg, caddr_t vaddr)
6027c478bd9Sstevel@tonic-gate {
6037c478bd9Sstevel@tonic-gate 	struct segkp_cache *freelist;
6047c478bd9Sstevel@tonic-gate 	struct segkp_data *kpd = NULL;
6057c478bd9Sstevel@tonic-gate 
6067c478bd9Sstevel@tonic-gate 	if ((kpd = segkp_find(seg, vaddr)) == NULL) {
6077c478bd9Sstevel@tonic-gate 		panic("segkp_release: null kpd");
6087c478bd9Sstevel@tonic-gate 		/*NOTREACHED*/
6097c478bd9Sstevel@tonic-gate 	}
6107c478bd9Sstevel@tonic-gate 
6117c478bd9Sstevel@tonic-gate 	if (kpd->kp_cookie != -1) {
6127c478bd9Sstevel@tonic-gate 		freelist = &segkp_cache[kpd->kp_cookie];
6137c478bd9Sstevel@tonic-gate 		mutex_enter(&segkp_lock);
6147c478bd9Sstevel@tonic-gate 		if (!segkp_indel && freelist->kpf_count < freelist->kpf_max) {
6157c478bd9Sstevel@tonic-gate 			segkp_delete(seg, kpd);
6167c478bd9Sstevel@tonic-gate 			kpd->kp_next = freelist->kpf_list;
6177c478bd9Sstevel@tonic-gate 			freelist->kpf_list = kpd;
6187c478bd9Sstevel@tonic-gate 			freelist->kpf_count++;
6197c478bd9Sstevel@tonic-gate 			mutex_exit(&segkp_lock);
6207c478bd9Sstevel@tonic-gate 			return;
6217c478bd9Sstevel@tonic-gate 		} else {
6227c478bd9Sstevel@tonic-gate 			mutex_exit(&segkp_lock);
6237c478bd9Sstevel@tonic-gate 			kpd->kp_cookie = -1;
6247c478bd9Sstevel@tonic-gate 		}
6257c478bd9Sstevel@tonic-gate 	}
6267c478bd9Sstevel@tonic-gate 	segkp_release_internal(seg, kpd, kpd->kp_len);
6277c478bd9Sstevel@tonic-gate }
6287c478bd9Sstevel@tonic-gate 
6297c478bd9Sstevel@tonic-gate /*
6307c478bd9Sstevel@tonic-gate  * Free the entire resource. segkp_unlock gets called with the start of the
6317c478bd9Sstevel@tonic-gate  * mapped portion of the resource. The length is the size of the mapped
6327c478bd9Sstevel@tonic-gate  * portion
6337c478bd9Sstevel@tonic-gate  */
6347c478bd9Sstevel@tonic-gate static void
6357c478bd9Sstevel@tonic-gate segkp_release_internal(struct seg *seg, struct segkp_data *kpd, size_t len)
6367c478bd9Sstevel@tonic-gate {
6377c478bd9Sstevel@tonic-gate 	caddr_t		va;
6387c478bd9Sstevel@tonic-gate 	long		i;
6397c478bd9Sstevel@tonic-gate 	long		redzone;
6407c478bd9Sstevel@tonic-gate 	size_t		np;
6417c478bd9Sstevel@tonic-gate 	page_t		*pp;
6427c478bd9Sstevel@tonic-gate 	struct vnode 	*vp;
6437c478bd9Sstevel@tonic-gate 	anoff_t		off;
6447c478bd9Sstevel@tonic-gate 	struct anon	*ap;
6457c478bd9Sstevel@tonic-gate 	pgcnt_t		segkpindex;
6467c478bd9Sstevel@tonic-gate 
6477c478bd9Sstevel@tonic-gate 	ASSERT(kpd != NULL);
6487c478bd9Sstevel@tonic-gate 	ASSERT((kpd->kp_flags & KPD_HASAMP) == 0 || kpd->kp_cookie == -1);
6497c478bd9Sstevel@tonic-gate 	np = btop(len);
6507c478bd9Sstevel@tonic-gate 
6517c478bd9Sstevel@tonic-gate 	/* Remove from active hash list */
6527c478bd9Sstevel@tonic-gate 	if (kpd->kp_cookie == -1) {
6537c478bd9Sstevel@tonic-gate 		mutex_enter(&segkp_lock);
6547c478bd9Sstevel@tonic-gate 		segkp_delete(seg, kpd);
6557c478bd9Sstevel@tonic-gate 		mutex_exit(&segkp_lock);
6567c478bd9Sstevel@tonic-gate 	}
6577c478bd9Sstevel@tonic-gate 
6587c478bd9Sstevel@tonic-gate 	/*
6597c478bd9Sstevel@tonic-gate 	 * Precompute redzone page index.
6607c478bd9Sstevel@tonic-gate 	 */
6617c478bd9Sstevel@tonic-gate 	redzone = -1;
6627c478bd9Sstevel@tonic-gate 	if (kpd->kp_flags & KPD_HASREDZONE)
6637c478bd9Sstevel@tonic-gate 		redzone = KPD_REDZONE(kpd);
6647c478bd9Sstevel@tonic-gate 
6657c478bd9Sstevel@tonic-gate 
6667c478bd9Sstevel@tonic-gate 	va = kpd->kp_base;
6677c478bd9Sstevel@tonic-gate 
6687c478bd9Sstevel@tonic-gate 	hat_unload(seg->s_as->a_hat, va, (np << PAGESHIFT),
6697c478bd9Sstevel@tonic-gate 	    ((kpd->kp_flags & KPD_LOCKED) ? HAT_UNLOAD_UNLOCK : HAT_UNLOAD));
6707c478bd9Sstevel@tonic-gate 	/*
6717c478bd9Sstevel@tonic-gate 	 * Free up those anon resources that are quiescent.
6727c478bd9Sstevel@tonic-gate 	 */
6737c478bd9Sstevel@tonic-gate 	if (segkp_fromheap)
6747c478bd9Sstevel@tonic-gate 		segkpindex = btop((uintptr_t)(va - kvseg.s_base));
6757c478bd9Sstevel@tonic-gate 	for (i = 0; i < np; i++, va += PAGESIZE) {
6767c478bd9Sstevel@tonic-gate 
6777c478bd9Sstevel@tonic-gate 		/*
6787c478bd9Sstevel@tonic-gate 		 * Clear the bit for this page from the bitmap.
6797c478bd9Sstevel@tonic-gate 		 */
6807c478bd9Sstevel@tonic-gate 		if (segkp_fromheap) {
6817c478bd9Sstevel@tonic-gate 			BT_ATOMIC_CLEAR(segkp_bitmap, segkpindex);
6827c478bd9Sstevel@tonic-gate 			segkpindex++;
6837c478bd9Sstevel@tonic-gate 		}
6847c478bd9Sstevel@tonic-gate 
6857c478bd9Sstevel@tonic-gate 		if (i == redzone)
6867c478bd9Sstevel@tonic-gate 			continue;
6877c478bd9Sstevel@tonic-gate 		if (kpd->kp_anon) {
6887c478bd9Sstevel@tonic-gate 			/*
6897c478bd9Sstevel@tonic-gate 			 * Free up anon resources and destroy the
6907c478bd9Sstevel@tonic-gate 			 * associated pages.
6917c478bd9Sstevel@tonic-gate 			 *
6927c478bd9Sstevel@tonic-gate 			 * Release the lock if there is one. Have to get the
6937c478bd9Sstevel@tonic-gate 			 * page to do this, unfortunately.
6947c478bd9Sstevel@tonic-gate 			 */
6957c478bd9Sstevel@tonic-gate 			if (kpd->kp_flags & KPD_LOCKED) {
6967c478bd9Sstevel@tonic-gate 				ap = anon_get_ptr(kpd->kp_anon,
6977c478bd9Sstevel@tonic-gate 				    kpd->kp_anon_idx + i);
6987c478bd9Sstevel@tonic-gate 				swap_xlate(ap, &vp, &off);
6997c478bd9Sstevel@tonic-gate 				/* Find the shared-locked page. */
7007c478bd9Sstevel@tonic-gate 				pp = page_find(vp, (u_offset_t)off);
7017c478bd9Sstevel@tonic-gate 				if (pp == NULL) {
7027c478bd9Sstevel@tonic-gate 					panic("segkp_release: "
7037c478bd9Sstevel@tonic-gate 					    "kp_anon: no page to unlock ");
7047c478bd9Sstevel@tonic-gate 					/*NOTREACHED*/
7057c478bd9Sstevel@tonic-gate 				}
7067c478bd9Sstevel@tonic-gate 				page_unlock(pp);
7077c478bd9Sstevel@tonic-gate 			}
7087c478bd9Sstevel@tonic-gate 			if ((kpd->kp_flags & KPD_HASAMP) == 0) {
7097c478bd9Sstevel@tonic-gate 				anon_free(kpd->kp_anon, kpd->kp_anon_idx + i,
7107c478bd9Sstevel@tonic-gate 				    PAGESIZE);
7110209230bSgjelinek 				anon_unresv_zone(PAGESIZE, NULL);
7120209230bSgjelinek 				atomic_add_long(&anon_segkp_pages_resv,
7130209230bSgjelinek 				    -1);
7147c478bd9Sstevel@tonic-gate 			}
7157c478bd9Sstevel@tonic-gate 			TRACE_5(TR_FAC_VM,
7167c478bd9Sstevel@tonic-gate 			    TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u",
7177c478bd9Sstevel@tonic-gate 			    kpd, va, PAGESIZE, 0, 0);
7187c478bd9Sstevel@tonic-gate 		} else {
7197c478bd9Sstevel@tonic-gate 			if (kpd->kp_flags & KPD_LOCKED) {
7207c478bd9Sstevel@tonic-gate 				pp = page_find(&kvp, (u_offset_t)(uintptr_t)va);
7217c478bd9Sstevel@tonic-gate 				if (pp == NULL) {
7227c478bd9Sstevel@tonic-gate 					panic("segkp_release: "
7237c478bd9Sstevel@tonic-gate 					    "no page to unlock");
7247c478bd9Sstevel@tonic-gate 					/*NOTREACHED*/
7257c478bd9Sstevel@tonic-gate 				}
7267c478bd9Sstevel@tonic-gate 				/*
7277c478bd9Sstevel@tonic-gate 				 * We should just upgrade the lock here
7287c478bd9Sstevel@tonic-gate 				 * but there is no upgrade that waits.
7297c478bd9Sstevel@tonic-gate 				 */
7307c478bd9Sstevel@tonic-gate 				page_unlock(pp);
7317c478bd9Sstevel@tonic-gate 			}
7327c478bd9Sstevel@tonic-gate 			pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)va,
7337c478bd9Sstevel@tonic-gate 			    SE_EXCL);
7347c478bd9Sstevel@tonic-gate 			if (pp != NULL)
7357c478bd9Sstevel@tonic-gate 				page_destroy(pp, 0);
7367c478bd9Sstevel@tonic-gate 		}
7377c478bd9Sstevel@tonic-gate 	}
7387c478bd9Sstevel@tonic-gate 
7397c478bd9Sstevel@tonic-gate 	/* If locked, release physical memory reservation */
7407c478bd9Sstevel@tonic-gate 	if (kpd->kp_flags & KPD_LOCKED) {
7417c478bd9Sstevel@tonic-gate 		pgcnt_t pages = btop(SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags));
7427c478bd9Sstevel@tonic-gate 		if ((kpd->kp_flags & KPD_NO_ANON) == 0)
7437c478bd9Sstevel@tonic-gate 			atomic_add_long(&anon_segkp_pages_locked, -pages);
7447c478bd9Sstevel@tonic-gate 		page_unresv(pages);
7457c478bd9Sstevel@tonic-gate 	}
7467c478bd9Sstevel@tonic-gate 
7477c478bd9Sstevel@tonic-gate 	vmem_free(SEGKP_VMEM(seg), kpd->kp_base, kpd->kp_len);
7487c478bd9Sstevel@tonic-gate 	kmem_free(kpd, sizeof (struct segkp_data));
7497c478bd9Sstevel@tonic-gate }
7507c478bd9Sstevel@tonic-gate 
7517c478bd9Sstevel@tonic-gate /*
7527c478bd9Sstevel@tonic-gate  * segkp_map_red() will check the current frame pointer against the
7537c478bd9Sstevel@tonic-gate  * stack base.  If the amount of stack remaining is questionable
7547c478bd9Sstevel@tonic-gate  * (less than red_minavail), then segkp_map_red() will map in the redzone
7557c478bd9Sstevel@tonic-gate  * and return 1.  Otherwise, it will return 0.  segkp_map_red() can
7567c478bd9Sstevel@tonic-gate  * _only_ be called when:
7577c478bd9Sstevel@tonic-gate  *
7587c478bd9Sstevel@tonic-gate  *   - it is safe to sleep on page_create_va().
7597c478bd9Sstevel@tonic-gate  *   - the caller is non-swappable.
7607c478bd9Sstevel@tonic-gate  *
7617c478bd9Sstevel@tonic-gate  * It is up to the caller to remember whether segkp_map_red() successfully
7627c478bd9Sstevel@tonic-gate  * mapped the redzone, and, if so, to call segkp_unmap_red() at a later
7637c478bd9Sstevel@tonic-gate  * time.  Note that the caller must _remain_ non-swappable until after
7647c478bd9Sstevel@tonic-gate  * calling segkp_unmap_red().
7657c478bd9Sstevel@tonic-gate  *
7667c478bd9Sstevel@tonic-gate  * Currently, this routine is only called from pagefault() (which necessarily
7677c478bd9Sstevel@tonic-gate  * satisfies the above conditions).
7687c478bd9Sstevel@tonic-gate  */
7697c478bd9Sstevel@tonic-gate #if defined(STACK_GROWTH_DOWN)
7707c478bd9Sstevel@tonic-gate int
7717c478bd9Sstevel@tonic-gate segkp_map_red(void)
7727c478bd9Sstevel@tonic-gate {
7737c478bd9Sstevel@tonic-gate 	uintptr_t fp = STACK_BIAS + (uintptr_t)getfp();
7747c478bd9Sstevel@tonic-gate #ifndef _LP64
7757c478bd9Sstevel@tonic-gate 	caddr_t stkbase;
7767c478bd9Sstevel@tonic-gate #endif
7777c478bd9Sstevel@tonic-gate 
7787c478bd9Sstevel@tonic-gate 	ASSERT(curthread->t_schedflag & TS_DONT_SWAP);
7797c478bd9Sstevel@tonic-gate 
7807c478bd9Sstevel@tonic-gate 	/*
7817c478bd9Sstevel@tonic-gate 	 * Optimize for the common case where we simply return.
7827c478bd9Sstevel@tonic-gate 	 */
7837c478bd9Sstevel@tonic-gate 	if ((curthread->t_red_pp == NULL) &&
7847c478bd9Sstevel@tonic-gate 	    (fp - (uintptr_t)curthread->t_stkbase >= red_minavail))
7857c478bd9Sstevel@tonic-gate 		return (0);
7867c478bd9Sstevel@tonic-gate 
7877c478bd9Sstevel@tonic-gate #if defined(_LP64)
7887c478bd9Sstevel@tonic-gate 	/*
7897c478bd9Sstevel@tonic-gate 	 * XXX	We probably need something better than this.
7907c478bd9Sstevel@tonic-gate 	 */
7917c478bd9Sstevel@tonic-gate 	panic("kernel stack overflow");
7927c478bd9Sstevel@tonic-gate 	/*NOTREACHED*/
7937c478bd9Sstevel@tonic-gate #else /* _LP64 */
7947c478bd9Sstevel@tonic-gate 	if (curthread->t_red_pp == NULL) {
7957c478bd9Sstevel@tonic-gate 		page_t *red_pp;
7967c478bd9Sstevel@tonic-gate 		struct seg kseg;
7977c478bd9Sstevel@tonic-gate 
7987c478bd9Sstevel@tonic-gate 		caddr_t red_va = (caddr_t)
7997c478bd9Sstevel@tonic-gate 		    (((uintptr_t)curthread->t_stkbase & (uintptr_t)PAGEMASK) -
8007c478bd9Sstevel@tonic-gate 		    PAGESIZE);
8017c478bd9Sstevel@tonic-gate 
8027c478bd9Sstevel@tonic-gate 		ASSERT(page_exists(&kvp, (u_offset_t)(uintptr_t)red_va) ==
8037c478bd9Sstevel@tonic-gate 		    NULL);
8047c478bd9Sstevel@tonic-gate 
8057c478bd9Sstevel@tonic-gate 		/*
8067c478bd9Sstevel@tonic-gate 		 * Allocate the physical for the red page.
8077c478bd9Sstevel@tonic-gate 		 */
8087c478bd9Sstevel@tonic-gate 		/*
8097c478bd9Sstevel@tonic-gate 		 * No PG_NORELOC here to avoid waits. Unlikely to get
8107c478bd9Sstevel@tonic-gate 		 * a relocate happening in the short time the page exists
8117c478bd9Sstevel@tonic-gate 		 * and it will be OK anyway.
8127c478bd9Sstevel@tonic-gate 		 */
8137c478bd9Sstevel@tonic-gate 
8147c478bd9Sstevel@tonic-gate 		kseg.s_as = &kas;
8157c478bd9Sstevel@tonic-gate 		red_pp = page_create_va(&kvp, (u_offset_t)(uintptr_t)red_va,
8167c478bd9Sstevel@tonic-gate 		    PAGESIZE, PG_WAIT | PG_EXCL, &kseg, red_va);
8177c478bd9Sstevel@tonic-gate 		ASSERT(red_pp != NULL);
8187c478bd9Sstevel@tonic-gate 
8197c478bd9Sstevel@tonic-gate 		/*
8207c478bd9Sstevel@tonic-gate 		 * So we now have a page to jam into the redzone...
8217c478bd9Sstevel@tonic-gate 		 */
8227c478bd9Sstevel@tonic-gate 		page_io_unlock(red_pp);
8237c478bd9Sstevel@tonic-gate 
8247c478bd9Sstevel@tonic-gate 		hat_memload(kas.a_hat, red_va, red_pp,
8257c478bd9Sstevel@tonic-gate 		    (PROT_READ|PROT_WRITE), HAT_LOAD_LOCK);
8267c478bd9Sstevel@tonic-gate 		page_downgrade(red_pp);
8277c478bd9Sstevel@tonic-gate 
8287c478bd9Sstevel@tonic-gate 		/*
8297c478bd9Sstevel@tonic-gate 		 * The page is left SE_SHARED locked so we can hold on to
8307c478bd9Sstevel@tonic-gate 		 * the page_t pointer.
8317c478bd9Sstevel@tonic-gate 		 */
8327c478bd9Sstevel@tonic-gate 		curthread->t_red_pp = red_pp;
8337c478bd9Sstevel@tonic-gate 
8347c478bd9Sstevel@tonic-gate 		atomic_add_32(&red_nmapped, 1);
8357c478bd9Sstevel@tonic-gate 		while (fp - (uintptr_t)curthread->t_stkbase < red_closest) {
8367c478bd9Sstevel@tonic-gate 			(void) cas32(&red_closest, red_closest,
8377c478bd9Sstevel@tonic-gate 			    (uint32_t)(fp - (uintptr_t)curthread->t_stkbase));
8387c478bd9Sstevel@tonic-gate 		}
8397c478bd9Sstevel@tonic-gate 		return (1);
8407c478bd9Sstevel@tonic-gate 	}
8417c478bd9Sstevel@tonic-gate 
8427c478bd9Sstevel@tonic-gate 	stkbase = (caddr_t)(((uintptr_t)curthread->t_stkbase &
8437c478bd9Sstevel@tonic-gate 	    (uintptr_t)PAGEMASK) - PAGESIZE);
8447c478bd9Sstevel@tonic-gate 
8457c478bd9Sstevel@tonic-gate 	atomic_add_32(&red_ndoubles, 1);
8467c478bd9Sstevel@tonic-gate 
8477c478bd9Sstevel@tonic-gate 	if (fp - (uintptr_t)stkbase < RED_DEEP_THRESHOLD) {
8487c478bd9Sstevel@tonic-gate 		/*
8497c478bd9Sstevel@tonic-gate 		 * Oh boy.  We're already deep within the mapped-in
8507c478bd9Sstevel@tonic-gate 		 * redzone page, and the caller is trying to prepare
8517c478bd9Sstevel@tonic-gate 		 * for a deep stack run.  We're running without a
8527c478bd9Sstevel@tonic-gate 		 * redzone right now:  if the caller plows off the
8537c478bd9Sstevel@tonic-gate 		 * end of the stack, it'll plow another thread or
8547c478bd9Sstevel@tonic-gate 		 * LWP structure.  That situation could result in
8557c478bd9Sstevel@tonic-gate 		 * a very hard-to-debug panic, so, in the spirit of
8567c478bd9Sstevel@tonic-gate 		 * recording the name of one's killer in one's own
8577c478bd9Sstevel@tonic-gate 		 * blood, we're going to record lbolt and the calling
8587c478bd9Sstevel@tonic-gate 		 * thread.
8597c478bd9Sstevel@tonic-gate 		 */
8607c478bd9Sstevel@tonic-gate 		red_deep_lbolt = lbolt;
8617c478bd9Sstevel@tonic-gate 		red_deep_thread = curthread;
8627c478bd9Sstevel@tonic-gate 	}
8637c478bd9Sstevel@tonic-gate 
8647c478bd9Sstevel@tonic-gate 	/*
8657c478bd9Sstevel@tonic-gate 	 * If this is a DEBUG kernel, and we've run too deep for comfort, toss.
8667c478bd9Sstevel@tonic-gate 	 */
8677c478bd9Sstevel@tonic-gate 	ASSERT(fp - (uintptr_t)stkbase >= RED_DEEP_THRESHOLD);
8687c478bd9Sstevel@tonic-gate 	return (0);
8697c478bd9Sstevel@tonic-gate #endif /* _LP64 */
8707c478bd9Sstevel@tonic-gate }
8717c478bd9Sstevel@tonic-gate 
8727c478bd9Sstevel@tonic-gate void
8737c478bd9Sstevel@tonic-gate segkp_unmap_red(void)
8747c478bd9Sstevel@tonic-gate {
8757c478bd9Sstevel@tonic-gate 	page_t *pp;
8767c478bd9Sstevel@tonic-gate 	caddr_t red_va = (caddr_t)(((uintptr_t)curthread->t_stkbase &
8777c478bd9Sstevel@tonic-gate 	    (uintptr_t)PAGEMASK) - PAGESIZE);
8787c478bd9Sstevel@tonic-gate 
8797c478bd9Sstevel@tonic-gate 	ASSERT(curthread->t_red_pp != NULL);
8807c478bd9Sstevel@tonic-gate 	ASSERT(curthread->t_schedflag & TS_DONT_SWAP);
8817c478bd9Sstevel@tonic-gate 
8827c478bd9Sstevel@tonic-gate 	/*
8837c478bd9Sstevel@tonic-gate 	 * Because we locked the mapping down, we can't simply rely
8847c478bd9Sstevel@tonic-gate 	 * on page_destroy() to clean everything up;  we need to call
8857c478bd9Sstevel@tonic-gate 	 * hat_unload() to explicitly unlock the mapping resources.
8867c478bd9Sstevel@tonic-gate 	 */
8877c478bd9Sstevel@tonic-gate 	hat_unload(kas.a_hat, red_va, PAGESIZE, HAT_UNLOAD_UNLOCK);
8887c478bd9Sstevel@tonic-gate 
8897c478bd9Sstevel@tonic-gate 	pp = curthread->t_red_pp;
8907c478bd9Sstevel@tonic-gate 
8917c478bd9Sstevel@tonic-gate 	ASSERT(pp == page_find(&kvp, (u_offset_t)(uintptr_t)red_va));
8927c478bd9Sstevel@tonic-gate 
8937c478bd9Sstevel@tonic-gate 	/*
8947c478bd9Sstevel@tonic-gate 	 * Need to upgrade the SE_SHARED lock to SE_EXCL.
8957c478bd9Sstevel@tonic-gate 	 */
8967c478bd9Sstevel@tonic-gate 	if (!page_tryupgrade(pp)) {
8977c478bd9Sstevel@tonic-gate 		/*
8987c478bd9Sstevel@tonic-gate 		 * As there is now wait for upgrade, release the
8997c478bd9Sstevel@tonic-gate 		 * SE_SHARED lock and wait for SE_EXCL.
9007c478bd9Sstevel@tonic-gate 		 */
9017c478bd9Sstevel@tonic-gate 		page_unlock(pp);
9027c478bd9Sstevel@tonic-gate 		pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)red_va, SE_EXCL);
9037c478bd9Sstevel@tonic-gate 		/* pp may be NULL here, hence the test below */
9047c478bd9Sstevel@tonic-gate 	}
9057c478bd9Sstevel@tonic-gate 
9067c478bd9Sstevel@tonic-gate 	/*
9077c478bd9Sstevel@tonic-gate 	 * Destroy the page, with dontfree set to zero (i.e. free it).
9087c478bd9Sstevel@tonic-gate 	 */
9097c478bd9Sstevel@tonic-gate 	if (pp != NULL)
9107c478bd9Sstevel@tonic-gate 		page_destroy(pp, 0);
9117c478bd9Sstevel@tonic-gate 	curthread->t_red_pp = NULL;
9127c478bd9Sstevel@tonic-gate }
9137c478bd9Sstevel@tonic-gate #else
9147c478bd9Sstevel@tonic-gate #error Red stacks only supported with downwards stack growth.
9157c478bd9Sstevel@tonic-gate #endif
9167c478bd9Sstevel@tonic-gate 
9177c478bd9Sstevel@tonic-gate /*
9187c478bd9Sstevel@tonic-gate  * Handle a fault on an address corresponding to one of the
9197c478bd9Sstevel@tonic-gate  * resources in the segkp segment.
9207c478bd9Sstevel@tonic-gate  */
9217c478bd9Sstevel@tonic-gate faultcode_t
9227c478bd9Sstevel@tonic-gate segkp_fault(
9237c478bd9Sstevel@tonic-gate 	struct hat	*hat,
9247c478bd9Sstevel@tonic-gate 	struct seg	*seg,
9257c478bd9Sstevel@tonic-gate 	caddr_t		vaddr,
9267c478bd9Sstevel@tonic-gate 	size_t		len,
9277c478bd9Sstevel@tonic-gate 	enum fault_type	type,
9287c478bd9Sstevel@tonic-gate 	enum seg_rw rw)
9297c478bd9Sstevel@tonic-gate {
9307c478bd9Sstevel@tonic-gate 	struct segkp_data	*kpd = NULL;
9317c478bd9Sstevel@tonic-gate 	int			err;
9327c478bd9Sstevel@tonic-gate 
9337c478bd9Sstevel@tonic-gate 	ASSERT(seg->s_as == &kas && RW_READ_HELD(&seg->s_as->a_lock));
9347c478bd9Sstevel@tonic-gate 
9357c478bd9Sstevel@tonic-gate 	/*
9367c478bd9Sstevel@tonic-gate 	 * Sanity checks.
9377c478bd9Sstevel@tonic-gate 	 */
9387c478bd9Sstevel@tonic-gate 	if (type == F_PROT) {
9397c478bd9Sstevel@tonic-gate 		panic("segkp_fault: unexpected F_PROT fault");
9407c478bd9Sstevel@tonic-gate 		/*NOTREACHED*/
9417c478bd9Sstevel@tonic-gate 	}
9427c478bd9Sstevel@tonic-gate 
9437c478bd9Sstevel@tonic-gate 	if ((kpd = segkp_find(seg, vaddr)) == NULL)
9447c478bd9Sstevel@tonic-gate 		return (FC_NOMAP);
9457c478bd9Sstevel@tonic-gate 
9467c478bd9Sstevel@tonic-gate 	mutex_enter(&kpd->kp_lock);
9477c478bd9Sstevel@tonic-gate 
9487c478bd9Sstevel@tonic-gate 	if (type == F_SOFTLOCK) {
9497c478bd9Sstevel@tonic-gate 		ASSERT(!(kpd->kp_flags & KPD_LOCKED));
9507c478bd9Sstevel@tonic-gate 		/*
9517c478bd9Sstevel@tonic-gate 		 * The F_SOFTLOCK case has more stringent
9527c478bd9Sstevel@tonic-gate 		 * range requirements: the given range must exactly coincide
9537c478bd9Sstevel@tonic-gate 		 * with the resource's mapped portion. Note reference to
9547c478bd9Sstevel@tonic-gate 		 * redzone is handled since vaddr would not equal base
9557c478bd9Sstevel@tonic-gate 		 */
9567c478bd9Sstevel@tonic-gate 		if (vaddr != stom(kpd->kp_base, kpd->kp_flags) ||
9577c478bd9Sstevel@tonic-gate 		    len != SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)) {
9587c478bd9Sstevel@tonic-gate 			mutex_exit(&kpd->kp_lock);
9597c478bd9Sstevel@tonic-gate 			return (FC_MAKE_ERR(EFAULT));
9607c478bd9Sstevel@tonic-gate 		}
9617c478bd9Sstevel@tonic-gate 
9627c478bd9Sstevel@tonic-gate 		if ((err = segkp_load(hat, seg, vaddr, len, kpd, KPD_LOCKED))) {
9637c478bd9Sstevel@tonic-gate 			mutex_exit(&kpd->kp_lock);
9647c478bd9Sstevel@tonic-gate 			return (FC_MAKE_ERR(err));
9657c478bd9Sstevel@tonic-gate 		}
9667c478bd9Sstevel@tonic-gate 		kpd->kp_flags |= KPD_LOCKED;
9677c478bd9Sstevel@tonic-gate 		mutex_exit(&kpd->kp_lock);
9687c478bd9Sstevel@tonic-gate 		return (0);
9697c478bd9Sstevel@tonic-gate 	}
9707c478bd9Sstevel@tonic-gate 
9717c478bd9Sstevel@tonic-gate 	if (type == F_INVAL) {
9727c478bd9Sstevel@tonic-gate 		ASSERT(!(kpd->kp_flags & KPD_NO_ANON));
9737c478bd9Sstevel@tonic-gate 
9747c478bd9Sstevel@tonic-gate 		/*
9757c478bd9Sstevel@tonic-gate 		 * Check if we touched the redzone. Somewhat optimistic
9767c478bd9Sstevel@tonic-gate 		 * here if we are touching the redzone of our own stack
9777c478bd9Sstevel@tonic-gate 		 * since we wouldn't have a stack to get this far...
9787c478bd9Sstevel@tonic-gate 		 */
9797c478bd9Sstevel@tonic-gate 		if ((kpd->kp_flags & KPD_HASREDZONE) &&
9807c478bd9Sstevel@tonic-gate 		    btop((uintptr_t)(vaddr - kpd->kp_base)) == KPD_REDZONE(kpd))
9817c478bd9Sstevel@tonic-gate 			panic("segkp_fault: accessing redzone");
9827c478bd9Sstevel@tonic-gate 
9837c478bd9Sstevel@tonic-gate 		/*
9847c478bd9Sstevel@tonic-gate 		 * This fault may occur while the page is being F_SOFTLOCK'ed.
9857c478bd9Sstevel@tonic-gate 		 * Return since a 2nd segkp_load is unnecessary and also would
9867c478bd9Sstevel@tonic-gate 		 * result in the page being locked twice and eventually
9877c478bd9Sstevel@tonic-gate 		 * hang the thread_reaper thread.
9887c478bd9Sstevel@tonic-gate 		 */
9897c478bd9Sstevel@tonic-gate 		if (kpd->kp_flags & KPD_LOCKED) {
9907c478bd9Sstevel@tonic-gate 			mutex_exit(&kpd->kp_lock);
9917c478bd9Sstevel@tonic-gate 			return (0);
9927c478bd9Sstevel@tonic-gate 		}
9937c478bd9Sstevel@tonic-gate 
9947c478bd9Sstevel@tonic-gate 		err = segkp_load(hat, seg, vaddr, len, kpd, kpd->kp_flags);
9957c478bd9Sstevel@tonic-gate 		mutex_exit(&kpd->kp_lock);
9967c478bd9Sstevel@tonic-gate 		return (err ? FC_MAKE_ERR(err) : 0);
9977c478bd9Sstevel@tonic-gate 	}
9987c478bd9Sstevel@tonic-gate 
9997c478bd9Sstevel@tonic-gate 	if (type == F_SOFTUNLOCK) {
10007c478bd9Sstevel@tonic-gate 		uint_t	flags;
10017c478bd9Sstevel@tonic-gate 
10027c478bd9Sstevel@tonic-gate 		/*
10037c478bd9Sstevel@tonic-gate 		 * Make sure the addr is LOCKED and it has anon backing
10047c478bd9Sstevel@tonic-gate 		 * before unlocking
10057c478bd9Sstevel@tonic-gate 		 */
10067c478bd9Sstevel@tonic-gate 		if ((kpd->kp_flags & (KPD_LOCKED|KPD_NO_ANON)) == KPD_NO_ANON) {
10077c478bd9Sstevel@tonic-gate 			panic("segkp_fault: bad unlock");
10087c478bd9Sstevel@tonic-gate 			/*NOTREACHED*/
10097c478bd9Sstevel@tonic-gate 		}
10107c478bd9Sstevel@tonic-gate 
10117c478bd9Sstevel@tonic-gate 		if (vaddr != stom(kpd->kp_base, kpd->kp_flags) ||
10127c478bd9Sstevel@tonic-gate 		    len != SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)) {
10137c478bd9Sstevel@tonic-gate 			panic("segkp_fault: bad range");
10147c478bd9Sstevel@tonic-gate 			/*NOTREACHED*/
10157c478bd9Sstevel@tonic-gate 		}
10167c478bd9Sstevel@tonic-gate 
10177c478bd9Sstevel@tonic-gate 		if (rw == S_WRITE)
10187c478bd9Sstevel@tonic-gate 			flags = kpd->kp_flags | KPD_WRITEDIRTY;
10197c478bd9Sstevel@tonic-gate 		else
10207c478bd9Sstevel@tonic-gate 			flags = kpd->kp_flags;
10217c478bd9Sstevel@tonic-gate 		err = segkp_unlock(hat, seg, vaddr, len, kpd, flags);
10227c478bd9Sstevel@tonic-gate 		kpd->kp_flags &= ~KPD_LOCKED;
10237c478bd9Sstevel@tonic-gate 		mutex_exit(&kpd->kp_lock);
10247c478bd9Sstevel@tonic-gate 		return (err ? FC_MAKE_ERR(err) : 0);
10257c478bd9Sstevel@tonic-gate 	}
10267c478bd9Sstevel@tonic-gate 	mutex_exit(&kpd->kp_lock);
10277c478bd9Sstevel@tonic-gate 	panic("segkp_fault: bogus fault type: %d\n", type);
10287c478bd9Sstevel@tonic-gate 	/*NOTREACHED*/
10297c478bd9Sstevel@tonic-gate }
10307c478bd9Sstevel@tonic-gate 
10317c478bd9Sstevel@tonic-gate /*
10327c478bd9Sstevel@tonic-gate  * Check that the given protections suffice over the range specified by
10337c478bd9Sstevel@tonic-gate  * vaddr and len.  For this segment type, the only issue is whether or
10347c478bd9Sstevel@tonic-gate  * not the range lies completely within the mapped part of an allocated
10357c478bd9Sstevel@tonic-gate  * resource.
10367c478bd9Sstevel@tonic-gate  */
10377c478bd9Sstevel@tonic-gate /* ARGSUSED */
10387c478bd9Sstevel@tonic-gate static int
10397c478bd9Sstevel@tonic-gate segkp_checkprot(struct seg *seg, caddr_t vaddr, size_t len, uint_t prot)
10407c478bd9Sstevel@tonic-gate {
10417c478bd9Sstevel@tonic-gate 	struct segkp_data *kpd = NULL;
10427c478bd9Sstevel@tonic-gate 	caddr_t mbase;
10437c478bd9Sstevel@tonic-gate 	size_t mlen;
10447c478bd9Sstevel@tonic-gate 
10457c478bd9Sstevel@tonic-gate 	if ((kpd = segkp_find(seg, vaddr)) == NULL)
10467c478bd9Sstevel@tonic-gate 		return (EACCES);
10477c478bd9Sstevel@tonic-gate 
10487c478bd9Sstevel@tonic-gate 	mutex_enter(&kpd->kp_lock);
10497c478bd9Sstevel@tonic-gate 	mbase = stom(kpd->kp_base, kpd->kp_flags);
10507c478bd9Sstevel@tonic-gate 	mlen = SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags);
10517c478bd9Sstevel@tonic-gate 	if (len > mlen || vaddr < mbase ||
10527c478bd9Sstevel@tonic-gate 	    ((vaddr + len) > (mbase + mlen))) {
10537c478bd9Sstevel@tonic-gate 		mutex_exit(&kpd->kp_lock);
10547c478bd9Sstevel@tonic-gate 		return (EACCES);
10557c478bd9Sstevel@tonic-gate 	}
10567c478bd9Sstevel@tonic-gate 	mutex_exit(&kpd->kp_lock);
10577c478bd9Sstevel@tonic-gate 	return (0);
10587c478bd9Sstevel@tonic-gate }
10597c478bd9Sstevel@tonic-gate 
10607c478bd9Sstevel@tonic-gate 
10617c478bd9Sstevel@tonic-gate /*
10627c478bd9Sstevel@tonic-gate  * Check to see if it makes sense to do kluster/read ahead to
10637c478bd9Sstevel@tonic-gate  * addr + delta relative to the mapping at addr.  We assume here
10647c478bd9Sstevel@tonic-gate  * that delta is a signed PAGESIZE'd multiple (which can be negative).
10657c478bd9Sstevel@tonic-gate  *
10667c478bd9Sstevel@tonic-gate  * For seg_u we always "approve" of this action from our standpoint.
10677c478bd9Sstevel@tonic-gate  */
10687c478bd9Sstevel@tonic-gate /*ARGSUSED*/
10697c478bd9Sstevel@tonic-gate static int
10707c478bd9Sstevel@tonic-gate segkp_kluster(struct seg *seg, caddr_t addr, ssize_t delta)
10717c478bd9Sstevel@tonic-gate {
10727c478bd9Sstevel@tonic-gate 	return (0);
10737c478bd9Sstevel@tonic-gate }
10747c478bd9Sstevel@tonic-gate 
10757c478bd9Sstevel@tonic-gate /*
10767c478bd9Sstevel@tonic-gate  * Load and possibly lock intra-slot resources in the range given by
10777c478bd9Sstevel@tonic-gate  * vaddr and len.
10787c478bd9Sstevel@tonic-gate  */
10797c478bd9Sstevel@tonic-gate static int
10807c478bd9Sstevel@tonic-gate segkp_load(
10817c478bd9Sstevel@tonic-gate 	struct hat *hat,
10827c478bd9Sstevel@tonic-gate 	struct seg *seg,
10837c478bd9Sstevel@tonic-gate 	caddr_t vaddr,
10847c478bd9Sstevel@tonic-gate 	size_t len,
10857c478bd9Sstevel@tonic-gate 	struct segkp_data *kpd,
10867c478bd9Sstevel@tonic-gate 	uint_t flags)
10877c478bd9Sstevel@tonic-gate {
10887c478bd9Sstevel@tonic-gate 	caddr_t va;
10897c478bd9Sstevel@tonic-gate 	caddr_t vlim;
10907c478bd9Sstevel@tonic-gate 	ulong_t i;
10917c478bd9Sstevel@tonic-gate 	uint_t lock;
10927c478bd9Sstevel@tonic-gate 
10937c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&kpd->kp_lock));
10947c478bd9Sstevel@tonic-gate 
10957c478bd9Sstevel@tonic-gate 	len = P2ROUNDUP(len, PAGESIZE);
10967c478bd9Sstevel@tonic-gate 
10977c478bd9Sstevel@tonic-gate 	/* If locking, reserve physical memory */
10987c478bd9Sstevel@tonic-gate 	if (flags & KPD_LOCKED) {
10997c478bd9Sstevel@tonic-gate 		pgcnt_t pages = btop(len);
11007c478bd9Sstevel@tonic-gate 		if ((kpd->kp_flags & KPD_NO_ANON) == 0)
11017c478bd9Sstevel@tonic-gate 			atomic_add_long(&anon_segkp_pages_locked, pages);
11027c478bd9Sstevel@tonic-gate 		(void) page_resv(pages, KM_SLEEP);
11037c478bd9Sstevel@tonic-gate 	}
11047c478bd9Sstevel@tonic-gate 
11057c478bd9Sstevel@tonic-gate 	/*
11067c478bd9Sstevel@tonic-gate 	 * Loop through the pages in the given range.
11077c478bd9Sstevel@tonic-gate 	 */
11087c478bd9Sstevel@tonic-gate 	va = (caddr_t)((uintptr_t)vaddr & (uintptr_t)PAGEMASK);
11097c478bd9Sstevel@tonic-gate 	vaddr = va;
11107c478bd9Sstevel@tonic-gate 	vlim = va + len;
11117c478bd9Sstevel@tonic-gate 	lock = flags & KPD_LOCKED;
11127c478bd9Sstevel@tonic-gate 	i = ((uintptr_t)(va - kpd->kp_base)) >> PAGESHIFT;
11137c478bd9Sstevel@tonic-gate 	for (; va < vlim; va += PAGESIZE, i++) {
11147c478bd9Sstevel@tonic-gate 		page_t		*pl[2];	/* second element NULL terminator */
11157c478bd9Sstevel@tonic-gate 		struct vnode    *vp;
11167c478bd9Sstevel@tonic-gate 		anoff_t		off;
11177c478bd9Sstevel@tonic-gate 		int		err;
11187c478bd9Sstevel@tonic-gate 		struct anon	*ap;
11197c478bd9Sstevel@tonic-gate 
11207c478bd9Sstevel@tonic-gate 		/*
11217c478bd9Sstevel@tonic-gate 		 * Summon the page.  If it's not resident, arrange
11227c478bd9Sstevel@tonic-gate 		 * for synchronous i/o to pull it in.
11237c478bd9Sstevel@tonic-gate 		 */
11247c478bd9Sstevel@tonic-gate 		ap = anon_get_ptr(kpd->kp_anon, kpd->kp_anon_idx + i);
11257c478bd9Sstevel@tonic-gate 		swap_xlate(ap, &vp, &off);
11267c478bd9Sstevel@tonic-gate 
11277c478bd9Sstevel@tonic-gate 		/*
11287c478bd9Sstevel@tonic-gate 		 * The returned page list will have exactly one entry,
11297c478bd9Sstevel@tonic-gate 		 * which is returned to us already kept.
11307c478bd9Sstevel@tonic-gate 		 */
11317c478bd9Sstevel@tonic-gate 		err = VOP_GETPAGE(vp, (offset_t)off, PAGESIZE, NULL,
1132*da6c28aaSamw 		    pl, PAGESIZE, seg, va, S_READ, kcred, NULL);
11337c478bd9Sstevel@tonic-gate 
11347c478bd9Sstevel@tonic-gate 		if (err) {
11357c478bd9Sstevel@tonic-gate 			/*
11367c478bd9Sstevel@tonic-gate 			 * Back out of what we've done so far.
11377c478bd9Sstevel@tonic-gate 			 */
11387c478bd9Sstevel@tonic-gate 			(void) segkp_unlock(hat, seg, vaddr,
11397c478bd9Sstevel@tonic-gate 			    (va - vaddr), kpd, flags);
11407c478bd9Sstevel@tonic-gate 			return (err);
11417c478bd9Sstevel@tonic-gate 		}
11427c478bd9Sstevel@tonic-gate 
11437c478bd9Sstevel@tonic-gate 		/*
11447c478bd9Sstevel@tonic-gate 		 * Load an MMU translation for the page.
11457c478bd9Sstevel@tonic-gate 		 */
11467c478bd9Sstevel@tonic-gate 		hat_memload(hat, va, pl[0], (PROT_READ|PROT_WRITE),
11477c478bd9Sstevel@tonic-gate 		    lock ? HAT_LOAD_LOCK : HAT_LOAD);
11487c478bd9Sstevel@tonic-gate 
11497c478bd9Sstevel@tonic-gate 		if (!lock) {
11507c478bd9Sstevel@tonic-gate 			/*
11517c478bd9Sstevel@tonic-gate 			 * Now, release "shared" lock on the page.
11527c478bd9Sstevel@tonic-gate 			 */
11537c478bd9Sstevel@tonic-gate 			page_unlock(pl[0]);
11547c478bd9Sstevel@tonic-gate 		}
11557c478bd9Sstevel@tonic-gate 	}
11567c478bd9Sstevel@tonic-gate 	return (0);
11577c478bd9Sstevel@tonic-gate }
11587c478bd9Sstevel@tonic-gate 
11597c478bd9Sstevel@tonic-gate /*
11607c478bd9Sstevel@tonic-gate  * At the very least unload the mmu-translations and unlock the range if locked
11617c478bd9Sstevel@tonic-gate  * Can be called with the following flag value KPD_WRITEDIRTY which specifies
11627c478bd9Sstevel@tonic-gate  * any dirty pages should be written to disk.
11637c478bd9Sstevel@tonic-gate  */
11647c478bd9Sstevel@tonic-gate static int
11657c478bd9Sstevel@tonic-gate segkp_unlock(
11667c478bd9Sstevel@tonic-gate 	struct hat *hat,
11677c478bd9Sstevel@tonic-gate 	struct seg *seg,
11687c478bd9Sstevel@tonic-gate 	caddr_t vaddr,
11697c478bd9Sstevel@tonic-gate 	size_t len,
11707c478bd9Sstevel@tonic-gate 	struct segkp_data *kpd,
11717c478bd9Sstevel@tonic-gate 	uint_t flags)
11727c478bd9Sstevel@tonic-gate {
11737c478bd9Sstevel@tonic-gate 	caddr_t va;
11747c478bd9Sstevel@tonic-gate 	caddr_t vlim;
11757c478bd9Sstevel@tonic-gate 	ulong_t i;
11767c478bd9Sstevel@tonic-gate 	struct page *pp;
11777c478bd9Sstevel@tonic-gate 	struct vnode *vp;
11787c478bd9Sstevel@tonic-gate 	anoff_t off;
11797c478bd9Sstevel@tonic-gate 	struct anon *ap;
11807c478bd9Sstevel@tonic-gate 
11817c478bd9Sstevel@tonic-gate #ifdef lint
11827c478bd9Sstevel@tonic-gate 	seg = seg;
11837c478bd9Sstevel@tonic-gate #endif /* lint */
11847c478bd9Sstevel@tonic-gate 
11857c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&kpd->kp_lock));
11867c478bd9Sstevel@tonic-gate 
11877c478bd9Sstevel@tonic-gate 	/*
11887c478bd9Sstevel@tonic-gate 	 * Loop through the pages in the given range. It is assumed
11897c478bd9Sstevel@tonic-gate 	 * segkp_unlock is called with page aligned base
11907c478bd9Sstevel@tonic-gate 	 */
11917c478bd9Sstevel@tonic-gate 	va = vaddr;
11927c478bd9Sstevel@tonic-gate 	vlim = va + len;
11937c478bd9Sstevel@tonic-gate 	i = ((uintptr_t)(va - kpd->kp_base)) >> PAGESHIFT;
11947c478bd9Sstevel@tonic-gate 	hat_unload(hat, va, len,
11957c478bd9Sstevel@tonic-gate 	    ((flags & KPD_LOCKED) ? HAT_UNLOAD_UNLOCK : HAT_UNLOAD));
11967c478bd9Sstevel@tonic-gate 	for (; va < vlim; va += PAGESIZE, i++) {
11977c478bd9Sstevel@tonic-gate 		/*
11987c478bd9Sstevel@tonic-gate 		 * Find the page associated with this part of the
11997c478bd9Sstevel@tonic-gate 		 * slot, tracking it down through its associated swap
12007c478bd9Sstevel@tonic-gate 		 * space.
12017c478bd9Sstevel@tonic-gate 		 */
12027c478bd9Sstevel@tonic-gate 		ap = anon_get_ptr(kpd->kp_anon, kpd->kp_anon_idx + i);
12037c478bd9Sstevel@tonic-gate 		swap_xlate(ap, &vp, &off);
12047c478bd9Sstevel@tonic-gate 
12057c478bd9Sstevel@tonic-gate 		if (flags & KPD_LOCKED) {
12067c478bd9Sstevel@tonic-gate 			if ((pp = page_find(vp, off)) == NULL) {
12077c478bd9Sstevel@tonic-gate 				if (flags & KPD_LOCKED) {
12087c478bd9Sstevel@tonic-gate 					panic("segkp_softunlock: missing page");
12097c478bd9Sstevel@tonic-gate 					/*NOTREACHED*/
12107c478bd9Sstevel@tonic-gate 				}
12117c478bd9Sstevel@tonic-gate 			}
12127c478bd9Sstevel@tonic-gate 		} else {
12137c478bd9Sstevel@tonic-gate 			/*
12147c478bd9Sstevel@tonic-gate 			 * Nothing to do if the slot is not locked and the
12157c478bd9Sstevel@tonic-gate 			 * page doesn't exist.
12167c478bd9Sstevel@tonic-gate 			 */
12177c478bd9Sstevel@tonic-gate 			if ((pp = page_lookup(vp, off, SE_SHARED)) == NULL)
12187c478bd9Sstevel@tonic-gate 				continue;
12197c478bd9Sstevel@tonic-gate 		}
12207c478bd9Sstevel@tonic-gate 
12217c478bd9Sstevel@tonic-gate 		/*
12227c478bd9Sstevel@tonic-gate 		 * If the page doesn't have any translations, is
12237c478bd9Sstevel@tonic-gate 		 * dirty and not being shared, then push it out
12247c478bd9Sstevel@tonic-gate 		 * asynchronously and avoid waiting for the
12257c478bd9Sstevel@tonic-gate 		 * pageout daemon to do it for us.
12267c478bd9Sstevel@tonic-gate 		 *
12277c478bd9Sstevel@tonic-gate 		 * XXX - Do we really need to get the "exclusive"
12287c478bd9Sstevel@tonic-gate 		 * lock via an upgrade?
12297c478bd9Sstevel@tonic-gate 		 */
12307c478bd9Sstevel@tonic-gate 		if ((flags & KPD_WRITEDIRTY) && !hat_page_is_mapped(pp) &&
12317c478bd9Sstevel@tonic-gate 		    hat_ismod(pp) && page_tryupgrade(pp)) {
12327c478bd9Sstevel@tonic-gate 			/*
12337c478bd9Sstevel@tonic-gate 			 * Hold the vnode before releasing the page lock to
12347c478bd9Sstevel@tonic-gate 			 * prevent it from being freed and re-used by some
12357c478bd9Sstevel@tonic-gate 			 * other thread.
12367c478bd9Sstevel@tonic-gate 			 */
12377c478bd9Sstevel@tonic-gate 			VN_HOLD(vp);
12387c478bd9Sstevel@tonic-gate 			page_unlock(pp);
12397c478bd9Sstevel@tonic-gate 
12407c478bd9Sstevel@tonic-gate 			/*
12417c478bd9Sstevel@tonic-gate 			 * Want most powerful credentials we can get so
12427c478bd9Sstevel@tonic-gate 			 * use kcred.
12437c478bd9Sstevel@tonic-gate 			 */
12447c478bd9Sstevel@tonic-gate 			(void) VOP_PUTPAGE(vp, (offset_t)off, PAGESIZE,
1245*da6c28aaSamw 			    B_ASYNC | B_FREE, kcred, NULL);
12467c478bd9Sstevel@tonic-gate 			VN_RELE(vp);
12477c478bd9Sstevel@tonic-gate 		} else {
12487c478bd9Sstevel@tonic-gate 			page_unlock(pp);
12497c478bd9Sstevel@tonic-gate 		}
12507c478bd9Sstevel@tonic-gate 	}
12517c478bd9Sstevel@tonic-gate 
12527c478bd9Sstevel@tonic-gate 	/* If unlocking, release physical memory */
12537c478bd9Sstevel@tonic-gate 	if (flags & KPD_LOCKED) {
12547c478bd9Sstevel@tonic-gate 		pgcnt_t pages = btopr(len);
12557c478bd9Sstevel@tonic-gate 		if ((kpd->kp_flags & KPD_NO_ANON) == 0)
12567c478bd9Sstevel@tonic-gate 			atomic_add_long(&anon_segkp_pages_locked, -pages);
12577c478bd9Sstevel@tonic-gate 		page_unresv(pages);
12587c478bd9Sstevel@tonic-gate 	}
12597c478bd9Sstevel@tonic-gate 	return (0);
12607c478bd9Sstevel@tonic-gate }
12617c478bd9Sstevel@tonic-gate 
12627c478bd9Sstevel@tonic-gate /*
12637c478bd9Sstevel@tonic-gate  * Insert the kpd in the hash table.
12647c478bd9Sstevel@tonic-gate  */
12657c478bd9Sstevel@tonic-gate static void
12667c478bd9Sstevel@tonic-gate segkp_insert(struct seg *seg, struct segkp_data *kpd)
12677c478bd9Sstevel@tonic-gate {
12687c478bd9Sstevel@tonic-gate 	struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
12697c478bd9Sstevel@tonic-gate 	int index;
12707c478bd9Sstevel@tonic-gate 
12717c478bd9Sstevel@tonic-gate 	/*
12727c478bd9Sstevel@tonic-gate 	 * Insert the kpd based on the address that will be returned
12737c478bd9Sstevel@tonic-gate 	 * via segkp_release.
12747c478bd9Sstevel@tonic-gate 	 */
12757c478bd9Sstevel@tonic-gate 	index = SEGKP_HASH(stom(kpd->kp_base, kpd->kp_flags));
12767c478bd9Sstevel@tonic-gate 	mutex_enter(&segkp_lock);
12777c478bd9Sstevel@tonic-gate 	kpd->kp_next = kpsd->kpsd_hash[index];
12787c478bd9Sstevel@tonic-gate 	kpsd->kpsd_hash[index] = kpd;
12797c478bd9Sstevel@tonic-gate 	mutex_exit(&segkp_lock);
12807c478bd9Sstevel@tonic-gate }
12817c478bd9Sstevel@tonic-gate 
12827c478bd9Sstevel@tonic-gate /*
12837c478bd9Sstevel@tonic-gate  * Remove kpd from the hash table.
12847c478bd9Sstevel@tonic-gate  */
12857c478bd9Sstevel@tonic-gate static void
12867c478bd9Sstevel@tonic-gate segkp_delete(struct seg *seg, struct segkp_data *kpd)
12877c478bd9Sstevel@tonic-gate {
12887c478bd9Sstevel@tonic-gate 	struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
12897c478bd9Sstevel@tonic-gate 	struct segkp_data **kpp;
12907c478bd9Sstevel@tonic-gate 	int index;
12917c478bd9Sstevel@tonic-gate 
12927c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&segkp_lock));
12937c478bd9Sstevel@tonic-gate 
12947c478bd9Sstevel@tonic-gate 	index = SEGKP_HASH(stom(kpd->kp_base, kpd->kp_flags));
12957c478bd9Sstevel@tonic-gate 	for (kpp = &kpsd->kpsd_hash[index];
12967c478bd9Sstevel@tonic-gate 	    *kpp != NULL; kpp = &((*kpp)->kp_next)) {
12977c478bd9Sstevel@tonic-gate 		if (*kpp == kpd) {
12987c478bd9Sstevel@tonic-gate 			*kpp = kpd->kp_next;
12997c478bd9Sstevel@tonic-gate 			return;
13007c478bd9Sstevel@tonic-gate 		}
13017c478bd9Sstevel@tonic-gate 	}
13027c478bd9Sstevel@tonic-gate 	panic("segkp_delete: unable to find element to delete");
13037c478bd9Sstevel@tonic-gate 	/*NOTREACHED*/
13047c478bd9Sstevel@tonic-gate }
13057c478bd9Sstevel@tonic-gate 
13067c478bd9Sstevel@tonic-gate /*
13077c478bd9Sstevel@tonic-gate  * Find the kpd associated with a vaddr.
13087c478bd9Sstevel@tonic-gate  *
13097c478bd9Sstevel@tonic-gate  * Most of the callers of segkp_find will pass the vaddr that
13107c478bd9Sstevel@tonic-gate  * hashes to the desired index, but there are cases where
13117c478bd9Sstevel@tonic-gate  * this is not true in which case we have to (potentially) scan
13127c478bd9Sstevel@tonic-gate  * the whole table looking for it. This should be very rare
13137c478bd9Sstevel@tonic-gate  * (e.g. a segkp_fault(F_INVAL) on an address somewhere in the
13147c478bd9Sstevel@tonic-gate  * middle of the segkp_data region).
13157c478bd9Sstevel@tonic-gate  */
13167c478bd9Sstevel@tonic-gate static struct segkp_data *
13177c478bd9Sstevel@tonic-gate segkp_find(struct seg *seg, caddr_t vaddr)
13187c478bd9Sstevel@tonic-gate {
13197c478bd9Sstevel@tonic-gate 	struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
13207c478bd9Sstevel@tonic-gate 	struct segkp_data *kpd;
13217c478bd9Sstevel@tonic-gate 	int	i;
13227c478bd9Sstevel@tonic-gate 	int	stop;
13237c478bd9Sstevel@tonic-gate 
13247c478bd9Sstevel@tonic-gate 	i = stop = SEGKP_HASH(vaddr);
13257c478bd9Sstevel@tonic-gate 	mutex_enter(&segkp_lock);
13267c478bd9Sstevel@tonic-gate 	do {
13277c478bd9Sstevel@tonic-gate 		for (kpd = kpsd->kpsd_hash[i]; kpd != NULL;
13287c478bd9Sstevel@tonic-gate 						kpd = kpd->kp_next) {
13297c478bd9Sstevel@tonic-gate 			if (vaddr >= kpd->kp_base &&
13307c478bd9Sstevel@tonic-gate 			    vaddr < kpd->kp_base + kpd->kp_len) {
13317c478bd9Sstevel@tonic-gate 				mutex_exit(&segkp_lock);
13327c478bd9Sstevel@tonic-gate 				return (kpd);
13337c478bd9Sstevel@tonic-gate 			}
13347c478bd9Sstevel@tonic-gate 		}
13357c478bd9Sstevel@tonic-gate 		if (--i < 0)
13367c478bd9Sstevel@tonic-gate 			i = SEGKP_HASHSZ - 1;	/* Wrap */
13377c478bd9Sstevel@tonic-gate 	} while (i != stop);
13387c478bd9Sstevel@tonic-gate 	mutex_exit(&segkp_lock);
13397c478bd9Sstevel@tonic-gate 	return (NULL);		/* Not found */
13407c478bd9Sstevel@tonic-gate }
13417c478bd9Sstevel@tonic-gate 
13427c478bd9Sstevel@tonic-gate /*
13437c478bd9Sstevel@tonic-gate  * returns size of swappable area.
13447c478bd9Sstevel@tonic-gate  */
13457c478bd9Sstevel@tonic-gate size_t
13467c478bd9Sstevel@tonic-gate swapsize(caddr_t v)
13477c478bd9Sstevel@tonic-gate {
13487c478bd9Sstevel@tonic-gate 	struct segkp_data *kpd;
13497c478bd9Sstevel@tonic-gate 
13507c478bd9Sstevel@tonic-gate 	if ((kpd = segkp_find(segkp, v)) != NULL)
13517c478bd9Sstevel@tonic-gate 		return (SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags));
13527c478bd9Sstevel@tonic-gate 	else
13537c478bd9Sstevel@tonic-gate 		return (NULL);
13547c478bd9Sstevel@tonic-gate }
13557c478bd9Sstevel@tonic-gate 
13567c478bd9Sstevel@tonic-gate /*
13577c478bd9Sstevel@tonic-gate  * Dump out all the active segkp pages
13587c478bd9Sstevel@tonic-gate  */
13597c478bd9Sstevel@tonic-gate static void
13607c478bd9Sstevel@tonic-gate segkp_dump(struct seg *seg)
13617c478bd9Sstevel@tonic-gate {
13627c478bd9Sstevel@tonic-gate 	int i;
13637c478bd9Sstevel@tonic-gate 	struct segkp_data *kpd;
13647c478bd9Sstevel@tonic-gate 	struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
13657c478bd9Sstevel@tonic-gate 
13667c478bd9Sstevel@tonic-gate 	for (i = 0; i < SEGKP_HASHSZ; i++) {
13677c478bd9Sstevel@tonic-gate 		for (kpd = kpsd->kpsd_hash[i];
13687c478bd9Sstevel@tonic-gate 		    kpd != NULL; kpd = kpd->kp_next) {
13697c478bd9Sstevel@tonic-gate 			pfn_t pfn;
13707c478bd9Sstevel@tonic-gate 			caddr_t addr;
13717c478bd9Sstevel@tonic-gate 			caddr_t eaddr;
13727c478bd9Sstevel@tonic-gate 
13737c478bd9Sstevel@tonic-gate 			addr = kpd->kp_base;
13747c478bd9Sstevel@tonic-gate 			eaddr = addr + kpd->kp_len;
13757c478bd9Sstevel@tonic-gate 			while (addr < eaddr) {
13767c478bd9Sstevel@tonic-gate 				ASSERT(seg->s_as == &kas);
13777c478bd9Sstevel@tonic-gate 				pfn = hat_getpfnum(seg->s_as->a_hat, addr);
13787c478bd9Sstevel@tonic-gate 				if (pfn != PFN_INVALID)
13797c478bd9Sstevel@tonic-gate 					dump_addpage(seg->s_as, addr, pfn);
13807c478bd9Sstevel@tonic-gate 				addr += PAGESIZE;
13817c478bd9Sstevel@tonic-gate 				dump_timeleft = dump_timeout;
13827c478bd9Sstevel@tonic-gate 			}
13837c478bd9Sstevel@tonic-gate 		}
13847c478bd9Sstevel@tonic-gate 	}
13857c478bd9Sstevel@tonic-gate }
13867c478bd9Sstevel@tonic-gate 
13877c478bd9Sstevel@tonic-gate /*ARGSUSED*/
13887c478bd9Sstevel@tonic-gate static int
13897c478bd9Sstevel@tonic-gate segkp_pagelock(struct seg *seg, caddr_t addr, size_t len,
13907c478bd9Sstevel@tonic-gate     struct page ***ppp, enum lock_type type, enum seg_rw rw)
13917c478bd9Sstevel@tonic-gate {
13927c478bd9Sstevel@tonic-gate 	return (ENOTSUP);
13937c478bd9Sstevel@tonic-gate }
13947c478bd9Sstevel@tonic-gate 
13957c478bd9Sstevel@tonic-gate /*ARGSUSED*/
13967c478bd9Sstevel@tonic-gate static int
13977c478bd9Sstevel@tonic-gate segkp_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
13987c478bd9Sstevel@tonic-gate {
13997c478bd9Sstevel@tonic-gate 	return (ENODEV);
14007c478bd9Sstevel@tonic-gate }
14017c478bd9Sstevel@tonic-gate 
14027c478bd9Sstevel@tonic-gate /*ARGSUSED*/
14037c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t	*
14047c478bd9Sstevel@tonic-gate segkp_getpolicy(struct seg *seg, caddr_t addr)
14057c478bd9Sstevel@tonic-gate {
14067c478bd9Sstevel@tonic-gate 	return (NULL);
14077c478bd9Sstevel@tonic-gate }
14087c478bd9Sstevel@tonic-gate 
14091bd5c35fSelowe /*ARGSUSED*/
14101bd5c35fSelowe static int
14111bd5c35fSelowe segkp_capable(struct seg *seg, segcapability_t capability)
14121bd5c35fSelowe {
14131bd5c35fSelowe 	return (0);
14141bd5c35fSelowe }
14151bd5c35fSelowe 
14167c478bd9Sstevel@tonic-gate #include <sys/mem_config.h>
14177c478bd9Sstevel@tonic-gate 
14187c478bd9Sstevel@tonic-gate /*ARGSUSED*/
14197c478bd9Sstevel@tonic-gate static void
14207c478bd9Sstevel@tonic-gate segkp_mem_config_post_add(void *arg, pgcnt_t delta_pages)
14217c478bd9Sstevel@tonic-gate {}
14227c478bd9Sstevel@tonic-gate 
14237c478bd9Sstevel@tonic-gate /*
14247c478bd9Sstevel@tonic-gate  * During memory delete, turn off caches so that pages are not held.
14257c478bd9Sstevel@tonic-gate  * A better solution may be to unlock the pages while they are
14267c478bd9Sstevel@tonic-gate  * in the cache so that they may be collected naturally.
14277c478bd9Sstevel@tonic-gate  */
14287c478bd9Sstevel@tonic-gate 
14297c478bd9Sstevel@tonic-gate /*ARGSUSED*/
14307c478bd9Sstevel@tonic-gate static int
14317c478bd9Sstevel@tonic-gate segkp_mem_config_pre_del(void *arg, pgcnt_t delta_pages)
14327c478bd9Sstevel@tonic-gate {
14337c478bd9Sstevel@tonic-gate 	atomic_add_32(&segkp_indel, 1);
14347c478bd9Sstevel@tonic-gate 	segkp_cache_free();
14357c478bd9Sstevel@tonic-gate 	return (0);
14367c478bd9Sstevel@tonic-gate }
14377c478bd9Sstevel@tonic-gate 
14387c478bd9Sstevel@tonic-gate /*ARGSUSED*/
14397c478bd9Sstevel@tonic-gate static void
14407c478bd9Sstevel@tonic-gate segkp_mem_config_post_del(void *arg, pgcnt_t delta_pages, int cancelled)
14417c478bd9Sstevel@tonic-gate {
14427c478bd9Sstevel@tonic-gate 	atomic_add_32(&segkp_indel, -1);
14437c478bd9Sstevel@tonic-gate }
14447c478bd9Sstevel@tonic-gate 
14457c478bd9Sstevel@tonic-gate static kphysm_setup_vector_t segkp_mem_config_vec = {
14467c478bd9Sstevel@tonic-gate 	KPHYSM_SETUP_VECTOR_VERSION,
14477c478bd9Sstevel@tonic-gate 	segkp_mem_config_post_add,
14487c478bd9Sstevel@tonic-gate 	segkp_mem_config_pre_del,
14497c478bd9Sstevel@tonic-gate 	segkp_mem_config_post_del,
14507c478bd9Sstevel@tonic-gate };
14517c478bd9Sstevel@tonic-gate 
14527c478bd9Sstevel@tonic-gate static void
14537c478bd9Sstevel@tonic-gate segkpinit_mem_config(struct seg *seg)
14547c478bd9Sstevel@tonic-gate {
14557c478bd9Sstevel@tonic-gate 	int ret;
14567c478bd9Sstevel@tonic-gate 
14577c478bd9Sstevel@tonic-gate 	ret = kphysm_setup_func_register(&segkp_mem_config_vec, (void *)seg);
14587c478bd9Sstevel@tonic-gate 	ASSERT(ret == 0);
14597c478bd9Sstevel@tonic-gate }
1460