17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 57c478bd9Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 67c478bd9Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 77c478bd9Sstevel@tonic-gate * with the License. 87c478bd9Sstevel@tonic-gate * 97c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 107c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 117c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 127c478bd9Sstevel@tonic-gate * and limitations under the License. 137c478bd9Sstevel@tonic-gate * 147c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 157c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 167c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 177c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 187c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 197c478bd9Sstevel@tonic-gate * 207c478bd9Sstevel@tonic-gate * CDDL HEADER END 217c478bd9Sstevel@tonic-gate */ 227c478bd9Sstevel@tonic-gate /* 23*1bd5c35fSelowe * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 247c478bd9Sstevel@tonic-gate * Use is subject to license terms. 257c478bd9Sstevel@tonic-gate */ 267c478bd9Sstevel@tonic-gate 277c478bd9Sstevel@tonic-gate /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 287c478bd9Sstevel@tonic-gate /* All Rights Reserved */ 297c478bd9Sstevel@tonic-gate 307c478bd9Sstevel@tonic-gate /* 317c478bd9Sstevel@tonic-gate * Portions of this source code were derived from Berkeley 4.3 BSD 327c478bd9Sstevel@tonic-gate * under license from the Regents of the University of California. 337c478bd9Sstevel@tonic-gate */ 347c478bd9Sstevel@tonic-gate 357c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 367c478bd9Sstevel@tonic-gate 377c478bd9Sstevel@tonic-gate /* 387c478bd9Sstevel@tonic-gate * segkp is a segment driver that administers the allocation and deallocation 397c478bd9Sstevel@tonic-gate * of pageable variable size chunks of kernel virtual address space. Each 407c478bd9Sstevel@tonic-gate * allocated resource is page-aligned. 417c478bd9Sstevel@tonic-gate * 427c478bd9Sstevel@tonic-gate * The user may specify whether the resource should be initialized to 0, 437c478bd9Sstevel@tonic-gate * include a redzone, or locked in memory. 447c478bd9Sstevel@tonic-gate */ 457c478bd9Sstevel@tonic-gate 467c478bd9Sstevel@tonic-gate #include <sys/types.h> 477c478bd9Sstevel@tonic-gate #include <sys/t_lock.h> 487c478bd9Sstevel@tonic-gate #include <sys/thread.h> 497c478bd9Sstevel@tonic-gate #include <sys/param.h> 507c478bd9Sstevel@tonic-gate #include <sys/errno.h> 517c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 527c478bd9Sstevel@tonic-gate #include <sys/systm.h> 537c478bd9Sstevel@tonic-gate #include <sys/buf.h> 547c478bd9Sstevel@tonic-gate #include <sys/mman.h> 557c478bd9Sstevel@tonic-gate #include <sys/vnode.h> 567c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 577c478bd9Sstevel@tonic-gate #include <sys/swap.h> 587c478bd9Sstevel@tonic-gate #include <sys/tuneable.h> 597c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 607c478bd9Sstevel@tonic-gate #include <sys/vmem.h> 617c478bd9Sstevel@tonic-gate #include <sys/cred.h> 627c478bd9Sstevel@tonic-gate #include <sys/dumphdr.h> 637c478bd9Sstevel@tonic-gate #include <sys/debug.h> 647c478bd9Sstevel@tonic-gate #include <sys/vtrace.h> 657c478bd9Sstevel@tonic-gate #include <sys/stack.h> 667c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 677c478bd9Sstevel@tonic-gate #include <sys/archsystm.h> 687c478bd9Sstevel@tonic-gate #include <sys/lgrp.h> 697c478bd9Sstevel@tonic-gate 707c478bd9Sstevel@tonic-gate #include <vm/as.h> 717c478bd9Sstevel@tonic-gate #include <vm/seg.h> 727c478bd9Sstevel@tonic-gate #include <vm/seg_kp.h> 737c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h> 747c478bd9Sstevel@tonic-gate #include <vm/anon.h> 757c478bd9Sstevel@tonic-gate #include <vm/page.h> 767c478bd9Sstevel@tonic-gate #include <vm/hat.h> 777c478bd9Sstevel@tonic-gate #include <sys/bitmap.h> 787c478bd9Sstevel@tonic-gate 797c478bd9Sstevel@tonic-gate /* 807c478bd9Sstevel@tonic-gate * Private seg op routines 817c478bd9Sstevel@tonic-gate */ 827c478bd9Sstevel@tonic-gate static void segkp_badop(void); 837c478bd9Sstevel@tonic-gate static void segkp_dump(struct seg *seg); 847c478bd9Sstevel@tonic-gate static int segkp_checkprot(struct seg *seg, caddr_t addr, size_t len, 857c478bd9Sstevel@tonic-gate uint_t prot); 867c478bd9Sstevel@tonic-gate static int segkp_kluster(struct seg *seg, caddr_t addr, ssize_t delta); 877c478bd9Sstevel@tonic-gate static int segkp_pagelock(struct seg *seg, caddr_t addr, size_t len, 887c478bd9Sstevel@tonic-gate struct page ***page, enum lock_type type, 897c478bd9Sstevel@tonic-gate enum seg_rw rw); 907c478bd9Sstevel@tonic-gate static void segkp_insert(struct seg *seg, struct segkp_data *kpd); 917c478bd9Sstevel@tonic-gate static void segkp_delete(struct seg *seg, struct segkp_data *kpd); 927c478bd9Sstevel@tonic-gate static caddr_t segkp_get_internal(struct seg *seg, size_t len, uint_t flags, 937c478bd9Sstevel@tonic-gate struct segkp_data **tkpd, struct anon_map *amp); 947c478bd9Sstevel@tonic-gate static void segkp_release_internal(struct seg *seg, 957c478bd9Sstevel@tonic-gate struct segkp_data *kpd, size_t len); 967c478bd9Sstevel@tonic-gate static int segkp_unlock(struct hat *hat, struct seg *seg, caddr_t vaddr, 977c478bd9Sstevel@tonic-gate size_t len, struct segkp_data *kpd, uint_t flags); 987c478bd9Sstevel@tonic-gate static int segkp_load(struct hat *hat, struct seg *seg, caddr_t vaddr, 997c478bd9Sstevel@tonic-gate size_t len, struct segkp_data *kpd, uint_t flags); 1007c478bd9Sstevel@tonic-gate static struct segkp_data *segkp_find(struct seg *seg, caddr_t vaddr); 1017c478bd9Sstevel@tonic-gate static int segkp_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp); 1027c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t *segkp_getpolicy(struct seg *seg, 1037c478bd9Sstevel@tonic-gate caddr_t addr); 104*1bd5c35fSelowe static int segkp_capable(struct seg *seg, segcapability_t capability); 1057c478bd9Sstevel@tonic-gate 1067c478bd9Sstevel@tonic-gate /* 1077c478bd9Sstevel@tonic-gate * Lock used to protect the hash table(s) and caches. 1087c478bd9Sstevel@tonic-gate */ 1097c478bd9Sstevel@tonic-gate static kmutex_t segkp_lock; 1107c478bd9Sstevel@tonic-gate 1117c478bd9Sstevel@tonic-gate /* 1127c478bd9Sstevel@tonic-gate * The segkp caches 1137c478bd9Sstevel@tonic-gate */ 1147c478bd9Sstevel@tonic-gate static struct segkp_cache segkp_cache[SEGKP_MAX_CACHE]; 1157c478bd9Sstevel@tonic-gate 1167c478bd9Sstevel@tonic-gate #define SEGKP_BADOP(t) (t(*)())segkp_badop 1177c478bd9Sstevel@tonic-gate 1187c478bd9Sstevel@tonic-gate /* 1197c478bd9Sstevel@tonic-gate * When there are fewer than red_minavail bytes left on the stack, 1207c478bd9Sstevel@tonic-gate * segkp_map_red() will map in the redzone (if called). 5000 seems 1217c478bd9Sstevel@tonic-gate * to work reasonably well... 1227c478bd9Sstevel@tonic-gate */ 1237c478bd9Sstevel@tonic-gate long red_minavail = 5000; 1247c478bd9Sstevel@tonic-gate 1257c478bd9Sstevel@tonic-gate /* 1267c478bd9Sstevel@tonic-gate * will be set to 1 for 32 bit x86 systems only, in startup.c 1277c478bd9Sstevel@tonic-gate */ 1287c478bd9Sstevel@tonic-gate int segkp_fromheap = 0; 1297c478bd9Sstevel@tonic-gate ulong_t *segkp_bitmap; 1307c478bd9Sstevel@tonic-gate 1317c478bd9Sstevel@tonic-gate /* 1327c478bd9Sstevel@tonic-gate * If segkp_map_red() is called with the redzone already mapped and 1337c478bd9Sstevel@tonic-gate * with less than RED_DEEP_THRESHOLD bytes available on the stack, 1347c478bd9Sstevel@tonic-gate * then the stack situation has become quite serious; if much more stack 1357c478bd9Sstevel@tonic-gate * is consumed, we have the potential of scrogging the next thread/LWP 1367c478bd9Sstevel@tonic-gate * structure. To help debug the "can't happen" panics which may 1377c478bd9Sstevel@tonic-gate * result from this condition, we record lbolt and the calling thread 1387c478bd9Sstevel@tonic-gate * in red_deep_lbolt and red_deep_thread respectively. 1397c478bd9Sstevel@tonic-gate */ 1407c478bd9Sstevel@tonic-gate #define RED_DEEP_THRESHOLD 2000 1417c478bd9Sstevel@tonic-gate 1427c478bd9Sstevel@tonic-gate clock_t red_deep_lbolt; 1437c478bd9Sstevel@tonic-gate kthread_t *red_deep_thread; 1447c478bd9Sstevel@tonic-gate 1457c478bd9Sstevel@tonic-gate uint32_t red_nmapped; 1467c478bd9Sstevel@tonic-gate uint32_t red_closest = UINT_MAX; 1477c478bd9Sstevel@tonic-gate uint32_t red_ndoubles; 1487c478bd9Sstevel@tonic-gate 1497c478bd9Sstevel@tonic-gate pgcnt_t anon_segkp_pages_locked; /* See vm/anon.h */ 1507c478bd9Sstevel@tonic-gate 1517c478bd9Sstevel@tonic-gate static struct seg_ops segkp_ops = { 1527c478bd9Sstevel@tonic-gate SEGKP_BADOP(int), /* dup */ 1537c478bd9Sstevel@tonic-gate SEGKP_BADOP(int), /* unmap */ 1547c478bd9Sstevel@tonic-gate SEGKP_BADOP(void), /* free */ 1557c478bd9Sstevel@tonic-gate segkp_fault, 1567c478bd9Sstevel@tonic-gate SEGKP_BADOP(faultcode_t), /* faulta */ 1577c478bd9Sstevel@tonic-gate SEGKP_BADOP(int), /* setprot */ 1587c478bd9Sstevel@tonic-gate segkp_checkprot, 1597c478bd9Sstevel@tonic-gate segkp_kluster, 1607c478bd9Sstevel@tonic-gate SEGKP_BADOP(size_t), /* swapout */ 1617c478bd9Sstevel@tonic-gate SEGKP_BADOP(int), /* sync */ 1627c478bd9Sstevel@tonic-gate SEGKP_BADOP(size_t), /* incore */ 1637c478bd9Sstevel@tonic-gate SEGKP_BADOP(int), /* lockop */ 1647c478bd9Sstevel@tonic-gate SEGKP_BADOP(int), /* getprot */ 1657c478bd9Sstevel@tonic-gate SEGKP_BADOP(u_offset_t), /* getoffset */ 1667c478bd9Sstevel@tonic-gate SEGKP_BADOP(int), /* gettype */ 1677c478bd9Sstevel@tonic-gate SEGKP_BADOP(int), /* getvp */ 1687c478bd9Sstevel@tonic-gate SEGKP_BADOP(int), /* advise */ 1697c478bd9Sstevel@tonic-gate segkp_dump, /* dump */ 1707c478bd9Sstevel@tonic-gate segkp_pagelock, /* pagelock */ 1717c478bd9Sstevel@tonic-gate SEGKP_BADOP(int), /* setpgsz */ 1727c478bd9Sstevel@tonic-gate segkp_getmemid, /* getmemid */ 1737c478bd9Sstevel@tonic-gate segkp_getpolicy, /* getpolicy */ 174*1bd5c35fSelowe segkp_capable, /* capable */ 1757c478bd9Sstevel@tonic-gate }; 1767c478bd9Sstevel@tonic-gate 1777c478bd9Sstevel@tonic-gate 1787c478bd9Sstevel@tonic-gate static void 1797c478bd9Sstevel@tonic-gate segkp_badop(void) 1807c478bd9Sstevel@tonic-gate { 1817c478bd9Sstevel@tonic-gate panic("segkp_badop"); 1827c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 1837c478bd9Sstevel@tonic-gate } 1847c478bd9Sstevel@tonic-gate 1857c478bd9Sstevel@tonic-gate static void segkpinit_mem_config(struct seg *); 1867c478bd9Sstevel@tonic-gate 1877c478bd9Sstevel@tonic-gate static uint32_t segkp_indel; 1887c478bd9Sstevel@tonic-gate 1897c478bd9Sstevel@tonic-gate /* 1907c478bd9Sstevel@tonic-gate * Allocate the segment specific private data struct and fill it in 1917c478bd9Sstevel@tonic-gate * with the per kp segment mutex, anon ptr. array and hash table. 1927c478bd9Sstevel@tonic-gate */ 1937c478bd9Sstevel@tonic-gate int 1947c478bd9Sstevel@tonic-gate segkp_create(struct seg *seg) 1957c478bd9Sstevel@tonic-gate { 1967c478bd9Sstevel@tonic-gate struct segkp_segdata *kpsd; 1977c478bd9Sstevel@tonic-gate size_t np; 1987c478bd9Sstevel@tonic-gate 1997c478bd9Sstevel@tonic-gate ASSERT(seg != NULL && seg->s_as == &kas); 2007c478bd9Sstevel@tonic-gate ASSERT(RW_WRITE_HELD(&seg->s_as->a_lock)); 2017c478bd9Sstevel@tonic-gate 2027c478bd9Sstevel@tonic-gate if (seg->s_size & PAGEOFFSET) { 2037c478bd9Sstevel@tonic-gate panic("Bad segkp size"); 2047c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 2057c478bd9Sstevel@tonic-gate } 2067c478bd9Sstevel@tonic-gate 2077c478bd9Sstevel@tonic-gate kpsd = kmem_zalloc(sizeof (struct segkp_segdata), KM_SLEEP); 2087c478bd9Sstevel@tonic-gate 2097c478bd9Sstevel@tonic-gate /* 2107c478bd9Sstevel@tonic-gate * Allocate the virtual memory for segkp and initialize it 2117c478bd9Sstevel@tonic-gate */ 2127c478bd9Sstevel@tonic-gate if (segkp_fromheap) { 2137c478bd9Sstevel@tonic-gate np = btop(kvseg.s_size); 2147c478bd9Sstevel@tonic-gate segkp_bitmap = kmem_zalloc(BT_SIZEOFMAP(np), KM_SLEEP); 2157c478bd9Sstevel@tonic-gate kpsd->kpsd_arena = vmem_create("segkp", NULL, 0, PAGESIZE, 2167c478bd9Sstevel@tonic-gate vmem_alloc, vmem_free, heap_arena, 5 * PAGESIZE, VM_SLEEP); 2177c478bd9Sstevel@tonic-gate } else { 2187c478bd9Sstevel@tonic-gate segkp_bitmap = NULL; 2197c478bd9Sstevel@tonic-gate np = btop(seg->s_size); 2207c478bd9Sstevel@tonic-gate kpsd->kpsd_arena = vmem_create("segkp", seg->s_base, 2217c478bd9Sstevel@tonic-gate seg->s_size, PAGESIZE, NULL, NULL, NULL, 5 * PAGESIZE, 2227c478bd9Sstevel@tonic-gate VM_SLEEP); 2237c478bd9Sstevel@tonic-gate } 2247c478bd9Sstevel@tonic-gate 2257c478bd9Sstevel@tonic-gate kpsd->kpsd_anon = anon_create(np, ANON_SLEEP | ANON_ALLOC_FORCE); 2267c478bd9Sstevel@tonic-gate 2277c478bd9Sstevel@tonic-gate kpsd->kpsd_hash = kmem_zalloc(SEGKP_HASHSZ * sizeof (struct segkp *), 2287c478bd9Sstevel@tonic-gate KM_SLEEP); 2297c478bd9Sstevel@tonic-gate seg->s_data = (void *)kpsd; 2307c478bd9Sstevel@tonic-gate seg->s_ops = &segkp_ops; 2317c478bd9Sstevel@tonic-gate segkpinit_mem_config(seg); 2327c478bd9Sstevel@tonic-gate return (0); 2337c478bd9Sstevel@tonic-gate } 2347c478bd9Sstevel@tonic-gate 2357c478bd9Sstevel@tonic-gate 2367c478bd9Sstevel@tonic-gate /* 2377c478bd9Sstevel@tonic-gate * Find a free 'freelist' and initialize it with the appropriate attributes 2387c478bd9Sstevel@tonic-gate */ 2397c478bd9Sstevel@tonic-gate void * 2407c478bd9Sstevel@tonic-gate segkp_cache_init(struct seg *seg, int maxsize, size_t len, uint_t flags) 2417c478bd9Sstevel@tonic-gate { 2427c478bd9Sstevel@tonic-gate int i; 2437c478bd9Sstevel@tonic-gate 2447c478bd9Sstevel@tonic-gate if ((flags & KPD_NO_ANON) && !(flags & KPD_LOCKED)) 2457c478bd9Sstevel@tonic-gate return ((void *)-1); 2467c478bd9Sstevel@tonic-gate 2477c478bd9Sstevel@tonic-gate mutex_enter(&segkp_lock); 2487c478bd9Sstevel@tonic-gate for (i = 0; i < SEGKP_MAX_CACHE; i++) { 2497c478bd9Sstevel@tonic-gate if (segkp_cache[i].kpf_inuse) 2507c478bd9Sstevel@tonic-gate continue; 2517c478bd9Sstevel@tonic-gate segkp_cache[i].kpf_inuse = 1; 2527c478bd9Sstevel@tonic-gate segkp_cache[i].kpf_max = maxsize; 2537c478bd9Sstevel@tonic-gate segkp_cache[i].kpf_flags = flags; 2547c478bd9Sstevel@tonic-gate segkp_cache[i].kpf_seg = seg; 2557c478bd9Sstevel@tonic-gate segkp_cache[i].kpf_len = len; 2567c478bd9Sstevel@tonic-gate mutex_exit(&segkp_lock); 2577c478bd9Sstevel@tonic-gate return ((void *)(uintptr_t)i); 2587c478bd9Sstevel@tonic-gate } 2597c478bd9Sstevel@tonic-gate mutex_exit(&segkp_lock); 2607c478bd9Sstevel@tonic-gate return ((void *)-1); 2617c478bd9Sstevel@tonic-gate } 2627c478bd9Sstevel@tonic-gate 2637c478bd9Sstevel@tonic-gate /* 2647c478bd9Sstevel@tonic-gate * Free all the cache resources. 2657c478bd9Sstevel@tonic-gate */ 2667c478bd9Sstevel@tonic-gate void 2677c478bd9Sstevel@tonic-gate segkp_cache_free(void) 2687c478bd9Sstevel@tonic-gate { 2697c478bd9Sstevel@tonic-gate struct segkp_data *kpd; 2707c478bd9Sstevel@tonic-gate struct seg *seg; 2717c478bd9Sstevel@tonic-gate int i; 2727c478bd9Sstevel@tonic-gate 2737c478bd9Sstevel@tonic-gate mutex_enter(&segkp_lock); 2747c478bd9Sstevel@tonic-gate for (i = 0; i < SEGKP_MAX_CACHE; i++) { 2757c478bd9Sstevel@tonic-gate if (!segkp_cache[i].kpf_inuse) 2767c478bd9Sstevel@tonic-gate continue; 2777c478bd9Sstevel@tonic-gate /* 2787c478bd9Sstevel@tonic-gate * Disconnect the freelist and process each element 2797c478bd9Sstevel@tonic-gate */ 2807c478bd9Sstevel@tonic-gate kpd = segkp_cache[i].kpf_list; 2817c478bd9Sstevel@tonic-gate seg = segkp_cache[i].kpf_seg; 2827c478bd9Sstevel@tonic-gate segkp_cache[i].kpf_list = NULL; 2837c478bd9Sstevel@tonic-gate segkp_cache[i].kpf_count = 0; 2847c478bd9Sstevel@tonic-gate mutex_exit(&segkp_lock); 2857c478bd9Sstevel@tonic-gate 2867c478bd9Sstevel@tonic-gate while (kpd != NULL) { 2877c478bd9Sstevel@tonic-gate struct segkp_data *next; 2887c478bd9Sstevel@tonic-gate 2897c478bd9Sstevel@tonic-gate next = kpd->kp_next; 2907c478bd9Sstevel@tonic-gate segkp_release_internal(seg, kpd, kpd->kp_len); 2917c478bd9Sstevel@tonic-gate kpd = next; 2927c478bd9Sstevel@tonic-gate } 2937c478bd9Sstevel@tonic-gate mutex_enter(&segkp_lock); 2947c478bd9Sstevel@tonic-gate } 2957c478bd9Sstevel@tonic-gate mutex_exit(&segkp_lock); 2967c478bd9Sstevel@tonic-gate } 2977c478bd9Sstevel@tonic-gate 2987c478bd9Sstevel@tonic-gate /* 2997c478bd9Sstevel@tonic-gate * There are 2 entries into segkp_get_internal. The first includes a cookie 3007c478bd9Sstevel@tonic-gate * used to access a pool of cached segkp resources. The second does not 3017c478bd9Sstevel@tonic-gate * use the cache. 3027c478bd9Sstevel@tonic-gate */ 3037c478bd9Sstevel@tonic-gate caddr_t 3047c478bd9Sstevel@tonic-gate segkp_get(struct seg *seg, size_t len, uint_t flags) 3057c478bd9Sstevel@tonic-gate { 3067c478bd9Sstevel@tonic-gate struct segkp_data *kpd = NULL; 3077c478bd9Sstevel@tonic-gate 3087c478bd9Sstevel@tonic-gate if (segkp_get_internal(seg, len, flags, &kpd, NULL) != NULL) { 3097c478bd9Sstevel@tonic-gate kpd->kp_cookie = -1; 3107c478bd9Sstevel@tonic-gate return (stom(kpd->kp_base, flags)); 3117c478bd9Sstevel@tonic-gate } 3127c478bd9Sstevel@tonic-gate return (NULL); 3137c478bd9Sstevel@tonic-gate } 3147c478bd9Sstevel@tonic-gate 3157c478bd9Sstevel@tonic-gate /* 3167c478bd9Sstevel@tonic-gate * Return a 'cached' segkp address 3177c478bd9Sstevel@tonic-gate */ 3187c478bd9Sstevel@tonic-gate caddr_t 3197c478bd9Sstevel@tonic-gate segkp_cache_get(void *cookie) 3207c478bd9Sstevel@tonic-gate { 3217c478bd9Sstevel@tonic-gate struct segkp_cache *freelist = NULL; 3227c478bd9Sstevel@tonic-gate struct segkp_data *kpd = NULL; 3237c478bd9Sstevel@tonic-gate int index = (int)(uintptr_t)cookie; 3247c478bd9Sstevel@tonic-gate struct seg *seg; 3257c478bd9Sstevel@tonic-gate size_t len; 3267c478bd9Sstevel@tonic-gate uint_t flags; 3277c478bd9Sstevel@tonic-gate 3287c478bd9Sstevel@tonic-gate if (index < 0 || index >= SEGKP_MAX_CACHE) 3297c478bd9Sstevel@tonic-gate return (NULL); 3307c478bd9Sstevel@tonic-gate freelist = &segkp_cache[index]; 3317c478bd9Sstevel@tonic-gate 3327c478bd9Sstevel@tonic-gate mutex_enter(&segkp_lock); 3337c478bd9Sstevel@tonic-gate seg = freelist->kpf_seg; 3347c478bd9Sstevel@tonic-gate flags = freelist->kpf_flags; 3357c478bd9Sstevel@tonic-gate if (freelist->kpf_list != NULL) { 3367c478bd9Sstevel@tonic-gate kpd = freelist->kpf_list; 3377c478bd9Sstevel@tonic-gate freelist->kpf_list = kpd->kp_next; 3387c478bd9Sstevel@tonic-gate freelist->kpf_count--; 3397c478bd9Sstevel@tonic-gate mutex_exit(&segkp_lock); 3407c478bd9Sstevel@tonic-gate kpd->kp_next = NULL; 3417c478bd9Sstevel@tonic-gate segkp_insert(seg, kpd); 3427c478bd9Sstevel@tonic-gate return (stom(kpd->kp_base, flags)); 3437c478bd9Sstevel@tonic-gate } 3447c478bd9Sstevel@tonic-gate len = freelist->kpf_len; 3457c478bd9Sstevel@tonic-gate mutex_exit(&segkp_lock); 3467c478bd9Sstevel@tonic-gate if (segkp_get_internal(seg, len, flags, &kpd, NULL) != NULL) { 3477c478bd9Sstevel@tonic-gate kpd->kp_cookie = index; 3487c478bd9Sstevel@tonic-gate return (stom(kpd->kp_base, flags)); 3497c478bd9Sstevel@tonic-gate } 3507c478bd9Sstevel@tonic-gate return (NULL); 3517c478bd9Sstevel@tonic-gate } 3527c478bd9Sstevel@tonic-gate 3537c478bd9Sstevel@tonic-gate caddr_t 3547c478bd9Sstevel@tonic-gate segkp_get_withanonmap( 3557c478bd9Sstevel@tonic-gate struct seg *seg, 3567c478bd9Sstevel@tonic-gate size_t len, 3577c478bd9Sstevel@tonic-gate uint_t flags, 3587c478bd9Sstevel@tonic-gate struct anon_map *amp) 3597c478bd9Sstevel@tonic-gate { 3607c478bd9Sstevel@tonic-gate struct segkp_data *kpd = NULL; 3617c478bd9Sstevel@tonic-gate 3627c478bd9Sstevel@tonic-gate ASSERT(amp != NULL); 3637c478bd9Sstevel@tonic-gate flags |= KPD_HASAMP; 3647c478bd9Sstevel@tonic-gate if (segkp_get_internal(seg, len, flags, &kpd, amp) != NULL) { 3657c478bd9Sstevel@tonic-gate kpd->kp_cookie = -1; 3667c478bd9Sstevel@tonic-gate return (stom(kpd->kp_base, flags)); 3677c478bd9Sstevel@tonic-gate } 3687c478bd9Sstevel@tonic-gate return (NULL); 3697c478bd9Sstevel@tonic-gate } 3707c478bd9Sstevel@tonic-gate 3717c478bd9Sstevel@tonic-gate /* 3727c478bd9Sstevel@tonic-gate * This does the real work of segkp allocation. 3737c478bd9Sstevel@tonic-gate * Return to client base addr. len must be page-aligned. A null value is 3747c478bd9Sstevel@tonic-gate * returned if there are no more vm resources (e.g. pages, swap). The len 3757c478bd9Sstevel@tonic-gate * and base recorded in the private data structure include the redzone 3767c478bd9Sstevel@tonic-gate * and the redzone length (if applicable). If the user requests a redzone 3777c478bd9Sstevel@tonic-gate * either the first or last page is left unmapped depending whether stacks 3787c478bd9Sstevel@tonic-gate * grow to low or high memory. 3797c478bd9Sstevel@tonic-gate * 3807c478bd9Sstevel@tonic-gate * The client may also specify a no-wait flag. If that is set then the 3817c478bd9Sstevel@tonic-gate * request will choose a non-blocking path when requesting resources. 3827c478bd9Sstevel@tonic-gate * The default is make the client wait. 3837c478bd9Sstevel@tonic-gate */ 3847c478bd9Sstevel@tonic-gate static caddr_t 3857c478bd9Sstevel@tonic-gate segkp_get_internal( 3867c478bd9Sstevel@tonic-gate struct seg *seg, 3877c478bd9Sstevel@tonic-gate size_t len, 3887c478bd9Sstevel@tonic-gate uint_t flags, 3897c478bd9Sstevel@tonic-gate struct segkp_data **tkpd, 3907c478bd9Sstevel@tonic-gate struct anon_map *amp) 3917c478bd9Sstevel@tonic-gate { 3927c478bd9Sstevel@tonic-gate struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data; 3937c478bd9Sstevel@tonic-gate struct segkp_data *kpd; 3947c478bd9Sstevel@tonic-gate caddr_t vbase = NULL; /* always first virtual, may not be mapped */ 3957c478bd9Sstevel@tonic-gate pgcnt_t np = 0; /* number of pages in the resource */ 3967c478bd9Sstevel@tonic-gate pgcnt_t segkpindex; 3977c478bd9Sstevel@tonic-gate long i; 3987c478bd9Sstevel@tonic-gate caddr_t va; 3997c478bd9Sstevel@tonic-gate pgcnt_t pages = 0; 4007c478bd9Sstevel@tonic-gate ulong_t anon_idx = 0; 4017c478bd9Sstevel@tonic-gate int kmflag = (flags & KPD_NOWAIT) ? KM_NOSLEEP : KM_SLEEP; 4027c478bd9Sstevel@tonic-gate caddr_t s_base = (segkp_fromheap) ? kvseg.s_base : seg->s_base; 4037c478bd9Sstevel@tonic-gate 4047c478bd9Sstevel@tonic-gate if (len & PAGEOFFSET) { 4057c478bd9Sstevel@tonic-gate panic("segkp_get: len is not page-aligned"); 4067c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 4077c478bd9Sstevel@tonic-gate } 4087c478bd9Sstevel@tonic-gate 4097c478bd9Sstevel@tonic-gate ASSERT(((flags & KPD_HASAMP) == 0) == (amp == NULL)); 4107c478bd9Sstevel@tonic-gate 4117c478bd9Sstevel@tonic-gate /* Only allow KPD_NO_ANON if we are going to lock it down */ 4127c478bd9Sstevel@tonic-gate if ((flags & (KPD_LOCKED|KPD_NO_ANON)) == KPD_NO_ANON) 4137c478bd9Sstevel@tonic-gate return (NULL); 4147c478bd9Sstevel@tonic-gate 4157c478bd9Sstevel@tonic-gate if ((kpd = kmem_zalloc(sizeof (struct segkp_data), kmflag)) == NULL) 4167c478bd9Sstevel@tonic-gate return (NULL); 4177c478bd9Sstevel@tonic-gate /* 4187c478bd9Sstevel@tonic-gate * Fix up the len to reflect the REDZONE if applicable 4197c478bd9Sstevel@tonic-gate */ 4207c478bd9Sstevel@tonic-gate if (flags & KPD_HASREDZONE) 4217c478bd9Sstevel@tonic-gate len += PAGESIZE; 4227c478bd9Sstevel@tonic-gate np = btop(len); 4237c478bd9Sstevel@tonic-gate 4247c478bd9Sstevel@tonic-gate vbase = vmem_alloc(SEGKP_VMEM(seg), len, kmflag | VM_BESTFIT); 4257c478bd9Sstevel@tonic-gate if (vbase == NULL) { 4267c478bd9Sstevel@tonic-gate kmem_free(kpd, sizeof (struct segkp_data)); 4277c478bd9Sstevel@tonic-gate return (NULL); 4287c478bd9Sstevel@tonic-gate } 4297c478bd9Sstevel@tonic-gate 4307c478bd9Sstevel@tonic-gate /* If locking, reserve physical memory */ 4317c478bd9Sstevel@tonic-gate if (flags & KPD_LOCKED) { 4327c478bd9Sstevel@tonic-gate pages = btop(SEGKP_MAPLEN(len, flags)); 4337c478bd9Sstevel@tonic-gate if (page_resv(pages, kmflag) == 0) { 4347c478bd9Sstevel@tonic-gate vmem_free(SEGKP_VMEM(seg), vbase, len); 4357c478bd9Sstevel@tonic-gate kmem_free(kpd, sizeof (struct segkp_data)); 4367c478bd9Sstevel@tonic-gate return (NULL); 4377c478bd9Sstevel@tonic-gate } 4387c478bd9Sstevel@tonic-gate if ((flags & KPD_NO_ANON) == 0) 4397c478bd9Sstevel@tonic-gate atomic_add_long(&anon_segkp_pages_locked, pages); 4407c478bd9Sstevel@tonic-gate } 4417c478bd9Sstevel@tonic-gate 4427c478bd9Sstevel@tonic-gate /* 4437c478bd9Sstevel@tonic-gate * Reserve sufficient swap space for this vm resource. We'll 4447c478bd9Sstevel@tonic-gate * actually allocate it in the loop below, but reserving it 4457c478bd9Sstevel@tonic-gate * here allows us to back out more gracefully than if we 4467c478bd9Sstevel@tonic-gate * had an allocation failure in the body of the loop. 4477c478bd9Sstevel@tonic-gate * 4487c478bd9Sstevel@tonic-gate * Note that we don't need swap space for the red zone page. 4497c478bd9Sstevel@tonic-gate */ 4507c478bd9Sstevel@tonic-gate if (amp != NULL) { 4517c478bd9Sstevel@tonic-gate ASSERT((flags & KPD_NO_ANON) == 0); 4527c478bd9Sstevel@tonic-gate /* The reserve has been done and the anon_hdr is separate. */ 4537c478bd9Sstevel@tonic-gate anon_idx = 0; 4547c478bd9Sstevel@tonic-gate kpd->kp_anon_idx = anon_idx; 4557c478bd9Sstevel@tonic-gate kpd->kp_anon = amp->ahp; 4567c478bd9Sstevel@tonic-gate 4577c478bd9Sstevel@tonic-gate TRACE_5(TR_FAC_VM, TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u", 4587c478bd9Sstevel@tonic-gate kpd, vbase, len, flags, 1); 4597c478bd9Sstevel@tonic-gate 4607c478bd9Sstevel@tonic-gate } else if ((flags & KPD_NO_ANON) == 0) { 4617c478bd9Sstevel@tonic-gate if (anon_resv(SEGKP_MAPLEN(len, flags)) == 0) { 4627c478bd9Sstevel@tonic-gate if (flags & KPD_LOCKED) { 4637c478bd9Sstevel@tonic-gate atomic_add_long(&anon_segkp_pages_locked, 4647c478bd9Sstevel@tonic-gate -pages); 4657c478bd9Sstevel@tonic-gate page_unresv(pages); 4667c478bd9Sstevel@tonic-gate } 4677c478bd9Sstevel@tonic-gate vmem_free(SEGKP_VMEM(seg), vbase, len); 4687c478bd9Sstevel@tonic-gate kmem_free(kpd, sizeof (struct segkp_data)); 4697c478bd9Sstevel@tonic-gate return (NULL); 4707c478bd9Sstevel@tonic-gate } 4717c478bd9Sstevel@tonic-gate anon_idx = ((uintptr_t)(vbase - s_base)) >> PAGESHIFT; 4727c478bd9Sstevel@tonic-gate kpd->kp_anon_idx = anon_idx; 4737c478bd9Sstevel@tonic-gate kpd->kp_anon = kpsd->kpsd_anon; 4747c478bd9Sstevel@tonic-gate 4757c478bd9Sstevel@tonic-gate TRACE_5(TR_FAC_VM, TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u", 4767c478bd9Sstevel@tonic-gate kpd, vbase, len, flags, 1); 4777c478bd9Sstevel@tonic-gate } else { 4787c478bd9Sstevel@tonic-gate kpd->kp_anon = NULL; 4797c478bd9Sstevel@tonic-gate kpd->kp_anon_idx = 0; 4807c478bd9Sstevel@tonic-gate } 4817c478bd9Sstevel@tonic-gate 4827c478bd9Sstevel@tonic-gate /* 4837c478bd9Sstevel@tonic-gate * Allocate page and anon resources for the virtual address range 4847c478bd9Sstevel@tonic-gate * except the redzone 4857c478bd9Sstevel@tonic-gate */ 4867c478bd9Sstevel@tonic-gate if (segkp_fromheap) 4877c478bd9Sstevel@tonic-gate segkpindex = btop((uintptr_t)(vbase - kvseg.s_base)); 4887c478bd9Sstevel@tonic-gate for (i = 0, va = vbase; i < np; i++, va += PAGESIZE) { 4897c478bd9Sstevel@tonic-gate page_t *pl[2]; 4907c478bd9Sstevel@tonic-gate struct vnode *vp; 4917c478bd9Sstevel@tonic-gate anoff_t off; 4927c478bd9Sstevel@tonic-gate int err; 4937c478bd9Sstevel@tonic-gate page_t *pp = NULL; 4947c478bd9Sstevel@tonic-gate 4957c478bd9Sstevel@tonic-gate /* 4967c478bd9Sstevel@tonic-gate * Mark this page to be a segkp page in the bitmap. 4977c478bd9Sstevel@tonic-gate */ 4987c478bd9Sstevel@tonic-gate if (segkp_fromheap) { 4997c478bd9Sstevel@tonic-gate BT_ATOMIC_SET(segkp_bitmap, segkpindex); 5007c478bd9Sstevel@tonic-gate segkpindex++; 5017c478bd9Sstevel@tonic-gate } 5027c478bd9Sstevel@tonic-gate 5037c478bd9Sstevel@tonic-gate /* 5047c478bd9Sstevel@tonic-gate * If this page is the red zone page, we don't need swap 5057c478bd9Sstevel@tonic-gate * space for it. Note that we skip over the code that 5067c478bd9Sstevel@tonic-gate * establishes MMU mappings, so that the page remains 5077c478bd9Sstevel@tonic-gate * invalid. 5087c478bd9Sstevel@tonic-gate */ 5097c478bd9Sstevel@tonic-gate if ((flags & KPD_HASREDZONE) && KPD_REDZONE(kpd) == i) 5107c478bd9Sstevel@tonic-gate continue; 5117c478bd9Sstevel@tonic-gate 5127c478bd9Sstevel@tonic-gate if (kpd->kp_anon != NULL) { 5137c478bd9Sstevel@tonic-gate struct anon *ap; 5147c478bd9Sstevel@tonic-gate 5157c478bd9Sstevel@tonic-gate ASSERT(anon_get_ptr(kpd->kp_anon, anon_idx + i) 5167c478bd9Sstevel@tonic-gate == NULL); 5177c478bd9Sstevel@tonic-gate /* 5187c478bd9Sstevel@tonic-gate * Determine the "vp" and "off" of the anon slot. 5197c478bd9Sstevel@tonic-gate */ 5207c478bd9Sstevel@tonic-gate ap = anon_alloc(NULL, 0); 5217c478bd9Sstevel@tonic-gate if (amp != NULL) 5227c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 5237c478bd9Sstevel@tonic-gate (void) anon_set_ptr(kpd->kp_anon, anon_idx + i, 5247c478bd9Sstevel@tonic-gate ap, ANON_SLEEP); 5257c478bd9Sstevel@tonic-gate if (amp != NULL) 5267c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 5277c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &off); 5287c478bd9Sstevel@tonic-gate 5297c478bd9Sstevel@tonic-gate /* 5307c478bd9Sstevel@tonic-gate * Create a page with the specified identity. The 5317c478bd9Sstevel@tonic-gate * page is returned with the "shared" lock held. 5327c478bd9Sstevel@tonic-gate */ 5337c478bd9Sstevel@tonic-gate err = VOP_GETPAGE(vp, (offset_t)off, PAGESIZE, 5347c478bd9Sstevel@tonic-gate NULL, pl, PAGESIZE, seg, va, S_CREATE, 5357c478bd9Sstevel@tonic-gate kcred); 5367c478bd9Sstevel@tonic-gate if (err) { 5377c478bd9Sstevel@tonic-gate /* 5387c478bd9Sstevel@tonic-gate * XXX - This should not fail. 5397c478bd9Sstevel@tonic-gate */ 5407c478bd9Sstevel@tonic-gate panic("segkp_get: no pages"); 5417c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 5427c478bd9Sstevel@tonic-gate } 5437c478bd9Sstevel@tonic-gate pp = pl[0]; 5447c478bd9Sstevel@tonic-gate } else { 5457c478bd9Sstevel@tonic-gate ASSERT(page_exists(&kvp, 5467c478bd9Sstevel@tonic-gate (u_offset_t)(uintptr_t)va) == NULL); 5477c478bd9Sstevel@tonic-gate 5487c478bd9Sstevel@tonic-gate if ((pp = page_create_va(&kvp, 5497c478bd9Sstevel@tonic-gate (u_offset_t)(uintptr_t)va, PAGESIZE, 5507c478bd9Sstevel@tonic-gate (flags & KPD_NOWAIT ? 0 : PG_WAIT) | PG_EXCL | 5517c478bd9Sstevel@tonic-gate PG_NORELOC, seg, va)) == NULL) { 5527c478bd9Sstevel@tonic-gate /* 5537c478bd9Sstevel@tonic-gate * Legitimize resource; then destroy it. 5547c478bd9Sstevel@tonic-gate * Easier than trying to unwind here. 5557c478bd9Sstevel@tonic-gate */ 5567c478bd9Sstevel@tonic-gate kpd->kp_flags = flags; 5577c478bd9Sstevel@tonic-gate kpd->kp_base = vbase; 5587c478bd9Sstevel@tonic-gate kpd->kp_len = len; 5597c478bd9Sstevel@tonic-gate segkp_release_internal(seg, kpd, va - vbase); 5607c478bd9Sstevel@tonic-gate return (NULL); 5617c478bd9Sstevel@tonic-gate } 5627c478bd9Sstevel@tonic-gate page_io_unlock(pp); 5637c478bd9Sstevel@tonic-gate } 5647c478bd9Sstevel@tonic-gate 5657c478bd9Sstevel@tonic-gate if (flags & KPD_ZERO) 5667c478bd9Sstevel@tonic-gate pagezero(pp, 0, PAGESIZE); 5677c478bd9Sstevel@tonic-gate 5687c478bd9Sstevel@tonic-gate /* 5697c478bd9Sstevel@tonic-gate * Load and lock an MMU translation for the page. 5707c478bd9Sstevel@tonic-gate */ 5717c478bd9Sstevel@tonic-gate hat_memload(seg->s_as->a_hat, va, pp, (PROT_READ|PROT_WRITE), 5727c478bd9Sstevel@tonic-gate ((flags & KPD_LOCKED) ? HAT_LOAD_LOCK : HAT_LOAD)); 5737c478bd9Sstevel@tonic-gate 5747c478bd9Sstevel@tonic-gate /* 5757c478bd9Sstevel@tonic-gate * Now, release lock on the page. 5767c478bd9Sstevel@tonic-gate */ 5777c478bd9Sstevel@tonic-gate if (flags & KPD_LOCKED) 5787c478bd9Sstevel@tonic-gate page_downgrade(pp); 5797c478bd9Sstevel@tonic-gate else 5807c478bd9Sstevel@tonic-gate page_unlock(pp); 5817c478bd9Sstevel@tonic-gate } 5827c478bd9Sstevel@tonic-gate 5837c478bd9Sstevel@tonic-gate kpd->kp_flags = flags; 5847c478bd9Sstevel@tonic-gate kpd->kp_base = vbase; 5857c478bd9Sstevel@tonic-gate kpd->kp_len = len; 5867c478bd9Sstevel@tonic-gate segkp_insert(seg, kpd); 5877c478bd9Sstevel@tonic-gate *tkpd = kpd; 5887c478bd9Sstevel@tonic-gate return (stom(kpd->kp_base, flags)); 5897c478bd9Sstevel@tonic-gate } 5907c478bd9Sstevel@tonic-gate 5917c478bd9Sstevel@tonic-gate /* 5927c478bd9Sstevel@tonic-gate * Release the resource to cache if the pool(designate by the cookie) 5937c478bd9Sstevel@tonic-gate * has less than the maximum allowable. If inserted in cache, 5947c478bd9Sstevel@tonic-gate * segkp_delete insures element is taken off of active list. 5957c478bd9Sstevel@tonic-gate */ 5967c478bd9Sstevel@tonic-gate void 5977c478bd9Sstevel@tonic-gate segkp_release(struct seg *seg, caddr_t vaddr) 5987c478bd9Sstevel@tonic-gate { 5997c478bd9Sstevel@tonic-gate struct segkp_cache *freelist; 6007c478bd9Sstevel@tonic-gate struct segkp_data *kpd = NULL; 6017c478bd9Sstevel@tonic-gate 6027c478bd9Sstevel@tonic-gate if ((kpd = segkp_find(seg, vaddr)) == NULL) { 6037c478bd9Sstevel@tonic-gate panic("segkp_release: null kpd"); 6047c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 6057c478bd9Sstevel@tonic-gate } 6067c478bd9Sstevel@tonic-gate 6077c478bd9Sstevel@tonic-gate if (kpd->kp_cookie != -1) { 6087c478bd9Sstevel@tonic-gate freelist = &segkp_cache[kpd->kp_cookie]; 6097c478bd9Sstevel@tonic-gate mutex_enter(&segkp_lock); 6107c478bd9Sstevel@tonic-gate if (!segkp_indel && freelist->kpf_count < freelist->kpf_max) { 6117c478bd9Sstevel@tonic-gate segkp_delete(seg, kpd); 6127c478bd9Sstevel@tonic-gate kpd->kp_next = freelist->kpf_list; 6137c478bd9Sstevel@tonic-gate freelist->kpf_list = kpd; 6147c478bd9Sstevel@tonic-gate freelist->kpf_count++; 6157c478bd9Sstevel@tonic-gate mutex_exit(&segkp_lock); 6167c478bd9Sstevel@tonic-gate return; 6177c478bd9Sstevel@tonic-gate } else { 6187c478bd9Sstevel@tonic-gate mutex_exit(&segkp_lock); 6197c478bd9Sstevel@tonic-gate kpd->kp_cookie = -1; 6207c478bd9Sstevel@tonic-gate } 6217c478bd9Sstevel@tonic-gate } 6227c478bd9Sstevel@tonic-gate segkp_release_internal(seg, kpd, kpd->kp_len); 6237c478bd9Sstevel@tonic-gate } 6247c478bd9Sstevel@tonic-gate 6257c478bd9Sstevel@tonic-gate /* 6267c478bd9Sstevel@tonic-gate * Free the entire resource. segkp_unlock gets called with the start of the 6277c478bd9Sstevel@tonic-gate * mapped portion of the resource. The length is the size of the mapped 6287c478bd9Sstevel@tonic-gate * portion 6297c478bd9Sstevel@tonic-gate */ 6307c478bd9Sstevel@tonic-gate static void 6317c478bd9Sstevel@tonic-gate segkp_release_internal(struct seg *seg, struct segkp_data *kpd, size_t len) 6327c478bd9Sstevel@tonic-gate { 6337c478bd9Sstevel@tonic-gate caddr_t va; 6347c478bd9Sstevel@tonic-gate long i; 6357c478bd9Sstevel@tonic-gate long redzone; 6367c478bd9Sstevel@tonic-gate size_t np; 6377c478bd9Sstevel@tonic-gate page_t *pp; 6387c478bd9Sstevel@tonic-gate struct vnode *vp; 6397c478bd9Sstevel@tonic-gate anoff_t off; 6407c478bd9Sstevel@tonic-gate struct anon *ap; 6417c478bd9Sstevel@tonic-gate pgcnt_t segkpindex; 6427c478bd9Sstevel@tonic-gate 6437c478bd9Sstevel@tonic-gate ASSERT(kpd != NULL); 6447c478bd9Sstevel@tonic-gate ASSERT((kpd->kp_flags & KPD_HASAMP) == 0 || kpd->kp_cookie == -1); 6457c478bd9Sstevel@tonic-gate np = btop(len); 6467c478bd9Sstevel@tonic-gate 6477c478bd9Sstevel@tonic-gate /* Remove from active hash list */ 6487c478bd9Sstevel@tonic-gate if (kpd->kp_cookie == -1) { 6497c478bd9Sstevel@tonic-gate mutex_enter(&segkp_lock); 6507c478bd9Sstevel@tonic-gate segkp_delete(seg, kpd); 6517c478bd9Sstevel@tonic-gate mutex_exit(&segkp_lock); 6527c478bd9Sstevel@tonic-gate } 6537c478bd9Sstevel@tonic-gate 6547c478bd9Sstevel@tonic-gate /* 6557c478bd9Sstevel@tonic-gate * Precompute redzone page index. 6567c478bd9Sstevel@tonic-gate */ 6577c478bd9Sstevel@tonic-gate redzone = -1; 6587c478bd9Sstevel@tonic-gate if (kpd->kp_flags & KPD_HASREDZONE) 6597c478bd9Sstevel@tonic-gate redzone = KPD_REDZONE(kpd); 6607c478bd9Sstevel@tonic-gate 6617c478bd9Sstevel@tonic-gate 6627c478bd9Sstevel@tonic-gate va = kpd->kp_base; 6637c478bd9Sstevel@tonic-gate 6647c478bd9Sstevel@tonic-gate hat_unload(seg->s_as->a_hat, va, (np << PAGESHIFT), 6657c478bd9Sstevel@tonic-gate ((kpd->kp_flags & KPD_LOCKED) ? HAT_UNLOAD_UNLOCK : HAT_UNLOAD)); 6667c478bd9Sstevel@tonic-gate /* 6677c478bd9Sstevel@tonic-gate * Free up those anon resources that are quiescent. 6687c478bd9Sstevel@tonic-gate */ 6697c478bd9Sstevel@tonic-gate if (segkp_fromheap) 6707c478bd9Sstevel@tonic-gate segkpindex = btop((uintptr_t)(va - kvseg.s_base)); 6717c478bd9Sstevel@tonic-gate for (i = 0; i < np; i++, va += PAGESIZE) { 6727c478bd9Sstevel@tonic-gate 6737c478bd9Sstevel@tonic-gate /* 6747c478bd9Sstevel@tonic-gate * Clear the bit for this page from the bitmap. 6757c478bd9Sstevel@tonic-gate */ 6767c478bd9Sstevel@tonic-gate if (segkp_fromheap) { 6777c478bd9Sstevel@tonic-gate BT_ATOMIC_CLEAR(segkp_bitmap, segkpindex); 6787c478bd9Sstevel@tonic-gate segkpindex++; 6797c478bd9Sstevel@tonic-gate } 6807c478bd9Sstevel@tonic-gate 6817c478bd9Sstevel@tonic-gate if (i == redzone) 6827c478bd9Sstevel@tonic-gate continue; 6837c478bd9Sstevel@tonic-gate if (kpd->kp_anon) { 6847c478bd9Sstevel@tonic-gate /* 6857c478bd9Sstevel@tonic-gate * Free up anon resources and destroy the 6867c478bd9Sstevel@tonic-gate * associated pages. 6877c478bd9Sstevel@tonic-gate * 6887c478bd9Sstevel@tonic-gate * Release the lock if there is one. Have to get the 6897c478bd9Sstevel@tonic-gate * page to do this, unfortunately. 6907c478bd9Sstevel@tonic-gate */ 6917c478bd9Sstevel@tonic-gate if (kpd->kp_flags & KPD_LOCKED) { 6927c478bd9Sstevel@tonic-gate ap = anon_get_ptr(kpd->kp_anon, 6937c478bd9Sstevel@tonic-gate kpd->kp_anon_idx + i); 6947c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &off); 6957c478bd9Sstevel@tonic-gate /* Find the shared-locked page. */ 6967c478bd9Sstevel@tonic-gate pp = page_find(vp, (u_offset_t)off); 6977c478bd9Sstevel@tonic-gate if (pp == NULL) { 6987c478bd9Sstevel@tonic-gate panic("segkp_release: " 6997c478bd9Sstevel@tonic-gate "kp_anon: no page to unlock "); 7007c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 7017c478bd9Sstevel@tonic-gate } 7027c478bd9Sstevel@tonic-gate page_unlock(pp); 7037c478bd9Sstevel@tonic-gate } 7047c478bd9Sstevel@tonic-gate if ((kpd->kp_flags & KPD_HASAMP) == 0) { 7057c478bd9Sstevel@tonic-gate anon_free(kpd->kp_anon, kpd->kp_anon_idx + i, 7067c478bd9Sstevel@tonic-gate PAGESIZE); 7077c478bd9Sstevel@tonic-gate anon_unresv(PAGESIZE); 7087c478bd9Sstevel@tonic-gate } 7097c478bd9Sstevel@tonic-gate TRACE_5(TR_FAC_VM, 7107c478bd9Sstevel@tonic-gate TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u", 7117c478bd9Sstevel@tonic-gate kpd, va, PAGESIZE, 0, 0); 7127c478bd9Sstevel@tonic-gate } else { 7137c478bd9Sstevel@tonic-gate if (kpd->kp_flags & KPD_LOCKED) { 7147c478bd9Sstevel@tonic-gate pp = page_find(&kvp, (u_offset_t)(uintptr_t)va); 7157c478bd9Sstevel@tonic-gate if (pp == NULL) { 7167c478bd9Sstevel@tonic-gate panic("segkp_release: " 7177c478bd9Sstevel@tonic-gate "no page to unlock"); 7187c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 7197c478bd9Sstevel@tonic-gate } 7207c478bd9Sstevel@tonic-gate /* 7217c478bd9Sstevel@tonic-gate * We should just upgrade the lock here 7227c478bd9Sstevel@tonic-gate * but there is no upgrade that waits. 7237c478bd9Sstevel@tonic-gate */ 7247c478bd9Sstevel@tonic-gate page_unlock(pp); 7257c478bd9Sstevel@tonic-gate } 7267c478bd9Sstevel@tonic-gate pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)va, 7277c478bd9Sstevel@tonic-gate SE_EXCL); 7287c478bd9Sstevel@tonic-gate if (pp != NULL) 7297c478bd9Sstevel@tonic-gate page_destroy(pp, 0); 7307c478bd9Sstevel@tonic-gate } 7317c478bd9Sstevel@tonic-gate } 7327c478bd9Sstevel@tonic-gate 7337c478bd9Sstevel@tonic-gate /* If locked, release physical memory reservation */ 7347c478bd9Sstevel@tonic-gate if (kpd->kp_flags & KPD_LOCKED) { 7357c478bd9Sstevel@tonic-gate pgcnt_t pages = btop(SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)); 7367c478bd9Sstevel@tonic-gate if ((kpd->kp_flags & KPD_NO_ANON) == 0) 7377c478bd9Sstevel@tonic-gate atomic_add_long(&anon_segkp_pages_locked, -pages); 7387c478bd9Sstevel@tonic-gate page_unresv(pages); 7397c478bd9Sstevel@tonic-gate } 7407c478bd9Sstevel@tonic-gate 7417c478bd9Sstevel@tonic-gate vmem_free(SEGKP_VMEM(seg), kpd->kp_base, kpd->kp_len); 7427c478bd9Sstevel@tonic-gate kmem_free(kpd, sizeof (struct segkp_data)); 7437c478bd9Sstevel@tonic-gate } 7447c478bd9Sstevel@tonic-gate 7457c478bd9Sstevel@tonic-gate /* 7467c478bd9Sstevel@tonic-gate * segkp_map_red() will check the current frame pointer against the 7477c478bd9Sstevel@tonic-gate * stack base. If the amount of stack remaining is questionable 7487c478bd9Sstevel@tonic-gate * (less than red_minavail), then segkp_map_red() will map in the redzone 7497c478bd9Sstevel@tonic-gate * and return 1. Otherwise, it will return 0. segkp_map_red() can 7507c478bd9Sstevel@tonic-gate * _only_ be called when: 7517c478bd9Sstevel@tonic-gate * 7527c478bd9Sstevel@tonic-gate * - it is safe to sleep on page_create_va(). 7537c478bd9Sstevel@tonic-gate * - the caller is non-swappable. 7547c478bd9Sstevel@tonic-gate * 7557c478bd9Sstevel@tonic-gate * It is up to the caller to remember whether segkp_map_red() successfully 7567c478bd9Sstevel@tonic-gate * mapped the redzone, and, if so, to call segkp_unmap_red() at a later 7577c478bd9Sstevel@tonic-gate * time. Note that the caller must _remain_ non-swappable until after 7587c478bd9Sstevel@tonic-gate * calling segkp_unmap_red(). 7597c478bd9Sstevel@tonic-gate * 7607c478bd9Sstevel@tonic-gate * Currently, this routine is only called from pagefault() (which necessarily 7617c478bd9Sstevel@tonic-gate * satisfies the above conditions). 7627c478bd9Sstevel@tonic-gate */ 7637c478bd9Sstevel@tonic-gate #if defined(STACK_GROWTH_DOWN) 7647c478bd9Sstevel@tonic-gate int 7657c478bd9Sstevel@tonic-gate segkp_map_red(void) 7667c478bd9Sstevel@tonic-gate { 7677c478bd9Sstevel@tonic-gate uintptr_t fp = STACK_BIAS + (uintptr_t)getfp(); 7687c478bd9Sstevel@tonic-gate #ifndef _LP64 7697c478bd9Sstevel@tonic-gate caddr_t stkbase; 7707c478bd9Sstevel@tonic-gate #endif 7717c478bd9Sstevel@tonic-gate 7727c478bd9Sstevel@tonic-gate ASSERT(curthread->t_schedflag & TS_DONT_SWAP); 7737c478bd9Sstevel@tonic-gate 7747c478bd9Sstevel@tonic-gate /* 7757c478bd9Sstevel@tonic-gate * Optimize for the common case where we simply return. 7767c478bd9Sstevel@tonic-gate */ 7777c478bd9Sstevel@tonic-gate if ((curthread->t_red_pp == NULL) && 7787c478bd9Sstevel@tonic-gate (fp - (uintptr_t)curthread->t_stkbase >= red_minavail)) 7797c478bd9Sstevel@tonic-gate return (0); 7807c478bd9Sstevel@tonic-gate 7817c478bd9Sstevel@tonic-gate #if defined(_LP64) 7827c478bd9Sstevel@tonic-gate /* 7837c478bd9Sstevel@tonic-gate * XXX We probably need something better than this. 7847c478bd9Sstevel@tonic-gate */ 7857c478bd9Sstevel@tonic-gate panic("kernel stack overflow"); 7867c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 7877c478bd9Sstevel@tonic-gate #else /* _LP64 */ 7887c478bd9Sstevel@tonic-gate if (curthread->t_red_pp == NULL) { 7897c478bd9Sstevel@tonic-gate page_t *red_pp; 7907c478bd9Sstevel@tonic-gate struct seg kseg; 7917c478bd9Sstevel@tonic-gate 7927c478bd9Sstevel@tonic-gate caddr_t red_va = (caddr_t) 7937c478bd9Sstevel@tonic-gate (((uintptr_t)curthread->t_stkbase & (uintptr_t)PAGEMASK) - 7947c478bd9Sstevel@tonic-gate PAGESIZE); 7957c478bd9Sstevel@tonic-gate 7967c478bd9Sstevel@tonic-gate ASSERT(page_exists(&kvp, (u_offset_t)(uintptr_t)red_va) == 7977c478bd9Sstevel@tonic-gate NULL); 7987c478bd9Sstevel@tonic-gate 7997c478bd9Sstevel@tonic-gate /* 8007c478bd9Sstevel@tonic-gate * Allocate the physical for the red page. 8017c478bd9Sstevel@tonic-gate */ 8027c478bd9Sstevel@tonic-gate /* 8037c478bd9Sstevel@tonic-gate * No PG_NORELOC here to avoid waits. Unlikely to get 8047c478bd9Sstevel@tonic-gate * a relocate happening in the short time the page exists 8057c478bd9Sstevel@tonic-gate * and it will be OK anyway. 8067c478bd9Sstevel@tonic-gate */ 8077c478bd9Sstevel@tonic-gate 8087c478bd9Sstevel@tonic-gate kseg.s_as = &kas; 8097c478bd9Sstevel@tonic-gate red_pp = page_create_va(&kvp, (u_offset_t)(uintptr_t)red_va, 8107c478bd9Sstevel@tonic-gate PAGESIZE, PG_WAIT | PG_EXCL, &kseg, red_va); 8117c478bd9Sstevel@tonic-gate ASSERT(red_pp != NULL); 8127c478bd9Sstevel@tonic-gate 8137c478bd9Sstevel@tonic-gate /* 8147c478bd9Sstevel@tonic-gate * So we now have a page to jam into the redzone... 8157c478bd9Sstevel@tonic-gate */ 8167c478bd9Sstevel@tonic-gate page_io_unlock(red_pp); 8177c478bd9Sstevel@tonic-gate 8187c478bd9Sstevel@tonic-gate hat_memload(kas.a_hat, red_va, red_pp, 8197c478bd9Sstevel@tonic-gate (PROT_READ|PROT_WRITE), HAT_LOAD_LOCK); 8207c478bd9Sstevel@tonic-gate page_downgrade(red_pp); 8217c478bd9Sstevel@tonic-gate 8227c478bd9Sstevel@tonic-gate /* 8237c478bd9Sstevel@tonic-gate * The page is left SE_SHARED locked so we can hold on to 8247c478bd9Sstevel@tonic-gate * the page_t pointer. 8257c478bd9Sstevel@tonic-gate */ 8267c478bd9Sstevel@tonic-gate curthread->t_red_pp = red_pp; 8277c478bd9Sstevel@tonic-gate 8287c478bd9Sstevel@tonic-gate atomic_add_32(&red_nmapped, 1); 8297c478bd9Sstevel@tonic-gate while (fp - (uintptr_t)curthread->t_stkbase < red_closest) { 8307c478bd9Sstevel@tonic-gate (void) cas32(&red_closest, red_closest, 8317c478bd9Sstevel@tonic-gate (uint32_t)(fp - (uintptr_t)curthread->t_stkbase)); 8327c478bd9Sstevel@tonic-gate } 8337c478bd9Sstevel@tonic-gate return (1); 8347c478bd9Sstevel@tonic-gate } 8357c478bd9Sstevel@tonic-gate 8367c478bd9Sstevel@tonic-gate stkbase = (caddr_t)(((uintptr_t)curthread->t_stkbase & 8377c478bd9Sstevel@tonic-gate (uintptr_t)PAGEMASK) - PAGESIZE); 8387c478bd9Sstevel@tonic-gate 8397c478bd9Sstevel@tonic-gate atomic_add_32(&red_ndoubles, 1); 8407c478bd9Sstevel@tonic-gate 8417c478bd9Sstevel@tonic-gate if (fp - (uintptr_t)stkbase < RED_DEEP_THRESHOLD) { 8427c478bd9Sstevel@tonic-gate /* 8437c478bd9Sstevel@tonic-gate * Oh boy. We're already deep within the mapped-in 8447c478bd9Sstevel@tonic-gate * redzone page, and the caller is trying to prepare 8457c478bd9Sstevel@tonic-gate * for a deep stack run. We're running without a 8467c478bd9Sstevel@tonic-gate * redzone right now: if the caller plows off the 8477c478bd9Sstevel@tonic-gate * end of the stack, it'll plow another thread or 8487c478bd9Sstevel@tonic-gate * LWP structure. That situation could result in 8497c478bd9Sstevel@tonic-gate * a very hard-to-debug panic, so, in the spirit of 8507c478bd9Sstevel@tonic-gate * recording the name of one's killer in one's own 8517c478bd9Sstevel@tonic-gate * blood, we're going to record lbolt and the calling 8527c478bd9Sstevel@tonic-gate * thread. 8537c478bd9Sstevel@tonic-gate */ 8547c478bd9Sstevel@tonic-gate red_deep_lbolt = lbolt; 8557c478bd9Sstevel@tonic-gate red_deep_thread = curthread; 8567c478bd9Sstevel@tonic-gate } 8577c478bd9Sstevel@tonic-gate 8587c478bd9Sstevel@tonic-gate /* 8597c478bd9Sstevel@tonic-gate * If this is a DEBUG kernel, and we've run too deep for comfort, toss. 8607c478bd9Sstevel@tonic-gate */ 8617c478bd9Sstevel@tonic-gate ASSERT(fp - (uintptr_t)stkbase >= RED_DEEP_THRESHOLD); 8627c478bd9Sstevel@tonic-gate return (0); 8637c478bd9Sstevel@tonic-gate #endif /* _LP64 */ 8647c478bd9Sstevel@tonic-gate } 8657c478bd9Sstevel@tonic-gate 8667c478bd9Sstevel@tonic-gate void 8677c478bd9Sstevel@tonic-gate segkp_unmap_red(void) 8687c478bd9Sstevel@tonic-gate { 8697c478bd9Sstevel@tonic-gate page_t *pp; 8707c478bd9Sstevel@tonic-gate caddr_t red_va = (caddr_t)(((uintptr_t)curthread->t_stkbase & 8717c478bd9Sstevel@tonic-gate (uintptr_t)PAGEMASK) - PAGESIZE); 8727c478bd9Sstevel@tonic-gate 8737c478bd9Sstevel@tonic-gate ASSERT(curthread->t_red_pp != NULL); 8747c478bd9Sstevel@tonic-gate ASSERT(curthread->t_schedflag & TS_DONT_SWAP); 8757c478bd9Sstevel@tonic-gate 8767c478bd9Sstevel@tonic-gate /* 8777c478bd9Sstevel@tonic-gate * Because we locked the mapping down, we can't simply rely 8787c478bd9Sstevel@tonic-gate * on page_destroy() to clean everything up; we need to call 8797c478bd9Sstevel@tonic-gate * hat_unload() to explicitly unlock the mapping resources. 8807c478bd9Sstevel@tonic-gate */ 8817c478bd9Sstevel@tonic-gate hat_unload(kas.a_hat, red_va, PAGESIZE, HAT_UNLOAD_UNLOCK); 8827c478bd9Sstevel@tonic-gate 8837c478bd9Sstevel@tonic-gate pp = curthread->t_red_pp; 8847c478bd9Sstevel@tonic-gate 8857c478bd9Sstevel@tonic-gate ASSERT(pp == page_find(&kvp, (u_offset_t)(uintptr_t)red_va)); 8867c478bd9Sstevel@tonic-gate 8877c478bd9Sstevel@tonic-gate /* 8887c478bd9Sstevel@tonic-gate * Need to upgrade the SE_SHARED lock to SE_EXCL. 8897c478bd9Sstevel@tonic-gate */ 8907c478bd9Sstevel@tonic-gate if (!page_tryupgrade(pp)) { 8917c478bd9Sstevel@tonic-gate /* 8927c478bd9Sstevel@tonic-gate * As there is now wait for upgrade, release the 8937c478bd9Sstevel@tonic-gate * SE_SHARED lock and wait for SE_EXCL. 8947c478bd9Sstevel@tonic-gate */ 8957c478bd9Sstevel@tonic-gate page_unlock(pp); 8967c478bd9Sstevel@tonic-gate pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)red_va, SE_EXCL); 8977c478bd9Sstevel@tonic-gate /* pp may be NULL here, hence the test below */ 8987c478bd9Sstevel@tonic-gate } 8997c478bd9Sstevel@tonic-gate 9007c478bd9Sstevel@tonic-gate /* 9017c478bd9Sstevel@tonic-gate * Destroy the page, with dontfree set to zero (i.e. free it). 9027c478bd9Sstevel@tonic-gate */ 9037c478bd9Sstevel@tonic-gate if (pp != NULL) 9047c478bd9Sstevel@tonic-gate page_destroy(pp, 0); 9057c478bd9Sstevel@tonic-gate curthread->t_red_pp = NULL; 9067c478bd9Sstevel@tonic-gate } 9077c478bd9Sstevel@tonic-gate #else 9087c478bd9Sstevel@tonic-gate #error Red stacks only supported with downwards stack growth. 9097c478bd9Sstevel@tonic-gate #endif 9107c478bd9Sstevel@tonic-gate 9117c478bd9Sstevel@tonic-gate /* 9127c478bd9Sstevel@tonic-gate * Handle a fault on an address corresponding to one of the 9137c478bd9Sstevel@tonic-gate * resources in the segkp segment. 9147c478bd9Sstevel@tonic-gate */ 9157c478bd9Sstevel@tonic-gate faultcode_t 9167c478bd9Sstevel@tonic-gate segkp_fault( 9177c478bd9Sstevel@tonic-gate struct hat *hat, 9187c478bd9Sstevel@tonic-gate struct seg *seg, 9197c478bd9Sstevel@tonic-gate caddr_t vaddr, 9207c478bd9Sstevel@tonic-gate size_t len, 9217c478bd9Sstevel@tonic-gate enum fault_type type, 9227c478bd9Sstevel@tonic-gate enum seg_rw rw) 9237c478bd9Sstevel@tonic-gate { 9247c478bd9Sstevel@tonic-gate struct segkp_data *kpd = NULL; 9257c478bd9Sstevel@tonic-gate int err; 9267c478bd9Sstevel@tonic-gate 9277c478bd9Sstevel@tonic-gate ASSERT(seg->s_as == &kas && RW_READ_HELD(&seg->s_as->a_lock)); 9287c478bd9Sstevel@tonic-gate 9297c478bd9Sstevel@tonic-gate /* 9307c478bd9Sstevel@tonic-gate * Sanity checks. 9317c478bd9Sstevel@tonic-gate */ 9327c478bd9Sstevel@tonic-gate if (type == F_PROT) { 9337c478bd9Sstevel@tonic-gate panic("segkp_fault: unexpected F_PROT fault"); 9347c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 9357c478bd9Sstevel@tonic-gate } 9367c478bd9Sstevel@tonic-gate 9377c478bd9Sstevel@tonic-gate if ((kpd = segkp_find(seg, vaddr)) == NULL) 9387c478bd9Sstevel@tonic-gate return (FC_NOMAP); 9397c478bd9Sstevel@tonic-gate 9407c478bd9Sstevel@tonic-gate mutex_enter(&kpd->kp_lock); 9417c478bd9Sstevel@tonic-gate 9427c478bd9Sstevel@tonic-gate if (type == F_SOFTLOCK) { 9437c478bd9Sstevel@tonic-gate ASSERT(!(kpd->kp_flags & KPD_LOCKED)); 9447c478bd9Sstevel@tonic-gate /* 9457c478bd9Sstevel@tonic-gate * The F_SOFTLOCK case has more stringent 9467c478bd9Sstevel@tonic-gate * range requirements: the given range must exactly coincide 9477c478bd9Sstevel@tonic-gate * with the resource's mapped portion. Note reference to 9487c478bd9Sstevel@tonic-gate * redzone is handled since vaddr would not equal base 9497c478bd9Sstevel@tonic-gate */ 9507c478bd9Sstevel@tonic-gate if (vaddr != stom(kpd->kp_base, kpd->kp_flags) || 9517c478bd9Sstevel@tonic-gate len != SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)) { 9527c478bd9Sstevel@tonic-gate mutex_exit(&kpd->kp_lock); 9537c478bd9Sstevel@tonic-gate return (FC_MAKE_ERR(EFAULT)); 9547c478bd9Sstevel@tonic-gate } 9557c478bd9Sstevel@tonic-gate 9567c478bd9Sstevel@tonic-gate if ((err = segkp_load(hat, seg, vaddr, len, kpd, KPD_LOCKED))) { 9577c478bd9Sstevel@tonic-gate mutex_exit(&kpd->kp_lock); 9587c478bd9Sstevel@tonic-gate return (FC_MAKE_ERR(err)); 9597c478bd9Sstevel@tonic-gate } 9607c478bd9Sstevel@tonic-gate kpd->kp_flags |= KPD_LOCKED; 9617c478bd9Sstevel@tonic-gate mutex_exit(&kpd->kp_lock); 9627c478bd9Sstevel@tonic-gate return (0); 9637c478bd9Sstevel@tonic-gate } 9647c478bd9Sstevel@tonic-gate 9657c478bd9Sstevel@tonic-gate if (type == F_INVAL) { 9667c478bd9Sstevel@tonic-gate ASSERT(!(kpd->kp_flags & KPD_NO_ANON)); 9677c478bd9Sstevel@tonic-gate 9687c478bd9Sstevel@tonic-gate /* 9697c478bd9Sstevel@tonic-gate * Check if we touched the redzone. Somewhat optimistic 9707c478bd9Sstevel@tonic-gate * here if we are touching the redzone of our own stack 9717c478bd9Sstevel@tonic-gate * since we wouldn't have a stack to get this far... 9727c478bd9Sstevel@tonic-gate */ 9737c478bd9Sstevel@tonic-gate if ((kpd->kp_flags & KPD_HASREDZONE) && 9747c478bd9Sstevel@tonic-gate btop((uintptr_t)(vaddr - kpd->kp_base)) == KPD_REDZONE(kpd)) 9757c478bd9Sstevel@tonic-gate panic("segkp_fault: accessing redzone"); 9767c478bd9Sstevel@tonic-gate 9777c478bd9Sstevel@tonic-gate /* 9787c478bd9Sstevel@tonic-gate * This fault may occur while the page is being F_SOFTLOCK'ed. 9797c478bd9Sstevel@tonic-gate * Return since a 2nd segkp_load is unnecessary and also would 9807c478bd9Sstevel@tonic-gate * result in the page being locked twice and eventually 9817c478bd9Sstevel@tonic-gate * hang the thread_reaper thread. 9827c478bd9Sstevel@tonic-gate */ 9837c478bd9Sstevel@tonic-gate if (kpd->kp_flags & KPD_LOCKED) { 9847c478bd9Sstevel@tonic-gate mutex_exit(&kpd->kp_lock); 9857c478bd9Sstevel@tonic-gate return (0); 9867c478bd9Sstevel@tonic-gate } 9877c478bd9Sstevel@tonic-gate 9887c478bd9Sstevel@tonic-gate err = segkp_load(hat, seg, vaddr, len, kpd, kpd->kp_flags); 9897c478bd9Sstevel@tonic-gate mutex_exit(&kpd->kp_lock); 9907c478bd9Sstevel@tonic-gate return (err ? FC_MAKE_ERR(err) : 0); 9917c478bd9Sstevel@tonic-gate } 9927c478bd9Sstevel@tonic-gate 9937c478bd9Sstevel@tonic-gate if (type == F_SOFTUNLOCK) { 9947c478bd9Sstevel@tonic-gate uint_t flags; 9957c478bd9Sstevel@tonic-gate 9967c478bd9Sstevel@tonic-gate /* 9977c478bd9Sstevel@tonic-gate * Make sure the addr is LOCKED and it has anon backing 9987c478bd9Sstevel@tonic-gate * before unlocking 9997c478bd9Sstevel@tonic-gate */ 10007c478bd9Sstevel@tonic-gate if ((kpd->kp_flags & (KPD_LOCKED|KPD_NO_ANON)) == KPD_NO_ANON) { 10017c478bd9Sstevel@tonic-gate panic("segkp_fault: bad unlock"); 10027c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 10037c478bd9Sstevel@tonic-gate } 10047c478bd9Sstevel@tonic-gate 10057c478bd9Sstevel@tonic-gate if (vaddr != stom(kpd->kp_base, kpd->kp_flags) || 10067c478bd9Sstevel@tonic-gate len != SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)) { 10077c478bd9Sstevel@tonic-gate panic("segkp_fault: bad range"); 10087c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 10097c478bd9Sstevel@tonic-gate } 10107c478bd9Sstevel@tonic-gate 10117c478bd9Sstevel@tonic-gate if (rw == S_WRITE) 10127c478bd9Sstevel@tonic-gate flags = kpd->kp_flags | KPD_WRITEDIRTY; 10137c478bd9Sstevel@tonic-gate else 10147c478bd9Sstevel@tonic-gate flags = kpd->kp_flags; 10157c478bd9Sstevel@tonic-gate err = segkp_unlock(hat, seg, vaddr, len, kpd, flags); 10167c478bd9Sstevel@tonic-gate kpd->kp_flags &= ~KPD_LOCKED; 10177c478bd9Sstevel@tonic-gate mutex_exit(&kpd->kp_lock); 10187c478bd9Sstevel@tonic-gate return (err ? FC_MAKE_ERR(err) : 0); 10197c478bd9Sstevel@tonic-gate } 10207c478bd9Sstevel@tonic-gate mutex_exit(&kpd->kp_lock); 10217c478bd9Sstevel@tonic-gate panic("segkp_fault: bogus fault type: %d\n", type); 10227c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 10237c478bd9Sstevel@tonic-gate } 10247c478bd9Sstevel@tonic-gate 10257c478bd9Sstevel@tonic-gate /* 10267c478bd9Sstevel@tonic-gate * Check that the given protections suffice over the range specified by 10277c478bd9Sstevel@tonic-gate * vaddr and len. For this segment type, the only issue is whether or 10287c478bd9Sstevel@tonic-gate * not the range lies completely within the mapped part of an allocated 10297c478bd9Sstevel@tonic-gate * resource. 10307c478bd9Sstevel@tonic-gate */ 10317c478bd9Sstevel@tonic-gate /* ARGSUSED */ 10327c478bd9Sstevel@tonic-gate static int 10337c478bd9Sstevel@tonic-gate segkp_checkprot(struct seg *seg, caddr_t vaddr, size_t len, uint_t prot) 10347c478bd9Sstevel@tonic-gate { 10357c478bd9Sstevel@tonic-gate struct segkp_data *kpd = NULL; 10367c478bd9Sstevel@tonic-gate caddr_t mbase; 10377c478bd9Sstevel@tonic-gate size_t mlen; 10387c478bd9Sstevel@tonic-gate 10397c478bd9Sstevel@tonic-gate if ((kpd = segkp_find(seg, vaddr)) == NULL) 10407c478bd9Sstevel@tonic-gate return (EACCES); 10417c478bd9Sstevel@tonic-gate 10427c478bd9Sstevel@tonic-gate mutex_enter(&kpd->kp_lock); 10437c478bd9Sstevel@tonic-gate mbase = stom(kpd->kp_base, kpd->kp_flags); 10447c478bd9Sstevel@tonic-gate mlen = SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags); 10457c478bd9Sstevel@tonic-gate if (len > mlen || vaddr < mbase || 10467c478bd9Sstevel@tonic-gate ((vaddr + len) > (mbase + mlen))) { 10477c478bd9Sstevel@tonic-gate mutex_exit(&kpd->kp_lock); 10487c478bd9Sstevel@tonic-gate return (EACCES); 10497c478bd9Sstevel@tonic-gate } 10507c478bd9Sstevel@tonic-gate mutex_exit(&kpd->kp_lock); 10517c478bd9Sstevel@tonic-gate return (0); 10527c478bd9Sstevel@tonic-gate } 10537c478bd9Sstevel@tonic-gate 10547c478bd9Sstevel@tonic-gate 10557c478bd9Sstevel@tonic-gate /* 10567c478bd9Sstevel@tonic-gate * Check to see if it makes sense to do kluster/read ahead to 10577c478bd9Sstevel@tonic-gate * addr + delta relative to the mapping at addr. We assume here 10587c478bd9Sstevel@tonic-gate * that delta is a signed PAGESIZE'd multiple (which can be negative). 10597c478bd9Sstevel@tonic-gate * 10607c478bd9Sstevel@tonic-gate * For seg_u we always "approve" of this action from our standpoint. 10617c478bd9Sstevel@tonic-gate */ 10627c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 10637c478bd9Sstevel@tonic-gate static int 10647c478bd9Sstevel@tonic-gate segkp_kluster(struct seg *seg, caddr_t addr, ssize_t delta) 10657c478bd9Sstevel@tonic-gate { 10667c478bd9Sstevel@tonic-gate return (0); 10677c478bd9Sstevel@tonic-gate } 10687c478bd9Sstevel@tonic-gate 10697c478bd9Sstevel@tonic-gate /* 10707c478bd9Sstevel@tonic-gate * Load and possibly lock intra-slot resources in the range given by 10717c478bd9Sstevel@tonic-gate * vaddr and len. 10727c478bd9Sstevel@tonic-gate */ 10737c478bd9Sstevel@tonic-gate static int 10747c478bd9Sstevel@tonic-gate segkp_load( 10757c478bd9Sstevel@tonic-gate struct hat *hat, 10767c478bd9Sstevel@tonic-gate struct seg *seg, 10777c478bd9Sstevel@tonic-gate caddr_t vaddr, 10787c478bd9Sstevel@tonic-gate size_t len, 10797c478bd9Sstevel@tonic-gate struct segkp_data *kpd, 10807c478bd9Sstevel@tonic-gate uint_t flags) 10817c478bd9Sstevel@tonic-gate { 10827c478bd9Sstevel@tonic-gate caddr_t va; 10837c478bd9Sstevel@tonic-gate caddr_t vlim; 10847c478bd9Sstevel@tonic-gate ulong_t i; 10857c478bd9Sstevel@tonic-gate uint_t lock; 10867c478bd9Sstevel@tonic-gate 10877c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&kpd->kp_lock)); 10887c478bd9Sstevel@tonic-gate 10897c478bd9Sstevel@tonic-gate len = P2ROUNDUP(len, PAGESIZE); 10907c478bd9Sstevel@tonic-gate 10917c478bd9Sstevel@tonic-gate /* If locking, reserve physical memory */ 10927c478bd9Sstevel@tonic-gate if (flags & KPD_LOCKED) { 10937c478bd9Sstevel@tonic-gate pgcnt_t pages = btop(len); 10947c478bd9Sstevel@tonic-gate if ((kpd->kp_flags & KPD_NO_ANON) == 0) 10957c478bd9Sstevel@tonic-gate atomic_add_long(&anon_segkp_pages_locked, pages); 10967c478bd9Sstevel@tonic-gate (void) page_resv(pages, KM_SLEEP); 10977c478bd9Sstevel@tonic-gate } 10987c478bd9Sstevel@tonic-gate 10997c478bd9Sstevel@tonic-gate /* 11007c478bd9Sstevel@tonic-gate * Loop through the pages in the given range. 11017c478bd9Sstevel@tonic-gate */ 11027c478bd9Sstevel@tonic-gate va = (caddr_t)((uintptr_t)vaddr & (uintptr_t)PAGEMASK); 11037c478bd9Sstevel@tonic-gate vaddr = va; 11047c478bd9Sstevel@tonic-gate vlim = va + len; 11057c478bd9Sstevel@tonic-gate lock = flags & KPD_LOCKED; 11067c478bd9Sstevel@tonic-gate i = ((uintptr_t)(va - kpd->kp_base)) >> PAGESHIFT; 11077c478bd9Sstevel@tonic-gate for (; va < vlim; va += PAGESIZE, i++) { 11087c478bd9Sstevel@tonic-gate page_t *pl[2]; /* second element NULL terminator */ 11097c478bd9Sstevel@tonic-gate struct vnode *vp; 11107c478bd9Sstevel@tonic-gate anoff_t off; 11117c478bd9Sstevel@tonic-gate int err; 11127c478bd9Sstevel@tonic-gate struct anon *ap; 11137c478bd9Sstevel@tonic-gate 11147c478bd9Sstevel@tonic-gate /* 11157c478bd9Sstevel@tonic-gate * Summon the page. If it's not resident, arrange 11167c478bd9Sstevel@tonic-gate * for synchronous i/o to pull it in. 11177c478bd9Sstevel@tonic-gate */ 11187c478bd9Sstevel@tonic-gate ap = anon_get_ptr(kpd->kp_anon, kpd->kp_anon_idx + i); 11197c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &off); 11207c478bd9Sstevel@tonic-gate 11217c478bd9Sstevel@tonic-gate /* 11227c478bd9Sstevel@tonic-gate * The returned page list will have exactly one entry, 11237c478bd9Sstevel@tonic-gate * which is returned to us already kept. 11247c478bd9Sstevel@tonic-gate */ 11257c478bd9Sstevel@tonic-gate err = VOP_GETPAGE(vp, (offset_t)off, PAGESIZE, NULL, 11267c478bd9Sstevel@tonic-gate pl, PAGESIZE, seg, va, S_READ, kcred); 11277c478bd9Sstevel@tonic-gate 11287c478bd9Sstevel@tonic-gate if (err) { 11297c478bd9Sstevel@tonic-gate /* 11307c478bd9Sstevel@tonic-gate * Back out of what we've done so far. 11317c478bd9Sstevel@tonic-gate */ 11327c478bd9Sstevel@tonic-gate (void) segkp_unlock(hat, seg, vaddr, 11337c478bd9Sstevel@tonic-gate (va - vaddr), kpd, flags); 11347c478bd9Sstevel@tonic-gate return (err); 11357c478bd9Sstevel@tonic-gate } 11367c478bd9Sstevel@tonic-gate 11377c478bd9Sstevel@tonic-gate /* 11387c478bd9Sstevel@tonic-gate * Load an MMU translation for the page. 11397c478bd9Sstevel@tonic-gate */ 11407c478bd9Sstevel@tonic-gate hat_memload(hat, va, pl[0], (PROT_READ|PROT_WRITE), 11417c478bd9Sstevel@tonic-gate lock ? HAT_LOAD_LOCK : HAT_LOAD); 11427c478bd9Sstevel@tonic-gate 11437c478bd9Sstevel@tonic-gate if (!lock) { 11447c478bd9Sstevel@tonic-gate /* 11457c478bd9Sstevel@tonic-gate * Now, release "shared" lock on the page. 11467c478bd9Sstevel@tonic-gate */ 11477c478bd9Sstevel@tonic-gate page_unlock(pl[0]); 11487c478bd9Sstevel@tonic-gate } 11497c478bd9Sstevel@tonic-gate } 11507c478bd9Sstevel@tonic-gate return (0); 11517c478bd9Sstevel@tonic-gate } 11527c478bd9Sstevel@tonic-gate 11537c478bd9Sstevel@tonic-gate /* 11547c478bd9Sstevel@tonic-gate * At the very least unload the mmu-translations and unlock the range if locked 11557c478bd9Sstevel@tonic-gate * Can be called with the following flag value KPD_WRITEDIRTY which specifies 11567c478bd9Sstevel@tonic-gate * any dirty pages should be written to disk. 11577c478bd9Sstevel@tonic-gate */ 11587c478bd9Sstevel@tonic-gate static int 11597c478bd9Sstevel@tonic-gate segkp_unlock( 11607c478bd9Sstevel@tonic-gate struct hat *hat, 11617c478bd9Sstevel@tonic-gate struct seg *seg, 11627c478bd9Sstevel@tonic-gate caddr_t vaddr, 11637c478bd9Sstevel@tonic-gate size_t len, 11647c478bd9Sstevel@tonic-gate struct segkp_data *kpd, 11657c478bd9Sstevel@tonic-gate uint_t flags) 11667c478bd9Sstevel@tonic-gate { 11677c478bd9Sstevel@tonic-gate caddr_t va; 11687c478bd9Sstevel@tonic-gate caddr_t vlim; 11697c478bd9Sstevel@tonic-gate ulong_t i; 11707c478bd9Sstevel@tonic-gate struct page *pp; 11717c478bd9Sstevel@tonic-gate struct vnode *vp; 11727c478bd9Sstevel@tonic-gate anoff_t off; 11737c478bd9Sstevel@tonic-gate struct anon *ap; 11747c478bd9Sstevel@tonic-gate 11757c478bd9Sstevel@tonic-gate #ifdef lint 11767c478bd9Sstevel@tonic-gate seg = seg; 11777c478bd9Sstevel@tonic-gate #endif /* lint */ 11787c478bd9Sstevel@tonic-gate 11797c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&kpd->kp_lock)); 11807c478bd9Sstevel@tonic-gate 11817c478bd9Sstevel@tonic-gate /* 11827c478bd9Sstevel@tonic-gate * Loop through the pages in the given range. It is assumed 11837c478bd9Sstevel@tonic-gate * segkp_unlock is called with page aligned base 11847c478bd9Sstevel@tonic-gate */ 11857c478bd9Sstevel@tonic-gate va = vaddr; 11867c478bd9Sstevel@tonic-gate vlim = va + len; 11877c478bd9Sstevel@tonic-gate i = ((uintptr_t)(va - kpd->kp_base)) >> PAGESHIFT; 11887c478bd9Sstevel@tonic-gate hat_unload(hat, va, len, 11897c478bd9Sstevel@tonic-gate ((flags & KPD_LOCKED) ? HAT_UNLOAD_UNLOCK : HAT_UNLOAD)); 11907c478bd9Sstevel@tonic-gate for (; va < vlim; va += PAGESIZE, i++) { 11917c478bd9Sstevel@tonic-gate /* 11927c478bd9Sstevel@tonic-gate * Find the page associated with this part of the 11937c478bd9Sstevel@tonic-gate * slot, tracking it down through its associated swap 11947c478bd9Sstevel@tonic-gate * space. 11957c478bd9Sstevel@tonic-gate */ 11967c478bd9Sstevel@tonic-gate ap = anon_get_ptr(kpd->kp_anon, kpd->kp_anon_idx + i); 11977c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &off); 11987c478bd9Sstevel@tonic-gate 11997c478bd9Sstevel@tonic-gate if (flags & KPD_LOCKED) { 12007c478bd9Sstevel@tonic-gate if ((pp = page_find(vp, off)) == NULL) { 12017c478bd9Sstevel@tonic-gate if (flags & KPD_LOCKED) { 12027c478bd9Sstevel@tonic-gate panic("segkp_softunlock: missing page"); 12037c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 12047c478bd9Sstevel@tonic-gate } 12057c478bd9Sstevel@tonic-gate } 12067c478bd9Sstevel@tonic-gate } else { 12077c478bd9Sstevel@tonic-gate /* 12087c478bd9Sstevel@tonic-gate * Nothing to do if the slot is not locked and the 12097c478bd9Sstevel@tonic-gate * page doesn't exist. 12107c478bd9Sstevel@tonic-gate */ 12117c478bd9Sstevel@tonic-gate if ((pp = page_lookup(vp, off, SE_SHARED)) == NULL) 12127c478bd9Sstevel@tonic-gate continue; 12137c478bd9Sstevel@tonic-gate } 12147c478bd9Sstevel@tonic-gate 12157c478bd9Sstevel@tonic-gate /* 12167c478bd9Sstevel@tonic-gate * If the page doesn't have any translations, is 12177c478bd9Sstevel@tonic-gate * dirty and not being shared, then push it out 12187c478bd9Sstevel@tonic-gate * asynchronously and avoid waiting for the 12197c478bd9Sstevel@tonic-gate * pageout daemon to do it for us. 12207c478bd9Sstevel@tonic-gate * 12217c478bd9Sstevel@tonic-gate * XXX - Do we really need to get the "exclusive" 12227c478bd9Sstevel@tonic-gate * lock via an upgrade? 12237c478bd9Sstevel@tonic-gate */ 12247c478bd9Sstevel@tonic-gate if ((flags & KPD_WRITEDIRTY) && !hat_page_is_mapped(pp) && 12257c478bd9Sstevel@tonic-gate hat_ismod(pp) && page_tryupgrade(pp)) { 12267c478bd9Sstevel@tonic-gate /* 12277c478bd9Sstevel@tonic-gate * Hold the vnode before releasing the page lock to 12287c478bd9Sstevel@tonic-gate * prevent it from being freed and re-used by some 12297c478bd9Sstevel@tonic-gate * other thread. 12307c478bd9Sstevel@tonic-gate */ 12317c478bd9Sstevel@tonic-gate VN_HOLD(vp); 12327c478bd9Sstevel@tonic-gate page_unlock(pp); 12337c478bd9Sstevel@tonic-gate 12347c478bd9Sstevel@tonic-gate /* 12357c478bd9Sstevel@tonic-gate * Want most powerful credentials we can get so 12367c478bd9Sstevel@tonic-gate * use kcred. 12377c478bd9Sstevel@tonic-gate */ 12387c478bd9Sstevel@tonic-gate (void) VOP_PUTPAGE(vp, (offset_t)off, PAGESIZE, 12397c478bd9Sstevel@tonic-gate B_ASYNC | B_FREE, kcred); 12407c478bd9Sstevel@tonic-gate VN_RELE(vp); 12417c478bd9Sstevel@tonic-gate } else { 12427c478bd9Sstevel@tonic-gate page_unlock(pp); 12437c478bd9Sstevel@tonic-gate } 12447c478bd9Sstevel@tonic-gate } 12457c478bd9Sstevel@tonic-gate 12467c478bd9Sstevel@tonic-gate /* If unlocking, release physical memory */ 12477c478bd9Sstevel@tonic-gate if (flags & KPD_LOCKED) { 12487c478bd9Sstevel@tonic-gate pgcnt_t pages = btopr(len); 12497c478bd9Sstevel@tonic-gate if ((kpd->kp_flags & KPD_NO_ANON) == 0) 12507c478bd9Sstevel@tonic-gate atomic_add_long(&anon_segkp_pages_locked, -pages); 12517c478bd9Sstevel@tonic-gate page_unresv(pages); 12527c478bd9Sstevel@tonic-gate } 12537c478bd9Sstevel@tonic-gate return (0); 12547c478bd9Sstevel@tonic-gate } 12557c478bd9Sstevel@tonic-gate 12567c478bd9Sstevel@tonic-gate /* 12577c478bd9Sstevel@tonic-gate * Insert the kpd in the hash table. 12587c478bd9Sstevel@tonic-gate */ 12597c478bd9Sstevel@tonic-gate static void 12607c478bd9Sstevel@tonic-gate segkp_insert(struct seg *seg, struct segkp_data *kpd) 12617c478bd9Sstevel@tonic-gate { 12627c478bd9Sstevel@tonic-gate struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data; 12637c478bd9Sstevel@tonic-gate int index; 12647c478bd9Sstevel@tonic-gate 12657c478bd9Sstevel@tonic-gate /* 12667c478bd9Sstevel@tonic-gate * Insert the kpd based on the address that will be returned 12677c478bd9Sstevel@tonic-gate * via segkp_release. 12687c478bd9Sstevel@tonic-gate */ 12697c478bd9Sstevel@tonic-gate index = SEGKP_HASH(stom(kpd->kp_base, kpd->kp_flags)); 12707c478bd9Sstevel@tonic-gate mutex_enter(&segkp_lock); 12717c478bd9Sstevel@tonic-gate kpd->kp_next = kpsd->kpsd_hash[index]; 12727c478bd9Sstevel@tonic-gate kpsd->kpsd_hash[index] = kpd; 12737c478bd9Sstevel@tonic-gate mutex_exit(&segkp_lock); 12747c478bd9Sstevel@tonic-gate } 12757c478bd9Sstevel@tonic-gate 12767c478bd9Sstevel@tonic-gate /* 12777c478bd9Sstevel@tonic-gate * Remove kpd from the hash table. 12787c478bd9Sstevel@tonic-gate */ 12797c478bd9Sstevel@tonic-gate static void 12807c478bd9Sstevel@tonic-gate segkp_delete(struct seg *seg, struct segkp_data *kpd) 12817c478bd9Sstevel@tonic-gate { 12827c478bd9Sstevel@tonic-gate struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data; 12837c478bd9Sstevel@tonic-gate struct segkp_data **kpp; 12847c478bd9Sstevel@tonic-gate int index; 12857c478bd9Sstevel@tonic-gate 12867c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&segkp_lock)); 12877c478bd9Sstevel@tonic-gate 12887c478bd9Sstevel@tonic-gate index = SEGKP_HASH(stom(kpd->kp_base, kpd->kp_flags)); 12897c478bd9Sstevel@tonic-gate for (kpp = &kpsd->kpsd_hash[index]; 12907c478bd9Sstevel@tonic-gate *kpp != NULL; kpp = &((*kpp)->kp_next)) { 12917c478bd9Sstevel@tonic-gate if (*kpp == kpd) { 12927c478bd9Sstevel@tonic-gate *kpp = kpd->kp_next; 12937c478bd9Sstevel@tonic-gate return; 12947c478bd9Sstevel@tonic-gate } 12957c478bd9Sstevel@tonic-gate } 12967c478bd9Sstevel@tonic-gate panic("segkp_delete: unable to find element to delete"); 12977c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 12987c478bd9Sstevel@tonic-gate } 12997c478bd9Sstevel@tonic-gate 13007c478bd9Sstevel@tonic-gate /* 13017c478bd9Sstevel@tonic-gate * Find the kpd associated with a vaddr. 13027c478bd9Sstevel@tonic-gate * 13037c478bd9Sstevel@tonic-gate * Most of the callers of segkp_find will pass the vaddr that 13047c478bd9Sstevel@tonic-gate * hashes to the desired index, but there are cases where 13057c478bd9Sstevel@tonic-gate * this is not true in which case we have to (potentially) scan 13067c478bd9Sstevel@tonic-gate * the whole table looking for it. This should be very rare 13077c478bd9Sstevel@tonic-gate * (e.g. a segkp_fault(F_INVAL) on an address somewhere in the 13087c478bd9Sstevel@tonic-gate * middle of the segkp_data region). 13097c478bd9Sstevel@tonic-gate */ 13107c478bd9Sstevel@tonic-gate static struct segkp_data * 13117c478bd9Sstevel@tonic-gate segkp_find(struct seg *seg, caddr_t vaddr) 13127c478bd9Sstevel@tonic-gate { 13137c478bd9Sstevel@tonic-gate struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data; 13147c478bd9Sstevel@tonic-gate struct segkp_data *kpd; 13157c478bd9Sstevel@tonic-gate int i; 13167c478bd9Sstevel@tonic-gate int stop; 13177c478bd9Sstevel@tonic-gate 13187c478bd9Sstevel@tonic-gate i = stop = SEGKP_HASH(vaddr); 13197c478bd9Sstevel@tonic-gate mutex_enter(&segkp_lock); 13207c478bd9Sstevel@tonic-gate do { 13217c478bd9Sstevel@tonic-gate for (kpd = kpsd->kpsd_hash[i]; kpd != NULL; 13227c478bd9Sstevel@tonic-gate kpd = kpd->kp_next) { 13237c478bd9Sstevel@tonic-gate if (vaddr >= kpd->kp_base && 13247c478bd9Sstevel@tonic-gate vaddr < kpd->kp_base + kpd->kp_len) { 13257c478bd9Sstevel@tonic-gate mutex_exit(&segkp_lock); 13267c478bd9Sstevel@tonic-gate return (kpd); 13277c478bd9Sstevel@tonic-gate } 13287c478bd9Sstevel@tonic-gate } 13297c478bd9Sstevel@tonic-gate if (--i < 0) 13307c478bd9Sstevel@tonic-gate i = SEGKP_HASHSZ - 1; /* Wrap */ 13317c478bd9Sstevel@tonic-gate } while (i != stop); 13327c478bd9Sstevel@tonic-gate mutex_exit(&segkp_lock); 13337c478bd9Sstevel@tonic-gate return (NULL); /* Not found */ 13347c478bd9Sstevel@tonic-gate } 13357c478bd9Sstevel@tonic-gate 13367c478bd9Sstevel@tonic-gate /* 13377c478bd9Sstevel@tonic-gate * returns size of swappable area. 13387c478bd9Sstevel@tonic-gate */ 13397c478bd9Sstevel@tonic-gate size_t 13407c478bd9Sstevel@tonic-gate swapsize(caddr_t v) 13417c478bd9Sstevel@tonic-gate { 13427c478bd9Sstevel@tonic-gate struct segkp_data *kpd; 13437c478bd9Sstevel@tonic-gate 13447c478bd9Sstevel@tonic-gate if ((kpd = segkp_find(segkp, v)) != NULL) 13457c478bd9Sstevel@tonic-gate return (SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)); 13467c478bd9Sstevel@tonic-gate else 13477c478bd9Sstevel@tonic-gate return (NULL); 13487c478bd9Sstevel@tonic-gate } 13497c478bd9Sstevel@tonic-gate 13507c478bd9Sstevel@tonic-gate /* 13517c478bd9Sstevel@tonic-gate * Dump out all the active segkp pages 13527c478bd9Sstevel@tonic-gate */ 13537c478bd9Sstevel@tonic-gate static void 13547c478bd9Sstevel@tonic-gate segkp_dump(struct seg *seg) 13557c478bd9Sstevel@tonic-gate { 13567c478bd9Sstevel@tonic-gate int i; 13577c478bd9Sstevel@tonic-gate struct segkp_data *kpd; 13587c478bd9Sstevel@tonic-gate struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data; 13597c478bd9Sstevel@tonic-gate 13607c478bd9Sstevel@tonic-gate for (i = 0; i < SEGKP_HASHSZ; i++) { 13617c478bd9Sstevel@tonic-gate for (kpd = kpsd->kpsd_hash[i]; 13627c478bd9Sstevel@tonic-gate kpd != NULL; kpd = kpd->kp_next) { 13637c478bd9Sstevel@tonic-gate pfn_t pfn; 13647c478bd9Sstevel@tonic-gate caddr_t addr; 13657c478bd9Sstevel@tonic-gate caddr_t eaddr; 13667c478bd9Sstevel@tonic-gate 13677c478bd9Sstevel@tonic-gate addr = kpd->kp_base; 13687c478bd9Sstevel@tonic-gate eaddr = addr + kpd->kp_len; 13697c478bd9Sstevel@tonic-gate while (addr < eaddr) { 13707c478bd9Sstevel@tonic-gate ASSERT(seg->s_as == &kas); 13717c478bd9Sstevel@tonic-gate pfn = hat_getpfnum(seg->s_as->a_hat, addr); 13727c478bd9Sstevel@tonic-gate if (pfn != PFN_INVALID) 13737c478bd9Sstevel@tonic-gate dump_addpage(seg->s_as, addr, pfn); 13747c478bd9Sstevel@tonic-gate addr += PAGESIZE; 13757c478bd9Sstevel@tonic-gate dump_timeleft = dump_timeout; 13767c478bd9Sstevel@tonic-gate } 13777c478bd9Sstevel@tonic-gate } 13787c478bd9Sstevel@tonic-gate } 13797c478bd9Sstevel@tonic-gate } 13807c478bd9Sstevel@tonic-gate 13817c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 13827c478bd9Sstevel@tonic-gate static int 13837c478bd9Sstevel@tonic-gate segkp_pagelock(struct seg *seg, caddr_t addr, size_t len, 13847c478bd9Sstevel@tonic-gate struct page ***ppp, enum lock_type type, enum seg_rw rw) 13857c478bd9Sstevel@tonic-gate { 13867c478bd9Sstevel@tonic-gate return (ENOTSUP); 13877c478bd9Sstevel@tonic-gate } 13887c478bd9Sstevel@tonic-gate 13897c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 13907c478bd9Sstevel@tonic-gate static int 13917c478bd9Sstevel@tonic-gate segkp_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp) 13927c478bd9Sstevel@tonic-gate { 13937c478bd9Sstevel@tonic-gate return (ENODEV); 13947c478bd9Sstevel@tonic-gate } 13957c478bd9Sstevel@tonic-gate 13967c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 13977c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t * 13987c478bd9Sstevel@tonic-gate segkp_getpolicy(struct seg *seg, caddr_t addr) 13997c478bd9Sstevel@tonic-gate { 14007c478bd9Sstevel@tonic-gate return (NULL); 14017c478bd9Sstevel@tonic-gate } 14027c478bd9Sstevel@tonic-gate 1403*1bd5c35fSelowe /*ARGSUSED*/ 1404*1bd5c35fSelowe static int 1405*1bd5c35fSelowe segkp_capable(struct seg *seg, segcapability_t capability) 1406*1bd5c35fSelowe { 1407*1bd5c35fSelowe return (0); 1408*1bd5c35fSelowe } 1409*1bd5c35fSelowe 14107c478bd9Sstevel@tonic-gate #include <sys/mem_config.h> 14117c478bd9Sstevel@tonic-gate 14127c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 14137c478bd9Sstevel@tonic-gate static void 14147c478bd9Sstevel@tonic-gate segkp_mem_config_post_add(void *arg, pgcnt_t delta_pages) 14157c478bd9Sstevel@tonic-gate {} 14167c478bd9Sstevel@tonic-gate 14177c478bd9Sstevel@tonic-gate /* 14187c478bd9Sstevel@tonic-gate * During memory delete, turn off caches so that pages are not held. 14197c478bd9Sstevel@tonic-gate * A better solution may be to unlock the pages while they are 14207c478bd9Sstevel@tonic-gate * in the cache so that they may be collected naturally. 14217c478bd9Sstevel@tonic-gate */ 14227c478bd9Sstevel@tonic-gate 14237c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 14247c478bd9Sstevel@tonic-gate static int 14257c478bd9Sstevel@tonic-gate segkp_mem_config_pre_del(void *arg, pgcnt_t delta_pages) 14267c478bd9Sstevel@tonic-gate { 14277c478bd9Sstevel@tonic-gate atomic_add_32(&segkp_indel, 1); 14287c478bd9Sstevel@tonic-gate segkp_cache_free(); 14297c478bd9Sstevel@tonic-gate return (0); 14307c478bd9Sstevel@tonic-gate } 14317c478bd9Sstevel@tonic-gate 14327c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 14337c478bd9Sstevel@tonic-gate static void 14347c478bd9Sstevel@tonic-gate segkp_mem_config_post_del(void *arg, pgcnt_t delta_pages, int cancelled) 14357c478bd9Sstevel@tonic-gate { 14367c478bd9Sstevel@tonic-gate atomic_add_32(&segkp_indel, -1); 14377c478bd9Sstevel@tonic-gate } 14387c478bd9Sstevel@tonic-gate 14397c478bd9Sstevel@tonic-gate static kphysm_setup_vector_t segkp_mem_config_vec = { 14407c478bd9Sstevel@tonic-gate KPHYSM_SETUP_VECTOR_VERSION, 14417c478bd9Sstevel@tonic-gate segkp_mem_config_post_add, 14427c478bd9Sstevel@tonic-gate segkp_mem_config_pre_del, 14437c478bd9Sstevel@tonic-gate segkp_mem_config_post_del, 14447c478bd9Sstevel@tonic-gate }; 14457c478bd9Sstevel@tonic-gate 14467c478bd9Sstevel@tonic-gate static void 14477c478bd9Sstevel@tonic-gate segkpinit_mem_config(struct seg *seg) 14487c478bd9Sstevel@tonic-gate { 14497c478bd9Sstevel@tonic-gate int ret; 14507c478bd9Sstevel@tonic-gate 14517c478bd9Sstevel@tonic-gate ret = kphysm_setup_func_register(&segkp_mem_config_vec, (void *)seg); 14527c478bd9Sstevel@tonic-gate ASSERT(ret == 0); 14537c478bd9Sstevel@tonic-gate } 1454