17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 50209230bSgjelinek * Common Development and Distribution License (the "License"). 60209230bSgjelinek * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22*da6c28aaSamw * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 277c478bd9Sstevel@tonic-gate /* All Rights Reserved */ 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate /* 307c478bd9Sstevel@tonic-gate * Portions of this source code were derived from Berkeley 4.3 BSD 317c478bd9Sstevel@tonic-gate * under license from the Regents of the University of California. 327c478bd9Sstevel@tonic-gate */ 337c478bd9Sstevel@tonic-gate 347c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 357c478bd9Sstevel@tonic-gate 367c478bd9Sstevel@tonic-gate /* 377c478bd9Sstevel@tonic-gate * segkp is a segment driver that administers the allocation and deallocation 387c478bd9Sstevel@tonic-gate * of pageable variable size chunks of kernel virtual address space. Each 397c478bd9Sstevel@tonic-gate * allocated resource is page-aligned. 407c478bd9Sstevel@tonic-gate * 417c478bd9Sstevel@tonic-gate * The user may specify whether the resource should be initialized to 0, 427c478bd9Sstevel@tonic-gate * include a redzone, or locked in memory. 437c478bd9Sstevel@tonic-gate */ 447c478bd9Sstevel@tonic-gate 457c478bd9Sstevel@tonic-gate #include <sys/types.h> 467c478bd9Sstevel@tonic-gate #include <sys/t_lock.h> 477c478bd9Sstevel@tonic-gate #include <sys/thread.h> 487c478bd9Sstevel@tonic-gate #include <sys/param.h> 497c478bd9Sstevel@tonic-gate #include <sys/errno.h> 507c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 517c478bd9Sstevel@tonic-gate #include <sys/systm.h> 527c478bd9Sstevel@tonic-gate #include <sys/buf.h> 537c478bd9Sstevel@tonic-gate #include <sys/mman.h> 547c478bd9Sstevel@tonic-gate #include <sys/vnode.h> 557c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 567c478bd9Sstevel@tonic-gate #include <sys/swap.h> 577c478bd9Sstevel@tonic-gate #include <sys/tuneable.h> 587c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 597c478bd9Sstevel@tonic-gate #include <sys/vmem.h> 607c478bd9Sstevel@tonic-gate #include <sys/cred.h> 617c478bd9Sstevel@tonic-gate #include <sys/dumphdr.h> 627c478bd9Sstevel@tonic-gate #include <sys/debug.h> 637c478bd9Sstevel@tonic-gate #include <sys/vtrace.h> 647c478bd9Sstevel@tonic-gate #include <sys/stack.h> 657c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 667c478bd9Sstevel@tonic-gate #include <sys/archsystm.h> 677c478bd9Sstevel@tonic-gate #include <sys/lgrp.h> 687c478bd9Sstevel@tonic-gate 697c478bd9Sstevel@tonic-gate #include <vm/as.h> 707c478bd9Sstevel@tonic-gate #include <vm/seg.h> 717c478bd9Sstevel@tonic-gate #include <vm/seg_kp.h> 727c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h> 737c478bd9Sstevel@tonic-gate #include <vm/anon.h> 747c478bd9Sstevel@tonic-gate #include <vm/page.h> 757c478bd9Sstevel@tonic-gate #include <vm/hat.h> 767c478bd9Sstevel@tonic-gate #include <sys/bitmap.h> 777c478bd9Sstevel@tonic-gate 787c478bd9Sstevel@tonic-gate /* 797c478bd9Sstevel@tonic-gate * Private seg op routines 807c478bd9Sstevel@tonic-gate */ 817c478bd9Sstevel@tonic-gate static void segkp_badop(void); 827c478bd9Sstevel@tonic-gate static void segkp_dump(struct seg *seg); 837c478bd9Sstevel@tonic-gate static int segkp_checkprot(struct seg *seg, caddr_t addr, size_t len, 847c478bd9Sstevel@tonic-gate uint_t prot); 857c478bd9Sstevel@tonic-gate static int segkp_kluster(struct seg *seg, caddr_t addr, ssize_t delta); 867c478bd9Sstevel@tonic-gate static int segkp_pagelock(struct seg *seg, caddr_t addr, size_t len, 877c478bd9Sstevel@tonic-gate struct page ***page, enum lock_type type, 887c478bd9Sstevel@tonic-gate enum seg_rw rw); 897c478bd9Sstevel@tonic-gate static void segkp_insert(struct seg *seg, struct segkp_data *kpd); 907c478bd9Sstevel@tonic-gate static void segkp_delete(struct seg *seg, struct segkp_data *kpd); 917c478bd9Sstevel@tonic-gate static caddr_t segkp_get_internal(struct seg *seg, size_t len, uint_t flags, 927c478bd9Sstevel@tonic-gate struct segkp_data **tkpd, struct anon_map *amp); 937c478bd9Sstevel@tonic-gate static void segkp_release_internal(struct seg *seg, 947c478bd9Sstevel@tonic-gate struct segkp_data *kpd, size_t len); 957c478bd9Sstevel@tonic-gate static int segkp_unlock(struct hat *hat, struct seg *seg, caddr_t vaddr, 967c478bd9Sstevel@tonic-gate size_t len, struct segkp_data *kpd, uint_t flags); 977c478bd9Sstevel@tonic-gate static int segkp_load(struct hat *hat, struct seg *seg, caddr_t vaddr, 987c478bd9Sstevel@tonic-gate size_t len, struct segkp_data *kpd, uint_t flags); 997c478bd9Sstevel@tonic-gate static struct segkp_data *segkp_find(struct seg *seg, caddr_t vaddr); 1007c478bd9Sstevel@tonic-gate static int segkp_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp); 1017c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t *segkp_getpolicy(struct seg *seg, 1027c478bd9Sstevel@tonic-gate caddr_t addr); 1031bd5c35fSelowe static int segkp_capable(struct seg *seg, segcapability_t capability); 1047c478bd9Sstevel@tonic-gate 1057c478bd9Sstevel@tonic-gate /* 1067c478bd9Sstevel@tonic-gate * Lock used to protect the hash table(s) and caches. 1077c478bd9Sstevel@tonic-gate */ 1087c478bd9Sstevel@tonic-gate static kmutex_t segkp_lock; 1097c478bd9Sstevel@tonic-gate 1107c478bd9Sstevel@tonic-gate /* 1117c478bd9Sstevel@tonic-gate * The segkp caches 1127c478bd9Sstevel@tonic-gate */ 1137c478bd9Sstevel@tonic-gate static struct segkp_cache segkp_cache[SEGKP_MAX_CACHE]; 1147c478bd9Sstevel@tonic-gate 1157c478bd9Sstevel@tonic-gate #define SEGKP_BADOP(t) (t(*)())segkp_badop 1167c478bd9Sstevel@tonic-gate 1177c478bd9Sstevel@tonic-gate /* 1187c478bd9Sstevel@tonic-gate * When there are fewer than red_minavail bytes left on the stack, 1197c478bd9Sstevel@tonic-gate * segkp_map_red() will map in the redzone (if called). 5000 seems 1207c478bd9Sstevel@tonic-gate * to work reasonably well... 1217c478bd9Sstevel@tonic-gate */ 1227c478bd9Sstevel@tonic-gate long red_minavail = 5000; 1237c478bd9Sstevel@tonic-gate 1247c478bd9Sstevel@tonic-gate /* 1257c478bd9Sstevel@tonic-gate * will be set to 1 for 32 bit x86 systems only, in startup.c 1267c478bd9Sstevel@tonic-gate */ 1277c478bd9Sstevel@tonic-gate int segkp_fromheap = 0; 1287c478bd9Sstevel@tonic-gate ulong_t *segkp_bitmap; 1297c478bd9Sstevel@tonic-gate 1307c478bd9Sstevel@tonic-gate /* 1317c478bd9Sstevel@tonic-gate * If segkp_map_red() is called with the redzone already mapped and 1327c478bd9Sstevel@tonic-gate * with less than RED_DEEP_THRESHOLD bytes available on the stack, 1337c478bd9Sstevel@tonic-gate * then the stack situation has become quite serious; if much more stack 1347c478bd9Sstevel@tonic-gate * is consumed, we have the potential of scrogging the next thread/LWP 1357c478bd9Sstevel@tonic-gate * structure. To help debug the "can't happen" panics which may 1367c478bd9Sstevel@tonic-gate * result from this condition, we record lbolt and the calling thread 1377c478bd9Sstevel@tonic-gate * in red_deep_lbolt and red_deep_thread respectively. 1387c478bd9Sstevel@tonic-gate */ 1397c478bd9Sstevel@tonic-gate #define RED_DEEP_THRESHOLD 2000 1407c478bd9Sstevel@tonic-gate 1417c478bd9Sstevel@tonic-gate clock_t red_deep_lbolt; 1427c478bd9Sstevel@tonic-gate kthread_t *red_deep_thread; 1437c478bd9Sstevel@tonic-gate 1447c478bd9Sstevel@tonic-gate uint32_t red_nmapped; 1457c478bd9Sstevel@tonic-gate uint32_t red_closest = UINT_MAX; 1467c478bd9Sstevel@tonic-gate uint32_t red_ndoubles; 1477c478bd9Sstevel@tonic-gate 1487c478bd9Sstevel@tonic-gate pgcnt_t anon_segkp_pages_locked; /* See vm/anon.h */ 1490209230bSgjelinek pgcnt_t anon_segkp_pages_resv; /* anon reserved by seg_kp */ 1507c478bd9Sstevel@tonic-gate 1517c478bd9Sstevel@tonic-gate static struct seg_ops segkp_ops = { 1527c478bd9Sstevel@tonic-gate SEGKP_BADOP(int), /* dup */ 1537c478bd9Sstevel@tonic-gate SEGKP_BADOP(int), /* unmap */ 1547c478bd9Sstevel@tonic-gate SEGKP_BADOP(void), /* free */ 1557c478bd9Sstevel@tonic-gate segkp_fault, 1567c478bd9Sstevel@tonic-gate SEGKP_BADOP(faultcode_t), /* faulta */ 1577c478bd9Sstevel@tonic-gate SEGKP_BADOP(int), /* setprot */ 1587c478bd9Sstevel@tonic-gate segkp_checkprot, 1597c478bd9Sstevel@tonic-gate segkp_kluster, 1607c478bd9Sstevel@tonic-gate SEGKP_BADOP(size_t), /* swapout */ 1617c478bd9Sstevel@tonic-gate SEGKP_BADOP(int), /* sync */ 1627c478bd9Sstevel@tonic-gate SEGKP_BADOP(size_t), /* incore */ 1637c478bd9Sstevel@tonic-gate SEGKP_BADOP(int), /* lockop */ 1647c478bd9Sstevel@tonic-gate SEGKP_BADOP(int), /* getprot */ 1657c478bd9Sstevel@tonic-gate SEGKP_BADOP(u_offset_t), /* getoffset */ 1667c478bd9Sstevel@tonic-gate SEGKP_BADOP(int), /* gettype */ 1677c478bd9Sstevel@tonic-gate SEGKP_BADOP(int), /* getvp */ 1687c478bd9Sstevel@tonic-gate SEGKP_BADOP(int), /* advise */ 1697c478bd9Sstevel@tonic-gate segkp_dump, /* dump */ 1707c478bd9Sstevel@tonic-gate segkp_pagelock, /* pagelock */ 1717c478bd9Sstevel@tonic-gate SEGKP_BADOP(int), /* setpgsz */ 1727c478bd9Sstevel@tonic-gate segkp_getmemid, /* getmemid */ 1737c478bd9Sstevel@tonic-gate segkp_getpolicy, /* getpolicy */ 1741bd5c35fSelowe segkp_capable, /* capable */ 1757c478bd9Sstevel@tonic-gate }; 1767c478bd9Sstevel@tonic-gate 1777c478bd9Sstevel@tonic-gate 1787c478bd9Sstevel@tonic-gate static void 1797c478bd9Sstevel@tonic-gate segkp_badop(void) 1807c478bd9Sstevel@tonic-gate { 1817c478bd9Sstevel@tonic-gate panic("segkp_badop"); 1827c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 1837c478bd9Sstevel@tonic-gate } 1847c478bd9Sstevel@tonic-gate 1857c478bd9Sstevel@tonic-gate static void segkpinit_mem_config(struct seg *); 1867c478bd9Sstevel@tonic-gate 1877c478bd9Sstevel@tonic-gate static uint32_t segkp_indel; 1887c478bd9Sstevel@tonic-gate 1897c478bd9Sstevel@tonic-gate /* 1907c478bd9Sstevel@tonic-gate * Allocate the segment specific private data struct and fill it in 1917c478bd9Sstevel@tonic-gate * with the per kp segment mutex, anon ptr. array and hash table. 1927c478bd9Sstevel@tonic-gate */ 1937c478bd9Sstevel@tonic-gate int 1947c478bd9Sstevel@tonic-gate segkp_create(struct seg *seg) 1957c478bd9Sstevel@tonic-gate { 1967c478bd9Sstevel@tonic-gate struct segkp_segdata *kpsd; 1977c478bd9Sstevel@tonic-gate size_t np; 1987c478bd9Sstevel@tonic-gate 1997c478bd9Sstevel@tonic-gate ASSERT(seg != NULL && seg->s_as == &kas); 2007c478bd9Sstevel@tonic-gate ASSERT(RW_WRITE_HELD(&seg->s_as->a_lock)); 2017c478bd9Sstevel@tonic-gate 2027c478bd9Sstevel@tonic-gate if (seg->s_size & PAGEOFFSET) { 2037c478bd9Sstevel@tonic-gate panic("Bad segkp size"); 2047c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 2057c478bd9Sstevel@tonic-gate } 2067c478bd9Sstevel@tonic-gate 2077c478bd9Sstevel@tonic-gate kpsd = kmem_zalloc(sizeof (struct segkp_segdata), KM_SLEEP); 2087c478bd9Sstevel@tonic-gate 2097c478bd9Sstevel@tonic-gate /* 2107c478bd9Sstevel@tonic-gate * Allocate the virtual memory for segkp and initialize it 2117c478bd9Sstevel@tonic-gate */ 2127c478bd9Sstevel@tonic-gate if (segkp_fromheap) { 2137c478bd9Sstevel@tonic-gate np = btop(kvseg.s_size); 2147c478bd9Sstevel@tonic-gate segkp_bitmap = kmem_zalloc(BT_SIZEOFMAP(np), KM_SLEEP); 2157c478bd9Sstevel@tonic-gate kpsd->kpsd_arena = vmem_create("segkp", NULL, 0, PAGESIZE, 2167c478bd9Sstevel@tonic-gate vmem_alloc, vmem_free, heap_arena, 5 * PAGESIZE, VM_SLEEP); 2177c478bd9Sstevel@tonic-gate } else { 2187c478bd9Sstevel@tonic-gate segkp_bitmap = NULL; 2197c478bd9Sstevel@tonic-gate np = btop(seg->s_size); 2207c478bd9Sstevel@tonic-gate kpsd->kpsd_arena = vmem_create("segkp", seg->s_base, 2217c478bd9Sstevel@tonic-gate seg->s_size, PAGESIZE, NULL, NULL, NULL, 5 * PAGESIZE, 2227c478bd9Sstevel@tonic-gate VM_SLEEP); 2237c478bd9Sstevel@tonic-gate } 2247c478bd9Sstevel@tonic-gate 2257c478bd9Sstevel@tonic-gate kpsd->kpsd_anon = anon_create(np, ANON_SLEEP | ANON_ALLOC_FORCE); 2267c478bd9Sstevel@tonic-gate 2277c478bd9Sstevel@tonic-gate kpsd->kpsd_hash = kmem_zalloc(SEGKP_HASHSZ * sizeof (struct segkp *), 2287c478bd9Sstevel@tonic-gate KM_SLEEP); 2297c478bd9Sstevel@tonic-gate seg->s_data = (void *)kpsd; 2307c478bd9Sstevel@tonic-gate seg->s_ops = &segkp_ops; 2317c478bd9Sstevel@tonic-gate segkpinit_mem_config(seg); 2327c478bd9Sstevel@tonic-gate return (0); 2337c478bd9Sstevel@tonic-gate } 2347c478bd9Sstevel@tonic-gate 2357c478bd9Sstevel@tonic-gate 2367c478bd9Sstevel@tonic-gate /* 2377c478bd9Sstevel@tonic-gate * Find a free 'freelist' and initialize it with the appropriate attributes 2387c478bd9Sstevel@tonic-gate */ 2397c478bd9Sstevel@tonic-gate void * 2407c478bd9Sstevel@tonic-gate segkp_cache_init(struct seg *seg, int maxsize, size_t len, uint_t flags) 2417c478bd9Sstevel@tonic-gate { 2427c478bd9Sstevel@tonic-gate int i; 2437c478bd9Sstevel@tonic-gate 2447c478bd9Sstevel@tonic-gate if ((flags & KPD_NO_ANON) && !(flags & KPD_LOCKED)) 2457c478bd9Sstevel@tonic-gate return ((void *)-1); 2467c478bd9Sstevel@tonic-gate 2477c478bd9Sstevel@tonic-gate mutex_enter(&segkp_lock); 2487c478bd9Sstevel@tonic-gate for (i = 0; i < SEGKP_MAX_CACHE; i++) { 2497c478bd9Sstevel@tonic-gate if (segkp_cache[i].kpf_inuse) 2507c478bd9Sstevel@tonic-gate continue; 2517c478bd9Sstevel@tonic-gate segkp_cache[i].kpf_inuse = 1; 2527c478bd9Sstevel@tonic-gate segkp_cache[i].kpf_max = maxsize; 2537c478bd9Sstevel@tonic-gate segkp_cache[i].kpf_flags = flags; 2547c478bd9Sstevel@tonic-gate segkp_cache[i].kpf_seg = seg; 2557c478bd9Sstevel@tonic-gate segkp_cache[i].kpf_len = len; 2567c478bd9Sstevel@tonic-gate mutex_exit(&segkp_lock); 2577c478bd9Sstevel@tonic-gate return ((void *)(uintptr_t)i); 2587c478bd9Sstevel@tonic-gate } 2597c478bd9Sstevel@tonic-gate mutex_exit(&segkp_lock); 2607c478bd9Sstevel@tonic-gate return ((void *)-1); 2617c478bd9Sstevel@tonic-gate } 2627c478bd9Sstevel@tonic-gate 2637c478bd9Sstevel@tonic-gate /* 2647c478bd9Sstevel@tonic-gate * Free all the cache resources. 2657c478bd9Sstevel@tonic-gate */ 2667c478bd9Sstevel@tonic-gate void 2677c478bd9Sstevel@tonic-gate segkp_cache_free(void) 2687c478bd9Sstevel@tonic-gate { 2697c478bd9Sstevel@tonic-gate struct segkp_data *kpd; 2707c478bd9Sstevel@tonic-gate struct seg *seg; 2717c478bd9Sstevel@tonic-gate int i; 2727c478bd9Sstevel@tonic-gate 2737c478bd9Sstevel@tonic-gate mutex_enter(&segkp_lock); 2747c478bd9Sstevel@tonic-gate for (i = 0; i < SEGKP_MAX_CACHE; i++) { 2757c478bd9Sstevel@tonic-gate if (!segkp_cache[i].kpf_inuse) 2767c478bd9Sstevel@tonic-gate continue; 2777c478bd9Sstevel@tonic-gate /* 2787c478bd9Sstevel@tonic-gate * Disconnect the freelist and process each element 2797c478bd9Sstevel@tonic-gate */ 2807c478bd9Sstevel@tonic-gate kpd = segkp_cache[i].kpf_list; 2817c478bd9Sstevel@tonic-gate seg = segkp_cache[i].kpf_seg; 2827c478bd9Sstevel@tonic-gate segkp_cache[i].kpf_list = NULL; 2837c478bd9Sstevel@tonic-gate segkp_cache[i].kpf_count = 0; 2847c478bd9Sstevel@tonic-gate mutex_exit(&segkp_lock); 2857c478bd9Sstevel@tonic-gate 2867c478bd9Sstevel@tonic-gate while (kpd != NULL) { 2877c478bd9Sstevel@tonic-gate struct segkp_data *next; 2887c478bd9Sstevel@tonic-gate 2897c478bd9Sstevel@tonic-gate next = kpd->kp_next; 2907c478bd9Sstevel@tonic-gate segkp_release_internal(seg, kpd, kpd->kp_len); 2917c478bd9Sstevel@tonic-gate kpd = next; 2927c478bd9Sstevel@tonic-gate } 2937c478bd9Sstevel@tonic-gate mutex_enter(&segkp_lock); 2947c478bd9Sstevel@tonic-gate } 2957c478bd9Sstevel@tonic-gate mutex_exit(&segkp_lock); 2967c478bd9Sstevel@tonic-gate } 2977c478bd9Sstevel@tonic-gate 2987c478bd9Sstevel@tonic-gate /* 2997c478bd9Sstevel@tonic-gate * There are 2 entries into segkp_get_internal. The first includes a cookie 3007c478bd9Sstevel@tonic-gate * used to access a pool of cached segkp resources. The second does not 3017c478bd9Sstevel@tonic-gate * use the cache. 3027c478bd9Sstevel@tonic-gate */ 3037c478bd9Sstevel@tonic-gate caddr_t 3047c478bd9Sstevel@tonic-gate segkp_get(struct seg *seg, size_t len, uint_t flags) 3057c478bd9Sstevel@tonic-gate { 3067c478bd9Sstevel@tonic-gate struct segkp_data *kpd = NULL; 3077c478bd9Sstevel@tonic-gate 3087c478bd9Sstevel@tonic-gate if (segkp_get_internal(seg, len, flags, &kpd, NULL) != NULL) { 3097c478bd9Sstevel@tonic-gate kpd->kp_cookie = -1; 3107c478bd9Sstevel@tonic-gate return (stom(kpd->kp_base, flags)); 3117c478bd9Sstevel@tonic-gate } 3127c478bd9Sstevel@tonic-gate return (NULL); 3137c478bd9Sstevel@tonic-gate } 3147c478bd9Sstevel@tonic-gate 3157c478bd9Sstevel@tonic-gate /* 3167c478bd9Sstevel@tonic-gate * Return a 'cached' segkp address 3177c478bd9Sstevel@tonic-gate */ 3187c478bd9Sstevel@tonic-gate caddr_t 3197c478bd9Sstevel@tonic-gate segkp_cache_get(void *cookie) 3207c478bd9Sstevel@tonic-gate { 3217c478bd9Sstevel@tonic-gate struct segkp_cache *freelist = NULL; 3227c478bd9Sstevel@tonic-gate struct segkp_data *kpd = NULL; 3237c478bd9Sstevel@tonic-gate int index = (int)(uintptr_t)cookie; 3247c478bd9Sstevel@tonic-gate struct seg *seg; 3257c478bd9Sstevel@tonic-gate size_t len; 3267c478bd9Sstevel@tonic-gate uint_t flags; 3277c478bd9Sstevel@tonic-gate 3287c478bd9Sstevel@tonic-gate if (index < 0 || index >= SEGKP_MAX_CACHE) 3297c478bd9Sstevel@tonic-gate return (NULL); 3307c478bd9Sstevel@tonic-gate freelist = &segkp_cache[index]; 3317c478bd9Sstevel@tonic-gate 3327c478bd9Sstevel@tonic-gate mutex_enter(&segkp_lock); 3337c478bd9Sstevel@tonic-gate seg = freelist->kpf_seg; 3347c478bd9Sstevel@tonic-gate flags = freelist->kpf_flags; 3357c478bd9Sstevel@tonic-gate if (freelist->kpf_list != NULL) { 3367c478bd9Sstevel@tonic-gate kpd = freelist->kpf_list; 3377c478bd9Sstevel@tonic-gate freelist->kpf_list = kpd->kp_next; 3387c478bd9Sstevel@tonic-gate freelist->kpf_count--; 3397c478bd9Sstevel@tonic-gate mutex_exit(&segkp_lock); 3407c478bd9Sstevel@tonic-gate kpd->kp_next = NULL; 3417c478bd9Sstevel@tonic-gate segkp_insert(seg, kpd); 3427c478bd9Sstevel@tonic-gate return (stom(kpd->kp_base, flags)); 3437c478bd9Sstevel@tonic-gate } 3447c478bd9Sstevel@tonic-gate len = freelist->kpf_len; 3457c478bd9Sstevel@tonic-gate mutex_exit(&segkp_lock); 3467c478bd9Sstevel@tonic-gate if (segkp_get_internal(seg, len, flags, &kpd, NULL) != NULL) { 3477c478bd9Sstevel@tonic-gate kpd->kp_cookie = index; 3487c478bd9Sstevel@tonic-gate return (stom(kpd->kp_base, flags)); 3497c478bd9Sstevel@tonic-gate } 3507c478bd9Sstevel@tonic-gate return (NULL); 3517c478bd9Sstevel@tonic-gate } 3527c478bd9Sstevel@tonic-gate 3537c478bd9Sstevel@tonic-gate caddr_t 3547c478bd9Sstevel@tonic-gate segkp_get_withanonmap( 3557c478bd9Sstevel@tonic-gate struct seg *seg, 3567c478bd9Sstevel@tonic-gate size_t len, 3577c478bd9Sstevel@tonic-gate uint_t flags, 3587c478bd9Sstevel@tonic-gate struct anon_map *amp) 3597c478bd9Sstevel@tonic-gate { 3607c478bd9Sstevel@tonic-gate struct segkp_data *kpd = NULL; 3617c478bd9Sstevel@tonic-gate 3627c478bd9Sstevel@tonic-gate ASSERT(amp != NULL); 3637c478bd9Sstevel@tonic-gate flags |= KPD_HASAMP; 3647c478bd9Sstevel@tonic-gate if (segkp_get_internal(seg, len, flags, &kpd, amp) != NULL) { 3657c478bd9Sstevel@tonic-gate kpd->kp_cookie = -1; 3667c478bd9Sstevel@tonic-gate return (stom(kpd->kp_base, flags)); 3677c478bd9Sstevel@tonic-gate } 3687c478bd9Sstevel@tonic-gate return (NULL); 3697c478bd9Sstevel@tonic-gate } 3707c478bd9Sstevel@tonic-gate 3717c478bd9Sstevel@tonic-gate /* 3727c478bd9Sstevel@tonic-gate * This does the real work of segkp allocation. 3737c478bd9Sstevel@tonic-gate * Return to client base addr. len must be page-aligned. A null value is 3747c478bd9Sstevel@tonic-gate * returned if there are no more vm resources (e.g. pages, swap). The len 3757c478bd9Sstevel@tonic-gate * and base recorded in the private data structure include the redzone 3767c478bd9Sstevel@tonic-gate * and the redzone length (if applicable). If the user requests a redzone 3777c478bd9Sstevel@tonic-gate * either the first or last page is left unmapped depending whether stacks 3787c478bd9Sstevel@tonic-gate * grow to low or high memory. 3797c478bd9Sstevel@tonic-gate * 3807c478bd9Sstevel@tonic-gate * The client may also specify a no-wait flag. If that is set then the 3817c478bd9Sstevel@tonic-gate * request will choose a non-blocking path when requesting resources. 3827c478bd9Sstevel@tonic-gate * The default is make the client wait. 3837c478bd9Sstevel@tonic-gate */ 3847c478bd9Sstevel@tonic-gate static caddr_t 3857c478bd9Sstevel@tonic-gate segkp_get_internal( 3867c478bd9Sstevel@tonic-gate struct seg *seg, 3877c478bd9Sstevel@tonic-gate size_t len, 3887c478bd9Sstevel@tonic-gate uint_t flags, 3897c478bd9Sstevel@tonic-gate struct segkp_data **tkpd, 3907c478bd9Sstevel@tonic-gate struct anon_map *amp) 3917c478bd9Sstevel@tonic-gate { 3927c478bd9Sstevel@tonic-gate struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data; 3937c478bd9Sstevel@tonic-gate struct segkp_data *kpd; 3947c478bd9Sstevel@tonic-gate caddr_t vbase = NULL; /* always first virtual, may not be mapped */ 3957c478bd9Sstevel@tonic-gate pgcnt_t np = 0; /* number of pages in the resource */ 3967c478bd9Sstevel@tonic-gate pgcnt_t segkpindex; 3977c478bd9Sstevel@tonic-gate long i; 3987c478bd9Sstevel@tonic-gate caddr_t va; 3997c478bd9Sstevel@tonic-gate pgcnt_t pages = 0; 4007c478bd9Sstevel@tonic-gate ulong_t anon_idx = 0; 4017c478bd9Sstevel@tonic-gate int kmflag = (flags & KPD_NOWAIT) ? KM_NOSLEEP : KM_SLEEP; 4027c478bd9Sstevel@tonic-gate caddr_t s_base = (segkp_fromheap) ? kvseg.s_base : seg->s_base; 4037c478bd9Sstevel@tonic-gate 4047c478bd9Sstevel@tonic-gate if (len & PAGEOFFSET) { 4057c478bd9Sstevel@tonic-gate panic("segkp_get: len is not page-aligned"); 4067c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 4077c478bd9Sstevel@tonic-gate } 4087c478bd9Sstevel@tonic-gate 4097c478bd9Sstevel@tonic-gate ASSERT(((flags & KPD_HASAMP) == 0) == (amp == NULL)); 4107c478bd9Sstevel@tonic-gate 4117c478bd9Sstevel@tonic-gate /* Only allow KPD_NO_ANON if we are going to lock it down */ 4127c478bd9Sstevel@tonic-gate if ((flags & (KPD_LOCKED|KPD_NO_ANON)) == KPD_NO_ANON) 4137c478bd9Sstevel@tonic-gate return (NULL); 4147c478bd9Sstevel@tonic-gate 4157c478bd9Sstevel@tonic-gate if ((kpd = kmem_zalloc(sizeof (struct segkp_data), kmflag)) == NULL) 4167c478bd9Sstevel@tonic-gate return (NULL); 4177c478bd9Sstevel@tonic-gate /* 4187c478bd9Sstevel@tonic-gate * Fix up the len to reflect the REDZONE if applicable 4197c478bd9Sstevel@tonic-gate */ 4207c478bd9Sstevel@tonic-gate if (flags & KPD_HASREDZONE) 4217c478bd9Sstevel@tonic-gate len += PAGESIZE; 4227c478bd9Sstevel@tonic-gate np = btop(len); 4237c478bd9Sstevel@tonic-gate 4247c478bd9Sstevel@tonic-gate vbase = vmem_alloc(SEGKP_VMEM(seg), len, kmflag | VM_BESTFIT); 4257c478bd9Sstevel@tonic-gate if (vbase == NULL) { 4267c478bd9Sstevel@tonic-gate kmem_free(kpd, sizeof (struct segkp_data)); 4277c478bd9Sstevel@tonic-gate return (NULL); 4287c478bd9Sstevel@tonic-gate } 4297c478bd9Sstevel@tonic-gate 4307c478bd9Sstevel@tonic-gate /* If locking, reserve physical memory */ 4317c478bd9Sstevel@tonic-gate if (flags & KPD_LOCKED) { 4327c478bd9Sstevel@tonic-gate pages = btop(SEGKP_MAPLEN(len, flags)); 4337c478bd9Sstevel@tonic-gate if (page_resv(pages, kmflag) == 0) { 4347c478bd9Sstevel@tonic-gate vmem_free(SEGKP_VMEM(seg), vbase, len); 4357c478bd9Sstevel@tonic-gate kmem_free(kpd, sizeof (struct segkp_data)); 4367c478bd9Sstevel@tonic-gate return (NULL); 4377c478bd9Sstevel@tonic-gate } 4387c478bd9Sstevel@tonic-gate if ((flags & KPD_NO_ANON) == 0) 4397c478bd9Sstevel@tonic-gate atomic_add_long(&anon_segkp_pages_locked, pages); 4407c478bd9Sstevel@tonic-gate } 4417c478bd9Sstevel@tonic-gate 4427c478bd9Sstevel@tonic-gate /* 4437c478bd9Sstevel@tonic-gate * Reserve sufficient swap space for this vm resource. We'll 4447c478bd9Sstevel@tonic-gate * actually allocate it in the loop below, but reserving it 4457c478bd9Sstevel@tonic-gate * here allows us to back out more gracefully than if we 4467c478bd9Sstevel@tonic-gate * had an allocation failure in the body of the loop. 4477c478bd9Sstevel@tonic-gate * 4487c478bd9Sstevel@tonic-gate * Note that we don't need swap space for the red zone page. 4497c478bd9Sstevel@tonic-gate */ 4507c478bd9Sstevel@tonic-gate if (amp != NULL) { 4510209230bSgjelinek /* 4520209230bSgjelinek * The swap reservation has been done, if required, and the 4530209230bSgjelinek * anon_hdr is separate. 4540209230bSgjelinek */ 4557c478bd9Sstevel@tonic-gate anon_idx = 0; 4567c478bd9Sstevel@tonic-gate kpd->kp_anon_idx = anon_idx; 4577c478bd9Sstevel@tonic-gate kpd->kp_anon = amp->ahp; 4587c478bd9Sstevel@tonic-gate 4597c478bd9Sstevel@tonic-gate TRACE_5(TR_FAC_VM, TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u", 4607c478bd9Sstevel@tonic-gate kpd, vbase, len, flags, 1); 4617c478bd9Sstevel@tonic-gate 4627c478bd9Sstevel@tonic-gate } else if ((flags & KPD_NO_ANON) == 0) { 4630209230bSgjelinek if (anon_resv_zone(SEGKP_MAPLEN(len, flags), NULL) == 0) { 4647c478bd9Sstevel@tonic-gate if (flags & KPD_LOCKED) { 4657c478bd9Sstevel@tonic-gate atomic_add_long(&anon_segkp_pages_locked, 4667c478bd9Sstevel@tonic-gate -pages); 4677c478bd9Sstevel@tonic-gate page_unresv(pages); 4687c478bd9Sstevel@tonic-gate } 4697c478bd9Sstevel@tonic-gate vmem_free(SEGKP_VMEM(seg), vbase, len); 4707c478bd9Sstevel@tonic-gate kmem_free(kpd, sizeof (struct segkp_data)); 4717c478bd9Sstevel@tonic-gate return (NULL); 4727c478bd9Sstevel@tonic-gate } 4730209230bSgjelinek atomic_add_long(&anon_segkp_pages_resv, 4740209230bSgjelinek btop(SEGKP_MAPLEN(len, flags))); 4757c478bd9Sstevel@tonic-gate anon_idx = ((uintptr_t)(vbase - s_base)) >> PAGESHIFT; 4767c478bd9Sstevel@tonic-gate kpd->kp_anon_idx = anon_idx; 4777c478bd9Sstevel@tonic-gate kpd->kp_anon = kpsd->kpsd_anon; 4787c478bd9Sstevel@tonic-gate 4797c478bd9Sstevel@tonic-gate TRACE_5(TR_FAC_VM, TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u", 4807c478bd9Sstevel@tonic-gate kpd, vbase, len, flags, 1); 4817c478bd9Sstevel@tonic-gate } else { 4827c478bd9Sstevel@tonic-gate kpd->kp_anon = NULL; 4837c478bd9Sstevel@tonic-gate kpd->kp_anon_idx = 0; 4847c478bd9Sstevel@tonic-gate } 4857c478bd9Sstevel@tonic-gate 4867c478bd9Sstevel@tonic-gate /* 4877c478bd9Sstevel@tonic-gate * Allocate page and anon resources for the virtual address range 4887c478bd9Sstevel@tonic-gate * except the redzone 4897c478bd9Sstevel@tonic-gate */ 4907c478bd9Sstevel@tonic-gate if (segkp_fromheap) 4917c478bd9Sstevel@tonic-gate segkpindex = btop((uintptr_t)(vbase - kvseg.s_base)); 4927c478bd9Sstevel@tonic-gate for (i = 0, va = vbase; i < np; i++, va += PAGESIZE) { 4937c478bd9Sstevel@tonic-gate page_t *pl[2]; 4947c478bd9Sstevel@tonic-gate struct vnode *vp; 4957c478bd9Sstevel@tonic-gate anoff_t off; 4967c478bd9Sstevel@tonic-gate int err; 4977c478bd9Sstevel@tonic-gate page_t *pp = NULL; 4987c478bd9Sstevel@tonic-gate 4997c478bd9Sstevel@tonic-gate /* 5007c478bd9Sstevel@tonic-gate * Mark this page to be a segkp page in the bitmap. 5017c478bd9Sstevel@tonic-gate */ 5027c478bd9Sstevel@tonic-gate if (segkp_fromheap) { 5037c478bd9Sstevel@tonic-gate BT_ATOMIC_SET(segkp_bitmap, segkpindex); 5047c478bd9Sstevel@tonic-gate segkpindex++; 5057c478bd9Sstevel@tonic-gate } 5067c478bd9Sstevel@tonic-gate 5077c478bd9Sstevel@tonic-gate /* 5087c478bd9Sstevel@tonic-gate * If this page is the red zone page, we don't need swap 5097c478bd9Sstevel@tonic-gate * space for it. Note that we skip over the code that 5107c478bd9Sstevel@tonic-gate * establishes MMU mappings, so that the page remains 5117c478bd9Sstevel@tonic-gate * invalid. 5127c478bd9Sstevel@tonic-gate */ 5137c478bd9Sstevel@tonic-gate if ((flags & KPD_HASREDZONE) && KPD_REDZONE(kpd) == i) 5147c478bd9Sstevel@tonic-gate continue; 5157c478bd9Sstevel@tonic-gate 5167c478bd9Sstevel@tonic-gate if (kpd->kp_anon != NULL) { 5177c478bd9Sstevel@tonic-gate struct anon *ap; 5187c478bd9Sstevel@tonic-gate 5197c478bd9Sstevel@tonic-gate ASSERT(anon_get_ptr(kpd->kp_anon, anon_idx + i) 5207c478bd9Sstevel@tonic-gate == NULL); 5217c478bd9Sstevel@tonic-gate /* 5227c478bd9Sstevel@tonic-gate * Determine the "vp" and "off" of the anon slot. 5237c478bd9Sstevel@tonic-gate */ 5247c478bd9Sstevel@tonic-gate ap = anon_alloc(NULL, 0); 5257c478bd9Sstevel@tonic-gate if (amp != NULL) 5267c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 5277c478bd9Sstevel@tonic-gate (void) anon_set_ptr(kpd->kp_anon, anon_idx + i, 5287c478bd9Sstevel@tonic-gate ap, ANON_SLEEP); 5297c478bd9Sstevel@tonic-gate if (amp != NULL) 5307c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 5317c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &off); 5327c478bd9Sstevel@tonic-gate 5337c478bd9Sstevel@tonic-gate /* 5347c478bd9Sstevel@tonic-gate * Create a page with the specified identity. The 5357c478bd9Sstevel@tonic-gate * page is returned with the "shared" lock held. 5367c478bd9Sstevel@tonic-gate */ 5377c478bd9Sstevel@tonic-gate err = VOP_GETPAGE(vp, (offset_t)off, PAGESIZE, 5387c478bd9Sstevel@tonic-gate NULL, pl, PAGESIZE, seg, va, S_CREATE, 539*da6c28aaSamw kcred, NULL); 5407c478bd9Sstevel@tonic-gate if (err) { 5417c478bd9Sstevel@tonic-gate /* 5427c478bd9Sstevel@tonic-gate * XXX - This should not fail. 5437c478bd9Sstevel@tonic-gate */ 5447c478bd9Sstevel@tonic-gate panic("segkp_get: no pages"); 5457c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 5467c478bd9Sstevel@tonic-gate } 5477c478bd9Sstevel@tonic-gate pp = pl[0]; 5487c478bd9Sstevel@tonic-gate } else { 5497c478bd9Sstevel@tonic-gate ASSERT(page_exists(&kvp, 5507c478bd9Sstevel@tonic-gate (u_offset_t)(uintptr_t)va) == NULL); 5517c478bd9Sstevel@tonic-gate 5527c478bd9Sstevel@tonic-gate if ((pp = page_create_va(&kvp, 5537c478bd9Sstevel@tonic-gate (u_offset_t)(uintptr_t)va, PAGESIZE, 5547c478bd9Sstevel@tonic-gate (flags & KPD_NOWAIT ? 0 : PG_WAIT) | PG_EXCL | 5557c478bd9Sstevel@tonic-gate PG_NORELOC, seg, va)) == NULL) { 5567c478bd9Sstevel@tonic-gate /* 5577c478bd9Sstevel@tonic-gate * Legitimize resource; then destroy it. 5587c478bd9Sstevel@tonic-gate * Easier than trying to unwind here. 5597c478bd9Sstevel@tonic-gate */ 5607c478bd9Sstevel@tonic-gate kpd->kp_flags = flags; 5617c478bd9Sstevel@tonic-gate kpd->kp_base = vbase; 5627c478bd9Sstevel@tonic-gate kpd->kp_len = len; 5637c478bd9Sstevel@tonic-gate segkp_release_internal(seg, kpd, va - vbase); 5647c478bd9Sstevel@tonic-gate return (NULL); 5657c478bd9Sstevel@tonic-gate } 5667c478bd9Sstevel@tonic-gate page_io_unlock(pp); 5677c478bd9Sstevel@tonic-gate } 5687c478bd9Sstevel@tonic-gate 5697c478bd9Sstevel@tonic-gate if (flags & KPD_ZERO) 5707c478bd9Sstevel@tonic-gate pagezero(pp, 0, PAGESIZE); 5717c478bd9Sstevel@tonic-gate 5727c478bd9Sstevel@tonic-gate /* 5737c478bd9Sstevel@tonic-gate * Load and lock an MMU translation for the page. 5747c478bd9Sstevel@tonic-gate */ 5757c478bd9Sstevel@tonic-gate hat_memload(seg->s_as->a_hat, va, pp, (PROT_READ|PROT_WRITE), 5767c478bd9Sstevel@tonic-gate ((flags & KPD_LOCKED) ? HAT_LOAD_LOCK : HAT_LOAD)); 5777c478bd9Sstevel@tonic-gate 5787c478bd9Sstevel@tonic-gate /* 5797c478bd9Sstevel@tonic-gate * Now, release lock on the page. 5807c478bd9Sstevel@tonic-gate */ 5817c478bd9Sstevel@tonic-gate if (flags & KPD_LOCKED) 5827c478bd9Sstevel@tonic-gate page_downgrade(pp); 5837c478bd9Sstevel@tonic-gate else 5847c478bd9Sstevel@tonic-gate page_unlock(pp); 5857c478bd9Sstevel@tonic-gate } 5867c478bd9Sstevel@tonic-gate 5877c478bd9Sstevel@tonic-gate kpd->kp_flags = flags; 5887c478bd9Sstevel@tonic-gate kpd->kp_base = vbase; 5897c478bd9Sstevel@tonic-gate kpd->kp_len = len; 5907c478bd9Sstevel@tonic-gate segkp_insert(seg, kpd); 5917c478bd9Sstevel@tonic-gate *tkpd = kpd; 5927c478bd9Sstevel@tonic-gate return (stom(kpd->kp_base, flags)); 5937c478bd9Sstevel@tonic-gate } 5947c478bd9Sstevel@tonic-gate 5957c478bd9Sstevel@tonic-gate /* 5967c478bd9Sstevel@tonic-gate * Release the resource to cache if the pool(designate by the cookie) 5977c478bd9Sstevel@tonic-gate * has less than the maximum allowable. If inserted in cache, 5987c478bd9Sstevel@tonic-gate * segkp_delete insures element is taken off of active list. 5997c478bd9Sstevel@tonic-gate */ 6007c478bd9Sstevel@tonic-gate void 6017c478bd9Sstevel@tonic-gate segkp_release(struct seg *seg, caddr_t vaddr) 6027c478bd9Sstevel@tonic-gate { 6037c478bd9Sstevel@tonic-gate struct segkp_cache *freelist; 6047c478bd9Sstevel@tonic-gate struct segkp_data *kpd = NULL; 6057c478bd9Sstevel@tonic-gate 6067c478bd9Sstevel@tonic-gate if ((kpd = segkp_find(seg, vaddr)) == NULL) { 6077c478bd9Sstevel@tonic-gate panic("segkp_release: null kpd"); 6087c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 6097c478bd9Sstevel@tonic-gate } 6107c478bd9Sstevel@tonic-gate 6117c478bd9Sstevel@tonic-gate if (kpd->kp_cookie != -1) { 6127c478bd9Sstevel@tonic-gate freelist = &segkp_cache[kpd->kp_cookie]; 6137c478bd9Sstevel@tonic-gate mutex_enter(&segkp_lock); 6147c478bd9Sstevel@tonic-gate if (!segkp_indel && freelist->kpf_count < freelist->kpf_max) { 6157c478bd9Sstevel@tonic-gate segkp_delete(seg, kpd); 6167c478bd9Sstevel@tonic-gate kpd->kp_next = freelist->kpf_list; 6177c478bd9Sstevel@tonic-gate freelist->kpf_list = kpd; 6187c478bd9Sstevel@tonic-gate freelist->kpf_count++; 6197c478bd9Sstevel@tonic-gate mutex_exit(&segkp_lock); 6207c478bd9Sstevel@tonic-gate return; 6217c478bd9Sstevel@tonic-gate } else { 6227c478bd9Sstevel@tonic-gate mutex_exit(&segkp_lock); 6237c478bd9Sstevel@tonic-gate kpd->kp_cookie = -1; 6247c478bd9Sstevel@tonic-gate } 6257c478bd9Sstevel@tonic-gate } 6267c478bd9Sstevel@tonic-gate segkp_release_internal(seg, kpd, kpd->kp_len); 6277c478bd9Sstevel@tonic-gate } 6287c478bd9Sstevel@tonic-gate 6297c478bd9Sstevel@tonic-gate /* 6307c478bd9Sstevel@tonic-gate * Free the entire resource. segkp_unlock gets called with the start of the 6317c478bd9Sstevel@tonic-gate * mapped portion of the resource. The length is the size of the mapped 6327c478bd9Sstevel@tonic-gate * portion 6337c478bd9Sstevel@tonic-gate */ 6347c478bd9Sstevel@tonic-gate static void 6357c478bd9Sstevel@tonic-gate segkp_release_internal(struct seg *seg, struct segkp_data *kpd, size_t len) 6367c478bd9Sstevel@tonic-gate { 6377c478bd9Sstevel@tonic-gate caddr_t va; 6387c478bd9Sstevel@tonic-gate long i; 6397c478bd9Sstevel@tonic-gate long redzone; 6407c478bd9Sstevel@tonic-gate size_t np; 6417c478bd9Sstevel@tonic-gate page_t *pp; 6427c478bd9Sstevel@tonic-gate struct vnode *vp; 6437c478bd9Sstevel@tonic-gate anoff_t off; 6447c478bd9Sstevel@tonic-gate struct anon *ap; 6457c478bd9Sstevel@tonic-gate pgcnt_t segkpindex; 6467c478bd9Sstevel@tonic-gate 6477c478bd9Sstevel@tonic-gate ASSERT(kpd != NULL); 6487c478bd9Sstevel@tonic-gate ASSERT((kpd->kp_flags & KPD_HASAMP) == 0 || kpd->kp_cookie == -1); 6497c478bd9Sstevel@tonic-gate np = btop(len); 6507c478bd9Sstevel@tonic-gate 6517c478bd9Sstevel@tonic-gate /* Remove from active hash list */ 6527c478bd9Sstevel@tonic-gate if (kpd->kp_cookie == -1) { 6537c478bd9Sstevel@tonic-gate mutex_enter(&segkp_lock); 6547c478bd9Sstevel@tonic-gate segkp_delete(seg, kpd); 6557c478bd9Sstevel@tonic-gate mutex_exit(&segkp_lock); 6567c478bd9Sstevel@tonic-gate } 6577c478bd9Sstevel@tonic-gate 6587c478bd9Sstevel@tonic-gate /* 6597c478bd9Sstevel@tonic-gate * Precompute redzone page index. 6607c478bd9Sstevel@tonic-gate */ 6617c478bd9Sstevel@tonic-gate redzone = -1; 6627c478bd9Sstevel@tonic-gate if (kpd->kp_flags & KPD_HASREDZONE) 6637c478bd9Sstevel@tonic-gate redzone = KPD_REDZONE(kpd); 6647c478bd9Sstevel@tonic-gate 6657c478bd9Sstevel@tonic-gate 6667c478bd9Sstevel@tonic-gate va = kpd->kp_base; 6677c478bd9Sstevel@tonic-gate 6687c478bd9Sstevel@tonic-gate hat_unload(seg->s_as->a_hat, va, (np << PAGESHIFT), 6697c478bd9Sstevel@tonic-gate ((kpd->kp_flags & KPD_LOCKED) ? HAT_UNLOAD_UNLOCK : HAT_UNLOAD)); 6707c478bd9Sstevel@tonic-gate /* 6717c478bd9Sstevel@tonic-gate * Free up those anon resources that are quiescent. 6727c478bd9Sstevel@tonic-gate */ 6737c478bd9Sstevel@tonic-gate if (segkp_fromheap) 6747c478bd9Sstevel@tonic-gate segkpindex = btop((uintptr_t)(va - kvseg.s_base)); 6757c478bd9Sstevel@tonic-gate for (i = 0; i < np; i++, va += PAGESIZE) { 6767c478bd9Sstevel@tonic-gate 6777c478bd9Sstevel@tonic-gate /* 6787c478bd9Sstevel@tonic-gate * Clear the bit for this page from the bitmap. 6797c478bd9Sstevel@tonic-gate */ 6807c478bd9Sstevel@tonic-gate if (segkp_fromheap) { 6817c478bd9Sstevel@tonic-gate BT_ATOMIC_CLEAR(segkp_bitmap, segkpindex); 6827c478bd9Sstevel@tonic-gate segkpindex++; 6837c478bd9Sstevel@tonic-gate } 6847c478bd9Sstevel@tonic-gate 6857c478bd9Sstevel@tonic-gate if (i == redzone) 6867c478bd9Sstevel@tonic-gate continue; 6877c478bd9Sstevel@tonic-gate if (kpd->kp_anon) { 6887c478bd9Sstevel@tonic-gate /* 6897c478bd9Sstevel@tonic-gate * Free up anon resources and destroy the 6907c478bd9Sstevel@tonic-gate * associated pages. 6917c478bd9Sstevel@tonic-gate * 6927c478bd9Sstevel@tonic-gate * Release the lock if there is one. Have to get the 6937c478bd9Sstevel@tonic-gate * page to do this, unfortunately. 6947c478bd9Sstevel@tonic-gate */ 6957c478bd9Sstevel@tonic-gate if (kpd->kp_flags & KPD_LOCKED) { 6967c478bd9Sstevel@tonic-gate ap = anon_get_ptr(kpd->kp_anon, 6977c478bd9Sstevel@tonic-gate kpd->kp_anon_idx + i); 6987c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &off); 6997c478bd9Sstevel@tonic-gate /* Find the shared-locked page. */ 7007c478bd9Sstevel@tonic-gate pp = page_find(vp, (u_offset_t)off); 7017c478bd9Sstevel@tonic-gate if (pp == NULL) { 7027c478bd9Sstevel@tonic-gate panic("segkp_release: " 7037c478bd9Sstevel@tonic-gate "kp_anon: no page to unlock "); 7047c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 7057c478bd9Sstevel@tonic-gate } 7067c478bd9Sstevel@tonic-gate page_unlock(pp); 7077c478bd9Sstevel@tonic-gate } 7087c478bd9Sstevel@tonic-gate if ((kpd->kp_flags & KPD_HASAMP) == 0) { 7097c478bd9Sstevel@tonic-gate anon_free(kpd->kp_anon, kpd->kp_anon_idx + i, 7107c478bd9Sstevel@tonic-gate PAGESIZE); 7110209230bSgjelinek anon_unresv_zone(PAGESIZE, NULL); 7120209230bSgjelinek atomic_add_long(&anon_segkp_pages_resv, 7130209230bSgjelinek -1); 7147c478bd9Sstevel@tonic-gate } 7157c478bd9Sstevel@tonic-gate TRACE_5(TR_FAC_VM, 7167c478bd9Sstevel@tonic-gate TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u", 7177c478bd9Sstevel@tonic-gate kpd, va, PAGESIZE, 0, 0); 7187c478bd9Sstevel@tonic-gate } else { 7197c478bd9Sstevel@tonic-gate if (kpd->kp_flags & KPD_LOCKED) { 7207c478bd9Sstevel@tonic-gate pp = page_find(&kvp, (u_offset_t)(uintptr_t)va); 7217c478bd9Sstevel@tonic-gate if (pp == NULL) { 7227c478bd9Sstevel@tonic-gate panic("segkp_release: " 7237c478bd9Sstevel@tonic-gate "no page to unlock"); 7247c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 7257c478bd9Sstevel@tonic-gate } 7267c478bd9Sstevel@tonic-gate /* 7277c478bd9Sstevel@tonic-gate * We should just upgrade the lock here 7287c478bd9Sstevel@tonic-gate * but there is no upgrade that waits. 7297c478bd9Sstevel@tonic-gate */ 7307c478bd9Sstevel@tonic-gate page_unlock(pp); 7317c478bd9Sstevel@tonic-gate } 7327c478bd9Sstevel@tonic-gate pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)va, 7337c478bd9Sstevel@tonic-gate SE_EXCL); 7347c478bd9Sstevel@tonic-gate if (pp != NULL) 7357c478bd9Sstevel@tonic-gate page_destroy(pp, 0); 7367c478bd9Sstevel@tonic-gate } 7377c478bd9Sstevel@tonic-gate } 7387c478bd9Sstevel@tonic-gate 7397c478bd9Sstevel@tonic-gate /* If locked, release physical memory reservation */ 7407c478bd9Sstevel@tonic-gate if (kpd->kp_flags & KPD_LOCKED) { 7417c478bd9Sstevel@tonic-gate pgcnt_t pages = btop(SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)); 7427c478bd9Sstevel@tonic-gate if ((kpd->kp_flags & KPD_NO_ANON) == 0) 7437c478bd9Sstevel@tonic-gate atomic_add_long(&anon_segkp_pages_locked, -pages); 7447c478bd9Sstevel@tonic-gate page_unresv(pages); 7457c478bd9Sstevel@tonic-gate } 7467c478bd9Sstevel@tonic-gate 7477c478bd9Sstevel@tonic-gate vmem_free(SEGKP_VMEM(seg), kpd->kp_base, kpd->kp_len); 7487c478bd9Sstevel@tonic-gate kmem_free(kpd, sizeof (struct segkp_data)); 7497c478bd9Sstevel@tonic-gate } 7507c478bd9Sstevel@tonic-gate 7517c478bd9Sstevel@tonic-gate /* 7527c478bd9Sstevel@tonic-gate * segkp_map_red() will check the current frame pointer against the 7537c478bd9Sstevel@tonic-gate * stack base. If the amount of stack remaining is questionable 7547c478bd9Sstevel@tonic-gate * (less than red_minavail), then segkp_map_red() will map in the redzone 7557c478bd9Sstevel@tonic-gate * and return 1. Otherwise, it will return 0. segkp_map_red() can 7567c478bd9Sstevel@tonic-gate * _only_ be called when: 7577c478bd9Sstevel@tonic-gate * 7587c478bd9Sstevel@tonic-gate * - it is safe to sleep on page_create_va(). 7597c478bd9Sstevel@tonic-gate * - the caller is non-swappable. 7607c478bd9Sstevel@tonic-gate * 7617c478bd9Sstevel@tonic-gate * It is up to the caller to remember whether segkp_map_red() successfully 7627c478bd9Sstevel@tonic-gate * mapped the redzone, and, if so, to call segkp_unmap_red() at a later 7637c478bd9Sstevel@tonic-gate * time. Note that the caller must _remain_ non-swappable until after 7647c478bd9Sstevel@tonic-gate * calling segkp_unmap_red(). 7657c478bd9Sstevel@tonic-gate * 7667c478bd9Sstevel@tonic-gate * Currently, this routine is only called from pagefault() (which necessarily 7677c478bd9Sstevel@tonic-gate * satisfies the above conditions). 7687c478bd9Sstevel@tonic-gate */ 7697c478bd9Sstevel@tonic-gate #if defined(STACK_GROWTH_DOWN) 7707c478bd9Sstevel@tonic-gate int 7717c478bd9Sstevel@tonic-gate segkp_map_red(void) 7727c478bd9Sstevel@tonic-gate { 7737c478bd9Sstevel@tonic-gate uintptr_t fp = STACK_BIAS + (uintptr_t)getfp(); 7747c478bd9Sstevel@tonic-gate #ifndef _LP64 7757c478bd9Sstevel@tonic-gate caddr_t stkbase; 7767c478bd9Sstevel@tonic-gate #endif 7777c478bd9Sstevel@tonic-gate 7787c478bd9Sstevel@tonic-gate ASSERT(curthread->t_schedflag & TS_DONT_SWAP); 7797c478bd9Sstevel@tonic-gate 7807c478bd9Sstevel@tonic-gate /* 7817c478bd9Sstevel@tonic-gate * Optimize for the common case where we simply return. 7827c478bd9Sstevel@tonic-gate */ 7837c478bd9Sstevel@tonic-gate if ((curthread->t_red_pp == NULL) && 7847c478bd9Sstevel@tonic-gate (fp - (uintptr_t)curthread->t_stkbase >= red_minavail)) 7857c478bd9Sstevel@tonic-gate return (0); 7867c478bd9Sstevel@tonic-gate 7877c478bd9Sstevel@tonic-gate #if defined(_LP64) 7887c478bd9Sstevel@tonic-gate /* 7897c478bd9Sstevel@tonic-gate * XXX We probably need something better than this. 7907c478bd9Sstevel@tonic-gate */ 7917c478bd9Sstevel@tonic-gate panic("kernel stack overflow"); 7927c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 7937c478bd9Sstevel@tonic-gate #else /* _LP64 */ 7947c478bd9Sstevel@tonic-gate if (curthread->t_red_pp == NULL) { 7957c478bd9Sstevel@tonic-gate page_t *red_pp; 7967c478bd9Sstevel@tonic-gate struct seg kseg; 7977c478bd9Sstevel@tonic-gate 7987c478bd9Sstevel@tonic-gate caddr_t red_va = (caddr_t) 7997c478bd9Sstevel@tonic-gate (((uintptr_t)curthread->t_stkbase & (uintptr_t)PAGEMASK) - 8007c478bd9Sstevel@tonic-gate PAGESIZE); 8017c478bd9Sstevel@tonic-gate 8027c478bd9Sstevel@tonic-gate ASSERT(page_exists(&kvp, (u_offset_t)(uintptr_t)red_va) == 8037c478bd9Sstevel@tonic-gate NULL); 8047c478bd9Sstevel@tonic-gate 8057c478bd9Sstevel@tonic-gate /* 8067c478bd9Sstevel@tonic-gate * Allocate the physical for the red page. 8077c478bd9Sstevel@tonic-gate */ 8087c478bd9Sstevel@tonic-gate /* 8097c478bd9Sstevel@tonic-gate * No PG_NORELOC here to avoid waits. Unlikely to get 8107c478bd9Sstevel@tonic-gate * a relocate happening in the short time the page exists 8117c478bd9Sstevel@tonic-gate * and it will be OK anyway. 8127c478bd9Sstevel@tonic-gate */ 8137c478bd9Sstevel@tonic-gate 8147c478bd9Sstevel@tonic-gate kseg.s_as = &kas; 8157c478bd9Sstevel@tonic-gate red_pp = page_create_va(&kvp, (u_offset_t)(uintptr_t)red_va, 8167c478bd9Sstevel@tonic-gate PAGESIZE, PG_WAIT | PG_EXCL, &kseg, red_va); 8177c478bd9Sstevel@tonic-gate ASSERT(red_pp != NULL); 8187c478bd9Sstevel@tonic-gate 8197c478bd9Sstevel@tonic-gate /* 8207c478bd9Sstevel@tonic-gate * So we now have a page to jam into the redzone... 8217c478bd9Sstevel@tonic-gate */ 8227c478bd9Sstevel@tonic-gate page_io_unlock(red_pp); 8237c478bd9Sstevel@tonic-gate 8247c478bd9Sstevel@tonic-gate hat_memload(kas.a_hat, red_va, red_pp, 8257c478bd9Sstevel@tonic-gate (PROT_READ|PROT_WRITE), HAT_LOAD_LOCK); 8267c478bd9Sstevel@tonic-gate page_downgrade(red_pp); 8277c478bd9Sstevel@tonic-gate 8287c478bd9Sstevel@tonic-gate /* 8297c478bd9Sstevel@tonic-gate * The page is left SE_SHARED locked so we can hold on to 8307c478bd9Sstevel@tonic-gate * the page_t pointer. 8317c478bd9Sstevel@tonic-gate */ 8327c478bd9Sstevel@tonic-gate curthread->t_red_pp = red_pp; 8337c478bd9Sstevel@tonic-gate 8347c478bd9Sstevel@tonic-gate atomic_add_32(&red_nmapped, 1); 8357c478bd9Sstevel@tonic-gate while (fp - (uintptr_t)curthread->t_stkbase < red_closest) { 8367c478bd9Sstevel@tonic-gate (void) cas32(&red_closest, red_closest, 8377c478bd9Sstevel@tonic-gate (uint32_t)(fp - (uintptr_t)curthread->t_stkbase)); 8387c478bd9Sstevel@tonic-gate } 8397c478bd9Sstevel@tonic-gate return (1); 8407c478bd9Sstevel@tonic-gate } 8417c478bd9Sstevel@tonic-gate 8427c478bd9Sstevel@tonic-gate stkbase = (caddr_t)(((uintptr_t)curthread->t_stkbase & 8437c478bd9Sstevel@tonic-gate (uintptr_t)PAGEMASK) - PAGESIZE); 8447c478bd9Sstevel@tonic-gate 8457c478bd9Sstevel@tonic-gate atomic_add_32(&red_ndoubles, 1); 8467c478bd9Sstevel@tonic-gate 8477c478bd9Sstevel@tonic-gate if (fp - (uintptr_t)stkbase < RED_DEEP_THRESHOLD) { 8487c478bd9Sstevel@tonic-gate /* 8497c478bd9Sstevel@tonic-gate * Oh boy. We're already deep within the mapped-in 8507c478bd9Sstevel@tonic-gate * redzone page, and the caller is trying to prepare 8517c478bd9Sstevel@tonic-gate * for a deep stack run. We're running without a 8527c478bd9Sstevel@tonic-gate * redzone right now: if the caller plows off the 8537c478bd9Sstevel@tonic-gate * end of the stack, it'll plow another thread or 8547c478bd9Sstevel@tonic-gate * LWP structure. That situation could result in 8557c478bd9Sstevel@tonic-gate * a very hard-to-debug panic, so, in the spirit of 8567c478bd9Sstevel@tonic-gate * recording the name of one's killer in one's own 8577c478bd9Sstevel@tonic-gate * blood, we're going to record lbolt and the calling 8587c478bd9Sstevel@tonic-gate * thread. 8597c478bd9Sstevel@tonic-gate */ 8607c478bd9Sstevel@tonic-gate red_deep_lbolt = lbolt; 8617c478bd9Sstevel@tonic-gate red_deep_thread = curthread; 8627c478bd9Sstevel@tonic-gate } 8637c478bd9Sstevel@tonic-gate 8647c478bd9Sstevel@tonic-gate /* 8657c478bd9Sstevel@tonic-gate * If this is a DEBUG kernel, and we've run too deep for comfort, toss. 8667c478bd9Sstevel@tonic-gate */ 8677c478bd9Sstevel@tonic-gate ASSERT(fp - (uintptr_t)stkbase >= RED_DEEP_THRESHOLD); 8687c478bd9Sstevel@tonic-gate return (0); 8697c478bd9Sstevel@tonic-gate #endif /* _LP64 */ 8707c478bd9Sstevel@tonic-gate } 8717c478bd9Sstevel@tonic-gate 8727c478bd9Sstevel@tonic-gate void 8737c478bd9Sstevel@tonic-gate segkp_unmap_red(void) 8747c478bd9Sstevel@tonic-gate { 8757c478bd9Sstevel@tonic-gate page_t *pp; 8767c478bd9Sstevel@tonic-gate caddr_t red_va = (caddr_t)(((uintptr_t)curthread->t_stkbase & 8777c478bd9Sstevel@tonic-gate (uintptr_t)PAGEMASK) - PAGESIZE); 8787c478bd9Sstevel@tonic-gate 8797c478bd9Sstevel@tonic-gate ASSERT(curthread->t_red_pp != NULL); 8807c478bd9Sstevel@tonic-gate ASSERT(curthread->t_schedflag & TS_DONT_SWAP); 8817c478bd9Sstevel@tonic-gate 8827c478bd9Sstevel@tonic-gate /* 8837c478bd9Sstevel@tonic-gate * Because we locked the mapping down, we can't simply rely 8847c478bd9Sstevel@tonic-gate * on page_destroy() to clean everything up; we need to call 8857c478bd9Sstevel@tonic-gate * hat_unload() to explicitly unlock the mapping resources. 8867c478bd9Sstevel@tonic-gate */ 8877c478bd9Sstevel@tonic-gate hat_unload(kas.a_hat, red_va, PAGESIZE, HAT_UNLOAD_UNLOCK); 8887c478bd9Sstevel@tonic-gate 8897c478bd9Sstevel@tonic-gate pp = curthread->t_red_pp; 8907c478bd9Sstevel@tonic-gate 8917c478bd9Sstevel@tonic-gate ASSERT(pp == page_find(&kvp, (u_offset_t)(uintptr_t)red_va)); 8927c478bd9Sstevel@tonic-gate 8937c478bd9Sstevel@tonic-gate /* 8947c478bd9Sstevel@tonic-gate * Need to upgrade the SE_SHARED lock to SE_EXCL. 8957c478bd9Sstevel@tonic-gate */ 8967c478bd9Sstevel@tonic-gate if (!page_tryupgrade(pp)) { 8977c478bd9Sstevel@tonic-gate /* 8987c478bd9Sstevel@tonic-gate * As there is now wait for upgrade, release the 8997c478bd9Sstevel@tonic-gate * SE_SHARED lock and wait for SE_EXCL. 9007c478bd9Sstevel@tonic-gate */ 9017c478bd9Sstevel@tonic-gate page_unlock(pp); 9027c478bd9Sstevel@tonic-gate pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)red_va, SE_EXCL); 9037c478bd9Sstevel@tonic-gate /* pp may be NULL here, hence the test below */ 9047c478bd9Sstevel@tonic-gate } 9057c478bd9Sstevel@tonic-gate 9067c478bd9Sstevel@tonic-gate /* 9077c478bd9Sstevel@tonic-gate * Destroy the page, with dontfree set to zero (i.e. free it). 9087c478bd9Sstevel@tonic-gate */ 9097c478bd9Sstevel@tonic-gate if (pp != NULL) 9107c478bd9Sstevel@tonic-gate page_destroy(pp, 0); 9117c478bd9Sstevel@tonic-gate curthread->t_red_pp = NULL; 9127c478bd9Sstevel@tonic-gate } 9137c478bd9Sstevel@tonic-gate #else 9147c478bd9Sstevel@tonic-gate #error Red stacks only supported with downwards stack growth. 9157c478bd9Sstevel@tonic-gate #endif 9167c478bd9Sstevel@tonic-gate 9177c478bd9Sstevel@tonic-gate /* 9187c478bd9Sstevel@tonic-gate * Handle a fault on an address corresponding to one of the 9197c478bd9Sstevel@tonic-gate * resources in the segkp segment. 9207c478bd9Sstevel@tonic-gate */ 9217c478bd9Sstevel@tonic-gate faultcode_t 9227c478bd9Sstevel@tonic-gate segkp_fault( 9237c478bd9Sstevel@tonic-gate struct hat *hat, 9247c478bd9Sstevel@tonic-gate struct seg *seg, 9257c478bd9Sstevel@tonic-gate caddr_t vaddr, 9267c478bd9Sstevel@tonic-gate size_t len, 9277c478bd9Sstevel@tonic-gate enum fault_type type, 9287c478bd9Sstevel@tonic-gate enum seg_rw rw) 9297c478bd9Sstevel@tonic-gate { 9307c478bd9Sstevel@tonic-gate struct segkp_data *kpd = NULL; 9317c478bd9Sstevel@tonic-gate int err; 9327c478bd9Sstevel@tonic-gate 9337c478bd9Sstevel@tonic-gate ASSERT(seg->s_as == &kas && RW_READ_HELD(&seg->s_as->a_lock)); 9347c478bd9Sstevel@tonic-gate 9357c478bd9Sstevel@tonic-gate /* 9367c478bd9Sstevel@tonic-gate * Sanity checks. 9377c478bd9Sstevel@tonic-gate */ 9387c478bd9Sstevel@tonic-gate if (type == F_PROT) { 9397c478bd9Sstevel@tonic-gate panic("segkp_fault: unexpected F_PROT fault"); 9407c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 9417c478bd9Sstevel@tonic-gate } 9427c478bd9Sstevel@tonic-gate 9437c478bd9Sstevel@tonic-gate if ((kpd = segkp_find(seg, vaddr)) == NULL) 9447c478bd9Sstevel@tonic-gate return (FC_NOMAP); 9457c478bd9Sstevel@tonic-gate 9467c478bd9Sstevel@tonic-gate mutex_enter(&kpd->kp_lock); 9477c478bd9Sstevel@tonic-gate 9487c478bd9Sstevel@tonic-gate if (type == F_SOFTLOCK) { 9497c478bd9Sstevel@tonic-gate ASSERT(!(kpd->kp_flags & KPD_LOCKED)); 9507c478bd9Sstevel@tonic-gate /* 9517c478bd9Sstevel@tonic-gate * The F_SOFTLOCK case has more stringent 9527c478bd9Sstevel@tonic-gate * range requirements: the given range must exactly coincide 9537c478bd9Sstevel@tonic-gate * with the resource's mapped portion. Note reference to 9547c478bd9Sstevel@tonic-gate * redzone is handled since vaddr would not equal base 9557c478bd9Sstevel@tonic-gate */ 9567c478bd9Sstevel@tonic-gate if (vaddr != stom(kpd->kp_base, kpd->kp_flags) || 9577c478bd9Sstevel@tonic-gate len != SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)) { 9587c478bd9Sstevel@tonic-gate mutex_exit(&kpd->kp_lock); 9597c478bd9Sstevel@tonic-gate return (FC_MAKE_ERR(EFAULT)); 9607c478bd9Sstevel@tonic-gate } 9617c478bd9Sstevel@tonic-gate 9627c478bd9Sstevel@tonic-gate if ((err = segkp_load(hat, seg, vaddr, len, kpd, KPD_LOCKED))) { 9637c478bd9Sstevel@tonic-gate mutex_exit(&kpd->kp_lock); 9647c478bd9Sstevel@tonic-gate return (FC_MAKE_ERR(err)); 9657c478bd9Sstevel@tonic-gate } 9667c478bd9Sstevel@tonic-gate kpd->kp_flags |= KPD_LOCKED; 9677c478bd9Sstevel@tonic-gate mutex_exit(&kpd->kp_lock); 9687c478bd9Sstevel@tonic-gate return (0); 9697c478bd9Sstevel@tonic-gate } 9707c478bd9Sstevel@tonic-gate 9717c478bd9Sstevel@tonic-gate if (type == F_INVAL) { 9727c478bd9Sstevel@tonic-gate ASSERT(!(kpd->kp_flags & KPD_NO_ANON)); 9737c478bd9Sstevel@tonic-gate 9747c478bd9Sstevel@tonic-gate /* 9757c478bd9Sstevel@tonic-gate * Check if we touched the redzone. Somewhat optimistic 9767c478bd9Sstevel@tonic-gate * here if we are touching the redzone of our own stack 9777c478bd9Sstevel@tonic-gate * since we wouldn't have a stack to get this far... 9787c478bd9Sstevel@tonic-gate */ 9797c478bd9Sstevel@tonic-gate if ((kpd->kp_flags & KPD_HASREDZONE) && 9807c478bd9Sstevel@tonic-gate btop((uintptr_t)(vaddr - kpd->kp_base)) == KPD_REDZONE(kpd)) 9817c478bd9Sstevel@tonic-gate panic("segkp_fault: accessing redzone"); 9827c478bd9Sstevel@tonic-gate 9837c478bd9Sstevel@tonic-gate /* 9847c478bd9Sstevel@tonic-gate * This fault may occur while the page is being F_SOFTLOCK'ed. 9857c478bd9Sstevel@tonic-gate * Return since a 2nd segkp_load is unnecessary and also would 9867c478bd9Sstevel@tonic-gate * result in the page being locked twice and eventually 9877c478bd9Sstevel@tonic-gate * hang the thread_reaper thread. 9887c478bd9Sstevel@tonic-gate */ 9897c478bd9Sstevel@tonic-gate if (kpd->kp_flags & KPD_LOCKED) { 9907c478bd9Sstevel@tonic-gate mutex_exit(&kpd->kp_lock); 9917c478bd9Sstevel@tonic-gate return (0); 9927c478bd9Sstevel@tonic-gate } 9937c478bd9Sstevel@tonic-gate 9947c478bd9Sstevel@tonic-gate err = segkp_load(hat, seg, vaddr, len, kpd, kpd->kp_flags); 9957c478bd9Sstevel@tonic-gate mutex_exit(&kpd->kp_lock); 9967c478bd9Sstevel@tonic-gate return (err ? FC_MAKE_ERR(err) : 0); 9977c478bd9Sstevel@tonic-gate } 9987c478bd9Sstevel@tonic-gate 9997c478bd9Sstevel@tonic-gate if (type == F_SOFTUNLOCK) { 10007c478bd9Sstevel@tonic-gate uint_t flags; 10017c478bd9Sstevel@tonic-gate 10027c478bd9Sstevel@tonic-gate /* 10037c478bd9Sstevel@tonic-gate * Make sure the addr is LOCKED and it has anon backing 10047c478bd9Sstevel@tonic-gate * before unlocking 10057c478bd9Sstevel@tonic-gate */ 10067c478bd9Sstevel@tonic-gate if ((kpd->kp_flags & (KPD_LOCKED|KPD_NO_ANON)) == KPD_NO_ANON) { 10077c478bd9Sstevel@tonic-gate panic("segkp_fault: bad unlock"); 10087c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 10097c478bd9Sstevel@tonic-gate } 10107c478bd9Sstevel@tonic-gate 10117c478bd9Sstevel@tonic-gate if (vaddr != stom(kpd->kp_base, kpd->kp_flags) || 10127c478bd9Sstevel@tonic-gate len != SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)) { 10137c478bd9Sstevel@tonic-gate panic("segkp_fault: bad range"); 10147c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 10157c478bd9Sstevel@tonic-gate } 10167c478bd9Sstevel@tonic-gate 10177c478bd9Sstevel@tonic-gate if (rw == S_WRITE) 10187c478bd9Sstevel@tonic-gate flags = kpd->kp_flags | KPD_WRITEDIRTY; 10197c478bd9Sstevel@tonic-gate else 10207c478bd9Sstevel@tonic-gate flags = kpd->kp_flags; 10217c478bd9Sstevel@tonic-gate err = segkp_unlock(hat, seg, vaddr, len, kpd, flags); 10227c478bd9Sstevel@tonic-gate kpd->kp_flags &= ~KPD_LOCKED; 10237c478bd9Sstevel@tonic-gate mutex_exit(&kpd->kp_lock); 10247c478bd9Sstevel@tonic-gate return (err ? FC_MAKE_ERR(err) : 0); 10257c478bd9Sstevel@tonic-gate } 10267c478bd9Sstevel@tonic-gate mutex_exit(&kpd->kp_lock); 10277c478bd9Sstevel@tonic-gate panic("segkp_fault: bogus fault type: %d\n", type); 10287c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 10297c478bd9Sstevel@tonic-gate } 10307c478bd9Sstevel@tonic-gate 10317c478bd9Sstevel@tonic-gate /* 10327c478bd9Sstevel@tonic-gate * Check that the given protections suffice over the range specified by 10337c478bd9Sstevel@tonic-gate * vaddr and len. For this segment type, the only issue is whether or 10347c478bd9Sstevel@tonic-gate * not the range lies completely within the mapped part of an allocated 10357c478bd9Sstevel@tonic-gate * resource. 10367c478bd9Sstevel@tonic-gate */ 10377c478bd9Sstevel@tonic-gate /* ARGSUSED */ 10387c478bd9Sstevel@tonic-gate static int 10397c478bd9Sstevel@tonic-gate segkp_checkprot(struct seg *seg, caddr_t vaddr, size_t len, uint_t prot) 10407c478bd9Sstevel@tonic-gate { 10417c478bd9Sstevel@tonic-gate struct segkp_data *kpd = NULL; 10427c478bd9Sstevel@tonic-gate caddr_t mbase; 10437c478bd9Sstevel@tonic-gate size_t mlen; 10447c478bd9Sstevel@tonic-gate 10457c478bd9Sstevel@tonic-gate if ((kpd = segkp_find(seg, vaddr)) == NULL) 10467c478bd9Sstevel@tonic-gate return (EACCES); 10477c478bd9Sstevel@tonic-gate 10487c478bd9Sstevel@tonic-gate mutex_enter(&kpd->kp_lock); 10497c478bd9Sstevel@tonic-gate mbase = stom(kpd->kp_base, kpd->kp_flags); 10507c478bd9Sstevel@tonic-gate mlen = SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags); 10517c478bd9Sstevel@tonic-gate if (len > mlen || vaddr < mbase || 10527c478bd9Sstevel@tonic-gate ((vaddr + len) > (mbase + mlen))) { 10537c478bd9Sstevel@tonic-gate mutex_exit(&kpd->kp_lock); 10547c478bd9Sstevel@tonic-gate return (EACCES); 10557c478bd9Sstevel@tonic-gate } 10567c478bd9Sstevel@tonic-gate mutex_exit(&kpd->kp_lock); 10577c478bd9Sstevel@tonic-gate return (0); 10587c478bd9Sstevel@tonic-gate } 10597c478bd9Sstevel@tonic-gate 10607c478bd9Sstevel@tonic-gate 10617c478bd9Sstevel@tonic-gate /* 10627c478bd9Sstevel@tonic-gate * Check to see if it makes sense to do kluster/read ahead to 10637c478bd9Sstevel@tonic-gate * addr + delta relative to the mapping at addr. We assume here 10647c478bd9Sstevel@tonic-gate * that delta is a signed PAGESIZE'd multiple (which can be negative). 10657c478bd9Sstevel@tonic-gate * 10667c478bd9Sstevel@tonic-gate * For seg_u we always "approve" of this action from our standpoint. 10677c478bd9Sstevel@tonic-gate */ 10687c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 10697c478bd9Sstevel@tonic-gate static int 10707c478bd9Sstevel@tonic-gate segkp_kluster(struct seg *seg, caddr_t addr, ssize_t delta) 10717c478bd9Sstevel@tonic-gate { 10727c478bd9Sstevel@tonic-gate return (0); 10737c478bd9Sstevel@tonic-gate } 10747c478bd9Sstevel@tonic-gate 10757c478bd9Sstevel@tonic-gate /* 10767c478bd9Sstevel@tonic-gate * Load and possibly lock intra-slot resources in the range given by 10777c478bd9Sstevel@tonic-gate * vaddr and len. 10787c478bd9Sstevel@tonic-gate */ 10797c478bd9Sstevel@tonic-gate static int 10807c478bd9Sstevel@tonic-gate segkp_load( 10817c478bd9Sstevel@tonic-gate struct hat *hat, 10827c478bd9Sstevel@tonic-gate struct seg *seg, 10837c478bd9Sstevel@tonic-gate caddr_t vaddr, 10847c478bd9Sstevel@tonic-gate size_t len, 10857c478bd9Sstevel@tonic-gate struct segkp_data *kpd, 10867c478bd9Sstevel@tonic-gate uint_t flags) 10877c478bd9Sstevel@tonic-gate { 10887c478bd9Sstevel@tonic-gate caddr_t va; 10897c478bd9Sstevel@tonic-gate caddr_t vlim; 10907c478bd9Sstevel@tonic-gate ulong_t i; 10917c478bd9Sstevel@tonic-gate uint_t lock; 10927c478bd9Sstevel@tonic-gate 10937c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&kpd->kp_lock)); 10947c478bd9Sstevel@tonic-gate 10957c478bd9Sstevel@tonic-gate len = P2ROUNDUP(len, PAGESIZE); 10967c478bd9Sstevel@tonic-gate 10977c478bd9Sstevel@tonic-gate /* If locking, reserve physical memory */ 10987c478bd9Sstevel@tonic-gate if (flags & KPD_LOCKED) { 10997c478bd9Sstevel@tonic-gate pgcnt_t pages = btop(len); 11007c478bd9Sstevel@tonic-gate if ((kpd->kp_flags & KPD_NO_ANON) == 0) 11017c478bd9Sstevel@tonic-gate atomic_add_long(&anon_segkp_pages_locked, pages); 11027c478bd9Sstevel@tonic-gate (void) page_resv(pages, KM_SLEEP); 11037c478bd9Sstevel@tonic-gate } 11047c478bd9Sstevel@tonic-gate 11057c478bd9Sstevel@tonic-gate /* 11067c478bd9Sstevel@tonic-gate * Loop through the pages in the given range. 11077c478bd9Sstevel@tonic-gate */ 11087c478bd9Sstevel@tonic-gate va = (caddr_t)((uintptr_t)vaddr & (uintptr_t)PAGEMASK); 11097c478bd9Sstevel@tonic-gate vaddr = va; 11107c478bd9Sstevel@tonic-gate vlim = va + len; 11117c478bd9Sstevel@tonic-gate lock = flags & KPD_LOCKED; 11127c478bd9Sstevel@tonic-gate i = ((uintptr_t)(va - kpd->kp_base)) >> PAGESHIFT; 11137c478bd9Sstevel@tonic-gate for (; va < vlim; va += PAGESIZE, i++) { 11147c478bd9Sstevel@tonic-gate page_t *pl[2]; /* second element NULL terminator */ 11157c478bd9Sstevel@tonic-gate struct vnode *vp; 11167c478bd9Sstevel@tonic-gate anoff_t off; 11177c478bd9Sstevel@tonic-gate int err; 11187c478bd9Sstevel@tonic-gate struct anon *ap; 11197c478bd9Sstevel@tonic-gate 11207c478bd9Sstevel@tonic-gate /* 11217c478bd9Sstevel@tonic-gate * Summon the page. If it's not resident, arrange 11227c478bd9Sstevel@tonic-gate * for synchronous i/o to pull it in. 11237c478bd9Sstevel@tonic-gate */ 11247c478bd9Sstevel@tonic-gate ap = anon_get_ptr(kpd->kp_anon, kpd->kp_anon_idx + i); 11257c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &off); 11267c478bd9Sstevel@tonic-gate 11277c478bd9Sstevel@tonic-gate /* 11287c478bd9Sstevel@tonic-gate * The returned page list will have exactly one entry, 11297c478bd9Sstevel@tonic-gate * which is returned to us already kept. 11307c478bd9Sstevel@tonic-gate */ 11317c478bd9Sstevel@tonic-gate err = VOP_GETPAGE(vp, (offset_t)off, PAGESIZE, NULL, 1132*da6c28aaSamw pl, PAGESIZE, seg, va, S_READ, kcred, NULL); 11337c478bd9Sstevel@tonic-gate 11347c478bd9Sstevel@tonic-gate if (err) { 11357c478bd9Sstevel@tonic-gate /* 11367c478bd9Sstevel@tonic-gate * Back out of what we've done so far. 11377c478bd9Sstevel@tonic-gate */ 11387c478bd9Sstevel@tonic-gate (void) segkp_unlock(hat, seg, vaddr, 11397c478bd9Sstevel@tonic-gate (va - vaddr), kpd, flags); 11407c478bd9Sstevel@tonic-gate return (err); 11417c478bd9Sstevel@tonic-gate } 11427c478bd9Sstevel@tonic-gate 11437c478bd9Sstevel@tonic-gate /* 11447c478bd9Sstevel@tonic-gate * Load an MMU translation for the page. 11457c478bd9Sstevel@tonic-gate */ 11467c478bd9Sstevel@tonic-gate hat_memload(hat, va, pl[0], (PROT_READ|PROT_WRITE), 11477c478bd9Sstevel@tonic-gate lock ? HAT_LOAD_LOCK : HAT_LOAD); 11487c478bd9Sstevel@tonic-gate 11497c478bd9Sstevel@tonic-gate if (!lock) { 11507c478bd9Sstevel@tonic-gate /* 11517c478bd9Sstevel@tonic-gate * Now, release "shared" lock on the page. 11527c478bd9Sstevel@tonic-gate */ 11537c478bd9Sstevel@tonic-gate page_unlock(pl[0]); 11547c478bd9Sstevel@tonic-gate } 11557c478bd9Sstevel@tonic-gate } 11567c478bd9Sstevel@tonic-gate return (0); 11577c478bd9Sstevel@tonic-gate } 11587c478bd9Sstevel@tonic-gate 11597c478bd9Sstevel@tonic-gate /* 11607c478bd9Sstevel@tonic-gate * At the very least unload the mmu-translations and unlock the range if locked 11617c478bd9Sstevel@tonic-gate * Can be called with the following flag value KPD_WRITEDIRTY which specifies 11627c478bd9Sstevel@tonic-gate * any dirty pages should be written to disk. 11637c478bd9Sstevel@tonic-gate */ 11647c478bd9Sstevel@tonic-gate static int 11657c478bd9Sstevel@tonic-gate segkp_unlock( 11667c478bd9Sstevel@tonic-gate struct hat *hat, 11677c478bd9Sstevel@tonic-gate struct seg *seg, 11687c478bd9Sstevel@tonic-gate caddr_t vaddr, 11697c478bd9Sstevel@tonic-gate size_t len, 11707c478bd9Sstevel@tonic-gate struct segkp_data *kpd, 11717c478bd9Sstevel@tonic-gate uint_t flags) 11727c478bd9Sstevel@tonic-gate { 11737c478bd9Sstevel@tonic-gate caddr_t va; 11747c478bd9Sstevel@tonic-gate caddr_t vlim; 11757c478bd9Sstevel@tonic-gate ulong_t i; 11767c478bd9Sstevel@tonic-gate struct page *pp; 11777c478bd9Sstevel@tonic-gate struct vnode *vp; 11787c478bd9Sstevel@tonic-gate anoff_t off; 11797c478bd9Sstevel@tonic-gate struct anon *ap; 11807c478bd9Sstevel@tonic-gate 11817c478bd9Sstevel@tonic-gate #ifdef lint 11827c478bd9Sstevel@tonic-gate seg = seg; 11837c478bd9Sstevel@tonic-gate #endif /* lint */ 11847c478bd9Sstevel@tonic-gate 11857c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&kpd->kp_lock)); 11867c478bd9Sstevel@tonic-gate 11877c478bd9Sstevel@tonic-gate /* 11887c478bd9Sstevel@tonic-gate * Loop through the pages in the given range. It is assumed 11897c478bd9Sstevel@tonic-gate * segkp_unlock is called with page aligned base 11907c478bd9Sstevel@tonic-gate */ 11917c478bd9Sstevel@tonic-gate va = vaddr; 11927c478bd9Sstevel@tonic-gate vlim = va + len; 11937c478bd9Sstevel@tonic-gate i = ((uintptr_t)(va - kpd->kp_base)) >> PAGESHIFT; 11947c478bd9Sstevel@tonic-gate hat_unload(hat, va, len, 11957c478bd9Sstevel@tonic-gate ((flags & KPD_LOCKED) ? HAT_UNLOAD_UNLOCK : HAT_UNLOAD)); 11967c478bd9Sstevel@tonic-gate for (; va < vlim; va += PAGESIZE, i++) { 11977c478bd9Sstevel@tonic-gate /* 11987c478bd9Sstevel@tonic-gate * Find the page associated with this part of the 11997c478bd9Sstevel@tonic-gate * slot, tracking it down through its associated swap 12007c478bd9Sstevel@tonic-gate * space. 12017c478bd9Sstevel@tonic-gate */ 12027c478bd9Sstevel@tonic-gate ap = anon_get_ptr(kpd->kp_anon, kpd->kp_anon_idx + i); 12037c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &off); 12047c478bd9Sstevel@tonic-gate 12057c478bd9Sstevel@tonic-gate if (flags & KPD_LOCKED) { 12067c478bd9Sstevel@tonic-gate if ((pp = page_find(vp, off)) == NULL) { 12077c478bd9Sstevel@tonic-gate if (flags & KPD_LOCKED) { 12087c478bd9Sstevel@tonic-gate panic("segkp_softunlock: missing page"); 12097c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 12107c478bd9Sstevel@tonic-gate } 12117c478bd9Sstevel@tonic-gate } 12127c478bd9Sstevel@tonic-gate } else { 12137c478bd9Sstevel@tonic-gate /* 12147c478bd9Sstevel@tonic-gate * Nothing to do if the slot is not locked and the 12157c478bd9Sstevel@tonic-gate * page doesn't exist. 12167c478bd9Sstevel@tonic-gate */ 12177c478bd9Sstevel@tonic-gate if ((pp = page_lookup(vp, off, SE_SHARED)) == NULL) 12187c478bd9Sstevel@tonic-gate continue; 12197c478bd9Sstevel@tonic-gate } 12207c478bd9Sstevel@tonic-gate 12217c478bd9Sstevel@tonic-gate /* 12227c478bd9Sstevel@tonic-gate * If the page doesn't have any translations, is 12237c478bd9Sstevel@tonic-gate * dirty and not being shared, then push it out 12247c478bd9Sstevel@tonic-gate * asynchronously and avoid waiting for the 12257c478bd9Sstevel@tonic-gate * pageout daemon to do it for us. 12267c478bd9Sstevel@tonic-gate * 12277c478bd9Sstevel@tonic-gate * XXX - Do we really need to get the "exclusive" 12287c478bd9Sstevel@tonic-gate * lock via an upgrade? 12297c478bd9Sstevel@tonic-gate */ 12307c478bd9Sstevel@tonic-gate if ((flags & KPD_WRITEDIRTY) && !hat_page_is_mapped(pp) && 12317c478bd9Sstevel@tonic-gate hat_ismod(pp) && page_tryupgrade(pp)) { 12327c478bd9Sstevel@tonic-gate /* 12337c478bd9Sstevel@tonic-gate * Hold the vnode before releasing the page lock to 12347c478bd9Sstevel@tonic-gate * prevent it from being freed and re-used by some 12357c478bd9Sstevel@tonic-gate * other thread. 12367c478bd9Sstevel@tonic-gate */ 12377c478bd9Sstevel@tonic-gate VN_HOLD(vp); 12387c478bd9Sstevel@tonic-gate page_unlock(pp); 12397c478bd9Sstevel@tonic-gate 12407c478bd9Sstevel@tonic-gate /* 12417c478bd9Sstevel@tonic-gate * Want most powerful credentials we can get so 12427c478bd9Sstevel@tonic-gate * use kcred. 12437c478bd9Sstevel@tonic-gate */ 12447c478bd9Sstevel@tonic-gate (void) VOP_PUTPAGE(vp, (offset_t)off, PAGESIZE, 1245*da6c28aaSamw B_ASYNC | B_FREE, kcred, NULL); 12467c478bd9Sstevel@tonic-gate VN_RELE(vp); 12477c478bd9Sstevel@tonic-gate } else { 12487c478bd9Sstevel@tonic-gate page_unlock(pp); 12497c478bd9Sstevel@tonic-gate } 12507c478bd9Sstevel@tonic-gate } 12517c478bd9Sstevel@tonic-gate 12527c478bd9Sstevel@tonic-gate /* If unlocking, release physical memory */ 12537c478bd9Sstevel@tonic-gate if (flags & KPD_LOCKED) { 12547c478bd9Sstevel@tonic-gate pgcnt_t pages = btopr(len); 12557c478bd9Sstevel@tonic-gate if ((kpd->kp_flags & KPD_NO_ANON) == 0) 12567c478bd9Sstevel@tonic-gate atomic_add_long(&anon_segkp_pages_locked, -pages); 12577c478bd9Sstevel@tonic-gate page_unresv(pages); 12587c478bd9Sstevel@tonic-gate } 12597c478bd9Sstevel@tonic-gate return (0); 12607c478bd9Sstevel@tonic-gate } 12617c478bd9Sstevel@tonic-gate 12627c478bd9Sstevel@tonic-gate /* 12637c478bd9Sstevel@tonic-gate * Insert the kpd in the hash table. 12647c478bd9Sstevel@tonic-gate */ 12657c478bd9Sstevel@tonic-gate static void 12667c478bd9Sstevel@tonic-gate segkp_insert(struct seg *seg, struct segkp_data *kpd) 12677c478bd9Sstevel@tonic-gate { 12687c478bd9Sstevel@tonic-gate struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data; 12697c478bd9Sstevel@tonic-gate int index; 12707c478bd9Sstevel@tonic-gate 12717c478bd9Sstevel@tonic-gate /* 12727c478bd9Sstevel@tonic-gate * Insert the kpd based on the address that will be returned 12737c478bd9Sstevel@tonic-gate * via segkp_release. 12747c478bd9Sstevel@tonic-gate */ 12757c478bd9Sstevel@tonic-gate index = SEGKP_HASH(stom(kpd->kp_base, kpd->kp_flags)); 12767c478bd9Sstevel@tonic-gate mutex_enter(&segkp_lock); 12777c478bd9Sstevel@tonic-gate kpd->kp_next = kpsd->kpsd_hash[index]; 12787c478bd9Sstevel@tonic-gate kpsd->kpsd_hash[index] = kpd; 12797c478bd9Sstevel@tonic-gate mutex_exit(&segkp_lock); 12807c478bd9Sstevel@tonic-gate } 12817c478bd9Sstevel@tonic-gate 12827c478bd9Sstevel@tonic-gate /* 12837c478bd9Sstevel@tonic-gate * Remove kpd from the hash table. 12847c478bd9Sstevel@tonic-gate */ 12857c478bd9Sstevel@tonic-gate static void 12867c478bd9Sstevel@tonic-gate segkp_delete(struct seg *seg, struct segkp_data *kpd) 12877c478bd9Sstevel@tonic-gate { 12887c478bd9Sstevel@tonic-gate struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data; 12897c478bd9Sstevel@tonic-gate struct segkp_data **kpp; 12907c478bd9Sstevel@tonic-gate int index; 12917c478bd9Sstevel@tonic-gate 12927c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&segkp_lock)); 12937c478bd9Sstevel@tonic-gate 12947c478bd9Sstevel@tonic-gate index = SEGKP_HASH(stom(kpd->kp_base, kpd->kp_flags)); 12957c478bd9Sstevel@tonic-gate for (kpp = &kpsd->kpsd_hash[index]; 12967c478bd9Sstevel@tonic-gate *kpp != NULL; kpp = &((*kpp)->kp_next)) { 12977c478bd9Sstevel@tonic-gate if (*kpp == kpd) { 12987c478bd9Sstevel@tonic-gate *kpp = kpd->kp_next; 12997c478bd9Sstevel@tonic-gate return; 13007c478bd9Sstevel@tonic-gate } 13017c478bd9Sstevel@tonic-gate } 13027c478bd9Sstevel@tonic-gate panic("segkp_delete: unable to find element to delete"); 13037c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 13047c478bd9Sstevel@tonic-gate } 13057c478bd9Sstevel@tonic-gate 13067c478bd9Sstevel@tonic-gate /* 13077c478bd9Sstevel@tonic-gate * Find the kpd associated with a vaddr. 13087c478bd9Sstevel@tonic-gate * 13097c478bd9Sstevel@tonic-gate * Most of the callers of segkp_find will pass the vaddr that 13107c478bd9Sstevel@tonic-gate * hashes to the desired index, but there are cases where 13117c478bd9Sstevel@tonic-gate * this is not true in which case we have to (potentially) scan 13127c478bd9Sstevel@tonic-gate * the whole table looking for it. This should be very rare 13137c478bd9Sstevel@tonic-gate * (e.g. a segkp_fault(F_INVAL) on an address somewhere in the 13147c478bd9Sstevel@tonic-gate * middle of the segkp_data region). 13157c478bd9Sstevel@tonic-gate */ 13167c478bd9Sstevel@tonic-gate static struct segkp_data * 13177c478bd9Sstevel@tonic-gate segkp_find(struct seg *seg, caddr_t vaddr) 13187c478bd9Sstevel@tonic-gate { 13197c478bd9Sstevel@tonic-gate struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data; 13207c478bd9Sstevel@tonic-gate struct segkp_data *kpd; 13217c478bd9Sstevel@tonic-gate int i; 13227c478bd9Sstevel@tonic-gate int stop; 13237c478bd9Sstevel@tonic-gate 13247c478bd9Sstevel@tonic-gate i = stop = SEGKP_HASH(vaddr); 13257c478bd9Sstevel@tonic-gate mutex_enter(&segkp_lock); 13267c478bd9Sstevel@tonic-gate do { 13277c478bd9Sstevel@tonic-gate for (kpd = kpsd->kpsd_hash[i]; kpd != NULL; 13287c478bd9Sstevel@tonic-gate kpd = kpd->kp_next) { 13297c478bd9Sstevel@tonic-gate if (vaddr >= kpd->kp_base && 13307c478bd9Sstevel@tonic-gate vaddr < kpd->kp_base + kpd->kp_len) { 13317c478bd9Sstevel@tonic-gate mutex_exit(&segkp_lock); 13327c478bd9Sstevel@tonic-gate return (kpd); 13337c478bd9Sstevel@tonic-gate } 13347c478bd9Sstevel@tonic-gate } 13357c478bd9Sstevel@tonic-gate if (--i < 0) 13367c478bd9Sstevel@tonic-gate i = SEGKP_HASHSZ - 1; /* Wrap */ 13377c478bd9Sstevel@tonic-gate } while (i != stop); 13387c478bd9Sstevel@tonic-gate mutex_exit(&segkp_lock); 13397c478bd9Sstevel@tonic-gate return (NULL); /* Not found */ 13407c478bd9Sstevel@tonic-gate } 13417c478bd9Sstevel@tonic-gate 13427c478bd9Sstevel@tonic-gate /* 13437c478bd9Sstevel@tonic-gate * returns size of swappable area. 13447c478bd9Sstevel@tonic-gate */ 13457c478bd9Sstevel@tonic-gate size_t 13467c478bd9Sstevel@tonic-gate swapsize(caddr_t v) 13477c478bd9Sstevel@tonic-gate { 13487c478bd9Sstevel@tonic-gate struct segkp_data *kpd; 13497c478bd9Sstevel@tonic-gate 13507c478bd9Sstevel@tonic-gate if ((kpd = segkp_find(segkp, v)) != NULL) 13517c478bd9Sstevel@tonic-gate return (SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)); 13527c478bd9Sstevel@tonic-gate else 13537c478bd9Sstevel@tonic-gate return (NULL); 13547c478bd9Sstevel@tonic-gate } 13557c478bd9Sstevel@tonic-gate 13567c478bd9Sstevel@tonic-gate /* 13577c478bd9Sstevel@tonic-gate * Dump out all the active segkp pages 13587c478bd9Sstevel@tonic-gate */ 13597c478bd9Sstevel@tonic-gate static void 13607c478bd9Sstevel@tonic-gate segkp_dump(struct seg *seg) 13617c478bd9Sstevel@tonic-gate { 13627c478bd9Sstevel@tonic-gate int i; 13637c478bd9Sstevel@tonic-gate struct segkp_data *kpd; 13647c478bd9Sstevel@tonic-gate struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data; 13657c478bd9Sstevel@tonic-gate 13667c478bd9Sstevel@tonic-gate for (i = 0; i < SEGKP_HASHSZ; i++) { 13677c478bd9Sstevel@tonic-gate for (kpd = kpsd->kpsd_hash[i]; 13687c478bd9Sstevel@tonic-gate kpd != NULL; kpd = kpd->kp_next) { 13697c478bd9Sstevel@tonic-gate pfn_t pfn; 13707c478bd9Sstevel@tonic-gate caddr_t addr; 13717c478bd9Sstevel@tonic-gate caddr_t eaddr; 13727c478bd9Sstevel@tonic-gate 13737c478bd9Sstevel@tonic-gate addr = kpd->kp_base; 13747c478bd9Sstevel@tonic-gate eaddr = addr + kpd->kp_len; 13757c478bd9Sstevel@tonic-gate while (addr < eaddr) { 13767c478bd9Sstevel@tonic-gate ASSERT(seg->s_as == &kas); 13777c478bd9Sstevel@tonic-gate pfn = hat_getpfnum(seg->s_as->a_hat, addr); 13787c478bd9Sstevel@tonic-gate if (pfn != PFN_INVALID) 13797c478bd9Sstevel@tonic-gate dump_addpage(seg->s_as, addr, pfn); 13807c478bd9Sstevel@tonic-gate addr += PAGESIZE; 13817c478bd9Sstevel@tonic-gate dump_timeleft = dump_timeout; 13827c478bd9Sstevel@tonic-gate } 13837c478bd9Sstevel@tonic-gate } 13847c478bd9Sstevel@tonic-gate } 13857c478bd9Sstevel@tonic-gate } 13867c478bd9Sstevel@tonic-gate 13877c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 13887c478bd9Sstevel@tonic-gate static int 13897c478bd9Sstevel@tonic-gate segkp_pagelock(struct seg *seg, caddr_t addr, size_t len, 13907c478bd9Sstevel@tonic-gate struct page ***ppp, enum lock_type type, enum seg_rw rw) 13917c478bd9Sstevel@tonic-gate { 13927c478bd9Sstevel@tonic-gate return (ENOTSUP); 13937c478bd9Sstevel@tonic-gate } 13947c478bd9Sstevel@tonic-gate 13957c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 13967c478bd9Sstevel@tonic-gate static int 13977c478bd9Sstevel@tonic-gate segkp_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp) 13987c478bd9Sstevel@tonic-gate { 13997c478bd9Sstevel@tonic-gate return (ENODEV); 14007c478bd9Sstevel@tonic-gate } 14017c478bd9Sstevel@tonic-gate 14027c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 14037c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t * 14047c478bd9Sstevel@tonic-gate segkp_getpolicy(struct seg *seg, caddr_t addr) 14057c478bd9Sstevel@tonic-gate { 14067c478bd9Sstevel@tonic-gate return (NULL); 14077c478bd9Sstevel@tonic-gate } 14087c478bd9Sstevel@tonic-gate 14091bd5c35fSelowe /*ARGSUSED*/ 14101bd5c35fSelowe static int 14111bd5c35fSelowe segkp_capable(struct seg *seg, segcapability_t capability) 14121bd5c35fSelowe { 14131bd5c35fSelowe return (0); 14141bd5c35fSelowe } 14151bd5c35fSelowe 14167c478bd9Sstevel@tonic-gate #include <sys/mem_config.h> 14177c478bd9Sstevel@tonic-gate 14187c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 14197c478bd9Sstevel@tonic-gate static void 14207c478bd9Sstevel@tonic-gate segkp_mem_config_post_add(void *arg, pgcnt_t delta_pages) 14217c478bd9Sstevel@tonic-gate {} 14227c478bd9Sstevel@tonic-gate 14237c478bd9Sstevel@tonic-gate /* 14247c478bd9Sstevel@tonic-gate * During memory delete, turn off caches so that pages are not held. 14257c478bd9Sstevel@tonic-gate * A better solution may be to unlock the pages while they are 14267c478bd9Sstevel@tonic-gate * in the cache so that they may be collected naturally. 14277c478bd9Sstevel@tonic-gate */ 14287c478bd9Sstevel@tonic-gate 14297c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 14307c478bd9Sstevel@tonic-gate static int 14317c478bd9Sstevel@tonic-gate segkp_mem_config_pre_del(void *arg, pgcnt_t delta_pages) 14327c478bd9Sstevel@tonic-gate { 14337c478bd9Sstevel@tonic-gate atomic_add_32(&segkp_indel, 1); 14347c478bd9Sstevel@tonic-gate segkp_cache_free(); 14357c478bd9Sstevel@tonic-gate return (0); 14367c478bd9Sstevel@tonic-gate } 14377c478bd9Sstevel@tonic-gate 14387c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 14397c478bd9Sstevel@tonic-gate static void 14407c478bd9Sstevel@tonic-gate segkp_mem_config_post_del(void *arg, pgcnt_t delta_pages, int cancelled) 14417c478bd9Sstevel@tonic-gate { 14427c478bd9Sstevel@tonic-gate atomic_add_32(&segkp_indel, -1); 14437c478bd9Sstevel@tonic-gate } 14447c478bd9Sstevel@tonic-gate 14457c478bd9Sstevel@tonic-gate static kphysm_setup_vector_t segkp_mem_config_vec = { 14467c478bd9Sstevel@tonic-gate KPHYSM_SETUP_VECTOR_VERSION, 14477c478bd9Sstevel@tonic-gate segkp_mem_config_post_add, 14487c478bd9Sstevel@tonic-gate segkp_mem_config_pre_del, 14497c478bd9Sstevel@tonic-gate segkp_mem_config_post_del, 14507c478bd9Sstevel@tonic-gate }; 14517c478bd9Sstevel@tonic-gate 14527c478bd9Sstevel@tonic-gate static void 14537c478bd9Sstevel@tonic-gate segkpinit_mem_config(struct seg *seg) 14547c478bd9Sstevel@tonic-gate { 14557c478bd9Sstevel@tonic-gate int ret; 14567c478bd9Sstevel@tonic-gate 14577c478bd9Sstevel@tonic-gate ret = kphysm_setup_func_register(&segkp_mem_config_vec, (void *)seg); 14587c478bd9Sstevel@tonic-gate ASSERT(ret == 0); 14597c478bd9Sstevel@tonic-gate } 1460