17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*7d692464Sdp * Common Development and Distribution License (the "License"). 6*7d692464Sdp * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22*7d692464Sdp * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 277c478bd9Sstevel@tonic-gate 287c478bd9Sstevel@tonic-gate /* 297c478bd9Sstevel@tonic-gate * Kernel memory allocator, as described in the following two papers: 307c478bd9Sstevel@tonic-gate * 317c478bd9Sstevel@tonic-gate * Jeff Bonwick, 327c478bd9Sstevel@tonic-gate * The Slab Allocator: An Object-Caching Kernel Memory Allocator. 337c478bd9Sstevel@tonic-gate * Proceedings of the Summer 1994 Usenix Conference. 347c478bd9Sstevel@tonic-gate * Available as /shared/sac/PSARC/1994/028/materials/kmem.pdf. 357c478bd9Sstevel@tonic-gate * 367c478bd9Sstevel@tonic-gate * Jeff Bonwick and Jonathan Adams, 377c478bd9Sstevel@tonic-gate * Magazines and vmem: Extending the Slab Allocator to Many CPUs and 387c478bd9Sstevel@tonic-gate * Arbitrary Resources. 397c478bd9Sstevel@tonic-gate * Proceedings of the 2001 Usenix Conference. 407c478bd9Sstevel@tonic-gate * Available as /shared/sac/PSARC/2000/550/materials/vmem.pdf. 417c478bd9Sstevel@tonic-gate */ 427c478bd9Sstevel@tonic-gate 437c478bd9Sstevel@tonic-gate #include <sys/kmem_impl.h> 447c478bd9Sstevel@tonic-gate #include <sys/vmem_impl.h> 457c478bd9Sstevel@tonic-gate #include <sys/param.h> 467c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 477c478bd9Sstevel@tonic-gate #include <sys/vm.h> 487c478bd9Sstevel@tonic-gate #include <sys/proc.h> 497c478bd9Sstevel@tonic-gate #include <sys/tuneable.h> 507c478bd9Sstevel@tonic-gate #include <sys/systm.h> 517c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 527c478bd9Sstevel@tonic-gate #include <sys/debug.h> 537c478bd9Sstevel@tonic-gate #include <sys/mutex.h> 547c478bd9Sstevel@tonic-gate #include <sys/bitmap.h> 557c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 567c478bd9Sstevel@tonic-gate #include <sys/kobj.h> 577c478bd9Sstevel@tonic-gate #include <sys/disp.h> 587c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h> 597c478bd9Sstevel@tonic-gate #include <sys/log.h> 607c478bd9Sstevel@tonic-gate #include <sys/callb.h> 617c478bd9Sstevel@tonic-gate #include <sys/taskq.h> 627c478bd9Sstevel@tonic-gate #include <sys/modctl.h> 637c478bd9Sstevel@tonic-gate #include <sys/reboot.h> 647c478bd9Sstevel@tonic-gate #include <sys/id32.h> 657c478bd9Sstevel@tonic-gate #include <sys/zone.h> 667c478bd9Sstevel@tonic-gate 677c478bd9Sstevel@tonic-gate extern void streams_msg_init(void); 687c478bd9Sstevel@tonic-gate extern int segkp_fromheap; 697c478bd9Sstevel@tonic-gate extern void segkp_cache_free(void); 707c478bd9Sstevel@tonic-gate 717c478bd9Sstevel@tonic-gate struct kmem_cache_kstat { 727c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_size; 737c478bd9Sstevel@tonic-gate kstat_named_t kmc_align; 747c478bd9Sstevel@tonic-gate kstat_named_t kmc_chunk_size; 757c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_size; 767c478bd9Sstevel@tonic-gate kstat_named_t kmc_alloc; 777c478bd9Sstevel@tonic-gate kstat_named_t kmc_alloc_fail; 787c478bd9Sstevel@tonic-gate kstat_named_t kmc_free; 797c478bd9Sstevel@tonic-gate kstat_named_t kmc_depot_alloc; 807c478bd9Sstevel@tonic-gate kstat_named_t kmc_depot_free; 817c478bd9Sstevel@tonic-gate kstat_named_t kmc_depot_contention; 827c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_alloc; 837c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_free; 847c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_constructed; 857c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_avail; 867c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_inuse; 877c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_total; 887c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_max; 897c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_create; 907c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_destroy; 917c478bd9Sstevel@tonic-gate kstat_named_t kmc_vmem_source; 927c478bd9Sstevel@tonic-gate kstat_named_t kmc_hash_size; 937c478bd9Sstevel@tonic-gate kstat_named_t kmc_hash_lookup_depth; 947c478bd9Sstevel@tonic-gate kstat_named_t kmc_hash_rescale; 957c478bd9Sstevel@tonic-gate kstat_named_t kmc_full_magazines; 967c478bd9Sstevel@tonic-gate kstat_named_t kmc_empty_magazines; 977c478bd9Sstevel@tonic-gate kstat_named_t kmc_magazine_size; 987c478bd9Sstevel@tonic-gate } kmem_cache_kstat = { 997c478bd9Sstevel@tonic-gate { "buf_size", KSTAT_DATA_UINT64 }, 1007c478bd9Sstevel@tonic-gate { "align", KSTAT_DATA_UINT64 }, 1017c478bd9Sstevel@tonic-gate { "chunk_size", KSTAT_DATA_UINT64 }, 1027c478bd9Sstevel@tonic-gate { "slab_size", KSTAT_DATA_UINT64 }, 1037c478bd9Sstevel@tonic-gate { "alloc", KSTAT_DATA_UINT64 }, 1047c478bd9Sstevel@tonic-gate { "alloc_fail", KSTAT_DATA_UINT64 }, 1057c478bd9Sstevel@tonic-gate { "free", KSTAT_DATA_UINT64 }, 1067c478bd9Sstevel@tonic-gate { "depot_alloc", KSTAT_DATA_UINT64 }, 1077c478bd9Sstevel@tonic-gate { "depot_free", KSTAT_DATA_UINT64 }, 1087c478bd9Sstevel@tonic-gate { "depot_contention", KSTAT_DATA_UINT64 }, 1097c478bd9Sstevel@tonic-gate { "slab_alloc", KSTAT_DATA_UINT64 }, 1107c478bd9Sstevel@tonic-gate { "slab_free", KSTAT_DATA_UINT64 }, 1117c478bd9Sstevel@tonic-gate { "buf_constructed", KSTAT_DATA_UINT64 }, 1127c478bd9Sstevel@tonic-gate { "buf_avail", KSTAT_DATA_UINT64 }, 1137c478bd9Sstevel@tonic-gate { "buf_inuse", KSTAT_DATA_UINT64 }, 1147c478bd9Sstevel@tonic-gate { "buf_total", KSTAT_DATA_UINT64 }, 1157c478bd9Sstevel@tonic-gate { "buf_max", KSTAT_DATA_UINT64 }, 1167c478bd9Sstevel@tonic-gate { "slab_create", KSTAT_DATA_UINT64 }, 1177c478bd9Sstevel@tonic-gate { "slab_destroy", KSTAT_DATA_UINT64 }, 1187c478bd9Sstevel@tonic-gate { "vmem_source", KSTAT_DATA_UINT64 }, 1197c478bd9Sstevel@tonic-gate { "hash_size", KSTAT_DATA_UINT64 }, 1207c478bd9Sstevel@tonic-gate { "hash_lookup_depth", KSTAT_DATA_UINT64 }, 1217c478bd9Sstevel@tonic-gate { "hash_rescale", KSTAT_DATA_UINT64 }, 1227c478bd9Sstevel@tonic-gate { "full_magazines", KSTAT_DATA_UINT64 }, 1237c478bd9Sstevel@tonic-gate { "empty_magazines", KSTAT_DATA_UINT64 }, 1247c478bd9Sstevel@tonic-gate { "magazine_size", KSTAT_DATA_UINT64 }, 1257c478bd9Sstevel@tonic-gate }; 1267c478bd9Sstevel@tonic-gate 1277c478bd9Sstevel@tonic-gate static kmutex_t kmem_cache_kstat_lock; 1287c478bd9Sstevel@tonic-gate 1297c478bd9Sstevel@tonic-gate /* 1307c478bd9Sstevel@tonic-gate * The default set of caches to back kmem_alloc(). 1317c478bd9Sstevel@tonic-gate * These sizes should be reevaluated periodically. 1327c478bd9Sstevel@tonic-gate * 1337c478bd9Sstevel@tonic-gate * We want allocations that are multiples of the coherency granularity 1347c478bd9Sstevel@tonic-gate * (64 bytes) to be satisfied from a cache which is a multiple of 64 1357c478bd9Sstevel@tonic-gate * bytes, so that it will be 64-byte aligned. For all multiples of 64, 1367c478bd9Sstevel@tonic-gate * the next kmem_cache_size greater than or equal to it must be a 1377c478bd9Sstevel@tonic-gate * multiple of 64. 1387c478bd9Sstevel@tonic-gate */ 1397c478bd9Sstevel@tonic-gate static const int kmem_alloc_sizes[] = { 1407c478bd9Sstevel@tonic-gate 1 * 8, 1417c478bd9Sstevel@tonic-gate 2 * 8, 1427c478bd9Sstevel@tonic-gate 3 * 8, 1437c478bd9Sstevel@tonic-gate 4 * 8, 5 * 8, 6 * 8, 7 * 8, 1447c478bd9Sstevel@tonic-gate 4 * 16, 5 * 16, 6 * 16, 7 * 16, 1457c478bd9Sstevel@tonic-gate 4 * 32, 5 * 32, 6 * 32, 7 * 32, 1467c478bd9Sstevel@tonic-gate 4 * 64, 5 * 64, 6 * 64, 7 * 64, 1477c478bd9Sstevel@tonic-gate 4 * 128, 5 * 128, 6 * 128, 7 * 128, 1487c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 7, 64), 1497c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 6, 64), 1507c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 5, 64), 1517c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 4, 64), 1527c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 3, 64), 1537c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 2, 64), 1547c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 1, 64), 1557c478bd9Sstevel@tonic-gate 4096 * 3, 1567c478bd9Sstevel@tonic-gate 8192 * 2, 1577c478bd9Sstevel@tonic-gate }; 1587c478bd9Sstevel@tonic-gate 1597c478bd9Sstevel@tonic-gate #define KMEM_MAXBUF 16384 1607c478bd9Sstevel@tonic-gate 1617c478bd9Sstevel@tonic-gate static kmem_cache_t *kmem_alloc_table[KMEM_MAXBUF >> KMEM_ALIGN_SHIFT]; 1627c478bd9Sstevel@tonic-gate 1637c478bd9Sstevel@tonic-gate static kmem_magtype_t kmem_magtype[] = { 1647c478bd9Sstevel@tonic-gate { 1, 8, 3200, 65536 }, 1657c478bd9Sstevel@tonic-gate { 3, 16, 256, 32768 }, 1667c478bd9Sstevel@tonic-gate { 7, 32, 64, 16384 }, 1677c478bd9Sstevel@tonic-gate { 15, 64, 0, 8192 }, 1687c478bd9Sstevel@tonic-gate { 31, 64, 0, 4096 }, 1697c478bd9Sstevel@tonic-gate { 47, 64, 0, 2048 }, 1707c478bd9Sstevel@tonic-gate { 63, 64, 0, 1024 }, 1717c478bd9Sstevel@tonic-gate { 95, 64, 0, 512 }, 1727c478bd9Sstevel@tonic-gate { 143, 64, 0, 0 }, 1737c478bd9Sstevel@tonic-gate }; 1747c478bd9Sstevel@tonic-gate 1757c478bd9Sstevel@tonic-gate static uint32_t kmem_reaping; 1767c478bd9Sstevel@tonic-gate static uint32_t kmem_reaping_idspace; 1777c478bd9Sstevel@tonic-gate 1787c478bd9Sstevel@tonic-gate /* 1797c478bd9Sstevel@tonic-gate * kmem tunables 1807c478bd9Sstevel@tonic-gate */ 1817c478bd9Sstevel@tonic-gate clock_t kmem_reap_interval; /* cache reaping rate [15 * HZ ticks] */ 1827c478bd9Sstevel@tonic-gate int kmem_depot_contention = 3; /* max failed tryenters per real interval */ 1837c478bd9Sstevel@tonic-gate pgcnt_t kmem_reapahead = 0; /* start reaping N pages before pageout */ 1847c478bd9Sstevel@tonic-gate int kmem_panic = 1; /* whether to panic on error */ 1857c478bd9Sstevel@tonic-gate int kmem_logging = 1; /* kmem_log_enter() override */ 1867c478bd9Sstevel@tonic-gate uint32_t kmem_mtbf = 0; /* mean time between failures [default: off] */ 1877c478bd9Sstevel@tonic-gate size_t kmem_transaction_log_size; /* transaction log size [2% of memory] */ 1887c478bd9Sstevel@tonic-gate size_t kmem_content_log_size; /* content log size [2% of memory] */ 1897c478bd9Sstevel@tonic-gate size_t kmem_failure_log_size; /* failure log [4 pages per CPU] */ 1907c478bd9Sstevel@tonic-gate size_t kmem_slab_log_size; /* slab create log [4 pages per CPU] */ 1917c478bd9Sstevel@tonic-gate size_t kmem_content_maxsave = 256; /* KMF_CONTENTS max bytes to log */ 1927c478bd9Sstevel@tonic-gate size_t kmem_lite_minsize = 0; /* minimum buffer size for KMF_LITE */ 1937c478bd9Sstevel@tonic-gate size_t kmem_lite_maxalign = 1024; /* maximum buffer alignment for KMF_LITE */ 1947c478bd9Sstevel@tonic-gate int kmem_lite_pcs = 4; /* number of PCs to store in KMF_LITE mode */ 1957c478bd9Sstevel@tonic-gate size_t kmem_maxverify; /* maximum bytes to inspect in debug routines */ 1967c478bd9Sstevel@tonic-gate size_t kmem_minfirewall; /* hardware-enforced redzone threshold */ 1977c478bd9Sstevel@tonic-gate 1987c478bd9Sstevel@tonic-gate #ifdef DEBUG 1997c478bd9Sstevel@tonic-gate int kmem_flags = KMF_AUDIT | KMF_DEADBEEF | KMF_REDZONE | KMF_CONTENTS; 2007c478bd9Sstevel@tonic-gate #else 2017c478bd9Sstevel@tonic-gate int kmem_flags = 0; 2027c478bd9Sstevel@tonic-gate #endif 2037c478bd9Sstevel@tonic-gate int kmem_ready; 2047c478bd9Sstevel@tonic-gate 2057c478bd9Sstevel@tonic-gate static kmem_cache_t *kmem_slab_cache; 2067c478bd9Sstevel@tonic-gate static kmem_cache_t *kmem_bufctl_cache; 2077c478bd9Sstevel@tonic-gate static kmem_cache_t *kmem_bufctl_audit_cache; 2087c478bd9Sstevel@tonic-gate 2097c478bd9Sstevel@tonic-gate static kmutex_t kmem_cache_lock; /* inter-cache linkage only */ 2107c478bd9Sstevel@tonic-gate kmem_cache_t kmem_null_cache; 2117c478bd9Sstevel@tonic-gate 2127c478bd9Sstevel@tonic-gate static taskq_t *kmem_taskq; 2137c478bd9Sstevel@tonic-gate static kmutex_t kmem_flags_lock; 2147c478bd9Sstevel@tonic-gate static vmem_t *kmem_metadata_arena; 2157c478bd9Sstevel@tonic-gate static vmem_t *kmem_msb_arena; /* arena for metadata caches */ 2167c478bd9Sstevel@tonic-gate static vmem_t *kmem_cache_arena; 2177c478bd9Sstevel@tonic-gate static vmem_t *kmem_hash_arena; 2187c478bd9Sstevel@tonic-gate static vmem_t *kmem_log_arena; 2197c478bd9Sstevel@tonic-gate static vmem_t *kmem_oversize_arena; 2207c478bd9Sstevel@tonic-gate static vmem_t *kmem_va_arena; 2217c478bd9Sstevel@tonic-gate static vmem_t *kmem_default_arena; 2227c478bd9Sstevel@tonic-gate static vmem_t *kmem_firewall_va_arena; 2237c478bd9Sstevel@tonic-gate static vmem_t *kmem_firewall_arena; 2247c478bd9Sstevel@tonic-gate 2257c478bd9Sstevel@tonic-gate kmem_log_header_t *kmem_transaction_log; 2267c478bd9Sstevel@tonic-gate kmem_log_header_t *kmem_content_log; 2277c478bd9Sstevel@tonic-gate kmem_log_header_t *kmem_failure_log; 2287c478bd9Sstevel@tonic-gate kmem_log_header_t *kmem_slab_log; 2297c478bd9Sstevel@tonic-gate 2307c478bd9Sstevel@tonic-gate static int kmem_lite_count; /* # of PCs in kmem_buftag_lite_t */ 2317c478bd9Sstevel@tonic-gate 2327c478bd9Sstevel@tonic-gate #define KMEM_BUFTAG_LITE_ENTER(bt, count, caller) \ 2337c478bd9Sstevel@tonic-gate if ((count) > 0) { \ 2347c478bd9Sstevel@tonic-gate pc_t *_s = ((kmem_buftag_lite_t *)(bt))->bt_history; \ 2357c478bd9Sstevel@tonic-gate pc_t *_e; \ 2367c478bd9Sstevel@tonic-gate /* memmove() the old entries down one notch */ \ 2377c478bd9Sstevel@tonic-gate for (_e = &_s[(count) - 1]; _e > _s; _e--) \ 2387c478bd9Sstevel@tonic-gate *_e = *(_e - 1); \ 2397c478bd9Sstevel@tonic-gate *_s = (uintptr_t)(caller); \ 2407c478bd9Sstevel@tonic-gate } 2417c478bd9Sstevel@tonic-gate 2427c478bd9Sstevel@tonic-gate #define KMERR_MODIFIED 0 /* buffer modified while on freelist */ 2437c478bd9Sstevel@tonic-gate #define KMERR_REDZONE 1 /* redzone violation (write past end of buf) */ 2447c478bd9Sstevel@tonic-gate #define KMERR_DUPFREE 2 /* freed a buffer twice */ 2457c478bd9Sstevel@tonic-gate #define KMERR_BADADDR 3 /* freed a bad (unallocated) address */ 2467c478bd9Sstevel@tonic-gate #define KMERR_BADBUFTAG 4 /* buftag corrupted */ 2477c478bd9Sstevel@tonic-gate #define KMERR_BADBUFCTL 5 /* bufctl corrupted */ 2487c478bd9Sstevel@tonic-gate #define KMERR_BADCACHE 6 /* freed a buffer to the wrong cache */ 2497c478bd9Sstevel@tonic-gate #define KMERR_BADSIZE 7 /* alloc size != free size */ 2507c478bd9Sstevel@tonic-gate #define KMERR_BADBASE 8 /* buffer base address wrong */ 2517c478bd9Sstevel@tonic-gate 2527c478bd9Sstevel@tonic-gate struct { 2537c478bd9Sstevel@tonic-gate hrtime_t kmp_timestamp; /* timestamp of panic */ 2547c478bd9Sstevel@tonic-gate int kmp_error; /* type of kmem error */ 2557c478bd9Sstevel@tonic-gate void *kmp_buffer; /* buffer that induced panic */ 2567c478bd9Sstevel@tonic-gate void *kmp_realbuf; /* real start address for buffer */ 2577c478bd9Sstevel@tonic-gate kmem_cache_t *kmp_cache; /* buffer's cache according to client */ 2587c478bd9Sstevel@tonic-gate kmem_cache_t *kmp_realcache; /* actual cache containing buffer */ 2597c478bd9Sstevel@tonic-gate kmem_slab_t *kmp_slab; /* slab accoring to kmem_findslab() */ 2607c478bd9Sstevel@tonic-gate kmem_bufctl_t *kmp_bufctl; /* bufctl */ 2617c478bd9Sstevel@tonic-gate } kmem_panic_info; 2627c478bd9Sstevel@tonic-gate 2637c478bd9Sstevel@tonic-gate 2647c478bd9Sstevel@tonic-gate static void 2657c478bd9Sstevel@tonic-gate copy_pattern(uint64_t pattern, void *buf_arg, size_t size) 2667c478bd9Sstevel@tonic-gate { 2677c478bd9Sstevel@tonic-gate uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 2687c478bd9Sstevel@tonic-gate uint64_t *buf = buf_arg; 2697c478bd9Sstevel@tonic-gate 2707c478bd9Sstevel@tonic-gate while (buf < bufend) 2717c478bd9Sstevel@tonic-gate *buf++ = pattern; 2727c478bd9Sstevel@tonic-gate } 2737c478bd9Sstevel@tonic-gate 2747c478bd9Sstevel@tonic-gate static void * 2757c478bd9Sstevel@tonic-gate verify_pattern(uint64_t pattern, void *buf_arg, size_t size) 2767c478bd9Sstevel@tonic-gate { 2777c478bd9Sstevel@tonic-gate uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 2787c478bd9Sstevel@tonic-gate uint64_t *buf; 2797c478bd9Sstevel@tonic-gate 2807c478bd9Sstevel@tonic-gate for (buf = buf_arg; buf < bufend; buf++) 2817c478bd9Sstevel@tonic-gate if (*buf != pattern) 2827c478bd9Sstevel@tonic-gate return (buf); 2837c478bd9Sstevel@tonic-gate return (NULL); 2847c478bd9Sstevel@tonic-gate } 2857c478bd9Sstevel@tonic-gate 2867c478bd9Sstevel@tonic-gate static void * 2877c478bd9Sstevel@tonic-gate verify_and_copy_pattern(uint64_t old, uint64_t new, void *buf_arg, size_t size) 2887c478bd9Sstevel@tonic-gate { 2897c478bd9Sstevel@tonic-gate uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 2907c478bd9Sstevel@tonic-gate uint64_t *buf; 2917c478bd9Sstevel@tonic-gate 2927c478bd9Sstevel@tonic-gate for (buf = buf_arg; buf < bufend; buf++) { 2937c478bd9Sstevel@tonic-gate if (*buf != old) { 2947c478bd9Sstevel@tonic-gate copy_pattern(old, buf_arg, 2957c478bd9Sstevel@tonic-gate (char *)buf - (char *)buf_arg); 2967c478bd9Sstevel@tonic-gate return (buf); 2977c478bd9Sstevel@tonic-gate } 2987c478bd9Sstevel@tonic-gate *buf = new; 2997c478bd9Sstevel@tonic-gate } 3007c478bd9Sstevel@tonic-gate 3017c478bd9Sstevel@tonic-gate return (NULL); 3027c478bd9Sstevel@tonic-gate } 3037c478bd9Sstevel@tonic-gate 3047c478bd9Sstevel@tonic-gate static void 3057c478bd9Sstevel@tonic-gate kmem_cache_applyall(void (*func)(kmem_cache_t *), taskq_t *tq, int tqflag) 3067c478bd9Sstevel@tonic-gate { 3077c478bd9Sstevel@tonic-gate kmem_cache_t *cp; 3087c478bd9Sstevel@tonic-gate 3097c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_lock); 3107c478bd9Sstevel@tonic-gate for (cp = kmem_null_cache.cache_next; cp != &kmem_null_cache; 3117c478bd9Sstevel@tonic-gate cp = cp->cache_next) 3127c478bd9Sstevel@tonic-gate if (tq != NULL) 3137c478bd9Sstevel@tonic-gate (void) taskq_dispatch(tq, (task_func_t *)func, cp, 3147c478bd9Sstevel@tonic-gate tqflag); 3157c478bd9Sstevel@tonic-gate else 3167c478bd9Sstevel@tonic-gate func(cp); 3177c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_lock); 3187c478bd9Sstevel@tonic-gate } 3197c478bd9Sstevel@tonic-gate 3207c478bd9Sstevel@tonic-gate static void 3217c478bd9Sstevel@tonic-gate kmem_cache_applyall_id(void (*func)(kmem_cache_t *), taskq_t *tq, int tqflag) 3227c478bd9Sstevel@tonic-gate { 3237c478bd9Sstevel@tonic-gate kmem_cache_t *cp; 3247c478bd9Sstevel@tonic-gate 3257c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_lock); 3267c478bd9Sstevel@tonic-gate for (cp = kmem_null_cache.cache_next; cp != &kmem_null_cache; 3277c478bd9Sstevel@tonic-gate cp = cp->cache_next) { 3287c478bd9Sstevel@tonic-gate if (!(cp->cache_cflags & KMC_IDENTIFIER)) 3297c478bd9Sstevel@tonic-gate continue; 3307c478bd9Sstevel@tonic-gate if (tq != NULL) 3317c478bd9Sstevel@tonic-gate (void) taskq_dispatch(tq, (task_func_t *)func, cp, 3327c478bd9Sstevel@tonic-gate tqflag); 3337c478bd9Sstevel@tonic-gate else 3347c478bd9Sstevel@tonic-gate func(cp); 3357c478bd9Sstevel@tonic-gate } 3367c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_lock); 3377c478bd9Sstevel@tonic-gate } 3387c478bd9Sstevel@tonic-gate 3397c478bd9Sstevel@tonic-gate /* 3407c478bd9Sstevel@tonic-gate * Debugging support. Given a buffer address, find its slab. 3417c478bd9Sstevel@tonic-gate */ 3427c478bd9Sstevel@tonic-gate static kmem_slab_t * 3437c478bd9Sstevel@tonic-gate kmem_findslab(kmem_cache_t *cp, void *buf) 3447c478bd9Sstevel@tonic-gate { 3457c478bd9Sstevel@tonic-gate kmem_slab_t *sp; 3467c478bd9Sstevel@tonic-gate 3477c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 3487c478bd9Sstevel@tonic-gate for (sp = cp->cache_nullslab.slab_next; 3497c478bd9Sstevel@tonic-gate sp != &cp->cache_nullslab; sp = sp->slab_next) { 3507c478bd9Sstevel@tonic-gate if (KMEM_SLAB_MEMBER(sp, buf)) { 3517c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 3527c478bd9Sstevel@tonic-gate return (sp); 3537c478bd9Sstevel@tonic-gate } 3547c478bd9Sstevel@tonic-gate } 3557c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 3567c478bd9Sstevel@tonic-gate 3577c478bd9Sstevel@tonic-gate return (NULL); 3587c478bd9Sstevel@tonic-gate } 3597c478bd9Sstevel@tonic-gate 3607c478bd9Sstevel@tonic-gate static void 3617c478bd9Sstevel@tonic-gate kmem_error(int error, kmem_cache_t *cparg, void *bufarg) 3627c478bd9Sstevel@tonic-gate { 3637c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = NULL; 3647c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp = NULL; 3657c478bd9Sstevel@tonic-gate kmem_cache_t *cp = cparg; 3667c478bd9Sstevel@tonic-gate kmem_slab_t *sp; 3677c478bd9Sstevel@tonic-gate uint64_t *off; 3687c478bd9Sstevel@tonic-gate void *buf = bufarg; 3697c478bd9Sstevel@tonic-gate 3707c478bd9Sstevel@tonic-gate kmem_logging = 0; /* stop logging when a bad thing happens */ 3717c478bd9Sstevel@tonic-gate 3727c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_timestamp = gethrtime(); 3737c478bd9Sstevel@tonic-gate 3747c478bd9Sstevel@tonic-gate sp = kmem_findslab(cp, buf); 3757c478bd9Sstevel@tonic-gate if (sp == NULL) { 3767c478bd9Sstevel@tonic-gate for (cp = kmem_null_cache.cache_prev; cp != &kmem_null_cache; 3777c478bd9Sstevel@tonic-gate cp = cp->cache_prev) { 3787c478bd9Sstevel@tonic-gate if ((sp = kmem_findslab(cp, buf)) != NULL) 3797c478bd9Sstevel@tonic-gate break; 3807c478bd9Sstevel@tonic-gate } 3817c478bd9Sstevel@tonic-gate } 3827c478bd9Sstevel@tonic-gate 3837c478bd9Sstevel@tonic-gate if (sp == NULL) { 3847c478bd9Sstevel@tonic-gate cp = NULL; 3857c478bd9Sstevel@tonic-gate error = KMERR_BADADDR; 3867c478bd9Sstevel@tonic-gate } else { 3877c478bd9Sstevel@tonic-gate if (cp != cparg) 3887c478bd9Sstevel@tonic-gate error = KMERR_BADCACHE; 3897c478bd9Sstevel@tonic-gate else 3907c478bd9Sstevel@tonic-gate buf = (char *)bufarg - ((uintptr_t)bufarg - 3917c478bd9Sstevel@tonic-gate (uintptr_t)sp->slab_base) % cp->cache_chunksize; 3927c478bd9Sstevel@tonic-gate if (buf != bufarg) 3937c478bd9Sstevel@tonic-gate error = KMERR_BADBASE; 3947c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_BUFTAG) 3957c478bd9Sstevel@tonic-gate btp = KMEM_BUFTAG(cp, buf); 3967c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) { 3977c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 3987c478bd9Sstevel@tonic-gate for (bcp = *KMEM_HASH(cp, buf); bcp; bcp = bcp->bc_next) 3997c478bd9Sstevel@tonic-gate if (bcp->bc_addr == buf) 4007c478bd9Sstevel@tonic-gate break; 4017c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 4027c478bd9Sstevel@tonic-gate if (bcp == NULL && btp != NULL) 4037c478bd9Sstevel@tonic-gate bcp = btp->bt_bufctl; 4047c478bd9Sstevel@tonic-gate if (kmem_findslab(cp->cache_bufctl_cache, bcp) == 4057c478bd9Sstevel@tonic-gate NULL || P2PHASE((uintptr_t)bcp, KMEM_ALIGN) || 4067c478bd9Sstevel@tonic-gate bcp->bc_addr != buf) { 4077c478bd9Sstevel@tonic-gate error = KMERR_BADBUFCTL; 4087c478bd9Sstevel@tonic-gate bcp = NULL; 4097c478bd9Sstevel@tonic-gate } 4107c478bd9Sstevel@tonic-gate } 4117c478bd9Sstevel@tonic-gate } 4127c478bd9Sstevel@tonic-gate 4137c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_error = error; 4147c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_buffer = bufarg; 4157c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_realbuf = buf; 4167c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_cache = cparg; 4177c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_realcache = cp; 4187c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_slab = sp; 4197c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_bufctl = bcp; 4207c478bd9Sstevel@tonic-gate 4217c478bd9Sstevel@tonic-gate printf("kernel memory allocator: "); 4227c478bd9Sstevel@tonic-gate 4237c478bd9Sstevel@tonic-gate switch (error) { 4247c478bd9Sstevel@tonic-gate 4257c478bd9Sstevel@tonic-gate case KMERR_MODIFIED: 4267c478bd9Sstevel@tonic-gate printf("buffer modified after being freed\n"); 4277c478bd9Sstevel@tonic-gate off = verify_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify); 4287c478bd9Sstevel@tonic-gate if (off == NULL) /* shouldn't happen */ 4297c478bd9Sstevel@tonic-gate off = buf; 4307c478bd9Sstevel@tonic-gate printf("modification occurred at offset 0x%lx " 4317c478bd9Sstevel@tonic-gate "(0x%llx replaced by 0x%llx)\n", 4327c478bd9Sstevel@tonic-gate (uintptr_t)off - (uintptr_t)buf, 4337c478bd9Sstevel@tonic-gate (longlong_t)KMEM_FREE_PATTERN, (longlong_t)*off); 4347c478bd9Sstevel@tonic-gate break; 4357c478bd9Sstevel@tonic-gate 4367c478bd9Sstevel@tonic-gate case KMERR_REDZONE: 4377c478bd9Sstevel@tonic-gate printf("redzone violation: write past end of buffer\n"); 4387c478bd9Sstevel@tonic-gate break; 4397c478bd9Sstevel@tonic-gate 4407c478bd9Sstevel@tonic-gate case KMERR_BADADDR: 4417c478bd9Sstevel@tonic-gate printf("invalid free: buffer not in cache\n"); 4427c478bd9Sstevel@tonic-gate break; 4437c478bd9Sstevel@tonic-gate 4447c478bd9Sstevel@tonic-gate case KMERR_DUPFREE: 4457c478bd9Sstevel@tonic-gate printf("duplicate free: buffer freed twice\n"); 4467c478bd9Sstevel@tonic-gate break; 4477c478bd9Sstevel@tonic-gate 4487c478bd9Sstevel@tonic-gate case KMERR_BADBUFTAG: 4497c478bd9Sstevel@tonic-gate printf("boundary tag corrupted\n"); 4507c478bd9Sstevel@tonic-gate printf("bcp ^ bxstat = %lx, should be %lx\n", 4517c478bd9Sstevel@tonic-gate (intptr_t)btp->bt_bufctl ^ btp->bt_bxstat, 4527c478bd9Sstevel@tonic-gate KMEM_BUFTAG_FREE); 4537c478bd9Sstevel@tonic-gate break; 4547c478bd9Sstevel@tonic-gate 4557c478bd9Sstevel@tonic-gate case KMERR_BADBUFCTL: 4567c478bd9Sstevel@tonic-gate printf("bufctl corrupted\n"); 4577c478bd9Sstevel@tonic-gate break; 4587c478bd9Sstevel@tonic-gate 4597c478bd9Sstevel@tonic-gate case KMERR_BADCACHE: 4607c478bd9Sstevel@tonic-gate printf("buffer freed to wrong cache\n"); 4617c478bd9Sstevel@tonic-gate printf("buffer was allocated from %s,\n", cp->cache_name); 4627c478bd9Sstevel@tonic-gate printf("caller attempting free to %s.\n", cparg->cache_name); 4637c478bd9Sstevel@tonic-gate break; 4647c478bd9Sstevel@tonic-gate 4657c478bd9Sstevel@tonic-gate case KMERR_BADSIZE: 4667c478bd9Sstevel@tonic-gate printf("bad free: free size (%u) != alloc size (%u)\n", 4677c478bd9Sstevel@tonic-gate KMEM_SIZE_DECODE(((uint32_t *)btp)[0]), 4687c478bd9Sstevel@tonic-gate KMEM_SIZE_DECODE(((uint32_t *)btp)[1])); 4697c478bd9Sstevel@tonic-gate break; 4707c478bd9Sstevel@tonic-gate 4717c478bd9Sstevel@tonic-gate case KMERR_BADBASE: 4727c478bd9Sstevel@tonic-gate printf("bad free: free address (%p) != alloc address (%p)\n", 4737c478bd9Sstevel@tonic-gate bufarg, buf); 4747c478bd9Sstevel@tonic-gate break; 4757c478bd9Sstevel@tonic-gate } 4767c478bd9Sstevel@tonic-gate 4777c478bd9Sstevel@tonic-gate printf("buffer=%p bufctl=%p cache: %s\n", 4787c478bd9Sstevel@tonic-gate bufarg, (void *)bcp, cparg->cache_name); 4797c478bd9Sstevel@tonic-gate 4807c478bd9Sstevel@tonic-gate if (bcp != NULL && (cp->cache_flags & KMF_AUDIT) && 4817c478bd9Sstevel@tonic-gate error != KMERR_BADBUFCTL) { 4827c478bd9Sstevel@tonic-gate int d; 4837c478bd9Sstevel@tonic-gate timestruc_t ts; 4847c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *bcap = (kmem_bufctl_audit_t *)bcp; 4857c478bd9Sstevel@tonic-gate 4867c478bd9Sstevel@tonic-gate hrt2ts(kmem_panic_info.kmp_timestamp - bcap->bc_timestamp, &ts); 4877c478bd9Sstevel@tonic-gate printf("previous transaction on buffer %p:\n", buf); 4887c478bd9Sstevel@tonic-gate printf("thread=%p time=T-%ld.%09ld slab=%p cache: %s\n", 4897c478bd9Sstevel@tonic-gate (void *)bcap->bc_thread, ts.tv_sec, ts.tv_nsec, 4907c478bd9Sstevel@tonic-gate (void *)sp, cp->cache_name); 4917c478bd9Sstevel@tonic-gate for (d = 0; d < MIN(bcap->bc_depth, KMEM_STACK_DEPTH); d++) { 4927c478bd9Sstevel@tonic-gate ulong_t off; 4937c478bd9Sstevel@tonic-gate char *sym = kobj_getsymname(bcap->bc_stack[d], &off); 4947c478bd9Sstevel@tonic-gate printf("%s+%lx\n", sym ? sym : "?", off); 4957c478bd9Sstevel@tonic-gate } 4967c478bd9Sstevel@tonic-gate } 4977c478bd9Sstevel@tonic-gate if (kmem_panic > 0) 4987c478bd9Sstevel@tonic-gate panic("kernel heap corruption detected"); 4997c478bd9Sstevel@tonic-gate if (kmem_panic == 0) 5007c478bd9Sstevel@tonic-gate debug_enter(NULL); 5017c478bd9Sstevel@tonic-gate kmem_logging = 1; /* resume logging */ 5027c478bd9Sstevel@tonic-gate } 5037c478bd9Sstevel@tonic-gate 5047c478bd9Sstevel@tonic-gate static kmem_log_header_t * 5057c478bd9Sstevel@tonic-gate kmem_log_init(size_t logsize) 5067c478bd9Sstevel@tonic-gate { 5077c478bd9Sstevel@tonic-gate kmem_log_header_t *lhp; 5087c478bd9Sstevel@tonic-gate int nchunks = 4 * max_ncpus; 5097c478bd9Sstevel@tonic-gate size_t lhsize = (size_t)&((kmem_log_header_t *)0)->lh_cpu[max_ncpus]; 5107c478bd9Sstevel@tonic-gate int i; 5117c478bd9Sstevel@tonic-gate 5127c478bd9Sstevel@tonic-gate /* 5137c478bd9Sstevel@tonic-gate * Make sure that lhp->lh_cpu[] is nicely aligned 5147c478bd9Sstevel@tonic-gate * to prevent false sharing of cache lines. 5157c478bd9Sstevel@tonic-gate */ 5167c478bd9Sstevel@tonic-gate lhsize = P2ROUNDUP(lhsize, KMEM_ALIGN); 5177c478bd9Sstevel@tonic-gate lhp = vmem_xalloc(kmem_log_arena, lhsize, 64, P2NPHASE(lhsize, 64), 0, 5187c478bd9Sstevel@tonic-gate NULL, NULL, VM_SLEEP); 5197c478bd9Sstevel@tonic-gate bzero(lhp, lhsize); 5207c478bd9Sstevel@tonic-gate 5217c478bd9Sstevel@tonic-gate mutex_init(&lhp->lh_lock, NULL, MUTEX_DEFAULT, NULL); 5227c478bd9Sstevel@tonic-gate lhp->lh_nchunks = nchunks; 5237c478bd9Sstevel@tonic-gate lhp->lh_chunksize = P2ROUNDUP(logsize / nchunks + 1, PAGESIZE); 5247c478bd9Sstevel@tonic-gate lhp->lh_base = vmem_alloc(kmem_log_arena, 5257c478bd9Sstevel@tonic-gate lhp->lh_chunksize * nchunks, VM_SLEEP); 5267c478bd9Sstevel@tonic-gate lhp->lh_free = vmem_alloc(kmem_log_arena, 5277c478bd9Sstevel@tonic-gate nchunks * sizeof (int), VM_SLEEP); 5287c478bd9Sstevel@tonic-gate bzero(lhp->lh_base, lhp->lh_chunksize * nchunks); 5297c478bd9Sstevel@tonic-gate 5307c478bd9Sstevel@tonic-gate for (i = 0; i < max_ncpus; i++) { 5317c478bd9Sstevel@tonic-gate kmem_cpu_log_header_t *clhp = &lhp->lh_cpu[i]; 5327c478bd9Sstevel@tonic-gate mutex_init(&clhp->clh_lock, NULL, MUTEX_DEFAULT, NULL); 5337c478bd9Sstevel@tonic-gate clhp->clh_chunk = i; 5347c478bd9Sstevel@tonic-gate } 5357c478bd9Sstevel@tonic-gate 5367c478bd9Sstevel@tonic-gate for (i = max_ncpus; i < nchunks; i++) 5377c478bd9Sstevel@tonic-gate lhp->lh_free[i] = i; 5387c478bd9Sstevel@tonic-gate 5397c478bd9Sstevel@tonic-gate lhp->lh_head = max_ncpus; 5407c478bd9Sstevel@tonic-gate lhp->lh_tail = 0; 5417c478bd9Sstevel@tonic-gate 5427c478bd9Sstevel@tonic-gate return (lhp); 5437c478bd9Sstevel@tonic-gate } 5447c478bd9Sstevel@tonic-gate 5457c478bd9Sstevel@tonic-gate static void * 5467c478bd9Sstevel@tonic-gate kmem_log_enter(kmem_log_header_t *lhp, void *data, size_t size) 5477c478bd9Sstevel@tonic-gate { 5487c478bd9Sstevel@tonic-gate void *logspace; 5497c478bd9Sstevel@tonic-gate kmem_cpu_log_header_t *clhp = &lhp->lh_cpu[CPU->cpu_seqid]; 5507c478bd9Sstevel@tonic-gate 5517c478bd9Sstevel@tonic-gate if (lhp == NULL || kmem_logging == 0 || panicstr) 5527c478bd9Sstevel@tonic-gate return (NULL); 5537c478bd9Sstevel@tonic-gate 5547c478bd9Sstevel@tonic-gate mutex_enter(&clhp->clh_lock); 5557c478bd9Sstevel@tonic-gate clhp->clh_hits++; 5567c478bd9Sstevel@tonic-gate if (size > clhp->clh_avail) { 5577c478bd9Sstevel@tonic-gate mutex_enter(&lhp->lh_lock); 5587c478bd9Sstevel@tonic-gate lhp->lh_hits++; 5597c478bd9Sstevel@tonic-gate lhp->lh_free[lhp->lh_tail] = clhp->clh_chunk; 5607c478bd9Sstevel@tonic-gate lhp->lh_tail = (lhp->lh_tail + 1) % lhp->lh_nchunks; 5617c478bd9Sstevel@tonic-gate clhp->clh_chunk = lhp->lh_free[lhp->lh_head]; 5627c478bd9Sstevel@tonic-gate lhp->lh_head = (lhp->lh_head + 1) % lhp->lh_nchunks; 5637c478bd9Sstevel@tonic-gate clhp->clh_current = lhp->lh_base + 5647c478bd9Sstevel@tonic-gate clhp->clh_chunk * lhp->lh_chunksize; 5657c478bd9Sstevel@tonic-gate clhp->clh_avail = lhp->lh_chunksize; 5667c478bd9Sstevel@tonic-gate if (size > lhp->lh_chunksize) 5677c478bd9Sstevel@tonic-gate size = lhp->lh_chunksize; 5687c478bd9Sstevel@tonic-gate mutex_exit(&lhp->lh_lock); 5697c478bd9Sstevel@tonic-gate } 5707c478bd9Sstevel@tonic-gate logspace = clhp->clh_current; 5717c478bd9Sstevel@tonic-gate clhp->clh_current += size; 5727c478bd9Sstevel@tonic-gate clhp->clh_avail -= size; 5737c478bd9Sstevel@tonic-gate bcopy(data, logspace, size); 5747c478bd9Sstevel@tonic-gate mutex_exit(&clhp->clh_lock); 5757c478bd9Sstevel@tonic-gate return (logspace); 5767c478bd9Sstevel@tonic-gate } 5777c478bd9Sstevel@tonic-gate 5787c478bd9Sstevel@tonic-gate #define KMEM_AUDIT(lp, cp, bcp) \ 5797c478bd9Sstevel@tonic-gate { \ 5807c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *_bcp = (kmem_bufctl_audit_t *)(bcp); \ 5817c478bd9Sstevel@tonic-gate _bcp->bc_timestamp = gethrtime(); \ 5827c478bd9Sstevel@tonic-gate _bcp->bc_thread = curthread; \ 5837c478bd9Sstevel@tonic-gate _bcp->bc_depth = getpcstack(_bcp->bc_stack, KMEM_STACK_DEPTH); \ 5847c478bd9Sstevel@tonic-gate _bcp->bc_lastlog = kmem_log_enter((lp), _bcp, sizeof (*_bcp)); \ 5857c478bd9Sstevel@tonic-gate } 5867c478bd9Sstevel@tonic-gate 5877c478bd9Sstevel@tonic-gate static void 5887c478bd9Sstevel@tonic-gate kmem_log_event(kmem_log_header_t *lp, kmem_cache_t *cp, 5897c478bd9Sstevel@tonic-gate kmem_slab_t *sp, void *addr) 5907c478bd9Sstevel@tonic-gate { 5917c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t bca; 5927c478bd9Sstevel@tonic-gate 5937c478bd9Sstevel@tonic-gate bzero(&bca, sizeof (kmem_bufctl_audit_t)); 5947c478bd9Sstevel@tonic-gate bca.bc_addr = addr; 5957c478bd9Sstevel@tonic-gate bca.bc_slab = sp; 5967c478bd9Sstevel@tonic-gate bca.bc_cache = cp; 5977c478bd9Sstevel@tonic-gate KMEM_AUDIT(lp, cp, &bca); 5987c478bd9Sstevel@tonic-gate } 5997c478bd9Sstevel@tonic-gate 6007c478bd9Sstevel@tonic-gate /* 6017c478bd9Sstevel@tonic-gate * Create a new slab for cache cp. 6027c478bd9Sstevel@tonic-gate */ 6037c478bd9Sstevel@tonic-gate static kmem_slab_t * 6047c478bd9Sstevel@tonic-gate kmem_slab_create(kmem_cache_t *cp, int kmflag) 6057c478bd9Sstevel@tonic-gate { 6067c478bd9Sstevel@tonic-gate size_t slabsize = cp->cache_slabsize; 6077c478bd9Sstevel@tonic-gate size_t chunksize = cp->cache_chunksize; 6087c478bd9Sstevel@tonic-gate int cache_flags = cp->cache_flags; 6097c478bd9Sstevel@tonic-gate size_t color, chunks; 6107c478bd9Sstevel@tonic-gate char *buf, *slab; 6117c478bd9Sstevel@tonic-gate kmem_slab_t *sp; 6127c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp; 6137c478bd9Sstevel@tonic-gate vmem_t *vmp = cp->cache_arena; 6147c478bd9Sstevel@tonic-gate 6157c478bd9Sstevel@tonic-gate color = cp->cache_color + cp->cache_align; 6167c478bd9Sstevel@tonic-gate if (color > cp->cache_maxcolor) 6177c478bd9Sstevel@tonic-gate color = cp->cache_mincolor; 6187c478bd9Sstevel@tonic-gate cp->cache_color = color; 6197c478bd9Sstevel@tonic-gate 6207c478bd9Sstevel@tonic-gate slab = vmem_alloc(vmp, slabsize, kmflag & KM_VMFLAGS); 6217c478bd9Sstevel@tonic-gate 6227c478bd9Sstevel@tonic-gate if (slab == NULL) 6237c478bd9Sstevel@tonic-gate goto vmem_alloc_failure; 6247c478bd9Sstevel@tonic-gate 6257c478bd9Sstevel@tonic-gate ASSERT(P2PHASE((uintptr_t)slab, vmp->vm_quantum) == 0); 6267c478bd9Sstevel@tonic-gate 6277c478bd9Sstevel@tonic-gate if (!(cp->cache_cflags & KMC_NOTOUCH)) 6287c478bd9Sstevel@tonic-gate copy_pattern(KMEM_UNINITIALIZED_PATTERN, slab, slabsize); 6297c478bd9Sstevel@tonic-gate 6307c478bd9Sstevel@tonic-gate if (cache_flags & KMF_HASH) { 6317c478bd9Sstevel@tonic-gate if ((sp = kmem_cache_alloc(kmem_slab_cache, kmflag)) == NULL) 6327c478bd9Sstevel@tonic-gate goto slab_alloc_failure; 6337c478bd9Sstevel@tonic-gate chunks = (slabsize - color) / chunksize; 6347c478bd9Sstevel@tonic-gate } else { 6357c478bd9Sstevel@tonic-gate sp = KMEM_SLAB(cp, slab); 6367c478bd9Sstevel@tonic-gate chunks = (slabsize - sizeof (kmem_slab_t) - color) / chunksize; 6377c478bd9Sstevel@tonic-gate } 6387c478bd9Sstevel@tonic-gate 6397c478bd9Sstevel@tonic-gate sp->slab_cache = cp; 6407c478bd9Sstevel@tonic-gate sp->slab_head = NULL; 6417c478bd9Sstevel@tonic-gate sp->slab_refcnt = 0; 6427c478bd9Sstevel@tonic-gate sp->slab_base = buf = slab + color; 6437c478bd9Sstevel@tonic-gate sp->slab_chunks = chunks; 6447c478bd9Sstevel@tonic-gate 6457c478bd9Sstevel@tonic-gate ASSERT(chunks > 0); 6467c478bd9Sstevel@tonic-gate while (chunks-- != 0) { 6477c478bd9Sstevel@tonic-gate if (cache_flags & KMF_HASH) { 6487c478bd9Sstevel@tonic-gate bcp = kmem_cache_alloc(cp->cache_bufctl_cache, kmflag); 6497c478bd9Sstevel@tonic-gate if (bcp == NULL) 6507c478bd9Sstevel@tonic-gate goto bufctl_alloc_failure; 6517c478bd9Sstevel@tonic-gate if (cache_flags & KMF_AUDIT) { 6527c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *bcap = 6537c478bd9Sstevel@tonic-gate (kmem_bufctl_audit_t *)bcp; 6547c478bd9Sstevel@tonic-gate bzero(bcap, sizeof (kmem_bufctl_audit_t)); 6557c478bd9Sstevel@tonic-gate bcap->bc_cache = cp; 6567c478bd9Sstevel@tonic-gate } 6577c478bd9Sstevel@tonic-gate bcp->bc_addr = buf; 6587c478bd9Sstevel@tonic-gate bcp->bc_slab = sp; 6597c478bd9Sstevel@tonic-gate } else { 6607c478bd9Sstevel@tonic-gate bcp = KMEM_BUFCTL(cp, buf); 6617c478bd9Sstevel@tonic-gate } 6627c478bd9Sstevel@tonic-gate if (cache_flags & KMF_BUFTAG) { 6637c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 6647c478bd9Sstevel@tonic-gate btp->bt_redzone = KMEM_REDZONE_PATTERN; 6657c478bd9Sstevel@tonic-gate btp->bt_bufctl = bcp; 6667c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE; 6677c478bd9Sstevel@tonic-gate if (cache_flags & KMF_DEADBEEF) { 6687c478bd9Sstevel@tonic-gate copy_pattern(KMEM_FREE_PATTERN, buf, 6697c478bd9Sstevel@tonic-gate cp->cache_verify); 6707c478bd9Sstevel@tonic-gate } 6717c478bd9Sstevel@tonic-gate } 6727c478bd9Sstevel@tonic-gate bcp->bc_next = sp->slab_head; 6737c478bd9Sstevel@tonic-gate sp->slab_head = bcp; 6747c478bd9Sstevel@tonic-gate buf += chunksize; 6757c478bd9Sstevel@tonic-gate } 6767c478bd9Sstevel@tonic-gate 6777c478bd9Sstevel@tonic-gate kmem_log_event(kmem_slab_log, cp, sp, slab); 6787c478bd9Sstevel@tonic-gate 6797c478bd9Sstevel@tonic-gate return (sp); 6807c478bd9Sstevel@tonic-gate 6817c478bd9Sstevel@tonic-gate bufctl_alloc_failure: 6827c478bd9Sstevel@tonic-gate 6837c478bd9Sstevel@tonic-gate while ((bcp = sp->slab_head) != NULL) { 6847c478bd9Sstevel@tonic-gate sp->slab_head = bcp->bc_next; 6857c478bd9Sstevel@tonic-gate kmem_cache_free(cp->cache_bufctl_cache, bcp); 6867c478bd9Sstevel@tonic-gate } 6877c478bd9Sstevel@tonic-gate kmem_cache_free(kmem_slab_cache, sp); 6887c478bd9Sstevel@tonic-gate 6897c478bd9Sstevel@tonic-gate slab_alloc_failure: 6907c478bd9Sstevel@tonic-gate 6917c478bd9Sstevel@tonic-gate vmem_free(vmp, slab, slabsize); 6927c478bd9Sstevel@tonic-gate 6937c478bd9Sstevel@tonic-gate vmem_alloc_failure: 6947c478bd9Sstevel@tonic-gate 6957c478bd9Sstevel@tonic-gate kmem_log_event(kmem_failure_log, cp, NULL, NULL); 6967c478bd9Sstevel@tonic-gate atomic_add_64(&cp->cache_alloc_fail, 1); 6977c478bd9Sstevel@tonic-gate 6987c478bd9Sstevel@tonic-gate return (NULL); 6997c478bd9Sstevel@tonic-gate } 7007c478bd9Sstevel@tonic-gate 7017c478bd9Sstevel@tonic-gate /* 7027c478bd9Sstevel@tonic-gate * Destroy a slab. 7037c478bd9Sstevel@tonic-gate */ 7047c478bd9Sstevel@tonic-gate static void 7057c478bd9Sstevel@tonic-gate kmem_slab_destroy(kmem_cache_t *cp, kmem_slab_t *sp) 7067c478bd9Sstevel@tonic-gate { 7077c478bd9Sstevel@tonic-gate vmem_t *vmp = cp->cache_arena; 7087c478bd9Sstevel@tonic-gate void *slab = (void *)P2ALIGN((uintptr_t)sp->slab_base, vmp->vm_quantum); 7097c478bd9Sstevel@tonic-gate 7107c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) { 7117c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp; 7127c478bd9Sstevel@tonic-gate while ((bcp = sp->slab_head) != NULL) { 7137c478bd9Sstevel@tonic-gate sp->slab_head = bcp->bc_next; 7147c478bd9Sstevel@tonic-gate kmem_cache_free(cp->cache_bufctl_cache, bcp); 7157c478bd9Sstevel@tonic-gate } 7167c478bd9Sstevel@tonic-gate kmem_cache_free(kmem_slab_cache, sp); 7177c478bd9Sstevel@tonic-gate } 7187c478bd9Sstevel@tonic-gate vmem_free(vmp, slab, cp->cache_slabsize); 7197c478bd9Sstevel@tonic-gate } 7207c478bd9Sstevel@tonic-gate 7217c478bd9Sstevel@tonic-gate /* 7227c478bd9Sstevel@tonic-gate * Allocate a raw (unconstructed) buffer from cp's slab layer. 7237c478bd9Sstevel@tonic-gate */ 7247c478bd9Sstevel@tonic-gate static void * 7257c478bd9Sstevel@tonic-gate kmem_slab_alloc(kmem_cache_t *cp, int kmflag) 7267c478bd9Sstevel@tonic-gate { 7277c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp, **hash_bucket; 7287c478bd9Sstevel@tonic-gate kmem_slab_t *sp; 7297c478bd9Sstevel@tonic-gate void *buf; 7307c478bd9Sstevel@tonic-gate 7317c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 7327c478bd9Sstevel@tonic-gate cp->cache_slab_alloc++; 7337c478bd9Sstevel@tonic-gate sp = cp->cache_freelist; 7347c478bd9Sstevel@tonic-gate ASSERT(sp->slab_cache == cp); 7357c478bd9Sstevel@tonic-gate if (sp->slab_head == NULL) { 7367c478bd9Sstevel@tonic-gate /* 7377c478bd9Sstevel@tonic-gate * The freelist is empty. Create a new slab. 7387c478bd9Sstevel@tonic-gate */ 7397c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 7407c478bd9Sstevel@tonic-gate if ((sp = kmem_slab_create(cp, kmflag)) == NULL) 7417c478bd9Sstevel@tonic-gate return (NULL); 7427c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 7437c478bd9Sstevel@tonic-gate cp->cache_slab_create++; 7447c478bd9Sstevel@tonic-gate if ((cp->cache_buftotal += sp->slab_chunks) > cp->cache_bufmax) 7457c478bd9Sstevel@tonic-gate cp->cache_bufmax = cp->cache_buftotal; 7467c478bd9Sstevel@tonic-gate sp->slab_next = cp->cache_freelist; 7477c478bd9Sstevel@tonic-gate sp->slab_prev = cp->cache_freelist->slab_prev; 7487c478bd9Sstevel@tonic-gate sp->slab_next->slab_prev = sp; 7497c478bd9Sstevel@tonic-gate sp->slab_prev->slab_next = sp; 7507c478bd9Sstevel@tonic-gate cp->cache_freelist = sp; 7517c478bd9Sstevel@tonic-gate } 7527c478bd9Sstevel@tonic-gate 7537c478bd9Sstevel@tonic-gate sp->slab_refcnt++; 7547c478bd9Sstevel@tonic-gate ASSERT(sp->slab_refcnt <= sp->slab_chunks); 7557c478bd9Sstevel@tonic-gate 7567c478bd9Sstevel@tonic-gate /* 7577c478bd9Sstevel@tonic-gate * If we're taking the last buffer in the slab, 7587c478bd9Sstevel@tonic-gate * remove the slab from the cache's freelist. 7597c478bd9Sstevel@tonic-gate */ 7607c478bd9Sstevel@tonic-gate bcp = sp->slab_head; 7617c478bd9Sstevel@tonic-gate if ((sp->slab_head = bcp->bc_next) == NULL) { 7627c478bd9Sstevel@tonic-gate cp->cache_freelist = sp->slab_next; 7637c478bd9Sstevel@tonic-gate ASSERT(sp->slab_refcnt == sp->slab_chunks); 7647c478bd9Sstevel@tonic-gate } 7657c478bd9Sstevel@tonic-gate 7667c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) { 7677c478bd9Sstevel@tonic-gate /* 7687c478bd9Sstevel@tonic-gate * Add buffer to allocated-address hash table. 7697c478bd9Sstevel@tonic-gate */ 7707c478bd9Sstevel@tonic-gate buf = bcp->bc_addr; 7717c478bd9Sstevel@tonic-gate hash_bucket = KMEM_HASH(cp, buf); 7727c478bd9Sstevel@tonic-gate bcp->bc_next = *hash_bucket; 7737c478bd9Sstevel@tonic-gate *hash_bucket = bcp; 7747c478bd9Sstevel@tonic-gate if ((cp->cache_flags & (KMF_AUDIT | KMF_BUFTAG)) == KMF_AUDIT) { 7757c478bd9Sstevel@tonic-gate KMEM_AUDIT(kmem_transaction_log, cp, bcp); 7767c478bd9Sstevel@tonic-gate } 7777c478bd9Sstevel@tonic-gate } else { 7787c478bd9Sstevel@tonic-gate buf = KMEM_BUF(cp, bcp); 7797c478bd9Sstevel@tonic-gate } 7807c478bd9Sstevel@tonic-gate 7817c478bd9Sstevel@tonic-gate ASSERT(KMEM_SLAB_MEMBER(sp, buf)); 7827c478bd9Sstevel@tonic-gate 7837c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 7847c478bd9Sstevel@tonic-gate 7857c478bd9Sstevel@tonic-gate return (buf); 7867c478bd9Sstevel@tonic-gate } 7877c478bd9Sstevel@tonic-gate 7887c478bd9Sstevel@tonic-gate /* 7897c478bd9Sstevel@tonic-gate * Free a raw (unconstructed) buffer to cp's slab layer. 7907c478bd9Sstevel@tonic-gate */ 7917c478bd9Sstevel@tonic-gate static void 7927c478bd9Sstevel@tonic-gate kmem_slab_free(kmem_cache_t *cp, void *buf) 7937c478bd9Sstevel@tonic-gate { 7947c478bd9Sstevel@tonic-gate kmem_slab_t *sp; 7957c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp, **prev_bcpp; 7967c478bd9Sstevel@tonic-gate 7977c478bd9Sstevel@tonic-gate ASSERT(buf != NULL); 7987c478bd9Sstevel@tonic-gate 7997c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 8007c478bd9Sstevel@tonic-gate cp->cache_slab_free++; 8017c478bd9Sstevel@tonic-gate 8027c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) { 8037c478bd9Sstevel@tonic-gate /* 8047c478bd9Sstevel@tonic-gate * Look up buffer in allocated-address hash table. 8057c478bd9Sstevel@tonic-gate */ 8067c478bd9Sstevel@tonic-gate prev_bcpp = KMEM_HASH(cp, buf); 8077c478bd9Sstevel@tonic-gate while ((bcp = *prev_bcpp) != NULL) { 8087c478bd9Sstevel@tonic-gate if (bcp->bc_addr == buf) { 8097c478bd9Sstevel@tonic-gate *prev_bcpp = bcp->bc_next; 8107c478bd9Sstevel@tonic-gate sp = bcp->bc_slab; 8117c478bd9Sstevel@tonic-gate break; 8127c478bd9Sstevel@tonic-gate } 8137c478bd9Sstevel@tonic-gate cp->cache_lookup_depth++; 8147c478bd9Sstevel@tonic-gate prev_bcpp = &bcp->bc_next; 8157c478bd9Sstevel@tonic-gate } 8167c478bd9Sstevel@tonic-gate } else { 8177c478bd9Sstevel@tonic-gate bcp = KMEM_BUFCTL(cp, buf); 8187c478bd9Sstevel@tonic-gate sp = KMEM_SLAB(cp, buf); 8197c478bd9Sstevel@tonic-gate } 8207c478bd9Sstevel@tonic-gate 8217c478bd9Sstevel@tonic-gate if (bcp == NULL || sp->slab_cache != cp || !KMEM_SLAB_MEMBER(sp, buf)) { 8227c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 8237c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADADDR, cp, buf); 8247c478bd9Sstevel@tonic-gate return; 8257c478bd9Sstevel@tonic-gate } 8267c478bd9Sstevel@tonic-gate 8277c478bd9Sstevel@tonic-gate if ((cp->cache_flags & (KMF_AUDIT | KMF_BUFTAG)) == KMF_AUDIT) { 8287c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_CONTENTS) 8297c478bd9Sstevel@tonic-gate ((kmem_bufctl_audit_t *)bcp)->bc_contents = 8307c478bd9Sstevel@tonic-gate kmem_log_enter(kmem_content_log, buf, 8317c478bd9Sstevel@tonic-gate cp->cache_contents); 8327c478bd9Sstevel@tonic-gate KMEM_AUDIT(kmem_transaction_log, cp, bcp); 8337c478bd9Sstevel@tonic-gate } 8347c478bd9Sstevel@tonic-gate 8357c478bd9Sstevel@tonic-gate /* 8367c478bd9Sstevel@tonic-gate * If this slab isn't currently on the freelist, put it there. 8377c478bd9Sstevel@tonic-gate */ 8387c478bd9Sstevel@tonic-gate if (sp->slab_head == NULL) { 8397c478bd9Sstevel@tonic-gate ASSERT(sp->slab_refcnt == sp->slab_chunks); 8407c478bd9Sstevel@tonic-gate ASSERT(cp->cache_freelist != sp); 8417c478bd9Sstevel@tonic-gate sp->slab_next->slab_prev = sp->slab_prev; 8427c478bd9Sstevel@tonic-gate sp->slab_prev->slab_next = sp->slab_next; 8437c478bd9Sstevel@tonic-gate sp->slab_next = cp->cache_freelist; 8447c478bd9Sstevel@tonic-gate sp->slab_prev = cp->cache_freelist->slab_prev; 8457c478bd9Sstevel@tonic-gate sp->slab_next->slab_prev = sp; 8467c478bd9Sstevel@tonic-gate sp->slab_prev->slab_next = sp; 8477c478bd9Sstevel@tonic-gate cp->cache_freelist = sp; 8487c478bd9Sstevel@tonic-gate } 8497c478bd9Sstevel@tonic-gate 8507c478bd9Sstevel@tonic-gate bcp->bc_next = sp->slab_head; 8517c478bd9Sstevel@tonic-gate sp->slab_head = bcp; 8527c478bd9Sstevel@tonic-gate 8537c478bd9Sstevel@tonic-gate ASSERT(sp->slab_refcnt >= 1); 8547c478bd9Sstevel@tonic-gate if (--sp->slab_refcnt == 0) { 8557c478bd9Sstevel@tonic-gate /* 8567c478bd9Sstevel@tonic-gate * There are no outstanding allocations from this slab, 8577c478bd9Sstevel@tonic-gate * so we can reclaim the memory. 8587c478bd9Sstevel@tonic-gate */ 8597c478bd9Sstevel@tonic-gate sp->slab_next->slab_prev = sp->slab_prev; 8607c478bd9Sstevel@tonic-gate sp->slab_prev->slab_next = sp->slab_next; 8617c478bd9Sstevel@tonic-gate if (sp == cp->cache_freelist) 8627c478bd9Sstevel@tonic-gate cp->cache_freelist = sp->slab_next; 8637c478bd9Sstevel@tonic-gate cp->cache_slab_destroy++; 8647c478bd9Sstevel@tonic-gate cp->cache_buftotal -= sp->slab_chunks; 8657c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 8667c478bd9Sstevel@tonic-gate kmem_slab_destroy(cp, sp); 8677c478bd9Sstevel@tonic-gate return; 8687c478bd9Sstevel@tonic-gate } 8697c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 8707c478bd9Sstevel@tonic-gate } 8717c478bd9Sstevel@tonic-gate 8727c478bd9Sstevel@tonic-gate static int 8737c478bd9Sstevel@tonic-gate kmem_cache_alloc_debug(kmem_cache_t *cp, void *buf, int kmflag, int construct, 8747c478bd9Sstevel@tonic-gate caddr_t caller) 8757c478bd9Sstevel@tonic-gate { 8767c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 8777c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *bcp = (kmem_bufctl_audit_t *)btp->bt_bufctl; 8787c478bd9Sstevel@tonic-gate uint32_t mtbf; 8797c478bd9Sstevel@tonic-gate 8807c478bd9Sstevel@tonic-gate if (btp->bt_bxstat != ((intptr_t)bcp ^ KMEM_BUFTAG_FREE)) { 8817c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADBUFTAG, cp, buf); 8827c478bd9Sstevel@tonic-gate return (-1); 8837c478bd9Sstevel@tonic-gate } 8847c478bd9Sstevel@tonic-gate 8857c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_ALLOC; 8867c478bd9Sstevel@tonic-gate 8877c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_HASH) && bcp->bc_addr != buf) { 8887c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADBUFCTL, cp, buf); 8897c478bd9Sstevel@tonic-gate return (-1); 8907c478bd9Sstevel@tonic-gate } 8917c478bd9Sstevel@tonic-gate 8927c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) { 8937c478bd9Sstevel@tonic-gate if (!construct && (cp->cache_flags & KMF_LITE)) { 8947c478bd9Sstevel@tonic-gate if (*(uint64_t *)buf != KMEM_FREE_PATTERN) { 8957c478bd9Sstevel@tonic-gate kmem_error(KMERR_MODIFIED, cp, buf); 8967c478bd9Sstevel@tonic-gate return (-1); 8977c478bd9Sstevel@tonic-gate } 8987c478bd9Sstevel@tonic-gate if (cp->cache_constructor != NULL) 8997c478bd9Sstevel@tonic-gate *(uint64_t *)buf = btp->bt_redzone; 9007c478bd9Sstevel@tonic-gate else 9017c478bd9Sstevel@tonic-gate *(uint64_t *)buf = KMEM_UNINITIALIZED_PATTERN; 9027c478bd9Sstevel@tonic-gate } else { 9037c478bd9Sstevel@tonic-gate construct = 1; 9047c478bd9Sstevel@tonic-gate if (verify_and_copy_pattern(KMEM_FREE_PATTERN, 9057c478bd9Sstevel@tonic-gate KMEM_UNINITIALIZED_PATTERN, buf, 9067c478bd9Sstevel@tonic-gate cp->cache_verify)) { 9077c478bd9Sstevel@tonic-gate kmem_error(KMERR_MODIFIED, cp, buf); 9087c478bd9Sstevel@tonic-gate return (-1); 9097c478bd9Sstevel@tonic-gate } 9107c478bd9Sstevel@tonic-gate } 9117c478bd9Sstevel@tonic-gate } 9127c478bd9Sstevel@tonic-gate btp->bt_redzone = KMEM_REDZONE_PATTERN; 9137c478bd9Sstevel@tonic-gate 9147c478bd9Sstevel@tonic-gate if ((mtbf = kmem_mtbf | cp->cache_mtbf) != 0 && 9157c478bd9Sstevel@tonic-gate gethrtime() % mtbf == 0 && 9167c478bd9Sstevel@tonic-gate (kmflag & (KM_NOSLEEP | KM_PANIC)) == KM_NOSLEEP) { 9177c478bd9Sstevel@tonic-gate kmem_log_event(kmem_failure_log, cp, NULL, NULL); 9187c478bd9Sstevel@tonic-gate if (!construct && cp->cache_destructor != NULL) 9197c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 9207c478bd9Sstevel@tonic-gate } else { 9217c478bd9Sstevel@tonic-gate mtbf = 0; 9227c478bd9Sstevel@tonic-gate } 9237c478bd9Sstevel@tonic-gate 9247c478bd9Sstevel@tonic-gate if (mtbf || (construct && cp->cache_constructor != NULL && 9257c478bd9Sstevel@tonic-gate cp->cache_constructor(buf, cp->cache_private, kmflag) != 0)) { 9267c478bd9Sstevel@tonic-gate atomic_add_64(&cp->cache_alloc_fail, 1); 9277c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE; 9287c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) 9297c478bd9Sstevel@tonic-gate copy_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify); 9307c478bd9Sstevel@tonic-gate kmem_slab_free(cp, buf); 9317c478bd9Sstevel@tonic-gate return (-1); 9327c478bd9Sstevel@tonic-gate } 9337c478bd9Sstevel@tonic-gate 9347c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_AUDIT) { 9357c478bd9Sstevel@tonic-gate KMEM_AUDIT(kmem_transaction_log, cp, bcp); 9367c478bd9Sstevel@tonic-gate } 9377c478bd9Sstevel@tonic-gate 9387c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_LITE) && 9397c478bd9Sstevel@tonic-gate !(cp->cache_cflags & KMC_KMEM_ALLOC)) { 9407c478bd9Sstevel@tonic-gate KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, caller); 9417c478bd9Sstevel@tonic-gate } 9427c478bd9Sstevel@tonic-gate 9437c478bd9Sstevel@tonic-gate return (0); 9447c478bd9Sstevel@tonic-gate } 9457c478bd9Sstevel@tonic-gate 9467c478bd9Sstevel@tonic-gate static int 9477c478bd9Sstevel@tonic-gate kmem_cache_free_debug(kmem_cache_t *cp, void *buf, caddr_t caller) 9487c478bd9Sstevel@tonic-gate { 9497c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 9507c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *bcp = (kmem_bufctl_audit_t *)btp->bt_bufctl; 9517c478bd9Sstevel@tonic-gate kmem_slab_t *sp; 9527c478bd9Sstevel@tonic-gate 9537c478bd9Sstevel@tonic-gate if (btp->bt_bxstat != ((intptr_t)bcp ^ KMEM_BUFTAG_ALLOC)) { 9547c478bd9Sstevel@tonic-gate if (btp->bt_bxstat == ((intptr_t)bcp ^ KMEM_BUFTAG_FREE)) { 9557c478bd9Sstevel@tonic-gate kmem_error(KMERR_DUPFREE, cp, buf); 9567c478bd9Sstevel@tonic-gate return (-1); 9577c478bd9Sstevel@tonic-gate } 9587c478bd9Sstevel@tonic-gate sp = kmem_findslab(cp, buf); 9597c478bd9Sstevel@tonic-gate if (sp == NULL || sp->slab_cache != cp) 9607c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADADDR, cp, buf); 9617c478bd9Sstevel@tonic-gate else 9627c478bd9Sstevel@tonic-gate kmem_error(KMERR_REDZONE, cp, buf); 9637c478bd9Sstevel@tonic-gate return (-1); 9647c478bd9Sstevel@tonic-gate } 9657c478bd9Sstevel@tonic-gate 9667c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE; 9677c478bd9Sstevel@tonic-gate 9687c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_HASH) && bcp->bc_addr != buf) { 9697c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADBUFCTL, cp, buf); 9707c478bd9Sstevel@tonic-gate return (-1); 9717c478bd9Sstevel@tonic-gate } 9727c478bd9Sstevel@tonic-gate 9737c478bd9Sstevel@tonic-gate if (btp->bt_redzone != KMEM_REDZONE_PATTERN) { 9747c478bd9Sstevel@tonic-gate kmem_error(KMERR_REDZONE, cp, buf); 9757c478bd9Sstevel@tonic-gate return (-1); 9767c478bd9Sstevel@tonic-gate } 9777c478bd9Sstevel@tonic-gate 9787c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_AUDIT) { 9797c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_CONTENTS) 9807c478bd9Sstevel@tonic-gate bcp->bc_contents = kmem_log_enter(kmem_content_log, 9817c478bd9Sstevel@tonic-gate buf, cp->cache_contents); 9827c478bd9Sstevel@tonic-gate KMEM_AUDIT(kmem_transaction_log, cp, bcp); 9837c478bd9Sstevel@tonic-gate } 9847c478bd9Sstevel@tonic-gate 9857c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_LITE) && 9867c478bd9Sstevel@tonic-gate !(cp->cache_cflags & KMC_KMEM_ALLOC)) { 9877c478bd9Sstevel@tonic-gate KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, caller); 9887c478bd9Sstevel@tonic-gate } 9897c478bd9Sstevel@tonic-gate 9907c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) { 9917c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) 9927c478bd9Sstevel@tonic-gate btp->bt_redzone = *(uint64_t *)buf; 9937c478bd9Sstevel@tonic-gate else if (cp->cache_destructor != NULL) 9947c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 9957c478bd9Sstevel@tonic-gate 9967c478bd9Sstevel@tonic-gate copy_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify); 9977c478bd9Sstevel@tonic-gate } 9987c478bd9Sstevel@tonic-gate 9997c478bd9Sstevel@tonic-gate return (0); 10007c478bd9Sstevel@tonic-gate } 10017c478bd9Sstevel@tonic-gate 10027c478bd9Sstevel@tonic-gate /* 10037c478bd9Sstevel@tonic-gate * Free each object in magazine mp to cp's slab layer, and free mp itself. 10047c478bd9Sstevel@tonic-gate */ 10057c478bd9Sstevel@tonic-gate static void 10067c478bd9Sstevel@tonic-gate kmem_magazine_destroy(kmem_cache_t *cp, kmem_magazine_t *mp, int nrounds) 10077c478bd9Sstevel@tonic-gate { 10087c478bd9Sstevel@tonic-gate int round; 10097c478bd9Sstevel@tonic-gate 10107c478bd9Sstevel@tonic-gate ASSERT(cp->cache_next == NULL || taskq_member(kmem_taskq, curthread)); 10117c478bd9Sstevel@tonic-gate 10127c478bd9Sstevel@tonic-gate for (round = 0; round < nrounds; round++) { 10137c478bd9Sstevel@tonic-gate void *buf = mp->mag_round[round]; 10147c478bd9Sstevel@tonic-gate 10157c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) { 10167c478bd9Sstevel@tonic-gate if (verify_pattern(KMEM_FREE_PATTERN, buf, 10177c478bd9Sstevel@tonic-gate cp->cache_verify) != NULL) { 10187c478bd9Sstevel@tonic-gate kmem_error(KMERR_MODIFIED, cp, buf); 10197c478bd9Sstevel@tonic-gate continue; 10207c478bd9Sstevel@tonic-gate } 10217c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_LITE) && 10227c478bd9Sstevel@tonic-gate cp->cache_destructor != NULL) { 10237c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 10247c478bd9Sstevel@tonic-gate *(uint64_t *)buf = btp->bt_redzone; 10257c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 10267c478bd9Sstevel@tonic-gate *(uint64_t *)buf = KMEM_FREE_PATTERN; 10277c478bd9Sstevel@tonic-gate } 10287c478bd9Sstevel@tonic-gate } else if (cp->cache_destructor != NULL) { 10297c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 10307c478bd9Sstevel@tonic-gate } 10317c478bd9Sstevel@tonic-gate 10327c478bd9Sstevel@tonic-gate kmem_slab_free(cp, buf); 10337c478bd9Sstevel@tonic-gate } 10347c478bd9Sstevel@tonic-gate ASSERT(KMEM_MAGAZINE_VALID(cp, mp)); 10357c478bd9Sstevel@tonic-gate kmem_cache_free(cp->cache_magtype->mt_cache, mp); 10367c478bd9Sstevel@tonic-gate } 10377c478bd9Sstevel@tonic-gate 10387c478bd9Sstevel@tonic-gate /* 10397c478bd9Sstevel@tonic-gate * Allocate a magazine from the depot. 10407c478bd9Sstevel@tonic-gate */ 10417c478bd9Sstevel@tonic-gate static kmem_magazine_t * 10427c478bd9Sstevel@tonic-gate kmem_depot_alloc(kmem_cache_t *cp, kmem_maglist_t *mlp) 10437c478bd9Sstevel@tonic-gate { 10447c478bd9Sstevel@tonic-gate kmem_magazine_t *mp; 10457c478bd9Sstevel@tonic-gate 10467c478bd9Sstevel@tonic-gate /* 10477c478bd9Sstevel@tonic-gate * If we can't get the depot lock without contention, 10487c478bd9Sstevel@tonic-gate * update our contention count. We use the depot 10497c478bd9Sstevel@tonic-gate * contention rate to determine whether we need to 10507c478bd9Sstevel@tonic-gate * increase the magazine size for better scalability. 10517c478bd9Sstevel@tonic-gate */ 10527c478bd9Sstevel@tonic-gate if (!mutex_tryenter(&cp->cache_depot_lock)) { 10537c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock); 10547c478bd9Sstevel@tonic-gate cp->cache_depot_contention++; 10557c478bd9Sstevel@tonic-gate } 10567c478bd9Sstevel@tonic-gate 10577c478bd9Sstevel@tonic-gate if ((mp = mlp->ml_list) != NULL) { 10587c478bd9Sstevel@tonic-gate ASSERT(KMEM_MAGAZINE_VALID(cp, mp)); 10597c478bd9Sstevel@tonic-gate mlp->ml_list = mp->mag_next; 10607c478bd9Sstevel@tonic-gate if (--mlp->ml_total < mlp->ml_min) 10617c478bd9Sstevel@tonic-gate mlp->ml_min = mlp->ml_total; 10627c478bd9Sstevel@tonic-gate mlp->ml_alloc++; 10637c478bd9Sstevel@tonic-gate } 10647c478bd9Sstevel@tonic-gate 10657c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock); 10667c478bd9Sstevel@tonic-gate 10677c478bd9Sstevel@tonic-gate return (mp); 10687c478bd9Sstevel@tonic-gate } 10697c478bd9Sstevel@tonic-gate 10707c478bd9Sstevel@tonic-gate /* 10717c478bd9Sstevel@tonic-gate * Free a magazine to the depot. 10727c478bd9Sstevel@tonic-gate */ 10737c478bd9Sstevel@tonic-gate static void 10747c478bd9Sstevel@tonic-gate kmem_depot_free(kmem_cache_t *cp, kmem_maglist_t *mlp, kmem_magazine_t *mp) 10757c478bd9Sstevel@tonic-gate { 10767c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock); 10777c478bd9Sstevel@tonic-gate ASSERT(KMEM_MAGAZINE_VALID(cp, mp)); 10787c478bd9Sstevel@tonic-gate mp->mag_next = mlp->ml_list; 10797c478bd9Sstevel@tonic-gate mlp->ml_list = mp; 10807c478bd9Sstevel@tonic-gate mlp->ml_total++; 10817c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock); 10827c478bd9Sstevel@tonic-gate } 10837c478bd9Sstevel@tonic-gate 10847c478bd9Sstevel@tonic-gate /* 10857c478bd9Sstevel@tonic-gate * Update the working set statistics for cp's depot. 10867c478bd9Sstevel@tonic-gate */ 10877c478bd9Sstevel@tonic-gate static void 10887c478bd9Sstevel@tonic-gate kmem_depot_ws_update(kmem_cache_t *cp) 10897c478bd9Sstevel@tonic-gate { 10907c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock); 10917c478bd9Sstevel@tonic-gate cp->cache_full.ml_reaplimit = cp->cache_full.ml_min; 10927c478bd9Sstevel@tonic-gate cp->cache_full.ml_min = cp->cache_full.ml_total; 10937c478bd9Sstevel@tonic-gate cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_min; 10947c478bd9Sstevel@tonic-gate cp->cache_empty.ml_min = cp->cache_empty.ml_total; 10957c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock); 10967c478bd9Sstevel@tonic-gate } 10977c478bd9Sstevel@tonic-gate 10987c478bd9Sstevel@tonic-gate /* 10997c478bd9Sstevel@tonic-gate * Reap all magazines that have fallen out of the depot's working set. 11007c478bd9Sstevel@tonic-gate */ 11017c478bd9Sstevel@tonic-gate static void 11027c478bd9Sstevel@tonic-gate kmem_depot_ws_reap(kmem_cache_t *cp) 11037c478bd9Sstevel@tonic-gate { 11047c478bd9Sstevel@tonic-gate long reap; 11057c478bd9Sstevel@tonic-gate kmem_magazine_t *mp; 11067c478bd9Sstevel@tonic-gate 11077c478bd9Sstevel@tonic-gate ASSERT(cp->cache_next == NULL || taskq_member(kmem_taskq, curthread)); 11087c478bd9Sstevel@tonic-gate 11097c478bd9Sstevel@tonic-gate reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min); 11107c478bd9Sstevel@tonic-gate while (reap-- && (mp = kmem_depot_alloc(cp, &cp->cache_full)) != NULL) 11117c478bd9Sstevel@tonic-gate kmem_magazine_destroy(cp, mp, cp->cache_magtype->mt_magsize); 11127c478bd9Sstevel@tonic-gate 11137c478bd9Sstevel@tonic-gate reap = MIN(cp->cache_empty.ml_reaplimit, cp->cache_empty.ml_min); 11147c478bd9Sstevel@tonic-gate while (reap-- && (mp = kmem_depot_alloc(cp, &cp->cache_empty)) != NULL) 11157c478bd9Sstevel@tonic-gate kmem_magazine_destroy(cp, mp, 0); 11167c478bd9Sstevel@tonic-gate } 11177c478bd9Sstevel@tonic-gate 11187c478bd9Sstevel@tonic-gate static void 11197c478bd9Sstevel@tonic-gate kmem_cpu_reload(kmem_cpu_cache_t *ccp, kmem_magazine_t *mp, int rounds) 11207c478bd9Sstevel@tonic-gate { 11217c478bd9Sstevel@tonic-gate ASSERT((ccp->cc_loaded == NULL && ccp->cc_rounds == -1) || 11227c478bd9Sstevel@tonic-gate (ccp->cc_loaded && ccp->cc_rounds + rounds == ccp->cc_magsize)); 11237c478bd9Sstevel@tonic-gate ASSERT(ccp->cc_magsize > 0); 11247c478bd9Sstevel@tonic-gate 11257c478bd9Sstevel@tonic-gate ccp->cc_ploaded = ccp->cc_loaded; 11267c478bd9Sstevel@tonic-gate ccp->cc_prounds = ccp->cc_rounds; 11277c478bd9Sstevel@tonic-gate ccp->cc_loaded = mp; 11287c478bd9Sstevel@tonic-gate ccp->cc_rounds = rounds; 11297c478bd9Sstevel@tonic-gate } 11307c478bd9Sstevel@tonic-gate 11317c478bd9Sstevel@tonic-gate /* 11327c478bd9Sstevel@tonic-gate * Allocate a constructed object from cache cp. 11337c478bd9Sstevel@tonic-gate */ 11347c478bd9Sstevel@tonic-gate void * 11357c478bd9Sstevel@tonic-gate kmem_cache_alloc(kmem_cache_t *cp, int kmflag) 11367c478bd9Sstevel@tonic-gate { 11377c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp = KMEM_CPU_CACHE(cp); 11387c478bd9Sstevel@tonic-gate kmem_magazine_t *fmp; 11397c478bd9Sstevel@tonic-gate void *buf; 11407c478bd9Sstevel@tonic-gate 11417c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 11427c478bd9Sstevel@tonic-gate for (;;) { 11437c478bd9Sstevel@tonic-gate /* 11447c478bd9Sstevel@tonic-gate * If there's an object available in the current CPU's 11457c478bd9Sstevel@tonic-gate * loaded magazine, just take it and return. 11467c478bd9Sstevel@tonic-gate */ 11477c478bd9Sstevel@tonic-gate if (ccp->cc_rounds > 0) { 11487c478bd9Sstevel@tonic-gate buf = ccp->cc_loaded->mag_round[--ccp->cc_rounds]; 11497c478bd9Sstevel@tonic-gate ccp->cc_alloc++; 11507c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 11517c478bd9Sstevel@tonic-gate if ((ccp->cc_flags & KMF_BUFTAG) && 11527c478bd9Sstevel@tonic-gate kmem_cache_alloc_debug(cp, buf, kmflag, 0, 11537c478bd9Sstevel@tonic-gate caller()) == -1) { 11547c478bd9Sstevel@tonic-gate if (kmflag & KM_NOSLEEP) 11557c478bd9Sstevel@tonic-gate return (NULL); 11567c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 11577c478bd9Sstevel@tonic-gate continue; 11587c478bd9Sstevel@tonic-gate } 11597c478bd9Sstevel@tonic-gate return (buf); 11607c478bd9Sstevel@tonic-gate } 11617c478bd9Sstevel@tonic-gate 11627c478bd9Sstevel@tonic-gate /* 11637c478bd9Sstevel@tonic-gate * The loaded magazine is empty. If the previously loaded 11647c478bd9Sstevel@tonic-gate * magazine was full, exchange them and try again. 11657c478bd9Sstevel@tonic-gate */ 11667c478bd9Sstevel@tonic-gate if (ccp->cc_prounds > 0) { 11677c478bd9Sstevel@tonic-gate kmem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds); 11687c478bd9Sstevel@tonic-gate continue; 11697c478bd9Sstevel@tonic-gate } 11707c478bd9Sstevel@tonic-gate 11717c478bd9Sstevel@tonic-gate /* 11727c478bd9Sstevel@tonic-gate * If the magazine layer is disabled, break out now. 11737c478bd9Sstevel@tonic-gate */ 11747c478bd9Sstevel@tonic-gate if (ccp->cc_magsize == 0) 11757c478bd9Sstevel@tonic-gate break; 11767c478bd9Sstevel@tonic-gate 11777c478bd9Sstevel@tonic-gate /* 11787c478bd9Sstevel@tonic-gate * Try to get a full magazine from the depot. 11797c478bd9Sstevel@tonic-gate */ 11807c478bd9Sstevel@tonic-gate fmp = kmem_depot_alloc(cp, &cp->cache_full); 11817c478bd9Sstevel@tonic-gate if (fmp != NULL) { 11827c478bd9Sstevel@tonic-gate if (ccp->cc_ploaded != NULL) 11837c478bd9Sstevel@tonic-gate kmem_depot_free(cp, &cp->cache_empty, 11847c478bd9Sstevel@tonic-gate ccp->cc_ploaded); 11857c478bd9Sstevel@tonic-gate kmem_cpu_reload(ccp, fmp, ccp->cc_magsize); 11867c478bd9Sstevel@tonic-gate continue; 11877c478bd9Sstevel@tonic-gate } 11887c478bd9Sstevel@tonic-gate 11897c478bd9Sstevel@tonic-gate /* 11907c478bd9Sstevel@tonic-gate * There are no full magazines in the depot, 11917c478bd9Sstevel@tonic-gate * so fall through to the slab layer. 11927c478bd9Sstevel@tonic-gate */ 11937c478bd9Sstevel@tonic-gate break; 11947c478bd9Sstevel@tonic-gate } 11957c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 11967c478bd9Sstevel@tonic-gate 11977c478bd9Sstevel@tonic-gate /* 11987c478bd9Sstevel@tonic-gate * We couldn't allocate a constructed object from the magazine layer, 11997c478bd9Sstevel@tonic-gate * so get a raw buffer from the slab layer and apply its constructor. 12007c478bd9Sstevel@tonic-gate */ 12017c478bd9Sstevel@tonic-gate buf = kmem_slab_alloc(cp, kmflag); 12027c478bd9Sstevel@tonic-gate 12037c478bd9Sstevel@tonic-gate if (buf == NULL) 12047c478bd9Sstevel@tonic-gate return (NULL); 12057c478bd9Sstevel@tonic-gate 12067c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_BUFTAG) { 12077c478bd9Sstevel@tonic-gate /* 12087c478bd9Sstevel@tonic-gate * Make kmem_cache_alloc_debug() apply the constructor for us. 12097c478bd9Sstevel@tonic-gate */ 12107c478bd9Sstevel@tonic-gate if (kmem_cache_alloc_debug(cp, buf, kmflag, 1, 12117c478bd9Sstevel@tonic-gate caller()) == -1) { 12127c478bd9Sstevel@tonic-gate if (kmflag & KM_NOSLEEP) 12137c478bd9Sstevel@tonic-gate return (NULL); 12147c478bd9Sstevel@tonic-gate /* 12157c478bd9Sstevel@tonic-gate * kmem_cache_alloc_debug() detected corruption 12167c478bd9Sstevel@tonic-gate * but didn't panic (kmem_panic <= 0). Try again. 12177c478bd9Sstevel@tonic-gate */ 12187c478bd9Sstevel@tonic-gate return (kmem_cache_alloc(cp, kmflag)); 12197c478bd9Sstevel@tonic-gate } 12207c478bd9Sstevel@tonic-gate return (buf); 12217c478bd9Sstevel@tonic-gate } 12227c478bd9Sstevel@tonic-gate 12237c478bd9Sstevel@tonic-gate if (cp->cache_constructor != NULL && 12247c478bd9Sstevel@tonic-gate cp->cache_constructor(buf, cp->cache_private, kmflag) != 0) { 12257c478bd9Sstevel@tonic-gate atomic_add_64(&cp->cache_alloc_fail, 1); 12267c478bd9Sstevel@tonic-gate kmem_slab_free(cp, buf); 12277c478bd9Sstevel@tonic-gate return (NULL); 12287c478bd9Sstevel@tonic-gate } 12297c478bd9Sstevel@tonic-gate 12307c478bd9Sstevel@tonic-gate return (buf); 12317c478bd9Sstevel@tonic-gate } 12327c478bd9Sstevel@tonic-gate 12337c478bd9Sstevel@tonic-gate /* 12347c478bd9Sstevel@tonic-gate * Free a constructed object to cache cp. 12357c478bd9Sstevel@tonic-gate */ 12367c478bd9Sstevel@tonic-gate void 12377c478bd9Sstevel@tonic-gate kmem_cache_free(kmem_cache_t *cp, void *buf) 12387c478bd9Sstevel@tonic-gate { 12397c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp = KMEM_CPU_CACHE(cp); 12407c478bd9Sstevel@tonic-gate kmem_magazine_t *emp; 12417c478bd9Sstevel@tonic-gate kmem_magtype_t *mtp; 12427c478bd9Sstevel@tonic-gate 12437c478bd9Sstevel@tonic-gate if (ccp->cc_flags & KMF_BUFTAG) 12447c478bd9Sstevel@tonic-gate if (kmem_cache_free_debug(cp, buf, caller()) == -1) 12457c478bd9Sstevel@tonic-gate return; 12467c478bd9Sstevel@tonic-gate 12477c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 12487c478bd9Sstevel@tonic-gate for (;;) { 12497c478bd9Sstevel@tonic-gate /* 12507c478bd9Sstevel@tonic-gate * If there's a slot available in the current CPU's 12517c478bd9Sstevel@tonic-gate * loaded magazine, just put the object there and return. 12527c478bd9Sstevel@tonic-gate */ 12537c478bd9Sstevel@tonic-gate if ((uint_t)ccp->cc_rounds < ccp->cc_magsize) { 12547c478bd9Sstevel@tonic-gate ccp->cc_loaded->mag_round[ccp->cc_rounds++] = buf; 12557c478bd9Sstevel@tonic-gate ccp->cc_free++; 12567c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 12577c478bd9Sstevel@tonic-gate return; 12587c478bd9Sstevel@tonic-gate } 12597c478bd9Sstevel@tonic-gate 12607c478bd9Sstevel@tonic-gate /* 12617c478bd9Sstevel@tonic-gate * The loaded magazine is full. If the previously loaded 12627c478bd9Sstevel@tonic-gate * magazine was empty, exchange them and try again. 12637c478bd9Sstevel@tonic-gate */ 12647c478bd9Sstevel@tonic-gate if (ccp->cc_prounds == 0) { 12657c478bd9Sstevel@tonic-gate kmem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds); 12667c478bd9Sstevel@tonic-gate continue; 12677c478bd9Sstevel@tonic-gate } 12687c478bd9Sstevel@tonic-gate 12697c478bd9Sstevel@tonic-gate /* 12707c478bd9Sstevel@tonic-gate * If the magazine layer is disabled, break out now. 12717c478bd9Sstevel@tonic-gate */ 12727c478bd9Sstevel@tonic-gate if (ccp->cc_magsize == 0) 12737c478bd9Sstevel@tonic-gate break; 12747c478bd9Sstevel@tonic-gate 12757c478bd9Sstevel@tonic-gate /* 12767c478bd9Sstevel@tonic-gate * Try to get an empty magazine from the depot. 12777c478bd9Sstevel@tonic-gate */ 12787c478bd9Sstevel@tonic-gate emp = kmem_depot_alloc(cp, &cp->cache_empty); 12797c478bd9Sstevel@tonic-gate if (emp != NULL) { 12807c478bd9Sstevel@tonic-gate if (ccp->cc_ploaded != NULL) 12817c478bd9Sstevel@tonic-gate kmem_depot_free(cp, &cp->cache_full, 12827c478bd9Sstevel@tonic-gate ccp->cc_ploaded); 12837c478bd9Sstevel@tonic-gate kmem_cpu_reload(ccp, emp, 0); 12847c478bd9Sstevel@tonic-gate continue; 12857c478bd9Sstevel@tonic-gate } 12867c478bd9Sstevel@tonic-gate 12877c478bd9Sstevel@tonic-gate /* 12887c478bd9Sstevel@tonic-gate * There are no empty magazines in the depot, 12897c478bd9Sstevel@tonic-gate * so try to allocate a new one. We must drop all locks 12907c478bd9Sstevel@tonic-gate * across kmem_cache_alloc() because lower layers may 12917c478bd9Sstevel@tonic-gate * attempt to allocate from this cache. 12927c478bd9Sstevel@tonic-gate */ 12937c478bd9Sstevel@tonic-gate mtp = cp->cache_magtype; 12947c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 12957c478bd9Sstevel@tonic-gate emp = kmem_cache_alloc(mtp->mt_cache, KM_NOSLEEP); 12967c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 12977c478bd9Sstevel@tonic-gate 12987c478bd9Sstevel@tonic-gate if (emp != NULL) { 12997c478bd9Sstevel@tonic-gate /* 13007c478bd9Sstevel@tonic-gate * We successfully allocated an empty magazine. 13017c478bd9Sstevel@tonic-gate * However, we had to drop ccp->cc_lock to do it, 13027c478bd9Sstevel@tonic-gate * so the cache's magazine size may have changed. 13037c478bd9Sstevel@tonic-gate * If so, free the magazine and try again. 13047c478bd9Sstevel@tonic-gate */ 13057c478bd9Sstevel@tonic-gate if (ccp->cc_magsize != mtp->mt_magsize) { 13067c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 13077c478bd9Sstevel@tonic-gate kmem_cache_free(mtp->mt_cache, emp); 13087c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 13097c478bd9Sstevel@tonic-gate continue; 13107c478bd9Sstevel@tonic-gate } 13117c478bd9Sstevel@tonic-gate 13127c478bd9Sstevel@tonic-gate /* 13137c478bd9Sstevel@tonic-gate * We got a magazine of the right size. Add it to 13147c478bd9Sstevel@tonic-gate * the depot and try the whole dance again. 13157c478bd9Sstevel@tonic-gate */ 13167c478bd9Sstevel@tonic-gate kmem_depot_free(cp, &cp->cache_empty, emp); 13177c478bd9Sstevel@tonic-gate continue; 13187c478bd9Sstevel@tonic-gate } 13197c478bd9Sstevel@tonic-gate 13207c478bd9Sstevel@tonic-gate /* 13217c478bd9Sstevel@tonic-gate * We couldn't allocate an empty magazine, 13227c478bd9Sstevel@tonic-gate * so fall through to the slab layer. 13237c478bd9Sstevel@tonic-gate */ 13247c478bd9Sstevel@tonic-gate break; 13257c478bd9Sstevel@tonic-gate } 13267c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 13277c478bd9Sstevel@tonic-gate 13287c478bd9Sstevel@tonic-gate /* 13297c478bd9Sstevel@tonic-gate * We couldn't free our constructed object to the magazine layer, 13307c478bd9Sstevel@tonic-gate * so apply its destructor and free it to the slab layer. 13317c478bd9Sstevel@tonic-gate * Note that if KMF_DEADBEEF is in effect and KMF_LITE is not, 13327c478bd9Sstevel@tonic-gate * kmem_cache_free_debug() will have already applied the destructor. 13337c478bd9Sstevel@tonic-gate */ 13347c478bd9Sstevel@tonic-gate if ((cp->cache_flags & (KMF_DEADBEEF | KMF_LITE)) != KMF_DEADBEEF && 13357c478bd9Sstevel@tonic-gate cp->cache_destructor != NULL) { 13367c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) { /* KMF_LITE implied */ 13377c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 13387c478bd9Sstevel@tonic-gate *(uint64_t *)buf = btp->bt_redzone; 13397c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 13407c478bd9Sstevel@tonic-gate *(uint64_t *)buf = KMEM_FREE_PATTERN; 13417c478bd9Sstevel@tonic-gate } else { 13427c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 13437c478bd9Sstevel@tonic-gate } 13447c478bd9Sstevel@tonic-gate } 13457c478bd9Sstevel@tonic-gate 13467c478bd9Sstevel@tonic-gate kmem_slab_free(cp, buf); 13477c478bd9Sstevel@tonic-gate } 13487c478bd9Sstevel@tonic-gate 13497c478bd9Sstevel@tonic-gate void * 13507c478bd9Sstevel@tonic-gate kmem_zalloc(size_t size, int kmflag) 13517c478bd9Sstevel@tonic-gate { 13527c478bd9Sstevel@tonic-gate size_t index = (size - 1) >> KMEM_ALIGN_SHIFT; 13537c478bd9Sstevel@tonic-gate void *buf; 13547c478bd9Sstevel@tonic-gate 13557c478bd9Sstevel@tonic-gate if (index < KMEM_MAXBUF >> KMEM_ALIGN_SHIFT) { 13567c478bd9Sstevel@tonic-gate kmem_cache_t *cp = kmem_alloc_table[index]; 13577c478bd9Sstevel@tonic-gate buf = kmem_cache_alloc(cp, kmflag); 13587c478bd9Sstevel@tonic-gate if (buf != NULL) { 13597c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_BUFTAG) { 13607c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 13617c478bd9Sstevel@tonic-gate ((uint8_t *)buf)[size] = KMEM_REDZONE_BYTE; 13627c478bd9Sstevel@tonic-gate ((uint32_t *)btp)[1] = KMEM_SIZE_ENCODE(size); 13637c478bd9Sstevel@tonic-gate 13647c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) { 13657c478bd9Sstevel@tonic-gate KMEM_BUFTAG_LITE_ENTER(btp, 13667c478bd9Sstevel@tonic-gate kmem_lite_count, caller()); 13677c478bd9Sstevel@tonic-gate } 13687c478bd9Sstevel@tonic-gate } 13697c478bd9Sstevel@tonic-gate bzero(buf, size); 13707c478bd9Sstevel@tonic-gate } 13717c478bd9Sstevel@tonic-gate } else { 13727c478bd9Sstevel@tonic-gate buf = kmem_alloc(size, kmflag); 13737c478bd9Sstevel@tonic-gate if (buf != NULL) 13747c478bd9Sstevel@tonic-gate bzero(buf, size); 13757c478bd9Sstevel@tonic-gate } 13767c478bd9Sstevel@tonic-gate return (buf); 13777c478bd9Sstevel@tonic-gate } 13787c478bd9Sstevel@tonic-gate 13797c478bd9Sstevel@tonic-gate void * 13807c478bd9Sstevel@tonic-gate kmem_alloc(size_t size, int kmflag) 13817c478bd9Sstevel@tonic-gate { 13827c478bd9Sstevel@tonic-gate size_t index = (size - 1) >> KMEM_ALIGN_SHIFT; 13837c478bd9Sstevel@tonic-gate void *buf; 13847c478bd9Sstevel@tonic-gate 13857c478bd9Sstevel@tonic-gate if (index < KMEM_MAXBUF >> KMEM_ALIGN_SHIFT) { 13867c478bd9Sstevel@tonic-gate kmem_cache_t *cp = kmem_alloc_table[index]; 13877c478bd9Sstevel@tonic-gate buf = kmem_cache_alloc(cp, kmflag); 13887c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_BUFTAG) && buf != NULL) { 13897c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 13907c478bd9Sstevel@tonic-gate ((uint8_t *)buf)[size] = KMEM_REDZONE_BYTE; 13917c478bd9Sstevel@tonic-gate ((uint32_t *)btp)[1] = KMEM_SIZE_ENCODE(size); 13927c478bd9Sstevel@tonic-gate 13937c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) { 13947c478bd9Sstevel@tonic-gate KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, 13957c478bd9Sstevel@tonic-gate caller()); 13967c478bd9Sstevel@tonic-gate } 13977c478bd9Sstevel@tonic-gate } 13987c478bd9Sstevel@tonic-gate return (buf); 13997c478bd9Sstevel@tonic-gate } 14007c478bd9Sstevel@tonic-gate if (size == 0) 14017c478bd9Sstevel@tonic-gate return (NULL); 14027c478bd9Sstevel@tonic-gate buf = vmem_alloc(kmem_oversize_arena, size, kmflag & KM_VMFLAGS); 14037c478bd9Sstevel@tonic-gate if (buf == NULL) 14047c478bd9Sstevel@tonic-gate kmem_log_event(kmem_failure_log, NULL, NULL, (void *)size); 14057c478bd9Sstevel@tonic-gate return (buf); 14067c478bd9Sstevel@tonic-gate } 14077c478bd9Sstevel@tonic-gate 14087c478bd9Sstevel@tonic-gate void 14097c478bd9Sstevel@tonic-gate kmem_free(void *buf, size_t size) 14107c478bd9Sstevel@tonic-gate { 14117c478bd9Sstevel@tonic-gate size_t index = (size - 1) >> KMEM_ALIGN_SHIFT; 14127c478bd9Sstevel@tonic-gate 14137c478bd9Sstevel@tonic-gate if (index < KMEM_MAXBUF >> KMEM_ALIGN_SHIFT) { 14147c478bd9Sstevel@tonic-gate kmem_cache_t *cp = kmem_alloc_table[index]; 14157c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_BUFTAG) { 14167c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 14177c478bd9Sstevel@tonic-gate uint32_t *ip = (uint32_t *)btp; 14187c478bd9Sstevel@tonic-gate if (ip[1] != KMEM_SIZE_ENCODE(size)) { 14197c478bd9Sstevel@tonic-gate if (*(uint64_t *)buf == KMEM_FREE_PATTERN) { 14207c478bd9Sstevel@tonic-gate kmem_error(KMERR_DUPFREE, cp, buf); 14217c478bd9Sstevel@tonic-gate return; 14227c478bd9Sstevel@tonic-gate } 14237c478bd9Sstevel@tonic-gate if (KMEM_SIZE_VALID(ip[1])) { 14247c478bd9Sstevel@tonic-gate ip[0] = KMEM_SIZE_ENCODE(size); 14257c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADSIZE, cp, buf); 14267c478bd9Sstevel@tonic-gate } else { 14277c478bd9Sstevel@tonic-gate kmem_error(KMERR_REDZONE, cp, buf); 14287c478bd9Sstevel@tonic-gate } 14297c478bd9Sstevel@tonic-gate return; 14307c478bd9Sstevel@tonic-gate } 14317c478bd9Sstevel@tonic-gate if (((uint8_t *)buf)[size] != KMEM_REDZONE_BYTE) { 14327c478bd9Sstevel@tonic-gate kmem_error(KMERR_REDZONE, cp, buf); 14337c478bd9Sstevel@tonic-gate return; 14347c478bd9Sstevel@tonic-gate } 14357c478bd9Sstevel@tonic-gate btp->bt_redzone = KMEM_REDZONE_PATTERN; 14367c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) { 14377c478bd9Sstevel@tonic-gate KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, 14387c478bd9Sstevel@tonic-gate caller()); 14397c478bd9Sstevel@tonic-gate } 14407c478bd9Sstevel@tonic-gate } 14417c478bd9Sstevel@tonic-gate kmem_cache_free(cp, buf); 14427c478bd9Sstevel@tonic-gate } else { 14437c478bd9Sstevel@tonic-gate if (buf == NULL && size == 0) 14447c478bd9Sstevel@tonic-gate return; 14457c478bd9Sstevel@tonic-gate vmem_free(kmem_oversize_arena, buf, size); 14467c478bd9Sstevel@tonic-gate } 14477c478bd9Sstevel@tonic-gate } 14487c478bd9Sstevel@tonic-gate 14497c478bd9Sstevel@tonic-gate void * 14507c478bd9Sstevel@tonic-gate kmem_firewall_va_alloc(vmem_t *vmp, size_t size, int vmflag) 14517c478bd9Sstevel@tonic-gate { 14527c478bd9Sstevel@tonic-gate size_t realsize = size + vmp->vm_quantum; 14537c478bd9Sstevel@tonic-gate void *addr; 14547c478bd9Sstevel@tonic-gate 14557c478bd9Sstevel@tonic-gate /* 14567c478bd9Sstevel@tonic-gate * Annoying edge case: if 'size' is just shy of ULONG_MAX, adding 14577c478bd9Sstevel@tonic-gate * vm_quantum will cause integer wraparound. Check for this, and 14587c478bd9Sstevel@tonic-gate * blow off the firewall page in this case. Note that such a 14597c478bd9Sstevel@tonic-gate * giant allocation (the entire kernel address space) can never 14607c478bd9Sstevel@tonic-gate * be satisfied, so it will either fail immediately (VM_NOSLEEP) 14617c478bd9Sstevel@tonic-gate * or sleep forever (VM_SLEEP). Thus, there is no need for a 14627c478bd9Sstevel@tonic-gate * corresponding check in kmem_firewall_va_free(). 14637c478bd9Sstevel@tonic-gate */ 14647c478bd9Sstevel@tonic-gate if (realsize < size) 14657c478bd9Sstevel@tonic-gate realsize = size; 14667c478bd9Sstevel@tonic-gate 14677c478bd9Sstevel@tonic-gate /* 14687c478bd9Sstevel@tonic-gate * While boot still owns resource management, make sure that this 14697c478bd9Sstevel@tonic-gate * redzone virtual address allocation is properly accounted for in 14707c478bd9Sstevel@tonic-gate * OBPs "virtual-memory" "available" lists because we're 14717c478bd9Sstevel@tonic-gate * effectively claiming them for a red zone. If we don't do this, 14727c478bd9Sstevel@tonic-gate * the available lists become too fragmented and too large for the 14737c478bd9Sstevel@tonic-gate * current boot/kernel memory list interface. 14747c478bd9Sstevel@tonic-gate */ 14757c478bd9Sstevel@tonic-gate addr = vmem_alloc(vmp, realsize, vmflag | VM_NEXTFIT); 14767c478bd9Sstevel@tonic-gate 14777c478bd9Sstevel@tonic-gate if (addr != NULL && kvseg.s_base == NULL && realsize != size) 14787c478bd9Sstevel@tonic-gate (void) boot_virt_alloc((char *)addr + size, vmp->vm_quantum); 14797c478bd9Sstevel@tonic-gate 14807c478bd9Sstevel@tonic-gate return (addr); 14817c478bd9Sstevel@tonic-gate } 14827c478bd9Sstevel@tonic-gate 14837c478bd9Sstevel@tonic-gate void 14847c478bd9Sstevel@tonic-gate kmem_firewall_va_free(vmem_t *vmp, void *addr, size_t size) 14857c478bd9Sstevel@tonic-gate { 14867c478bd9Sstevel@tonic-gate ASSERT((kvseg.s_base == NULL ? 14877c478bd9Sstevel@tonic-gate va_to_pfn((char *)addr + size) : 14887c478bd9Sstevel@tonic-gate hat_getpfnum(kas.a_hat, (caddr_t)addr + size)) == PFN_INVALID); 14897c478bd9Sstevel@tonic-gate 14907c478bd9Sstevel@tonic-gate vmem_free(vmp, addr, size + vmp->vm_quantum); 14917c478bd9Sstevel@tonic-gate } 14927c478bd9Sstevel@tonic-gate 14937c478bd9Sstevel@tonic-gate /* 14947c478bd9Sstevel@tonic-gate * Try to allocate at least `size' bytes of memory without sleeping or 14957c478bd9Sstevel@tonic-gate * panicking. Return actual allocated size in `asize'. If allocation failed, 14967c478bd9Sstevel@tonic-gate * try final allocation with sleep or panic allowed. 14977c478bd9Sstevel@tonic-gate */ 14987c478bd9Sstevel@tonic-gate void * 14997c478bd9Sstevel@tonic-gate kmem_alloc_tryhard(size_t size, size_t *asize, int kmflag) 15007c478bd9Sstevel@tonic-gate { 15017c478bd9Sstevel@tonic-gate void *p; 15027c478bd9Sstevel@tonic-gate 15037c478bd9Sstevel@tonic-gate *asize = P2ROUNDUP(size, KMEM_ALIGN); 15047c478bd9Sstevel@tonic-gate do { 15057c478bd9Sstevel@tonic-gate p = kmem_alloc(*asize, (kmflag | KM_NOSLEEP) & ~KM_PANIC); 15067c478bd9Sstevel@tonic-gate if (p != NULL) 15077c478bd9Sstevel@tonic-gate return (p); 15087c478bd9Sstevel@tonic-gate *asize += KMEM_ALIGN; 15097c478bd9Sstevel@tonic-gate } while (*asize <= PAGESIZE); 15107c478bd9Sstevel@tonic-gate 15117c478bd9Sstevel@tonic-gate *asize = P2ROUNDUP(size, KMEM_ALIGN); 15127c478bd9Sstevel@tonic-gate return (kmem_alloc(*asize, kmflag)); 15137c478bd9Sstevel@tonic-gate } 15147c478bd9Sstevel@tonic-gate 15157c478bd9Sstevel@tonic-gate /* 15167c478bd9Sstevel@tonic-gate * Reclaim all unused memory from a cache. 15177c478bd9Sstevel@tonic-gate */ 15187c478bd9Sstevel@tonic-gate static void 15197c478bd9Sstevel@tonic-gate kmem_cache_reap(kmem_cache_t *cp) 15207c478bd9Sstevel@tonic-gate { 15217c478bd9Sstevel@tonic-gate /* 15227c478bd9Sstevel@tonic-gate * Ask the cache's owner to free some memory if possible. 15237c478bd9Sstevel@tonic-gate * The idea is to handle things like the inode cache, which 15247c478bd9Sstevel@tonic-gate * typically sits on a bunch of memory that it doesn't truly 15257c478bd9Sstevel@tonic-gate * *need*. Reclaim policy is entirely up to the owner; this 15267c478bd9Sstevel@tonic-gate * callback is just an advisory plea for help. 15277c478bd9Sstevel@tonic-gate */ 15287c478bd9Sstevel@tonic-gate if (cp->cache_reclaim != NULL) 15297c478bd9Sstevel@tonic-gate cp->cache_reclaim(cp->cache_private); 15307c478bd9Sstevel@tonic-gate 15317c478bd9Sstevel@tonic-gate kmem_depot_ws_reap(cp); 15327c478bd9Sstevel@tonic-gate } 15337c478bd9Sstevel@tonic-gate 15347c478bd9Sstevel@tonic-gate static void 15357c478bd9Sstevel@tonic-gate kmem_reap_timeout(void *flag_arg) 15367c478bd9Sstevel@tonic-gate { 15377c478bd9Sstevel@tonic-gate uint32_t *flag = (uint32_t *)flag_arg; 15387c478bd9Sstevel@tonic-gate 15397c478bd9Sstevel@tonic-gate ASSERT(flag == &kmem_reaping || flag == &kmem_reaping_idspace); 15407c478bd9Sstevel@tonic-gate *flag = 0; 15417c478bd9Sstevel@tonic-gate } 15427c478bd9Sstevel@tonic-gate 15437c478bd9Sstevel@tonic-gate static void 15447c478bd9Sstevel@tonic-gate kmem_reap_done(void *flag) 15457c478bd9Sstevel@tonic-gate { 15467c478bd9Sstevel@tonic-gate (void) timeout(kmem_reap_timeout, flag, kmem_reap_interval); 15477c478bd9Sstevel@tonic-gate } 15487c478bd9Sstevel@tonic-gate 15497c478bd9Sstevel@tonic-gate static void 15507c478bd9Sstevel@tonic-gate kmem_reap_start(void *flag) 15517c478bd9Sstevel@tonic-gate { 15527c478bd9Sstevel@tonic-gate ASSERT(flag == &kmem_reaping || flag == &kmem_reaping_idspace); 15537c478bd9Sstevel@tonic-gate 15547c478bd9Sstevel@tonic-gate if (flag == &kmem_reaping) { 15557c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_reap, kmem_taskq, TQ_NOSLEEP); 15567c478bd9Sstevel@tonic-gate /* 15577c478bd9Sstevel@tonic-gate * if we have segkp under heap, reap segkp cache. 15587c478bd9Sstevel@tonic-gate */ 15597c478bd9Sstevel@tonic-gate if (segkp_fromheap) 15607c478bd9Sstevel@tonic-gate segkp_cache_free(); 15617c478bd9Sstevel@tonic-gate } 15627c478bd9Sstevel@tonic-gate else 15637c478bd9Sstevel@tonic-gate kmem_cache_applyall_id(kmem_cache_reap, kmem_taskq, TQ_NOSLEEP); 15647c478bd9Sstevel@tonic-gate 15657c478bd9Sstevel@tonic-gate /* 15667c478bd9Sstevel@tonic-gate * We use taskq_dispatch() to schedule a timeout to clear 15677c478bd9Sstevel@tonic-gate * the flag so that kmem_reap() becomes self-throttling: 15687c478bd9Sstevel@tonic-gate * we won't reap again until the current reap completes *and* 15697c478bd9Sstevel@tonic-gate * at least kmem_reap_interval ticks have elapsed. 15707c478bd9Sstevel@tonic-gate */ 15717c478bd9Sstevel@tonic-gate if (!taskq_dispatch(kmem_taskq, kmem_reap_done, flag, TQ_NOSLEEP)) 15727c478bd9Sstevel@tonic-gate kmem_reap_done(flag); 15737c478bd9Sstevel@tonic-gate } 15747c478bd9Sstevel@tonic-gate 15757c478bd9Sstevel@tonic-gate static void 15767c478bd9Sstevel@tonic-gate kmem_reap_common(void *flag_arg) 15777c478bd9Sstevel@tonic-gate { 15787c478bd9Sstevel@tonic-gate uint32_t *flag = (uint32_t *)flag_arg; 15797c478bd9Sstevel@tonic-gate 15807c478bd9Sstevel@tonic-gate if (MUTEX_HELD(&kmem_cache_lock) || kmem_taskq == NULL || 15817c478bd9Sstevel@tonic-gate cas32(flag, 0, 1) != 0) 15827c478bd9Sstevel@tonic-gate return; 15837c478bd9Sstevel@tonic-gate 15847c478bd9Sstevel@tonic-gate /* 15857c478bd9Sstevel@tonic-gate * It may not be kosher to do memory allocation when a reap is called 15867c478bd9Sstevel@tonic-gate * is called (for example, if vmem_populate() is in the call chain). 15877c478bd9Sstevel@tonic-gate * So we start the reap going with a TQ_NOALLOC dispatch. If the 15887c478bd9Sstevel@tonic-gate * dispatch fails, we reset the flag, and the next reap will try again. 15897c478bd9Sstevel@tonic-gate */ 15907c478bd9Sstevel@tonic-gate if (!taskq_dispatch(kmem_taskq, kmem_reap_start, flag, TQ_NOALLOC)) 15917c478bd9Sstevel@tonic-gate *flag = 0; 15927c478bd9Sstevel@tonic-gate } 15937c478bd9Sstevel@tonic-gate 15947c478bd9Sstevel@tonic-gate /* 15957c478bd9Sstevel@tonic-gate * Reclaim all unused memory from all caches. Called from the VM system 15967c478bd9Sstevel@tonic-gate * when memory gets tight. 15977c478bd9Sstevel@tonic-gate */ 15987c478bd9Sstevel@tonic-gate void 15997c478bd9Sstevel@tonic-gate kmem_reap(void) 16007c478bd9Sstevel@tonic-gate { 16017c478bd9Sstevel@tonic-gate kmem_reap_common(&kmem_reaping); 16027c478bd9Sstevel@tonic-gate } 16037c478bd9Sstevel@tonic-gate 16047c478bd9Sstevel@tonic-gate /* 16057c478bd9Sstevel@tonic-gate * Reclaim all unused memory from identifier arenas, called when a vmem 16067c478bd9Sstevel@tonic-gate * arena not back by memory is exhausted. Since reaping memory-backed caches 16077c478bd9Sstevel@tonic-gate * cannot help with identifier exhaustion, we avoid both a large amount of 16087c478bd9Sstevel@tonic-gate * work and unwanted side-effects from reclaim callbacks. 16097c478bd9Sstevel@tonic-gate */ 16107c478bd9Sstevel@tonic-gate void 16117c478bd9Sstevel@tonic-gate kmem_reap_idspace(void) 16127c478bd9Sstevel@tonic-gate { 16137c478bd9Sstevel@tonic-gate kmem_reap_common(&kmem_reaping_idspace); 16147c478bd9Sstevel@tonic-gate } 16157c478bd9Sstevel@tonic-gate 16167c478bd9Sstevel@tonic-gate /* 16177c478bd9Sstevel@tonic-gate * Purge all magazines from a cache and set its magazine limit to zero. 16187c478bd9Sstevel@tonic-gate * All calls are serialized by the kmem_taskq lock, except for the final 16197c478bd9Sstevel@tonic-gate * call from kmem_cache_destroy(). 16207c478bd9Sstevel@tonic-gate */ 16217c478bd9Sstevel@tonic-gate static void 16227c478bd9Sstevel@tonic-gate kmem_cache_magazine_purge(kmem_cache_t *cp) 16237c478bd9Sstevel@tonic-gate { 16247c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp; 16257c478bd9Sstevel@tonic-gate kmem_magazine_t *mp, *pmp; 16267c478bd9Sstevel@tonic-gate int rounds, prounds, cpu_seqid; 16277c478bd9Sstevel@tonic-gate 16287c478bd9Sstevel@tonic-gate ASSERT(cp->cache_next == NULL || taskq_member(kmem_taskq, curthread)); 16297c478bd9Sstevel@tonic-gate ASSERT(MUTEX_NOT_HELD(&cp->cache_lock)); 16307c478bd9Sstevel@tonic-gate 16317c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 16327c478bd9Sstevel@tonic-gate ccp = &cp->cache_cpu[cpu_seqid]; 16337c478bd9Sstevel@tonic-gate 16347c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 16357c478bd9Sstevel@tonic-gate mp = ccp->cc_loaded; 16367c478bd9Sstevel@tonic-gate pmp = ccp->cc_ploaded; 16377c478bd9Sstevel@tonic-gate rounds = ccp->cc_rounds; 16387c478bd9Sstevel@tonic-gate prounds = ccp->cc_prounds; 16397c478bd9Sstevel@tonic-gate ccp->cc_loaded = NULL; 16407c478bd9Sstevel@tonic-gate ccp->cc_ploaded = NULL; 16417c478bd9Sstevel@tonic-gate ccp->cc_rounds = -1; 16427c478bd9Sstevel@tonic-gate ccp->cc_prounds = -1; 16437c478bd9Sstevel@tonic-gate ccp->cc_magsize = 0; 16447c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 16457c478bd9Sstevel@tonic-gate 16467c478bd9Sstevel@tonic-gate if (mp) 16477c478bd9Sstevel@tonic-gate kmem_magazine_destroy(cp, mp, rounds); 16487c478bd9Sstevel@tonic-gate if (pmp) 16497c478bd9Sstevel@tonic-gate kmem_magazine_destroy(cp, pmp, prounds); 16507c478bd9Sstevel@tonic-gate } 16517c478bd9Sstevel@tonic-gate 16527c478bd9Sstevel@tonic-gate /* 16537c478bd9Sstevel@tonic-gate * Updating the working set statistics twice in a row has the 16547c478bd9Sstevel@tonic-gate * effect of setting the working set size to zero, so everything 16557c478bd9Sstevel@tonic-gate * is eligible for reaping. 16567c478bd9Sstevel@tonic-gate */ 16577c478bd9Sstevel@tonic-gate kmem_depot_ws_update(cp); 16587c478bd9Sstevel@tonic-gate kmem_depot_ws_update(cp); 16597c478bd9Sstevel@tonic-gate 16607c478bd9Sstevel@tonic-gate kmem_depot_ws_reap(cp); 16617c478bd9Sstevel@tonic-gate } 16627c478bd9Sstevel@tonic-gate 16637c478bd9Sstevel@tonic-gate /* 16647c478bd9Sstevel@tonic-gate * Enable per-cpu magazines on a cache. 16657c478bd9Sstevel@tonic-gate */ 16667c478bd9Sstevel@tonic-gate static void 16677c478bd9Sstevel@tonic-gate kmem_cache_magazine_enable(kmem_cache_t *cp) 16687c478bd9Sstevel@tonic-gate { 16697c478bd9Sstevel@tonic-gate int cpu_seqid; 16707c478bd9Sstevel@tonic-gate 16717c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_NOMAGAZINE) 16727c478bd9Sstevel@tonic-gate return; 16737c478bd9Sstevel@tonic-gate 16747c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 16757c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 16767c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 16777c478bd9Sstevel@tonic-gate ccp->cc_magsize = cp->cache_magtype->mt_magsize; 16787c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 16797c478bd9Sstevel@tonic-gate } 16807c478bd9Sstevel@tonic-gate 16817c478bd9Sstevel@tonic-gate } 16827c478bd9Sstevel@tonic-gate 1683fa9e4066Sahrens /* 1684fa9e4066Sahrens * Reap (almost) everything right now. See kmem_cache_magazine_purge() 1685fa9e4066Sahrens * for explanation of the back-to-back kmem_depot_ws_update() calls. 1686fa9e4066Sahrens */ 1687fa9e4066Sahrens void 1688fa9e4066Sahrens kmem_cache_reap_now(kmem_cache_t *cp) 1689fa9e4066Sahrens { 1690fa9e4066Sahrens kmem_depot_ws_update(cp); 1691fa9e4066Sahrens kmem_depot_ws_update(cp); 1692fa9e4066Sahrens 1693fa9e4066Sahrens (void) taskq_dispatch(kmem_taskq, 1694fa9e4066Sahrens (task_func_t *)kmem_depot_ws_reap, cp, TQ_SLEEP); 1695fa9e4066Sahrens taskq_wait(kmem_taskq); 1696fa9e4066Sahrens } 1697fa9e4066Sahrens 16987c478bd9Sstevel@tonic-gate /* 16997c478bd9Sstevel@tonic-gate * Recompute a cache's magazine size. The trade-off is that larger magazines 17007c478bd9Sstevel@tonic-gate * provide a higher transfer rate with the depot, while smaller magazines 17017c478bd9Sstevel@tonic-gate * reduce memory consumption. Magazine resizing is an expensive operation; 17027c478bd9Sstevel@tonic-gate * it should not be done frequently. 17037c478bd9Sstevel@tonic-gate * 17047c478bd9Sstevel@tonic-gate * Changes to the magazine size are serialized by the kmem_taskq lock. 17057c478bd9Sstevel@tonic-gate * 17067c478bd9Sstevel@tonic-gate * Note: at present this only grows the magazine size. It might be useful 17077c478bd9Sstevel@tonic-gate * to allow shrinkage too. 17087c478bd9Sstevel@tonic-gate */ 17097c478bd9Sstevel@tonic-gate static void 17107c478bd9Sstevel@tonic-gate kmem_cache_magazine_resize(kmem_cache_t *cp) 17117c478bd9Sstevel@tonic-gate { 17127c478bd9Sstevel@tonic-gate kmem_magtype_t *mtp = cp->cache_magtype; 17137c478bd9Sstevel@tonic-gate 17147c478bd9Sstevel@tonic-gate ASSERT(taskq_member(kmem_taskq, curthread)); 17157c478bd9Sstevel@tonic-gate 17167c478bd9Sstevel@tonic-gate if (cp->cache_chunksize < mtp->mt_maxbuf) { 17177c478bd9Sstevel@tonic-gate kmem_cache_magazine_purge(cp); 17187c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock); 17197c478bd9Sstevel@tonic-gate cp->cache_magtype = ++mtp; 17207c478bd9Sstevel@tonic-gate cp->cache_depot_contention_prev = 17217c478bd9Sstevel@tonic-gate cp->cache_depot_contention + INT_MAX; 17227c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock); 17237c478bd9Sstevel@tonic-gate kmem_cache_magazine_enable(cp); 17247c478bd9Sstevel@tonic-gate } 17257c478bd9Sstevel@tonic-gate } 17267c478bd9Sstevel@tonic-gate 17277c478bd9Sstevel@tonic-gate /* 17287c478bd9Sstevel@tonic-gate * Rescale a cache's hash table, so that the table size is roughly the 17297c478bd9Sstevel@tonic-gate * cache size. We want the average lookup time to be extremely small. 17307c478bd9Sstevel@tonic-gate */ 17317c478bd9Sstevel@tonic-gate static void 17327c478bd9Sstevel@tonic-gate kmem_hash_rescale(kmem_cache_t *cp) 17337c478bd9Sstevel@tonic-gate { 17347c478bd9Sstevel@tonic-gate kmem_bufctl_t **old_table, **new_table, *bcp; 17357c478bd9Sstevel@tonic-gate size_t old_size, new_size, h; 17367c478bd9Sstevel@tonic-gate 17377c478bd9Sstevel@tonic-gate ASSERT(taskq_member(kmem_taskq, curthread)); 17387c478bd9Sstevel@tonic-gate 17397c478bd9Sstevel@tonic-gate new_size = MAX(KMEM_HASH_INITIAL, 17407c478bd9Sstevel@tonic-gate 1 << (highbit(3 * cp->cache_buftotal + 4) - 2)); 17417c478bd9Sstevel@tonic-gate old_size = cp->cache_hash_mask + 1; 17427c478bd9Sstevel@tonic-gate 17437c478bd9Sstevel@tonic-gate if ((old_size >> 1) <= new_size && new_size <= (old_size << 1)) 17447c478bd9Sstevel@tonic-gate return; 17457c478bd9Sstevel@tonic-gate 17467c478bd9Sstevel@tonic-gate new_table = vmem_alloc(kmem_hash_arena, new_size * sizeof (void *), 17477c478bd9Sstevel@tonic-gate VM_NOSLEEP); 17487c478bd9Sstevel@tonic-gate if (new_table == NULL) 17497c478bd9Sstevel@tonic-gate return; 17507c478bd9Sstevel@tonic-gate bzero(new_table, new_size * sizeof (void *)); 17517c478bd9Sstevel@tonic-gate 17527c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 17537c478bd9Sstevel@tonic-gate 17547c478bd9Sstevel@tonic-gate old_size = cp->cache_hash_mask + 1; 17557c478bd9Sstevel@tonic-gate old_table = cp->cache_hash_table; 17567c478bd9Sstevel@tonic-gate 17577c478bd9Sstevel@tonic-gate cp->cache_hash_mask = new_size - 1; 17587c478bd9Sstevel@tonic-gate cp->cache_hash_table = new_table; 17597c478bd9Sstevel@tonic-gate cp->cache_rescale++; 17607c478bd9Sstevel@tonic-gate 17617c478bd9Sstevel@tonic-gate for (h = 0; h < old_size; h++) { 17627c478bd9Sstevel@tonic-gate bcp = old_table[h]; 17637c478bd9Sstevel@tonic-gate while (bcp != NULL) { 17647c478bd9Sstevel@tonic-gate void *addr = bcp->bc_addr; 17657c478bd9Sstevel@tonic-gate kmem_bufctl_t *next_bcp = bcp->bc_next; 17667c478bd9Sstevel@tonic-gate kmem_bufctl_t **hash_bucket = KMEM_HASH(cp, addr); 17677c478bd9Sstevel@tonic-gate bcp->bc_next = *hash_bucket; 17687c478bd9Sstevel@tonic-gate *hash_bucket = bcp; 17697c478bd9Sstevel@tonic-gate bcp = next_bcp; 17707c478bd9Sstevel@tonic-gate } 17717c478bd9Sstevel@tonic-gate } 17727c478bd9Sstevel@tonic-gate 17737c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 17747c478bd9Sstevel@tonic-gate 17757c478bd9Sstevel@tonic-gate vmem_free(kmem_hash_arena, old_table, old_size * sizeof (void *)); 17767c478bd9Sstevel@tonic-gate } 17777c478bd9Sstevel@tonic-gate 17787c478bd9Sstevel@tonic-gate /* 17797c478bd9Sstevel@tonic-gate * Perform periodic maintenance on a cache: hash rescaling, 17807c478bd9Sstevel@tonic-gate * depot working-set update, and magazine resizing. 17817c478bd9Sstevel@tonic-gate */ 17827c478bd9Sstevel@tonic-gate static void 17837c478bd9Sstevel@tonic-gate kmem_cache_update(kmem_cache_t *cp) 17847c478bd9Sstevel@tonic-gate { 17857c478bd9Sstevel@tonic-gate int need_hash_rescale = 0; 17867c478bd9Sstevel@tonic-gate int need_magazine_resize = 0; 17877c478bd9Sstevel@tonic-gate 17887c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&kmem_cache_lock)); 17897c478bd9Sstevel@tonic-gate 17907c478bd9Sstevel@tonic-gate /* 17917c478bd9Sstevel@tonic-gate * If the cache has become much larger or smaller than its hash table, 17927c478bd9Sstevel@tonic-gate * fire off a request to rescale the hash table. 17937c478bd9Sstevel@tonic-gate */ 17947c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 17957c478bd9Sstevel@tonic-gate 17967c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_HASH) && 17977c478bd9Sstevel@tonic-gate (cp->cache_buftotal > (cp->cache_hash_mask << 1) || 17987c478bd9Sstevel@tonic-gate (cp->cache_buftotal < (cp->cache_hash_mask >> 1) && 17997c478bd9Sstevel@tonic-gate cp->cache_hash_mask > KMEM_HASH_INITIAL))) 18007c478bd9Sstevel@tonic-gate need_hash_rescale = 1; 18017c478bd9Sstevel@tonic-gate 18027c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 18037c478bd9Sstevel@tonic-gate 18047c478bd9Sstevel@tonic-gate /* 18057c478bd9Sstevel@tonic-gate * Update the depot working set statistics. 18067c478bd9Sstevel@tonic-gate */ 18077c478bd9Sstevel@tonic-gate kmem_depot_ws_update(cp); 18087c478bd9Sstevel@tonic-gate 18097c478bd9Sstevel@tonic-gate /* 18107c478bd9Sstevel@tonic-gate * If there's a lot of contention in the depot, 18117c478bd9Sstevel@tonic-gate * increase the magazine size. 18127c478bd9Sstevel@tonic-gate */ 18137c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock); 18147c478bd9Sstevel@tonic-gate 18157c478bd9Sstevel@tonic-gate if (cp->cache_chunksize < cp->cache_magtype->mt_maxbuf && 18167c478bd9Sstevel@tonic-gate (int)(cp->cache_depot_contention - 18177c478bd9Sstevel@tonic-gate cp->cache_depot_contention_prev) > kmem_depot_contention) 18187c478bd9Sstevel@tonic-gate need_magazine_resize = 1; 18197c478bd9Sstevel@tonic-gate 18207c478bd9Sstevel@tonic-gate cp->cache_depot_contention_prev = cp->cache_depot_contention; 18217c478bd9Sstevel@tonic-gate 18227c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock); 18237c478bd9Sstevel@tonic-gate 18247c478bd9Sstevel@tonic-gate if (need_hash_rescale) 18257c478bd9Sstevel@tonic-gate (void) taskq_dispatch(kmem_taskq, 18267c478bd9Sstevel@tonic-gate (task_func_t *)kmem_hash_rescale, cp, TQ_NOSLEEP); 18277c478bd9Sstevel@tonic-gate 18287c478bd9Sstevel@tonic-gate if (need_magazine_resize) 18297c478bd9Sstevel@tonic-gate (void) taskq_dispatch(kmem_taskq, 18307c478bd9Sstevel@tonic-gate (task_func_t *)kmem_cache_magazine_resize, cp, TQ_NOSLEEP); 18317c478bd9Sstevel@tonic-gate } 18327c478bd9Sstevel@tonic-gate 18337c478bd9Sstevel@tonic-gate static void 18347c478bd9Sstevel@tonic-gate kmem_update_timeout(void *dummy) 18357c478bd9Sstevel@tonic-gate { 18367c478bd9Sstevel@tonic-gate static void kmem_update(void *); 18377c478bd9Sstevel@tonic-gate 18387c478bd9Sstevel@tonic-gate (void) timeout(kmem_update, dummy, kmem_reap_interval); 18397c478bd9Sstevel@tonic-gate } 18407c478bd9Sstevel@tonic-gate 18417c478bd9Sstevel@tonic-gate static void 18427c478bd9Sstevel@tonic-gate kmem_update(void *dummy) 18437c478bd9Sstevel@tonic-gate { 18447c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_update, NULL, TQ_NOSLEEP); 18457c478bd9Sstevel@tonic-gate 18467c478bd9Sstevel@tonic-gate /* 18477c478bd9Sstevel@tonic-gate * We use taskq_dispatch() to reschedule the timeout so that 18487c478bd9Sstevel@tonic-gate * kmem_update() becomes self-throttling: it won't schedule 18497c478bd9Sstevel@tonic-gate * new tasks until all previous tasks have completed. 18507c478bd9Sstevel@tonic-gate */ 18517c478bd9Sstevel@tonic-gate if (!taskq_dispatch(kmem_taskq, kmem_update_timeout, dummy, TQ_NOSLEEP)) 18527c478bd9Sstevel@tonic-gate kmem_update_timeout(NULL); 18537c478bd9Sstevel@tonic-gate } 18547c478bd9Sstevel@tonic-gate 18557c478bd9Sstevel@tonic-gate static int 18567c478bd9Sstevel@tonic-gate kmem_cache_kstat_update(kstat_t *ksp, int rw) 18577c478bd9Sstevel@tonic-gate { 18587c478bd9Sstevel@tonic-gate struct kmem_cache_kstat *kmcp = &kmem_cache_kstat; 18597c478bd9Sstevel@tonic-gate kmem_cache_t *cp = ksp->ks_private; 18607c478bd9Sstevel@tonic-gate kmem_slab_t *sp; 18617c478bd9Sstevel@tonic-gate uint64_t cpu_buf_avail; 18627c478bd9Sstevel@tonic-gate uint64_t buf_avail = 0; 18637c478bd9Sstevel@tonic-gate int cpu_seqid; 18647c478bd9Sstevel@tonic-gate 18657c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&kmem_cache_kstat_lock)); 18667c478bd9Sstevel@tonic-gate 18677c478bd9Sstevel@tonic-gate if (rw == KSTAT_WRITE) 18687c478bd9Sstevel@tonic-gate return (EACCES); 18697c478bd9Sstevel@tonic-gate 18707c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 18717c478bd9Sstevel@tonic-gate 18727c478bd9Sstevel@tonic-gate kmcp->kmc_alloc_fail.value.ui64 = cp->cache_alloc_fail; 18737c478bd9Sstevel@tonic-gate kmcp->kmc_alloc.value.ui64 = cp->cache_slab_alloc; 18747c478bd9Sstevel@tonic-gate kmcp->kmc_free.value.ui64 = cp->cache_slab_free; 18757c478bd9Sstevel@tonic-gate kmcp->kmc_slab_alloc.value.ui64 = cp->cache_slab_alloc; 18767c478bd9Sstevel@tonic-gate kmcp->kmc_slab_free.value.ui64 = cp->cache_slab_free; 18777c478bd9Sstevel@tonic-gate 18787c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 18797c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 18807c478bd9Sstevel@tonic-gate 18817c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 18827c478bd9Sstevel@tonic-gate 18837c478bd9Sstevel@tonic-gate cpu_buf_avail = 0; 18847c478bd9Sstevel@tonic-gate if (ccp->cc_rounds > 0) 18857c478bd9Sstevel@tonic-gate cpu_buf_avail += ccp->cc_rounds; 18867c478bd9Sstevel@tonic-gate if (ccp->cc_prounds > 0) 18877c478bd9Sstevel@tonic-gate cpu_buf_avail += ccp->cc_prounds; 18887c478bd9Sstevel@tonic-gate 18897c478bd9Sstevel@tonic-gate kmcp->kmc_alloc.value.ui64 += ccp->cc_alloc; 18907c478bd9Sstevel@tonic-gate kmcp->kmc_free.value.ui64 += ccp->cc_free; 18917c478bd9Sstevel@tonic-gate buf_avail += cpu_buf_avail; 18927c478bd9Sstevel@tonic-gate 18937c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 18947c478bd9Sstevel@tonic-gate } 18957c478bd9Sstevel@tonic-gate 18967c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock); 18977c478bd9Sstevel@tonic-gate 18987c478bd9Sstevel@tonic-gate kmcp->kmc_depot_alloc.value.ui64 = cp->cache_full.ml_alloc; 18997c478bd9Sstevel@tonic-gate kmcp->kmc_depot_free.value.ui64 = cp->cache_empty.ml_alloc; 19007c478bd9Sstevel@tonic-gate kmcp->kmc_depot_contention.value.ui64 = cp->cache_depot_contention; 19017c478bd9Sstevel@tonic-gate kmcp->kmc_full_magazines.value.ui64 = cp->cache_full.ml_total; 19027c478bd9Sstevel@tonic-gate kmcp->kmc_empty_magazines.value.ui64 = cp->cache_empty.ml_total; 19037c478bd9Sstevel@tonic-gate kmcp->kmc_magazine_size.value.ui64 = 19047c478bd9Sstevel@tonic-gate (cp->cache_flags & KMF_NOMAGAZINE) ? 19057c478bd9Sstevel@tonic-gate 0 : cp->cache_magtype->mt_magsize; 19067c478bd9Sstevel@tonic-gate 19077c478bd9Sstevel@tonic-gate kmcp->kmc_alloc.value.ui64 += cp->cache_full.ml_alloc; 19087c478bd9Sstevel@tonic-gate kmcp->kmc_free.value.ui64 += cp->cache_empty.ml_alloc; 19097c478bd9Sstevel@tonic-gate buf_avail += cp->cache_full.ml_total * cp->cache_magtype->mt_magsize; 19107c478bd9Sstevel@tonic-gate 19117c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock); 19127c478bd9Sstevel@tonic-gate 19137c478bd9Sstevel@tonic-gate kmcp->kmc_buf_size.value.ui64 = cp->cache_bufsize; 19147c478bd9Sstevel@tonic-gate kmcp->kmc_align.value.ui64 = cp->cache_align; 19157c478bd9Sstevel@tonic-gate kmcp->kmc_chunk_size.value.ui64 = cp->cache_chunksize; 19167c478bd9Sstevel@tonic-gate kmcp->kmc_slab_size.value.ui64 = cp->cache_slabsize; 19177c478bd9Sstevel@tonic-gate kmcp->kmc_buf_constructed.value.ui64 = buf_avail; 19187c478bd9Sstevel@tonic-gate for (sp = cp->cache_freelist; sp != &cp->cache_nullslab; 19197c478bd9Sstevel@tonic-gate sp = sp->slab_next) 19207c478bd9Sstevel@tonic-gate buf_avail += sp->slab_chunks - sp->slab_refcnt; 19217c478bd9Sstevel@tonic-gate kmcp->kmc_buf_avail.value.ui64 = buf_avail; 19227c478bd9Sstevel@tonic-gate kmcp->kmc_buf_inuse.value.ui64 = cp->cache_buftotal - buf_avail; 19237c478bd9Sstevel@tonic-gate kmcp->kmc_buf_total.value.ui64 = cp->cache_buftotal; 19247c478bd9Sstevel@tonic-gate kmcp->kmc_buf_max.value.ui64 = cp->cache_bufmax; 19257c478bd9Sstevel@tonic-gate kmcp->kmc_slab_create.value.ui64 = cp->cache_slab_create; 19267c478bd9Sstevel@tonic-gate kmcp->kmc_slab_destroy.value.ui64 = cp->cache_slab_destroy; 19277c478bd9Sstevel@tonic-gate kmcp->kmc_hash_size.value.ui64 = (cp->cache_flags & KMF_HASH) ? 19287c478bd9Sstevel@tonic-gate cp->cache_hash_mask + 1 : 0; 19297c478bd9Sstevel@tonic-gate kmcp->kmc_hash_lookup_depth.value.ui64 = cp->cache_lookup_depth; 19307c478bd9Sstevel@tonic-gate kmcp->kmc_hash_rescale.value.ui64 = cp->cache_rescale; 19317c478bd9Sstevel@tonic-gate kmcp->kmc_vmem_source.value.ui64 = cp->cache_arena->vm_id; 19327c478bd9Sstevel@tonic-gate 19337c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 19347c478bd9Sstevel@tonic-gate return (0); 19357c478bd9Sstevel@tonic-gate } 19367c478bd9Sstevel@tonic-gate 19377c478bd9Sstevel@tonic-gate /* 19387c478bd9Sstevel@tonic-gate * Return a named statistic about a particular cache. 19397c478bd9Sstevel@tonic-gate * This shouldn't be called very often, so it's currently designed for 19407c478bd9Sstevel@tonic-gate * simplicity (leverages existing kstat support) rather than efficiency. 19417c478bd9Sstevel@tonic-gate */ 19427c478bd9Sstevel@tonic-gate uint64_t 19437c478bd9Sstevel@tonic-gate kmem_cache_stat(kmem_cache_t *cp, char *name) 19447c478bd9Sstevel@tonic-gate { 19457c478bd9Sstevel@tonic-gate int i; 19467c478bd9Sstevel@tonic-gate kstat_t *ksp = cp->cache_kstat; 19477c478bd9Sstevel@tonic-gate kstat_named_t *knp = (kstat_named_t *)&kmem_cache_kstat; 19487c478bd9Sstevel@tonic-gate uint64_t value = 0; 19497c478bd9Sstevel@tonic-gate 19507c478bd9Sstevel@tonic-gate if (ksp != NULL) { 19517c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_kstat_lock); 19527c478bd9Sstevel@tonic-gate (void) kmem_cache_kstat_update(ksp, KSTAT_READ); 19537c478bd9Sstevel@tonic-gate for (i = 0; i < ksp->ks_ndata; i++) { 19547c478bd9Sstevel@tonic-gate if (strcmp(knp[i].name, name) == 0) { 19557c478bd9Sstevel@tonic-gate value = knp[i].value.ui64; 19567c478bd9Sstevel@tonic-gate break; 19577c478bd9Sstevel@tonic-gate } 19587c478bd9Sstevel@tonic-gate } 19597c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_kstat_lock); 19607c478bd9Sstevel@tonic-gate } 19617c478bd9Sstevel@tonic-gate return (value); 19627c478bd9Sstevel@tonic-gate } 19637c478bd9Sstevel@tonic-gate 19647c478bd9Sstevel@tonic-gate /* 19657c478bd9Sstevel@tonic-gate * Return an estimate of currently available kernel heap memory. 19667c478bd9Sstevel@tonic-gate * On 32-bit systems, physical memory may exceed virtual memory, 19677c478bd9Sstevel@tonic-gate * we just truncate the result at 1GB. 19687c478bd9Sstevel@tonic-gate */ 19697c478bd9Sstevel@tonic-gate size_t 19707c478bd9Sstevel@tonic-gate kmem_avail(void) 19717c478bd9Sstevel@tonic-gate { 19727c478bd9Sstevel@tonic-gate spgcnt_t rmem = availrmem - tune.t_minarmem; 19737c478bd9Sstevel@tonic-gate spgcnt_t fmem = freemem - minfree; 19747c478bd9Sstevel@tonic-gate 19757c478bd9Sstevel@tonic-gate return ((size_t)ptob(MIN(MAX(MIN(rmem, fmem), 0), 19767c478bd9Sstevel@tonic-gate 1 << (30 - PAGESHIFT)))); 19777c478bd9Sstevel@tonic-gate } 19787c478bd9Sstevel@tonic-gate 19797c478bd9Sstevel@tonic-gate /* 19807c478bd9Sstevel@tonic-gate * Return the maximum amount of memory that is (in theory) allocatable 19817c478bd9Sstevel@tonic-gate * from the heap. This may be used as an estimate only since there 19827c478bd9Sstevel@tonic-gate * is no guarentee this space will still be available when an allocation 19837c478bd9Sstevel@tonic-gate * request is made, nor that the space may be allocated in one big request 19847c478bd9Sstevel@tonic-gate * due to kernel heap fragmentation. 19857c478bd9Sstevel@tonic-gate */ 19867c478bd9Sstevel@tonic-gate size_t 19877c478bd9Sstevel@tonic-gate kmem_maxavail(void) 19887c478bd9Sstevel@tonic-gate { 19897c478bd9Sstevel@tonic-gate spgcnt_t pmem = availrmem - tune.t_minarmem; 19907c478bd9Sstevel@tonic-gate spgcnt_t vmem = btop(vmem_size(heap_arena, VMEM_FREE)); 19917c478bd9Sstevel@tonic-gate 19927c478bd9Sstevel@tonic-gate return ((size_t)ptob(MAX(MIN(pmem, vmem), 0))); 19937c478bd9Sstevel@tonic-gate } 19947c478bd9Sstevel@tonic-gate 1995fa9e4066Sahrens /* 1996fa9e4066Sahrens * Indicate whether memory-intensive kmem debugging is enabled. 1997fa9e4066Sahrens */ 1998fa9e4066Sahrens int 1999fa9e4066Sahrens kmem_debugging(void) 2000fa9e4066Sahrens { 2001fa9e4066Sahrens return (kmem_flags & (KMF_AUDIT | KMF_REDZONE)); 2002fa9e4066Sahrens } 2003fa9e4066Sahrens 20047c478bd9Sstevel@tonic-gate kmem_cache_t * 20057c478bd9Sstevel@tonic-gate kmem_cache_create( 20067c478bd9Sstevel@tonic-gate char *name, /* descriptive name for this cache */ 20077c478bd9Sstevel@tonic-gate size_t bufsize, /* size of the objects it manages */ 20087c478bd9Sstevel@tonic-gate size_t align, /* required object alignment */ 20097c478bd9Sstevel@tonic-gate int (*constructor)(void *, void *, int), /* object constructor */ 20107c478bd9Sstevel@tonic-gate void (*destructor)(void *, void *), /* object destructor */ 20117c478bd9Sstevel@tonic-gate void (*reclaim)(void *), /* memory reclaim callback */ 20127c478bd9Sstevel@tonic-gate void *private, /* pass-thru arg for constr/destr/reclaim */ 20137c478bd9Sstevel@tonic-gate vmem_t *vmp, /* vmem source for slab allocation */ 20147c478bd9Sstevel@tonic-gate int cflags) /* cache creation flags */ 20157c478bd9Sstevel@tonic-gate { 20167c478bd9Sstevel@tonic-gate int cpu_seqid; 20177c478bd9Sstevel@tonic-gate size_t chunksize; 20187c478bd9Sstevel@tonic-gate kmem_cache_t *cp, *cnext, *cprev; 20197c478bd9Sstevel@tonic-gate kmem_magtype_t *mtp; 20207c478bd9Sstevel@tonic-gate size_t csize = KMEM_CACHE_SIZE(max_ncpus); 20217c478bd9Sstevel@tonic-gate 20227c478bd9Sstevel@tonic-gate #ifdef DEBUG 20237c478bd9Sstevel@tonic-gate /* 20247c478bd9Sstevel@tonic-gate * Cache names should conform to the rules for valid C identifiers 20257c478bd9Sstevel@tonic-gate */ 20267c478bd9Sstevel@tonic-gate if (!strident_valid(name)) { 20277c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, 20287c478bd9Sstevel@tonic-gate "kmem_cache_create: '%s' is an invalid cache name\n" 20297c478bd9Sstevel@tonic-gate "cache names must conform to the rules for " 20307c478bd9Sstevel@tonic-gate "C identifiers\n", name); 20317c478bd9Sstevel@tonic-gate } 20327c478bd9Sstevel@tonic-gate #endif /* DEBUG */ 20337c478bd9Sstevel@tonic-gate 20347c478bd9Sstevel@tonic-gate if (vmp == NULL) 20357c478bd9Sstevel@tonic-gate vmp = kmem_default_arena; 20367c478bd9Sstevel@tonic-gate 20377c478bd9Sstevel@tonic-gate /* 20387c478bd9Sstevel@tonic-gate * If this kmem cache has an identifier vmem arena as its source, mark 20397c478bd9Sstevel@tonic-gate * it such to allow kmem_reap_idspace(). 20407c478bd9Sstevel@tonic-gate */ 20417c478bd9Sstevel@tonic-gate ASSERT(!(cflags & KMC_IDENTIFIER)); /* consumer should not set this */ 20427c478bd9Sstevel@tonic-gate if (vmp->vm_cflags & VMC_IDENTIFIER) 20437c478bd9Sstevel@tonic-gate cflags |= KMC_IDENTIFIER; 20447c478bd9Sstevel@tonic-gate 20457c478bd9Sstevel@tonic-gate /* 20467c478bd9Sstevel@tonic-gate * Get a kmem_cache structure. We arrange that cp->cache_cpu[] 20477c478bd9Sstevel@tonic-gate * is aligned on a KMEM_CPU_CACHE_SIZE boundary to prevent 20487c478bd9Sstevel@tonic-gate * false sharing of per-CPU data. 20497c478bd9Sstevel@tonic-gate */ 20507c478bd9Sstevel@tonic-gate cp = vmem_xalloc(kmem_cache_arena, csize, KMEM_CPU_CACHE_SIZE, 20517c478bd9Sstevel@tonic-gate P2NPHASE(csize, KMEM_CPU_CACHE_SIZE), 0, NULL, NULL, VM_SLEEP); 20527c478bd9Sstevel@tonic-gate bzero(cp, csize); 20537c478bd9Sstevel@tonic-gate 20547c478bd9Sstevel@tonic-gate if (align == 0) 20557c478bd9Sstevel@tonic-gate align = KMEM_ALIGN; 20567c478bd9Sstevel@tonic-gate 20577c478bd9Sstevel@tonic-gate /* 20587c478bd9Sstevel@tonic-gate * If we're not at least KMEM_ALIGN aligned, we can't use free 20597c478bd9Sstevel@tonic-gate * memory to hold bufctl information (because we can't safely 20607c478bd9Sstevel@tonic-gate * perform word loads and stores on it). 20617c478bd9Sstevel@tonic-gate */ 20627c478bd9Sstevel@tonic-gate if (align < KMEM_ALIGN) 20637c478bd9Sstevel@tonic-gate cflags |= KMC_NOTOUCH; 20647c478bd9Sstevel@tonic-gate 20657c478bd9Sstevel@tonic-gate if ((align & (align - 1)) != 0 || align > vmp->vm_quantum) 20667c478bd9Sstevel@tonic-gate panic("kmem_cache_create: bad alignment %lu", align); 20677c478bd9Sstevel@tonic-gate 20687c478bd9Sstevel@tonic-gate mutex_enter(&kmem_flags_lock); 20697c478bd9Sstevel@tonic-gate if (kmem_flags & KMF_RANDOMIZE) 20707c478bd9Sstevel@tonic-gate kmem_flags = (((kmem_flags | ~KMF_RANDOM) + 1) & KMF_RANDOM) | 20717c478bd9Sstevel@tonic-gate KMF_RANDOMIZE; 20727c478bd9Sstevel@tonic-gate cp->cache_flags = (kmem_flags | cflags) & KMF_DEBUG; 20737c478bd9Sstevel@tonic-gate mutex_exit(&kmem_flags_lock); 20747c478bd9Sstevel@tonic-gate 20757c478bd9Sstevel@tonic-gate /* 20767c478bd9Sstevel@tonic-gate * Make sure all the various flags are reasonable. 20777c478bd9Sstevel@tonic-gate */ 20787c478bd9Sstevel@tonic-gate ASSERT(!(cflags & KMC_NOHASH) || !(cflags & KMC_NOTOUCH)); 20797c478bd9Sstevel@tonic-gate 20807c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) { 20817c478bd9Sstevel@tonic-gate if (bufsize >= kmem_lite_minsize && 20827c478bd9Sstevel@tonic-gate align <= kmem_lite_maxalign && 20837c478bd9Sstevel@tonic-gate P2PHASE(bufsize, kmem_lite_maxalign) != 0) { 20847c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_BUFTAG; 20857c478bd9Sstevel@tonic-gate cp->cache_flags &= ~(KMF_AUDIT | KMF_FIREWALL); 20867c478bd9Sstevel@tonic-gate } else { 20877c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_DEBUG; 20887c478bd9Sstevel@tonic-gate } 20897c478bd9Sstevel@tonic-gate } 20907c478bd9Sstevel@tonic-gate 20917c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) 20927c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_REDZONE; 20937c478bd9Sstevel@tonic-gate 20947c478bd9Sstevel@tonic-gate if ((cflags & KMC_QCACHE) && (cp->cache_flags & KMF_AUDIT)) 20957c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_NOMAGAZINE; 20967c478bd9Sstevel@tonic-gate 20977c478bd9Sstevel@tonic-gate if (cflags & KMC_NODEBUG) 20987c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_DEBUG; 20997c478bd9Sstevel@tonic-gate 21007c478bd9Sstevel@tonic-gate if (cflags & KMC_NOTOUCH) 21017c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_TOUCH; 21027c478bd9Sstevel@tonic-gate 21037c478bd9Sstevel@tonic-gate if (cflags & KMC_NOHASH) 21047c478bd9Sstevel@tonic-gate cp->cache_flags &= ~(KMF_AUDIT | KMF_FIREWALL); 21057c478bd9Sstevel@tonic-gate 21067c478bd9Sstevel@tonic-gate if (cflags & KMC_NOMAGAZINE) 21077c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_NOMAGAZINE; 21087c478bd9Sstevel@tonic-gate 21097c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_AUDIT) && !(cflags & KMC_NOTOUCH)) 21107c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_REDZONE; 21117c478bd9Sstevel@tonic-gate 21127c478bd9Sstevel@tonic-gate if (!(cp->cache_flags & KMF_AUDIT)) 21137c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_CONTENTS; 21147c478bd9Sstevel@tonic-gate 21157c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_BUFTAG) && bufsize >= kmem_minfirewall && 21167c478bd9Sstevel@tonic-gate !(cp->cache_flags & KMF_LITE) && !(cflags & KMC_NOHASH)) 21177c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_FIREWALL; 21187c478bd9Sstevel@tonic-gate 21197c478bd9Sstevel@tonic-gate if (vmp != kmem_default_arena || kmem_firewall_arena == NULL) 21207c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_FIREWALL; 21217c478bd9Sstevel@tonic-gate 21227c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_FIREWALL) { 21237c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_BUFTAG; 21247c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_NOMAGAZINE; 21257c478bd9Sstevel@tonic-gate ASSERT(vmp == kmem_default_arena); 21267c478bd9Sstevel@tonic-gate vmp = kmem_firewall_arena; 21277c478bd9Sstevel@tonic-gate } 21287c478bd9Sstevel@tonic-gate 21297c478bd9Sstevel@tonic-gate /* 21307c478bd9Sstevel@tonic-gate * Set cache properties. 21317c478bd9Sstevel@tonic-gate */ 21327c478bd9Sstevel@tonic-gate (void) strncpy(cp->cache_name, name, KMEM_CACHE_NAMELEN); 21337c478bd9Sstevel@tonic-gate strident_canon(cp->cache_name, KMEM_CACHE_NAMELEN); 21347c478bd9Sstevel@tonic-gate cp->cache_bufsize = bufsize; 21357c478bd9Sstevel@tonic-gate cp->cache_align = align; 21367c478bd9Sstevel@tonic-gate cp->cache_constructor = constructor; 21377c478bd9Sstevel@tonic-gate cp->cache_destructor = destructor; 21387c478bd9Sstevel@tonic-gate cp->cache_reclaim = reclaim; 21397c478bd9Sstevel@tonic-gate cp->cache_private = private; 21407c478bd9Sstevel@tonic-gate cp->cache_arena = vmp; 21417c478bd9Sstevel@tonic-gate cp->cache_cflags = cflags; 21427c478bd9Sstevel@tonic-gate 21437c478bd9Sstevel@tonic-gate /* 21447c478bd9Sstevel@tonic-gate * Determine the chunk size. 21457c478bd9Sstevel@tonic-gate */ 21467c478bd9Sstevel@tonic-gate chunksize = bufsize; 21477c478bd9Sstevel@tonic-gate 21487c478bd9Sstevel@tonic-gate if (align >= KMEM_ALIGN) { 21497c478bd9Sstevel@tonic-gate chunksize = P2ROUNDUP(chunksize, KMEM_ALIGN); 21507c478bd9Sstevel@tonic-gate cp->cache_bufctl = chunksize - KMEM_ALIGN; 21517c478bd9Sstevel@tonic-gate } 21527c478bd9Sstevel@tonic-gate 21537c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_BUFTAG) { 21547c478bd9Sstevel@tonic-gate cp->cache_bufctl = chunksize; 21557c478bd9Sstevel@tonic-gate cp->cache_buftag = chunksize; 21567c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) 21577c478bd9Sstevel@tonic-gate chunksize += KMEM_BUFTAG_LITE_SIZE(kmem_lite_count); 21587c478bd9Sstevel@tonic-gate else 21597c478bd9Sstevel@tonic-gate chunksize += sizeof (kmem_buftag_t); 21607c478bd9Sstevel@tonic-gate } 21617c478bd9Sstevel@tonic-gate 21627c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) { 21637c478bd9Sstevel@tonic-gate cp->cache_verify = MIN(cp->cache_buftag, kmem_maxverify); 21647c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) 21657c478bd9Sstevel@tonic-gate cp->cache_verify = sizeof (uint64_t); 21667c478bd9Sstevel@tonic-gate } 21677c478bd9Sstevel@tonic-gate 21687c478bd9Sstevel@tonic-gate cp->cache_contents = MIN(cp->cache_bufctl, kmem_content_maxsave); 21697c478bd9Sstevel@tonic-gate 21707c478bd9Sstevel@tonic-gate cp->cache_chunksize = chunksize = P2ROUNDUP(chunksize, align); 21717c478bd9Sstevel@tonic-gate 21727c478bd9Sstevel@tonic-gate /* 21737c478bd9Sstevel@tonic-gate * Now that we know the chunk size, determine the optimal slab size. 21747c478bd9Sstevel@tonic-gate */ 21757c478bd9Sstevel@tonic-gate if (vmp == kmem_firewall_arena) { 21767c478bd9Sstevel@tonic-gate cp->cache_slabsize = P2ROUNDUP(chunksize, vmp->vm_quantum); 21777c478bd9Sstevel@tonic-gate cp->cache_mincolor = cp->cache_slabsize - chunksize; 21787c478bd9Sstevel@tonic-gate cp->cache_maxcolor = cp->cache_mincolor; 21797c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_HASH; 21807c478bd9Sstevel@tonic-gate ASSERT(!(cp->cache_flags & KMF_BUFTAG)); 21817c478bd9Sstevel@tonic-gate } else if ((cflags & KMC_NOHASH) || (!(cflags & KMC_NOTOUCH) && 21827c478bd9Sstevel@tonic-gate !(cp->cache_flags & KMF_AUDIT) && 21837c478bd9Sstevel@tonic-gate chunksize < vmp->vm_quantum / KMEM_VOID_FRACTION)) { 21847c478bd9Sstevel@tonic-gate cp->cache_slabsize = vmp->vm_quantum; 21857c478bd9Sstevel@tonic-gate cp->cache_mincolor = 0; 21867c478bd9Sstevel@tonic-gate cp->cache_maxcolor = 21877c478bd9Sstevel@tonic-gate (cp->cache_slabsize - sizeof (kmem_slab_t)) % chunksize; 21887c478bd9Sstevel@tonic-gate ASSERT(chunksize + sizeof (kmem_slab_t) <= cp->cache_slabsize); 21897c478bd9Sstevel@tonic-gate ASSERT(!(cp->cache_flags & KMF_AUDIT)); 21907c478bd9Sstevel@tonic-gate } else { 21917c478bd9Sstevel@tonic-gate size_t chunks, bestfit, waste, slabsize; 21927c478bd9Sstevel@tonic-gate size_t minwaste = LONG_MAX; 21937c478bd9Sstevel@tonic-gate 21947c478bd9Sstevel@tonic-gate for (chunks = 1; chunks <= KMEM_VOID_FRACTION; chunks++) { 21957c478bd9Sstevel@tonic-gate slabsize = P2ROUNDUP(chunksize * chunks, 21967c478bd9Sstevel@tonic-gate vmp->vm_quantum); 21977c478bd9Sstevel@tonic-gate chunks = slabsize / chunksize; 21987c478bd9Sstevel@tonic-gate waste = (slabsize % chunksize) / chunks; 21997c478bd9Sstevel@tonic-gate if (waste < minwaste) { 22007c478bd9Sstevel@tonic-gate minwaste = waste; 22017c478bd9Sstevel@tonic-gate bestfit = slabsize; 22027c478bd9Sstevel@tonic-gate } 22037c478bd9Sstevel@tonic-gate } 22047c478bd9Sstevel@tonic-gate if (cflags & KMC_QCACHE) 22057c478bd9Sstevel@tonic-gate bestfit = VMEM_QCACHE_SLABSIZE(vmp->vm_qcache_max); 22067c478bd9Sstevel@tonic-gate cp->cache_slabsize = bestfit; 22077c478bd9Sstevel@tonic-gate cp->cache_mincolor = 0; 22087c478bd9Sstevel@tonic-gate cp->cache_maxcolor = bestfit % chunksize; 22097c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_HASH; 22107c478bd9Sstevel@tonic-gate } 22117c478bd9Sstevel@tonic-gate 22127c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) { 22137c478bd9Sstevel@tonic-gate ASSERT(!(cflags & KMC_NOHASH)); 22147c478bd9Sstevel@tonic-gate cp->cache_bufctl_cache = (cp->cache_flags & KMF_AUDIT) ? 22157c478bd9Sstevel@tonic-gate kmem_bufctl_audit_cache : kmem_bufctl_cache; 22167c478bd9Sstevel@tonic-gate } 22177c478bd9Sstevel@tonic-gate 22187c478bd9Sstevel@tonic-gate if (cp->cache_maxcolor >= vmp->vm_quantum) 22197c478bd9Sstevel@tonic-gate cp->cache_maxcolor = vmp->vm_quantum - 1; 22207c478bd9Sstevel@tonic-gate 22217c478bd9Sstevel@tonic-gate cp->cache_color = cp->cache_mincolor; 22227c478bd9Sstevel@tonic-gate 22237c478bd9Sstevel@tonic-gate /* 22247c478bd9Sstevel@tonic-gate * Initialize the rest of the slab layer. 22257c478bd9Sstevel@tonic-gate */ 22267c478bd9Sstevel@tonic-gate mutex_init(&cp->cache_lock, NULL, MUTEX_DEFAULT, NULL); 22277c478bd9Sstevel@tonic-gate 22287c478bd9Sstevel@tonic-gate cp->cache_freelist = &cp->cache_nullslab; 22297c478bd9Sstevel@tonic-gate cp->cache_nullslab.slab_cache = cp; 22307c478bd9Sstevel@tonic-gate cp->cache_nullslab.slab_refcnt = -1; 22317c478bd9Sstevel@tonic-gate cp->cache_nullslab.slab_next = &cp->cache_nullslab; 22327c478bd9Sstevel@tonic-gate cp->cache_nullslab.slab_prev = &cp->cache_nullslab; 22337c478bd9Sstevel@tonic-gate 22347c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) { 22357c478bd9Sstevel@tonic-gate cp->cache_hash_table = vmem_alloc(kmem_hash_arena, 22367c478bd9Sstevel@tonic-gate KMEM_HASH_INITIAL * sizeof (void *), VM_SLEEP); 22377c478bd9Sstevel@tonic-gate bzero(cp->cache_hash_table, 22387c478bd9Sstevel@tonic-gate KMEM_HASH_INITIAL * sizeof (void *)); 22397c478bd9Sstevel@tonic-gate cp->cache_hash_mask = KMEM_HASH_INITIAL - 1; 22407c478bd9Sstevel@tonic-gate cp->cache_hash_shift = highbit((ulong_t)chunksize) - 1; 22417c478bd9Sstevel@tonic-gate } 22427c478bd9Sstevel@tonic-gate 22437c478bd9Sstevel@tonic-gate /* 22447c478bd9Sstevel@tonic-gate * Initialize the depot. 22457c478bd9Sstevel@tonic-gate */ 22467c478bd9Sstevel@tonic-gate mutex_init(&cp->cache_depot_lock, NULL, MUTEX_DEFAULT, NULL); 22477c478bd9Sstevel@tonic-gate 22487c478bd9Sstevel@tonic-gate for (mtp = kmem_magtype; chunksize <= mtp->mt_minbuf; mtp++) 22497c478bd9Sstevel@tonic-gate continue; 22507c478bd9Sstevel@tonic-gate 22517c478bd9Sstevel@tonic-gate cp->cache_magtype = mtp; 22527c478bd9Sstevel@tonic-gate 22537c478bd9Sstevel@tonic-gate /* 22547c478bd9Sstevel@tonic-gate * Initialize the CPU layer. 22557c478bd9Sstevel@tonic-gate */ 22567c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 22577c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 22587c478bd9Sstevel@tonic-gate mutex_init(&ccp->cc_lock, NULL, MUTEX_DEFAULT, NULL); 22597c478bd9Sstevel@tonic-gate ccp->cc_flags = cp->cache_flags; 22607c478bd9Sstevel@tonic-gate ccp->cc_rounds = -1; 22617c478bd9Sstevel@tonic-gate ccp->cc_prounds = -1; 22627c478bd9Sstevel@tonic-gate } 22637c478bd9Sstevel@tonic-gate 22647c478bd9Sstevel@tonic-gate /* 22657c478bd9Sstevel@tonic-gate * Create the cache's kstats. 22667c478bd9Sstevel@tonic-gate */ 22677c478bd9Sstevel@tonic-gate if ((cp->cache_kstat = kstat_create("unix", 0, cp->cache_name, 22687c478bd9Sstevel@tonic-gate "kmem_cache", KSTAT_TYPE_NAMED, 22697c478bd9Sstevel@tonic-gate sizeof (kmem_cache_kstat) / sizeof (kstat_named_t), 22707c478bd9Sstevel@tonic-gate KSTAT_FLAG_VIRTUAL)) != NULL) { 22717c478bd9Sstevel@tonic-gate cp->cache_kstat->ks_data = &kmem_cache_kstat; 22727c478bd9Sstevel@tonic-gate cp->cache_kstat->ks_update = kmem_cache_kstat_update; 22737c478bd9Sstevel@tonic-gate cp->cache_kstat->ks_private = cp; 22747c478bd9Sstevel@tonic-gate cp->cache_kstat->ks_lock = &kmem_cache_kstat_lock; 22757c478bd9Sstevel@tonic-gate kstat_install(cp->cache_kstat); 22767c478bd9Sstevel@tonic-gate } 22777c478bd9Sstevel@tonic-gate 22787c478bd9Sstevel@tonic-gate /* 22797c478bd9Sstevel@tonic-gate * Add the cache to the global list. This makes it visible 22807c478bd9Sstevel@tonic-gate * to kmem_update(), so the cache must be ready for business. 22817c478bd9Sstevel@tonic-gate */ 22827c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_lock); 22837c478bd9Sstevel@tonic-gate cp->cache_next = cnext = &kmem_null_cache; 22847c478bd9Sstevel@tonic-gate cp->cache_prev = cprev = kmem_null_cache.cache_prev; 22857c478bd9Sstevel@tonic-gate cnext->cache_prev = cp; 22867c478bd9Sstevel@tonic-gate cprev->cache_next = cp; 22877c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_lock); 22887c478bd9Sstevel@tonic-gate 22897c478bd9Sstevel@tonic-gate if (kmem_ready) 22907c478bd9Sstevel@tonic-gate kmem_cache_magazine_enable(cp); 22917c478bd9Sstevel@tonic-gate 22927c478bd9Sstevel@tonic-gate return (cp); 22937c478bd9Sstevel@tonic-gate } 22947c478bd9Sstevel@tonic-gate 22957c478bd9Sstevel@tonic-gate void 22967c478bd9Sstevel@tonic-gate kmem_cache_destroy(kmem_cache_t *cp) 22977c478bd9Sstevel@tonic-gate { 22987c478bd9Sstevel@tonic-gate int cpu_seqid; 22997c478bd9Sstevel@tonic-gate 23007c478bd9Sstevel@tonic-gate /* 23017c478bd9Sstevel@tonic-gate * Remove the cache from the global cache list so that no one else 23027c478bd9Sstevel@tonic-gate * can schedule tasks on its behalf, wait for any pending tasks to 23037c478bd9Sstevel@tonic-gate * complete, purge the cache, and then destroy it. 23047c478bd9Sstevel@tonic-gate */ 23057c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_lock); 23067c478bd9Sstevel@tonic-gate cp->cache_prev->cache_next = cp->cache_next; 23077c478bd9Sstevel@tonic-gate cp->cache_next->cache_prev = cp->cache_prev; 23087c478bd9Sstevel@tonic-gate cp->cache_prev = cp->cache_next = NULL; 23097c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_lock); 23107c478bd9Sstevel@tonic-gate 23117c478bd9Sstevel@tonic-gate if (kmem_taskq != NULL) 23127c478bd9Sstevel@tonic-gate taskq_wait(kmem_taskq); 23137c478bd9Sstevel@tonic-gate 23147c478bd9Sstevel@tonic-gate kmem_cache_magazine_purge(cp); 23157c478bd9Sstevel@tonic-gate 23167c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 23177c478bd9Sstevel@tonic-gate if (cp->cache_buftotal != 0) 23187c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "kmem_cache_destroy: '%s' (%p) not empty", 23197c478bd9Sstevel@tonic-gate cp->cache_name, (void *)cp); 23207c478bd9Sstevel@tonic-gate cp->cache_reclaim = NULL; 23217c478bd9Sstevel@tonic-gate /* 23227c478bd9Sstevel@tonic-gate * The cache is now dead. There should be no further activity. 23237c478bd9Sstevel@tonic-gate * We enforce this by setting land mines in the constructor and 23247c478bd9Sstevel@tonic-gate * destructor routines that induce a kernel text fault if invoked. 23257c478bd9Sstevel@tonic-gate */ 23267c478bd9Sstevel@tonic-gate cp->cache_constructor = (int (*)(void *, void *, int))1; 23277c478bd9Sstevel@tonic-gate cp->cache_destructor = (void (*)(void *, void *))2; 23287c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 23297c478bd9Sstevel@tonic-gate 23307c478bd9Sstevel@tonic-gate kstat_delete(cp->cache_kstat); 23317c478bd9Sstevel@tonic-gate 23327c478bd9Sstevel@tonic-gate if (cp->cache_hash_table != NULL) 23337c478bd9Sstevel@tonic-gate vmem_free(kmem_hash_arena, cp->cache_hash_table, 23347c478bd9Sstevel@tonic-gate (cp->cache_hash_mask + 1) * sizeof (void *)); 23357c478bd9Sstevel@tonic-gate 23367c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) 23377c478bd9Sstevel@tonic-gate mutex_destroy(&cp->cache_cpu[cpu_seqid].cc_lock); 23387c478bd9Sstevel@tonic-gate 23397c478bd9Sstevel@tonic-gate mutex_destroy(&cp->cache_depot_lock); 23407c478bd9Sstevel@tonic-gate mutex_destroy(&cp->cache_lock); 23417c478bd9Sstevel@tonic-gate 23427c478bd9Sstevel@tonic-gate vmem_free(kmem_cache_arena, cp, KMEM_CACHE_SIZE(max_ncpus)); 23437c478bd9Sstevel@tonic-gate } 23447c478bd9Sstevel@tonic-gate 23457c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 23467c478bd9Sstevel@tonic-gate static int 23477c478bd9Sstevel@tonic-gate kmem_cpu_setup(cpu_setup_t what, int id, void *arg) 23487c478bd9Sstevel@tonic-gate { 23497c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 23507c478bd9Sstevel@tonic-gate if (what == CPU_UNCONFIG) { 23517c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_magazine_purge, 23527c478bd9Sstevel@tonic-gate kmem_taskq, TQ_SLEEP); 23537c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_magazine_enable, 23547c478bd9Sstevel@tonic-gate kmem_taskq, TQ_SLEEP); 23557c478bd9Sstevel@tonic-gate } 23567c478bd9Sstevel@tonic-gate return (0); 23577c478bd9Sstevel@tonic-gate } 23587c478bd9Sstevel@tonic-gate 23597c478bd9Sstevel@tonic-gate static void 23607c478bd9Sstevel@tonic-gate kmem_cache_init(int pass, int use_large_pages) 23617c478bd9Sstevel@tonic-gate { 23627c478bd9Sstevel@tonic-gate int i; 23637c478bd9Sstevel@tonic-gate size_t size; 23647c478bd9Sstevel@tonic-gate kmem_cache_t *cp; 23657c478bd9Sstevel@tonic-gate kmem_magtype_t *mtp; 23667c478bd9Sstevel@tonic-gate char name[KMEM_CACHE_NAMELEN + 1]; 23677c478bd9Sstevel@tonic-gate 23687c478bd9Sstevel@tonic-gate for (i = 0; i < sizeof (kmem_magtype) / sizeof (*mtp); i++) { 23697c478bd9Sstevel@tonic-gate mtp = &kmem_magtype[i]; 23707c478bd9Sstevel@tonic-gate (void) sprintf(name, "kmem_magazine_%d", mtp->mt_magsize); 23717c478bd9Sstevel@tonic-gate mtp->mt_cache = kmem_cache_create(name, 23727c478bd9Sstevel@tonic-gate (mtp->mt_magsize + 1) * sizeof (void *), 23737c478bd9Sstevel@tonic-gate mtp->mt_align, NULL, NULL, NULL, NULL, 23747c478bd9Sstevel@tonic-gate kmem_msb_arena, KMC_NOHASH); 23757c478bd9Sstevel@tonic-gate } 23767c478bd9Sstevel@tonic-gate 23777c478bd9Sstevel@tonic-gate kmem_slab_cache = kmem_cache_create("kmem_slab_cache", 23787c478bd9Sstevel@tonic-gate sizeof (kmem_slab_t), 0, NULL, NULL, NULL, NULL, 23797c478bd9Sstevel@tonic-gate kmem_msb_arena, KMC_NOHASH); 23807c478bd9Sstevel@tonic-gate 23817c478bd9Sstevel@tonic-gate kmem_bufctl_cache = kmem_cache_create("kmem_bufctl_cache", 23827c478bd9Sstevel@tonic-gate sizeof (kmem_bufctl_t), 0, NULL, NULL, NULL, NULL, 23837c478bd9Sstevel@tonic-gate kmem_msb_arena, KMC_NOHASH); 23847c478bd9Sstevel@tonic-gate 23857c478bd9Sstevel@tonic-gate kmem_bufctl_audit_cache = kmem_cache_create("kmem_bufctl_audit_cache", 23867c478bd9Sstevel@tonic-gate sizeof (kmem_bufctl_audit_t), 0, NULL, NULL, NULL, NULL, 23877c478bd9Sstevel@tonic-gate kmem_msb_arena, KMC_NOHASH); 23887c478bd9Sstevel@tonic-gate 23897c478bd9Sstevel@tonic-gate if (pass == 2) { 23907c478bd9Sstevel@tonic-gate kmem_va_arena = vmem_create("kmem_va", 23917c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE, 23927c478bd9Sstevel@tonic-gate vmem_alloc, vmem_free, heap_arena, 23937c478bd9Sstevel@tonic-gate 8 * PAGESIZE, VM_SLEEP); 23947c478bd9Sstevel@tonic-gate 23957c478bd9Sstevel@tonic-gate if (use_large_pages) { 23967c478bd9Sstevel@tonic-gate kmem_default_arena = vmem_xcreate("kmem_default", 23977c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE, 23987c478bd9Sstevel@tonic-gate segkmem_alloc_lp, segkmem_free_lp, kmem_va_arena, 23997c478bd9Sstevel@tonic-gate 0, VM_SLEEP); 24007c478bd9Sstevel@tonic-gate } else { 24017c478bd9Sstevel@tonic-gate kmem_default_arena = vmem_create("kmem_default", 24027c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE, 24037c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, kmem_va_arena, 24047c478bd9Sstevel@tonic-gate 0, VM_SLEEP); 24057c478bd9Sstevel@tonic-gate } 24067c478bd9Sstevel@tonic-gate } else { 24077c478bd9Sstevel@tonic-gate /* 24087c478bd9Sstevel@tonic-gate * During the first pass, the kmem_alloc_* caches 24097c478bd9Sstevel@tonic-gate * are treated as metadata. 24107c478bd9Sstevel@tonic-gate */ 24117c478bd9Sstevel@tonic-gate kmem_default_arena = kmem_msb_arena; 24127c478bd9Sstevel@tonic-gate } 24137c478bd9Sstevel@tonic-gate 24147c478bd9Sstevel@tonic-gate /* 24157c478bd9Sstevel@tonic-gate * Set up the default caches to back kmem_alloc() 24167c478bd9Sstevel@tonic-gate */ 24177c478bd9Sstevel@tonic-gate size = KMEM_ALIGN; 24187c478bd9Sstevel@tonic-gate for (i = 0; i < sizeof (kmem_alloc_sizes) / sizeof (int); i++) { 24197c478bd9Sstevel@tonic-gate size_t align = KMEM_ALIGN; 24207c478bd9Sstevel@tonic-gate size_t cache_size = kmem_alloc_sizes[i]; 24217c478bd9Sstevel@tonic-gate /* 24227c478bd9Sstevel@tonic-gate * If they allocate a multiple of the coherency granularity, 24237c478bd9Sstevel@tonic-gate * they get a coherency-granularity-aligned address. 24247c478bd9Sstevel@tonic-gate */ 24257c478bd9Sstevel@tonic-gate if (IS_P2ALIGNED(cache_size, 64)) 24267c478bd9Sstevel@tonic-gate align = 64; 24277c478bd9Sstevel@tonic-gate if (IS_P2ALIGNED(cache_size, PAGESIZE)) 24287c478bd9Sstevel@tonic-gate align = PAGESIZE; 24297c478bd9Sstevel@tonic-gate (void) sprintf(name, "kmem_alloc_%lu", cache_size); 24307c478bd9Sstevel@tonic-gate cp = kmem_cache_create(name, cache_size, align, 24317c478bd9Sstevel@tonic-gate NULL, NULL, NULL, NULL, NULL, KMC_KMEM_ALLOC); 24327c478bd9Sstevel@tonic-gate while (size <= cache_size) { 24337c478bd9Sstevel@tonic-gate kmem_alloc_table[(size - 1) >> KMEM_ALIGN_SHIFT] = cp; 24347c478bd9Sstevel@tonic-gate size += KMEM_ALIGN; 24357c478bd9Sstevel@tonic-gate } 24367c478bd9Sstevel@tonic-gate } 24377c478bd9Sstevel@tonic-gate } 24387c478bd9Sstevel@tonic-gate 24397c478bd9Sstevel@tonic-gate void 24407c478bd9Sstevel@tonic-gate kmem_init(void) 24417c478bd9Sstevel@tonic-gate { 24427c478bd9Sstevel@tonic-gate kmem_cache_t *cp; 24437c478bd9Sstevel@tonic-gate int old_kmem_flags = kmem_flags; 24447c478bd9Sstevel@tonic-gate int use_large_pages = 0; 24457c478bd9Sstevel@tonic-gate size_t maxverify, minfirewall; 24467c478bd9Sstevel@tonic-gate 24477c478bd9Sstevel@tonic-gate kstat_init(); 24487c478bd9Sstevel@tonic-gate 24497c478bd9Sstevel@tonic-gate /* 24507c478bd9Sstevel@tonic-gate * Small-memory systems (< 24 MB) can't handle kmem_flags overhead. 24517c478bd9Sstevel@tonic-gate */ 24527c478bd9Sstevel@tonic-gate if (physmem < btop(24 << 20) && !(old_kmem_flags & KMF_STICKY)) 24537c478bd9Sstevel@tonic-gate kmem_flags = 0; 24547c478bd9Sstevel@tonic-gate 24557c478bd9Sstevel@tonic-gate /* 24567c478bd9Sstevel@tonic-gate * Don't do firewalled allocations if the heap is less than 1TB 24577c478bd9Sstevel@tonic-gate * (i.e. on a 32-bit kernel) 24587c478bd9Sstevel@tonic-gate * The resulting VM_NEXTFIT allocations would create too much 24597c478bd9Sstevel@tonic-gate * fragmentation in a small heap. 24607c478bd9Sstevel@tonic-gate */ 24617c478bd9Sstevel@tonic-gate #if defined(_LP64) 24627c478bd9Sstevel@tonic-gate maxverify = minfirewall = PAGESIZE / 2; 24637c478bd9Sstevel@tonic-gate #else 24647c478bd9Sstevel@tonic-gate maxverify = minfirewall = ULONG_MAX; 24657c478bd9Sstevel@tonic-gate #endif 24667c478bd9Sstevel@tonic-gate 24677c478bd9Sstevel@tonic-gate /* LINTED */ 24687c478bd9Sstevel@tonic-gate ASSERT(sizeof (kmem_cpu_cache_t) == KMEM_CPU_CACHE_SIZE); 24697c478bd9Sstevel@tonic-gate 24707c478bd9Sstevel@tonic-gate kmem_null_cache.cache_next = &kmem_null_cache; 24717c478bd9Sstevel@tonic-gate kmem_null_cache.cache_prev = &kmem_null_cache; 24727c478bd9Sstevel@tonic-gate 24737c478bd9Sstevel@tonic-gate kmem_metadata_arena = vmem_create("kmem_metadata", NULL, 0, PAGESIZE, 24747c478bd9Sstevel@tonic-gate vmem_alloc, vmem_free, heap_arena, 8 * PAGESIZE, 24757c478bd9Sstevel@tonic-gate VM_SLEEP | VMC_NO_QCACHE); 24767c478bd9Sstevel@tonic-gate 24777c478bd9Sstevel@tonic-gate kmem_msb_arena = vmem_create("kmem_msb", NULL, 0, 24787c478bd9Sstevel@tonic-gate PAGESIZE, segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, 24797c478bd9Sstevel@tonic-gate VM_SLEEP); 24807c478bd9Sstevel@tonic-gate 24817c478bd9Sstevel@tonic-gate kmem_cache_arena = vmem_create("kmem_cache", NULL, 0, KMEM_ALIGN, 24827c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, VM_SLEEP); 24837c478bd9Sstevel@tonic-gate 24847c478bd9Sstevel@tonic-gate kmem_hash_arena = vmem_create("kmem_hash", NULL, 0, KMEM_ALIGN, 24857c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, VM_SLEEP); 24867c478bd9Sstevel@tonic-gate 24877c478bd9Sstevel@tonic-gate kmem_log_arena = vmem_create("kmem_log", NULL, 0, KMEM_ALIGN, 24887c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP); 24897c478bd9Sstevel@tonic-gate 24907c478bd9Sstevel@tonic-gate kmem_firewall_va_arena = vmem_create("kmem_firewall_va", 24917c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE, 24927c478bd9Sstevel@tonic-gate kmem_firewall_va_alloc, kmem_firewall_va_free, heap_arena, 24937c478bd9Sstevel@tonic-gate 0, VM_SLEEP); 24947c478bd9Sstevel@tonic-gate 24957c478bd9Sstevel@tonic-gate kmem_firewall_arena = vmem_create("kmem_firewall", NULL, 0, PAGESIZE, 24967c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, kmem_firewall_va_arena, 0, VM_SLEEP); 24977c478bd9Sstevel@tonic-gate 24987c478bd9Sstevel@tonic-gate /* temporary oversize arena for mod_read_system_file */ 24997c478bd9Sstevel@tonic-gate kmem_oversize_arena = vmem_create("kmem_oversize", NULL, 0, PAGESIZE, 25007c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP); 25017c478bd9Sstevel@tonic-gate 25027c478bd9Sstevel@tonic-gate kmem_null_cache.cache_next = &kmem_null_cache; 25037c478bd9Sstevel@tonic-gate kmem_null_cache.cache_prev = &kmem_null_cache; 25047c478bd9Sstevel@tonic-gate 25057c478bd9Sstevel@tonic-gate kmem_reap_interval = 15 * hz; 25067c478bd9Sstevel@tonic-gate 25077c478bd9Sstevel@tonic-gate /* 25087c478bd9Sstevel@tonic-gate * Read /etc/system. This is a chicken-and-egg problem because 25097c478bd9Sstevel@tonic-gate * kmem_flags may be set in /etc/system, but mod_read_system_file() 25107c478bd9Sstevel@tonic-gate * needs to use the allocator. The simplest solution is to create 25117c478bd9Sstevel@tonic-gate * all the standard kmem caches, read /etc/system, destroy all the 25127c478bd9Sstevel@tonic-gate * caches we just created, and then create them all again in light 25137c478bd9Sstevel@tonic-gate * of the (possibly) new kmem_flags and other kmem tunables. 25147c478bd9Sstevel@tonic-gate */ 25157c478bd9Sstevel@tonic-gate kmem_cache_init(1, 0); 25167c478bd9Sstevel@tonic-gate 25177c478bd9Sstevel@tonic-gate mod_read_system_file(boothowto & RB_ASKNAME); 25187c478bd9Sstevel@tonic-gate 25197c478bd9Sstevel@tonic-gate while ((cp = kmem_null_cache.cache_prev) != &kmem_null_cache) 25207c478bd9Sstevel@tonic-gate kmem_cache_destroy(cp); 25217c478bd9Sstevel@tonic-gate 25227c478bd9Sstevel@tonic-gate vmem_destroy(kmem_oversize_arena); 25237c478bd9Sstevel@tonic-gate 25247c478bd9Sstevel@tonic-gate if (old_kmem_flags & KMF_STICKY) 25257c478bd9Sstevel@tonic-gate kmem_flags = old_kmem_flags; 25267c478bd9Sstevel@tonic-gate 25277c478bd9Sstevel@tonic-gate if (!(kmem_flags & KMF_AUDIT)) 25287c478bd9Sstevel@tonic-gate vmem_seg_size = offsetof(vmem_seg_t, vs_thread); 25297c478bd9Sstevel@tonic-gate 25307c478bd9Sstevel@tonic-gate if (kmem_maxverify == 0) 25317c478bd9Sstevel@tonic-gate kmem_maxverify = maxverify; 25327c478bd9Sstevel@tonic-gate 25337c478bd9Sstevel@tonic-gate if (kmem_minfirewall == 0) 25347c478bd9Sstevel@tonic-gate kmem_minfirewall = minfirewall; 25357c478bd9Sstevel@tonic-gate 25367c478bd9Sstevel@tonic-gate /* 25377c478bd9Sstevel@tonic-gate * give segkmem a chance to figure out if we are using large pages 25387c478bd9Sstevel@tonic-gate * for the kernel heap 25397c478bd9Sstevel@tonic-gate */ 25407c478bd9Sstevel@tonic-gate use_large_pages = segkmem_lpsetup(); 25417c478bd9Sstevel@tonic-gate 25427c478bd9Sstevel@tonic-gate /* 25437c478bd9Sstevel@tonic-gate * To protect against corruption, we keep the actual number of callers 25447c478bd9Sstevel@tonic-gate * KMF_LITE records seperate from the tunable. We arbitrarily clamp 25457c478bd9Sstevel@tonic-gate * to 16, since the overhead for small buffers quickly gets out of 25467c478bd9Sstevel@tonic-gate * hand. 25477c478bd9Sstevel@tonic-gate * 25487c478bd9Sstevel@tonic-gate * The real limit would depend on the needs of the largest KMC_NOHASH 25497c478bd9Sstevel@tonic-gate * cache. 25507c478bd9Sstevel@tonic-gate */ 25517c478bd9Sstevel@tonic-gate kmem_lite_count = MIN(MAX(0, kmem_lite_pcs), 16); 25527c478bd9Sstevel@tonic-gate kmem_lite_pcs = kmem_lite_count; 25537c478bd9Sstevel@tonic-gate 25547c478bd9Sstevel@tonic-gate /* 25557c478bd9Sstevel@tonic-gate * Normally, we firewall oversized allocations when possible, but 25567c478bd9Sstevel@tonic-gate * if we are using large pages for kernel memory, and we don't have 25577c478bd9Sstevel@tonic-gate * any non-LITE debugging flags set, we want to allocate oversized 25587c478bd9Sstevel@tonic-gate * buffers from large pages, and so skip the firewalling. 25597c478bd9Sstevel@tonic-gate */ 25607c478bd9Sstevel@tonic-gate if (use_large_pages && 25617c478bd9Sstevel@tonic-gate ((kmem_flags & KMF_LITE) || !(kmem_flags & KMF_DEBUG))) { 25627c478bd9Sstevel@tonic-gate kmem_oversize_arena = vmem_xcreate("kmem_oversize", NULL, 0, 25637c478bd9Sstevel@tonic-gate PAGESIZE, segkmem_alloc_lp, segkmem_free_lp, heap_arena, 25647c478bd9Sstevel@tonic-gate 0, VM_SLEEP); 25657c478bd9Sstevel@tonic-gate } else { 25667c478bd9Sstevel@tonic-gate kmem_oversize_arena = vmem_create("kmem_oversize", 25677c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE, 25687c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, kmem_minfirewall < ULONG_MAX? 25697c478bd9Sstevel@tonic-gate kmem_firewall_va_arena : heap_arena, 0, VM_SLEEP); 25707c478bd9Sstevel@tonic-gate } 25717c478bd9Sstevel@tonic-gate 25727c478bd9Sstevel@tonic-gate kmem_cache_init(2, use_large_pages); 25737c478bd9Sstevel@tonic-gate 25747c478bd9Sstevel@tonic-gate if (kmem_flags & (KMF_AUDIT | KMF_RANDOMIZE)) { 25757c478bd9Sstevel@tonic-gate if (kmem_transaction_log_size == 0) 25767c478bd9Sstevel@tonic-gate kmem_transaction_log_size = kmem_maxavail() / 50; 25777c478bd9Sstevel@tonic-gate kmem_transaction_log = kmem_log_init(kmem_transaction_log_size); 25787c478bd9Sstevel@tonic-gate } 25797c478bd9Sstevel@tonic-gate 25807c478bd9Sstevel@tonic-gate if (kmem_flags & (KMF_CONTENTS | KMF_RANDOMIZE)) { 25817c478bd9Sstevel@tonic-gate if (kmem_content_log_size == 0) 25827c478bd9Sstevel@tonic-gate kmem_content_log_size = kmem_maxavail() / 50; 25837c478bd9Sstevel@tonic-gate kmem_content_log = kmem_log_init(kmem_content_log_size); 25847c478bd9Sstevel@tonic-gate } 25857c478bd9Sstevel@tonic-gate 25867c478bd9Sstevel@tonic-gate kmem_failure_log = kmem_log_init(kmem_failure_log_size); 25877c478bd9Sstevel@tonic-gate 25887c478bd9Sstevel@tonic-gate kmem_slab_log = kmem_log_init(kmem_slab_log_size); 25897c478bd9Sstevel@tonic-gate 25907c478bd9Sstevel@tonic-gate /* 25917c478bd9Sstevel@tonic-gate * Initialize STREAMS message caches so allocb() is available. 25927c478bd9Sstevel@tonic-gate * This allows us to initialize the logging framework (cmn_err(9F), 25937c478bd9Sstevel@tonic-gate * strlog(9F), etc) so we can start recording messages. 25947c478bd9Sstevel@tonic-gate */ 25957c478bd9Sstevel@tonic-gate streams_msg_init(); 2596*7d692464Sdp 25977c478bd9Sstevel@tonic-gate /* 25987c478bd9Sstevel@tonic-gate * Initialize the ZSD framework in Zones so modules loaded henceforth 25997c478bd9Sstevel@tonic-gate * can register their callbacks. 26007c478bd9Sstevel@tonic-gate */ 26017c478bd9Sstevel@tonic-gate zone_zsd_init(); 26027c478bd9Sstevel@tonic-gate log_init(); 26037c478bd9Sstevel@tonic-gate taskq_init(); 26047c478bd9Sstevel@tonic-gate 2605*7d692464Sdp /* 2606*7d692464Sdp * Warn about invalid or dangerous values of kmem_flags. 2607*7d692464Sdp * Always warn about unsupported values. 2608*7d692464Sdp */ 2609*7d692464Sdp if (((kmem_flags & ~(KMF_AUDIT | KMF_DEADBEEF | KMF_REDZONE | 2610*7d692464Sdp KMF_CONTENTS | KMF_LITE)) != 0) || 2611*7d692464Sdp ((kmem_flags & KMF_LITE) && kmem_flags != KMF_LITE)) 2612*7d692464Sdp cmn_err(CE_WARN, "kmem_flags set to unsupported value 0x%x. " 2613*7d692464Sdp "See the Solaris Tunable Parameters Reference Manual.", 2614*7d692464Sdp kmem_flags); 2615*7d692464Sdp 2616*7d692464Sdp #ifdef DEBUG 2617*7d692464Sdp if ((kmem_flags & KMF_DEBUG) == 0) 2618*7d692464Sdp cmn_err(CE_NOTE, "kmem debugging disabled."); 2619*7d692464Sdp #else 2620*7d692464Sdp /* 2621*7d692464Sdp * For non-debug kernels, the only "normal" flags are 0, KMF_LITE, 2622*7d692464Sdp * KMF_REDZONE, and KMF_CONTENTS (the last because it is only enabled 2623*7d692464Sdp * if KMF_AUDIT is set). We should warn the user about the performance 2624*7d692464Sdp * penalty of KMF_AUDIT or KMF_DEADBEEF if they are set and KMF_LITE 2625*7d692464Sdp * isn't set (since that disables AUDIT). 2626*7d692464Sdp */ 2627*7d692464Sdp if (!(kmem_flags & KMF_LITE) && 2628*7d692464Sdp (kmem_flags & (KMF_AUDIT | KMF_DEADBEEF)) != 0) 2629*7d692464Sdp cmn_err(CE_WARN, "High-overhead kmem debugging features " 2630*7d692464Sdp "enabled (kmem_flags = 0x%x). Performance degradation " 2631*7d692464Sdp "and large memory overhead possible. See the Solaris " 2632*7d692464Sdp "Tunable Parameters Reference Manual.", kmem_flags); 2633*7d692464Sdp #endif /* not DEBUG */ 2634*7d692464Sdp 26357c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_magazine_enable, NULL, TQ_SLEEP); 26367c478bd9Sstevel@tonic-gate 26377c478bd9Sstevel@tonic-gate kmem_ready = 1; 26387c478bd9Sstevel@tonic-gate 26397c478bd9Sstevel@tonic-gate /* 26407c478bd9Sstevel@tonic-gate * Initialize the platform-specific aligned/DMA memory allocator. 26417c478bd9Sstevel@tonic-gate */ 26427c478bd9Sstevel@tonic-gate ka_init(); 26437c478bd9Sstevel@tonic-gate 26447c478bd9Sstevel@tonic-gate /* 26457c478bd9Sstevel@tonic-gate * Initialize 32-bit ID cache. 26467c478bd9Sstevel@tonic-gate */ 26477c478bd9Sstevel@tonic-gate id32_init(); 26487c478bd9Sstevel@tonic-gate } 26497c478bd9Sstevel@tonic-gate 26507c478bd9Sstevel@tonic-gate void 26517c478bd9Sstevel@tonic-gate kmem_thread_init(void) 26527c478bd9Sstevel@tonic-gate { 26537c478bd9Sstevel@tonic-gate kmem_taskq = taskq_create_instance("kmem_taskq", 0, 1, minclsyspri, 26547c478bd9Sstevel@tonic-gate 300, INT_MAX, TASKQ_PREPOPULATE); 26557c478bd9Sstevel@tonic-gate } 26567c478bd9Sstevel@tonic-gate 26577c478bd9Sstevel@tonic-gate void 26587c478bd9Sstevel@tonic-gate kmem_mp_init(void) 26597c478bd9Sstevel@tonic-gate { 26607c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 26617c478bd9Sstevel@tonic-gate register_cpu_setup_func(kmem_cpu_setup, NULL); 26627c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 26637c478bd9Sstevel@tonic-gate 26647c478bd9Sstevel@tonic-gate kmem_update_timeout(NULL); 26657c478bd9Sstevel@tonic-gate } 2666