17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 57d692464Sdp * Common Development and Distribution License (the "License"). 67d692464Sdp * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22*f4b3ec61Sdh * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 277c478bd9Sstevel@tonic-gate 287c478bd9Sstevel@tonic-gate /* 297c478bd9Sstevel@tonic-gate * Kernel memory allocator, as described in the following two papers: 307c478bd9Sstevel@tonic-gate * 317c478bd9Sstevel@tonic-gate * Jeff Bonwick, 327c478bd9Sstevel@tonic-gate * The Slab Allocator: An Object-Caching Kernel Memory Allocator. 337c478bd9Sstevel@tonic-gate * Proceedings of the Summer 1994 Usenix Conference. 347c478bd9Sstevel@tonic-gate * Available as /shared/sac/PSARC/1994/028/materials/kmem.pdf. 357c478bd9Sstevel@tonic-gate * 367c478bd9Sstevel@tonic-gate * Jeff Bonwick and Jonathan Adams, 377c478bd9Sstevel@tonic-gate * Magazines and vmem: Extending the Slab Allocator to Many CPUs and 387c478bd9Sstevel@tonic-gate * Arbitrary Resources. 397c478bd9Sstevel@tonic-gate * Proceedings of the 2001 Usenix Conference. 407c478bd9Sstevel@tonic-gate * Available as /shared/sac/PSARC/2000/550/materials/vmem.pdf. 417c478bd9Sstevel@tonic-gate */ 427c478bd9Sstevel@tonic-gate 437c478bd9Sstevel@tonic-gate #include <sys/kmem_impl.h> 447c478bd9Sstevel@tonic-gate #include <sys/vmem_impl.h> 457c478bd9Sstevel@tonic-gate #include <sys/param.h> 467c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 477c478bd9Sstevel@tonic-gate #include <sys/vm.h> 487c478bd9Sstevel@tonic-gate #include <sys/proc.h> 497c478bd9Sstevel@tonic-gate #include <sys/tuneable.h> 507c478bd9Sstevel@tonic-gate #include <sys/systm.h> 517c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 527c478bd9Sstevel@tonic-gate #include <sys/debug.h> 537c478bd9Sstevel@tonic-gate #include <sys/mutex.h> 547c478bd9Sstevel@tonic-gate #include <sys/bitmap.h> 557c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 567c478bd9Sstevel@tonic-gate #include <sys/kobj.h> 577c478bd9Sstevel@tonic-gate #include <sys/disp.h> 587c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h> 597c478bd9Sstevel@tonic-gate #include <sys/log.h> 607c478bd9Sstevel@tonic-gate #include <sys/callb.h> 617c478bd9Sstevel@tonic-gate #include <sys/taskq.h> 627c478bd9Sstevel@tonic-gate #include <sys/modctl.h> 637c478bd9Sstevel@tonic-gate #include <sys/reboot.h> 647c478bd9Sstevel@tonic-gate #include <sys/id32.h> 657c478bd9Sstevel@tonic-gate #include <sys/zone.h> 66*f4b3ec61Sdh #include <sys/netstack.h> 677c478bd9Sstevel@tonic-gate 687c478bd9Sstevel@tonic-gate extern void streams_msg_init(void); 697c478bd9Sstevel@tonic-gate extern int segkp_fromheap; 707c478bd9Sstevel@tonic-gate extern void segkp_cache_free(void); 717c478bd9Sstevel@tonic-gate 727c478bd9Sstevel@tonic-gate struct kmem_cache_kstat { 737c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_size; 747c478bd9Sstevel@tonic-gate kstat_named_t kmc_align; 757c478bd9Sstevel@tonic-gate kstat_named_t kmc_chunk_size; 767c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_size; 777c478bd9Sstevel@tonic-gate kstat_named_t kmc_alloc; 787c478bd9Sstevel@tonic-gate kstat_named_t kmc_alloc_fail; 797c478bd9Sstevel@tonic-gate kstat_named_t kmc_free; 807c478bd9Sstevel@tonic-gate kstat_named_t kmc_depot_alloc; 817c478bd9Sstevel@tonic-gate kstat_named_t kmc_depot_free; 827c478bd9Sstevel@tonic-gate kstat_named_t kmc_depot_contention; 837c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_alloc; 847c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_free; 857c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_constructed; 867c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_avail; 877c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_inuse; 887c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_total; 897c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_max; 907c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_create; 917c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_destroy; 927c478bd9Sstevel@tonic-gate kstat_named_t kmc_vmem_source; 937c478bd9Sstevel@tonic-gate kstat_named_t kmc_hash_size; 947c478bd9Sstevel@tonic-gate kstat_named_t kmc_hash_lookup_depth; 957c478bd9Sstevel@tonic-gate kstat_named_t kmc_hash_rescale; 967c478bd9Sstevel@tonic-gate kstat_named_t kmc_full_magazines; 977c478bd9Sstevel@tonic-gate kstat_named_t kmc_empty_magazines; 987c478bd9Sstevel@tonic-gate kstat_named_t kmc_magazine_size; 997c478bd9Sstevel@tonic-gate } kmem_cache_kstat = { 1007c478bd9Sstevel@tonic-gate { "buf_size", KSTAT_DATA_UINT64 }, 1017c478bd9Sstevel@tonic-gate { "align", KSTAT_DATA_UINT64 }, 1027c478bd9Sstevel@tonic-gate { "chunk_size", KSTAT_DATA_UINT64 }, 1037c478bd9Sstevel@tonic-gate { "slab_size", KSTAT_DATA_UINT64 }, 1047c478bd9Sstevel@tonic-gate { "alloc", KSTAT_DATA_UINT64 }, 1057c478bd9Sstevel@tonic-gate { "alloc_fail", KSTAT_DATA_UINT64 }, 1067c478bd9Sstevel@tonic-gate { "free", KSTAT_DATA_UINT64 }, 1077c478bd9Sstevel@tonic-gate { "depot_alloc", KSTAT_DATA_UINT64 }, 1087c478bd9Sstevel@tonic-gate { "depot_free", KSTAT_DATA_UINT64 }, 1097c478bd9Sstevel@tonic-gate { "depot_contention", KSTAT_DATA_UINT64 }, 1107c478bd9Sstevel@tonic-gate { "slab_alloc", KSTAT_DATA_UINT64 }, 1117c478bd9Sstevel@tonic-gate { "slab_free", KSTAT_DATA_UINT64 }, 1127c478bd9Sstevel@tonic-gate { "buf_constructed", KSTAT_DATA_UINT64 }, 1137c478bd9Sstevel@tonic-gate { "buf_avail", KSTAT_DATA_UINT64 }, 1147c478bd9Sstevel@tonic-gate { "buf_inuse", KSTAT_DATA_UINT64 }, 1157c478bd9Sstevel@tonic-gate { "buf_total", KSTAT_DATA_UINT64 }, 1167c478bd9Sstevel@tonic-gate { "buf_max", KSTAT_DATA_UINT64 }, 1177c478bd9Sstevel@tonic-gate { "slab_create", KSTAT_DATA_UINT64 }, 1187c478bd9Sstevel@tonic-gate { "slab_destroy", KSTAT_DATA_UINT64 }, 1197c478bd9Sstevel@tonic-gate { "vmem_source", KSTAT_DATA_UINT64 }, 1207c478bd9Sstevel@tonic-gate { "hash_size", KSTAT_DATA_UINT64 }, 1217c478bd9Sstevel@tonic-gate { "hash_lookup_depth", KSTAT_DATA_UINT64 }, 1227c478bd9Sstevel@tonic-gate { "hash_rescale", KSTAT_DATA_UINT64 }, 1237c478bd9Sstevel@tonic-gate { "full_magazines", KSTAT_DATA_UINT64 }, 1247c478bd9Sstevel@tonic-gate { "empty_magazines", KSTAT_DATA_UINT64 }, 1257c478bd9Sstevel@tonic-gate { "magazine_size", KSTAT_DATA_UINT64 }, 1267c478bd9Sstevel@tonic-gate }; 1277c478bd9Sstevel@tonic-gate 1287c478bd9Sstevel@tonic-gate static kmutex_t kmem_cache_kstat_lock; 1297c478bd9Sstevel@tonic-gate 1307c478bd9Sstevel@tonic-gate /* 1317c478bd9Sstevel@tonic-gate * The default set of caches to back kmem_alloc(). 1327c478bd9Sstevel@tonic-gate * These sizes should be reevaluated periodically. 1337c478bd9Sstevel@tonic-gate * 1347c478bd9Sstevel@tonic-gate * We want allocations that are multiples of the coherency granularity 1357c478bd9Sstevel@tonic-gate * (64 bytes) to be satisfied from a cache which is a multiple of 64 1367c478bd9Sstevel@tonic-gate * bytes, so that it will be 64-byte aligned. For all multiples of 64, 1377c478bd9Sstevel@tonic-gate * the next kmem_cache_size greater than or equal to it must be a 1387c478bd9Sstevel@tonic-gate * multiple of 64. 1397c478bd9Sstevel@tonic-gate */ 1407c478bd9Sstevel@tonic-gate static const int kmem_alloc_sizes[] = { 1417c478bd9Sstevel@tonic-gate 1 * 8, 1427c478bd9Sstevel@tonic-gate 2 * 8, 1437c478bd9Sstevel@tonic-gate 3 * 8, 1447c478bd9Sstevel@tonic-gate 4 * 8, 5 * 8, 6 * 8, 7 * 8, 1457c478bd9Sstevel@tonic-gate 4 * 16, 5 * 16, 6 * 16, 7 * 16, 1467c478bd9Sstevel@tonic-gate 4 * 32, 5 * 32, 6 * 32, 7 * 32, 1477c478bd9Sstevel@tonic-gate 4 * 64, 5 * 64, 6 * 64, 7 * 64, 1487c478bd9Sstevel@tonic-gate 4 * 128, 5 * 128, 6 * 128, 7 * 128, 1497c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 7, 64), 1507c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 6, 64), 1517c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 5, 64), 1527c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 4, 64), 1537c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 3, 64), 1547c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 2, 64), 1557c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 1, 64), 1567c478bd9Sstevel@tonic-gate 4096 * 3, 1577c478bd9Sstevel@tonic-gate 8192 * 2, 158ad23a2dbSjohansen 8192 * 3, 159ad23a2dbSjohansen 8192 * 4, 1607c478bd9Sstevel@tonic-gate }; 1617c478bd9Sstevel@tonic-gate 162ad23a2dbSjohansen #define KMEM_MAXBUF 32768 1637c478bd9Sstevel@tonic-gate 1647c478bd9Sstevel@tonic-gate static kmem_cache_t *kmem_alloc_table[KMEM_MAXBUF >> KMEM_ALIGN_SHIFT]; 1657c478bd9Sstevel@tonic-gate 1667c478bd9Sstevel@tonic-gate static kmem_magtype_t kmem_magtype[] = { 1677c478bd9Sstevel@tonic-gate { 1, 8, 3200, 65536 }, 1687c478bd9Sstevel@tonic-gate { 3, 16, 256, 32768 }, 1697c478bd9Sstevel@tonic-gate { 7, 32, 64, 16384 }, 1707c478bd9Sstevel@tonic-gate { 15, 64, 0, 8192 }, 1717c478bd9Sstevel@tonic-gate { 31, 64, 0, 4096 }, 1727c478bd9Sstevel@tonic-gate { 47, 64, 0, 2048 }, 1737c478bd9Sstevel@tonic-gate { 63, 64, 0, 1024 }, 1747c478bd9Sstevel@tonic-gate { 95, 64, 0, 512 }, 1757c478bd9Sstevel@tonic-gate { 143, 64, 0, 0 }, 1767c478bd9Sstevel@tonic-gate }; 1777c478bd9Sstevel@tonic-gate 1787c478bd9Sstevel@tonic-gate static uint32_t kmem_reaping; 1797c478bd9Sstevel@tonic-gate static uint32_t kmem_reaping_idspace; 1807c478bd9Sstevel@tonic-gate 1817c478bd9Sstevel@tonic-gate /* 1827c478bd9Sstevel@tonic-gate * kmem tunables 1837c478bd9Sstevel@tonic-gate */ 1847c478bd9Sstevel@tonic-gate clock_t kmem_reap_interval; /* cache reaping rate [15 * HZ ticks] */ 1857c478bd9Sstevel@tonic-gate int kmem_depot_contention = 3; /* max failed tryenters per real interval */ 1867c478bd9Sstevel@tonic-gate pgcnt_t kmem_reapahead = 0; /* start reaping N pages before pageout */ 1877c478bd9Sstevel@tonic-gate int kmem_panic = 1; /* whether to panic on error */ 1887c478bd9Sstevel@tonic-gate int kmem_logging = 1; /* kmem_log_enter() override */ 1897c478bd9Sstevel@tonic-gate uint32_t kmem_mtbf = 0; /* mean time between failures [default: off] */ 1907c478bd9Sstevel@tonic-gate size_t kmem_transaction_log_size; /* transaction log size [2% of memory] */ 1917c478bd9Sstevel@tonic-gate size_t kmem_content_log_size; /* content log size [2% of memory] */ 1927c478bd9Sstevel@tonic-gate size_t kmem_failure_log_size; /* failure log [4 pages per CPU] */ 1937c478bd9Sstevel@tonic-gate size_t kmem_slab_log_size; /* slab create log [4 pages per CPU] */ 1947c478bd9Sstevel@tonic-gate size_t kmem_content_maxsave = 256; /* KMF_CONTENTS max bytes to log */ 1957c478bd9Sstevel@tonic-gate size_t kmem_lite_minsize = 0; /* minimum buffer size for KMF_LITE */ 1967c478bd9Sstevel@tonic-gate size_t kmem_lite_maxalign = 1024; /* maximum buffer alignment for KMF_LITE */ 1977c478bd9Sstevel@tonic-gate int kmem_lite_pcs = 4; /* number of PCs to store in KMF_LITE mode */ 1987c478bd9Sstevel@tonic-gate size_t kmem_maxverify; /* maximum bytes to inspect in debug routines */ 1997c478bd9Sstevel@tonic-gate size_t kmem_minfirewall; /* hardware-enforced redzone threshold */ 2007c478bd9Sstevel@tonic-gate 2017c478bd9Sstevel@tonic-gate #ifdef DEBUG 2027c478bd9Sstevel@tonic-gate int kmem_flags = KMF_AUDIT | KMF_DEADBEEF | KMF_REDZONE | KMF_CONTENTS; 2037c478bd9Sstevel@tonic-gate #else 2047c478bd9Sstevel@tonic-gate int kmem_flags = 0; 2057c478bd9Sstevel@tonic-gate #endif 2067c478bd9Sstevel@tonic-gate int kmem_ready; 2077c478bd9Sstevel@tonic-gate 2087c478bd9Sstevel@tonic-gate static kmem_cache_t *kmem_slab_cache; 2097c478bd9Sstevel@tonic-gate static kmem_cache_t *kmem_bufctl_cache; 2107c478bd9Sstevel@tonic-gate static kmem_cache_t *kmem_bufctl_audit_cache; 2117c478bd9Sstevel@tonic-gate 2127c478bd9Sstevel@tonic-gate static kmutex_t kmem_cache_lock; /* inter-cache linkage only */ 2137c478bd9Sstevel@tonic-gate kmem_cache_t kmem_null_cache; 2147c478bd9Sstevel@tonic-gate 2157c478bd9Sstevel@tonic-gate static taskq_t *kmem_taskq; 2167c478bd9Sstevel@tonic-gate static kmutex_t kmem_flags_lock; 2177c478bd9Sstevel@tonic-gate static vmem_t *kmem_metadata_arena; 2187c478bd9Sstevel@tonic-gate static vmem_t *kmem_msb_arena; /* arena for metadata caches */ 2197c478bd9Sstevel@tonic-gate static vmem_t *kmem_cache_arena; 2207c478bd9Sstevel@tonic-gate static vmem_t *kmem_hash_arena; 2217c478bd9Sstevel@tonic-gate static vmem_t *kmem_log_arena; 2227c478bd9Sstevel@tonic-gate static vmem_t *kmem_oversize_arena; 2237c478bd9Sstevel@tonic-gate static vmem_t *kmem_va_arena; 2247c478bd9Sstevel@tonic-gate static vmem_t *kmem_default_arena; 2257c478bd9Sstevel@tonic-gate static vmem_t *kmem_firewall_va_arena; 2267c478bd9Sstevel@tonic-gate static vmem_t *kmem_firewall_arena; 2277c478bd9Sstevel@tonic-gate 2287c478bd9Sstevel@tonic-gate kmem_log_header_t *kmem_transaction_log; 2297c478bd9Sstevel@tonic-gate kmem_log_header_t *kmem_content_log; 2307c478bd9Sstevel@tonic-gate kmem_log_header_t *kmem_failure_log; 2317c478bd9Sstevel@tonic-gate kmem_log_header_t *kmem_slab_log; 2327c478bd9Sstevel@tonic-gate 2337c478bd9Sstevel@tonic-gate static int kmem_lite_count; /* # of PCs in kmem_buftag_lite_t */ 2347c478bd9Sstevel@tonic-gate 2357c478bd9Sstevel@tonic-gate #define KMEM_BUFTAG_LITE_ENTER(bt, count, caller) \ 2367c478bd9Sstevel@tonic-gate if ((count) > 0) { \ 2377c478bd9Sstevel@tonic-gate pc_t *_s = ((kmem_buftag_lite_t *)(bt))->bt_history; \ 2387c478bd9Sstevel@tonic-gate pc_t *_e; \ 2397c478bd9Sstevel@tonic-gate /* memmove() the old entries down one notch */ \ 2407c478bd9Sstevel@tonic-gate for (_e = &_s[(count) - 1]; _e > _s; _e--) \ 2417c478bd9Sstevel@tonic-gate *_e = *(_e - 1); \ 2427c478bd9Sstevel@tonic-gate *_s = (uintptr_t)(caller); \ 2437c478bd9Sstevel@tonic-gate } 2447c478bd9Sstevel@tonic-gate 2457c478bd9Sstevel@tonic-gate #define KMERR_MODIFIED 0 /* buffer modified while on freelist */ 2467c478bd9Sstevel@tonic-gate #define KMERR_REDZONE 1 /* redzone violation (write past end of buf) */ 2477c478bd9Sstevel@tonic-gate #define KMERR_DUPFREE 2 /* freed a buffer twice */ 2487c478bd9Sstevel@tonic-gate #define KMERR_BADADDR 3 /* freed a bad (unallocated) address */ 2497c478bd9Sstevel@tonic-gate #define KMERR_BADBUFTAG 4 /* buftag corrupted */ 2507c478bd9Sstevel@tonic-gate #define KMERR_BADBUFCTL 5 /* bufctl corrupted */ 2517c478bd9Sstevel@tonic-gate #define KMERR_BADCACHE 6 /* freed a buffer to the wrong cache */ 2527c478bd9Sstevel@tonic-gate #define KMERR_BADSIZE 7 /* alloc size != free size */ 2537c478bd9Sstevel@tonic-gate #define KMERR_BADBASE 8 /* buffer base address wrong */ 2547c478bd9Sstevel@tonic-gate 2557c478bd9Sstevel@tonic-gate struct { 2567c478bd9Sstevel@tonic-gate hrtime_t kmp_timestamp; /* timestamp of panic */ 2577c478bd9Sstevel@tonic-gate int kmp_error; /* type of kmem error */ 2587c478bd9Sstevel@tonic-gate void *kmp_buffer; /* buffer that induced panic */ 2597c478bd9Sstevel@tonic-gate void *kmp_realbuf; /* real start address for buffer */ 2607c478bd9Sstevel@tonic-gate kmem_cache_t *kmp_cache; /* buffer's cache according to client */ 2617c478bd9Sstevel@tonic-gate kmem_cache_t *kmp_realcache; /* actual cache containing buffer */ 2627c478bd9Sstevel@tonic-gate kmem_slab_t *kmp_slab; /* slab accoring to kmem_findslab() */ 2637c478bd9Sstevel@tonic-gate kmem_bufctl_t *kmp_bufctl; /* bufctl */ 2647c478bd9Sstevel@tonic-gate } kmem_panic_info; 2657c478bd9Sstevel@tonic-gate 2667c478bd9Sstevel@tonic-gate 2677c478bd9Sstevel@tonic-gate static void 2687c478bd9Sstevel@tonic-gate copy_pattern(uint64_t pattern, void *buf_arg, size_t size) 2697c478bd9Sstevel@tonic-gate { 2707c478bd9Sstevel@tonic-gate uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 2717c478bd9Sstevel@tonic-gate uint64_t *buf = buf_arg; 2727c478bd9Sstevel@tonic-gate 2737c478bd9Sstevel@tonic-gate while (buf < bufend) 2747c478bd9Sstevel@tonic-gate *buf++ = pattern; 2757c478bd9Sstevel@tonic-gate } 2767c478bd9Sstevel@tonic-gate 2777c478bd9Sstevel@tonic-gate static void * 2787c478bd9Sstevel@tonic-gate verify_pattern(uint64_t pattern, void *buf_arg, size_t size) 2797c478bd9Sstevel@tonic-gate { 2807c478bd9Sstevel@tonic-gate uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 2817c478bd9Sstevel@tonic-gate uint64_t *buf; 2827c478bd9Sstevel@tonic-gate 2837c478bd9Sstevel@tonic-gate for (buf = buf_arg; buf < bufend; buf++) 2847c478bd9Sstevel@tonic-gate if (*buf != pattern) 2857c478bd9Sstevel@tonic-gate return (buf); 2867c478bd9Sstevel@tonic-gate return (NULL); 2877c478bd9Sstevel@tonic-gate } 2887c478bd9Sstevel@tonic-gate 2897c478bd9Sstevel@tonic-gate static void * 2907c478bd9Sstevel@tonic-gate verify_and_copy_pattern(uint64_t old, uint64_t new, void *buf_arg, size_t size) 2917c478bd9Sstevel@tonic-gate { 2927c478bd9Sstevel@tonic-gate uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 2937c478bd9Sstevel@tonic-gate uint64_t *buf; 2947c478bd9Sstevel@tonic-gate 2957c478bd9Sstevel@tonic-gate for (buf = buf_arg; buf < bufend; buf++) { 2967c478bd9Sstevel@tonic-gate if (*buf != old) { 2977c478bd9Sstevel@tonic-gate copy_pattern(old, buf_arg, 2987c478bd9Sstevel@tonic-gate (char *)buf - (char *)buf_arg); 2997c478bd9Sstevel@tonic-gate return (buf); 3007c478bd9Sstevel@tonic-gate } 3017c478bd9Sstevel@tonic-gate *buf = new; 3027c478bd9Sstevel@tonic-gate } 3037c478bd9Sstevel@tonic-gate 3047c478bd9Sstevel@tonic-gate return (NULL); 3057c478bd9Sstevel@tonic-gate } 3067c478bd9Sstevel@tonic-gate 3077c478bd9Sstevel@tonic-gate static void 3087c478bd9Sstevel@tonic-gate kmem_cache_applyall(void (*func)(kmem_cache_t *), taskq_t *tq, int tqflag) 3097c478bd9Sstevel@tonic-gate { 3107c478bd9Sstevel@tonic-gate kmem_cache_t *cp; 3117c478bd9Sstevel@tonic-gate 3127c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_lock); 3137c478bd9Sstevel@tonic-gate for (cp = kmem_null_cache.cache_next; cp != &kmem_null_cache; 3147c478bd9Sstevel@tonic-gate cp = cp->cache_next) 3157c478bd9Sstevel@tonic-gate if (tq != NULL) 3167c478bd9Sstevel@tonic-gate (void) taskq_dispatch(tq, (task_func_t *)func, cp, 3177c478bd9Sstevel@tonic-gate tqflag); 3187c478bd9Sstevel@tonic-gate else 3197c478bd9Sstevel@tonic-gate func(cp); 3207c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_lock); 3217c478bd9Sstevel@tonic-gate } 3227c478bd9Sstevel@tonic-gate 3237c478bd9Sstevel@tonic-gate static void 3247c478bd9Sstevel@tonic-gate kmem_cache_applyall_id(void (*func)(kmem_cache_t *), taskq_t *tq, int tqflag) 3257c478bd9Sstevel@tonic-gate { 3267c478bd9Sstevel@tonic-gate kmem_cache_t *cp; 3277c478bd9Sstevel@tonic-gate 3287c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_lock); 3297c478bd9Sstevel@tonic-gate for (cp = kmem_null_cache.cache_next; cp != &kmem_null_cache; 3307c478bd9Sstevel@tonic-gate cp = cp->cache_next) { 3317c478bd9Sstevel@tonic-gate if (!(cp->cache_cflags & KMC_IDENTIFIER)) 3327c478bd9Sstevel@tonic-gate continue; 3337c478bd9Sstevel@tonic-gate if (tq != NULL) 3347c478bd9Sstevel@tonic-gate (void) taskq_dispatch(tq, (task_func_t *)func, cp, 3357c478bd9Sstevel@tonic-gate tqflag); 3367c478bd9Sstevel@tonic-gate else 3377c478bd9Sstevel@tonic-gate func(cp); 3387c478bd9Sstevel@tonic-gate } 3397c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_lock); 3407c478bd9Sstevel@tonic-gate } 3417c478bd9Sstevel@tonic-gate 3427c478bd9Sstevel@tonic-gate /* 3437c478bd9Sstevel@tonic-gate * Debugging support. Given a buffer address, find its slab. 3447c478bd9Sstevel@tonic-gate */ 3457c478bd9Sstevel@tonic-gate static kmem_slab_t * 3467c478bd9Sstevel@tonic-gate kmem_findslab(kmem_cache_t *cp, void *buf) 3477c478bd9Sstevel@tonic-gate { 3487c478bd9Sstevel@tonic-gate kmem_slab_t *sp; 3497c478bd9Sstevel@tonic-gate 3507c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 3517c478bd9Sstevel@tonic-gate for (sp = cp->cache_nullslab.slab_next; 3527c478bd9Sstevel@tonic-gate sp != &cp->cache_nullslab; sp = sp->slab_next) { 3537c478bd9Sstevel@tonic-gate if (KMEM_SLAB_MEMBER(sp, buf)) { 3547c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 3557c478bd9Sstevel@tonic-gate return (sp); 3567c478bd9Sstevel@tonic-gate } 3577c478bd9Sstevel@tonic-gate } 3587c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 3597c478bd9Sstevel@tonic-gate 3607c478bd9Sstevel@tonic-gate return (NULL); 3617c478bd9Sstevel@tonic-gate } 3627c478bd9Sstevel@tonic-gate 3637c478bd9Sstevel@tonic-gate static void 3647c478bd9Sstevel@tonic-gate kmem_error(int error, kmem_cache_t *cparg, void *bufarg) 3657c478bd9Sstevel@tonic-gate { 3667c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = NULL; 3677c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp = NULL; 3687c478bd9Sstevel@tonic-gate kmem_cache_t *cp = cparg; 3697c478bd9Sstevel@tonic-gate kmem_slab_t *sp; 3707c478bd9Sstevel@tonic-gate uint64_t *off; 3717c478bd9Sstevel@tonic-gate void *buf = bufarg; 3727c478bd9Sstevel@tonic-gate 3737c478bd9Sstevel@tonic-gate kmem_logging = 0; /* stop logging when a bad thing happens */ 3747c478bd9Sstevel@tonic-gate 3757c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_timestamp = gethrtime(); 3767c478bd9Sstevel@tonic-gate 3777c478bd9Sstevel@tonic-gate sp = kmem_findslab(cp, buf); 3787c478bd9Sstevel@tonic-gate if (sp == NULL) { 3797c478bd9Sstevel@tonic-gate for (cp = kmem_null_cache.cache_prev; cp != &kmem_null_cache; 3807c478bd9Sstevel@tonic-gate cp = cp->cache_prev) { 3817c478bd9Sstevel@tonic-gate if ((sp = kmem_findslab(cp, buf)) != NULL) 3827c478bd9Sstevel@tonic-gate break; 3837c478bd9Sstevel@tonic-gate } 3847c478bd9Sstevel@tonic-gate } 3857c478bd9Sstevel@tonic-gate 3867c478bd9Sstevel@tonic-gate if (sp == NULL) { 3877c478bd9Sstevel@tonic-gate cp = NULL; 3887c478bd9Sstevel@tonic-gate error = KMERR_BADADDR; 3897c478bd9Sstevel@tonic-gate } else { 3907c478bd9Sstevel@tonic-gate if (cp != cparg) 3917c478bd9Sstevel@tonic-gate error = KMERR_BADCACHE; 3927c478bd9Sstevel@tonic-gate else 3937c478bd9Sstevel@tonic-gate buf = (char *)bufarg - ((uintptr_t)bufarg - 3947c478bd9Sstevel@tonic-gate (uintptr_t)sp->slab_base) % cp->cache_chunksize; 3957c478bd9Sstevel@tonic-gate if (buf != bufarg) 3967c478bd9Sstevel@tonic-gate error = KMERR_BADBASE; 3977c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_BUFTAG) 3987c478bd9Sstevel@tonic-gate btp = KMEM_BUFTAG(cp, buf); 3997c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) { 4007c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 4017c478bd9Sstevel@tonic-gate for (bcp = *KMEM_HASH(cp, buf); bcp; bcp = bcp->bc_next) 4027c478bd9Sstevel@tonic-gate if (bcp->bc_addr == buf) 4037c478bd9Sstevel@tonic-gate break; 4047c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 4057c478bd9Sstevel@tonic-gate if (bcp == NULL && btp != NULL) 4067c478bd9Sstevel@tonic-gate bcp = btp->bt_bufctl; 4077c478bd9Sstevel@tonic-gate if (kmem_findslab(cp->cache_bufctl_cache, bcp) == 4087c478bd9Sstevel@tonic-gate NULL || P2PHASE((uintptr_t)bcp, KMEM_ALIGN) || 4097c478bd9Sstevel@tonic-gate bcp->bc_addr != buf) { 4107c478bd9Sstevel@tonic-gate error = KMERR_BADBUFCTL; 4117c478bd9Sstevel@tonic-gate bcp = NULL; 4127c478bd9Sstevel@tonic-gate } 4137c478bd9Sstevel@tonic-gate } 4147c478bd9Sstevel@tonic-gate } 4157c478bd9Sstevel@tonic-gate 4167c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_error = error; 4177c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_buffer = bufarg; 4187c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_realbuf = buf; 4197c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_cache = cparg; 4207c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_realcache = cp; 4217c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_slab = sp; 4227c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_bufctl = bcp; 4237c478bd9Sstevel@tonic-gate 4247c478bd9Sstevel@tonic-gate printf("kernel memory allocator: "); 4257c478bd9Sstevel@tonic-gate 4267c478bd9Sstevel@tonic-gate switch (error) { 4277c478bd9Sstevel@tonic-gate 4287c478bd9Sstevel@tonic-gate case KMERR_MODIFIED: 4297c478bd9Sstevel@tonic-gate printf("buffer modified after being freed\n"); 4307c478bd9Sstevel@tonic-gate off = verify_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify); 4317c478bd9Sstevel@tonic-gate if (off == NULL) /* shouldn't happen */ 4327c478bd9Sstevel@tonic-gate off = buf; 4337c478bd9Sstevel@tonic-gate printf("modification occurred at offset 0x%lx " 4347c478bd9Sstevel@tonic-gate "(0x%llx replaced by 0x%llx)\n", 4357c478bd9Sstevel@tonic-gate (uintptr_t)off - (uintptr_t)buf, 4367c478bd9Sstevel@tonic-gate (longlong_t)KMEM_FREE_PATTERN, (longlong_t)*off); 4377c478bd9Sstevel@tonic-gate break; 4387c478bd9Sstevel@tonic-gate 4397c478bd9Sstevel@tonic-gate case KMERR_REDZONE: 4407c478bd9Sstevel@tonic-gate printf("redzone violation: write past end of buffer\n"); 4417c478bd9Sstevel@tonic-gate break; 4427c478bd9Sstevel@tonic-gate 4437c478bd9Sstevel@tonic-gate case KMERR_BADADDR: 4447c478bd9Sstevel@tonic-gate printf("invalid free: buffer not in cache\n"); 4457c478bd9Sstevel@tonic-gate break; 4467c478bd9Sstevel@tonic-gate 4477c478bd9Sstevel@tonic-gate case KMERR_DUPFREE: 4487c478bd9Sstevel@tonic-gate printf("duplicate free: buffer freed twice\n"); 4497c478bd9Sstevel@tonic-gate break; 4507c478bd9Sstevel@tonic-gate 4517c478bd9Sstevel@tonic-gate case KMERR_BADBUFTAG: 4527c478bd9Sstevel@tonic-gate printf("boundary tag corrupted\n"); 4537c478bd9Sstevel@tonic-gate printf("bcp ^ bxstat = %lx, should be %lx\n", 4547c478bd9Sstevel@tonic-gate (intptr_t)btp->bt_bufctl ^ btp->bt_bxstat, 4557c478bd9Sstevel@tonic-gate KMEM_BUFTAG_FREE); 4567c478bd9Sstevel@tonic-gate break; 4577c478bd9Sstevel@tonic-gate 4587c478bd9Sstevel@tonic-gate case KMERR_BADBUFCTL: 4597c478bd9Sstevel@tonic-gate printf("bufctl corrupted\n"); 4607c478bd9Sstevel@tonic-gate break; 4617c478bd9Sstevel@tonic-gate 4627c478bd9Sstevel@tonic-gate case KMERR_BADCACHE: 4637c478bd9Sstevel@tonic-gate printf("buffer freed to wrong cache\n"); 4647c478bd9Sstevel@tonic-gate printf("buffer was allocated from %s,\n", cp->cache_name); 4657c478bd9Sstevel@tonic-gate printf("caller attempting free to %s.\n", cparg->cache_name); 4667c478bd9Sstevel@tonic-gate break; 4677c478bd9Sstevel@tonic-gate 4687c478bd9Sstevel@tonic-gate case KMERR_BADSIZE: 4697c478bd9Sstevel@tonic-gate printf("bad free: free size (%u) != alloc size (%u)\n", 4707c478bd9Sstevel@tonic-gate KMEM_SIZE_DECODE(((uint32_t *)btp)[0]), 4717c478bd9Sstevel@tonic-gate KMEM_SIZE_DECODE(((uint32_t *)btp)[1])); 4727c478bd9Sstevel@tonic-gate break; 4737c478bd9Sstevel@tonic-gate 4747c478bd9Sstevel@tonic-gate case KMERR_BADBASE: 4757c478bd9Sstevel@tonic-gate printf("bad free: free address (%p) != alloc address (%p)\n", 4767c478bd9Sstevel@tonic-gate bufarg, buf); 4777c478bd9Sstevel@tonic-gate break; 4787c478bd9Sstevel@tonic-gate } 4797c478bd9Sstevel@tonic-gate 4807c478bd9Sstevel@tonic-gate printf("buffer=%p bufctl=%p cache: %s\n", 4817c478bd9Sstevel@tonic-gate bufarg, (void *)bcp, cparg->cache_name); 4827c478bd9Sstevel@tonic-gate 4837c478bd9Sstevel@tonic-gate if (bcp != NULL && (cp->cache_flags & KMF_AUDIT) && 4847c478bd9Sstevel@tonic-gate error != KMERR_BADBUFCTL) { 4857c478bd9Sstevel@tonic-gate int d; 4867c478bd9Sstevel@tonic-gate timestruc_t ts; 4877c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *bcap = (kmem_bufctl_audit_t *)bcp; 4887c478bd9Sstevel@tonic-gate 4897c478bd9Sstevel@tonic-gate hrt2ts(kmem_panic_info.kmp_timestamp - bcap->bc_timestamp, &ts); 4907c478bd9Sstevel@tonic-gate printf("previous transaction on buffer %p:\n", buf); 4917c478bd9Sstevel@tonic-gate printf("thread=%p time=T-%ld.%09ld slab=%p cache: %s\n", 4927c478bd9Sstevel@tonic-gate (void *)bcap->bc_thread, ts.tv_sec, ts.tv_nsec, 4937c478bd9Sstevel@tonic-gate (void *)sp, cp->cache_name); 4947c478bd9Sstevel@tonic-gate for (d = 0; d < MIN(bcap->bc_depth, KMEM_STACK_DEPTH); d++) { 4957c478bd9Sstevel@tonic-gate ulong_t off; 4967c478bd9Sstevel@tonic-gate char *sym = kobj_getsymname(bcap->bc_stack[d], &off); 4977c478bd9Sstevel@tonic-gate printf("%s+%lx\n", sym ? sym : "?", off); 4987c478bd9Sstevel@tonic-gate } 4997c478bd9Sstevel@tonic-gate } 5007c478bd9Sstevel@tonic-gate if (kmem_panic > 0) 5017c478bd9Sstevel@tonic-gate panic("kernel heap corruption detected"); 5027c478bd9Sstevel@tonic-gate if (kmem_panic == 0) 5037c478bd9Sstevel@tonic-gate debug_enter(NULL); 5047c478bd9Sstevel@tonic-gate kmem_logging = 1; /* resume logging */ 5057c478bd9Sstevel@tonic-gate } 5067c478bd9Sstevel@tonic-gate 5077c478bd9Sstevel@tonic-gate static kmem_log_header_t * 5087c478bd9Sstevel@tonic-gate kmem_log_init(size_t logsize) 5097c478bd9Sstevel@tonic-gate { 5107c478bd9Sstevel@tonic-gate kmem_log_header_t *lhp; 5117c478bd9Sstevel@tonic-gate int nchunks = 4 * max_ncpus; 5127c478bd9Sstevel@tonic-gate size_t lhsize = (size_t)&((kmem_log_header_t *)0)->lh_cpu[max_ncpus]; 5137c478bd9Sstevel@tonic-gate int i; 5147c478bd9Sstevel@tonic-gate 5157c478bd9Sstevel@tonic-gate /* 5167c478bd9Sstevel@tonic-gate * Make sure that lhp->lh_cpu[] is nicely aligned 5177c478bd9Sstevel@tonic-gate * to prevent false sharing of cache lines. 5187c478bd9Sstevel@tonic-gate */ 5197c478bd9Sstevel@tonic-gate lhsize = P2ROUNDUP(lhsize, KMEM_ALIGN); 5207c478bd9Sstevel@tonic-gate lhp = vmem_xalloc(kmem_log_arena, lhsize, 64, P2NPHASE(lhsize, 64), 0, 5217c478bd9Sstevel@tonic-gate NULL, NULL, VM_SLEEP); 5227c478bd9Sstevel@tonic-gate bzero(lhp, lhsize); 5237c478bd9Sstevel@tonic-gate 5247c478bd9Sstevel@tonic-gate mutex_init(&lhp->lh_lock, NULL, MUTEX_DEFAULT, NULL); 5257c478bd9Sstevel@tonic-gate lhp->lh_nchunks = nchunks; 5267c478bd9Sstevel@tonic-gate lhp->lh_chunksize = P2ROUNDUP(logsize / nchunks + 1, PAGESIZE); 5277c478bd9Sstevel@tonic-gate lhp->lh_base = vmem_alloc(kmem_log_arena, 5287c478bd9Sstevel@tonic-gate lhp->lh_chunksize * nchunks, VM_SLEEP); 5297c478bd9Sstevel@tonic-gate lhp->lh_free = vmem_alloc(kmem_log_arena, 5307c478bd9Sstevel@tonic-gate nchunks * sizeof (int), VM_SLEEP); 5317c478bd9Sstevel@tonic-gate bzero(lhp->lh_base, lhp->lh_chunksize * nchunks); 5327c478bd9Sstevel@tonic-gate 5337c478bd9Sstevel@tonic-gate for (i = 0; i < max_ncpus; i++) { 5347c478bd9Sstevel@tonic-gate kmem_cpu_log_header_t *clhp = &lhp->lh_cpu[i]; 5357c478bd9Sstevel@tonic-gate mutex_init(&clhp->clh_lock, NULL, MUTEX_DEFAULT, NULL); 5367c478bd9Sstevel@tonic-gate clhp->clh_chunk = i; 5377c478bd9Sstevel@tonic-gate } 5387c478bd9Sstevel@tonic-gate 5397c478bd9Sstevel@tonic-gate for (i = max_ncpus; i < nchunks; i++) 5407c478bd9Sstevel@tonic-gate lhp->lh_free[i] = i; 5417c478bd9Sstevel@tonic-gate 5427c478bd9Sstevel@tonic-gate lhp->lh_head = max_ncpus; 5437c478bd9Sstevel@tonic-gate lhp->lh_tail = 0; 5447c478bd9Sstevel@tonic-gate 5457c478bd9Sstevel@tonic-gate return (lhp); 5467c478bd9Sstevel@tonic-gate } 5477c478bd9Sstevel@tonic-gate 5487c478bd9Sstevel@tonic-gate static void * 5497c478bd9Sstevel@tonic-gate kmem_log_enter(kmem_log_header_t *lhp, void *data, size_t size) 5507c478bd9Sstevel@tonic-gate { 5517c478bd9Sstevel@tonic-gate void *logspace; 5527c478bd9Sstevel@tonic-gate kmem_cpu_log_header_t *clhp = &lhp->lh_cpu[CPU->cpu_seqid]; 5537c478bd9Sstevel@tonic-gate 5547c478bd9Sstevel@tonic-gate if (lhp == NULL || kmem_logging == 0 || panicstr) 5557c478bd9Sstevel@tonic-gate return (NULL); 5567c478bd9Sstevel@tonic-gate 5577c478bd9Sstevel@tonic-gate mutex_enter(&clhp->clh_lock); 5587c478bd9Sstevel@tonic-gate clhp->clh_hits++; 5597c478bd9Sstevel@tonic-gate if (size > clhp->clh_avail) { 5607c478bd9Sstevel@tonic-gate mutex_enter(&lhp->lh_lock); 5617c478bd9Sstevel@tonic-gate lhp->lh_hits++; 5627c478bd9Sstevel@tonic-gate lhp->lh_free[lhp->lh_tail] = clhp->clh_chunk; 5637c478bd9Sstevel@tonic-gate lhp->lh_tail = (lhp->lh_tail + 1) % lhp->lh_nchunks; 5647c478bd9Sstevel@tonic-gate clhp->clh_chunk = lhp->lh_free[lhp->lh_head]; 5657c478bd9Sstevel@tonic-gate lhp->lh_head = (lhp->lh_head + 1) % lhp->lh_nchunks; 5667c478bd9Sstevel@tonic-gate clhp->clh_current = lhp->lh_base + 5677c478bd9Sstevel@tonic-gate clhp->clh_chunk * lhp->lh_chunksize; 5687c478bd9Sstevel@tonic-gate clhp->clh_avail = lhp->lh_chunksize; 5697c478bd9Sstevel@tonic-gate if (size > lhp->lh_chunksize) 5707c478bd9Sstevel@tonic-gate size = lhp->lh_chunksize; 5717c478bd9Sstevel@tonic-gate mutex_exit(&lhp->lh_lock); 5727c478bd9Sstevel@tonic-gate } 5737c478bd9Sstevel@tonic-gate logspace = clhp->clh_current; 5747c478bd9Sstevel@tonic-gate clhp->clh_current += size; 5757c478bd9Sstevel@tonic-gate clhp->clh_avail -= size; 5767c478bd9Sstevel@tonic-gate bcopy(data, logspace, size); 5777c478bd9Sstevel@tonic-gate mutex_exit(&clhp->clh_lock); 5787c478bd9Sstevel@tonic-gate return (logspace); 5797c478bd9Sstevel@tonic-gate } 5807c478bd9Sstevel@tonic-gate 5817c478bd9Sstevel@tonic-gate #define KMEM_AUDIT(lp, cp, bcp) \ 5827c478bd9Sstevel@tonic-gate { \ 5837c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *_bcp = (kmem_bufctl_audit_t *)(bcp); \ 5847c478bd9Sstevel@tonic-gate _bcp->bc_timestamp = gethrtime(); \ 5857c478bd9Sstevel@tonic-gate _bcp->bc_thread = curthread; \ 5867c478bd9Sstevel@tonic-gate _bcp->bc_depth = getpcstack(_bcp->bc_stack, KMEM_STACK_DEPTH); \ 5877c478bd9Sstevel@tonic-gate _bcp->bc_lastlog = kmem_log_enter((lp), _bcp, sizeof (*_bcp)); \ 5887c478bd9Sstevel@tonic-gate } 5897c478bd9Sstevel@tonic-gate 5907c478bd9Sstevel@tonic-gate static void 5917c478bd9Sstevel@tonic-gate kmem_log_event(kmem_log_header_t *lp, kmem_cache_t *cp, 5927c478bd9Sstevel@tonic-gate kmem_slab_t *sp, void *addr) 5937c478bd9Sstevel@tonic-gate { 5947c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t bca; 5957c478bd9Sstevel@tonic-gate 5967c478bd9Sstevel@tonic-gate bzero(&bca, sizeof (kmem_bufctl_audit_t)); 5977c478bd9Sstevel@tonic-gate bca.bc_addr = addr; 5987c478bd9Sstevel@tonic-gate bca.bc_slab = sp; 5997c478bd9Sstevel@tonic-gate bca.bc_cache = cp; 6007c478bd9Sstevel@tonic-gate KMEM_AUDIT(lp, cp, &bca); 6017c478bd9Sstevel@tonic-gate } 6027c478bd9Sstevel@tonic-gate 6037c478bd9Sstevel@tonic-gate /* 6047c478bd9Sstevel@tonic-gate * Create a new slab for cache cp. 6057c478bd9Sstevel@tonic-gate */ 6067c478bd9Sstevel@tonic-gate static kmem_slab_t * 6077c478bd9Sstevel@tonic-gate kmem_slab_create(kmem_cache_t *cp, int kmflag) 6087c478bd9Sstevel@tonic-gate { 6097c478bd9Sstevel@tonic-gate size_t slabsize = cp->cache_slabsize; 6107c478bd9Sstevel@tonic-gate size_t chunksize = cp->cache_chunksize; 6117c478bd9Sstevel@tonic-gate int cache_flags = cp->cache_flags; 6127c478bd9Sstevel@tonic-gate size_t color, chunks; 6137c478bd9Sstevel@tonic-gate char *buf, *slab; 6147c478bd9Sstevel@tonic-gate kmem_slab_t *sp; 6157c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp; 6167c478bd9Sstevel@tonic-gate vmem_t *vmp = cp->cache_arena; 6177c478bd9Sstevel@tonic-gate 6187c478bd9Sstevel@tonic-gate color = cp->cache_color + cp->cache_align; 6197c478bd9Sstevel@tonic-gate if (color > cp->cache_maxcolor) 6207c478bd9Sstevel@tonic-gate color = cp->cache_mincolor; 6217c478bd9Sstevel@tonic-gate cp->cache_color = color; 6227c478bd9Sstevel@tonic-gate 6237c478bd9Sstevel@tonic-gate slab = vmem_alloc(vmp, slabsize, kmflag & KM_VMFLAGS); 6247c478bd9Sstevel@tonic-gate 6257c478bd9Sstevel@tonic-gate if (slab == NULL) 6267c478bd9Sstevel@tonic-gate goto vmem_alloc_failure; 6277c478bd9Sstevel@tonic-gate 6287c478bd9Sstevel@tonic-gate ASSERT(P2PHASE((uintptr_t)slab, vmp->vm_quantum) == 0); 6297c478bd9Sstevel@tonic-gate 6307c478bd9Sstevel@tonic-gate if (!(cp->cache_cflags & KMC_NOTOUCH)) 6317c478bd9Sstevel@tonic-gate copy_pattern(KMEM_UNINITIALIZED_PATTERN, slab, slabsize); 6327c478bd9Sstevel@tonic-gate 6337c478bd9Sstevel@tonic-gate if (cache_flags & KMF_HASH) { 6347c478bd9Sstevel@tonic-gate if ((sp = kmem_cache_alloc(kmem_slab_cache, kmflag)) == NULL) 6357c478bd9Sstevel@tonic-gate goto slab_alloc_failure; 6367c478bd9Sstevel@tonic-gate chunks = (slabsize - color) / chunksize; 6377c478bd9Sstevel@tonic-gate } else { 6387c478bd9Sstevel@tonic-gate sp = KMEM_SLAB(cp, slab); 6397c478bd9Sstevel@tonic-gate chunks = (slabsize - sizeof (kmem_slab_t) - color) / chunksize; 6407c478bd9Sstevel@tonic-gate } 6417c478bd9Sstevel@tonic-gate 6427c478bd9Sstevel@tonic-gate sp->slab_cache = cp; 6437c478bd9Sstevel@tonic-gate sp->slab_head = NULL; 6447c478bd9Sstevel@tonic-gate sp->slab_refcnt = 0; 6457c478bd9Sstevel@tonic-gate sp->slab_base = buf = slab + color; 6467c478bd9Sstevel@tonic-gate sp->slab_chunks = chunks; 6477c478bd9Sstevel@tonic-gate 6487c478bd9Sstevel@tonic-gate ASSERT(chunks > 0); 6497c478bd9Sstevel@tonic-gate while (chunks-- != 0) { 6507c478bd9Sstevel@tonic-gate if (cache_flags & KMF_HASH) { 6517c478bd9Sstevel@tonic-gate bcp = kmem_cache_alloc(cp->cache_bufctl_cache, kmflag); 6527c478bd9Sstevel@tonic-gate if (bcp == NULL) 6537c478bd9Sstevel@tonic-gate goto bufctl_alloc_failure; 6547c478bd9Sstevel@tonic-gate if (cache_flags & KMF_AUDIT) { 6557c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *bcap = 6567c478bd9Sstevel@tonic-gate (kmem_bufctl_audit_t *)bcp; 6577c478bd9Sstevel@tonic-gate bzero(bcap, sizeof (kmem_bufctl_audit_t)); 6587c478bd9Sstevel@tonic-gate bcap->bc_cache = cp; 6597c478bd9Sstevel@tonic-gate } 6607c478bd9Sstevel@tonic-gate bcp->bc_addr = buf; 6617c478bd9Sstevel@tonic-gate bcp->bc_slab = sp; 6627c478bd9Sstevel@tonic-gate } else { 6637c478bd9Sstevel@tonic-gate bcp = KMEM_BUFCTL(cp, buf); 6647c478bd9Sstevel@tonic-gate } 6657c478bd9Sstevel@tonic-gate if (cache_flags & KMF_BUFTAG) { 6667c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 6677c478bd9Sstevel@tonic-gate btp->bt_redzone = KMEM_REDZONE_PATTERN; 6687c478bd9Sstevel@tonic-gate btp->bt_bufctl = bcp; 6697c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE; 6707c478bd9Sstevel@tonic-gate if (cache_flags & KMF_DEADBEEF) { 6717c478bd9Sstevel@tonic-gate copy_pattern(KMEM_FREE_PATTERN, buf, 6727c478bd9Sstevel@tonic-gate cp->cache_verify); 6737c478bd9Sstevel@tonic-gate } 6747c478bd9Sstevel@tonic-gate } 6757c478bd9Sstevel@tonic-gate bcp->bc_next = sp->slab_head; 6767c478bd9Sstevel@tonic-gate sp->slab_head = bcp; 6777c478bd9Sstevel@tonic-gate buf += chunksize; 6787c478bd9Sstevel@tonic-gate } 6797c478bd9Sstevel@tonic-gate 6807c478bd9Sstevel@tonic-gate kmem_log_event(kmem_slab_log, cp, sp, slab); 6817c478bd9Sstevel@tonic-gate 6827c478bd9Sstevel@tonic-gate return (sp); 6837c478bd9Sstevel@tonic-gate 6847c478bd9Sstevel@tonic-gate bufctl_alloc_failure: 6857c478bd9Sstevel@tonic-gate 6867c478bd9Sstevel@tonic-gate while ((bcp = sp->slab_head) != NULL) { 6877c478bd9Sstevel@tonic-gate sp->slab_head = bcp->bc_next; 6887c478bd9Sstevel@tonic-gate kmem_cache_free(cp->cache_bufctl_cache, bcp); 6897c478bd9Sstevel@tonic-gate } 6907c478bd9Sstevel@tonic-gate kmem_cache_free(kmem_slab_cache, sp); 6917c478bd9Sstevel@tonic-gate 6927c478bd9Sstevel@tonic-gate slab_alloc_failure: 6937c478bd9Sstevel@tonic-gate 6947c478bd9Sstevel@tonic-gate vmem_free(vmp, slab, slabsize); 6957c478bd9Sstevel@tonic-gate 6967c478bd9Sstevel@tonic-gate vmem_alloc_failure: 6977c478bd9Sstevel@tonic-gate 6987c478bd9Sstevel@tonic-gate kmem_log_event(kmem_failure_log, cp, NULL, NULL); 6997c478bd9Sstevel@tonic-gate atomic_add_64(&cp->cache_alloc_fail, 1); 7007c478bd9Sstevel@tonic-gate 7017c478bd9Sstevel@tonic-gate return (NULL); 7027c478bd9Sstevel@tonic-gate } 7037c478bd9Sstevel@tonic-gate 7047c478bd9Sstevel@tonic-gate /* 7057c478bd9Sstevel@tonic-gate * Destroy a slab. 7067c478bd9Sstevel@tonic-gate */ 7077c478bd9Sstevel@tonic-gate static void 7087c478bd9Sstevel@tonic-gate kmem_slab_destroy(kmem_cache_t *cp, kmem_slab_t *sp) 7097c478bd9Sstevel@tonic-gate { 7107c478bd9Sstevel@tonic-gate vmem_t *vmp = cp->cache_arena; 7117c478bd9Sstevel@tonic-gate void *slab = (void *)P2ALIGN((uintptr_t)sp->slab_base, vmp->vm_quantum); 7127c478bd9Sstevel@tonic-gate 7137c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) { 7147c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp; 7157c478bd9Sstevel@tonic-gate while ((bcp = sp->slab_head) != NULL) { 7167c478bd9Sstevel@tonic-gate sp->slab_head = bcp->bc_next; 7177c478bd9Sstevel@tonic-gate kmem_cache_free(cp->cache_bufctl_cache, bcp); 7187c478bd9Sstevel@tonic-gate } 7197c478bd9Sstevel@tonic-gate kmem_cache_free(kmem_slab_cache, sp); 7207c478bd9Sstevel@tonic-gate } 7217c478bd9Sstevel@tonic-gate vmem_free(vmp, slab, cp->cache_slabsize); 7227c478bd9Sstevel@tonic-gate } 7237c478bd9Sstevel@tonic-gate 7247c478bd9Sstevel@tonic-gate /* 7257c478bd9Sstevel@tonic-gate * Allocate a raw (unconstructed) buffer from cp's slab layer. 7267c478bd9Sstevel@tonic-gate */ 7277c478bd9Sstevel@tonic-gate static void * 7287c478bd9Sstevel@tonic-gate kmem_slab_alloc(kmem_cache_t *cp, int kmflag) 7297c478bd9Sstevel@tonic-gate { 7307c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp, **hash_bucket; 7317c478bd9Sstevel@tonic-gate kmem_slab_t *sp; 7327c478bd9Sstevel@tonic-gate void *buf; 7337c478bd9Sstevel@tonic-gate 7347c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 7357c478bd9Sstevel@tonic-gate cp->cache_slab_alloc++; 7367c478bd9Sstevel@tonic-gate sp = cp->cache_freelist; 7377c478bd9Sstevel@tonic-gate ASSERT(sp->slab_cache == cp); 7387c478bd9Sstevel@tonic-gate if (sp->slab_head == NULL) { 7397c478bd9Sstevel@tonic-gate /* 7407c478bd9Sstevel@tonic-gate * The freelist is empty. Create a new slab. 7417c478bd9Sstevel@tonic-gate */ 7427c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 7437c478bd9Sstevel@tonic-gate if ((sp = kmem_slab_create(cp, kmflag)) == NULL) 7447c478bd9Sstevel@tonic-gate return (NULL); 7457c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 7467c478bd9Sstevel@tonic-gate cp->cache_slab_create++; 7477c478bd9Sstevel@tonic-gate if ((cp->cache_buftotal += sp->slab_chunks) > cp->cache_bufmax) 7487c478bd9Sstevel@tonic-gate cp->cache_bufmax = cp->cache_buftotal; 7497c478bd9Sstevel@tonic-gate sp->slab_next = cp->cache_freelist; 7507c478bd9Sstevel@tonic-gate sp->slab_prev = cp->cache_freelist->slab_prev; 7517c478bd9Sstevel@tonic-gate sp->slab_next->slab_prev = sp; 7527c478bd9Sstevel@tonic-gate sp->slab_prev->slab_next = sp; 7537c478bd9Sstevel@tonic-gate cp->cache_freelist = sp; 7547c478bd9Sstevel@tonic-gate } 7557c478bd9Sstevel@tonic-gate 7567c478bd9Sstevel@tonic-gate sp->slab_refcnt++; 7577c478bd9Sstevel@tonic-gate ASSERT(sp->slab_refcnt <= sp->slab_chunks); 7587c478bd9Sstevel@tonic-gate 7597c478bd9Sstevel@tonic-gate /* 7607c478bd9Sstevel@tonic-gate * If we're taking the last buffer in the slab, 7617c478bd9Sstevel@tonic-gate * remove the slab from the cache's freelist. 7627c478bd9Sstevel@tonic-gate */ 7637c478bd9Sstevel@tonic-gate bcp = sp->slab_head; 7647c478bd9Sstevel@tonic-gate if ((sp->slab_head = bcp->bc_next) == NULL) { 7657c478bd9Sstevel@tonic-gate cp->cache_freelist = sp->slab_next; 7667c478bd9Sstevel@tonic-gate ASSERT(sp->slab_refcnt == sp->slab_chunks); 7677c478bd9Sstevel@tonic-gate } 7687c478bd9Sstevel@tonic-gate 7697c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) { 7707c478bd9Sstevel@tonic-gate /* 7717c478bd9Sstevel@tonic-gate * Add buffer to allocated-address hash table. 7727c478bd9Sstevel@tonic-gate */ 7737c478bd9Sstevel@tonic-gate buf = bcp->bc_addr; 7747c478bd9Sstevel@tonic-gate hash_bucket = KMEM_HASH(cp, buf); 7757c478bd9Sstevel@tonic-gate bcp->bc_next = *hash_bucket; 7767c478bd9Sstevel@tonic-gate *hash_bucket = bcp; 7777c478bd9Sstevel@tonic-gate if ((cp->cache_flags & (KMF_AUDIT | KMF_BUFTAG)) == KMF_AUDIT) { 7787c478bd9Sstevel@tonic-gate KMEM_AUDIT(kmem_transaction_log, cp, bcp); 7797c478bd9Sstevel@tonic-gate } 7807c478bd9Sstevel@tonic-gate } else { 7817c478bd9Sstevel@tonic-gate buf = KMEM_BUF(cp, bcp); 7827c478bd9Sstevel@tonic-gate } 7837c478bd9Sstevel@tonic-gate 7847c478bd9Sstevel@tonic-gate ASSERT(KMEM_SLAB_MEMBER(sp, buf)); 7857c478bd9Sstevel@tonic-gate 7867c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 7877c478bd9Sstevel@tonic-gate 7887c478bd9Sstevel@tonic-gate return (buf); 7897c478bd9Sstevel@tonic-gate } 7907c478bd9Sstevel@tonic-gate 7917c478bd9Sstevel@tonic-gate /* 7927c478bd9Sstevel@tonic-gate * Free a raw (unconstructed) buffer to cp's slab layer. 7937c478bd9Sstevel@tonic-gate */ 7947c478bd9Sstevel@tonic-gate static void 7957c478bd9Sstevel@tonic-gate kmem_slab_free(kmem_cache_t *cp, void *buf) 7967c478bd9Sstevel@tonic-gate { 7977c478bd9Sstevel@tonic-gate kmem_slab_t *sp; 7987c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp, **prev_bcpp; 7997c478bd9Sstevel@tonic-gate 8007c478bd9Sstevel@tonic-gate ASSERT(buf != NULL); 8017c478bd9Sstevel@tonic-gate 8027c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 8037c478bd9Sstevel@tonic-gate cp->cache_slab_free++; 8047c478bd9Sstevel@tonic-gate 8057c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) { 8067c478bd9Sstevel@tonic-gate /* 8077c478bd9Sstevel@tonic-gate * Look up buffer in allocated-address hash table. 8087c478bd9Sstevel@tonic-gate */ 8097c478bd9Sstevel@tonic-gate prev_bcpp = KMEM_HASH(cp, buf); 8107c478bd9Sstevel@tonic-gate while ((bcp = *prev_bcpp) != NULL) { 8117c478bd9Sstevel@tonic-gate if (bcp->bc_addr == buf) { 8127c478bd9Sstevel@tonic-gate *prev_bcpp = bcp->bc_next; 8137c478bd9Sstevel@tonic-gate sp = bcp->bc_slab; 8147c478bd9Sstevel@tonic-gate break; 8157c478bd9Sstevel@tonic-gate } 8167c478bd9Sstevel@tonic-gate cp->cache_lookup_depth++; 8177c478bd9Sstevel@tonic-gate prev_bcpp = &bcp->bc_next; 8187c478bd9Sstevel@tonic-gate } 8197c478bd9Sstevel@tonic-gate } else { 8207c478bd9Sstevel@tonic-gate bcp = KMEM_BUFCTL(cp, buf); 8217c478bd9Sstevel@tonic-gate sp = KMEM_SLAB(cp, buf); 8227c478bd9Sstevel@tonic-gate } 8237c478bd9Sstevel@tonic-gate 8247c478bd9Sstevel@tonic-gate if (bcp == NULL || sp->slab_cache != cp || !KMEM_SLAB_MEMBER(sp, buf)) { 8257c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 8267c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADADDR, cp, buf); 8277c478bd9Sstevel@tonic-gate return; 8287c478bd9Sstevel@tonic-gate } 8297c478bd9Sstevel@tonic-gate 8307c478bd9Sstevel@tonic-gate if ((cp->cache_flags & (KMF_AUDIT | KMF_BUFTAG)) == KMF_AUDIT) { 8317c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_CONTENTS) 8327c478bd9Sstevel@tonic-gate ((kmem_bufctl_audit_t *)bcp)->bc_contents = 8337c478bd9Sstevel@tonic-gate kmem_log_enter(kmem_content_log, buf, 8347c478bd9Sstevel@tonic-gate cp->cache_contents); 8357c478bd9Sstevel@tonic-gate KMEM_AUDIT(kmem_transaction_log, cp, bcp); 8367c478bd9Sstevel@tonic-gate } 8377c478bd9Sstevel@tonic-gate 8387c478bd9Sstevel@tonic-gate /* 8397c478bd9Sstevel@tonic-gate * If this slab isn't currently on the freelist, put it there. 8407c478bd9Sstevel@tonic-gate */ 8417c478bd9Sstevel@tonic-gate if (sp->slab_head == NULL) { 8427c478bd9Sstevel@tonic-gate ASSERT(sp->slab_refcnt == sp->slab_chunks); 8437c478bd9Sstevel@tonic-gate ASSERT(cp->cache_freelist != sp); 8447c478bd9Sstevel@tonic-gate sp->slab_next->slab_prev = sp->slab_prev; 8457c478bd9Sstevel@tonic-gate sp->slab_prev->slab_next = sp->slab_next; 8467c478bd9Sstevel@tonic-gate sp->slab_next = cp->cache_freelist; 8477c478bd9Sstevel@tonic-gate sp->slab_prev = cp->cache_freelist->slab_prev; 8487c478bd9Sstevel@tonic-gate sp->slab_next->slab_prev = sp; 8497c478bd9Sstevel@tonic-gate sp->slab_prev->slab_next = sp; 8507c478bd9Sstevel@tonic-gate cp->cache_freelist = sp; 8517c478bd9Sstevel@tonic-gate } 8527c478bd9Sstevel@tonic-gate 8537c478bd9Sstevel@tonic-gate bcp->bc_next = sp->slab_head; 8547c478bd9Sstevel@tonic-gate sp->slab_head = bcp; 8557c478bd9Sstevel@tonic-gate 8567c478bd9Sstevel@tonic-gate ASSERT(sp->slab_refcnt >= 1); 8577c478bd9Sstevel@tonic-gate if (--sp->slab_refcnt == 0) { 8587c478bd9Sstevel@tonic-gate /* 8597c478bd9Sstevel@tonic-gate * There are no outstanding allocations from this slab, 8607c478bd9Sstevel@tonic-gate * so we can reclaim the memory. 8617c478bd9Sstevel@tonic-gate */ 8627c478bd9Sstevel@tonic-gate sp->slab_next->slab_prev = sp->slab_prev; 8637c478bd9Sstevel@tonic-gate sp->slab_prev->slab_next = sp->slab_next; 8647c478bd9Sstevel@tonic-gate if (sp == cp->cache_freelist) 8657c478bd9Sstevel@tonic-gate cp->cache_freelist = sp->slab_next; 8667c478bd9Sstevel@tonic-gate cp->cache_slab_destroy++; 8677c478bd9Sstevel@tonic-gate cp->cache_buftotal -= sp->slab_chunks; 8687c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 8697c478bd9Sstevel@tonic-gate kmem_slab_destroy(cp, sp); 8707c478bd9Sstevel@tonic-gate return; 8717c478bd9Sstevel@tonic-gate } 8727c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 8737c478bd9Sstevel@tonic-gate } 8747c478bd9Sstevel@tonic-gate 8757c478bd9Sstevel@tonic-gate static int 8767c478bd9Sstevel@tonic-gate kmem_cache_alloc_debug(kmem_cache_t *cp, void *buf, int kmflag, int construct, 8777c478bd9Sstevel@tonic-gate caddr_t caller) 8787c478bd9Sstevel@tonic-gate { 8797c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 8807c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *bcp = (kmem_bufctl_audit_t *)btp->bt_bufctl; 8817c478bd9Sstevel@tonic-gate uint32_t mtbf; 8827c478bd9Sstevel@tonic-gate 8837c478bd9Sstevel@tonic-gate if (btp->bt_bxstat != ((intptr_t)bcp ^ KMEM_BUFTAG_FREE)) { 8847c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADBUFTAG, cp, buf); 8857c478bd9Sstevel@tonic-gate return (-1); 8867c478bd9Sstevel@tonic-gate } 8877c478bd9Sstevel@tonic-gate 8887c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_ALLOC; 8897c478bd9Sstevel@tonic-gate 8907c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_HASH) && bcp->bc_addr != buf) { 8917c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADBUFCTL, cp, buf); 8927c478bd9Sstevel@tonic-gate return (-1); 8937c478bd9Sstevel@tonic-gate } 8947c478bd9Sstevel@tonic-gate 8957c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) { 8967c478bd9Sstevel@tonic-gate if (!construct && (cp->cache_flags & KMF_LITE)) { 8977c478bd9Sstevel@tonic-gate if (*(uint64_t *)buf != KMEM_FREE_PATTERN) { 8987c478bd9Sstevel@tonic-gate kmem_error(KMERR_MODIFIED, cp, buf); 8997c478bd9Sstevel@tonic-gate return (-1); 9007c478bd9Sstevel@tonic-gate } 9017c478bd9Sstevel@tonic-gate if (cp->cache_constructor != NULL) 9027c478bd9Sstevel@tonic-gate *(uint64_t *)buf = btp->bt_redzone; 9037c478bd9Sstevel@tonic-gate else 9047c478bd9Sstevel@tonic-gate *(uint64_t *)buf = KMEM_UNINITIALIZED_PATTERN; 9057c478bd9Sstevel@tonic-gate } else { 9067c478bd9Sstevel@tonic-gate construct = 1; 9077c478bd9Sstevel@tonic-gate if (verify_and_copy_pattern(KMEM_FREE_PATTERN, 9087c478bd9Sstevel@tonic-gate KMEM_UNINITIALIZED_PATTERN, buf, 9097c478bd9Sstevel@tonic-gate cp->cache_verify)) { 9107c478bd9Sstevel@tonic-gate kmem_error(KMERR_MODIFIED, cp, buf); 9117c478bd9Sstevel@tonic-gate return (-1); 9127c478bd9Sstevel@tonic-gate } 9137c478bd9Sstevel@tonic-gate } 9147c478bd9Sstevel@tonic-gate } 9157c478bd9Sstevel@tonic-gate btp->bt_redzone = KMEM_REDZONE_PATTERN; 9167c478bd9Sstevel@tonic-gate 9177c478bd9Sstevel@tonic-gate if ((mtbf = kmem_mtbf | cp->cache_mtbf) != 0 && 9187c478bd9Sstevel@tonic-gate gethrtime() % mtbf == 0 && 9197c478bd9Sstevel@tonic-gate (kmflag & (KM_NOSLEEP | KM_PANIC)) == KM_NOSLEEP) { 9207c478bd9Sstevel@tonic-gate kmem_log_event(kmem_failure_log, cp, NULL, NULL); 9217c478bd9Sstevel@tonic-gate if (!construct && cp->cache_destructor != NULL) 9227c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 9237c478bd9Sstevel@tonic-gate } else { 9247c478bd9Sstevel@tonic-gate mtbf = 0; 9257c478bd9Sstevel@tonic-gate } 9267c478bd9Sstevel@tonic-gate 9277c478bd9Sstevel@tonic-gate if (mtbf || (construct && cp->cache_constructor != NULL && 9287c478bd9Sstevel@tonic-gate cp->cache_constructor(buf, cp->cache_private, kmflag) != 0)) { 9297c478bd9Sstevel@tonic-gate atomic_add_64(&cp->cache_alloc_fail, 1); 9307c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE; 9317c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) 9327c478bd9Sstevel@tonic-gate copy_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify); 9337c478bd9Sstevel@tonic-gate kmem_slab_free(cp, buf); 9347c478bd9Sstevel@tonic-gate return (-1); 9357c478bd9Sstevel@tonic-gate } 9367c478bd9Sstevel@tonic-gate 9377c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_AUDIT) { 9387c478bd9Sstevel@tonic-gate KMEM_AUDIT(kmem_transaction_log, cp, bcp); 9397c478bd9Sstevel@tonic-gate } 9407c478bd9Sstevel@tonic-gate 9417c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_LITE) && 9427c478bd9Sstevel@tonic-gate !(cp->cache_cflags & KMC_KMEM_ALLOC)) { 9437c478bd9Sstevel@tonic-gate KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, caller); 9447c478bd9Sstevel@tonic-gate } 9457c478bd9Sstevel@tonic-gate 9467c478bd9Sstevel@tonic-gate return (0); 9477c478bd9Sstevel@tonic-gate } 9487c478bd9Sstevel@tonic-gate 9497c478bd9Sstevel@tonic-gate static int 9507c478bd9Sstevel@tonic-gate kmem_cache_free_debug(kmem_cache_t *cp, void *buf, caddr_t caller) 9517c478bd9Sstevel@tonic-gate { 9527c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 9537c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *bcp = (kmem_bufctl_audit_t *)btp->bt_bufctl; 9547c478bd9Sstevel@tonic-gate kmem_slab_t *sp; 9557c478bd9Sstevel@tonic-gate 9567c478bd9Sstevel@tonic-gate if (btp->bt_bxstat != ((intptr_t)bcp ^ KMEM_BUFTAG_ALLOC)) { 9577c478bd9Sstevel@tonic-gate if (btp->bt_bxstat == ((intptr_t)bcp ^ KMEM_BUFTAG_FREE)) { 9587c478bd9Sstevel@tonic-gate kmem_error(KMERR_DUPFREE, cp, buf); 9597c478bd9Sstevel@tonic-gate return (-1); 9607c478bd9Sstevel@tonic-gate } 9617c478bd9Sstevel@tonic-gate sp = kmem_findslab(cp, buf); 9627c478bd9Sstevel@tonic-gate if (sp == NULL || sp->slab_cache != cp) 9637c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADADDR, cp, buf); 9647c478bd9Sstevel@tonic-gate else 9657c478bd9Sstevel@tonic-gate kmem_error(KMERR_REDZONE, cp, buf); 9667c478bd9Sstevel@tonic-gate return (-1); 9677c478bd9Sstevel@tonic-gate } 9687c478bd9Sstevel@tonic-gate 9697c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE; 9707c478bd9Sstevel@tonic-gate 9717c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_HASH) && bcp->bc_addr != buf) { 9727c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADBUFCTL, cp, buf); 9737c478bd9Sstevel@tonic-gate return (-1); 9747c478bd9Sstevel@tonic-gate } 9757c478bd9Sstevel@tonic-gate 9767c478bd9Sstevel@tonic-gate if (btp->bt_redzone != KMEM_REDZONE_PATTERN) { 9777c478bd9Sstevel@tonic-gate kmem_error(KMERR_REDZONE, cp, buf); 9787c478bd9Sstevel@tonic-gate return (-1); 9797c478bd9Sstevel@tonic-gate } 9807c478bd9Sstevel@tonic-gate 9817c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_AUDIT) { 9827c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_CONTENTS) 9837c478bd9Sstevel@tonic-gate bcp->bc_contents = kmem_log_enter(kmem_content_log, 9847c478bd9Sstevel@tonic-gate buf, cp->cache_contents); 9857c478bd9Sstevel@tonic-gate KMEM_AUDIT(kmem_transaction_log, cp, bcp); 9867c478bd9Sstevel@tonic-gate } 9877c478bd9Sstevel@tonic-gate 9887c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_LITE) && 9897c478bd9Sstevel@tonic-gate !(cp->cache_cflags & KMC_KMEM_ALLOC)) { 9907c478bd9Sstevel@tonic-gate KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, caller); 9917c478bd9Sstevel@tonic-gate } 9927c478bd9Sstevel@tonic-gate 9937c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) { 9947c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) 9957c478bd9Sstevel@tonic-gate btp->bt_redzone = *(uint64_t *)buf; 9967c478bd9Sstevel@tonic-gate else if (cp->cache_destructor != NULL) 9977c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 9987c478bd9Sstevel@tonic-gate 9997c478bd9Sstevel@tonic-gate copy_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify); 10007c478bd9Sstevel@tonic-gate } 10017c478bd9Sstevel@tonic-gate 10027c478bd9Sstevel@tonic-gate return (0); 10037c478bd9Sstevel@tonic-gate } 10047c478bd9Sstevel@tonic-gate 10057c478bd9Sstevel@tonic-gate /* 10067c478bd9Sstevel@tonic-gate * Free each object in magazine mp to cp's slab layer, and free mp itself. 10077c478bd9Sstevel@tonic-gate */ 10087c478bd9Sstevel@tonic-gate static void 10097c478bd9Sstevel@tonic-gate kmem_magazine_destroy(kmem_cache_t *cp, kmem_magazine_t *mp, int nrounds) 10107c478bd9Sstevel@tonic-gate { 10117c478bd9Sstevel@tonic-gate int round; 10127c478bd9Sstevel@tonic-gate 10137c478bd9Sstevel@tonic-gate ASSERT(cp->cache_next == NULL || taskq_member(kmem_taskq, curthread)); 10147c478bd9Sstevel@tonic-gate 10157c478bd9Sstevel@tonic-gate for (round = 0; round < nrounds; round++) { 10167c478bd9Sstevel@tonic-gate void *buf = mp->mag_round[round]; 10177c478bd9Sstevel@tonic-gate 10187c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) { 10197c478bd9Sstevel@tonic-gate if (verify_pattern(KMEM_FREE_PATTERN, buf, 10207c478bd9Sstevel@tonic-gate cp->cache_verify) != NULL) { 10217c478bd9Sstevel@tonic-gate kmem_error(KMERR_MODIFIED, cp, buf); 10227c478bd9Sstevel@tonic-gate continue; 10237c478bd9Sstevel@tonic-gate } 10247c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_LITE) && 10257c478bd9Sstevel@tonic-gate cp->cache_destructor != NULL) { 10267c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 10277c478bd9Sstevel@tonic-gate *(uint64_t *)buf = btp->bt_redzone; 10287c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 10297c478bd9Sstevel@tonic-gate *(uint64_t *)buf = KMEM_FREE_PATTERN; 10307c478bd9Sstevel@tonic-gate } 10317c478bd9Sstevel@tonic-gate } else if (cp->cache_destructor != NULL) { 10327c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 10337c478bd9Sstevel@tonic-gate } 10347c478bd9Sstevel@tonic-gate 10357c478bd9Sstevel@tonic-gate kmem_slab_free(cp, buf); 10367c478bd9Sstevel@tonic-gate } 10377c478bd9Sstevel@tonic-gate ASSERT(KMEM_MAGAZINE_VALID(cp, mp)); 10387c478bd9Sstevel@tonic-gate kmem_cache_free(cp->cache_magtype->mt_cache, mp); 10397c478bd9Sstevel@tonic-gate } 10407c478bd9Sstevel@tonic-gate 10417c478bd9Sstevel@tonic-gate /* 10427c478bd9Sstevel@tonic-gate * Allocate a magazine from the depot. 10437c478bd9Sstevel@tonic-gate */ 10447c478bd9Sstevel@tonic-gate static kmem_magazine_t * 10457c478bd9Sstevel@tonic-gate kmem_depot_alloc(kmem_cache_t *cp, kmem_maglist_t *mlp) 10467c478bd9Sstevel@tonic-gate { 10477c478bd9Sstevel@tonic-gate kmem_magazine_t *mp; 10487c478bd9Sstevel@tonic-gate 10497c478bd9Sstevel@tonic-gate /* 10507c478bd9Sstevel@tonic-gate * If we can't get the depot lock without contention, 10517c478bd9Sstevel@tonic-gate * update our contention count. We use the depot 10527c478bd9Sstevel@tonic-gate * contention rate to determine whether we need to 10537c478bd9Sstevel@tonic-gate * increase the magazine size for better scalability. 10547c478bd9Sstevel@tonic-gate */ 10557c478bd9Sstevel@tonic-gate if (!mutex_tryenter(&cp->cache_depot_lock)) { 10567c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock); 10577c478bd9Sstevel@tonic-gate cp->cache_depot_contention++; 10587c478bd9Sstevel@tonic-gate } 10597c478bd9Sstevel@tonic-gate 10607c478bd9Sstevel@tonic-gate if ((mp = mlp->ml_list) != NULL) { 10617c478bd9Sstevel@tonic-gate ASSERT(KMEM_MAGAZINE_VALID(cp, mp)); 10627c478bd9Sstevel@tonic-gate mlp->ml_list = mp->mag_next; 10637c478bd9Sstevel@tonic-gate if (--mlp->ml_total < mlp->ml_min) 10647c478bd9Sstevel@tonic-gate mlp->ml_min = mlp->ml_total; 10657c478bd9Sstevel@tonic-gate mlp->ml_alloc++; 10667c478bd9Sstevel@tonic-gate } 10677c478bd9Sstevel@tonic-gate 10687c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock); 10697c478bd9Sstevel@tonic-gate 10707c478bd9Sstevel@tonic-gate return (mp); 10717c478bd9Sstevel@tonic-gate } 10727c478bd9Sstevel@tonic-gate 10737c478bd9Sstevel@tonic-gate /* 10747c478bd9Sstevel@tonic-gate * Free a magazine to the depot. 10757c478bd9Sstevel@tonic-gate */ 10767c478bd9Sstevel@tonic-gate static void 10777c478bd9Sstevel@tonic-gate kmem_depot_free(kmem_cache_t *cp, kmem_maglist_t *mlp, kmem_magazine_t *mp) 10787c478bd9Sstevel@tonic-gate { 10797c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock); 10807c478bd9Sstevel@tonic-gate ASSERT(KMEM_MAGAZINE_VALID(cp, mp)); 10817c478bd9Sstevel@tonic-gate mp->mag_next = mlp->ml_list; 10827c478bd9Sstevel@tonic-gate mlp->ml_list = mp; 10837c478bd9Sstevel@tonic-gate mlp->ml_total++; 10847c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock); 10857c478bd9Sstevel@tonic-gate } 10867c478bd9Sstevel@tonic-gate 10877c478bd9Sstevel@tonic-gate /* 10887c478bd9Sstevel@tonic-gate * Update the working set statistics for cp's depot. 10897c478bd9Sstevel@tonic-gate */ 10907c478bd9Sstevel@tonic-gate static void 10917c478bd9Sstevel@tonic-gate kmem_depot_ws_update(kmem_cache_t *cp) 10927c478bd9Sstevel@tonic-gate { 10937c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock); 10947c478bd9Sstevel@tonic-gate cp->cache_full.ml_reaplimit = cp->cache_full.ml_min; 10957c478bd9Sstevel@tonic-gate cp->cache_full.ml_min = cp->cache_full.ml_total; 10967c478bd9Sstevel@tonic-gate cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_min; 10977c478bd9Sstevel@tonic-gate cp->cache_empty.ml_min = cp->cache_empty.ml_total; 10987c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock); 10997c478bd9Sstevel@tonic-gate } 11007c478bd9Sstevel@tonic-gate 11017c478bd9Sstevel@tonic-gate /* 11027c478bd9Sstevel@tonic-gate * Reap all magazines that have fallen out of the depot's working set. 11037c478bd9Sstevel@tonic-gate */ 11047c478bd9Sstevel@tonic-gate static void 11057c478bd9Sstevel@tonic-gate kmem_depot_ws_reap(kmem_cache_t *cp) 11067c478bd9Sstevel@tonic-gate { 11077c478bd9Sstevel@tonic-gate long reap; 11087c478bd9Sstevel@tonic-gate kmem_magazine_t *mp; 11097c478bd9Sstevel@tonic-gate 11107c478bd9Sstevel@tonic-gate ASSERT(cp->cache_next == NULL || taskq_member(kmem_taskq, curthread)); 11117c478bd9Sstevel@tonic-gate 11127c478bd9Sstevel@tonic-gate reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min); 11137c478bd9Sstevel@tonic-gate while (reap-- && (mp = kmem_depot_alloc(cp, &cp->cache_full)) != NULL) 11147c478bd9Sstevel@tonic-gate kmem_magazine_destroy(cp, mp, cp->cache_magtype->mt_magsize); 11157c478bd9Sstevel@tonic-gate 11167c478bd9Sstevel@tonic-gate reap = MIN(cp->cache_empty.ml_reaplimit, cp->cache_empty.ml_min); 11177c478bd9Sstevel@tonic-gate while (reap-- && (mp = kmem_depot_alloc(cp, &cp->cache_empty)) != NULL) 11187c478bd9Sstevel@tonic-gate kmem_magazine_destroy(cp, mp, 0); 11197c478bd9Sstevel@tonic-gate } 11207c478bd9Sstevel@tonic-gate 11217c478bd9Sstevel@tonic-gate static void 11227c478bd9Sstevel@tonic-gate kmem_cpu_reload(kmem_cpu_cache_t *ccp, kmem_magazine_t *mp, int rounds) 11237c478bd9Sstevel@tonic-gate { 11247c478bd9Sstevel@tonic-gate ASSERT((ccp->cc_loaded == NULL && ccp->cc_rounds == -1) || 11257c478bd9Sstevel@tonic-gate (ccp->cc_loaded && ccp->cc_rounds + rounds == ccp->cc_magsize)); 11267c478bd9Sstevel@tonic-gate ASSERT(ccp->cc_magsize > 0); 11277c478bd9Sstevel@tonic-gate 11287c478bd9Sstevel@tonic-gate ccp->cc_ploaded = ccp->cc_loaded; 11297c478bd9Sstevel@tonic-gate ccp->cc_prounds = ccp->cc_rounds; 11307c478bd9Sstevel@tonic-gate ccp->cc_loaded = mp; 11317c478bd9Sstevel@tonic-gate ccp->cc_rounds = rounds; 11327c478bd9Sstevel@tonic-gate } 11337c478bd9Sstevel@tonic-gate 11347c478bd9Sstevel@tonic-gate /* 11357c478bd9Sstevel@tonic-gate * Allocate a constructed object from cache cp. 11367c478bd9Sstevel@tonic-gate */ 11377c478bd9Sstevel@tonic-gate void * 11387c478bd9Sstevel@tonic-gate kmem_cache_alloc(kmem_cache_t *cp, int kmflag) 11397c478bd9Sstevel@tonic-gate { 11407c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp = KMEM_CPU_CACHE(cp); 11417c478bd9Sstevel@tonic-gate kmem_magazine_t *fmp; 11427c478bd9Sstevel@tonic-gate void *buf; 11437c478bd9Sstevel@tonic-gate 11447c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 11457c478bd9Sstevel@tonic-gate for (;;) { 11467c478bd9Sstevel@tonic-gate /* 11477c478bd9Sstevel@tonic-gate * If there's an object available in the current CPU's 11487c478bd9Sstevel@tonic-gate * loaded magazine, just take it and return. 11497c478bd9Sstevel@tonic-gate */ 11507c478bd9Sstevel@tonic-gate if (ccp->cc_rounds > 0) { 11517c478bd9Sstevel@tonic-gate buf = ccp->cc_loaded->mag_round[--ccp->cc_rounds]; 11527c478bd9Sstevel@tonic-gate ccp->cc_alloc++; 11537c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 11547c478bd9Sstevel@tonic-gate if ((ccp->cc_flags & KMF_BUFTAG) && 11557c478bd9Sstevel@tonic-gate kmem_cache_alloc_debug(cp, buf, kmflag, 0, 11567c478bd9Sstevel@tonic-gate caller()) == -1) { 11577c478bd9Sstevel@tonic-gate if (kmflag & KM_NOSLEEP) 11587c478bd9Sstevel@tonic-gate return (NULL); 11597c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 11607c478bd9Sstevel@tonic-gate continue; 11617c478bd9Sstevel@tonic-gate } 11627c478bd9Sstevel@tonic-gate return (buf); 11637c478bd9Sstevel@tonic-gate } 11647c478bd9Sstevel@tonic-gate 11657c478bd9Sstevel@tonic-gate /* 11667c478bd9Sstevel@tonic-gate * The loaded magazine is empty. If the previously loaded 11677c478bd9Sstevel@tonic-gate * magazine was full, exchange them and try again. 11687c478bd9Sstevel@tonic-gate */ 11697c478bd9Sstevel@tonic-gate if (ccp->cc_prounds > 0) { 11707c478bd9Sstevel@tonic-gate kmem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds); 11717c478bd9Sstevel@tonic-gate continue; 11727c478bd9Sstevel@tonic-gate } 11737c478bd9Sstevel@tonic-gate 11747c478bd9Sstevel@tonic-gate /* 11757c478bd9Sstevel@tonic-gate * If the magazine layer is disabled, break out now. 11767c478bd9Sstevel@tonic-gate */ 11777c478bd9Sstevel@tonic-gate if (ccp->cc_magsize == 0) 11787c478bd9Sstevel@tonic-gate break; 11797c478bd9Sstevel@tonic-gate 11807c478bd9Sstevel@tonic-gate /* 11817c478bd9Sstevel@tonic-gate * Try to get a full magazine from the depot. 11827c478bd9Sstevel@tonic-gate */ 11837c478bd9Sstevel@tonic-gate fmp = kmem_depot_alloc(cp, &cp->cache_full); 11847c478bd9Sstevel@tonic-gate if (fmp != NULL) { 11857c478bd9Sstevel@tonic-gate if (ccp->cc_ploaded != NULL) 11867c478bd9Sstevel@tonic-gate kmem_depot_free(cp, &cp->cache_empty, 11877c478bd9Sstevel@tonic-gate ccp->cc_ploaded); 11887c478bd9Sstevel@tonic-gate kmem_cpu_reload(ccp, fmp, ccp->cc_magsize); 11897c478bd9Sstevel@tonic-gate continue; 11907c478bd9Sstevel@tonic-gate } 11917c478bd9Sstevel@tonic-gate 11927c478bd9Sstevel@tonic-gate /* 11937c478bd9Sstevel@tonic-gate * There are no full magazines in the depot, 11947c478bd9Sstevel@tonic-gate * so fall through to the slab layer. 11957c478bd9Sstevel@tonic-gate */ 11967c478bd9Sstevel@tonic-gate break; 11977c478bd9Sstevel@tonic-gate } 11987c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 11997c478bd9Sstevel@tonic-gate 12007c478bd9Sstevel@tonic-gate /* 12017c478bd9Sstevel@tonic-gate * We couldn't allocate a constructed object from the magazine layer, 12027c478bd9Sstevel@tonic-gate * so get a raw buffer from the slab layer and apply its constructor. 12037c478bd9Sstevel@tonic-gate */ 12047c478bd9Sstevel@tonic-gate buf = kmem_slab_alloc(cp, kmflag); 12057c478bd9Sstevel@tonic-gate 12067c478bd9Sstevel@tonic-gate if (buf == NULL) 12077c478bd9Sstevel@tonic-gate return (NULL); 12087c478bd9Sstevel@tonic-gate 12097c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_BUFTAG) { 12107c478bd9Sstevel@tonic-gate /* 12117c478bd9Sstevel@tonic-gate * Make kmem_cache_alloc_debug() apply the constructor for us. 12127c478bd9Sstevel@tonic-gate */ 12137c478bd9Sstevel@tonic-gate if (kmem_cache_alloc_debug(cp, buf, kmflag, 1, 12147c478bd9Sstevel@tonic-gate caller()) == -1) { 12157c478bd9Sstevel@tonic-gate if (kmflag & KM_NOSLEEP) 12167c478bd9Sstevel@tonic-gate return (NULL); 12177c478bd9Sstevel@tonic-gate /* 12187c478bd9Sstevel@tonic-gate * kmem_cache_alloc_debug() detected corruption 12197c478bd9Sstevel@tonic-gate * but didn't panic (kmem_panic <= 0). Try again. 12207c478bd9Sstevel@tonic-gate */ 12217c478bd9Sstevel@tonic-gate return (kmem_cache_alloc(cp, kmflag)); 12227c478bd9Sstevel@tonic-gate } 12237c478bd9Sstevel@tonic-gate return (buf); 12247c478bd9Sstevel@tonic-gate } 12257c478bd9Sstevel@tonic-gate 12267c478bd9Sstevel@tonic-gate if (cp->cache_constructor != NULL && 12277c478bd9Sstevel@tonic-gate cp->cache_constructor(buf, cp->cache_private, kmflag) != 0) { 12287c478bd9Sstevel@tonic-gate atomic_add_64(&cp->cache_alloc_fail, 1); 12297c478bd9Sstevel@tonic-gate kmem_slab_free(cp, buf); 12307c478bd9Sstevel@tonic-gate return (NULL); 12317c478bd9Sstevel@tonic-gate } 12327c478bd9Sstevel@tonic-gate 12337c478bd9Sstevel@tonic-gate return (buf); 12347c478bd9Sstevel@tonic-gate } 12357c478bd9Sstevel@tonic-gate 12367c478bd9Sstevel@tonic-gate /* 12377c478bd9Sstevel@tonic-gate * Free a constructed object to cache cp. 12387c478bd9Sstevel@tonic-gate */ 12397c478bd9Sstevel@tonic-gate void 12407c478bd9Sstevel@tonic-gate kmem_cache_free(kmem_cache_t *cp, void *buf) 12417c478bd9Sstevel@tonic-gate { 12427c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp = KMEM_CPU_CACHE(cp); 12437c478bd9Sstevel@tonic-gate kmem_magazine_t *emp; 12447c478bd9Sstevel@tonic-gate kmem_magtype_t *mtp; 12457c478bd9Sstevel@tonic-gate 12467c478bd9Sstevel@tonic-gate if (ccp->cc_flags & KMF_BUFTAG) 12477c478bd9Sstevel@tonic-gate if (kmem_cache_free_debug(cp, buf, caller()) == -1) 12487c478bd9Sstevel@tonic-gate return; 12497c478bd9Sstevel@tonic-gate 12507c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 12517c478bd9Sstevel@tonic-gate for (;;) { 12527c478bd9Sstevel@tonic-gate /* 12537c478bd9Sstevel@tonic-gate * If there's a slot available in the current CPU's 12547c478bd9Sstevel@tonic-gate * loaded magazine, just put the object there and return. 12557c478bd9Sstevel@tonic-gate */ 12567c478bd9Sstevel@tonic-gate if ((uint_t)ccp->cc_rounds < ccp->cc_magsize) { 12577c478bd9Sstevel@tonic-gate ccp->cc_loaded->mag_round[ccp->cc_rounds++] = buf; 12587c478bd9Sstevel@tonic-gate ccp->cc_free++; 12597c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 12607c478bd9Sstevel@tonic-gate return; 12617c478bd9Sstevel@tonic-gate } 12627c478bd9Sstevel@tonic-gate 12637c478bd9Sstevel@tonic-gate /* 12647c478bd9Sstevel@tonic-gate * The loaded magazine is full. If the previously loaded 12657c478bd9Sstevel@tonic-gate * magazine was empty, exchange them and try again. 12667c478bd9Sstevel@tonic-gate */ 12677c478bd9Sstevel@tonic-gate if (ccp->cc_prounds == 0) { 12687c478bd9Sstevel@tonic-gate kmem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds); 12697c478bd9Sstevel@tonic-gate continue; 12707c478bd9Sstevel@tonic-gate } 12717c478bd9Sstevel@tonic-gate 12727c478bd9Sstevel@tonic-gate /* 12737c478bd9Sstevel@tonic-gate * If the magazine layer is disabled, break out now. 12747c478bd9Sstevel@tonic-gate */ 12757c478bd9Sstevel@tonic-gate if (ccp->cc_magsize == 0) 12767c478bd9Sstevel@tonic-gate break; 12777c478bd9Sstevel@tonic-gate 12787c478bd9Sstevel@tonic-gate /* 12797c478bd9Sstevel@tonic-gate * Try to get an empty magazine from the depot. 12807c478bd9Sstevel@tonic-gate */ 12817c478bd9Sstevel@tonic-gate emp = kmem_depot_alloc(cp, &cp->cache_empty); 12827c478bd9Sstevel@tonic-gate if (emp != NULL) { 12837c478bd9Sstevel@tonic-gate if (ccp->cc_ploaded != NULL) 12847c478bd9Sstevel@tonic-gate kmem_depot_free(cp, &cp->cache_full, 12857c478bd9Sstevel@tonic-gate ccp->cc_ploaded); 12867c478bd9Sstevel@tonic-gate kmem_cpu_reload(ccp, emp, 0); 12877c478bd9Sstevel@tonic-gate continue; 12887c478bd9Sstevel@tonic-gate } 12897c478bd9Sstevel@tonic-gate 12907c478bd9Sstevel@tonic-gate /* 12917c478bd9Sstevel@tonic-gate * There are no empty magazines in the depot, 12927c478bd9Sstevel@tonic-gate * so try to allocate a new one. We must drop all locks 12937c478bd9Sstevel@tonic-gate * across kmem_cache_alloc() because lower layers may 12947c478bd9Sstevel@tonic-gate * attempt to allocate from this cache. 12957c478bd9Sstevel@tonic-gate */ 12967c478bd9Sstevel@tonic-gate mtp = cp->cache_magtype; 12977c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 12987c478bd9Sstevel@tonic-gate emp = kmem_cache_alloc(mtp->mt_cache, KM_NOSLEEP); 12997c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 13007c478bd9Sstevel@tonic-gate 13017c478bd9Sstevel@tonic-gate if (emp != NULL) { 13027c478bd9Sstevel@tonic-gate /* 13037c478bd9Sstevel@tonic-gate * We successfully allocated an empty magazine. 13047c478bd9Sstevel@tonic-gate * However, we had to drop ccp->cc_lock to do it, 13057c478bd9Sstevel@tonic-gate * so the cache's magazine size may have changed. 13067c478bd9Sstevel@tonic-gate * If so, free the magazine and try again. 13077c478bd9Sstevel@tonic-gate */ 13087c478bd9Sstevel@tonic-gate if (ccp->cc_magsize != mtp->mt_magsize) { 13097c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 13107c478bd9Sstevel@tonic-gate kmem_cache_free(mtp->mt_cache, emp); 13117c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 13127c478bd9Sstevel@tonic-gate continue; 13137c478bd9Sstevel@tonic-gate } 13147c478bd9Sstevel@tonic-gate 13157c478bd9Sstevel@tonic-gate /* 13167c478bd9Sstevel@tonic-gate * We got a magazine of the right size. Add it to 13177c478bd9Sstevel@tonic-gate * the depot and try the whole dance again. 13187c478bd9Sstevel@tonic-gate */ 13197c478bd9Sstevel@tonic-gate kmem_depot_free(cp, &cp->cache_empty, emp); 13207c478bd9Sstevel@tonic-gate continue; 13217c478bd9Sstevel@tonic-gate } 13227c478bd9Sstevel@tonic-gate 13237c478bd9Sstevel@tonic-gate /* 13247c478bd9Sstevel@tonic-gate * We couldn't allocate an empty magazine, 13257c478bd9Sstevel@tonic-gate * so fall through to the slab layer. 13267c478bd9Sstevel@tonic-gate */ 13277c478bd9Sstevel@tonic-gate break; 13287c478bd9Sstevel@tonic-gate } 13297c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 13307c478bd9Sstevel@tonic-gate 13317c478bd9Sstevel@tonic-gate /* 13327c478bd9Sstevel@tonic-gate * We couldn't free our constructed object to the magazine layer, 13337c478bd9Sstevel@tonic-gate * so apply its destructor and free it to the slab layer. 13347c478bd9Sstevel@tonic-gate * Note that if KMF_DEADBEEF is in effect and KMF_LITE is not, 13357c478bd9Sstevel@tonic-gate * kmem_cache_free_debug() will have already applied the destructor. 13367c478bd9Sstevel@tonic-gate */ 13377c478bd9Sstevel@tonic-gate if ((cp->cache_flags & (KMF_DEADBEEF | KMF_LITE)) != KMF_DEADBEEF && 13387c478bd9Sstevel@tonic-gate cp->cache_destructor != NULL) { 13397c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) { /* KMF_LITE implied */ 13407c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 13417c478bd9Sstevel@tonic-gate *(uint64_t *)buf = btp->bt_redzone; 13427c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 13437c478bd9Sstevel@tonic-gate *(uint64_t *)buf = KMEM_FREE_PATTERN; 13447c478bd9Sstevel@tonic-gate } else { 13457c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 13467c478bd9Sstevel@tonic-gate } 13477c478bd9Sstevel@tonic-gate } 13487c478bd9Sstevel@tonic-gate 13497c478bd9Sstevel@tonic-gate kmem_slab_free(cp, buf); 13507c478bd9Sstevel@tonic-gate } 13517c478bd9Sstevel@tonic-gate 13527c478bd9Sstevel@tonic-gate void * 13537c478bd9Sstevel@tonic-gate kmem_zalloc(size_t size, int kmflag) 13547c478bd9Sstevel@tonic-gate { 13557c478bd9Sstevel@tonic-gate size_t index = (size - 1) >> KMEM_ALIGN_SHIFT; 13567c478bd9Sstevel@tonic-gate void *buf; 13577c478bd9Sstevel@tonic-gate 13587c478bd9Sstevel@tonic-gate if (index < KMEM_MAXBUF >> KMEM_ALIGN_SHIFT) { 13597c478bd9Sstevel@tonic-gate kmem_cache_t *cp = kmem_alloc_table[index]; 13607c478bd9Sstevel@tonic-gate buf = kmem_cache_alloc(cp, kmflag); 13617c478bd9Sstevel@tonic-gate if (buf != NULL) { 13627c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_BUFTAG) { 13637c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 13647c478bd9Sstevel@tonic-gate ((uint8_t *)buf)[size] = KMEM_REDZONE_BYTE; 13657c478bd9Sstevel@tonic-gate ((uint32_t *)btp)[1] = KMEM_SIZE_ENCODE(size); 13667c478bd9Sstevel@tonic-gate 13677c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) { 13687c478bd9Sstevel@tonic-gate KMEM_BUFTAG_LITE_ENTER(btp, 13697c478bd9Sstevel@tonic-gate kmem_lite_count, caller()); 13707c478bd9Sstevel@tonic-gate } 13717c478bd9Sstevel@tonic-gate } 13727c478bd9Sstevel@tonic-gate bzero(buf, size); 13737c478bd9Sstevel@tonic-gate } 13747c478bd9Sstevel@tonic-gate } else { 13757c478bd9Sstevel@tonic-gate buf = kmem_alloc(size, kmflag); 13767c478bd9Sstevel@tonic-gate if (buf != NULL) 13777c478bd9Sstevel@tonic-gate bzero(buf, size); 13787c478bd9Sstevel@tonic-gate } 13797c478bd9Sstevel@tonic-gate return (buf); 13807c478bd9Sstevel@tonic-gate } 13817c478bd9Sstevel@tonic-gate 13827c478bd9Sstevel@tonic-gate void * 13837c478bd9Sstevel@tonic-gate kmem_alloc(size_t size, int kmflag) 13847c478bd9Sstevel@tonic-gate { 13857c478bd9Sstevel@tonic-gate size_t index = (size - 1) >> KMEM_ALIGN_SHIFT; 13867c478bd9Sstevel@tonic-gate void *buf; 13877c478bd9Sstevel@tonic-gate 13887c478bd9Sstevel@tonic-gate if (index < KMEM_MAXBUF >> KMEM_ALIGN_SHIFT) { 13897c478bd9Sstevel@tonic-gate kmem_cache_t *cp = kmem_alloc_table[index]; 13907c478bd9Sstevel@tonic-gate buf = kmem_cache_alloc(cp, kmflag); 13917c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_BUFTAG) && buf != NULL) { 13927c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 13937c478bd9Sstevel@tonic-gate ((uint8_t *)buf)[size] = KMEM_REDZONE_BYTE; 13947c478bd9Sstevel@tonic-gate ((uint32_t *)btp)[1] = KMEM_SIZE_ENCODE(size); 13957c478bd9Sstevel@tonic-gate 13967c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) { 13977c478bd9Sstevel@tonic-gate KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, 13987c478bd9Sstevel@tonic-gate caller()); 13997c478bd9Sstevel@tonic-gate } 14007c478bd9Sstevel@tonic-gate } 14017c478bd9Sstevel@tonic-gate return (buf); 14027c478bd9Sstevel@tonic-gate } 14037c478bd9Sstevel@tonic-gate if (size == 0) 14047c478bd9Sstevel@tonic-gate return (NULL); 14057c478bd9Sstevel@tonic-gate buf = vmem_alloc(kmem_oversize_arena, size, kmflag & KM_VMFLAGS); 14067c478bd9Sstevel@tonic-gate if (buf == NULL) 14077c478bd9Sstevel@tonic-gate kmem_log_event(kmem_failure_log, NULL, NULL, (void *)size); 14087c478bd9Sstevel@tonic-gate return (buf); 14097c478bd9Sstevel@tonic-gate } 14107c478bd9Sstevel@tonic-gate 14117c478bd9Sstevel@tonic-gate void 14127c478bd9Sstevel@tonic-gate kmem_free(void *buf, size_t size) 14137c478bd9Sstevel@tonic-gate { 14147c478bd9Sstevel@tonic-gate size_t index = (size - 1) >> KMEM_ALIGN_SHIFT; 14157c478bd9Sstevel@tonic-gate 14167c478bd9Sstevel@tonic-gate if (index < KMEM_MAXBUF >> KMEM_ALIGN_SHIFT) { 14177c478bd9Sstevel@tonic-gate kmem_cache_t *cp = kmem_alloc_table[index]; 14187c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_BUFTAG) { 14197c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 14207c478bd9Sstevel@tonic-gate uint32_t *ip = (uint32_t *)btp; 14217c478bd9Sstevel@tonic-gate if (ip[1] != KMEM_SIZE_ENCODE(size)) { 14227c478bd9Sstevel@tonic-gate if (*(uint64_t *)buf == KMEM_FREE_PATTERN) { 14237c478bd9Sstevel@tonic-gate kmem_error(KMERR_DUPFREE, cp, buf); 14247c478bd9Sstevel@tonic-gate return; 14257c478bd9Sstevel@tonic-gate } 14267c478bd9Sstevel@tonic-gate if (KMEM_SIZE_VALID(ip[1])) { 14277c478bd9Sstevel@tonic-gate ip[0] = KMEM_SIZE_ENCODE(size); 14287c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADSIZE, cp, buf); 14297c478bd9Sstevel@tonic-gate } else { 14307c478bd9Sstevel@tonic-gate kmem_error(KMERR_REDZONE, cp, buf); 14317c478bd9Sstevel@tonic-gate } 14327c478bd9Sstevel@tonic-gate return; 14337c478bd9Sstevel@tonic-gate } 14347c478bd9Sstevel@tonic-gate if (((uint8_t *)buf)[size] != KMEM_REDZONE_BYTE) { 14357c478bd9Sstevel@tonic-gate kmem_error(KMERR_REDZONE, cp, buf); 14367c478bd9Sstevel@tonic-gate return; 14377c478bd9Sstevel@tonic-gate } 14387c478bd9Sstevel@tonic-gate btp->bt_redzone = KMEM_REDZONE_PATTERN; 14397c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) { 14407c478bd9Sstevel@tonic-gate KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, 14417c478bd9Sstevel@tonic-gate caller()); 14427c478bd9Sstevel@tonic-gate } 14437c478bd9Sstevel@tonic-gate } 14447c478bd9Sstevel@tonic-gate kmem_cache_free(cp, buf); 14457c478bd9Sstevel@tonic-gate } else { 14467c478bd9Sstevel@tonic-gate if (buf == NULL && size == 0) 14477c478bd9Sstevel@tonic-gate return; 14487c478bd9Sstevel@tonic-gate vmem_free(kmem_oversize_arena, buf, size); 14497c478bd9Sstevel@tonic-gate } 14507c478bd9Sstevel@tonic-gate } 14517c478bd9Sstevel@tonic-gate 14527c478bd9Sstevel@tonic-gate void * 14537c478bd9Sstevel@tonic-gate kmem_firewall_va_alloc(vmem_t *vmp, size_t size, int vmflag) 14547c478bd9Sstevel@tonic-gate { 14557c478bd9Sstevel@tonic-gate size_t realsize = size + vmp->vm_quantum; 14567c478bd9Sstevel@tonic-gate void *addr; 14577c478bd9Sstevel@tonic-gate 14587c478bd9Sstevel@tonic-gate /* 14597c478bd9Sstevel@tonic-gate * Annoying edge case: if 'size' is just shy of ULONG_MAX, adding 14607c478bd9Sstevel@tonic-gate * vm_quantum will cause integer wraparound. Check for this, and 14617c478bd9Sstevel@tonic-gate * blow off the firewall page in this case. Note that such a 14627c478bd9Sstevel@tonic-gate * giant allocation (the entire kernel address space) can never 14637c478bd9Sstevel@tonic-gate * be satisfied, so it will either fail immediately (VM_NOSLEEP) 14647c478bd9Sstevel@tonic-gate * or sleep forever (VM_SLEEP). Thus, there is no need for a 14657c478bd9Sstevel@tonic-gate * corresponding check in kmem_firewall_va_free(). 14667c478bd9Sstevel@tonic-gate */ 14677c478bd9Sstevel@tonic-gate if (realsize < size) 14687c478bd9Sstevel@tonic-gate realsize = size; 14697c478bd9Sstevel@tonic-gate 14707c478bd9Sstevel@tonic-gate /* 14717c478bd9Sstevel@tonic-gate * While boot still owns resource management, make sure that this 14727c478bd9Sstevel@tonic-gate * redzone virtual address allocation is properly accounted for in 14737c478bd9Sstevel@tonic-gate * OBPs "virtual-memory" "available" lists because we're 14747c478bd9Sstevel@tonic-gate * effectively claiming them for a red zone. If we don't do this, 14757c478bd9Sstevel@tonic-gate * the available lists become too fragmented and too large for the 14767c478bd9Sstevel@tonic-gate * current boot/kernel memory list interface. 14777c478bd9Sstevel@tonic-gate */ 14787c478bd9Sstevel@tonic-gate addr = vmem_alloc(vmp, realsize, vmflag | VM_NEXTFIT); 14797c478bd9Sstevel@tonic-gate 14807c478bd9Sstevel@tonic-gate if (addr != NULL && kvseg.s_base == NULL && realsize != size) 14817c478bd9Sstevel@tonic-gate (void) boot_virt_alloc((char *)addr + size, vmp->vm_quantum); 14827c478bd9Sstevel@tonic-gate 14837c478bd9Sstevel@tonic-gate return (addr); 14847c478bd9Sstevel@tonic-gate } 14857c478bd9Sstevel@tonic-gate 14867c478bd9Sstevel@tonic-gate void 14877c478bd9Sstevel@tonic-gate kmem_firewall_va_free(vmem_t *vmp, void *addr, size_t size) 14887c478bd9Sstevel@tonic-gate { 14897c478bd9Sstevel@tonic-gate ASSERT((kvseg.s_base == NULL ? 14907c478bd9Sstevel@tonic-gate va_to_pfn((char *)addr + size) : 14917c478bd9Sstevel@tonic-gate hat_getpfnum(kas.a_hat, (caddr_t)addr + size)) == PFN_INVALID); 14927c478bd9Sstevel@tonic-gate 14937c478bd9Sstevel@tonic-gate vmem_free(vmp, addr, size + vmp->vm_quantum); 14947c478bd9Sstevel@tonic-gate } 14957c478bd9Sstevel@tonic-gate 14967c478bd9Sstevel@tonic-gate /* 14977c478bd9Sstevel@tonic-gate * Try to allocate at least `size' bytes of memory without sleeping or 14987c478bd9Sstevel@tonic-gate * panicking. Return actual allocated size in `asize'. If allocation failed, 14997c478bd9Sstevel@tonic-gate * try final allocation with sleep or panic allowed. 15007c478bd9Sstevel@tonic-gate */ 15017c478bd9Sstevel@tonic-gate void * 15027c478bd9Sstevel@tonic-gate kmem_alloc_tryhard(size_t size, size_t *asize, int kmflag) 15037c478bd9Sstevel@tonic-gate { 15047c478bd9Sstevel@tonic-gate void *p; 15057c478bd9Sstevel@tonic-gate 15067c478bd9Sstevel@tonic-gate *asize = P2ROUNDUP(size, KMEM_ALIGN); 15077c478bd9Sstevel@tonic-gate do { 15087c478bd9Sstevel@tonic-gate p = kmem_alloc(*asize, (kmflag | KM_NOSLEEP) & ~KM_PANIC); 15097c478bd9Sstevel@tonic-gate if (p != NULL) 15107c478bd9Sstevel@tonic-gate return (p); 15117c478bd9Sstevel@tonic-gate *asize += KMEM_ALIGN; 15127c478bd9Sstevel@tonic-gate } while (*asize <= PAGESIZE); 15137c478bd9Sstevel@tonic-gate 15147c478bd9Sstevel@tonic-gate *asize = P2ROUNDUP(size, KMEM_ALIGN); 15157c478bd9Sstevel@tonic-gate return (kmem_alloc(*asize, kmflag)); 15167c478bd9Sstevel@tonic-gate } 15177c478bd9Sstevel@tonic-gate 15187c478bd9Sstevel@tonic-gate /* 15197c478bd9Sstevel@tonic-gate * Reclaim all unused memory from a cache. 15207c478bd9Sstevel@tonic-gate */ 15217c478bd9Sstevel@tonic-gate static void 15227c478bd9Sstevel@tonic-gate kmem_cache_reap(kmem_cache_t *cp) 15237c478bd9Sstevel@tonic-gate { 15247c478bd9Sstevel@tonic-gate /* 15257c478bd9Sstevel@tonic-gate * Ask the cache's owner to free some memory if possible. 15267c478bd9Sstevel@tonic-gate * The idea is to handle things like the inode cache, which 15277c478bd9Sstevel@tonic-gate * typically sits on a bunch of memory that it doesn't truly 15287c478bd9Sstevel@tonic-gate * *need*. Reclaim policy is entirely up to the owner; this 15297c478bd9Sstevel@tonic-gate * callback is just an advisory plea for help. 15307c478bd9Sstevel@tonic-gate */ 15317c478bd9Sstevel@tonic-gate if (cp->cache_reclaim != NULL) 15327c478bd9Sstevel@tonic-gate cp->cache_reclaim(cp->cache_private); 15337c478bd9Sstevel@tonic-gate 15347c478bd9Sstevel@tonic-gate kmem_depot_ws_reap(cp); 15357c478bd9Sstevel@tonic-gate } 15367c478bd9Sstevel@tonic-gate 15377c478bd9Sstevel@tonic-gate static void 15387c478bd9Sstevel@tonic-gate kmem_reap_timeout(void *flag_arg) 15397c478bd9Sstevel@tonic-gate { 15407c478bd9Sstevel@tonic-gate uint32_t *flag = (uint32_t *)flag_arg; 15417c478bd9Sstevel@tonic-gate 15427c478bd9Sstevel@tonic-gate ASSERT(flag == &kmem_reaping || flag == &kmem_reaping_idspace); 15437c478bd9Sstevel@tonic-gate *flag = 0; 15447c478bd9Sstevel@tonic-gate } 15457c478bd9Sstevel@tonic-gate 15467c478bd9Sstevel@tonic-gate static void 15477c478bd9Sstevel@tonic-gate kmem_reap_done(void *flag) 15487c478bd9Sstevel@tonic-gate { 15497c478bd9Sstevel@tonic-gate (void) timeout(kmem_reap_timeout, flag, kmem_reap_interval); 15507c478bd9Sstevel@tonic-gate } 15517c478bd9Sstevel@tonic-gate 15527c478bd9Sstevel@tonic-gate static void 15537c478bd9Sstevel@tonic-gate kmem_reap_start(void *flag) 15547c478bd9Sstevel@tonic-gate { 15557c478bd9Sstevel@tonic-gate ASSERT(flag == &kmem_reaping || flag == &kmem_reaping_idspace); 15567c478bd9Sstevel@tonic-gate 15577c478bd9Sstevel@tonic-gate if (flag == &kmem_reaping) { 15587c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_reap, kmem_taskq, TQ_NOSLEEP); 15597c478bd9Sstevel@tonic-gate /* 15607c478bd9Sstevel@tonic-gate * if we have segkp under heap, reap segkp cache. 15617c478bd9Sstevel@tonic-gate */ 15627c478bd9Sstevel@tonic-gate if (segkp_fromheap) 15637c478bd9Sstevel@tonic-gate segkp_cache_free(); 15647c478bd9Sstevel@tonic-gate } 15657c478bd9Sstevel@tonic-gate else 15667c478bd9Sstevel@tonic-gate kmem_cache_applyall_id(kmem_cache_reap, kmem_taskq, TQ_NOSLEEP); 15677c478bd9Sstevel@tonic-gate 15687c478bd9Sstevel@tonic-gate /* 15697c478bd9Sstevel@tonic-gate * We use taskq_dispatch() to schedule a timeout to clear 15707c478bd9Sstevel@tonic-gate * the flag so that kmem_reap() becomes self-throttling: 15717c478bd9Sstevel@tonic-gate * we won't reap again until the current reap completes *and* 15727c478bd9Sstevel@tonic-gate * at least kmem_reap_interval ticks have elapsed. 15737c478bd9Sstevel@tonic-gate */ 15747c478bd9Sstevel@tonic-gate if (!taskq_dispatch(kmem_taskq, kmem_reap_done, flag, TQ_NOSLEEP)) 15757c478bd9Sstevel@tonic-gate kmem_reap_done(flag); 15767c478bd9Sstevel@tonic-gate } 15777c478bd9Sstevel@tonic-gate 15787c478bd9Sstevel@tonic-gate static void 15797c478bd9Sstevel@tonic-gate kmem_reap_common(void *flag_arg) 15807c478bd9Sstevel@tonic-gate { 15817c478bd9Sstevel@tonic-gate uint32_t *flag = (uint32_t *)flag_arg; 15827c478bd9Sstevel@tonic-gate 15837c478bd9Sstevel@tonic-gate if (MUTEX_HELD(&kmem_cache_lock) || kmem_taskq == NULL || 15847c478bd9Sstevel@tonic-gate cas32(flag, 0, 1) != 0) 15857c478bd9Sstevel@tonic-gate return; 15867c478bd9Sstevel@tonic-gate 15877c478bd9Sstevel@tonic-gate /* 15887c478bd9Sstevel@tonic-gate * It may not be kosher to do memory allocation when a reap is called 15897c478bd9Sstevel@tonic-gate * is called (for example, if vmem_populate() is in the call chain). 15907c478bd9Sstevel@tonic-gate * So we start the reap going with a TQ_NOALLOC dispatch. If the 15917c478bd9Sstevel@tonic-gate * dispatch fails, we reset the flag, and the next reap will try again. 15927c478bd9Sstevel@tonic-gate */ 15937c478bd9Sstevel@tonic-gate if (!taskq_dispatch(kmem_taskq, kmem_reap_start, flag, TQ_NOALLOC)) 15947c478bd9Sstevel@tonic-gate *flag = 0; 15957c478bd9Sstevel@tonic-gate } 15967c478bd9Sstevel@tonic-gate 15977c478bd9Sstevel@tonic-gate /* 15987c478bd9Sstevel@tonic-gate * Reclaim all unused memory from all caches. Called from the VM system 15997c478bd9Sstevel@tonic-gate * when memory gets tight. 16007c478bd9Sstevel@tonic-gate */ 16017c478bd9Sstevel@tonic-gate void 16027c478bd9Sstevel@tonic-gate kmem_reap(void) 16037c478bd9Sstevel@tonic-gate { 16047c478bd9Sstevel@tonic-gate kmem_reap_common(&kmem_reaping); 16057c478bd9Sstevel@tonic-gate } 16067c478bd9Sstevel@tonic-gate 16077c478bd9Sstevel@tonic-gate /* 16087c478bd9Sstevel@tonic-gate * Reclaim all unused memory from identifier arenas, called when a vmem 16097c478bd9Sstevel@tonic-gate * arena not back by memory is exhausted. Since reaping memory-backed caches 16107c478bd9Sstevel@tonic-gate * cannot help with identifier exhaustion, we avoid both a large amount of 16117c478bd9Sstevel@tonic-gate * work and unwanted side-effects from reclaim callbacks. 16127c478bd9Sstevel@tonic-gate */ 16137c478bd9Sstevel@tonic-gate void 16147c478bd9Sstevel@tonic-gate kmem_reap_idspace(void) 16157c478bd9Sstevel@tonic-gate { 16167c478bd9Sstevel@tonic-gate kmem_reap_common(&kmem_reaping_idspace); 16177c478bd9Sstevel@tonic-gate } 16187c478bd9Sstevel@tonic-gate 16197c478bd9Sstevel@tonic-gate /* 16207c478bd9Sstevel@tonic-gate * Purge all magazines from a cache and set its magazine limit to zero. 16217c478bd9Sstevel@tonic-gate * All calls are serialized by the kmem_taskq lock, except for the final 16227c478bd9Sstevel@tonic-gate * call from kmem_cache_destroy(). 16237c478bd9Sstevel@tonic-gate */ 16247c478bd9Sstevel@tonic-gate static void 16257c478bd9Sstevel@tonic-gate kmem_cache_magazine_purge(kmem_cache_t *cp) 16267c478bd9Sstevel@tonic-gate { 16277c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp; 16287c478bd9Sstevel@tonic-gate kmem_magazine_t *mp, *pmp; 16297c478bd9Sstevel@tonic-gate int rounds, prounds, cpu_seqid; 16307c478bd9Sstevel@tonic-gate 16317c478bd9Sstevel@tonic-gate ASSERT(cp->cache_next == NULL || taskq_member(kmem_taskq, curthread)); 16327c478bd9Sstevel@tonic-gate ASSERT(MUTEX_NOT_HELD(&cp->cache_lock)); 16337c478bd9Sstevel@tonic-gate 16347c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 16357c478bd9Sstevel@tonic-gate ccp = &cp->cache_cpu[cpu_seqid]; 16367c478bd9Sstevel@tonic-gate 16377c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 16387c478bd9Sstevel@tonic-gate mp = ccp->cc_loaded; 16397c478bd9Sstevel@tonic-gate pmp = ccp->cc_ploaded; 16407c478bd9Sstevel@tonic-gate rounds = ccp->cc_rounds; 16417c478bd9Sstevel@tonic-gate prounds = ccp->cc_prounds; 16427c478bd9Sstevel@tonic-gate ccp->cc_loaded = NULL; 16437c478bd9Sstevel@tonic-gate ccp->cc_ploaded = NULL; 16447c478bd9Sstevel@tonic-gate ccp->cc_rounds = -1; 16457c478bd9Sstevel@tonic-gate ccp->cc_prounds = -1; 16467c478bd9Sstevel@tonic-gate ccp->cc_magsize = 0; 16477c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 16487c478bd9Sstevel@tonic-gate 16497c478bd9Sstevel@tonic-gate if (mp) 16507c478bd9Sstevel@tonic-gate kmem_magazine_destroy(cp, mp, rounds); 16517c478bd9Sstevel@tonic-gate if (pmp) 16527c478bd9Sstevel@tonic-gate kmem_magazine_destroy(cp, pmp, prounds); 16537c478bd9Sstevel@tonic-gate } 16547c478bd9Sstevel@tonic-gate 16557c478bd9Sstevel@tonic-gate /* 16567c478bd9Sstevel@tonic-gate * Updating the working set statistics twice in a row has the 16577c478bd9Sstevel@tonic-gate * effect of setting the working set size to zero, so everything 16587c478bd9Sstevel@tonic-gate * is eligible for reaping. 16597c478bd9Sstevel@tonic-gate */ 16607c478bd9Sstevel@tonic-gate kmem_depot_ws_update(cp); 16617c478bd9Sstevel@tonic-gate kmem_depot_ws_update(cp); 16627c478bd9Sstevel@tonic-gate 16637c478bd9Sstevel@tonic-gate kmem_depot_ws_reap(cp); 16647c478bd9Sstevel@tonic-gate } 16657c478bd9Sstevel@tonic-gate 16667c478bd9Sstevel@tonic-gate /* 16677c478bd9Sstevel@tonic-gate * Enable per-cpu magazines on a cache. 16687c478bd9Sstevel@tonic-gate */ 16697c478bd9Sstevel@tonic-gate static void 16707c478bd9Sstevel@tonic-gate kmem_cache_magazine_enable(kmem_cache_t *cp) 16717c478bd9Sstevel@tonic-gate { 16727c478bd9Sstevel@tonic-gate int cpu_seqid; 16737c478bd9Sstevel@tonic-gate 16747c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_NOMAGAZINE) 16757c478bd9Sstevel@tonic-gate return; 16767c478bd9Sstevel@tonic-gate 16777c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 16787c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 16797c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 16807c478bd9Sstevel@tonic-gate ccp->cc_magsize = cp->cache_magtype->mt_magsize; 16817c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 16827c478bd9Sstevel@tonic-gate } 16837c478bd9Sstevel@tonic-gate 16847c478bd9Sstevel@tonic-gate } 16857c478bd9Sstevel@tonic-gate 1686fa9e4066Sahrens /* 1687fa9e4066Sahrens * Reap (almost) everything right now. See kmem_cache_magazine_purge() 1688fa9e4066Sahrens * for explanation of the back-to-back kmem_depot_ws_update() calls. 1689fa9e4066Sahrens */ 1690fa9e4066Sahrens void 1691fa9e4066Sahrens kmem_cache_reap_now(kmem_cache_t *cp) 1692fa9e4066Sahrens { 1693fa9e4066Sahrens kmem_depot_ws_update(cp); 1694fa9e4066Sahrens kmem_depot_ws_update(cp); 1695fa9e4066Sahrens 1696fa9e4066Sahrens (void) taskq_dispatch(kmem_taskq, 1697fa9e4066Sahrens (task_func_t *)kmem_depot_ws_reap, cp, TQ_SLEEP); 1698fa9e4066Sahrens taskq_wait(kmem_taskq); 1699fa9e4066Sahrens } 1700fa9e4066Sahrens 17017c478bd9Sstevel@tonic-gate /* 17027c478bd9Sstevel@tonic-gate * Recompute a cache's magazine size. The trade-off is that larger magazines 17037c478bd9Sstevel@tonic-gate * provide a higher transfer rate with the depot, while smaller magazines 17047c478bd9Sstevel@tonic-gate * reduce memory consumption. Magazine resizing is an expensive operation; 17057c478bd9Sstevel@tonic-gate * it should not be done frequently. 17067c478bd9Sstevel@tonic-gate * 17077c478bd9Sstevel@tonic-gate * Changes to the magazine size are serialized by the kmem_taskq lock. 17087c478bd9Sstevel@tonic-gate * 17097c478bd9Sstevel@tonic-gate * Note: at present this only grows the magazine size. It might be useful 17107c478bd9Sstevel@tonic-gate * to allow shrinkage too. 17117c478bd9Sstevel@tonic-gate */ 17127c478bd9Sstevel@tonic-gate static void 17137c478bd9Sstevel@tonic-gate kmem_cache_magazine_resize(kmem_cache_t *cp) 17147c478bd9Sstevel@tonic-gate { 17157c478bd9Sstevel@tonic-gate kmem_magtype_t *mtp = cp->cache_magtype; 17167c478bd9Sstevel@tonic-gate 17177c478bd9Sstevel@tonic-gate ASSERT(taskq_member(kmem_taskq, curthread)); 17187c478bd9Sstevel@tonic-gate 17197c478bd9Sstevel@tonic-gate if (cp->cache_chunksize < mtp->mt_maxbuf) { 17207c478bd9Sstevel@tonic-gate kmem_cache_magazine_purge(cp); 17217c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock); 17227c478bd9Sstevel@tonic-gate cp->cache_magtype = ++mtp; 17237c478bd9Sstevel@tonic-gate cp->cache_depot_contention_prev = 17247c478bd9Sstevel@tonic-gate cp->cache_depot_contention + INT_MAX; 17257c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock); 17267c478bd9Sstevel@tonic-gate kmem_cache_magazine_enable(cp); 17277c478bd9Sstevel@tonic-gate } 17287c478bd9Sstevel@tonic-gate } 17297c478bd9Sstevel@tonic-gate 17307c478bd9Sstevel@tonic-gate /* 17317c478bd9Sstevel@tonic-gate * Rescale a cache's hash table, so that the table size is roughly the 17327c478bd9Sstevel@tonic-gate * cache size. We want the average lookup time to be extremely small. 17337c478bd9Sstevel@tonic-gate */ 17347c478bd9Sstevel@tonic-gate static void 17357c478bd9Sstevel@tonic-gate kmem_hash_rescale(kmem_cache_t *cp) 17367c478bd9Sstevel@tonic-gate { 17377c478bd9Sstevel@tonic-gate kmem_bufctl_t **old_table, **new_table, *bcp; 17387c478bd9Sstevel@tonic-gate size_t old_size, new_size, h; 17397c478bd9Sstevel@tonic-gate 17407c478bd9Sstevel@tonic-gate ASSERT(taskq_member(kmem_taskq, curthread)); 17417c478bd9Sstevel@tonic-gate 17427c478bd9Sstevel@tonic-gate new_size = MAX(KMEM_HASH_INITIAL, 17437c478bd9Sstevel@tonic-gate 1 << (highbit(3 * cp->cache_buftotal + 4) - 2)); 17447c478bd9Sstevel@tonic-gate old_size = cp->cache_hash_mask + 1; 17457c478bd9Sstevel@tonic-gate 17467c478bd9Sstevel@tonic-gate if ((old_size >> 1) <= new_size && new_size <= (old_size << 1)) 17477c478bd9Sstevel@tonic-gate return; 17487c478bd9Sstevel@tonic-gate 17497c478bd9Sstevel@tonic-gate new_table = vmem_alloc(kmem_hash_arena, new_size * sizeof (void *), 17507c478bd9Sstevel@tonic-gate VM_NOSLEEP); 17517c478bd9Sstevel@tonic-gate if (new_table == NULL) 17527c478bd9Sstevel@tonic-gate return; 17537c478bd9Sstevel@tonic-gate bzero(new_table, new_size * sizeof (void *)); 17547c478bd9Sstevel@tonic-gate 17557c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 17567c478bd9Sstevel@tonic-gate 17577c478bd9Sstevel@tonic-gate old_size = cp->cache_hash_mask + 1; 17587c478bd9Sstevel@tonic-gate old_table = cp->cache_hash_table; 17597c478bd9Sstevel@tonic-gate 17607c478bd9Sstevel@tonic-gate cp->cache_hash_mask = new_size - 1; 17617c478bd9Sstevel@tonic-gate cp->cache_hash_table = new_table; 17627c478bd9Sstevel@tonic-gate cp->cache_rescale++; 17637c478bd9Sstevel@tonic-gate 17647c478bd9Sstevel@tonic-gate for (h = 0; h < old_size; h++) { 17657c478bd9Sstevel@tonic-gate bcp = old_table[h]; 17667c478bd9Sstevel@tonic-gate while (bcp != NULL) { 17677c478bd9Sstevel@tonic-gate void *addr = bcp->bc_addr; 17687c478bd9Sstevel@tonic-gate kmem_bufctl_t *next_bcp = bcp->bc_next; 17697c478bd9Sstevel@tonic-gate kmem_bufctl_t **hash_bucket = KMEM_HASH(cp, addr); 17707c478bd9Sstevel@tonic-gate bcp->bc_next = *hash_bucket; 17717c478bd9Sstevel@tonic-gate *hash_bucket = bcp; 17727c478bd9Sstevel@tonic-gate bcp = next_bcp; 17737c478bd9Sstevel@tonic-gate } 17747c478bd9Sstevel@tonic-gate } 17757c478bd9Sstevel@tonic-gate 17767c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 17777c478bd9Sstevel@tonic-gate 17787c478bd9Sstevel@tonic-gate vmem_free(kmem_hash_arena, old_table, old_size * sizeof (void *)); 17797c478bd9Sstevel@tonic-gate } 17807c478bd9Sstevel@tonic-gate 17817c478bd9Sstevel@tonic-gate /* 17827c478bd9Sstevel@tonic-gate * Perform periodic maintenance on a cache: hash rescaling, 17837c478bd9Sstevel@tonic-gate * depot working-set update, and magazine resizing. 17847c478bd9Sstevel@tonic-gate */ 17857c478bd9Sstevel@tonic-gate static void 17867c478bd9Sstevel@tonic-gate kmem_cache_update(kmem_cache_t *cp) 17877c478bd9Sstevel@tonic-gate { 17887c478bd9Sstevel@tonic-gate int need_hash_rescale = 0; 17897c478bd9Sstevel@tonic-gate int need_magazine_resize = 0; 17907c478bd9Sstevel@tonic-gate 17917c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&kmem_cache_lock)); 17927c478bd9Sstevel@tonic-gate 17937c478bd9Sstevel@tonic-gate /* 17947c478bd9Sstevel@tonic-gate * If the cache has become much larger or smaller than its hash table, 17957c478bd9Sstevel@tonic-gate * fire off a request to rescale the hash table. 17967c478bd9Sstevel@tonic-gate */ 17977c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 17987c478bd9Sstevel@tonic-gate 17997c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_HASH) && 18007c478bd9Sstevel@tonic-gate (cp->cache_buftotal > (cp->cache_hash_mask << 1) || 18017c478bd9Sstevel@tonic-gate (cp->cache_buftotal < (cp->cache_hash_mask >> 1) && 18027c478bd9Sstevel@tonic-gate cp->cache_hash_mask > KMEM_HASH_INITIAL))) 18037c478bd9Sstevel@tonic-gate need_hash_rescale = 1; 18047c478bd9Sstevel@tonic-gate 18057c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 18067c478bd9Sstevel@tonic-gate 18077c478bd9Sstevel@tonic-gate /* 18087c478bd9Sstevel@tonic-gate * Update the depot working set statistics. 18097c478bd9Sstevel@tonic-gate */ 18107c478bd9Sstevel@tonic-gate kmem_depot_ws_update(cp); 18117c478bd9Sstevel@tonic-gate 18127c478bd9Sstevel@tonic-gate /* 18137c478bd9Sstevel@tonic-gate * If there's a lot of contention in the depot, 18147c478bd9Sstevel@tonic-gate * increase the magazine size. 18157c478bd9Sstevel@tonic-gate */ 18167c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock); 18177c478bd9Sstevel@tonic-gate 18187c478bd9Sstevel@tonic-gate if (cp->cache_chunksize < cp->cache_magtype->mt_maxbuf && 18197c478bd9Sstevel@tonic-gate (int)(cp->cache_depot_contention - 18207c478bd9Sstevel@tonic-gate cp->cache_depot_contention_prev) > kmem_depot_contention) 18217c478bd9Sstevel@tonic-gate need_magazine_resize = 1; 18227c478bd9Sstevel@tonic-gate 18237c478bd9Sstevel@tonic-gate cp->cache_depot_contention_prev = cp->cache_depot_contention; 18247c478bd9Sstevel@tonic-gate 18257c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock); 18267c478bd9Sstevel@tonic-gate 18277c478bd9Sstevel@tonic-gate if (need_hash_rescale) 18287c478bd9Sstevel@tonic-gate (void) taskq_dispatch(kmem_taskq, 18297c478bd9Sstevel@tonic-gate (task_func_t *)kmem_hash_rescale, cp, TQ_NOSLEEP); 18307c478bd9Sstevel@tonic-gate 18317c478bd9Sstevel@tonic-gate if (need_magazine_resize) 18327c478bd9Sstevel@tonic-gate (void) taskq_dispatch(kmem_taskq, 18337c478bd9Sstevel@tonic-gate (task_func_t *)kmem_cache_magazine_resize, cp, TQ_NOSLEEP); 18347c478bd9Sstevel@tonic-gate } 18357c478bd9Sstevel@tonic-gate 18367c478bd9Sstevel@tonic-gate static void 18377c478bd9Sstevel@tonic-gate kmem_update_timeout(void *dummy) 18387c478bd9Sstevel@tonic-gate { 18397c478bd9Sstevel@tonic-gate static void kmem_update(void *); 18407c478bd9Sstevel@tonic-gate 18417c478bd9Sstevel@tonic-gate (void) timeout(kmem_update, dummy, kmem_reap_interval); 18427c478bd9Sstevel@tonic-gate } 18437c478bd9Sstevel@tonic-gate 18447c478bd9Sstevel@tonic-gate static void 18457c478bd9Sstevel@tonic-gate kmem_update(void *dummy) 18467c478bd9Sstevel@tonic-gate { 18477c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_update, NULL, TQ_NOSLEEP); 18487c478bd9Sstevel@tonic-gate 18497c478bd9Sstevel@tonic-gate /* 18507c478bd9Sstevel@tonic-gate * We use taskq_dispatch() to reschedule the timeout so that 18517c478bd9Sstevel@tonic-gate * kmem_update() becomes self-throttling: it won't schedule 18527c478bd9Sstevel@tonic-gate * new tasks until all previous tasks have completed. 18537c478bd9Sstevel@tonic-gate */ 18547c478bd9Sstevel@tonic-gate if (!taskq_dispatch(kmem_taskq, kmem_update_timeout, dummy, TQ_NOSLEEP)) 18557c478bd9Sstevel@tonic-gate kmem_update_timeout(NULL); 18567c478bd9Sstevel@tonic-gate } 18577c478bd9Sstevel@tonic-gate 18587c478bd9Sstevel@tonic-gate static int 18597c478bd9Sstevel@tonic-gate kmem_cache_kstat_update(kstat_t *ksp, int rw) 18607c478bd9Sstevel@tonic-gate { 18617c478bd9Sstevel@tonic-gate struct kmem_cache_kstat *kmcp = &kmem_cache_kstat; 18627c478bd9Sstevel@tonic-gate kmem_cache_t *cp = ksp->ks_private; 18637c478bd9Sstevel@tonic-gate kmem_slab_t *sp; 18647c478bd9Sstevel@tonic-gate uint64_t cpu_buf_avail; 18657c478bd9Sstevel@tonic-gate uint64_t buf_avail = 0; 18667c478bd9Sstevel@tonic-gate int cpu_seqid; 18677c478bd9Sstevel@tonic-gate 18687c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&kmem_cache_kstat_lock)); 18697c478bd9Sstevel@tonic-gate 18707c478bd9Sstevel@tonic-gate if (rw == KSTAT_WRITE) 18717c478bd9Sstevel@tonic-gate return (EACCES); 18727c478bd9Sstevel@tonic-gate 18737c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 18747c478bd9Sstevel@tonic-gate 18757c478bd9Sstevel@tonic-gate kmcp->kmc_alloc_fail.value.ui64 = cp->cache_alloc_fail; 18767c478bd9Sstevel@tonic-gate kmcp->kmc_alloc.value.ui64 = cp->cache_slab_alloc; 18777c478bd9Sstevel@tonic-gate kmcp->kmc_free.value.ui64 = cp->cache_slab_free; 18787c478bd9Sstevel@tonic-gate kmcp->kmc_slab_alloc.value.ui64 = cp->cache_slab_alloc; 18797c478bd9Sstevel@tonic-gate kmcp->kmc_slab_free.value.ui64 = cp->cache_slab_free; 18807c478bd9Sstevel@tonic-gate 18817c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 18827c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 18837c478bd9Sstevel@tonic-gate 18847c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 18857c478bd9Sstevel@tonic-gate 18867c478bd9Sstevel@tonic-gate cpu_buf_avail = 0; 18877c478bd9Sstevel@tonic-gate if (ccp->cc_rounds > 0) 18887c478bd9Sstevel@tonic-gate cpu_buf_avail += ccp->cc_rounds; 18897c478bd9Sstevel@tonic-gate if (ccp->cc_prounds > 0) 18907c478bd9Sstevel@tonic-gate cpu_buf_avail += ccp->cc_prounds; 18917c478bd9Sstevel@tonic-gate 18927c478bd9Sstevel@tonic-gate kmcp->kmc_alloc.value.ui64 += ccp->cc_alloc; 18937c478bd9Sstevel@tonic-gate kmcp->kmc_free.value.ui64 += ccp->cc_free; 18947c478bd9Sstevel@tonic-gate buf_avail += cpu_buf_avail; 18957c478bd9Sstevel@tonic-gate 18967c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 18977c478bd9Sstevel@tonic-gate } 18987c478bd9Sstevel@tonic-gate 18997c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock); 19007c478bd9Sstevel@tonic-gate 19017c478bd9Sstevel@tonic-gate kmcp->kmc_depot_alloc.value.ui64 = cp->cache_full.ml_alloc; 19027c478bd9Sstevel@tonic-gate kmcp->kmc_depot_free.value.ui64 = cp->cache_empty.ml_alloc; 19037c478bd9Sstevel@tonic-gate kmcp->kmc_depot_contention.value.ui64 = cp->cache_depot_contention; 19047c478bd9Sstevel@tonic-gate kmcp->kmc_full_magazines.value.ui64 = cp->cache_full.ml_total; 19057c478bd9Sstevel@tonic-gate kmcp->kmc_empty_magazines.value.ui64 = cp->cache_empty.ml_total; 19067c478bd9Sstevel@tonic-gate kmcp->kmc_magazine_size.value.ui64 = 19077c478bd9Sstevel@tonic-gate (cp->cache_flags & KMF_NOMAGAZINE) ? 19087c478bd9Sstevel@tonic-gate 0 : cp->cache_magtype->mt_magsize; 19097c478bd9Sstevel@tonic-gate 19107c478bd9Sstevel@tonic-gate kmcp->kmc_alloc.value.ui64 += cp->cache_full.ml_alloc; 19117c478bd9Sstevel@tonic-gate kmcp->kmc_free.value.ui64 += cp->cache_empty.ml_alloc; 19127c478bd9Sstevel@tonic-gate buf_avail += cp->cache_full.ml_total * cp->cache_magtype->mt_magsize; 19137c478bd9Sstevel@tonic-gate 19147c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock); 19157c478bd9Sstevel@tonic-gate 19167c478bd9Sstevel@tonic-gate kmcp->kmc_buf_size.value.ui64 = cp->cache_bufsize; 19177c478bd9Sstevel@tonic-gate kmcp->kmc_align.value.ui64 = cp->cache_align; 19187c478bd9Sstevel@tonic-gate kmcp->kmc_chunk_size.value.ui64 = cp->cache_chunksize; 19197c478bd9Sstevel@tonic-gate kmcp->kmc_slab_size.value.ui64 = cp->cache_slabsize; 19207c478bd9Sstevel@tonic-gate kmcp->kmc_buf_constructed.value.ui64 = buf_avail; 19217c478bd9Sstevel@tonic-gate for (sp = cp->cache_freelist; sp != &cp->cache_nullslab; 19227c478bd9Sstevel@tonic-gate sp = sp->slab_next) 19237c478bd9Sstevel@tonic-gate buf_avail += sp->slab_chunks - sp->slab_refcnt; 19247c478bd9Sstevel@tonic-gate kmcp->kmc_buf_avail.value.ui64 = buf_avail; 19257c478bd9Sstevel@tonic-gate kmcp->kmc_buf_inuse.value.ui64 = cp->cache_buftotal - buf_avail; 19267c478bd9Sstevel@tonic-gate kmcp->kmc_buf_total.value.ui64 = cp->cache_buftotal; 19277c478bd9Sstevel@tonic-gate kmcp->kmc_buf_max.value.ui64 = cp->cache_bufmax; 19287c478bd9Sstevel@tonic-gate kmcp->kmc_slab_create.value.ui64 = cp->cache_slab_create; 19297c478bd9Sstevel@tonic-gate kmcp->kmc_slab_destroy.value.ui64 = cp->cache_slab_destroy; 19307c478bd9Sstevel@tonic-gate kmcp->kmc_hash_size.value.ui64 = (cp->cache_flags & KMF_HASH) ? 19317c478bd9Sstevel@tonic-gate cp->cache_hash_mask + 1 : 0; 19327c478bd9Sstevel@tonic-gate kmcp->kmc_hash_lookup_depth.value.ui64 = cp->cache_lookup_depth; 19337c478bd9Sstevel@tonic-gate kmcp->kmc_hash_rescale.value.ui64 = cp->cache_rescale; 19347c478bd9Sstevel@tonic-gate kmcp->kmc_vmem_source.value.ui64 = cp->cache_arena->vm_id; 19357c478bd9Sstevel@tonic-gate 19367c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 19377c478bd9Sstevel@tonic-gate return (0); 19387c478bd9Sstevel@tonic-gate } 19397c478bd9Sstevel@tonic-gate 19407c478bd9Sstevel@tonic-gate /* 19417c478bd9Sstevel@tonic-gate * Return a named statistic about a particular cache. 19427c478bd9Sstevel@tonic-gate * This shouldn't be called very often, so it's currently designed for 19437c478bd9Sstevel@tonic-gate * simplicity (leverages existing kstat support) rather than efficiency. 19447c478bd9Sstevel@tonic-gate */ 19457c478bd9Sstevel@tonic-gate uint64_t 19467c478bd9Sstevel@tonic-gate kmem_cache_stat(kmem_cache_t *cp, char *name) 19477c478bd9Sstevel@tonic-gate { 19487c478bd9Sstevel@tonic-gate int i; 19497c478bd9Sstevel@tonic-gate kstat_t *ksp = cp->cache_kstat; 19507c478bd9Sstevel@tonic-gate kstat_named_t *knp = (kstat_named_t *)&kmem_cache_kstat; 19517c478bd9Sstevel@tonic-gate uint64_t value = 0; 19527c478bd9Sstevel@tonic-gate 19537c478bd9Sstevel@tonic-gate if (ksp != NULL) { 19547c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_kstat_lock); 19557c478bd9Sstevel@tonic-gate (void) kmem_cache_kstat_update(ksp, KSTAT_READ); 19567c478bd9Sstevel@tonic-gate for (i = 0; i < ksp->ks_ndata; i++) { 19577c478bd9Sstevel@tonic-gate if (strcmp(knp[i].name, name) == 0) { 19587c478bd9Sstevel@tonic-gate value = knp[i].value.ui64; 19597c478bd9Sstevel@tonic-gate break; 19607c478bd9Sstevel@tonic-gate } 19617c478bd9Sstevel@tonic-gate } 19627c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_kstat_lock); 19637c478bd9Sstevel@tonic-gate } 19647c478bd9Sstevel@tonic-gate return (value); 19657c478bd9Sstevel@tonic-gate } 19667c478bd9Sstevel@tonic-gate 19677c478bd9Sstevel@tonic-gate /* 19687c478bd9Sstevel@tonic-gate * Return an estimate of currently available kernel heap memory. 19697c478bd9Sstevel@tonic-gate * On 32-bit systems, physical memory may exceed virtual memory, 19707c478bd9Sstevel@tonic-gate * we just truncate the result at 1GB. 19717c478bd9Sstevel@tonic-gate */ 19727c478bd9Sstevel@tonic-gate size_t 19737c478bd9Sstevel@tonic-gate kmem_avail(void) 19747c478bd9Sstevel@tonic-gate { 19757c478bd9Sstevel@tonic-gate spgcnt_t rmem = availrmem - tune.t_minarmem; 19767c478bd9Sstevel@tonic-gate spgcnt_t fmem = freemem - minfree; 19777c478bd9Sstevel@tonic-gate 19787c478bd9Sstevel@tonic-gate return ((size_t)ptob(MIN(MAX(MIN(rmem, fmem), 0), 19797c478bd9Sstevel@tonic-gate 1 << (30 - PAGESHIFT)))); 19807c478bd9Sstevel@tonic-gate } 19817c478bd9Sstevel@tonic-gate 19827c478bd9Sstevel@tonic-gate /* 19837c478bd9Sstevel@tonic-gate * Return the maximum amount of memory that is (in theory) allocatable 19847c478bd9Sstevel@tonic-gate * from the heap. This may be used as an estimate only since there 19857c478bd9Sstevel@tonic-gate * is no guarentee this space will still be available when an allocation 19867c478bd9Sstevel@tonic-gate * request is made, nor that the space may be allocated in one big request 19877c478bd9Sstevel@tonic-gate * due to kernel heap fragmentation. 19887c478bd9Sstevel@tonic-gate */ 19897c478bd9Sstevel@tonic-gate size_t 19907c478bd9Sstevel@tonic-gate kmem_maxavail(void) 19917c478bd9Sstevel@tonic-gate { 19927c478bd9Sstevel@tonic-gate spgcnt_t pmem = availrmem - tune.t_minarmem; 19937c478bd9Sstevel@tonic-gate spgcnt_t vmem = btop(vmem_size(heap_arena, VMEM_FREE)); 19947c478bd9Sstevel@tonic-gate 19957c478bd9Sstevel@tonic-gate return ((size_t)ptob(MAX(MIN(pmem, vmem), 0))); 19967c478bd9Sstevel@tonic-gate } 19977c478bd9Sstevel@tonic-gate 1998fa9e4066Sahrens /* 1999fa9e4066Sahrens * Indicate whether memory-intensive kmem debugging is enabled. 2000fa9e4066Sahrens */ 2001fa9e4066Sahrens int 2002fa9e4066Sahrens kmem_debugging(void) 2003fa9e4066Sahrens { 2004fa9e4066Sahrens return (kmem_flags & (KMF_AUDIT | KMF_REDZONE)); 2005fa9e4066Sahrens } 2006fa9e4066Sahrens 20077c478bd9Sstevel@tonic-gate kmem_cache_t * 20087c478bd9Sstevel@tonic-gate kmem_cache_create( 20097c478bd9Sstevel@tonic-gate char *name, /* descriptive name for this cache */ 20107c478bd9Sstevel@tonic-gate size_t bufsize, /* size of the objects it manages */ 20117c478bd9Sstevel@tonic-gate size_t align, /* required object alignment */ 20127c478bd9Sstevel@tonic-gate int (*constructor)(void *, void *, int), /* object constructor */ 20137c478bd9Sstevel@tonic-gate void (*destructor)(void *, void *), /* object destructor */ 20147c478bd9Sstevel@tonic-gate void (*reclaim)(void *), /* memory reclaim callback */ 20157c478bd9Sstevel@tonic-gate void *private, /* pass-thru arg for constr/destr/reclaim */ 20167c478bd9Sstevel@tonic-gate vmem_t *vmp, /* vmem source for slab allocation */ 20177c478bd9Sstevel@tonic-gate int cflags) /* cache creation flags */ 20187c478bd9Sstevel@tonic-gate { 20197c478bd9Sstevel@tonic-gate int cpu_seqid; 20207c478bd9Sstevel@tonic-gate size_t chunksize; 20217c478bd9Sstevel@tonic-gate kmem_cache_t *cp, *cnext, *cprev; 20227c478bd9Sstevel@tonic-gate kmem_magtype_t *mtp; 20237c478bd9Sstevel@tonic-gate size_t csize = KMEM_CACHE_SIZE(max_ncpus); 20247c478bd9Sstevel@tonic-gate 20257c478bd9Sstevel@tonic-gate #ifdef DEBUG 20267c478bd9Sstevel@tonic-gate /* 20277c478bd9Sstevel@tonic-gate * Cache names should conform to the rules for valid C identifiers 20287c478bd9Sstevel@tonic-gate */ 20297c478bd9Sstevel@tonic-gate if (!strident_valid(name)) { 20307c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, 20317c478bd9Sstevel@tonic-gate "kmem_cache_create: '%s' is an invalid cache name\n" 20327c478bd9Sstevel@tonic-gate "cache names must conform to the rules for " 20337c478bd9Sstevel@tonic-gate "C identifiers\n", name); 20347c478bd9Sstevel@tonic-gate } 20357c478bd9Sstevel@tonic-gate #endif /* DEBUG */ 20367c478bd9Sstevel@tonic-gate 20377c478bd9Sstevel@tonic-gate if (vmp == NULL) 20387c478bd9Sstevel@tonic-gate vmp = kmem_default_arena; 20397c478bd9Sstevel@tonic-gate 20407c478bd9Sstevel@tonic-gate /* 20417c478bd9Sstevel@tonic-gate * If this kmem cache has an identifier vmem arena as its source, mark 20427c478bd9Sstevel@tonic-gate * it such to allow kmem_reap_idspace(). 20437c478bd9Sstevel@tonic-gate */ 20447c478bd9Sstevel@tonic-gate ASSERT(!(cflags & KMC_IDENTIFIER)); /* consumer should not set this */ 20457c478bd9Sstevel@tonic-gate if (vmp->vm_cflags & VMC_IDENTIFIER) 20467c478bd9Sstevel@tonic-gate cflags |= KMC_IDENTIFIER; 20477c478bd9Sstevel@tonic-gate 20487c478bd9Sstevel@tonic-gate /* 20497c478bd9Sstevel@tonic-gate * Get a kmem_cache structure. We arrange that cp->cache_cpu[] 20507c478bd9Sstevel@tonic-gate * is aligned on a KMEM_CPU_CACHE_SIZE boundary to prevent 20517c478bd9Sstevel@tonic-gate * false sharing of per-CPU data. 20527c478bd9Sstevel@tonic-gate */ 20537c478bd9Sstevel@tonic-gate cp = vmem_xalloc(kmem_cache_arena, csize, KMEM_CPU_CACHE_SIZE, 20547c478bd9Sstevel@tonic-gate P2NPHASE(csize, KMEM_CPU_CACHE_SIZE), 0, NULL, NULL, VM_SLEEP); 20557c478bd9Sstevel@tonic-gate bzero(cp, csize); 20567c478bd9Sstevel@tonic-gate 20577c478bd9Sstevel@tonic-gate if (align == 0) 20587c478bd9Sstevel@tonic-gate align = KMEM_ALIGN; 20597c478bd9Sstevel@tonic-gate 20607c478bd9Sstevel@tonic-gate /* 20617c478bd9Sstevel@tonic-gate * If we're not at least KMEM_ALIGN aligned, we can't use free 20627c478bd9Sstevel@tonic-gate * memory to hold bufctl information (because we can't safely 20637c478bd9Sstevel@tonic-gate * perform word loads and stores on it). 20647c478bd9Sstevel@tonic-gate */ 20657c478bd9Sstevel@tonic-gate if (align < KMEM_ALIGN) 20667c478bd9Sstevel@tonic-gate cflags |= KMC_NOTOUCH; 20677c478bd9Sstevel@tonic-gate 20687c478bd9Sstevel@tonic-gate if ((align & (align - 1)) != 0 || align > vmp->vm_quantum) 20697c478bd9Sstevel@tonic-gate panic("kmem_cache_create: bad alignment %lu", align); 20707c478bd9Sstevel@tonic-gate 20717c478bd9Sstevel@tonic-gate mutex_enter(&kmem_flags_lock); 20727c478bd9Sstevel@tonic-gate if (kmem_flags & KMF_RANDOMIZE) 20737c478bd9Sstevel@tonic-gate kmem_flags = (((kmem_flags | ~KMF_RANDOM) + 1) & KMF_RANDOM) | 20747c478bd9Sstevel@tonic-gate KMF_RANDOMIZE; 20757c478bd9Sstevel@tonic-gate cp->cache_flags = (kmem_flags | cflags) & KMF_DEBUG; 20767c478bd9Sstevel@tonic-gate mutex_exit(&kmem_flags_lock); 20777c478bd9Sstevel@tonic-gate 20787c478bd9Sstevel@tonic-gate /* 20797c478bd9Sstevel@tonic-gate * Make sure all the various flags are reasonable. 20807c478bd9Sstevel@tonic-gate */ 20817c478bd9Sstevel@tonic-gate ASSERT(!(cflags & KMC_NOHASH) || !(cflags & KMC_NOTOUCH)); 20827c478bd9Sstevel@tonic-gate 20837c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) { 20847c478bd9Sstevel@tonic-gate if (bufsize >= kmem_lite_minsize && 20857c478bd9Sstevel@tonic-gate align <= kmem_lite_maxalign && 20867c478bd9Sstevel@tonic-gate P2PHASE(bufsize, kmem_lite_maxalign) != 0) { 20877c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_BUFTAG; 20887c478bd9Sstevel@tonic-gate cp->cache_flags &= ~(KMF_AUDIT | KMF_FIREWALL); 20897c478bd9Sstevel@tonic-gate } else { 20907c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_DEBUG; 20917c478bd9Sstevel@tonic-gate } 20927c478bd9Sstevel@tonic-gate } 20937c478bd9Sstevel@tonic-gate 20947c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) 20957c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_REDZONE; 20967c478bd9Sstevel@tonic-gate 20977c478bd9Sstevel@tonic-gate if ((cflags & KMC_QCACHE) && (cp->cache_flags & KMF_AUDIT)) 20987c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_NOMAGAZINE; 20997c478bd9Sstevel@tonic-gate 21007c478bd9Sstevel@tonic-gate if (cflags & KMC_NODEBUG) 21017c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_DEBUG; 21027c478bd9Sstevel@tonic-gate 21037c478bd9Sstevel@tonic-gate if (cflags & KMC_NOTOUCH) 21047c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_TOUCH; 21057c478bd9Sstevel@tonic-gate 21067c478bd9Sstevel@tonic-gate if (cflags & KMC_NOHASH) 21077c478bd9Sstevel@tonic-gate cp->cache_flags &= ~(KMF_AUDIT | KMF_FIREWALL); 21087c478bd9Sstevel@tonic-gate 21097c478bd9Sstevel@tonic-gate if (cflags & KMC_NOMAGAZINE) 21107c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_NOMAGAZINE; 21117c478bd9Sstevel@tonic-gate 21127c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_AUDIT) && !(cflags & KMC_NOTOUCH)) 21137c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_REDZONE; 21147c478bd9Sstevel@tonic-gate 21157c478bd9Sstevel@tonic-gate if (!(cp->cache_flags & KMF_AUDIT)) 21167c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_CONTENTS; 21177c478bd9Sstevel@tonic-gate 21187c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_BUFTAG) && bufsize >= kmem_minfirewall && 21197c478bd9Sstevel@tonic-gate !(cp->cache_flags & KMF_LITE) && !(cflags & KMC_NOHASH)) 21207c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_FIREWALL; 21217c478bd9Sstevel@tonic-gate 21227c478bd9Sstevel@tonic-gate if (vmp != kmem_default_arena || kmem_firewall_arena == NULL) 21237c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_FIREWALL; 21247c478bd9Sstevel@tonic-gate 21257c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_FIREWALL) { 21267c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_BUFTAG; 21277c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_NOMAGAZINE; 21287c478bd9Sstevel@tonic-gate ASSERT(vmp == kmem_default_arena); 21297c478bd9Sstevel@tonic-gate vmp = kmem_firewall_arena; 21307c478bd9Sstevel@tonic-gate } 21317c478bd9Sstevel@tonic-gate 21327c478bd9Sstevel@tonic-gate /* 21337c478bd9Sstevel@tonic-gate * Set cache properties. 21347c478bd9Sstevel@tonic-gate */ 21357c478bd9Sstevel@tonic-gate (void) strncpy(cp->cache_name, name, KMEM_CACHE_NAMELEN); 21367c478bd9Sstevel@tonic-gate strident_canon(cp->cache_name, KMEM_CACHE_NAMELEN); 21377c478bd9Sstevel@tonic-gate cp->cache_bufsize = bufsize; 21387c478bd9Sstevel@tonic-gate cp->cache_align = align; 21397c478bd9Sstevel@tonic-gate cp->cache_constructor = constructor; 21407c478bd9Sstevel@tonic-gate cp->cache_destructor = destructor; 21417c478bd9Sstevel@tonic-gate cp->cache_reclaim = reclaim; 21427c478bd9Sstevel@tonic-gate cp->cache_private = private; 21437c478bd9Sstevel@tonic-gate cp->cache_arena = vmp; 21447c478bd9Sstevel@tonic-gate cp->cache_cflags = cflags; 21457c478bd9Sstevel@tonic-gate 21467c478bd9Sstevel@tonic-gate /* 21477c478bd9Sstevel@tonic-gate * Determine the chunk size. 21487c478bd9Sstevel@tonic-gate */ 21497c478bd9Sstevel@tonic-gate chunksize = bufsize; 21507c478bd9Sstevel@tonic-gate 21517c478bd9Sstevel@tonic-gate if (align >= KMEM_ALIGN) { 21527c478bd9Sstevel@tonic-gate chunksize = P2ROUNDUP(chunksize, KMEM_ALIGN); 21537c478bd9Sstevel@tonic-gate cp->cache_bufctl = chunksize - KMEM_ALIGN; 21547c478bd9Sstevel@tonic-gate } 21557c478bd9Sstevel@tonic-gate 21567c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_BUFTAG) { 21577c478bd9Sstevel@tonic-gate cp->cache_bufctl = chunksize; 21587c478bd9Sstevel@tonic-gate cp->cache_buftag = chunksize; 21597c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) 21607c478bd9Sstevel@tonic-gate chunksize += KMEM_BUFTAG_LITE_SIZE(kmem_lite_count); 21617c478bd9Sstevel@tonic-gate else 21627c478bd9Sstevel@tonic-gate chunksize += sizeof (kmem_buftag_t); 21637c478bd9Sstevel@tonic-gate } 21647c478bd9Sstevel@tonic-gate 21657c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) { 21667c478bd9Sstevel@tonic-gate cp->cache_verify = MIN(cp->cache_buftag, kmem_maxverify); 21677c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) 21687c478bd9Sstevel@tonic-gate cp->cache_verify = sizeof (uint64_t); 21697c478bd9Sstevel@tonic-gate } 21707c478bd9Sstevel@tonic-gate 21717c478bd9Sstevel@tonic-gate cp->cache_contents = MIN(cp->cache_bufctl, kmem_content_maxsave); 21727c478bd9Sstevel@tonic-gate 21737c478bd9Sstevel@tonic-gate cp->cache_chunksize = chunksize = P2ROUNDUP(chunksize, align); 21747c478bd9Sstevel@tonic-gate 21757c478bd9Sstevel@tonic-gate /* 21767c478bd9Sstevel@tonic-gate * Now that we know the chunk size, determine the optimal slab size. 21777c478bd9Sstevel@tonic-gate */ 21787c478bd9Sstevel@tonic-gate if (vmp == kmem_firewall_arena) { 21797c478bd9Sstevel@tonic-gate cp->cache_slabsize = P2ROUNDUP(chunksize, vmp->vm_quantum); 21807c478bd9Sstevel@tonic-gate cp->cache_mincolor = cp->cache_slabsize - chunksize; 21817c478bd9Sstevel@tonic-gate cp->cache_maxcolor = cp->cache_mincolor; 21827c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_HASH; 21837c478bd9Sstevel@tonic-gate ASSERT(!(cp->cache_flags & KMF_BUFTAG)); 21847c478bd9Sstevel@tonic-gate } else if ((cflags & KMC_NOHASH) || (!(cflags & KMC_NOTOUCH) && 21857c478bd9Sstevel@tonic-gate !(cp->cache_flags & KMF_AUDIT) && 21867c478bd9Sstevel@tonic-gate chunksize < vmp->vm_quantum / KMEM_VOID_FRACTION)) { 21877c478bd9Sstevel@tonic-gate cp->cache_slabsize = vmp->vm_quantum; 21887c478bd9Sstevel@tonic-gate cp->cache_mincolor = 0; 21897c478bd9Sstevel@tonic-gate cp->cache_maxcolor = 21907c478bd9Sstevel@tonic-gate (cp->cache_slabsize - sizeof (kmem_slab_t)) % chunksize; 21917c478bd9Sstevel@tonic-gate ASSERT(chunksize + sizeof (kmem_slab_t) <= cp->cache_slabsize); 21927c478bd9Sstevel@tonic-gate ASSERT(!(cp->cache_flags & KMF_AUDIT)); 21937c478bd9Sstevel@tonic-gate } else { 21947c478bd9Sstevel@tonic-gate size_t chunks, bestfit, waste, slabsize; 21957c478bd9Sstevel@tonic-gate size_t minwaste = LONG_MAX; 21967c478bd9Sstevel@tonic-gate 21977c478bd9Sstevel@tonic-gate for (chunks = 1; chunks <= KMEM_VOID_FRACTION; chunks++) { 21987c478bd9Sstevel@tonic-gate slabsize = P2ROUNDUP(chunksize * chunks, 21997c478bd9Sstevel@tonic-gate vmp->vm_quantum); 22007c478bd9Sstevel@tonic-gate chunks = slabsize / chunksize; 22017c478bd9Sstevel@tonic-gate waste = (slabsize % chunksize) / chunks; 22027c478bd9Sstevel@tonic-gate if (waste < minwaste) { 22037c478bd9Sstevel@tonic-gate minwaste = waste; 22047c478bd9Sstevel@tonic-gate bestfit = slabsize; 22057c478bd9Sstevel@tonic-gate } 22067c478bd9Sstevel@tonic-gate } 22077c478bd9Sstevel@tonic-gate if (cflags & KMC_QCACHE) 22087c478bd9Sstevel@tonic-gate bestfit = VMEM_QCACHE_SLABSIZE(vmp->vm_qcache_max); 22097c478bd9Sstevel@tonic-gate cp->cache_slabsize = bestfit; 22107c478bd9Sstevel@tonic-gate cp->cache_mincolor = 0; 22117c478bd9Sstevel@tonic-gate cp->cache_maxcolor = bestfit % chunksize; 22127c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_HASH; 22137c478bd9Sstevel@tonic-gate } 22147c478bd9Sstevel@tonic-gate 22157c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) { 22167c478bd9Sstevel@tonic-gate ASSERT(!(cflags & KMC_NOHASH)); 22177c478bd9Sstevel@tonic-gate cp->cache_bufctl_cache = (cp->cache_flags & KMF_AUDIT) ? 22187c478bd9Sstevel@tonic-gate kmem_bufctl_audit_cache : kmem_bufctl_cache; 22197c478bd9Sstevel@tonic-gate } 22207c478bd9Sstevel@tonic-gate 22217c478bd9Sstevel@tonic-gate if (cp->cache_maxcolor >= vmp->vm_quantum) 22227c478bd9Sstevel@tonic-gate cp->cache_maxcolor = vmp->vm_quantum - 1; 22237c478bd9Sstevel@tonic-gate 22247c478bd9Sstevel@tonic-gate cp->cache_color = cp->cache_mincolor; 22257c478bd9Sstevel@tonic-gate 22267c478bd9Sstevel@tonic-gate /* 22277c478bd9Sstevel@tonic-gate * Initialize the rest of the slab layer. 22287c478bd9Sstevel@tonic-gate */ 22297c478bd9Sstevel@tonic-gate mutex_init(&cp->cache_lock, NULL, MUTEX_DEFAULT, NULL); 22307c478bd9Sstevel@tonic-gate 22317c478bd9Sstevel@tonic-gate cp->cache_freelist = &cp->cache_nullslab; 22327c478bd9Sstevel@tonic-gate cp->cache_nullslab.slab_cache = cp; 22337c478bd9Sstevel@tonic-gate cp->cache_nullslab.slab_refcnt = -1; 22347c478bd9Sstevel@tonic-gate cp->cache_nullslab.slab_next = &cp->cache_nullslab; 22357c478bd9Sstevel@tonic-gate cp->cache_nullslab.slab_prev = &cp->cache_nullslab; 22367c478bd9Sstevel@tonic-gate 22377c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) { 22387c478bd9Sstevel@tonic-gate cp->cache_hash_table = vmem_alloc(kmem_hash_arena, 22397c478bd9Sstevel@tonic-gate KMEM_HASH_INITIAL * sizeof (void *), VM_SLEEP); 22407c478bd9Sstevel@tonic-gate bzero(cp->cache_hash_table, 22417c478bd9Sstevel@tonic-gate KMEM_HASH_INITIAL * sizeof (void *)); 22427c478bd9Sstevel@tonic-gate cp->cache_hash_mask = KMEM_HASH_INITIAL - 1; 22437c478bd9Sstevel@tonic-gate cp->cache_hash_shift = highbit((ulong_t)chunksize) - 1; 22447c478bd9Sstevel@tonic-gate } 22457c478bd9Sstevel@tonic-gate 22467c478bd9Sstevel@tonic-gate /* 22477c478bd9Sstevel@tonic-gate * Initialize the depot. 22487c478bd9Sstevel@tonic-gate */ 22497c478bd9Sstevel@tonic-gate mutex_init(&cp->cache_depot_lock, NULL, MUTEX_DEFAULT, NULL); 22507c478bd9Sstevel@tonic-gate 22517c478bd9Sstevel@tonic-gate for (mtp = kmem_magtype; chunksize <= mtp->mt_minbuf; mtp++) 22527c478bd9Sstevel@tonic-gate continue; 22537c478bd9Sstevel@tonic-gate 22547c478bd9Sstevel@tonic-gate cp->cache_magtype = mtp; 22557c478bd9Sstevel@tonic-gate 22567c478bd9Sstevel@tonic-gate /* 22577c478bd9Sstevel@tonic-gate * Initialize the CPU layer. 22587c478bd9Sstevel@tonic-gate */ 22597c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 22607c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 22617c478bd9Sstevel@tonic-gate mutex_init(&ccp->cc_lock, NULL, MUTEX_DEFAULT, NULL); 22627c478bd9Sstevel@tonic-gate ccp->cc_flags = cp->cache_flags; 22637c478bd9Sstevel@tonic-gate ccp->cc_rounds = -1; 22647c478bd9Sstevel@tonic-gate ccp->cc_prounds = -1; 22657c478bd9Sstevel@tonic-gate } 22667c478bd9Sstevel@tonic-gate 22677c478bd9Sstevel@tonic-gate /* 22687c478bd9Sstevel@tonic-gate * Create the cache's kstats. 22697c478bd9Sstevel@tonic-gate */ 22707c478bd9Sstevel@tonic-gate if ((cp->cache_kstat = kstat_create("unix", 0, cp->cache_name, 22717c478bd9Sstevel@tonic-gate "kmem_cache", KSTAT_TYPE_NAMED, 22727c478bd9Sstevel@tonic-gate sizeof (kmem_cache_kstat) / sizeof (kstat_named_t), 22737c478bd9Sstevel@tonic-gate KSTAT_FLAG_VIRTUAL)) != NULL) { 22747c478bd9Sstevel@tonic-gate cp->cache_kstat->ks_data = &kmem_cache_kstat; 22757c478bd9Sstevel@tonic-gate cp->cache_kstat->ks_update = kmem_cache_kstat_update; 22767c478bd9Sstevel@tonic-gate cp->cache_kstat->ks_private = cp; 22777c478bd9Sstevel@tonic-gate cp->cache_kstat->ks_lock = &kmem_cache_kstat_lock; 22787c478bd9Sstevel@tonic-gate kstat_install(cp->cache_kstat); 22797c478bd9Sstevel@tonic-gate } 22807c478bd9Sstevel@tonic-gate 22817c478bd9Sstevel@tonic-gate /* 22827c478bd9Sstevel@tonic-gate * Add the cache to the global list. This makes it visible 22837c478bd9Sstevel@tonic-gate * to kmem_update(), so the cache must be ready for business. 22847c478bd9Sstevel@tonic-gate */ 22857c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_lock); 22867c478bd9Sstevel@tonic-gate cp->cache_next = cnext = &kmem_null_cache; 22877c478bd9Sstevel@tonic-gate cp->cache_prev = cprev = kmem_null_cache.cache_prev; 22887c478bd9Sstevel@tonic-gate cnext->cache_prev = cp; 22897c478bd9Sstevel@tonic-gate cprev->cache_next = cp; 22907c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_lock); 22917c478bd9Sstevel@tonic-gate 22927c478bd9Sstevel@tonic-gate if (kmem_ready) 22937c478bd9Sstevel@tonic-gate kmem_cache_magazine_enable(cp); 22947c478bd9Sstevel@tonic-gate 22957c478bd9Sstevel@tonic-gate return (cp); 22967c478bd9Sstevel@tonic-gate } 22977c478bd9Sstevel@tonic-gate 22987c478bd9Sstevel@tonic-gate void 22997c478bd9Sstevel@tonic-gate kmem_cache_destroy(kmem_cache_t *cp) 23007c478bd9Sstevel@tonic-gate { 23017c478bd9Sstevel@tonic-gate int cpu_seqid; 23027c478bd9Sstevel@tonic-gate 23037c478bd9Sstevel@tonic-gate /* 23047c478bd9Sstevel@tonic-gate * Remove the cache from the global cache list so that no one else 23057c478bd9Sstevel@tonic-gate * can schedule tasks on its behalf, wait for any pending tasks to 23067c478bd9Sstevel@tonic-gate * complete, purge the cache, and then destroy it. 23077c478bd9Sstevel@tonic-gate */ 23087c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_lock); 23097c478bd9Sstevel@tonic-gate cp->cache_prev->cache_next = cp->cache_next; 23107c478bd9Sstevel@tonic-gate cp->cache_next->cache_prev = cp->cache_prev; 23117c478bd9Sstevel@tonic-gate cp->cache_prev = cp->cache_next = NULL; 23127c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_lock); 23137c478bd9Sstevel@tonic-gate 23147c478bd9Sstevel@tonic-gate if (kmem_taskq != NULL) 23157c478bd9Sstevel@tonic-gate taskq_wait(kmem_taskq); 23167c478bd9Sstevel@tonic-gate 23177c478bd9Sstevel@tonic-gate kmem_cache_magazine_purge(cp); 23187c478bd9Sstevel@tonic-gate 23197c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 23207c478bd9Sstevel@tonic-gate if (cp->cache_buftotal != 0) 23217c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "kmem_cache_destroy: '%s' (%p) not empty", 23227c478bd9Sstevel@tonic-gate cp->cache_name, (void *)cp); 23237c478bd9Sstevel@tonic-gate cp->cache_reclaim = NULL; 23247c478bd9Sstevel@tonic-gate /* 23257c478bd9Sstevel@tonic-gate * The cache is now dead. There should be no further activity. 23267c478bd9Sstevel@tonic-gate * We enforce this by setting land mines in the constructor and 23277c478bd9Sstevel@tonic-gate * destructor routines that induce a kernel text fault if invoked. 23287c478bd9Sstevel@tonic-gate */ 23297c478bd9Sstevel@tonic-gate cp->cache_constructor = (int (*)(void *, void *, int))1; 23307c478bd9Sstevel@tonic-gate cp->cache_destructor = (void (*)(void *, void *))2; 23317c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 23327c478bd9Sstevel@tonic-gate 23337c478bd9Sstevel@tonic-gate kstat_delete(cp->cache_kstat); 23347c478bd9Sstevel@tonic-gate 23357c478bd9Sstevel@tonic-gate if (cp->cache_hash_table != NULL) 23367c478bd9Sstevel@tonic-gate vmem_free(kmem_hash_arena, cp->cache_hash_table, 23377c478bd9Sstevel@tonic-gate (cp->cache_hash_mask + 1) * sizeof (void *)); 23387c478bd9Sstevel@tonic-gate 23397c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) 23407c478bd9Sstevel@tonic-gate mutex_destroy(&cp->cache_cpu[cpu_seqid].cc_lock); 23417c478bd9Sstevel@tonic-gate 23427c478bd9Sstevel@tonic-gate mutex_destroy(&cp->cache_depot_lock); 23437c478bd9Sstevel@tonic-gate mutex_destroy(&cp->cache_lock); 23447c478bd9Sstevel@tonic-gate 23457c478bd9Sstevel@tonic-gate vmem_free(kmem_cache_arena, cp, KMEM_CACHE_SIZE(max_ncpus)); 23467c478bd9Sstevel@tonic-gate } 23477c478bd9Sstevel@tonic-gate 23487c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 23497c478bd9Sstevel@tonic-gate static int 23507c478bd9Sstevel@tonic-gate kmem_cpu_setup(cpu_setup_t what, int id, void *arg) 23517c478bd9Sstevel@tonic-gate { 23527c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 23537c478bd9Sstevel@tonic-gate if (what == CPU_UNCONFIG) { 23547c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_magazine_purge, 23557c478bd9Sstevel@tonic-gate kmem_taskq, TQ_SLEEP); 23567c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_magazine_enable, 23577c478bd9Sstevel@tonic-gate kmem_taskq, TQ_SLEEP); 23587c478bd9Sstevel@tonic-gate } 23597c478bd9Sstevel@tonic-gate return (0); 23607c478bd9Sstevel@tonic-gate } 23617c478bd9Sstevel@tonic-gate 23627c478bd9Sstevel@tonic-gate static void 23637c478bd9Sstevel@tonic-gate kmem_cache_init(int pass, int use_large_pages) 23647c478bd9Sstevel@tonic-gate { 23657c478bd9Sstevel@tonic-gate int i; 23667c478bd9Sstevel@tonic-gate size_t size; 23677c478bd9Sstevel@tonic-gate kmem_cache_t *cp; 23687c478bd9Sstevel@tonic-gate kmem_magtype_t *mtp; 23697c478bd9Sstevel@tonic-gate char name[KMEM_CACHE_NAMELEN + 1]; 23707c478bd9Sstevel@tonic-gate 23717c478bd9Sstevel@tonic-gate for (i = 0; i < sizeof (kmem_magtype) / sizeof (*mtp); i++) { 23727c478bd9Sstevel@tonic-gate mtp = &kmem_magtype[i]; 23737c478bd9Sstevel@tonic-gate (void) sprintf(name, "kmem_magazine_%d", mtp->mt_magsize); 23747c478bd9Sstevel@tonic-gate mtp->mt_cache = kmem_cache_create(name, 23757c478bd9Sstevel@tonic-gate (mtp->mt_magsize + 1) * sizeof (void *), 23767c478bd9Sstevel@tonic-gate mtp->mt_align, NULL, NULL, NULL, NULL, 23777c478bd9Sstevel@tonic-gate kmem_msb_arena, KMC_NOHASH); 23787c478bd9Sstevel@tonic-gate } 23797c478bd9Sstevel@tonic-gate 23807c478bd9Sstevel@tonic-gate kmem_slab_cache = kmem_cache_create("kmem_slab_cache", 23817c478bd9Sstevel@tonic-gate sizeof (kmem_slab_t), 0, NULL, NULL, NULL, NULL, 23827c478bd9Sstevel@tonic-gate kmem_msb_arena, KMC_NOHASH); 23837c478bd9Sstevel@tonic-gate 23847c478bd9Sstevel@tonic-gate kmem_bufctl_cache = kmem_cache_create("kmem_bufctl_cache", 23857c478bd9Sstevel@tonic-gate sizeof (kmem_bufctl_t), 0, NULL, NULL, NULL, NULL, 23867c478bd9Sstevel@tonic-gate kmem_msb_arena, KMC_NOHASH); 23877c478bd9Sstevel@tonic-gate 23887c478bd9Sstevel@tonic-gate kmem_bufctl_audit_cache = kmem_cache_create("kmem_bufctl_audit_cache", 23897c478bd9Sstevel@tonic-gate sizeof (kmem_bufctl_audit_t), 0, NULL, NULL, NULL, NULL, 23907c478bd9Sstevel@tonic-gate kmem_msb_arena, KMC_NOHASH); 23917c478bd9Sstevel@tonic-gate 23927c478bd9Sstevel@tonic-gate if (pass == 2) { 23937c478bd9Sstevel@tonic-gate kmem_va_arena = vmem_create("kmem_va", 23947c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE, 23957c478bd9Sstevel@tonic-gate vmem_alloc, vmem_free, heap_arena, 23967c478bd9Sstevel@tonic-gate 8 * PAGESIZE, VM_SLEEP); 23977c478bd9Sstevel@tonic-gate 23987c478bd9Sstevel@tonic-gate if (use_large_pages) { 23997c478bd9Sstevel@tonic-gate kmem_default_arena = vmem_xcreate("kmem_default", 24007c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE, 24017c478bd9Sstevel@tonic-gate segkmem_alloc_lp, segkmem_free_lp, kmem_va_arena, 24027c478bd9Sstevel@tonic-gate 0, VM_SLEEP); 24037c478bd9Sstevel@tonic-gate } else { 24047c478bd9Sstevel@tonic-gate kmem_default_arena = vmem_create("kmem_default", 24057c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE, 24067c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, kmem_va_arena, 24077c478bd9Sstevel@tonic-gate 0, VM_SLEEP); 24087c478bd9Sstevel@tonic-gate } 24097c478bd9Sstevel@tonic-gate } else { 24107c478bd9Sstevel@tonic-gate /* 24117c478bd9Sstevel@tonic-gate * During the first pass, the kmem_alloc_* caches 24127c478bd9Sstevel@tonic-gate * are treated as metadata. 24137c478bd9Sstevel@tonic-gate */ 24147c478bd9Sstevel@tonic-gate kmem_default_arena = kmem_msb_arena; 24157c478bd9Sstevel@tonic-gate } 24167c478bd9Sstevel@tonic-gate 24177c478bd9Sstevel@tonic-gate /* 24187c478bd9Sstevel@tonic-gate * Set up the default caches to back kmem_alloc() 24197c478bd9Sstevel@tonic-gate */ 24207c478bd9Sstevel@tonic-gate size = KMEM_ALIGN; 24217c478bd9Sstevel@tonic-gate for (i = 0; i < sizeof (kmem_alloc_sizes) / sizeof (int); i++) { 24227c478bd9Sstevel@tonic-gate size_t align = KMEM_ALIGN; 24237c478bd9Sstevel@tonic-gate size_t cache_size = kmem_alloc_sizes[i]; 24247c478bd9Sstevel@tonic-gate /* 24257c478bd9Sstevel@tonic-gate * If they allocate a multiple of the coherency granularity, 24267c478bd9Sstevel@tonic-gate * they get a coherency-granularity-aligned address. 24277c478bd9Sstevel@tonic-gate */ 24287c478bd9Sstevel@tonic-gate if (IS_P2ALIGNED(cache_size, 64)) 24297c478bd9Sstevel@tonic-gate align = 64; 24307c478bd9Sstevel@tonic-gate if (IS_P2ALIGNED(cache_size, PAGESIZE)) 24317c478bd9Sstevel@tonic-gate align = PAGESIZE; 24327c478bd9Sstevel@tonic-gate (void) sprintf(name, "kmem_alloc_%lu", cache_size); 24337c478bd9Sstevel@tonic-gate cp = kmem_cache_create(name, cache_size, align, 24347c478bd9Sstevel@tonic-gate NULL, NULL, NULL, NULL, NULL, KMC_KMEM_ALLOC); 24357c478bd9Sstevel@tonic-gate while (size <= cache_size) { 24367c478bd9Sstevel@tonic-gate kmem_alloc_table[(size - 1) >> KMEM_ALIGN_SHIFT] = cp; 24377c478bd9Sstevel@tonic-gate size += KMEM_ALIGN; 24387c478bd9Sstevel@tonic-gate } 24397c478bd9Sstevel@tonic-gate } 24407c478bd9Sstevel@tonic-gate } 24417c478bd9Sstevel@tonic-gate 24427c478bd9Sstevel@tonic-gate void 24437c478bd9Sstevel@tonic-gate kmem_init(void) 24447c478bd9Sstevel@tonic-gate { 24457c478bd9Sstevel@tonic-gate kmem_cache_t *cp; 24467c478bd9Sstevel@tonic-gate int old_kmem_flags = kmem_flags; 24477c478bd9Sstevel@tonic-gate int use_large_pages = 0; 24487c478bd9Sstevel@tonic-gate size_t maxverify, minfirewall; 24497c478bd9Sstevel@tonic-gate 24507c478bd9Sstevel@tonic-gate kstat_init(); 24517c478bd9Sstevel@tonic-gate 24527c478bd9Sstevel@tonic-gate /* 24537c478bd9Sstevel@tonic-gate * Small-memory systems (< 24 MB) can't handle kmem_flags overhead. 24547c478bd9Sstevel@tonic-gate */ 24557c478bd9Sstevel@tonic-gate if (physmem < btop(24 << 20) && !(old_kmem_flags & KMF_STICKY)) 24567c478bd9Sstevel@tonic-gate kmem_flags = 0; 24577c478bd9Sstevel@tonic-gate 24587c478bd9Sstevel@tonic-gate /* 24597c478bd9Sstevel@tonic-gate * Don't do firewalled allocations if the heap is less than 1TB 24607c478bd9Sstevel@tonic-gate * (i.e. on a 32-bit kernel) 24617c478bd9Sstevel@tonic-gate * The resulting VM_NEXTFIT allocations would create too much 24627c478bd9Sstevel@tonic-gate * fragmentation in a small heap. 24637c478bd9Sstevel@tonic-gate */ 24647c478bd9Sstevel@tonic-gate #if defined(_LP64) 24657c478bd9Sstevel@tonic-gate maxverify = minfirewall = PAGESIZE / 2; 24667c478bd9Sstevel@tonic-gate #else 24677c478bd9Sstevel@tonic-gate maxverify = minfirewall = ULONG_MAX; 24687c478bd9Sstevel@tonic-gate #endif 24697c478bd9Sstevel@tonic-gate 24707c478bd9Sstevel@tonic-gate /* LINTED */ 24717c478bd9Sstevel@tonic-gate ASSERT(sizeof (kmem_cpu_cache_t) == KMEM_CPU_CACHE_SIZE); 24727c478bd9Sstevel@tonic-gate 24737c478bd9Sstevel@tonic-gate kmem_null_cache.cache_next = &kmem_null_cache; 24747c478bd9Sstevel@tonic-gate kmem_null_cache.cache_prev = &kmem_null_cache; 24757c478bd9Sstevel@tonic-gate 24767c478bd9Sstevel@tonic-gate kmem_metadata_arena = vmem_create("kmem_metadata", NULL, 0, PAGESIZE, 24777c478bd9Sstevel@tonic-gate vmem_alloc, vmem_free, heap_arena, 8 * PAGESIZE, 24787c478bd9Sstevel@tonic-gate VM_SLEEP | VMC_NO_QCACHE); 24797c478bd9Sstevel@tonic-gate 24807c478bd9Sstevel@tonic-gate kmem_msb_arena = vmem_create("kmem_msb", NULL, 0, 24817c478bd9Sstevel@tonic-gate PAGESIZE, segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, 24827c478bd9Sstevel@tonic-gate VM_SLEEP); 24837c478bd9Sstevel@tonic-gate 24847c478bd9Sstevel@tonic-gate kmem_cache_arena = vmem_create("kmem_cache", NULL, 0, KMEM_ALIGN, 24857c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, VM_SLEEP); 24867c478bd9Sstevel@tonic-gate 24877c478bd9Sstevel@tonic-gate kmem_hash_arena = vmem_create("kmem_hash", NULL, 0, KMEM_ALIGN, 24887c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, VM_SLEEP); 24897c478bd9Sstevel@tonic-gate 24907c478bd9Sstevel@tonic-gate kmem_log_arena = vmem_create("kmem_log", NULL, 0, KMEM_ALIGN, 24917c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP); 24927c478bd9Sstevel@tonic-gate 24937c478bd9Sstevel@tonic-gate kmem_firewall_va_arena = vmem_create("kmem_firewall_va", 24947c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE, 24957c478bd9Sstevel@tonic-gate kmem_firewall_va_alloc, kmem_firewall_va_free, heap_arena, 24967c478bd9Sstevel@tonic-gate 0, VM_SLEEP); 24977c478bd9Sstevel@tonic-gate 24987c478bd9Sstevel@tonic-gate kmem_firewall_arena = vmem_create("kmem_firewall", NULL, 0, PAGESIZE, 24997c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, kmem_firewall_va_arena, 0, VM_SLEEP); 25007c478bd9Sstevel@tonic-gate 25017c478bd9Sstevel@tonic-gate /* temporary oversize arena for mod_read_system_file */ 25027c478bd9Sstevel@tonic-gate kmem_oversize_arena = vmem_create("kmem_oversize", NULL, 0, PAGESIZE, 25037c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP); 25047c478bd9Sstevel@tonic-gate 25057c478bd9Sstevel@tonic-gate kmem_null_cache.cache_next = &kmem_null_cache; 25067c478bd9Sstevel@tonic-gate kmem_null_cache.cache_prev = &kmem_null_cache; 25077c478bd9Sstevel@tonic-gate 25087c478bd9Sstevel@tonic-gate kmem_reap_interval = 15 * hz; 25097c478bd9Sstevel@tonic-gate 25107c478bd9Sstevel@tonic-gate /* 25117c478bd9Sstevel@tonic-gate * Read /etc/system. This is a chicken-and-egg problem because 25127c478bd9Sstevel@tonic-gate * kmem_flags may be set in /etc/system, but mod_read_system_file() 25137c478bd9Sstevel@tonic-gate * needs to use the allocator. The simplest solution is to create 25147c478bd9Sstevel@tonic-gate * all the standard kmem caches, read /etc/system, destroy all the 25157c478bd9Sstevel@tonic-gate * caches we just created, and then create them all again in light 25167c478bd9Sstevel@tonic-gate * of the (possibly) new kmem_flags and other kmem tunables. 25177c478bd9Sstevel@tonic-gate */ 25187c478bd9Sstevel@tonic-gate kmem_cache_init(1, 0); 25197c478bd9Sstevel@tonic-gate 25207c478bd9Sstevel@tonic-gate mod_read_system_file(boothowto & RB_ASKNAME); 25217c478bd9Sstevel@tonic-gate 25227c478bd9Sstevel@tonic-gate while ((cp = kmem_null_cache.cache_prev) != &kmem_null_cache) 25237c478bd9Sstevel@tonic-gate kmem_cache_destroy(cp); 25247c478bd9Sstevel@tonic-gate 25257c478bd9Sstevel@tonic-gate vmem_destroy(kmem_oversize_arena); 25267c478bd9Sstevel@tonic-gate 25277c478bd9Sstevel@tonic-gate if (old_kmem_flags & KMF_STICKY) 25287c478bd9Sstevel@tonic-gate kmem_flags = old_kmem_flags; 25297c478bd9Sstevel@tonic-gate 25307c478bd9Sstevel@tonic-gate if (!(kmem_flags & KMF_AUDIT)) 25317c478bd9Sstevel@tonic-gate vmem_seg_size = offsetof(vmem_seg_t, vs_thread); 25327c478bd9Sstevel@tonic-gate 25337c478bd9Sstevel@tonic-gate if (kmem_maxverify == 0) 25347c478bd9Sstevel@tonic-gate kmem_maxverify = maxverify; 25357c478bd9Sstevel@tonic-gate 25367c478bd9Sstevel@tonic-gate if (kmem_minfirewall == 0) 25377c478bd9Sstevel@tonic-gate kmem_minfirewall = minfirewall; 25387c478bd9Sstevel@tonic-gate 25397c478bd9Sstevel@tonic-gate /* 25407c478bd9Sstevel@tonic-gate * give segkmem a chance to figure out if we are using large pages 25417c478bd9Sstevel@tonic-gate * for the kernel heap 25427c478bd9Sstevel@tonic-gate */ 25437c478bd9Sstevel@tonic-gate use_large_pages = segkmem_lpsetup(); 25447c478bd9Sstevel@tonic-gate 25457c478bd9Sstevel@tonic-gate /* 25467c478bd9Sstevel@tonic-gate * To protect against corruption, we keep the actual number of callers 25477c478bd9Sstevel@tonic-gate * KMF_LITE records seperate from the tunable. We arbitrarily clamp 25487c478bd9Sstevel@tonic-gate * to 16, since the overhead for small buffers quickly gets out of 25497c478bd9Sstevel@tonic-gate * hand. 25507c478bd9Sstevel@tonic-gate * 25517c478bd9Sstevel@tonic-gate * The real limit would depend on the needs of the largest KMC_NOHASH 25527c478bd9Sstevel@tonic-gate * cache. 25537c478bd9Sstevel@tonic-gate */ 25547c478bd9Sstevel@tonic-gate kmem_lite_count = MIN(MAX(0, kmem_lite_pcs), 16); 25557c478bd9Sstevel@tonic-gate kmem_lite_pcs = kmem_lite_count; 25567c478bd9Sstevel@tonic-gate 25577c478bd9Sstevel@tonic-gate /* 25587c478bd9Sstevel@tonic-gate * Normally, we firewall oversized allocations when possible, but 25597c478bd9Sstevel@tonic-gate * if we are using large pages for kernel memory, and we don't have 25607c478bd9Sstevel@tonic-gate * any non-LITE debugging flags set, we want to allocate oversized 25617c478bd9Sstevel@tonic-gate * buffers from large pages, and so skip the firewalling. 25627c478bd9Sstevel@tonic-gate */ 25637c478bd9Sstevel@tonic-gate if (use_large_pages && 25647c478bd9Sstevel@tonic-gate ((kmem_flags & KMF_LITE) || !(kmem_flags & KMF_DEBUG))) { 25657c478bd9Sstevel@tonic-gate kmem_oversize_arena = vmem_xcreate("kmem_oversize", NULL, 0, 25667c478bd9Sstevel@tonic-gate PAGESIZE, segkmem_alloc_lp, segkmem_free_lp, heap_arena, 25677c478bd9Sstevel@tonic-gate 0, VM_SLEEP); 25687c478bd9Sstevel@tonic-gate } else { 25697c478bd9Sstevel@tonic-gate kmem_oversize_arena = vmem_create("kmem_oversize", 25707c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE, 25717c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, kmem_minfirewall < ULONG_MAX? 25727c478bd9Sstevel@tonic-gate kmem_firewall_va_arena : heap_arena, 0, VM_SLEEP); 25737c478bd9Sstevel@tonic-gate } 25747c478bd9Sstevel@tonic-gate 25757c478bd9Sstevel@tonic-gate kmem_cache_init(2, use_large_pages); 25767c478bd9Sstevel@tonic-gate 25777c478bd9Sstevel@tonic-gate if (kmem_flags & (KMF_AUDIT | KMF_RANDOMIZE)) { 25787c478bd9Sstevel@tonic-gate if (kmem_transaction_log_size == 0) 25797c478bd9Sstevel@tonic-gate kmem_transaction_log_size = kmem_maxavail() / 50; 25807c478bd9Sstevel@tonic-gate kmem_transaction_log = kmem_log_init(kmem_transaction_log_size); 25817c478bd9Sstevel@tonic-gate } 25827c478bd9Sstevel@tonic-gate 25837c478bd9Sstevel@tonic-gate if (kmem_flags & (KMF_CONTENTS | KMF_RANDOMIZE)) { 25847c478bd9Sstevel@tonic-gate if (kmem_content_log_size == 0) 25857c478bd9Sstevel@tonic-gate kmem_content_log_size = kmem_maxavail() / 50; 25867c478bd9Sstevel@tonic-gate kmem_content_log = kmem_log_init(kmem_content_log_size); 25877c478bd9Sstevel@tonic-gate } 25887c478bd9Sstevel@tonic-gate 25897c478bd9Sstevel@tonic-gate kmem_failure_log = kmem_log_init(kmem_failure_log_size); 25907c478bd9Sstevel@tonic-gate 25917c478bd9Sstevel@tonic-gate kmem_slab_log = kmem_log_init(kmem_slab_log_size); 25927c478bd9Sstevel@tonic-gate 25937c478bd9Sstevel@tonic-gate /* 25947c478bd9Sstevel@tonic-gate * Initialize STREAMS message caches so allocb() is available. 25957c478bd9Sstevel@tonic-gate * This allows us to initialize the logging framework (cmn_err(9F), 25967c478bd9Sstevel@tonic-gate * strlog(9F), etc) so we can start recording messages. 25977c478bd9Sstevel@tonic-gate */ 25987c478bd9Sstevel@tonic-gate streams_msg_init(); 25997d692464Sdp 26007c478bd9Sstevel@tonic-gate /* 26017c478bd9Sstevel@tonic-gate * Initialize the ZSD framework in Zones so modules loaded henceforth 26027c478bd9Sstevel@tonic-gate * can register their callbacks. 26037c478bd9Sstevel@tonic-gate */ 26047c478bd9Sstevel@tonic-gate zone_zsd_init(); 2605*f4b3ec61Sdh 26067c478bd9Sstevel@tonic-gate log_init(); 26077c478bd9Sstevel@tonic-gate taskq_init(); 26087c478bd9Sstevel@tonic-gate 26097d692464Sdp /* 26107d692464Sdp * Warn about invalid or dangerous values of kmem_flags. 26117d692464Sdp * Always warn about unsupported values. 26127d692464Sdp */ 26137d692464Sdp if (((kmem_flags & ~(KMF_AUDIT | KMF_DEADBEEF | KMF_REDZONE | 26147d692464Sdp KMF_CONTENTS | KMF_LITE)) != 0) || 26157d692464Sdp ((kmem_flags & KMF_LITE) && kmem_flags != KMF_LITE)) 26167d692464Sdp cmn_err(CE_WARN, "kmem_flags set to unsupported value 0x%x. " 26177d692464Sdp "See the Solaris Tunable Parameters Reference Manual.", 26187d692464Sdp kmem_flags); 26197d692464Sdp 26207d692464Sdp #ifdef DEBUG 26217d692464Sdp if ((kmem_flags & KMF_DEBUG) == 0) 26227d692464Sdp cmn_err(CE_NOTE, "kmem debugging disabled."); 26237d692464Sdp #else 26247d692464Sdp /* 26257d692464Sdp * For non-debug kernels, the only "normal" flags are 0, KMF_LITE, 26267d692464Sdp * KMF_REDZONE, and KMF_CONTENTS (the last because it is only enabled 26277d692464Sdp * if KMF_AUDIT is set). We should warn the user about the performance 26287d692464Sdp * penalty of KMF_AUDIT or KMF_DEADBEEF if they are set and KMF_LITE 26297d692464Sdp * isn't set (since that disables AUDIT). 26307d692464Sdp */ 26317d692464Sdp if (!(kmem_flags & KMF_LITE) && 26327d692464Sdp (kmem_flags & (KMF_AUDIT | KMF_DEADBEEF)) != 0) 26337d692464Sdp cmn_err(CE_WARN, "High-overhead kmem debugging features " 26347d692464Sdp "enabled (kmem_flags = 0x%x). Performance degradation " 26357d692464Sdp "and large memory overhead possible. See the Solaris " 26367d692464Sdp "Tunable Parameters Reference Manual.", kmem_flags); 26377d692464Sdp #endif /* not DEBUG */ 26387d692464Sdp 26397c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_magazine_enable, NULL, TQ_SLEEP); 26407c478bd9Sstevel@tonic-gate 26417c478bd9Sstevel@tonic-gate kmem_ready = 1; 26427c478bd9Sstevel@tonic-gate 26437c478bd9Sstevel@tonic-gate /* 26447c478bd9Sstevel@tonic-gate * Initialize the platform-specific aligned/DMA memory allocator. 26457c478bd9Sstevel@tonic-gate */ 26467c478bd9Sstevel@tonic-gate ka_init(); 26477c478bd9Sstevel@tonic-gate 26487c478bd9Sstevel@tonic-gate /* 26497c478bd9Sstevel@tonic-gate * Initialize 32-bit ID cache. 26507c478bd9Sstevel@tonic-gate */ 26517c478bd9Sstevel@tonic-gate id32_init(); 2652*f4b3ec61Sdh 2653*f4b3ec61Sdh /* 2654*f4b3ec61Sdh * Initialize the networking stack so modules loaded can 2655*f4b3ec61Sdh * register their callbacks. 2656*f4b3ec61Sdh */ 2657*f4b3ec61Sdh netstack_init(); 26587c478bd9Sstevel@tonic-gate } 26597c478bd9Sstevel@tonic-gate 26607c478bd9Sstevel@tonic-gate void 26617c478bd9Sstevel@tonic-gate kmem_thread_init(void) 26627c478bd9Sstevel@tonic-gate { 26637c478bd9Sstevel@tonic-gate kmem_taskq = taskq_create_instance("kmem_taskq", 0, 1, minclsyspri, 26647c478bd9Sstevel@tonic-gate 300, INT_MAX, TASKQ_PREPOPULATE); 26657c478bd9Sstevel@tonic-gate } 26667c478bd9Sstevel@tonic-gate 26677c478bd9Sstevel@tonic-gate void 26687c478bd9Sstevel@tonic-gate kmem_mp_init(void) 26697c478bd9Sstevel@tonic-gate { 26707c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 26717c478bd9Sstevel@tonic-gate register_cpu_setup_func(kmem_cpu_setup, NULL); 26727c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 26737c478bd9Sstevel@tonic-gate 26747c478bd9Sstevel@tonic-gate kmem_update_timeout(NULL); 26757c478bd9Sstevel@tonic-gate } 2676