17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5789d94c2Sjwadams * Common Development and Distribution License (the "License"). 6789d94c2Sjwadams * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 21e8031f0aSraf 227c478bd9Sstevel@tonic-gate /* 23a574db85Sraf * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 247c478bd9Sstevel@tonic-gate * Use is subject to license terms. 257c478bd9Sstevel@tonic-gate */ 267c478bd9Sstevel@tonic-gate 274f364e7cSRobert Mustacchi /* 28b1e2e3fbSRobert Mustacchi * Copyright (c) 2019 Joyent, Inc. 29831abf2cSDan Kimmel * Copyright (c) 2015 by Delphix. All rights reserved. 304f364e7cSRobert Mustacchi */ 314f364e7cSRobert Mustacchi 327c478bd9Sstevel@tonic-gate /* 337c478bd9Sstevel@tonic-gate * based on usr/src/uts/common/os/kmem.c r1.64 from 2001/12/18 347c478bd9Sstevel@tonic-gate * 357c478bd9Sstevel@tonic-gate * The slab allocator, as described in the following two papers: 367c478bd9Sstevel@tonic-gate * 377c478bd9Sstevel@tonic-gate * Jeff Bonwick, 387c478bd9Sstevel@tonic-gate * The Slab Allocator: An Object-Caching Kernel Memory Allocator. 397c478bd9Sstevel@tonic-gate * Proceedings of the Summer 1994 Usenix Conference. 407c478bd9Sstevel@tonic-gate * Available as /shared/sac/PSARC/1994/028/materials/kmem.pdf. 417c478bd9Sstevel@tonic-gate * 427c478bd9Sstevel@tonic-gate * Jeff Bonwick and Jonathan Adams, 437c478bd9Sstevel@tonic-gate * Magazines and vmem: Extending the Slab Allocator to Many CPUs and 447c478bd9Sstevel@tonic-gate * Arbitrary Resources. 457c478bd9Sstevel@tonic-gate * Proceedings of the 2001 Usenix Conference. 467c478bd9Sstevel@tonic-gate * Available as /shared/sac/PSARC/2000/550/materials/vmem.pdf. 477c478bd9Sstevel@tonic-gate * 487c478bd9Sstevel@tonic-gate * 1. Overview 497c478bd9Sstevel@tonic-gate * ----------- 504f364e7cSRobert Mustacchi * umem is very close to kmem in implementation. There are seven major 517c478bd9Sstevel@tonic-gate * areas of divergence: 527c478bd9Sstevel@tonic-gate * 537c478bd9Sstevel@tonic-gate * * Initialization 547c478bd9Sstevel@tonic-gate * 557c478bd9Sstevel@tonic-gate * * CPU handling 567c478bd9Sstevel@tonic-gate * 577c478bd9Sstevel@tonic-gate * * umem_update() 587c478bd9Sstevel@tonic-gate * 597c478bd9Sstevel@tonic-gate * * KM_SLEEP v.s. UMEM_NOFAIL 607c478bd9Sstevel@tonic-gate * 611c326e94Sjwadams * * lock ordering 627c478bd9Sstevel@tonic-gate * 634f364e7cSRobert Mustacchi * * changing UMEM_MAXBUF 644f364e7cSRobert Mustacchi * 654f364e7cSRobert Mustacchi * * Per-thread caching for malloc/free 664f364e7cSRobert Mustacchi * 677c478bd9Sstevel@tonic-gate * 2. Initialization 687c478bd9Sstevel@tonic-gate * ----------------- 697c478bd9Sstevel@tonic-gate * kmem is initialized early on in boot, and knows that no one will call 707c478bd9Sstevel@tonic-gate * into it before it is ready. umem does not have these luxuries. Instead, 717c478bd9Sstevel@tonic-gate * initialization is divided into two phases: 727c478bd9Sstevel@tonic-gate * 737c478bd9Sstevel@tonic-gate * * library initialization, and 747c478bd9Sstevel@tonic-gate * 757c478bd9Sstevel@tonic-gate * * first use 767c478bd9Sstevel@tonic-gate * 777c478bd9Sstevel@tonic-gate * umem's full initialization happens at the time of the first allocation 787c478bd9Sstevel@tonic-gate * request (via malloc() and friends, umem_alloc(), or umem_zalloc()), 797c478bd9Sstevel@tonic-gate * or the first call to umem_cache_create(). 807c478bd9Sstevel@tonic-gate * 817c478bd9Sstevel@tonic-gate * umem_free(), and umem_cache_alloc() do not require special handling, 827c478bd9Sstevel@tonic-gate * since the only way to get valid arguments for them is to successfully 837c478bd9Sstevel@tonic-gate * call a function from the first group. 847c478bd9Sstevel@tonic-gate * 857c478bd9Sstevel@tonic-gate * 2.1. Library Initialization: umem_startup() 867c478bd9Sstevel@tonic-gate * ------------------------------------------- 877c478bd9Sstevel@tonic-gate * umem_startup() is libumem.so's .init section. It calls pthread_atfork() 887c478bd9Sstevel@tonic-gate * to install the handlers necessary for umem's Fork1-Safety. Because of 897c478bd9Sstevel@tonic-gate * race condition issues, all other pre-umem_init() initialization is done 907c478bd9Sstevel@tonic-gate * statically (i.e. by the dynamic linker). 917c478bd9Sstevel@tonic-gate * 927c478bd9Sstevel@tonic-gate * For standalone use, umem_startup() returns everything to its initial 937c478bd9Sstevel@tonic-gate * state. 947c478bd9Sstevel@tonic-gate * 957c478bd9Sstevel@tonic-gate * 2.2. First use: umem_init() 967c478bd9Sstevel@tonic-gate * ------------------------------ 977c478bd9Sstevel@tonic-gate * The first time any memory allocation function is used, we have to 987c478bd9Sstevel@tonic-gate * create the backing caches and vmem arenas which are needed for it. 997c478bd9Sstevel@tonic-gate * umem_init() is the central point for that task. When it completes, 1007c478bd9Sstevel@tonic-gate * umem_ready is either UMEM_READY (all set) or UMEM_READY_INIT_FAILED (unable 1017c478bd9Sstevel@tonic-gate * to initialize, probably due to lack of memory). 1027c478bd9Sstevel@tonic-gate * 1037c478bd9Sstevel@tonic-gate * There are four different paths from which umem_init() is called: 1047c478bd9Sstevel@tonic-gate * 1057c478bd9Sstevel@tonic-gate * * from umem_alloc() or umem_zalloc(), with 0 < size < UMEM_MAXBUF, 1067c478bd9Sstevel@tonic-gate * 1077c478bd9Sstevel@tonic-gate * * from umem_alloc() or umem_zalloc(), with size > UMEM_MAXBUF, 1087c478bd9Sstevel@tonic-gate * 1097c478bd9Sstevel@tonic-gate * * from umem_cache_create(), and 1107c478bd9Sstevel@tonic-gate * 1117c478bd9Sstevel@tonic-gate * * from memalign(), with align > UMEM_ALIGN. 1127c478bd9Sstevel@tonic-gate * 1137c478bd9Sstevel@tonic-gate * The last three just check if umem is initialized, and call umem_init() 1147c478bd9Sstevel@tonic-gate * if it is not. For performance reasons, the first case is more complicated. 1157c478bd9Sstevel@tonic-gate * 1167c478bd9Sstevel@tonic-gate * 2.2.1. umem_alloc()/umem_zalloc(), with 0 < size < UMEM_MAXBUF 1177c478bd9Sstevel@tonic-gate * ----------------------------------------------------------------- 1187c478bd9Sstevel@tonic-gate * In this case, umem_cache_alloc(&umem_null_cache, ...) is called. 1197c478bd9Sstevel@tonic-gate * There is special case code in which causes any allocation on 1207c478bd9Sstevel@tonic-gate * &umem_null_cache to fail by returning (NULL), regardless of the 1217c478bd9Sstevel@tonic-gate * flags argument. 1227c478bd9Sstevel@tonic-gate * 1237c478bd9Sstevel@tonic-gate * So umem_cache_alloc() returns NULL, and umem_alloc()/umem_zalloc() call 1247c478bd9Sstevel@tonic-gate * umem_alloc_retry(). umem_alloc_retry() sees that the allocation 1257c478bd9Sstevel@tonic-gate * was agains &umem_null_cache, and calls umem_init(). 1267c478bd9Sstevel@tonic-gate * 1277c478bd9Sstevel@tonic-gate * If initialization is successful, umem_alloc_retry() returns 1, which 1287c478bd9Sstevel@tonic-gate * causes umem_alloc()/umem_zalloc() to start over, which causes it to load 1297c478bd9Sstevel@tonic-gate * the (now valid) cache pointer from umem_alloc_table. 1307c478bd9Sstevel@tonic-gate * 1317c478bd9Sstevel@tonic-gate * 2.2.2. Dealing with race conditions 1327c478bd9Sstevel@tonic-gate * ----------------------------------- 1337c478bd9Sstevel@tonic-gate * There are a couple race conditions resulting from the initialization 1347c478bd9Sstevel@tonic-gate * code that we have to guard against: 1357c478bd9Sstevel@tonic-gate * 1367c478bd9Sstevel@tonic-gate * * In umem_cache_create(), there is a special UMC_INTERNAL cflag 1377c478bd9Sstevel@tonic-gate * that is passed for caches created during initialization. It 1387c478bd9Sstevel@tonic-gate * is illegal for a user to try to create a UMC_INTERNAL cache. 1397c478bd9Sstevel@tonic-gate * This allows initialization to proceed, but any other 1407c478bd9Sstevel@tonic-gate * umem_cache_create()s will block by calling umem_init(). 1417c478bd9Sstevel@tonic-gate * 1427c478bd9Sstevel@tonic-gate * * Since umem_null_cache has a 1-element cache_cpu, it's cache_cpu_mask 1437c478bd9Sstevel@tonic-gate * is always zero. umem_cache_alloc uses cp->cache_cpu_mask to 1447c478bd9Sstevel@tonic-gate * mask the cpu number. This prevents a race between grabbing a 1457c478bd9Sstevel@tonic-gate * cache pointer out of umem_alloc_table and growing the cpu array. 1467c478bd9Sstevel@tonic-gate * 1477c478bd9Sstevel@tonic-gate * 1487c478bd9Sstevel@tonic-gate * 3. CPU handling 1497c478bd9Sstevel@tonic-gate * --------------- 1507c478bd9Sstevel@tonic-gate * kmem uses the CPU's sequence number to determine which "cpu cache" to 1517c478bd9Sstevel@tonic-gate * use for an allocation. Currently, there is no way to get the sequence 1527c478bd9Sstevel@tonic-gate * number in userspace. 1537c478bd9Sstevel@tonic-gate * 1547c478bd9Sstevel@tonic-gate * umem keeps track of cpu information in umem_cpus, an array of umem_max_ncpus 1557c478bd9Sstevel@tonic-gate * umem_cpu_t structures. CURCPU() is a a "hint" function, which we then mask 1567c478bd9Sstevel@tonic-gate * with either umem_cpu_mask or cp->cache_cpu_mask to find the actual "cpu" id. 1577c478bd9Sstevel@tonic-gate * The mechanics of this is all in the CPU(mask) macro. 1587c478bd9Sstevel@tonic-gate * 1597c478bd9Sstevel@tonic-gate * Currently, umem uses _lwp_self() as its hint. 1607c478bd9Sstevel@tonic-gate * 1617c478bd9Sstevel@tonic-gate * 1627c478bd9Sstevel@tonic-gate * 4. The update thread 1637c478bd9Sstevel@tonic-gate * -------------------- 1647c478bd9Sstevel@tonic-gate * kmem uses a task queue, kmem_taskq, to do periodic maintenance on 1657c478bd9Sstevel@tonic-gate * every kmem cache. vmem has a periodic timeout for hash table resizing. 1667c478bd9Sstevel@tonic-gate * The kmem_taskq also provides a separate context for kmem_cache_reap()'s 1677c478bd9Sstevel@tonic-gate * to be done in, avoiding issues of the context of kmem_reap() callers. 1687c478bd9Sstevel@tonic-gate * 1697c478bd9Sstevel@tonic-gate * Instead, umem has the concept of "updates", which are asynchronous requests 1707c478bd9Sstevel@tonic-gate * for work attached to single caches. All caches with pending work are 1717c478bd9Sstevel@tonic-gate * on a doubly linked list rooted at the umem_null_cache. All update state 1727c478bd9Sstevel@tonic-gate * is protected by the umem_update_lock mutex, and the umem_update_cv is used 1737c478bd9Sstevel@tonic-gate * for notification between threads. 1747c478bd9Sstevel@tonic-gate * 1757c478bd9Sstevel@tonic-gate * 4.1. Cache states with regards to updates 1767c478bd9Sstevel@tonic-gate * ----------------------------------------- 1777c478bd9Sstevel@tonic-gate * A given cache is in one of three states: 1787c478bd9Sstevel@tonic-gate * 1797c478bd9Sstevel@tonic-gate * Inactive cache_uflags is zero, cache_u{next,prev} are NULL 1807c478bd9Sstevel@tonic-gate * 1817c478bd9Sstevel@tonic-gate * Work Requested cache_uflags is non-zero (but UMU_ACTIVE is not set), 1827c478bd9Sstevel@tonic-gate * cache_u{next,prev} link the cache onto the global 1837c478bd9Sstevel@tonic-gate * update list 1847c478bd9Sstevel@tonic-gate * 1857c478bd9Sstevel@tonic-gate * Active cache_uflags has UMU_ACTIVE set, cache_u{next,prev} 1867c478bd9Sstevel@tonic-gate * are NULL, and either umem_update_thr or 1877c478bd9Sstevel@tonic-gate * umem_st_update_thr are actively doing work on the 1887c478bd9Sstevel@tonic-gate * cache. 1897c478bd9Sstevel@tonic-gate * 1907c478bd9Sstevel@tonic-gate * An update can be added to any cache in any state -- if the cache is 1917c478bd9Sstevel@tonic-gate * Inactive, it transitions to being Work Requested. If the cache is 1927c478bd9Sstevel@tonic-gate * Active, the worker will notice the new update and act on it before 1937c478bd9Sstevel@tonic-gate * transitioning the cache to the Inactive state. 1947c478bd9Sstevel@tonic-gate * 1957c478bd9Sstevel@tonic-gate * If a cache is in the Active state, UMU_NOTIFY can be set, which asks 1967c478bd9Sstevel@tonic-gate * the worker to broadcast the umem_update_cv when it has finished. 1977c478bd9Sstevel@tonic-gate * 1987c478bd9Sstevel@tonic-gate * 4.2. Update interface 1997c478bd9Sstevel@tonic-gate * --------------------- 2007c478bd9Sstevel@tonic-gate * umem_add_update() adds an update to a particular cache. 2017c478bd9Sstevel@tonic-gate * umem_updateall() adds an update to all caches. 2027c478bd9Sstevel@tonic-gate * umem_remove_updates() returns a cache to the Inactive state. 2037c478bd9Sstevel@tonic-gate * 2047c478bd9Sstevel@tonic-gate * umem_process_updates() process all caches in the Work Requested state. 2057c478bd9Sstevel@tonic-gate * 2067c478bd9Sstevel@tonic-gate * 4.3. Reaping 2077c478bd9Sstevel@tonic-gate * ------------ 2087c478bd9Sstevel@tonic-gate * When umem_reap() is called (at the time of heap growth), it schedule 2097c478bd9Sstevel@tonic-gate * UMU_REAP updates on every cache. It then checks to see if the update 2107c478bd9Sstevel@tonic-gate * thread exists (umem_update_thr != 0). If it is, it broadcasts 2117c478bd9Sstevel@tonic-gate * the umem_update_cv to wake the update thread up, and returns. 2127c478bd9Sstevel@tonic-gate * 2137c478bd9Sstevel@tonic-gate * If the update thread does not exist (umem_update_thr == 0), and the 2147c478bd9Sstevel@tonic-gate * program currently has multiple threads, umem_reap() attempts to create 2157c478bd9Sstevel@tonic-gate * a new update thread. 2167c478bd9Sstevel@tonic-gate * 2177c478bd9Sstevel@tonic-gate * If the process is not multithreaded, or the creation fails, umem_reap() 2187c478bd9Sstevel@tonic-gate * calls umem_st_update() to do an inline update. 2197c478bd9Sstevel@tonic-gate * 2207c478bd9Sstevel@tonic-gate * 4.4. The update thread 2217c478bd9Sstevel@tonic-gate * ---------------------- 2227c478bd9Sstevel@tonic-gate * The update thread spends most of its time in cond_timedwait() on the 2237c478bd9Sstevel@tonic-gate * umem_update_cv. It wakes up under two conditions: 2247c478bd9Sstevel@tonic-gate * 2257c478bd9Sstevel@tonic-gate * * The timedwait times out, in which case it needs to run a global 2267c478bd9Sstevel@tonic-gate * update, or 2277c478bd9Sstevel@tonic-gate * 2287c478bd9Sstevel@tonic-gate * * someone cond_broadcast(3THR)s the umem_update_cv, in which case 2297c478bd9Sstevel@tonic-gate * it needs to check if there are any caches in the Work Requested 2307c478bd9Sstevel@tonic-gate * state. 2317c478bd9Sstevel@tonic-gate * 2327c478bd9Sstevel@tonic-gate * When it is time for another global update, umem calls umem_cache_update() 2337c478bd9Sstevel@tonic-gate * on every cache, then calls vmem_update(), which tunes the vmem structures. 2347c478bd9Sstevel@tonic-gate * umem_cache_update() can request further work using umem_add_update(). 2357c478bd9Sstevel@tonic-gate * 2367c478bd9Sstevel@tonic-gate * After any work from the global update completes, the update timer is 2377c478bd9Sstevel@tonic-gate * reset to umem_reap_interval seconds in the future. This makes the 2387c478bd9Sstevel@tonic-gate * updates self-throttling. 2397c478bd9Sstevel@tonic-gate * 2407c478bd9Sstevel@tonic-gate * Reaps are similarly self-throttling. After a UMU_REAP update has 2417c478bd9Sstevel@tonic-gate * been scheduled on all caches, umem_reap() sets a flag and wakes up the 2427c478bd9Sstevel@tonic-gate * update thread. The update thread notices the flag, and resets the 2437c478bd9Sstevel@tonic-gate * reap state. 2447c478bd9Sstevel@tonic-gate * 2457c478bd9Sstevel@tonic-gate * 4.5. Inline updates 2467c478bd9Sstevel@tonic-gate * ------------------- 2477c478bd9Sstevel@tonic-gate * If the update thread is not running, umem_st_update() is used instead. It 2487c478bd9Sstevel@tonic-gate * immediately does a global update (as above), then calls 2497c478bd9Sstevel@tonic-gate * umem_process_updates() to process both the reaps that umem_reap() added and 2507c478bd9Sstevel@tonic-gate * any work generated by the global update. Afterwards, it resets the reap 2517c478bd9Sstevel@tonic-gate * state. 2527c478bd9Sstevel@tonic-gate * 2537c478bd9Sstevel@tonic-gate * While the umem_st_update() is running, umem_st_update_thr holds the thread 2547c478bd9Sstevel@tonic-gate * id of the thread performing the update. 2557c478bd9Sstevel@tonic-gate * 2567c478bd9Sstevel@tonic-gate * 4.6. Updates and fork1() 2577c478bd9Sstevel@tonic-gate * ------------------------ 2587c478bd9Sstevel@tonic-gate * umem has fork1() pre- and post-handlers which lock up (and release) every 2597c478bd9Sstevel@tonic-gate * mutex in every cache. They also lock up the umem_update_lock. Since 2607c478bd9Sstevel@tonic-gate * fork1() only copies over a single lwp, other threads (including the update 2617c478bd9Sstevel@tonic-gate * thread) could have been actively using a cache in the parent. This 2627c478bd9Sstevel@tonic-gate * can lead to inconsistencies in the child process. 2637c478bd9Sstevel@tonic-gate * 2647c478bd9Sstevel@tonic-gate * Because we locked all of the mutexes, the only possible inconsistancies are: 2657c478bd9Sstevel@tonic-gate * 2667c478bd9Sstevel@tonic-gate * * a umem_cache_alloc() could leak its buffer. 2677c478bd9Sstevel@tonic-gate * 2687c478bd9Sstevel@tonic-gate * * a caller of umem_depot_alloc() could leak a magazine, and all the 2697c478bd9Sstevel@tonic-gate * buffers contained in it. 2707c478bd9Sstevel@tonic-gate * 2717c478bd9Sstevel@tonic-gate * * a cache could be in the Active update state. In the child, there 2727c478bd9Sstevel@tonic-gate * would be no thread actually working on it. 2737c478bd9Sstevel@tonic-gate * 2747c478bd9Sstevel@tonic-gate * * a umem_hash_rescale() could leak the new hash table. 2757c478bd9Sstevel@tonic-gate * 2767c478bd9Sstevel@tonic-gate * * a umem_magazine_resize() could be in progress. 2777c478bd9Sstevel@tonic-gate * 2787c478bd9Sstevel@tonic-gate * * a umem_reap() could be in progress. 2797c478bd9Sstevel@tonic-gate * 2807c478bd9Sstevel@tonic-gate * The memory leaks we can't do anything about. umem_release_child() resets 2817c478bd9Sstevel@tonic-gate * the update state, moves any caches in the Active state to the Work Requested 2827c478bd9Sstevel@tonic-gate * state. This might cause some updates to be re-run, but UMU_REAP and 2837c478bd9Sstevel@tonic-gate * UMU_HASH_RESCALE are effectively idempotent, and the worst that can 2847c478bd9Sstevel@tonic-gate * happen from umem_magazine_resize() is resizing the magazine twice in close 2857c478bd9Sstevel@tonic-gate * succession. 2867c478bd9Sstevel@tonic-gate * 2877c478bd9Sstevel@tonic-gate * Much of the cleanup in umem_release_child() is skipped if 2887c478bd9Sstevel@tonic-gate * umem_st_update_thr == thr_self(). This is so that applications which call 2897c478bd9Sstevel@tonic-gate * fork1() from a cache callback does not break. Needless to say, any such 2907c478bd9Sstevel@tonic-gate * application is tremendously broken. 2917c478bd9Sstevel@tonic-gate * 2927c478bd9Sstevel@tonic-gate * 2937c478bd9Sstevel@tonic-gate * 5. KM_SLEEP v.s. UMEM_NOFAIL 2947c478bd9Sstevel@tonic-gate * ---------------------------- 2957c478bd9Sstevel@tonic-gate * Allocations against kmem and vmem have two basic modes: SLEEP and 2967c478bd9Sstevel@tonic-gate * NOSLEEP. A sleeping allocation is will go to sleep (waiting for 2977c478bd9Sstevel@tonic-gate * more memory) instead of failing (returning NULL). 2987c478bd9Sstevel@tonic-gate * 2997c478bd9Sstevel@tonic-gate * SLEEP allocations presume an extremely multithreaded model, with 3007c478bd9Sstevel@tonic-gate * a lot of allocation and deallocation activity. umem cannot presume 3017c478bd9Sstevel@tonic-gate * that its clients have any particular type of behavior. Instead, 3027c478bd9Sstevel@tonic-gate * it provides two types of allocations: 3037c478bd9Sstevel@tonic-gate * 3047c478bd9Sstevel@tonic-gate * * UMEM_DEFAULT, equivalent to KM_NOSLEEP (i.e. return NULL on 3057c478bd9Sstevel@tonic-gate * failure) 3067c478bd9Sstevel@tonic-gate * 3077c478bd9Sstevel@tonic-gate * * UMEM_NOFAIL, which, on failure, calls an optional callback 3087c478bd9Sstevel@tonic-gate * (registered with umem_nofail_callback()). 3097c478bd9Sstevel@tonic-gate * 3107c478bd9Sstevel@tonic-gate * The callback is invoked with no locks held, and can do an arbitrary 3117c478bd9Sstevel@tonic-gate * amount of work. It then has a choice between: 3127c478bd9Sstevel@tonic-gate * 3137c478bd9Sstevel@tonic-gate * * Returning UMEM_CALLBACK_RETRY, which will cause the allocation 3147c478bd9Sstevel@tonic-gate * to be restarted. 3157c478bd9Sstevel@tonic-gate * 3167c478bd9Sstevel@tonic-gate * * Returning UMEM_CALLBACK_EXIT(status), which will cause exit(2) 3177c478bd9Sstevel@tonic-gate * to be invoked with status. If multiple threads attempt to do 3187c478bd9Sstevel@tonic-gate * this simultaneously, only one will call exit(2). 3197c478bd9Sstevel@tonic-gate * 3207c478bd9Sstevel@tonic-gate * * Doing some kind of non-local exit (thr_exit(3thr), longjmp(3C), 3217c478bd9Sstevel@tonic-gate * etc.) 3227c478bd9Sstevel@tonic-gate * 3237c478bd9Sstevel@tonic-gate * The default callback returns UMEM_CALLBACK_EXIT(255). 3247c478bd9Sstevel@tonic-gate * 3257c478bd9Sstevel@tonic-gate * To have these callbacks without risk of state corruption (in the case of 3267c478bd9Sstevel@tonic-gate * a non-local exit), we have to ensure that the callbacks get invoked 3277c478bd9Sstevel@tonic-gate * close to the original allocation, with no inconsistent state or held 3287c478bd9Sstevel@tonic-gate * locks. The following steps are taken: 3297c478bd9Sstevel@tonic-gate * 3307c478bd9Sstevel@tonic-gate * * All invocations of vmem are VM_NOSLEEP. 3317c478bd9Sstevel@tonic-gate * 3327c478bd9Sstevel@tonic-gate * * All constructor callbacks (which can themselves to allocations) 3337c478bd9Sstevel@tonic-gate * are passed UMEM_DEFAULT as their required allocation argument. This 3347c478bd9Sstevel@tonic-gate * way, the constructor will fail, allowing the highest-level allocation 3357c478bd9Sstevel@tonic-gate * invoke the nofail callback. 3367c478bd9Sstevel@tonic-gate * 3377c478bd9Sstevel@tonic-gate * If a constructor callback _does_ do a UMEM_NOFAIL allocation, and 3387c478bd9Sstevel@tonic-gate * the nofail callback does a non-local exit, we will leak the 3397c478bd9Sstevel@tonic-gate * partially-constructed buffer. 3401c326e94Sjwadams * 3411c326e94Sjwadams * 3421c326e94Sjwadams * 6. Lock Ordering 3431c326e94Sjwadams * ---------------- 3441c326e94Sjwadams * umem has a few more locks than kmem does, mostly in the update path. The 3451c326e94Sjwadams * overall lock ordering (earlier locks must be acquired first) is: 3461c326e94Sjwadams * 3471c326e94Sjwadams * umem_init_lock 3481c326e94Sjwadams * 3491c326e94Sjwadams * vmem_list_lock 3501c326e94Sjwadams * vmem_nosleep_lock.vmpl_mutex 3511c326e94Sjwadams * vmem_t's: 3521c326e94Sjwadams * vm_lock 353789d94c2Sjwadams * sbrk_lock 3541c326e94Sjwadams * 3551c326e94Sjwadams * umem_cache_lock 3561c326e94Sjwadams * umem_update_lock 3571c326e94Sjwadams * umem_flags_lock 3581c326e94Sjwadams * umem_cache_t's: 3591c326e94Sjwadams * cache_cpu[*].cc_lock 3601c326e94Sjwadams * cache_depot_lock 3611c326e94Sjwadams * cache_lock 3621c326e94Sjwadams * umem_log_header_t's: 3631c326e94Sjwadams * lh_cpu[*].clh_lock 3641c326e94Sjwadams * lh_lock 36538849194SRobert Mustacchi * 36638849194SRobert Mustacchi * 7. Changing UMEM_MAXBUF 36738849194SRobert Mustacchi * ----------------------- 36838849194SRobert Mustacchi * 36938849194SRobert Mustacchi * When changing UMEM_MAXBUF extra care has to be taken. It is not sufficient to 37038849194SRobert Mustacchi * simply increase this number. First, one must update the umem_alloc_table to 37138849194SRobert Mustacchi * have the appropriate number of entires based upon the new size. If this is 37238849194SRobert Mustacchi * not done, this will lead to libumem blowing an assertion. 37338849194SRobert Mustacchi * 37438849194SRobert Mustacchi * The second place to update, which is not required, is the umem_alloc_sizes. 37538849194SRobert Mustacchi * These determine the default cache sizes that we're going to support. 3764f364e7cSRobert Mustacchi * 3774f364e7cSRobert Mustacchi * 8. Per-thread caching for malloc/free 3784f364e7cSRobert Mustacchi * ------------------------------------- 3794f364e7cSRobert Mustacchi * 3804f364e7cSRobert Mustacchi * "Time is an illusion. Lunchtime doubly so." -- Douglas Adams 3814f364e7cSRobert Mustacchi * 3824f364e7cSRobert Mustacchi * Time may be an illusion, but CPU cycles aren't. While libumem is designed 3834f364e7cSRobert Mustacchi * to be a highly scalable allocator, that scalability comes with a fixed cycle 3844f364e7cSRobert Mustacchi * penalty even in the absence of contention: libumem must acquire (and release 3854f364e7cSRobert Mustacchi * a per-CPU lock for each allocation. When contention is low and malloc(3C) 3864f364e7cSRobert Mustacchi * frequency is high, this overhead can dominate execution time. To alleviate 3874f364e7cSRobert Mustacchi * this, we allow for per-thread caching, a lock-free means of caching recent 3884f364e7cSRobert Mustacchi * deallocations on a per-thread basis for use in satisfying subsequent calls 3894f364e7cSRobert Mustacchi * 3904f364e7cSRobert Mustacchi * In addition to improving performance, we also want to: 3914f364e7cSRobert Mustacchi * * Minimize fragmentation 3924f364e7cSRobert Mustacchi * * Not add additional memory overhead (no larger malloc tags) 3934f364e7cSRobert Mustacchi * 3944f364e7cSRobert Mustacchi * In the ulwp_t of each thread there is a private data structure called a 3954f364e7cSRobert Mustacchi * umem_t that looks like: 3964f364e7cSRobert Mustacchi * 3974f364e7cSRobert Mustacchi * typedef struct { 398*14702342SRobert Mustacchi * size_t tm_size; 399*14702342SRobert Mustacchi * void *tm_roots[NTMEMBASE]; (Currently 16) 4004f364e7cSRobert Mustacchi * } tmem_t; 4014f364e7cSRobert Mustacchi * 4024f364e7cSRobert Mustacchi * Each of the roots is treated as the head of a linked list. Each entry in the 4034f364e7cSRobert Mustacchi * list can be thought of as a void ** which points to the next entry, until one 4044f364e7cSRobert Mustacchi * of them points to NULL. If the head points to NULL, the list is empty. 4054f364e7cSRobert Mustacchi * 4064f364e7cSRobert Mustacchi * Each head corresponds to a umem_cache. Currently there is a linear mapping 4074f364e7cSRobert Mustacchi * where the first root corresponds to the first cache, second root to the 4084f364e7cSRobert Mustacchi * second cache, etc. This works because every allocation that malloc makes to 4094f364e7cSRobert Mustacchi * umem_alloc that can be satisified by a umem_cache will actually return a 4104f364e7cSRobert Mustacchi * number of bytes equal to the size of that cache. Because of this property and 4114f364e7cSRobert Mustacchi * a one to one mapping between caches and roots we can guarantee that every 4124f364e7cSRobert Mustacchi * entry in a given root's list will be able to satisfy the same requests as the 4134f364e7cSRobert Mustacchi * corresponding cache. 4144f364e7cSRobert Mustacchi * 4154f364e7cSRobert Mustacchi * The choice of sixteen roots is based on where we believe we get the biggest 4164f364e7cSRobert Mustacchi * bang for our buck. The per-thread caches will cache up to 256 byte and 448 4174f364e7cSRobert Mustacchi * byte allocations on ILP32 and LP64 respectively. Generally applications plan 4184f364e7cSRobert Mustacchi * more carefully how they do larger allocations than smaller ones. Therefore 4194f364e7cSRobert Mustacchi * sixteen roots is a reasonable compromise between the amount of additional 4204f364e7cSRobert Mustacchi * overhead per thread, and the likelihood of a program to benefit from it. 4214f364e7cSRobert Mustacchi * 4224f364e7cSRobert Mustacchi * The maximum amount of memory that can be cached in each thread is determined 4234f364e7cSRobert Mustacchi * by the perthread_cache UMEM_OPTION. It corresponds to the umem_ptc_size 4244f364e7cSRobert Mustacchi * value. The default value for this is currently 1 MB. Once umem_init() has 4254f364e7cSRobert Mustacchi * finished this cannot be directly tuned without directly modifying the 4264f364e7cSRobert Mustacchi * instruction text. If, upon calling free(3C), the amount cached would exceed 4274f364e7cSRobert Mustacchi * this maximum, we instead actually return the buffer to the umem_cache instead 4284f364e7cSRobert Mustacchi * of holding onto it in the thread. 4294f364e7cSRobert Mustacchi * 4304f364e7cSRobert Mustacchi * When a thread calls malloc(3C) it first determines which umem_cache it 4314f364e7cSRobert Mustacchi * would be serviced by. If the allocation is not covered by ptcumem it goes to 4324f364e7cSRobert Mustacchi * the normal malloc instead. Next, it checks if the tmem_root's list is empty 4334f364e7cSRobert Mustacchi * or not. If it is empty, we instead go and allocate the memory from 4344f364e7cSRobert Mustacchi * umem_alloc. If it is not empty, we remove the head of the list, set the 4354f364e7cSRobert Mustacchi * appropriate malloc tags, and return that buffer. 4364f364e7cSRobert Mustacchi * 4374f364e7cSRobert Mustacchi * When a thread calls free(3C) it first looks at the malloc tag and if it is 4384f364e7cSRobert Mustacchi * invalid or the allocation exceeds the largest cache in ptcumem and sends it 4394f364e7cSRobert Mustacchi * off to the original free() to handle and clean up appropriately. Next, it 4404f364e7cSRobert Mustacchi * checks if the allocation size is covered by one of the per-thread roots and 4414f364e7cSRobert Mustacchi * if it isn't, it passes it off to the original free() to be released. Finally, 4424f364e7cSRobert Mustacchi * before it inserts this buffer as the head, it checks if adding this buffer 4434f364e7cSRobert Mustacchi * would put the thread over its maximum cache size. If it would, it frees the 4444f364e7cSRobert Mustacchi * buffer back to the umem_cache. Otherwise it increments the threads total 4454f364e7cSRobert Mustacchi * cached amount and makes the buffer the new head of the appropriate tm_root. 4464f364e7cSRobert Mustacchi * 4474f364e7cSRobert Mustacchi * When a thread exits, all of the buffers that it has in its per-thread cache 4484f364e7cSRobert Mustacchi * will be passed to umem_free() and returned to the appropriate umem_cache. 4494f364e7cSRobert Mustacchi * 4504f364e7cSRobert Mustacchi * 8.1 Handling addition and removal of umem_caches 4514f364e7cSRobert Mustacchi * ------------------------------------------------ 4524f364e7cSRobert Mustacchi * 4534f364e7cSRobert Mustacchi * The set of umem_caches that are used to back calls to umem_alloc() and 4544f364e7cSRobert Mustacchi * ultimately malloc() are determined at program execution time. The default set 4554f364e7cSRobert Mustacchi * of caches is defined below in umem_alloc_sizes[]. Various umem_options exist 4564f364e7cSRobert Mustacchi * that modify the set of caches: size_add, size_clear, and size_remove. Because 4574f364e7cSRobert Mustacchi * the set of caches can only be determined once umem_init() has been called and 4584f364e7cSRobert Mustacchi * we have the additional goals of minimizing additional fragmentation and 4594f364e7cSRobert Mustacchi * metadata space overhead in the malloc tags, this forces our hand to go down a 4604f364e7cSRobert Mustacchi * slightly different path: the one tread by fasttrap and trapstat. 4614f364e7cSRobert Mustacchi * 4624f364e7cSRobert Mustacchi * During umem_init we're going to dynamically construct a new version of 4634f364e7cSRobert Mustacchi * malloc(3C) and free(3C) that utilizes the known cache sizes and then ensure 4644f364e7cSRobert Mustacchi * that ptcmalloc and ptcfree replace malloc and free as entries in the plt. If 4654f364e7cSRobert Mustacchi * ptcmalloc and ptcfree cannot handle a request, they simply jump to the 4664f364e7cSRobert Mustacchi * original libumem implementations. 4674f364e7cSRobert Mustacchi * 4684f364e7cSRobert Mustacchi * After creating all of the umem_caches, but before making them visible, 4694f364e7cSRobert Mustacchi * umem_cache_init checks that umem_genasm_supported is non-zero. This value is 4704f364e7cSRobert Mustacchi * set by each architecture in $ARCH/umem_genasm.c to indicate whether or not 4714f364e7cSRobert Mustacchi * they support this. If the value is zero, then this process is skipped. 4724f364e7cSRobert Mustacchi * Similarly, if the cache size has been tuned to zero by UMEM_OPTIONS, then 4734f364e7cSRobert Mustacchi * this is also skipped. 4744f364e7cSRobert Mustacchi * 4754f364e7cSRobert Mustacchi * In umem_genasm.c, each architecture's implementation implements a single 4764f364e7cSRobert Mustacchi * function called umem_genasm() that is responsible for generating the 4774f364e7cSRobert Mustacchi * appropriate versions of ptcmalloc() and ptcfree(), placing them in the 4784f364e7cSRobert Mustacchi * appropriate memory location, and finally doing the switch from malloc() and 4794f364e7cSRobert Mustacchi * free() to ptcmalloc() and ptcfree(). Once the change has been made, there is 4804f364e7cSRobert Mustacchi * no way to switch back, short of restarting the program or modifying program 4814f364e7cSRobert Mustacchi * text with mdb. 4824f364e7cSRobert Mustacchi * 4834f364e7cSRobert Mustacchi * 8.2 Modifying the Procedure Linkage Table (PLT) 4844f364e7cSRobert Mustacchi * ----------------------------------------------- 4854f364e7cSRobert Mustacchi * 4864f364e7cSRobert Mustacchi * The last piece of this puzzle is how we actually jam ptcmalloc() into the 4874f364e7cSRobert Mustacchi * PLT. To handle this, we have defined two functions, _malloc and _free and 4884f364e7cSRobert Mustacchi * used a special mapfile directive to place them into the a readable, 4894f364e7cSRobert Mustacchi * writeable, and executable segment. Next we use a standard #pragma weak for 4904f364e7cSRobert Mustacchi * malloc and free and direct them to those symbols. By default, those symbols 4914f364e7cSRobert Mustacchi * have text defined as nops for our generated functions and when they're 4924f364e7cSRobert Mustacchi * invoked, they jump to the default malloc and free functions. 4934f364e7cSRobert Mustacchi * 4944f364e7cSRobert Mustacchi * When umem_genasm() is called, it goes through and generates new malloc() and 4954f364e7cSRobert Mustacchi * free() functions in the text provided for by _malloc and _free just after the 4964f364e7cSRobert Mustacchi * jump. Once both have been successfully generated, umem_genasm() nops over the 4974f364e7cSRobert Mustacchi * original jump so that we now call into the genasm versions of these 4984f364e7cSRobert Mustacchi * functions. 4994f364e7cSRobert Mustacchi * 5004f364e7cSRobert Mustacchi * 8.3 umem_genasm() 5014f364e7cSRobert Mustacchi * ----------------- 5024f364e7cSRobert Mustacchi * 5034f364e7cSRobert Mustacchi * umem_genasm() is currently implemented for i386 and amd64. This section 5044f364e7cSRobert Mustacchi * describes the theory behind the construction. For specific byte code to 5054f364e7cSRobert Mustacchi * assembly instructions and niceish C and asm versions of ptcmalloc and 5064f364e7cSRobert Mustacchi * ptcfree, see the individual umem_genasm.c files. The layout consists of the 5074f364e7cSRobert Mustacchi * following sections: 5084f364e7cSRobert Mustacchi * 5094f364e7cSRobert Mustacchi * o. function-specfic prologue 5104f364e7cSRobert Mustacchi * o. function-generic cache-selecting elements 5114f364e7cSRobert Mustacchi * o. function-specific epilogue 5124f364e7cSRobert Mustacchi * 5134f364e7cSRobert Mustacchi * There are three different generic cache elements that exist: 5144f364e7cSRobert Mustacchi * 5154f364e7cSRobert Mustacchi * o. the last or only cache 5164f364e7cSRobert Mustacchi * o. the intermediary caches if more than two 5174f364e7cSRobert Mustacchi * o. the first one if more than one cache 5184f364e7cSRobert Mustacchi * 5194f364e7cSRobert Mustacchi * The malloc and free prologues and epilogues mimic the necessary portions of 5204f364e7cSRobert Mustacchi * libumem's malloc and free. This includes things like checking for size 5214f364e7cSRobert Mustacchi * overflow, setting and verifying the malloc tags. 5224f364e7cSRobert Mustacchi * 5234f364e7cSRobert Mustacchi * It is an important constraint that these functions do not make use of the 5244f364e7cSRobert Mustacchi * call instruction. The only jmp outside of the individual functions is to the 5254f364e7cSRobert Mustacchi * original libumem malloc and free respectively. Because doing things like 5264f364e7cSRobert Mustacchi * setting errno or raising an internal umem error on improper malloc tags would 5274f364e7cSRobert Mustacchi * require using calls into the PLT, whenever we encounter one of those cases we 5284f364e7cSRobert Mustacchi * just jump to the original malloc and free functions reusing the same stack 5294f364e7cSRobert Mustacchi * frame. 5304f364e7cSRobert Mustacchi * 5314f364e7cSRobert Mustacchi * Each of the above sections, the three caches, and the malloc and free 5324f364e7cSRobert Mustacchi * prologue and epilogue are implemented as blocks of machine code with the 5334f364e7cSRobert Mustacchi * corresponding assembly in comments. There are known offsets into each block 5344f364e7cSRobert Mustacchi * that corresponds to locations of data and addresses that we only know at run 5354f364e7cSRobert Mustacchi * time. These blocks are copied as necessary and the blanks filled in 5364f364e7cSRobert Mustacchi * appropriately. 5374f364e7cSRobert Mustacchi * 5384f364e7cSRobert Mustacchi * As mentioned in section 8.2, the trampoline library uses specifically named 5394f364e7cSRobert Mustacchi * variables to communicate the buffers and size to use. These variables are: 5404f364e7cSRobert Mustacchi * 5414f364e7cSRobert Mustacchi * o. umem_genasm_mptr: The buffer for ptcmalloc 5424f364e7cSRobert Mustacchi * o. umem_genasm_msize: The size in bytes of the above buffer 5434f364e7cSRobert Mustacchi * o. umem_genasm_fptr: The buffer for ptcfree 5444f364e7cSRobert Mustacchi * o. umem_genasm_fsize: The size in bytes of the above buffer 5454f364e7cSRobert Mustacchi * 5464f364e7cSRobert Mustacchi * Finally, to enable the generated assembly we need to remove the previous jump 5474f364e7cSRobert Mustacchi * to the actual malloc that exists at the start of these buffers. On x86, this 5484f364e7cSRobert Mustacchi * is a five byte region. We could zero out the jump offset to be a jmp +0, but 5494f364e7cSRobert Mustacchi * using nops can be faster. We specifically use a single five byte nop on x86 5504f364e7cSRobert Mustacchi * as it is faster. When porting ptcumem to other architectures, the various 5514f364e7cSRobert Mustacchi * opcode changes and options should be analyzed. 5524f364e7cSRobert Mustacchi * 5534f364e7cSRobert Mustacchi * 8.4 Interface with libc.so 5544f364e7cSRobert Mustacchi * -------------------------- 5554f364e7cSRobert Mustacchi * 5564f364e7cSRobert Mustacchi * The tmem_t structure as described in the beginning of section 8, is part of a 5574f364e7cSRobert Mustacchi * private interface with libc. There are three functions that exist to cover 5584f364e7cSRobert Mustacchi * this. They are not documented in man pages or header files. They are in the 5594f364e7cSRobert Mustacchi * SUNWprivate part of libc's mapfile. 5604f364e7cSRobert Mustacchi * 5614f364e7cSRobert Mustacchi * o. _tmem_get_base(void) 5624f364e7cSRobert Mustacchi * 563*14702342SRobert Mustacchi * Returns the offset from the ulwp_t (curthread) to the tmem_t structure. 564*14702342SRobert Mustacchi * This is a constant for all threads and is effectively a way to to do 565*14702342SRobert Mustacchi * ::offsetof ulwp_t ul_tmem without having to know the specifics of the 566*14702342SRobert Mustacchi * structure outside of libc. 5674f364e7cSRobert Mustacchi * 5684f364e7cSRobert Mustacchi * o. _tmem_get_nentries(void) 5694f364e7cSRobert Mustacchi * 5704f364e7cSRobert Mustacchi * Returns the number of roots that exist in the tmem_t. This is one part 5714f364e7cSRobert Mustacchi * of the cap on the number of umem_caches that we can back with tmem. 5724f364e7cSRobert Mustacchi * 5734f364e7cSRobert Mustacchi * o. _tmem_set_cleanup(void (*)(void *, int)) 5744f364e7cSRobert Mustacchi * 5754f364e7cSRobert Mustacchi * This sets a clean up handler that gets called back when a thread exits. 5764f364e7cSRobert Mustacchi * There is one call per buffer, the void * is a pointer to the buffer on 5774f364e7cSRobert Mustacchi * the list, the int is the index into the roots array for this buffer. 5784f364e7cSRobert Mustacchi * 5794f364e7cSRobert Mustacchi * 8.5 Tuning and disabling per-thread caching 5804f364e7cSRobert Mustacchi * ------------------------------------------- 5814f364e7cSRobert Mustacchi * 5824f364e7cSRobert Mustacchi * There is only one tunable for per-thread caching: the amount of memory each 5834f364e7cSRobert Mustacchi * thread should be able to cache. This is specified via the perthread_cache 5844f364e7cSRobert Mustacchi * UMEM_OPTION option. No attempt is made to to sanity check the specified 5854f364e7cSRobert Mustacchi * value; the limit is simply the maximum value of a size_t. 5864f364e7cSRobert Mustacchi * 5874f364e7cSRobert Mustacchi * If the perthread_cache UMEM_OPTION is set to zero, nomagazines was requested, 5884f364e7cSRobert Mustacchi * or UMEM_DEBUG has been turned on then we will never call into umem_genasm; 5894f364e7cSRobert Mustacchi * however, the trampoline audit library and jump will still be in place. 5904f364e7cSRobert Mustacchi * 5914f364e7cSRobert Mustacchi * 8.6 Observing efficacy of per-thread caching 5924f364e7cSRobert Mustacchi * -------------------------------------------- 5934f364e7cSRobert Mustacchi * 5944f364e7cSRobert Mustacchi * To understand the efficacy of per-thread caching, use the ::umastat dcmd 5954f364e7cSRobert Mustacchi * to see the percentage of capacity consumed on a per-thread basis, the 5964f364e7cSRobert Mustacchi * degree to which each umem cache contributes to per-thread cache consumption, 5974f364e7cSRobert Mustacchi * and the number of buffers in per-thread caches on a per-umem cache basis. 5984f364e7cSRobert Mustacchi * If more detail is required, the specific buffers in a per-thread cache can 5994f364e7cSRobert Mustacchi * be iterated over with the umem_ptc_* walkers. (These walkers allow an 6004f364e7cSRobert Mustacchi * optional ulwp_t to be specified to iterate only over a particular thread's 6014f364e7cSRobert Mustacchi * cache.) 6027c478bd9Sstevel@tonic-gate */ 6037c478bd9Sstevel@tonic-gate 6047c478bd9Sstevel@tonic-gate #include <umem_impl.h> 6057c478bd9Sstevel@tonic-gate #include <sys/vmem_impl_user.h> 6067c478bd9Sstevel@tonic-gate #include "umem_base.h" 6077c478bd9Sstevel@tonic-gate #include "vmem_base.h" 6087c478bd9Sstevel@tonic-gate 6097c478bd9Sstevel@tonic-gate #include <sys/processor.h> 6107c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 6117c478bd9Sstevel@tonic-gate 6127c478bd9Sstevel@tonic-gate #include <alloca.h> 6137c478bd9Sstevel@tonic-gate #include <errno.h> 6147c478bd9Sstevel@tonic-gate #include <limits.h> 6157c478bd9Sstevel@tonic-gate #include <stdio.h> 6167c478bd9Sstevel@tonic-gate #include <stdlib.h> 6177c478bd9Sstevel@tonic-gate #include <string.h> 6187c478bd9Sstevel@tonic-gate #include <strings.h> 6197c478bd9Sstevel@tonic-gate #include <signal.h> 6207c478bd9Sstevel@tonic-gate #include <unistd.h> 6217c478bd9Sstevel@tonic-gate #include <atomic.h> 6227c478bd9Sstevel@tonic-gate 6237c478bd9Sstevel@tonic-gate #include "misc.h" 6247c478bd9Sstevel@tonic-gate 6257c478bd9Sstevel@tonic-gate #define UMEM_VMFLAGS(umflag) (VM_NOSLEEP) 6267c478bd9Sstevel@tonic-gate 6277c478bd9Sstevel@tonic-gate size_t pagesize; 6287c478bd9Sstevel@tonic-gate 6297c478bd9Sstevel@tonic-gate /* 6307c478bd9Sstevel@tonic-gate * The default set of caches to back umem_alloc(). 6317c478bd9Sstevel@tonic-gate * These sizes should be reevaluated periodically. 6327c478bd9Sstevel@tonic-gate * 6337c478bd9Sstevel@tonic-gate * We want allocations that are multiples of the coherency granularity 6347c478bd9Sstevel@tonic-gate * (64 bytes) to be satisfied from a cache which is a multiple of 64 6357c478bd9Sstevel@tonic-gate * bytes, so that it will be 64-byte aligned. For all multiples of 64, 6367c478bd9Sstevel@tonic-gate * the next kmem_cache_size greater than or equal to it must be a 6377c478bd9Sstevel@tonic-gate * multiple of 64. 638789d94c2Sjwadams * 639789d94c2Sjwadams * This table must be in sorted order, from smallest to highest. The 640789d94c2Sjwadams * highest slot must be UMEM_MAXBUF, and every slot afterwards must be 641789d94c2Sjwadams * zero. 6427c478bd9Sstevel@tonic-gate */ 643789d94c2Sjwadams static int umem_alloc_sizes[] = { 6447c478bd9Sstevel@tonic-gate #ifdef _LP64 6457c478bd9Sstevel@tonic-gate 1 * 8, 6467c478bd9Sstevel@tonic-gate 1 * 16, 6477c478bd9Sstevel@tonic-gate 2 * 16, 6487c478bd9Sstevel@tonic-gate 3 * 16, 6497c478bd9Sstevel@tonic-gate #else 6507c478bd9Sstevel@tonic-gate 1 * 8, 6517c478bd9Sstevel@tonic-gate 2 * 8, 6527c478bd9Sstevel@tonic-gate 3 * 8, 6537c478bd9Sstevel@tonic-gate 4 * 8, 5 * 8, 6 * 8, 7 * 8, 6547c478bd9Sstevel@tonic-gate #endif 6557c478bd9Sstevel@tonic-gate 4 * 16, 5 * 16, 6 * 16, 7 * 16, 6567c478bd9Sstevel@tonic-gate 4 * 32, 5 * 32, 6 * 32, 7 * 32, 6577c478bd9Sstevel@tonic-gate 4 * 64, 5 * 64, 6 * 64, 7 * 64, 6587c478bd9Sstevel@tonic-gate 4 * 128, 5 * 128, 6 * 128, 7 * 128, 6597c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 7, 64), 6607c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 6, 64), 6617c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 5, 64), 662789d94c2Sjwadams P2ALIGN(8192 / 4, 64), 2304, 6637c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 3, 64), 664789d94c2Sjwadams P2ALIGN(8192 / 2, 64), 4544, 665789d94c2Sjwadams P2ALIGN(8192 / 1, 64), 9216, 6667c478bd9Sstevel@tonic-gate 4096 * 3, 66738849194SRobert Mustacchi 8192 * 2, /* = 8192 * 2 */ 66838849194SRobert Mustacchi 24576, 32768, 40960, 49152, 57344, 65536, 73728, 81920, 66938849194SRobert Mustacchi 90112, 98304, 106496, 114688, 122880, UMEM_MAXBUF, /* 128k */ 670789d94c2Sjwadams /* 24 slots for user expansion */ 671789d94c2Sjwadams 0, 0, 0, 0, 0, 0, 0, 0, 672789d94c2Sjwadams 0, 0, 0, 0, 0, 0, 0, 0, 673789d94c2Sjwadams 0, 0, 0, 0, 0, 0, 0, 0, 6747c478bd9Sstevel@tonic-gate }; 6757c478bd9Sstevel@tonic-gate #define NUM_ALLOC_SIZES (sizeof (umem_alloc_sizes) / sizeof (*umem_alloc_sizes)) 6767c478bd9Sstevel@tonic-gate 6777c478bd9Sstevel@tonic-gate static umem_magtype_t umem_magtype[] = { 6787c478bd9Sstevel@tonic-gate { 1, 8, 3200, 65536 }, 6797c478bd9Sstevel@tonic-gate { 3, 16, 256, 32768 }, 6807c478bd9Sstevel@tonic-gate { 7, 32, 64, 16384 }, 6817c478bd9Sstevel@tonic-gate { 15, 64, 0, 8192 }, 6827c478bd9Sstevel@tonic-gate { 31, 64, 0, 4096 }, 6837c478bd9Sstevel@tonic-gate { 47, 64, 0, 2048 }, 6847c478bd9Sstevel@tonic-gate { 63, 64, 0, 1024 }, 6857c478bd9Sstevel@tonic-gate { 95, 64, 0, 512 }, 6867c478bd9Sstevel@tonic-gate { 143, 64, 0, 0 }, 6877c478bd9Sstevel@tonic-gate }; 6887c478bd9Sstevel@tonic-gate 6897c478bd9Sstevel@tonic-gate /* 6907c478bd9Sstevel@tonic-gate * umem tunables 6917c478bd9Sstevel@tonic-gate */ 6927c478bd9Sstevel@tonic-gate uint32_t umem_max_ncpus; /* # of CPU caches. */ 6937c478bd9Sstevel@tonic-gate 6947c478bd9Sstevel@tonic-gate uint32_t umem_stack_depth = 15; /* # stack frames in a bufctl_audit */ 6957c478bd9Sstevel@tonic-gate uint32_t umem_reap_interval = 10; /* max reaping rate (seconds) */ 6967c478bd9Sstevel@tonic-gate uint_t umem_depot_contention = 2; /* max failed trylocks per real interval */ 6977c478bd9Sstevel@tonic-gate uint_t umem_abort = 1; /* whether to abort on error */ 6987c478bd9Sstevel@tonic-gate uint_t umem_output = 0; /* whether to write to standard error */ 6997c478bd9Sstevel@tonic-gate uint_t umem_logging = 0; /* umem_log_enter() override */ 7007c478bd9Sstevel@tonic-gate uint32_t umem_mtbf = 0; /* mean time between failures [default: off] */ 7017c478bd9Sstevel@tonic-gate size_t umem_transaction_log_size; /* size of transaction log */ 7027c478bd9Sstevel@tonic-gate size_t umem_content_log_size; /* size of content log */ 7037c478bd9Sstevel@tonic-gate size_t umem_failure_log_size; /* failure log [4 pages per CPU] */ 7047c478bd9Sstevel@tonic-gate size_t umem_slab_log_size; /* slab create log [4 pages per CPU] */ 7057c478bd9Sstevel@tonic-gate size_t umem_content_maxsave = 256; /* UMF_CONTENTS max bytes to log */ 7067c478bd9Sstevel@tonic-gate size_t umem_lite_minsize = 0; /* minimum buffer size for UMF_LITE */ 7077c478bd9Sstevel@tonic-gate size_t umem_lite_maxalign = 1024; /* maximum buffer alignment for UMF_LITE */ 7087c478bd9Sstevel@tonic-gate size_t umem_maxverify; /* maximum bytes to inspect in debug routines */ 7097c478bd9Sstevel@tonic-gate size_t umem_minfirewall; /* hardware-enforced redzone threshold */ 7104f364e7cSRobert Mustacchi size_t umem_ptc_size = 1048576; /* size of per-thread cache (in bytes) */ 7117c478bd9Sstevel@tonic-gate 7127c478bd9Sstevel@tonic-gate uint_t umem_flags = 0; 7134f364e7cSRobert Mustacchi uintptr_t umem_tmem_off; 7147c478bd9Sstevel@tonic-gate 7157c478bd9Sstevel@tonic-gate mutex_t umem_init_lock; /* locks initialization */ 7167c478bd9Sstevel@tonic-gate cond_t umem_init_cv; /* initialization CV */ 7177c478bd9Sstevel@tonic-gate thread_t umem_init_thr; /* thread initializing */ 7187c478bd9Sstevel@tonic-gate int umem_init_env_ready; /* environ pre-initted */ 7197c478bd9Sstevel@tonic-gate int umem_ready = UMEM_READY_STARTUP; 7207c478bd9Sstevel@tonic-gate 7214f364e7cSRobert Mustacchi int umem_ptc_enabled; /* per-thread caching enabled */ 7224f364e7cSRobert Mustacchi 7237c478bd9Sstevel@tonic-gate static umem_nofail_callback_t *nofail_callback; 7247c478bd9Sstevel@tonic-gate static mutex_t umem_nofail_exit_lock; 7257c478bd9Sstevel@tonic-gate static thread_t umem_nofail_exit_thr; 7267c478bd9Sstevel@tonic-gate 7277c478bd9Sstevel@tonic-gate static umem_cache_t *umem_slab_cache; 7287c478bd9Sstevel@tonic-gate static umem_cache_t *umem_bufctl_cache; 7297c478bd9Sstevel@tonic-gate static umem_cache_t *umem_bufctl_audit_cache; 7307c478bd9Sstevel@tonic-gate 7317c478bd9Sstevel@tonic-gate mutex_t umem_flags_lock; 7327c478bd9Sstevel@tonic-gate 7337c478bd9Sstevel@tonic-gate static vmem_t *heap_arena; 7347c478bd9Sstevel@tonic-gate static vmem_alloc_t *heap_alloc; 7357c478bd9Sstevel@tonic-gate static vmem_free_t *heap_free; 7367c478bd9Sstevel@tonic-gate 7377c478bd9Sstevel@tonic-gate static vmem_t *umem_internal_arena; 7387c478bd9Sstevel@tonic-gate static vmem_t *umem_cache_arena; 7397c478bd9Sstevel@tonic-gate static vmem_t *umem_hash_arena; 7407c478bd9Sstevel@tonic-gate static vmem_t *umem_log_arena; 7417c478bd9Sstevel@tonic-gate static vmem_t *umem_oversize_arena; 7427c478bd9Sstevel@tonic-gate static vmem_t *umem_va_arena; 7437c478bd9Sstevel@tonic-gate static vmem_t *umem_default_arena; 7447c478bd9Sstevel@tonic-gate static vmem_t *umem_firewall_va_arena; 7457c478bd9Sstevel@tonic-gate static vmem_t *umem_firewall_arena; 7467c478bd9Sstevel@tonic-gate 7477c478bd9Sstevel@tonic-gate vmem_t *umem_memalign_arena; 7487c478bd9Sstevel@tonic-gate 7497c478bd9Sstevel@tonic-gate umem_log_header_t *umem_transaction_log; 7507c478bd9Sstevel@tonic-gate umem_log_header_t *umem_content_log; 7517c478bd9Sstevel@tonic-gate umem_log_header_t *umem_failure_log; 7527c478bd9Sstevel@tonic-gate umem_log_header_t *umem_slab_log; 7537c478bd9Sstevel@tonic-gate 7547257d1b4Sraf #define CPUHINT() (thr_self()) 7557c478bd9Sstevel@tonic-gate #define CPUHINT_MAX() INT_MAX 7567c478bd9Sstevel@tonic-gate 7577c478bd9Sstevel@tonic-gate #define CPU(mask) (umem_cpus + (CPUHINT() & (mask))) 7587c478bd9Sstevel@tonic-gate static umem_cpu_t umem_startup_cpu = { /* initial, single, cpu */ 7597c478bd9Sstevel@tonic-gate UMEM_CACHE_SIZE(0), 7607c478bd9Sstevel@tonic-gate 0 7617c478bd9Sstevel@tonic-gate }; 7627c478bd9Sstevel@tonic-gate 7637c478bd9Sstevel@tonic-gate static uint32_t umem_cpu_mask = 0; /* global cpu mask */ 7647c478bd9Sstevel@tonic-gate static umem_cpu_t *umem_cpus = &umem_startup_cpu; /* cpu list */ 7657c478bd9Sstevel@tonic-gate 7667c478bd9Sstevel@tonic-gate volatile uint32_t umem_reaping; 7677c478bd9Sstevel@tonic-gate 7687c478bd9Sstevel@tonic-gate thread_t umem_update_thr; 7697c478bd9Sstevel@tonic-gate struct timeval umem_update_next; /* timeofday of next update */ 7707c478bd9Sstevel@tonic-gate volatile thread_t umem_st_update_thr; /* only used when single-thd */ 7717c478bd9Sstevel@tonic-gate 7727c478bd9Sstevel@tonic-gate #define IN_UPDATE() (thr_self() == umem_update_thr || \ 7737c478bd9Sstevel@tonic-gate thr_self() == umem_st_update_thr) 7747c478bd9Sstevel@tonic-gate #define IN_REAP() IN_UPDATE() 7757c478bd9Sstevel@tonic-gate 7767c478bd9Sstevel@tonic-gate mutex_t umem_update_lock; /* cache_u{next,prev,flags} */ 7777c478bd9Sstevel@tonic-gate cond_t umem_update_cv; 7787c478bd9Sstevel@tonic-gate 7797c478bd9Sstevel@tonic-gate volatile hrtime_t umem_reap_next; /* min hrtime of next reap */ 7807c478bd9Sstevel@tonic-gate 7817c478bd9Sstevel@tonic-gate mutex_t umem_cache_lock; /* inter-cache linkage only */ 7827c478bd9Sstevel@tonic-gate 7837c478bd9Sstevel@tonic-gate #ifdef UMEM_STANDALONE 7847c478bd9Sstevel@tonic-gate umem_cache_t umem_null_cache; 7857c478bd9Sstevel@tonic-gate static const umem_cache_t umem_null_cache_template = { 7867c478bd9Sstevel@tonic-gate #else 7877c478bd9Sstevel@tonic-gate umem_cache_t umem_null_cache = { 7887c478bd9Sstevel@tonic-gate #endif 7897c478bd9Sstevel@tonic-gate 0, 0, 0, 0, 0, 7907c478bd9Sstevel@tonic-gate 0, 0, 7917c478bd9Sstevel@tonic-gate 0, 0, 7927c478bd9Sstevel@tonic-gate 0, 0, 7937c478bd9Sstevel@tonic-gate "invalid_cache", 7947c478bd9Sstevel@tonic-gate 0, 0, 7957c478bd9Sstevel@tonic-gate NULL, NULL, NULL, NULL, 7967c478bd9Sstevel@tonic-gate NULL, 7977c478bd9Sstevel@tonic-gate 0, 0, 0, 0, 7987c478bd9Sstevel@tonic-gate &umem_null_cache, &umem_null_cache, 7997c478bd9Sstevel@tonic-gate &umem_null_cache, &umem_null_cache, 8007c478bd9Sstevel@tonic-gate 0, 8017c478bd9Sstevel@tonic-gate DEFAULTMUTEX, /* start of slab layer */ 8027c478bd9Sstevel@tonic-gate 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8037c478bd9Sstevel@tonic-gate &umem_null_cache.cache_nullslab, 8047c478bd9Sstevel@tonic-gate { 8057c478bd9Sstevel@tonic-gate &umem_null_cache, 8067c478bd9Sstevel@tonic-gate NULL, 8077c478bd9Sstevel@tonic-gate &umem_null_cache.cache_nullslab, 8087c478bd9Sstevel@tonic-gate &umem_null_cache.cache_nullslab, 8097c478bd9Sstevel@tonic-gate NULL, 8107c478bd9Sstevel@tonic-gate -1, 8117c478bd9Sstevel@tonic-gate 0 8127c478bd9Sstevel@tonic-gate }, 8137c478bd9Sstevel@tonic-gate NULL, 8147c478bd9Sstevel@tonic-gate NULL, 8157c478bd9Sstevel@tonic-gate DEFAULTMUTEX, /* start of depot layer */ 8167c478bd9Sstevel@tonic-gate NULL, { 8177c478bd9Sstevel@tonic-gate NULL, 0, 0, 0, 0 8187c478bd9Sstevel@tonic-gate }, { 8197c478bd9Sstevel@tonic-gate NULL, 0, 0, 0, 0 8207c478bd9Sstevel@tonic-gate }, { 8217c478bd9Sstevel@tonic-gate { 8227c478bd9Sstevel@tonic-gate DEFAULTMUTEX, /* start of CPU cache */ 8237c478bd9Sstevel@tonic-gate 0, 0, NULL, NULL, -1, -1, 0 8247c478bd9Sstevel@tonic-gate } 8257c478bd9Sstevel@tonic-gate } 8267c478bd9Sstevel@tonic-gate }; 8277c478bd9Sstevel@tonic-gate 8287c478bd9Sstevel@tonic-gate #define ALLOC_TABLE_4 \ 8297c478bd9Sstevel@tonic-gate &umem_null_cache, &umem_null_cache, &umem_null_cache, &umem_null_cache 8307c478bd9Sstevel@tonic-gate 8317c478bd9Sstevel@tonic-gate #define ALLOC_TABLE_64 \ 8327c478bd9Sstevel@tonic-gate ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, \ 8337c478bd9Sstevel@tonic-gate ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, \ 8347c478bd9Sstevel@tonic-gate ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, \ 8357c478bd9Sstevel@tonic-gate ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4 8367c478bd9Sstevel@tonic-gate 8377c478bd9Sstevel@tonic-gate #define ALLOC_TABLE_1024 \ 8387c478bd9Sstevel@tonic-gate ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, \ 8397c478bd9Sstevel@tonic-gate ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, \ 8407c478bd9Sstevel@tonic-gate ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, \ 8417c478bd9Sstevel@tonic-gate ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64 8427c478bd9Sstevel@tonic-gate 8437c478bd9Sstevel@tonic-gate static umem_cache_t *umem_alloc_table[UMEM_MAXBUF >> UMEM_ALIGN_SHIFT] = { 84438849194SRobert Mustacchi ALLOC_TABLE_1024, 84538849194SRobert Mustacchi ALLOC_TABLE_1024, 84638849194SRobert Mustacchi ALLOC_TABLE_1024, 84738849194SRobert Mustacchi ALLOC_TABLE_1024, 84838849194SRobert Mustacchi ALLOC_TABLE_1024, 84938849194SRobert Mustacchi ALLOC_TABLE_1024, 85038849194SRobert Mustacchi ALLOC_TABLE_1024, 85138849194SRobert Mustacchi ALLOC_TABLE_1024, 85238849194SRobert Mustacchi ALLOC_TABLE_1024, 85338849194SRobert Mustacchi ALLOC_TABLE_1024, 85438849194SRobert Mustacchi ALLOC_TABLE_1024, 85538849194SRobert Mustacchi ALLOC_TABLE_1024, 85638849194SRobert Mustacchi ALLOC_TABLE_1024, 85738849194SRobert Mustacchi ALLOC_TABLE_1024, 8587c478bd9Sstevel@tonic-gate ALLOC_TABLE_1024, 8597c478bd9Sstevel@tonic-gate ALLOC_TABLE_1024 8607c478bd9Sstevel@tonic-gate }; 8617c478bd9Sstevel@tonic-gate 8627c478bd9Sstevel@tonic-gate 8637c478bd9Sstevel@tonic-gate /* Used to constrain audit-log stack traces */ 8647c478bd9Sstevel@tonic-gate caddr_t umem_min_stack; 8657c478bd9Sstevel@tonic-gate caddr_t umem_max_stack; 8667c478bd9Sstevel@tonic-gate 8677c478bd9Sstevel@tonic-gate 8687c478bd9Sstevel@tonic-gate #define UMERR_MODIFIED 0 /* buffer modified while on freelist */ 8697c478bd9Sstevel@tonic-gate #define UMERR_REDZONE 1 /* redzone violation (write past end of buf) */ 8707c478bd9Sstevel@tonic-gate #define UMERR_DUPFREE 2 /* freed a buffer twice */ 8717c478bd9Sstevel@tonic-gate #define UMERR_BADADDR 3 /* freed a bad (unallocated) address */ 8727c478bd9Sstevel@tonic-gate #define UMERR_BADBUFTAG 4 /* buftag corrupted */ 8737c478bd9Sstevel@tonic-gate #define UMERR_BADBUFCTL 5 /* bufctl corrupted */ 8747c478bd9Sstevel@tonic-gate #define UMERR_BADCACHE 6 /* freed a buffer to the wrong cache */ 8757c478bd9Sstevel@tonic-gate #define UMERR_BADSIZE 7 /* alloc size != free size */ 8767c478bd9Sstevel@tonic-gate #define UMERR_BADBASE 8 /* buffer base address wrong */ 8777c478bd9Sstevel@tonic-gate 8787c478bd9Sstevel@tonic-gate struct { 8797c478bd9Sstevel@tonic-gate hrtime_t ump_timestamp; /* timestamp of error */ 8807c478bd9Sstevel@tonic-gate int ump_error; /* type of umem error (UMERR_*) */ 8817c478bd9Sstevel@tonic-gate void *ump_buffer; /* buffer that induced abort */ 8827c478bd9Sstevel@tonic-gate void *ump_realbuf; /* real start address for buffer */ 8837c478bd9Sstevel@tonic-gate umem_cache_t *ump_cache; /* buffer's cache according to client */ 8847c478bd9Sstevel@tonic-gate umem_cache_t *ump_realcache; /* actual cache containing buffer */ 8857c478bd9Sstevel@tonic-gate umem_slab_t *ump_slab; /* slab accoring to umem_findslab() */ 8867c478bd9Sstevel@tonic-gate umem_bufctl_t *ump_bufctl; /* bufctl */ 8877c478bd9Sstevel@tonic-gate } umem_abort_info; 8887c478bd9Sstevel@tonic-gate 8897c478bd9Sstevel@tonic-gate static void 8907c478bd9Sstevel@tonic-gate copy_pattern(uint64_t pattern, void *buf_arg, size_t size) 8917c478bd9Sstevel@tonic-gate { 8927c478bd9Sstevel@tonic-gate uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 8937c478bd9Sstevel@tonic-gate uint64_t *buf = buf_arg; 8947c478bd9Sstevel@tonic-gate 8957c478bd9Sstevel@tonic-gate while (buf < bufend) 8967c478bd9Sstevel@tonic-gate *buf++ = pattern; 8977c478bd9Sstevel@tonic-gate } 8987c478bd9Sstevel@tonic-gate 8997c478bd9Sstevel@tonic-gate static void * 9007c478bd9Sstevel@tonic-gate verify_pattern(uint64_t pattern, void *buf_arg, size_t size) 9017c478bd9Sstevel@tonic-gate { 9027c478bd9Sstevel@tonic-gate uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 9037c478bd9Sstevel@tonic-gate uint64_t *buf; 9047c478bd9Sstevel@tonic-gate 9057c478bd9Sstevel@tonic-gate for (buf = buf_arg; buf < bufend; buf++) 9067c478bd9Sstevel@tonic-gate if (*buf != pattern) 9077c478bd9Sstevel@tonic-gate return (buf); 9087c478bd9Sstevel@tonic-gate return (NULL); 9097c478bd9Sstevel@tonic-gate } 9107c478bd9Sstevel@tonic-gate 9117c478bd9Sstevel@tonic-gate static void * 9127c478bd9Sstevel@tonic-gate verify_and_copy_pattern(uint64_t old, uint64_t new, void *buf_arg, size_t size) 9137c478bd9Sstevel@tonic-gate { 9147c478bd9Sstevel@tonic-gate uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 9157c478bd9Sstevel@tonic-gate uint64_t *buf; 9167c478bd9Sstevel@tonic-gate 9177c478bd9Sstevel@tonic-gate for (buf = buf_arg; buf < bufend; buf++) { 9187c478bd9Sstevel@tonic-gate if (*buf != old) { 9197c478bd9Sstevel@tonic-gate copy_pattern(old, buf_arg, 9207c478bd9Sstevel@tonic-gate (char *)buf - (char *)buf_arg); 9217c478bd9Sstevel@tonic-gate return (buf); 9227c478bd9Sstevel@tonic-gate } 9237c478bd9Sstevel@tonic-gate *buf = new; 9247c478bd9Sstevel@tonic-gate } 9257c478bd9Sstevel@tonic-gate 9267c478bd9Sstevel@tonic-gate return (NULL); 9277c478bd9Sstevel@tonic-gate } 9287c478bd9Sstevel@tonic-gate 9297c478bd9Sstevel@tonic-gate void 9307c478bd9Sstevel@tonic-gate umem_cache_applyall(void (*func)(umem_cache_t *)) 9317c478bd9Sstevel@tonic-gate { 9327c478bd9Sstevel@tonic-gate umem_cache_t *cp; 9337c478bd9Sstevel@tonic-gate 9347c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_cache_lock); 9357c478bd9Sstevel@tonic-gate for (cp = umem_null_cache.cache_next; cp != &umem_null_cache; 9367c478bd9Sstevel@tonic-gate cp = cp->cache_next) 9377c478bd9Sstevel@tonic-gate func(cp); 9387c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_cache_lock); 9397c478bd9Sstevel@tonic-gate } 9407c478bd9Sstevel@tonic-gate 9417c478bd9Sstevel@tonic-gate static void 9427c478bd9Sstevel@tonic-gate umem_add_update_unlocked(umem_cache_t *cp, int flags) 9437c478bd9Sstevel@tonic-gate { 9447c478bd9Sstevel@tonic-gate umem_cache_t *cnext, *cprev; 9457c478bd9Sstevel@tonic-gate 9467c478bd9Sstevel@tonic-gate flags &= ~UMU_ACTIVE; 9477c478bd9Sstevel@tonic-gate 9487c478bd9Sstevel@tonic-gate if (!flags) 9497c478bd9Sstevel@tonic-gate return; 9507c478bd9Sstevel@tonic-gate 9517c478bd9Sstevel@tonic-gate if (cp->cache_uflags & UMU_ACTIVE) { 9527c478bd9Sstevel@tonic-gate cp->cache_uflags |= flags; 9537c478bd9Sstevel@tonic-gate } else { 9547c478bd9Sstevel@tonic-gate if (cp->cache_unext != NULL) { 9557c478bd9Sstevel@tonic-gate ASSERT(cp->cache_uflags != 0); 9567c478bd9Sstevel@tonic-gate cp->cache_uflags |= flags; 9577c478bd9Sstevel@tonic-gate } else { 9587c478bd9Sstevel@tonic-gate ASSERT(cp->cache_uflags == 0); 9597c478bd9Sstevel@tonic-gate cp->cache_uflags = flags; 9607c478bd9Sstevel@tonic-gate cp->cache_unext = cnext = &umem_null_cache; 9617c478bd9Sstevel@tonic-gate cp->cache_uprev = cprev = umem_null_cache.cache_uprev; 9627c478bd9Sstevel@tonic-gate cnext->cache_uprev = cp; 9637c478bd9Sstevel@tonic-gate cprev->cache_unext = cp; 9647c478bd9Sstevel@tonic-gate } 9657c478bd9Sstevel@tonic-gate } 9667c478bd9Sstevel@tonic-gate } 9677c478bd9Sstevel@tonic-gate 9687c478bd9Sstevel@tonic-gate static void 9697c478bd9Sstevel@tonic-gate umem_add_update(umem_cache_t *cp, int flags) 9707c478bd9Sstevel@tonic-gate { 9717c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_update_lock); 9727c478bd9Sstevel@tonic-gate 9737c478bd9Sstevel@tonic-gate umem_add_update_unlocked(cp, flags); 9747c478bd9Sstevel@tonic-gate 9757c478bd9Sstevel@tonic-gate if (!IN_UPDATE()) 9767c478bd9Sstevel@tonic-gate (void) cond_broadcast(&umem_update_cv); 9777c478bd9Sstevel@tonic-gate 9787c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_update_lock); 9797c478bd9Sstevel@tonic-gate } 9807c478bd9Sstevel@tonic-gate 9817c478bd9Sstevel@tonic-gate /* 9827c478bd9Sstevel@tonic-gate * Remove a cache from the update list, waiting for any in-progress work to 9837c478bd9Sstevel@tonic-gate * complete first. 9847c478bd9Sstevel@tonic-gate */ 9857c478bd9Sstevel@tonic-gate static void 9867c478bd9Sstevel@tonic-gate umem_remove_updates(umem_cache_t *cp) 9877c478bd9Sstevel@tonic-gate { 9887c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_update_lock); 9897c478bd9Sstevel@tonic-gate 9907c478bd9Sstevel@tonic-gate /* 9917c478bd9Sstevel@tonic-gate * Get it out of the active state 9927c478bd9Sstevel@tonic-gate */ 9937c478bd9Sstevel@tonic-gate while (cp->cache_uflags & UMU_ACTIVE) { 994a574db85Sraf int cancel_state; 995a574db85Sraf 9967c478bd9Sstevel@tonic-gate ASSERT(cp->cache_unext == NULL); 9977c478bd9Sstevel@tonic-gate 9987c478bd9Sstevel@tonic-gate cp->cache_uflags |= UMU_NOTIFY; 9997c478bd9Sstevel@tonic-gate 10007c478bd9Sstevel@tonic-gate /* 10017c478bd9Sstevel@tonic-gate * Make sure the update state is sane, before we wait 10027c478bd9Sstevel@tonic-gate */ 10037c478bd9Sstevel@tonic-gate ASSERT(umem_update_thr != 0 || umem_st_update_thr != 0); 10047c478bd9Sstevel@tonic-gate ASSERT(umem_update_thr != thr_self() && 10057c478bd9Sstevel@tonic-gate umem_st_update_thr != thr_self()); 10067c478bd9Sstevel@tonic-gate 1007a574db85Sraf (void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, 1008a574db85Sraf &cancel_state); 1009a574db85Sraf (void) cond_wait(&umem_update_cv, &umem_update_lock); 1010a574db85Sraf (void) pthread_setcancelstate(cancel_state, NULL); 10117c478bd9Sstevel@tonic-gate } 10127c478bd9Sstevel@tonic-gate /* 10137c478bd9Sstevel@tonic-gate * Get it out of the Work Requested state 10147c478bd9Sstevel@tonic-gate */ 10157c478bd9Sstevel@tonic-gate if (cp->cache_unext != NULL) { 10167c478bd9Sstevel@tonic-gate cp->cache_uprev->cache_unext = cp->cache_unext; 10177c478bd9Sstevel@tonic-gate cp->cache_unext->cache_uprev = cp->cache_uprev; 10187c478bd9Sstevel@tonic-gate cp->cache_uprev = cp->cache_unext = NULL; 10197c478bd9Sstevel@tonic-gate cp->cache_uflags = 0; 10207c478bd9Sstevel@tonic-gate } 10217c478bd9Sstevel@tonic-gate /* 10227c478bd9Sstevel@tonic-gate * Make sure it is in the Inactive state 10237c478bd9Sstevel@tonic-gate */ 10247c478bd9Sstevel@tonic-gate ASSERT(cp->cache_unext == NULL && cp->cache_uflags == 0); 10257c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_update_lock); 10267c478bd9Sstevel@tonic-gate } 10277c478bd9Sstevel@tonic-gate 10287c478bd9Sstevel@tonic-gate static void 10297c478bd9Sstevel@tonic-gate umem_updateall(int flags) 10307c478bd9Sstevel@tonic-gate { 10317c478bd9Sstevel@tonic-gate umem_cache_t *cp; 10327c478bd9Sstevel@tonic-gate 10337c478bd9Sstevel@tonic-gate /* 10347c478bd9Sstevel@tonic-gate * NOTE: To prevent deadlock, umem_cache_lock is always acquired first. 10357c478bd9Sstevel@tonic-gate * 10367c478bd9Sstevel@tonic-gate * (umem_add_update is called from things run via umem_cache_applyall) 10377c478bd9Sstevel@tonic-gate */ 10387c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_cache_lock); 10397c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_update_lock); 10407c478bd9Sstevel@tonic-gate 10417c478bd9Sstevel@tonic-gate for (cp = umem_null_cache.cache_next; cp != &umem_null_cache; 10427c478bd9Sstevel@tonic-gate cp = cp->cache_next) 10437c478bd9Sstevel@tonic-gate umem_add_update_unlocked(cp, flags); 10447c478bd9Sstevel@tonic-gate 10457c478bd9Sstevel@tonic-gate if (!IN_UPDATE()) 10467c478bd9Sstevel@tonic-gate (void) cond_broadcast(&umem_update_cv); 10477c478bd9Sstevel@tonic-gate 10487c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_update_lock); 10497c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_cache_lock); 10507c478bd9Sstevel@tonic-gate } 10517c478bd9Sstevel@tonic-gate 10527c478bd9Sstevel@tonic-gate /* 10537c478bd9Sstevel@tonic-gate * Debugging support. Given a buffer address, find its slab. 10547c478bd9Sstevel@tonic-gate */ 10557c478bd9Sstevel@tonic-gate static umem_slab_t * 10567c478bd9Sstevel@tonic-gate umem_findslab(umem_cache_t *cp, void *buf) 10577c478bd9Sstevel@tonic-gate { 10587c478bd9Sstevel@tonic-gate umem_slab_t *sp; 10597c478bd9Sstevel@tonic-gate 10607c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_lock); 10617c478bd9Sstevel@tonic-gate for (sp = cp->cache_nullslab.slab_next; 10627c478bd9Sstevel@tonic-gate sp != &cp->cache_nullslab; sp = sp->slab_next) { 10637c478bd9Sstevel@tonic-gate if (UMEM_SLAB_MEMBER(sp, buf)) { 10647c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 10657c478bd9Sstevel@tonic-gate return (sp); 10667c478bd9Sstevel@tonic-gate } 10677c478bd9Sstevel@tonic-gate } 10687c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 10697c478bd9Sstevel@tonic-gate 10707c478bd9Sstevel@tonic-gate return (NULL); 10717c478bd9Sstevel@tonic-gate } 10727c478bd9Sstevel@tonic-gate 10737c478bd9Sstevel@tonic-gate static void 10747c478bd9Sstevel@tonic-gate umem_error(int error, umem_cache_t *cparg, void *bufarg) 10757c478bd9Sstevel@tonic-gate { 10767c478bd9Sstevel@tonic-gate umem_buftag_t *btp = NULL; 10777c478bd9Sstevel@tonic-gate umem_bufctl_t *bcp = NULL; 10787c478bd9Sstevel@tonic-gate umem_cache_t *cp = cparg; 10797c478bd9Sstevel@tonic-gate umem_slab_t *sp; 10807c478bd9Sstevel@tonic-gate uint64_t *off; 10817c478bd9Sstevel@tonic-gate void *buf = bufarg; 10827c478bd9Sstevel@tonic-gate 10837c478bd9Sstevel@tonic-gate int old_logging = umem_logging; 10847c478bd9Sstevel@tonic-gate 10857c478bd9Sstevel@tonic-gate umem_logging = 0; /* stop logging when a bad thing happens */ 10867c478bd9Sstevel@tonic-gate 10877c478bd9Sstevel@tonic-gate umem_abort_info.ump_timestamp = gethrtime(); 10887c478bd9Sstevel@tonic-gate 10897c478bd9Sstevel@tonic-gate sp = umem_findslab(cp, buf); 10907c478bd9Sstevel@tonic-gate if (sp == NULL) { 10917c478bd9Sstevel@tonic-gate for (cp = umem_null_cache.cache_prev; cp != &umem_null_cache; 10927c478bd9Sstevel@tonic-gate cp = cp->cache_prev) { 10937c478bd9Sstevel@tonic-gate if ((sp = umem_findslab(cp, buf)) != NULL) 10947c478bd9Sstevel@tonic-gate break; 10957c478bd9Sstevel@tonic-gate } 10967c478bd9Sstevel@tonic-gate } 10977c478bd9Sstevel@tonic-gate 10987c478bd9Sstevel@tonic-gate if (sp == NULL) { 10997c478bd9Sstevel@tonic-gate cp = NULL; 11007c478bd9Sstevel@tonic-gate error = UMERR_BADADDR; 11017c478bd9Sstevel@tonic-gate } else { 11027c478bd9Sstevel@tonic-gate if (cp != cparg) 11037c478bd9Sstevel@tonic-gate error = UMERR_BADCACHE; 11047c478bd9Sstevel@tonic-gate else 11057c478bd9Sstevel@tonic-gate buf = (char *)bufarg - ((uintptr_t)bufarg - 11067c478bd9Sstevel@tonic-gate (uintptr_t)sp->slab_base) % cp->cache_chunksize; 11077c478bd9Sstevel@tonic-gate if (buf != bufarg) 11087c478bd9Sstevel@tonic-gate error = UMERR_BADBASE; 11097c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_BUFTAG) 11107c478bd9Sstevel@tonic-gate btp = UMEM_BUFTAG(cp, buf); 11117c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_HASH) { 11127c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_lock); 11137c478bd9Sstevel@tonic-gate for (bcp = *UMEM_HASH(cp, buf); bcp; bcp = bcp->bc_next) 11147c478bd9Sstevel@tonic-gate if (bcp->bc_addr == buf) 11157c478bd9Sstevel@tonic-gate break; 11167c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 11177c478bd9Sstevel@tonic-gate if (bcp == NULL && btp != NULL) 11187c478bd9Sstevel@tonic-gate bcp = btp->bt_bufctl; 11197c478bd9Sstevel@tonic-gate if (umem_findslab(cp->cache_bufctl_cache, bcp) == 11207c478bd9Sstevel@tonic-gate NULL || P2PHASE((uintptr_t)bcp, UMEM_ALIGN) || 11217c478bd9Sstevel@tonic-gate bcp->bc_addr != buf) { 11227c478bd9Sstevel@tonic-gate error = UMERR_BADBUFCTL; 11237c478bd9Sstevel@tonic-gate bcp = NULL; 11247c478bd9Sstevel@tonic-gate } 11257c478bd9Sstevel@tonic-gate } 11267c478bd9Sstevel@tonic-gate } 11277c478bd9Sstevel@tonic-gate 11287c478bd9Sstevel@tonic-gate umem_abort_info.ump_error = error; 11297c478bd9Sstevel@tonic-gate umem_abort_info.ump_buffer = bufarg; 11307c478bd9Sstevel@tonic-gate umem_abort_info.ump_realbuf = buf; 11317c478bd9Sstevel@tonic-gate umem_abort_info.ump_cache = cparg; 11327c478bd9Sstevel@tonic-gate umem_abort_info.ump_realcache = cp; 11337c478bd9Sstevel@tonic-gate umem_abort_info.ump_slab = sp; 11347c478bd9Sstevel@tonic-gate umem_abort_info.ump_bufctl = bcp; 11357c478bd9Sstevel@tonic-gate 11367c478bd9Sstevel@tonic-gate umem_printf("umem allocator: "); 11377c478bd9Sstevel@tonic-gate 11387c478bd9Sstevel@tonic-gate switch (error) { 11397c478bd9Sstevel@tonic-gate 11407c478bd9Sstevel@tonic-gate case UMERR_MODIFIED: 11417c478bd9Sstevel@tonic-gate umem_printf("buffer modified after being freed\n"); 11427c478bd9Sstevel@tonic-gate off = verify_pattern(UMEM_FREE_PATTERN, buf, cp->cache_verify); 11437c478bd9Sstevel@tonic-gate if (off == NULL) /* shouldn't happen */ 11447c478bd9Sstevel@tonic-gate off = buf; 11457c478bd9Sstevel@tonic-gate umem_printf("modification occurred at offset 0x%lx " 11467c478bd9Sstevel@tonic-gate "(0x%llx replaced by 0x%llx)\n", 11477c478bd9Sstevel@tonic-gate (uintptr_t)off - (uintptr_t)buf, 11487c478bd9Sstevel@tonic-gate (longlong_t)UMEM_FREE_PATTERN, (longlong_t)*off); 11497c478bd9Sstevel@tonic-gate break; 11507c478bd9Sstevel@tonic-gate 11517c478bd9Sstevel@tonic-gate case UMERR_REDZONE: 11527c478bd9Sstevel@tonic-gate umem_printf("redzone violation: write past end of buffer\n"); 11537c478bd9Sstevel@tonic-gate break; 11547c478bd9Sstevel@tonic-gate 11557c478bd9Sstevel@tonic-gate case UMERR_BADADDR: 11567c478bd9Sstevel@tonic-gate umem_printf("invalid free: buffer not in cache\n"); 11577c478bd9Sstevel@tonic-gate break; 11587c478bd9Sstevel@tonic-gate 11597c478bd9Sstevel@tonic-gate case UMERR_DUPFREE: 11607c478bd9Sstevel@tonic-gate umem_printf("duplicate free: buffer freed twice\n"); 11617c478bd9Sstevel@tonic-gate break; 11627c478bd9Sstevel@tonic-gate 11637c478bd9Sstevel@tonic-gate case UMERR_BADBUFTAG: 11647c478bd9Sstevel@tonic-gate umem_printf("boundary tag corrupted\n"); 11657c478bd9Sstevel@tonic-gate umem_printf("bcp ^ bxstat = %lx, should be %lx\n", 11667c478bd9Sstevel@tonic-gate (intptr_t)btp->bt_bufctl ^ btp->bt_bxstat, 11677c478bd9Sstevel@tonic-gate UMEM_BUFTAG_FREE); 11687c478bd9Sstevel@tonic-gate break; 11697c478bd9Sstevel@tonic-gate 11707c478bd9Sstevel@tonic-gate case UMERR_BADBUFCTL: 11717c478bd9Sstevel@tonic-gate umem_printf("bufctl corrupted\n"); 11727c478bd9Sstevel@tonic-gate break; 11737c478bd9Sstevel@tonic-gate 11747c478bd9Sstevel@tonic-gate case UMERR_BADCACHE: 11757c478bd9Sstevel@tonic-gate umem_printf("buffer freed to wrong cache\n"); 11767c478bd9Sstevel@tonic-gate umem_printf("buffer was allocated from %s,\n", cp->cache_name); 11777c478bd9Sstevel@tonic-gate umem_printf("caller attempting free to %s.\n", 11787c478bd9Sstevel@tonic-gate cparg->cache_name); 11797c478bd9Sstevel@tonic-gate break; 11807c478bd9Sstevel@tonic-gate 11817c478bd9Sstevel@tonic-gate case UMERR_BADSIZE: 11827c478bd9Sstevel@tonic-gate umem_printf("bad free: free size (%u) != alloc size (%u)\n", 11837c478bd9Sstevel@tonic-gate UMEM_SIZE_DECODE(((uint32_t *)btp)[0]), 11847c478bd9Sstevel@tonic-gate UMEM_SIZE_DECODE(((uint32_t *)btp)[1])); 11857c478bd9Sstevel@tonic-gate break; 11867c478bd9Sstevel@tonic-gate 11877c478bd9Sstevel@tonic-gate case UMERR_BADBASE: 11887c478bd9Sstevel@tonic-gate umem_printf("bad free: free address (%p) != alloc address " 11897c478bd9Sstevel@tonic-gate "(%p)\n", bufarg, buf); 11907c478bd9Sstevel@tonic-gate break; 11917c478bd9Sstevel@tonic-gate } 11927c478bd9Sstevel@tonic-gate 11937c478bd9Sstevel@tonic-gate umem_printf("buffer=%p bufctl=%p cache: %s\n", 11947c478bd9Sstevel@tonic-gate bufarg, (void *)bcp, cparg->cache_name); 11957c478bd9Sstevel@tonic-gate 11967c478bd9Sstevel@tonic-gate if (bcp != NULL && (cp->cache_flags & UMF_AUDIT) && 11977c478bd9Sstevel@tonic-gate error != UMERR_BADBUFCTL) { 11987c478bd9Sstevel@tonic-gate int d; 11997c478bd9Sstevel@tonic-gate timespec_t ts; 12007c478bd9Sstevel@tonic-gate hrtime_t diff; 12017c478bd9Sstevel@tonic-gate umem_bufctl_audit_t *bcap = (umem_bufctl_audit_t *)bcp; 12027c478bd9Sstevel@tonic-gate 12037c478bd9Sstevel@tonic-gate diff = umem_abort_info.ump_timestamp - bcap->bc_timestamp; 12047c478bd9Sstevel@tonic-gate ts.tv_sec = diff / NANOSEC; 12057c478bd9Sstevel@tonic-gate ts.tv_nsec = diff % NANOSEC; 12067c478bd9Sstevel@tonic-gate 12077c478bd9Sstevel@tonic-gate umem_printf("previous transaction on buffer %p:\n", buf); 12087c478bd9Sstevel@tonic-gate umem_printf("thread=%p time=T-%ld.%09ld slab=%p cache: %s\n", 12097c478bd9Sstevel@tonic-gate (void *)(intptr_t)bcap->bc_thread, ts.tv_sec, ts.tv_nsec, 12107c478bd9Sstevel@tonic-gate (void *)sp, cp->cache_name); 12117c478bd9Sstevel@tonic-gate for (d = 0; d < MIN(bcap->bc_depth, umem_stack_depth); d++) { 12127c478bd9Sstevel@tonic-gate (void) print_sym((void *)bcap->bc_stack[d]); 12137c478bd9Sstevel@tonic-gate umem_printf("\n"); 12147c478bd9Sstevel@tonic-gate } 12157c478bd9Sstevel@tonic-gate } 12167c478bd9Sstevel@tonic-gate 12177c478bd9Sstevel@tonic-gate umem_err_recoverable("umem: heap corruption detected"); 12187c478bd9Sstevel@tonic-gate 12197c478bd9Sstevel@tonic-gate umem_logging = old_logging; /* resume logging */ 12207c478bd9Sstevel@tonic-gate } 12217c478bd9Sstevel@tonic-gate 12227c478bd9Sstevel@tonic-gate void 12237c478bd9Sstevel@tonic-gate umem_nofail_callback(umem_nofail_callback_t *cb) 12247c478bd9Sstevel@tonic-gate { 12257c478bd9Sstevel@tonic-gate nofail_callback = cb; 12267c478bd9Sstevel@tonic-gate } 12277c478bd9Sstevel@tonic-gate 12287c478bd9Sstevel@tonic-gate static int 12297c478bd9Sstevel@tonic-gate umem_alloc_retry(umem_cache_t *cp, int umflag) 12307c478bd9Sstevel@tonic-gate { 12317c478bd9Sstevel@tonic-gate if (cp == &umem_null_cache) { 12327c478bd9Sstevel@tonic-gate if (umem_init()) 12337c478bd9Sstevel@tonic-gate return (1); /* retry */ 12347c478bd9Sstevel@tonic-gate /* 12357c478bd9Sstevel@tonic-gate * Initialization failed. Do normal failure processing. 12367c478bd9Sstevel@tonic-gate */ 12377c478bd9Sstevel@tonic-gate } 1238831abf2cSDan Kimmel if (umem_flags & UMF_CHECKNULL) { 1239831abf2cSDan Kimmel umem_err_recoverable("umem: out of heap space"); 1240831abf2cSDan Kimmel } 12417c478bd9Sstevel@tonic-gate if (umflag & UMEM_NOFAIL) { 12427c478bd9Sstevel@tonic-gate int def_result = UMEM_CALLBACK_EXIT(255); 12437c478bd9Sstevel@tonic-gate int result = def_result; 12447c478bd9Sstevel@tonic-gate umem_nofail_callback_t *callback = nofail_callback; 12457c478bd9Sstevel@tonic-gate 12467c478bd9Sstevel@tonic-gate if (callback != NULL) 12477c478bd9Sstevel@tonic-gate result = callback(); 12487c478bd9Sstevel@tonic-gate 12497c478bd9Sstevel@tonic-gate if (result == UMEM_CALLBACK_RETRY) 12507c478bd9Sstevel@tonic-gate return (1); 12517c478bd9Sstevel@tonic-gate 12527c478bd9Sstevel@tonic-gate if ((result & ~0xFF) != UMEM_CALLBACK_EXIT(0)) { 12537c478bd9Sstevel@tonic-gate log_message("nofail callback returned %x\n", result); 12547c478bd9Sstevel@tonic-gate result = def_result; 12557c478bd9Sstevel@tonic-gate } 12567c478bd9Sstevel@tonic-gate 12577c478bd9Sstevel@tonic-gate /* 12587c478bd9Sstevel@tonic-gate * only one thread will call exit 12597c478bd9Sstevel@tonic-gate */ 12607c478bd9Sstevel@tonic-gate if (umem_nofail_exit_thr == thr_self()) 12617c478bd9Sstevel@tonic-gate umem_panic("recursive UMEM_CALLBACK_EXIT()\n"); 12627c478bd9Sstevel@tonic-gate 12637c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_nofail_exit_lock); 12647c478bd9Sstevel@tonic-gate umem_nofail_exit_thr = thr_self(); 12657c478bd9Sstevel@tonic-gate exit(result & 0xFF); 12667c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 12677c478bd9Sstevel@tonic-gate } 12687c478bd9Sstevel@tonic-gate return (0); 12697c478bd9Sstevel@tonic-gate } 12707c478bd9Sstevel@tonic-gate 12717c478bd9Sstevel@tonic-gate static umem_log_header_t * 12727c478bd9Sstevel@tonic-gate umem_log_init(size_t logsize) 12737c478bd9Sstevel@tonic-gate { 12747c478bd9Sstevel@tonic-gate umem_log_header_t *lhp; 12757c478bd9Sstevel@tonic-gate int nchunks = 4 * umem_max_ncpus; 12767c478bd9Sstevel@tonic-gate size_t lhsize = offsetof(umem_log_header_t, lh_cpu[umem_max_ncpus]); 12777c478bd9Sstevel@tonic-gate int i; 12787c478bd9Sstevel@tonic-gate 12797c478bd9Sstevel@tonic-gate if (logsize == 0) 12807c478bd9Sstevel@tonic-gate return (NULL); 12817c478bd9Sstevel@tonic-gate 12827c478bd9Sstevel@tonic-gate /* 12837c478bd9Sstevel@tonic-gate * Make sure that lhp->lh_cpu[] is nicely aligned 12847c478bd9Sstevel@tonic-gate * to prevent false sharing of cache lines. 12857c478bd9Sstevel@tonic-gate */ 12867c478bd9Sstevel@tonic-gate lhsize = P2ROUNDUP(lhsize, UMEM_ALIGN); 12877c478bd9Sstevel@tonic-gate lhp = vmem_xalloc(umem_log_arena, lhsize, 64, P2NPHASE(lhsize, 64), 0, 12887c478bd9Sstevel@tonic-gate NULL, NULL, VM_NOSLEEP); 12897c478bd9Sstevel@tonic-gate if (lhp == NULL) 12907c478bd9Sstevel@tonic-gate goto fail; 12917c478bd9Sstevel@tonic-gate 12927c478bd9Sstevel@tonic-gate bzero(lhp, lhsize); 12937c478bd9Sstevel@tonic-gate 12947c478bd9Sstevel@tonic-gate (void) mutex_init(&lhp->lh_lock, USYNC_THREAD, NULL); 12957c478bd9Sstevel@tonic-gate lhp->lh_nchunks = nchunks; 12967c478bd9Sstevel@tonic-gate lhp->lh_chunksize = P2ROUNDUP(logsize / nchunks, PAGESIZE); 12977c478bd9Sstevel@tonic-gate if (lhp->lh_chunksize == 0) 12987c478bd9Sstevel@tonic-gate lhp->lh_chunksize = PAGESIZE; 12997c478bd9Sstevel@tonic-gate 13007c478bd9Sstevel@tonic-gate lhp->lh_base = vmem_alloc(umem_log_arena, 13017c478bd9Sstevel@tonic-gate lhp->lh_chunksize * nchunks, VM_NOSLEEP); 13027c478bd9Sstevel@tonic-gate if (lhp->lh_base == NULL) 13037c478bd9Sstevel@tonic-gate goto fail; 13047c478bd9Sstevel@tonic-gate 13057c478bd9Sstevel@tonic-gate lhp->lh_free = vmem_alloc(umem_log_arena, 13067c478bd9Sstevel@tonic-gate nchunks * sizeof (int), VM_NOSLEEP); 13077c478bd9Sstevel@tonic-gate if (lhp->lh_free == NULL) 13087c478bd9Sstevel@tonic-gate goto fail; 13097c478bd9Sstevel@tonic-gate 13107c478bd9Sstevel@tonic-gate bzero(lhp->lh_base, lhp->lh_chunksize * nchunks); 13117c478bd9Sstevel@tonic-gate 13127c478bd9Sstevel@tonic-gate for (i = 0; i < umem_max_ncpus; i++) { 13137c478bd9Sstevel@tonic-gate umem_cpu_log_header_t *clhp = &lhp->lh_cpu[i]; 13147c478bd9Sstevel@tonic-gate (void) mutex_init(&clhp->clh_lock, USYNC_THREAD, NULL); 13157c478bd9Sstevel@tonic-gate clhp->clh_chunk = i; 13167c478bd9Sstevel@tonic-gate } 13177c478bd9Sstevel@tonic-gate 13187c478bd9Sstevel@tonic-gate for (i = umem_max_ncpus; i < nchunks; i++) 13197c478bd9Sstevel@tonic-gate lhp->lh_free[i] = i; 13207c478bd9Sstevel@tonic-gate 13217c478bd9Sstevel@tonic-gate lhp->lh_head = umem_max_ncpus; 13227c478bd9Sstevel@tonic-gate lhp->lh_tail = 0; 13237c478bd9Sstevel@tonic-gate 13247c478bd9Sstevel@tonic-gate return (lhp); 13257c478bd9Sstevel@tonic-gate 13267c478bd9Sstevel@tonic-gate fail: 13277c478bd9Sstevel@tonic-gate if (lhp != NULL) { 13287c478bd9Sstevel@tonic-gate if (lhp->lh_base != NULL) 13297c478bd9Sstevel@tonic-gate vmem_free(umem_log_arena, lhp->lh_base, 13307c478bd9Sstevel@tonic-gate lhp->lh_chunksize * nchunks); 13317c478bd9Sstevel@tonic-gate 13327c478bd9Sstevel@tonic-gate vmem_xfree(umem_log_arena, lhp, lhsize); 13337c478bd9Sstevel@tonic-gate } 13347c478bd9Sstevel@tonic-gate return (NULL); 13357c478bd9Sstevel@tonic-gate } 13367c478bd9Sstevel@tonic-gate 13377c478bd9Sstevel@tonic-gate static void * 13387c478bd9Sstevel@tonic-gate umem_log_enter(umem_log_header_t *lhp, void *data, size_t size) 13397c478bd9Sstevel@tonic-gate { 13407c478bd9Sstevel@tonic-gate void *logspace; 1341b1e2e3fbSRobert Mustacchi umem_cpu_log_header_t *clhp; 13427c478bd9Sstevel@tonic-gate 13437c478bd9Sstevel@tonic-gate if (lhp == NULL || umem_logging == 0) 13447c478bd9Sstevel@tonic-gate return (NULL); 13457c478bd9Sstevel@tonic-gate 1346b1e2e3fbSRobert Mustacchi clhp = &lhp->lh_cpu[CPU(umem_cpu_mask)->cpu_number]; 1347b1e2e3fbSRobert Mustacchi 13487c478bd9Sstevel@tonic-gate (void) mutex_lock(&clhp->clh_lock); 13497c478bd9Sstevel@tonic-gate clhp->clh_hits++; 13507c478bd9Sstevel@tonic-gate if (size > clhp->clh_avail) { 13517c478bd9Sstevel@tonic-gate (void) mutex_lock(&lhp->lh_lock); 13527c478bd9Sstevel@tonic-gate lhp->lh_hits++; 13537c478bd9Sstevel@tonic-gate lhp->lh_free[lhp->lh_tail] = clhp->clh_chunk; 13547c478bd9Sstevel@tonic-gate lhp->lh_tail = (lhp->lh_tail + 1) % lhp->lh_nchunks; 13557c478bd9Sstevel@tonic-gate clhp->clh_chunk = lhp->lh_free[lhp->lh_head]; 13567c478bd9Sstevel@tonic-gate lhp->lh_head = (lhp->lh_head + 1) % lhp->lh_nchunks; 13577c478bd9Sstevel@tonic-gate clhp->clh_current = lhp->lh_base + 13587c478bd9Sstevel@tonic-gate clhp->clh_chunk * lhp->lh_chunksize; 13597c478bd9Sstevel@tonic-gate clhp->clh_avail = lhp->lh_chunksize; 13607c478bd9Sstevel@tonic-gate if (size > lhp->lh_chunksize) 13617c478bd9Sstevel@tonic-gate size = lhp->lh_chunksize; 13627c478bd9Sstevel@tonic-gate (void) mutex_unlock(&lhp->lh_lock); 13637c478bd9Sstevel@tonic-gate } 13647c478bd9Sstevel@tonic-gate logspace = clhp->clh_current; 13657c478bd9Sstevel@tonic-gate clhp->clh_current += size; 13667c478bd9Sstevel@tonic-gate clhp->clh_avail -= size; 13677c478bd9Sstevel@tonic-gate bcopy(data, logspace, size); 13687c478bd9Sstevel@tonic-gate (void) mutex_unlock(&clhp->clh_lock); 13697c478bd9Sstevel@tonic-gate return (logspace); 13707c478bd9Sstevel@tonic-gate } 13717c478bd9Sstevel@tonic-gate 13727c478bd9Sstevel@tonic-gate #define UMEM_AUDIT(lp, cp, bcp) \ 13737c478bd9Sstevel@tonic-gate { \ 13747c478bd9Sstevel@tonic-gate umem_bufctl_audit_t *_bcp = (umem_bufctl_audit_t *)(bcp); \ 13757c478bd9Sstevel@tonic-gate _bcp->bc_timestamp = gethrtime(); \ 13767c478bd9Sstevel@tonic-gate _bcp->bc_thread = thr_self(); \ 13777c478bd9Sstevel@tonic-gate _bcp->bc_depth = getpcstack(_bcp->bc_stack, umem_stack_depth, \ 13787c478bd9Sstevel@tonic-gate (cp != NULL) && (cp->cache_flags & UMF_CHECKSIGNAL)); \ 13797c478bd9Sstevel@tonic-gate _bcp->bc_lastlog = umem_log_enter((lp), _bcp, \ 13807c478bd9Sstevel@tonic-gate UMEM_BUFCTL_AUDIT_SIZE); \ 13817c478bd9Sstevel@tonic-gate } 13827c478bd9Sstevel@tonic-gate 13837c478bd9Sstevel@tonic-gate static void 13847c478bd9Sstevel@tonic-gate umem_log_event(umem_log_header_t *lp, umem_cache_t *cp, 1385831abf2cSDan Kimmel umem_slab_t *sp, void *addr) 13867c478bd9Sstevel@tonic-gate { 13877c478bd9Sstevel@tonic-gate umem_bufctl_audit_t *bcp; 13887c478bd9Sstevel@tonic-gate UMEM_LOCAL_BUFCTL_AUDIT(&bcp); 13897c478bd9Sstevel@tonic-gate 13907c478bd9Sstevel@tonic-gate bzero(bcp, UMEM_BUFCTL_AUDIT_SIZE); 13917c478bd9Sstevel@tonic-gate bcp->bc_addr = addr; 13927c478bd9Sstevel@tonic-gate bcp->bc_slab = sp; 13937c478bd9Sstevel@tonic-gate bcp->bc_cache = cp; 13947c478bd9Sstevel@tonic-gate UMEM_AUDIT(lp, cp, bcp); 13957c478bd9Sstevel@tonic-gate } 13967c478bd9Sstevel@tonic-gate 13977c478bd9Sstevel@tonic-gate /* 13987c478bd9Sstevel@tonic-gate * Create a new slab for cache cp. 13997c478bd9Sstevel@tonic-gate */ 14007c478bd9Sstevel@tonic-gate static umem_slab_t * 14017c478bd9Sstevel@tonic-gate umem_slab_create(umem_cache_t *cp, int umflag) 14027c478bd9Sstevel@tonic-gate { 14037c478bd9Sstevel@tonic-gate size_t slabsize = cp->cache_slabsize; 14047c478bd9Sstevel@tonic-gate size_t chunksize = cp->cache_chunksize; 14057c478bd9Sstevel@tonic-gate int cache_flags = cp->cache_flags; 14067c478bd9Sstevel@tonic-gate size_t color, chunks; 14077c478bd9Sstevel@tonic-gate char *buf, *slab; 14087c478bd9Sstevel@tonic-gate umem_slab_t *sp; 14097c478bd9Sstevel@tonic-gate umem_bufctl_t *bcp; 14107c478bd9Sstevel@tonic-gate vmem_t *vmp = cp->cache_arena; 14117c478bd9Sstevel@tonic-gate 14127c478bd9Sstevel@tonic-gate color = cp->cache_color + cp->cache_align; 14137c478bd9Sstevel@tonic-gate if (color > cp->cache_maxcolor) 14147c478bd9Sstevel@tonic-gate color = cp->cache_mincolor; 14157c478bd9Sstevel@tonic-gate cp->cache_color = color; 14167c478bd9Sstevel@tonic-gate 14177c478bd9Sstevel@tonic-gate slab = vmem_alloc(vmp, slabsize, UMEM_VMFLAGS(umflag)); 14187c478bd9Sstevel@tonic-gate 14197c478bd9Sstevel@tonic-gate if (slab == NULL) 14207c478bd9Sstevel@tonic-gate goto vmem_alloc_failure; 14217c478bd9Sstevel@tonic-gate 14227c478bd9Sstevel@tonic-gate ASSERT(P2PHASE((uintptr_t)slab, vmp->vm_quantum) == 0); 14237c478bd9Sstevel@tonic-gate 14247c478bd9Sstevel@tonic-gate if (!(cp->cache_cflags & UMC_NOTOUCH) && 14257c478bd9Sstevel@tonic-gate (cp->cache_flags & UMF_DEADBEEF)) 14267c478bd9Sstevel@tonic-gate copy_pattern(UMEM_UNINITIALIZED_PATTERN, slab, slabsize); 14277c478bd9Sstevel@tonic-gate 14287c478bd9Sstevel@tonic-gate if (cache_flags & UMF_HASH) { 14297c478bd9Sstevel@tonic-gate if ((sp = _umem_cache_alloc(umem_slab_cache, umflag)) == NULL) 14307c478bd9Sstevel@tonic-gate goto slab_alloc_failure; 14317c478bd9Sstevel@tonic-gate chunks = (slabsize - color) / chunksize; 14327c478bd9Sstevel@tonic-gate } else { 14337c478bd9Sstevel@tonic-gate sp = UMEM_SLAB(cp, slab); 14347c478bd9Sstevel@tonic-gate chunks = (slabsize - sizeof (umem_slab_t) - color) / chunksize; 14357c478bd9Sstevel@tonic-gate } 14367c478bd9Sstevel@tonic-gate 14377c478bd9Sstevel@tonic-gate sp->slab_cache = cp; 14387c478bd9Sstevel@tonic-gate sp->slab_head = NULL; 14397c478bd9Sstevel@tonic-gate sp->slab_refcnt = 0; 14407c478bd9Sstevel@tonic-gate sp->slab_base = buf = slab + color; 14417c478bd9Sstevel@tonic-gate sp->slab_chunks = chunks; 14427c478bd9Sstevel@tonic-gate 14437c478bd9Sstevel@tonic-gate ASSERT(chunks > 0); 14447c478bd9Sstevel@tonic-gate while (chunks-- != 0) { 14457c478bd9Sstevel@tonic-gate if (cache_flags & UMF_HASH) { 14467c478bd9Sstevel@tonic-gate bcp = _umem_cache_alloc(cp->cache_bufctl_cache, umflag); 14477c478bd9Sstevel@tonic-gate if (bcp == NULL) 14487c478bd9Sstevel@tonic-gate goto bufctl_alloc_failure; 14497c478bd9Sstevel@tonic-gate if (cache_flags & UMF_AUDIT) { 14507c478bd9Sstevel@tonic-gate umem_bufctl_audit_t *bcap = 14517c478bd9Sstevel@tonic-gate (umem_bufctl_audit_t *)bcp; 14527c478bd9Sstevel@tonic-gate bzero(bcap, UMEM_BUFCTL_AUDIT_SIZE); 14537c478bd9Sstevel@tonic-gate bcap->bc_cache = cp; 14547c478bd9Sstevel@tonic-gate } 14557c478bd9Sstevel@tonic-gate bcp->bc_addr = buf; 14567c478bd9Sstevel@tonic-gate bcp->bc_slab = sp; 14577c478bd9Sstevel@tonic-gate } else { 14587c478bd9Sstevel@tonic-gate bcp = UMEM_BUFCTL(cp, buf); 14597c478bd9Sstevel@tonic-gate } 14607c478bd9Sstevel@tonic-gate if (cache_flags & UMF_BUFTAG) { 14617c478bd9Sstevel@tonic-gate umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); 14627c478bd9Sstevel@tonic-gate btp->bt_redzone = UMEM_REDZONE_PATTERN; 14637c478bd9Sstevel@tonic-gate btp->bt_bufctl = bcp; 14647c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ UMEM_BUFTAG_FREE; 14657c478bd9Sstevel@tonic-gate if (cache_flags & UMF_DEADBEEF) { 14667c478bd9Sstevel@tonic-gate copy_pattern(UMEM_FREE_PATTERN, buf, 14677c478bd9Sstevel@tonic-gate cp->cache_verify); 14687c478bd9Sstevel@tonic-gate } 14697c478bd9Sstevel@tonic-gate } 14707c478bd9Sstevel@tonic-gate bcp->bc_next = sp->slab_head; 14717c478bd9Sstevel@tonic-gate sp->slab_head = bcp; 14727c478bd9Sstevel@tonic-gate buf += chunksize; 14737c478bd9Sstevel@tonic-gate } 14747c478bd9Sstevel@tonic-gate 14757c478bd9Sstevel@tonic-gate umem_log_event(umem_slab_log, cp, sp, slab); 14767c478bd9Sstevel@tonic-gate 14777c478bd9Sstevel@tonic-gate return (sp); 14787c478bd9Sstevel@tonic-gate 14797c478bd9Sstevel@tonic-gate bufctl_alloc_failure: 14807c478bd9Sstevel@tonic-gate 14817c478bd9Sstevel@tonic-gate while ((bcp = sp->slab_head) != NULL) { 14827c478bd9Sstevel@tonic-gate sp->slab_head = bcp->bc_next; 14837c478bd9Sstevel@tonic-gate _umem_cache_free(cp->cache_bufctl_cache, bcp); 14847c478bd9Sstevel@tonic-gate } 14857c478bd9Sstevel@tonic-gate _umem_cache_free(umem_slab_cache, sp); 14867c478bd9Sstevel@tonic-gate 14877c478bd9Sstevel@tonic-gate slab_alloc_failure: 14887c478bd9Sstevel@tonic-gate 14897c478bd9Sstevel@tonic-gate vmem_free(vmp, slab, slabsize); 14907c478bd9Sstevel@tonic-gate 14917c478bd9Sstevel@tonic-gate vmem_alloc_failure: 14927c478bd9Sstevel@tonic-gate 14937c478bd9Sstevel@tonic-gate umem_log_event(umem_failure_log, cp, NULL, NULL); 14947c478bd9Sstevel@tonic-gate atomic_add_64(&cp->cache_alloc_fail, 1); 14957c478bd9Sstevel@tonic-gate 14967c478bd9Sstevel@tonic-gate return (NULL); 14977c478bd9Sstevel@tonic-gate } 14987c478bd9Sstevel@tonic-gate 14997c478bd9Sstevel@tonic-gate /* 15007c478bd9Sstevel@tonic-gate * Destroy a slab. 15017c478bd9Sstevel@tonic-gate */ 15027c478bd9Sstevel@tonic-gate static void 15037c478bd9Sstevel@tonic-gate umem_slab_destroy(umem_cache_t *cp, umem_slab_t *sp) 15047c478bd9Sstevel@tonic-gate { 15057c478bd9Sstevel@tonic-gate vmem_t *vmp = cp->cache_arena; 15067c478bd9Sstevel@tonic-gate void *slab = (void *)P2ALIGN((uintptr_t)sp->slab_base, vmp->vm_quantum); 15077c478bd9Sstevel@tonic-gate 15087c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_HASH) { 15097c478bd9Sstevel@tonic-gate umem_bufctl_t *bcp; 15107c478bd9Sstevel@tonic-gate while ((bcp = sp->slab_head) != NULL) { 15117c478bd9Sstevel@tonic-gate sp->slab_head = bcp->bc_next; 15127c478bd9Sstevel@tonic-gate _umem_cache_free(cp->cache_bufctl_cache, bcp); 15137c478bd9Sstevel@tonic-gate } 15147c478bd9Sstevel@tonic-gate _umem_cache_free(umem_slab_cache, sp); 15157c478bd9Sstevel@tonic-gate } 15167c478bd9Sstevel@tonic-gate vmem_free(vmp, slab, cp->cache_slabsize); 15177c478bd9Sstevel@tonic-gate } 15187c478bd9Sstevel@tonic-gate 15197c478bd9Sstevel@tonic-gate /* 15207c478bd9Sstevel@tonic-gate * Allocate a raw (unconstructed) buffer from cp's slab layer. 15217c478bd9Sstevel@tonic-gate */ 15227c478bd9Sstevel@tonic-gate static void * 15237c478bd9Sstevel@tonic-gate umem_slab_alloc(umem_cache_t *cp, int umflag) 15247c478bd9Sstevel@tonic-gate { 15257c478bd9Sstevel@tonic-gate umem_bufctl_t *bcp, **hash_bucket; 15267c478bd9Sstevel@tonic-gate umem_slab_t *sp; 15277c478bd9Sstevel@tonic-gate void *buf; 15287c478bd9Sstevel@tonic-gate 15297c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_lock); 15307c478bd9Sstevel@tonic-gate cp->cache_slab_alloc++; 15317c478bd9Sstevel@tonic-gate sp = cp->cache_freelist; 15327c478bd9Sstevel@tonic-gate ASSERT(sp->slab_cache == cp); 15337c478bd9Sstevel@tonic-gate if (sp->slab_head == NULL) { 15347c478bd9Sstevel@tonic-gate /* 15357c478bd9Sstevel@tonic-gate * The freelist is empty. Create a new slab. 15367c478bd9Sstevel@tonic-gate */ 15377c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 15387c478bd9Sstevel@tonic-gate if (cp == &umem_null_cache) 15397c478bd9Sstevel@tonic-gate return (NULL); 15407c478bd9Sstevel@tonic-gate if ((sp = umem_slab_create(cp, umflag)) == NULL) 15417c478bd9Sstevel@tonic-gate return (NULL); 15427c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_lock); 15437c478bd9Sstevel@tonic-gate cp->cache_slab_create++; 15447c478bd9Sstevel@tonic-gate if ((cp->cache_buftotal += sp->slab_chunks) > cp->cache_bufmax) 15457c478bd9Sstevel@tonic-gate cp->cache_bufmax = cp->cache_buftotal; 15467c478bd9Sstevel@tonic-gate sp->slab_next = cp->cache_freelist; 15477c478bd9Sstevel@tonic-gate sp->slab_prev = cp->cache_freelist->slab_prev; 15487c478bd9Sstevel@tonic-gate sp->slab_next->slab_prev = sp; 15497c478bd9Sstevel@tonic-gate sp->slab_prev->slab_next = sp; 15507c478bd9Sstevel@tonic-gate cp->cache_freelist = sp; 15517c478bd9Sstevel@tonic-gate } 15527c478bd9Sstevel@tonic-gate 15537c478bd9Sstevel@tonic-gate sp->slab_refcnt++; 15547c478bd9Sstevel@tonic-gate ASSERT(sp->slab_refcnt <= sp->slab_chunks); 15557c478bd9Sstevel@tonic-gate 15567c478bd9Sstevel@tonic-gate /* 15577c478bd9Sstevel@tonic-gate * If we're taking the last buffer in the slab, 15587c478bd9Sstevel@tonic-gate * remove the slab from the cache's freelist. 15597c478bd9Sstevel@tonic-gate */ 15607c478bd9Sstevel@tonic-gate bcp = sp->slab_head; 15617c478bd9Sstevel@tonic-gate if ((sp->slab_head = bcp->bc_next) == NULL) { 15627c478bd9Sstevel@tonic-gate cp->cache_freelist = sp->slab_next; 15637c478bd9Sstevel@tonic-gate ASSERT(sp->slab_refcnt == sp->slab_chunks); 15647c478bd9Sstevel@tonic-gate } 15657c478bd9Sstevel@tonic-gate 15667c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_HASH) { 15677c478bd9Sstevel@tonic-gate /* 15687c478bd9Sstevel@tonic-gate * Add buffer to allocated-address hash table. 15697c478bd9Sstevel@tonic-gate */ 15707c478bd9Sstevel@tonic-gate buf = bcp->bc_addr; 15717c478bd9Sstevel@tonic-gate hash_bucket = UMEM_HASH(cp, buf); 15727c478bd9Sstevel@tonic-gate bcp->bc_next = *hash_bucket; 15737c478bd9Sstevel@tonic-gate *hash_bucket = bcp; 15747c478bd9Sstevel@tonic-gate if ((cp->cache_flags & (UMF_AUDIT | UMF_BUFTAG)) == UMF_AUDIT) { 15757c478bd9Sstevel@tonic-gate UMEM_AUDIT(umem_transaction_log, cp, bcp); 15767c478bd9Sstevel@tonic-gate } 15777c478bd9Sstevel@tonic-gate } else { 15787c478bd9Sstevel@tonic-gate buf = UMEM_BUF(cp, bcp); 15797c478bd9Sstevel@tonic-gate } 15807c478bd9Sstevel@tonic-gate 15817c478bd9Sstevel@tonic-gate ASSERT(UMEM_SLAB_MEMBER(sp, buf)); 15827c478bd9Sstevel@tonic-gate 15837c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 15847c478bd9Sstevel@tonic-gate 15857c478bd9Sstevel@tonic-gate return (buf); 15867c478bd9Sstevel@tonic-gate } 15877c478bd9Sstevel@tonic-gate 15887c478bd9Sstevel@tonic-gate /* 15897c478bd9Sstevel@tonic-gate * Free a raw (unconstructed) buffer to cp's slab layer. 15907c478bd9Sstevel@tonic-gate */ 15917c478bd9Sstevel@tonic-gate static void 15927c478bd9Sstevel@tonic-gate umem_slab_free(umem_cache_t *cp, void *buf) 15937c478bd9Sstevel@tonic-gate { 15947c478bd9Sstevel@tonic-gate umem_slab_t *sp; 15957c478bd9Sstevel@tonic-gate umem_bufctl_t *bcp, **prev_bcpp; 15967c478bd9Sstevel@tonic-gate 15977c478bd9Sstevel@tonic-gate ASSERT(buf != NULL); 15987c478bd9Sstevel@tonic-gate 15997c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_lock); 16007c478bd9Sstevel@tonic-gate cp->cache_slab_free++; 16017c478bd9Sstevel@tonic-gate 16027c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_HASH) { 16037c478bd9Sstevel@tonic-gate /* 16047c478bd9Sstevel@tonic-gate * Look up buffer in allocated-address hash table. 16057c478bd9Sstevel@tonic-gate */ 16067c478bd9Sstevel@tonic-gate prev_bcpp = UMEM_HASH(cp, buf); 16077c478bd9Sstevel@tonic-gate while ((bcp = *prev_bcpp) != NULL) { 16087c478bd9Sstevel@tonic-gate if (bcp->bc_addr == buf) { 16097c478bd9Sstevel@tonic-gate *prev_bcpp = bcp->bc_next; 16107c478bd9Sstevel@tonic-gate sp = bcp->bc_slab; 16117c478bd9Sstevel@tonic-gate break; 16127c478bd9Sstevel@tonic-gate } 16137c478bd9Sstevel@tonic-gate cp->cache_lookup_depth++; 16147c478bd9Sstevel@tonic-gate prev_bcpp = &bcp->bc_next; 16157c478bd9Sstevel@tonic-gate } 16167c478bd9Sstevel@tonic-gate } else { 16177c478bd9Sstevel@tonic-gate bcp = UMEM_BUFCTL(cp, buf); 16187c478bd9Sstevel@tonic-gate sp = UMEM_SLAB(cp, buf); 16197c478bd9Sstevel@tonic-gate } 16207c478bd9Sstevel@tonic-gate 16217c478bd9Sstevel@tonic-gate if (bcp == NULL || sp->slab_cache != cp || !UMEM_SLAB_MEMBER(sp, buf)) { 16227c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 16237c478bd9Sstevel@tonic-gate umem_error(UMERR_BADADDR, cp, buf); 16247c478bd9Sstevel@tonic-gate return; 16257c478bd9Sstevel@tonic-gate } 16267c478bd9Sstevel@tonic-gate 16277c478bd9Sstevel@tonic-gate if ((cp->cache_flags & (UMF_AUDIT | UMF_BUFTAG)) == UMF_AUDIT) { 16287c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_CONTENTS) 16297c478bd9Sstevel@tonic-gate ((umem_bufctl_audit_t *)bcp)->bc_contents = 16307c478bd9Sstevel@tonic-gate umem_log_enter(umem_content_log, buf, 16317c478bd9Sstevel@tonic-gate cp->cache_contents); 16327c478bd9Sstevel@tonic-gate UMEM_AUDIT(umem_transaction_log, cp, bcp); 16337c478bd9Sstevel@tonic-gate } 16347c478bd9Sstevel@tonic-gate 16357c478bd9Sstevel@tonic-gate /* 16367c478bd9Sstevel@tonic-gate * If this slab isn't currently on the freelist, put it there. 16377c478bd9Sstevel@tonic-gate */ 16387c478bd9Sstevel@tonic-gate if (sp->slab_head == NULL) { 16397c478bd9Sstevel@tonic-gate ASSERT(sp->slab_refcnt == sp->slab_chunks); 16407c478bd9Sstevel@tonic-gate ASSERT(cp->cache_freelist != sp); 16417c478bd9Sstevel@tonic-gate sp->slab_next->slab_prev = sp->slab_prev; 16427c478bd9Sstevel@tonic-gate sp->slab_prev->slab_next = sp->slab_next; 16437c478bd9Sstevel@tonic-gate sp->slab_next = cp->cache_freelist; 16447c478bd9Sstevel@tonic-gate sp->slab_prev = cp->cache_freelist->slab_prev; 16457c478bd9Sstevel@tonic-gate sp->slab_next->slab_prev = sp; 16467c478bd9Sstevel@tonic-gate sp->slab_prev->slab_next = sp; 16477c478bd9Sstevel@tonic-gate cp->cache_freelist = sp; 16487c478bd9Sstevel@tonic-gate } 16497c478bd9Sstevel@tonic-gate 16507c478bd9Sstevel@tonic-gate bcp->bc_next = sp->slab_head; 16517c478bd9Sstevel@tonic-gate sp->slab_head = bcp; 16527c478bd9Sstevel@tonic-gate 16537c478bd9Sstevel@tonic-gate ASSERT(sp->slab_refcnt >= 1); 16547c478bd9Sstevel@tonic-gate if (--sp->slab_refcnt == 0) { 16557c478bd9Sstevel@tonic-gate /* 16567c478bd9Sstevel@tonic-gate * There are no outstanding allocations from this slab, 16577c478bd9Sstevel@tonic-gate * so we can reclaim the memory. 16587c478bd9Sstevel@tonic-gate */ 16597c478bd9Sstevel@tonic-gate sp->slab_next->slab_prev = sp->slab_prev; 16607c478bd9Sstevel@tonic-gate sp->slab_prev->slab_next = sp->slab_next; 16617c478bd9Sstevel@tonic-gate if (sp == cp->cache_freelist) 16627c478bd9Sstevel@tonic-gate cp->cache_freelist = sp->slab_next; 16637c478bd9Sstevel@tonic-gate cp->cache_slab_destroy++; 16647c478bd9Sstevel@tonic-gate cp->cache_buftotal -= sp->slab_chunks; 16657c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 16667c478bd9Sstevel@tonic-gate umem_slab_destroy(cp, sp); 16677c478bd9Sstevel@tonic-gate return; 16687c478bd9Sstevel@tonic-gate } 16697c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 16707c478bd9Sstevel@tonic-gate } 16717c478bd9Sstevel@tonic-gate 16727c478bd9Sstevel@tonic-gate static int 16737c478bd9Sstevel@tonic-gate umem_cache_alloc_debug(umem_cache_t *cp, void *buf, int umflag) 16747c478bd9Sstevel@tonic-gate { 16757c478bd9Sstevel@tonic-gate umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); 16767c478bd9Sstevel@tonic-gate umem_bufctl_audit_t *bcp = (umem_bufctl_audit_t *)btp->bt_bufctl; 16777c478bd9Sstevel@tonic-gate uint32_t mtbf; 16787c478bd9Sstevel@tonic-gate int flags_nfatal; 16797c478bd9Sstevel@tonic-gate 16807c478bd9Sstevel@tonic-gate if (btp->bt_bxstat != ((intptr_t)bcp ^ UMEM_BUFTAG_FREE)) { 16817c478bd9Sstevel@tonic-gate umem_error(UMERR_BADBUFTAG, cp, buf); 16827c478bd9Sstevel@tonic-gate return (-1); 16837c478bd9Sstevel@tonic-gate } 16847c478bd9Sstevel@tonic-gate 16857c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ UMEM_BUFTAG_ALLOC; 16867c478bd9Sstevel@tonic-gate 16877c478bd9Sstevel@tonic-gate if ((cp->cache_flags & UMF_HASH) && bcp->bc_addr != buf) { 16887c478bd9Sstevel@tonic-gate umem_error(UMERR_BADBUFCTL, cp, buf); 16897c478bd9Sstevel@tonic-gate return (-1); 16907c478bd9Sstevel@tonic-gate } 16917c478bd9Sstevel@tonic-gate 16927c478bd9Sstevel@tonic-gate btp->bt_redzone = UMEM_REDZONE_PATTERN; 16937c478bd9Sstevel@tonic-gate 16947c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_DEADBEEF) { 16957c478bd9Sstevel@tonic-gate if (verify_and_copy_pattern(UMEM_FREE_PATTERN, 16967c478bd9Sstevel@tonic-gate UMEM_UNINITIALIZED_PATTERN, buf, cp->cache_verify)) { 16977c478bd9Sstevel@tonic-gate umem_error(UMERR_MODIFIED, cp, buf); 16987c478bd9Sstevel@tonic-gate return (-1); 16997c478bd9Sstevel@tonic-gate } 17007c478bd9Sstevel@tonic-gate } 17017c478bd9Sstevel@tonic-gate 17027c478bd9Sstevel@tonic-gate if ((mtbf = umem_mtbf | cp->cache_mtbf) != 0 && 17037c478bd9Sstevel@tonic-gate gethrtime() % mtbf == 0 && 17047c478bd9Sstevel@tonic-gate (umflag & (UMEM_FATAL_FLAGS)) == 0) { 17057c478bd9Sstevel@tonic-gate umem_log_event(umem_failure_log, cp, NULL, NULL); 17067c478bd9Sstevel@tonic-gate } else { 17077c478bd9Sstevel@tonic-gate mtbf = 0; 17087c478bd9Sstevel@tonic-gate } 17097c478bd9Sstevel@tonic-gate 17107c478bd9Sstevel@tonic-gate /* 17117c478bd9Sstevel@tonic-gate * We do not pass fatal flags on to the constructor. This prevents 17127c478bd9Sstevel@tonic-gate * leaking buffers in the event of a subordinate constructor failing. 17137c478bd9Sstevel@tonic-gate */ 17147c478bd9Sstevel@tonic-gate flags_nfatal = UMEM_DEFAULT; 17157c478bd9Sstevel@tonic-gate if (mtbf || (cp->cache_constructor != NULL && 17167c478bd9Sstevel@tonic-gate cp->cache_constructor(buf, cp->cache_private, flags_nfatal) != 0)) { 17177c478bd9Sstevel@tonic-gate atomic_add_64(&cp->cache_alloc_fail, 1); 17187c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ UMEM_BUFTAG_FREE; 17197c478bd9Sstevel@tonic-gate copy_pattern(UMEM_FREE_PATTERN, buf, cp->cache_verify); 17207c478bd9Sstevel@tonic-gate umem_slab_free(cp, buf); 17217c478bd9Sstevel@tonic-gate return (-1); 17227c478bd9Sstevel@tonic-gate } 17237c478bd9Sstevel@tonic-gate 17247c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_AUDIT) { 17257c478bd9Sstevel@tonic-gate UMEM_AUDIT(umem_transaction_log, cp, bcp); 17267c478bd9Sstevel@tonic-gate } 17277c478bd9Sstevel@tonic-gate 17287c478bd9Sstevel@tonic-gate return (0); 17297c478bd9Sstevel@tonic-gate } 17307c478bd9Sstevel@tonic-gate 17317c478bd9Sstevel@tonic-gate static int 17327c478bd9Sstevel@tonic-gate umem_cache_free_debug(umem_cache_t *cp, void *buf) 17337c478bd9Sstevel@tonic-gate { 17347c478bd9Sstevel@tonic-gate umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); 17357c478bd9Sstevel@tonic-gate umem_bufctl_audit_t *bcp = (umem_bufctl_audit_t *)btp->bt_bufctl; 17367c478bd9Sstevel@tonic-gate umem_slab_t *sp; 17377c478bd9Sstevel@tonic-gate 17387c478bd9Sstevel@tonic-gate if (btp->bt_bxstat != ((intptr_t)bcp ^ UMEM_BUFTAG_ALLOC)) { 17397c478bd9Sstevel@tonic-gate if (btp->bt_bxstat == ((intptr_t)bcp ^ UMEM_BUFTAG_FREE)) { 17407c478bd9Sstevel@tonic-gate umem_error(UMERR_DUPFREE, cp, buf); 17417c478bd9Sstevel@tonic-gate return (-1); 17427c478bd9Sstevel@tonic-gate } 17437c478bd9Sstevel@tonic-gate sp = umem_findslab(cp, buf); 17447c478bd9Sstevel@tonic-gate if (sp == NULL || sp->slab_cache != cp) 17457c478bd9Sstevel@tonic-gate umem_error(UMERR_BADADDR, cp, buf); 17467c478bd9Sstevel@tonic-gate else 17477c478bd9Sstevel@tonic-gate umem_error(UMERR_REDZONE, cp, buf); 17487c478bd9Sstevel@tonic-gate return (-1); 17497c478bd9Sstevel@tonic-gate } 17507c478bd9Sstevel@tonic-gate 17517c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ UMEM_BUFTAG_FREE; 17527c478bd9Sstevel@tonic-gate 17537c478bd9Sstevel@tonic-gate if ((cp->cache_flags & UMF_HASH) && bcp->bc_addr != buf) { 17547c478bd9Sstevel@tonic-gate umem_error(UMERR_BADBUFCTL, cp, buf); 17557c478bd9Sstevel@tonic-gate return (-1); 17567c478bd9Sstevel@tonic-gate } 17577c478bd9Sstevel@tonic-gate 17587c478bd9Sstevel@tonic-gate if (btp->bt_redzone != UMEM_REDZONE_PATTERN) { 17597c478bd9Sstevel@tonic-gate umem_error(UMERR_REDZONE, cp, buf); 17607c478bd9Sstevel@tonic-gate return (-1); 17617c478bd9Sstevel@tonic-gate } 17627c478bd9Sstevel@tonic-gate 17637c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_AUDIT) { 17647c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_CONTENTS) 17657c478bd9Sstevel@tonic-gate bcp->bc_contents = umem_log_enter(umem_content_log, 17667c478bd9Sstevel@tonic-gate buf, cp->cache_contents); 17677c478bd9Sstevel@tonic-gate UMEM_AUDIT(umem_transaction_log, cp, bcp); 17687c478bd9Sstevel@tonic-gate } 17697c478bd9Sstevel@tonic-gate 17707c478bd9Sstevel@tonic-gate if (cp->cache_destructor != NULL) 17717c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 17727c478bd9Sstevel@tonic-gate 17737c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_DEADBEEF) 17747c478bd9Sstevel@tonic-gate copy_pattern(UMEM_FREE_PATTERN, buf, cp->cache_verify); 17757c478bd9Sstevel@tonic-gate 17767c478bd9Sstevel@tonic-gate return (0); 17777c478bd9Sstevel@tonic-gate } 17787c478bd9Sstevel@tonic-gate 17797c478bd9Sstevel@tonic-gate /* 17807c478bd9Sstevel@tonic-gate * Free each object in magazine mp to cp's slab layer, and free mp itself. 17817c478bd9Sstevel@tonic-gate */ 17827c478bd9Sstevel@tonic-gate static void 17837c478bd9Sstevel@tonic-gate umem_magazine_destroy(umem_cache_t *cp, umem_magazine_t *mp, int nrounds) 17847c478bd9Sstevel@tonic-gate { 17857c478bd9Sstevel@tonic-gate int round; 17867c478bd9Sstevel@tonic-gate 17877c478bd9Sstevel@tonic-gate ASSERT(cp->cache_next == NULL || IN_UPDATE()); 17887c478bd9Sstevel@tonic-gate 17897c478bd9Sstevel@tonic-gate for (round = 0; round < nrounds; round++) { 17907c478bd9Sstevel@tonic-gate void *buf = mp->mag_round[round]; 17917c478bd9Sstevel@tonic-gate 17927c478bd9Sstevel@tonic-gate if ((cp->cache_flags & UMF_DEADBEEF) && 17937c478bd9Sstevel@tonic-gate verify_pattern(UMEM_FREE_PATTERN, buf, 17947c478bd9Sstevel@tonic-gate cp->cache_verify) != NULL) { 17957c478bd9Sstevel@tonic-gate umem_error(UMERR_MODIFIED, cp, buf); 17967c478bd9Sstevel@tonic-gate continue; 17977c478bd9Sstevel@tonic-gate } 17987c478bd9Sstevel@tonic-gate 17997c478bd9Sstevel@tonic-gate if (!(cp->cache_flags & UMF_BUFTAG) && 18007c478bd9Sstevel@tonic-gate cp->cache_destructor != NULL) 18017c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 18027c478bd9Sstevel@tonic-gate 18037c478bd9Sstevel@tonic-gate umem_slab_free(cp, buf); 18047c478bd9Sstevel@tonic-gate } 18057c478bd9Sstevel@tonic-gate ASSERT(UMEM_MAGAZINE_VALID(cp, mp)); 18067c478bd9Sstevel@tonic-gate _umem_cache_free(cp->cache_magtype->mt_cache, mp); 18077c478bd9Sstevel@tonic-gate } 18087c478bd9Sstevel@tonic-gate 18097c478bd9Sstevel@tonic-gate /* 18107c478bd9Sstevel@tonic-gate * Allocate a magazine from the depot. 18117c478bd9Sstevel@tonic-gate */ 18127c478bd9Sstevel@tonic-gate static umem_magazine_t * 18137c478bd9Sstevel@tonic-gate umem_depot_alloc(umem_cache_t *cp, umem_maglist_t *mlp) 18147c478bd9Sstevel@tonic-gate { 18157c478bd9Sstevel@tonic-gate umem_magazine_t *mp; 18167c478bd9Sstevel@tonic-gate 18177c478bd9Sstevel@tonic-gate /* 18187c478bd9Sstevel@tonic-gate * If we can't get the depot lock without contention, 18197c478bd9Sstevel@tonic-gate * update our contention count. We use the depot 18207c478bd9Sstevel@tonic-gate * contention rate to determine whether we need to 18217c478bd9Sstevel@tonic-gate * increase the magazine size for better scalability. 18227c478bd9Sstevel@tonic-gate */ 18237c478bd9Sstevel@tonic-gate if (mutex_trylock(&cp->cache_depot_lock) != 0) { 18247c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_depot_lock); 18257c478bd9Sstevel@tonic-gate cp->cache_depot_contention++; 18267c478bd9Sstevel@tonic-gate } 18277c478bd9Sstevel@tonic-gate 18287c478bd9Sstevel@tonic-gate if ((mp = mlp->ml_list) != NULL) { 18297c478bd9Sstevel@tonic-gate ASSERT(UMEM_MAGAZINE_VALID(cp, mp)); 18307c478bd9Sstevel@tonic-gate mlp->ml_list = mp->mag_next; 18317c478bd9Sstevel@tonic-gate if (--mlp->ml_total < mlp->ml_min) 18327c478bd9Sstevel@tonic-gate mlp->ml_min = mlp->ml_total; 18337c478bd9Sstevel@tonic-gate mlp->ml_alloc++; 18347c478bd9Sstevel@tonic-gate } 18357c478bd9Sstevel@tonic-gate 18367c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_depot_lock); 18377c478bd9Sstevel@tonic-gate 18387c478bd9Sstevel@tonic-gate return (mp); 18397c478bd9Sstevel@tonic-gate } 18407c478bd9Sstevel@tonic-gate 18417c478bd9Sstevel@tonic-gate /* 18427c478bd9Sstevel@tonic-gate * Free a magazine to the depot. 18437c478bd9Sstevel@tonic-gate */ 18447c478bd9Sstevel@tonic-gate static void 18457c478bd9Sstevel@tonic-gate umem_depot_free(umem_cache_t *cp, umem_maglist_t *mlp, umem_magazine_t *mp) 18467c478bd9Sstevel@tonic-gate { 18477c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_depot_lock); 18487c478bd9Sstevel@tonic-gate ASSERT(UMEM_MAGAZINE_VALID(cp, mp)); 18497c478bd9Sstevel@tonic-gate mp->mag_next = mlp->ml_list; 18507c478bd9Sstevel@tonic-gate mlp->ml_list = mp; 18517c478bd9Sstevel@tonic-gate mlp->ml_total++; 18527c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_depot_lock); 18537c478bd9Sstevel@tonic-gate } 18547c478bd9Sstevel@tonic-gate 18557c478bd9Sstevel@tonic-gate /* 18567c478bd9Sstevel@tonic-gate * Update the working set statistics for cp's depot. 18577c478bd9Sstevel@tonic-gate */ 18587c478bd9Sstevel@tonic-gate static void 18597c478bd9Sstevel@tonic-gate umem_depot_ws_update(umem_cache_t *cp) 18607c478bd9Sstevel@tonic-gate { 18617c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_depot_lock); 18627c478bd9Sstevel@tonic-gate cp->cache_full.ml_reaplimit = cp->cache_full.ml_min; 18637c478bd9Sstevel@tonic-gate cp->cache_full.ml_min = cp->cache_full.ml_total; 18647c478bd9Sstevel@tonic-gate cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_min; 18657c478bd9Sstevel@tonic-gate cp->cache_empty.ml_min = cp->cache_empty.ml_total; 18667c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_depot_lock); 18677c478bd9Sstevel@tonic-gate } 18687c478bd9Sstevel@tonic-gate 18697c478bd9Sstevel@tonic-gate /* 18707c478bd9Sstevel@tonic-gate * Reap all magazines that have fallen out of the depot's working set. 18717c478bd9Sstevel@tonic-gate */ 18727c478bd9Sstevel@tonic-gate static void 18737c478bd9Sstevel@tonic-gate umem_depot_ws_reap(umem_cache_t *cp) 18747c478bd9Sstevel@tonic-gate { 18757c478bd9Sstevel@tonic-gate long reap; 18767c478bd9Sstevel@tonic-gate umem_magazine_t *mp; 18777c478bd9Sstevel@tonic-gate 18787c478bd9Sstevel@tonic-gate ASSERT(cp->cache_next == NULL || IN_REAP()); 18797c478bd9Sstevel@tonic-gate 18807c478bd9Sstevel@tonic-gate reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min); 18817c478bd9Sstevel@tonic-gate while (reap-- && (mp = umem_depot_alloc(cp, &cp->cache_full)) != NULL) 18827c478bd9Sstevel@tonic-gate umem_magazine_destroy(cp, mp, cp->cache_magtype->mt_magsize); 18837c478bd9Sstevel@tonic-gate 18847c478bd9Sstevel@tonic-gate reap = MIN(cp->cache_empty.ml_reaplimit, cp->cache_empty.ml_min); 18857c478bd9Sstevel@tonic-gate while (reap-- && (mp = umem_depot_alloc(cp, &cp->cache_empty)) != NULL) 18867c478bd9Sstevel@tonic-gate umem_magazine_destroy(cp, mp, 0); 18877c478bd9Sstevel@tonic-gate } 18887c478bd9Sstevel@tonic-gate 18897c478bd9Sstevel@tonic-gate static void 18907c478bd9Sstevel@tonic-gate umem_cpu_reload(umem_cpu_cache_t *ccp, umem_magazine_t *mp, int rounds) 18917c478bd9Sstevel@tonic-gate { 18927c478bd9Sstevel@tonic-gate ASSERT((ccp->cc_loaded == NULL && ccp->cc_rounds == -1) || 18937c478bd9Sstevel@tonic-gate (ccp->cc_loaded && ccp->cc_rounds + rounds == ccp->cc_magsize)); 18947c478bd9Sstevel@tonic-gate ASSERT(ccp->cc_magsize > 0); 18957c478bd9Sstevel@tonic-gate 18967c478bd9Sstevel@tonic-gate ccp->cc_ploaded = ccp->cc_loaded; 18977c478bd9Sstevel@tonic-gate ccp->cc_prounds = ccp->cc_rounds; 18987c478bd9Sstevel@tonic-gate ccp->cc_loaded = mp; 18997c478bd9Sstevel@tonic-gate ccp->cc_rounds = rounds; 19007c478bd9Sstevel@tonic-gate } 19017c478bd9Sstevel@tonic-gate 19027c478bd9Sstevel@tonic-gate /* 19037c478bd9Sstevel@tonic-gate * Allocate a constructed object from cache cp. 19047c478bd9Sstevel@tonic-gate */ 19057c478bd9Sstevel@tonic-gate #pragma weak umem_cache_alloc = _umem_cache_alloc 19067c478bd9Sstevel@tonic-gate void * 19077c478bd9Sstevel@tonic-gate _umem_cache_alloc(umem_cache_t *cp, int umflag) 19087c478bd9Sstevel@tonic-gate { 19097c478bd9Sstevel@tonic-gate umem_cpu_cache_t *ccp; 19107c478bd9Sstevel@tonic-gate umem_magazine_t *fmp; 19117c478bd9Sstevel@tonic-gate void *buf; 19127c478bd9Sstevel@tonic-gate int flags_nfatal; 19137c478bd9Sstevel@tonic-gate 19147c478bd9Sstevel@tonic-gate retry: 19157c478bd9Sstevel@tonic-gate ccp = UMEM_CPU_CACHE(cp, CPU(cp->cache_cpu_mask)); 19167c478bd9Sstevel@tonic-gate (void) mutex_lock(&ccp->cc_lock); 19177c478bd9Sstevel@tonic-gate for (;;) { 19187c478bd9Sstevel@tonic-gate /* 19197c478bd9Sstevel@tonic-gate * If there's an object available in the current CPU's 19207c478bd9Sstevel@tonic-gate * loaded magazine, just take it and return. 19217c478bd9Sstevel@tonic-gate */ 19227c478bd9Sstevel@tonic-gate if (ccp->cc_rounds > 0) { 19237c478bd9Sstevel@tonic-gate buf = ccp->cc_loaded->mag_round[--ccp->cc_rounds]; 19247c478bd9Sstevel@tonic-gate ccp->cc_alloc++; 19257c478bd9Sstevel@tonic-gate (void) mutex_unlock(&ccp->cc_lock); 19267c478bd9Sstevel@tonic-gate if ((ccp->cc_flags & UMF_BUFTAG) && 19277c478bd9Sstevel@tonic-gate umem_cache_alloc_debug(cp, buf, umflag) == -1) { 19287c478bd9Sstevel@tonic-gate if (umem_alloc_retry(cp, umflag)) { 19297c478bd9Sstevel@tonic-gate goto retry; 19307c478bd9Sstevel@tonic-gate } 19317c478bd9Sstevel@tonic-gate 19327c478bd9Sstevel@tonic-gate return (NULL); 19337c478bd9Sstevel@tonic-gate } 19347c478bd9Sstevel@tonic-gate return (buf); 19357c478bd9Sstevel@tonic-gate } 19367c478bd9Sstevel@tonic-gate 19377c478bd9Sstevel@tonic-gate /* 19387c478bd9Sstevel@tonic-gate * The loaded magazine is empty. If the previously loaded 19397c478bd9Sstevel@tonic-gate * magazine was full, exchange them and try again. 19407c478bd9Sstevel@tonic-gate */ 19417c478bd9Sstevel@tonic-gate if (ccp->cc_prounds > 0) { 19427c478bd9Sstevel@tonic-gate umem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds); 19437c478bd9Sstevel@tonic-gate continue; 19447c478bd9Sstevel@tonic-gate } 19457c478bd9Sstevel@tonic-gate 19467c478bd9Sstevel@tonic-gate /* 19477c478bd9Sstevel@tonic-gate * If the magazine layer is disabled, break out now. 19487c478bd9Sstevel@tonic-gate */ 19497c478bd9Sstevel@tonic-gate if (ccp->cc_magsize == 0) 19507c478bd9Sstevel@tonic-gate break; 19517c478bd9Sstevel@tonic-gate 19527c478bd9Sstevel@tonic-gate /* 19537c478bd9Sstevel@tonic-gate * Try to get a full magazine from the depot. 19547c478bd9Sstevel@tonic-gate */ 19557c478bd9Sstevel@tonic-gate fmp = umem_depot_alloc(cp, &cp->cache_full); 19567c478bd9Sstevel@tonic-gate if (fmp != NULL) { 19577c478bd9Sstevel@tonic-gate if (ccp->cc_ploaded != NULL) 19587c478bd9Sstevel@tonic-gate umem_depot_free(cp, &cp->cache_empty, 19597c478bd9Sstevel@tonic-gate ccp->cc_ploaded); 19607c478bd9Sstevel@tonic-gate umem_cpu_reload(ccp, fmp, ccp->cc_magsize); 19617c478bd9Sstevel@tonic-gate continue; 19627c478bd9Sstevel@tonic-gate } 19637c478bd9Sstevel@tonic-gate 19647c478bd9Sstevel@tonic-gate /* 19657c478bd9Sstevel@tonic-gate * There are no full magazines in the depot, 19667c478bd9Sstevel@tonic-gate * so fall through to the slab layer. 19677c478bd9Sstevel@tonic-gate */ 19687c478bd9Sstevel@tonic-gate break; 19697c478bd9Sstevel@tonic-gate } 19707c478bd9Sstevel@tonic-gate (void) mutex_unlock(&ccp->cc_lock); 19717c478bd9Sstevel@tonic-gate 19727c478bd9Sstevel@tonic-gate /* 19737c478bd9Sstevel@tonic-gate * We couldn't allocate a constructed object from the magazine layer, 19747c478bd9Sstevel@tonic-gate * so get a raw buffer from the slab layer and apply its constructor. 19757c478bd9Sstevel@tonic-gate */ 19767c478bd9Sstevel@tonic-gate buf = umem_slab_alloc(cp, umflag); 19777c478bd9Sstevel@tonic-gate 19787c478bd9Sstevel@tonic-gate if (buf == NULL) { 19797c478bd9Sstevel@tonic-gate if (cp == &umem_null_cache) 19807c478bd9Sstevel@tonic-gate return (NULL); 19817c478bd9Sstevel@tonic-gate if (umem_alloc_retry(cp, umflag)) { 19827c478bd9Sstevel@tonic-gate goto retry; 19837c478bd9Sstevel@tonic-gate } 19847c478bd9Sstevel@tonic-gate 19857c478bd9Sstevel@tonic-gate return (NULL); 19867c478bd9Sstevel@tonic-gate } 19877c478bd9Sstevel@tonic-gate 19887c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_BUFTAG) { 19897c478bd9Sstevel@tonic-gate /* 19907c478bd9Sstevel@tonic-gate * Let umem_cache_alloc_debug() apply the constructor for us. 19917c478bd9Sstevel@tonic-gate */ 19927c478bd9Sstevel@tonic-gate if (umem_cache_alloc_debug(cp, buf, umflag) == -1) { 19937c478bd9Sstevel@tonic-gate if (umem_alloc_retry(cp, umflag)) { 19947c478bd9Sstevel@tonic-gate goto retry; 19957c478bd9Sstevel@tonic-gate } 19967c478bd9Sstevel@tonic-gate return (NULL); 19977c478bd9Sstevel@tonic-gate } 19987c478bd9Sstevel@tonic-gate return (buf); 19997c478bd9Sstevel@tonic-gate } 20007c478bd9Sstevel@tonic-gate 20017c478bd9Sstevel@tonic-gate /* 20027c478bd9Sstevel@tonic-gate * We do not pass fatal flags on to the constructor. This prevents 20037c478bd9Sstevel@tonic-gate * leaking buffers in the event of a subordinate constructor failing. 20047c478bd9Sstevel@tonic-gate */ 20057c478bd9Sstevel@tonic-gate flags_nfatal = UMEM_DEFAULT; 20067c478bd9Sstevel@tonic-gate if (cp->cache_constructor != NULL && 20077c478bd9Sstevel@tonic-gate cp->cache_constructor(buf, cp->cache_private, flags_nfatal) != 0) { 20087c478bd9Sstevel@tonic-gate atomic_add_64(&cp->cache_alloc_fail, 1); 20097c478bd9Sstevel@tonic-gate umem_slab_free(cp, buf); 20107c478bd9Sstevel@tonic-gate 20117c478bd9Sstevel@tonic-gate if (umem_alloc_retry(cp, umflag)) { 20127c478bd9Sstevel@tonic-gate goto retry; 20137c478bd9Sstevel@tonic-gate } 20147c478bd9Sstevel@tonic-gate return (NULL); 20157c478bd9Sstevel@tonic-gate } 20167c478bd9Sstevel@tonic-gate 20177c478bd9Sstevel@tonic-gate return (buf); 20187c478bd9Sstevel@tonic-gate } 20197c478bd9Sstevel@tonic-gate 20207c478bd9Sstevel@tonic-gate /* 20217c478bd9Sstevel@tonic-gate * Free a constructed object to cache cp. 20227c478bd9Sstevel@tonic-gate */ 20237c478bd9Sstevel@tonic-gate #pragma weak umem_cache_free = _umem_cache_free 20247c478bd9Sstevel@tonic-gate void 20257c478bd9Sstevel@tonic-gate _umem_cache_free(umem_cache_t *cp, void *buf) 20267c478bd9Sstevel@tonic-gate { 20277c478bd9Sstevel@tonic-gate umem_cpu_cache_t *ccp = UMEM_CPU_CACHE(cp, CPU(cp->cache_cpu_mask)); 20287c478bd9Sstevel@tonic-gate umem_magazine_t *emp; 20297c478bd9Sstevel@tonic-gate umem_magtype_t *mtp; 20307c478bd9Sstevel@tonic-gate 20317c478bd9Sstevel@tonic-gate if (ccp->cc_flags & UMF_BUFTAG) 20327c478bd9Sstevel@tonic-gate if (umem_cache_free_debug(cp, buf) == -1) 20337c478bd9Sstevel@tonic-gate return; 20347c478bd9Sstevel@tonic-gate 20357c478bd9Sstevel@tonic-gate (void) mutex_lock(&ccp->cc_lock); 20367c478bd9Sstevel@tonic-gate for (;;) { 20377c478bd9Sstevel@tonic-gate /* 20387c478bd9Sstevel@tonic-gate * If there's a slot available in the current CPU's 20397c478bd9Sstevel@tonic-gate * loaded magazine, just put the object there and return. 20407c478bd9Sstevel@tonic-gate */ 20417c478bd9Sstevel@tonic-gate if ((uint_t)ccp->cc_rounds < ccp->cc_magsize) { 20427c478bd9Sstevel@tonic-gate ccp->cc_loaded->mag_round[ccp->cc_rounds++] = buf; 20437c478bd9Sstevel@tonic-gate ccp->cc_free++; 20447c478bd9Sstevel@tonic-gate (void) mutex_unlock(&ccp->cc_lock); 20457c478bd9Sstevel@tonic-gate return; 20467c478bd9Sstevel@tonic-gate } 20477c478bd9Sstevel@tonic-gate 20487c478bd9Sstevel@tonic-gate /* 20497c478bd9Sstevel@tonic-gate * The loaded magazine is full. If the previously loaded 20507c478bd9Sstevel@tonic-gate * magazine was empty, exchange them and try again. 20517c478bd9Sstevel@tonic-gate */ 20527c478bd9Sstevel@tonic-gate if (ccp->cc_prounds == 0) { 20537c478bd9Sstevel@tonic-gate umem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds); 20547c478bd9Sstevel@tonic-gate continue; 20557c478bd9Sstevel@tonic-gate } 20567c478bd9Sstevel@tonic-gate 20577c478bd9Sstevel@tonic-gate /* 20587c478bd9Sstevel@tonic-gate * If the magazine layer is disabled, break out now. 20597c478bd9Sstevel@tonic-gate */ 20607c478bd9Sstevel@tonic-gate if (ccp->cc_magsize == 0) 20617c478bd9Sstevel@tonic-gate break; 20627c478bd9Sstevel@tonic-gate 20637c478bd9Sstevel@tonic-gate /* 20647c478bd9Sstevel@tonic-gate * Try to get an empty magazine from the depot. 20657c478bd9Sstevel@tonic-gate */ 20667c478bd9Sstevel@tonic-gate emp = umem_depot_alloc(cp, &cp->cache_empty); 20677c478bd9Sstevel@tonic-gate if (emp != NULL) { 20687c478bd9Sstevel@tonic-gate if (ccp->cc_ploaded != NULL) 20697c478bd9Sstevel@tonic-gate umem_depot_free(cp, &cp->cache_full, 20707c478bd9Sstevel@tonic-gate ccp->cc_ploaded); 20717c478bd9Sstevel@tonic-gate umem_cpu_reload(ccp, emp, 0); 20727c478bd9Sstevel@tonic-gate continue; 20737c478bd9Sstevel@tonic-gate } 20747c478bd9Sstevel@tonic-gate 20757c478bd9Sstevel@tonic-gate /* 20767c478bd9Sstevel@tonic-gate * There are no empty magazines in the depot, 20777c478bd9Sstevel@tonic-gate * so try to allocate a new one. We must drop all locks 20787c478bd9Sstevel@tonic-gate * across umem_cache_alloc() because lower layers may 20797c478bd9Sstevel@tonic-gate * attempt to allocate from this cache. 20807c478bd9Sstevel@tonic-gate */ 20817c478bd9Sstevel@tonic-gate mtp = cp->cache_magtype; 20827c478bd9Sstevel@tonic-gate (void) mutex_unlock(&ccp->cc_lock); 20837c478bd9Sstevel@tonic-gate emp = _umem_cache_alloc(mtp->mt_cache, UMEM_DEFAULT); 20847c478bd9Sstevel@tonic-gate (void) mutex_lock(&ccp->cc_lock); 20857c478bd9Sstevel@tonic-gate 20867c478bd9Sstevel@tonic-gate if (emp != NULL) { 20877c478bd9Sstevel@tonic-gate /* 20887c478bd9Sstevel@tonic-gate * We successfully allocated an empty magazine. 20897c478bd9Sstevel@tonic-gate * However, we had to drop ccp->cc_lock to do it, 20907c478bd9Sstevel@tonic-gate * so the cache's magazine size may have changed. 20917c478bd9Sstevel@tonic-gate * If so, free the magazine and try again. 20927c478bd9Sstevel@tonic-gate */ 20937c478bd9Sstevel@tonic-gate if (ccp->cc_magsize != mtp->mt_magsize) { 20947c478bd9Sstevel@tonic-gate (void) mutex_unlock(&ccp->cc_lock); 20957c478bd9Sstevel@tonic-gate _umem_cache_free(mtp->mt_cache, emp); 20967c478bd9Sstevel@tonic-gate (void) mutex_lock(&ccp->cc_lock); 20977c478bd9Sstevel@tonic-gate continue; 20987c478bd9Sstevel@tonic-gate } 20997c478bd9Sstevel@tonic-gate 21007c478bd9Sstevel@tonic-gate /* 21017c478bd9Sstevel@tonic-gate * We got a magazine of the right size. Add it to 21027c478bd9Sstevel@tonic-gate * the depot and try the whole dance again. 21037c478bd9Sstevel@tonic-gate */ 21047c478bd9Sstevel@tonic-gate umem_depot_free(cp, &cp->cache_empty, emp); 21057c478bd9Sstevel@tonic-gate continue; 21067c478bd9Sstevel@tonic-gate } 21077c478bd9Sstevel@tonic-gate 21087c478bd9Sstevel@tonic-gate /* 21097c478bd9Sstevel@tonic-gate * We couldn't allocate an empty magazine, 21107c478bd9Sstevel@tonic-gate * so fall through to the slab layer. 21117c478bd9Sstevel@tonic-gate */ 21127c478bd9Sstevel@tonic-gate break; 21137c478bd9Sstevel@tonic-gate } 21147c478bd9Sstevel@tonic-gate (void) mutex_unlock(&ccp->cc_lock); 21157c478bd9Sstevel@tonic-gate 21167c478bd9Sstevel@tonic-gate /* 21177c478bd9Sstevel@tonic-gate * We couldn't free our constructed object to the magazine layer, 21187c478bd9Sstevel@tonic-gate * so apply its destructor and free it to the slab layer. 21197c478bd9Sstevel@tonic-gate * Note that if UMF_BUFTAG is in effect, umem_cache_free_debug() 21207c478bd9Sstevel@tonic-gate * will have already applied the destructor. 21217c478bd9Sstevel@tonic-gate */ 21227c478bd9Sstevel@tonic-gate if (!(cp->cache_flags & UMF_BUFTAG) && cp->cache_destructor != NULL) 21237c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 21247c478bd9Sstevel@tonic-gate 21257c478bd9Sstevel@tonic-gate umem_slab_free(cp, buf); 21267c478bd9Sstevel@tonic-gate } 21277c478bd9Sstevel@tonic-gate 21287c478bd9Sstevel@tonic-gate #pragma weak umem_zalloc = _umem_zalloc 21297c478bd9Sstevel@tonic-gate void * 21307c478bd9Sstevel@tonic-gate _umem_zalloc(size_t size, int umflag) 21317c478bd9Sstevel@tonic-gate { 21327c478bd9Sstevel@tonic-gate size_t index = (size - 1) >> UMEM_ALIGN_SHIFT; 21337c478bd9Sstevel@tonic-gate void *buf; 21347c478bd9Sstevel@tonic-gate 21357c478bd9Sstevel@tonic-gate retry: 21367c478bd9Sstevel@tonic-gate if (index < UMEM_MAXBUF >> UMEM_ALIGN_SHIFT) { 21377c478bd9Sstevel@tonic-gate umem_cache_t *cp = umem_alloc_table[index]; 21387c478bd9Sstevel@tonic-gate buf = _umem_cache_alloc(cp, umflag); 21397c478bd9Sstevel@tonic-gate if (buf != NULL) { 21407c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_BUFTAG) { 21417c478bd9Sstevel@tonic-gate umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); 21427c478bd9Sstevel@tonic-gate ((uint8_t *)buf)[size] = UMEM_REDZONE_BYTE; 21437c478bd9Sstevel@tonic-gate ((uint32_t *)btp)[1] = UMEM_SIZE_ENCODE(size); 21447c478bd9Sstevel@tonic-gate } 21457c478bd9Sstevel@tonic-gate bzero(buf, size); 21467c478bd9Sstevel@tonic-gate } else if (umem_alloc_retry(cp, umflag)) 21477c478bd9Sstevel@tonic-gate goto retry; 21487c478bd9Sstevel@tonic-gate } else { 21497c478bd9Sstevel@tonic-gate buf = _umem_alloc(size, umflag); /* handles failure */ 21507c478bd9Sstevel@tonic-gate if (buf != NULL) 21517c478bd9Sstevel@tonic-gate bzero(buf, size); 21527c478bd9Sstevel@tonic-gate } 21537c478bd9Sstevel@tonic-gate return (buf); 21547c478bd9Sstevel@tonic-gate } 21557c478bd9Sstevel@tonic-gate 21567c478bd9Sstevel@tonic-gate #pragma weak umem_alloc = _umem_alloc 21577c478bd9Sstevel@tonic-gate void * 21587c478bd9Sstevel@tonic-gate _umem_alloc(size_t size, int umflag) 21597c478bd9Sstevel@tonic-gate { 21607c478bd9Sstevel@tonic-gate size_t index = (size - 1) >> UMEM_ALIGN_SHIFT; 21617c478bd9Sstevel@tonic-gate void *buf; 21627c478bd9Sstevel@tonic-gate umem_alloc_retry: 21637c478bd9Sstevel@tonic-gate if (index < UMEM_MAXBUF >> UMEM_ALIGN_SHIFT) { 21647c478bd9Sstevel@tonic-gate umem_cache_t *cp = umem_alloc_table[index]; 21657c478bd9Sstevel@tonic-gate buf = _umem_cache_alloc(cp, umflag); 21667c478bd9Sstevel@tonic-gate if ((cp->cache_flags & UMF_BUFTAG) && buf != NULL) { 21677c478bd9Sstevel@tonic-gate umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); 21687c478bd9Sstevel@tonic-gate ((uint8_t *)buf)[size] = UMEM_REDZONE_BYTE; 21697c478bd9Sstevel@tonic-gate ((uint32_t *)btp)[1] = UMEM_SIZE_ENCODE(size); 21707c478bd9Sstevel@tonic-gate } 21717c478bd9Sstevel@tonic-gate if (buf == NULL && umem_alloc_retry(cp, umflag)) 21727c478bd9Sstevel@tonic-gate goto umem_alloc_retry; 21737c478bd9Sstevel@tonic-gate return (buf); 21747c478bd9Sstevel@tonic-gate } 21757c478bd9Sstevel@tonic-gate if (size == 0) 21767c478bd9Sstevel@tonic-gate return (NULL); 21777c478bd9Sstevel@tonic-gate if (umem_oversize_arena == NULL) { 21787c478bd9Sstevel@tonic-gate if (umem_init()) 21797c478bd9Sstevel@tonic-gate ASSERT(umem_oversize_arena != NULL); 21807c478bd9Sstevel@tonic-gate else 21817c478bd9Sstevel@tonic-gate return (NULL); 21827c478bd9Sstevel@tonic-gate } 21837c478bd9Sstevel@tonic-gate buf = vmem_alloc(umem_oversize_arena, size, UMEM_VMFLAGS(umflag)); 21847c478bd9Sstevel@tonic-gate if (buf == NULL) { 21857c478bd9Sstevel@tonic-gate umem_log_event(umem_failure_log, NULL, NULL, (void *)size); 21867c478bd9Sstevel@tonic-gate if (umem_alloc_retry(NULL, umflag)) 21877c478bd9Sstevel@tonic-gate goto umem_alloc_retry; 21887c478bd9Sstevel@tonic-gate } 21897c478bd9Sstevel@tonic-gate return (buf); 21907c478bd9Sstevel@tonic-gate } 21917c478bd9Sstevel@tonic-gate 21927c478bd9Sstevel@tonic-gate #pragma weak umem_alloc_align = _umem_alloc_align 21937c478bd9Sstevel@tonic-gate void * 21947c478bd9Sstevel@tonic-gate _umem_alloc_align(size_t size, size_t align, int umflag) 21957c478bd9Sstevel@tonic-gate { 21967c478bd9Sstevel@tonic-gate void *buf; 21977c478bd9Sstevel@tonic-gate 21987c478bd9Sstevel@tonic-gate if (size == 0) 21997c478bd9Sstevel@tonic-gate return (NULL); 22007c478bd9Sstevel@tonic-gate if ((align & (align - 1)) != 0) 22017c478bd9Sstevel@tonic-gate return (NULL); 22027c478bd9Sstevel@tonic-gate if (align < UMEM_ALIGN) 22037c478bd9Sstevel@tonic-gate align = UMEM_ALIGN; 22047c478bd9Sstevel@tonic-gate 22057c478bd9Sstevel@tonic-gate umem_alloc_align_retry: 22067c478bd9Sstevel@tonic-gate if (umem_memalign_arena == NULL) { 22077c478bd9Sstevel@tonic-gate if (umem_init()) 22087c478bd9Sstevel@tonic-gate ASSERT(umem_oversize_arena != NULL); 22097c478bd9Sstevel@tonic-gate else 22107c478bd9Sstevel@tonic-gate return (NULL); 22117c478bd9Sstevel@tonic-gate } 22127c478bd9Sstevel@tonic-gate buf = vmem_xalloc(umem_memalign_arena, size, align, 0, 0, NULL, NULL, 22137c478bd9Sstevel@tonic-gate UMEM_VMFLAGS(umflag)); 22147c478bd9Sstevel@tonic-gate if (buf == NULL) { 22157c478bd9Sstevel@tonic-gate umem_log_event(umem_failure_log, NULL, NULL, (void *)size); 22167c478bd9Sstevel@tonic-gate if (umem_alloc_retry(NULL, umflag)) 22177c478bd9Sstevel@tonic-gate goto umem_alloc_align_retry; 22187c478bd9Sstevel@tonic-gate } 22197c478bd9Sstevel@tonic-gate return (buf); 22207c478bd9Sstevel@tonic-gate } 22217c478bd9Sstevel@tonic-gate 22227c478bd9Sstevel@tonic-gate #pragma weak umem_free = _umem_free 22237c478bd9Sstevel@tonic-gate void 22247c478bd9Sstevel@tonic-gate _umem_free(void *buf, size_t size) 22257c478bd9Sstevel@tonic-gate { 22267c478bd9Sstevel@tonic-gate size_t index = (size - 1) >> UMEM_ALIGN_SHIFT; 22277c478bd9Sstevel@tonic-gate 22287c478bd9Sstevel@tonic-gate if (index < UMEM_MAXBUF >> UMEM_ALIGN_SHIFT) { 22297c478bd9Sstevel@tonic-gate umem_cache_t *cp = umem_alloc_table[index]; 22307c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_BUFTAG) { 22317c478bd9Sstevel@tonic-gate umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); 22327c478bd9Sstevel@tonic-gate uint32_t *ip = (uint32_t *)btp; 22337c478bd9Sstevel@tonic-gate if (ip[1] != UMEM_SIZE_ENCODE(size)) { 22347c478bd9Sstevel@tonic-gate if (*(uint64_t *)buf == UMEM_FREE_PATTERN) { 22357c478bd9Sstevel@tonic-gate umem_error(UMERR_DUPFREE, cp, buf); 22367c478bd9Sstevel@tonic-gate return; 22377c478bd9Sstevel@tonic-gate } 22387c478bd9Sstevel@tonic-gate if (UMEM_SIZE_VALID(ip[1])) { 22397c478bd9Sstevel@tonic-gate ip[0] = UMEM_SIZE_ENCODE(size); 22407c478bd9Sstevel@tonic-gate umem_error(UMERR_BADSIZE, cp, buf); 22417c478bd9Sstevel@tonic-gate } else { 22427c478bd9Sstevel@tonic-gate umem_error(UMERR_REDZONE, cp, buf); 22437c478bd9Sstevel@tonic-gate } 22447c478bd9Sstevel@tonic-gate return; 22457c478bd9Sstevel@tonic-gate } 22467c478bd9Sstevel@tonic-gate if (((uint8_t *)buf)[size] != UMEM_REDZONE_BYTE) { 22477c478bd9Sstevel@tonic-gate umem_error(UMERR_REDZONE, cp, buf); 22487c478bd9Sstevel@tonic-gate return; 22497c478bd9Sstevel@tonic-gate } 22507c478bd9Sstevel@tonic-gate btp->bt_redzone = UMEM_REDZONE_PATTERN; 22517c478bd9Sstevel@tonic-gate } 22527c478bd9Sstevel@tonic-gate _umem_cache_free(cp, buf); 22537c478bd9Sstevel@tonic-gate } else { 22547c478bd9Sstevel@tonic-gate if (buf == NULL && size == 0) 22557c478bd9Sstevel@tonic-gate return; 22567c478bd9Sstevel@tonic-gate vmem_free(umem_oversize_arena, buf, size); 22577c478bd9Sstevel@tonic-gate } 22587c478bd9Sstevel@tonic-gate } 22597c478bd9Sstevel@tonic-gate 22607c478bd9Sstevel@tonic-gate #pragma weak umem_free_align = _umem_free_align 22617c478bd9Sstevel@tonic-gate void 22627c478bd9Sstevel@tonic-gate _umem_free_align(void *buf, size_t size) 22637c478bd9Sstevel@tonic-gate { 22647c478bd9Sstevel@tonic-gate if (buf == NULL && size == 0) 22657c478bd9Sstevel@tonic-gate return; 22667c478bd9Sstevel@tonic-gate vmem_xfree(umem_memalign_arena, buf, size); 22677c478bd9Sstevel@tonic-gate } 22687c478bd9Sstevel@tonic-gate 22697c478bd9Sstevel@tonic-gate static void * 22707c478bd9Sstevel@tonic-gate umem_firewall_va_alloc(vmem_t *vmp, size_t size, int vmflag) 22717c478bd9Sstevel@tonic-gate { 22727c478bd9Sstevel@tonic-gate size_t realsize = size + vmp->vm_quantum; 22737c478bd9Sstevel@tonic-gate 22747c478bd9Sstevel@tonic-gate /* 22757c478bd9Sstevel@tonic-gate * Annoying edge case: if 'size' is just shy of ULONG_MAX, adding 22767c478bd9Sstevel@tonic-gate * vm_quantum will cause integer wraparound. Check for this, and 22777c478bd9Sstevel@tonic-gate * blow off the firewall page in this case. Note that such a 22787c478bd9Sstevel@tonic-gate * giant allocation (the entire address space) can never be 22797c478bd9Sstevel@tonic-gate * satisfied, so it will either fail immediately (VM_NOSLEEP) 22807c478bd9Sstevel@tonic-gate * or sleep forever (VM_SLEEP). Thus, there is no need for a 22817c478bd9Sstevel@tonic-gate * corresponding check in umem_firewall_va_free(). 22827c478bd9Sstevel@tonic-gate */ 22837c478bd9Sstevel@tonic-gate if (realsize < size) 22847c478bd9Sstevel@tonic-gate realsize = size; 22857c478bd9Sstevel@tonic-gate 22867c478bd9Sstevel@tonic-gate return (vmem_alloc(vmp, realsize, vmflag | VM_NEXTFIT)); 22877c478bd9Sstevel@tonic-gate } 22887c478bd9Sstevel@tonic-gate 22897c478bd9Sstevel@tonic-gate static void 22907c478bd9Sstevel@tonic-gate umem_firewall_va_free(vmem_t *vmp, void *addr, size_t size) 22917c478bd9Sstevel@tonic-gate { 22927c478bd9Sstevel@tonic-gate vmem_free(vmp, addr, size + vmp->vm_quantum); 22937c478bd9Sstevel@tonic-gate } 22947c478bd9Sstevel@tonic-gate 22957c478bd9Sstevel@tonic-gate /* 22967c478bd9Sstevel@tonic-gate * Reclaim all unused memory from a cache. 22977c478bd9Sstevel@tonic-gate */ 22987c478bd9Sstevel@tonic-gate static void 22997c478bd9Sstevel@tonic-gate umem_cache_reap(umem_cache_t *cp) 23007c478bd9Sstevel@tonic-gate { 23017c478bd9Sstevel@tonic-gate /* 23027c478bd9Sstevel@tonic-gate * Ask the cache's owner to free some memory if possible. 23037c478bd9Sstevel@tonic-gate * The idea is to handle things like the inode cache, which 23047c478bd9Sstevel@tonic-gate * typically sits on a bunch of memory that it doesn't truly 23057c478bd9Sstevel@tonic-gate * *need*. Reclaim policy is entirely up to the owner; this 23067c478bd9Sstevel@tonic-gate * callback is just an advisory plea for help. 23077c478bd9Sstevel@tonic-gate */ 23087c478bd9Sstevel@tonic-gate if (cp->cache_reclaim != NULL) 23097c478bd9Sstevel@tonic-gate cp->cache_reclaim(cp->cache_private); 23107c478bd9Sstevel@tonic-gate 23117c478bd9Sstevel@tonic-gate umem_depot_ws_reap(cp); 23127c478bd9Sstevel@tonic-gate } 23137c478bd9Sstevel@tonic-gate 23147c478bd9Sstevel@tonic-gate /* 23157c478bd9Sstevel@tonic-gate * Purge all magazines from a cache and set its magazine limit to zero. 23167c478bd9Sstevel@tonic-gate * All calls are serialized by being done by the update thread, except for 23177c478bd9Sstevel@tonic-gate * the final call from umem_cache_destroy(). 23187c478bd9Sstevel@tonic-gate */ 23197c478bd9Sstevel@tonic-gate static void 23207c478bd9Sstevel@tonic-gate umem_cache_magazine_purge(umem_cache_t *cp) 23217c478bd9Sstevel@tonic-gate { 23227c478bd9Sstevel@tonic-gate umem_cpu_cache_t *ccp; 23237c478bd9Sstevel@tonic-gate umem_magazine_t *mp, *pmp; 23247c478bd9Sstevel@tonic-gate int rounds, prounds, cpu_seqid; 23257c478bd9Sstevel@tonic-gate 23267c478bd9Sstevel@tonic-gate ASSERT(cp->cache_next == NULL || IN_UPDATE()); 23277c478bd9Sstevel@tonic-gate 23287c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < umem_max_ncpus; cpu_seqid++) { 23297c478bd9Sstevel@tonic-gate ccp = &cp->cache_cpu[cpu_seqid]; 23307c478bd9Sstevel@tonic-gate 23317c478bd9Sstevel@tonic-gate (void) mutex_lock(&ccp->cc_lock); 23327c478bd9Sstevel@tonic-gate mp = ccp->cc_loaded; 23337c478bd9Sstevel@tonic-gate pmp = ccp->cc_ploaded; 23347c478bd9Sstevel@tonic-gate rounds = ccp->cc_rounds; 23357c478bd9Sstevel@tonic-gate prounds = ccp->cc_prounds; 23367c478bd9Sstevel@tonic-gate ccp->cc_loaded = NULL; 23377c478bd9Sstevel@tonic-gate ccp->cc_ploaded = NULL; 23387c478bd9Sstevel@tonic-gate ccp->cc_rounds = -1; 23397c478bd9Sstevel@tonic-gate ccp->cc_prounds = -1; 23407c478bd9Sstevel@tonic-gate ccp->cc_magsize = 0; 23417c478bd9Sstevel@tonic-gate (void) mutex_unlock(&ccp->cc_lock); 23427c478bd9Sstevel@tonic-gate 23437c478bd9Sstevel@tonic-gate if (mp) 23447c478bd9Sstevel@tonic-gate umem_magazine_destroy(cp, mp, rounds); 23457c478bd9Sstevel@tonic-gate if (pmp) 23467c478bd9Sstevel@tonic-gate umem_magazine_destroy(cp, pmp, prounds); 23477c478bd9Sstevel@tonic-gate } 23487c478bd9Sstevel@tonic-gate 23497c478bd9Sstevel@tonic-gate /* 23507c478bd9Sstevel@tonic-gate * Updating the working set statistics twice in a row has the 23517c478bd9Sstevel@tonic-gate * effect of setting the working set size to zero, so everything 23527c478bd9Sstevel@tonic-gate * is eligible for reaping. 23537c478bd9Sstevel@tonic-gate */ 23547c478bd9Sstevel@tonic-gate umem_depot_ws_update(cp); 23557c478bd9Sstevel@tonic-gate umem_depot_ws_update(cp); 23567c478bd9Sstevel@tonic-gate 23577c478bd9Sstevel@tonic-gate umem_depot_ws_reap(cp); 23587c478bd9Sstevel@tonic-gate } 23597c478bd9Sstevel@tonic-gate 23607c478bd9Sstevel@tonic-gate /* 23617c478bd9Sstevel@tonic-gate * Enable per-cpu magazines on a cache. 23627c478bd9Sstevel@tonic-gate */ 23637c478bd9Sstevel@tonic-gate static void 23647c478bd9Sstevel@tonic-gate umem_cache_magazine_enable(umem_cache_t *cp) 23657c478bd9Sstevel@tonic-gate { 23667c478bd9Sstevel@tonic-gate int cpu_seqid; 23677c478bd9Sstevel@tonic-gate 23687c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_NOMAGAZINE) 23697c478bd9Sstevel@tonic-gate return; 23707c478bd9Sstevel@tonic-gate 23717c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < umem_max_ncpus; cpu_seqid++) { 23727c478bd9Sstevel@tonic-gate umem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 23737c478bd9Sstevel@tonic-gate (void) mutex_lock(&ccp->cc_lock); 23747c478bd9Sstevel@tonic-gate ccp->cc_magsize = cp->cache_magtype->mt_magsize; 23757c478bd9Sstevel@tonic-gate (void) mutex_unlock(&ccp->cc_lock); 23767c478bd9Sstevel@tonic-gate } 23777c478bd9Sstevel@tonic-gate 23787c478bd9Sstevel@tonic-gate } 23797c478bd9Sstevel@tonic-gate 23807c478bd9Sstevel@tonic-gate /* 23817c478bd9Sstevel@tonic-gate * Recompute a cache's magazine size. The trade-off is that larger magazines 23827c478bd9Sstevel@tonic-gate * provide a higher transfer rate with the depot, while smaller magazines 23837c478bd9Sstevel@tonic-gate * reduce memory consumption. Magazine resizing is an expensive operation; 23847c478bd9Sstevel@tonic-gate * it should not be done frequently. 23857c478bd9Sstevel@tonic-gate * 23867c478bd9Sstevel@tonic-gate * Changes to the magazine size are serialized by only having one thread 23877c478bd9Sstevel@tonic-gate * doing updates. (the update thread) 23887c478bd9Sstevel@tonic-gate * 23897c478bd9Sstevel@tonic-gate * Note: at present this only grows the magazine size. It might be useful 23907c478bd9Sstevel@tonic-gate * to allow shrinkage too. 23917c478bd9Sstevel@tonic-gate */ 23927c478bd9Sstevel@tonic-gate static void 23937c478bd9Sstevel@tonic-gate umem_cache_magazine_resize(umem_cache_t *cp) 23947c478bd9Sstevel@tonic-gate { 23957c478bd9Sstevel@tonic-gate umem_magtype_t *mtp = cp->cache_magtype; 23967c478bd9Sstevel@tonic-gate 23977c478bd9Sstevel@tonic-gate ASSERT(IN_UPDATE()); 23987c478bd9Sstevel@tonic-gate 23997c478bd9Sstevel@tonic-gate if (cp->cache_chunksize < mtp->mt_maxbuf) { 24007c478bd9Sstevel@tonic-gate umem_cache_magazine_purge(cp); 24017c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_depot_lock); 24027c478bd9Sstevel@tonic-gate cp->cache_magtype = ++mtp; 24037c478bd9Sstevel@tonic-gate cp->cache_depot_contention_prev = 24047c478bd9Sstevel@tonic-gate cp->cache_depot_contention + INT_MAX; 24057c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_depot_lock); 24067c478bd9Sstevel@tonic-gate umem_cache_magazine_enable(cp); 24077c478bd9Sstevel@tonic-gate } 24087c478bd9Sstevel@tonic-gate } 24097c478bd9Sstevel@tonic-gate 24107c478bd9Sstevel@tonic-gate /* 24117c478bd9Sstevel@tonic-gate * Rescale a cache's hash table, so that the table size is roughly the 24127c478bd9Sstevel@tonic-gate * cache size. We want the average lookup time to be extremely small. 24137c478bd9Sstevel@tonic-gate */ 24147c478bd9Sstevel@tonic-gate static void 24157c478bd9Sstevel@tonic-gate umem_hash_rescale(umem_cache_t *cp) 24167c478bd9Sstevel@tonic-gate { 24177c478bd9Sstevel@tonic-gate umem_bufctl_t **old_table, **new_table, *bcp; 24187c478bd9Sstevel@tonic-gate size_t old_size, new_size, h; 24197c478bd9Sstevel@tonic-gate 24207c478bd9Sstevel@tonic-gate ASSERT(IN_UPDATE()); 24217c478bd9Sstevel@tonic-gate 24227c478bd9Sstevel@tonic-gate new_size = MAX(UMEM_HASH_INITIAL, 24237c478bd9Sstevel@tonic-gate 1 << (highbit(3 * cp->cache_buftotal + 4) - 2)); 24247c478bd9Sstevel@tonic-gate old_size = cp->cache_hash_mask + 1; 24257c478bd9Sstevel@tonic-gate 24267c478bd9Sstevel@tonic-gate if ((old_size >> 1) <= new_size && new_size <= (old_size << 1)) 24277c478bd9Sstevel@tonic-gate return; 24287c478bd9Sstevel@tonic-gate 24297c478bd9Sstevel@tonic-gate new_table = vmem_alloc(umem_hash_arena, new_size * sizeof (void *), 24307c478bd9Sstevel@tonic-gate VM_NOSLEEP); 24317c478bd9Sstevel@tonic-gate if (new_table == NULL) 24327c478bd9Sstevel@tonic-gate return; 24337c478bd9Sstevel@tonic-gate bzero(new_table, new_size * sizeof (void *)); 24347c478bd9Sstevel@tonic-gate 24357c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_lock); 24367c478bd9Sstevel@tonic-gate 24377c478bd9Sstevel@tonic-gate old_size = cp->cache_hash_mask + 1; 24387c478bd9Sstevel@tonic-gate old_table = cp->cache_hash_table; 24397c478bd9Sstevel@tonic-gate 24407c478bd9Sstevel@tonic-gate cp->cache_hash_mask = new_size - 1; 24417c478bd9Sstevel@tonic-gate cp->cache_hash_table = new_table; 24427c478bd9Sstevel@tonic-gate cp->cache_rescale++; 24437c478bd9Sstevel@tonic-gate 24447c478bd9Sstevel@tonic-gate for (h = 0; h < old_size; h++) { 24457c478bd9Sstevel@tonic-gate bcp = old_table[h]; 24467c478bd9Sstevel@tonic-gate while (bcp != NULL) { 24477c478bd9Sstevel@tonic-gate void *addr = bcp->bc_addr; 24487c478bd9Sstevel@tonic-gate umem_bufctl_t *next_bcp = bcp->bc_next; 24497c478bd9Sstevel@tonic-gate umem_bufctl_t **hash_bucket = UMEM_HASH(cp, addr); 24507c478bd9Sstevel@tonic-gate bcp->bc_next = *hash_bucket; 24517c478bd9Sstevel@tonic-gate *hash_bucket = bcp; 24527c478bd9Sstevel@tonic-gate bcp = next_bcp; 24537c478bd9Sstevel@tonic-gate } 24547c478bd9Sstevel@tonic-gate } 24557c478bd9Sstevel@tonic-gate 24567c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 24577c478bd9Sstevel@tonic-gate 24587c478bd9Sstevel@tonic-gate vmem_free(umem_hash_arena, old_table, old_size * sizeof (void *)); 24597c478bd9Sstevel@tonic-gate } 24607c478bd9Sstevel@tonic-gate 24617c478bd9Sstevel@tonic-gate /* 24627c478bd9Sstevel@tonic-gate * Perform periodic maintenance on a cache: hash rescaling, 24637c478bd9Sstevel@tonic-gate * depot working-set update, and magazine resizing. 24647c478bd9Sstevel@tonic-gate */ 24657c478bd9Sstevel@tonic-gate void 24667c478bd9Sstevel@tonic-gate umem_cache_update(umem_cache_t *cp) 24677c478bd9Sstevel@tonic-gate { 24687c478bd9Sstevel@tonic-gate int update_flags = 0; 24697c478bd9Sstevel@tonic-gate 24707c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&umem_cache_lock)); 24717c478bd9Sstevel@tonic-gate 24727c478bd9Sstevel@tonic-gate /* 24737c478bd9Sstevel@tonic-gate * If the cache has become much larger or smaller than its hash table, 24747c478bd9Sstevel@tonic-gate * fire off a request to rescale the hash table. 24757c478bd9Sstevel@tonic-gate */ 24767c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_lock); 24777c478bd9Sstevel@tonic-gate 24787c478bd9Sstevel@tonic-gate if ((cp->cache_flags & UMF_HASH) && 24797c478bd9Sstevel@tonic-gate (cp->cache_buftotal > (cp->cache_hash_mask << 1) || 24807c478bd9Sstevel@tonic-gate (cp->cache_buftotal < (cp->cache_hash_mask >> 1) && 24817c478bd9Sstevel@tonic-gate cp->cache_hash_mask > UMEM_HASH_INITIAL))) 24827c478bd9Sstevel@tonic-gate update_flags |= UMU_HASH_RESCALE; 24837c478bd9Sstevel@tonic-gate 24847c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 24857c478bd9Sstevel@tonic-gate 24867c478bd9Sstevel@tonic-gate /* 24877c478bd9Sstevel@tonic-gate * Update the depot working set statistics. 24887c478bd9Sstevel@tonic-gate */ 24897c478bd9Sstevel@tonic-gate umem_depot_ws_update(cp); 24907c478bd9Sstevel@tonic-gate 24917c478bd9Sstevel@tonic-gate /* 24927c478bd9Sstevel@tonic-gate * If there's a lot of contention in the depot, 24937c478bd9Sstevel@tonic-gate * increase the magazine size. 24947c478bd9Sstevel@tonic-gate */ 24957c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_depot_lock); 24967c478bd9Sstevel@tonic-gate 24977c478bd9Sstevel@tonic-gate if (cp->cache_chunksize < cp->cache_magtype->mt_maxbuf && 24987c478bd9Sstevel@tonic-gate (int)(cp->cache_depot_contention - 24997c478bd9Sstevel@tonic-gate cp->cache_depot_contention_prev) > umem_depot_contention) 25007c478bd9Sstevel@tonic-gate update_flags |= UMU_MAGAZINE_RESIZE; 25017c478bd9Sstevel@tonic-gate 25027c478bd9Sstevel@tonic-gate cp->cache_depot_contention_prev = cp->cache_depot_contention; 25037c478bd9Sstevel@tonic-gate 25047c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_depot_lock); 25057c478bd9Sstevel@tonic-gate 25067c478bd9Sstevel@tonic-gate if (update_flags) 25077c478bd9Sstevel@tonic-gate umem_add_update(cp, update_flags); 25087c478bd9Sstevel@tonic-gate } 25097c478bd9Sstevel@tonic-gate 25107c478bd9Sstevel@tonic-gate /* 25117c478bd9Sstevel@tonic-gate * Runs all pending updates. 25127c478bd9Sstevel@tonic-gate * 25137c478bd9Sstevel@tonic-gate * The update lock must be held on entrance, and will be held on exit. 25147c478bd9Sstevel@tonic-gate */ 25157c478bd9Sstevel@tonic-gate void 25167c478bd9Sstevel@tonic-gate umem_process_updates(void) 25177c478bd9Sstevel@tonic-gate { 25187c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&umem_update_lock)); 25197c478bd9Sstevel@tonic-gate 25207c478bd9Sstevel@tonic-gate while (umem_null_cache.cache_unext != &umem_null_cache) { 25217c478bd9Sstevel@tonic-gate int notify = 0; 25227c478bd9Sstevel@tonic-gate umem_cache_t *cp = umem_null_cache.cache_unext; 25237c478bd9Sstevel@tonic-gate 25247c478bd9Sstevel@tonic-gate cp->cache_uprev->cache_unext = cp->cache_unext; 25257c478bd9Sstevel@tonic-gate cp->cache_unext->cache_uprev = cp->cache_uprev; 25267c478bd9Sstevel@tonic-gate cp->cache_uprev = cp->cache_unext = NULL; 25277c478bd9Sstevel@tonic-gate 25287c478bd9Sstevel@tonic-gate ASSERT(!(cp->cache_uflags & UMU_ACTIVE)); 25297c478bd9Sstevel@tonic-gate 25307c478bd9Sstevel@tonic-gate while (cp->cache_uflags) { 25317c478bd9Sstevel@tonic-gate int uflags = (cp->cache_uflags |= UMU_ACTIVE); 25327c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_update_lock); 25337c478bd9Sstevel@tonic-gate 25347c478bd9Sstevel@tonic-gate /* 25357c478bd9Sstevel@tonic-gate * The order here is important. Each step can speed up 25367c478bd9Sstevel@tonic-gate * later steps. 25377c478bd9Sstevel@tonic-gate */ 25387c478bd9Sstevel@tonic-gate 25397c478bd9Sstevel@tonic-gate if (uflags & UMU_HASH_RESCALE) 25407c478bd9Sstevel@tonic-gate umem_hash_rescale(cp); 25417c478bd9Sstevel@tonic-gate 25427c478bd9Sstevel@tonic-gate if (uflags & UMU_MAGAZINE_RESIZE) 25437c478bd9Sstevel@tonic-gate umem_cache_magazine_resize(cp); 25447c478bd9Sstevel@tonic-gate 25457c478bd9Sstevel@tonic-gate if (uflags & UMU_REAP) 25467c478bd9Sstevel@tonic-gate umem_cache_reap(cp); 25477c478bd9Sstevel@tonic-gate 25487c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_update_lock); 25497c478bd9Sstevel@tonic-gate 25507c478bd9Sstevel@tonic-gate /* 25517c478bd9Sstevel@tonic-gate * check if anyone has requested notification 25527c478bd9Sstevel@tonic-gate */ 25537c478bd9Sstevel@tonic-gate if (cp->cache_uflags & UMU_NOTIFY) { 25547c478bd9Sstevel@tonic-gate uflags |= UMU_NOTIFY; 25557c478bd9Sstevel@tonic-gate notify = 1; 25567c478bd9Sstevel@tonic-gate } 25577c478bd9Sstevel@tonic-gate cp->cache_uflags &= ~uflags; 25587c478bd9Sstevel@tonic-gate } 25597c478bd9Sstevel@tonic-gate if (notify) 25607c478bd9Sstevel@tonic-gate (void) cond_broadcast(&umem_update_cv); 25617c478bd9Sstevel@tonic-gate } 25627c478bd9Sstevel@tonic-gate } 25637c478bd9Sstevel@tonic-gate 25647c478bd9Sstevel@tonic-gate #ifndef UMEM_STANDALONE 25657c478bd9Sstevel@tonic-gate static void 25667c478bd9Sstevel@tonic-gate umem_st_update(void) 25677c478bd9Sstevel@tonic-gate { 25687c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&umem_update_lock)); 25697c478bd9Sstevel@tonic-gate ASSERT(umem_update_thr == 0 && umem_st_update_thr == 0); 25707c478bd9Sstevel@tonic-gate 25717c478bd9Sstevel@tonic-gate umem_st_update_thr = thr_self(); 25727c478bd9Sstevel@tonic-gate 25737c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_update_lock); 25747c478bd9Sstevel@tonic-gate 25757c478bd9Sstevel@tonic-gate vmem_update(NULL); 25767c478bd9Sstevel@tonic-gate umem_cache_applyall(umem_cache_update); 25777c478bd9Sstevel@tonic-gate 25787c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_update_lock); 25797c478bd9Sstevel@tonic-gate 25807c478bd9Sstevel@tonic-gate umem_process_updates(); /* does all of the requested work */ 25817c478bd9Sstevel@tonic-gate 25827c478bd9Sstevel@tonic-gate umem_reap_next = gethrtime() + 25837c478bd9Sstevel@tonic-gate (hrtime_t)umem_reap_interval * NANOSEC; 25847c478bd9Sstevel@tonic-gate 25857c478bd9Sstevel@tonic-gate umem_reaping = UMEM_REAP_DONE; 25867c478bd9Sstevel@tonic-gate 25877c478bd9Sstevel@tonic-gate umem_st_update_thr = 0; 25887c478bd9Sstevel@tonic-gate } 25897c478bd9Sstevel@tonic-gate #endif 25907c478bd9Sstevel@tonic-gate 25917c478bd9Sstevel@tonic-gate /* 25927c478bd9Sstevel@tonic-gate * Reclaim all unused memory from all caches. Called from vmem when memory 25937c478bd9Sstevel@tonic-gate * gets tight. Must be called with no locks held. 25947c478bd9Sstevel@tonic-gate * 25957c478bd9Sstevel@tonic-gate * This just requests a reap on all caches, and notifies the update thread. 25967c478bd9Sstevel@tonic-gate */ 25977c478bd9Sstevel@tonic-gate void 25987c478bd9Sstevel@tonic-gate umem_reap(void) 25997c478bd9Sstevel@tonic-gate { 26007c478bd9Sstevel@tonic-gate #ifndef UMEM_STANDALONE 26017c478bd9Sstevel@tonic-gate extern int __nthreads(void); 26027c478bd9Sstevel@tonic-gate #endif 26037c478bd9Sstevel@tonic-gate 26047c478bd9Sstevel@tonic-gate if (umem_ready != UMEM_READY || umem_reaping != UMEM_REAP_DONE || 26057c478bd9Sstevel@tonic-gate gethrtime() < umem_reap_next) 26067c478bd9Sstevel@tonic-gate return; 26077c478bd9Sstevel@tonic-gate 26087c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_update_lock); 26097c478bd9Sstevel@tonic-gate 26107c478bd9Sstevel@tonic-gate if (umem_reaping != UMEM_REAP_DONE || gethrtime() < umem_reap_next) { 26117c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_update_lock); 26127c478bd9Sstevel@tonic-gate return; 26137c478bd9Sstevel@tonic-gate } 26147c478bd9Sstevel@tonic-gate umem_reaping = UMEM_REAP_ADDING; /* lock out other reaps */ 26157c478bd9Sstevel@tonic-gate 26167c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_update_lock); 26177c478bd9Sstevel@tonic-gate 26187c478bd9Sstevel@tonic-gate umem_updateall(UMU_REAP); 26197c478bd9Sstevel@tonic-gate 26207c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_update_lock); 26217c478bd9Sstevel@tonic-gate 26227c478bd9Sstevel@tonic-gate umem_reaping = UMEM_REAP_ACTIVE; 26237c478bd9Sstevel@tonic-gate 26247c478bd9Sstevel@tonic-gate /* Standalone is single-threaded */ 26257c478bd9Sstevel@tonic-gate #ifndef UMEM_STANDALONE 26267c478bd9Sstevel@tonic-gate if (umem_update_thr == 0) { 26277c478bd9Sstevel@tonic-gate /* 26287c478bd9Sstevel@tonic-gate * The update thread does not exist. If the process is 26297c478bd9Sstevel@tonic-gate * multi-threaded, create it. If not, or the creation fails, 26307c478bd9Sstevel@tonic-gate * do the update processing inline. 26317c478bd9Sstevel@tonic-gate */ 26327c478bd9Sstevel@tonic-gate ASSERT(umem_st_update_thr == 0); 26337c478bd9Sstevel@tonic-gate 26347c478bd9Sstevel@tonic-gate if (__nthreads() <= 1 || umem_create_update_thread() == 0) 26357c478bd9Sstevel@tonic-gate umem_st_update(); 26367c478bd9Sstevel@tonic-gate } 26377c478bd9Sstevel@tonic-gate 26387c478bd9Sstevel@tonic-gate (void) cond_broadcast(&umem_update_cv); /* wake up the update thread */ 26397c478bd9Sstevel@tonic-gate #endif 26407c478bd9Sstevel@tonic-gate 26417c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_update_lock); 26427c478bd9Sstevel@tonic-gate } 26437c478bd9Sstevel@tonic-gate 26447c478bd9Sstevel@tonic-gate umem_cache_t * 26457c478bd9Sstevel@tonic-gate umem_cache_create( 26467c478bd9Sstevel@tonic-gate char *name, /* descriptive name for this cache */ 26477c478bd9Sstevel@tonic-gate size_t bufsize, /* size of the objects it manages */ 26487c478bd9Sstevel@tonic-gate size_t align, /* required object alignment */ 26497c478bd9Sstevel@tonic-gate umem_constructor_t *constructor, /* object constructor */ 26507c478bd9Sstevel@tonic-gate umem_destructor_t *destructor, /* object destructor */ 26517c478bd9Sstevel@tonic-gate umem_reclaim_t *reclaim, /* memory reclaim callback */ 26527c478bd9Sstevel@tonic-gate void *private, /* pass-thru arg for constr/destr/reclaim */ 26537c478bd9Sstevel@tonic-gate vmem_t *vmp, /* vmem source for slab allocation */ 26547c478bd9Sstevel@tonic-gate int cflags) /* cache creation flags */ 26557c478bd9Sstevel@tonic-gate { 26567c478bd9Sstevel@tonic-gate int cpu_seqid; 26577c478bd9Sstevel@tonic-gate size_t chunksize; 26587c478bd9Sstevel@tonic-gate umem_cache_t *cp, *cnext, *cprev; 26597c478bd9Sstevel@tonic-gate umem_magtype_t *mtp; 26607c478bd9Sstevel@tonic-gate size_t csize; 26617c478bd9Sstevel@tonic-gate size_t phase; 26627c478bd9Sstevel@tonic-gate 26637c478bd9Sstevel@tonic-gate /* 26647c478bd9Sstevel@tonic-gate * The init thread is allowed to create internal and quantum caches. 26657c478bd9Sstevel@tonic-gate * 26667c478bd9Sstevel@tonic-gate * Other threads must wait until until initialization is complete. 26677c478bd9Sstevel@tonic-gate */ 26687c478bd9Sstevel@tonic-gate if (umem_init_thr == thr_self()) 26697c478bd9Sstevel@tonic-gate ASSERT((cflags & (UMC_INTERNAL | UMC_QCACHE)) != 0); 26707c478bd9Sstevel@tonic-gate else { 26717c478bd9Sstevel@tonic-gate ASSERT(!(cflags & UMC_INTERNAL)); 26727c478bd9Sstevel@tonic-gate if (umem_ready != UMEM_READY && umem_init() == 0) { 26737c478bd9Sstevel@tonic-gate errno = EAGAIN; 26747c478bd9Sstevel@tonic-gate return (NULL); 26757c478bd9Sstevel@tonic-gate } 26767c478bd9Sstevel@tonic-gate } 26777c478bd9Sstevel@tonic-gate 26787c478bd9Sstevel@tonic-gate csize = UMEM_CACHE_SIZE(umem_max_ncpus); 26797c478bd9Sstevel@tonic-gate phase = P2NPHASE(csize, UMEM_CPU_CACHE_SIZE); 26807c478bd9Sstevel@tonic-gate 26817c478bd9Sstevel@tonic-gate if (vmp == NULL) 26827c478bd9Sstevel@tonic-gate vmp = umem_default_arena; 26837c478bd9Sstevel@tonic-gate 26847c478bd9Sstevel@tonic-gate ASSERT(P2PHASE(phase, UMEM_ALIGN) == 0); 26857c478bd9Sstevel@tonic-gate 26867c478bd9Sstevel@tonic-gate /* 26877c478bd9Sstevel@tonic-gate * Check that the arguments are reasonable 26887c478bd9Sstevel@tonic-gate */ 26897c478bd9Sstevel@tonic-gate if ((align & (align - 1)) != 0 || align > vmp->vm_quantum || 26907c478bd9Sstevel@tonic-gate ((cflags & UMC_NOHASH) && (cflags & UMC_NOTOUCH)) || 26917c478bd9Sstevel@tonic-gate name == NULL || bufsize == 0) { 26927c478bd9Sstevel@tonic-gate errno = EINVAL; 26937c478bd9Sstevel@tonic-gate return (NULL); 26947c478bd9Sstevel@tonic-gate } 26957c478bd9Sstevel@tonic-gate 26967c478bd9Sstevel@tonic-gate /* 26977c478bd9Sstevel@tonic-gate * If align == 0, we set it to the minimum required alignment. 26987c478bd9Sstevel@tonic-gate * 26997c478bd9Sstevel@tonic-gate * If align < UMEM_ALIGN, we round it up to UMEM_ALIGN, unless 27007c478bd9Sstevel@tonic-gate * UMC_NOTOUCH was passed. 27017c478bd9Sstevel@tonic-gate */ 27027c478bd9Sstevel@tonic-gate if (align == 0) { 27037c478bd9Sstevel@tonic-gate if (P2ROUNDUP(bufsize, UMEM_ALIGN) >= UMEM_SECOND_ALIGN) 27047c478bd9Sstevel@tonic-gate align = UMEM_SECOND_ALIGN; 27057c478bd9Sstevel@tonic-gate else 27067c478bd9Sstevel@tonic-gate align = UMEM_ALIGN; 27077c478bd9Sstevel@tonic-gate } else if (align < UMEM_ALIGN && (cflags & UMC_NOTOUCH) == 0) 27087c478bd9Sstevel@tonic-gate align = UMEM_ALIGN; 27097c478bd9Sstevel@tonic-gate 27107c478bd9Sstevel@tonic-gate 27117c478bd9Sstevel@tonic-gate /* 27127c478bd9Sstevel@tonic-gate * Get a umem_cache structure. We arrange that cp->cache_cpu[] 27137c478bd9Sstevel@tonic-gate * is aligned on a UMEM_CPU_CACHE_SIZE boundary to prevent 27147c478bd9Sstevel@tonic-gate * false sharing of per-CPU data. 27157c478bd9Sstevel@tonic-gate */ 27167c478bd9Sstevel@tonic-gate cp = vmem_xalloc(umem_cache_arena, csize, UMEM_CPU_CACHE_SIZE, phase, 27177c478bd9Sstevel@tonic-gate 0, NULL, NULL, VM_NOSLEEP); 27187c478bd9Sstevel@tonic-gate 27197c478bd9Sstevel@tonic-gate if (cp == NULL) { 27207c478bd9Sstevel@tonic-gate errno = EAGAIN; 27217c478bd9Sstevel@tonic-gate return (NULL); 27227c478bd9Sstevel@tonic-gate } 27237c478bd9Sstevel@tonic-gate 27247c478bd9Sstevel@tonic-gate bzero(cp, csize); 27257c478bd9Sstevel@tonic-gate 27267c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_flags_lock); 27277c478bd9Sstevel@tonic-gate if (umem_flags & UMF_RANDOMIZE) 27287c478bd9Sstevel@tonic-gate umem_flags = (((umem_flags | ~UMF_RANDOM) + 1) & UMF_RANDOM) | 27297c478bd9Sstevel@tonic-gate UMF_RANDOMIZE; 27307c478bd9Sstevel@tonic-gate cp->cache_flags = umem_flags | (cflags & UMF_DEBUG); 27317c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_flags_lock); 27327c478bd9Sstevel@tonic-gate 27337c478bd9Sstevel@tonic-gate /* 27347c478bd9Sstevel@tonic-gate * Make sure all the various flags are reasonable. 27357c478bd9Sstevel@tonic-gate */ 27367c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_LITE) { 27377c478bd9Sstevel@tonic-gate if (bufsize >= umem_lite_minsize && 27387c478bd9Sstevel@tonic-gate align <= umem_lite_maxalign && 27397c478bd9Sstevel@tonic-gate P2PHASE(bufsize, umem_lite_maxalign) != 0) { 27407c478bd9Sstevel@tonic-gate cp->cache_flags |= UMF_BUFTAG; 27417c478bd9Sstevel@tonic-gate cp->cache_flags &= ~(UMF_AUDIT | UMF_FIREWALL); 27427c478bd9Sstevel@tonic-gate } else { 27437c478bd9Sstevel@tonic-gate cp->cache_flags &= ~UMF_DEBUG; 27447c478bd9Sstevel@tonic-gate } 27457c478bd9Sstevel@tonic-gate } 27467c478bd9Sstevel@tonic-gate 27477c478bd9Sstevel@tonic-gate if ((cflags & UMC_QCACHE) && (cp->cache_flags & UMF_AUDIT)) 27487c478bd9Sstevel@tonic-gate cp->cache_flags |= UMF_NOMAGAZINE; 27497c478bd9Sstevel@tonic-gate 27507c478bd9Sstevel@tonic-gate if (cflags & UMC_NODEBUG) 27517c478bd9Sstevel@tonic-gate cp->cache_flags &= ~UMF_DEBUG; 27527c478bd9Sstevel@tonic-gate 27537c478bd9Sstevel@tonic-gate if (cflags & UMC_NOTOUCH) 27547c478bd9Sstevel@tonic-gate cp->cache_flags &= ~UMF_TOUCH; 27557c478bd9Sstevel@tonic-gate 27567c478bd9Sstevel@tonic-gate if (cflags & UMC_NOHASH) 27577c478bd9Sstevel@tonic-gate cp->cache_flags &= ~(UMF_AUDIT | UMF_FIREWALL); 27587c478bd9Sstevel@tonic-gate 27597c478bd9Sstevel@tonic-gate if (cflags & UMC_NOMAGAZINE) 27607c478bd9Sstevel@tonic-gate cp->cache_flags |= UMF_NOMAGAZINE; 27617c478bd9Sstevel@tonic-gate 27627c478bd9Sstevel@tonic-gate if ((cp->cache_flags & UMF_AUDIT) && !(cflags & UMC_NOTOUCH)) 27637c478bd9Sstevel@tonic-gate cp->cache_flags |= UMF_REDZONE; 27647c478bd9Sstevel@tonic-gate 27657c478bd9Sstevel@tonic-gate if ((cp->cache_flags & UMF_BUFTAG) && bufsize >= umem_minfirewall && 27667c478bd9Sstevel@tonic-gate !(cp->cache_flags & UMF_LITE) && !(cflags & UMC_NOHASH)) 27677c478bd9Sstevel@tonic-gate cp->cache_flags |= UMF_FIREWALL; 27687c478bd9Sstevel@tonic-gate 27697c478bd9Sstevel@tonic-gate if (vmp != umem_default_arena || umem_firewall_arena == NULL) 27707c478bd9Sstevel@tonic-gate cp->cache_flags &= ~UMF_FIREWALL; 27717c478bd9Sstevel@tonic-gate 27727c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_FIREWALL) { 27737c478bd9Sstevel@tonic-gate cp->cache_flags &= ~UMF_BUFTAG; 27747c478bd9Sstevel@tonic-gate cp->cache_flags |= UMF_NOMAGAZINE; 27757c478bd9Sstevel@tonic-gate ASSERT(vmp == umem_default_arena); 27767c478bd9Sstevel@tonic-gate vmp = umem_firewall_arena; 27777c478bd9Sstevel@tonic-gate } 27787c478bd9Sstevel@tonic-gate 27797c478bd9Sstevel@tonic-gate /* 27807c478bd9Sstevel@tonic-gate * Set cache properties. 27817c478bd9Sstevel@tonic-gate */ 27827c478bd9Sstevel@tonic-gate (void) strncpy(cp->cache_name, name, sizeof (cp->cache_name) - 1); 27837c478bd9Sstevel@tonic-gate cp->cache_bufsize = bufsize; 27847c478bd9Sstevel@tonic-gate cp->cache_align = align; 27857c478bd9Sstevel@tonic-gate cp->cache_constructor = constructor; 27867c478bd9Sstevel@tonic-gate cp->cache_destructor = destructor; 27877c478bd9Sstevel@tonic-gate cp->cache_reclaim = reclaim; 27887c478bd9Sstevel@tonic-gate cp->cache_private = private; 27897c478bd9Sstevel@tonic-gate cp->cache_arena = vmp; 27907c478bd9Sstevel@tonic-gate cp->cache_cflags = cflags; 27917c478bd9Sstevel@tonic-gate cp->cache_cpu_mask = umem_cpu_mask; 27927c478bd9Sstevel@tonic-gate 27937c478bd9Sstevel@tonic-gate /* 27947c478bd9Sstevel@tonic-gate * Determine the chunk size. 27957c478bd9Sstevel@tonic-gate */ 27967c478bd9Sstevel@tonic-gate chunksize = bufsize; 27977c478bd9Sstevel@tonic-gate 27987c478bd9Sstevel@tonic-gate if (align >= UMEM_ALIGN) { 27997c478bd9Sstevel@tonic-gate chunksize = P2ROUNDUP(chunksize, UMEM_ALIGN); 28007c478bd9Sstevel@tonic-gate cp->cache_bufctl = chunksize - UMEM_ALIGN; 28017c478bd9Sstevel@tonic-gate } 28027c478bd9Sstevel@tonic-gate 28037c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_BUFTAG) { 28047c478bd9Sstevel@tonic-gate cp->cache_bufctl = chunksize; 28057c478bd9Sstevel@tonic-gate cp->cache_buftag = chunksize; 28067c478bd9Sstevel@tonic-gate chunksize += sizeof (umem_buftag_t); 28077c478bd9Sstevel@tonic-gate } 28087c478bd9Sstevel@tonic-gate 28097c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_DEADBEEF) { 28107c478bd9Sstevel@tonic-gate cp->cache_verify = MIN(cp->cache_buftag, umem_maxverify); 28117c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_LITE) 28127c478bd9Sstevel@tonic-gate cp->cache_verify = MIN(cp->cache_verify, UMEM_ALIGN); 28137c478bd9Sstevel@tonic-gate } 28147c478bd9Sstevel@tonic-gate 28157c478bd9Sstevel@tonic-gate cp->cache_contents = MIN(cp->cache_bufctl, umem_content_maxsave); 28167c478bd9Sstevel@tonic-gate 28177c478bd9Sstevel@tonic-gate cp->cache_chunksize = chunksize = P2ROUNDUP(chunksize, align); 28187c478bd9Sstevel@tonic-gate 28197c478bd9Sstevel@tonic-gate if (chunksize < bufsize) { 28207c478bd9Sstevel@tonic-gate errno = ENOMEM; 28217c478bd9Sstevel@tonic-gate goto fail; 28227c478bd9Sstevel@tonic-gate } 28237c478bd9Sstevel@tonic-gate 28247c478bd9Sstevel@tonic-gate /* 28257c478bd9Sstevel@tonic-gate * Now that we know the chunk size, determine the optimal slab size. 28267c478bd9Sstevel@tonic-gate */ 28277c478bd9Sstevel@tonic-gate if (vmp == umem_firewall_arena) { 28287c478bd9Sstevel@tonic-gate cp->cache_slabsize = P2ROUNDUP(chunksize, vmp->vm_quantum); 28297c478bd9Sstevel@tonic-gate cp->cache_mincolor = cp->cache_slabsize - chunksize; 28307c478bd9Sstevel@tonic-gate cp->cache_maxcolor = cp->cache_mincolor; 28317c478bd9Sstevel@tonic-gate cp->cache_flags |= UMF_HASH; 28327c478bd9Sstevel@tonic-gate ASSERT(!(cp->cache_flags & UMF_BUFTAG)); 28337c478bd9Sstevel@tonic-gate } else if ((cflags & UMC_NOHASH) || (!(cflags & UMC_NOTOUCH) && 28347c478bd9Sstevel@tonic-gate !(cp->cache_flags & UMF_AUDIT) && 28357c478bd9Sstevel@tonic-gate chunksize < vmp->vm_quantum / UMEM_VOID_FRACTION)) { 28367c478bd9Sstevel@tonic-gate cp->cache_slabsize = vmp->vm_quantum; 28377c478bd9Sstevel@tonic-gate cp->cache_mincolor = 0; 28387c478bd9Sstevel@tonic-gate cp->cache_maxcolor = 28397c478bd9Sstevel@tonic-gate (cp->cache_slabsize - sizeof (umem_slab_t)) % chunksize; 28407c478bd9Sstevel@tonic-gate 28417c478bd9Sstevel@tonic-gate if (chunksize + sizeof (umem_slab_t) > cp->cache_slabsize) { 28427c478bd9Sstevel@tonic-gate errno = EINVAL; 28437c478bd9Sstevel@tonic-gate goto fail; 28447c478bd9Sstevel@tonic-gate } 28457c478bd9Sstevel@tonic-gate ASSERT(!(cp->cache_flags & UMF_AUDIT)); 28467c478bd9Sstevel@tonic-gate } else { 2847b1e2e3fbSRobert Mustacchi size_t chunks, waste, slabsize; 28487c478bd9Sstevel@tonic-gate size_t minwaste = LONG_MAX; 2849b1e2e3fbSRobert Mustacchi size_t bestfit = SIZE_MAX; 28507c478bd9Sstevel@tonic-gate 28517c478bd9Sstevel@tonic-gate for (chunks = 1; chunks <= UMEM_VOID_FRACTION; chunks++) { 28527c478bd9Sstevel@tonic-gate slabsize = P2ROUNDUP(chunksize * chunks, 28537c478bd9Sstevel@tonic-gate vmp->vm_quantum); 28547c478bd9Sstevel@tonic-gate /* 28557c478bd9Sstevel@tonic-gate * check for overflow 28567c478bd9Sstevel@tonic-gate */ 28577c478bd9Sstevel@tonic-gate if ((slabsize / chunks) < chunksize) { 28587c478bd9Sstevel@tonic-gate errno = ENOMEM; 28597c478bd9Sstevel@tonic-gate goto fail; 28607c478bd9Sstevel@tonic-gate } 28617c478bd9Sstevel@tonic-gate chunks = slabsize / chunksize; 28627c478bd9Sstevel@tonic-gate waste = (slabsize % chunksize) / chunks; 28637c478bd9Sstevel@tonic-gate if (waste < minwaste) { 28647c478bd9Sstevel@tonic-gate minwaste = waste; 28657c478bd9Sstevel@tonic-gate bestfit = slabsize; 28667c478bd9Sstevel@tonic-gate } 28677c478bd9Sstevel@tonic-gate } 28687c478bd9Sstevel@tonic-gate if (cflags & UMC_QCACHE) 28697c478bd9Sstevel@tonic-gate bestfit = MAX(1 << highbit(3 * vmp->vm_qcache_max), 64); 2870b1e2e3fbSRobert Mustacchi if (bestfit == SIZE_MAX) { 2871b1e2e3fbSRobert Mustacchi errno = ENOMEM; 2872b1e2e3fbSRobert Mustacchi goto fail; 2873b1e2e3fbSRobert Mustacchi } 28747c478bd9Sstevel@tonic-gate cp->cache_slabsize = bestfit; 28757c478bd9Sstevel@tonic-gate cp->cache_mincolor = 0; 28767c478bd9Sstevel@tonic-gate cp->cache_maxcolor = bestfit % chunksize; 28777c478bd9Sstevel@tonic-gate cp->cache_flags |= UMF_HASH; 28787c478bd9Sstevel@tonic-gate } 28797c478bd9Sstevel@tonic-gate 28807c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_HASH) { 28817c478bd9Sstevel@tonic-gate ASSERT(!(cflags & UMC_NOHASH)); 28827c478bd9Sstevel@tonic-gate cp->cache_bufctl_cache = (cp->cache_flags & UMF_AUDIT) ? 28837c478bd9Sstevel@tonic-gate umem_bufctl_audit_cache : umem_bufctl_cache; 28847c478bd9Sstevel@tonic-gate } 28857c478bd9Sstevel@tonic-gate 28867c478bd9Sstevel@tonic-gate if (cp->cache_maxcolor >= vmp->vm_quantum) 28877c478bd9Sstevel@tonic-gate cp->cache_maxcolor = vmp->vm_quantum - 1; 28887c478bd9Sstevel@tonic-gate 28897c478bd9Sstevel@tonic-gate cp->cache_color = cp->cache_mincolor; 28907c478bd9Sstevel@tonic-gate 28917c478bd9Sstevel@tonic-gate /* 28927c478bd9Sstevel@tonic-gate * Initialize the rest of the slab layer. 28937c478bd9Sstevel@tonic-gate */ 28947c478bd9Sstevel@tonic-gate (void) mutex_init(&cp->cache_lock, USYNC_THREAD, NULL); 28957c478bd9Sstevel@tonic-gate 28967c478bd9Sstevel@tonic-gate cp->cache_freelist = &cp->cache_nullslab; 28977c478bd9Sstevel@tonic-gate cp->cache_nullslab.slab_cache = cp; 28987c478bd9Sstevel@tonic-gate cp->cache_nullslab.slab_refcnt = -1; 28997c478bd9Sstevel@tonic-gate cp->cache_nullslab.slab_next = &cp->cache_nullslab; 29007c478bd9Sstevel@tonic-gate cp->cache_nullslab.slab_prev = &cp->cache_nullslab; 29017c478bd9Sstevel@tonic-gate 29027c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_HASH) { 29037c478bd9Sstevel@tonic-gate cp->cache_hash_table = vmem_alloc(umem_hash_arena, 29047c478bd9Sstevel@tonic-gate UMEM_HASH_INITIAL * sizeof (void *), VM_NOSLEEP); 29057c478bd9Sstevel@tonic-gate if (cp->cache_hash_table == NULL) { 29067c478bd9Sstevel@tonic-gate errno = EAGAIN; 29077c478bd9Sstevel@tonic-gate goto fail_lock; 29087c478bd9Sstevel@tonic-gate } 29097c478bd9Sstevel@tonic-gate bzero(cp->cache_hash_table, 29107c478bd9Sstevel@tonic-gate UMEM_HASH_INITIAL * sizeof (void *)); 29117c478bd9Sstevel@tonic-gate cp->cache_hash_mask = UMEM_HASH_INITIAL - 1; 29127c478bd9Sstevel@tonic-gate cp->cache_hash_shift = highbit((ulong_t)chunksize) - 1; 29137c478bd9Sstevel@tonic-gate } 29147c478bd9Sstevel@tonic-gate 29157c478bd9Sstevel@tonic-gate /* 29167c478bd9Sstevel@tonic-gate * Initialize the depot. 29177c478bd9Sstevel@tonic-gate */ 29187c478bd9Sstevel@tonic-gate (void) mutex_init(&cp->cache_depot_lock, USYNC_THREAD, NULL); 29197c478bd9Sstevel@tonic-gate 29207c478bd9Sstevel@tonic-gate for (mtp = umem_magtype; chunksize <= mtp->mt_minbuf; mtp++) 29217c478bd9Sstevel@tonic-gate continue; 29227c478bd9Sstevel@tonic-gate 29237c478bd9Sstevel@tonic-gate cp->cache_magtype = mtp; 29247c478bd9Sstevel@tonic-gate 29257c478bd9Sstevel@tonic-gate /* 29267c478bd9Sstevel@tonic-gate * Initialize the CPU layer. 29277c478bd9Sstevel@tonic-gate */ 29287c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < umem_max_ncpus; cpu_seqid++) { 29297c478bd9Sstevel@tonic-gate umem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 29307c478bd9Sstevel@tonic-gate (void) mutex_init(&ccp->cc_lock, USYNC_THREAD, NULL); 29317c478bd9Sstevel@tonic-gate ccp->cc_flags = cp->cache_flags; 29327c478bd9Sstevel@tonic-gate ccp->cc_rounds = -1; 29337c478bd9Sstevel@tonic-gate ccp->cc_prounds = -1; 29347c478bd9Sstevel@tonic-gate } 29357c478bd9Sstevel@tonic-gate 29367c478bd9Sstevel@tonic-gate /* 29377c478bd9Sstevel@tonic-gate * Add the cache to the global list. This makes it visible 29387c478bd9Sstevel@tonic-gate * to umem_update(), so the cache must be ready for business. 29397c478bd9Sstevel@tonic-gate */ 29407c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_cache_lock); 29417c478bd9Sstevel@tonic-gate cp->cache_next = cnext = &umem_null_cache; 29427c478bd9Sstevel@tonic-gate cp->cache_prev = cprev = umem_null_cache.cache_prev; 29437c478bd9Sstevel@tonic-gate cnext->cache_prev = cp; 29447c478bd9Sstevel@tonic-gate cprev->cache_next = cp; 29457c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_cache_lock); 29467c478bd9Sstevel@tonic-gate 29477c478bd9Sstevel@tonic-gate if (umem_ready == UMEM_READY) 29487c478bd9Sstevel@tonic-gate umem_cache_magazine_enable(cp); 29497c478bd9Sstevel@tonic-gate 29507c478bd9Sstevel@tonic-gate return (cp); 29517c478bd9Sstevel@tonic-gate 29527c478bd9Sstevel@tonic-gate fail_lock: 29537c478bd9Sstevel@tonic-gate (void) mutex_destroy(&cp->cache_lock); 29547c478bd9Sstevel@tonic-gate fail: 29557c478bd9Sstevel@tonic-gate vmem_xfree(umem_cache_arena, cp, csize); 29567c478bd9Sstevel@tonic-gate return (NULL); 29577c478bd9Sstevel@tonic-gate } 29587c478bd9Sstevel@tonic-gate 29597c478bd9Sstevel@tonic-gate void 29607c478bd9Sstevel@tonic-gate umem_cache_destroy(umem_cache_t *cp) 29617c478bd9Sstevel@tonic-gate { 29627c478bd9Sstevel@tonic-gate int cpu_seqid; 29637c478bd9Sstevel@tonic-gate 29647c478bd9Sstevel@tonic-gate /* 29657c478bd9Sstevel@tonic-gate * Remove the cache from the global cache list so that no new updates 29667c478bd9Sstevel@tonic-gate * will be scheduled on its behalf, wait for any pending tasks to 29677c478bd9Sstevel@tonic-gate * complete, purge the cache, and then destroy it. 29687c478bd9Sstevel@tonic-gate */ 29697c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_cache_lock); 29707c478bd9Sstevel@tonic-gate cp->cache_prev->cache_next = cp->cache_next; 29717c478bd9Sstevel@tonic-gate cp->cache_next->cache_prev = cp->cache_prev; 29727c478bd9Sstevel@tonic-gate cp->cache_prev = cp->cache_next = NULL; 29737c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_cache_lock); 29747c478bd9Sstevel@tonic-gate 29757c478bd9Sstevel@tonic-gate umem_remove_updates(cp); 29767c478bd9Sstevel@tonic-gate 29777c478bd9Sstevel@tonic-gate umem_cache_magazine_purge(cp); 29787c478bd9Sstevel@tonic-gate 29797c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_lock); 29807c478bd9Sstevel@tonic-gate if (cp->cache_buftotal != 0) 29817c478bd9Sstevel@tonic-gate log_message("umem_cache_destroy: '%s' (%p) not empty\n", 29827c478bd9Sstevel@tonic-gate cp->cache_name, (void *)cp); 29837c478bd9Sstevel@tonic-gate cp->cache_reclaim = NULL; 29847c478bd9Sstevel@tonic-gate /* 29857c478bd9Sstevel@tonic-gate * The cache is now dead. There should be no further activity. 29867c478bd9Sstevel@tonic-gate * We enforce this by setting land mines in the constructor and 29877c478bd9Sstevel@tonic-gate * destructor routines that induce a segmentation fault if invoked. 29887c478bd9Sstevel@tonic-gate */ 29897c478bd9Sstevel@tonic-gate cp->cache_constructor = (umem_constructor_t *)1; 29907c478bd9Sstevel@tonic-gate cp->cache_destructor = (umem_destructor_t *)2; 29917c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 29927c478bd9Sstevel@tonic-gate 29937c478bd9Sstevel@tonic-gate if (cp->cache_hash_table != NULL) 29947c478bd9Sstevel@tonic-gate vmem_free(umem_hash_arena, cp->cache_hash_table, 29957c478bd9Sstevel@tonic-gate (cp->cache_hash_mask + 1) * sizeof (void *)); 29967c478bd9Sstevel@tonic-gate 29977c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < umem_max_ncpus; cpu_seqid++) 29987c478bd9Sstevel@tonic-gate (void) mutex_destroy(&cp->cache_cpu[cpu_seqid].cc_lock); 29997c478bd9Sstevel@tonic-gate 30007c478bd9Sstevel@tonic-gate (void) mutex_destroy(&cp->cache_depot_lock); 30017c478bd9Sstevel@tonic-gate (void) mutex_destroy(&cp->cache_lock); 30027c478bd9Sstevel@tonic-gate 30037c478bd9Sstevel@tonic-gate vmem_free(umem_cache_arena, cp, UMEM_CACHE_SIZE(umem_max_ncpus)); 30047c478bd9Sstevel@tonic-gate } 30057c478bd9Sstevel@tonic-gate 3006789d94c2Sjwadams void 3007789d94c2Sjwadams umem_alloc_sizes_clear(void) 3008789d94c2Sjwadams { 3009789d94c2Sjwadams int i; 3010789d94c2Sjwadams 3011789d94c2Sjwadams umem_alloc_sizes[0] = UMEM_MAXBUF; 3012789d94c2Sjwadams for (i = 1; i < NUM_ALLOC_SIZES; i++) 3013789d94c2Sjwadams umem_alloc_sizes[i] = 0; 3014789d94c2Sjwadams } 3015789d94c2Sjwadams 3016789d94c2Sjwadams void 3017789d94c2Sjwadams umem_alloc_sizes_add(size_t size_arg) 3018789d94c2Sjwadams { 3019789d94c2Sjwadams int i, j; 3020789d94c2Sjwadams size_t size = size_arg; 3021789d94c2Sjwadams 3022789d94c2Sjwadams if (size == 0) { 3023789d94c2Sjwadams log_message("size_add: cannot add zero-sized cache\n", 3024789d94c2Sjwadams size, UMEM_MAXBUF); 3025789d94c2Sjwadams return; 3026789d94c2Sjwadams } 3027789d94c2Sjwadams 3028789d94c2Sjwadams if (size > UMEM_MAXBUF) { 3029789d94c2Sjwadams log_message("size_add: %ld > %d, cannot add\n", size, 3030789d94c2Sjwadams UMEM_MAXBUF); 3031789d94c2Sjwadams return; 3032789d94c2Sjwadams } 3033789d94c2Sjwadams 3034789d94c2Sjwadams if (umem_alloc_sizes[NUM_ALLOC_SIZES - 1] != 0) { 3035789d94c2Sjwadams log_message("size_add: no space in alloc_table for %d\n", 3036789d94c2Sjwadams size); 3037789d94c2Sjwadams return; 3038789d94c2Sjwadams } 3039789d94c2Sjwadams 3040789d94c2Sjwadams if (P2PHASE(size, UMEM_ALIGN) != 0) { 3041789d94c2Sjwadams size = P2ROUNDUP(size, UMEM_ALIGN); 3042789d94c2Sjwadams log_message("size_add: rounding %d up to %d\n", size_arg, 3043789d94c2Sjwadams size); 3044789d94c2Sjwadams } 3045789d94c2Sjwadams 3046789d94c2Sjwadams for (i = 0; i < NUM_ALLOC_SIZES; i++) { 3047789d94c2Sjwadams int cur = umem_alloc_sizes[i]; 3048789d94c2Sjwadams if (cur == size) { 3049789d94c2Sjwadams log_message("size_add: %ld already in table\n", 3050789d94c2Sjwadams size); 3051789d94c2Sjwadams return; 3052789d94c2Sjwadams } 3053789d94c2Sjwadams if (cur > size) 3054789d94c2Sjwadams break; 3055789d94c2Sjwadams } 3056789d94c2Sjwadams 3057789d94c2Sjwadams for (j = NUM_ALLOC_SIZES - 1; j > i; j--) 3058789d94c2Sjwadams umem_alloc_sizes[j] = umem_alloc_sizes[j-1]; 3059789d94c2Sjwadams umem_alloc_sizes[i] = size; 3060789d94c2Sjwadams } 3061789d94c2Sjwadams 3062789d94c2Sjwadams void 3063789d94c2Sjwadams umem_alloc_sizes_remove(size_t size) 3064789d94c2Sjwadams { 3065789d94c2Sjwadams int i; 3066789d94c2Sjwadams 3067789d94c2Sjwadams if (size == UMEM_MAXBUF) { 3068789d94c2Sjwadams log_message("size_remove: cannot remove %ld\n", size); 3069789d94c2Sjwadams return; 3070789d94c2Sjwadams } 3071789d94c2Sjwadams 3072789d94c2Sjwadams for (i = 0; i < NUM_ALLOC_SIZES; i++) { 3073789d94c2Sjwadams int cur = umem_alloc_sizes[i]; 3074789d94c2Sjwadams if (cur == size) 3075789d94c2Sjwadams break; 3076789d94c2Sjwadams else if (cur > size || cur == 0) { 3077789d94c2Sjwadams log_message("size_remove: %ld not found in table\n", 3078789d94c2Sjwadams size); 3079789d94c2Sjwadams return; 3080789d94c2Sjwadams } 3081789d94c2Sjwadams } 3082789d94c2Sjwadams 3083789d94c2Sjwadams for (; i + 1 < NUM_ALLOC_SIZES; i++) 3084789d94c2Sjwadams umem_alloc_sizes[i] = umem_alloc_sizes[i+1]; 3085789d94c2Sjwadams umem_alloc_sizes[i] = 0; 3086789d94c2Sjwadams } 3087789d94c2Sjwadams 30884f364e7cSRobert Mustacchi /* 30894f364e7cSRobert Mustacchi * We've been called back from libc to indicate that thread is terminating and 30904f364e7cSRobert Mustacchi * that it needs to release the per-thread memory that it has. We get to know 30914f364e7cSRobert Mustacchi * which entry in the thread's tmem array the allocation came from. Currently 30924f364e7cSRobert Mustacchi * this refers to first n umem_caches which makes this a pretty simple indexing 30934f364e7cSRobert Mustacchi * job. 30944f364e7cSRobert Mustacchi */ 30954f364e7cSRobert Mustacchi static void 30964f364e7cSRobert Mustacchi umem_cache_tmem_cleanup(void *buf, int entry) 30974f364e7cSRobert Mustacchi { 30984f364e7cSRobert Mustacchi size_t size; 30994f364e7cSRobert Mustacchi umem_cache_t *cp; 31004f364e7cSRobert Mustacchi 31014f364e7cSRobert Mustacchi size = umem_alloc_sizes[entry]; 31024f364e7cSRobert Mustacchi cp = umem_alloc_table[(size - 1) >> UMEM_ALIGN_SHIFT]; 31034f364e7cSRobert Mustacchi _umem_cache_free(cp, buf); 31044f364e7cSRobert Mustacchi } 31054f364e7cSRobert Mustacchi 31067c478bd9Sstevel@tonic-gate static int 31077c478bd9Sstevel@tonic-gate umem_cache_init(void) 31087c478bd9Sstevel@tonic-gate { 31097c478bd9Sstevel@tonic-gate int i; 31107c478bd9Sstevel@tonic-gate size_t size, max_size; 31117c478bd9Sstevel@tonic-gate umem_cache_t *cp; 31127c478bd9Sstevel@tonic-gate umem_magtype_t *mtp; 31137c478bd9Sstevel@tonic-gate char name[UMEM_CACHE_NAMELEN + 1]; 31147c478bd9Sstevel@tonic-gate umem_cache_t *umem_alloc_caches[NUM_ALLOC_SIZES]; 31157c478bd9Sstevel@tonic-gate 31167c478bd9Sstevel@tonic-gate for (i = 0; i < sizeof (umem_magtype) / sizeof (*mtp); i++) { 31177c478bd9Sstevel@tonic-gate mtp = &umem_magtype[i]; 31187c478bd9Sstevel@tonic-gate (void) snprintf(name, sizeof (name), "umem_magazine_%d", 31197c478bd9Sstevel@tonic-gate mtp->mt_magsize); 31207c478bd9Sstevel@tonic-gate mtp->mt_cache = umem_cache_create(name, 31217c478bd9Sstevel@tonic-gate (mtp->mt_magsize + 1) * sizeof (void *), 31227c478bd9Sstevel@tonic-gate mtp->mt_align, NULL, NULL, NULL, NULL, 31237c478bd9Sstevel@tonic-gate umem_internal_arena, UMC_NOHASH | UMC_INTERNAL); 31247c478bd9Sstevel@tonic-gate if (mtp->mt_cache == NULL) 31257c478bd9Sstevel@tonic-gate return (0); 31267c478bd9Sstevel@tonic-gate } 31277c478bd9Sstevel@tonic-gate 31287c478bd9Sstevel@tonic-gate umem_slab_cache = umem_cache_create("umem_slab_cache", 31297c478bd9Sstevel@tonic-gate sizeof (umem_slab_t), 0, NULL, NULL, NULL, NULL, 31307c478bd9Sstevel@tonic-gate umem_internal_arena, UMC_NOHASH | UMC_INTERNAL); 31317c478bd9Sstevel@tonic-gate 31327c478bd9Sstevel@tonic-gate if (umem_slab_cache == NULL) 31337c478bd9Sstevel@tonic-gate return (0); 31347c478bd9Sstevel@tonic-gate 31357c478bd9Sstevel@tonic-gate umem_bufctl_cache = umem_cache_create("umem_bufctl_cache", 31367c478bd9Sstevel@tonic-gate sizeof (umem_bufctl_t), 0, NULL, NULL, NULL, NULL, 31377c478bd9Sstevel@tonic-gate umem_internal_arena, UMC_NOHASH | UMC_INTERNAL); 31387c478bd9Sstevel@tonic-gate 31397c478bd9Sstevel@tonic-gate if (umem_bufctl_cache == NULL) 31407c478bd9Sstevel@tonic-gate return (0); 31417c478bd9Sstevel@tonic-gate 31427c478bd9Sstevel@tonic-gate /* 31437c478bd9Sstevel@tonic-gate * The size of the umem_bufctl_audit structure depends upon 31447c478bd9Sstevel@tonic-gate * umem_stack_depth. See umem_impl.h for details on the size 31457c478bd9Sstevel@tonic-gate * restrictions. 31467c478bd9Sstevel@tonic-gate */ 31477c478bd9Sstevel@tonic-gate 31487c478bd9Sstevel@tonic-gate size = UMEM_BUFCTL_AUDIT_SIZE_DEPTH(umem_stack_depth); 31497c478bd9Sstevel@tonic-gate max_size = UMEM_BUFCTL_AUDIT_MAX_SIZE; 31507c478bd9Sstevel@tonic-gate 31517c478bd9Sstevel@tonic-gate if (size > max_size) { /* too large -- truncate */ 31527c478bd9Sstevel@tonic-gate int max_frames = UMEM_MAX_STACK_DEPTH; 31537c478bd9Sstevel@tonic-gate 31547c478bd9Sstevel@tonic-gate ASSERT(UMEM_BUFCTL_AUDIT_SIZE_DEPTH(max_frames) <= max_size); 31557c478bd9Sstevel@tonic-gate 31567c478bd9Sstevel@tonic-gate umem_stack_depth = max_frames; 31577c478bd9Sstevel@tonic-gate size = UMEM_BUFCTL_AUDIT_SIZE_DEPTH(umem_stack_depth); 31587c478bd9Sstevel@tonic-gate } 31597c478bd9Sstevel@tonic-gate 31607c478bd9Sstevel@tonic-gate umem_bufctl_audit_cache = umem_cache_create("umem_bufctl_audit_cache", 31617c478bd9Sstevel@tonic-gate size, 0, NULL, NULL, NULL, NULL, umem_internal_arena, 31627c478bd9Sstevel@tonic-gate UMC_NOHASH | UMC_INTERNAL); 31637c478bd9Sstevel@tonic-gate 31647c478bd9Sstevel@tonic-gate if (umem_bufctl_audit_cache == NULL) 31657c478bd9Sstevel@tonic-gate return (0); 31667c478bd9Sstevel@tonic-gate 31677c478bd9Sstevel@tonic-gate if (vmem_backend & VMEM_BACKEND_MMAP) 31687c478bd9Sstevel@tonic-gate umem_va_arena = vmem_create("umem_va", 31697c478bd9Sstevel@tonic-gate NULL, 0, pagesize, 31707c478bd9Sstevel@tonic-gate vmem_alloc, vmem_free, heap_arena, 31717c478bd9Sstevel@tonic-gate 8 * pagesize, VM_NOSLEEP); 31727c478bd9Sstevel@tonic-gate else 31737c478bd9Sstevel@tonic-gate umem_va_arena = heap_arena; 31747c478bd9Sstevel@tonic-gate 31757c478bd9Sstevel@tonic-gate if (umem_va_arena == NULL) 31767c478bd9Sstevel@tonic-gate return (0); 31777c478bd9Sstevel@tonic-gate 31787c478bd9Sstevel@tonic-gate umem_default_arena = vmem_create("umem_default", 31797c478bd9Sstevel@tonic-gate NULL, 0, pagesize, 31807c478bd9Sstevel@tonic-gate heap_alloc, heap_free, umem_va_arena, 31817c478bd9Sstevel@tonic-gate 0, VM_NOSLEEP); 31827c478bd9Sstevel@tonic-gate 31837c478bd9Sstevel@tonic-gate if (umem_default_arena == NULL) 31847c478bd9Sstevel@tonic-gate return (0); 31857c478bd9Sstevel@tonic-gate 31867c478bd9Sstevel@tonic-gate /* 31877c478bd9Sstevel@tonic-gate * make sure the umem_alloc table initializer is correct 31887c478bd9Sstevel@tonic-gate */ 31897c478bd9Sstevel@tonic-gate i = sizeof (umem_alloc_table) / sizeof (*umem_alloc_table); 31907c478bd9Sstevel@tonic-gate ASSERT(umem_alloc_table[i - 1] == &umem_null_cache); 31917c478bd9Sstevel@tonic-gate 31927c478bd9Sstevel@tonic-gate /* 31937c478bd9Sstevel@tonic-gate * Create the default caches to back umem_alloc() 31947c478bd9Sstevel@tonic-gate */ 31957c478bd9Sstevel@tonic-gate for (i = 0; i < NUM_ALLOC_SIZES; i++) { 31967c478bd9Sstevel@tonic-gate size_t cache_size = umem_alloc_sizes[i]; 31977c478bd9Sstevel@tonic-gate size_t align = 0; 3198789d94c2Sjwadams 3199789d94c2Sjwadams if (cache_size == 0) 3200789d94c2Sjwadams break; /* 0 terminates the list */ 3201789d94c2Sjwadams 32027c478bd9Sstevel@tonic-gate /* 32037c478bd9Sstevel@tonic-gate * If they allocate a multiple of the coherency granularity, 32047c478bd9Sstevel@tonic-gate * they get a coherency-granularity-aligned address. 32057c478bd9Sstevel@tonic-gate */ 32067c478bd9Sstevel@tonic-gate if (IS_P2ALIGNED(cache_size, 64)) 32077c478bd9Sstevel@tonic-gate align = 64; 32087c478bd9Sstevel@tonic-gate if (IS_P2ALIGNED(cache_size, pagesize)) 32097c478bd9Sstevel@tonic-gate align = pagesize; 32107c478bd9Sstevel@tonic-gate (void) snprintf(name, sizeof (name), "umem_alloc_%lu", 32117c478bd9Sstevel@tonic-gate (long)cache_size); 32127c478bd9Sstevel@tonic-gate 32137c478bd9Sstevel@tonic-gate cp = umem_cache_create(name, cache_size, align, 32147c478bd9Sstevel@tonic-gate NULL, NULL, NULL, NULL, NULL, UMC_INTERNAL); 32157c478bd9Sstevel@tonic-gate if (cp == NULL) 32167c478bd9Sstevel@tonic-gate return (0); 32177c478bd9Sstevel@tonic-gate 32187c478bd9Sstevel@tonic-gate umem_alloc_caches[i] = cp; 32197c478bd9Sstevel@tonic-gate } 32207c478bd9Sstevel@tonic-gate 32214f364e7cSRobert Mustacchi umem_tmem_off = _tmem_get_base(); 32224f364e7cSRobert Mustacchi _tmem_set_cleanup(umem_cache_tmem_cleanup); 32234f364e7cSRobert Mustacchi 3224b1e2e3fbSRobert Mustacchi #ifndef UMEM_STANDALONE 32254f364e7cSRobert Mustacchi if (umem_genasm_supported && !(umem_flags & UMF_DEBUG) && 32264f364e7cSRobert Mustacchi !(umem_flags & UMF_NOMAGAZINE) && 32274f364e7cSRobert Mustacchi umem_ptc_size > 0) { 32284f364e7cSRobert Mustacchi umem_ptc_enabled = umem_genasm(umem_alloc_sizes, 3229b1e2e3fbSRobert Mustacchi umem_alloc_caches, i) ? 1 : 0; 32304f364e7cSRobert Mustacchi } 3231b1e2e3fbSRobert Mustacchi #else 3232b1e2e3fbSRobert Mustacchi umem_ptc_enabled = 0; 3233b1e2e3fbSRobert Mustacchi #endif 32344f364e7cSRobert Mustacchi 32357c478bd9Sstevel@tonic-gate /* 32367c478bd9Sstevel@tonic-gate * Initialization cannot fail at this point. Make the caches 32377c478bd9Sstevel@tonic-gate * visible to umem_alloc() and friends. 32387c478bd9Sstevel@tonic-gate */ 32397c478bd9Sstevel@tonic-gate size = UMEM_ALIGN; 32407c478bd9Sstevel@tonic-gate for (i = 0; i < NUM_ALLOC_SIZES; i++) { 32417c478bd9Sstevel@tonic-gate size_t cache_size = umem_alloc_sizes[i]; 32427c478bd9Sstevel@tonic-gate 3243789d94c2Sjwadams if (cache_size == 0) 3244789d94c2Sjwadams break; /* 0 terminates the list */ 3245789d94c2Sjwadams 32467c478bd9Sstevel@tonic-gate cp = umem_alloc_caches[i]; 32477c478bd9Sstevel@tonic-gate 32487c478bd9Sstevel@tonic-gate while (size <= cache_size) { 32497c478bd9Sstevel@tonic-gate umem_alloc_table[(size - 1) >> UMEM_ALIGN_SHIFT] = cp; 32507c478bd9Sstevel@tonic-gate size += UMEM_ALIGN; 32517c478bd9Sstevel@tonic-gate } 32527c478bd9Sstevel@tonic-gate } 3253789d94c2Sjwadams ASSERT(size - UMEM_ALIGN == UMEM_MAXBUF); 32547c478bd9Sstevel@tonic-gate return (1); 32557c478bd9Sstevel@tonic-gate } 32567c478bd9Sstevel@tonic-gate 32577c478bd9Sstevel@tonic-gate /* 32587c478bd9Sstevel@tonic-gate * umem_startup() is called early on, and must be called explicitly if we're 32597c478bd9Sstevel@tonic-gate * the standalone version. 32607c478bd9Sstevel@tonic-gate */ 32617c478bd9Sstevel@tonic-gate #ifdef UMEM_STANDALONE 32627c478bd9Sstevel@tonic-gate void 32637c478bd9Sstevel@tonic-gate #else 32647c478bd9Sstevel@tonic-gate #pragma init(umem_startup) 32657c478bd9Sstevel@tonic-gate static void 32667c478bd9Sstevel@tonic-gate #endif 32677c478bd9Sstevel@tonic-gate umem_startup(caddr_t start, size_t len, size_t pagesize, caddr_t minstack, 32687c478bd9Sstevel@tonic-gate caddr_t maxstack) 32697c478bd9Sstevel@tonic-gate { 32707c478bd9Sstevel@tonic-gate #ifdef UMEM_STANDALONE 32717c478bd9Sstevel@tonic-gate int idx; 32727c478bd9Sstevel@tonic-gate /* Standalone doesn't fork */ 32737c478bd9Sstevel@tonic-gate #else 32747c478bd9Sstevel@tonic-gate umem_forkhandler_init(); /* register the fork handler */ 32757c478bd9Sstevel@tonic-gate #endif 32767c478bd9Sstevel@tonic-gate 32777c478bd9Sstevel@tonic-gate #ifdef __lint 32787c478bd9Sstevel@tonic-gate /* make lint happy */ 32797c478bd9Sstevel@tonic-gate minstack = maxstack; 32807c478bd9Sstevel@tonic-gate #endif 32817c478bd9Sstevel@tonic-gate 32827c478bd9Sstevel@tonic-gate #ifdef UMEM_STANDALONE 32837c478bd9Sstevel@tonic-gate umem_ready = UMEM_READY_STARTUP; 32847c478bd9Sstevel@tonic-gate umem_init_env_ready = 0; 32857c478bd9Sstevel@tonic-gate 32867c478bd9Sstevel@tonic-gate umem_min_stack = minstack; 32877c478bd9Sstevel@tonic-gate umem_max_stack = maxstack; 32887c478bd9Sstevel@tonic-gate 32897c478bd9Sstevel@tonic-gate nofail_callback = NULL; 32907c478bd9Sstevel@tonic-gate umem_slab_cache = NULL; 32917c478bd9Sstevel@tonic-gate umem_bufctl_cache = NULL; 32927c478bd9Sstevel@tonic-gate umem_bufctl_audit_cache = NULL; 32937c478bd9Sstevel@tonic-gate heap_arena = NULL; 32947c478bd9Sstevel@tonic-gate heap_alloc = NULL; 32957c478bd9Sstevel@tonic-gate heap_free = NULL; 32967c478bd9Sstevel@tonic-gate umem_internal_arena = NULL; 32977c478bd9Sstevel@tonic-gate umem_cache_arena = NULL; 32987c478bd9Sstevel@tonic-gate umem_hash_arena = NULL; 32997c478bd9Sstevel@tonic-gate umem_log_arena = NULL; 33007c478bd9Sstevel@tonic-gate umem_oversize_arena = NULL; 33017c478bd9Sstevel@tonic-gate umem_va_arena = NULL; 33027c478bd9Sstevel@tonic-gate umem_default_arena = NULL; 33037c478bd9Sstevel@tonic-gate umem_firewall_va_arena = NULL; 33047c478bd9Sstevel@tonic-gate umem_firewall_arena = NULL; 33057c478bd9Sstevel@tonic-gate umem_memalign_arena = NULL; 33067c478bd9Sstevel@tonic-gate umem_transaction_log = NULL; 33077c478bd9Sstevel@tonic-gate umem_content_log = NULL; 33087c478bd9Sstevel@tonic-gate umem_failure_log = NULL; 33097c478bd9Sstevel@tonic-gate umem_slab_log = NULL; 33107c478bd9Sstevel@tonic-gate umem_cpu_mask = 0; 33117c478bd9Sstevel@tonic-gate 33127c478bd9Sstevel@tonic-gate umem_cpus = &umem_startup_cpu; 33137c478bd9Sstevel@tonic-gate umem_startup_cpu.cpu_cache_offset = UMEM_CACHE_SIZE(0); 33147c478bd9Sstevel@tonic-gate umem_startup_cpu.cpu_number = 0; 33157c478bd9Sstevel@tonic-gate 33167c478bd9Sstevel@tonic-gate bcopy(&umem_null_cache_template, &umem_null_cache, 33177c478bd9Sstevel@tonic-gate sizeof (umem_cache_t)); 33187c478bd9Sstevel@tonic-gate 33197c478bd9Sstevel@tonic-gate for (idx = 0; idx < (UMEM_MAXBUF >> UMEM_ALIGN_SHIFT); idx++) 33207c478bd9Sstevel@tonic-gate umem_alloc_table[idx] = &umem_null_cache; 33217c478bd9Sstevel@tonic-gate #endif 33227c478bd9Sstevel@tonic-gate 33237c478bd9Sstevel@tonic-gate /* 33247c478bd9Sstevel@tonic-gate * Perform initialization specific to the way we've been compiled 33257c478bd9Sstevel@tonic-gate * (library or standalone) 33267c478bd9Sstevel@tonic-gate */ 33277c478bd9Sstevel@tonic-gate umem_type_init(start, len, pagesize); 33287c478bd9Sstevel@tonic-gate 33297c478bd9Sstevel@tonic-gate vmem_startup(); 33307c478bd9Sstevel@tonic-gate } 33317c478bd9Sstevel@tonic-gate 33327c478bd9Sstevel@tonic-gate int 33337c478bd9Sstevel@tonic-gate umem_init(void) 33347c478bd9Sstevel@tonic-gate { 33357c478bd9Sstevel@tonic-gate size_t maxverify, minfirewall; 33367c478bd9Sstevel@tonic-gate size_t size; 33377c478bd9Sstevel@tonic-gate int idx; 33387c478bd9Sstevel@tonic-gate umem_cpu_t *new_cpus; 33397c478bd9Sstevel@tonic-gate 33407c478bd9Sstevel@tonic-gate vmem_t *memalign_arena, *oversize_arena; 33417c478bd9Sstevel@tonic-gate 33427c478bd9Sstevel@tonic-gate if (thr_self() != umem_init_thr) { 33437c478bd9Sstevel@tonic-gate /* 33447c478bd9Sstevel@tonic-gate * The usual case -- non-recursive invocation of umem_init(). 33457c478bd9Sstevel@tonic-gate */ 33467c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_init_lock); 33477c478bd9Sstevel@tonic-gate if (umem_ready != UMEM_READY_STARTUP) { 33487c478bd9Sstevel@tonic-gate /* 33497c478bd9Sstevel@tonic-gate * someone else beat us to initializing umem. Wait 33507c478bd9Sstevel@tonic-gate * for them to complete, then return. 33517c478bd9Sstevel@tonic-gate */ 3352a574db85Sraf while (umem_ready == UMEM_READY_INITING) { 3353a574db85Sraf int cancel_state; 3354a574db85Sraf 3355a574db85Sraf (void) pthread_setcancelstate( 3356a574db85Sraf PTHREAD_CANCEL_DISABLE, &cancel_state); 3357a574db85Sraf (void) cond_wait(&umem_init_cv, 33587c478bd9Sstevel@tonic-gate &umem_init_lock); 3359a574db85Sraf (void) pthread_setcancelstate( 3360a574db85Sraf cancel_state, NULL); 3361a574db85Sraf } 33627c478bd9Sstevel@tonic-gate ASSERT(umem_ready == UMEM_READY || 33637c478bd9Sstevel@tonic-gate umem_ready == UMEM_READY_INIT_FAILED); 33647c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_init_lock); 33657c478bd9Sstevel@tonic-gate return (umem_ready == UMEM_READY); 33667c478bd9Sstevel@tonic-gate } 33677c478bd9Sstevel@tonic-gate 33687c478bd9Sstevel@tonic-gate ASSERT(umem_ready == UMEM_READY_STARTUP); 33697c478bd9Sstevel@tonic-gate ASSERT(umem_init_env_ready == 0); 33707c478bd9Sstevel@tonic-gate 33717c478bd9Sstevel@tonic-gate umem_ready = UMEM_READY_INITING; 33727c478bd9Sstevel@tonic-gate umem_init_thr = thr_self(); 33737c478bd9Sstevel@tonic-gate 33747c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_init_lock); 33757c478bd9Sstevel@tonic-gate umem_setup_envvars(0); /* can recurse -- see below */ 33767c478bd9Sstevel@tonic-gate if (umem_init_env_ready) { 33777c478bd9Sstevel@tonic-gate /* 33787c478bd9Sstevel@tonic-gate * initialization was completed already 33797c478bd9Sstevel@tonic-gate */ 33807c478bd9Sstevel@tonic-gate ASSERT(umem_ready == UMEM_READY || 33817c478bd9Sstevel@tonic-gate umem_ready == UMEM_READY_INIT_FAILED); 33827c478bd9Sstevel@tonic-gate ASSERT(umem_init_thr == 0); 33837c478bd9Sstevel@tonic-gate return (umem_ready == UMEM_READY); 33847c478bd9Sstevel@tonic-gate } 33857c478bd9Sstevel@tonic-gate } else if (!umem_init_env_ready) { 33867c478bd9Sstevel@tonic-gate /* 33877c478bd9Sstevel@tonic-gate * The umem_setup_envvars() call (above) makes calls into 33887c478bd9Sstevel@tonic-gate * the dynamic linker and directly into user-supplied code. 33897c478bd9Sstevel@tonic-gate * Since we cannot know what that code will do, we could be 33907c478bd9Sstevel@tonic-gate * recursively invoked (by, say, a malloc() call in the code 33917c478bd9Sstevel@tonic-gate * itself, or in a (C++) _init section it causes to be fired). 33927c478bd9Sstevel@tonic-gate * 33937c478bd9Sstevel@tonic-gate * This code is where we end up if such recursion occurs. We 33947c478bd9Sstevel@tonic-gate * first clean up any partial results in the envvar code, then 33957c478bd9Sstevel@tonic-gate * proceed to finish initialization processing in the recursive 33967c478bd9Sstevel@tonic-gate * call. The original call will notice this, and return 33977c478bd9Sstevel@tonic-gate * immediately. 33987c478bd9Sstevel@tonic-gate */ 33997c478bd9Sstevel@tonic-gate umem_setup_envvars(1); /* clean up any partial state */ 34007c478bd9Sstevel@tonic-gate } else { 34017c478bd9Sstevel@tonic-gate umem_panic( 34027c478bd9Sstevel@tonic-gate "recursive allocation while initializing umem\n"); 34037c478bd9Sstevel@tonic-gate } 34047c478bd9Sstevel@tonic-gate umem_init_env_ready = 1; 34057c478bd9Sstevel@tonic-gate 34067c478bd9Sstevel@tonic-gate /* 34077c478bd9Sstevel@tonic-gate * From this point until we finish, recursion into umem_init() will 34087c478bd9Sstevel@tonic-gate * cause a umem_panic(). 34097c478bd9Sstevel@tonic-gate */ 34107c478bd9Sstevel@tonic-gate maxverify = minfirewall = ULONG_MAX; 34117c478bd9Sstevel@tonic-gate 34127c478bd9Sstevel@tonic-gate /* LINTED constant condition */ 34137c478bd9Sstevel@tonic-gate if (sizeof (umem_cpu_cache_t) != UMEM_CPU_CACHE_SIZE) { 34147c478bd9Sstevel@tonic-gate umem_panic("sizeof (umem_cpu_cache_t) = %d, should be %d\n", 34157c478bd9Sstevel@tonic-gate sizeof (umem_cpu_cache_t), UMEM_CPU_CACHE_SIZE); 34167c478bd9Sstevel@tonic-gate } 34177c478bd9Sstevel@tonic-gate 34187c478bd9Sstevel@tonic-gate umem_max_ncpus = umem_get_max_ncpus(); 34197c478bd9Sstevel@tonic-gate 34207c478bd9Sstevel@tonic-gate /* 34217c478bd9Sstevel@tonic-gate * load tunables from environment 34227c478bd9Sstevel@tonic-gate */ 34237c478bd9Sstevel@tonic-gate umem_process_envvars(); 34247c478bd9Sstevel@tonic-gate 34257c478bd9Sstevel@tonic-gate if (issetugid()) 34267c478bd9Sstevel@tonic-gate umem_mtbf = 0; 34277c478bd9Sstevel@tonic-gate 34287c478bd9Sstevel@tonic-gate /* 34297c478bd9Sstevel@tonic-gate * set up vmem 34307c478bd9Sstevel@tonic-gate */ 34317c478bd9Sstevel@tonic-gate if (!(umem_flags & UMF_AUDIT)) 34327c478bd9Sstevel@tonic-gate vmem_no_debug(); 34337c478bd9Sstevel@tonic-gate 34347c478bd9Sstevel@tonic-gate heap_arena = vmem_heap_arena(&heap_alloc, &heap_free); 34357c478bd9Sstevel@tonic-gate 34367c478bd9Sstevel@tonic-gate pagesize = heap_arena->vm_quantum; 34377c478bd9Sstevel@tonic-gate 34387c478bd9Sstevel@tonic-gate umem_internal_arena = vmem_create("umem_internal", NULL, 0, pagesize, 34397c478bd9Sstevel@tonic-gate heap_alloc, heap_free, heap_arena, 0, VM_NOSLEEP); 34407c478bd9Sstevel@tonic-gate 34417c478bd9Sstevel@tonic-gate umem_default_arena = umem_internal_arena; 34427c478bd9Sstevel@tonic-gate 34437c478bd9Sstevel@tonic-gate if (umem_internal_arena == NULL) 34447c478bd9Sstevel@tonic-gate goto fail; 34457c478bd9Sstevel@tonic-gate 34467c478bd9Sstevel@tonic-gate umem_cache_arena = vmem_create("umem_cache", NULL, 0, UMEM_ALIGN, 34477c478bd9Sstevel@tonic-gate vmem_alloc, vmem_free, umem_internal_arena, 0, VM_NOSLEEP); 34487c478bd9Sstevel@tonic-gate 34497c478bd9Sstevel@tonic-gate umem_hash_arena = vmem_create("umem_hash", NULL, 0, UMEM_ALIGN, 34507c478bd9Sstevel@tonic-gate vmem_alloc, vmem_free, umem_internal_arena, 0, VM_NOSLEEP); 34517c478bd9Sstevel@tonic-gate 34527c478bd9Sstevel@tonic-gate umem_log_arena = vmem_create("umem_log", NULL, 0, UMEM_ALIGN, 34537c478bd9Sstevel@tonic-gate heap_alloc, heap_free, heap_arena, 0, VM_NOSLEEP); 34547c478bd9Sstevel@tonic-gate 34557c478bd9Sstevel@tonic-gate umem_firewall_va_arena = vmem_create("umem_firewall_va", 34567c478bd9Sstevel@tonic-gate NULL, 0, pagesize, 34577c478bd9Sstevel@tonic-gate umem_firewall_va_alloc, umem_firewall_va_free, heap_arena, 34587c478bd9Sstevel@tonic-gate 0, VM_NOSLEEP); 34597c478bd9Sstevel@tonic-gate 34607c478bd9Sstevel@tonic-gate if (umem_cache_arena == NULL || umem_hash_arena == NULL || 34617c478bd9Sstevel@tonic-gate umem_log_arena == NULL || umem_firewall_va_arena == NULL) 34627c478bd9Sstevel@tonic-gate goto fail; 34637c478bd9Sstevel@tonic-gate 34647c478bd9Sstevel@tonic-gate umem_firewall_arena = vmem_create("umem_firewall", NULL, 0, pagesize, 34657c478bd9Sstevel@tonic-gate heap_alloc, heap_free, umem_firewall_va_arena, 0, 34667c478bd9Sstevel@tonic-gate VM_NOSLEEP); 34677c478bd9Sstevel@tonic-gate 34687c478bd9Sstevel@tonic-gate if (umem_firewall_arena == NULL) 34697c478bd9Sstevel@tonic-gate goto fail; 34707c478bd9Sstevel@tonic-gate 34717c478bd9Sstevel@tonic-gate oversize_arena = vmem_create("umem_oversize", NULL, 0, pagesize, 34727c478bd9Sstevel@tonic-gate heap_alloc, heap_free, minfirewall < ULONG_MAX ? 34737c478bd9Sstevel@tonic-gate umem_firewall_va_arena : heap_arena, 0, VM_NOSLEEP); 34747c478bd9Sstevel@tonic-gate 34757c478bd9Sstevel@tonic-gate memalign_arena = vmem_create("umem_memalign", NULL, 0, UMEM_ALIGN, 34767c478bd9Sstevel@tonic-gate heap_alloc, heap_free, minfirewall < ULONG_MAX ? 34777c478bd9Sstevel@tonic-gate umem_firewall_va_arena : heap_arena, 0, VM_NOSLEEP); 34787c478bd9Sstevel@tonic-gate 34797c478bd9Sstevel@tonic-gate if (oversize_arena == NULL || memalign_arena == NULL) 34807c478bd9Sstevel@tonic-gate goto fail; 34817c478bd9Sstevel@tonic-gate 34827c478bd9Sstevel@tonic-gate if (umem_max_ncpus > CPUHINT_MAX()) 34837c478bd9Sstevel@tonic-gate umem_max_ncpus = CPUHINT_MAX(); 34847c478bd9Sstevel@tonic-gate 34857c478bd9Sstevel@tonic-gate while ((umem_max_ncpus & (umem_max_ncpus - 1)) != 0) 34867c478bd9Sstevel@tonic-gate umem_max_ncpus++; 34877c478bd9Sstevel@tonic-gate 34887c478bd9Sstevel@tonic-gate if (umem_max_ncpus == 0) 34897c478bd9Sstevel@tonic-gate umem_max_ncpus = 1; 34907c478bd9Sstevel@tonic-gate 34917c478bd9Sstevel@tonic-gate size = umem_max_ncpus * sizeof (umem_cpu_t); 34927c478bd9Sstevel@tonic-gate new_cpus = vmem_alloc(umem_internal_arena, size, VM_NOSLEEP); 34937c478bd9Sstevel@tonic-gate if (new_cpus == NULL) 34947c478bd9Sstevel@tonic-gate goto fail; 34957c478bd9Sstevel@tonic-gate 34967c478bd9Sstevel@tonic-gate bzero(new_cpus, size); 34977c478bd9Sstevel@tonic-gate for (idx = 0; idx < umem_max_ncpus; idx++) { 34987c478bd9Sstevel@tonic-gate new_cpus[idx].cpu_number = idx; 34997c478bd9Sstevel@tonic-gate new_cpus[idx].cpu_cache_offset = UMEM_CACHE_SIZE(idx); 35007c478bd9Sstevel@tonic-gate } 35017c478bd9Sstevel@tonic-gate umem_cpus = new_cpus; 35027c478bd9Sstevel@tonic-gate umem_cpu_mask = (umem_max_ncpus - 1); 35037c478bd9Sstevel@tonic-gate 35047c478bd9Sstevel@tonic-gate if (umem_maxverify == 0) 35057c478bd9Sstevel@tonic-gate umem_maxverify = maxverify; 35067c478bd9Sstevel@tonic-gate 35077c478bd9Sstevel@tonic-gate if (umem_minfirewall == 0) 35087c478bd9Sstevel@tonic-gate umem_minfirewall = minfirewall; 35097c478bd9Sstevel@tonic-gate 35107c478bd9Sstevel@tonic-gate /* 35117c478bd9Sstevel@tonic-gate * Set up updating and reaping 35127c478bd9Sstevel@tonic-gate */ 35137c478bd9Sstevel@tonic-gate umem_reap_next = gethrtime() + NANOSEC; 35147c478bd9Sstevel@tonic-gate 35157c478bd9Sstevel@tonic-gate #ifndef UMEM_STANDALONE 35167c478bd9Sstevel@tonic-gate (void) gettimeofday(&umem_update_next, NULL); 35177c478bd9Sstevel@tonic-gate #endif 35187c478bd9Sstevel@tonic-gate 35197c478bd9Sstevel@tonic-gate /* 35207c478bd9Sstevel@tonic-gate * Set up logging -- failure here is okay, since it will just disable 35217c478bd9Sstevel@tonic-gate * the logs 35227c478bd9Sstevel@tonic-gate */ 35237c478bd9Sstevel@tonic-gate if (umem_logging) { 35247c478bd9Sstevel@tonic-gate umem_transaction_log = umem_log_init(umem_transaction_log_size); 35257c478bd9Sstevel@tonic-gate umem_content_log = umem_log_init(umem_content_log_size); 35267c478bd9Sstevel@tonic-gate umem_failure_log = umem_log_init(umem_failure_log_size); 35277c478bd9Sstevel@tonic-gate umem_slab_log = umem_log_init(umem_slab_log_size); 35287c478bd9Sstevel@tonic-gate } 35297c478bd9Sstevel@tonic-gate 35307c478bd9Sstevel@tonic-gate /* 35317c478bd9Sstevel@tonic-gate * Set up caches -- if successful, initialization cannot fail, since 35327c478bd9Sstevel@tonic-gate * allocations from other threads can now succeed. 35337c478bd9Sstevel@tonic-gate */ 35347c478bd9Sstevel@tonic-gate if (umem_cache_init() == 0) { 35357c478bd9Sstevel@tonic-gate log_message("unable to create initial caches\n"); 35367c478bd9Sstevel@tonic-gate goto fail; 35377c478bd9Sstevel@tonic-gate } 35387c478bd9Sstevel@tonic-gate umem_oversize_arena = oversize_arena; 35397c478bd9Sstevel@tonic-gate umem_memalign_arena = memalign_arena; 35407c478bd9Sstevel@tonic-gate 35417c478bd9Sstevel@tonic-gate umem_cache_applyall(umem_cache_magazine_enable); 35427c478bd9Sstevel@tonic-gate 35437c478bd9Sstevel@tonic-gate /* 35447c478bd9Sstevel@tonic-gate * initialization done, ready to go 35457c478bd9Sstevel@tonic-gate */ 35467c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_init_lock); 35477c478bd9Sstevel@tonic-gate umem_ready = UMEM_READY; 35487c478bd9Sstevel@tonic-gate umem_init_thr = 0; 35497c478bd9Sstevel@tonic-gate (void) cond_broadcast(&umem_init_cv); 35507c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_init_lock); 35517c478bd9Sstevel@tonic-gate return (1); 35527c478bd9Sstevel@tonic-gate 35537c478bd9Sstevel@tonic-gate fail: 35547c478bd9Sstevel@tonic-gate log_message("umem initialization failed\n"); 35557c478bd9Sstevel@tonic-gate 35567c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_init_lock); 35577c478bd9Sstevel@tonic-gate umem_ready = UMEM_READY_INIT_FAILED; 35587c478bd9Sstevel@tonic-gate umem_init_thr = 0; 35597c478bd9Sstevel@tonic-gate (void) cond_broadcast(&umem_init_cv); 35607c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_init_lock); 35617c478bd9Sstevel@tonic-gate return (0); 35627c478bd9Sstevel@tonic-gate } 3563*14702342SRobert Mustacchi 3564*14702342SRobert Mustacchi void 3565*14702342SRobert Mustacchi umem_setmtbf(uint32_t mtbf) 3566*14702342SRobert Mustacchi { 3567*14702342SRobert Mustacchi umem_mtbf = mtbf; 3568*14702342SRobert Mustacchi } 3569