17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5789d94c2Sjwadams * Common Development and Distribution License (the "License"). 6789d94c2Sjwadams * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 21e8031f0aSraf 227c478bd9Sstevel@tonic-gate /* 23a574db85Sraf * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 24*38849194SRobert Mustacchi * Copyright 2012 Joyent, Inc. All rights reserved. 257c478bd9Sstevel@tonic-gate * Use is subject to license terms. 267c478bd9Sstevel@tonic-gate */ 277c478bd9Sstevel@tonic-gate 287c478bd9Sstevel@tonic-gate /* 297c478bd9Sstevel@tonic-gate * based on usr/src/uts/common/os/kmem.c r1.64 from 2001/12/18 307c478bd9Sstevel@tonic-gate * 317c478bd9Sstevel@tonic-gate * The slab allocator, as described in the following two papers: 327c478bd9Sstevel@tonic-gate * 337c478bd9Sstevel@tonic-gate * Jeff Bonwick, 347c478bd9Sstevel@tonic-gate * The Slab Allocator: An Object-Caching Kernel Memory Allocator. 357c478bd9Sstevel@tonic-gate * Proceedings of the Summer 1994 Usenix Conference. 367c478bd9Sstevel@tonic-gate * Available as /shared/sac/PSARC/1994/028/materials/kmem.pdf. 377c478bd9Sstevel@tonic-gate * 387c478bd9Sstevel@tonic-gate * Jeff Bonwick and Jonathan Adams, 397c478bd9Sstevel@tonic-gate * Magazines and vmem: Extending the Slab Allocator to Many CPUs and 407c478bd9Sstevel@tonic-gate * Arbitrary Resources. 417c478bd9Sstevel@tonic-gate * Proceedings of the 2001 Usenix Conference. 427c478bd9Sstevel@tonic-gate * Available as /shared/sac/PSARC/2000/550/materials/vmem.pdf. 437c478bd9Sstevel@tonic-gate * 447c478bd9Sstevel@tonic-gate * 1. Overview 457c478bd9Sstevel@tonic-gate * ----------- 467c478bd9Sstevel@tonic-gate * umem is very close to kmem in implementation. There are four major 477c478bd9Sstevel@tonic-gate * areas of divergence: 487c478bd9Sstevel@tonic-gate * 497c478bd9Sstevel@tonic-gate * * Initialization 507c478bd9Sstevel@tonic-gate * 517c478bd9Sstevel@tonic-gate * * CPU handling 527c478bd9Sstevel@tonic-gate * 537c478bd9Sstevel@tonic-gate * * umem_update() 547c478bd9Sstevel@tonic-gate * 557c478bd9Sstevel@tonic-gate * * KM_SLEEP v.s. UMEM_NOFAIL 567c478bd9Sstevel@tonic-gate * 571c326e94Sjwadams * * lock ordering 587c478bd9Sstevel@tonic-gate * 597c478bd9Sstevel@tonic-gate * 2. Initialization 607c478bd9Sstevel@tonic-gate * ----------------- 617c478bd9Sstevel@tonic-gate * kmem is initialized early on in boot, and knows that no one will call 627c478bd9Sstevel@tonic-gate * into it before it is ready. umem does not have these luxuries. Instead, 637c478bd9Sstevel@tonic-gate * initialization is divided into two phases: 647c478bd9Sstevel@tonic-gate * 657c478bd9Sstevel@tonic-gate * * library initialization, and 667c478bd9Sstevel@tonic-gate * 677c478bd9Sstevel@tonic-gate * * first use 687c478bd9Sstevel@tonic-gate * 697c478bd9Sstevel@tonic-gate * umem's full initialization happens at the time of the first allocation 707c478bd9Sstevel@tonic-gate * request (via malloc() and friends, umem_alloc(), or umem_zalloc()), 717c478bd9Sstevel@tonic-gate * or the first call to umem_cache_create(). 727c478bd9Sstevel@tonic-gate * 737c478bd9Sstevel@tonic-gate * umem_free(), and umem_cache_alloc() do not require special handling, 747c478bd9Sstevel@tonic-gate * since the only way to get valid arguments for them is to successfully 757c478bd9Sstevel@tonic-gate * call a function from the first group. 767c478bd9Sstevel@tonic-gate * 777c478bd9Sstevel@tonic-gate * 2.1. Library Initialization: umem_startup() 787c478bd9Sstevel@tonic-gate * ------------------------------------------- 797c478bd9Sstevel@tonic-gate * umem_startup() is libumem.so's .init section. It calls pthread_atfork() 807c478bd9Sstevel@tonic-gate * to install the handlers necessary for umem's Fork1-Safety. Because of 817c478bd9Sstevel@tonic-gate * race condition issues, all other pre-umem_init() initialization is done 827c478bd9Sstevel@tonic-gate * statically (i.e. by the dynamic linker). 837c478bd9Sstevel@tonic-gate * 847c478bd9Sstevel@tonic-gate * For standalone use, umem_startup() returns everything to its initial 857c478bd9Sstevel@tonic-gate * state. 867c478bd9Sstevel@tonic-gate * 877c478bd9Sstevel@tonic-gate * 2.2. First use: umem_init() 887c478bd9Sstevel@tonic-gate * ------------------------------ 897c478bd9Sstevel@tonic-gate * The first time any memory allocation function is used, we have to 907c478bd9Sstevel@tonic-gate * create the backing caches and vmem arenas which are needed for it. 917c478bd9Sstevel@tonic-gate * umem_init() is the central point for that task. When it completes, 927c478bd9Sstevel@tonic-gate * umem_ready is either UMEM_READY (all set) or UMEM_READY_INIT_FAILED (unable 937c478bd9Sstevel@tonic-gate * to initialize, probably due to lack of memory). 947c478bd9Sstevel@tonic-gate * 957c478bd9Sstevel@tonic-gate * There are four different paths from which umem_init() is called: 967c478bd9Sstevel@tonic-gate * 977c478bd9Sstevel@tonic-gate * * from umem_alloc() or umem_zalloc(), with 0 < size < UMEM_MAXBUF, 987c478bd9Sstevel@tonic-gate * 997c478bd9Sstevel@tonic-gate * * from umem_alloc() or umem_zalloc(), with size > UMEM_MAXBUF, 1007c478bd9Sstevel@tonic-gate * 1017c478bd9Sstevel@tonic-gate * * from umem_cache_create(), and 1027c478bd9Sstevel@tonic-gate * 1037c478bd9Sstevel@tonic-gate * * from memalign(), with align > UMEM_ALIGN. 1047c478bd9Sstevel@tonic-gate * 1057c478bd9Sstevel@tonic-gate * The last three just check if umem is initialized, and call umem_init() 1067c478bd9Sstevel@tonic-gate * if it is not. For performance reasons, the first case is more complicated. 1077c478bd9Sstevel@tonic-gate * 1087c478bd9Sstevel@tonic-gate * 2.2.1. umem_alloc()/umem_zalloc(), with 0 < size < UMEM_MAXBUF 1097c478bd9Sstevel@tonic-gate * ----------------------------------------------------------------- 1107c478bd9Sstevel@tonic-gate * In this case, umem_cache_alloc(&umem_null_cache, ...) is called. 1117c478bd9Sstevel@tonic-gate * There is special case code in which causes any allocation on 1127c478bd9Sstevel@tonic-gate * &umem_null_cache to fail by returning (NULL), regardless of the 1137c478bd9Sstevel@tonic-gate * flags argument. 1147c478bd9Sstevel@tonic-gate * 1157c478bd9Sstevel@tonic-gate * So umem_cache_alloc() returns NULL, and umem_alloc()/umem_zalloc() call 1167c478bd9Sstevel@tonic-gate * umem_alloc_retry(). umem_alloc_retry() sees that the allocation 1177c478bd9Sstevel@tonic-gate * was agains &umem_null_cache, and calls umem_init(). 1187c478bd9Sstevel@tonic-gate * 1197c478bd9Sstevel@tonic-gate * If initialization is successful, umem_alloc_retry() returns 1, which 1207c478bd9Sstevel@tonic-gate * causes umem_alloc()/umem_zalloc() to start over, which causes it to load 1217c478bd9Sstevel@tonic-gate * the (now valid) cache pointer from umem_alloc_table. 1227c478bd9Sstevel@tonic-gate * 1237c478bd9Sstevel@tonic-gate * 2.2.2. Dealing with race conditions 1247c478bd9Sstevel@tonic-gate * ----------------------------------- 1257c478bd9Sstevel@tonic-gate * There are a couple race conditions resulting from the initialization 1267c478bd9Sstevel@tonic-gate * code that we have to guard against: 1277c478bd9Sstevel@tonic-gate * 1287c478bd9Sstevel@tonic-gate * * In umem_cache_create(), there is a special UMC_INTERNAL cflag 1297c478bd9Sstevel@tonic-gate * that is passed for caches created during initialization. It 1307c478bd9Sstevel@tonic-gate * is illegal for a user to try to create a UMC_INTERNAL cache. 1317c478bd9Sstevel@tonic-gate * This allows initialization to proceed, but any other 1327c478bd9Sstevel@tonic-gate * umem_cache_create()s will block by calling umem_init(). 1337c478bd9Sstevel@tonic-gate * 1347c478bd9Sstevel@tonic-gate * * Since umem_null_cache has a 1-element cache_cpu, it's cache_cpu_mask 1357c478bd9Sstevel@tonic-gate * is always zero. umem_cache_alloc uses cp->cache_cpu_mask to 1367c478bd9Sstevel@tonic-gate * mask the cpu number. This prevents a race between grabbing a 1377c478bd9Sstevel@tonic-gate * cache pointer out of umem_alloc_table and growing the cpu array. 1387c478bd9Sstevel@tonic-gate * 1397c478bd9Sstevel@tonic-gate * 1407c478bd9Sstevel@tonic-gate * 3. CPU handling 1417c478bd9Sstevel@tonic-gate * --------------- 1427c478bd9Sstevel@tonic-gate * kmem uses the CPU's sequence number to determine which "cpu cache" to 1437c478bd9Sstevel@tonic-gate * use for an allocation. Currently, there is no way to get the sequence 1447c478bd9Sstevel@tonic-gate * number in userspace. 1457c478bd9Sstevel@tonic-gate * 1467c478bd9Sstevel@tonic-gate * umem keeps track of cpu information in umem_cpus, an array of umem_max_ncpus 1477c478bd9Sstevel@tonic-gate * umem_cpu_t structures. CURCPU() is a a "hint" function, which we then mask 1487c478bd9Sstevel@tonic-gate * with either umem_cpu_mask or cp->cache_cpu_mask to find the actual "cpu" id. 1497c478bd9Sstevel@tonic-gate * The mechanics of this is all in the CPU(mask) macro. 1507c478bd9Sstevel@tonic-gate * 1517c478bd9Sstevel@tonic-gate * Currently, umem uses _lwp_self() as its hint. 1527c478bd9Sstevel@tonic-gate * 1537c478bd9Sstevel@tonic-gate * 1547c478bd9Sstevel@tonic-gate * 4. The update thread 1557c478bd9Sstevel@tonic-gate * -------------------- 1567c478bd9Sstevel@tonic-gate * kmem uses a task queue, kmem_taskq, to do periodic maintenance on 1577c478bd9Sstevel@tonic-gate * every kmem cache. vmem has a periodic timeout for hash table resizing. 1587c478bd9Sstevel@tonic-gate * The kmem_taskq also provides a separate context for kmem_cache_reap()'s 1597c478bd9Sstevel@tonic-gate * to be done in, avoiding issues of the context of kmem_reap() callers. 1607c478bd9Sstevel@tonic-gate * 1617c478bd9Sstevel@tonic-gate * Instead, umem has the concept of "updates", which are asynchronous requests 1627c478bd9Sstevel@tonic-gate * for work attached to single caches. All caches with pending work are 1637c478bd9Sstevel@tonic-gate * on a doubly linked list rooted at the umem_null_cache. All update state 1647c478bd9Sstevel@tonic-gate * is protected by the umem_update_lock mutex, and the umem_update_cv is used 1657c478bd9Sstevel@tonic-gate * for notification between threads. 1667c478bd9Sstevel@tonic-gate * 1677c478bd9Sstevel@tonic-gate * 4.1. Cache states with regards to updates 1687c478bd9Sstevel@tonic-gate * ----------------------------------------- 1697c478bd9Sstevel@tonic-gate * A given cache is in one of three states: 1707c478bd9Sstevel@tonic-gate * 1717c478bd9Sstevel@tonic-gate * Inactive cache_uflags is zero, cache_u{next,prev} are NULL 1727c478bd9Sstevel@tonic-gate * 1737c478bd9Sstevel@tonic-gate * Work Requested cache_uflags is non-zero (but UMU_ACTIVE is not set), 1747c478bd9Sstevel@tonic-gate * cache_u{next,prev} link the cache onto the global 1757c478bd9Sstevel@tonic-gate * update list 1767c478bd9Sstevel@tonic-gate * 1777c478bd9Sstevel@tonic-gate * Active cache_uflags has UMU_ACTIVE set, cache_u{next,prev} 1787c478bd9Sstevel@tonic-gate * are NULL, and either umem_update_thr or 1797c478bd9Sstevel@tonic-gate * umem_st_update_thr are actively doing work on the 1807c478bd9Sstevel@tonic-gate * cache. 1817c478bd9Sstevel@tonic-gate * 1827c478bd9Sstevel@tonic-gate * An update can be added to any cache in any state -- if the cache is 1837c478bd9Sstevel@tonic-gate * Inactive, it transitions to being Work Requested. If the cache is 1847c478bd9Sstevel@tonic-gate * Active, the worker will notice the new update and act on it before 1857c478bd9Sstevel@tonic-gate * transitioning the cache to the Inactive state. 1867c478bd9Sstevel@tonic-gate * 1877c478bd9Sstevel@tonic-gate * If a cache is in the Active state, UMU_NOTIFY can be set, which asks 1887c478bd9Sstevel@tonic-gate * the worker to broadcast the umem_update_cv when it has finished. 1897c478bd9Sstevel@tonic-gate * 1907c478bd9Sstevel@tonic-gate * 4.2. Update interface 1917c478bd9Sstevel@tonic-gate * --------------------- 1927c478bd9Sstevel@tonic-gate * umem_add_update() adds an update to a particular cache. 1937c478bd9Sstevel@tonic-gate * umem_updateall() adds an update to all caches. 1947c478bd9Sstevel@tonic-gate * umem_remove_updates() returns a cache to the Inactive state. 1957c478bd9Sstevel@tonic-gate * 1967c478bd9Sstevel@tonic-gate * umem_process_updates() process all caches in the Work Requested state. 1977c478bd9Sstevel@tonic-gate * 1987c478bd9Sstevel@tonic-gate * 4.3. Reaping 1997c478bd9Sstevel@tonic-gate * ------------ 2007c478bd9Sstevel@tonic-gate * When umem_reap() is called (at the time of heap growth), it schedule 2017c478bd9Sstevel@tonic-gate * UMU_REAP updates on every cache. It then checks to see if the update 2027c478bd9Sstevel@tonic-gate * thread exists (umem_update_thr != 0). If it is, it broadcasts 2037c478bd9Sstevel@tonic-gate * the umem_update_cv to wake the update thread up, and returns. 2047c478bd9Sstevel@tonic-gate * 2057c478bd9Sstevel@tonic-gate * If the update thread does not exist (umem_update_thr == 0), and the 2067c478bd9Sstevel@tonic-gate * program currently has multiple threads, umem_reap() attempts to create 2077c478bd9Sstevel@tonic-gate * a new update thread. 2087c478bd9Sstevel@tonic-gate * 2097c478bd9Sstevel@tonic-gate * If the process is not multithreaded, or the creation fails, umem_reap() 2107c478bd9Sstevel@tonic-gate * calls umem_st_update() to do an inline update. 2117c478bd9Sstevel@tonic-gate * 2127c478bd9Sstevel@tonic-gate * 4.4. The update thread 2137c478bd9Sstevel@tonic-gate * ---------------------- 2147c478bd9Sstevel@tonic-gate * The update thread spends most of its time in cond_timedwait() on the 2157c478bd9Sstevel@tonic-gate * umem_update_cv. It wakes up under two conditions: 2167c478bd9Sstevel@tonic-gate * 2177c478bd9Sstevel@tonic-gate * * The timedwait times out, in which case it needs to run a global 2187c478bd9Sstevel@tonic-gate * update, or 2197c478bd9Sstevel@tonic-gate * 2207c478bd9Sstevel@tonic-gate * * someone cond_broadcast(3THR)s the umem_update_cv, in which case 2217c478bd9Sstevel@tonic-gate * it needs to check if there are any caches in the Work Requested 2227c478bd9Sstevel@tonic-gate * state. 2237c478bd9Sstevel@tonic-gate * 2247c478bd9Sstevel@tonic-gate * When it is time for another global update, umem calls umem_cache_update() 2257c478bd9Sstevel@tonic-gate * on every cache, then calls vmem_update(), which tunes the vmem structures. 2267c478bd9Sstevel@tonic-gate * umem_cache_update() can request further work using umem_add_update(). 2277c478bd9Sstevel@tonic-gate * 2287c478bd9Sstevel@tonic-gate * After any work from the global update completes, the update timer is 2297c478bd9Sstevel@tonic-gate * reset to umem_reap_interval seconds in the future. This makes the 2307c478bd9Sstevel@tonic-gate * updates self-throttling. 2317c478bd9Sstevel@tonic-gate * 2327c478bd9Sstevel@tonic-gate * Reaps are similarly self-throttling. After a UMU_REAP update has 2337c478bd9Sstevel@tonic-gate * been scheduled on all caches, umem_reap() sets a flag and wakes up the 2347c478bd9Sstevel@tonic-gate * update thread. The update thread notices the flag, and resets the 2357c478bd9Sstevel@tonic-gate * reap state. 2367c478bd9Sstevel@tonic-gate * 2377c478bd9Sstevel@tonic-gate * 4.5. Inline updates 2387c478bd9Sstevel@tonic-gate * ------------------- 2397c478bd9Sstevel@tonic-gate * If the update thread is not running, umem_st_update() is used instead. It 2407c478bd9Sstevel@tonic-gate * immediately does a global update (as above), then calls 2417c478bd9Sstevel@tonic-gate * umem_process_updates() to process both the reaps that umem_reap() added and 2427c478bd9Sstevel@tonic-gate * any work generated by the global update. Afterwards, it resets the reap 2437c478bd9Sstevel@tonic-gate * state. 2447c478bd9Sstevel@tonic-gate * 2457c478bd9Sstevel@tonic-gate * While the umem_st_update() is running, umem_st_update_thr holds the thread 2467c478bd9Sstevel@tonic-gate * id of the thread performing the update. 2477c478bd9Sstevel@tonic-gate * 2487c478bd9Sstevel@tonic-gate * 4.6. Updates and fork1() 2497c478bd9Sstevel@tonic-gate * ------------------------ 2507c478bd9Sstevel@tonic-gate * umem has fork1() pre- and post-handlers which lock up (and release) every 2517c478bd9Sstevel@tonic-gate * mutex in every cache. They also lock up the umem_update_lock. Since 2527c478bd9Sstevel@tonic-gate * fork1() only copies over a single lwp, other threads (including the update 2537c478bd9Sstevel@tonic-gate * thread) could have been actively using a cache in the parent. This 2547c478bd9Sstevel@tonic-gate * can lead to inconsistencies in the child process. 2557c478bd9Sstevel@tonic-gate * 2567c478bd9Sstevel@tonic-gate * Because we locked all of the mutexes, the only possible inconsistancies are: 2577c478bd9Sstevel@tonic-gate * 2587c478bd9Sstevel@tonic-gate * * a umem_cache_alloc() could leak its buffer. 2597c478bd9Sstevel@tonic-gate * 2607c478bd9Sstevel@tonic-gate * * a caller of umem_depot_alloc() could leak a magazine, and all the 2617c478bd9Sstevel@tonic-gate * buffers contained in it. 2627c478bd9Sstevel@tonic-gate * 2637c478bd9Sstevel@tonic-gate * * a cache could be in the Active update state. In the child, there 2647c478bd9Sstevel@tonic-gate * would be no thread actually working on it. 2657c478bd9Sstevel@tonic-gate * 2667c478bd9Sstevel@tonic-gate * * a umem_hash_rescale() could leak the new hash table. 2677c478bd9Sstevel@tonic-gate * 2687c478bd9Sstevel@tonic-gate * * a umem_magazine_resize() could be in progress. 2697c478bd9Sstevel@tonic-gate * 2707c478bd9Sstevel@tonic-gate * * a umem_reap() could be in progress. 2717c478bd9Sstevel@tonic-gate * 2727c478bd9Sstevel@tonic-gate * The memory leaks we can't do anything about. umem_release_child() resets 2737c478bd9Sstevel@tonic-gate * the update state, moves any caches in the Active state to the Work Requested 2747c478bd9Sstevel@tonic-gate * state. This might cause some updates to be re-run, but UMU_REAP and 2757c478bd9Sstevel@tonic-gate * UMU_HASH_RESCALE are effectively idempotent, and the worst that can 2767c478bd9Sstevel@tonic-gate * happen from umem_magazine_resize() is resizing the magazine twice in close 2777c478bd9Sstevel@tonic-gate * succession. 2787c478bd9Sstevel@tonic-gate * 2797c478bd9Sstevel@tonic-gate * Much of the cleanup in umem_release_child() is skipped if 2807c478bd9Sstevel@tonic-gate * umem_st_update_thr == thr_self(). This is so that applications which call 2817c478bd9Sstevel@tonic-gate * fork1() from a cache callback does not break. Needless to say, any such 2827c478bd9Sstevel@tonic-gate * application is tremendously broken. 2837c478bd9Sstevel@tonic-gate * 2847c478bd9Sstevel@tonic-gate * 2857c478bd9Sstevel@tonic-gate * 5. KM_SLEEP v.s. UMEM_NOFAIL 2867c478bd9Sstevel@tonic-gate * ---------------------------- 2877c478bd9Sstevel@tonic-gate * Allocations against kmem and vmem have two basic modes: SLEEP and 2887c478bd9Sstevel@tonic-gate * NOSLEEP. A sleeping allocation is will go to sleep (waiting for 2897c478bd9Sstevel@tonic-gate * more memory) instead of failing (returning NULL). 2907c478bd9Sstevel@tonic-gate * 2917c478bd9Sstevel@tonic-gate * SLEEP allocations presume an extremely multithreaded model, with 2927c478bd9Sstevel@tonic-gate * a lot of allocation and deallocation activity. umem cannot presume 2937c478bd9Sstevel@tonic-gate * that its clients have any particular type of behavior. Instead, 2947c478bd9Sstevel@tonic-gate * it provides two types of allocations: 2957c478bd9Sstevel@tonic-gate * 2967c478bd9Sstevel@tonic-gate * * UMEM_DEFAULT, equivalent to KM_NOSLEEP (i.e. return NULL on 2977c478bd9Sstevel@tonic-gate * failure) 2987c478bd9Sstevel@tonic-gate * 2997c478bd9Sstevel@tonic-gate * * UMEM_NOFAIL, which, on failure, calls an optional callback 3007c478bd9Sstevel@tonic-gate * (registered with umem_nofail_callback()). 3017c478bd9Sstevel@tonic-gate * 3027c478bd9Sstevel@tonic-gate * The callback is invoked with no locks held, and can do an arbitrary 3037c478bd9Sstevel@tonic-gate * amount of work. It then has a choice between: 3047c478bd9Sstevel@tonic-gate * 3057c478bd9Sstevel@tonic-gate * * Returning UMEM_CALLBACK_RETRY, which will cause the allocation 3067c478bd9Sstevel@tonic-gate * to be restarted. 3077c478bd9Sstevel@tonic-gate * 3087c478bd9Sstevel@tonic-gate * * Returning UMEM_CALLBACK_EXIT(status), which will cause exit(2) 3097c478bd9Sstevel@tonic-gate * to be invoked with status. If multiple threads attempt to do 3107c478bd9Sstevel@tonic-gate * this simultaneously, only one will call exit(2). 3117c478bd9Sstevel@tonic-gate * 3127c478bd9Sstevel@tonic-gate * * Doing some kind of non-local exit (thr_exit(3thr), longjmp(3C), 3137c478bd9Sstevel@tonic-gate * etc.) 3147c478bd9Sstevel@tonic-gate * 3157c478bd9Sstevel@tonic-gate * The default callback returns UMEM_CALLBACK_EXIT(255). 3167c478bd9Sstevel@tonic-gate * 3177c478bd9Sstevel@tonic-gate * To have these callbacks without risk of state corruption (in the case of 3187c478bd9Sstevel@tonic-gate * a non-local exit), we have to ensure that the callbacks get invoked 3197c478bd9Sstevel@tonic-gate * close to the original allocation, with no inconsistent state or held 3207c478bd9Sstevel@tonic-gate * locks. The following steps are taken: 3217c478bd9Sstevel@tonic-gate * 3227c478bd9Sstevel@tonic-gate * * All invocations of vmem are VM_NOSLEEP. 3237c478bd9Sstevel@tonic-gate * 3247c478bd9Sstevel@tonic-gate * * All constructor callbacks (which can themselves to allocations) 3257c478bd9Sstevel@tonic-gate * are passed UMEM_DEFAULT as their required allocation argument. This 3267c478bd9Sstevel@tonic-gate * way, the constructor will fail, allowing the highest-level allocation 3277c478bd9Sstevel@tonic-gate * invoke the nofail callback. 3287c478bd9Sstevel@tonic-gate * 3297c478bd9Sstevel@tonic-gate * If a constructor callback _does_ do a UMEM_NOFAIL allocation, and 3307c478bd9Sstevel@tonic-gate * the nofail callback does a non-local exit, we will leak the 3317c478bd9Sstevel@tonic-gate * partially-constructed buffer. 3321c326e94Sjwadams * 3331c326e94Sjwadams * 3341c326e94Sjwadams * 6. Lock Ordering 3351c326e94Sjwadams * ---------------- 3361c326e94Sjwadams * umem has a few more locks than kmem does, mostly in the update path. The 3371c326e94Sjwadams * overall lock ordering (earlier locks must be acquired first) is: 3381c326e94Sjwadams * 3391c326e94Sjwadams * umem_init_lock 3401c326e94Sjwadams * 3411c326e94Sjwadams * vmem_list_lock 3421c326e94Sjwadams * vmem_nosleep_lock.vmpl_mutex 3431c326e94Sjwadams * vmem_t's: 3441c326e94Sjwadams * vm_lock 345789d94c2Sjwadams * sbrk_lock 3461c326e94Sjwadams * 3471c326e94Sjwadams * umem_cache_lock 3481c326e94Sjwadams * umem_update_lock 3491c326e94Sjwadams * umem_flags_lock 3501c326e94Sjwadams * umem_cache_t's: 3511c326e94Sjwadams * cache_cpu[*].cc_lock 3521c326e94Sjwadams * cache_depot_lock 3531c326e94Sjwadams * cache_lock 3541c326e94Sjwadams * umem_log_header_t's: 3551c326e94Sjwadams * lh_cpu[*].clh_lock 3561c326e94Sjwadams * lh_lock 357*38849194SRobert Mustacchi * 358*38849194SRobert Mustacchi * 7. Changing UMEM_MAXBUF 359*38849194SRobert Mustacchi * ----------------------- 360*38849194SRobert Mustacchi * 361*38849194SRobert Mustacchi * When changing UMEM_MAXBUF extra care has to be taken. It is not sufficient to 362*38849194SRobert Mustacchi * simply increase this number. First, one must update the umem_alloc_table to 363*38849194SRobert Mustacchi * have the appropriate number of entires based upon the new size. If this is 364*38849194SRobert Mustacchi * not done, this will lead to libumem blowing an assertion. 365*38849194SRobert Mustacchi * 366*38849194SRobert Mustacchi * The second place to update, which is not required, is the umem_alloc_sizes. 367*38849194SRobert Mustacchi * These determine the default cache sizes that we're going to support. 3687c478bd9Sstevel@tonic-gate */ 3697c478bd9Sstevel@tonic-gate 3707c478bd9Sstevel@tonic-gate #include <umem_impl.h> 3717c478bd9Sstevel@tonic-gate #include <sys/vmem_impl_user.h> 3727c478bd9Sstevel@tonic-gate #include "umem_base.h" 3737c478bd9Sstevel@tonic-gate #include "vmem_base.h" 3747c478bd9Sstevel@tonic-gate 3757c478bd9Sstevel@tonic-gate #include <sys/processor.h> 3767c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 3777c478bd9Sstevel@tonic-gate 3787c478bd9Sstevel@tonic-gate #include <alloca.h> 3797c478bd9Sstevel@tonic-gate #include <errno.h> 3807c478bd9Sstevel@tonic-gate #include <limits.h> 3817c478bd9Sstevel@tonic-gate #include <stdio.h> 3827c478bd9Sstevel@tonic-gate #include <stdlib.h> 3837c478bd9Sstevel@tonic-gate #include <string.h> 3847c478bd9Sstevel@tonic-gate #include <strings.h> 3857c478bd9Sstevel@tonic-gate #include <signal.h> 3867c478bd9Sstevel@tonic-gate #include <unistd.h> 3877c478bd9Sstevel@tonic-gate #include <atomic.h> 3887c478bd9Sstevel@tonic-gate 3897c478bd9Sstevel@tonic-gate #include "misc.h" 3907c478bd9Sstevel@tonic-gate 3917c478bd9Sstevel@tonic-gate #define UMEM_VMFLAGS(umflag) (VM_NOSLEEP) 3927c478bd9Sstevel@tonic-gate 3937c478bd9Sstevel@tonic-gate size_t pagesize; 3947c478bd9Sstevel@tonic-gate 3957c478bd9Sstevel@tonic-gate /* 3967c478bd9Sstevel@tonic-gate * The default set of caches to back umem_alloc(). 3977c478bd9Sstevel@tonic-gate * These sizes should be reevaluated periodically. 3987c478bd9Sstevel@tonic-gate * 3997c478bd9Sstevel@tonic-gate * We want allocations that are multiples of the coherency granularity 4007c478bd9Sstevel@tonic-gate * (64 bytes) to be satisfied from a cache which is a multiple of 64 4017c478bd9Sstevel@tonic-gate * bytes, so that it will be 64-byte aligned. For all multiples of 64, 4027c478bd9Sstevel@tonic-gate * the next kmem_cache_size greater than or equal to it must be a 4037c478bd9Sstevel@tonic-gate * multiple of 64. 404789d94c2Sjwadams * 405789d94c2Sjwadams * This table must be in sorted order, from smallest to highest. The 406789d94c2Sjwadams * highest slot must be UMEM_MAXBUF, and every slot afterwards must be 407789d94c2Sjwadams * zero. 4087c478bd9Sstevel@tonic-gate */ 409789d94c2Sjwadams static int umem_alloc_sizes[] = { 4107c478bd9Sstevel@tonic-gate #ifdef _LP64 4117c478bd9Sstevel@tonic-gate 1 * 8, 4127c478bd9Sstevel@tonic-gate 1 * 16, 4137c478bd9Sstevel@tonic-gate 2 * 16, 4147c478bd9Sstevel@tonic-gate 3 * 16, 4157c478bd9Sstevel@tonic-gate #else 4167c478bd9Sstevel@tonic-gate 1 * 8, 4177c478bd9Sstevel@tonic-gate 2 * 8, 4187c478bd9Sstevel@tonic-gate 3 * 8, 4197c478bd9Sstevel@tonic-gate 4 * 8, 5 * 8, 6 * 8, 7 * 8, 4207c478bd9Sstevel@tonic-gate #endif 4217c478bd9Sstevel@tonic-gate 4 * 16, 5 * 16, 6 * 16, 7 * 16, 4227c478bd9Sstevel@tonic-gate 4 * 32, 5 * 32, 6 * 32, 7 * 32, 4237c478bd9Sstevel@tonic-gate 4 * 64, 5 * 64, 6 * 64, 7 * 64, 4247c478bd9Sstevel@tonic-gate 4 * 128, 5 * 128, 6 * 128, 7 * 128, 4257c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 7, 64), 4267c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 6, 64), 4277c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 5, 64), 428789d94c2Sjwadams P2ALIGN(8192 / 4, 64), 2304, 4297c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 3, 64), 430789d94c2Sjwadams P2ALIGN(8192 / 2, 64), 4544, 431789d94c2Sjwadams P2ALIGN(8192 / 1, 64), 9216, 4327c478bd9Sstevel@tonic-gate 4096 * 3, 433*38849194SRobert Mustacchi 8192 * 2, /* = 8192 * 2 */ 434*38849194SRobert Mustacchi 24576, 32768, 40960, 49152, 57344, 65536, 73728, 81920, 435*38849194SRobert Mustacchi 90112, 98304, 106496, 114688, 122880, UMEM_MAXBUF, /* 128k */ 436789d94c2Sjwadams /* 24 slots for user expansion */ 437789d94c2Sjwadams 0, 0, 0, 0, 0, 0, 0, 0, 438789d94c2Sjwadams 0, 0, 0, 0, 0, 0, 0, 0, 439789d94c2Sjwadams 0, 0, 0, 0, 0, 0, 0, 0, 4407c478bd9Sstevel@tonic-gate }; 4417c478bd9Sstevel@tonic-gate #define NUM_ALLOC_SIZES (sizeof (umem_alloc_sizes) / sizeof (*umem_alloc_sizes)) 4427c478bd9Sstevel@tonic-gate 4437c478bd9Sstevel@tonic-gate static umem_magtype_t umem_magtype[] = { 4447c478bd9Sstevel@tonic-gate { 1, 8, 3200, 65536 }, 4457c478bd9Sstevel@tonic-gate { 3, 16, 256, 32768 }, 4467c478bd9Sstevel@tonic-gate { 7, 32, 64, 16384 }, 4477c478bd9Sstevel@tonic-gate { 15, 64, 0, 8192 }, 4487c478bd9Sstevel@tonic-gate { 31, 64, 0, 4096 }, 4497c478bd9Sstevel@tonic-gate { 47, 64, 0, 2048 }, 4507c478bd9Sstevel@tonic-gate { 63, 64, 0, 1024 }, 4517c478bd9Sstevel@tonic-gate { 95, 64, 0, 512 }, 4527c478bd9Sstevel@tonic-gate { 143, 64, 0, 0 }, 4537c478bd9Sstevel@tonic-gate }; 4547c478bd9Sstevel@tonic-gate 4557c478bd9Sstevel@tonic-gate /* 4567c478bd9Sstevel@tonic-gate * umem tunables 4577c478bd9Sstevel@tonic-gate */ 4587c478bd9Sstevel@tonic-gate uint32_t umem_max_ncpus; /* # of CPU caches. */ 4597c478bd9Sstevel@tonic-gate 4607c478bd9Sstevel@tonic-gate uint32_t umem_stack_depth = 15; /* # stack frames in a bufctl_audit */ 4617c478bd9Sstevel@tonic-gate uint32_t umem_reap_interval = 10; /* max reaping rate (seconds) */ 4627c478bd9Sstevel@tonic-gate uint_t umem_depot_contention = 2; /* max failed trylocks per real interval */ 4637c478bd9Sstevel@tonic-gate uint_t umem_abort = 1; /* whether to abort on error */ 4647c478bd9Sstevel@tonic-gate uint_t umem_output = 0; /* whether to write to standard error */ 4657c478bd9Sstevel@tonic-gate uint_t umem_logging = 0; /* umem_log_enter() override */ 4667c478bd9Sstevel@tonic-gate uint32_t umem_mtbf = 0; /* mean time between failures [default: off] */ 4677c478bd9Sstevel@tonic-gate size_t umem_transaction_log_size; /* size of transaction log */ 4687c478bd9Sstevel@tonic-gate size_t umem_content_log_size; /* size of content log */ 4697c478bd9Sstevel@tonic-gate size_t umem_failure_log_size; /* failure log [4 pages per CPU] */ 4707c478bd9Sstevel@tonic-gate size_t umem_slab_log_size; /* slab create log [4 pages per CPU] */ 4717c478bd9Sstevel@tonic-gate size_t umem_content_maxsave = 256; /* UMF_CONTENTS max bytes to log */ 4727c478bd9Sstevel@tonic-gate size_t umem_lite_minsize = 0; /* minimum buffer size for UMF_LITE */ 4737c478bd9Sstevel@tonic-gate size_t umem_lite_maxalign = 1024; /* maximum buffer alignment for UMF_LITE */ 4747c478bd9Sstevel@tonic-gate size_t umem_maxverify; /* maximum bytes to inspect in debug routines */ 4757c478bd9Sstevel@tonic-gate size_t umem_minfirewall; /* hardware-enforced redzone threshold */ 4767c478bd9Sstevel@tonic-gate 4777c478bd9Sstevel@tonic-gate uint_t umem_flags = 0; 4787c478bd9Sstevel@tonic-gate 4797c478bd9Sstevel@tonic-gate mutex_t umem_init_lock; /* locks initialization */ 4807c478bd9Sstevel@tonic-gate cond_t umem_init_cv; /* initialization CV */ 4817c478bd9Sstevel@tonic-gate thread_t umem_init_thr; /* thread initializing */ 4827c478bd9Sstevel@tonic-gate int umem_init_env_ready; /* environ pre-initted */ 4837c478bd9Sstevel@tonic-gate int umem_ready = UMEM_READY_STARTUP; 4847c478bd9Sstevel@tonic-gate 4857c478bd9Sstevel@tonic-gate static umem_nofail_callback_t *nofail_callback; 4867c478bd9Sstevel@tonic-gate static mutex_t umem_nofail_exit_lock; 4877c478bd9Sstevel@tonic-gate static thread_t umem_nofail_exit_thr; 4887c478bd9Sstevel@tonic-gate 4897c478bd9Sstevel@tonic-gate static umem_cache_t *umem_slab_cache; 4907c478bd9Sstevel@tonic-gate static umem_cache_t *umem_bufctl_cache; 4917c478bd9Sstevel@tonic-gate static umem_cache_t *umem_bufctl_audit_cache; 4927c478bd9Sstevel@tonic-gate 4937c478bd9Sstevel@tonic-gate mutex_t umem_flags_lock; 4947c478bd9Sstevel@tonic-gate 4957c478bd9Sstevel@tonic-gate static vmem_t *heap_arena; 4967c478bd9Sstevel@tonic-gate static vmem_alloc_t *heap_alloc; 4977c478bd9Sstevel@tonic-gate static vmem_free_t *heap_free; 4987c478bd9Sstevel@tonic-gate 4997c478bd9Sstevel@tonic-gate static vmem_t *umem_internal_arena; 5007c478bd9Sstevel@tonic-gate static vmem_t *umem_cache_arena; 5017c478bd9Sstevel@tonic-gate static vmem_t *umem_hash_arena; 5027c478bd9Sstevel@tonic-gate static vmem_t *umem_log_arena; 5037c478bd9Sstevel@tonic-gate static vmem_t *umem_oversize_arena; 5047c478bd9Sstevel@tonic-gate static vmem_t *umem_va_arena; 5057c478bd9Sstevel@tonic-gate static vmem_t *umem_default_arena; 5067c478bd9Sstevel@tonic-gate static vmem_t *umem_firewall_va_arena; 5077c478bd9Sstevel@tonic-gate static vmem_t *umem_firewall_arena; 5087c478bd9Sstevel@tonic-gate 5097c478bd9Sstevel@tonic-gate vmem_t *umem_memalign_arena; 5107c478bd9Sstevel@tonic-gate 5117c478bd9Sstevel@tonic-gate umem_log_header_t *umem_transaction_log; 5127c478bd9Sstevel@tonic-gate umem_log_header_t *umem_content_log; 5137c478bd9Sstevel@tonic-gate umem_log_header_t *umem_failure_log; 5147c478bd9Sstevel@tonic-gate umem_log_header_t *umem_slab_log; 5157c478bd9Sstevel@tonic-gate 5167257d1b4Sraf #define CPUHINT() (thr_self()) 5177c478bd9Sstevel@tonic-gate #define CPUHINT_MAX() INT_MAX 5187c478bd9Sstevel@tonic-gate 5197c478bd9Sstevel@tonic-gate #define CPU(mask) (umem_cpus + (CPUHINT() & (mask))) 5207c478bd9Sstevel@tonic-gate static umem_cpu_t umem_startup_cpu = { /* initial, single, cpu */ 5217c478bd9Sstevel@tonic-gate UMEM_CACHE_SIZE(0), 5227c478bd9Sstevel@tonic-gate 0 5237c478bd9Sstevel@tonic-gate }; 5247c478bd9Sstevel@tonic-gate 5257c478bd9Sstevel@tonic-gate static uint32_t umem_cpu_mask = 0; /* global cpu mask */ 5267c478bd9Sstevel@tonic-gate static umem_cpu_t *umem_cpus = &umem_startup_cpu; /* cpu list */ 5277c478bd9Sstevel@tonic-gate 5287c478bd9Sstevel@tonic-gate volatile uint32_t umem_reaping; 5297c478bd9Sstevel@tonic-gate 5307c478bd9Sstevel@tonic-gate thread_t umem_update_thr; 5317c478bd9Sstevel@tonic-gate struct timeval umem_update_next; /* timeofday of next update */ 5327c478bd9Sstevel@tonic-gate volatile thread_t umem_st_update_thr; /* only used when single-thd */ 5337c478bd9Sstevel@tonic-gate 5347c478bd9Sstevel@tonic-gate #define IN_UPDATE() (thr_self() == umem_update_thr || \ 5357c478bd9Sstevel@tonic-gate thr_self() == umem_st_update_thr) 5367c478bd9Sstevel@tonic-gate #define IN_REAP() IN_UPDATE() 5377c478bd9Sstevel@tonic-gate 5387c478bd9Sstevel@tonic-gate mutex_t umem_update_lock; /* cache_u{next,prev,flags} */ 5397c478bd9Sstevel@tonic-gate cond_t umem_update_cv; 5407c478bd9Sstevel@tonic-gate 5417c478bd9Sstevel@tonic-gate volatile hrtime_t umem_reap_next; /* min hrtime of next reap */ 5427c478bd9Sstevel@tonic-gate 5437c478bd9Sstevel@tonic-gate mutex_t umem_cache_lock; /* inter-cache linkage only */ 5447c478bd9Sstevel@tonic-gate 5457c478bd9Sstevel@tonic-gate #ifdef UMEM_STANDALONE 5467c478bd9Sstevel@tonic-gate umem_cache_t umem_null_cache; 5477c478bd9Sstevel@tonic-gate static const umem_cache_t umem_null_cache_template = { 5487c478bd9Sstevel@tonic-gate #else 5497c478bd9Sstevel@tonic-gate umem_cache_t umem_null_cache = { 5507c478bd9Sstevel@tonic-gate #endif 5517c478bd9Sstevel@tonic-gate 0, 0, 0, 0, 0, 5527c478bd9Sstevel@tonic-gate 0, 0, 5537c478bd9Sstevel@tonic-gate 0, 0, 5547c478bd9Sstevel@tonic-gate 0, 0, 5557c478bd9Sstevel@tonic-gate "invalid_cache", 5567c478bd9Sstevel@tonic-gate 0, 0, 5577c478bd9Sstevel@tonic-gate NULL, NULL, NULL, NULL, 5587c478bd9Sstevel@tonic-gate NULL, 5597c478bd9Sstevel@tonic-gate 0, 0, 0, 0, 5607c478bd9Sstevel@tonic-gate &umem_null_cache, &umem_null_cache, 5617c478bd9Sstevel@tonic-gate &umem_null_cache, &umem_null_cache, 5627c478bd9Sstevel@tonic-gate 0, 5637c478bd9Sstevel@tonic-gate DEFAULTMUTEX, /* start of slab layer */ 5647c478bd9Sstevel@tonic-gate 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5657c478bd9Sstevel@tonic-gate &umem_null_cache.cache_nullslab, 5667c478bd9Sstevel@tonic-gate { 5677c478bd9Sstevel@tonic-gate &umem_null_cache, 5687c478bd9Sstevel@tonic-gate NULL, 5697c478bd9Sstevel@tonic-gate &umem_null_cache.cache_nullslab, 5707c478bd9Sstevel@tonic-gate &umem_null_cache.cache_nullslab, 5717c478bd9Sstevel@tonic-gate NULL, 5727c478bd9Sstevel@tonic-gate -1, 5737c478bd9Sstevel@tonic-gate 0 5747c478bd9Sstevel@tonic-gate }, 5757c478bd9Sstevel@tonic-gate NULL, 5767c478bd9Sstevel@tonic-gate NULL, 5777c478bd9Sstevel@tonic-gate DEFAULTMUTEX, /* start of depot layer */ 5787c478bd9Sstevel@tonic-gate NULL, { 5797c478bd9Sstevel@tonic-gate NULL, 0, 0, 0, 0 5807c478bd9Sstevel@tonic-gate }, { 5817c478bd9Sstevel@tonic-gate NULL, 0, 0, 0, 0 5827c478bd9Sstevel@tonic-gate }, { 5837c478bd9Sstevel@tonic-gate { 5847c478bd9Sstevel@tonic-gate DEFAULTMUTEX, /* start of CPU cache */ 5857c478bd9Sstevel@tonic-gate 0, 0, NULL, NULL, -1, -1, 0 5867c478bd9Sstevel@tonic-gate } 5877c478bd9Sstevel@tonic-gate } 5887c478bd9Sstevel@tonic-gate }; 5897c478bd9Sstevel@tonic-gate 5907c478bd9Sstevel@tonic-gate #define ALLOC_TABLE_4 \ 5917c478bd9Sstevel@tonic-gate &umem_null_cache, &umem_null_cache, &umem_null_cache, &umem_null_cache 5927c478bd9Sstevel@tonic-gate 5937c478bd9Sstevel@tonic-gate #define ALLOC_TABLE_64 \ 5947c478bd9Sstevel@tonic-gate ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, \ 5957c478bd9Sstevel@tonic-gate ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, \ 5967c478bd9Sstevel@tonic-gate ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, \ 5977c478bd9Sstevel@tonic-gate ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4 5987c478bd9Sstevel@tonic-gate 5997c478bd9Sstevel@tonic-gate #define ALLOC_TABLE_1024 \ 6007c478bd9Sstevel@tonic-gate ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, \ 6017c478bd9Sstevel@tonic-gate ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, \ 6027c478bd9Sstevel@tonic-gate ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, \ 6037c478bd9Sstevel@tonic-gate ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64 6047c478bd9Sstevel@tonic-gate 6057c478bd9Sstevel@tonic-gate static umem_cache_t *umem_alloc_table[UMEM_MAXBUF >> UMEM_ALIGN_SHIFT] = { 606*38849194SRobert Mustacchi ALLOC_TABLE_1024, 607*38849194SRobert Mustacchi ALLOC_TABLE_1024, 608*38849194SRobert Mustacchi ALLOC_TABLE_1024, 609*38849194SRobert Mustacchi ALLOC_TABLE_1024, 610*38849194SRobert Mustacchi ALLOC_TABLE_1024, 611*38849194SRobert Mustacchi ALLOC_TABLE_1024, 612*38849194SRobert Mustacchi ALLOC_TABLE_1024, 613*38849194SRobert Mustacchi ALLOC_TABLE_1024, 614*38849194SRobert Mustacchi ALLOC_TABLE_1024, 615*38849194SRobert Mustacchi ALLOC_TABLE_1024, 616*38849194SRobert Mustacchi ALLOC_TABLE_1024, 617*38849194SRobert Mustacchi ALLOC_TABLE_1024, 618*38849194SRobert Mustacchi ALLOC_TABLE_1024, 619*38849194SRobert Mustacchi ALLOC_TABLE_1024, 6207c478bd9Sstevel@tonic-gate ALLOC_TABLE_1024, 6217c478bd9Sstevel@tonic-gate ALLOC_TABLE_1024 6227c478bd9Sstevel@tonic-gate }; 6237c478bd9Sstevel@tonic-gate 6247c478bd9Sstevel@tonic-gate 6257c478bd9Sstevel@tonic-gate /* Used to constrain audit-log stack traces */ 6267c478bd9Sstevel@tonic-gate caddr_t umem_min_stack; 6277c478bd9Sstevel@tonic-gate caddr_t umem_max_stack; 6287c478bd9Sstevel@tonic-gate 6297c478bd9Sstevel@tonic-gate 6307c478bd9Sstevel@tonic-gate #define UMERR_MODIFIED 0 /* buffer modified while on freelist */ 6317c478bd9Sstevel@tonic-gate #define UMERR_REDZONE 1 /* redzone violation (write past end of buf) */ 6327c478bd9Sstevel@tonic-gate #define UMERR_DUPFREE 2 /* freed a buffer twice */ 6337c478bd9Sstevel@tonic-gate #define UMERR_BADADDR 3 /* freed a bad (unallocated) address */ 6347c478bd9Sstevel@tonic-gate #define UMERR_BADBUFTAG 4 /* buftag corrupted */ 6357c478bd9Sstevel@tonic-gate #define UMERR_BADBUFCTL 5 /* bufctl corrupted */ 6367c478bd9Sstevel@tonic-gate #define UMERR_BADCACHE 6 /* freed a buffer to the wrong cache */ 6377c478bd9Sstevel@tonic-gate #define UMERR_BADSIZE 7 /* alloc size != free size */ 6387c478bd9Sstevel@tonic-gate #define UMERR_BADBASE 8 /* buffer base address wrong */ 6397c478bd9Sstevel@tonic-gate 6407c478bd9Sstevel@tonic-gate struct { 6417c478bd9Sstevel@tonic-gate hrtime_t ump_timestamp; /* timestamp of error */ 6427c478bd9Sstevel@tonic-gate int ump_error; /* type of umem error (UMERR_*) */ 6437c478bd9Sstevel@tonic-gate void *ump_buffer; /* buffer that induced abort */ 6447c478bd9Sstevel@tonic-gate void *ump_realbuf; /* real start address for buffer */ 6457c478bd9Sstevel@tonic-gate umem_cache_t *ump_cache; /* buffer's cache according to client */ 6467c478bd9Sstevel@tonic-gate umem_cache_t *ump_realcache; /* actual cache containing buffer */ 6477c478bd9Sstevel@tonic-gate umem_slab_t *ump_slab; /* slab accoring to umem_findslab() */ 6487c478bd9Sstevel@tonic-gate umem_bufctl_t *ump_bufctl; /* bufctl */ 6497c478bd9Sstevel@tonic-gate } umem_abort_info; 6507c478bd9Sstevel@tonic-gate 6517c478bd9Sstevel@tonic-gate static void 6527c478bd9Sstevel@tonic-gate copy_pattern(uint64_t pattern, void *buf_arg, size_t size) 6537c478bd9Sstevel@tonic-gate { 6547c478bd9Sstevel@tonic-gate uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 6557c478bd9Sstevel@tonic-gate uint64_t *buf = buf_arg; 6567c478bd9Sstevel@tonic-gate 6577c478bd9Sstevel@tonic-gate while (buf < bufend) 6587c478bd9Sstevel@tonic-gate *buf++ = pattern; 6597c478bd9Sstevel@tonic-gate } 6607c478bd9Sstevel@tonic-gate 6617c478bd9Sstevel@tonic-gate static void * 6627c478bd9Sstevel@tonic-gate verify_pattern(uint64_t pattern, void *buf_arg, size_t size) 6637c478bd9Sstevel@tonic-gate { 6647c478bd9Sstevel@tonic-gate uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 6657c478bd9Sstevel@tonic-gate uint64_t *buf; 6667c478bd9Sstevel@tonic-gate 6677c478bd9Sstevel@tonic-gate for (buf = buf_arg; buf < bufend; buf++) 6687c478bd9Sstevel@tonic-gate if (*buf != pattern) 6697c478bd9Sstevel@tonic-gate return (buf); 6707c478bd9Sstevel@tonic-gate return (NULL); 6717c478bd9Sstevel@tonic-gate } 6727c478bd9Sstevel@tonic-gate 6737c478bd9Sstevel@tonic-gate static void * 6747c478bd9Sstevel@tonic-gate verify_and_copy_pattern(uint64_t old, uint64_t new, void *buf_arg, size_t size) 6757c478bd9Sstevel@tonic-gate { 6767c478bd9Sstevel@tonic-gate uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 6777c478bd9Sstevel@tonic-gate uint64_t *buf; 6787c478bd9Sstevel@tonic-gate 6797c478bd9Sstevel@tonic-gate for (buf = buf_arg; buf < bufend; buf++) { 6807c478bd9Sstevel@tonic-gate if (*buf != old) { 6817c478bd9Sstevel@tonic-gate copy_pattern(old, buf_arg, 6827c478bd9Sstevel@tonic-gate (char *)buf - (char *)buf_arg); 6837c478bd9Sstevel@tonic-gate return (buf); 6847c478bd9Sstevel@tonic-gate } 6857c478bd9Sstevel@tonic-gate *buf = new; 6867c478bd9Sstevel@tonic-gate } 6877c478bd9Sstevel@tonic-gate 6887c478bd9Sstevel@tonic-gate return (NULL); 6897c478bd9Sstevel@tonic-gate } 6907c478bd9Sstevel@tonic-gate 6917c478bd9Sstevel@tonic-gate void 6927c478bd9Sstevel@tonic-gate umem_cache_applyall(void (*func)(umem_cache_t *)) 6937c478bd9Sstevel@tonic-gate { 6947c478bd9Sstevel@tonic-gate umem_cache_t *cp; 6957c478bd9Sstevel@tonic-gate 6967c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_cache_lock); 6977c478bd9Sstevel@tonic-gate for (cp = umem_null_cache.cache_next; cp != &umem_null_cache; 6987c478bd9Sstevel@tonic-gate cp = cp->cache_next) 6997c478bd9Sstevel@tonic-gate func(cp); 7007c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_cache_lock); 7017c478bd9Sstevel@tonic-gate } 7027c478bd9Sstevel@tonic-gate 7037c478bd9Sstevel@tonic-gate static void 7047c478bd9Sstevel@tonic-gate umem_add_update_unlocked(umem_cache_t *cp, int flags) 7057c478bd9Sstevel@tonic-gate { 7067c478bd9Sstevel@tonic-gate umem_cache_t *cnext, *cprev; 7077c478bd9Sstevel@tonic-gate 7087c478bd9Sstevel@tonic-gate flags &= ~UMU_ACTIVE; 7097c478bd9Sstevel@tonic-gate 7107c478bd9Sstevel@tonic-gate if (!flags) 7117c478bd9Sstevel@tonic-gate return; 7127c478bd9Sstevel@tonic-gate 7137c478bd9Sstevel@tonic-gate if (cp->cache_uflags & UMU_ACTIVE) { 7147c478bd9Sstevel@tonic-gate cp->cache_uflags |= flags; 7157c478bd9Sstevel@tonic-gate } else { 7167c478bd9Sstevel@tonic-gate if (cp->cache_unext != NULL) { 7177c478bd9Sstevel@tonic-gate ASSERT(cp->cache_uflags != 0); 7187c478bd9Sstevel@tonic-gate cp->cache_uflags |= flags; 7197c478bd9Sstevel@tonic-gate } else { 7207c478bd9Sstevel@tonic-gate ASSERT(cp->cache_uflags == 0); 7217c478bd9Sstevel@tonic-gate cp->cache_uflags = flags; 7227c478bd9Sstevel@tonic-gate cp->cache_unext = cnext = &umem_null_cache; 7237c478bd9Sstevel@tonic-gate cp->cache_uprev = cprev = umem_null_cache.cache_uprev; 7247c478bd9Sstevel@tonic-gate cnext->cache_uprev = cp; 7257c478bd9Sstevel@tonic-gate cprev->cache_unext = cp; 7267c478bd9Sstevel@tonic-gate } 7277c478bd9Sstevel@tonic-gate } 7287c478bd9Sstevel@tonic-gate } 7297c478bd9Sstevel@tonic-gate 7307c478bd9Sstevel@tonic-gate static void 7317c478bd9Sstevel@tonic-gate umem_add_update(umem_cache_t *cp, int flags) 7327c478bd9Sstevel@tonic-gate { 7337c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_update_lock); 7347c478bd9Sstevel@tonic-gate 7357c478bd9Sstevel@tonic-gate umem_add_update_unlocked(cp, flags); 7367c478bd9Sstevel@tonic-gate 7377c478bd9Sstevel@tonic-gate if (!IN_UPDATE()) 7387c478bd9Sstevel@tonic-gate (void) cond_broadcast(&umem_update_cv); 7397c478bd9Sstevel@tonic-gate 7407c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_update_lock); 7417c478bd9Sstevel@tonic-gate } 7427c478bd9Sstevel@tonic-gate 7437c478bd9Sstevel@tonic-gate /* 7447c478bd9Sstevel@tonic-gate * Remove a cache from the update list, waiting for any in-progress work to 7457c478bd9Sstevel@tonic-gate * complete first. 7467c478bd9Sstevel@tonic-gate */ 7477c478bd9Sstevel@tonic-gate static void 7487c478bd9Sstevel@tonic-gate umem_remove_updates(umem_cache_t *cp) 7497c478bd9Sstevel@tonic-gate { 7507c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_update_lock); 7517c478bd9Sstevel@tonic-gate 7527c478bd9Sstevel@tonic-gate /* 7537c478bd9Sstevel@tonic-gate * Get it out of the active state 7547c478bd9Sstevel@tonic-gate */ 7557c478bd9Sstevel@tonic-gate while (cp->cache_uflags & UMU_ACTIVE) { 756a574db85Sraf int cancel_state; 757a574db85Sraf 7587c478bd9Sstevel@tonic-gate ASSERT(cp->cache_unext == NULL); 7597c478bd9Sstevel@tonic-gate 7607c478bd9Sstevel@tonic-gate cp->cache_uflags |= UMU_NOTIFY; 7617c478bd9Sstevel@tonic-gate 7627c478bd9Sstevel@tonic-gate /* 7637c478bd9Sstevel@tonic-gate * Make sure the update state is sane, before we wait 7647c478bd9Sstevel@tonic-gate */ 7657c478bd9Sstevel@tonic-gate ASSERT(umem_update_thr != 0 || umem_st_update_thr != 0); 7667c478bd9Sstevel@tonic-gate ASSERT(umem_update_thr != thr_self() && 7677c478bd9Sstevel@tonic-gate umem_st_update_thr != thr_self()); 7687c478bd9Sstevel@tonic-gate 769a574db85Sraf (void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, 770a574db85Sraf &cancel_state); 771a574db85Sraf (void) cond_wait(&umem_update_cv, &umem_update_lock); 772a574db85Sraf (void) pthread_setcancelstate(cancel_state, NULL); 7737c478bd9Sstevel@tonic-gate } 7747c478bd9Sstevel@tonic-gate /* 7757c478bd9Sstevel@tonic-gate * Get it out of the Work Requested state 7767c478bd9Sstevel@tonic-gate */ 7777c478bd9Sstevel@tonic-gate if (cp->cache_unext != NULL) { 7787c478bd9Sstevel@tonic-gate cp->cache_uprev->cache_unext = cp->cache_unext; 7797c478bd9Sstevel@tonic-gate cp->cache_unext->cache_uprev = cp->cache_uprev; 7807c478bd9Sstevel@tonic-gate cp->cache_uprev = cp->cache_unext = NULL; 7817c478bd9Sstevel@tonic-gate cp->cache_uflags = 0; 7827c478bd9Sstevel@tonic-gate } 7837c478bd9Sstevel@tonic-gate /* 7847c478bd9Sstevel@tonic-gate * Make sure it is in the Inactive state 7857c478bd9Sstevel@tonic-gate */ 7867c478bd9Sstevel@tonic-gate ASSERT(cp->cache_unext == NULL && cp->cache_uflags == 0); 7877c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_update_lock); 7887c478bd9Sstevel@tonic-gate } 7897c478bd9Sstevel@tonic-gate 7907c478bd9Sstevel@tonic-gate static void 7917c478bd9Sstevel@tonic-gate umem_updateall(int flags) 7927c478bd9Sstevel@tonic-gate { 7937c478bd9Sstevel@tonic-gate umem_cache_t *cp; 7947c478bd9Sstevel@tonic-gate 7957c478bd9Sstevel@tonic-gate /* 7967c478bd9Sstevel@tonic-gate * NOTE: To prevent deadlock, umem_cache_lock is always acquired first. 7977c478bd9Sstevel@tonic-gate * 7987c478bd9Sstevel@tonic-gate * (umem_add_update is called from things run via umem_cache_applyall) 7997c478bd9Sstevel@tonic-gate */ 8007c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_cache_lock); 8017c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_update_lock); 8027c478bd9Sstevel@tonic-gate 8037c478bd9Sstevel@tonic-gate for (cp = umem_null_cache.cache_next; cp != &umem_null_cache; 8047c478bd9Sstevel@tonic-gate cp = cp->cache_next) 8057c478bd9Sstevel@tonic-gate umem_add_update_unlocked(cp, flags); 8067c478bd9Sstevel@tonic-gate 8077c478bd9Sstevel@tonic-gate if (!IN_UPDATE()) 8087c478bd9Sstevel@tonic-gate (void) cond_broadcast(&umem_update_cv); 8097c478bd9Sstevel@tonic-gate 8107c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_update_lock); 8117c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_cache_lock); 8127c478bd9Sstevel@tonic-gate } 8137c478bd9Sstevel@tonic-gate 8147c478bd9Sstevel@tonic-gate /* 8157c478bd9Sstevel@tonic-gate * Debugging support. Given a buffer address, find its slab. 8167c478bd9Sstevel@tonic-gate */ 8177c478bd9Sstevel@tonic-gate static umem_slab_t * 8187c478bd9Sstevel@tonic-gate umem_findslab(umem_cache_t *cp, void *buf) 8197c478bd9Sstevel@tonic-gate { 8207c478bd9Sstevel@tonic-gate umem_slab_t *sp; 8217c478bd9Sstevel@tonic-gate 8227c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_lock); 8237c478bd9Sstevel@tonic-gate for (sp = cp->cache_nullslab.slab_next; 8247c478bd9Sstevel@tonic-gate sp != &cp->cache_nullslab; sp = sp->slab_next) { 8257c478bd9Sstevel@tonic-gate if (UMEM_SLAB_MEMBER(sp, buf)) { 8267c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 8277c478bd9Sstevel@tonic-gate return (sp); 8287c478bd9Sstevel@tonic-gate } 8297c478bd9Sstevel@tonic-gate } 8307c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 8317c478bd9Sstevel@tonic-gate 8327c478bd9Sstevel@tonic-gate return (NULL); 8337c478bd9Sstevel@tonic-gate } 8347c478bd9Sstevel@tonic-gate 8357c478bd9Sstevel@tonic-gate static void 8367c478bd9Sstevel@tonic-gate umem_error(int error, umem_cache_t *cparg, void *bufarg) 8377c478bd9Sstevel@tonic-gate { 8387c478bd9Sstevel@tonic-gate umem_buftag_t *btp = NULL; 8397c478bd9Sstevel@tonic-gate umem_bufctl_t *bcp = NULL; 8407c478bd9Sstevel@tonic-gate umem_cache_t *cp = cparg; 8417c478bd9Sstevel@tonic-gate umem_slab_t *sp; 8427c478bd9Sstevel@tonic-gate uint64_t *off; 8437c478bd9Sstevel@tonic-gate void *buf = bufarg; 8447c478bd9Sstevel@tonic-gate 8457c478bd9Sstevel@tonic-gate int old_logging = umem_logging; 8467c478bd9Sstevel@tonic-gate 8477c478bd9Sstevel@tonic-gate umem_logging = 0; /* stop logging when a bad thing happens */ 8487c478bd9Sstevel@tonic-gate 8497c478bd9Sstevel@tonic-gate umem_abort_info.ump_timestamp = gethrtime(); 8507c478bd9Sstevel@tonic-gate 8517c478bd9Sstevel@tonic-gate sp = umem_findslab(cp, buf); 8527c478bd9Sstevel@tonic-gate if (sp == NULL) { 8537c478bd9Sstevel@tonic-gate for (cp = umem_null_cache.cache_prev; cp != &umem_null_cache; 8547c478bd9Sstevel@tonic-gate cp = cp->cache_prev) { 8557c478bd9Sstevel@tonic-gate if ((sp = umem_findslab(cp, buf)) != NULL) 8567c478bd9Sstevel@tonic-gate break; 8577c478bd9Sstevel@tonic-gate } 8587c478bd9Sstevel@tonic-gate } 8597c478bd9Sstevel@tonic-gate 8607c478bd9Sstevel@tonic-gate if (sp == NULL) { 8617c478bd9Sstevel@tonic-gate cp = NULL; 8627c478bd9Sstevel@tonic-gate error = UMERR_BADADDR; 8637c478bd9Sstevel@tonic-gate } else { 8647c478bd9Sstevel@tonic-gate if (cp != cparg) 8657c478bd9Sstevel@tonic-gate error = UMERR_BADCACHE; 8667c478bd9Sstevel@tonic-gate else 8677c478bd9Sstevel@tonic-gate buf = (char *)bufarg - ((uintptr_t)bufarg - 8687c478bd9Sstevel@tonic-gate (uintptr_t)sp->slab_base) % cp->cache_chunksize; 8697c478bd9Sstevel@tonic-gate if (buf != bufarg) 8707c478bd9Sstevel@tonic-gate error = UMERR_BADBASE; 8717c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_BUFTAG) 8727c478bd9Sstevel@tonic-gate btp = UMEM_BUFTAG(cp, buf); 8737c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_HASH) { 8747c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_lock); 8757c478bd9Sstevel@tonic-gate for (bcp = *UMEM_HASH(cp, buf); bcp; bcp = bcp->bc_next) 8767c478bd9Sstevel@tonic-gate if (bcp->bc_addr == buf) 8777c478bd9Sstevel@tonic-gate break; 8787c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 8797c478bd9Sstevel@tonic-gate if (bcp == NULL && btp != NULL) 8807c478bd9Sstevel@tonic-gate bcp = btp->bt_bufctl; 8817c478bd9Sstevel@tonic-gate if (umem_findslab(cp->cache_bufctl_cache, bcp) == 8827c478bd9Sstevel@tonic-gate NULL || P2PHASE((uintptr_t)bcp, UMEM_ALIGN) || 8837c478bd9Sstevel@tonic-gate bcp->bc_addr != buf) { 8847c478bd9Sstevel@tonic-gate error = UMERR_BADBUFCTL; 8857c478bd9Sstevel@tonic-gate bcp = NULL; 8867c478bd9Sstevel@tonic-gate } 8877c478bd9Sstevel@tonic-gate } 8887c478bd9Sstevel@tonic-gate } 8897c478bd9Sstevel@tonic-gate 8907c478bd9Sstevel@tonic-gate umem_abort_info.ump_error = error; 8917c478bd9Sstevel@tonic-gate umem_abort_info.ump_buffer = bufarg; 8927c478bd9Sstevel@tonic-gate umem_abort_info.ump_realbuf = buf; 8937c478bd9Sstevel@tonic-gate umem_abort_info.ump_cache = cparg; 8947c478bd9Sstevel@tonic-gate umem_abort_info.ump_realcache = cp; 8957c478bd9Sstevel@tonic-gate umem_abort_info.ump_slab = sp; 8967c478bd9Sstevel@tonic-gate umem_abort_info.ump_bufctl = bcp; 8977c478bd9Sstevel@tonic-gate 8987c478bd9Sstevel@tonic-gate umem_printf("umem allocator: "); 8997c478bd9Sstevel@tonic-gate 9007c478bd9Sstevel@tonic-gate switch (error) { 9017c478bd9Sstevel@tonic-gate 9027c478bd9Sstevel@tonic-gate case UMERR_MODIFIED: 9037c478bd9Sstevel@tonic-gate umem_printf("buffer modified after being freed\n"); 9047c478bd9Sstevel@tonic-gate off = verify_pattern(UMEM_FREE_PATTERN, buf, cp->cache_verify); 9057c478bd9Sstevel@tonic-gate if (off == NULL) /* shouldn't happen */ 9067c478bd9Sstevel@tonic-gate off = buf; 9077c478bd9Sstevel@tonic-gate umem_printf("modification occurred at offset 0x%lx " 9087c478bd9Sstevel@tonic-gate "(0x%llx replaced by 0x%llx)\n", 9097c478bd9Sstevel@tonic-gate (uintptr_t)off - (uintptr_t)buf, 9107c478bd9Sstevel@tonic-gate (longlong_t)UMEM_FREE_PATTERN, (longlong_t)*off); 9117c478bd9Sstevel@tonic-gate break; 9127c478bd9Sstevel@tonic-gate 9137c478bd9Sstevel@tonic-gate case UMERR_REDZONE: 9147c478bd9Sstevel@tonic-gate umem_printf("redzone violation: write past end of buffer\n"); 9157c478bd9Sstevel@tonic-gate break; 9167c478bd9Sstevel@tonic-gate 9177c478bd9Sstevel@tonic-gate case UMERR_BADADDR: 9187c478bd9Sstevel@tonic-gate umem_printf("invalid free: buffer not in cache\n"); 9197c478bd9Sstevel@tonic-gate break; 9207c478bd9Sstevel@tonic-gate 9217c478bd9Sstevel@tonic-gate case UMERR_DUPFREE: 9227c478bd9Sstevel@tonic-gate umem_printf("duplicate free: buffer freed twice\n"); 9237c478bd9Sstevel@tonic-gate break; 9247c478bd9Sstevel@tonic-gate 9257c478bd9Sstevel@tonic-gate case UMERR_BADBUFTAG: 9267c478bd9Sstevel@tonic-gate umem_printf("boundary tag corrupted\n"); 9277c478bd9Sstevel@tonic-gate umem_printf("bcp ^ bxstat = %lx, should be %lx\n", 9287c478bd9Sstevel@tonic-gate (intptr_t)btp->bt_bufctl ^ btp->bt_bxstat, 9297c478bd9Sstevel@tonic-gate UMEM_BUFTAG_FREE); 9307c478bd9Sstevel@tonic-gate break; 9317c478bd9Sstevel@tonic-gate 9327c478bd9Sstevel@tonic-gate case UMERR_BADBUFCTL: 9337c478bd9Sstevel@tonic-gate umem_printf("bufctl corrupted\n"); 9347c478bd9Sstevel@tonic-gate break; 9357c478bd9Sstevel@tonic-gate 9367c478bd9Sstevel@tonic-gate case UMERR_BADCACHE: 9377c478bd9Sstevel@tonic-gate umem_printf("buffer freed to wrong cache\n"); 9387c478bd9Sstevel@tonic-gate umem_printf("buffer was allocated from %s,\n", cp->cache_name); 9397c478bd9Sstevel@tonic-gate umem_printf("caller attempting free to %s.\n", 9407c478bd9Sstevel@tonic-gate cparg->cache_name); 9417c478bd9Sstevel@tonic-gate break; 9427c478bd9Sstevel@tonic-gate 9437c478bd9Sstevel@tonic-gate case UMERR_BADSIZE: 9447c478bd9Sstevel@tonic-gate umem_printf("bad free: free size (%u) != alloc size (%u)\n", 9457c478bd9Sstevel@tonic-gate UMEM_SIZE_DECODE(((uint32_t *)btp)[0]), 9467c478bd9Sstevel@tonic-gate UMEM_SIZE_DECODE(((uint32_t *)btp)[1])); 9477c478bd9Sstevel@tonic-gate break; 9487c478bd9Sstevel@tonic-gate 9497c478bd9Sstevel@tonic-gate case UMERR_BADBASE: 9507c478bd9Sstevel@tonic-gate umem_printf("bad free: free address (%p) != alloc address " 9517c478bd9Sstevel@tonic-gate "(%p)\n", bufarg, buf); 9527c478bd9Sstevel@tonic-gate break; 9537c478bd9Sstevel@tonic-gate } 9547c478bd9Sstevel@tonic-gate 9557c478bd9Sstevel@tonic-gate umem_printf("buffer=%p bufctl=%p cache: %s\n", 9567c478bd9Sstevel@tonic-gate bufarg, (void *)bcp, cparg->cache_name); 9577c478bd9Sstevel@tonic-gate 9587c478bd9Sstevel@tonic-gate if (bcp != NULL && (cp->cache_flags & UMF_AUDIT) && 9597c478bd9Sstevel@tonic-gate error != UMERR_BADBUFCTL) { 9607c478bd9Sstevel@tonic-gate int d; 9617c478bd9Sstevel@tonic-gate timespec_t ts; 9627c478bd9Sstevel@tonic-gate hrtime_t diff; 9637c478bd9Sstevel@tonic-gate umem_bufctl_audit_t *bcap = (umem_bufctl_audit_t *)bcp; 9647c478bd9Sstevel@tonic-gate 9657c478bd9Sstevel@tonic-gate diff = umem_abort_info.ump_timestamp - bcap->bc_timestamp; 9667c478bd9Sstevel@tonic-gate ts.tv_sec = diff / NANOSEC; 9677c478bd9Sstevel@tonic-gate ts.tv_nsec = diff % NANOSEC; 9687c478bd9Sstevel@tonic-gate 9697c478bd9Sstevel@tonic-gate umem_printf("previous transaction on buffer %p:\n", buf); 9707c478bd9Sstevel@tonic-gate umem_printf("thread=%p time=T-%ld.%09ld slab=%p cache: %s\n", 9717c478bd9Sstevel@tonic-gate (void *)(intptr_t)bcap->bc_thread, ts.tv_sec, ts.tv_nsec, 9727c478bd9Sstevel@tonic-gate (void *)sp, cp->cache_name); 9737c478bd9Sstevel@tonic-gate for (d = 0; d < MIN(bcap->bc_depth, umem_stack_depth); d++) { 9747c478bd9Sstevel@tonic-gate (void) print_sym((void *)bcap->bc_stack[d]); 9757c478bd9Sstevel@tonic-gate umem_printf("\n"); 9767c478bd9Sstevel@tonic-gate } 9777c478bd9Sstevel@tonic-gate } 9787c478bd9Sstevel@tonic-gate 9797c478bd9Sstevel@tonic-gate umem_err_recoverable("umem: heap corruption detected"); 9807c478bd9Sstevel@tonic-gate 9817c478bd9Sstevel@tonic-gate umem_logging = old_logging; /* resume logging */ 9827c478bd9Sstevel@tonic-gate } 9837c478bd9Sstevel@tonic-gate 9847c478bd9Sstevel@tonic-gate void 9857c478bd9Sstevel@tonic-gate umem_nofail_callback(umem_nofail_callback_t *cb) 9867c478bd9Sstevel@tonic-gate { 9877c478bd9Sstevel@tonic-gate nofail_callback = cb; 9887c478bd9Sstevel@tonic-gate } 9897c478bd9Sstevel@tonic-gate 9907c478bd9Sstevel@tonic-gate static int 9917c478bd9Sstevel@tonic-gate umem_alloc_retry(umem_cache_t *cp, int umflag) 9927c478bd9Sstevel@tonic-gate { 9937c478bd9Sstevel@tonic-gate if (cp == &umem_null_cache) { 9947c478bd9Sstevel@tonic-gate if (umem_init()) 9957c478bd9Sstevel@tonic-gate return (1); /* retry */ 9967c478bd9Sstevel@tonic-gate /* 9977c478bd9Sstevel@tonic-gate * Initialization failed. Do normal failure processing. 9987c478bd9Sstevel@tonic-gate */ 9997c478bd9Sstevel@tonic-gate } 10007c478bd9Sstevel@tonic-gate if (umflag & UMEM_NOFAIL) { 10017c478bd9Sstevel@tonic-gate int def_result = UMEM_CALLBACK_EXIT(255); 10027c478bd9Sstevel@tonic-gate int result = def_result; 10037c478bd9Sstevel@tonic-gate umem_nofail_callback_t *callback = nofail_callback; 10047c478bd9Sstevel@tonic-gate 10057c478bd9Sstevel@tonic-gate if (callback != NULL) 10067c478bd9Sstevel@tonic-gate result = callback(); 10077c478bd9Sstevel@tonic-gate 10087c478bd9Sstevel@tonic-gate if (result == UMEM_CALLBACK_RETRY) 10097c478bd9Sstevel@tonic-gate return (1); 10107c478bd9Sstevel@tonic-gate 10117c478bd9Sstevel@tonic-gate if ((result & ~0xFF) != UMEM_CALLBACK_EXIT(0)) { 10127c478bd9Sstevel@tonic-gate log_message("nofail callback returned %x\n", result); 10137c478bd9Sstevel@tonic-gate result = def_result; 10147c478bd9Sstevel@tonic-gate } 10157c478bd9Sstevel@tonic-gate 10167c478bd9Sstevel@tonic-gate /* 10177c478bd9Sstevel@tonic-gate * only one thread will call exit 10187c478bd9Sstevel@tonic-gate */ 10197c478bd9Sstevel@tonic-gate if (umem_nofail_exit_thr == thr_self()) 10207c478bd9Sstevel@tonic-gate umem_panic("recursive UMEM_CALLBACK_EXIT()\n"); 10217c478bd9Sstevel@tonic-gate 10227c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_nofail_exit_lock); 10237c478bd9Sstevel@tonic-gate umem_nofail_exit_thr = thr_self(); 10247c478bd9Sstevel@tonic-gate exit(result & 0xFF); 10257c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 10267c478bd9Sstevel@tonic-gate } 10277c478bd9Sstevel@tonic-gate return (0); 10287c478bd9Sstevel@tonic-gate } 10297c478bd9Sstevel@tonic-gate 10307c478bd9Sstevel@tonic-gate static umem_log_header_t * 10317c478bd9Sstevel@tonic-gate umem_log_init(size_t logsize) 10327c478bd9Sstevel@tonic-gate { 10337c478bd9Sstevel@tonic-gate umem_log_header_t *lhp; 10347c478bd9Sstevel@tonic-gate int nchunks = 4 * umem_max_ncpus; 10357c478bd9Sstevel@tonic-gate size_t lhsize = offsetof(umem_log_header_t, lh_cpu[umem_max_ncpus]); 10367c478bd9Sstevel@tonic-gate int i; 10377c478bd9Sstevel@tonic-gate 10387c478bd9Sstevel@tonic-gate if (logsize == 0) 10397c478bd9Sstevel@tonic-gate return (NULL); 10407c478bd9Sstevel@tonic-gate 10417c478bd9Sstevel@tonic-gate /* 10427c478bd9Sstevel@tonic-gate * Make sure that lhp->lh_cpu[] is nicely aligned 10437c478bd9Sstevel@tonic-gate * to prevent false sharing of cache lines. 10447c478bd9Sstevel@tonic-gate */ 10457c478bd9Sstevel@tonic-gate lhsize = P2ROUNDUP(lhsize, UMEM_ALIGN); 10467c478bd9Sstevel@tonic-gate lhp = vmem_xalloc(umem_log_arena, lhsize, 64, P2NPHASE(lhsize, 64), 0, 10477c478bd9Sstevel@tonic-gate NULL, NULL, VM_NOSLEEP); 10487c478bd9Sstevel@tonic-gate if (lhp == NULL) 10497c478bd9Sstevel@tonic-gate goto fail; 10507c478bd9Sstevel@tonic-gate 10517c478bd9Sstevel@tonic-gate bzero(lhp, lhsize); 10527c478bd9Sstevel@tonic-gate 10537c478bd9Sstevel@tonic-gate (void) mutex_init(&lhp->lh_lock, USYNC_THREAD, NULL); 10547c478bd9Sstevel@tonic-gate lhp->lh_nchunks = nchunks; 10557c478bd9Sstevel@tonic-gate lhp->lh_chunksize = P2ROUNDUP(logsize / nchunks, PAGESIZE); 10567c478bd9Sstevel@tonic-gate if (lhp->lh_chunksize == 0) 10577c478bd9Sstevel@tonic-gate lhp->lh_chunksize = PAGESIZE; 10587c478bd9Sstevel@tonic-gate 10597c478bd9Sstevel@tonic-gate lhp->lh_base = vmem_alloc(umem_log_arena, 10607c478bd9Sstevel@tonic-gate lhp->lh_chunksize * nchunks, VM_NOSLEEP); 10617c478bd9Sstevel@tonic-gate if (lhp->lh_base == NULL) 10627c478bd9Sstevel@tonic-gate goto fail; 10637c478bd9Sstevel@tonic-gate 10647c478bd9Sstevel@tonic-gate lhp->lh_free = vmem_alloc(umem_log_arena, 10657c478bd9Sstevel@tonic-gate nchunks * sizeof (int), VM_NOSLEEP); 10667c478bd9Sstevel@tonic-gate if (lhp->lh_free == NULL) 10677c478bd9Sstevel@tonic-gate goto fail; 10687c478bd9Sstevel@tonic-gate 10697c478bd9Sstevel@tonic-gate bzero(lhp->lh_base, lhp->lh_chunksize * nchunks); 10707c478bd9Sstevel@tonic-gate 10717c478bd9Sstevel@tonic-gate for (i = 0; i < umem_max_ncpus; i++) { 10727c478bd9Sstevel@tonic-gate umem_cpu_log_header_t *clhp = &lhp->lh_cpu[i]; 10737c478bd9Sstevel@tonic-gate (void) mutex_init(&clhp->clh_lock, USYNC_THREAD, NULL); 10747c478bd9Sstevel@tonic-gate clhp->clh_chunk = i; 10757c478bd9Sstevel@tonic-gate } 10767c478bd9Sstevel@tonic-gate 10777c478bd9Sstevel@tonic-gate for (i = umem_max_ncpus; i < nchunks; i++) 10787c478bd9Sstevel@tonic-gate lhp->lh_free[i] = i; 10797c478bd9Sstevel@tonic-gate 10807c478bd9Sstevel@tonic-gate lhp->lh_head = umem_max_ncpus; 10817c478bd9Sstevel@tonic-gate lhp->lh_tail = 0; 10827c478bd9Sstevel@tonic-gate 10837c478bd9Sstevel@tonic-gate return (lhp); 10847c478bd9Sstevel@tonic-gate 10857c478bd9Sstevel@tonic-gate fail: 10867c478bd9Sstevel@tonic-gate if (lhp != NULL) { 10877c478bd9Sstevel@tonic-gate if (lhp->lh_base != NULL) 10887c478bd9Sstevel@tonic-gate vmem_free(umem_log_arena, lhp->lh_base, 10897c478bd9Sstevel@tonic-gate lhp->lh_chunksize * nchunks); 10907c478bd9Sstevel@tonic-gate 10917c478bd9Sstevel@tonic-gate vmem_xfree(umem_log_arena, lhp, lhsize); 10927c478bd9Sstevel@tonic-gate } 10937c478bd9Sstevel@tonic-gate return (NULL); 10947c478bd9Sstevel@tonic-gate } 10957c478bd9Sstevel@tonic-gate 10967c478bd9Sstevel@tonic-gate static void * 10977c478bd9Sstevel@tonic-gate umem_log_enter(umem_log_header_t *lhp, void *data, size_t size) 10987c478bd9Sstevel@tonic-gate { 10997c478bd9Sstevel@tonic-gate void *logspace; 11007c478bd9Sstevel@tonic-gate umem_cpu_log_header_t *clhp = 11017c478bd9Sstevel@tonic-gate &lhp->lh_cpu[CPU(umem_cpu_mask)->cpu_number]; 11027c478bd9Sstevel@tonic-gate 11037c478bd9Sstevel@tonic-gate if (lhp == NULL || umem_logging == 0) 11047c478bd9Sstevel@tonic-gate return (NULL); 11057c478bd9Sstevel@tonic-gate 11067c478bd9Sstevel@tonic-gate (void) mutex_lock(&clhp->clh_lock); 11077c478bd9Sstevel@tonic-gate clhp->clh_hits++; 11087c478bd9Sstevel@tonic-gate if (size > clhp->clh_avail) { 11097c478bd9Sstevel@tonic-gate (void) mutex_lock(&lhp->lh_lock); 11107c478bd9Sstevel@tonic-gate lhp->lh_hits++; 11117c478bd9Sstevel@tonic-gate lhp->lh_free[lhp->lh_tail] = clhp->clh_chunk; 11127c478bd9Sstevel@tonic-gate lhp->lh_tail = (lhp->lh_tail + 1) % lhp->lh_nchunks; 11137c478bd9Sstevel@tonic-gate clhp->clh_chunk = lhp->lh_free[lhp->lh_head]; 11147c478bd9Sstevel@tonic-gate lhp->lh_head = (lhp->lh_head + 1) % lhp->lh_nchunks; 11157c478bd9Sstevel@tonic-gate clhp->clh_current = lhp->lh_base + 11167c478bd9Sstevel@tonic-gate clhp->clh_chunk * lhp->lh_chunksize; 11177c478bd9Sstevel@tonic-gate clhp->clh_avail = lhp->lh_chunksize; 11187c478bd9Sstevel@tonic-gate if (size > lhp->lh_chunksize) 11197c478bd9Sstevel@tonic-gate size = lhp->lh_chunksize; 11207c478bd9Sstevel@tonic-gate (void) mutex_unlock(&lhp->lh_lock); 11217c478bd9Sstevel@tonic-gate } 11227c478bd9Sstevel@tonic-gate logspace = clhp->clh_current; 11237c478bd9Sstevel@tonic-gate clhp->clh_current += size; 11247c478bd9Sstevel@tonic-gate clhp->clh_avail -= size; 11257c478bd9Sstevel@tonic-gate bcopy(data, logspace, size); 11267c478bd9Sstevel@tonic-gate (void) mutex_unlock(&clhp->clh_lock); 11277c478bd9Sstevel@tonic-gate return (logspace); 11287c478bd9Sstevel@tonic-gate } 11297c478bd9Sstevel@tonic-gate 11307c478bd9Sstevel@tonic-gate #define UMEM_AUDIT(lp, cp, bcp) \ 11317c478bd9Sstevel@tonic-gate { \ 11327c478bd9Sstevel@tonic-gate umem_bufctl_audit_t *_bcp = (umem_bufctl_audit_t *)(bcp); \ 11337c478bd9Sstevel@tonic-gate _bcp->bc_timestamp = gethrtime(); \ 11347c478bd9Sstevel@tonic-gate _bcp->bc_thread = thr_self(); \ 11357c478bd9Sstevel@tonic-gate _bcp->bc_depth = getpcstack(_bcp->bc_stack, umem_stack_depth, \ 11367c478bd9Sstevel@tonic-gate (cp != NULL) && (cp->cache_flags & UMF_CHECKSIGNAL)); \ 11377c478bd9Sstevel@tonic-gate _bcp->bc_lastlog = umem_log_enter((lp), _bcp, \ 11387c478bd9Sstevel@tonic-gate UMEM_BUFCTL_AUDIT_SIZE); \ 11397c478bd9Sstevel@tonic-gate } 11407c478bd9Sstevel@tonic-gate 11417c478bd9Sstevel@tonic-gate static void 11427c478bd9Sstevel@tonic-gate umem_log_event(umem_log_header_t *lp, umem_cache_t *cp, 11437c478bd9Sstevel@tonic-gate umem_slab_t *sp, void *addr) 11447c478bd9Sstevel@tonic-gate { 11457c478bd9Sstevel@tonic-gate umem_bufctl_audit_t *bcp; 11467c478bd9Sstevel@tonic-gate UMEM_LOCAL_BUFCTL_AUDIT(&bcp); 11477c478bd9Sstevel@tonic-gate 11487c478bd9Sstevel@tonic-gate bzero(bcp, UMEM_BUFCTL_AUDIT_SIZE); 11497c478bd9Sstevel@tonic-gate bcp->bc_addr = addr; 11507c478bd9Sstevel@tonic-gate bcp->bc_slab = sp; 11517c478bd9Sstevel@tonic-gate bcp->bc_cache = cp; 11527c478bd9Sstevel@tonic-gate UMEM_AUDIT(lp, cp, bcp); 11537c478bd9Sstevel@tonic-gate } 11547c478bd9Sstevel@tonic-gate 11557c478bd9Sstevel@tonic-gate /* 11567c478bd9Sstevel@tonic-gate * Create a new slab for cache cp. 11577c478bd9Sstevel@tonic-gate */ 11587c478bd9Sstevel@tonic-gate static umem_slab_t * 11597c478bd9Sstevel@tonic-gate umem_slab_create(umem_cache_t *cp, int umflag) 11607c478bd9Sstevel@tonic-gate { 11617c478bd9Sstevel@tonic-gate size_t slabsize = cp->cache_slabsize; 11627c478bd9Sstevel@tonic-gate size_t chunksize = cp->cache_chunksize; 11637c478bd9Sstevel@tonic-gate int cache_flags = cp->cache_flags; 11647c478bd9Sstevel@tonic-gate size_t color, chunks; 11657c478bd9Sstevel@tonic-gate char *buf, *slab; 11667c478bd9Sstevel@tonic-gate umem_slab_t *sp; 11677c478bd9Sstevel@tonic-gate umem_bufctl_t *bcp; 11687c478bd9Sstevel@tonic-gate vmem_t *vmp = cp->cache_arena; 11697c478bd9Sstevel@tonic-gate 11707c478bd9Sstevel@tonic-gate color = cp->cache_color + cp->cache_align; 11717c478bd9Sstevel@tonic-gate if (color > cp->cache_maxcolor) 11727c478bd9Sstevel@tonic-gate color = cp->cache_mincolor; 11737c478bd9Sstevel@tonic-gate cp->cache_color = color; 11747c478bd9Sstevel@tonic-gate 11757c478bd9Sstevel@tonic-gate slab = vmem_alloc(vmp, slabsize, UMEM_VMFLAGS(umflag)); 11767c478bd9Sstevel@tonic-gate 11777c478bd9Sstevel@tonic-gate if (slab == NULL) 11787c478bd9Sstevel@tonic-gate goto vmem_alloc_failure; 11797c478bd9Sstevel@tonic-gate 11807c478bd9Sstevel@tonic-gate ASSERT(P2PHASE((uintptr_t)slab, vmp->vm_quantum) == 0); 11817c478bd9Sstevel@tonic-gate 11827c478bd9Sstevel@tonic-gate if (!(cp->cache_cflags & UMC_NOTOUCH) && 11837c478bd9Sstevel@tonic-gate (cp->cache_flags & UMF_DEADBEEF)) 11847c478bd9Sstevel@tonic-gate copy_pattern(UMEM_UNINITIALIZED_PATTERN, slab, slabsize); 11857c478bd9Sstevel@tonic-gate 11867c478bd9Sstevel@tonic-gate if (cache_flags & UMF_HASH) { 11877c478bd9Sstevel@tonic-gate if ((sp = _umem_cache_alloc(umem_slab_cache, umflag)) == NULL) 11887c478bd9Sstevel@tonic-gate goto slab_alloc_failure; 11897c478bd9Sstevel@tonic-gate chunks = (slabsize - color) / chunksize; 11907c478bd9Sstevel@tonic-gate } else { 11917c478bd9Sstevel@tonic-gate sp = UMEM_SLAB(cp, slab); 11927c478bd9Sstevel@tonic-gate chunks = (slabsize - sizeof (umem_slab_t) - color) / chunksize; 11937c478bd9Sstevel@tonic-gate } 11947c478bd9Sstevel@tonic-gate 11957c478bd9Sstevel@tonic-gate sp->slab_cache = cp; 11967c478bd9Sstevel@tonic-gate sp->slab_head = NULL; 11977c478bd9Sstevel@tonic-gate sp->slab_refcnt = 0; 11987c478bd9Sstevel@tonic-gate sp->slab_base = buf = slab + color; 11997c478bd9Sstevel@tonic-gate sp->slab_chunks = chunks; 12007c478bd9Sstevel@tonic-gate 12017c478bd9Sstevel@tonic-gate ASSERT(chunks > 0); 12027c478bd9Sstevel@tonic-gate while (chunks-- != 0) { 12037c478bd9Sstevel@tonic-gate if (cache_flags & UMF_HASH) { 12047c478bd9Sstevel@tonic-gate bcp = _umem_cache_alloc(cp->cache_bufctl_cache, umflag); 12057c478bd9Sstevel@tonic-gate if (bcp == NULL) 12067c478bd9Sstevel@tonic-gate goto bufctl_alloc_failure; 12077c478bd9Sstevel@tonic-gate if (cache_flags & UMF_AUDIT) { 12087c478bd9Sstevel@tonic-gate umem_bufctl_audit_t *bcap = 12097c478bd9Sstevel@tonic-gate (umem_bufctl_audit_t *)bcp; 12107c478bd9Sstevel@tonic-gate bzero(bcap, UMEM_BUFCTL_AUDIT_SIZE); 12117c478bd9Sstevel@tonic-gate bcap->bc_cache = cp; 12127c478bd9Sstevel@tonic-gate } 12137c478bd9Sstevel@tonic-gate bcp->bc_addr = buf; 12147c478bd9Sstevel@tonic-gate bcp->bc_slab = sp; 12157c478bd9Sstevel@tonic-gate } else { 12167c478bd9Sstevel@tonic-gate bcp = UMEM_BUFCTL(cp, buf); 12177c478bd9Sstevel@tonic-gate } 12187c478bd9Sstevel@tonic-gate if (cache_flags & UMF_BUFTAG) { 12197c478bd9Sstevel@tonic-gate umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); 12207c478bd9Sstevel@tonic-gate btp->bt_redzone = UMEM_REDZONE_PATTERN; 12217c478bd9Sstevel@tonic-gate btp->bt_bufctl = bcp; 12227c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ UMEM_BUFTAG_FREE; 12237c478bd9Sstevel@tonic-gate if (cache_flags & UMF_DEADBEEF) { 12247c478bd9Sstevel@tonic-gate copy_pattern(UMEM_FREE_PATTERN, buf, 12257c478bd9Sstevel@tonic-gate cp->cache_verify); 12267c478bd9Sstevel@tonic-gate } 12277c478bd9Sstevel@tonic-gate } 12287c478bd9Sstevel@tonic-gate bcp->bc_next = sp->slab_head; 12297c478bd9Sstevel@tonic-gate sp->slab_head = bcp; 12307c478bd9Sstevel@tonic-gate buf += chunksize; 12317c478bd9Sstevel@tonic-gate } 12327c478bd9Sstevel@tonic-gate 12337c478bd9Sstevel@tonic-gate umem_log_event(umem_slab_log, cp, sp, slab); 12347c478bd9Sstevel@tonic-gate 12357c478bd9Sstevel@tonic-gate return (sp); 12367c478bd9Sstevel@tonic-gate 12377c478bd9Sstevel@tonic-gate bufctl_alloc_failure: 12387c478bd9Sstevel@tonic-gate 12397c478bd9Sstevel@tonic-gate while ((bcp = sp->slab_head) != NULL) { 12407c478bd9Sstevel@tonic-gate sp->slab_head = bcp->bc_next; 12417c478bd9Sstevel@tonic-gate _umem_cache_free(cp->cache_bufctl_cache, bcp); 12427c478bd9Sstevel@tonic-gate } 12437c478bd9Sstevel@tonic-gate _umem_cache_free(umem_slab_cache, sp); 12447c478bd9Sstevel@tonic-gate 12457c478bd9Sstevel@tonic-gate slab_alloc_failure: 12467c478bd9Sstevel@tonic-gate 12477c478bd9Sstevel@tonic-gate vmem_free(vmp, slab, slabsize); 12487c478bd9Sstevel@tonic-gate 12497c478bd9Sstevel@tonic-gate vmem_alloc_failure: 12507c478bd9Sstevel@tonic-gate 12517c478bd9Sstevel@tonic-gate umem_log_event(umem_failure_log, cp, NULL, NULL); 12527c478bd9Sstevel@tonic-gate atomic_add_64(&cp->cache_alloc_fail, 1); 12537c478bd9Sstevel@tonic-gate 12547c478bd9Sstevel@tonic-gate return (NULL); 12557c478bd9Sstevel@tonic-gate } 12567c478bd9Sstevel@tonic-gate 12577c478bd9Sstevel@tonic-gate /* 12587c478bd9Sstevel@tonic-gate * Destroy a slab. 12597c478bd9Sstevel@tonic-gate */ 12607c478bd9Sstevel@tonic-gate static void 12617c478bd9Sstevel@tonic-gate umem_slab_destroy(umem_cache_t *cp, umem_slab_t *sp) 12627c478bd9Sstevel@tonic-gate { 12637c478bd9Sstevel@tonic-gate vmem_t *vmp = cp->cache_arena; 12647c478bd9Sstevel@tonic-gate void *slab = (void *)P2ALIGN((uintptr_t)sp->slab_base, vmp->vm_quantum); 12657c478bd9Sstevel@tonic-gate 12667c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_HASH) { 12677c478bd9Sstevel@tonic-gate umem_bufctl_t *bcp; 12687c478bd9Sstevel@tonic-gate while ((bcp = sp->slab_head) != NULL) { 12697c478bd9Sstevel@tonic-gate sp->slab_head = bcp->bc_next; 12707c478bd9Sstevel@tonic-gate _umem_cache_free(cp->cache_bufctl_cache, bcp); 12717c478bd9Sstevel@tonic-gate } 12727c478bd9Sstevel@tonic-gate _umem_cache_free(umem_slab_cache, sp); 12737c478bd9Sstevel@tonic-gate } 12747c478bd9Sstevel@tonic-gate vmem_free(vmp, slab, cp->cache_slabsize); 12757c478bd9Sstevel@tonic-gate } 12767c478bd9Sstevel@tonic-gate 12777c478bd9Sstevel@tonic-gate /* 12787c478bd9Sstevel@tonic-gate * Allocate a raw (unconstructed) buffer from cp's slab layer. 12797c478bd9Sstevel@tonic-gate */ 12807c478bd9Sstevel@tonic-gate static void * 12817c478bd9Sstevel@tonic-gate umem_slab_alloc(umem_cache_t *cp, int umflag) 12827c478bd9Sstevel@tonic-gate { 12837c478bd9Sstevel@tonic-gate umem_bufctl_t *bcp, **hash_bucket; 12847c478bd9Sstevel@tonic-gate umem_slab_t *sp; 12857c478bd9Sstevel@tonic-gate void *buf; 12867c478bd9Sstevel@tonic-gate 12877c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_lock); 12887c478bd9Sstevel@tonic-gate cp->cache_slab_alloc++; 12897c478bd9Sstevel@tonic-gate sp = cp->cache_freelist; 12907c478bd9Sstevel@tonic-gate ASSERT(sp->slab_cache == cp); 12917c478bd9Sstevel@tonic-gate if (sp->slab_head == NULL) { 12927c478bd9Sstevel@tonic-gate /* 12937c478bd9Sstevel@tonic-gate * The freelist is empty. Create a new slab. 12947c478bd9Sstevel@tonic-gate */ 12957c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 12967c478bd9Sstevel@tonic-gate if (cp == &umem_null_cache) 12977c478bd9Sstevel@tonic-gate return (NULL); 12987c478bd9Sstevel@tonic-gate if ((sp = umem_slab_create(cp, umflag)) == NULL) 12997c478bd9Sstevel@tonic-gate return (NULL); 13007c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_lock); 13017c478bd9Sstevel@tonic-gate cp->cache_slab_create++; 13027c478bd9Sstevel@tonic-gate if ((cp->cache_buftotal += sp->slab_chunks) > cp->cache_bufmax) 13037c478bd9Sstevel@tonic-gate cp->cache_bufmax = cp->cache_buftotal; 13047c478bd9Sstevel@tonic-gate sp->slab_next = cp->cache_freelist; 13057c478bd9Sstevel@tonic-gate sp->slab_prev = cp->cache_freelist->slab_prev; 13067c478bd9Sstevel@tonic-gate sp->slab_next->slab_prev = sp; 13077c478bd9Sstevel@tonic-gate sp->slab_prev->slab_next = sp; 13087c478bd9Sstevel@tonic-gate cp->cache_freelist = sp; 13097c478bd9Sstevel@tonic-gate } 13107c478bd9Sstevel@tonic-gate 13117c478bd9Sstevel@tonic-gate sp->slab_refcnt++; 13127c478bd9Sstevel@tonic-gate ASSERT(sp->slab_refcnt <= sp->slab_chunks); 13137c478bd9Sstevel@tonic-gate 13147c478bd9Sstevel@tonic-gate /* 13157c478bd9Sstevel@tonic-gate * If we're taking the last buffer in the slab, 13167c478bd9Sstevel@tonic-gate * remove the slab from the cache's freelist. 13177c478bd9Sstevel@tonic-gate */ 13187c478bd9Sstevel@tonic-gate bcp = sp->slab_head; 13197c478bd9Sstevel@tonic-gate if ((sp->slab_head = bcp->bc_next) == NULL) { 13207c478bd9Sstevel@tonic-gate cp->cache_freelist = sp->slab_next; 13217c478bd9Sstevel@tonic-gate ASSERT(sp->slab_refcnt == sp->slab_chunks); 13227c478bd9Sstevel@tonic-gate } 13237c478bd9Sstevel@tonic-gate 13247c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_HASH) { 13257c478bd9Sstevel@tonic-gate /* 13267c478bd9Sstevel@tonic-gate * Add buffer to allocated-address hash table. 13277c478bd9Sstevel@tonic-gate */ 13287c478bd9Sstevel@tonic-gate buf = bcp->bc_addr; 13297c478bd9Sstevel@tonic-gate hash_bucket = UMEM_HASH(cp, buf); 13307c478bd9Sstevel@tonic-gate bcp->bc_next = *hash_bucket; 13317c478bd9Sstevel@tonic-gate *hash_bucket = bcp; 13327c478bd9Sstevel@tonic-gate if ((cp->cache_flags & (UMF_AUDIT | UMF_BUFTAG)) == UMF_AUDIT) { 13337c478bd9Sstevel@tonic-gate UMEM_AUDIT(umem_transaction_log, cp, bcp); 13347c478bd9Sstevel@tonic-gate } 13357c478bd9Sstevel@tonic-gate } else { 13367c478bd9Sstevel@tonic-gate buf = UMEM_BUF(cp, bcp); 13377c478bd9Sstevel@tonic-gate } 13387c478bd9Sstevel@tonic-gate 13397c478bd9Sstevel@tonic-gate ASSERT(UMEM_SLAB_MEMBER(sp, buf)); 13407c478bd9Sstevel@tonic-gate 13417c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 13427c478bd9Sstevel@tonic-gate 13437c478bd9Sstevel@tonic-gate return (buf); 13447c478bd9Sstevel@tonic-gate } 13457c478bd9Sstevel@tonic-gate 13467c478bd9Sstevel@tonic-gate /* 13477c478bd9Sstevel@tonic-gate * Free a raw (unconstructed) buffer to cp's slab layer. 13487c478bd9Sstevel@tonic-gate */ 13497c478bd9Sstevel@tonic-gate static void 13507c478bd9Sstevel@tonic-gate umem_slab_free(umem_cache_t *cp, void *buf) 13517c478bd9Sstevel@tonic-gate { 13527c478bd9Sstevel@tonic-gate umem_slab_t *sp; 13537c478bd9Sstevel@tonic-gate umem_bufctl_t *bcp, **prev_bcpp; 13547c478bd9Sstevel@tonic-gate 13557c478bd9Sstevel@tonic-gate ASSERT(buf != NULL); 13567c478bd9Sstevel@tonic-gate 13577c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_lock); 13587c478bd9Sstevel@tonic-gate cp->cache_slab_free++; 13597c478bd9Sstevel@tonic-gate 13607c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_HASH) { 13617c478bd9Sstevel@tonic-gate /* 13627c478bd9Sstevel@tonic-gate * Look up buffer in allocated-address hash table. 13637c478bd9Sstevel@tonic-gate */ 13647c478bd9Sstevel@tonic-gate prev_bcpp = UMEM_HASH(cp, buf); 13657c478bd9Sstevel@tonic-gate while ((bcp = *prev_bcpp) != NULL) { 13667c478bd9Sstevel@tonic-gate if (bcp->bc_addr == buf) { 13677c478bd9Sstevel@tonic-gate *prev_bcpp = bcp->bc_next; 13687c478bd9Sstevel@tonic-gate sp = bcp->bc_slab; 13697c478bd9Sstevel@tonic-gate break; 13707c478bd9Sstevel@tonic-gate } 13717c478bd9Sstevel@tonic-gate cp->cache_lookup_depth++; 13727c478bd9Sstevel@tonic-gate prev_bcpp = &bcp->bc_next; 13737c478bd9Sstevel@tonic-gate } 13747c478bd9Sstevel@tonic-gate } else { 13757c478bd9Sstevel@tonic-gate bcp = UMEM_BUFCTL(cp, buf); 13767c478bd9Sstevel@tonic-gate sp = UMEM_SLAB(cp, buf); 13777c478bd9Sstevel@tonic-gate } 13787c478bd9Sstevel@tonic-gate 13797c478bd9Sstevel@tonic-gate if (bcp == NULL || sp->slab_cache != cp || !UMEM_SLAB_MEMBER(sp, buf)) { 13807c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 13817c478bd9Sstevel@tonic-gate umem_error(UMERR_BADADDR, cp, buf); 13827c478bd9Sstevel@tonic-gate return; 13837c478bd9Sstevel@tonic-gate } 13847c478bd9Sstevel@tonic-gate 13857c478bd9Sstevel@tonic-gate if ((cp->cache_flags & (UMF_AUDIT | UMF_BUFTAG)) == UMF_AUDIT) { 13867c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_CONTENTS) 13877c478bd9Sstevel@tonic-gate ((umem_bufctl_audit_t *)bcp)->bc_contents = 13887c478bd9Sstevel@tonic-gate umem_log_enter(umem_content_log, buf, 13897c478bd9Sstevel@tonic-gate cp->cache_contents); 13907c478bd9Sstevel@tonic-gate UMEM_AUDIT(umem_transaction_log, cp, bcp); 13917c478bd9Sstevel@tonic-gate } 13927c478bd9Sstevel@tonic-gate 13937c478bd9Sstevel@tonic-gate /* 13947c478bd9Sstevel@tonic-gate * If this slab isn't currently on the freelist, put it there. 13957c478bd9Sstevel@tonic-gate */ 13967c478bd9Sstevel@tonic-gate if (sp->slab_head == NULL) { 13977c478bd9Sstevel@tonic-gate ASSERT(sp->slab_refcnt == sp->slab_chunks); 13987c478bd9Sstevel@tonic-gate ASSERT(cp->cache_freelist != sp); 13997c478bd9Sstevel@tonic-gate sp->slab_next->slab_prev = sp->slab_prev; 14007c478bd9Sstevel@tonic-gate sp->slab_prev->slab_next = sp->slab_next; 14017c478bd9Sstevel@tonic-gate sp->slab_next = cp->cache_freelist; 14027c478bd9Sstevel@tonic-gate sp->slab_prev = cp->cache_freelist->slab_prev; 14037c478bd9Sstevel@tonic-gate sp->slab_next->slab_prev = sp; 14047c478bd9Sstevel@tonic-gate sp->slab_prev->slab_next = sp; 14057c478bd9Sstevel@tonic-gate cp->cache_freelist = sp; 14067c478bd9Sstevel@tonic-gate } 14077c478bd9Sstevel@tonic-gate 14087c478bd9Sstevel@tonic-gate bcp->bc_next = sp->slab_head; 14097c478bd9Sstevel@tonic-gate sp->slab_head = bcp; 14107c478bd9Sstevel@tonic-gate 14117c478bd9Sstevel@tonic-gate ASSERT(sp->slab_refcnt >= 1); 14127c478bd9Sstevel@tonic-gate if (--sp->slab_refcnt == 0) { 14137c478bd9Sstevel@tonic-gate /* 14147c478bd9Sstevel@tonic-gate * There are no outstanding allocations from this slab, 14157c478bd9Sstevel@tonic-gate * so we can reclaim the memory. 14167c478bd9Sstevel@tonic-gate */ 14177c478bd9Sstevel@tonic-gate sp->slab_next->slab_prev = sp->slab_prev; 14187c478bd9Sstevel@tonic-gate sp->slab_prev->slab_next = sp->slab_next; 14197c478bd9Sstevel@tonic-gate if (sp == cp->cache_freelist) 14207c478bd9Sstevel@tonic-gate cp->cache_freelist = sp->slab_next; 14217c478bd9Sstevel@tonic-gate cp->cache_slab_destroy++; 14227c478bd9Sstevel@tonic-gate cp->cache_buftotal -= sp->slab_chunks; 14237c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 14247c478bd9Sstevel@tonic-gate umem_slab_destroy(cp, sp); 14257c478bd9Sstevel@tonic-gate return; 14267c478bd9Sstevel@tonic-gate } 14277c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 14287c478bd9Sstevel@tonic-gate } 14297c478bd9Sstevel@tonic-gate 14307c478bd9Sstevel@tonic-gate static int 14317c478bd9Sstevel@tonic-gate umem_cache_alloc_debug(umem_cache_t *cp, void *buf, int umflag) 14327c478bd9Sstevel@tonic-gate { 14337c478bd9Sstevel@tonic-gate umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); 14347c478bd9Sstevel@tonic-gate umem_bufctl_audit_t *bcp = (umem_bufctl_audit_t *)btp->bt_bufctl; 14357c478bd9Sstevel@tonic-gate uint32_t mtbf; 14367c478bd9Sstevel@tonic-gate int flags_nfatal; 14377c478bd9Sstevel@tonic-gate 14387c478bd9Sstevel@tonic-gate if (btp->bt_bxstat != ((intptr_t)bcp ^ UMEM_BUFTAG_FREE)) { 14397c478bd9Sstevel@tonic-gate umem_error(UMERR_BADBUFTAG, cp, buf); 14407c478bd9Sstevel@tonic-gate return (-1); 14417c478bd9Sstevel@tonic-gate } 14427c478bd9Sstevel@tonic-gate 14437c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ UMEM_BUFTAG_ALLOC; 14447c478bd9Sstevel@tonic-gate 14457c478bd9Sstevel@tonic-gate if ((cp->cache_flags & UMF_HASH) && bcp->bc_addr != buf) { 14467c478bd9Sstevel@tonic-gate umem_error(UMERR_BADBUFCTL, cp, buf); 14477c478bd9Sstevel@tonic-gate return (-1); 14487c478bd9Sstevel@tonic-gate } 14497c478bd9Sstevel@tonic-gate 14507c478bd9Sstevel@tonic-gate btp->bt_redzone = UMEM_REDZONE_PATTERN; 14517c478bd9Sstevel@tonic-gate 14527c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_DEADBEEF) { 14537c478bd9Sstevel@tonic-gate if (verify_and_copy_pattern(UMEM_FREE_PATTERN, 14547c478bd9Sstevel@tonic-gate UMEM_UNINITIALIZED_PATTERN, buf, cp->cache_verify)) { 14557c478bd9Sstevel@tonic-gate umem_error(UMERR_MODIFIED, cp, buf); 14567c478bd9Sstevel@tonic-gate return (-1); 14577c478bd9Sstevel@tonic-gate } 14587c478bd9Sstevel@tonic-gate } 14597c478bd9Sstevel@tonic-gate 14607c478bd9Sstevel@tonic-gate if ((mtbf = umem_mtbf | cp->cache_mtbf) != 0 && 14617c478bd9Sstevel@tonic-gate gethrtime() % mtbf == 0 && 14627c478bd9Sstevel@tonic-gate (umflag & (UMEM_FATAL_FLAGS)) == 0) { 14637c478bd9Sstevel@tonic-gate umem_log_event(umem_failure_log, cp, NULL, NULL); 14647c478bd9Sstevel@tonic-gate } else { 14657c478bd9Sstevel@tonic-gate mtbf = 0; 14667c478bd9Sstevel@tonic-gate } 14677c478bd9Sstevel@tonic-gate 14687c478bd9Sstevel@tonic-gate /* 14697c478bd9Sstevel@tonic-gate * We do not pass fatal flags on to the constructor. This prevents 14707c478bd9Sstevel@tonic-gate * leaking buffers in the event of a subordinate constructor failing. 14717c478bd9Sstevel@tonic-gate */ 14727c478bd9Sstevel@tonic-gate flags_nfatal = UMEM_DEFAULT; 14737c478bd9Sstevel@tonic-gate if (mtbf || (cp->cache_constructor != NULL && 14747c478bd9Sstevel@tonic-gate cp->cache_constructor(buf, cp->cache_private, flags_nfatal) != 0)) { 14757c478bd9Sstevel@tonic-gate atomic_add_64(&cp->cache_alloc_fail, 1); 14767c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ UMEM_BUFTAG_FREE; 14777c478bd9Sstevel@tonic-gate copy_pattern(UMEM_FREE_PATTERN, buf, cp->cache_verify); 14787c478bd9Sstevel@tonic-gate umem_slab_free(cp, buf); 14797c478bd9Sstevel@tonic-gate return (-1); 14807c478bd9Sstevel@tonic-gate } 14817c478bd9Sstevel@tonic-gate 14827c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_AUDIT) { 14837c478bd9Sstevel@tonic-gate UMEM_AUDIT(umem_transaction_log, cp, bcp); 14847c478bd9Sstevel@tonic-gate } 14857c478bd9Sstevel@tonic-gate 14867c478bd9Sstevel@tonic-gate return (0); 14877c478bd9Sstevel@tonic-gate } 14887c478bd9Sstevel@tonic-gate 14897c478bd9Sstevel@tonic-gate static int 14907c478bd9Sstevel@tonic-gate umem_cache_free_debug(umem_cache_t *cp, void *buf) 14917c478bd9Sstevel@tonic-gate { 14927c478bd9Sstevel@tonic-gate umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); 14937c478bd9Sstevel@tonic-gate umem_bufctl_audit_t *bcp = (umem_bufctl_audit_t *)btp->bt_bufctl; 14947c478bd9Sstevel@tonic-gate umem_slab_t *sp; 14957c478bd9Sstevel@tonic-gate 14967c478bd9Sstevel@tonic-gate if (btp->bt_bxstat != ((intptr_t)bcp ^ UMEM_BUFTAG_ALLOC)) { 14977c478bd9Sstevel@tonic-gate if (btp->bt_bxstat == ((intptr_t)bcp ^ UMEM_BUFTAG_FREE)) { 14987c478bd9Sstevel@tonic-gate umem_error(UMERR_DUPFREE, cp, buf); 14997c478bd9Sstevel@tonic-gate return (-1); 15007c478bd9Sstevel@tonic-gate } 15017c478bd9Sstevel@tonic-gate sp = umem_findslab(cp, buf); 15027c478bd9Sstevel@tonic-gate if (sp == NULL || sp->slab_cache != cp) 15037c478bd9Sstevel@tonic-gate umem_error(UMERR_BADADDR, cp, buf); 15047c478bd9Sstevel@tonic-gate else 15057c478bd9Sstevel@tonic-gate umem_error(UMERR_REDZONE, cp, buf); 15067c478bd9Sstevel@tonic-gate return (-1); 15077c478bd9Sstevel@tonic-gate } 15087c478bd9Sstevel@tonic-gate 15097c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ UMEM_BUFTAG_FREE; 15107c478bd9Sstevel@tonic-gate 15117c478bd9Sstevel@tonic-gate if ((cp->cache_flags & UMF_HASH) && bcp->bc_addr != buf) { 15127c478bd9Sstevel@tonic-gate umem_error(UMERR_BADBUFCTL, cp, buf); 15137c478bd9Sstevel@tonic-gate return (-1); 15147c478bd9Sstevel@tonic-gate } 15157c478bd9Sstevel@tonic-gate 15167c478bd9Sstevel@tonic-gate if (btp->bt_redzone != UMEM_REDZONE_PATTERN) { 15177c478bd9Sstevel@tonic-gate umem_error(UMERR_REDZONE, cp, buf); 15187c478bd9Sstevel@tonic-gate return (-1); 15197c478bd9Sstevel@tonic-gate } 15207c478bd9Sstevel@tonic-gate 15217c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_AUDIT) { 15227c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_CONTENTS) 15237c478bd9Sstevel@tonic-gate bcp->bc_contents = umem_log_enter(umem_content_log, 15247c478bd9Sstevel@tonic-gate buf, cp->cache_contents); 15257c478bd9Sstevel@tonic-gate UMEM_AUDIT(umem_transaction_log, cp, bcp); 15267c478bd9Sstevel@tonic-gate } 15277c478bd9Sstevel@tonic-gate 15287c478bd9Sstevel@tonic-gate if (cp->cache_destructor != NULL) 15297c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 15307c478bd9Sstevel@tonic-gate 15317c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_DEADBEEF) 15327c478bd9Sstevel@tonic-gate copy_pattern(UMEM_FREE_PATTERN, buf, cp->cache_verify); 15337c478bd9Sstevel@tonic-gate 15347c478bd9Sstevel@tonic-gate return (0); 15357c478bd9Sstevel@tonic-gate } 15367c478bd9Sstevel@tonic-gate 15377c478bd9Sstevel@tonic-gate /* 15387c478bd9Sstevel@tonic-gate * Free each object in magazine mp to cp's slab layer, and free mp itself. 15397c478bd9Sstevel@tonic-gate */ 15407c478bd9Sstevel@tonic-gate static void 15417c478bd9Sstevel@tonic-gate umem_magazine_destroy(umem_cache_t *cp, umem_magazine_t *mp, int nrounds) 15427c478bd9Sstevel@tonic-gate { 15437c478bd9Sstevel@tonic-gate int round; 15447c478bd9Sstevel@tonic-gate 15457c478bd9Sstevel@tonic-gate ASSERT(cp->cache_next == NULL || IN_UPDATE()); 15467c478bd9Sstevel@tonic-gate 15477c478bd9Sstevel@tonic-gate for (round = 0; round < nrounds; round++) { 15487c478bd9Sstevel@tonic-gate void *buf = mp->mag_round[round]; 15497c478bd9Sstevel@tonic-gate 15507c478bd9Sstevel@tonic-gate if ((cp->cache_flags & UMF_DEADBEEF) && 15517c478bd9Sstevel@tonic-gate verify_pattern(UMEM_FREE_PATTERN, buf, 15527c478bd9Sstevel@tonic-gate cp->cache_verify) != NULL) { 15537c478bd9Sstevel@tonic-gate umem_error(UMERR_MODIFIED, cp, buf); 15547c478bd9Sstevel@tonic-gate continue; 15557c478bd9Sstevel@tonic-gate } 15567c478bd9Sstevel@tonic-gate 15577c478bd9Sstevel@tonic-gate if (!(cp->cache_flags & UMF_BUFTAG) && 15587c478bd9Sstevel@tonic-gate cp->cache_destructor != NULL) 15597c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 15607c478bd9Sstevel@tonic-gate 15617c478bd9Sstevel@tonic-gate umem_slab_free(cp, buf); 15627c478bd9Sstevel@tonic-gate } 15637c478bd9Sstevel@tonic-gate ASSERT(UMEM_MAGAZINE_VALID(cp, mp)); 15647c478bd9Sstevel@tonic-gate _umem_cache_free(cp->cache_magtype->mt_cache, mp); 15657c478bd9Sstevel@tonic-gate } 15667c478bd9Sstevel@tonic-gate 15677c478bd9Sstevel@tonic-gate /* 15687c478bd9Sstevel@tonic-gate * Allocate a magazine from the depot. 15697c478bd9Sstevel@tonic-gate */ 15707c478bd9Sstevel@tonic-gate static umem_magazine_t * 15717c478bd9Sstevel@tonic-gate umem_depot_alloc(umem_cache_t *cp, umem_maglist_t *mlp) 15727c478bd9Sstevel@tonic-gate { 15737c478bd9Sstevel@tonic-gate umem_magazine_t *mp; 15747c478bd9Sstevel@tonic-gate 15757c478bd9Sstevel@tonic-gate /* 15767c478bd9Sstevel@tonic-gate * If we can't get the depot lock without contention, 15777c478bd9Sstevel@tonic-gate * update our contention count. We use the depot 15787c478bd9Sstevel@tonic-gate * contention rate to determine whether we need to 15797c478bd9Sstevel@tonic-gate * increase the magazine size for better scalability. 15807c478bd9Sstevel@tonic-gate */ 15817c478bd9Sstevel@tonic-gate if (mutex_trylock(&cp->cache_depot_lock) != 0) { 15827c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_depot_lock); 15837c478bd9Sstevel@tonic-gate cp->cache_depot_contention++; 15847c478bd9Sstevel@tonic-gate } 15857c478bd9Sstevel@tonic-gate 15867c478bd9Sstevel@tonic-gate if ((mp = mlp->ml_list) != NULL) { 15877c478bd9Sstevel@tonic-gate ASSERT(UMEM_MAGAZINE_VALID(cp, mp)); 15887c478bd9Sstevel@tonic-gate mlp->ml_list = mp->mag_next; 15897c478bd9Sstevel@tonic-gate if (--mlp->ml_total < mlp->ml_min) 15907c478bd9Sstevel@tonic-gate mlp->ml_min = mlp->ml_total; 15917c478bd9Sstevel@tonic-gate mlp->ml_alloc++; 15927c478bd9Sstevel@tonic-gate } 15937c478bd9Sstevel@tonic-gate 15947c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_depot_lock); 15957c478bd9Sstevel@tonic-gate 15967c478bd9Sstevel@tonic-gate return (mp); 15977c478bd9Sstevel@tonic-gate } 15987c478bd9Sstevel@tonic-gate 15997c478bd9Sstevel@tonic-gate /* 16007c478bd9Sstevel@tonic-gate * Free a magazine to the depot. 16017c478bd9Sstevel@tonic-gate */ 16027c478bd9Sstevel@tonic-gate static void 16037c478bd9Sstevel@tonic-gate umem_depot_free(umem_cache_t *cp, umem_maglist_t *mlp, umem_magazine_t *mp) 16047c478bd9Sstevel@tonic-gate { 16057c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_depot_lock); 16067c478bd9Sstevel@tonic-gate ASSERT(UMEM_MAGAZINE_VALID(cp, mp)); 16077c478bd9Sstevel@tonic-gate mp->mag_next = mlp->ml_list; 16087c478bd9Sstevel@tonic-gate mlp->ml_list = mp; 16097c478bd9Sstevel@tonic-gate mlp->ml_total++; 16107c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_depot_lock); 16117c478bd9Sstevel@tonic-gate } 16127c478bd9Sstevel@tonic-gate 16137c478bd9Sstevel@tonic-gate /* 16147c478bd9Sstevel@tonic-gate * Update the working set statistics for cp's depot. 16157c478bd9Sstevel@tonic-gate */ 16167c478bd9Sstevel@tonic-gate static void 16177c478bd9Sstevel@tonic-gate umem_depot_ws_update(umem_cache_t *cp) 16187c478bd9Sstevel@tonic-gate { 16197c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_depot_lock); 16207c478bd9Sstevel@tonic-gate cp->cache_full.ml_reaplimit = cp->cache_full.ml_min; 16217c478bd9Sstevel@tonic-gate cp->cache_full.ml_min = cp->cache_full.ml_total; 16227c478bd9Sstevel@tonic-gate cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_min; 16237c478bd9Sstevel@tonic-gate cp->cache_empty.ml_min = cp->cache_empty.ml_total; 16247c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_depot_lock); 16257c478bd9Sstevel@tonic-gate } 16267c478bd9Sstevel@tonic-gate 16277c478bd9Sstevel@tonic-gate /* 16287c478bd9Sstevel@tonic-gate * Reap all magazines that have fallen out of the depot's working set. 16297c478bd9Sstevel@tonic-gate */ 16307c478bd9Sstevel@tonic-gate static void 16317c478bd9Sstevel@tonic-gate umem_depot_ws_reap(umem_cache_t *cp) 16327c478bd9Sstevel@tonic-gate { 16337c478bd9Sstevel@tonic-gate long reap; 16347c478bd9Sstevel@tonic-gate umem_magazine_t *mp; 16357c478bd9Sstevel@tonic-gate 16367c478bd9Sstevel@tonic-gate ASSERT(cp->cache_next == NULL || IN_REAP()); 16377c478bd9Sstevel@tonic-gate 16387c478bd9Sstevel@tonic-gate reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min); 16397c478bd9Sstevel@tonic-gate while (reap-- && (mp = umem_depot_alloc(cp, &cp->cache_full)) != NULL) 16407c478bd9Sstevel@tonic-gate umem_magazine_destroy(cp, mp, cp->cache_magtype->mt_magsize); 16417c478bd9Sstevel@tonic-gate 16427c478bd9Sstevel@tonic-gate reap = MIN(cp->cache_empty.ml_reaplimit, cp->cache_empty.ml_min); 16437c478bd9Sstevel@tonic-gate while (reap-- && (mp = umem_depot_alloc(cp, &cp->cache_empty)) != NULL) 16447c478bd9Sstevel@tonic-gate umem_magazine_destroy(cp, mp, 0); 16457c478bd9Sstevel@tonic-gate } 16467c478bd9Sstevel@tonic-gate 16477c478bd9Sstevel@tonic-gate static void 16487c478bd9Sstevel@tonic-gate umem_cpu_reload(umem_cpu_cache_t *ccp, umem_magazine_t *mp, int rounds) 16497c478bd9Sstevel@tonic-gate { 16507c478bd9Sstevel@tonic-gate ASSERT((ccp->cc_loaded == NULL && ccp->cc_rounds == -1) || 16517c478bd9Sstevel@tonic-gate (ccp->cc_loaded && ccp->cc_rounds + rounds == ccp->cc_magsize)); 16527c478bd9Sstevel@tonic-gate ASSERT(ccp->cc_magsize > 0); 16537c478bd9Sstevel@tonic-gate 16547c478bd9Sstevel@tonic-gate ccp->cc_ploaded = ccp->cc_loaded; 16557c478bd9Sstevel@tonic-gate ccp->cc_prounds = ccp->cc_rounds; 16567c478bd9Sstevel@tonic-gate ccp->cc_loaded = mp; 16577c478bd9Sstevel@tonic-gate ccp->cc_rounds = rounds; 16587c478bd9Sstevel@tonic-gate } 16597c478bd9Sstevel@tonic-gate 16607c478bd9Sstevel@tonic-gate /* 16617c478bd9Sstevel@tonic-gate * Allocate a constructed object from cache cp. 16627c478bd9Sstevel@tonic-gate */ 16637c478bd9Sstevel@tonic-gate #pragma weak umem_cache_alloc = _umem_cache_alloc 16647c478bd9Sstevel@tonic-gate void * 16657c478bd9Sstevel@tonic-gate _umem_cache_alloc(umem_cache_t *cp, int umflag) 16667c478bd9Sstevel@tonic-gate { 16677c478bd9Sstevel@tonic-gate umem_cpu_cache_t *ccp; 16687c478bd9Sstevel@tonic-gate umem_magazine_t *fmp; 16697c478bd9Sstevel@tonic-gate void *buf; 16707c478bd9Sstevel@tonic-gate int flags_nfatal; 16717c478bd9Sstevel@tonic-gate 16727c478bd9Sstevel@tonic-gate retry: 16737c478bd9Sstevel@tonic-gate ccp = UMEM_CPU_CACHE(cp, CPU(cp->cache_cpu_mask)); 16747c478bd9Sstevel@tonic-gate (void) mutex_lock(&ccp->cc_lock); 16757c478bd9Sstevel@tonic-gate for (;;) { 16767c478bd9Sstevel@tonic-gate /* 16777c478bd9Sstevel@tonic-gate * If there's an object available in the current CPU's 16787c478bd9Sstevel@tonic-gate * loaded magazine, just take it and return. 16797c478bd9Sstevel@tonic-gate */ 16807c478bd9Sstevel@tonic-gate if (ccp->cc_rounds > 0) { 16817c478bd9Sstevel@tonic-gate buf = ccp->cc_loaded->mag_round[--ccp->cc_rounds]; 16827c478bd9Sstevel@tonic-gate ccp->cc_alloc++; 16837c478bd9Sstevel@tonic-gate (void) mutex_unlock(&ccp->cc_lock); 16847c478bd9Sstevel@tonic-gate if ((ccp->cc_flags & UMF_BUFTAG) && 16857c478bd9Sstevel@tonic-gate umem_cache_alloc_debug(cp, buf, umflag) == -1) { 16867c478bd9Sstevel@tonic-gate if (umem_alloc_retry(cp, umflag)) { 16877c478bd9Sstevel@tonic-gate goto retry; 16887c478bd9Sstevel@tonic-gate } 16897c478bd9Sstevel@tonic-gate 16907c478bd9Sstevel@tonic-gate return (NULL); 16917c478bd9Sstevel@tonic-gate } 16927c478bd9Sstevel@tonic-gate return (buf); 16937c478bd9Sstevel@tonic-gate } 16947c478bd9Sstevel@tonic-gate 16957c478bd9Sstevel@tonic-gate /* 16967c478bd9Sstevel@tonic-gate * The loaded magazine is empty. If the previously loaded 16977c478bd9Sstevel@tonic-gate * magazine was full, exchange them and try again. 16987c478bd9Sstevel@tonic-gate */ 16997c478bd9Sstevel@tonic-gate if (ccp->cc_prounds > 0) { 17007c478bd9Sstevel@tonic-gate umem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds); 17017c478bd9Sstevel@tonic-gate continue; 17027c478bd9Sstevel@tonic-gate } 17037c478bd9Sstevel@tonic-gate 17047c478bd9Sstevel@tonic-gate /* 17057c478bd9Sstevel@tonic-gate * If the magazine layer is disabled, break out now. 17067c478bd9Sstevel@tonic-gate */ 17077c478bd9Sstevel@tonic-gate if (ccp->cc_magsize == 0) 17087c478bd9Sstevel@tonic-gate break; 17097c478bd9Sstevel@tonic-gate 17107c478bd9Sstevel@tonic-gate /* 17117c478bd9Sstevel@tonic-gate * Try to get a full magazine from the depot. 17127c478bd9Sstevel@tonic-gate */ 17137c478bd9Sstevel@tonic-gate fmp = umem_depot_alloc(cp, &cp->cache_full); 17147c478bd9Sstevel@tonic-gate if (fmp != NULL) { 17157c478bd9Sstevel@tonic-gate if (ccp->cc_ploaded != NULL) 17167c478bd9Sstevel@tonic-gate umem_depot_free(cp, &cp->cache_empty, 17177c478bd9Sstevel@tonic-gate ccp->cc_ploaded); 17187c478bd9Sstevel@tonic-gate umem_cpu_reload(ccp, fmp, ccp->cc_magsize); 17197c478bd9Sstevel@tonic-gate continue; 17207c478bd9Sstevel@tonic-gate } 17217c478bd9Sstevel@tonic-gate 17227c478bd9Sstevel@tonic-gate /* 17237c478bd9Sstevel@tonic-gate * There are no full magazines in the depot, 17247c478bd9Sstevel@tonic-gate * so fall through to the slab layer. 17257c478bd9Sstevel@tonic-gate */ 17267c478bd9Sstevel@tonic-gate break; 17277c478bd9Sstevel@tonic-gate } 17287c478bd9Sstevel@tonic-gate (void) mutex_unlock(&ccp->cc_lock); 17297c478bd9Sstevel@tonic-gate 17307c478bd9Sstevel@tonic-gate /* 17317c478bd9Sstevel@tonic-gate * We couldn't allocate a constructed object from the magazine layer, 17327c478bd9Sstevel@tonic-gate * so get a raw buffer from the slab layer and apply its constructor. 17337c478bd9Sstevel@tonic-gate */ 17347c478bd9Sstevel@tonic-gate buf = umem_slab_alloc(cp, umflag); 17357c478bd9Sstevel@tonic-gate 17367c478bd9Sstevel@tonic-gate if (buf == NULL) { 17377c478bd9Sstevel@tonic-gate if (cp == &umem_null_cache) 17387c478bd9Sstevel@tonic-gate return (NULL); 17397c478bd9Sstevel@tonic-gate if (umem_alloc_retry(cp, umflag)) { 17407c478bd9Sstevel@tonic-gate goto retry; 17417c478bd9Sstevel@tonic-gate } 17427c478bd9Sstevel@tonic-gate 17437c478bd9Sstevel@tonic-gate return (NULL); 17447c478bd9Sstevel@tonic-gate } 17457c478bd9Sstevel@tonic-gate 17467c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_BUFTAG) { 17477c478bd9Sstevel@tonic-gate /* 17487c478bd9Sstevel@tonic-gate * Let umem_cache_alloc_debug() apply the constructor for us. 17497c478bd9Sstevel@tonic-gate */ 17507c478bd9Sstevel@tonic-gate if (umem_cache_alloc_debug(cp, buf, umflag) == -1) { 17517c478bd9Sstevel@tonic-gate if (umem_alloc_retry(cp, umflag)) { 17527c478bd9Sstevel@tonic-gate goto retry; 17537c478bd9Sstevel@tonic-gate } 17547c478bd9Sstevel@tonic-gate return (NULL); 17557c478bd9Sstevel@tonic-gate } 17567c478bd9Sstevel@tonic-gate return (buf); 17577c478bd9Sstevel@tonic-gate } 17587c478bd9Sstevel@tonic-gate 17597c478bd9Sstevel@tonic-gate /* 17607c478bd9Sstevel@tonic-gate * We do not pass fatal flags on to the constructor. This prevents 17617c478bd9Sstevel@tonic-gate * leaking buffers in the event of a subordinate constructor failing. 17627c478bd9Sstevel@tonic-gate */ 17637c478bd9Sstevel@tonic-gate flags_nfatal = UMEM_DEFAULT; 17647c478bd9Sstevel@tonic-gate if (cp->cache_constructor != NULL && 17657c478bd9Sstevel@tonic-gate cp->cache_constructor(buf, cp->cache_private, flags_nfatal) != 0) { 17667c478bd9Sstevel@tonic-gate atomic_add_64(&cp->cache_alloc_fail, 1); 17677c478bd9Sstevel@tonic-gate umem_slab_free(cp, buf); 17687c478bd9Sstevel@tonic-gate 17697c478bd9Sstevel@tonic-gate if (umem_alloc_retry(cp, umflag)) { 17707c478bd9Sstevel@tonic-gate goto retry; 17717c478bd9Sstevel@tonic-gate } 17727c478bd9Sstevel@tonic-gate return (NULL); 17737c478bd9Sstevel@tonic-gate } 17747c478bd9Sstevel@tonic-gate 17757c478bd9Sstevel@tonic-gate return (buf); 17767c478bd9Sstevel@tonic-gate } 17777c478bd9Sstevel@tonic-gate 17787c478bd9Sstevel@tonic-gate /* 17797c478bd9Sstevel@tonic-gate * Free a constructed object to cache cp. 17807c478bd9Sstevel@tonic-gate */ 17817c478bd9Sstevel@tonic-gate #pragma weak umem_cache_free = _umem_cache_free 17827c478bd9Sstevel@tonic-gate void 17837c478bd9Sstevel@tonic-gate _umem_cache_free(umem_cache_t *cp, void *buf) 17847c478bd9Sstevel@tonic-gate { 17857c478bd9Sstevel@tonic-gate umem_cpu_cache_t *ccp = UMEM_CPU_CACHE(cp, CPU(cp->cache_cpu_mask)); 17867c478bd9Sstevel@tonic-gate umem_magazine_t *emp; 17877c478bd9Sstevel@tonic-gate umem_magtype_t *mtp; 17887c478bd9Sstevel@tonic-gate 17897c478bd9Sstevel@tonic-gate if (ccp->cc_flags & UMF_BUFTAG) 17907c478bd9Sstevel@tonic-gate if (umem_cache_free_debug(cp, buf) == -1) 17917c478bd9Sstevel@tonic-gate return; 17927c478bd9Sstevel@tonic-gate 17937c478bd9Sstevel@tonic-gate (void) mutex_lock(&ccp->cc_lock); 17947c478bd9Sstevel@tonic-gate for (;;) { 17957c478bd9Sstevel@tonic-gate /* 17967c478bd9Sstevel@tonic-gate * If there's a slot available in the current CPU's 17977c478bd9Sstevel@tonic-gate * loaded magazine, just put the object there and return. 17987c478bd9Sstevel@tonic-gate */ 17997c478bd9Sstevel@tonic-gate if ((uint_t)ccp->cc_rounds < ccp->cc_magsize) { 18007c478bd9Sstevel@tonic-gate ccp->cc_loaded->mag_round[ccp->cc_rounds++] = buf; 18017c478bd9Sstevel@tonic-gate ccp->cc_free++; 18027c478bd9Sstevel@tonic-gate (void) mutex_unlock(&ccp->cc_lock); 18037c478bd9Sstevel@tonic-gate return; 18047c478bd9Sstevel@tonic-gate } 18057c478bd9Sstevel@tonic-gate 18067c478bd9Sstevel@tonic-gate /* 18077c478bd9Sstevel@tonic-gate * The loaded magazine is full. If the previously loaded 18087c478bd9Sstevel@tonic-gate * magazine was empty, exchange them and try again. 18097c478bd9Sstevel@tonic-gate */ 18107c478bd9Sstevel@tonic-gate if (ccp->cc_prounds == 0) { 18117c478bd9Sstevel@tonic-gate umem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds); 18127c478bd9Sstevel@tonic-gate continue; 18137c478bd9Sstevel@tonic-gate } 18147c478bd9Sstevel@tonic-gate 18157c478bd9Sstevel@tonic-gate /* 18167c478bd9Sstevel@tonic-gate * If the magazine layer is disabled, break out now. 18177c478bd9Sstevel@tonic-gate */ 18187c478bd9Sstevel@tonic-gate if (ccp->cc_magsize == 0) 18197c478bd9Sstevel@tonic-gate break; 18207c478bd9Sstevel@tonic-gate 18217c478bd9Sstevel@tonic-gate /* 18227c478bd9Sstevel@tonic-gate * Try to get an empty magazine from the depot. 18237c478bd9Sstevel@tonic-gate */ 18247c478bd9Sstevel@tonic-gate emp = umem_depot_alloc(cp, &cp->cache_empty); 18257c478bd9Sstevel@tonic-gate if (emp != NULL) { 18267c478bd9Sstevel@tonic-gate if (ccp->cc_ploaded != NULL) 18277c478bd9Sstevel@tonic-gate umem_depot_free(cp, &cp->cache_full, 18287c478bd9Sstevel@tonic-gate ccp->cc_ploaded); 18297c478bd9Sstevel@tonic-gate umem_cpu_reload(ccp, emp, 0); 18307c478bd9Sstevel@tonic-gate continue; 18317c478bd9Sstevel@tonic-gate } 18327c478bd9Sstevel@tonic-gate 18337c478bd9Sstevel@tonic-gate /* 18347c478bd9Sstevel@tonic-gate * There are no empty magazines in the depot, 18357c478bd9Sstevel@tonic-gate * so try to allocate a new one. We must drop all locks 18367c478bd9Sstevel@tonic-gate * across umem_cache_alloc() because lower layers may 18377c478bd9Sstevel@tonic-gate * attempt to allocate from this cache. 18387c478bd9Sstevel@tonic-gate */ 18397c478bd9Sstevel@tonic-gate mtp = cp->cache_magtype; 18407c478bd9Sstevel@tonic-gate (void) mutex_unlock(&ccp->cc_lock); 18417c478bd9Sstevel@tonic-gate emp = _umem_cache_alloc(mtp->mt_cache, UMEM_DEFAULT); 18427c478bd9Sstevel@tonic-gate (void) mutex_lock(&ccp->cc_lock); 18437c478bd9Sstevel@tonic-gate 18447c478bd9Sstevel@tonic-gate if (emp != NULL) { 18457c478bd9Sstevel@tonic-gate /* 18467c478bd9Sstevel@tonic-gate * We successfully allocated an empty magazine. 18477c478bd9Sstevel@tonic-gate * However, we had to drop ccp->cc_lock to do it, 18487c478bd9Sstevel@tonic-gate * so the cache's magazine size may have changed. 18497c478bd9Sstevel@tonic-gate * If so, free the magazine and try again. 18507c478bd9Sstevel@tonic-gate */ 18517c478bd9Sstevel@tonic-gate if (ccp->cc_magsize != mtp->mt_magsize) { 18527c478bd9Sstevel@tonic-gate (void) mutex_unlock(&ccp->cc_lock); 18537c478bd9Sstevel@tonic-gate _umem_cache_free(mtp->mt_cache, emp); 18547c478bd9Sstevel@tonic-gate (void) mutex_lock(&ccp->cc_lock); 18557c478bd9Sstevel@tonic-gate continue; 18567c478bd9Sstevel@tonic-gate } 18577c478bd9Sstevel@tonic-gate 18587c478bd9Sstevel@tonic-gate /* 18597c478bd9Sstevel@tonic-gate * We got a magazine of the right size. Add it to 18607c478bd9Sstevel@tonic-gate * the depot and try the whole dance again. 18617c478bd9Sstevel@tonic-gate */ 18627c478bd9Sstevel@tonic-gate umem_depot_free(cp, &cp->cache_empty, emp); 18637c478bd9Sstevel@tonic-gate continue; 18647c478bd9Sstevel@tonic-gate } 18657c478bd9Sstevel@tonic-gate 18667c478bd9Sstevel@tonic-gate /* 18677c478bd9Sstevel@tonic-gate * We couldn't allocate an empty magazine, 18687c478bd9Sstevel@tonic-gate * so fall through to the slab layer. 18697c478bd9Sstevel@tonic-gate */ 18707c478bd9Sstevel@tonic-gate break; 18717c478bd9Sstevel@tonic-gate } 18727c478bd9Sstevel@tonic-gate (void) mutex_unlock(&ccp->cc_lock); 18737c478bd9Sstevel@tonic-gate 18747c478bd9Sstevel@tonic-gate /* 18757c478bd9Sstevel@tonic-gate * We couldn't free our constructed object to the magazine layer, 18767c478bd9Sstevel@tonic-gate * so apply its destructor and free it to the slab layer. 18777c478bd9Sstevel@tonic-gate * Note that if UMF_BUFTAG is in effect, umem_cache_free_debug() 18787c478bd9Sstevel@tonic-gate * will have already applied the destructor. 18797c478bd9Sstevel@tonic-gate */ 18807c478bd9Sstevel@tonic-gate if (!(cp->cache_flags & UMF_BUFTAG) && cp->cache_destructor != NULL) 18817c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 18827c478bd9Sstevel@tonic-gate 18837c478bd9Sstevel@tonic-gate umem_slab_free(cp, buf); 18847c478bd9Sstevel@tonic-gate } 18857c478bd9Sstevel@tonic-gate 18867c478bd9Sstevel@tonic-gate #pragma weak umem_zalloc = _umem_zalloc 18877c478bd9Sstevel@tonic-gate void * 18887c478bd9Sstevel@tonic-gate _umem_zalloc(size_t size, int umflag) 18897c478bd9Sstevel@tonic-gate { 18907c478bd9Sstevel@tonic-gate size_t index = (size - 1) >> UMEM_ALIGN_SHIFT; 18917c478bd9Sstevel@tonic-gate void *buf; 18927c478bd9Sstevel@tonic-gate 18937c478bd9Sstevel@tonic-gate retry: 18947c478bd9Sstevel@tonic-gate if (index < UMEM_MAXBUF >> UMEM_ALIGN_SHIFT) { 18957c478bd9Sstevel@tonic-gate umem_cache_t *cp = umem_alloc_table[index]; 18967c478bd9Sstevel@tonic-gate buf = _umem_cache_alloc(cp, umflag); 18977c478bd9Sstevel@tonic-gate if (buf != NULL) { 18987c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_BUFTAG) { 18997c478bd9Sstevel@tonic-gate umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); 19007c478bd9Sstevel@tonic-gate ((uint8_t *)buf)[size] = UMEM_REDZONE_BYTE; 19017c478bd9Sstevel@tonic-gate ((uint32_t *)btp)[1] = UMEM_SIZE_ENCODE(size); 19027c478bd9Sstevel@tonic-gate } 19037c478bd9Sstevel@tonic-gate bzero(buf, size); 19047c478bd9Sstevel@tonic-gate } else if (umem_alloc_retry(cp, umflag)) 19057c478bd9Sstevel@tonic-gate goto retry; 19067c478bd9Sstevel@tonic-gate } else { 19077c478bd9Sstevel@tonic-gate buf = _umem_alloc(size, umflag); /* handles failure */ 19087c478bd9Sstevel@tonic-gate if (buf != NULL) 19097c478bd9Sstevel@tonic-gate bzero(buf, size); 19107c478bd9Sstevel@tonic-gate } 19117c478bd9Sstevel@tonic-gate return (buf); 19127c478bd9Sstevel@tonic-gate } 19137c478bd9Sstevel@tonic-gate 19147c478bd9Sstevel@tonic-gate #pragma weak umem_alloc = _umem_alloc 19157c478bd9Sstevel@tonic-gate void * 19167c478bd9Sstevel@tonic-gate _umem_alloc(size_t size, int umflag) 19177c478bd9Sstevel@tonic-gate { 19187c478bd9Sstevel@tonic-gate size_t index = (size - 1) >> UMEM_ALIGN_SHIFT; 19197c478bd9Sstevel@tonic-gate void *buf; 19207c478bd9Sstevel@tonic-gate umem_alloc_retry: 19217c478bd9Sstevel@tonic-gate if (index < UMEM_MAXBUF >> UMEM_ALIGN_SHIFT) { 19227c478bd9Sstevel@tonic-gate umem_cache_t *cp = umem_alloc_table[index]; 19237c478bd9Sstevel@tonic-gate buf = _umem_cache_alloc(cp, umflag); 19247c478bd9Sstevel@tonic-gate if ((cp->cache_flags & UMF_BUFTAG) && buf != NULL) { 19257c478bd9Sstevel@tonic-gate umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); 19267c478bd9Sstevel@tonic-gate ((uint8_t *)buf)[size] = UMEM_REDZONE_BYTE; 19277c478bd9Sstevel@tonic-gate ((uint32_t *)btp)[1] = UMEM_SIZE_ENCODE(size); 19287c478bd9Sstevel@tonic-gate } 19297c478bd9Sstevel@tonic-gate if (buf == NULL && umem_alloc_retry(cp, umflag)) 19307c478bd9Sstevel@tonic-gate goto umem_alloc_retry; 19317c478bd9Sstevel@tonic-gate return (buf); 19327c478bd9Sstevel@tonic-gate } 19337c478bd9Sstevel@tonic-gate if (size == 0) 19347c478bd9Sstevel@tonic-gate return (NULL); 19357c478bd9Sstevel@tonic-gate if (umem_oversize_arena == NULL) { 19367c478bd9Sstevel@tonic-gate if (umem_init()) 19377c478bd9Sstevel@tonic-gate ASSERT(umem_oversize_arena != NULL); 19387c478bd9Sstevel@tonic-gate else 19397c478bd9Sstevel@tonic-gate return (NULL); 19407c478bd9Sstevel@tonic-gate } 19417c478bd9Sstevel@tonic-gate buf = vmem_alloc(umem_oversize_arena, size, UMEM_VMFLAGS(umflag)); 19427c478bd9Sstevel@tonic-gate if (buf == NULL) { 19437c478bd9Sstevel@tonic-gate umem_log_event(umem_failure_log, NULL, NULL, (void *)size); 19447c478bd9Sstevel@tonic-gate if (umem_alloc_retry(NULL, umflag)) 19457c478bd9Sstevel@tonic-gate goto umem_alloc_retry; 19467c478bd9Sstevel@tonic-gate } 19477c478bd9Sstevel@tonic-gate return (buf); 19487c478bd9Sstevel@tonic-gate } 19497c478bd9Sstevel@tonic-gate 19507c478bd9Sstevel@tonic-gate #pragma weak umem_alloc_align = _umem_alloc_align 19517c478bd9Sstevel@tonic-gate void * 19527c478bd9Sstevel@tonic-gate _umem_alloc_align(size_t size, size_t align, int umflag) 19537c478bd9Sstevel@tonic-gate { 19547c478bd9Sstevel@tonic-gate void *buf; 19557c478bd9Sstevel@tonic-gate 19567c478bd9Sstevel@tonic-gate if (size == 0) 19577c478bd9Sstevel@tonic-gate return (NULL); 19587c478bd9Sstevel@tonic-gate if ((align & (align - 1)) != 0) 19597c478bd9Sstevel@tonic-gate return (NULL); 19607c478bd9Sstevel@tonic-gate if (align < UMEM_ALIGN) 19617c478bd9Sstevel@tonic-gate align = UMEM_ALIGN; 19627c478bd9Sstevel@tonic-gate 19637c478bd9Sstevel@tonic-gate umem_alloc_align_retry: 19647c478bd9Sstevel@tonic-gate if (umem_memalign_arena == NULL) { 19657c478bd9Sstevel@tonic-gate if (umem_init()) 19667c478bd9Sstevel@tonic-gate ASSERT(umem_oversize_arena != NULL); 19677c478bd9Sstevel@tonic-gate else 19687c478bd9Sstevel@tonic-gate return (NULL); 19697c478bd9Sstevel@tonic-gate } 19707c478bd9Sstevel@tonic-gate buf = vmem_xalloc(umem_memalign_arena, size, align, 0, 0, NULL, NULL, 19717c478bd9Sstevel@tonic-gate UMEM_VMFLAGS(umflag)); 19727c478bd9Sstevel@tonic-gate if (buf == NULL) { 19737c478bd9Sstevel@tonic-gate umem_log_event(umem_failure_log, NULL, NULL, (void *)size); 19747c478bd9Sstevel@tonic-gate if (umem_alloc_retry(NULL, umflag)) 19757c478bd9Sstevel@tonic-gate goto umem_alloc_align_retry; 19767c478bd9Sstevel@tonic-gate } 19777c478bd9Sstevel@tonic-gate return (buf); 19787c478bd9Sstevel@tonic-gate } 19797c478bd9Sstevel@tonic-gate 19807c478bd9Sstevel@tonic-gate #pragma weak umem_free = _umem_free 19817c478bd9Sstevel@tonic-gate void 19827c478bd9Sstevel@tonic-gate _umem_free(void *buf, size_t size) 19837c478bd9Sstevel@tonic-gate { 19847c478bd9Sstevel@tonic-gate size_t index = (size - 1) >> UMEM_ALIGN_SHIFT; 19857c478bd9Sstevel@tonic-gate 19867c478bd9Sstevel@tonic-gate if (index < UMEM_MAXBUF >> UMEM_ALIGN_SHIFT) { 19877c478bd9Sstevel@tonic-gate umem_cache_t *cp = umem_alloc_table[index]; 19887c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_BUFTAG) { 19897c478bd9Sstevel@tonic-gate umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); 19907c478bd9Sstevel@tonic-gate uint32_t *ip = (uint32_t *)btp; 19917c478bd9Sstevel@tonic-gate if (ip[1] != UMEM_SIZE_ENCODE(size)) { 19927c478bd9Sstevel@tonic-gate if (*(uint64_t *)buf == UMEM_FREE_PATTERN) { 19937c478bd9Sstevel@tonic-gate umem_error(UMERR_DUPFREE, cp, buf); 19947c478bd9Sstevel@tonic-gate return; 19957c478bd9Sstevel@tonic-gate } 19967c478bd9Sstevel@tonic-gate if (UMEM_SIZE_VALID(ip[1])) { 19977c478bd9Sstevel@tonic-gate ip[0] = UMEM_SIZE_ENCODE(size); 19987c478bd9Sstevel@tonic-gate umem_error(UMERR_BADSIZE, cp, buf); 19997c478bd9Sstevel@tonic-gate } else { 20007c478bd9Sstevel@tonic-gate umem_error(UMERR_REDZONE, cp, buf); 20017c478bd9Sstevel@tonic-gate } 20027c478bd9Sstevel@tonic-gate return; 20037c478bd9Sstevel@tonic-gate } 20047c478bd9Sstevel@tonic-gate if (((uint8_t *)buf)[size] != UMEM_REDZONE_BYTE) { 20057c478bd9Sstevel@tonic-gate umem_error(UMERR_REDZONE, cp, buf); 20067c478bd9Sstevel@tonic-gate return; 20077c478bd9Sstevel@tonic-gate } 20087c478bd9Sstevel@tonic-gate btp->bt_redzone = UMEM_REDZONE_PATTERN; 20097c478bd9Sstevel@tonic-gate } 20107c478bd9Sstevel@tonic-gate _umem_cache_free(cp, buf); 20117c478bd9Sstevel@tonic-gate } else { 20127c478bd9Sstevel@tonic-gate if (buf == NULL && size == 0) 20137c478bd9Sstevel@tonic-gate return; 20147c478bd9Sstevel@tonic-gate vmem_free(umem_oversize_arena, buf, size); 20157c478bd9Sstevel@tonic-gate } 20167c478bd9Sstevel@tonic-gate } 20177c478bd9Sstevel@tonic-gate 20187c478bd9Sstevel@tonic-gate #pragma weak umem_free_align = _umem_free_align 20197c478bd9Sstevel@tonic-gate void 20207c478bd9Sstevel@tonic-gate _umem_free_align(void *buf, size_t size) 20217c478bd9Sstevel@tonic-gate { 20227c478bd9Sstevel@tonic-gate if (buf == NULL && size == 0) 20237c478bd9Sstevel@tonic-gate return; 20247c478bd9Sstevel@tonic-gate vmem_xfree(umem_memalign_arena, buf, size); 20257c478bd9Sstevel@tonic-gate } 20267c478bd9Sstevel@tonic-gate 20277c478bd9Sstevel@tonic-gate static void * 20287c478bd9Sstevel@tonic-gate umem_firewall_va_alloc(vmem_t *vmp, size_t size, int vmflag) 20297c478bd9Sstevel@tonic-gate { 20307c478bd9Sstevel@tonic-gate size_t realsize = size + vmp->vm_quantum; 20317c478bd9Sstevel@tonic-gate 20327c478bd9Sstevel@tonic-gate /* 20337c478bd9Sstevel@tonic-gate * Annoying edge case: if 'size' is just shy of ULONG_MAX, adding 20347c478bd9Sstevel@tonic-gate * vm_quantum will cause integer wraparound. Check for this, and 20357c478bd9Sstevel@tonic-gate * blow off the firewall page in this case. Note that such a 20367c478bd9Sstevel@tonic-gate * giant allocation (the entire address space) can never be 20377c478bd9Sstevel@tonic-gate * satisfied, so it will either fail immediately (VM_NOSLEEP) 20387c478bd9Sstevel@tonic-gate * or sleep forever (VM_SLEEP). Thus, there is no need for a 20397c478bd9Sstevel@tonic-gate * corresponding check in umem_firewall_va_free(). 20407c478bd9Sstevel@tonic-gate */ 20417c478bd9Sstevel@tonic-gate if (realsize < size) 20427c478bd9Sstevel@tonic-gate realsize = size; 20437c478bd9Sstevel@tonic-gate 20447c478bd9Sstevel@tonic-gate return (vmem_alloc(vmp, realsize, vmflag | VM_NEXTFIT)); 20457c478bd9Sstevel@tonic-gate } 20467c478bd9Sstevel@tonic-gate 20477c478bd9Sstevel@tonic-gate static void 20487c478bd9Sstevel@tonic-gate umem_firewall_va_free(vmem_t *vmp, void *addr, size_t size) 20497c478bd9Sstevel@tonic-gate { 20507c478bd9Sstevel@tonic-gate vmem_free(vmp, addr, size + vmp->vm_quantum); 20517c478bd9Sstevel@tonic-gate } 20527c478bd9Sstevel@tonic-gate 20537c478bd9Sstevel@tonic-gate /* 20547c478bd9Sstevel@tonic-gate * Reclaim all unused memory from a cache. 20557c478bd9Sstevel@tonic-gate */ 20567c478bd9Sstevel@tonic-gate static void 20577c478bd9Sstevel@tonic-gate umem_cache_reap(umem_cache_t *cp) 20587c478bd9Sstevel@tonic-gate { 20597c478bd9Sstevel@tonic-gate /* 20607c478bd9Sstevel@tonic-gate * Ask the cache's owner to free some memory if possible. 20617c478bd9Sstevel@tonic-gate * The idea is to handle things like the inode cache, which 20627c478bd9Sstevel@tonic-gate * typically sits on a bunch of memory that it doesn't truly 20637c478bd9Sstevel@tonic-gate * *need*. Reclaim policy is entirely up to the owner; this 20647c478bd9Sstevel@tonic-gate * callback is just an advisory plea for help. 20657c478bd9Sstevel@tonic-gate */ 20667c478bd9Sstevel@tonic-gate if (cp->cache_reclaim != NULL) 20677c478bd9Sstevel@tonic-gate cp->cache_reclaim(cp->cache_private); 20687c478bd9Sstevel@tonic-gate 20697c478bd9Sstevel@tonic-gate umem_depot_ws_reap(cp); 20707c478bd9Sstevel@tonic-gate } 20717c478bd9Sstevel@tonic-gate 20727c478bd9Sstevel@tonic-gate /* 20737c478bd9Sstevel@tonic-gate * Purge all magazines from a cache and set its magazine limit to zero. 20747c478bd9Sstevel@tonic-gate * All calls are serialized by being done by the update thread, except for 20757c478bd9Sstevel@tonic-gate * the final call from umem_cache_destroy(). 20767c478bd9Sstevel@tonic-gate */ 20777c478bd9Sstevel@tonic-gate static void 20787c478bd9Sstevel@tonic-gate umem_cache_magazine_purge(umem_cache_t *cp) 20797c478bd9Sstevel@tonic-gate { 20807c478bd9Sstevel@tonic-gate umem_cpu_cache_t *ccp; 20817c478bd9Sstevel@tonic-gate umem_magazine_t *mp, *pmp; 20827c478bd9Sstevel@tonic-gate int rounds, prounds, cpu_seqid; 20837c478bd9Sstevel@tonic-gate 20847c478bd9Sstevel@tonic-gate ASSERT(cp->cache_next == NULL || IN_UPDATE()); 20857c478bd9Sstevel@tonic-gate 20867c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < umem_max_ncpus; cpu_seqid++) { 20877c478bd9Sstevel@tonic-gate ccp = &cp->cache_cpu[cpu_seqid]; 20887c478bd9Sstevel@tonic-gate 20897c478bd9Sstevel@tonic-gate (void) mutex_lock(&ccp->cc_lock); 20907c478bd9Sstevel@tonic-gate mp = ccp->cc_loaded; 20917c478bd9Sstevel@tonic-gate pmp = ccp->cc_ploaded; 20927c478bd9Sstevel@tonic-gate rounds = ccp->cc_rounds; 20937c478bd9Sstevel@tonic-gate prounds = ccp->cc_prounds; 20947c478bd9Sstevel@tonic-gate ccp->cc_loaded = NULL; 20957c478bd9Sstevel@tonic-gate ccp->cc_ploaded = NULL; 20967c478bd9Sstevel@tonic-gate ccp->cc_rounds = -1; 20977c478bd9Sstevel@tonic-gate ccp->cc_prounds = -1; 20987c478bd9Sstevel@tonic-gate ccp->cc_magsize = 0; 20997c478bd9Sstevel@tonic-gate (void) mutex_unlock(&ccp->cc_lock); 21007c478bd9Sstevel@tonic-gate 21017c478bd9Sstevel@tonic-gate if (mp) 21027c478bd9Sstevel@tonic-gate umem_magazine_destroy(cp, mp, rounds); 21037c478bd9Sstevel@tonic-gate if (pmp) 21047c478bd9Sstevel@tonic-gate umem_magazine_destroy(cp, pmp, prounds); 21057c478bd9Sstevel@tonic-gate } 21067c478bd9Sstevel@tonic-gate 21077c478bd9Sstevel@tonic-gate /* 21087c478bd9Sstevel@tonic-gate * Updating the working set statistics twice in a row has the 21097c478bd9Sstevel@tonic-gate * effect of setting the working set size to zero, so everything 21107c478bd9Sstevel@tonic-gate * is eligible for reaping. 21117c478bd9Sstevel@tonic-gate */ 21127c478bd9Sstevel@tonic-gate umem_depot_ws_update(cp); 21137c478bd9Sstevel@tonic-gate umem_depot_ws_update(cp); 21147c478bd9Sstevel@tonic-gate 21157c478bd9Sstevel@tonic-gate umem_depot_ws_reap(cp); 21167c478bd9Sstevel@tonic-gate } 21177c478bd9Sstevel@tonic-gate 21187c478bd9Sstevel@tonic-gate /* 21197c478bd9Sstevel@tonic-gate * Enable per-cpu magazines on a cache. 21207c478bd9Sstevel@tonic-gate */ 21217c478bd9Sstevel@tonic-gate static void 21227c478bd9Sstevel@tonic-gate umem_cache_magazine_enable(umem_cache_t *cp) 21237c478bd9Sstevel@tonic-gate { 21247c478bd9Sstevel@tonic-gate int cpu_seqid; 21257c478bd9Sstevel@tonic-gate 21267c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_NOMAGAZINE) 21277c478bd9Sstevel@tonic-gate return; 21287c478bd9Sstevel@tonic-gate 21297c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < umem_max_ncpus; cpu_seqid++) { 21307c478bd9Sstevel@tonic-gate umem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 21317c478bd9Sstevel@tonic-gate (void) mutex_lock(&ccp->cc_lock); 21327c478bd9Sstevel@tonic-gate ccp->cc_magsize = cp->cache_magtype->mt_magsize; 21337c478bd9Sstevel@tonic-gate (void) mutex_unlock(&ccp->cc_lock); 21347c478bd9Sstevel@tonic-gate } 21357c478bd9Sstevel@tonic-gate 21367c478bd9Sstevel@tonic-gate } 21377c478bd9Sstevel@tonic-gate 21387c478bd9Sstevel@tonic-gate /* 21397c478bd9Sstevel@tonic-gate * Recompute a cache's magazine size. The trade-off is that larger magazines 21407c478bd9Sstevel@tonic-gate * provide a higher transfer rate with the depot, while smaller magazines 21417c478bd9Sstevel@tonic-gate * reduce memory consumption. Magazine resizing is an expensive operation; 21427c478bd9Sstevel@tonic-gate * it should not be done frequently. 21437c478bd9Sstevel@tonic-gate * 21447c478bd9Sstevel@tonic-gate * Changes to the magazine size are serialized by only having one thread 21457c478bd9Sstevel@tonic-gate * doing updates. (the update thread) 21467c478bd9Sstevel@tonic-gate * 21477c478bd9Sstevel@tonic-gate * Note: at present this only grows the magazine size. It might be useful 21487c478bd9Sstevel@tonic-gate * to allow shrinkage too. 21497c478bd9Sstevel@tonic-gate */ 21507c478bd9Sstevel@tonic-gate static void 21517c478bd9Sstevel@tonic-gate umem_cache_magazine_resize(umem_cache_t *cp) 21527c478bd9Sstevel@tonic-gate { 21537c478bd9Sstevel@tonic-gate umem_magtype_t *mtp = cp->cache_magtype; 21547c478bd9Sstevel@tonic-gate 21557c478bd9Sstevel@tonic-gate ASSERT(IN_UPDATE()); 21567c478bd9Sstevel@tonic-gate 21577c478bd9Sstevel@tonic-gate if (cp->cache_chunksize < mtp->mt_maxbuf) { 21587c478bd9Sstevel@tonic-gate umem_cache_magazine_purge(cp); 21597c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_depot_lock); 21607c478bd9Sstevel@tonic-gate cp->cache_magtype = ++mtp; 21617c478bd9Sstevel@tonic-gate cp->cache_depot_contention_prev = 21627c478bd9Sstevel@tonic-gate cp->cache_depot_contention + INT_MAX; 21637c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_depot_lock); 21647c478bd9Sstevel@tonic-gate umem_cache_magazine_enable(cp); 21657c478bd9Sstevel@tonic-gate } 21667c478bd9Sstevel@tonic-gate } 21677c478bd9Sstevel@tonic-gate 21687c478bd9Sstevel@tonic-gate /* 21697c478bd9Sstevel@tonic-gate * Rescale a cache's hash table, so that the table size is roughly the 21707c478bd9Sstevel@tonic-gate * cache size. We want the average lookup time to be extremely small. 21717c478bd9Sstevel@tonic-gate */ 21727c478bd9Sstevel@tonic-gate static void 21737c478bd9Sstevel@tonic-gate umem_hash_rescale(umem_cache_t *cp) 21747c478bd9Sstevel@tonic-gate { 21757c478bd9Sstevel@tonic-gate umem_bufctl_t **old_table, **new_table, *bcp; 21767c478bd9Sstevel@tonic-gate size_t old_size, new_size, h; 21777c478bd9Sstevel@tonic-gate 21787c478bd9Sstevel@tonic-gate ASSERT(IN_UPDATE()); 21797c478bd9Sstevel@tonic-gate 21807c478bd9Sstevel@tonic-gate new_size = MAX(UMEM_HASH_INITIAL, 21817c478bd9Sstevel@tonic-gate 1 << (highbit(3 * cp->cache_buftotal + 4) - 2)); 21827c478bd9Sstevel@tonic-gate old_size = cp->cache_hash_mask + 1; 21837c478bd9Sstevel@tonic-gate 21847c478bd9Sstevel@tonic-gate if ((old_size >> 1) <= new_size && new_size <= (old_size << 1)) 21857c478bd9Sstevel@tonic-gate return; 21867c478bd9Sstevel@tonic-gate 21877c478bd9Sstevel@tonic-gate new_table = vmem_alloc(umem_hash_arena, new_size * sizeof (void *), 21887c478bd9Sstevel@tonic-gate VM_NOSLEEP); 21897c478bd9Sstevel@tonic-gate if (new_table == NULL) 21907c478bd9Sstevel@tonic-gate return; 21917c478bd9Sstevel@tonic-gate bzero(new_table, new_size * sizeof (void *)); 21927c478bd9Sstevel@tonic-gate 21937c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_lock); 21947c478bd9Sstevel@tonic-gate 21957c478bd9Sstevel@tonic-gate old_size = cp->cache_hash_mask + 1; 21967c478bd9Sstevel@tonic-gate old_table = cp->cache_hash_table; 21977c478bd9Sstevel@tonic-gate 21987c478bd9Sstevel@tonic-gate cp->cache_hash_mask = new_size - 1; 21997c478bd9Sstevel@tonic-gate cp->cache_hash_table = new_table; 22007c478bd9Sstevel@tonic-gate cp->cache_rescale++; 22017c478bd9Sstevel@tonic-gate 22027c478bd9Sstevel@tonic-gate for (h = 0; h < old_size; h++) { 22037c478bd9Sstevel@tonic-gate bcp = old_table[h]; 22047c478bd9Sstevel@tonic-gate while (bcp != NULL) { 22057c478bd9Sstevel@tonic-gate void *addr = bcp->bc_addr; 22067c478bd9Sstevel@tonic-gate umem_bufctl_t *next_bcp = bcp->bc_next; 22077c478bd9Sstevel@tonic-gate umem_bufctl_t **hash_bucket = UMEM_HASH(cp, addr); 22087c478bd9Sstevel@tonic-gate bcp->bc_next = *hash_bucket; 22097c478bd9Sstevel@tonic-gate *hash_bucket = bcp; 22107c478bd9Sstevel@tonic-gate bcp = next_bcp; 22117c478bd9Sstevel@tonic-gate } 22127c478bd9Sstevel@tonic-gate } 22137c478bd9Sstevel@tonic-gate 22147c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 22157c478bd9Sstevel@tonic-gate 22167c478bd9Sstevel@tonic-gate vmem_free(umem_hash_arena, old_table, old_size * sizeof (void *)); 22177c478bd9Sstevel@tonic-gate } 22187c478bd9Sstevel@tonic-gate 22197c478bd9Sstevel@tonic-gate /* 22207c478bd9Sstevel@tonic-gate * Perform periodic maintenance on a cache: hash rescaling, 22217c478bd9Sstevel@tonic-gate * depot working-set update, and magazine resizing. 22227c478bd9Sstevel@tonic-gate */ 22237c478bd9Sstevel@tonic-gate void 22247c478bd9Sstevel@tonic-gate umem_cache_update(umem_cache_t *cp) 22257c478bd9Sstevel@tonic-gate { 22267c478bd9Sstevel@tonic-gate int update_flags = 0; 22277c478bd9Sstevel@tonic-gate 22287c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&umem_cache_lock)); 22297c478bd9Sstevel@tonic-gate 22307c478bd9Sstevel@tonic-gate /* 22317c478bd9Sstevel@tonic-gate * If the cache has become much larger or smaller than its hash table, 22327c478bd9Sstevel@tonic-gate * fire off a request to rescale the hash table. 22337c478bd9Sstevel@tonic-gate */ 22347c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_lock); 22357c478bd9Sstevel@tonic-gate 22367c478bd9Sstevel@tonic-gate if ((cp->cache_flags & UMF_HASH) && 22377c478bd9Sstevel@tonic-gate (cp->cache_buftotal > (cp->cache_hash_mask << 1) || 22387c478bd9Sstevel@tonic-gate (cp->cache_buftotal < (cp->cache_hash_mask >> 1) && 22397c478bd9Sstevel@tonic-gate cp->cache_hash_mask > UMEM_HASH_INITIAL))) 22407c478bd9Sstevel@tonic-gate update_flags |= UMU_HASH_RESCALE; 22417c478bd9Sstevel@tonic-gate 22427c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 22437c478bd9Sstevel@tonic-gate 22447c478bd9Sstevel@tonic-gate /* 22457c478bd9Sstevel@tonic-gate * Update the depot working set statistics. 22467c478bd9Sstevel@tonic-gate */ 22477c478bd9Sstevel@tonic-gate umem_depot_ws_update(cp); 22487c478bd9Sstevel@tonic-gate 22497c478bd9Sstevel@tonic-gate /* 22507c478bd9Sstevel@tonic-gate * If there's a lot of contention in the depot, 22517c478bd9Sstevel@tonic-gate * increase the magazine size. 22527c478bd9Sstevel@tonic-gate */ 22537c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_depot_lock); 22547c478bd9Sstevel@tonic-gate 22557c478bd9Sstevel@tonic-gate if (cp->cache_chunksize < cp->cache_magtype->mt_maxbuf && 22567c478bd9Sstevel@tonic-gate (int)(cp->cache_depot_contention - 22577c478bd9Sstevel@tonic-gate cp->cache_depot_contention_prev) > umem_depot_contention) 22587c478bd9Sstevel@tonic-gate update_flags |= UMU_MAGAZINE_RESIZE; 22597c478bd9Sstevel@tonic-gate 22607c478bd9Sstevel@tonic-gate cp->cache_depot_contention_prev = cp->cache_depot_contention; 22617c478bd9Sstevel@tonic-gate 22627c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_depot_lock); 22637c478bd9Sstevel@tonic-gate 22647c478bd9Sstevel@tonic-gate if (update_flags) 22657c478bd9Sstevel@tonic-gate umem_add_update(cp, update_flags); 22667c478bd9Sstevel@tonic-gate } 22677c478bd9Sstevel@tonic-gate 22687c478bd9Sstevel@tonic-gate /* 22697c478bd9Sstevel@tonic-gate * Runs all pending updates. 22707c478bd9Sstevel@tonic-gate * 22717c478bd9Sstevel@tonic-gate * The update lock must be held on entrance, and will be held on exit. 22727c478bd9Sstevel@tonic-gate */ 22737c478bd9Sstevel@tonic-gate void 22747c478bd9Sstevel@tonic-gate umem_process_updates(void) 22757c478bd9Sstevel@tonic-gate { 22767c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&umem_update_lock)); 22777c478bd9Sstevel@tonic-gate 22787c478bd9Sstevel@tonic-gate while (umem_null_cache.cache_unext != &umem_null_cache) { 22797c478bd9Sstevel@tonic-gate int notify = 0; 22807c478bd9Sstevel@tonic-gate umem_cache_t *cp = umem_null_cache.cache_unext; 22817c478bd9Sstevel@tonic-gate 22827c478bd9Sstevel@tonic-gate cp->cache_uprev->cache_unext = cp->cache_unext; 22837c478bd9Sstevel@tonic-gate cp->cache_unext->cache_uprev = cp->cache_uprev; 22847c478bd9Sstevel@tonic-gate cp->cache_uprev = cp->cache_unext = NULL; 22857c478bd9Sstevel@tonic-gate 22867c478bd9Sstevel@tonic-gate ASSERT(!(cp->cache_uflags & UMU_ACTIVE)); 22877c478bd9Sstevel@tonic-gate 22887c478bd9Sstevel@tonic-gate while (cp->cache_uflags) { 22897c478bd9Sstevel@tonic-gate int uflags = (cp->cache_uflags |= UMU_ACTIVE); 22907c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_update_lock); 22917c478bd9Sstevel@tonic-gate 22927c478bd9Sstevel@tonic-gate /* 22937c478bd9Sstevel@tonic-gate * The order here is important. Each step can speed up 22947c478bd9Sstevel@tonic-gate * later steps. 22957c478bd9Sstevel@tonic-gate */ 22967c478bd9Sstevel@tonic-gate 22977c478bd9Sstevel@tonic-gate if (uflags & UMU_HASH_RESCALE) 22987c478bd9Sstevel@tonic-gate umem_hash_rescale(cp); 22997c478bd9Sstevel@tonic-gate 23007c478bd9Sstevel@tonic-gate if (uflags & UMU_MAGAZINE_RESIZE) 23017c478bd9Sstevel@tonic-gate umem_cache_magazine_resize(cp); 23027c478bd9Sstevel@tonic-gate 23037c478bd9Sstevel@tonic-gate if (uflags & UMU_REAP) 23047c478bd9Sstevel@tonic-gate umem_cache_reap(cp); 23057c478bd9Sstevel@tonic-gate 23067c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_update_lock); 23077c478bd9Sstevel@tonic-gate 23087c478bd9Sstevel@tonic-gate /* 23097c478bd9Sstevel@tonic-gate * check if anyone has requested notification 23107c478bd9Sstevel@tonic-gate */ 23117c478bd9Sstevel@tonic-gate if (cp->cache_uflags & UMU_NOTIFY) { 23127c478bd9Sstevel@tonic-gate uflags |= UMU_NOTIFY; 23137c478bd9Sstevel@tonic-gate notify = 1; 23147c478bd9Sstevel@tonic-gate } 23157c478bd9Sstevel@tonic-gate cp->cache_uflags &= ~uflags; 23167c478bd9Sstevel@tonic-gate } 23177c478bd9Sstevel@tonic-gate if (notify) 23187c478bd9Sstevel@tonic-gate (void) cond_broadcast(&umem_update_cv); 23197c478bd9Sstevel@tonic-gate } 23207c478bd9Sstevel@tonic-gate } 23217c478bd9Sstevel@tonic-gate 23227c478bd9Sstevel@tonic-gate #ifndef UMEM_STANDALONE 23237c478bd9Sstevel@tonic-gate static void 23247c478bd9Sstevel@tonic-gate umem_st_update(void) 23257c478bd9Sstevel@tonic-gate { 23267c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&umem_update_lock)); 23277c478bd9Sstevel@tonic-gate ASSERT(umem_update_thr == 0 && umem_st_update_thr == 0); 23287c478bd9Sstevel@tonic-gate 23297c478bd9Sstevel@tonic-gate umem_st_update_thr = thr_self(); 23307c478bd9Sstevel@tonic-gate 23317c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_update_lock); 23327c478bd9Sstevel@tonic-gate 23337c478bd9Sstevel@tonic-gate vmem_update(NULL); 23347c478bd9Sstevel@tonic-gate umem_cache_applyall(umem_cache_update); 23357c478bd9Sstevel@tonic-gate 23367c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_update_lock); 23377c478bd9Sstevel@tonic-gate 23387c478bd9Sstevel@tonic-gate umem_process_updates(); /* does all of the requested work */ 23397c478bd9Sstevel@tonic-gate 23407c478bd9Sstevel@tonic-gate umem_reap_next = gethrtime() + 23417c478bd9Sstevel@tonic-gate (hrtime_t)umem_reap_interval * NANOSEC; 23427c478bd9Sstevel@tonic-gate 23437c478bd9Sstevel@tonic-gate umem_reaping = UMEM_REAP_DONE; 23447c478bd9Sstevel@tonic-gate 23457c478bd9Sstevel@tonic-gate umem_st_update_thr = 0; 23467c478bd9Sstevel@tonic-gate } 23477c478bd9Sstevel@tonic-gate #endif 23487c478bd9Sstevel@tonic-gate 23497c478bd9Sstevel@tonic-gate /* 23507c478bd9Sstevel@tonic-gate * Reclaim all unused memory from all caches. Called from vmem when memory 23517c478bd9Sstevel@tonic-gate * gets tight. Must be called with no locks held. 23527c478bd9Sstevel@tonic-gate * 23537c478bd9Sstevel@tonic-gate * This just requests a reap on all caches, and notifies the update thread. 23547c478bd9Sstevel@tonic-gate */ 23557c478bd9Sstevel@tonic-gate void 23567c478bd9Sstevel@tonic-gate umem_reap(void) 23577c478bd9Sstevel@tonic-gate { 23587c478bd9Sstevel@tonic-gate #ifndef UMEM_STANDALONE 23597c478bd9Sstevel@tonic-gate extern int __nthreads(void); 23607c478bd9Sstevel@tonic-gate #endif 23617c478bd9Sstevel@tonic-gate 23627c478bd9Sstevel@tonic-gate if (umem_ready != UMEM_READY || umem_reaping != UMEM_REAP_DONE || 23637c478bd9Sstevel@tonic-gate gethrtime() < umem_reap_next) 23647c478bd9Sstevel@tonic-gate return; 23657c478bd9Sstevel@tonic-gate 23667c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_update_lock); 23677c478bd9Sstevel@tonic-gate 23687c478bd9Sstevel@tonic-gate if (umem_reaping != UMEM_REAP_DONE || gethrtime() < umem_reap_next) { 23697c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_update_lock); 23707c478bd9Sstevel@tonic-gate return; 23717c478bd9Sstevel@tonic-gate } 23727c478bd9Sstevel@tonic-gate umem_reaping = UMEM_REAP_ADDING; /* lock out other reaps */ 23737c478bd9Sstevel@tonic-gate 23747c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_update_lock); 23757c478bd9Sstevel@tonic-gate 23767c478bd9Sstevel@tonic-gate umem_updateall(UMU_REAP); 23777c478bd9Sstevel@tonic-gate 23787c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_update_lock); 23797c478bd9Sstevel@tonic-gate 23807c478bd9Sstevel@tonic-gate umem_reaping = UMEM_REAP_ACTIVE; 23817c478bd9Sstevel@tonic-gate 23827c478bd9Sstevel@tonic-gate /* Standalone is single-threaded */ 23837c478bd9Sstevel@tonic-gate #ifndef UMEM_STANDALONE 23847c478bd9Sstevel@tonic-gate if (umem_update_thr == 0) { 23857c478bd9Sstevel@tonic-gate /* 23867c478bd9Sstevel@tonic-gate * The update thread does not exist. If the process is 23877c478bd9Sstevel@tonic-gate * multi-threaded, create it. If not, or the creation fails, 23887c478bd9Sstevel@tonic-gate * do the update processing inline. 23897c478bd9Sstevel@tonic-gate */ 23907c478bd9Sstevel@tonic-gate ASSERT(umem_st_update_thr == 0); 23917c478bd9Sstevel@tonic-gate 23927c478bd9Sstevel@tonic-gate if (__nthreads() <= 1 || umem_create_update_thread() == 0) 23937c478bd9Sstevel@tonic-gate umem_st_update(); 23947c478bd9Sstevel@tonic-gate } 23957c478bd9Sstevel@tonic-gate 23967c478bd9Sstevel@tonic-gate (void) cond_broadcast(&umem_update_cv); /* wake up the update thread */ 23977c478bd9Sstevel@tonic-gate #endif 23987c478bd9Sstevel@tonic-gate 23997c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_update_lock); 24007c478bd9Sstevel@tonic-gate } 24017c478bd9Sstevel@tonic-gate 24027c478bd9Sstevel@tonic-gate umem_cache_t * 24037c478bd9Sstevel@tonic-gate umem_cache_create( 24047c478bd9Sstevel@tonic-gate char *name, /* descriptive name for this cache */ 24057c478bd9Sstevel@tonic-gate size_t bufsize, /* size of the objects it manages */ 24067c478bd9Sstevel@tonic-gate size_t align, /* required object alignment */ 24077c478bd9Sstevel@tonic-gate umem_constructor_t *constructor, /* object constructor */ 24087c478bd9Sstevel@tonic-gate umem_destructor_t *destructor, /* object destructor */ 24097c478bd9Sstevel@tonic-gate umem_reclaim_t *reclaim, /* memory reclaim callback */ 24107c478bd9Sstevel@tonic-gate void *private, /* pass-thru arg for constr/destr/reclaim */ 24117c478bd9Sstevel@tonic-gate vmem_t *vmp, /* vmem source for slab allocation */ 24127c478bd9Sstevel@tonic-gate int cflags) /* cache creation flags */ 24137c478bd9Sstevel@tonic-gate { 24147c478bd9Sstevel@tonic-gate int cpu_seqid; 24157c478bd9Sstevel@tonic-gate size_t chunksize; 24167c478bd9Sstevel@tonic-gate umem_cache_t *cp, *cnext, *cprev; 24177c478bd9Sstevel@tonic-gate umem_magtype_t *mtp; 24187c478bd9Sstevel@tonic-gate size_t csize; 24197c478bd9Sstevel@tonic-gate size_t phase; 24207c478bd9Sstevel@tonic-gate 24217c478bd9Sstevel@tonic-gate /* 24227c478bd9Sstevel@tonic-gate * The init thread is allowed to create internal and quantum caches. 24237c478bd9Sstevel@tonic-gate * 24247c478bd9Sstevel@tonic-gate * Other threads must wait until until initialization is complete. 24257c478bd9Sstevel@tonic-gate */ 24267c478bd9Sstevel@tonic-gate if (umem_init_thr == thr_self()) 24277c478bd9Sstevel@tonic-gate ASSERT((cflags & (UMC_INTERNAL | UMC_QCACHE)) != 0); 24287c478bd9Sstevel@tonic-gate else { 24297c478bd9Sstevel@tonic-gate ASSERT(!(cflags & UMC_INTERNAL)); 24307c478bd9Sstevel@tonic-gate if (umem_ready != UMEM_READY && umem_init() == 0) { 24317c478bd9Sstevel@tonic-gate errno = EAGAIN; 24327c478bd9Sstevel@tonic-gate return (NULL); 24337c478bd9Sstevel@tonic-gate } 24347c478bd9Sstevel@tonic-gate } 24357c478bd9Sstevel@tonic-gate 24367c478bd9Sstevel@tonic-gate csize = UMEM_CACHE_SIZE(umem_max_ncpus); 24377c478bd9Sstevel@tonic-gate phase = P2NPHASE(csize, UMEM_CPU_CACHE_SIZE); 24387c478bd9Sstevel@tonic-gate 24397c478bd9Sstevel@tonic-gate if (vmp == NULL) 24407c478bd9Sstevel@tonic-gate vmp = umem_default_arena; 24417c478bd9Sstevel@tonic-gate 24427c478bd9Sstevel@tonic-gate ASSERT(P2PHASE(phase, UMEM_ALIGN) == 0); 24437c478bd9Sstevel@tonic-gate 24447c478bd9Sstevel@tonic-gate /* 24457c478bd9Sstevel@tonic-gate * Check that the arguments are reasonable 24467c478bd9Sstevel@tonic-gate */ 24477c478bd9Sstevel@tonic-gate if ((align & (align - 1)) != 0 || align > vmp->vm_quantum || 24487c478bd9Sstevel@tonic-gate ((cflags & UMC_NOHASH) && (cflags & UMC_NOTOUCH)) || 24497c478bd9Sstevel@tonic-gate name == NULL || bufsize == 0) { 24507c478bd9Sstevel@tonic-gate errno = EINVAL; 24517c478bd9Sstevel@tonic-gate return (NULL); 24527c478bd9Sstevel@tonic-gate } 24537c478bd9Sstevel@tonic-gate 24547c478bd9Sstevel@tonic-gate /* 24557c478bd9Sstevel@tonic-gate * If align == 0, we set it to the minimum required alignment. 24567c478bd9Sstevel@tonic-gate * 24577c478bd9Sstevel@tonic-gate * If align < UMEM_ALIGN, we round it up to UMEM_ALIGN, unless 24587c478bd9Sstevel@tonic-gate * UMC_NOTOUCH was passed. 24597c478bd9Sstevel@tonic-gate */ 24607c478bd9Sstevel@tonic-gate if (align == 0) { 24617c478bd9Sstevel@tonic-gate if (P2ROUNDUP(bufsize, UMEM_ALIGN) >= UMEM_SECOND_ALIGN) 24627c478bd9Sstevel@tonic-gate align = UMEM_SECOND_ALIGN; 24637c478bd9Sstevel@tonic-gate else 24647c478bd9Sstevel@tonic-gate align = UMEM_ALIGN; 24657c478bd9Sstevel@tonic-gate } else if (align < UMEM_ALIGN && (cflags & UMC_NOTOUCH) == 0) 24667c478bd9Sstevel@tonic-gate align = UMEM_ALIGN; 24677c478bd9Sstevel@tonic-gate 24687c478bd9Sstevel@tonic-gate 24697c478bd9Sstevel@tonic-gate /* 24707c478bd9Sstevel@tonic-gate * Get a umem_cache structure. We arrange that cp->cache_cpu[] 24717c478bd9Sstevel@tonic-gate * is aligned on a UMEM_CPU_CACHE_SIZE boundary to prevent 24727c478bd9Sstevel@tonic-gate * false sharing of per-CPU data. 24737c478bd9Sstevel@tonic-gate */ 24747c478bd9Sstevel@tonic-gate cp = vmem_xalloc(umem_cache_arena, csize, UMEM_CPU_CACHE_SIZE, phase, 24757c478bd9Sstevel@tonic-gate 0, NULL, NULL, VM_NOSLEEP); 24767c478bd9Sstevel@tonic-gate 24777c478bd9Sstevel@tonic-gate if (cp == NULL) { 24787c478bd9Sstevel@tonic-gate errno = EAGAIN; 24797c478bd9Sstevel@tonic-gate return (NULL); 24807c478bd9Sstevel@tonic-gate } 24817c478bd9Sstevel@tonic-gate 24827c478bd9Sstevel@tonic-gate bzero(cp, csize); 24837c478bd9Sstevel@tonic-gate 24847c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_flags_lock); 24857c478bd9Sstevel@tonic-gate if (umem_flags & UMF_RANDOMIZE) 24867c478bd9Sstevel@tonic-gate umem_flags = (((umem_flags | ~UMF_RANDOM) + 1) & UMF_RANDOM) | 24877c478bd9Sstevel@tonic-gate UMF_RANDOMIZE; 24887c478bd9Sstevel@tonic-gate cp->cache_flags = umem_flags | (cflags & UMF_DEBUG); 24897c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_flags_lock); 24907c478bd9Sstevel@tonic-gate 24917c478bd9Sstevel@tonic-gate /* 24927c478bd9Sstevel@tonic-gate * Make sure all the various flags are reasonable. 24937c478bd9Sstevel@tonic-gate */ 24947c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_LITE) { 24957c478bd9Sstevel@tonic-gate if (bufsize >= umem_lite_minsize && 24967c478bd9Sstevel@tonic-gate align <= umem_lite_maxalign && 24977c478bd9Sstevel@tonic-gate P2PHASE(bufsize, umem_lite_maxalign) != 0) { 24987c478bd9Sstevel@tonic-gate cp->cache_flags |= UMF_BUFTAG; 24997c478bd9Sstevel@tonic-gate cp->cache_flags &= ~(UMF_AUDIT | UMF_FIREWALL); 25007c478bd9Sstevel@tonic-gate } else { 25017c478bd9Sstevel@tonic-gate cp->cache_flags &= ~UMF_DEBUG; 25027c478bd9Sstevel@tonic-gate } 25037c478bd9Sstevel@tonic-gate } 25047c478bd9Sstevel@tonic-gate 25057c478bd9Sstevel@tonic-gate if ((cflags & UMC_QCACHE) && (cp->cache_flags & UMF_AUDIT)) 25067c478bd9Sstevel@tonic-gate cp->cache_flags |= UMF_NOMAGAZINE; 25077c478bd9Sstevel@tonic-gate 25087c478bd9Sstevel@tonic-gate if (cflags & UMC_NODEBUG) 25097c478bd9Sstevel@tonic-gate cp->cache_flags &= ~UMF_DEBUG; 25107c478bd9Sstevel@tonic-gate 25117c478bd9Sstevel@tonic-gate if (cflags & UMC_NOTOUCH) 25127c478bd9Sstevel@tonic-gate cp->cache_flags &= ~UMF_TOUCH; 25137c478bd9Sstevel@tonic-gate 25147c478bd9Sstevel@tonic-gate if (cflags & UMC_NOHASH) 25157c478bd9Sstevel@tonic-gate cp->cache_flags &= ~(UMF_AUDIT | UMF_FIREWALL); 25167c478bd9Sstevel@tonic-gate 25177c478bd9Sstevel@tonic-gate if (cflags & UMC_NOMAGAZINE) 25187c478bd9Sstevel@tonic-gate cp->cache_flags |= UMF_NOMAGAZINE; 25197c478bd9Sstevel@tonic-gate 25207c478bd9Sstevel@tonic-gate if ((cp->cache_flags & UMF_AUDIT) && !(cflags & UMC_NOTOUCH)) 25217c478bd9Sstevel@tonic-gate cp->cache_flags |= UMF_REDZONE; 25227c478bd9Sstevel@tonic-gate 25237c478bd9Sstevel@tonic-gate if ((cp->cache_flags & UMF_BUFTAG) && bufsize >= umem_minfirewall && 25247c478bd9Sstevel@tonic-gate !(cp->cache_flags & UMF_LITE) && !(cflags & UMC_NOHASH)) 25257c478bd9Sstevel@tonic-gate cp->cache_flags |= UMF_FIREWALL; 25267c478bd9Sstevel@tonic-gate 25277c478bd9Sstevel@tonic-gate if (vmp != umem_default_arena || umem_firewall_arena == NULL) 25287c478bd9Sstevel@tonic-gate cp->cache_flags &= ~UMF_FIREWALL; 25297c478bd9Sstevel@tonic-gate 25307c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_FIREWALL) { 25317c478bd9Sstevel@tonic-gate cp->cache_flags &= ~UMF_BUFTAG; 25327c478bd9Sstevel@tonic-gate cp->cache_flags |= UMF_NOMAGAZINE; 25337c478bd9Sstevel@tonic-gate ASSERT(vmp == umem_default_arena); 25347c478bd9Sstevel@tonic-gate vmp = umem_firewall_arena; 25357c478bd9Sstevel@tonic-gate } 25367c478bd9Sstevel@tonic-gate 25377c478bd9Sstevel@tonic-gate /* 25387c478bd9Sstevel@tonic-gate * Set cache properties. 25397c478bd9Sstevel@tonic-gate */ 25407c478bd9Sstevel@tonic-gate (void) strncpy(cp->cache_name, name, sizeof (cp->cache_name) - 1); 25417c478bd9Sstevel@tonic-gate cp->cache_bufsize = bufsize; 25427c478bd9Sstevel@tonic-gate cp->cache_align = align; 25437c478bd9Sstevel@tonic-gate cp->cache_constructor = constructor; 25447c478bd9Sstevel@tonic-gate cp->cache_destructor = destructor; 25457c478bd9Sstevel@tonic-gate cp->cache_reclaim = reclaim; 25467c478bd9Sstevel@tonic-gate cp->cache_private = private; 25477c478bd9Sstevel@tonic-gate cp->cache_arena = vmp; 25487c478bd9Sstevel@tonic-gate cp->cache_cflags = cflags; 25497c478bd9Sstevel@tonic-gate cp->cache_cpu_mask = umem_cpu_mask; 25507c478bd9Sstevel@tonic-gate 25517c478bd9Sstevel@tonic-gate /* 25527c478bd9Sstevel@tonic-gate * Determine the chunk size. 25537c478bd9Sstevel@tonic-gate */ 25547c478bd9Sstevel@tonic-gate chunksize = bufsize; 25557c478bd9Sstevel@tonic-gate 25567c478bd9Sstevel@tonic-gate if (align >= UMEM_ALIGN) { 25577c478bd9Sstevel@tonic-gate chunksize = P2ROUNDUP(chunksize, UMEM_ALIGN); 25587c478bd9Sstevel@tonic-gate cp->cache_bufctl = chunksize - UMEM_ALIGN; 25597c478bd9Sstevel@tonic-gate } 25607c478bd9Sstevel@tonic-gate 25617c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_BUFTAG) { 25627c478bd9Sstevel@tonic-gate cp->cache_bufctl = chunksize; 25637c478bd9Sstevel@tonic-gate cp->cache_buftag = chunksize; 25647c478bd9Sstevel@tonic-gate chunksize += sizeof (umem_buftag_t); 25657c478bd9Sstevel@tonic-gate } 25667c478bd9Sstevel@tonic-gate 25677c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_DEADBEEF) { 25687c478bd9Sstevel@tonic-gate cp->cache_verify = MIN(cp->cache_buftag, umem_maxverify); 25697c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_LITE) 25707c478bd9Sstevel@tonic-gate cp->cache_verify = MIN(cp->cache_verify, UMEM_ALIGN); 25717c478bd9Sstevel@tonic-gate } 25727c478bd9Sstevel@tonic-gate 25737c478bd9Sstevel@tonic-gate cp->cache_contents = MIN(cp->cache_bufctl, umem_content_maxsave); 25747c478bd9Sstevel@tonic-gate 25757c478bd9Sstevel@tonic-gate cp->cache_chunksize = chunksize = P2ROUNDUP(chunksize, align); 25767c478bd9Sstevel@tonic-gate 25777c478bd9Sstevel@tonic-gate if (chunksize < bufsize) { 25787c478bd9Sstevel@tonic-gate errno = ENOMEM; 25797c478bd9Sstevel@tonic-gate goto fail; 25807c478bd9Sstevel@tonic-gate } 25817c478bd9Sstevel@tonic-gate 25827c478bd9Sstevel@tonic-gate /* 25837c478bd9Sstevel@tonic-gate * Now that we know the chunk size, determine the optimal slab size. 25847c478bd9Sstevel@tonic-gate */ 25857c478bd9Sstevel@tonic-gate if (vmp == umem_firewall_arena) { 25867c478bd9Sstevel@tonic-gate cp->cache_slabsize = P2ROUNDUP(chunksize, vmp->vm_quantum); 25877c478bd9Sstevel@tonic-gate cp->cache_mincolor = cp->cache_slabsize - chunksize; 25887c478bd9Sstevel@tonic-gate cp->cache_maxcolor = cp->cache_mincolor; 25897c478bd9Sstevel@tonic-gate cp->cache_flags |= UMF_HASH; 25907c478bd9Sstevel@tonic-gate ASSERT(!(cp->cache_flags & UMF_BUFTAG)); 25917c478bd9Sstevel@tonic-gate } else if ((cflags & UMC_NOHASH) || (!(cflags & UMC_NOTOUCH) && 25927c478bd9Sstevel@tonic-gate !(cp->cache_flags & UMF_AUDIT) && 25937c478bd9Sstevel@tonic-gate chunksize < vmp->vm_quantum / UMEM_VOID_FRACTION)) { 25947c478bd9Sstevel@tonic-gate cp->cache_slabsize = vmp->vm_quantum; 25957c478bd9Sstevel@tonic-gate cp->cache_mincolor = 0; 25967c478bd9Sstevel@tonic-gate cp->cache_maxcolor = 25977c478bd9Sstevel@tonic-gate (cp->cache_slabsize - sizeof (umem_slab_t)) % chunksize; 25987c478bd9Sstevel@tonic-gate 25997c478bd9Sstevel@tonic-gate if (chunksize + sizeof (umem_slab_t) > cp->cache_slabsize) { 26007c478bd9Sstevel@tonic-gate errno = EINVAL; 26017c478bd9Sstevel@tonic-gate goto fail; 26027c478bd9Sstevel@tonic-gate } 26037c478bd9Sstevel@tonic-gate ASSERT(!(cp->cache_flags & UMF_AUDIT)); 26047c478bd9Sstevel@tonic-gate } else { 26057c478bd9Sstevel@tonic-gate size_t chunks, bestfit, waste, slabsize; 26067c478bd9Sstevel@tonic-gate size_t minwaste = LONG_MAX; 26077c478bd9Sstevel@tonic-gate 26087c478bd9Sstevel@tonic-gate for (chunks = 1; chunks <= UMEM_VOID_FRACTION; chunks++) { 26097c478bd9Sstevel@tonic-gate slabsize = P2ROUNDUP(chunksize * chunks, 26107c478bd9Sstevel@tonic-gate vmp->vm_quantum); 26117c478bd9Sstevel@tonic-gate /* 26127c478bd9Sstevel@tonic-gate * check for overflow 26137c478bd9Sstevel@tonic-gate */ 26147c478bd9Sstevel@tonic-gate if ((slabsize / chunks) < chunksize) { 26157c478bd9Sstevel@tonic-gate errno = ENOMEM; 26167c478bd9Sstevel@tonic-gate goto fail; 26177c478bd9Sstevel@tonic-gate } 26187c478bd9Sstevel@tonic-gate chunks = slabsize / chunksize; 26197c478bd9Sstevel@tonic-gate waste = (slabsize % chunksize) / chunks; 26207c478bd9Sstevel@tonic-gate if (waste < minwaste) { 26217c478bd9Sstevel@tonic-gate minwaste = waste; 26227c478bd9Sstevel@tonic-gate bestfit = slabsize; 26237c478bd9Sstevel@tonic-gate } 26247c478bd9Sstevel@tonic-gate } 26257c478bd9Sstevel@tonic-gate if (cflags & UMC_QCACHE) 26267c478bd9Sstevel@tonic-gate bestfit = MAX(1 << highbit(3 * vmp->vm_qcache_max), 64); 26277c478bd9Sstevel@tonic-gate cp->cache_slabsize = bestfit; 26287c478bd9Sstevel@tonic-gate cp->cache_mincolor = 0; 26297c478bd9Sstevel@tonic-gate cp->cache_maxcolor = bestfit % chunksize; 26307c478bd9Sstevel@tonic-gate cp->cache_flags |= UMF_HASH; 26317c478bd9Sstevel@tonic-gate } 26327c478bd9Sstevel@tonic-gate 26337c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_HASH) { 26347c478bd9Sstevel@tonic-gate ASSERT(!(cflags & UMC_NOHASH)); 26357c478bd9Sstevel@tonic-gate cp->cache_bufctl_cache = (cp->cache_flags & UMF_AUDIT) ? 26367c478bd9Sstevel@tonic-gate umem_bufctl_audit_cache : umem_bufctl_cache; 26377c478bd9Sstevel@tonic-gate } 26387c478bd9Sstevel@tonic-gate 26397c478bd9Sstevel@tonic-gate if (cp->cache_maxcolor >= vmp->vm_quantum) 26407c478bd9Sstevel@tonic-gate cp->cache_maxcolor = vmp->vm_quantum - 1; 26417c478bd9Sstevel@tonic-gate 26427c478bd9Sstevel@tonic-gate cp->cache_color = cp->cache_mincolor; 26437c478bd9Sstevel@tonic-gate 26447c478bd9Sstevel@tonic-gate /* 26457c478bd9Sstevel@tonic-gate * Initialize the rest of the slab layer. 26467c478bd9Sstevel@tonic-gate */ 26477c478bd9Sstevel@tonic-gate (void) mutex_init(&cp->cache_lock, USYNC_THREAD, NULL); 26487c478bd9Sstevel@tonic-gate 26497c478bd9Sstevel@tonic-gate cp->cache_freelist = &cp->cache_nullslab; 26507c478bd9Sstevel@tonic-gate cp->cache_nullslab.slab_cache = cp; 26517c478bd9Sstevel@tonic-gate cp->cache_nullslab.slab_refcnt = -1; 26527c478bd9Sstevel@tonic-gate cp->cache_nullslab.slab_next = &cp->cache_nullslab; 26537c478bd9Sstevel@tonic-gate cp->cache_nullslab.slab_prev = &cp->cache_nullslab; 26547c478bd9Sstevel@tonic-gate 26557c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_HASH) { 26567c478bd9Sstevel@tonic-gate cp->cache_hash_table = vmem_alloc(umem_hash_arena, 26577c478bd9Sstevel@tonic-gate UMEM_HASH_INITIAL * sizeof (void *), VM_NOSLEEP); 26587c478bd9Sstevel@tonic-gate if (cp->cache_hash_table == NULL) { 26597c478bd9Sstevel@tonic-gate errno = EAGAIN; 26607c478bd9Sstevel@tonic-gate goto fail_lock; 26617c478bd9Sstevel@tonic-gate } 26627c478bd9Sstevel@tonic-gate bzero(cp->cache_hash_table, 26637c478bd9Sstevel@tonic-gate UMEM_HASH_INITIAL * sizeof (void *)); 26647c478bd9Sstevel@tonic-gate cp->cache_hash_mask = UMEM_HASH_INITIAL - 1; 26657c478bd9Sstevel@tonic-gate cp->cache_hash_shift = highbit((ulong_t)chunksize) - 1; 26667c478bd9Sstevel@tonic-gate } 26677c478bd9Sstevel@tonic-gate 26687c478bd9Sstevel@tonic-gate /* 26697c478bd9Sstevel@tonic-gate * Initialize the depot. 26707c478bd9Sstevel@tonic-gate */ 26717c478bd9Sstevel@tonic-gate (void) mutex_init(&cp->cache_depot_lock, USYNC_THREAD, NULL); 26727c478bd9Sstevel@tonic-gate 26737c478bd9Sstevel@tonic-gate for (mtp = umem_magtype; chunksize <= mtp->mt_minbuf; mtp++) 26747c478bd9Sstevel@tonic-gate continue; 26757c478bd9Sstevel@tonic-gate 26767c478bd9Sstevel@tonic-gate cp->cache_magtype = mtp; 26777c478bd9Sstevel@tonic-gate 26787c478bd9Sstevel@tonic-gate /* 26797c478bd9Sstevel@tonic-gate * Initialize the CPU layer. 26807c478bd9Sstevel@tonic-gate */ 26817c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < umem_max_ncpus; cpu_seqid++) { 26827c478bd9Sstevel@tonic-gate umem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 26837c478bd9Sstevel@tonic-gate (void) mutex_init(&ccp->cc_lock, USYNC_THREAD, NULL); 26847c478bd9Sstevel@tonic-gate ccp->cc_flags = cp->cache_flags; 26857c478bd9Sstevel@tonic-gate ccp->cc_rounds = -1; 26867c478bd9Sstevel@tonic-gate ccp->cc_prounds = -1; 26877c478bd9Sstevel@tonic-gate } 26887c478bd9Sstevel@tonic-gate 26897c478bd9Sstevel@tonic-gate /* 26907c478bd9Sstevel@tonic-gate * Add the cache to the global list. This makes it visible 26917c478bd9Sstevel@tonic-gate * to umem_update(), so the cache must be ready for business. 26927c478bd9Sstevel@tonic-gate */ 26937c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_cache_lock); 26947c478bd9Sstevel@tonic-gate cp->cache_next = cnext = &umem_null_cache; 26957c478bd9Sstevel@tonic-gate cp->cache_prev = cprev = umem_null_cache.cache_prev; 26967c478bd9Sstevel@tonic-gate cnext->cache_prev = cp; 26977c478bd9Sstevel@tonic-gate cprev->cache_next = cp; 26987c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_cache_lock); 26997c478bd9Sstevel@tonic-gate 27007c478bd9Sstevel@tonic-gate if (umem_ready == UMEM_READY) 27017c478bd9Sstevel@tonic-gate umem_cache_magazine_enable(cp); 27027c478bd9Sstevel@tonic-gate 27037c478bd9Sstevel@tonic-gate return (cp); 27047c478bd9Sstevel@tonic-gate 27057c478bd9Sstevel@tonic-gate fail_lock: 27067c478bd9Sstevel@tonic-gate (void) mutex_destroy(&cp->cache_lock); 27077c478bd9Sstevel@tonic-gate fail: 27087c478bd9Sstevel@tonic-gate vmem_xfree(umem_cache_arena, cp, csize); 27097c478bd9Sstevel@tonic-gate return (NULL); 27107c478bd9Sstevel@tonic-gate } 27117c478bd9Sstevel@tonic-gate 27127c478bd9Sstevel@tonic-gate void 27137c478bd9Sstevel@tonic-gate umem_cache_destroy(umem_cache_t *cp) 27147c478bd9Sstevel@tonic-gate { 27157c478bd9Sstevel@tonic-gate int cpu_seqid; 27167c478bd9Sstevel@tonic-gate 27177c478bd9Sstevel@tonic-gate /* 27187c478bd9Sstevel@tonic-gate * Remove the cache from the global cache list so that no new updates 27197c478bd9Sstevel@tonic-gate * will be scheduled on its behalf, wait for any pending tasks to 27207c478bd9Sstevel@tonic-gate * complete, purge the cache, and then destroy it. 27217c478bd9Sstevel@tonic-gate */ 27227c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_cache_lock); 27237c478bd9Sstevel@tonic-gate cp->cache_prev->cache_next = cp->cache_next; 27247c478bd9Sstevel@tonic-gate cp->cache_next->cache_prev = cp->cache_prev; 27257c478bd9Sstevel@tonic-gate cp->cache_prev = cp->cache_next = NULL; 27267c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_cache_lock); 27277c478bd9Sstevel@tonic-gate 27287c478bd9Sstevel@tonic-gate umem_remove_updates(cp); 27297c478bd9Sstevel@tonic-gate 27307c478bd9Sstevel@tonic-gate umem_cache_magazine_purge(cp); 27317c478bd9Sstevel@tonic-gate 27327c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_lock); 27337c478bd9Sstevel@tonic-gate if (cp->cache_buftotal != 0) 27347c478bd9Sstevel@tonic-gate log_message("umem_cache_destroy: '%s' (%p) not empty\n", 27357c478bd9Sstevel@tonic-gate cp->cache_name, (void *)cp); 27367c478bd9Sstevel@tonic-gate cp->cache_reclaim = NULL; 27377c478bd9Sstevel@tonic-gate /* 27387c478bd9Sstevel@tonic-gate * The cache is now dead. There should be no further activity. 27397c478bd9Sstevel@tonic-gate * We enforce this by setting land mines in the constructor and 27407c478bd9Sstevel@tonic-gate * destructor routines that induce a segmentation fault if invoked. 27417c478bd9Sstevel@tonic-gate */ 27427c478bd9Sstevel@tonic-gate cp->cache_constructor = (umem_constructor_t *)1; 27437c478bd9Sstevel@tonic-gate cp->cache_destructor = (umem_destructor_t *)2; 27447c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 27457c478bd9Sstevel@tonic-gate 27467c478bd9Sstevel@tonic-gate if (cp->cache_hash_table != NULL) 27477c478bd9Sstevel@tonic-gate vmem_free(umem_hash_arena, cp->cache_hash_table, 27487c478bd9Sstevel@tonic-gate (cp->cache_hash_mask + 1) * sizeof (void *)); 27497c478bd9Sstevel@tonic-gate 27507c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < umem_max_ncpus; cpu_seqid++) 27517c478bd9Sstevel@tonic-gate (void) mutex_destroy(&cp->cache_cpu[cpu_seqid].cc_lock); 27527c478bd9Sstevel@tonic-gate 27537c478bd9Sstevel@tonic-gate (void) mutex_destroy(&cp->cache_depot_lock); 27547c478bd9Sstevel@tonic-gate (void) mutex_destroy(&cp->cache_lock); 27557c478bd9Sstevel@tonic-gate 27567c478bd9Sstevel@tonic-gate vmem_free(umem_cache_arena, cp, UMEM_CACHE_SIZE(umem_max_ncpus)); 27577c478bd9Sstevel@tonic-gate } 27587c478bd9Sstevel@tonic-gate 2759789d94c2Sjwadams void 2760789d94c2Sjwadams umem_alloc_sizes_clear(void) 2761789d94c2Sjwadams { 2762789d94c2Sjwadams int i; 2763789d94c2Sjwadams 2764789d94c2Sjwadams umem_alloc_sizes[0] = UMEM_MAXBUF; 2765789d94c2Sjwadams for (i = 1; i < NUM_ALLOC_SIZES; i++) 2766789d94c2Sjwadams umem_alloc_sizes[i] = 0; 2767789d94c2Sjwadams } 2768789d94c2Sjwadams 2769789d94c2Sjwadams void 2770789d94c2Sjwadams umem_alloc_sizes_add(size_t size_arg) 2771789d94c2Sjwadams { 2772789d94c2Sjwadams int i, j; 2773789d94c2Sjwadams size_t size = size_arg; 2774789d94c2Sjwadams 2775789d94c2Sjwadams if (size == 0) { 2776789d94c2Sjwadams log_message("size_add: cannot add zero-sized cache\n", 2777789d94c2Sjwadams size, UMEM_MAXBUF); 2778789d94c2Sjwadams return; 2779789d94c2Sjwadams } 2780789d94c2Sjwadams 2781789d94c2Sjwadams if (size > UMEM_MAXBUF) { 2782789d94c2Sjwadams log_message("size_add: %ld > %d, cannot add\n", size, 2783789d94c2Sjwadams UMEM_MAXBUF); 2784789d94c2Sjwadams return; 2785789d94c2Sjwadams } 2786789d94c2Sjwadams 2787789d94c2Sjwadams if (umem_alloc_sizes[NUM_ALLOC_SIZES - 1] != 0) { 2788789d94c2Sjwadams log_message("size_add: no space in alloc_table for %d\n", 2789789d94c2Sjwadams size); 2790789d94c2Sjwadams return; 2791789d94c2Sjwadams } 2792789d94c2Sjwadams 2793789d94c2Sjwadams if (P2PHASE(size, UMEM_ALIGN) != 0) { 2794789d94c2Sjwadams size = P2ROUNDUP(size, UMEM_ALIGN); 2795789d94c2Sjwadams log_message("size_add: rounding %d up to %d\n", size_arg, 2796789d94c2Sjwadams size); 2797789d94c2Sjwadams } 2798789d94c2Sjwadams 2799789d94c2Sjwadams for (i = 0; i < NUM_ALLOC_SIZES; i++) { 2800789d94c2Sjwadams int cur = umem_alloc_sizes[i]; 2801789d94c2Sjwadams if (cur == size) { 2802789d94c2Sjwadams log_message("size_add: %ld already in table\n", 2803789d94c2Sjwadams size); 2804789d94c2Sjwadams return; 2805789d94c2Sjwadams } 2806789d94c2Sjwadams if (cur > size) 2807789d94c2Sjwadams break; 2808789d94c2Sjwadams } 2809789d94c2Sjwadams 2810789d94c2Sjwadams for (j = NUM_ALLOC_SIZES - 1; j > i; j--) 2811789d94c2Sjwadams umem_alloc_sizes[j] = umem_alloc_sizes[j-1]; 2812789d94c2Sjwadams umem_alloc_sizes[i] = size; 2813789d94c2Sjwadams } 2814789d94c2Sjwadams 2815789d94c2Sjwadams void 2816789d94c2Sjwadams umem_alloc_sizes_remove(size_t size) 2817789d94c2Sjwadams { 2818789d94c2Sjwadams int i; 2819789d94c2Sjwadams 2820789d94c2Sjwadams if (size == UMEM_MAXBUF) { 2821789d94c2Sjwadams log_message("size_remove: cannot remove %ld\n", size); 2822789d94c2Sjwadams return; 2823789d94c2Sjwadams } 2824789d94c2Sjwadams 2825789d94c2Sjwadams for (i = 0; i < NUM_ALLOC_SIZES; i++) { 2826789d94c2Sjwadams int cur = umem_alloc_sizes[i]; 2827789d94c2Sjwadams if (cur == size) 2828789d94c2Sjwadams break; 2829789d94c2Sjwadams else if (cur > size || cur == 0) { 2830789d94c2Sjwadams log_message("size_remove: %ld not found in table\n", 2831789d94c2Sjwadams size); 2832789d94c2Sjwadams return; 2833789d94c2Sjwadams } 2834789d94c2Sjwadams } 2835789d94c2Sjwadams 2836789d94c2Sjwadams for (; i + 1 < NUM_ALLOC_SIZES; i++) 2837789d94c2Sjwadams umem_alloc_sizes[i] = umem_alloc_sizes[i+1]; 2838789d94c2Sjwadams umem_alloc_sizes[i] = 0; 2839789d94c2Sjwadams } 2840789d94c2Sjwadams 28417c478bd9Sstevel@tonic-gate static int 28427c478bd9Sstevel@tonic-gate umem_cache_init(void) 28437c478bd9Sstevel@tonic-gate { 28447c478bd9Sstevel@tonic-gate int i; 28457c478bd9Sstevel@tonic-gate size_t size, max_size; 28467c478bd9Sstevel@tonic-gate umem_cache_t *cp; 28477c478bd9Sstevel@tonic-gate umem_magtype_t *mtp; 28487c478bd9Sstevel@tonic-gate char name[UMEM_CACHE_NAMELEN + 1]; 28497c478bd9Sstevel@tonic-gate umem_cache_t *umem_alloc_caches[NUM_ALLOC_SIZES]; 28507c478bd9Sstevel@tonic-gate 28517c478bd9Sstevel@tonic-gate for (i = 0; i < sizeof (umem_magtype) / sizeof (*mtp); i++) { 28527c478bd9Sstevel@tonic-gate mtp = &umem_magtype[i]; 28537c478bd9Sstevel@tonic-gate (void) snprintf(name, sizeof (name), "umem_magazine_%d", 28547c478bd9Sstevel@tonic-gate mtp->mt_magsize); 28557c478bd9Sstevel@tonic-gate mtp->mt_cache = umem_cache_create(name, 28567c478bd9Sstevel@tonic-gate (mtp->mt_magsize + 1) * sizeof (void *), 28577c478bd9Sstevel@tonic-gate mtp->mt_align, NULL, NULL, NULL, NULL, 28587c478bd9Sstevel@tonic-gate umem_internal_arena, UMC_NOHASH | UMC_INTERNAL); 28597c478bd9Sstevel@tonic-gate if (mtp->mt_cache == NULL) 28607c478bd9Sstevel@tonic-gate return (0); 28617c478bd9Sstevel@tonic-gate } 28627c478bd9Sstevel@tonic-gate 28637c478bd9Sstevel@tonic-gate umem_slab_cache = umem_cache_create("umem_slab_cache", 28647c478bd9Sstevel@tonic-gate sizeof (umem_slab_t), 0, NULL, NULL, NULL, NULL, 28657c478bd9Sstevel@tonic-gate umem_internal_arena, UMC_NOHASH | UMC_INTERNAL); 28667c478bd9Sstevel@tonic-gate 28677c478bd9Sstevel@tonic-gate if (umem_slab_cache == NULL) 28687c478bd9Sstevel@tonic-gate return (0); 28697c478bd9Sstevel@tonic-gate 28707c478bd9Sstevel@tonic-gate umem_bufctl_cache = umem_cache_create("umem_bufctl_cache", 28717c478bd9Sstevel@tonic-gate sizeof (umem_bufctl_t), 0, NULL, NULL, NULL, NULL, 28727c478bd9Sstevel@tonic-gate umem_internal_arena, UMC_NOHASH | UMC_INTERNAL); 28737c478bd9Sstevel@tonic-gate 28747c478bd9Sstevel@tonic-gate if (umem_bufctl_cache == NULL) 28757c478bd9Sstevel@tonic-gate return (0); 28767c478bd9Sstevel@tonic-gate 28777c478bd9Sstevel@tonic-gate /* 28787c478bd9Sstevel@tonic-gate * The size of the umem_bufctl_audit structure depends upon 28797c478bd9Sstevel@tonic-gate * umem_stack_depth. See umem_impl.h for details on the size 28807c478bd9Sstevel@tonic-gate * restrictions. 28817c478bd9Sstevel@tonic-gate */ 28827c478bd9Sstevel@tonic-gate 28837c478bd9Sstevel@tonic-gate size = UMEM_BUFCTL_AUDIT_SIZE_DEPTH(umem_stack_depth); 28847c478bd9Sstevel@tonic-gate max_size = UMEM_BUFCTL_AUDIT_MAX_SIZE; 28857c478bd9Sstevel@tonic-gate 28867c478bd9Sstevel@tonic-gate if (size > max_size) { /* too large -- truncate */ 28877c478bd9Sstevel@tonic-gate int max_frames = UMEM_MAX_STACK_DEPTH; 28887c478bd9Sstevel@tonic-gate 28897c478bd9Sstevel@tonic-gate ASSERT(UMEM_BUFCTL_AUDIT_SIZE_DEPTH(max_frames) <= max_size); 28907c478bd9Sstevel@tonic-gate 28917c478bd9Sstevel@tonic-gate umem_stack_depth = max_frames; 28927c478bd9Sstevel@tonic-gate size = UMEM_BUFCTL_AUDIT_SIZE_DEPTH(umem_stack_depth); 28937c478bd9Sstevel@tonic-gate } 28947c478bd9Sstevel@tonic-gate 28957c478bd9Sstevel@tonic-gate umem_bufctl_audit_cache = umem_cache_create("umem_bufctl_audit_cache", 28967c478bd9Sstevel@tonic-gate size, 0, NULL, NULL, NULL, NULL, umem_internal_arena, 28977c478bd9Sstevel@tonic-gate UMC_NOHASH | UMC_INTERNAL); 28987c478bd9Sstevel@tonic-gate 28997c478bd9Sstevel@tonic-gate if (umem_bufctl_audit_cache == NULL) 29007c478bd9Sstevel@tonic-gate return (0); 29017c478bd9Sstevel@tonic-gate 29027c478bd9Sstevel@tonic-gate if (vmem_backend & VMEM_BACKEND_MMAP) 29037c478bd9Sstevel@tonic-gate umem_va_arena = vmem_create("umem_va", 29047c478bd9Sstevel@tonic-gate NULL, 0, pagesize, 29057c478bd9Sstevel@tonic-gate vmem_alloc, vmem_free, heap_arena, 29067c478bd9Sstevel@tonic-gate 8 * pagesize, VM_NOSLEEP); 29077c478bd9Sstevel@tonic-gate else 29087c478bd9Sstevel@tonic-gate umem_va_arena = heap_arena; 29097c478bd9Sstevel@tonic-gate 29107c478bd9Sstevel@tonic-gate if (umem_va_arena == NULL) 29117c478bd9Sstevel@tonic-gate return (0); 29127c478bd9Sstevel@tonic-gate 29137c478bd9Sstevel@tonic-gate umem_default_arena = vmem_create("umem_default", 29147c478bd9Sstevel@tonic-gate NULL, 0, pagesize, 29157c478bd9Sstevel@tonic-gate heap_alloc, heap_free, umem_va_arena, 29167c478bd9Sstevel@tonic-gate 0, VM_NOSLEEP); 29177c478bd9Sstevel@tonic-gate 29187c478bd9Sstevel@tonic-gate if (umem_default_arena == NULL) 29197c478bd9Sstevel@tonic-gate return (0); 29207c478bd9Sstevel@tonic-gate 29217c478bd9Sstevel@tonic-gate /* 29227c478bd9Sstevel@tonic-gate * make sure the umem_alloc table initializer is correct 29237c478bd9Sstevel@tonic-gate */ 29247c478bd9Sstevel@tonic-gate i = sizeof (umem_alloc_table) / sizeof (*umem_alloc_table); 29257c478bd9Sstevel@tonic-gate ASSERT(umem_alloc_table[i - 1] == &umem_null_cache); 29267c478bd9Sstevel@tonic-gate 29277c478bd9Sstevel@tonic-gate /* 29287c478bd9Sstevel@tonic-gate * Create the default caches to back umem_alloc() 29297c478bd9Sstevel@tonic-gate */ 29307c478bd9Sstevel@tonic-gate for (i = 0; i < NUM_ALLOC_SIZES; i++) { 29317c478bd9Sstevel@tonic-gate size_t cache_size = umem_alloc_sizes[i]; 29327c478bd9Sstevel@tonic-gate size_t align = 0; 2933789d94c2Sjwadams 2934789d94c2Sjwadams if (cache_size == 0) 2935789d94c2Sjwadams break; /* 0 terminates the list */ 2936789d94c2Sjwadams 29377c478bd9Sstevel@tonic-gate /* 29387c478bd9Sstevel@tonic-gate * If they allocate a multiple of the coherency granularity, 29397c478bd9Sstevel@tonic-gate * they get a coherency-granularity-aligned address. 29407c478bd9Sstevel@tonic-gate */ 29417c478bd9Sstevel@tonic-gate if (IS_P2ALIGNED(cache_size, 64)) 29427c478bd9Sstevel@tonic-gate align = 64; 29437c478bd9Sstevel@tonic-gate if (IS_P2ALIGNED(cache_size, pagesize)) 29447c478bd9Sstevel@tonic-gate align = pagesize; 29457c478bd9Sstevel@tonic-gate (void) snprintf(name, sizeof (name), "umem_alloc_%lu", 29467c478bd9Sstevel@tonic-gate (long)cache_size); 29477c478bd9Sstevel@tonic-gate 29487c478bd9Sstevel@tonic-gate cp = umem_cache_create(name, cache_size, align, 29497c478bd9Sstevel@tonic-gate NULL, NULL, NULL, NULL, NULL, UMC_INTERNAL); 29507c478bd9Sstevel@tonic-gate if (cp == NULL) 29517c478bd9Sstevel@tonic-gate return (0); 29527c478bd9Sstevel@tonic-gate 29537c478bd9Sstevel@tonic-gate umem_alloc_caches[i] = cp; 29547c478bd9Sstevel@tonic-gate } 29557c478bd9Sstevel@tonic-gate 29567c478bd9Sstevel@tonic-gate /* 29577c478bd9Sstevel@tonic-gate * Initialization cannot fail at this point. Make the caches 29587c478bd9Sstevel@tonic-gate * visible to umem_alloc() and friends. 29597c478bd9Sstevel@tonic-gate */ 29607c478bd9Sstevel@tonic-gate size = UMEM_ALIGN; 29617c478bd9Sstevel@tonic-gate for (i = 0; i < NUM_ALLOC_SIZES; i++) { 29627c478bd9Sstevel@tonic-gate size_t cache_size = umem_alloc_sizes[i]; 29637c478bd9Sstevel@tonic-gate 2964789d94c2Sjwadams if (cache_size == 0) 2965789d94c2Sjwadams break; /* 0 terminates the list */ 2966789d94c2Sjwadams 29677c478bd9Sstevel@tonic-gate cp = umem_alloc_caches[i]; 29687c478bd9Sstevel@tonic-gate 29697c478bd9Sstevel@tonic-gate while (size <= cache_size) { 29707c478bd9Sstevel@tonic-gate umem_alloc_table[(size - 1) >> UMEM_ALIGN_SHIFT] = cp; 29717c478bd9Sstevel@tonic-gate size += UMEM_ALIGN; 29727c478bd9Sstevel@tonic-gate } 29737c478bd9Sstevel@tonic-gate } 2974789d94c2Sjwadams ASSERT(size - UMEM_ALIGN == UMEM_MAXBUF); 29757c478bd9Sstevel@tonic-gate return (1); 29767c478bd9Sstevel@tonic-gate } 29777c478bd9Sstevel@tonic-gate 29787c478bd9Sstevel@tonic-gate /* 29797c478bd9Sstevel@tonic-gate * umem_startup() is called early on, and must be called explicitly if we're 29807c478bd9Sstevel@tonic-gate * the standalone version. 29817c478bd9Sstevel@tonic-gate */ 29827c478bd9Sstevel@tonic-gate #ifdef UMEM_STANDALONE 29837c478bd9Sstevel@tonic-gate void 29847c478bd9Sstevel@tonic-gate #else 29857c478bd9Sstevel@tonic-gate #pragma init(umem_startup) 29867c478bd9Sstevel@tonic-gate static void 29877c478bd9Sstevel@tonic-gate #endif 29887c478bd9Sstevel@tonic-gate umem_startup(caddr_t start, size_t len, size_t pagesize, caddr_t minstack, 29897c478bd9Sstevel@tonic-gate caddr_t maxstack) 29907c478bd9Sstevel@tonic-gate { 29917c478bd9Sstevel@tonic-gate #ifdef UMEM_STANDALONE 29927c478bd9Sstevel@tonic-gate int idx; 29937c478bd9Sstevel@tonic-gate /* Standalone doesn't fork */ 29947c478bd9Sstevel@tonic-gate #else 29957c478bd9Sstevel@tonic-gate umem_forkhandler_init(); /* register the fork handler */ 29967c478bd9Sstevel@tonic-gate #endif 29977c478bd9Sstevel@tonic-gate 29987c478bd9Sstevel@tonic-gate #ifdef __lint 29997c478bd9Sstevel@tonic-gate /* make lint happy */ 30007c478bd9Sstevel@tonic-gate minstack = maxstack; 30017c478bd9Sstevel@tonic-gate #endif 30027c478bd9Sstevel@tonic-gate 30037c478bd9Sstevel@tonic-gate #ifdef UMEM_STANDALONE 30047c478bd9Sstevel@tonic-gate umem_ready = UMEM_READY_STARTUP; 30057c478bd9Sstevel@tonic-gate umem_init_env_ready = 0; 30067c478bd9Sstevel@tonic-gate 30077c478bd9Sstevel@tonic-gate umem_min_stack = minstack; 30087c478bd9Sstevel@tonic-gate umem_max_stack = maxstack; 30097c478bd9Sstevel@tonic-gate 30107c478bd9Sstevel@tonic-gate nofail_callback = NULL; 30117c478bd9Sstevel@tonic-gate umem_slab_cache = NULL; 30127c478bd9Sstevel@tonic-gate umem_bufctl_cache = NULL; 30137c478bd9Sstevel@tonic-gate umem_bufctl_audit_cache = NULL; 30147c478bd9Sstevel@tonic-gate heap_arena = NULL; 30157c478bd9Sstevel@tonic-gate heap_alloc = NULL; 30167c478bd9Sstevel@tonic-gate heap_free = NULL; 30177c478bd9Sstevel@tonic-gate umem_internal_arena = NULL; 30187c478bd9Sstevel@tonic-gate umem_cache_arena = NULL; 30197c478bd9Sstevel@tonic-gate umem_hash_arena = NULL; 30207c478bd9Sstevel@tonic-gate umem_log_arena = NULL; 30217c478bd9Sstevel@tonic-gate umem_oversize_arena = NULL; 30227c478bd9Sstevel@tonic-gate umem_va_arena = NULL; 30237c478bd9Sstevel@tonic-gate umem_default_arena = NULL; 30247c478bd9Sstevel@tonic-gate umem_firewall_va_arena = NULL; 30257c478bd9Sstevel@tonic-gate umem_firewall_arena = NULL; 30267c478bd9Sstevel@tonic-gate umem_memalign_arena = NULL; 30277c478bd9Sstevel@tonic-gate umem_transaction_log = NULL; 30287c478bd9Sstevel@tonic-gate umem_content_log = NULL; 30297c478bd9Sstevel@tonic-gate umem_failure_log = NULL; 30307c478bd9Sstevel@tonic-gate umem_slab_log = NULL; 30317c478bd9Sstevel@tonic-gate umem_cpu_mask = 0; 30327c478bd9Sstevel@tonic-gate 30337c478bd9Sstevel@tonic-gate umem_cpus = &umem_startup_cpu; 30347c478bd9Sstevel@tonic-gate umem_startup_cpu.cpu_cache_offset = UMEM_CACHE_SIZE(0); 30357c478bd9Sstevel@tonic-gate umem_startup_cpu.cpu_number = 0; 30367c478bd9Sstevel@tonic-gate 30377c478bd9Sstevel@tonic-gate bcopy(&umem_null_cache_template, &umem_null_cache, 30387c478bd9Sstevel@tonic-gate sizeof (umem_cache_t)); 30397c478bd9Sstevel@tonic-gate 30407c478bd9Sstevel@tonic-gate for (idx = 0; idx < (UMEM_MAXBUF >> UMEM_ALIGN_SHIFT); idx++) 30417c478bd9Sstevel@tonic-gate umem_alloc_table[idx] = &umem_null_cache; 30427c478bd9Sstevel@tonic-gate #endif 30437c478bd9Sstevel@tonic-gate 30447c478bd9Sstevel@tonic-gate /* 30457c478bd9Sstevel@tonic-gate * Perform initialization specific to the way we've been compiled 30467c478bd9Sstevel@tonic-gate * (library or standalone) 30477c478bd9Sstevel@tonic-gate */ 30487c478bd9Sstevel@tonic-gate umem_type_init(start, len, pagesize); 30497c478bd9Sstevel@tonic-gate 30507c478bd9Sstevel@tonic-gate vmem_startup(); 30517c478bd9Sstevel@tonic-gate } 30527c478bd9Sstevel@tonic-gate 30537c478bd9Sstevel@tonic-gate int 30547c478bd9Sstevel@tonic-gate umem_init(void) 30557c478bd9Sstevel@tonic-gate { 30567c478bd9Sstevel@tonic-gate size_t maxverify, minfirewall; 30577c478bd9Sstevel@tonic-gate size_t size; 30587c478bd9Sstevel@tonic-gate int idx; 30597c478bd9Sstevel@tonic-gate umem_cpu_t *new_cpus; 30607c478bd9Sstevel@tonic-gate 30617c478bd9Sstevel@tonic-gate vmem_t *memalign_arena, *oversize_arena; 30627c478bd9Sstevel@tonic-gate 30637c478bd9Sstevel@tonic-gate if (thr_self() != umem_init_thr) { 30647c478bd9Sstevel@tonic-gate /* 30657c478bd9Sstevel@tonic-gate * The usual case -- non-recursive invocation of umem_init(). 30667c478bd9Sstevel@tonic-gate */ 30677c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_init_lock); 30687c478bd9Sstevel@tonic-gate if (umem_ready != UMEM_READY_STARTUP) { 30697c478bd9Sstevel@tonic-gate /* 30707c478bd9Sstevel@tonic-gate * someone else beat us to initializing umem. Wait 30717c478bd9Sstevel@tonic-gate * for them to complete, then return. 30727c478bd9Sstevel@tonic-gate */ 3073a574db85Sraf while (umem_ready == UMEM_READY_INITING) { 3074a574db85Sraf int cancel_state; 3075a574db85Sraf 3076a574db85Sraf (void) pthread_setcancelstate( 3077a574db85Sraf PTHREAD_CANCEL_DISABLE, &cancel_state); 3078a574db85Sraf (void) cond_wait(&umem_init_cv, 30797c478bd9Sstevel@tonic-gate &umem_init_lock); 3080a574db85Sraf (void) pthread_setcancelstate( 3081a574db85Sraf cancel_state, NULL); 3082a574db85Sraf } 30837c478bd9Sstevel@tonic-gate ASSERT(umem_ready == UMEM_READY || 30847c478bd9Sstevel@tonic-gate umem_ready == UMEM_READY_INIT_FAILED); 30857c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_init_lock); 30867c478bd9Sstevel@tonic-gate return (umem_ready == UMEM_READY); 30877c478bd9Sstevel@tonic-gate } 30887c478bd9Sstevel@tonic-gate 30897c478bd9Sstevel@tonic-gate ASSERT(umem_ready == UMEM_READY_STARTUP); 30907c478bd9Sstevel@tonic-gate ASSERT(umem_init_env_ready == 0); 30917c478bd9Sstevel@tonic-gate 30927c478bd9Sstevel@tonic-gate umem_ready = UMEM_READY_INITING; 30937c478bd9Sstevel@tonic-gate umem_init_thr = thr_self(); 30947c478bd9Sstevel@tonic-gate 30957c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_init_lock); 30967c478bd9Sstevel@tonic-gate umem_setup_envvars(0); /* can recurse -- see below */ 30977c478bd9Sstevel@tonic-gate if (umem_init_env_ready) { 30987c478bd9Sstevel@tonic-gate /* 30997c478bd9Sstevel@tonic-gate * initialization was completed already 31007c478bd9Sstevel@tonic-gate */ 31017c478bd9Sstevel@tonic-gate ASSERT(umem_ready == UMEM_READY || 31027c478bd9Sstevel@tonic-gate umem_ready == UMEM_READY_INIT_FAILED); 31037c478bd9Sstevel@tonic-gate ASSERT(umem_init_thr == 0); 31047c478bd9Sstevel@tonic-gate return (umem_ready == UMEM_READY); 31057c478bd9Sstevel@tonic-gate } 31067c478bd9Sstevel@tonic-gate } else if (!umem_init_env_ready) { 31077c478bd9Sstevel@tonic-gate /* 31087c478bd9Sstevel@tonic-gate * The umem_setup_envvars() call (above) makes calls into 31097c478bd9Sstevel@tonic-gate * the dynamic linker and directly into user-supplied code. 31107c478bd9Sstevel@tonic-gate * Since we cannot know what that code will do, we could be 31117c478bd9Sstevel@tonic-gate * recursively invoked (by, say, a malloc() call in the code 31127c478bd9Sstevel@tonic-gate * itself, or in a (C++) _init section it causes to be fired). 31137c478bd9Sstevel@tonic-gate * 31147c478bd9Sstevel@tonic-gate * This code is where we end up if such recursion occurs. We 31157c478bd9Sstevel@tonic-gate * first clean up any partial results in the envvar code, then 31167c478bd9Sstevel@tonic-gate * proceed to finish initialization processing in the recursive 31177c478bd9Sstevel@tonic-gate * call. The original call will notice this, and return 31187c478bd9Sstevel@tonic-gate * immediately. 31197c478bd9Sstevel@tonic-gate */ 31207c478bd9Sstevel@tonic-gate umem_setup_envvars(1); /* clean up any partial state */ 31217c478bd9Sstevel@tonic-gate } else { 31227c478bd9Sstevel@tonic-gate umem_panic( 31237c478bd9Sstevel@tonic-gate "recursive allocation while initializing umem\n"); 31247c478bd9Sstevel@tonic-gate } 31257c478bd9Sstevel@tonic-gate umem_init_env_ready = 1; 31267c478bd9Sstevel@tonic-gate 31277c478bd9Sstevel@tonic-gate /* 31287c478bd9Sstevel@tonic-gate * From this point until we finish, recursion into umem_init() will 31297c478bd9Sstevel@tonic-gate * cause a umem_panic(). 31307c478bd9Sstevel@tonic-gate */ 31317c478bd9Sstevel@tonic-gate maxverify = minfirewall = ULONG_MAX; 31327c478bd9Sstevel@tonic-gate 31337c478bd9Sstevel@tonic-gate /* LINTED constant condition */ 31347c478bd9Sstevel@tonic-gate if (sizeof (umem_cpu_cache_t) != UMEM_CPU_CACHE_SIZE) { 31357c478bd9Sstevel@tonic-gate umem_panic("sizeof (umem_cpu_cache_t) = %d, should be %d\n", 31367c478bd9Sstevel@tonic-gate sizeof (umem_cpu_cache_t), UMEM_CPU_CACHE_SIZE); 31377c478bd9Sstevel@tonic-gate } 31387c478bd9Sstevel@tonic-gate 31397c478bd9Sstevel@tonic-gate umem_max_ncpus = umem_get_max_ncpus(); 31407c478bd9Sstevel@tonic-gate 31417c478bd9Sstevel@tonic-gate /* 31427c478bd9Sstevel@tonic-gate * load tunables from environment 31437c478bd9Sstevel@tonic-gate */ 31447c478bd9Sstevel@tonic-gate umem_process_envvars(); 31457c478bd9Sstevel@tonic-gate 31467c478bd9Sstevel@tonic-gate if (issetugid()) 31477c478bd9Sstevel@tonic-gate umem_mtbf = 0; 31487c478bd9Sstevel@tonic-gate 31497c478bd9Sstevel@tonic-gate /* 31507c478bd9Sstevel@tonic-gate * set up vmem 31517c478bd9Sstevel@tonic-gate */ 31527c478bd9Sstevel@tonic-gate if (!(umem_flags & UMF_AUDIT)) 31537c478bd9Sstevel@tonic-gate vmem_no_debug(); 31547c478bd9Sstevel@tonic-gate 31557c478bd9Sstevel@tonic-gate heap_arena = vmem_heap_arena(&heap_alloc, &heap_free); 31567c478bd9Sstevel@tonic-gate 31577c478bd9Sstevel@tonic-gate pagesize = heap_arena->vm_quantum; 31587c478bd9Sstevel@tonic-gate 31597c478bd9Sstevel@tonic-gate umem_internal_arena = vmem_create("umem_internal", NULL, 0, pagesize, 31607c478bd9Sstevel@tonic-gate heap_alloc, heap_free, heap_arena, 0, VM_NOSLEEP); 31617c478bd9Sstevel@tonic-gate 31627c478bd9Sstevel@tonic-gate umem_default_arena = umem_internal_arena; 31637c478bd9Sstevel@tonic-gate 31647c478bd9Sstevel@tonic-gate if (umem_internal_arena == NULL) 31657c478bd9Sstevel@tonic-gate goto fail; 31667c478bd9Sstevel@tonic-gate 31677c478bd9Sstevel@tonic-gate umem_cache_arena = vmem_create("umem_cache", NULL, 0, UMEM_ALIGN, 31687c478bd9Sstevel@tonic-gate vmem_alloc, vmem_free, umem_internal_arena, 0, VM_NOSLEEP); 31697c478bd9Sstevel@tonic-gate 31707c478bd9Sstevel@tonic-gate umem_hash_arena = vmem_create("umem_hash", NULL, 0, UMEM_ALIGN, 31717c478bd9Sstevel@tonic-gate vmem_alloc, vmem_free, umem_internal_arena, 0, VM_NOSLEEP); 31727c478bd9Sstevel@tonic-gate 31737c478bd9Sstevel@tonic-gate umem_log_arena = vmem_create("umem_log", NULL, 0, UMEM_ALIGN, 31747c478bd9Sstevel@tonic-gate heap_alloc, heap_free, heap_arena, 0, VM_NOSLEEP); 31757c478bd9Sstevel@tonic-gate 31767c478bd9Sstevel@tonic-gate umem_firewall_va_arena = vmem_create("umem_firewall_va", 31777c478bd9Sstevel@tonic-gate NULL, 0, pagesize, 31787c478bd9Sstevel@tonic-gate umem_firewall_va_alloc, umem_firewall_va_free, heap_arena, 31797c478bd9Sstevel@tonic-gate 0, VM_NOSLEEP); 31807c478bd9Sstevel@tonic-gate 31817c478bd9Sstevel@tonic-gate if (umem_cache_arena == NULL || umem_hash_arena == NULL || 31827c478bd9Sstevel@tonic-gate umem_log_arena == NULL || umem_firewall_va_arena == NULL) 31837c478bd9Sstevel@tonic-gate goto fail; 31847c478bd9Sstevel@tonic-gate 31857c478bd9Sstevel@tonic-gate umem_firewall_arena = vmem_create("umem_firewall", NULL, 0, pagesize, 31867c478bd9Sstevel@tonic-gate heap_alloc, heap_free, umem_firewall_va_arena, 0, 31877c478bd9Sstevel@tonic-gate VM_NOSLEEP); 31887c478bd9Sstevel@tonic-gate 31897c478bd9Sstevel@tonic-gate if (umem_firewall_arena == NULL) 31907c478bd9Sstevel@tonic-gate goto fail; 31917c478bd9Sstevel@tonic-gate 31927c478bd9Sstevel@tonic-gate oversize_arena = vmem_create("umem_oversize", NULL, 0, pagesize, 31937c478bd9Sstevel@tonic-gate heap_alloc, heap_free, minfirewall < ULONG_MAX ? 31947c478bd9Sstevel@tonic-gate umem_firewall_va_arena : heap_arena, 0, VM_NOSLEEP); 31957c478bd9Sstevel@tonic-gate 31967c478bd9Sstevel@tonic-gate memalign_arena = vmem_create("umem_memalign", NULL, 0, UMEM_ALIGN, 31977c478bd9Sstevel@tonic-gate heap_alloc, heap_free, minfirewall < ULONG_MAX ? 31987c478bd9Sstevel@tonic-gate umem_firewall_va_arena : heap_arena, 0, VM_NOSLEEP); 31997c478bd9Sstevel@tonic-gate 32007c478bd9Sstevel@tonic-gate if (oversize_arena == NULL || memalign_arena == NULL) 32017c478bd9Sstevel@tonic-gate goto fail; 32027c478bd9Sstevel@tonic-gate 32037c478bd9Sstevel@tonic-gate if (umem_max_ncpus > CPUHINT_MAX()) 32047c478bd9Sstevel@tonic-gate umem_max_ncpus = CPUHINT_MAX(); 32057c478bd9Sstevel@tonic-gate 32067c478bd9Sstevel@tonic-gate while ((umem_max_ncpus & (umem_max_ncpus - 1)) != 0) 32077c478bd9Sstevel@tonic-gate umem_max_ncpus++; 32087c478bd9Sstevel@tonic-gate 32097c478bd9Sstevel@tonic-gate if (umem_max_ncpus == 0) 32107c478bd9Sstevel@tonic-gate umem_max_ncpus = 1; 32117c478bd9Sstevel@tonic-gate 32127c478bd9Sstevel@tonic-gate size = umem_max_ncpus * sizeof (umem_cpu_t); 32137c478bd9Sstevel@tonic-gate new_cpus = vmem_alloc(umem_internal_arena, size, VM_NOSLEEP); 32147c478bd9Sstevel@tonic-gate if (new_cpus == NULL) 32157c478bd9Sstevel@tonic-gate goto fail; 32167c478bd9Sstevel@tonic-gate 32177c478bd9Sstevel@tonic-gate bzero(new_cpus, size); 32187c478bd9Sstevel@tonic-gate for (idx = 0; idx < umem_max_ncpus; idx++) { 32197c478bd9Sstevel@tonic-gate new_cpus[idx].cpu_number = idx; 32207c478bd9Sstevel@tonic-gate new_cpus[idx].cpu_cache_offset = UMEM_CACHE_SIZE(idx); 32217c478bd9Sstevel@tonic-gate } 32227c478bd9Sstevel@tonic-gate umem_cpus = new_cpus; 32237c478bd9Sstevel@tonic-gate umem_cpu_mask = (umem_max_ncpus - 1); 32247c478bd9Sstevel@tonic-gate 32257c478bd9Sstevel@tonic-gate if (umem_maxverify == 0) 32267c478bd9Sstevel@tonic-gate umem_maxverify = maxverify; 32277c478bd9Sstevel@tonic-gate 32287c478bd9Sstevel@tonic-gate if (umem_minfirewall == 0) 32297c478bd9Sstevel@tonic-gate umem_minfirewall = minfirewall; 32307c478bd9Sstevel@tonic-gate 32317c478bd9Sstevel@tonic-gate /* 32327c478bd9Sstevel@tonic-gate * Set up updating and reaping 32337c478bd9Sstevel@tonic-gate */ 32347c478bd9Sstevel@tonic-gate umem_reap_next = gethrtime() + NANOSEC; 32357c478bd9Sstevel@tonic-gate 32367c478bd9Sstevel@tonic-gate #ifndef UMEM_STANDALONE 32377c478bd9Sstevel@tonic-gate (void) gettimeofday(&umem_update_next, NULL); 32387c478bd9Sstevel@tonic-gate #endif 32397c478bd9Sstevel@tonic-gate 32407c478bd9Sstevel@tonic-gate /* 32417c478bd9Sstevel@tonic-gate * Set up logging -- failure here is okay, since it will just disable 32427c478bd9Sstevel@tonic-gate * the logs 32437c478bd9Sstevel@tonic-gate */ 32447c478bd9Sstevel@tonic-gate if (umem_logging) { 32457c478bd9Sstevel@tonic-gate umem_transaction_log = umem_log_init(umem_transaction_log_size); 32467c478bd9Sstevel@tonic-gate umem_content_log = umem_log_init(umem_content_log_size); 32477c478bd9Sstevel@tonic-gate umem_failure_log = umem_log_init(umem_failure_log_size); 32487c478bd9Sstevel@tonic-gate umem_slab_log = umem_log_init(umem_slab_log_size); 32497c478bd9Sstevel@tonic-gate } 32507c478bd9Sstevel@tonic-gate 32517c478bd9Sstevel@tonic-gate /* 32527c478bd9Sstevel@tonic-gate * Set up caches -- if successful, initialization cannot fail, since 32537c478bd9Sstevel@tonic-gate * allocations from other threads can now succeed. 32547c478bd9Sstevel@tonic-gate */ 32557c478bd9Sstevel@tonic-gate if (umem_cache_init() == 0) { 32567c478bd9Sstevel@tonic-gate log_message("unable to create initial caches\n"); 32577c478bd9Sstevel@tonic-gate goto fail; 32587c478bd9Sstevel@tonic-gate } 32597c478bd9Sstevel@tonic-gate umem_oversize_arena = oversize_arena; 32607c478bd9Sstevel@tonic-gate umem_memalign_arena = memalign_arena; 32617c478bd9Sstevel@tonic-gate 32627c478bd9Sstevel@tonic-gate umem_cache_applyall(umem_cache_magazine_enable); 32637c478bd9Sstevel@tonic-gate 32647c478bd9Sstevel@tonic-gate /* 32657c478bd9Sstevel@tonic-gate * initialization done, ready to go 32667c478bd9Sstevel@tonic-gate */ 32677c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_init_lock); 32687c478bd9Sstevel@tonic-gate umem_ready = UMEM_READY; 32697c478bd9Sstevel@tonic-gate umem_init_thr = 0; 32707c478bd9Sstevel@tonic-gate (void) cond_broadcast(&umem_init_cv); 32717c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_init_lock); 32727c478bd9Sstevel@tonic-gate return (1); 32737c478bd9Sstevel@tonic-gate 32747c478bd9Sstevel@tonic-gate fail: 32757c478bd9Sstevel@tonic-gate log_message("umem initialization failed\n"); 32767c478bd9Sstevel@tonic-gate 32777c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_init_lock); 32787c478bd9Sstevel@tonic-gate umem_ready = UMEM_READY_INIT_FAILED; 32797c478bd9Sstevel@tonic-gate umem_init_thr = 0; 32807c478bd9Sstevel@tonic-gate (void) cond_broadcast(&umem_init_cv); 32817c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_init_lock); 32827c478bd9Sstevel@tonic-gate return (0); 32837c478bd9Sstevel@tonic-gate } 3284