xref: /illumos-gate/usr/src/lib/libumem/common/umem.c (revision 9f160f41)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5789d94c2Sjwadams  * Common Development and Distribution License (the "License").
6789d94c2Sjwadams  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
21e8031f0aSraf 
227c478bd9Sstevel@tonic-gate /*
23a574db85Sraf  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
247c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
257c478bd9Sstevel@tonic-gate  */
267c478bd9Sstevel@tonic-gate 
274f364e7cSRobert Mustacchi /*
28b1e2e3fbSRobert Mustacchi  * Copyright (c) 2019 Joyent, Inc.
29831abf2cSDan Kimmel  * Copyright (c) 2015 by Delphix. All rights reserved.
304f364e7cSRobert Mustacchi  */
314f364e7cSRobert Mustacchi 
327c478bd9Sstevel@tonic-gate /*
337c478bd9Sstevel@tonic-gate  * based on usr/src/uts/common/os/kmem.c r1.64 from 2001/12/18
347c478bd9Sstevel@tonic-gate  *
357c478bd9Sstevel@tonic-gate  * The slab allocator, as described in the following two papers:
367c478bd9Sstevel@tonic-gate  *
377c478bd9Sstevel@tonic-gate  *	Jeff Bonwick,
387c478bd9Sstevel@tonic-gate  *	The Slab Allocator: An Object-Caching Kernel Memory Allocator.
397c478bd9Sstevel@tonic-gate  *	Proceedings of the Summer 1994 Usenix Conference.
407c478bd9Sstevel@tonic-gate  *	Available as /shared/sac/PSARC/1994/028/materials/kmem.pdf.
417c478bd9Sstevel@tonic-gate  *
427c478bd9Sstevel@tonic-gate  *	Jeff Bonwick and Jonathan Adams,
437c478bd9Sstevel@tonic-gate  *	Magazines and vmem: Extending the Slab Allocator to Many CPUs and
447c478bd9Sstevel@tonic-gate  *	Arbitrary Resources.
457c478bd9Sstevel@tonic-gate  *	Proceedings of the 2001 Usenix Conference.
467c478bd9Sstevel@tonic-gate  *	Available as /shared/sac/PSARC/2000/550/materials/vmem.pdf.
477c478bd9Sstevel@tonic-gate  *
487c478bd9Sstevel@tonic-gate  * 1. Overview
497c478bd9Sstevel@tonic-gate  * -----------
504f364e7cSRobert Mustacchi  * umem is very close to kmem in implementation.  There are seven major
517c478bd9Sstevel@tonic-gate  * areas of divergence:
527c478bd9Sstevel@tonic-gate  *
537c478bd9Sstevel@tonic-gate  *	* Initialization
547c478bd9Sstevel@tonic-gate  *
557c478bd9Sstevel@tonic-gate  *	* CPU handling
567c478bd9Sstevel@tonic-gate  *
577c478bd9Sstevel@tonic-gate  *	* umem_update()
587c478bd9Sstevel@tonic-gate  *
597c478bd9Sstevel@tonic-gate  *	* KM_SLEEP v.s. UMEM_NOFAIL
607c478bd9Sstevel@tonic-gate  *
611c326e94Sjwadams  *	* lock ordering
627c478bd9Sstevel@tonic-gate  *
634f364e7cSRobert Mustacchi  *	* changing UMEM_MAXBUF
644f364e7cSRobert Mustacchi  *
654f364e7cSRobert Mustacchi  *	* Per-thread caching for malloc/free
664f364e7cSRobert Mustacchi  *
677c478bd9Sstevel@tonic-gate  * 2. Initialization
687c478bd9Sstevel@tonic-gate  * -----------------
697c478bd9Sstevel@tonic-gate  * kmem is initialized early on in boot, and knows that no one will call
707c478bd9Sstevel@tonic-gate  * into it before it is ready.  umem does not have these luxuries. Instead,
717c478bd9Sstevel@tonic-gate  * initialization is divided into two phases:
727c478bd9Sstevel@tonic-gate  *
737c478bd9Sstevel@tonic-gate  *	* library initialization, and
747c478bd9Sstevel@tonic-gate  *
757c478bd9Sstevel@tonic-gate  *	* first use
767c478bd9Sstevel@tonic-gate  *
777c478bd9Sstevel@tonic-gate  * umem's full initialization happens at the time of the first allocation
787c478bd9Sstevel@tonic-gate  * request (via malloc() and friends, umem_alloc(), or umem_zalloc()),
797c478bd9Sstevel@tonic-gate  * or the first call to umem_cache_create().
807c478bd9Sstevel@tonic-gate  *
817c478bd9Sstevel@tonic-gate  * umem_free(), and umem_cache_alloc() do not require special handling,
827c478bd9Sstevel@tonic-gate  * since the only way to get valid arguments for them is to successfully
837c478bd9Sstevel@tonic-gate  * call a function from the first group.
847c478bd9Sstevel@tonic-gate  *
857c478bd9Sstevel@tonic-gate  * 2.1. Library Initialization: umem_startup()
867c478bd9Sstevel@tonic-gate  * -------------------------------------------
877c478bd9Sstevel@tonic-gate  * umem_startup() is libumem.so's .init section.  It calls pthread_atfork()
887c478bd9Sstevel@tonic-gate  * to install the handlers necessary for umem's Fork1-Safety.  Because of
897c478bd9Sstevel@tonic-gate  * race condition issues, all other pre-umem_init() initialization is done
907c478bd9Sstevel@tonic-gate  * statically (i.e. by the dynamic linker).
917c478bd9Sstevel@tonic-gate  *
927c478bd9Sstevel@tonic-gate  * For standalone use, umem_startup() returns everything to its initial
937c478bd9Sstevel@tonic-gate  * state.
947c478bd9Sstevel@tonic-gate  *
957c478bd9Sstevel@tonic-gate  * 2.2. First use: umem_init()
967c478bd9Sstevel@tonic-gate  * ------------------------------
977c478bd9Sstevel@tonic-gate  * The first time any memory allocation function is used, we have to
987c478bd9Sstevel@tonic-gate  * create the backing caches and vmem arenas which are needed for it.
997c478bd9Sstevel@tonic-gate  * umem_init() is the central point for that task.  When it completes,
1007c478bd9Sstevel@tonic-gate  * umem_ready is either UMEM_READY (all set) or UMEM_READY_INIT_FAILED (unable
1017c478bd9Sstevel@tonic-gate  * to initialize, probably due to lack of memory).
1027c478bd9Sstevel@tonic-gate  *
1037c478bd9Sstevel@tonic-gate  * There are four different paths from which umem_init() is called:
1047c478bd9Sstevel@tonic-gate  *
1057c478bd9Sstevel@tonic-gate  *	* from umem_alloc() or umem_zalloc(), with 0 < size < UMEM_MAXBUF,
1067c478bd9Sstevel@tonic-gate  *
1077c478bd9Sstevel@tonic-gate  *	* from umem_alloc() or umem_zalloc(), with size > UMEM_MAXBUF,
1087c478bd9Sstevel@tonic-gate  *
1097c478bd9Sstevel@tonic-gate  *	* from umem_cache_create(), and
1107c478bd9Sstevel@tonic-gate  *
1117c478bd9Sstevel@tonic-gate  *	* from memalign(), with align > UMEM_ALIGN.
1127c478bd9Sstevel@tonic-gate  *
1137c478bd9Sstevel@tonic-gate  * The last three just check if umem is initialized, and call umem_init()
1147c478bd9Sstevel@tonic-gate  * if it is not.  For performance reasons, the first case is more complicated.
1157c478bd9Sstevel@tonic-gate  *
1167c478bd9Sstevel@tonic-gate  * 2.2.1. umem_alloc()/umem_zalloc(), with 0 < size < UMEM_MAXBUF
1177c478bd9Sstevel@tonic-gate  * -----------------------------------------------------------------
1187c478bd9Sstevel@tonic-gate  * In this case, umem_cache_alloc(&umem_null_cache, ...) is called.
1197c478bd9Sstevel@tonic-gate  * There is special case code in which causes any allocation on
1207c478bd9Sstevel@tonic-gate  * &umem_null_cache to fail by returning (NULL), regardless of the
1217c478bd9Sstevel@tonic-gate  * flags argument.
1227c478bd9Sstevel@tonic-gate  *
1237c478bd9Sstevel@tonic-gate  * So umem_cache_alloc() returns NULL, and umem_alloc()/umem_zalloc() call
1247c478bd9Sstevel@tonic-gate  * umem_alloc_retry().  umem_alloc_retry() sees that the allocation
1257c478bd9Sstevel@tonic-gate  * was agains &umem_null_cache, and calls umem_init().
1267c478bd9Sstevel@tonic-gate  *
1277c478bd9Sstevel@tonic-gate  * If initialization is successful, umem_alloc_retry() returns 1, which
1287c478bd9Sstevel@tonic-gate  * causes umem_alloc()/umem_zalloc() to start over, which causes it to load
1297c478bd9Sstevel@tonic-gate  * the (now valid) cache pointer from umem_alloc_table.
1307c478bd9Sstevel@tonic-gate  *
1317c478bd9Sstevel@tonic-gate  * 2.2.2. Dealing with race conditions
1327c478bd9Sstevel@tonic-gate  * -----------------------------------
1337c478bd9Sstevel@tonic-gate  * There are a couple race conditions resulting from the initialization
1347c478bd9Sstevel@tonic-gate  * code that we have to guard against:
1357c478bd9Sstevel@tonic-gate  *
1367c478bd9Sstevel@tonic-gate  *	* In umem_cache_create(), there is a special UMC_INTERNAL cflag
1377c478bd9Sstevel@tonic-gate  *	that is passed for caches created during initialization.  It
1387c478bd9Sstevel@tonic-gate  *	is illegal for a user to try to create a UMC_INTERNAL cache.
1397c478bd9Sstevel@tonic-gate  *	This allows initialization to proceed, but any other
1407c478bd9Sstevel@tonic-gate  *	umem_cache_create()s will block by calling umem_init().
1417c478bd9Sstevel@tonic-gate  *
1427c478bd9Sstevel@tonic-gate  *	* Since umem_null_cache has a 1-element cache_cpu, it's cache_cpu_mask
1437c478bd9Sstevel@tonic-gate  *	is always zero.  umem_cache_alloc uses cp->cache_cpu_mask to
1447c478bd9Sstevel@tonic-gate  *	mask the cpu number.  This prevents a race between grabbing a
1457c478bd9Sstevel@tonic-gate  *	cache pointer out of umem_alloc_table and growing the cpu array.
1467c478bd9Sstevel@tonic-gate  *
1477c478bd9Sstevel@tonic-gate  *
1487c478bd9Sstevel@tonic-gate  * 3. CPU handling
1497c478bd9Sstevel@tonic-gate  * ---------------
1507c478bd9Sstevel@tonic-gate  * kmem uses the CPU's sequence number to determine which "cpu cache" to
1517c478bd9Sstevel@tonic-gate  * use for an allocation.  Currently, there is no way to get the sequence
1527c478bd9Sstevel@tonic-gate  * number in userspace.
1537c478bd9Sstevel@tonic-gate  *
1547c478bd9Sstevel@tonic-gate  * umem keeps track of cpu information in umem_cpus, an array of umem_max_ncpus
1557c478bd9Sstevel@tonic-gate  * umem_cpu_t structures.  CURCPU() is a a "hint" function, which we then mask
1567c478bd9Sstevel@tonic-gate  * with either umem_cpu_mask or cp->cache_cpu_mask to find the actual "cpu" id.
1577c478bd9Sstevel@tonic-gate  * The mechanics of this is all in the CPU(mask) macro.
1587c478bd9Sstevel@tonic-gate  *
1597c478bd9Sstevel@tonic-gate  * Currently, umem uses _lwp_self() as its hint.
1607c478bd9Sstevel@tonic-gate  *
1617c478bd9Sstevel@tonic-gate  *
1627c478bd9Sstevel@tonic-gate  * 4. The update thread
1637c478bd9Sstevel@tonic-gate  * --------------------
1647c478bd9Sstevel@tonic-gate  * kmem uses a task queue, kmem_taskq, to do periodic maintenance on
1657c478bd9Sstevel@tonic-gate  * every kmem cache.  vmem has a periodic timeout for hash table resizing.
1667c478bd9Sstevel@tonic-gate  * The kmem_taskq also provides a separate context for kmem_cache_reap()'s
1677c478bd9Sstevel@tonic-gate  * to be done in, avoiding issues of the context of kmem_reap() callers.
1687c478bd9Sstevel@tonic-gate  *
1697c478bd9Sstevel@tonic-gate  * Instead, umem has the concept of "updates", which are asynchronous requests
1707c478bd9Sstevel@tonic-gate  * for work attached to single caches.  All caches with pending work are
1717c478bd9Sstevel@tonic-gate  * on a doubly linked list rooted at the umem_null_cache.  All update state
1727c478bd9Sstevel@tonic-gate  * is protected by the umem_update_lock mutex, and the umem_update_cv is used
1737c478bd9Sstevel@tonic-gate  * for notification between threads.
1747c478bd9Sstevel@tonic-gate  *
1757c478bd9Sstevel@tonic-gate  * 4.1. Cache states with regards to updates
1767c478bd9Sstevel@tonic-gate  * -----------------------------------------
1777c478bd9Sstevel@tonic-gate  * A given cache is in one of three states:
1787c478bd9Sstevel@tonic-gate  *
1797c478bd9Sstevel@tonic-gate  * Inactive		cache_uflags is zero, cache_u{next,prev} are NULL
1807c478bd9Sstevel@tonic-gate  *
1817c478bd9Sstevel@tonic-gate  * Work Requested	cache_uflags is non-zero (but UMU_ACTIVE is not set),
1827c478bd9Sstevel@tonic-gate  *			cache_u{next,prev} link the cache onto the global
1837c478bd9Sstevel@tonic-gate  *			update list
1847c478bd9Sstevel@tonic-gate  *
1857c478bd9Sstevel@tonic-gate  * Active		cache_uflags has UMU_ACTIVE set, cache_u{next,prev}
1867c478bd9Sstevel@tonic-gate  *			are NULL, and either umem_update_thr or
1877c478bd9Sstevel@tonic-gate  *			umem_st_update_thr are actively doing work on the
1887c478bd9Sstevel@tonic-gate  *			cache.
1897c478bd9Sstevel@tonic-gate  *
1907c478bd9Sstevel@tonic-gate  * An update can be added to any cache in any state -- if the cache is
1917c478bd9Sstevel@tonic-gate  * Inactive, it transitions to being Work Requested.  If the cache is
1927c478bd9Sstevel@tonic-gate  * Active, the worker will notice the new update and act on it before
1937c478bd9Sstevel@tonic-gate  * transitioning the cache to the Inactive state.
1947c478bd9Sstevel@tonic-gate  *
1957c478bd9Sstevel@tonic-gate  * If a cache is in the Active state, UMU_NOTIFY can be set, which asks
1967c478bd9Sstevel@tonic-gate  * the worker to broadcast the umem_update_cv when it has finished.
1977c478bd9Sstevel@tonic-gate  *
1987c478bd9Sstevel@tonic-gate  * 4.2. Update interface
1997c478bd9Sstevel@tonic-gate  * ---------------------
2007c478bd9Sstevel@tonic-gate  * umem_add_update() adds an update to a particular cache.
2017c478bd9Sstevel@tonic-gate  * umem_updateall() adds an update to all caches.
2027c478bd9Sstevel@tonic-gate  * umem_remove_updates() returns a cache to the Inactive state.
2037c478bd9Sstevel@tonic-gate  *
2047c478bd9Sstevel@tonic-gate  * umem_process_updates() process all caches in the Work Requested state.
2057c478bd9Sstevel@tonic-gate  *
2067c478bd9Sstevel@tonic-gate  * 4.3. Reaping
2077c478bd9Sstevel@tonic-gate  * ------------
2087c478bd9Sstevel@tonic-gate  * When umem_reap() is called (at the time of heap growth), it schedule
2097c478bd9Sstevel@tonic-gate  * UMU_REAP updates on every cache.  It then checks to see if the update
2107c478bd9Sstevel@tonic-gate  * thread exists (umem_update_thr != 0).  If it is, it broadcasts
2117c478bd9Sstevel@tonic-gate  * the umem_update_cv to wake the update thread up, and returns.
2127c478bd9Sstevel@tonic-gate  *
2137c478bd9Sstevel@tonic-gate  * If the update thread does not exist (umem_update_thr == 0), and the
2147c478bd9Sstevel@tonic-gate  * program currently has multiple threads, umem_reap() attempts to create
2157c478bd9Sstevel@tonic-gate  * a new update thread.
2167c478bd9Sstevel@tonic-gate  *
2177c478bd9Sstevel@tonic-gate  * If the process is not multithreaded, or the creation fails, umem_reap()
2187c478bd9Sstevel@tonic-gate  * calls umem_st_update() to do an inline update.
2197c478bd9Sstevel@tonic-gate  *
2207c478bd9Sstevel@tonic-gate  * 4.4. The update thread
2217c478bd9Sstevel@tonic-gate  * ----------------------
2227c478bd9Sstevel@tonic-gate  * The update thread spends most of its time in cond_timedwait() on the
2237c478bd9Sstevel@tonic-gate  * umem_update_cv.  It wakes up under two conditions:
2247c478bd9Sstevel@tonic-gate  *
2257c478bd9Sstevel@tonic-gate  *	* The timedwait times out, in which case it needs to run a global
2267c478bd9Sstevel@tonic-gate  *	update, or
2277c478bd9Sstevel@tonic-gate  *
2287c478bd9Sstevel@tonic-gate  *	* someone cond_broadcast(3THR)s the umem_update_cv, in which case
2297c478bd9Sstevel@tonic-gate  *	it needs to check if there are any caches in the Work Requested
2307c478bd9Sstevel@tonic-gate  *	state.
2317c478bd9Sstevel@tonic-gate  *
2327c478bd9Sstevel@tonic-gate  * When it is time for another global update, umem calls umem_cache_update()
2337c478bd9Sstevel@tonic-gate  * on every cache, then calls vmem_update(), which tunes the vmem structures.
2347c478bd9Sstevel@tonic-gate  * umem_cache_update() can request further work using umem_add_update().
2357c478bd9Sstevel@tonic-gate  *
2367c478bd9Sstevel@tonic-gate  * After any work from the global update completes, the update timer is
2377c478bd9Sstevel@tonic-gate  * reset to umem_reap_interval seconds in the future.  This makes the
2387c478bd9Sstevel@tonic-gate  * updates self-throttling.
2397c478bd9Sstevel@tonic-gate  *
2407c478bd9Sstevel@tonic-gate  * Reaps are similarly self-throttling.  After a UMU_REAP update has
2417c478bd9Sstevel@tonic-gate  * been scheduled on all caches, umem_reap() sets a flag and wakes up the
2427c478bd9Sstevel@tonic-gate  * update thread.  The update thread notices the flag, and resets the
2437c478bd9Sstevel@tonic-gate  * reap state.
2447c478bd9Sstevel@tonic-gate  *
2457c478bd9Sstevel@tonic-gate  * 4.5. Inline updates
2467c478bd9Sstevel@tonic-gate  * -------------------
2477c478bd9Sstevel@tonic-gate  * If the update thread is not running, umem_st_update() is used instead.  It
2487c478bd9Sstevel@tonic-gate  * immediately does a global update (as above), then calls
2497c478bd9Sstevel@tonic-gate  * umem_process_updates() to process both the reaps that umem_reap() added and
2507c478bd9Sstevel@tonic-gate  * any work generated by the global update.  Afterwards, it resets the reap
2517c478bd9Sstevel@tonic-gate  * state.
2527c478bd9Sstevel@tonic-gate  *
2537c478bd9Sstevel@tonic-gate  * While the umem_st_update() is running, umem_st_update_thr holds the thread
2547c478bd9Sstevel@tonic-gate  * id of the thread performing the update.
2557c478bd9Sstevel@tonic-gate  *
2567c478bd9Sstevel@tonic-gate  * 4.6. Updates and fork1()
2577c478bd9Sstevel@tonic-gate  * ------------------------
2587c478bd9Sstevel@tonic-gate  * umem has fork1() pre- and post-handlers which lock up (and release) every
2597c478bd9Sstevel@tonic-gate  * mutex in every cache.  They also lock up the umem_update_lock.  Since
2607c478bd9Sstevel@tonic-gate  * fork1() only copies over a single lwp, other threads (including the update
2617c478bd9Sstevel@tonic-gate  * thread) could have been actively using a cache in the parent.  This
2627c478bd9Sstevel@tonic-gate  * can lead to inconsistencies in the child process.
2637c478bd9Sstevel@tonic-gate  *
2647c478bd9Sstevel@tonic-gate  * Because we locked all of the mutexes, the only possible inconsistancies are:
2657c478bd9Sstevel@tonic-gate  *
2667c478bd9Sstevel@tonic-gate  *	* a umem_cache_alloc() could leak its buffer.
2677c478bd9Sstevel@tonic-gate  *
2687c478bd9Sstevel@tonic-gate  *	* a caller of umem_depot_alloc() could leak a magazine, and all the
2697c478bd9Sstevel@tonic-gate  *	buffers contained in it.
2707c478bd9Sstevel@tonic-gate  *
2717c478bd9Sstevel@tonic-gate  *	* a cache could be in the Active update state.  In the child, there
2727c478bd9Sstevel@tonic-gate  *	would be no thread actually working on it.
2737c478bd9Sstevel@tonic-gate  *
2747c478bd9Sstevel@tonic-gate  *	* a umem_hash_rescale() could leak the new hash table.
2757c478bd9Sstevel@tonic-gate  *
2767c478bd9Sstevel@tonic-gate  *	* a umem_magazine_resize() could be in progress.
2777c478bd9Sstevel@tonic-gate  *
2787c478bd9Sstevel@tonic-gate  *	* a umem_reap() could be in progress.
2797c478bd9Sstevel@tonic-gate  *
2807c478bd9Sstevel@tonic-gate  * The memory leaks we can't do anything about.  umem_release_child() resets
2817c478bd9Sstevel@tonic-gate  * the update state, moves any caches in the Active state to the Work Requested
2827c478bd9Sstevel@tonic-gate  * state.  This might cause some updates to be re-run, but UMU_REAP and
2837c478bd9Sstevel@tonic-gate  * UMU_HASH_RESCALE are effectively idempotent, and the worst that can
2847c478bd9Sstevel@tonic-gate  * happen from umem_magazine_resize() is resizing the magazine twice in close
2857c478bd9Sstevel@tonic-gate  * succession.
2867c478bd9Sstevel@tonic-gate  *
2877c478bd9Sstevel@tonic-gate  * Much of the cleanup in umem_release_child() is skipped if
2887c478bd9Sstevel@tonic-gate  * umem_st_update_thr == thr_self().  This is so that applications which call
2897c478bd9Sstevel@tonic-gate  * fork1() from a cache callback does not break.  Needless to say, any such
2907c478bd9Sstevel@tonic-gate  * application is tremendously broken.
2917c478bd9Sstevel@tonic-gate  *
2927c478bd9Sstevel@tonic-gate  *
2937c478bd9Sstevel@tonic-gate  * 5. KM_SLEEP v.s. UMEM_NOFAIL
2947c478bd9Sstevel@tonic-gate  * ----------------------------
2957c478bd9Sstevel@tonic-gate  * Allocations against kmem and vmem have two basic modes:  SLEEP and
2967c478bd9Sstevel@tonic-gate  * NOSLEEP.  A sleeping allocation is will go to sleep (waiting for
2977c478bd9Sstevel@tonic-gate  * more memory) instead of failing (returning NULL).
2987c478bd9Sstevel@tonic-gate  *
2997c478bd9Sstevel@tonic-gate  * SLEEP allocations presume an extremely multithreaded model, with
3007c478bd9Sstevel@tonic-gate  * a lot of allocation and deallocation activity.  umem cannot presume
3017c478bd9Sstevel@tonic-gate  * that its clients have any particular type of behavior.  Instead,
3027c478bd9Sstevel@tonic-gate  * it provides two types of allocations:
3037c478bd9Sstevel@tonic-gate  *
3047c478bd9Sstevel@tonic-gate  *	* UMEM_DEFAULT, equivalent to KM_NOSLEEP (i.e. return NULL on
3057c478bd9Sstevel@tonic-gate  *	failure)
3067c478bd9Sstevel@tonic-gate  *
3077c478bd9Sstevel@tonic-gate  *	* UMEM_NOFAIL, which, on failure, calls an optional callback
3087c478bd9Sstevel@tonic-gate  *	(registered with umem_nofail_callback()).
3097c478bd9Sstevel@tonic-gate  *
3107c478bd9Sstevel@tonic-gate  * The callback is invoked with no locks held, and can do an arbitrary
3117c478bd9Sstevel@tonic-gate  * amount of work.  It then has a choice between:
3127c478bd9Sstevel@tonic-gate  *
3137c478bd9Sstevel@tonic-gate  *	* Returning UMEM_CALLBACK_RETRY, which will cause the allocation
3147c478bd9Sstevel@tonic-gate  *	to be restarted.
3157c478bd9Sstevel@tonic-gate  *
3167c478bd9Sstevel@tonic-gate  *	* Returning UMEM_CALLBACK_EXIT(status), which will cause exit(2)
3177c478bd9Sstevel@tonic-gate  *	to be invoked with status.  If multiple threads attempt to do
3187c478bd9Sstevel@tonic-gate  *	this simultaneously, only one will call exit(2).
3197c478bd9Sstevel@tonic-gate  *
3207c478bd9Sstevel@tonic-gate  *	* Doing some kind of non-local exit (thr_exit(3thr), longjmp(3C),
3217c478bd9Sstevel@tonic-gate  *	etc.)
3227c478bd9Sstevel@tonic-gate  *
3237c478bd9Sstevel@tonic-gate  * The default callback returns UMEM_CALLBACK_EXIT(255).
3247c478bd9Sstevel@tonic-gate  *
3257c478bd9Sstevel@tonic-gate  * To have these callbacks without risk of state corruption (in the case of
3267c478bd9Sstevel@tonic-gate  * a non-local exit), we have to ensure that the callbacks get invoked
3277c478bd9Sstevel@tonic-gate  * close to the original allocation, with no inconsistent state or held
3287c478bd9Sstevel@tonic-gate  * locks.  The following steps are taken:
3297c478bd9Sstevel@tonic-gate  *
3307c478bd9Sstevel@tonic-gate  *	* All invocations of vmem are VM_NOSLEEP.
3317c478bd9Sstevel@tonic-gate  *
3327c478bd9Sstevel@tonic-gate  *	* All constructor callbacks (which can themselves to allocations)
3337c478bd9Sstevel@tonic-gate  *	are passed UMEM_DEFAULT as their required allocation argument.  This
3347c478bd9Sstevel@tonic-gate  *	way, the constructor will fail, allowing the highest-level allocation
3357c478bd9Sstevel@tonic-gate  *	invoke the nofail callback.
3367c478bd9Sstevel@tonic-gate  *
3377c478bd9Sstevel@tonic-gate  *	If a constructor callback _does_ do a UMEM_NOFAIL allocation, and
3387c478bd9Sstevel@tonic-gate  *	the nofail callback does a non-local exit, we will leak the
3397c478bd9Sstevel@tonic-gate  *	partially-constructed buffer.
3401c326e94Sjwadams  *
3411c326e94Sjwadams  *
3421c326e94Sjwadams  * 6. Lock Ordering
3431c326e94Sjwadams  * ----------------
3441c326e94Sjwadams  * umem has a few more locks than kmem does, mostly in the update path.  The
3451c326e94Sjwadams  * overall lock ordering (earlier locks must be acquired first) is:
3461c326e94Sjwadams  *
3471c326e94Sjwadams  *	umem_init_lock
3481c326e94Sjwadams  *
3491c326e94Sjwadams  *	vmem_list_lock
3501c326e94Sjwadams  *	vmem_nosleep_lock.vmpl_mutex
3511c326e94Sjwadams  *	vmem_t's:
3521c326e94Sjwadams  *		vm_lock
353789d94c2Sjwadams  *	sbrk_lock
3541c326e94Sjwadams  *
3551c326e94Sjwadams  *	umem_cache_lock
3561c326e94Sjwadams  *	umem_update_lock
3571c326e94Sjwadams  *	umem_flags_lock
3581c326e94Sjwadams  *	umem_cache_t's:
3591c326e94Sjwadams  *		cache_cpu[*].cc_lock
3601c326e94Sjwadams  *		cache_depot_lock
3611c326e94Sjwadams  *		cache_lock
3621c326e94Sjwadams  *	umem_log_header_t's:
3631c326e94Sjwadams  *		lh_cpu[*].clh_lock
3641c326e94Sjwadams  *		lh_lock
36538849194SRobert Mustacchi  *
36638849194SRobert Mustacchi  * 7. Changing UMEM_MAXBUF
36738849194SRobert Mustacchi  * -----------------------
36838849194SRobert Mustacchi  *
36938849194SRobert Mustacchi  * When changing UMEM_MAXBUF extra care has to be taken. It is not sufficient to
37038849194SRobert Mustacchi  * simply increase this number. First, one must update the umem_alloc_table to
37138849194SRobert Mustacchi  * have the appropriate number of entires based upon the new size. If this is
37238849194SRobert Mustacchi  * not done, this will lead to libumem blowing an assertion.
37338849194SRobert Mustacchi  *
37438849194SRobert Mustacchi  * The second place to update, which is not required, is the umem_alloc_sizes.
37538849194SRobert Mustacchi  * These determine the default cache sizes that we're going to support.
3764f364e7cSRobert Mustacchi  *
3774f364e7cSRobert Mustacchi  * 8. Per-thread caching for malloc/free
3784f364e7cSRobert Mustacchi  * -------------------------------------
3794f364e7cSRobert Mustacchi  *
3804f364e7cSRobert Mustacchi  * "Time is an illusion. Lunchtime doubly so." -- Douglas Adams
3814f364e7cSRobert Mustacchi  *
3824f364e7cSRobert Mustacchi  * Time may be an illusion, but CPU cycles aren't.  While libumem is designed
3834f364e7cSRobert Mustacchi  * to be a highly scalable allocator, that scalability comes with a fixed cycle
3844f364e7cSRobert Mustacchi  * penalty even in the absence of contention: libumem must acquire (and release
3854f364e7cSRobert Mustacchi  * a per-CPU lock for each allocation.  When contention is low and malloc(3C)
3864f364e7cSRobert Mustacchi  * frequency is high, this overhead can dominate execution time.  To alleviate
3874f364e7cSRobert Mustacchi  * this, we allow for per-thread caching, a lock-free means of caching recent
3884f364e7cSRobert Mustacchi  * deallocations on a per-thread basis for use in satisfying subsequent calls
3894f364e7cSRobert Mustacchi  *
3904f364e7cSRobert Mustacchi  * In addition to improving performance, we also want to:
3914f364e7cSRobert Mustacchi  *	* Minimize fragmentation
3924f364e7cSRobert Mustacchi  *	* Not add additional memory overhead (no larger malloc tags)
3934f364e7cSRobert Mustacchi  *
3944f364e7cSRobert Mustacchi  * In the ulwp_t of each thread there is a private data structure called a
3954f364e7cSRobert Mustacchi  * umem_t that looks like:
3964f364e7cSRobert Mustacchi  *
3974f364e7cSRobert Mustacchi  * typedef struct {
39814702342SRobert Mustacchi  *	size_t	tm_size;
39914702342SRobert Mustacchi  *	void	*tm_roots[NTMEMBASE];  (Currently 16)
4004f364e7cSRobert Mustacchi  * } tmem_t;
4014f364e7cSRobert Mustacchi  *
4024f364e7cSRobert Mustacchi  * Each of the roots is treated as the head of a linked list. Each entry in the
4034f364e7cSRobert Mustacchi  * list can be thought of as a void ** which points to the next entry, until one
4044f364e7cSRobert Mustacchi  * of them points to NULL. If the head points to NULL, the list is empty.
4054f364e7cSRobert Mustacchi  *
4064f364e7cSRobert Mustacchi  * Each head corresponds to a umem_cache. Currently there is a linear mapping
4074f364e7cSRobert Mustacchi  * where the first root corresponds to the first cache, second root to the
4084f364e7cSRobert Mustacchi  * second cache, etc. This works because every allocation that malloc makes to
4094f364e7cSRobert Mustacchi  * umem_alloc that can be satisified by a umem_cache will actually return a
4104f364e7cSRobert Mustacchi  * number of bytes equal to the size of that cache. Because of this property and
4114f364e7cSRobert Mustacchi  * a one to one mapping between caches and roots we can guarantee that every
4124f364e7cSRobert Mustacchi  * entry in a given root's list will be able to satisfy the same requests as the
4134f364e7cSRobert Mustacchi  * corresponding cache.
4144f364e7cSRobert Mustacchi  *
4154f364e7cSRobert Mustacchi  * The choice of sixteen roots is based on where we believe we get the biggest
4164f364e7cSRobert Mustacchi  * bang for our buck. The per-thread caches will cache up to 256 byte and 448
4174f364e7cSRobert Mustacchi  * byte allocations on ILP32 and LP64 respectively. Generally applications plan
4184f364e7cSRobert Mustacchi  * more carefully how they do larger allocations than smaller ones. Therefore
4194f364e7cSRobert Mustacchi  * sixteen roots is a reasonable compromise between the amount of additional
4204f364e7cSRobert Mustacchi  * overhead per thread, and the likelihood of a program to benefit from it.
4214f364e7cSRobert Mustacchi  *
4224f364e7cSRobert Mustacchi  * The maximum amount of memory that can be cached in each thread is determined
4234f364e7cSRobert Mustacchi  * by the perthread_cache UMEM_OPTION. It corresponds to the umem_ptc_size
4244f364e7cSRobert Mustacchi  * value. The default value for this is currently 1 MB. Once umem_init() has
4254f364e7cSRobert Mustacchi  * finished this cannot be directly tuned without directly modifying the
4264f364e7cSRobert Mustacchi  * instruction text. If, upon calling free(3C), the amount cached would exceed
4274f364e7cSRobert Mustacchi  * this maximum, we instead actually return the buffer to the umem_cache instead
4284f364e7cSRobert Mustacchi  * of holding onto it in the thread.
4294f364e7cSRobert Mustacchi  *
4304f364e7cSRobert Mustacchi  * When a thread calls malloc(3C) it first determines which umem_cache it
4314f364e7cSRobert Mustacchi  * would be serviced by. If the allocation is not covered by ptcumem it goes to
4324f364e7cSRobert Mustacchi  * the normal malloc instead.  Next, it checks if the tmem_root's list is empty
4334f364e7cSRobert Mustacchi  * or not. If it is empty, we instead go and allocate the memory from
4344f364e7cSRobert Mustacchi  * umem_alloc. If it is not empty, we remove the head of the list, set the
4354f364e7cSRobert Mustacchi  * appropriate malloc tags, and return that buffer.
4364f364e7cSRobert Mustacchi  *
4374f364e7cSRobert Mustacchi  * When a thread calls free(3C) it first looks at the malloc tag and if it is
4384f364e7cSRobert Mustacchi  * invalid or the allocation exceeds the largest cache in ptcumem and sends it
4394f364e7cSRobert Mustacchi  * off to the original free() to handle and clean up appropriately. Next, it
4404f364e7cSRobert Mustacchi  * checks if the allocation size is covered by one of the per-thread roots and
4414f364e7cSRobert Mustacchi  * if it isn't, it passes it off to the original free() to be released. Finally,
4424f364e7cSRobert Mustacchi  * before it inserts this buffer as the head, it checks if adding this buffer
4434f364e7cSRobert Mustacchi  * would put the thread over its maximum cache size. If it would, it frees the
4444f364e7cSRobert Mustacchi  * buffer back to the umem_cache. Otherwise it increments the threads total
4454f364e7cSRobert Mustacchi  * cached amount and makes the buffer the new head of the appropriate tm_root.
4464f364e7cSRobert Mustacchi  *
4474f364e7cSRobert Mustacchi  * When a thread exits, all of the buffers that it has in its per-thread cache
4484f364e7cSRobert Mustacchi  * will be passed to umem_free() and returned to the appropriate umem_cache.
4494f364e7cSRobert Mustacchi  *
4504f364e7cSRobert Mustacchi  * 8.1 Handling addition and removal of umem_caches
4514f364e7cSRobert Mustacchi  * ------------------------------------------------
4524f364e7cSRobert Mustacchi  *
4534f364e7cSRobert Mustacchi  * The set of umem_caches that are used to back calls to umem_alloc() and
4544f364e7cSRobert Mustacchi  * ultimately malloc() are determined at program execution time. The default set
4554f364e7cSRobert Mustacchi  * of caches is defined below in umem_alloc_sizes[]. Various umem_options exist
4564f364e7cSRobert Mustacchi  * that modify the set of caches: size_add, size_clear, and size_remove. Because
4574f364e7cSRobert Mustacchi  * the set of caches can only be determined once umem_init() has been called and
4584f364e7cSRobert Mustacchi  * we have the additional goals of minimizing additional fragmentation and
4594f364e7cSRobert Mustacchi  * metadata space overhead in the malloc tags, this forces our hand to go down a
4604f364e7cSRobert Mustacchi  * slightly different path: the one tread by fasttrap and trapstat.
4614f364e7cSRobert Mustacchi  *
4624f364e7cSRobert Mustacchi  * During umem_init we're going to dynamically construct a new version of
4634f364e7cSRobert Mustacchi  * malloc(3C) and free(3C) that utilizes the known cache sizes and then ensure
4644f364e7cSRobert Mustacchi  * that ptcmalloc and ptcfree replace malloc and free as entries in the plt. If
4654f364e7cSRobert Mustacchi  * ptcmalloc and ptcfree cannot handle a request, they simply jump to the
4664f364e7cSRobert Mustacchi  * original libumem implementations.
4674f364e7cSRobert Mustacchi  *
4684f364e7cSRobert Mustacchi  * After creating all of the umem_caches, but before making them visible,
4694f364e7cSRobert Mustacchi  * umem_cache_init checks that umem_genasm_supported is non-zero. This value is
4704f364e7cSRobert Mustacchi  * set by each architecture in $ARCH/umem_genasm.c to indicate whether or not
4714f364e7cSRobert Mustacchi  * they support this. If the value is zero, then this process is skipped.
4724f364e7cSRobert Mustacchi  * Similarly, if the cache size has been tuned to zero by UMEM_OPTIONS, then
4734f364e7cSRobert Mustacchi  * this is also skipped.
4744f364e7cSRobert Mustacchi  *
4754f364e7cSRobert Mustacchi  * In umem_genasm.c, each architecture's implementation implements a single
4764f364e7cSRobert Mustacchi  * function called umem_genasm() that is responsible for generating the
4774f364e7cSRobert Mustacchi  * appropriate versions of ptcmalloc() and ptcfree(), placing them in the
4784f364e7cSRobert Mustacchi  * appropriate memory location, and finally doing the switch from malloc() and
4794f364e7cSRobert Mustacchi  * free() to ptcmalloc() and ptcfree().  Once the change has been made, there is
4804f364e7cSRobert Mustacchi  * no way to switch back, short of restarting the program or modifying program
4814f364e7cSRobert Mustacchi  * text with mdb.
4824f364e7cSRobert Mustacchi  *
4834f364e7cSRobert Mustacchi  * 8.2 Modifying the Procedure Linkage Table (PLT)
4844f364e7cSRobert Mustacchi  * -----------------------------------------------
4854f364e7cSRobert Mustacchi  *
4864f364e7cSRobert Mustacchi  * The last piece of this puzzle is how we actually jam ptcmalloc() into the
487*9f160f41SRichard Lowe  * PLT.  To handle this, we have defined two functions, _malloc and _free, we
488*9f160f41SRichard Lowe  * use a standard #pragma weak for malloc and free and direct them to those
489*9f160f41SRichard Lowe  * symbols. By default, those symbols have text defined as nops for our
490*9f160f41SRichard Lowe  * generated functions and when they're invoked, they jump to the default
491*9f160f41SRichard Lowe  * malloc and free functions.
492*9f160f41SRichard Lowe  *
493*9f160f41SRichard Lowe  * When umem_genasm() is called, it makes _malloc and _free writeable and goes
494*9f160f41SRichard Lowe  * through and updates the text provided for by _malloc and _free just after
495*9f160f41SRichard Lowe  * the jump. Once both have been successfully generated, umem_genasm() nops
496*9f160f41SRichard Lowe  * over the original jump so that we now call into the genasm versions of
497*9f160f41SRichard Lowe  * these functions, and makes the functions read-only once again.
4984f364e7cSRobert Mustacchi  *
4994f364e7cSRobert Mustacchi  * 8.3 umem_genasm()
5004f364e7cSRobert Mustacchi  * -----------------
5014f364e7cSRobert Mustacchi  *
5024f364e7cSRobert Mustacchi  * umem_genasm() is currently implemented for i386 and amd64. This section
5034f364e7cSRobert Mustacchi  * describes the theory behind the construction. For specific byte code to
5044f364e7cSRobert Mustacchi  * assembly instructions and niceish C and asm versions of ptcmalloc and
5054f364e7cSRobert Mustacchi  * ptcfree, see the individual umem_genasm.c files. The layout consists of the
5064f364e7cSRobert Mustacchi  * following sections:
5074f364e7cSRobert Mustacchi  *
5084f364e7cSRobert Mustacchi  *	o. function-specfic prologue
5094f364e7cSRobert Mustacchi  *	o. function-generic cache-selecting elements
5104f364e7cSRobert Mustacchi  *	o. function-specific epilogue
5114f364e7cSRobert Mustacchi  *
5124f364e7cSRobert Mustacchi  * There are three different generic cache elements that exist:
5134f364e7cSRobert Mustacchi  *
5144f364e7cSRobert Mustacchi  *	o. the last or only cache
5154f364e7cSRobert Mustacchi  *	o. the intermediary caches if more than two
5164f364e7cSRobert Mustacchi  *	o. the first one if more than one cache
5174f364e7cSRobert Mustacchi  *
5184f364e7cSRobert Mustacchi  * The malloc and free prologues and epilogues mimic the necessary portions of
5194f364e7cSRobert Mustacchi  * libumem's malloc and free. This includes things like checking for size
5204f364e7cSRobert Mustacchi  * overflow, setting and verifying the malloc tags.
5214f364e7cSRobert Mustacchi  *
5224f364e7cSRobert Mustacchi  * It is an important constraint that these functions do not make use of the
5234f364e7cSRobert Mustacchi  * call instruction. The only jmp outside of the individual functions is to the
5244f364e7cSRobert Mustacchi  * original libumem malloc and free respectively. Because doing things like
5254f364e7cSRobert Mustacchi  * setting errno or raising an internal umem error on improper malloc tags would
5264f364e7cSRobert Mustacchi  * require using calls into the PLT, whenever we encounter one of those cases we
5274f364e7cSRobert Mustacchi  * just jump to the original malloc and free functions reusing the same stack
5284f364e7cSRobert Mustacchi  * frame.
5294f364e7cSRobert Mustacchi  *
5304f364e7cSRobert Mustacchi  * Each of the above sections, the three caches, and the malloc and free
5314f364e7cSRobert Mustacchi  * prologue and epilogue are implemented as blocks of machine code with the
5324f364e7cSRobert Mustacchi  * corresponding assembly in comments. There are known offsets into each block
5334f364e7cSRobert Mustacchi  * that corresponds to locations of data and addresses that we only know at run
5344f364e7cSRobert Mustacchi  * time. These blocks are copied as necessary and the blanks filled in
5354f364e7cSRobert Mustacchi  * appropriately.
5364f364e7cSRobert Mustacchi  *
5374f364e7cSRobert Mustacchi  * As mentioned in section 8.2, the trampoline library uses specifically named
5384f364e7cSRobert Mustacchi  * variables to communicate the buffers and size to use. These variables are:
5394f364e7cSRobert Mustacchi  *
5404f364e7cSRobert Mustacchi  *	o. umem_genasm_mptr: The buffer for ptcmalloc
5414f364e7cSRobert Mustacchi  *	o. umem_genasm_msize: The size in bytes of the above buffer
5424f364e7cSRobert Mustacchi  *	o. umem_genasm_fptr: The buffer for ptcfree
5434f364e7cSRobert Mustacchi  *	o. umem_genasm_fsize: The size in bytes of the above buffer
5444f364e7cSRobert Mustacchi  *
5454f364e7cSRobert Mustacchi  * Finally, to enable the generated assembly we need to remove the previous jump
5464f364e7cSRobert Mustacchi  * to the actual malloc that exists at the start of these buffers. On x86, this
5474f364e7cSRobert Mustacchi  * is a five byte region. We could zero out the jump offset to be a jmp +0, but
5484f364e7cSRobert Mustacchi  * using nops can be faster. We specifically use a single five byte nop on x86
5494f364e7cSRobert Mustacchi  * as it is faster. When porting ptcumem to other architectures, the various
5504f364e7cSRobert Mustacchi  * opcode changes and options should be analyzed.
5514f364e7cSRobert Mustacchi  *
5524f364e7cSRobert Mustacchi  * 8.4 Interface with libc.so
5534f364e7cSRobert Mustacchi  * --------------------------
5544f364e7cSRobert Mustacchi  *
5554f364e7cSRobert Mustacchi  * The tmem_t structure as described in the beginning of section 8, is part of a
5564f364e7cSRobert Mustacchi  * private interface with libc. There are three functions that exist to cover
5574f364e7cSRobert Mustacchi  * this. They are not documented in man pages or header files. They are in the
5584f364e7cSRobert Mustacchi  * SUNWprivate part of libc's mapfile.
5594f364e7cSRobert Mustacchi  *
5604f364e7cSRobert Mustacchi  *	o. _tmem_get_base(void)
5614f364e7cSRobert Mustacchi  *
56214702342SRobert Mustacchi  *	Returns the offset from the ulwp_t (curthread) to the tmem_t structure.
56314702342SRobert Mustacchi  *	This is a constant for all threads and is effectively a way to to do
56414702342SRobert Mustacchi  *	::offsetof ulwp_t ul_tmem without having to know the specifics of the
56514702342SRobert Mustacchi  *	structure outside of libc.
5664f364e7cSRobert Mustacchi  *
5674f364e7cSRobert Mustacchi  *	o. _tmem_get_nentries(void)
5684f364e7cSRobert Mustacchi  *
5694f364e7cSRobert Mustacchi  *	Returns the number of roots that exist in the tmem_t. This is one part
5704f364e7cSRobert Mustacchi  *	of the cap on the number of umem_caches that we can back with tmem.
5714f364e7cSRobert Mustacchi  *
5724f364e7cSRobert Mustacchi  *	o. _tmem_set_cleanup(void (*)(void *, int))
5734f364e7cSRobert Mustacchi  *
5744f364e7cSRobert Mustacchi  *	This sets a clean up handler that gets called back when a thread exits.
5754f364e7cSRobert Mustacchi  *	There is one call per buffer, the void * is a pointer to the buffer on
5764f364e7cSRobert Mustacchi  *	the list, the int is the index into the roots array for this buffer.
5774f364e7cSRobert Mustacchi  *
5784f364e7cSRobert Mustacchi  * 8.5 Tuning and disabling per-thread caching
5794f364e7cSRobert Mustacchi  * -------------------------------------------
5804f364e7cSRobert Mustacchi  *
5814f364e7cSRobert Mustacchi  * There is only one tunable for per-thread caching:  the amount of memory each
5824f364e7cSRobert Mustacchi  * thread should be able to cache.  This is specified via the perthread_cache
5834f364e7cSRobert Mustacchi  * UMEM_OPTION option.  No attempt is made to to sanity check the specified
5844f364e7cSRobert Mustacchi  * value; the limit is simply the maximum value of a size_t.
5854f364e7cSRobert Mustacchi  *
5864f364e7cSRobert Mustacchi  * If the perthread_cache UMEM_OPTION is set to zero, nomagazines was requested,
5874f364e7cSRobert Mustacchi  * or UMEM_DEBUG has been turned on then we will never call into umem_genasm;
5884f364e7cSRobert Mustacchi  * however, the trampoline audit library and jump will still be in place.
5894f364e7cSRobert Mustacchi  *
5904f364e7cSRobert Mustacchi  * 8.6 Observing efficacy of per-thread caching
5914f364e7cSRobert Mustacchi  * --------------------------------------------
5924f364e7cSRobert Mustacchi  *
5934f364e7cSRobert Mustacchi  * To understand the efficacy of per-thread caching, use the ::umastat dcmd
5944f364e7cSRobert Mustacchi  * to see the percentage of capacity consumed on a per-thread basis, the
5954f364e7cSRobert Mustacchi  * degree to which each umem cache contributes to per-thread cache consumption,
5964f364e7cSRobert Mustacchi  * and the number of buffers in per-thread caches on a per-umem cache basis.
5974f364e7cSRobert Mustacchi  * If more detail is required, the specific buffers in a per-thread cache can
5984f364e7cSRobert Mustacchi  * be iterated over with the umem_ptc_* walkers. (These walkers allow an
5994f364e7cSRobert Mustacchi  * optional ulwp_t to be specified to iterate only over a particular thread's
6004f364e7cSRobert Mustacchi  * cache.)
6017c478bd9Sstevel@tonic-gate  */
6027c478bd9Sstevel@tonic-gate 
6037c478bd9Sstevel@tonic-gate #include <umem_impl.h>
6047c478bd9Sstevel@tonic-gate #include <sys/vmem_impl_user.h>
6057c478bd9Sstevel@tonic-gate #include "umem_base.h"
6067c478bd9Sstevel@tonic-gate #include "vmem_base.h"
6077c478bd9Sstevel@tonic-gate 
6087c478bd9Sstevel@tonic-gate #include <sys/processor.h>
6097c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
6107c478bd9Sstevel@tonic-gate 
6117c478bd9Sstevel@tonic-gate #include <alloca.h>
6127c478bd9Sstevel@tonic-gate #include <errno.h>
6137c478bd9Sstevel@tonic-gate #include <limits.h>
6147c478bd9Sstevel@tonic-gate #include <stdio.h>
6157c478bd9Sstevel@tonic-gate #include <stdlib.h>
6167c478bd9Sstevel@tonic-gate #include <string.h>
6177c478bd9Sstevel@tonic-gate #include <strings.h>
6187c478bd9Sstevel@tonic-gate #include <signal.h>
6197c478bd9Sstevel@tonic-gate #include <unistd.h>
6207c478bd9Sstevel@tonic-gate #include <atomic.h>
6217c478bd9Sstevel@tonic-gate 
6227c478bd9Sstevel@tonic-gate #include "misc.h"
6237c478bd9Sstevel@tonic-gate 
6247c478bd9Sstevel@tonic-gate #define	UMEM_VMFLAGS(umflag)	(VM_NOSLEEP)
6257c478bd9Sstevel@tonic-gate 
6267c478bd9Sstevel@tonic-gate size_t pagesize;
6277c478bd9Sstevel@tonic-gate 
6287c478bd9Sstevel@tonic-gate /*
6297c478bd9Sstevel@tonic-gate  * The default set of caches to back umem_alloc().
6307c478bd9Sstevel@tonic-gate  * These sizes should be reevaluated periodically.
6317c478bd9Sstevel@tonic-gate  *
6327c478bd9Sstevel@tonic-gate  * We want allocations that are multiples of the coherency granularity
6337c478bd9Sstevel@tonic-gate  * (64 bytes) to be satisfied from a cache which is a multiple of 64
6347c478bd9Sstevel@tonic-gate  * bytes, so that it will be 64-byte aligned.  For all multiples of 64,
6357c478bd9Sstevel@tonic-gate  * the next kmem_cache_size greater than or equal to it must be a
6367c478bd9Sstevel@tonic-gate  * multiple of 64.
637789d94c2Sjwadams  *
638789d94c2Sjwadams  * This table must be in sorted order, from smallest to highest.  The
639789d94c2Sjwadams  * highest slot must be UMEM_MAXBUF, and every slot afterwards must be
640789d94c2Sjwadams  * zero.
6417c478bd9Sstevel@tonic-gate  */
642789d94c2Sjwadams static int umem_alloc_sizes[] = {
6437c478bd9Sstevel@tonic-gate #ifdef _LP64
6447c478bd9Sstevel@tonic-gate 	1 * 8,
6457c478bd9Sstevel@tonic-gate 	1 * 16,
6467c478bd9Sstevel@tonic-gate 	2 * 16,
6477c478bd9Sstevel@tonic-gate 	3 * 16,
6487c478bd9Sstevel@tonic-gate #else
6497c478bd9Sstevel@tonic-gate 	1 * 8,
6507c478bd9Sstevel@tonic-gate 	2 * 8,
6517c478bd9Sstevel@tonic-gate 	3 * 8,
6527c478bd9Sstevel@tonic-gate 	4 * 8,		5 * 8,		6 * 8,		7 * 8,
6537c478bd9Sstevel@tonic-gate #endif
6547c478bd9Sstevel@tonic-gate 	4 * 16,		5 * 16,		6 * 16,		7 * 16,
6557c478bd9Sstevel@tonic-gate 	4 * 32,		5 * 32,		6 * 32,		7 * 32,
6567c478bd9Sstevel@tonic-gate 	4 * 64,		5 * 64,		6 * 64,		7 * 64,
6577c478bd9Sstevel@tonic-gate 	4 * 128,	5 * 128,	6 * 128,	7 * 128,
6587c478bd9Sstevel@tonic-gate 	P2ALIGN(8192 / 7, 64),
6597c478bd9Sstevel@tonic-gate 	P2ALIGN(8192 / 6, 64),
6607c478bd9Sstevel@tonic-gate 	P2ALIGN(8192 / 5, 64),
661789d94c2Sjwadams 	P2ALIGN(8192 / 4, 64), 2304,
6627c478bd9Sstevel@tonic-gate 	P2ALIGN(8192 / 3, 64),
663789d94c2Sjwadams 	P2ALIGN(8192 / 2, 64), 4544,
664789d94c2Sjwadams 	P2ALIGN(8192 / 1, 64), 9216,
6657c478bd9Sstevel@tonic-gate 	4096 * 3,
66638849194SRobert Mustacchi 	8192 * 2,				/* = 8192 * 2 */
66738849194SRobert Mustacchi 	24576, 32768, 40960, 49152, 57344, 65536, 73728, 81920,
66838849194SRobert Mustacchi 	90112, 98304, 106496, 114688, 122880, UMEM_MAXBUF, /* 128k */
669789d94c2Sjwadams 	/* 24 slots for user expansion */
670789d94c2Sjwadams 	0, 0, 0, 0, 0, 0, 0, 0,
671789d94c2Sjwadams 	0, 0, 0, 0, 0, 0, 0, 0,
672789d94c2Sjwadams 	0, 0, 0, 0, 0, 0, 0, 0,
6737c478bd9Sstevel@tonic-gate };
6747c478bd9Sstevel@tonic-gate #define	NUM_ALLOC_SIZES (sizeof (umem_alloc_sizes) / sizeof (*umem_alloc_sizes))
6757c478bd9Sstevel@tonic-gate 
6767c478bd9Sstevel@tonic-gate static umem_magtype_t umem_magtype[] = {
6777c478bd9Sstevel@tonic-gate 	{ 1,	8,	3200,	65536	},
6787c478bd9Sstevel@tonic-gate 	{ 3,	16,	256,	32768	},
6797c478bd9Sstevel@tonic-gate 	{ 7,	32,	64,	16384	},
6807c478bd9Sstevel@tonic-gate 	{ 15,	64,	0,	8192	},
6817c478bd9Sstevel@tonic-gate 	{ 31,	64,	0,	4096	},
6827c478bd9Sstevel@tonic-gate 	{ 47,	64,	0,	2048	},
6837c478bd9Sstevel@tonic-gate 	{ 63,	64,	0,	1024	},
6847c478bd9Sstevel@tonic-gate 	{ 95,	64,	0,	512	},
6857c478bd9Sstevel@tonic-gate 	{ 143,	64,	0,	0	},
6867c478bd9Sstevel@tonic-gate };
6877c478bd9Sstevel@tonic-gate 
6887c478bd9Sstevel@tonic-gate /*
6897c478bd9Sstevel@tonic-gate  * umem tunables
6907c478bd9Sstevel@tonic-gate  */
6917c478bd9Sstevel@tonic-gate uint32_t umem_max_ncpus;	/* # of CPU caches. */
6927c478bd9Sstevel@tonic-gate 
6937c478bd9Sstevel@tonic-gate uint32_t umem_stack_depth = 15; /* # stack frames in a bufctl_audit */
6947c478bd9Sstevel@tonic-gate uint32_t umem_reap_interval = 10; /* max reaping rate (seconds) */
6957c478bd9Sstevel@tonic-gate uint_t umem_depot_contention = 2; /* max failed trylocks per real interval */
6967c478bd9Sstevel@tonic-gate uint_t umem_abort = 1;		/* whether to abort on error */
6977c478bd9Sstevel@tonic-gate uint_t umem_output = 0;		/* whether to write to standard error */
6987c478bd9Sstevel@tonic-gate uint_t umem_logging = 0;	/* umem_log_enter() override */
6997c478bd9Sstevel@tonic-gate uint32_t umem_mtbf = 0;		/* mean time between failures [default: off] */
7007c478bd9Sstevel@tonic-gate size_t umem_transaction_log_size; /* size of transaction log */
7017c478bd9Sstevel@tonic-gate size_t umem_content_log_size;	/* size of content log */
7027c478bd9Sstevel@tonic-gate size_t umem_failure_log_size;	/* failure log [4 pages per CPU] */
7037c478bd9Sstevel@tonic-gate size_t umem_slab_log_size;	/* slab create log [4 pages per CPU] */
7047c478bd9Sstevel@tonic-gate size_t umem_content_maxsave = 256; /* UMF_CONTENTS max bytes to log */
7057c478bd9Sstevel@tonic-gate size_t umem_lite_minsize = 0;	/* minimum buffer size for UMF_LITE */
7067c478bd9Sstevel@tonic-gate size_t umem_lite_maxalign = 1024; /* maximum buffer alignment for UMF_LITE */
7077c478bd9Sstevel@tonic-gate size_t umem_maxverify;		/* maximum bytes to inspect in debug routines */
7087c478bd9Sstevel@tonic-gate size_t umem_minfirewall;	/* hardware-enforced redzone threshold */
7094f364e7cSRobert Mustacchi size_t umem_ptc_size = 1048576;	/* size of per-thread cache (in bytes) */
7107c478bd9Sstevel@tonic-gate 
7117c478bd9Sstevel@tonic-gate uint_t umem_flags = 0;
7124f364e7cSRobert Mustacchi uintptr_t umem_tmem_off;
7137c478bd9Sstevel@tonic-gate 
7147c478bd9Sstevel@tonic-gate mutex_t			umem_init_lock;		/* locks initialization */
7157c478bd9Sstevel@tonic-gate cond_t			umem_init_cv;		/* initialization CV */
7167c478bd9Sstevel@tonic-gate thread_t		umem_init_thr;		/* thread initializing */
7177c478bd9Sstevel@tonic-gate int			umem_init_env_ready;	/* environ pre-initted */
7187c478bd9Sstevel@tonic-gate int			umem_ready = UMEM_READY_STARTUP;
7197c478bd9Sstevel@tonic-gate 
7204f364e7cSRobert Mustacchi int			umem_ptc_enabled;	/* per-thread caching enabled */
7214f364e7cSRobert Mustacchi 
7227c478bd9Sstevel@tonic-gate static umem_nofail_callback_t *nofail_callback;
7237c478bd9Sstevel@tonic-gate static mutex_t		umem_nofail_exit_lock;
7247c478bd9Sstevel@tonic-gate static thread_t		umem_nofail_exit_thr;
7257c478bd9Sstevel@tonic-gate 
7267c478bd9Sstevel@tonic-gate static umem_cache_t	*umem_slab_cache;
7277c478bd9Sstevel@tonic-gate static umem_cache_t	*umem_bufctl_cache;
7287c478bd9Sstevel@tonic-gate static umem_cache_t	*umem_bufctl_audit_cache;
7297c478bd9Sstevel@tonic-gate 
7307c478bd9Sstevel@tonic-gate mutex_t			umem_flags_lock;
7317c478bd9Sstevel@tonic-gate 
7327c478bd9Sstevel@tonic-gate static vmem_t		*heap_arena;
7337c478bd9Sstevel@tonic-gate static vmem_alloc_t	*heap_alloc;
7347c478bd9Sstevel@tonic-gate static vmem_free_t	*heap_free;
7357c478bd9Sstevel@tonic-gate 
7367c478bd9Sstevel@tonic-gate static vmem_t		*umem_internal_arena;
7377c478bd9Sstevel@tonic-gate static vmem_t		*umem_cache_arena;
7387c478bd9Sstevel@tonic-gate static vmem_t		*umem_hash_arena;
7397c478bd9Sstevel@tonic-gate static vmem_t		*umem_log_arena;
7407c478bd9Sstevel@tonic-gate static vmem_t		*umem_oversize_arena;
7417c478bd9Sstevel@tonic-gate static vmem_t		*umem_va_arena;
7427c478bd9Sstevel@tonic-gate static vmem_t		*umem_default_arena;
7437c478bd9Sstevel@tonic-gate static vmem_t		*umem_firewall_va_arena;
7447c478bd9Sstevel@tonic-gate static vmem_t		*umem_firewall_arena;
7457c478bd9Sstevel@tonic-gate 
7467c478bd9Sstevel@tonic-gate vmem_t			*umem_memalign_arena;
7477c478bd9Sstevel@tonic-gate 
7487c478bd9Sstevel@tonic-gate umem_log_header_t *umem_transaction_log;
7497c478bd9Sstevel@tonic-gate umem_log_header_t *umem_content_log;
7507c478bd9Sstevel@tonic-gate umem_log_header_t *umem_failure_log;
7517c478bd9Sstevel@tonic-gate umem_log_header_t *umem_slab_log;
7527c478bd9Sstevel@tonic-gate 
7537257d1b4Sraf #define	CPUHINT()		(thr_self())
7547c478bd9Sstevel@tonic-gate #define	CPUHINT_MAX()		INT_MAX
7557c478bd9Sstevel@tonic-gate 
7567c478bd9Sstevel@tonic-gate #define	CPU(mask)		(umem_cpus + (CPUHINT() & (mask)))
7577c478bd9Sstevel@tonic-gate static umem_cpu_t umem_startup_cpu = {	/* initial, single, cpu */
7587c478bd9Sstevel@tonic-gate 	UMEM_CACHE_SIZE(0),
7597c478bd9Sstevel@tonic-gate 	0
7607c478bd9Sstevel@tonic-gate };
7617c478bd9Sstevel@tonic-gate 
7627c478bd9Sstevel@tonic-gate static uint32_t umem_cpu_mask = 0;			/* global cpu mask */
7637c478bd9Sstevel@tonic-gate static umem_cpu_t *umem_cpus = &umem_startup_cpu;	/* cpu list */
7647c478bd9Sstevel@tonic-gate 
7657c478bd9Sstevel@tonic-gate volatile uint32_t umem_reaping;
7667c478bd9Sstevel@tonic-gate 
7677c478bd9Sstevel@tonic-gate thread_t		umem_update_thr;
7687c478bd9Sstevel@tonic-gate struct timeval		umem_update_next;	/* timeofday of next update */
7697c478bd9Sstevel@tonic-gate volatile thread_t	umem_st_update_thr;	/* only used when single-thd */
7707c478bd9Sstevel@tonic-gate 
7717c478bd9Sstevel@tonic-gate #define	IN_UPDATE()	(thr_self() == umem_update_thr || \
7727c478bd9Sstevel@tonic-gate 			    thr_self() == umem_st_update_thr)
7737c478bd9Sstevel@tonic-gate #define	IN_REAP()	IN_UPDATE()
7747c478bd9Sstevel@tonic-gate 
7757c478bd9Sstevel@tonic-gate mutex_t			umem_update_lock;	/* cache_u{next,prev,flags} */
7767c478bd9Sstevel@tonic-gate cond_t			umem_update_cv;
7777c478bd9Sstevel@tonic-gate 
7787c478bd9Sstevel@tonic-gate volatile hrtime_t umem_reap_next;	/* min hrtime of next reap */
7797c478bd9Sstevel@tonic-gate 
7807c478bd9Sstevel@tonic-gate mutex_t			umem_cache_lock;	/* inter-cache linkage only */
7817c478bd9Sstevel@tonic-gate 
7827c478bd9Sstevel@tonic-gate #ifdef UMEM_STANDALONE
7837c478bd9Sstevel@tonic-gate umem_cache_t		umem_null_cache;
7847c478bd9Sstevel@tonic-gate static const umem_cache_t umem_null_cache_template = {
7857c478bd9Sstevel@tonic-gate #else
7867c478bd9Sstevel@tonic-gate umem_cache_t		umem_null_cache = {
7877c478bd9Sstevel@tonic-gate #endif
7887c478bd9Sstevel@tonic-gate 	0, 0, 0, 0, 0,
7897c478bd9Sstevel@tonic-gate 	0, 0,
7907c478bd9Sstevel@tonic-gate 	0, 0,
7917c478bd9Sstevel@tonic-gate 	0, 0,
7927c478bd9Sstevel@tonic-gate 	"invalid_cache",
7937c478bd9Sstevel@tonic-gate 	0, 0,
7947c478bd9Sstevel@tonic-gate 	NULL, NULL, NULL, NULL,
7957c478bd9Sstevel@tonic-gate 	NULL,
7967c478bd9Sstevel@tonic-gate 	0, 0, 0, 0,
7977c478bd9Sstevel@tonic-gate 	&umem_null_cache, &umem_null_cache,
7987c478bd9Sstevel@tonic-gate 	&umem_null_cache, &umem_null_cache,
7997c478bd9Sstevel@tonic-gate 	0,
8007c478bd9Sstevel@tonic-gate 	DEFAULTMUTEX,				/* start of slab layer */
8017c478bd9Sstevel@tonic-gate 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
8027c478bd9Sstevel@tonic-gate 	&umem_null_cache.cache_nullslab,
8037c478bd9Sstevel@tonic-gate 	{
8047c478bd9Sstevel@tonic-gate 		&umem_null_cache,
8057c478bd9Sstevel@tonic-gate 		NULL,
8067c478bd9Sstevel@tonic-gate 		&umem_null_cache.cache_nullslab,
8077c478bd9Sstevel@tonic-gate 		&umem_null_cache.cache_nullslab,
8087c478bd9Sstevel@tonic-gate 		NULL,
8097c478bd9Sstevel@tonic-gate 		-1,
8107c478bd9Sstevel@tonic-gate 		0
8117c478bd9Sstevel@tonic-gate 	},
8127c478bd9Sstevel@tonic-gate 	NULL,
8137c478bd9Sstevel@tonic-gate 	NULL,
8147c478bd9Sstevel@tonic-gate 	DEFAULTMUTEX,				/* start of depot layer */
8157c478bd9Sstevel@tonic-gate 	NULL, {
8167c478bd9Sstevel@tonic-gate 		NULL, 0, 0, 0, 0
8177c478bd9Sstevel@tonic-gate 	}, {
8187c478bd9Sstevel@tonic-gate 		NULL, 0, 0, 0, 0
8197c478bd9Sstevel@tonic-gate 	}, {
8207c478bd9Sstevel@tonic-gate 		{
8217c478bd9Sstevel@tonic-gate 			DEFAULTMUTEX,		/* start of CPU cache */
8227c478bd9Sstevel@tonic-gate 			0, 0, NULL, NULL, -1, -1, 0
8237c478bd9Sstevel@tonic-gate 		}
8247c478bd9Sstevel@tonic-gate 	}
8257c478bd9Sstevel@tonic-gate };
8267c478bd9Sstevel@tonic-gate 
8277c478bd9Sstevel@tonic-gate #define	ALLOC_TABLE_4 \
8287c478bd9Sstevel@tonic-gate 	&umem_null_cache, &umem_null_cache, &umem_null_cache, &umem_null_cache
8297c478bd9Sstevel@tonic-gate 
8307c478bd9Sstevel@tonic-gate #define	ALLOC_TABLE_64 \
8317c478bd9Sstevel@tonic-gate 	ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, \
8327c478bd9Sstevel@tonic-gate 	ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, \
8337c478bd9Sstevel@tonic-gate 	ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, \
8347c478bd9Sstevel@tonic-gate 	ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4
8357c478bd9Sstevel@tonic-gate 
8367c478bd9Sstevel@tonic-gate #define	ALLOC_TABLE_1024 \
8377c478bd9Sstevel@tonic-gate 	ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, \
8387c478bd9Sstevel@tonic-gate 	ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, \
8397c478bd9Sstevel@tonic-gate 	ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, \
8407c478bd9Sstevel@tonic-gate 	ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64
8417c478bd9Sstevel@tonic-gate 
8427c478bd9Sstevel@tonic-gate static umem_cache_t *umem_alloc_table[UMEM_MAXBUF >> UMEM_ALIGN_SHIFT] = {
84338849194SRobert Mustacchi 	ALLOC_TABLE_1024,
84438849194SRobert Mustacchi 	ALLOC_TABLE_1024,
84538849194SRobert Mustacchi 	ALLOC_TABLE_1024,
84638849194SRobert Mustacchi 	ALLOC_TABLE_1024,
84738849194SRobert Mustacchi 	ALLOC_TABLE_1024,
84838849194SRobert Mustacchi 	ALLOC_TABLE_1024,
84938849194SRobert Mustacchi 	ALLOC_TABLE_1024,
85038849194SRobert Mustacchi 	ALLOC_TABLE_1024,
85138849194SRobert Mustacchi 	ALLOC_TABLE_1024,
85238849194SRobert Mustacchi 	ALLOC_TABLE_1024,
85338849194SRobert Mustacchi 	ALLOC_TABLE_1024,
85438849194SRobert Mustacchi 	ALLOC_TABLE_1024,
85538849194SRobert Mustacchi 	ALLOC_TABLE_1024,
85638849194SRobert Mustacchi 	ALLOC_TABLE_1024,
8577c478bd9Sstevel@tonic-gate 	ALLOC_TABLE_1024,
8587c478bd9Sstevel@tonic-gate 	ALLOC_TABLE_1024
8597c478bd9Sstevel@tonic-gate };
8607c478bd9Sstevel@tonic-gate 
8617c478bd9Sstevel@tonic-gate 
8627c478bd9Sstevel@tonic-gate /* Used to constrain audit-log stack traces */
8637c478bd9Sstevel@tonic-gate caddr_t			umem_min_stack;
8647c478bd9Sstevel@tonic-gate caddr_t			umem_max_stack;
8657c478bd9Sstevel@tonic-gate 
8667c478bd9Sstevel@tonic-gate 
8677c478bd9Sstevel@tonic-gate #define	UMERR_MODIFIED	0	/* buffer modified while on freelist */
8687c478bd9Sstevel@tonic-gate #define	UMERR_REDZONE	1	/* redzone violation (write past end of buf) */
8697c478bd9Sstevel@tonic-gate #define	UMERR_DUPFREE	2	/* freed a buffer twice */
8707c478bd9Sstevel@tonic-gate #define	UMERR_BADADDR	3	/* freed a bad (unallocated) address */
8717c478bd9Sstevel@tonic-gate #define	UMERR_BADBUFTAG	4	/* buftag corrupted */
8727c478bd9Sstevel@tonic-gate #define	UMERR_BADBUFCTL	5	/* bufctl corrupted */
8737c478bd9Sstevel@tonic-gate #define	UMERR_BADCACHE	6	/* freed a buffer to the wrong cache */
8747c478bd9Sstevel@tonic-gate #define	UMERR_BADSIZE	7	/* alloc size != free size */
8757c478bd9Sstevel@tonic-gate #define	UMERR_BADBASE	8	/* buffer base address wrong */
8767c478bd9Sstevel@tonic-gate 
8777c478bd9Sstevel@tonic-gate struct {
8787c478bd9Sstevel@tonic-gate 	hrtime_t	ump_timestamp;	/* timestamp of error */
8797c478bd9Sstevel@tonic-gate 	int		ump_error;	/* type of umem error (UMERR_*) */
8807c478bd9Sstevel@tonic-gate 	void		*ump_buffer;	/* buffer that induced abort */
8817c478bd9Sstevel@tonic-gate 	void		*ump_realbuf;	/* real start address for buffer */
8827c478bd9Sstevel@tonic-gate 	umem_cache_t	*ump_cache;	/* buffer's cache according to client */
8837c478bd9Sstevel@tonic-gate 	umem_cache_t	*ump_realcache;	/* actual cache containing buffer */
8847c478bd9Sstevel@tonic-gate 	umem_slab_t	*ump_slab;	/* slab accoring to umem_findslab() */
8857c478bd9Sstevel@tonic-gate 	umem_bufctl_t	*ump_bufctl;	/* bufctl */
8867c478bd9Sstevel@tonic-gate } umem_abort_info;
8877c478bd9Sstevel@tonic-gate 
8887c478bd9Sstevel@tonic-gate static void
copy_pattern(uint64_t pattern,void * buf_arg,size_t size)8897c478bd9Sstevel@tonic-gate copy_pattern(uint64_t pattern, void *buf_arg, size_t size)
8907c478bd9Sstevel@tonic-gate {
8917c478bd9Sstevel@tonic-gate 	uint64_t *bufend = (uint64_t *)((char *)buf_arg + size);
8927c478bd9Sstevel@tonic-gate 	uint64_t *buf = buf_arg;
8937c478bd9Sstevel@tonic-gate 
8947c478bd9Sstevel@tonic-gate 	while (buf < bufend)
8957c478bd9Sstevel@tonic-gate 		*buf++ = pattern;
8967c478bd9Sstevel@tonic-gate }
8977c478bd9Sstevel@tonic-gate 
8987c478bd9Sstevel@tonic-gate static void *
verify_pattern(uint64_t pattern,void * buf_arg,size_t size)8997c478bd9Sstevel@tonic-gate verify_pattern(uint64_t pattern, void *buf_arg, size_t size)
9007c478bd9Sstevel@tonic-gate {
9017c478bd9Sstevel@tonic-gate 	uint64_t *bufend = (uint64_t *)((char *)buf_arg + size);
9027c478bd9Sstevel@tonic-gate 	uint64_t *buf;
9037c478bd9Sstevel@tonic-gate 
9047c478bd9Sstevel@tonic-gate 	for (buf = buf_arg; buf < bufend; buf++)
9057c478bd9Sstevel@tonic-gate 		if (*buf != pattern)
9067c478bd9Sstevel@tonic-gate 			return (buf);
9077c478bd9Sstevel@tonic-gate 	return (NULL);
9087c478bd9Sstevel@tonic-gate }
9097c478bd9Sstevel@tonic-gate 
9107c478bd9Sstevel@tonic-gate static void *
verify_and_copy_pattern(uint64_t old,uint64_t new,void * buf_arg,size_t size)9117c478bd9Sstevel@tonic-gate verify_and_copy_pattern(uint64_t old, uint64_t new, void *buf_arg, size_t size)
9127c478bd9Sstevel@tonic-gate {
9137c478bd9Sstevel@tonic-gate 	uint64_t *bufend = (uint64_t *)((char *)buf_arg + size);
9147c478bd9Sstevel@tonic-gate 	uint64_t *buf;
9157c478bd9Sstevel@tonic-gate 
9167c478bd9Sstevel@tonic-gate 	for (buf = buf_arg; buf < bufend; buf++) {
9177c478bd9Sstevel@tonic-gate 		if (*buf != old) {
9187c478bd9Sstevel@tonic-gate 			copy_pattern(old, buf_arg,
9197c478bd9Sstevel@tonic-gate 			    (char *)buf - (char *)buf_arg);
9207c478bd9Sstevel@tonic-gate 			return (buf);
9217c478bd9Sstevel@tonic-gate 		}
9227c478bd9Sstevel@tonic-gate 		*buf = new;
9237c478bd9Sstevel@tonic-gate 	}
9247c478bd9Sstevel@tonic-gate 
9257c478bd9Sstevel@tonic-gate 	return (NULL);
9267c478bd9Sstevel@tonic-gate }
9277c478bd9Sstevel@tonic-gate 
9287c478bd9Sstevel@tonic-gate void
umem_cache_applyall(void (* func)(umem_cache_t *))9297c478bd9Sstevel@tonic-gate umem_cache_applyall(void (*func)(umem_cache_t *))
9307c478bd9Sstevel@tonic-gate {
9317c478bd9Sstevel@tonic-gate 	umem_cache_t *cp;
9327c478bd9Sstevel@tonic-gate 
9337c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&umem_cache_lock);
9347c478bd9Sstevel@tonic-gate 	for (cp = umem_null_cache.cache_next; cp != &umem_null_cache;
9357c478bd9Sstevel@tonic-gate 	    cp = cp->cache_next)
9367c478bd9Sstevel@tonic-gate 		func(cp);
9377c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&umem_cache_lock);
9387c478bd9Sstevel@tonic-gate }
9397c478bd9Sstevel@tonic-gate 
9407c478bd9Sstevel@tonic-gate static void
umem_add_update_unlocked(umem_cache_t * cp,int flags)9417c478bd9Sstevel@tonic-gate umem_add_update_unlocked(umem_cache_t *cp, int flags)
9427c478bd9Sstevel@tonic-gate {
9437c478bd9Sstevel@tonic-gate 	umem_cache_t *cnext, *cprev;
9447c478bd9Sstevel@tonic-gate 
9457c478bd9Sstevel@tonic-gate 	flags &= ~UMU_ACTIVE;
9467c478bd9Sstevel@tonic-gate 
9477c478bd9Sstevel@tonic-gate 	if (!flags)
9487c478bd9Sstevel@tonic-gate 		return;
9497c478bd9Sstevel@tonic-gate 
9507c478bd9Sstevel@tonic-gate 	if (cp->cache_uflags & UMU_ACTIVE) {
9517c478bd9Sstevel@tonic-gate 		cp->cache_uflags |= flags;
9527c478bd9Sstevel@tonic-gate 	} else {
9537c478bd9Sstevel@tonic-gate 		if (cp->cache_unext != NULL) {
9547c478bd9Sstevel@tonic-gate 			ASSERT(cp->cache_uflags != 0);
9557c478bd9Sstevel@tonic-gate 			cp->cache_uflags |= flags;
9567c478bd9Sstevel@tonic-gate 		} else {
9577c478bd9Sstevel@tonic-gate 			ASSERT(cp->cache_uflags == 0);
9587c478bd9Sstevel@tonic-gate 			cp->cache_uflags = flags;
9597c478bd9Sstevel@tonic-gate 			cp->cache_unext = cnext = &umem_null_cache;
9607c478bd9Sstevel@tonic-gate 			cp->cache_uprev = cprev = umem_null_cache.cache_uprev;
9617c478bd9Sstevel@tonic-gate 			cnext->cache_uprev = cp;
9627c478bd9Sstevel@tonic-gate 			cprev->cache_unext = cp;
9637c478bd9Sstevel@tonic-gate 		}
9647c478bd9Sstevel@tonic-gate 	}
9657c478bd9Sstevel@tonic-gate }
9667c478bd9Sstevel@tonic-gate 
9677c478bd9Sstevel@tonic-gate static void
umem_add_update(umem_cache_t * cp,int flags)9687c478bd9Sstevel@tonic-gate umem_add_update(umem_cache_t *cp, int flags)
9697c478bd9Sstevel@tonic-gate {
9707c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&umem_update_lock);
9717c478bd9Sstevel@tonic-gate 
9727c478bd9Sstevel@tonic-gate 	umem_add_update_unlocked(cp, flags);
9737c478bd9Sstevel@tonic-gate 
9747c478bd9Sstevel@tonic-gate 	if (!IN_UPDATE())
9757c478bd9Sstevel@tonic-gate 		(void) cond_broadcast(&umem_update_cv);
9767c478bd9Sstevel@tonic-gate 
9777c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&umem_update_lock);
9787c478bd9Sstevel@tonic-gate }
9797c478bd9Sstevel@tonic-gate 
9807c478bd9Sstevel@tonic-gate /*
9817c478bd9Sstevel@tonic-gate  * Remove a cache from the update list, waiting for any in-progress work to
9827c478bd9Sstevel@tonic-gate  * complete first.
9837c478bd9Sstevel@tonic-gate  */
9847c478bd9Sstevel@tonic-gate static void
umem_remove_updates(umem_cache_t * cp)9857c478bd9Sstevel@tonic-gate umem_remove_updates(umem_cache_t *cp)
9867c478bd9Sstevel@tonic-gate {
9877c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&umem_update_lock);
9887c478bd9Sstevel@tonic-gate 
9897c478bd9Sstevel@tonic-gate 	/*
9907c478bd9Sstevel@tonic-gate 	 * Get it out of the active state
9917c478bd9Sstevel@tonic-gate 	 */
9927c478bd9Sstevel@tonic-gate 	while (cp->cache_uflags & UMU_ACTIVE) {
993a574db85Sraf 		int cancel_state;
994a574db85Sraf 
9957c478bd9Sstevel@tonic-gate 		ASSERT(cp->cache_unext == NULL);
9967c478bd9Sstevel@tonic-gate 
9977c478bd9Sstevel@tonic-gate 		cp->cache_uflags |= UMU_NOTIFY;
9987c478bd9Sstevel@tonic-gate 
9997c478bd9Sstevel@tonic-gate 		/*
10007c478bd9Sstevel@tonic-gate 		 * Make sure the update state is sane, before we wait
10017c478bd9Sstevel@tonic-gate 		 */
10027c478bd9Sstevel@tonic-gate 		ASSERT(umem_update_thr != 0 || umem_st_update_thr != 0);
10037c478bd9Sstevel@tonic-gate 		ASSERT(umem_update_thr != thr_self() &&
10047c478bd9Sstevel@tonic-gate 		    umem_st_update_thr != thr_self());
10057c478bd9Sstevel@tonic-gate 
1006a574db85Sraf 		(void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE,
1007a574db85Sraf 		    &cancel_state);
1008a574db85Sraf 		(void) cond_wait(&umem_update_cv, &umem_update_lock);
1009a574db85Sraf 		(void) pthread_setcancelstate(cancel_state, NULL);
10107c478bd9Sstevel@tonic-gate 	}
10117c478bd9Sstevel@tonic-gate 	/*
10127c478bd9Sstevel@tonic-gate 	 * Get it out of the Work Requested state
10137c478bd9Sstevel@tonic-gate 	 */
10147c478bd9Sstevel@tonic-gate 	if (cp->cache_unext != NULL) {
10157c478bd9Sstevel@tonic-gate 		cp->cache_uprev->cache_unext = cp->cache_unext;
10167c478bd9Sstevel@tonic-gate 		cp->cache_unext->cache_uprev = cp->cache_uprev;
10177c478bd9Sstevel@tonic-gate 		cp->cache_uprev = cp->cache_unext = NULL;
10187c478bd9Sstevel@tonic-gate 		cp->cache_uflags = 0;
10197c478bd9Sstevel@tonic-gate 	}
10207c478bd9Sstevel@tonic-gate 	/*
10217c478bd9Sstevel@tonic-gate 	 * Make sure it is in the Inactive state
10227c478bd9Sstevel@tonic-gate 	 */
10237c478bd9Sstevel@tonic-gate 	ASSERT(cp->cache_unext == NULL && cp->cache_uflags == 0);
10247c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&umem_update_lock);
10257c478bd9Sstevel@tonic-gate }
10267c478bd9Sstevel@tonic-gate 
10277c478bd9Sstevel@tonic-gate static void
umem_updateall(int flags)10287c478bd9Sstevel@tonic-gate umem_updateall(int flags)
10297c478bd9Sstevel@tonic-gate {
10307c478bd9Sstevel@tonic-gate 	umem_cache_t *cp;
10317c478bd9Sstevel@tonic-gate 
10327c478bd9Sstevel@tonic-gate 	/*
10337c478bd9Sstevel@tonic-gate 	 * NOTE:  To prevent deadlock, umem_cache_lock is always acquired first.
10347c478bd9Sstevel@tonic-gate 	 *
10357c478bd9Sstevel@tonic-gate 	 * (umem_add_update is called from things run via umem_cache_applyall)
10367c478bd9Sstevel@tonic-gate 	 */
10377c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&umem_cache_lock);
10387c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&umem_update_lock);
10397c478bd9Sstevel@tonic-gate 
10407c478bd9Sstevel@tonic-gate 	for (cp = umem_null_cache.cache_next; cp != &umem_null_cache;
10417c478bd9Sstevel@tonic-gate 	    cp = cp->cache_next)
10427c478bd9Sstevel@tonic-gate 		umem_add_update_unlocked(cp, flags);
10437c478bd9Sstevel@tonic-gate 
10447c478bd9Sstevel@tonic-gate 	if (!IN_UPDATE())
10457c478bd9Sstevel@tonic-gate 		(void) cond_broadcast(&umem_update_cv);
10467c478bd9Sstevel@tonic-gate 
10477c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&umem_update_lock);
10487c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&umem_cache_lock);
10497c478bd9Sstevel@tonic-gate }
10507c478bd9Sstevel@tonic-gate 
10517c478bd9Sstevel@tonic-gate /*
10527c478bd9Sstevel@tonic-gate  * Debugging support.  Given a buffer address, find its slab.
10537c478bd9Sstevel@tonic-gate  */
10547c478bd9Sstevel@tonic-gate static umem_slab_t *
umem_findslab(umem_cache_t * cp,void * buf)10557c478bd9Sstevel@tonic-gate umem_findslab(umem_cache_t *cp, void *buf)
10567c478bd9Sstevel@tonic-gate {
10577c478bd9Sstevel@tonic-gate 	umem_slab_t *sp;
10587c478bd9Sstevel@tonic-gate 
10597c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&cp->cache_lock);
10607c478bd9Sstevel@tonic-gate 	for (sp = cp->cache_nullslab.slab_next;
10617c478bd9Sstevel@tonic-gate 	    sp != &cp->cache_nullslab; sp = sp->slab_next) {
10627c478bd9Sstevel@tonic-gate 		if (UMEM_SLAB_MEMBER(sp, buf)) {
10637c478bd9Sstevel@tonic-gate 			(void) mutex_unlock(&cp->cache_lock);
10647c478bd9Sstevel@tonic-gate 			return (sp);
10657c478bd9Sstevel@tonic-gate 		}
10667c478bd9Sstevel@tonic-gate 	}
10677c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&cp->cache_lock);
10687c478bd9Sstevel@tonic-gate 
10697c478bd9Sstevel@tonic-gate 	return (NULL);
10707c478bd9Sstevel@tonic-gate }
10717c478bd9Sstevel@tonic-gate 
10727c478bd9Sstevel@tonic-gate static void
umem_error(int error,umem_cache_t * cparg,void * bufarg)10737c478bd9Sstevel@tonic-gate umem_error(int error, umem_cache_t *cparg, void *bufarg)
10747c478bd9Sstevel@tonic-gate {
10757c478bd9Sstevel@tonic-gate 	umem_buftag_t *btp = NULL;
10767c478bd9Sstevel@tonic-gate 	umem_bufctl_t *bcp = NULL;
10777c478bd9Sstevel@tonic-gate 	umem_cache_t *cp = cparg;
10787c478bd9Sstevel@tonic-gate 	umem_slab_t *sp;
10797c478bd9Sstevel@tonic-gate 	uint64_t *off;
10807c478bd9Sstevel@tonic-gate 	void *buf = bufarg;
10817c478bd9Sstevel@tonic-gate 
10827c478bd9Sstevel@tonic-gate 	int old_logging = umem_logging;
10837c478bd9Sstevel@tonic-gate 
10847c478bd9Sstevel@tonic-gate 	umem_logging = 0;	/* stop logging when a bad thing happens */
10857c478bd9Sstevel@tonic-gate 
10867c478bd9Sstevel@tonic-gate 	umem_abort_info.ump_timestamp = gethrtime();
10877c478bd9Sstevel@tonic-gate 
10887c478bd9Sstevel@tonic-gate 	sp = umem_findslab(cp, buf);
10897c478bd9Sstevel@tonic-gate 	if (sp == NULL) {
10907c478bd9Sstevel@tonic-gate 		for (cp = umem_null_cache.cache_prev; cp != &umem_null_cache;
10917c478bd9Sstevel@tonic-gate 		    cp = cp->cache_prev) {
10927c478bd9Sstevel@tonic-gate 			if ((sp = umem_findslab(cp, buf)) != NULL)
10937c478bd9Sstevel@tonic-gate 				break;
10947c478bd9Sstevel@tonic-gate 		}
10957c478bd9Sstevel@tonic-gate 	}
10967c478bd9Sstevel@tonic-gate 
10977c478bd9Sstevel@tonic-gate 	if (sp == NULL) {
10987c478bd9Sstevel@tonic-gate 		cp = NULL;
10997c478bd9Sstevel@tonic-gate 		error = UMERR_BADADDR;
11007c478bd9Sstevel@tonic-gate 	} else {
11017c478bd9Sstevel@tonic-gate 		if (cp != cparg)
11027c478bd9Sstevel@tonic-gate 			error = UMERR_BADCACHE;
11037c478bd9Sstevel@tonic-gate 		else
11047c478bd9Sstevel@tonic-gate 			buf = (char *)bufarg - ((uintptr_t)bufarg -
11057c478bd9Sstevel@tonic-gate 			    (uintptr_t)sp->slab_base) % cp->cache_chunksize;
11067c478bd9Sstevel@tonic-gate 		if (buf != bufarg)
11077c478bd9Sstevel@tonic-gate 			error = UMERR_BADBASE;
11087c478bd9Sstevel@tonic-gate 		if (cp->cache_flags & UMF_BUFTAG)
11097c478bd9Sstevel@tonic-gate 			btp = UMEM_BUFTAG(cp, buf);
11107c478bd9Sstevel@tonic-gate 		if (cp->cache_flags & UMF_HASH) {
11117c478bd9Sstevel@tonic-gate 			(void) mutex_lock(&cp->cache_lock);
11127c478bd9Sstevel@tonic-gate 			for (bcp = *UMEM_HASH(cp, buf); bcp; bcp = bcp->bc_next)
11137c478bd9Sstevel@tonic-gate 				if (bcp->bc_addr == buf)
11147c478bd9Sstevel@tonic-gate 					break;
11157c478bd9Sstevel@tonic-gate 			(void) mutex_unlock(&cp->cache_lock);
11167c478bd9Sstevel@tonic-gate 			if (bcp == NULL && btp != NULL)
11177c478bd9Sstevel@tonic-gate 				bcp = btp->bt_bufctl;
11187c478bd9Sstevel@tonic-gate 			if (umem_findslab(cp->cache_bufctl_cache, bcp) ==
11197c478bd9Sstevel@tonic-gate 			    NULL || P2PHASE((uintptr_t)bcp, UMEM_ALIGN) ||
11207c478bd9Sstevel@tonic-gate 			    bcp->bc_addr != buf) {
11217c478bd9Sstevel@tonic-gate 				error = UMERR_BADBUFCTL;
11227c478bd9Sstevel@tonic-gate 				bcp = NULL;
11237c478bd9Sstevel@tonic-gate 			}
11247c478bd9Sstevel@tonic-gate 		}
11257c478bd9Sstevel@tonic-gate 	}
11267c478bd9Sstevel@tonic-gate 
11277c478bd9Sstevel@tonic-gate 	umem_abort_info.ump_error = error;
11287c478bd9Sstevel@tonic-gate 	umem_abort_info.ump_buffer = bufarg;
11297c478bd9Sstevel@tonic-gate 	umem_abort_info.ump_realbuf = buf;
11307c478bd9Sstevel@tonic-gate 	umem_abort_info.ump_cache = cparg;
11317c478bd9Sstevel@tonic-gate 	umem_abort_info.ump_realcache = cp;
11327c478bd9Sstevel@tonic-gate 	umem_abort_info.ump_slab = sp;
11337c478bd9Sstevel@tonic-gate 	umem_abort_info.ump_bufctl = bcp;
11347c478bd9Sstevel@tonic-gate 
11357c478bd9Sstevel@tonic-gate 	umem_printf("umem allocator: ");
11367c478bd9Sstevel@tonic-gate 
11377c478bd9Sstevel@tonic-gate 	switch (error) {
11387c478bd9Sstevel@tonic-gate 
11397c478bd9Sstevel@tonic-gate 	case UMERR_MODIFIED:
11407c478bd9Sstevel@tonic-gate 		umem_printf("buffer modified after being freed\n");
11417c478bd9Sstevel@tonic-gate 		off = verify_pattern(UMEM_FREE_PATTERN, buf, cp->cache_verify);
11427c478bd9Sstevel@tonic-gate 		if (off == NULL)	/* shouldn't happen */
11437c478bd9Sstevel@tonic-gate 			off = buf;
11447c478bd9Sstevel@tonic-gate 		umem_printf("modification occurred at offset 0x%lx "
11457c478bd9Sstevel@tonic-gate 		    "(0x%llx replaced by 0x%llx)\n",
11467c478bd9Sstevel@tonic-gate 		    (uintptr_t)off - (uintptr_t)buf,
11477c478bd9Sstevel@tonic-gate 		    (longlong_t)UMEM_FREE_PATTERN, (longlong_t)*off);
11487c478bd9Sstevel@tonic-gate 		break;
11497c478bd9Sstevel@tonic-gate 
11507c478bd9Sstevel@tonic-gate 	case UMERR_REDZONE:
11517c478bd9Sstevel@tonic-gate 		umem_printf("redzone violation: write past end of buffer\n");
11527c478bd9Sstevel@tonic-gate 		break;
11537c478bd9Sstevel@tonic-gate 
11547c478bd9Sstevel@tonic-gate 	case UMERR_BADADDR:
11557c478bd9Sstevel@tonic-gate 		umem_printf("invalid free: buffer not in cache\n");
11567c478bd9Sstevel@tonic-gate 		break;
11577c478bd9Sstevel@tonic-gate 
11587c478bd9Sstevel@tonic-gate 	case UMERR_DUPFREE:
11597c478bd9Sstevel@tonic-gate 		umem_printf("duplicate free: buffer freed twice\n");
11607c478bd9Sstevel@tonic-gate 		break;
11617c478bd9Sstevel@tonic-gate 
11627c478bd9Sstevel@tonic-gate 	case UMERR_BADBUFTAG:
11637c478bd9Sstevel@tonic-gate 		umem_printf("boundary tag corrupted\n");
11647c478bd9Sstevel@tonic-gate 		umem_printf("bcp ^ bxstat = %lx, should be %lx\n",
11657c478bd9Sstevel@tonic-gate 		    (intptr_t)btp->bt_bufctl ^ btp->bt_bxstat,
11667c478bd9Sstevel@tonic-gate 		    UMEM_BUFTAG_FREE);
11677c478bd9Sstevel@tonic-gate 		break;
11687c478bd9Sstevel@tonic-gate 
11697c478bd9Sstevel@tonic-gate 	case UMERR_BADBUFCTL:
11707c478bd9Sstevel@tonic-gate 		umem_printf("bufctl corrupted\n");
11717c478bd9Sstevel@tonic-gate 		break;
11727c478bd9Sstevel@tonic-gate 
11737c478bd9Sstevel@tonic-gate 	case UMERR_BADCACHE:
11747c478bd9Sstevel@tonic-gate 		umem_printf("buffer freed to wrong cache\n");
11757c478bd9Sstevel@tonic-gate 		umem_printf("buffer was allocated from %s,\n", cp->cache_name);
11767c478bd9Sstevel@tonic-gate 		umem_printf("caller attempting free to %s.\n",
11777c478bd9Sstevel@tonic-gate 		    cparg->cache_name);
11787c478bd9Sstevel@tonic-gate 		break;
11797c478bd9Sstevel@tonic-gate 
11807c478bd9Sstevel@tonic-gate 	case UMERR_BADSIZE:
11817c478bd9Sstevel@tonic-gate 		umem_printf("bad free: free size (%u) != alloc size (%u)\n",
11827c478bd9Sstevel@tonic-gate 		    UMEM_SIZE_DECODE(((uint32_t *)btp)[0]),
11837c478bd9Sstevel@tonic-gate 		    UMEM_SIZE_DECODE(((uint32_t *)btp)[1]));
11847c478bd9Sstevel@tonic-gate 		break;
11857c478bd9Sstevel@tonic-gate 
11867c478bd9Sstevel@tonic-gate 	case UMERR_BADBASE:
11877c478bd9Sstevel@tonic-gate 		umem_printf("bad free: free address (%p) != alloc address "
11887c478bd9Sstevel@tonic-gate 		    "(%p)\n", bufarg, buf);
11897c478bd9Sstevel@tonic-gate 		break;
11907c478bd9Sstevel@tonic-gate 	}
11917c478bd9Sstevel@tonic-gate 
11927c478bd9Sstevel@tonic-gate 	umem_printf("buffer=%p  bufctl=%p  cache: %s\n",
11937c478bd9Sstevel@tonic-gate 	    bufarg, (void *)bcp, cparg->cache_name);
11947c478bd9Sstevel@tonic-gate 
11957c478bd9Sstevel@tonic-gate 	if (bcp != NULL && (cp->cache_flags & UMF_AUDIT) &&
11967c478bd9Sstevel@tonic-gate 	    error != UMERR_BADBUFCTL) {
11977c478bd9Sstevel@tonic-gate 		int d;
11987c478bd9Sstevel@tonic-gate 		timespec_t ts;
11997c478bd9Sstevel@tonic-gate 		hrtime_t diff;
12007c478bd9Sstevel@tonic-gate 		umem_bufctl_audit_t *bcap = (umem_bufctl_audit_t *)bcp;
12017c478bd9Sstevel@tonic-gate 
12027c478bd9Sstevel@tonic-gate 		diff = umem_abort_info.ump_timestamp - bcap->bc_timestamp;
12037c478bd9Sstevel@tonic-gate 		ts.tv_sec = diff / NANOSEC;
12047c478bd9Sstevel@tonic-gate 		ts.tv_nsec = diff % NANOSEC;
12057c478bd9Sstevel@tonic-gate 
12067c478bd9Sstevel@tonic-gate 		umem_printf("previous transaction on buffer %p:\n", buf);
12077c478bd9Sstevel@tonic-gate 		umem_printf("thread=%p  time=T-%ld.%09ld  slab=%p  cache: %s\n",
12087c478bd9Sstevel@tonic-gate 		    (void *)(intptr_t)bcap->bc_thread, ts.tv_sec, ts.tv_nsec,
12097c478bd9Sstevel@tonic-gate 		    (void *)sp, cp->cache_name);
12107c478bd9Sstevel@tonic-gate 		for (d = 0; d < MIN(bcap->bc_depth, umem_stack_depth); d++) {
12117c478bd9Sstevel@tonic-gate 			(void) print_sym((void *)bcap->bc_stack[d]);
12127c478bd9Sstevel@tonic-gate 			umem_printf("\n");
12137c478bd9Sstevel@tonic-gate 		}
12147c478bd9Sstevel@tonic-gate 	}
12157c478bd9Sstevel@tonic-gate 
12167c478bd9Sstevel@tonic-gate 	umem_err_recoverable("umem: heap corruption detected");
12177c478bd9Sstevel@tonic-gate 
12187c478bd9Sstevel@tonic-gate 	umem_logging = old_logging;	/* resume logging */
12197c478bd9Sstevel@tonic-gate }
12207c478bd9Sstevel@tonic-gate 
12217c478bd9Sstevel@tonic-gate void
umem_nofail_callback(umem_nofail_callback_t * cb)12227c478bd9Sstevel@tonic-gate umem_nofail_callback(umem_nofail_callback_t *cb)
12237c478bd9Sstevel@tonic-gate {
12247c478bd9Sstevel@tonic-gate 	nofail_callback = cb;
12257c478bd9Sstevel@tonic-gate }
12267c478bd9Sstevel@tonic-gate 
12277c478bd9Sstevel@tonic-gate static int
umem_alloc_retry(umem_cache_t * cp,int umflag)12287c478bd9Sstevel@tonic-gate umem_alloc_retry(umem_cache_t *cp, int umflag)
12297c478bd9Sstevel@tonic-gate {
12307c478bd9Sstevel@tonic-gate 	if (cp == &umem_null_cache) {
12317c478bd9Sstevel@tonic-gate 		if (umem_init())
12327c478bd9Sstevel@tonic-gate 			return (1);				/* retry */
12337c478bd9Sstevel@tonic-gate 		/*
12347c478bd9Sstevel@tonic-gate 		 * Initialization failed.  Do normal failure processing.
12357c478bd9Sstevel@tonic-gate 		 */
12367c478bd9Sstevel@tonic-gate 	}
1237831abf2cSDan Kimmel 	if (umem_flags & UMF_CHECKNULL) {
1238831abf2cSDan Kimmel 		umem_err_recoverable("umem: out of heap space");
1239831abf2cSDan Kimmel 	}
12407c478bd9Sstevel@tonic-gate 	if (umflag & UMEM_NOFAIL) {
12417c478bd9Sstevel@tonic-gate 		int def_result = UMEM_CALLBACK_EXIT(255);
12427c478bd9Sstevel@tonic-gate 		int result = def_result;
12437c478bd9Sstevel@tonic-gate 		umem_nofail_callback_t *callback = nofail_callback;
12447c478bd9Sstevel@tonic-gate 
12457c478bd9Sstevel@tonic-gate 		if (callback != NULL)
12467c478bd9Sstevel@tonic-gate 			result = callback();
12477c478bd9Sstevel@tonic-gate 
12487c478bd9Sstevel@tonic-gate 		if (result == UMEM_CALLBACK_RETRY)
12497c478bd9Sstevel@tonic-gate 			return (1);
12507c478bd9Sstevel@tonic-gate 
12517c478bd9Sstevel@tonic-gate 		if ((result & ~0xFF) != UMEM_CALLBACK_EXIT(0)) {
12527c478bd9Sstevel@tonic-gate 			log_message("nofail callback returned %x\n", result);
12537c478bd9Sstevel@tonic-gate 			result = def_result;
12547c478bd9Sstevel@tonic-gate 		}
12557c478bd9Sstevel@tonic-gate 
12567c478bd9Sstevel@tonic-gate 		/*
12577c478bd9Sstevel@tonic-gate 		 * only one thread will call exit
12587c478bd9Sstevel@tonic-gate 		 */
12597c478bd9Sstevel@tonic-gate 		if (umem_nofail_exit_thr == thr_self())
12607c478bd9Sstevel@tonic-gate 			umem_panic("recursive UMEM_CALLBACK_EXIT()\n");
12617c478bd9Sstevel@tonic-gate 
12627c478bd9Sstevel@tonic-gate 		(void) mutex_lock(&umem_nofail_exit_lock);
12637c478bd9Sstevel@tonic-gate 		umem_nofail_exit_thr = thr_self();
12647c478bd9Sstevel@tonic-gate 		exit(result & 0xFF);
12657c478bd9Sstevel@tonic-gate 		/*NOTREACHED*/
12667c478bd9Sstevel@tonic-gate 	}
12677c478bd9Sstevel@tonic-gate 	return (0);
12687c478bd9Sstevel@tonic-gate }
12697c478bd9Sstevel@tonic-gate 
12707c478bd9Sstevel@tonic-gate static umem_log_header_t *
umem_log_init(size_t logsize)12717c478bd9Sstevel@tonic-gate umem_log_init(size_t logsize)
12727c478bd9Sstevel@tonic-gate {
12737c478bd9Sstevel@tonic-gate 	umem_log_header_t *lhp;
12747c478bd9Sstevel@tonic-gate 	int nchunks = 4 * umem_max_ncpus;
12757c478bd9Sstevel@tonic-gate 	size_t lhsize = offsetof(umem_log_header_t, lh_cpu[umem_max_ncpus]);
12767c478bd9Sstevel@tonic-gate 	int i;
12777c478bd9Sstevel@tonic-gate 
12787c478bd9Sstevel@tonic-gate 	if (logsize == 0)
12797c478bd9Sstevel@tonic-gate 		return (NULL);
12807c478bd9Sstevel@tonic-gate 
12817c478bd9Sstevel@tonic-gate 	/*
12827c478bd9Sstevel@tonic-gate 	 * Make sure that lhp->lh_cpu[] is nicely aligned
12837c478bd9Sstevel@tonic-gate 	 * to prevent false sharing of cache lines.
12847c478bd9Sstevel@tonic-gate 	 */
12857c478bd9Sstevel@tonic-gate 	lhsize = P2ROUNDUP(lhsize, UMEM_ALIGN);
12867c478bd9Sstevel@tonic-gate 	lhp = vmem_xalloc(umem_log_arena, lhsize, 64, P2NPHASE(lhsize, 64), 0,
12877c478bd9Sstevel@tonic-gate 	    NULL, NULL, VM_NOSLEEP);
12887c478bd9Sstevel@tonic-gate 	if (lhp == NULL)
12897c478bd9Sstevel@tonic-gate 		goto fail;
12907c478bd9Sstevel@tonic-gate 
12917c478bd9Sstevel@tonic-gate 	bzero(lhp, lhsize);
12927c478bd9Sstevel@tonic-gate 
12937c478bd9Sstevel@tonic-gate 	(void) mutex_init(&lhp->lh_lock, USYNC_THREAD, NULL);
12947c478bd9Sstevel@tonic-gate 	lhp->lh_nchunks = nchunks;
12957c478bd9Sstevel@tonic-gate 	lhp->lh_chunksize = P2ROUNDUP(logsize / nchunks, PAGESIZE);
12967c478bd9Sstevel@tonic-gate 	if (lhp->lh_chunksize == 0)
12977c478bd9Sstevel@tonic-gate 		lhp->lh_chunksize = PAGESIZE;
12987c478bd9Sstevel@tonic-gate 
12997c478bd9Sstevel@tonic-gate 	lhp->lh_base = vmem_alloc(umem_log_arena,
13007c478bd9Sstevel@tonic-gate 	    lhp->lh_chunksize * nchunks, VM_NOSLEEP);
13017c478bd9Sstevel@tonic-gate 	if (lhp->lh_base == NULL)
13027c478bd9Sstevel@tonic-gate 		goto fail;
13037c478bd9Sstevel@tonic-gate 
13047c478bd9Sstevel@tonic-gate 	lhp->lh_free = vmem_alloc(umem_log_arena,
13057c478bd9Sstevel@tonic-gate 	    nchunks * sizeof (int), VM_NOSLEEP);
13067c478bd9Sstevel@tonic-gate 	if (lhp->lh_free == NULL)
13077c478bd9Sstevel@tonic-gate 		goto fail;
13087c478bd9Sstevel@tonic-gate 
13097c478bd9Sstevel@tonic-gate 	bzero(lhp->lh_base, lhp->lh_chunksize * nchunks);
13107c478bd9Sstevel@tonic-gate 
13117c478bd9Sstevel@tonic-gate 	for (i = 0; i < umem_max_ncpus; i++) {
13127c478bd9Sstevel@tonic-gate 		umem_cpu_log_header_t *clhp = &lhp->lh_cpu[i];
13137c478bd9Sstevel@tonic-gate 		(void) mutex_init(&clhp->clh_lock, USYNC_THREAD, NULL);
13147c478bd9Sstevel@tonic-gate 		clhp->clh_chunk = i;
13157c478bd9Sstevel@tonic-gate 	}
13167c478bd9Sstevel@tonic-gate 
13177c478bd9Sstevel@tonic-gate 	for (i = umem_max_ncpus; i < nchunks; i++)
13187c478bd9Sstevel@tonic-gate 		lhp->lh_free[i] = i;
13197c478bd9Sstevel@tonic-gate 
13207c478bd9Sstevel@tonic-gate 	lhp->lh_head = umem_max_ncpus;
13217c478bd9Sstevel@tonic-gate 	lhp->lh_tail = 0;
13227c478bd9Sstevel@tonic-gate 
13237c478bd9Sstevel@tonic-gate 	return (lhp);
13247c478bd9Sstevel@tonic-gate 
13257c478bd9Sstevel@tonic-gate fail:
13267c478bd9Sstevel@tonic-gate 	if (lhp != NULL) {
13277c478bd9Sstevel@tonic-gate 		if (lhp->lh_base != NULL)
13287c478bd9Sstevel@tonic-gate 			vmem_free(umem_log_arena, lhp->lh_base,
13297c478bd9Sstevel@tonic-gate 			    lhp->lh_chunksize * nchunks);
13307c478bd9Sstevel@tonic-gate 
13317c478bd9Sstevel@tonic-gate 		vmem_xfree(umem_log_arena, lhp, lhsize);
13327c478bd9Sstevel@tonic-gate 	}
13337c478bd9Sstevel@tonic-gate 	return (NULL);
13347c478bd9Sstevel@tonic-gate }
13357c478bd9Sstevel@tonic-gate 
13367c478bd9Sstevel@tonic-gate static void *
umem_log_enter(umem_log_header_t * lhp,void * data,size_t size)13377c478bd9Sstevel@tonic-gate umem_log_enter(umem_log_header_t *lhp, void *data, size_t size)
13387c478bd9Sstevel@tonic-gate {
13397c478bd9Sstevel@tonic-gate 	void *logspace;
1340b1e2e3fbSRobert Mustacchi 	umem_cpu_log_header_t *clhp;
13417c478bd9Sstevel@tonic-gate 
13427c478bd9Sstevel@tonic-gate 	if (lhp == NULL || umem_logging == 0)
13437c478bd9Sstevel@tonic-gate 		return (NULL);
13447c478bd9Sstevel@tonic-gate 
1345b1e2e3fbSRobert Mustacchi 	clhp = &lhp->lh_cpu[CPU(umem_cpu_mask)->cpu_number];
1346b1e2e3fbSRobert Mustacchi 
13477c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&clhp->clh_lock);
13487c478bd9Sstevel@tonic-gate 	clhp->clh_hits++;
13497c478bd9Sstevel@tonic-gate 	if (size > clhp->clh_avail) {
13507c478bd9Sstevel@tonic-gate 		(void) mutex_lock(&lhp->lh_lock);
13517c478bd9Sstevel@tonic-gate 		lhp->lh_hits++;
13527c478bd9Sstevel@tonic-gate 		lhp->lh_free[lhp->lh_tail] = clhp->clh_chunk;
13537c478bd9Sstevel@tonic-gate 		lhp->lh_tail = (lhp->lh_tail + 1) % lhp->lh_nchunks;
13547c478bd9Sstevel@tonic-gate 		clhp->clh_chunk = lhp->lh_free[lhp->lh_head];
13557c478bd9Sstevel@tonic-gate 		lhp->lh_head = (lhp->lh_head + 1) % lhp->lh_nchunks;
13567c478bd9Sstevel@tonic-gate 		clhp->clh_current = lhp->lh_base +
13577c478bd9Sstevel@tonic-gate 		    clhp->clh_chunk * lhp->lh_chunksize;
13587c478bd9Sstevel@tonic-gate 		clhp->clh_avail = lhp->lh_chunksize;
13597c478bd9Sstevel@tonic-gate 		if (size > lhp->lh_chunksize)
13607c478bd9Sstevel@tonic-gate 			size = lhp->lh_chunksize;
13617c478bd9Sstevel@tonic-gate 		(void) mutex_unlock(&lhp->lh_lock);
13627c478bd9Sstevel@tonic-gate 	}
13637c478bd9Sstevel@tonic-gate 	logspace = clhp->clh_current;
13647c478bd9Sstevel@tonic-gate 	clhp->clh_current += size;
13657c478bd9Sstevel@tonic-gate 	clhp->clh_avail -= size;
13667c478bd9Sstevel@tonic-gate 	bcopy(data, logspace, size);
13677c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&clhp->clh_lock);
13687c478bd9Sstevel@tonic-gate 	return (logspace);
13697c478bd9Sstevel@tonic-gate }
13707c478bd9Sstevel@tonic-gate 
13717c478bd9Sstevel@tonic-gate #define	UMEM_AUDIT(lp, cp, bcp)						\
13727c478bd9Sstevel@tonic-gate {									\
13737c478bd9Sstevel@tonic-gate 	umem_bufctl_audit_t *_bcp = (umem_bufctl_audit_t *)(bcp);	\
13747c478bd9Sstevel@tonic-gate 	_bcp->bc_timestamp = gethrtime();				\
13757c478bd9Sstevel@tonic-gate 	_bcp->bc_thread = thr_self();					\
13767c478bd9Sstevel@tonic-gate 	_bcp->bc_depth = getpcstack(_bcp->bc_stack, umem_stack_depth,	\
13777c478bd9Sstevel@tonic-gate 	    (cp != NULL) && (cp->cache_flags & UMF_CHECKSIGNAL));	\
13787c478bd9Sstevel@tonic-gate 	_bcp->bc_lastlog = umem_log_enter((lp), _bcp,			\
13797c478bd9Sstevel@tonic-gate 	    UMEM_BUFCTL_AUDIT_SIZE);					\
13807c478bd9Sstevel@tonic-gate }
13817c478bd9Sstevel@tonic-gate 
13827c478bd9Sstevel@tonic-gate static void
umem_log_event(umem_log_header_t * lp,umem_cache_t * cp,umem_slab_t * sp,void * addr)13837c478bd9Sstevel@tonic-gate umem_log_event(umem_log_header_t *lp, umem_cache_t *cp,
1384831abf2cSDan Kimmel     umem_slab_t *sp, void *addr)
13857c478bd9Sstevel@tonic-gate {
13867c478bd9Sstevel@tonic-gate 	umem_bufctl_audit_t *bcp;
13877c478bd9Sstevel@tonic-gate 	UMEM_LOCAL_BUFCTL_AUDIT(&bcp);
13887c478bd9Sstevel@tonic-gate 
13897c478bd9Sstevel@tonic-gate 	bzero(bcp, UMEM_BUFCTL_AUDIT_SIZE);
13907c478bd9Sstevel@tonic-gate 	bcp->bc_addr = addr;
13917c478bd9Sstevel@tonic-gate 	bcp->bc_slab = sp;
13927c478bd9Sstevel@tonic-gate 	bcp->bc_cache = cp;
13937c478bd9Sstevel@tonic-gate 	UMEM_AUDIT(lp, cp, bcp);
13947c478bd9Sstevel@tonic-gate }
13957c478bd9Sstevel@tonic-gate 
13967c478bd9Sstevel@tonic-gate /*
13977c478bd9Sstevel@tonic-gate  * Create a new slab for cache cp.
13987c478bd9Sstevel@tonic-gate  */
13997c478bd9Sstevel@tonic-gate static umem_slab_t *
umem_slab_create(umem_cache_t * cp,int umflag)14007c478bd9Sstevel@tonic-gate umem_slab_create(umem_cache_t *cp, int umflag)
14017c478bd9Sstevel@tonic-gate {
14027c478bd9Sstevel@tonic-gate 	size_t slabsize = cp->cache_slabsize;
14037c478bd9Sstevel@tonic-gate 	size_t chunksize = cp->cache_chunksize;
14047c478bd9Sstevel@tonic-gate 	int cache_flags = cp->cache_flags;
14057c478bd9Sstevel@tonic-gate 	size_t color, chunks;
14067c478bd9Sstevel@tonic-gate 	char *buf, *slab;
14077c478bd9Sstevel@tonic-gate 	umem_slab_t *sp;
14087c478bd9Sstevel@tonic-gate 	umem_bufctl_t *bcp;
14097c478bd9Sstevel@tonic-gate 	vmem_t *vmp = cp->cache_arena;
14107c478bd9Sstevel@tonic-gate 
14117c478bd9Sstevel@tonic-gate 	color = cp->cache_color + cp->cache_align;
14127c478bd9Sstevel@tonic-gate 	if (color > cp->cache_maxcolor)
14137c478bd9Sstevel@tonic-gate 		color = cp->cache_mincolor;
14147c478bd9Sstevel@tonic-gate 	cp->cache_color = color;
14157c478bd9Sstevel@tonic-gate 
14167c478bd9Sstevel@tonic-gate 	slab = vmem_alloc(vmp, slabsize, UMEM_VMFLAGS(umflag));
14177c478bd9Sstevel@tonic-gate 
14187c478bd9Sstevel@tonic-gate 	if (slab == NULL)
14197c478bd9Sstevel@tonic-gate 		goto vmem_alloc_failure;
14207c478bd9Sstevel@tonic-gate 
14217c478bd9Sstevel@tonic-gate 	ASSERT(P2PHASE((uintptr_t)slab, vmp->vm_quantum) == 0);
14227c478bd9Sstevel@tonic-gate 
14237c478bd9Sstevel@tonic-gate 	if (!(cp->cache_cflags & UMC_NOTOUCH) &&
14247c478bd9Sstevel@tonic-gate 	    (cp->cache_flags & UMF_DEADBEEF))
14257c478bd9Sstevel@tonic-gate 		copy_pattern(UMEM_UNINITIALIZED_PATTERN, slab, slabsize);
14267c478bd9Sstevel@tonic-gate 
14277c478bd9Sstevel@tonic-gate 	if (cache_flags & UMF_HASH) {
14287c478bd9Sstevel@tonic-gate 		if ((sp = _umem_cache_alloc(umem_slab_cache, umflag)) == NULL)
14297c478bd9Sstevel@tonic-gate 			goto slab_alloc_failure;
14307c478bd9Sstevel@tonic-gate 		chunks = (slabsize - color) / chunksize;
14317c478bd9Sstevel@tonic-gate 	} else {
14327c478bd9Sstevel@tonic-gate 		sp = UMEM_SLAB(cp, slab);
14337c478bd9Sstevel@tonic-gate 		chunks = (slabsize - sizeof (umem_slab_t) - color) / chunksize;
14347c478bd9Sstevel@tonic-gate 	}
14357c478bd9Sstevel@tonic-gate 
14367c478bd9Sstevel@tonic-gate 	sp->slab_cache	= cp;
14377c478bd9Sstevel@tonic-gate 	sp->slab_head	= NULL;
14387c478bd9Sstevel@tonic-gate 	sp->slab_refcnt	= 0;
14397c478bd9Sstevel@tonic-gate 	sp->slab_base	= buf = slab + color;
14407c478bd9Sstevel@tonic-gate 	sp->slab_chunks	= chunks;
14417c478bd9Sstevel@tonic-gate 
14427c478bd9Sstevel@tonic-gate 	ASSERT(chunks > 0);
14437c478bd9Sstevel@tonic-gate 	while (chunks-- != 0) {
14447c478bd9Sstevel@tonic-gate 		if (cache_flags & UMF_HASH) {
14457c478bd9Sstevel@tonic-gate 			bcp = _umem_cache_alloc(cp->cache_bufctl_cache, umflag);
14467c478bd9Sstevel@tonic-gate 			if (bcp == NULL)
14477c478bd9Sstevel@tonic-gate 				goto bufctl_alloc_failure;
14487c478bd9Sstevel@tonic-gate 			if (cache_flags & UMF_AUDIT) {
14497c478bd9Sstevel@tonic-gate 				umem_bufctl_audit_t *bcap =
14507c478bd9Sstevel@tonic-gate 				    (umem_bufctl_audit_t *)bcp;
14517c478bd9Sstevel@tonic-gate 				bzero(bcap, UMEM_BUFCTL_AUDIT_SIZE);
14527c478bd9Sstevel@tonic-gate 				bcap->bc_cache = cp;
14537c478bd9Sstevel@tonic-gate 			}
14547c478bd9Sstevel@tonic-gate 			bcp->bc_addr = buf;
14557c478bd9Sstevel@tonic-gate 			bcp->bc_slab = sp;
14567c478bd9Sstevel@tonic-gate 		} else {
14577c478bd9Sstevel@tonic-gate 			bcp = UMEM_BUFCTL(cp, buf);
14587c478bd9Sstevel@tonic-gate 		}
14597c478bd9Sstevel@tonic-gate 		if (cache_flags & UMF_BUFTAG) {
14607c478bd9Sstevel@tonic-gate 			umem_buftag_t *btp = UMEM_BUFTAG(cp, buf);
14617c478bd9Sstevel@tonic-gate 			btp->bt_redzone = UMEM_REDZONE_PATTERN;
14627c478bd9Sstevel@tonic-gate 			btp->bt_bufctl = bcp;
14637c478bd9Sstevel@tonic-gate 			btp->bt_bxstat = (intptr_t)bcp ^ UMEM_BUFTAG_FREE;
14647c478bd9Sstevel@tonic-gate 			if (cache_flags & UMF_DEADBEEF) {
14657c478bd9Sstevel@tonic-gate 				copy_pattern(UMEM_FREE_PATTERN, buf,
14667c478bd9Sstevel@tonic-gate 				    cp->cache_verify);
14677c478bd9Sstevel@tonic-gate 			}
14687c478bd9Sstevel@tonic-gate 		}
14697c478bd9Sstevel@tonic-gate 		bcp->bc_next = sp->slab_head;
14707c478bd9Sstevel@tonic-gate 		sp->slab_head = bcp;
14717c478bd9Sstevel@tonic-gate 		buf += chunksize;
14727c478bd9Sstevel@tonic-gate 	}
14737c478bd9Sstevel@tonic-gate 
14747c478bd9Sstevel@tonic-gate 	umem_log_event(umem_slab_log, cp, sp, slab);
14757c478bd9Sstevel@tonic-gate 
14767c478bd9Sstevel@tonic-gate 	return (sp);
14777c478bd9Sstevel@tonic-gate 
14787c478bd9Sstevel@tonic-gate bufctl_alloc_failure:
14797c478bd9Sstevel@tonic-gate 
14807c478bd9Sstevel@tonic-gate 	while ((bcp = sp->slab_head) != NULL) {
14817c478bd9Sstevel@tonic-gate 		sp->slab_head = bcp->bc_next;
14827c478bd9Sstevel@tonic-gate 		_umem_cache_free(cp->cache_bufctl_cache, bcp);
14837c478bd9Sstevel@tonic-gate 	}
14847c478bd9Sstevel@tonic-gate 	_umem_cache_free(umem_slab_cache, sp);
14857c478bd9Sstevel@tonic-gate 
14867c478bd9Sstevel@tonic-gate slab_alloc_failure:
14877c478bd9Sstevel@tonic-gate 
14887c478bd9Sstevel@tonic-gate 	vmem_free(vmp, slab, slabsize);
14897c478bd9Sstevel@tonic-gate 
14907c478bd9Sstevel@tonic-gate vmem_alloc_failure:
14917c478bd9Sstevel@tonic-gate 
14927c478bd9Sstevel@tonic-gate 	umem_log_event(umem_failure_log, cp, NULL, NULL);
14937c478bd9Sstevel@tonic-gate 	atomic_add_64(&cp->cache_alloc_fail, 1);
14947c478bd9Sstevel@tonic-gate 
14957c478bd9Sstevel@tonic-gate 	return (NULL);
14967c478bd9Sstevel@tonic-gate }
14977c478bd9Sstevel@tonic-gate 
14987c478bd9Sstevel@tonic-gate /*
14997c478bd9Sstevel@tonic-gate  * Destroy a slab.
15007c478bd9Sstevel@tonic-gate  */
15017c478bd9Sstevel@tonic-gate static void
umem_slab_destroy(umem_cache_t * cp,umem_slab_t * sp)15027c478bd9Sstevel@tonic-gate umem_slab_destroy(umem_cache_t *cp, umem_slab_t *sp)
15037c478bd9Sstevel@tonic-gate {
15047c478bd9Sstevel@tonic-gate 	vmem_t *vmp = cp->cache_arena;
15057c478bd9Sstevel@tonic-gate 	void *slab = (void *)P2ALIGN((uintptr_t)sp->slab_base, vmp->vm_quantum);
15067c478bd9Sstevel@tonic-gate 
15077c478bd9Sstevel@tonic-gate 	if (cp->cache_flags & UMF_HASH) {
15087c478bd9Sstevel@tonic-gate 		umem_bufctl_t *bcp;
15097c478bd9Sstevel@tonic-gate 		while ((bcp = sp->slab_head) != NULL) {
15107c478bd9Sstevel@tonic-gate 			sp->slab_head = bcp->bc_next;
15117c478bd9Sstevel@tonic-gate 			_umem_cache_free(cp->cache_bufctl_cache, bcp);
15127c478bd9Sstevel@tonic-gate 		}
15137c478bd9Sstevel@tonic-gate 		_umem_cache_free(umem_slab_cache, sp);
15147c478bd9Sstevel@tonic-gate 	}
15157c478bd9Sstevel@tonic-gate 	vmem_free(vmp, slab, cp->cache_slabsize);
15167c478bd9Sstevel@tonic-gate }
15177c478bd9Sstevel@tonic-gate 
15187c478bd9Sstevel@tonic-gate /*
15197c478bd9Sstevel@tonic-gate  * Allocate a raw (unconstructed) buffer from cp's slab layer.
15207c478bd9Sstevel@tonic-gate  */
15217c478bd9Sstevel@tonic-gate static void *
umem_slab_alloc(umem_cache_t * cp,int umflag)15227c478bd9Sstevel@tonic-gate umem_slab_alloc(umem_cache_t *cp, int umflag)
15237c478bd9Sstevel@tonic-gate {
15247c478bd9Sstevel@tonic-gate 	umem_bufctl_t *bcp, **hash_bucket;
15257c478bd9Sstevel@tonic-gate 	umem_slab_t *sp;
15267c478bd9Sstevel@tonic-gate 	void *buf;
15277c478bd9Sstevel@tonic-gate 
15287c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&cp->cache_lock);
15297c478bd9Sstevel@tonic-gate 	cp->cache_slab_alloc++;
15307c478bd9Sstevel@tonic-gate 	sp = cp->cache_freelist;
15317c478bd9Sstevel@tonic-gate 	ASSERT(sp->slab_cache == cp);
15327c478bd9Sstevel@tonic-gate 	if (sp->slab_head == NULL) {
15337c478bd9Sstevel@tonic-gate 		/*
15347c478bd9Sstevel@tonic-gate 		 * The freelist is empty.  Create a new slab.
15357c478bd9Sstevel@tonic-gate 		 */
15367c478bd9Sstevel@tonic-gate 		(void) mutex_unlock(&cp->cache_lock);
15377c478bd9Sstevel@tonic-gate 		if (cp == &umem_null_cache)
15387c478bd9Sstevel@tonic-gate 			return (NULL);
15397c478bd9Sstevel@tonic-gate 		if ((sp = umem_slab_create(cp, umflag)) == NULL)
15407c478bd9Sstevel@tonic-gate 			return (NULL);
15417c478bd9Sstevel@tonic-gate 		(void) mutex_lock(&cp->cache_lock);
15427c478bd9Sstevel@tonic-gate 		cp->cache_slab_create++;
15437c478bd9Sstevel@tonic-gate 		if ((cp->cache_buftotal += sp->slab_chunks) > cp->cache_bufmax)
15447c478bd9Sstevel@tonic-gate 			cp->cache_bufmax = cp->cache_buftotal;
15457c478bd9Sstevel@tonic-gate 		sp->slab_next = cp->cache_freelist;
15467c478bd9Sstevel@tonic-gate 		sp->slab_prev = cp->cache_freelist->slab_prev;
15477c478bd9Sstevel@tonic-gate 		sp->slab_next->slab_prev = sp;
15487c478bd9Sstevel@tonic-gate 		sp->slab_prev->slab_next = sp;
15497c478bd9Sstevel@tonic-gate 		cp->cache_freelist = sp;
15507c478bd9Sstevel@tonic-gate 	}
15517c478bd9Sstevel@tonic-gate 
15527c478bd9Sstevel@tonic-gate 	sp->slab_refcnt++;
15537c478bd9Sstevel@tonic-gate 	ASSERT(sp->slab_refcnt <= sp->slab_chunks);
15547c478bd9Sstevel@tonic-gate 
15557c478bd9Sstevel@tonic-gate 	/*
15567c478bd9Sstevel@tonic-gate 	 * If we're taking the last buffer in the slab,
15577c478bd9Sstevel@tonic-gate 	 * remove the slab from the cache's freelist.
15587c478bd9Sstevel@tonic-gate 	 */
15597c478bd9Sstevel@tonic-gate 	bcp = sp->slab_head;
15607c478bd9Sstevel@tonic-gate 	if ((sp->slab_head = bcp->bc_next) == NULL) {
15617c478bd9Sstevel@tonic-gate 		cp->cache_freelist = sp->slab_next;
15627c478bd9Sstevel@tonic-gate 		ASSERT(sp->slab_refcnt == sp->slab_chunks);
15637c478bd9Sstevel@tonic-gate 	}
15647c478bd9Sstevel@tonic-gate 
15657c478bd9Sstevel@tonic-gate 	if (cp->cache_flags & UMF_HASH) {
15667c478bd9Sstevel@tonic-gate 		/*
15677c478bd9Sstevel@tonic-gate 		 * Add buffer to allocated-address hash table.
15687c478bd9Sstevel@tonic-gate 		 */
15697c478bd9Sstevel@tonic-gate 		buf = bcp->bc_addr;
15707c478bd9Sstevel@tonic-gate 		hash_bucket = UMEM_HASH(cp, buf);
15717c478bd9Sstevel@tonic-gate 		bcp->bc_next = *hash_bucket;
15727c478bd9Sstevel@tonic-gate 		*hash_bucket = bcp;
15737c478bd9Sstevel@tonic-gate 		if ((cp->cache_flags & (UMF_AUDIT | UMF_BUFTAG)) == UMF_AUDIT) {
15747c478bd9Sstevel@tonic-gate 			UMEM_AUDIT(umem_transaction_log, cp, bcp);
15757c478bd9Sstevel@tonic-gate 		}
15767c478bd9Sstevel@tonic-gate 	} else {
15777c478bd9Sstevel@tonic-gate 		buf = UMEM_BUF(cp, bcp);
15787c478bd9Sstevel@tonic-gate 	}
15797c478bd9Sstevel@tonic-gate 
15807c478bd9Sstevel@tonic-gate 	ASSERT(UMEM_SLAB_MEMBER(sp, buf));
15817c478bd9Sstevel@tonic-gate 
15827c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&cp->cache_lock);
15837c478bd9Sstevel@tonic-gate 
15847c478bd9Sstevel@tonic-gate 	return (buf);
15857c478bd9Sstevel@tonic-gate }
15867c478bd9Sstevel@tonic-gate 
15877c478bd9Sstevel@tonic-gate /*
15887c478bd9Sstevel@tonic-gate  * Free a raw (unconstructed) buffer to cp's slab layer.
15897c478bd9Sstevel@tonic-gate  */
15907c478bd9Sstevel@tonic-gate static void
umem_slab_free(umem_cache_t * cp,void * buf)15917c478bd9Sstevel@tonic-gate umem_slab_free(umem_cache_t *cp, void *buf)
15927c478bd9Sstevel@tonic-gate {
15937c478bd9Sstevel@tonic-gate 	umem_slab_t *sp;
15947c478bd9Sstevel@tonic-gate 	umem_bufctl_t *bcp, **prev_bcpp;
15957c478bd9Sstevel@tonic-gate 
15967c478bd9Sstevel@tonic-gate 	ASSERT(buf != NULL);
15977c478bd9Sstevel@tonic-gate 
15987c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&cp->cache_lock);
15997c478bd9Sstevel@tonic-gate 	cp->cache_slab_free++;
16007c478bd9Sstevel@tonic-gate 
16017c478bd9Sstevel@tonic-gate 	if (cp->cache_flags & UMF_HASH) {
16027c478bd9Sstevel@tonic-gate 		/*
16037c478bd9Sstevel@tonic-gate 		 * Look up buffer in allocated-address hash table.
16047c478bd9Sstevel@tonic-gate 		 */
16057c478bd9Sstevel@tonic-gate 		prev_bcpp = UMEM_HASH(cp, buf);
16067c478bd9Sstevel@tonic-gate 		while ((bcp = *prev_bcpp) != NULL) {
16077c478bd9Sstevel@tonic-gate 			if (bcp->bc_addr == buf) {
16087c478bd9Sstevel@tonic-gate 				*prev_bcpp = bcp->bc_next;
16097c478bd9Sstevel@tonic-gate 				sp = bcp->bc_slab;
16107c478bd9Sstevel@tonic-gate 				break;
16117c478bd9Sstevel@tonic-gate 			}
16127c478bd9Sstevel@tonic-gate 			cp->cache_lookup_depth++;
16137c478bd9Sstevel@tonic-gate 			prev_bcpp = &bcp->bc_next;
16147c478bd9Sstevel@tonic-gate 		}
16157c478bd9Sstevel@tonic-gate 	} else {
16167c478bd9Sstevel@tonic-gate 		bcp = UMEM_BUFCTL(cp, buf);
16177c478bd9Sstevel@tonic-gate 		sp = UMEM_SLAB(cp, buf);
16187c478bd9Sstevel@tonic-gate 	}
16197c478bd9Sstevel@tonic-gate 
16207c478bd9Sstevel@tonic-gate 	if (bcp == NULL || sp->slab_cache != cp || !UMEM_SLAB_MEMBER(sp, buf)) {
16217c478bd9Sstevel@tonic-gate 		(void) mutex_unlock(&cp->cache_lock);
16227c478bd9Sstevel@tonic-gate 		umem_error(UMERR_BADADDR, cp, buf);
16237c478bd9Sstevel@tonic-gate 		return;
16247c478bd9Sstevel@tonic-gate 	}
16257c478bd9Sstevel@tonic-gate 
16267c478bd9Sstevel@tonic-gate 	if ((cp->cache_flags & (UMF_AUDIT | UMF_BUFTAG)) == UMF_AUDIT) {
16277c478bd9Sstevel@tonic-gate 		if (cp->cache_flags & UMF_CONTENTS)
16287c478bd9Sstevel@tonic-gate 			((umem_bufctl_audit_t *)bcp)->bc_contents =
16297c478bd9Sstevel@tonic-gate 			    umem_log_enter(umem_content_log, buf,
16307c478bd9Sstevel@tonic-gate 			    cp->cache_contents);
16317c478bd9Sstevel@tonic-gate 		UMEM_AUDIT(umem_transaction_log, cp, bcp);
16327c478bd9Sstevel@tonic-gate 	}
16337c478bd9Sstevel@tonic-gate 
16347c478bd9Sstevel@tonic-gate 	/*
16357c478bd9Sstevel@tonic-gate 	 * If this slab isn't currently on the freelist, put it there.
16367c478bd9Sstevel@tonic-gate 	 */
16377c478bd9Sstevel@tonic-gate 	if (sp->slab_head == NULL) {
16387c478bd9Sstevel@tonic-gate 		ASSERT(sp->slab_refcnt == sp->slab_chunks);
16397c478bd9Sstevel@tonic-gate 		ASSERT(cp->cache_freelist != sp);
16407c478bd9Sstevel@tonic-gate 		sp->slab_next->slab_prev = sp->slab_prev;
16417c478bd9Sstevel@tonic-gate 		sp->slab_prev->slab_next = sp->slab_next;
16427c478bd9Sstevel@tonic-gate 		sp->slab_next = cp->cache_freelist;
16437c478bd9Sstevel@tonic-gate 		sp->slab_prev = cp->cache_freelist->slab_prev;
16447c478bd9Sstevel@tonic-gate 		sp->slab_next->slab_prev = sp;
16457c478bd9Sstevel@tonic-gate 		sp->slab_prev->slab_next = sp;
16467c478bd9Sstevel@tonic-gate 		cp->cache_freelist = sp;
16477c478bd9Sstevel@tonic-gate 	}
16487c478bd9Sstevel@tonic-gate 
16497c478bd9Sstevel@tonic-gate 	bcp->bc_next = sp->slab_head;
16507c478bd9Sstevel@tonic-gate 	sp->slab_head = bcp;
16517c478bd9Sstevel@tonic-gate 
16527c478bd9Sstevel@tonic-gate 	ASSERT(sp->slab_refcnt >= 1);
16537c478bd9Sstevel@tonic-gate 	if (--sp->slab_refcnt == 0) {
16547c478bd9Sstevel@tonic-gate 		/*
16557c478bd9Sstevel@tonic-gate 		 * There are no outstanding allocations from this slab,
16567c478bd9Sstevel@tonic-gate 		 * so we can reclaim the memory.
16577c478bd9Sstevel@tonic-gate 		 */
16587c478bd9Sstevel@tonic-gate 		sp->slab_next->slab_prev = sp->slab_prev;
16597c478bd9Sstevel@tonic-gate 		sp->slab_prev->slab_next = sp->slab_next;
16607c478bd9Sstevel@tonic-gate 		if (sp == cp->cache_freelist)
16617c478bd9Sstevel@tonic-gate 			cp->cache_freelist = sp->slab_next;
16627c478bd9Sstevel@tonic-gate 		cp->cache_slab_destroy++;
16637c478bd9Sstevel@tonic-gate 		cp->cache_buftotal -= sp->slab_chunks;
16647c478bd9Sstevel@tonic-gate 		(void) mutex_unlock(&cp->cache_lock);
16657c478bd9Sstevel@tonic-gate 		umem_slab_destroy(cp, sp);
16667c478bd9Sstevel@tonic-gate 		return;
16677c478bd9Sstevel@tonic-gate 	}
16687c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&cp->cache_lock);
16697c478bd9Sstevel@tonic-gate }
16707c478bd9Sstevel@tonic-gate 
16717c478bd9Sstevel@tonic-gate static int
umem_cache_alloc_debug(umem_cache_t * cp,void * buf,int umflag)16727c478bd9Sstevel@tonic-gate umem_cache_alloc_debug(umem_cache_t *cp, void *buf, int umflag)
16737c478bd9Sstevel@tonic-gate {
16747c478bd9Sstevel@tonic-gate 	umem_buftag_t *btp = UMEM_BUFTAG(cp, buf);
16757c478bd9Sstevel@tonic-gate 	umem_bufctl_audit_t *bcp = (umem_bufctl_audit_t *)btp->bt_bufctl;
16767c478bd9Sstevel@tonic-gate 	uint32_t mtbf;
16777c478bd9Sstevel@tonic-gate 	int flags_nfatal;
16787c478bd9Sstevel@tonic-gate 
16797c478bd9Sstevel@tonic-gate 	if (btp->bt_bxstat != ((intptr_t)bcp ^ UMEM_BUFTAG_FREE)) {
16807c478bd9Sstevel@tonic-gate 		umem_error(UMERR_BADBUFTAG, cp, buf);
16817c478bd9Sstevel@tonic-gate 		return (-1);
16827c478bd9Sstevel@tonic-gate 	}
16837c478bd9Sstevel@tonic-gate 
16847c478bd9Sstevel@tonic-gate 	btp->bt_bxstat = (intptr_t)bcp ^ UMEM_BUFTAG_ALLOC;
16857c478bd9Sstevel@tonic-gate 
16867c478bd9Sstevel@tonic-gate 	if ((cp->cache_flags & UMF_HASH) && bcp->bc_addr != buf) {
16877c478bd9Sstevel@tonic-gate 		umem_error(UMERR_BADBUFCTL, cp, buf);
16887c478bd9Sstevel@tonic-gate 		return (-1);
16897c478bd9Sstevel@tonic-gate 	}
16907c478bd9Sstevel@tonic-gate 
16917c478bd9Sstevel@tonic-gate 	btp->bt_redzone = UMEM_REDZONE_PATTERN;
16927c478bd9Sstevel@tonic-gate 
16937c478bd9Sstevel@tonic-gate 	if (cp->cache_flags & UMF_DEADBEEF) {
16947c478bd9Sstevel@tonic-gate 		if (verify_and_copy_pattern(UMEM_FREE_PATTERN,
16957c478bd9Sstevel@tonic-gate 		    UMEM_UNINITIALIZED_PATTERN, buf, cp->cache_verify)) {
16967c478bd9Sstevel@tonic-gate 			umem_error(UMERR_MODIFIED, cp, buf);
16977c478bd9Sstevel@tonic-gate 			return (-1);
16987c478bd9Sstevel@tonic-gate 		}
16997c478bd9Sstevel@tonic-gate 	}
17007c478bd9Sstevel@tonic-gate 
17017c478bd9Sstevel@tonic-gate 	if ((mtbf = umem_mtbf | cp->cache_mtbf) != 0 &&
17027c478bd9Sstevel@tonic-gate 	    gethrtime() % mtbf == 0 &&
17037c478bd9Sstevel@tonic-gate 	    (umflag & (UMEM_FATAL_FLAGS)) == 0) {
17047c478bd9Sstevel@tonic-gate 		umem_log_event(umem_failure_log, cp, NULL, NULL);
17057c478bd9Sstevel@tonic-gate 	} else {
17067c478bd9Sstevel@tonic-gate 		mtbf = 0;
17077c478bd9Sstevel@tonic-gate 	}
17087c478bd9Sstevel@tonic-gate 
17097c478bd9Sstevel@tonic-gate 	/*
17107c478bd9Sstevel@tonic-gate 	 * We do not pass fatal flags on to the constructor.  This prevents
17117c478bd9Sstevel@tonic-gate 	 * leaking buffers in the event of a subordinate constructor failing.
17127c478bd9Sstevel@tonic-gate 	 */
17137c478bd9Sstevel@tonic-gate 	flags_nfatal = UMEM_DEFAULT;
17147c478bd9Sstevel@tonic-gate 	if (mtbf || (cp->cache_constructor != NULL &&
17157c478bd9Sstevel@tonic-gate 	    cp->cache_constructor(buf, cp->cache_private, flags_nfatal) != 0)) {
17167c478bd9Sstevel@tonic-gate 		atomic_add_64(&cp->cache_alloc_fail, 1);
17177c478bd9Sstevel@tonic-gate 		btp->bt_bxstat = (intptr_t)bcp ^ UMEM_BUFTAG_FREE;
17187c478bd9Sstevel@tonic-gate 		copy_pattern(UMEM_FREE_PATTERN, buf, cp->cache_verify);
17197c478bd9Sstevel@tonic-gate 		umem_slab_free(cp, buf);
17207c478bd9Sstevel@tonic-gate 		return (-1);
17217c478bd9Sstevel@tonic-gate 	}
17227c478bd9Sstevel@tonic-gate 
17237c478bd9Sstevel@tonic-gate 	if (cp->cache_flags & UMF_AUDIT) {
17247c478bd9Sstevel@tonic-gate 		UMEM_AUDIT(umem_transaction_log, cp, bcp);
17257c478bd9Sstevel@tonic-gate 	}
17267c478bd9Sstevel@tonic-gate 
17277c478bd9Sstevel@tonic-gate 	return (0);
17287c478bd9Sstevel@tonic-gate }
17297c478bd9Sstevel@tonic-gate 
17307c478bd9Sstevel@tonic-gate static int
umem_cache_free_debug(umem_cache_t * cp,void * buf)17317c478bd9Sstevel@tonic-gate umem_cache_free_debug(umem_cache_t *cp, void *buf)
17327c478bd9Sstevel@tonic-gate {
17337c478bd9Sstevel@tonic-gate 	umem_buftag_t *btp = UMEM_BUFTAG(cp, buf);
17347c478bd9Sstevel@tonic-gate 	umem_bufctl_audit_t *bcp = (umem_bufctl_audit_t *)btp->bt_bufctl;
17357c478bd9Sstevel@tonic-gate 	umem_slab_t *sp;
17367c478bd9Sstevel@tonic-gate 
17377c478bd9Sstevel@tonic-gate 	if (btp->bt_bxstat != ((intptr_t)bcp ^ UMEM_BUFTAG_ALLOC)) {
17387c478bd9Sstevel@tonic-gate 		if (btp->bt_bxstat == ((intptr_t)bcp ^ UMEM_BUFTAG_FREE)) {
17397c478bd9Sstevel@tonic-gate 			umem_error(UMERR_DUPFREE, cp, buf);
17407c478bd9Sstevel@tonic-gate 			return (-1);
17417c478bd9Sstevel@tonic-gate 		}
17427c478bd9Sstevel@tonic-gate 		sp = umem_findslab(cp, buf);
17437c478bd9Sstevel@tonic-gate 		if (sp == NULL || sp->slab_cache != cp)
17447c478bd9Sstevel@tonic-gate 			umem_error(UMERR_BADADDR, cp, buf);
17457c478bd9Sstevel@tonic-gate 		else
17467c478bd9Sstevel@tonic-gate 			umem_error(UMERR_REDZONE, cp, buf);
17477c478bd9Sstevel@tonic-gate 		return (-1);
17487c478bd9Sstevel@tonic-gate 	}
17497c478bd9Sstevel@tonic-gate 
17507c478bd9Sstevel@tonic-gate 	btp->bt_bxstat = (intptr_t)bcp ^ UMEM_BUFTAG_FREE;
17517c478bd9Sstevel@tonic-gate 
17527c478bd9Sstevel@tonic-gate 	if ((cp->cache_flags & UMF_HASH) && bcp->bc_addr != buf) {
17537c478bd9Sstevel@tonic-gate 		umem_error(UMERR_BADBUFCTL, cp, buf);
17547c478bd9Sstevel@tonic-gate 		return (-1);
17557c478bd9Sstevel@tonic-gate 	}
17567c478bd9Sstevel@tonic-gate 
17577c478bd9Sstevel@tonic-gate 	if (btp->bt_redzone != UMEM_REDZONE_PATTERN) {
17587c478bd9Sstevel@tonic-gate 		umem_error(UMERR_REDZONE, cp, buf);
17597c478bd9Sstevel@tonic-gate 		return (-1);
17607c478bd9Sstevel@tonic-gate 	}
17617c478bd9Sstevel@tonic-gate 
17627c478bd9Sstevel@tonic-gate 	if (cp->cache_flags & UMF_AUDIT) {
17637c478bd9Sstevel@tonic-gate 		if (cp->cache_flags & UMF_CONTENTS)
17647c478bd9Sstevel@tonic-gate 			bcp->bc_contents = umem_log_enter(umem_content_log,
17657c478bd9Sstevel@tonic-gate 			    buf, cp->cache_contents);
17667c478bd9Sstevel@tonic-gate 		UMEM_AUDIT(umem_transaction_log, cp, bcp);
17677c478bd9Sstevel@tonic-gate 	}
17687c478bd9Sstevel@tonic-gate 
17697c478bd9Sstevel@tonic-gate 	if (cp->cache_destructor != NULL)
17707c478bd9Sstevel@tonic-gate 		cp->cache_destructor(buf, cp->cache_private);
17717c478bd9Sstevel@tonic-gate 
17727c478bd9Sstevel@tonic-gate 	if (cp->cache_flags & UMF_DEADBEEF)
17737c478bd9Sstevel@tonic-gate 		copy_pattern(UMEM_FREE_PATTERN, buf, cp->cache_verify);
17747c478bd9Sstevel@tonic-gate 
17757c478bd9Sstevel@tonic-gate 	return (0);
17767c478bd9Sstevel@tonic-gate }
17777c478bd9Sstevel@tonic-gate 
17787c478bd9Sstevel@tonic-gate /*
17797c478bd9Sstevel@tonic-gate  * Free each object in magazine mp to cp's slab layer, and free mp itself.
17807c478bd9Sstevel@tonic-gate  */
17817c478bd9Sstevel@tonic-gate static void
umem_magazine_destroy(umem_cache_t * cp,umem_magazine_t * mp,int nrounds)17827c478bd9Sstevel@tonic-gate umem_magazine_destroy(umem_cache_t *cp, umem_magazine_t *mp, int nrounds)
17837c478bd9Sstevel@tonic-gate {
17847c478bd9Sstevel@tonic-gate 	int round;
17857c478bd9Sstevel@tonic-gate 
17867c478bd9Sstevel@tonic-gate 	ASSERT(cp->cache_next == NULL || IN_UPDATE());
17877c478bd9Sstevel@tonic-gate 
17887c478bd9Sstevel@tonic-gate 	for (round = 0; round < nrounds; round++) {
17897c478bd9Sstevel@tonic-gate 		void *buf = mp->mag_round[round];
17907c478bd9Sstevel@tonic-gate 
17917c478bd9Sstevel@tonic-gate 		if ((cp->cache_flags & UMF_DEADBEEF) &&
17927c478bd9Sstevel@tonic-gate 		    verify_pattern(UMEM_FREE_PATTERN, buf,
17937c478bd9Sstevel@tonic-gate 		    cp->cache_verify) != NULL) {
17947c478bd9Sstevel@tonic-gate 			umem_error(UMERR_MODIFIED, cp, buf);
17957c478bd9Sstevel@tonic-gate 			continue;
17967c478bd9Sstevel@tonic-gate 		}
17977c478bd9Sstevel@tonic-gate 
17987c478bd9Sstevel@tonic-gate 		if (!(cp->cache_flags & UMF_BUFTAG) &&
17997c478bd9Sstevel@tonic-gate 		    cp->cache_destructor != NULL)
18007c478bd9Sstevel@tonic-gate 			cp->cache_destructor(buf, cp->cache_private);
18017c478bd9Sstevel@tonic-gate 
18027c478bd9Sstevel@tonic-gate 		umem_slab_free(cp, buf);
18037c478bd9Sstevel@tonic-gate 	}
18047c478bd9Sstevel@tonic-gate 	ASSERT(UMEM_MAGAZINE_VALID(cp, mp));
18057c478bd9Sstevel@tonic-gate 	_umem_cache_free(cp->cache_magtype->mt_cache, mp);
18067c478bd9Sstevel@tonic-gate }
18077c478bd9Sstevel@tonic-gate 
18087c478bd9Sstevel@tonic-gate /*
18097c478bd9Sstevel@tonic-gate  * Allocate a magazine from the depot.
18107c478bd9Sstevel@tonic-gate  */
18117c478bd9Sstevel@tonic-gate static umem_magazine_t *
umem_depot_alloc(umem_cache_t * cp,umem_maglist_t * mlp)18127c478bd9Sstevel@tonic-gate umem_depot_alloc(umem_cache_t *cp, umem_maglist_t *mlp)
18137c478bd9Sstevel@tonic-gate {
18147c478bd9Sstevel@tonic-gate 	umem_magazine_t *mp;
18157c478bd9Sstevel@tonic-gate 
18167c478bd9Sstevel@tonic-gate 	/*
18177c478bd9Sstevel@tonic-gate 	 * If we can't get the depot lock without contention,
18187c478bd9Sstevel@tonic-gate 	 * update our contention count.  We use the depot
18197c478bd9Sstevel@tonic-gate 	 * contention rate to determine whether we need to
18207c478bd9Sstevel@tonic-gate 	 * increase the magazine size for better scalability.
18217c478bd9Sstevel@tonic-gate 	 */
18227c478bd9Sstevel@tonic-gate 	if (mutex_trylock(&cp->cache_depot_lock) != 0) {
18237c478bd9Sstevel@tonic-gate 		(void) mutex_lock(&cp->cache_depot_lock);
18247c478bd9Sstevel@tonic-gate 		cp->cache_depot_contention++;
18257c478bd9Sstevel@tonic-gate 	}
18267c478bd9Sstevel@tonic-gate 
18277c478bd9Sstevel@tonic-gate 	if ((mp = mlp->ml_list) != NULL) {
18287c478bd9Sstevel@tonic-gate 		ASSERT(UMEM_MAGAZINE_VALID(cp, mp));
18297c478bd9Sstevel@tonic-gate 		mlp->ml_list = mp->mag_next;
18307c478bd9Sstevel@tonic-gate 		if (--mlp->ml_total < mlp->ml_min)
18317c478bd9Sstevel@tonic-gate 			mlp->ml_min = mlp->ml_total;
18327c478bd9Sstevel@tonic-gate 		mlp->ml_alloc++;
18337c478bd9Sstevel@tonic-gate 	}
18347c478bd9Sstevel@tonic-gate 
18357c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&cp->cache_depot_lock);
18367c478bd9Sstevel@tonic-gate 
18377c478bd9Sstevel@tonic-gate 	return (mp);
18387c478bd9Sstevel@tonic-gate }
18397c478bd9Sstevel@tonic-gate 
18407c478bd9Sstevel@tonic-gate /*
18417c478bd9Sstevel@tonic-gate  * Free a magazine to the depot.
18427c478bd9Sstevel@tonic-gate  */
18437c478bd9Sstevel@tonic-gate static void
umem_depot_free(umem_cache_t * cp,umem_maglist_t * mlp,umem_magazine_t * mp)18447c478bd9Sstevel@tonic-gate umem_depot_free(umem_cache_t *cp, umem_maglist_t *mlp, umem_magazine_t *mp)
18457c478bd9Sstevel@tonic-gate {
18467c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&cp->cache_depot_lock);
18477c478bd9Sstevel@tonic-gate 	ASSERT(UMEM_MAGAZINE_VALID(cp, mp));
18487c478bd9Sstevel@tonic-gate 	mp->mag_next = mlp->ml_list;
18497c478bd9Sstevel@tonic-gate 	mlp->ml_list = mp;
18507c478bd9Sstevel@tonic-gate 	mlp->ml_total++;
18517c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&cp->cache_depot_lock);
18527c478bd9Sstevel@tonic-gate }
18537c478bd9Sstevel@tonic-gate 
18547c478bd9Sstevel@tonic-gate /*
18557c478bd9Sstevel@tonic-gate  * Update the working set statistics for cp's depot.
18567c478bd9Sstevel@tonic-gate  */
18577c478bd9Sstevel@tonic-gate static void
umem_depot_ws_update(umem_cache_t * cp)18587c478bd9Sstevel@tonic-gate umem_depot_ws_update(umem_cache_t *cp)
18597c478bd9Sstevel@tonic-gate {
18607c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&cp->cache_depot_lock);
18617c478bd9Sstevel@tonic-gate 	cp->cache_full.ml_reaplimit = cp->cache_full.ml_min;
18627c478bd9Sstevel@tonic-gate 	cp->cache_full.ml_min = cp->cache_full.ml_total;
18637c478bd9Sstevel@tonic-gate 	cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_min;
18647c478bd9Sstevel@tonic-gate 	cp->cache_empty.ml_min = cp->cache_empty.ml_total;
18657c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&cp->cache_depot_lock);
18667c478bd9Sstevel@tonic-gate }
18677c478bd9Sstevel@tonic-gate 
18687c478bd9Sstevel@tonic-gate /*
18697c478bd9Sstevel@tonic-gate  * Reap all magazines that have fallen out of the depot's working set.
18707c478bd9Sstevel@tonic-gate  */
18717c478bd9Sstevel@tonic-gate static void
umem_depot_ws_reap(umem_cache_t * cp)18727c478bd9Sstevel@tonic-gate umem_depot_ws_reap(umem_cache_t *cp)
18737c478bd9Sstevel@tonic-gate {
18747c478bd9Sstevel@tonic-gate 	long reap;
18757c478bd9Sstevel@tonic-gate 	umem_magazine_t *mp;
18767c478bd9Sstevel@tonic-gate 
18777c478bd9Sstevel@tonic-gate 	ASSERT(cp->cache_next == NULL || IN_REAP());
18787c478bd9Sstevel@tonic-gate 
18797c478bd9Sstevel@tonic-gate 	reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min);
18807c478bd9Sstevel@tonic-gate 	while (reap-- && (mp = umem_depot_alloc(cp, &cp->cache_full)) != NULL)
18817c478bd9Sstevel@tonic-gate 		umem_magazine_destroy(cp, mp, cp->cache_magtype->mt_magsize);
18827c478bd9Sstevel@tonic-gate 
18837c478bd9Sstevel@tonic-gate 	reap = MIN(cp->cache_empty.ml_reaplimit, cp->cache_empty.ml_min);
18847c478bd9Sstevel@tonic-gate 	while (reap-- && (mp = umem_depot_alloc(cp, &cp->cache_empty)) != NULL)
18857c478bd9Sstevel@tonic-gate 		umem_magazine_destroy(cp, mp, 0);
18867c478bd9Sstevel@tonic-gate }
18877c478bd9Sstevel@tonic-gate 
18887c478bd9Sstevel@tonic-gate static void
umem_cpu_reload(umem_cpu_cache_t * ccp,umem_magazine_t * mp,int rounds)18897c478bd9Sstevel@tonic-gate umem_cpu_reload(umem_cpu_cache_t *ccp, umem_magazine_t *mp, int rounds)
18907c478bd9Sstevel@tonic-gate {
18917c478bd9Sstevel@tonic-gate 	ASSERT((ccp->cc_loaded == NULL && ccp->cc_rounds == -1) ||
18927c478bd9Sstevel@tonic-gate 	    (ccp->cc_loaded && ccp->cc_rounds + rounds == ccp->cc_magsize));
18937c478bd9Sstevel@tonic-gate 	ASSERT(ccp->cc_magsize > 0);
18947c478bd9Sstevel@tonic-gate 
18957c478bd9Sstevel@tonic-gate 	ccp->cc_ploaded = ccp->cc_loaded;
18967c478bd9Sstevel@tonic-gate 	ccp->cc_prounds = ccp->cc_rounds;
18977c478bd9Sstevel@tonic-gate 	ccp->cc_loaded = mp;
18987c478bd9Sstevel@tonic-gate 	ccp->cc_rounds = rounds;
18997c478bd9Sstevel@tonic-gate }
19007c478bd9Sstevel@tonic-gate 
19017c478bd9Sstevel@tonic-gate /*
19027c478bd9Sstevel@tonic-gate  * Allocate a constructed object from cache cp.
19037c478bd9Sstevel@tonic-gate  */
19047c478bd9Sstevel@tonic-gate #pragma weak umem_cache_alloc = _umem_cache_alloc
19057c478bd9Sstevel@tonic-gate void *
_umem_cache_alloc(umem_cache_t * cp,int umflag)19067c478bd9Sstevel@tonic-gate _umem_cache_alloc(umem_cache_t *cp, int umflag)
19077c478bd9Sstevel@tonic-gate {
19087c478bd9Sstevel@tonic-gate 	umem_cpu_cache_t *ccp;
19097c478bd9Sstevel@tonic-gate 	umem_magazine_t *fmp;
19107c478bd9Sstevel@tonic-gate 	void *buf;
19117c478bd9Sstevel@tonic-gate 	int flags_nfatal;
19127c478bd9Sstevel@tonic-gate 
19137c478bd9Sstevel@tonic-gate retry:
19147c478bd9Sstevel@tonic-gate 	ccp = UMEM_CPU_CACHE(cp, CPU(cp->cache_cpu_mask));
19157c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&ccp->cc_lock);
19167c478bd9Sstevel@tonic-gate 	for (;;) {
19177c478bd9Sstevel@tonic-gate 		/*
19187c478bd9Sstevel@tonic-gate 		 * If there's an object available in the current CPU's
19197c478bd9Sstevel@tonic-gate 		 * loaded magazine, just take it and return.
19207c478bd9Sstevel@tonic-gate 		 */
19217c478bd9Sstevel@tonic-gate 		if (ccp->cc_rounds > 0) {
19227c478bd9Sstevel@tonic-gate 			buf = ccp->cc_loaded->mag_round[--ccp->cc_rounds];
19237c478bd9Sstevel@tonic-gate 			ccp->cc_alloc++;
19247c478bd9Sstevel@tonic-gate 			(void) mutex_unlock(&ccp->cc_lock);
19257c478bd9Sstevel@tonic-gate 			if ((ccp->cc_flags & UMF_BUFTAG) &&
19267c478bd9Sstevel@tonic-gate 			    umem_cache_alloc_debug(cp, buf, umflag) == -1) {
19277c478bd9Sstevel@tonic-gate 				if (umem_alloc_retry(cp, umflag)) {
19287c478bd9Sstevel@tonic-gate 					goto retry;
19297c478bd9Sstevel@tonic-gate 				}
19307c478bd9Sstevel@tonic-gate 
19317c478bd9Sstevel@tonic-gate 				return (NULL);
19327c478bd9Sstevel@tonic-gate 			}
19337c478bd9Sstevel@tonic-gate 			return (buf);
19347c478bd9Sstevel@tonic-gate 		}
19357c478bd9Sstevel@tonic-gate 
19367c478bd9Sstevel@tonic-gate 		/*
19377c478bd9Sstevel@tonic-gate 		 * The loaded magazine is empty.  If the previously loaded
19387c478bd9Sstevel@tonic-gate 		 * magazine was full, exchange them and try again.
19397c478bd9Sstevel@tonic-gate 		 */
19407c478bd9Sstevel@tonic-gate 		if (ccp->cc_prounds > 0) {
19417c478bd9Sstevel@tonic-gate 			umem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds);
19427c478bd9Sstevel@tonic-gate 			continue;
19437c478bd9Sstevel@tonic-gate 		}
19447c478bd9Sstevel@tonic-gate 
19457c478bd9Sstevel@tonic-gate 		/*
19467c478bd9Sstevel@tonic-gate 		 * If the magazine layer is disabled, break out now.
19477c478bd9Sstevel@tonic-gate 		 */
19487c478bd9Sstevel@tonic-gate 		if (ccp->cc_magsize == 0)
19497c478bd9Sstevel@tonic-gate 			break;
19507c478bd9Sstevel@tonic-gate 
19517c478bd9Sstevel@tonic-gate 		/*
19527c478bd9Sstevel@tonic-gate 		 * Try to get a full magazine from the depot.
19537c478bd9Sstevel@tonic-gate 		 */
19547c478bd9Sstevel@tonic-gate 		fmp = umem_depot_alloc(cp, &cp->cache_full);
19557c478bd9Sstevel@tonic-gate 		if (fmp != NULL) {
19567c478bd9Sstevel@tonic-gate 			if (ccp->cc_ploaded != NULL)
19577c478bd9Sstevel@tonic-gate 				umem_depot_free(cp, &cp->cache_empty,
19587c478bd9Sstevel@tonic-gate 				    ccp->cc_ploaded);
19597c478bd9Sstevel@tonic-gate 			umem_cpu_reload(ccp, fmp, ccp->cc_magsize);
19607c478bd9Sstevel@tonic-gate 			continue;
19617c478bd9Sstevel@tonic-gate 		}
19627c478bd9Sstevel@tonic-gate 
19637c478bd9Sstevel@tonic-gate 		/*
19647c478bd9Sstevel@tonic-gate 		 * There are no full magazines in the depot,
19657c478bd9Sstevel@tonic-gate 		 * so fall through to the slab layer.
19667c478bd9Sstevel@tonic-gate 		 */
19677c478bd9Sstevel@tonic-gate 		break;
19687c478bd9Sstevel@tonic-gate 	}
19697c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&ccp->cc_lock);
19707c478bd9Sstevel@tonic-gate 
19717c478bd9Sstevel@tonic-gate 	/*
19727c478bd9Sstevel@tonic-gate 	 * We couldn't allocate a constructed object from the magazine layer,
19737c478bd9Sstevel@tonic-gate 	 * so get a raw buffer from the slab layer and apply its constructor.
19747c478bd9Sstevel@tonic-gate 	 */
19757c478bd9Sstevel@tonic-gate 	buf = umem_slab_alloc(cp, umflag);
19767c478bd9Sstevel@tonic-gate 
19777c478bd9Sstevel@tonic-gate 	if (buf == NULL) {
19787c478bd9Sstevel@tonic-gate 		if (cp == &umem_null_cache)
19797c478bd9Sstevel@tonic-gate 			return (NULL);
19807c478bd9Sstevel@tonic-gate 		if (umem_alloc_retry(cp, umflag)) {
19817c478bd9Sstevel@tonic-gate 			goto retry;
19827c478bd9Sstevel@tonic-gate 		}
19837c478bd9Sstevel@tonic-gate 
19847c478bd9Sstevel@tonic-gate 		return (NULL);
19857c478bd9Sstevel@tonic-gate 	}
19867c478bd9Sstevel@tonic-gate 
19877c478bd9Sstevel@tonic-gate 	if (cp->cache_flags & UMF_BUFTAG) {
19887c478bd9Sstevel@tonic-gate 		/*
19897c478bd9Sstevel@tonic-gate 		 * Let umem_cache_alloc_debug() apply the constructor for us.
19907c478bd9Sstevel@tonic-gate 		 */
19917c478bd9Sstevel@tonic-gate 		if (umem_cache_alloc_debug(cp, buf, umflag) == -1) {
19927c478bd9Sstevel@tonic-gate 			if (umem_alloc_retry(cp, umflag)) {
19937c478bd9Sstevel@tonic-gate 				goto retry;
19947c478bd9Sstevel@tonic-gate 			}
19957c478bd9Sstevel@tonic-gate 			return (NULL);
19967c478bd9Sstevel@tonic-gate 		}
19977c478bd9Sstevel@tonic-gate 		return (buf);
19987c478bd9Sstevel@tonic-gate 	}
19997c478bd9Sstevel@tonic-gate 
20007c478bd9Sstevel@tonic-gate 	/*
20017c478bd9Sstevel@tonic-gate 	 * We do not pass fatal flags on to the constructor.  This prevents
20027c478bd9Sstevel@tonic-gate 	 * leaking buffers in the event of a subordinate constructor failing.
20037c478bd9Sstevel@tonic-gate 	 */
20047c478bd9Sstevel@tonic-gate 	flags_nfatal = UMEM_DEFAULT;
20057c478bd9Sstevel@tonic-gate 	if (cp->cache_constructor != NULL &&
20067c478bd9Sstevel@tonic-gate 	    cp->cache_constructor(buf, cp->cache_private, flags_nfatal) != 0) {
20077c478bd9Sstevel@tonic-gate 		atomic_add_64(&cp->cache_alloc_fail, 1);
20087c478bd9Sstevel@tonic-gate 		umem_slab_free(cp, buf);
20097c478bd9Sstevel@tonic-gate 
20107c478bd9Sstevel@tonic-gate 		if (umem_alloc_retry(cp, umflag)) {
20117c478bd9Sstevel@tonic-gate 			goto retry;
20127c478bd9Sstevel@tonic-gate 		}
20137c478bd9Sstevel@tonic-gate 		return (NULL);
20147c478bd9Sstevel@tonic-gate 	}
20157c478bd9Sstevel@tonic-gate 
20167c478bd9Sstevel@tonic-gate 	return (buf);
20177c478bd9Sstevel@tonic-gate }
20187c478bd9Sstevel@tonic-gate 
20197c478bd9Sstevel@tonic-gate /*
20207c478bd9Sstevel@tonic-gate  * Free a constructed object to cache cp.
20217c478bd9Sstevel@tonic-gate  */
20227c478bd9Sstevel@tonic-gate #pragma weak umem_cache_free = _umem_cache_free
20237c478bd9Sstevel@tonic-gate void
_umem_cache_free(umem_cache_t * cp,void * buf)20247c478bd9Sstevel@tonic-gate _umem_cache_free(umem_cache_t *cp, void *buf)
20257c478bd9Sstevel@tonic-gate {
20267c478bd9Sstevel@tonic-gate 	umem_cpu_cache_t *ccp = UMEM_CPU_CACHE(cp, CPU(cp->cache_cpu_mask));
20277c478bd9Sstevel@tonic-gate 	umem_magazine_t *emp;
20287c478bd9Sstevel@tonic-gate 	umem_magtype_t *mtp;
20297c478bd9Sstevel@tonic-gate 
20307c478bd9Sstevel@tonic-gate 	if (ccp->cc_flags & UMF_BUFTAG)
20317c478bd9Sstevel@tonic-gate 		if (umem_cache_free_debug(cp, buf) == -1)
20327c478bd9Sstevel@tonic-gate 			return;
20337c478bd9Sstevel@tonic-gate 
20347c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&ccp->cc_lock);
20357c478bd9Sstevel@tonic-gate 	for (;;) {
20367c478bd9Sstevel@tonic-gate 		/*
20377c478bd9Sstevel@tonic-gate 		 * If there's a slot available in the current CPU's
20387c478bd9Sstevel@tonic-gate 		 * loaded magazine, just put the object there and return.
20397c478bd9Sstevel@tonic-gate 		 */
20407c478bd9Sstevel@tonic-gate 		if ((uint_t)ccp->cc_rounds < ccp->cc_magsize) {
20417c478bd9Sstevel@tonic-gate 			ccp->cc_loaded->mag_round[ccp->cc_rounds++] = buf;
20427c478bd9Sstevel@tonic-gate 			ccp->cc_free++;
20437c478bd9Sstevel@tonic-gate 			(void) mutex_unlock(&ccp->cc_lock);
20447c478bd9Sstevel@tonic-gate 			return;
20457c478bd9Sstevel@tonic-gate 		}
20467c478bd9Sstevel@tonic-gate 
20477c478bd9Sstevel@tonic-gate 		/*
20487c478bd9Sstevel@tonic-gate 		 * The loaded magazine is full.  If the previously loaded
20497c478bd9Sstevel@tonic-gate 		 * magazine was empty, exchange them and try again.
20507c478bd9Sstevel@tonic-gate 		 */
20517c478bd9Sstevel@tonic-gate 		if (ccp->cc_prounds == 0) {
20527c478bd9Sstevel@tonic-gate 			umem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds);
20537c478bd9Sstevel@tonic-gate 			continue;
20547c478bd9Sstevel@tonic-gate 		}
20557c478bd9Sstevel@tonic-gate 
20567c478bd9Sstevel@tonic-gate 		/*
20577c478bd9Sstevel@tonic-gate 		 * If the magazine layer is disabled, break out now.
20587c478bd9Sstevel@tonic-gate 		 */
20597c478bd9Sstevel@tonic-gate 		if (ccp->cc_magsize == 0)
20607c478bd9Sstevel@tonic-gate 			break;
20617c478bd9Sstevel@tonic-gate 
20627c478bd9Sstevel@tonic-gate 		/*
20637c478bd9Sstevel@tonic-gate 		 * Try to get an empty magazine from the depot.
20647c478bd9Sstevel@tonic-gate 		 */
20657c478bd9Sstevel@tonic-gate 		emp = umem_depot_alloc(cp, &cp->cache_empty);
20667c478bd9Sstevel@tonic-gate 		if (emp != NULL) {
20677c478bd9Sstevel@tonic-gate 			if (ccp->cc_ploaded != NULL)
20687c478bd9Sstevel@tonic-gate 				umem_depot_free(cp, &cp->cache_full,
20697c478bd9Sstevel@tonic-gate 				    ccp->cc_ploaded);
20707c478bd9Sstevel@tonic-gate 			umem_cpu_reload(ccp, emp, 0);
20717c478bd9Sstevel@tonic-gate 			continue;
20727c478bd9Sstevel@tonic-gate 		}
20737c478bd9Sstevel@tonic-gate 
20747c478bd9Sstevel@tonic-gate 		/*
20757c478bd9Sstevel@tonic-gate 		 * There are no empty magazines in the depot,
20767c478bd9Sstevel@tonic-gate 		 * so try to allocate a new one.  We must drop all locks
20777c478bd9Sstevel@tonic-gate 		 * across umem_cache_alloc() because lower layers may
20787c478bd9Sstevel@tonic-gate 		 * attempt to allocate from this cache.
20797c478bd9Sstevel@tonic-gate 		 */
20807c478bd9Sstevel@tonic-gate 		mtp = cp->cache_magtype;
20817c478bd9Sstevel@tonic-gate 		(void) mutex_unlock(&ccp->cc_lock);
20827c478bd9Sstevel@tonic-gate 		emp = _umem_cache_alloc(mtp->mt_cache, UMEM_DEFAULT);
20837c478bd9Sstevel@tonic-gate 		(void) mutex_lock(&ccp->cc_lock);
20847c478bd9Sstevel@tonic-gate 
20857c478bd9Sstevel@tonic-gate 		if (emp != NULL) {
20867c478bd9Sstevel@tonic-gate 			/*
20877c478bd9Sstevel@tonic-gate 			 * We successfully allocated an empty magazine.
20887c478bd9Sstevel@tonic-gate 			 * However, we had to drop ccp->cc_lock to do it,
20897c478bd9Sstevel@tonic-gate 			 * so the cache's magazine size may have changed.
20907c478bd9Sstevel@tonic-gate 			 * If so, free the magazine and try again.
20917c478bd9Sstevel@tonic-gate 			 */
20927c478bd9Sstevel@tonic-gate 			if (ccp->cc_magsize != mtp->mt_magsize) {
20937c478bd9Sstevel@tonic-gate 				(void) mutex_unlock(&ccp->cc_lock);
20947c478bd9Sstevel@tonic-gate 				_umem_cache_free(mtp->mt_cache, emp);
20957c478bd9Sstevel@tonic-gate 				(void) mutex_lock(&ccp->cc_lock);
20967c478bd9Sstevel@tonic-gate 				continue;
20977c478bd9Sstevel@tonic-gate 			}
20987c478bd9Sstevel@tonic-gate 
20997c478bd9Sstevel@tonic-gate 			/*
21007c478bd9Sstevel@tonic-gate 			 * We got a magazine of the right size.  Add it to
21017c478bd9Sstevel@tonic-gate 			 * the depot and try the whole dance again.
21027c478bd9Sstevel@tonic-gate 			 */
21037c478bd9Sstevel@tonic-gate 			umem_depot_free(cp, &cp->cache_empty, emp);
21047c478bd9Sstevel@tonic-gate 			continue;
21057c478bd9Sstevel@tonic-gate 		}
21067c478bd9Sstevel@tonic-gate 
21077c478bd9Sstevel@tonic-gate 		/*
21087c478bd9Sstevel@tonic-gate 		 * We couldn't allocate an empty magazine,
21097c478bd9Sstevel@tonic-gate 		 * so fall through to the slab layer.
21107c478bd9Sstevel@tonic-gate 		 */
21117c478bd9Sstevel@tonic-gate 		break;
21127c478bd9Sstevel@tonic-gate 	}
21137c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&ccp->cc_lock);
21147c478bd9Sstevel@tonic-gate 
21157c478bd9Sstevel@tonic-gate 	/*
21167c478bd9Sstevel@tonic-gate 	 * We couldn't free our constructed object to the magazine layer,
21177c478bd9Sstevel@tonic-gate 	 * so apply its destructor and free it to the slab layer.
21187c478bd9Sstevel@tonic-gate 	 * Note that if UMF_BUFTAG is in effect, umem_cache_free_debug()
21197c478bd9Sstevel@tonic-gate 	 * will have already applied the destructor.
21207c478bd9Sstevel@tonic-gate 	 */
21217c478bd9Sstevel@tonic-gate 	if (!(cp->cache_flags & UMF_BUFTAG) && cp->cache_destructor != NULL)
21227c478bd9Sstevel@tonic-gate 		cp->cache_destructor(buf, cp->cache_private);
21237c478bd9Sstevel@tonic-gate 
21247c478bd9Sstevel@tonic-gate 	umem_slab_free(cp, buf);
21257c478bd9Sstevel@tonic-gate }
21267c478bd9Sstevel@tonic-gate 
21277c478bd9Sstevel@tonic-gate #pragma weak umem_zalloc = _umem_zalloc
21287c478bd9Sstevel@tonic-gate void *
_umem_zalloc(size_t size,int umflag)21297c478bd9Sstevel@tonic-gate _umem_zalloc(size_t size, int umflag)
21307c478bd9Sstevel@tonic-gate {
21317c478bd9Sstevel@tonic-gate 	size_t index = (size - 1) >> UMEM_ALIGN_SHIFT;
21327c478bd9Sstevel@tonic-gate 	void *buf;
21337c478bd9Sstevel@tonic-gate 
21347c478bd9Sstevel@tonic-gate retry:
21357c478bd9Sstevel@tonic-gate 	if (index < UMEM_MAXBUF >> UMEM_ALIGN_SHIFT) {
21367c478bd9Sstevel@tonic-gate 		umem_cache_t *cp = umem_alloc_table[index];
21377c478bd9Sstevel@tonic-gate 		buf = _umem_cache_alloc(cp, umflag);
21387c478bd9Sstevel@tonic-gate 		if (buf != NULL) {
21397c478bd9Sstevel@tonic-gate 			if (cp->cache_flags & UMF_BUFTAG) {
21407c478bd9Sstevel@tonic-gate 				umem_buftag_t *btp = UMEM_BUFTAG(cp, buf);
21417c478bd9Sstevel@tonic-gate 				((uint8_t *)buf)[size] = UMEM_REDZONE_BYTE;
21427c478bd9Sstevel@tonic-gate 				((uint32_t *)btp)[1] = UMEM_SIZE_ENCODE(size);
21437c478bd9Sstevel@tonic-gate 			}
21447c478bd9Sstevel@tonic-gate 			bzero(buf, size);
21457c478bd9Sstevel@tonic-gate 		} else if (umem_alloc_retry(cp, umflag))
21467c478bd9Sstevel@tonic-gate 			goto retry;
21477c478bd9Sstevel@tonic-gate 	} else {
21487c478bd9Sstevel@tonic-gate 		buf = _umem_alloc(size, umflag);	/* handles failure */
21497c478bd9Sstevel@tonic-gate 		if (buf != NULL)
21507c478bd9Sstevel@tonic-gate 			bzero(buf, size);
21517c478bd9Sstevel@tonic-gate 	}
21527c478bd9Sstevel@tonic-gate 	return (buf);
21537c478bd9Sstevel@tonic-gate }
21547c478bd9Sstevel@tonic-gate 
21557c478bd9Sstevel@tonic-gate #pragma weak umem_alloc = _umem_alloc
21567c478bd9Sstevel@tonic-gate void *
_umem_alloc(size_t size,int umflag)21577c478bd9Sstevel@tonic-gate _umem_alloc(size_t size, int umflag)
21587c478bd9Sstevel@tonic-gate {
21597c478bd9Sstevel@tonic-gate 	size_t index = (size - 1) >> UMEM_ALIGN_SHIFT;
21607c478bd9Sstevel@tonic-gate 	void *buf;
21617c478bd9Sstevel@tonic-gate umem_alloc_retry:
21627c478bd9Sstevel@tonic-gate 	if (index < UMEM_MAXBUF >> UMEM_ALIGN_SHIFT) {
21637c478bd9Sstevel@tonic-gate 		umem_cache_t *cp = umem_alloc_table[index];
21647c478bd9Sstevel@tonic-gate 		buf = _umem_cache_alloc(cp, umflag);
21657c478bd9Sstevel@tonic-gate 		if ((cp->cache_flags & UMF_BUFTAG) && buf != NULL) {
21667c478bd9Sstevel@tonic-gate 			umem_buftag_t *btp = UMEM_BUFTAG(cp, buf);
21677c478bd9Sstevel@tonic-gate 			((uint8_t *)buf)[size] = UMEM_REDZONE_BYTE;
21687c478bd9Sstevel@tonic-gate 			((uint32_t *)btp)[1] = UMEM_SIZE_ENCODE(size);
21697c478bd9Sstevel@tonic-gate 		}
21707c478bd9Sstevel@tonic-gate 		if (buf == NULL && umem_alloc_retry(cp, umflag))
21717c478bd9Sstevel@tonic-gate 			goto umem_alloc_retry;
21727c478bd9Sstevel@tonic-gate 		return (buf);
21737c478bd9Sstevel@tonic-gate 	}
21747c478bd9Sstevel@tonic-gate 	if (size == 0)
21757c478bd9Sstevel@tonic-gate 		return (NULL);
21767c478bd9Sstevel@tonic-gate 	if (umem_oversize_arena == NULL) {
21777c478bd9Sstevel@tonic-gate 		if (umem_init())
21787c478bd9Sstevel@tonic-gate 			ASSERT(umem_oversize_arena != NULL);
21797c478bd9Sstevel@tonic-gate 		else
21807c478bd9Sstevel@tonic-gate 			return (NULL);
21817c478bd9Sstevel@tonic-gate 	}
21827c478bd9Sstevel@tonic-gate 	buf = vmem_alloc(umem_oversize_arena, size, UMEM_VMFLAGS(umflag));
21837c478bd9Sstevel@tonic-gate 	if (buf == NULL) {
21847c478bd9Sstevel@tonic-gate 		umem_log_event(umem_failure_log, NULL, NULL, (void *)size);
21857c478bd9Sstevel@tonic-gate 		if (umem_alloc_retry(NULL, umflag))
21867c478bd9Sstevel@tonic-gate 			goto umem_alloc_retry;
21877c478bd9Sstevel@tonic-gate 	}
21887c478bd9Sstevel@tonic-gate 	return (buf);
21897c478bd9Sstevel@tonic-gate }
21907c478bd9Sstevel@tonic-gate 
21917c478bd9Sstevel@tonic-gate #pragma weak umem_alloc_align = _umem_alloc_align
21927c478bd9Sstevel@tonic-gate void *
_umem_alloc_align(size_t size,size_t align,int umflag)21937c478bd9Sstevel@tonic-gate _umem_alloc_align(size_t size, size_t align, int umflag)
21947c478bd9Sstevel@tonic-gate {
21957c478bd9Sstevel@tonic-gate 	void *buf;
21967c478bd9Sstevel@tonic-gate 
21977c478bd9Sstevel@tonic-gate 	if (size == 0)
21987c478bd9Sstevel@tonic-gate 		return (NULL);
21997c478bd9Sstevel@tonic-gate 	if ((align & (align - 1)) != 0)
22007c478bd9Sstevel@tonic-gate 		return (NULL);
22017c478bd9Sstevel@tonic-gate 	if (align < UMEM_ALIGN)
22027c478bd9Sstevel@tonic-gate 		align = UMEM_ALIGN;
22037c478bd9Sstevel@tonic-gate 
22047c478bd9Sstevel@tonic-gate umem_alloc_align_retry:
22057c478bd9Sstevel@tonic-gate 	if (umem_memalign_arena == NULL) {
22067c478bd9Sstevel@tonic-gate 		if (umem_init())
22077c478bd9Sstevel@tonic-gate 			ASSERT(umem_oversize_arena != NULL);
22087c478bd9Sstevel@tonic-gate 		else
22097c478bd9Sstevel@tonic-gate 			return (NULL);
22107c478bd9Sstevel@tonic-gate 	}
22117c478bd9Sstevel@tonic-gate 	buf = vmem_xalloc(umem_memalign_arena, size, align, 0, 0, NULL, NULL,
22127c478bd9Sstevel@tonic-gate 	    UMEM_VMFLAGS(umflag));
22137c478bd9Sstevel@tonic-gate 	if (buf == NULL) {
22147c478bd9Sstevel@tonic-gate 		umem_log_event(umem_failure_log, NULL, NULL, (void *)size);
22157c478bd9Sstevel@tonic-gate 		if (umem_alloc_retry(NULL, umflag))
22167c478bd9Sstevel@tonic-gate 			goto umem_alloc_align_retry;
22177c478bd9Sstevel@tonic-gate 	}
22187c478bd9Sstevel@tonic-gate 	return (buf);
22197c478bd9Sstevel@tonic-gate }
22207c478bd9Sstevel@tonic-gate 
22217c478bd9Sstevel@tonic-gate #pragma weak umem_free = _umem_free
22227c478bd9Sstevel@tonic-gate void
_umem_free(void * buf,size_t size)22237c478bd9Sstevel@tonic-gate _umem_free(void *buf, size_t size)
22247c478bd9Sstevel@tonic-gate {
22257c478bd9Sstevel@tonic-gate 	size_t index = (size - 1) >> UMEM_ALIGN_SHIFT;
22267c478bd9Sstevel@tonic-gate 
22277c478bd9Sstevel@tonic-gate 	if (index < UMEM_MAXBUF >> UMEM_ALIGN_SHIFT) {
22287c478bd9Sstevel@tonic-gate 		umem_cache_t *cp = umem_alloc_table[index];
22297c478bd9Sstevel@tonic-gate 		if (cp->cache_flags & UMF_BUFTAG) {
22307c478bd9Sstevel@tonic-gate 			umem_buftag_t *btp = UMEM_BUFTAG(cp, buf);
22317c478bd9Sstevel@tonic-gate 			uint32_t *ip = (uint32_t *)btp;
22327c478bd9Sstevel@tonic-gate 			if (ip[1] != UMEM_SIZE_ENCODE(size)) {
22337c478bd9Sstevel@tonic-gate 				if (*(uint64_t *)buf == UMEM_FREE_PATTERN) {
22347c478bd9Sstevel@tonic-gate 					umem_error(UMERR_DUPFREE, cp, buf);
22357c478bd9Sstevel@tonic-gate 					return;
22367c478bd9Sstevel@tonic-gate 				}
22377c478bd9Sstevel@tonic-gate 				if (UMEM_SIZE_VALID(ip[1])) {
22387c478bd9Sstevel@tonic-gate 					ip[0] = UMEM_SIZE_ENCODE(size);
22397c478bd9Sstevel@tonic-gate 					umem_error(UMERR_BADSIZE, cp, buf);
22407c478bd9Sstevel@tonic-gate 				} else {
22417c478bd9Sstevel@tonic-gate 					umem_error(UMERR_REDZONE, cp, buf);
22427c478bd9Sstevel@tonic-gate 				}
22437c478bd9Sstevel@tonic-gate 				return;
22447c478bd9Sstevel@tonic-gate 			}
22457c478bd9Sstevel@tonic-gate 			if (((uint8_t *)buf)[size] != UMEM_REDZONE_BYTE) {
22467c478bd9Sstevel@tonic-gate 				umem_error(UMERR_REDZONE, cp, buf);
22477c478bd9Sstevel@tonic-gate 				return;
22487c478bd9Sstevel@tonic-gate 			}
22497c478bd9Sstevel@tonic-gate 			btp->bt_redzone = UMEM_REDZONE_PATTERN;
22507c478bd9Sstevel@tonic-gate 		}
22517c478bd9Sstevel@tonic-gate 		_umem_cache_free(cp, buf);
22527c478bd9Sstevel@tonic-gate 	} else {
22537c478bd9Sstevel@tonic-gate 		if (buf == NULL && size == 0)
22547c478bd9Sstevel@tonic-gate 			return;
22557c478bd9Sstevel@tonic-gate 		vmem_free(umem_oversize_arena, buf, size);
22567c478bd9Sstevel@tonic-gate 	}
22577c478bd9Sstevel@tonic-gate }
22587c478bd9Sstevel@tonic-gate 
22597c478bd9Sstevel@tonic-gate #pragma weak umem_free_align = _umem_free_align
22607c478bd9Sstevel@tonic-gate void
_umem_free_align(void * buf,size_t size)22617c478bd9Sstevel@tonic-gate _umem_free_align(void *buf, size_t size)
22627c478bd9Sstevel@tonic-gate {
22637c478bd9Sstevel@tonic-gate 	if (buf == NULL && size == 0)
22647c478bd9Sstevel@tonic-gate 		return;
22657c478bd9Sstevel@tonic-gate 	vmem_xfree(umem_memalign_arena, buf, size);
22667c478bd9Sstevel@tonic-gate }
22677c478bd9Sstevel@tonic-gate 
22687c478bd9Sstevel@tonic-gate static void *
umem_firewall_va_alloc(vmem_t * vmp,size_t size,int vmflag)22697c478bd9Sstevel@tonic-gate umem_firewall_va_alloc(vmem_t *vmp, size_t size, int vmflag)
22707c478bd9Sstevel@tonic-gate {
22717c478bd9Sstevel@tonic-gate 	size_t realsize = size + vmp->vm_quantum;
22727c478bd9Sstevel@tonic-gate 
22737c478bd9Sstevel@tonic-gate 	/*
22747c478bd9Sstevel@tonic-gate 	 * Annoying edge case: if 'size' is just shy of ULONG_MAX, adding
22757c478bd9Sstevel@tonic-gate 	 * vm_quantum will cause integer wraparound.  Check for this, and
22767c478bd9Sstevel@tonic-gate 	 * blow off the firewall page in this case.  Note that such a
22777c478bd9Sstevel@tonic-gate 	 * giant allocation (the entire address space) can never be
22787c478bd9Sstevel@tonic-gate 	 * satisfied, so it will either fail immediately (VM_NOSLEEP)
22797c478bd9Sstevel@tonic-gate 	 * or sleep forever (VM_SLEEP).  Thus, there is no need for a
22807c478bd9Sstevel@tonic-gate 	 * corresponding check in umem_firewall_va_free().
22817c478bd9Sstevel@tonic-gate 	 */
22827c478bd9Sstevel@tonic-gate 	if (realsize < size)
22837c478bd9Sstevel@tonic-gate 		realsize = size;
22847c478bd9Sstevel@tonic-gate 
22857c478bd9Sstevel@tonic-gate 	return (vmem_alloc(vmp, realsize, vmflag | VM_NEXTFIT));
22867c478bd9Sstevel@tonic-gate }
22877c478bd9Sstevel@tonic-gate 
22887c478bd9Sstevel@tonic-gate static void
umem_firewall_va_free(vmem_t * vmp,void * addr,size_t size)22897c478bd9Sstevel@tonic-gate umem_firewall_va_free(vmem_t *vmp, void *addr, size_t size)
22907c478bd9Sstevel@tonic-gate {
22917c478bd9Sstevel@tonic-gate 	vmem_free(vmp, addr, size + vmp->vm_quantum);
22927c478bd9Sstevel@tonic-gate }
22937c478bd9Sstevel@tonic-gate 
22947c478bd9Sstevel@tonic-gate /*
22957c478bd9Sstevel@tonic-gate  * Reclaim all unused memory from a cache.
22967c478bd9Sstevel@tonic-gate  */
22977c478bd9Sstevel@tonic-gate static void
umem_cache_reap(umem_cache_t * cp)22987c478bd9Sstevel@tonic-gate umem_cache_reap(umem_cache_t *cp)
22997c478bd9Sstevel@tonic-gate {
23007c478bd9Sstevel@tonic-gate 	/*
23017c478bd9Sstevel@tonic-gate 	 * Ask the cache's owner to free some memory if possible.
23027c478bd9Sstevel@tonic-gate 	 * The idea is to handle things like the inode cache, which
23037c478bd9Sstevel@tonic-gate 	 * typically sits on a bunch of memory that it doesn't truly
23047c478bd9Sstevel@tonic-gate 	 * *need*.  Reclaim policy is entirely up to the owner; this
23057c478bd9Sstevel@tonic-gate 	 * callback is just an advisory plea for help.
23067c478bd9Sstevel@tonic-gate 	 */
23077c478bd9Sstevel@tonic-gate 	if (cp->cache_reclaim != NULL)
23087c478bd9Sstevel@tonic-gate 		cp->cache_reclaim(cp->cache_private);
23097c478bd9Sstevel@tonic-gate 
23107c478bd9Sstevel@tonic-gate 	umem_depot_ws_reap(cp);
23117c478bd9Sstevel@tonic-gate }
23127c478bd9Sstevel@tonic-gate 
23137c478bd9Sstevel@tonic-gate /*
23147c478bd9Sstevel@tonic-gate  * Purge all magazines from a cache and set its magazine limit to zero.
23157c478bd9Sstevel@tonic-gate  * All calls are serialized by being done by the update thread, except for
23167c478bd9Sstevel@tonic-gate  * the final call from umem_cache_destroy().
23177c478bd9Sstevel@tonic-gate  */
23187c478bd9Sstevel@tonic-gate static void
umem_cache_magazine_purge(umem_cache_t * cp)23197c478bd9Sstevel@tonic-gate umem_cache_magazine_purge(umem_cache_t *cp)
23207c478bd9Sstevel@tonic-gate {
23217c478bd9Sstevel@tonic-gate 	umem_cpu_cache_t *ccp;
23227c478bd9Sstevel@tonic-gate 	umem_magazine_t *mp, *pmp;
23237c478bd9Sstevel@tonic-gate 	int rounds, prounds, cpu_seqid;
23247c478bd9Sstevel@tonic-gate 
23257c478bd9Sstevel@tonic-gate 	ASSERT(cp->cache_next == NULL || IN_UPDATE());
23267c478bd9Sstevel@tonic-gate 
23277c478bd9Sstevel@tonic-gate 	for (cpu_seqid = 0; cpu_seqid < umem_max_ncpus; cpu_seqid++) {
23287c478bd9Sstevel@tonic-gate 		ccp = &cp->cache_cpu[cpu_seqid];
23297c478bd9Sstevel@tonic-gate 
23307c478bd9Sstevel@tonic-gate 		(void) mutex_lock(&ccp->cc_lock);
23317c478bd9Sstevel@tonic-gate 		mp = ccp->cc_loaded;
23327c478bd9Sstevel@tonic-gate 		pmp = ccp->cc_ploaded;
23337c478bd9Sstevel@tonic-gate 		rounds = ccp->cc_rounds;
23347c478bd9Sstevel@tonic-gate 		prounds = ccp->cc_prounds;
23357c478bd9Sstevel@tonic-gate 		ccp->cc_loaded = NULL;
23367c478bd9Sstevel@tonic-gate 		ccp->cc_ploaded = NULL;
23377c478bd9Sstevel@tonic-gate 		ccp->cc_rounds = -1;
23387c478bd9Sstevel@tonic-gate 		ccp->cc_prounds = -1;
23397c478bd9Sstevel@tonic-gate 		ccp->cc_magsize = 0;
23407c478bd9Sstevel@tonic-gate 		(void) mutex_unlock(&ccp->cc_lock);
23417c478bd9Sstevel@tonic-gate 
23427c478bd9Sstevel@tonic-gate 		if (mp)
23437c478bd9Sstevel@tonic-gate 			umem_magazine_destroy(cp, mp, rounds);
23447c478bd9Sstevel@tonic-gate 		if (pmp)
23457c478bd9Sstevel@tonic-gate 			umem_magazine_destroy(cp, pmp, prounds);
23467c478bd9Sstevel@tonic-gate 	}
23477c478bd9Sstevel@tonic-gate 
23487c478bd9Sstevel@tonic-gate 	/*
23497c478bd9Sstevel@tonic-gate 	 * Updating the working set statistics twice in a row has the
23507c478bd9Sstevel@tonic-gate 	 * effect of setting the working set size to zero, so everything
23517c478bd9Sstevel@tonic-gate 	 * is eligible for reaping.
23527c478bd9Sstevel@tonic-gate 	 */
23537c478bd9Sstevel@tonic-gate 	umem_depot_ws_update(cp);
23547c478bd9Sstevel@tonic-gate 	umem_depot_ws_update(cp);
23557c478bd9Sstevel@tonic-gate 
23567c478bd9Sstevel@tonic-gate 	umem_depot_ws_reap(cp);
23577c478bd9Sstevel@tonic-gate }
23587c478bd9Sstevel@tonic-gate 
23597c478bd9Sstevel@tonic-gate /*
23607c478bd9Sstevel@tonic-gate  * Enable per-cpu magazines on a cache.
23617c478bd9Sstevel@tonic-gate  */
23627c478bd9Sstevel@tonic-gate static void
umem_cache_magazine_enable(umem_cache_t * cp)23637c478bd9Sstevel@tonic-gate umem_cache_magazine_enable(umem_cache_t *cp)
23647c478bd9Sstevel@tonic-gate {
23657c478bd9Sstevel@tonic-gate 	int cpu_seqid;
23667c478bd9Sstevel@tonic-gate 
23677c478bd9Sstevel@tonic-gate 	if (cp->cache_flags & UMF_NOMAGAZINE)
23687c478bd9Sstevel@tonic-gate 		return;
23697c478bd9Sstevel@tonic-gate 
23707c478bd9Sstevel@tonic-gate 	for (cpu_seqid = 0; cpu_seqid < umem_max_ncpus; cpu_seqid++) {
23717c478bd9Sstevel@tonic-gate 		umem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid];
23727c478bd9Sstevel@tonic-gate 		(void) mutex_lock(&ccp->cc_lock);
23737c478bd9Sstevel@tonic-gate 		ccp->cc_magsize = cp->cache_magtype->mt_magsize;
23747c478bd9Sstevel@tonic-gate 		(void) mutex_unlock(&ccp->cc_lock);
23757c478bd9Sstevel@tonic-gate 	}
23767c478bd9Sstevel@tonic-gate 
23777c478bd9Sstevel@tonic-gate }
23787c478bd9Sstevel@tonic-gate 
23797c478bd9Sstevel@tonic-gate /*
23807c478bd9Sstevel@tonic-gate  * Recompute a cache's magazine size.  The trade-off is that larger magazines
23817c478bd9Sstevel@tonic-gate  * provide a higher transfer rate with the depot, while smaller magazines
23827c478bd9Sstevel@tonic-gate  * reduce memory consumption.  Magazine resizing is an expensive operation;
23837c478bd9Sstevel@tonic-gate  * it should not be done frequently.
23847c478bd9Sstevel@tonic-gate  *
23857c478bd9Sstevel@tonic-gate  * Changes to the magazine size are serialized by only having one thread
23867c478bd9Sstevel@tonic-gate  * doing updates. (the update thread)
23877c478bd9Sstevel@tonic-gate  *
23887c478bd9Sstevel@tonic-gate  * Note: at present this only grows the magazine size.  It might be useful
23897c478bd9Sstevel@tonic-gate  * to allow shrinkage too.
23907c478bd9Sstevel@tonic-gate  */
23917c478bd9Sstevel@tonic-gate static void
umem_cache_magazine_resize(umem_cache_t * cp)23927c478bd9Sstevel@tonic-gate umem_cache_magazine_resize(umem_cache_t *cp)
23937c478bd9Sstevel@tonic-gate {
23947c478bd9Sstevel@tonic-gate 	umem_magtype_t *mtp = cp->cache_magtype;
23957c478bd9Sstevel@tonic-gate 
23967c478bd9Sstevel@tonic-gate 	ASSERT(IN_UPDATE());
23977c478bd9Sstevel@tonic-gate 
23987c478bd9Sstevel@tonic-gate 	if (cp->cache_chunksize < mtp->mt_maxbuf) {
23997c478bd9Sstevel@tonic-gate 		umem_cache_magazine_purge(cp);
24007c478bd9Sstevel@tonic-gate 		(void) mutex_lock(&cp->cache_depot_lock);
24017c478bd9Sstevel@tonic-gate 		cp->cache_magtype = ++mtp;
24027c478bd9Sstevel@tonic-gate 		cp->cache_depot_contention_prev =
24037c478bd9Sstevel@tonic-gate 		    cp->cache_depot_contention + INT_MAX;
24047c478bd9Sstevel@tonic-gate 		(void) mutex_unlock(&cp->cache_depot_lock);
24057c478bd9Sstevel@tonic-gate 		umem_cache_magazine_enable(cp);
24067c478bd9Sstevel@tonic-gate 	}
24077c478bd9Sstevel@tonic-gate }
24087c478bd9Sstevel@tonic-gate 
24097c478bd9Sstevel@tonic-gate /*
24107c478bd9Sstevel@tonic-gate  * Rescale a cache's hash table, so that the table size is roughly the
24117c478bd9Sstevel@tonic-gate  * cache size.  We want the average lookup time to be extremely small.
24127c478bd9Sstevel@tonic-gate  */
24137c478bd9Sstevel@tonic-gate static void
umem_hash_rescale(umem_cache_t * cp)24147c478bd9Sstevel@tonic-gate umem_hash_rescale(umem_cache_t *cp)
24157c478bd9Sstevel@tonic-gate {
24167c478bd9Sstevel@tonic-gate 	umem_bufctl_t **old_table, **new_table, *bcp;
24177c478bd9Sstevel@tonic-gate 	size_t old_size, new_size, h;
24187c478bd9Sstevel@tonic-gate 
24197c478bd9Sstevel@tonic-gate 	ASSERT(IN_UPDATE());
24207c478bd9Sstevel@tonic-gate 
24217c478bd9Sstevel@tonic-gate 	new_size = MAX(UMEM_HASH_INITIAL,
24227c478bd9Sstevel@tonic-gate 	    1 << (highbit(3 * cp->cache_buftotal + 4) - 2));
24237c478bd9Sstevel@tonic-gate 	old_size = cp->cache_hash_mask + 1;
24247c478bd9Sstevel@tonic-gate 
24257c478bd9Sstevel@tonic-gate 	if ((old_size >> 1) <= new_size && new_size <= (old_size << 1))
24267c478bd9Sstevel@tonic-gate 		return;
24277c478bd9Sstevel@tonic-gate 
24287c478bd9Sstevel@tonic-gate 	new_table = vmem_alloc(umem_hash_arena, new_size * sizeof (void *),
24297c478bd9Sstevel@tonic-gate 	    VM_NOSLEEP);
24307c478bd9Sstevel@tonic-gate 	if (new_table == NULL)
24317c478bd9Sstevel@tonic-gate 		return;
24327c478bd9Sstevel@tonic-gate 	bzero(new_table, new_size * sizeof (void *));
24337c478bd9Sstevel@tonic-gate 
24347c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&cp->cache_lock);
24357c478bd9Sstevel@tonic-gate 
24367c478bd9Sstevel@tonic-gate 	old_size = cp->cache_hash_mask + 1;
24377c478bd9Sstevel@tonic-gate 	old_table = cp->cache_hash_table;
24387c478bd9Sstevel@tonic-gate 
24397c478bd9Sstevel@tonic-gate 	cp->cache_hash_mask = new_size - 1;
24407c478bd9Sstevel@tonic-gate 	cp->cache_hash_table = new_table;
24417c478bd9Sstevel@tonic-gate 	cp->cache_rescale++;
24427c478bd9Sstevel@tonic-gate 
24437c478bd9Sstevel@tonic-gate 	for (h = 0; h < old_size; h++) {
24447c478bd9Sstevel@tonic-gate 		bcp = old_table[h];
24457c478bd9Sstevel@tonic-gate 		while (bcp != NULL) {
24467c478bd9Sstevel@tonic-gate 			void *addr = bcp->bc_addr;
24477c478bd9Sstevel@tonic-gate 			umem_bufctl_t *next_bcp = bcp->bc_next;
24487c478bd9Sstevel@tonic-gate 			umem_bufctl_t **hash_bucket = UMEM_HASH(cp, addr);
24497c478bd9Sstevel@tonic-gate 			bcp->bc_next = *hash_bucket;
24507c478bd9Sstevel@tonic-gate 			*hash_bucket = bcp;
24517c478bd9Sstevel@tonic-gate 			bcp = next_bcp;
24527c478bd9Sstevel@tonic-gate 		}
24537c478bd9Sstevel@tonic-gate 	}
24547c478bd9Sstevel@tonic-gate 
24557c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&cp->cache_lock);
24567c478bd9Sstevel@tonic-gate 
24577c478bd9Sstevel@tonic-gate 	vmem_free(umem_hash_arena, old_table, old_size * sizeof (void *));
24587c478bd9Sstevel@tonic-gate }
24597c478bd9Sstevel@tonic-gate 
24607c478bd9Sstevel@tonic-gate /*
24617c478bd9Sstevel@tonic-gate  * Perform periodic maintenance on a cache: hash rescaling,
24627c478bd9Sstevel@tonic-gate  * depot working-set update, and magazine resizing.
24637c478bd9Sstevel@tonic-gate  */
24647c478bd9Sstevel@tonic-gate void
umem_cache_update(umem_cache_t * cp)24657c478bd9Sstevel@tonic-gate umem_cache_update(umem_cache_t *cp)
24667c478bd9Sstevel@tonic-gate {
24677c478bd9Sstevel@tonic-gate 	int update_flags = 0;
24687c478bd9Sstevel@tonic-gate 
24697c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&umem_cache_lock));
24707c478bd9Sstevel@tonic-gate 
24717c478bd9Sstevel@tonic-gate 	/*
24727c478bd9Sstevel@tonic-gate 	 * If the cache has become much larger or smaller than its hash table,
24737c478bd9Sstevel@tonic-gate 	 * fire off a request to rescale the hash table.
24747c478bd9Sstevel@tonic-gate 	 */
24757c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&cp->cache_lock);
24767c478bd9Sstevel@tonic-gate 
24777c478bd9Sstevel@tonic-gate 	if ((cp->cache_flags & UMF_HASH) &&
24787c478bd9Sstevel@tonic-gate 	    (cp->cache_buftotal > (cp->cache_hash_mask << 1) ||
24797c478bd9Sstevel@tonic-gate 	    (cp->cache_buftotal < (cp->cache_hash_mask >> 1) &&
24807c478bd9Sstevel@tonic-gate 	    cp->cache_hash_mask > UMEM_HASH_INITIAL)))
24817c478bd9Sstevel@tonic-gate 		update_flags |= UMU_HASH_RESCALE;
24827c478bd9Sstevel@tonic-gate 
24837c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&cp->cache_lock);
24847c478bd9Sstevel@tonic-gate 
24857c478bd9Sstevel@tonic-gate 	/*
24867c478bd9Sstevel@tonic-gate 	 * Update the depot working set statistics.
24877c478bd9Sstevel@tonic-gate 	 */
24887c478bd9Sstevel@tonic-gate 	umem_depot_ws_update(cp);
24897c478bd9Sstevel@tonic-gate 
24907c478bd9Sstevel@tonic-gate 	/*
24917c478bd9Sstevel@tonic-gate 	 * If there's a lot of contention in the depot,
24927c478bd9Sstevel@tonic-gate 	 * increase the magazine size.
24937c478bd9Sstevel@tonic-gate 	 */
24947c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&cp->cache_depot_lock);
24957c478bd9Sstevel@tonic-gate 
24967c478bd9Sstevel@tonic-gate 	if (cp->cache_chunksize < cp->cache_magtype->mt_maxbuf &&
24977c478bd9Sstevel@tonic-gate 	    (int)(cp->cache_depot_contention -
24987c478bd9Sstevel@tonic-gate 	    cp->cache_depot_contention_prev) > umem_depot_contention)
24997c478bd9Sstevel@tonic-gate 		update_flags |= UMU_MAGAZINE_RESIZE;
25007c478bd9Sstevel@tonic-gate 
25017c478bd9Sstevel@tonic-gate 	cp->cache_depot_contention_prev = cp->cache_depot_contention;
25027c478bd9Sstevel@tonic-gate 
25037c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&cp->cache_depot_lock);
25047c478bd9Sstevel@tonic-gate 
25057c478bd9Sstevel@tonic-gate 	if (update_flags)
25067c478bd9Sstevel@tonic-gate 		umem_add_update(cp, update_flags);
25077c478bd9Sstevel@tonic-gate }
25087c478bd9Sstevel@tonic-gate 
25097c478bd9Sstevel@tonic-gate /*
25107c478bd9Sstevel@tonic-gate  * Runs all pending updates.
25117c478bd9Sstevel@tonic-gate  *
25127c478bd9Sstevel@tonic-gate  * The update lock must be held on entrance, and will be held on exit.
25137c478bd9Sstevel@tonic-gate  */
25147c478bd9Sstevel@tonic-gate void
umem_process_updates(void)25157c478bd9Sstevel@tonic-gate umem_process_updates(void)
25167c478bd9Sstevel@tonic-gate {
25177c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&umem_update_lock));
25187c478bd9Sstevel@tonic-gate 
25197c478bd9Sstevel@tonic-gate 	while (umem_null_cache.cache_unext != &umem_null_cache) {
25207c478bd9Sstevel@tonic-gate 		int notify = 0;
25217c478bd9Sstevel@tonic-gate 		umem_cache_t *cp = umem_null_cache.cache_unext;
25227c478bd9Sstevel@tonic-gate 
25237c478bd9Sstevel@tonic-gate 		cp->cache_uprev->cache_unext = cp->cache_unext;
25247c478bd9Sstevel@tonic-gate 		cp->cache_unext->cache_uprev = cp->cache_uprev;
25257c478bd9Sstevel@tonic-gate 		cp->cache_uprev = cp->cache_unext = NULL;
25267c478bd9Sstevel@tonic-gate 
25277c478bd9Sstevel@tonic-gate 		ASSERT(!(cp->cache_uflags & UMU_ACTIVE));
25287c478bd9Sstevel@tonic-gate 
25297c478bd9Sstevel@tonic-gate 		while (cp->cache_uflags) {
25307c478bd9Sstevel@tonic-gate 			int uflags = (cp->cache_uflags |= UMU_ACTIVE);
25317c478bd9Sstevel@tonic-gate 			(void) mutex_unlock(&umem_update_lock);
25327c478bd9Sstevel@tonic-gate 
25337c478bd9Sstevel@tonic-gate 			/*
25347c478bd9Sstevel@tonic-gate 			 * The order here is important.  Each step can speed up
25357c478bd9Sstevel@tonic-gate 			 * later steps.
25367c478bd9Sstevel@tonic-gate 			 */
25377c478bd9Sstevel@tonic-gate 
25387c478bd9Sstevel@tonic-gate 			if (uflags & UMU_HASH_RESCALE)
25397c478bd9Sstevel@tonic-gate 				umem_hash_rescale(cp);
25407c478bd9Sstevel@tonic-gate 
25417c478bd9Sstevel@tonic-gate 			if (uflags & UMU_MAGAZINE_RESIZE)
25427c478bd9Sstevel@tonic-gate 				umem_cache_magazine_resize(cp);
25437c478bd9Sstevel@tonic-gate 
25447c478bd9Sstevel@tonic-gate 			if (uflags & UMU_REAP)
25457c478bd9Sstevel@tonic-gate 				umem_cache_reap(cp);
25467c478bd9Sstevel@tonic-gate 
25477c478bd9Sstevel@tonic-gate 			(void) mutex_lock(&umem_update_lock);
25487c478bd9Sstevel@tonic-gate 
25497c478bd9Sstevel@tonic-gate 			/*
25507c478bd9Sstevel@tonic-gate 			 * check if anyone has requested notification
25517c478bd9Sstevel@tonic-gate 			 */
25527c478bd9Sstevel@tonic-gate 			if (cp->cache_uflags & UMU_NOTIFY) {
25537c478bd9Sstevel@tonic-gate 				uflags |= UMU_NOTIFY;
25547c478bd9Sstevel@tonic-gate 				notify = 1;
25557c478bd9Sstevel@tonic-gate 			}
25567c478bd9Sstevel@tonic-gate 			cp->cache_uflags &= ~uflags;
25577c478bd9Sstevel@tonic-gate 		}
25587c478bd9Sstevel@tonic-gate 		if (notify)
25597c478bd9Sstevel@tonic-gate 			(void) cond_broadcast(&umem_update_cv);
25607c478bd9Sstevel@tonic-gate 	}
25617c478bd9Sstevel@tonic-gate }
25627c478bd9Sstevel@tonic-gate 
25637c478bd9Sstevel@tonic-gate #ifndef UMEM_STANDALONE
25647c478bd9Sstevel@tonic-gate static void
umem_st_update(void)25657c478bd9Sstevel@tonic-gate umem_st_update(void)
25667c478bd9Sstevel@tonic-gate {
25677c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&umem_update_lock));
25687c478bd9Sstevel@tonic-gate 	ASSERT(umem_update_thr == 0 && umem_st_update_thr == 0);
25697c478bd9Sstevel@tonic-gate 
25707c478bd9Sstevel@tonic-gate 	umem_st_update_thr = thr_self();
25717c478bd9Sstevel@tonic-gate 
25727c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&umem_update_lock);
25737c478bd9Sstevel@tonic-gate 
25747c478bd9Sstevel@tonic-gate 	vmem_update(NULL);
25757c478bd9Sstevel@tonic-gate 	umem_cache_applyall(umem_cache_update);
25767c478bd9Sstevel@tonic-gate 
25777c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&umem_update_lock);
25787c478bd9Sstevel@tonic-gate 
25797c478bd9Sstevel@tonic-gate 	umem_process_updates();	/* does all of the requested work */
25807c478bd9Sstevel@tonic-gate 
25817c478bd9Sstevel@tonic-gate 	umem_reap_next = gethrtime() +
25827c478bd9Sstevel@tonic-gate 	    (hrtime_t)umem_reap_interval * NANOSEC;
25837c478bd9Sstevel@tonic-gate 
25847c478bd9Sstevel@tonic-gate 	umem_reaping = UMEM_REAP_DONE;
25857c478bd9Sstevel@tonic-gate 
25867c478bd9Sstevel@tonic-gate 	umem_st_update_thr = 0;
25877c478bd9Sstevel@tonic-gate }
25887c478bd9Sstevel@tonic-gate #endif
25897c478bd9Sstevel@tonic-gate 
25907c478bd9Sstevel@tonic-gate /*
25917c478bd9Sstevel@tonic-gate  * Reclaim all unused memory from all caches.  Called from vmem when memory
25927c478bd9Sstevel@tonic-gate  * gets tight.  Must be called with no locks held.
25937c478bd9Sstevel@tonic-gate  *
25947c478bd9Sstevel@tonic-gate  * This just requests a reap on all caches, and notifies the update thread.
25957c478bd9Sstevel@tonic-gate  */
25967c478bd9Sstevel@tonic-gate void
umem_reap(void)25977c478bd9Sstevel@tonic-gate umem_reap(void)
25987c478bd9Sstevel@tonic-gate {
25997c478bd9Sstevel@tonic-gate #ifndef UMEM_STANDALONE
26007c478bd9Sstevel@tonic-gate 	extern int __nthreads(void);
26017c478bd9Sstevel@tonic-gate #endif
26027c478bd9Sstevel@tonic-gate 
26037c478bd9Sstevel@tonic-gate 	if (umem_ready != UMEM_READY || umem_reaping != UMEM_REAP_DONE ||
26047c478bd9Sstevel@tonic-gate 	    gethrtime() < umem_reap_next)
26057c478bd9Sstevel@tonic-gate 		return;
26067c478bd9Sstevel@tonic-gate 
26077c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&umem_update_lock);
26087c478bd9Sstevel@tonic-gate 
26097c478bd9Sstevel@tonic-gate 	if (umem_reaping != UMEM_REAP_DONE || gethrtime() < umem_reap_next) {
26107c478bd9Sstevel@tonic-gate 		(void) mutex_unlock(&umem_update_lock);
26117c478bd9Sstevel@tonic-gate 		return;
26127c478bd9Sstevel@tonic-gate 	}
26137c478bd9Sstevel@tonic-gate 	umem_reaping = UMEM_REAP_ADDING;	/* lock out other reaps */
26147c478bd9Sstevel@tonic-gate 
26157c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&umem_update_lock);
26167c478bd9Sstevel@tonic-gate 
26177c478bd9Sstevel@tonic-gate 	umem_updateall(UMU_REAP);
26187c478bd9Sstevel@tonic-gate 
26197c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&umem_update_lock);
26207c478bd9Sstevel@tonic-gate 
26217c478bd9Sstevel@tonic-gate 	umem_reaping = UMEM_REAP_ACTIVE;
26227c478bd9Sstevel@tonic-gate 
26237c478bd9Sstevel@tonic-gate 	/* Standalone is single-threaded */
26247c478bd9Sstevel@tonic-gate #ifndef UMEM_STANDALONE
26257c478bd9Sstevel@tonic-gate 	if (umem_update_thr == 0) {
26267c478bd9Sstevel@tonic-gate 		/*
26277c478bd9Sstevel@tonic-gate 		 * The update thread does not exist.  If the process is
26287c478bd9Sstevel@tonic-gate 		 * multi-threaded, create it.  If not, or the creation fails,
26297c478bd9Sstevel@tonic-gate 		 * do the update processing inline.
26307c478bd9Sstevel@tonic-gate 		 */
26317c478bd9Sstevel@tonic-gate 		ASSERT(umem_st_update_thr == 0);
26327c478bd9Sstevel@tonic-gate 
26337c478bd9Sstevel@tonic-gate 		if (__nthreads() <= 1 || umem_create_update_thread() == 0)
26347c478bd9Sstevel@tonic-gate 			umem_st_update();
26357c478bd9Sstevel@tonic-gate 	}
26367c478bd9Sstevel@tonic-gate 
26377c478bd9Sstevel@tonic-gate 	(void) cond_broadcast(&umem_update_cv);	/* wake up the update thread */
26387c478bd9Sstevel@tonic-gate #endif
26397c478bd9Sstevel@tonic-gate 
26407c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&umem_update_lock);
26417c478bd9Sstevel@tonic-gate }
26427c478bd9Sstevel@tonic-gate 
26437c478bd9Sstevel@tonic-gate umem_cache_t *
umem_cache_create(char * name,size_t bufsize,size_t align,umem_constructor_t * constructor,umem_destructor_t * destructor,umem_reclaim_t * reclaim,void * private,vmem_t * vmp,int cflags)26447c478bd9Sstevel@tonic-gate umem_cache_create(
26457c478bd9Sstevel@tonic-gate 	char *name,		/* descriptive name for this cache */
26467c478bd9Sstevel@tonic-gate 	size_t bufsize,		/* size of the objects it manages */
26477c478bd9Sstevel@tonic-gate 	size_t align,		/* required object alignment */
26487c478bd9Sstevel@tonic-gate 	umem_constructor_t *constructor, /* object constructor */
26497c478bd9Sstevel@tonic-gate 	umem_destructor_t *destructor, /* object destructor */
26507c478bd9Sstevel@tonic-gate 	umem_reclaim_t *reclaim, /* memory reclaim callback */
26517c478bd9Sstevel@tonic-gate 	void *private,		/* pass-thru arg for constr/destr/reclaim */
26527c478bd9Sstevel@tonic-gate 	vmem_t *vmp,		/* vmem source for slab allocation */
26537c478bd9Sstevel@tonic-gate 	int cflags)		/* cache creation flags */
26547c478bd9Sstevel@tonic-gate {
26557c478bd9Sstevel@tonic-gate 	int cpu_seqid;
26567c478bd9Sstevel@tonic-gate 	size_t chunksize;
26577c478bd9Sstevel@tonic-gate 	umem_cache_t *cp, *cnext, *cprev;
26587c478bd9Sstevel@tonic-gate 	umem_magtype_t *mtp;
26597c478bd9Sstevel@tonic-gate 	size_t csize;
26607c478bd9Sstevel@tonic-gate 	size_t phase;
26617c478bd9Sstevel@tonic-gate 
26627c478bd9Sstevel@tonic-gate 	/*
26637c478bd9Sstevel@tonic-gate 	 * The init thread is allowed to create internal and quantum caches.
26647c478bd9Sstevel@tonic-gate 	 *
26657c478bd9Sstevel@tonic-gate 	 * Other threads must wait until until initialization is complete.
26667c478bd9Sstevel@tonic-gate 	 */
26677c478bd9Sstevel@tonic-gate 	if (umem_init_thr == thr_self())
26687c478bd9Sstevel@tonic-gate 		ASSERT((cflags & (UMC_INTERNAL | UMC_QCACHE)) != 0);
26697c478bd9Sstevel@tonic-gate 	else {
26707c478bd9Sstevel@tonic-gate 		ASSERT(!(cflags & UMC_INTERNAL));
26717c478bd9Sstevel@tonic-gate 		if (umem_ready != UMEM_READY && umem_init() == 0) {
26727c478bd9Sstevel@tonic-gate 			errno = EAGAIN;
26737c478bd9Sstevel@tonic-gate 			return (NULL);
26747c478bd9Sstevel@tonic-gate 		}
26757c478bd9Sstevel@tonic-gate 	}
26767c478bd9Sstevel@tonic-gate 
26777c478bd9Sstevel@tonic-gate 	csize = UMEM_CACHE_SIZE(umem_max_ncpus);
26787c478bd9Sstevel@tonic-gate 	phase = P2NPHASE(csize, UMEM_CPU_CACHE_SIZE);
26797c478bd9Sstevel@tonic-gate 
26807c478bd9Sstevel@tonic-gate 	if (vmp == NULL)
26817c478bd9Sstevel@tonic-gate 		vmp = umem_default_arena;
26827c478bd9Sstevel@tonic-gate 
26837c478bd9Sstevel@tonic-gate 	ASSERT(P2PHASE(phase, UMEM_ALIGN) == 0);
26847c478bd9Sstevel@tonic-gate 
26857c478bd9Sstevel@tonic-gate 	/*
26867c478bd9Sstevel@tonic-gate 	 * Check that the arguments are reasonable
26877c478bd9Sstevel@tonic-gate 	 */
26887c478bd9Sstevel@tonic-gate 	if ((align & (align - 1)) != 0 || align > vmp->vm_quantum ||
26897c478bd9Sstevel@tonic-gate 	    ((cflags & UMC_NOHASH) && (cflags & UMC_NOTOUCH)) ||
26907c478bd9Sstevel@tonic-gate 	    name == NULL || bufsize == 0) {
26917c478bd9Sstevel@tonic-gate 		errno = EINVAL;
26927c478bd9Sstevel@tonic-gate 		return (NULL);
26937c478bd9Sstevel@tonic-gate 	}
26947c478bd9Sstevel@tonic-gate 
26957c478bd9Sstevel@tonic-gate 	/*
26967c478bd9Sstevel@tonic-gate 	 * If align == 0, we set it to the minimum required alignment.
26977c478bd9Sstevel@tonic-gate 	 *
26987c478bd9Sstevel@tonic-gate 	 * If align < UMEM_ALIGN, we round it up to UMEM_ALIGN, unless
26997c478bd9Sstevel@tonic-gate 	 * UMC_NOTOUCH was passed.
27007c478bd9Sstevel@tonic-gate 	 */
27017c478bd9Sstevel@tonic-gate 	if (align == 0) {
27027c478bd9Sstevel@tonic-gate 		if (P2ROUNDUP(bufsize, UMEM_ALIGN) >= UMEM_SECOND_ALIGN)
27037c478bd9Sstevel@tonic-gate 			align = UMEM_SECOND_ALIGN;
27047c478bd9Sstevel@tonic-gate 		else
27057c478bd9Sstevel@tonic-gate 			align = UMEM_ALIGN;
27067c478bd9Sstevel@tonic-gate 	} else if (align < UMEM_ALIGN && (cflags & UMC_NOTOUCH) == 0)
27077c478bd9Sstevel@tonic-gate 		align = UMEM_ALIGN;
27087c478bd9Sstevel@tonic-gate 
27097c478bd9Sstevel@tonic-gate 
27107c478bd9Sstevel@tonic-gate 	/*
27117c478bd9Sstevel@tonic-gate 	 * Get a umem_cache structure.  We arrange that cp->cache_cpu[]
27127c478bd9Sstevel@tonic-gate 	 * is aligned on a UMEM_CPU_CACHE_SIZE boundary to prevent
27137c478bd9Sstevel@tonic-gate 	 * false sharing of per-CPU data.
27147c478bd9Sstevel@tonic-gate 	 */
27157c478bd9Sstevel@tonic-gate 	cp = vmem_xalloc(umem_cache_arena, csize, UMEM_CPU_CACHE_SIZE, phase,
27167c478bd9Sstevel@tonic-gate 	    0, NULL, NULL, VM_NOSLEEP);
27177c478bd9Sstevel@tonic-gate 
27187c478bd9Sstevel@tonic-gate 	if (cp == NULL) {
27197c478bd9Sstevel@tonic-gate 		errno = EAGAIN;
27207c478bd9Sstevel@tonic-gate 		return (NULL);
27217c478bd9Sstevel@tonic-gate 	}
27227c478bd9Sstevel@tonic-gate 
27237c478bd9Sstevel@tonic-gate 	bzero(cp, csize);
27247c478bd9Sstevel@tonic-gate 
27257c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&umem_flags_lock);
27267c478bd9Sstevel@tonic-gate 	if (umem_flags & UMF_RANDOMIZE)
27277c478bd9Sstevel@tonic-gate 		umem_flags = (((umem_flags | ~UMF_RANDOM) + 1) & UMF_RANDOM) |
27287c478bd9Sstevel@tonic-gate 		    UMF_RANDOMIZE;
27297c478bd9Sstevel@tonic-gate 	cp->cache_flags = umem_flags | (cflags & UMF_DEBUG);
27307c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&umem_flags_lock);
27317c478bd9Sstevel@tonic-gate 
27327c478bd9Sstevel@tonic-gate 	/*
27337c478bd9Sstevel@tonic-gate 	 * Make sure all the various flags are reasonable.
27347c478bd9Sstevel@tonic-gate 	 */
27357c478bd9Sstevel@tonic-gate 	if (cp->cache_flags & UMF_LITE) {
27367c478bd9Sstevel@tonic-gate 		if (bufsize >= umem_lite_minsize &&
27377c478bd9Sstevel@tonic-gate 		    align <= umem_lite_maxalign &&
27387c478bd9Sstevel@tonic-gate 		    P2PHASE(bufsize, umem_lite_maxalign) != 0) {
27397c478bd9Sstevel@tonic-gate 			cp->cache_flags |= UMF_BUFTAG;
27407c478bd9Sstevel@tonic-gate 			cp->cache_flags &= ~(UMF_AUDIT | UMF_FIREWALL);
27417c478bd9Sstevel@tonic-gate 		} else {
27427c478bd9Sstevel@tonic-gate 			cp->cache_flags &= ~UMF_DEBUG;
27437c478bd9Sstevel@tonic-gate 		}
27447c478bd9Sstevel@tonic-gate 	}
27457c478bd9Sstevel@tonic-gate 
27467c478bd9Sstevel@tonic-gate 	if ((cflags & UMC_QCACHE) && (cp->cache_flags & UMF_AUDIT))
27477c478bd9Sstevel@tonic-gate 		cp->cache_flags |= UMF_NOMAGAZINE;
27487c478bd9Sstevel@tonic-gate 
27497c478bd9Sstevel@tonic-gate 	if (cflags & UMC_NODEBUG)
27507c478bd9Sstevel@tonic-gate 		cp->cache_flags &= ~UMF_DEBUG;
27517c478bd9Sstevel@tonic-gate 
27527c478bd9Sstevel@tonic-gate 	if (cflags & UMC_NOTOUCH)
27537c478bd9Sstevel@tonic-gate 		cp->cache_flags &= ~UMF_TOUCH;
27547c478bd9Sstevel@tonic-gate 
27557c478bd9Sstevel@tonic-gate 	if (cflags & UMC_NOHASH)
27567c478bd9Sstevel@tonic-gate 		cp->cache_flags &= ~(UMF_AUDIT | UMF_FIREWALL);
27577c478bd9Sstevel@tonic-gate 
27587c478bd9Sstevel@tonic-gate 	if (cflags & UMC_NOMAGAZINE)
27597c478bd9Sstevel@tonic-gate 		cp->cache_flags |= UMF_NOMAGAZINE;
27607c478bd9Sstevel@tonic-gate 
27617c478bd9Sstevel@tonic-gate 	if ((cp->cache_flags & UMF_AUDIT) && !(cflags & UMC_NOTOUCH))
27627c478bd9Sstevel@tonic-gate 		cp->cache_flags |= UMF_REDZONE;
27637c478bd9Sstevel@tonic-gate 
27647c478bd9Sstevel@tonic-gate 	if ((cp->cache_flags & UMF_BUFTAG) && bufsize >= umem_minfirewall &&
27657c478bd9Sstevel@tonic-gate 	    !(cp->cache_flags & UMF_LITE) && !(cflags & UMC_NOHASH))
27667c478bd9Sstevel@tonic-gate 		cp->cache_flags |= UMF_FIREWALL;
27677c478bd9Sstevel@tonic-gate 
27687c478bd9Sstevel@tonic-gate 	if (vmp != umem_default_arena || umem_firewall_arena == NULL)
27697c478bd9Sstevel@tonic-gate 		cp->cache_flags &= ~UMF_FIREWALL;
27707c478bd9Sstevel@tonic-gate 
27717c478bd9Sstevel@tonic-gate 	if (cp->cache_flags & UMF_FIREWALL) {
27727c478bd9Sstevel@tonic-gate 		cp->cache_flags &= ~UMF_BUFTAG;
27737c478bd9Sstevel@tonic-gate 		cp->cache_flags |= UMF_NOMAGAZINE;
27747c478bd9Sstevel@tonic-gate 		ASSERT(vmp == umem_default_arena);
27757c478bd9Sstevel@tonic-gate 		vmp = umem_firewall_arena;
27767c478bd9Sstevel@tonic-gate 	}
27777c478bd9Sstevel@tonic-gate 
27787c478bd9Sstevel@tonic-gate 	/*
27797c478bd9Sstevel@tonic-gate 	 * Set cache properties.
27807c478bd9Sstevel@tonic-gate 	 */
27817c478bd9Sstevel@tonic-gate 	(void) strncpy(cp->cache_name, name, sizeof (cp->cache_name) - 1);
27827c478bd9Sstevel@tonic-gate 	cp->cache_bufsize = bufsize;
27837c478bd9Sstevel@tonic-gate 	cp->cache_align = align;
27847c478bd9Sstevel@tonic-gate 	cp->cache_constructor = constructor;
27857c478bd9Sstevel@tonic-gate 	cp->cache_destructor = destructor;
27867c478bd9Sstevel@tonic-gate 	cp->cache_reclaim = reclaim;
27877c478bd9Sstevel@tonic-gate 	cp->cache_private = private;
27887c478bd9Sstevel@tonic-gate 	cp->cache_arena = vmp;
27897c478bd9Sstevel@tonic-gate 	cp->cache_cflags = cflags;
27907c478bd9Sstevel@tonic-gate 	cp->cache_cpu_mask = umem_cpu_mask;
27917c478bd9Sstevel@tonic-gate 
27927c478bd9Sstevel@tonic-gate 	/*
27937c478bd9Sstevel@tonic-gate 	 * Determine the chunk size.
27947c478bd9Sstevel@tonic-gate 	 */
27957c478bd9Sstevel@tonic-gate 	chunksize = bufsize;
27967c478bd9Sstevel@tonic-gate 
27977c478bd9Sstevel@tonic-gate 	if (align >= UMEM_ALIGN) {
27987c478bd9Sstevel@tonic-gate 		chunksize = P2ROUNDUP(chunksize, UMEM_ALIGN);
27997c478bd9Sstevel@tonic-gate 		cp->cache_bufctl = chunksize - UMEM_ALIGN;
28007c478bd9Sstevel@tonic-gate 	}
28017c478bd9Sstevel@tonic-gate 
28027c478bd9Sstevel@tonic-gate 	if (cp->cache_flags & UMF_BUFTAG) {
28037c478bd9Sstevel@tonic-gate 		cp->cache_bufctl = chunksize;
28047c478bd9Sstevel@tonic-gate 		cp->cache_buftag = chunksize;
28057c478bd9Sstevel@tonic-gate 		chunksize += sizeof (umem_buftag_t);
28067c478bd9Sstevel@tonic-gate 	}
28077c478bd9Sstevel@tonic-gate 
28087c478bd9Sstevel@tonic-gate 	if (cp->cache_flags & UMF_DEADBEEF) {
28097c478bd9Sstevel@tonic-gate 		cp->cache_verify = MIN(cp->cache_buftag, umem_maxverify);
28107c478bd9Sstevel@tonic-gate 		if (cp->cache_flags & UMF_LITE)
28117c478bd9Sstevel@tonic-gate 			cp->cache_verify = MIN(cp->cache_verify, UMEM_ALIGN);
28127c478bd9Sstevel@tonic-gate 	}
28137c478bd9Sstevel@tonic-gate 
28147c478bd9Sstevel@tonic-gate 	cp->cache_contents = MIN(cp->cache_bufctl, umem_content_maxsave);
28157c478bd9Sstevel@tonic-gate 
28167c478bd9Sstevel@tonic-gate 	cp->cache_chunksize = chunksize = P2ROUNDUP(chunksize, align);
28177c478bd9Sstevel@tonic-gate 
28187c478bd9Sstevel@tonic-gate 	if (chunksize < bufsize) {
28197c478bd9Sstevel@tonic-gate 		errno = ENOMEM;
28207c478bd9Sstevel@tonic-gate 		goto fail;
28217c478bd9Sstevel@tonic-gate 	}
28227c478bd9Sstevel@tonic-gate 
28237c478bd9Sstevel@tonic-gate 	/*
28247c478bd9Sstevel@tonic-gate 	 * Now that we know the chunk size, determine the optimal slab size.
28257c478bd9Sstevel@tonic-gate 	 */
28267c478bd9Sstevel@tonic-gate 	if (vmp == umem_firewall_arena) {
28277c478bd9Sstevel@tonic-gate 		cp->cache_slabsize = P2ROUNDUP(chunksize, vmp->vm_quantum);
28287c478bd9Sstevel@tonic-gate 		cp->cache_mincolor = cp->cache_slabsize - chunksize;
28297c478bd9Sstevel@tonic-gate 		cp->cache_maxcolor = cp->cache_mincolor;
28307c478bd9Sstevel@tonic-gate 		cp->cache_flags |= UMF_HASH;
28317c478bd9Sstevel@tonic-gate 		ASSERT(!(cp->cache_flags & UMF_BUFTAG));
28327c478bd9Sstevel@tonic-gate 	} else if ((cflags & UMC_NOHASH) || (!(cflags & UMC_NOTOUCH) &&
28337c478bd9Sstevel@tonic-gate 	    !(cp->cache_flags & UMF_AUDIT) &&
28347c478bd9Sstevel@tonic-gate 	    chunksize < vmp->vm_quantum / UMEM_VOID_FRACTION)) {
28357c478bd9Sstevel@tonic-gate 		cp->cache_slabsize = vmp->vm_quantum;
28367c478bd9Sstevel@tonic-gate 		cp->cache_mincolor = 0;
28377c478bd9Sstevel@tonic-gate 		cp->cache_maxcolor =
28387c478bd9Sstevel@tonic-gate 		    (cp->cache_slabsize - sizeof (umem_slab_t)) % chunksize;
28397c478bd9Sstevel@tonic-gate 
28407c478bd9Sstevel@tonic-gate 		if (chunksize + sizeof (umem_slab_t) > cp->cache_slabsize) {
28417c478bd9Sstevel@tonic-gate 			errno = EINVAL;
28427c478bd9Sstevel@tonic-gate 			goto fail;
28437c478bd9Sstevel@tonic-gate 		}
28447c478bd9Sstevel@tonic-gate 		ASSERT(!(cp->cache_flags & UMF_AUDIT));
28457c478bd9Sstevel@tonic-gate 	} else {
2846b1e2e3fbSRobert Mustacchi 		size_t chunks, waste, slabsize;
28477c478bd9Sstevel@tonic-gate 		size_t minwaste = LONG_MAX;
2848b1e2e3fbSRobert Mustacchi 		size_t bestfit = SIZE_MAX;
28497c478bd9Sstevel@tonic-gate 
28507c478bd9Sstevel@tonic-gate 		for (chunks = 1; chunks <= UMEM_VOID_FRACTION; chunks++) {
28517c478bd9Sstevel@tonic-gate 			slabsize = P2ROUNDUP(chunksize * chunks,
28527c478bd9Sstevel@tonic-gate 			    vmp->vm_quantum);
28537c478bd9Sstevel@tonic-gate 			/*
28547c478bd9Sstevel@tonic-gate 			 * check for overflow
28557c478bd9Sstevel@tonic-gate 			 */
28567c478bd9Sstevel@tonic-gate 			if ((slabsize / chunks) < chunksize) {
28577c478bd9Sstevel@tonic-gate 				errno = ENOMEM;
28587c478bd9Sstevel@tonic-gate 				goto fail;
28597c478bd9Sstevel@tonic-gate 			}
28607c478bd9Sstevel@tonic-gate 			chunks = slabsize / chunksize;
28617c478bd9Sstevel@tonic-gate 			waste = (slabsize % chunksize) / chunks;
28627c478bd9Sstevel@tonic-gate 			if (waste < minwaste) {
28637c478bd9Sstevel@tonic-gate 				minwaste = waste;
28647c478bd9Sstevel@tonic-gate 				bestfit = slabsize;
28657c478bd9Sstevel@tonic-gate 			}
28667c478bd9Sstevel@tonic-gate 		}
28677c478bd9Sstevel@tonic-gate 		if (cflags & UMC_QCACHE)
28687c478bd9Sstevel@tonic-gate 			bestfit = MAX(1 << highbit(3 * vmp->vm_qcache_max), 64);
2869b1e2e3fbSRobert Mustacchi 		if (bestfit == SIZE_MAX) {
2870b1e2e3fbSRobert Mustacchi 			errno = ENOMEM;
2871b1e2e3fbSRobert Mustacchi 			goto fail;
2872b1e2e3fbSRobert Mustacchi 		}
28737c478bd9Sstevel@tonic-gate 		cp->cache_slabsize = bestfit;
28747c478bd9Sstevel@tonic-gate 		cp->cache_mincolor = 0;
28757c478bd9Sstevel@tonic-gate 		cp->cache_maxcolor = bestfit % chunksize;
28767c478bd9Sstevel@tonic-gate 		cp->cache_flags |= UMF_HASH;
28777c478bd9Sstevel@tonic-gate 	}
28787c478bd9Sstevel@tonic-gate 
28797c478bd9Sstevel@tonic-gate 	if (cp->cache_flags & UMF_HASH) {
28807c478bd9Sstevel@tonic-gate 		ASSERT(!(cflags & UMC_NOHASH));
28817c478bd9Sstevel@tonic-gate 		cp->cache_bufctl_cache = (cp->cache_flags & UMF_AUDIT) ?
28827c478bd9Sstevel@tonic-gate 		    umem_bufctl_audit_cache : umem_bufctl_cache;
28837c478bd9Sstevel@tonic-gate 	}
28847c478bd9Sstevel@tonic-gate 
28857c478bd9Sstevel@tonic-gate 	if (cp->cache_maxcolor >= vmp->vm_quantum)
28867c478bd9Sstevel@tonic-gate 		cp->cache_maxcolor = vmp->vm_quantum - 1;
28877c478bd9Sstevel@tonic-gate 
28887c478bd9Sstevel@tonic-gate 	cp->cache_color = cp->cache_mincolor;
28897c478bd9Sstevel@tonic-gate 
28907c478bd9Sstevel@tonic-gate 	/*
28917c478bd9Sstevel@tonic-gate 	 * Initialize the rest of the slab layer.
28927c478bd9Sstevel@tonic-gate 	 */
28937c478bd9Sstevel@tonic-gate 	(void) mutex_init(&cp->cache_lock, USYNC_THREAD, NULL);
28947c478bd9Sstevel@tonic-gate 
28957c478bd9Sstevel@tonic-gate 	cp->cache_freelist = &cp->cache_nullslab;
28967c478bd9Sstevel@tonic-gate 	cp->cache_nullslab.slab_cache = cp;
28977c478bd9Sstevel@tonic-gate 	cp->cache_nullslab.slab_refcnt = -1;
28987c478bd9Sstevel@tonic-gate 	cp->cache_nullslab.slab_next = &cp->cache_nullslab;
28997c478bd9Sstevel@tonic-gate 	cp->cache_nullslab.slab_prev = &cp->cache_nullslab;
29007c478bd9Sstevel@tonic-gate 
29017c478bd9Sstevel@tonic-gate 	if (cp->cache_flags & UMF_HASH) {
29027c478bd9Sstevel@tonic-gate 		cp->cache_hash_table = vmem_alloc(umem_hash_arena,
29037c478bd9Sstevel@tonic-gate 		    UMEM_HASH_INITIAL * sizeof (void *), VM_NOSLEEP);
29047c478bd9Sstevel@tonic-gate 		if (cp->cache_hash_table == NULL) {
29057c478bd9Sstevel@tonic-gate 			errno = EAGAIN;
29067c478bd9Sstevel@tonic-gate 			goto fail_lock;
29077c478bd9Sstevel@tonic-gate 		}
29087c478bd9Sstevel@tonic-gate 		bzero(cp->cache_hash_table,
29097c478bd9Sstevel@tonic-gate 		    UMEM_HASH_INITIAL * sizeof (void *));
29107c478bd9Sstevel@tonic-gate 		cp->cache_hash_mask = UMEM_HASH_INITIAL - 1;
29117c478bd9Sstevel@tonic-gate 		cp->cache_hash_shift = highbit((ulong_t)chunksize) - 1;
29127c478bd9Sstevel@tonic-gate 	}
29137c478bd9Sstevel@tonic-gate 
29147c478bd9Sstevel@tonic-gate 	/*
29157c478bd9Sstevel@tonic-gate 	 * Initialize the depot.
29167c478bd9Sstevel@tonic-gate 	 */
29177c478bd9Sstevel@tonic-gate 	(void) mutex_init(&cp->cache_depot_lock, USYNC_THREAD, NULL);
29187c478bd9Sstevel@tonic-gate 
29197c478bd9Sstevel@tonic-gate 	for (mtp = umem_magtype; chunksize <= mtp->mt_minbuf; mtp++)
29207c478bd9Sstevel@tonic-gate 		continue;
29217c478bd9Sstevel@tonic-gate 
29227c478bd9Sstevel@tonic-gate 	cp->cache_magtype = mtp;
29237c478bd9Sstevel@tonic-gate 
29247c478bd9Sstevel@tonic-gate 	/*
29257c478bd9Sstevel@tonic-gate 	 * Initialize the CPU layer.
29267c478bd9Sstevel@tonic-gate 	 */
29277c478bd9Sstevel@tonic-gate 	for (cpu_seqid = 0; cpu_seqid < umem_max_ncpus; cpu_seqid++) {
29287c478bd9Sstevel@tonic-gate 		umem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid];
29297c478bd9Sstevel@tonic-gate 		(void) mutex_init(&ccp->cc_lock, USYNC_THREAD, NULL);
29307c478bd9Sstevel@tonic-gate 		ccp->cc_flags = cp->cache_flags;
29317c478bd9Sstevel@tonic-gate 		ccp->cc_rounds = -1;
29327c478bd9Sstevel@tonic-gate 		ccp->cc_prounds = -1;
29337c478bd9Sstevel@tonic-gate 	}
29347c478bd9Sstevel@tonic-gate 
29357c478bd9Sstevel@tonic-gate 	/*
29367c478bd9Sstevel@tonic-gate 	 * Add the cache to the global list.  This makes it visible
29377c478bd9Sstevel@tonic-gate 	 * to umem_update(), so the cache must be ready for business.
29387c478bd9Sstevel@tonic-gate 	 */
29397c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&umem_cache_lock);
29407c478bd9Sstevel@tonic-gate 	cp->cache_next = cnext = &umem_null_cache;
29417c478bd9Sstevel@tonic-gate 	cp->cache_prev = cprev = umem_null_cache.cache_prev;
29427c478bd9Sstevel@tonic-gate 	cnext->cache_prev = cp;
29437c478bd9Sstevel@tonic-gate 	cprev->cache_next = cp;
29447c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&umem_cache_lock);
29457c478bd9Sstevel@tonic-gate 
29467c478bd9Sstevel@tonic-gate 	if (umem_ready == UMEM_READY)
29477c478bd9Sstevel@tonic-gate 		umem_cache_magazine_enable(cp);
29487c478bd9Sstevel@tonic-gate 
29497c478bd9Sstevel@tonic-gate 	return (cp);
29507c478bd9Sstevel@tonic-gate 
29517c478bd9Sstevel@tonic-gate fail_lock:
29527c478bd9Sstevel@tonic-gate 	(void) mutex_destroy(&cp->cache_lock);
29537c478bd9Sstevel@tonic-gate fail:
29547c478bd9Sstevel@tonic-gate 	vmem_xfree(umem_cache_arena, cp, csize);
29557c478bd9Sstevel@tonic-gate 	return (NULL);
29567c478bd9Sstevel@tonic-gate }
29577c478bd9Sstevel@tonic-gate 
29587c478bd9Sstevel@tonic-gate void
umem_cache_destroy(umem_cache_t * cp)29597c478bd9Sstevel@tonic-gate umem_cache_destroy(umem_cache_t *cp)
29607c478bd9Sstevel@tonic-gate {
29617c478bd9Sstevel@tonic-gate 	int cpu_seqid;
29627c478bd9Sstevel@tonic-gate 
29637c478bd9Sstevel@tonic-gate 	/*
29647c478bd9Sstevel@tonic-gate 	 * Remove the cache from the global cache list so that no new updates
29657c478bd9Sstevel@tonic-gate 	 * will be scheduled on its behalf, wait for any pending tasks to
29667c478bd9Sstevel@tonic-gate 	 * complete, purge the cache, and then destroy it.
29677c478bd9Sstevel@tonic-gate 	 */
29687c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&umem_cache_lock);
29697c478bd9Sstevel@tonic-gate 	cp->cache_prev->cache_next = cp->cache_next;
29707c478bd9Sstevel@tonic-gate 	cp->cache_next->cache_prev = cp->cache_prev;
29717c478bd9Sstevel@tonic-gate 	cp->cache_prev = cp->cache_next = NULL;
29727c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&umem_cache_lock);
29737c478bd9Sstevel@tonic-gate 
29747c478bd9Sstevel@tonic-gate 	umem_remove_updates(cp);
29757c478bd9Sstevel@tonic-gate 
29767c478bd9Sstevel@tonic-gate 	umem_cache_magazine_purge(cp);
29777c478bd9Sstevel@tonic-gate 
29787c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&cp->cache_lock);
29797c478bd9Sstevel@tonic-gate 	if (cp->cache_buftotal != 0)
29807c478bd9Sstevel@tonic-gate 		log_message("umem_cache_destroy: '%s' (%p) not empty\n",
29817c478bd9Sstevel@tonic-gate 		    cp->cache_name, (void *)cp);
29827c478bd9Sstevel@tonic-gate 	cp->cache_reclaim = NULL;
29837c478bd9Sstevel@tonic-gate 	/*
29847c478bd9Sstevel@tonic-gate 	 * The cache is now dead.  There should be no further activity.
29857c478bd9Sstevel@tonic-gate 	 * We enforce this by setting land mines in the constructor and
29867c478bd9Sstevel@tonic-gate 	 * destructor routines that induce a segmentation fault if invoked.
29877c478bd9Sstevel@tonic-gate 	 */
29887c478bd9Sstevel@tonic-gate 	cp->cache_constructor = (umem_constructor_t *)1;
29897c478bd9Sstevel@tonic-gate 	cp->cache_destructor = (umem_destructor_t *)2;
29907c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&cp->cache_lock);
29917c478bd9Sstevel@tonic-gate 
29927c478bd9Sstevel@tonic-gate 	if (cp->cache_hash_table != NULL)
29937c478bd9Sstevel@tonic-gate 		vmem_free(umem_hash_arena, cp->cache_hash_table,
29947c478bd9Sstevel@tonic-gate 		    (cp->cache_hash_mask + 1) * sizeof (void *));
29957c478bd9Sstevel@tonic-gate 
29967c478bd9Sstevel@tonic-gate 	for (cpu_seqid = 0; cpu_seqid < umem_max_ncpus; cpu_seqid++)
29977c478bd9Sstevel@tonic-gate 		(void) mutex_destroy(&cp->cache_cpu[cpu_seqid].cc_lock);
29987c478bd9Sstevel@tonic-gate 
29997c478bd9Sstevel@tonic-gate 	(void) mutex_destroy(&cp->cache_depot_lock);
30007c478bd9Sstevel@tonic-gate 	(void) mutex_destroy(&cp->cache_lock);
30017c478bd9Sstevel@tonic-gate 
30027c478bd9Sstevel@tonic-gate 	vmem_free(umem_cache_arena, cp, UMEM_CACHE_SIZE(umem_max_ncpus));
30037c478bd9Sstevel@tonic-gate }
30047c478bd9Sstevel@tonic-gate 
3005789d94c2Sjwadams void
umem_alloc_sizes_clear(void)3006789d94c2Sjwadams umem_alloc_sizes_clear(void)
3007789d94c2Sjwadams {
3008789d94c2Sjwadams 	int i;
3009789d94c2Sjwadams 
3010789d94c2Sjwadams 	umem_alloc_sizes[0] = UMEM_MAXBUF;
3011789d94c2Sjwadams 	for (i = 1; i < NUM_ALLOC_SIZES; i++)
3012789d94c2Sjwadams 		umem_alloc_sizes[i] = 0;
3013789d94c2Sjwadams }
3014789d94c2Sjwadams 
3015789d94c2Sjwadams void
umem_alloc_sizes_add(size_t size_arg)3016789d94c2Sjwadams umem_alloc_sizes_add(size_t size_arg)
3017789d94c2Sjwadams {
3018789d94c2Sjwadams 	int i, j;
3019789d94c2Sjwadams 	size_t size = size_arg;
3020789d94c2Sjwadams 
3021789d94c2Sjwadams 	if (size == 0) {
3022789d94c2Sjwadams 		log_message("size_add: cannot add zero-sized cache\n",
3023789d94c2Sjwadams 		    size, UMEM_MAXBUF);
3024789d94c2Sjwadams 		return;
3025789d94c2Sjwadams 	}
3026789d94c2Sjwadams 
3027789d94c2Sjwadams 	if (size > UMEM_MAXBUF) {
3028789d94c2Sjwadams 		log_message("size_add: %ld > %d, cannot add\n", size,
3029789d94c2Sjwadams 		    UMEM_MAXBUF);
3030789d94c2Sjwadams 		return;
3031789d94c2Sjwadams 	}
3032789d94c2Sjwadams 
3033789d94c2Sjwadams 	if (umem_alloc_sizes[NUM_ALLOC_SIZES - 1] != 0) {
3034789d94c2Sjwadams 		log_message("size_add: no space in alloc_table for %d\n",
3035789d94c2Sjwadams 		    size);
3036789d94c2Sjwadams 		return;
3037789d94c2Sjwadams 	}
3038789d94c2Sjwadams 
3039789d94c2Sjwadams 	if (P2PHASE(size, UMEM_ALIGN) != 0) {
3040789d94c2Sjwadams 		size = P2ROUNDUP(size, UMEM_ALIGN);
3041789d94c2Sjwadams 		log_message("size_add: rounding %d up to %d\n", size_arg,
3042789d94c2Sjwadams 		    size);
3043789d94c2Sjwadams 	}
3044789d94c2Sjwadams 
3045789d94c2Sjwadams 	for (i = 0; i < NUM_ALLOC_SIZES; i++) {
3046789d94c2Sjwadams 		int cur = umem_alloc_sizes[i];
3047789d94c2Sjwadams 		if (cur == size) {
3048789d94c2Sjwadams 			log_message("size_add: %ld already in table\n",
3049789d94c2Sjwadams 			    size);
3050789d94c2Sjwadams 			return;
3051789d94c2Sjwadams 		}
3052789d94c2Sjwadams 		if (cur > size)
3053789d94c2Sjwadams 			break;
3054789d94c2Sjwadams 	}
3055789d94c2Sjwadams 
3056789d94c2Sjwadams 	for (j = NUM_ALLOC_SIZES - 1; j > i; j--)
3057789d94c2Sjwadams 		umem_alloc_sizes[j] = umem_alloc_sizes[j-1];
3058789d94c2Sjwadams 	umem_alloc_sizes[i] = size;
3059789d94c2Sjwadams }
3060789d94c2Sjwadams 
3061789d94c2Sjwadams void
umem_alloc_sizes_remove(size_t size)3062789d94c2Sjwadams umem_alloc_sizes_remove(size_t size)
3063789d94c2Sjwadams {
3064789d94c2Sjwadams 	int i;
3065789d94c2Sjwadams 
3066789d94c2Sjwadams 	if (size == UMEM_MAXBUF) {
3067789d94c2Sjwadams 		log_message("size_remove: cannot remove %ld\n", size);
3068789d94c2Sjwadams 		return;
3069789d94c2Sjwadams 	}
3070789d94c2Sjwadams 
3071789d94c2Sjwadams 	for (i = 0; i < NUM_ALLOC_SIZES; i++) {
3072789d94c2Sjwadams 		int cur = umem_alloc_sizes[i];
3073789d94c2Sjwadams 		if (cur == size)
3074789d94c2Sjwadams 			break;
3075789d94c2Sjwadams 		else if (cur > size || cur == 0) {
3076789d94c2Sjwadams 			log_message("size_remove: %ld not found in table\n",
3077789d94c2Sjwadams 			    size);
3078789d94c2Sjwadams 			return;
3079789d94c2Sjwadams 		}
3080789d94c2Sjwadams 	}
3081789d94c2Sjwadams 
3082789d94c2Sjwadams 	for (; i + 1 < NUM_ALLOC_SIZES; i++)
3083789d94c2Sjwadams 		umem_alloc_sizes[i] = umem_alloc_sizes[i+1];
3084789d94c2Sjwadams 	umem_alloc_sizes[i] = 0;
3085789d94c2Sjwadams }
3086789d94c2Sjwadams 
30874f364e7cSRobert Mustacchi /*
30884f364e7cSRobert Mustacchi  * We've been called back from libc to indicate that thread is terminating and
30894f364e7cSRobert Mustacchi  * that it needs to release the per-thread memory that it has. We get to know
30904f364e7cSRobert Mustacchi  * which entry in the thread's tmem array the allocation came from. Currently
30914f364e7cSRobert Mustacchi  * this refers to first n umem_caches which makes this a pretty simple indexing
30924f364e7cSRobert Mustacchi  * job.
30934f364e7cSRobert Mustacchi  */
30944f364e7cSRobert Mustacchi static void
umem_cache_tmem_cleanup(void * buf,int entry)30954f364e7cSRobert Mustacchi umem_cache_tmem_cleanup(void *buf, int entry)
30964f364e7cSRobert Mustacchi {
30974f364e7cSRobert Mustacchi 	size_t size;
30984f364e7cSRobert Mustacchi 	umem_cache_t *cp;
30994f364e7cSRobert Mustacchi 
31004f364e7cSRobert Mustacchi 	size = umem_alloc_sizes[entry];
31014f364e7cSRobert Mustacchi 	cp = umem_alloc_table[(size - 1) >> UMEM_ALIGN_SHIFT];
31024f364e7cSRobert Mustacchi 	_umem_cache_free(cp, buf);
31034f364e7cSRobert Mustacchi }
31044f364e7cSRobert Mustacchi 
31057c478bd9Sstevel@tonic-gate static int
umem_cache_init(void)31067c478bd9Sstevel@tonic-gate umem_cache_init(void)
31077c478bd9Sstevel@tonic-gate {
31087c478bd9Sstevel@tonic-gate 	int i;
31097c478bd9Sstevel@tonic-gate 	size_t size, max_size;
31107c478bd9Sstevel@tonic-gate 	umem_cache_t *cp;
31117c478bd9Sstevel@tonic-gate 	umem_magtype_t *mtp;
31127c478bd9Sstevel@tonic-gate 	char name[UMEM_CACHE_NAMELEN + 1];
31137c478bd9Sstevel@tonic-gate 	umem_cache_t *umem_alloc_caches[NUM_ALLOC_SIZES];
31147c478bd9Sstevel@tonic-gate 
31157c478bd9Sstevel@tonic-gate 	for (i = 0; i < sizeof (umem_magtype) / sizeof (*mtp); i++) {
31167c478bd9Sstevel@tonic-gate 		mtp = &umem_magtype[i];
31177c478bd9Sstevel@tonic-gate 		(void) snprintf(name, sizeof (name), "umem_magazine_%d",
31187c478bd9Sstevel@tonic-gate 		    mtp->mt_magsize);
31197c478bd9Sstevel@tonic-gate 		mtp->mt_cache = umem_cache_create(name,
31207c478bd9Sstevel@tonic-gate 		    (mtp->mt_magsize + 1) * sizeof (void *),
31217c478bd9Sstevel@tonic-gate 		    mtp->mt_align, NULL, NULL, NULL, NULL,
31227c478bd9Sstevel@tonic-gate 		    umem_internal_arena, UMC_NOHASH | UMC_INTERNAL);
31237c478bd9Sstevel@tonic-gate 		if (mtp->mt_cache == NULL)
31247c478bd9Sstevel@tonic-gate 			return (0);
31257c478bd9Sstevel@tonic-gate 	}
31267c478bd9Sstevel@tonic-gate 
31277c478bd9Sstevel@tonic-gate 	umem_slab_cache = umem_cache_create("umem_slab_cache",
31287c478bd9Sstevel@tonic-gate 	    sizeof (umem_slab_t), 0, NULL, NULL, NULL, NULL,
31297c478bd9Sstevel@tonic-gate 	    umem_internal_arena, UMC_NOHASH | UMC_INTERNAL);
31307c478bd9Sstevel@tonic-gate 
31317c478bd9Sstevel@tonic-gate 	if (umem_slab_cache == NULL)
31327c478bd9Sstevel@tonic-gate 		return (0);
31337c478bd9Sstevel@tonic-gate 
31347c478bd9Sstevel@tonic-gate 	umem_bufctl_cache = umem_cache_create("umem_bufctl_cache",
31357c478bd9Sstevel@tonic-gate 	    sizeof (umem_bufctl_t), 0, NULL, NULL, NULL, NULL,
31367c478bd9Sstevel@tonic-gate 	    umem_internal_arena, UMC_NOHASH | UMC_INTERNAL);
31377c478bd9Sstevel@tonic-gate 
31387c478bd9Sstevel@tonic-gate 	if (umem_bufctl_cache == NULL)
31397c478bd9Sstevel@tonic-gate 		return (0);
31407c478bd9Sstevel@tonic-gate 
31417c478bd9Sstevel@tonic-gate 	/*
31427c478bd9Sstevel@tonic-gate 	 * The size of the umem_bufctl_audit structure depends upon
31437c478bd9Sstevel@tonic-gate 	 * umem_stack_depth.   See umem_impl.h for details on the size
31447c478bd9Sstevel@tonic-gate 	 * restrictions.
31457c478bd9Sstevel@tonic-gate 	 */
31467c478bd9Sstevel@tonic-gate 
31477c478bd9Sstevel@tonic-gate 	size = UMEM_BUFCTL_AUDIT_SIZE_DEPTH(umem_stack_depth);
31487c478bd9Sstevel@tonic-gate 	max_size = UMEM_BUFCTL_AUDIT_MAX_SIZE;
31497c478bd9Sstevel@tonic-gate 
31507c478bd9Sstevel@tonic-gate 	if (size > max_size) {			/* too large -- truncate */
31517c478bd9Sstevel@tonic-gate 		int max_frames = UMEM_MAX_STACK_DEPTH;
31527c478bd9Sstevel@tonic-gate 
31537c478bd9Sstevel@tonic-gate 		ASSERT(UMEM_BUFCTL_AUDIT_SIZE_DEPTH(max_frames) <= max_size);
31547c478bd9Sstevel@tonic-gate 
31557c478bd9Sstevel@tonic-gate 		umem_stack_depth = max_frames;
31567c478bd9Sstevel@tonic-gate 		size = UMEM_BUFCTL_AUDIT_SIZE_DEPTH(umem_stack_depth);
31577c478bd9Sstevel@tonic-gate 	}
31587c478bd9Sstevel@tonic-gate 
31597c478bd9Sstevel@tonic-gate 	umem_bufctl_audit_cache = umem_cache_create("umem_bufctl_audit_cache",
31607c478bd9Sstevel@tonic-gate 	    size, 0, NULL, NULL, NULL, NULL, umem_internal_arena,
31617c478bd9Sstevel@tonic-gate 	    UMC_NOHASH | UMC_INTERNAL);
31627c478bd9Sstevel@tonic-gate 
31637c478bd9Sstevel@tonic-gate 	if (umem_bufctl_audit_cache == NULL)
31647c478bd9Sstevel@tonic-gate 		return (0);
31657c478bd9Sstevel@tonic-gate 
31667c478bd9Sstevel@tonic-gate 	if (vmem_backend & VMEM_BACKEND_MMAP)
31677c478bd9Sstevel@tonic-gate 		umem_va_arena = vmem_create("umem_va",
31687c478bd9Sstevel@tonic-gate 		    NULL, 0, pagesize,
31697c478bd9Sstevel@tonic-gate 		    vmem_alloc, vmem_free, heap_arena,
31707c478bd9Sstevel@tonic-gate 		    8 * pagesize, VM_NOSLEEP);
31717c478bd9Sstevel@tonic-gate 	else
31727c478bd9Sstevel@tonic-gate 		umem_va_arena = heap_arena;
31737c478bd9Sstevel@tonic-gate 
31747c478bd9Sstevel@tonic-gate 	if (umem_va_arena == NULL)
31757c478bd9Sstevel@tonic-gate 		return (0);
31767c478bd9Sstevel@tonic-gate 
31777c478bd9Sstevel@tonic-gate 	umem_default_arena = vmem_create("umem_default",
31787c478bd9Sstevel@tonic-gate 	    NULL, 0, pagesize,
31797c478bd9Sstevel@tonic-gate 	    heap_alloc, heap_free, umem_va_arena,
31807c478bd9Sstevel@tonic-gate 	    0, VM_NOSLEEP);
31817c478bd9Sstevel@tonic-gate 
31827c478bd9Sstevel@tonic-gate 	if (umem_default_arena == NULL)
31837c478bd9Sstevel@tonic-gate 		return (0);
31847c478bd9Sstevel@tonic-gate 
31857c478bd9Sstevel@tonic-gate 	/*
31867c478bd9Sstevel@tonic-gate 	 * make sure the umem_alloc table initializer is correct
31877c478bd9Sstevel@tonic-gate 	 */
31887c478bd9Sstevel@tonic-gate 	i = sizeof (umem_alloc_table) / sizeof (*umem_alloc_table);
31897c478bd9Sstevel@tonic-gate 	ASSERT(umem_alloc_table[i - 1] == &umem_null_cache);
31907c478bd9Sstevel@tonic-gate 
31917c478bd9Sstevel@tonic-gate 	/*
31927c478bd9Sstevel@tonic-gate 	 * Create the default caches to back umem_alloc()
31937c478bd9Sstevel@tonic-gate 	 */
31947c478bd9Sstevel@tonic-gate 	for (i = 0; i < NUM_ALLOC_SIZES; i++) {
31957c478bd9Sstevel@tonic-gate 		size_t cache_size = umem_alloc_sizes[i];
31967c478bd9Sstevel@tonic-gate 		size_t align = 0;
3197789d94c2Sjwadams 
3198789d94c2Sjwadams 		if (cache_size == 0)
3199789d94c2Sjwadams 			break;		/* 0 terminates the list */
3200789d94c2Sjwadams 
32017c478bd9Sstevel@tonic-gate 		/*
32027c478bd9Sstevel@tonic-gate 		 * If they allocate a multiple of the coherency granularity,
32037c478bd9Sstevel@tonic-gate 		 * they get a coherency-granularity-aligned address.
32047c478bd9Sstevel@tonic-gate 		 */
32057c478bd9Sstevel@tonic-gate 		if (IS_P2ALIGNED(cache_size, 64))
32067c478bd9Sstevel@tonic-gate 			align = 64;
32077c478bd9Sstevel@tonic-gate 		if (IS_P2ALIGNED(cache_size, pagesize))
32087c478bd9Sstevel@tonic-gate 			align = pagesize;
32097c478bd9Sstevel@tonic-gate 		(void) snprintf(name, sizeof (name), "umem_alloc_%lu",
32107c478bd9Sstevel@tonic-gate 		    (long)cache_size);
32117c478bd9Sstevel@tonic-gate 
32127c478bd9Sstevel@tonic-gate 		cp = umem_cache_create(name, cache_size, align,
32137c478bd9Sstevel@tonic-gate 		    NULL, NULL, NULL, NULL, NULL, UMC_INTERNAL);
32147c478bd9Sstevel@tonic-gate 		if (cp == NULL)
32157c478bd9Sstevel@tonic-gate 			return (0);
32167c478bd9Sstevel@tonic-gate 
32177c478bd9Sstevel@tonic-gate 		umem_alloc_caches[i] = cp;
32187c478bd9Sstevel@tonic-gate 	}
32197c478bd9Sstevel@tonic-gate 
32204f364e7cSRobert Mustacchi 	umem_tmem_off = _tmem_get_base();
32214f364e7cSRobert Mustacchi 	_tmem_set_cleanup(umem_cache_tmem_cleanup);
32224f364e7cSRobert Mustacchi 
3223b1e2e3fbSRobert Mustacchi #ifndef	UMEM_STANDALONE
32244f364e7cSRobert Mustacchi 	if (umem_genasm_supported && !(umem_flags & UMF_DEBUG) &&
32254f364e7cSRobert Mustacchi 	    !(umem_flags & UMF_NOMAGAZINE) &&
32264f364e7cSRobert Mustacchi 	    umem_ptc_size > 0) {
32274f364e7cSRobert Mustacchi 		umem_ptc_enabled = umem_genasm(umem_alloc_sizes,
3228b1e2e3fbSRobert Mustacchi 		    umem_alloc_caches, i) ? 1 : 0;
32294f364e7cSRobert Mustacchi 	}
3230b1e2e3fbSRobert Mustacchi #else
3231b1e2e3fbSRobert Mustacchi 	umem_ptc_enabled = 0;
3232b1e2e3fbSRobert Mustacchi #endif
32334f364e7cSRobert Mustacchi 
32347c478bd9Sstevel@tonic-gate 	/*
32357c478bd9Sstevel@tonic-gate 	 * Initialization cannot fail at this point.  Make the caches
32367c478bd9Sstevel@tonic-gate 	 * visible to umem_alloc() and friends.
32377c478bd9Sstevel@tonic-gate 	 */
32387c478bd9Sstevel@tonic-gate 	size = UMEM_ALIGN;
32397c478bd9Sstevel@tonic-gate 	for (i = 0; i < NUM_ALLOC_SIZES; i++) {
32407c478bd9Sstevel@tonic-gate 		size_t cache_size = umem_alloc_sizes[i];
32417c478bd9Sstevel@tonic-gate 
3242789d94c2Sjwadams 		if (cache_size == 0)
3243789d94c2Sjwadams 			break;		/* 0 terminates the list */
3244789d94c2Sjwadams 
32457c478bd9Sstevel@tonic-gate 		cp = umem_alloc_caches[i];
32467c478bd9Sstevel@tonic-gate 
32477c478bd9Sstevel@tonic-gate 		while (size <= cache_size) {
32487c478bd9Sstevel@tonic-gate 			umem_alloc_table[(size - 1) >> UMEM_ALIGN_SHIFT] = cp;
32497c478bd9Sstevel@tonic-gate 			size += UMEM_ALIGN;
32507c478bd9Sstevel@tonic-gate 		}
32517c478bd9Sstevel@tonic-gate 	}
3252789d94c2Sjwadams 	ASSERT(size - UMEM_ALIGN == UMEM_MAXBUF);
32537c478bd9Sstevel@tonic-gate 	return (1);
32547c478bd9Sstevel@tonic-gate }
32557c478bd9Sstevel@tonic-gate 
32567c478bd9Sstevel@tonic-gate /*
32577c478bd9Sstevel@tonic-gate  * umem_startup() is called early on, and must be called explicitly if we're
32587c478bd9Sstevel@tonic-gate  * the standalone version.
32597c478bd9Sstevel@tonic-gate  */
32607c478bd9Sstevel@tonic-gate #ifdef UMEM_STANDALONE
32617c478bd9Sstevel@tonic-gate void
32627c478bd9Sstevel@tonic-gate #else
32637c478bd9Sstevel@tonic-gate #pragma init(umem_startup)
32647c478bd9Sstevel@tonic-gate static void
32657c478bd9Sstevel@tonic-gate #endif
umem_startup(caddr_t start,size_t len,size_t pagesize,caddr_t minstack,caddr_t maxstack)32667c478bd9Sstevel@tonic-gate umem_startup(caddr_t start, size_t len, size_t pagesize, caddr_t minstack,
32677c478bd9Sstevel@tonic-gate     caddr_t maxstack)
32687c478bd9Sstevel@tonic-gate {
32697c478bd9Sstevel@tonic-gate #ifdef UMEM_STANDALONE
32707c478bd9Sstevel@tonic-gate 	int idx;
32717c478bd9Sstevel@tonic-gate 	/* Standalone doesn't fork */
32727c478bd9Sstevel@tonic-gate #else
32737c478bd9Sstevel@tonic-gate 	umem_forkhandler_init(); /* register the fork handler */
32747c478bd9Sstevel@tonic-gate #endif
32757c478bd9Sstevel@tonic-gate 
32767c478bd9Sstevel@tonic-gate #ifdef __lint
32777c478bd9Sstevel@tonic-gate 	/* make lint happy */
32787c478bd9Sstevel@tonic-gate 	minstack = maxstack;
32797c478bd9Sstevel@tonic-gate #endif
32807c478bd9Sstevel@tonic-gate 
32817c478bd9Sstevel@tonic-gate #ifdef UMEM_STANDALONE
32827c478bd9Sstevel@tonic-gate 	umem_ready = UMEM_READY_STARTUP;
32837c478bd9Sstevel@tonic-gate 	umem_init_env_ready = 0;
32847c478bd9Sstevel@tonic-gate 
32857c478bd9Sstevel@tonic-gate 	umem_min_stack = minstack;
32867c478bd9Sstevel@tonic-gate 	umem_max_stack = maxstack;
32877c478bd9Sstevel@tonic-gate 
32887c478bd9Sstevel@tonic-gate 	nofail_callback = NULL;
32897c478bd9Sstevel@tonic-gate 	umem_slab_cache = NULL;
32907c478bd9Sstevel@tonic-gate 	umem_bufctl_cache = NULL;
32917c478bd9Sstevel@tonic-gate 	umem_bufctl_audit_cache = NULL;
32927c478bd9Sstevel@tonic-gate 	heap_arena = NULL;
32937c478bd9Sstevel@tonic-gate 	heap_alloc = NULL;
32947c478bd9Sstevel@tonic-gate 	heap_free = NULL;
32957c478bd9Sstevel@tonic-gate 	umem_internal_arena = NULL;
32967c478bd9Sstevel@tonic-gate 	umem_cache_arena = NULL;
32977c478bd9Sstevel@tonic-gate 	umem_hash_arena = NULL;
32987c478bd9Sstevel@tonic-gate 	umem_log_arena = NULL;
32997c478bd9Sstevel@tonic-gate 	umem_oversize_arena = NULL;
33007c478bd9Sstevel@tonic-gate 	umem_va_arena = NULL;
33017c478bd9Sstevel@tonic-gate 	umem_default_arena = NULL;
33027c478bd9Sstevel@tonic-gate 	umem_firewall_va_arena = NULL;
33037c478bd9Sstevel@tonic-gate 	umem_firewall_arena = NULL;
33047c478bd9Sstevel@tonic-gate 	umem_memalign_arena = NULL;
33057c478bd9Sstevel@tonic-gate 	umem_transaction_log = NULL;
33067c478bd9Sstevel@tonic-gate 	umem_content_log = NULL;
33077c478bd9Sstevel@tonic-gate 	umem_failure_log = NULL;
33087c478bd9Sstevel@tonic-gate 	umem_slab_log = NULL;
33097c478bd9Sstevel@tonic-gate 	umem_cpu_mask = 0;
33107c478bd9Sstevel@tonic-gate 
33117c478bd9Sstevel@tonic-gate 	umem_cpus = &umem_startup_cpu;
33127c478bd9Sstevel@tonic-gate 	umem_startup_cpu.cpu_cache_offset = UMEM_CACHE_SIZE(0);
33137c478bd9Sstevel@tonic-gate 	umem_startup_cpu.cpu_number = 0;
33147c478bd9Sstevel@tonic-gate 
33157c478bd9Sstevel@tonic-gate 	bcopy(&umem_null_cache_template, &umem_null_cache,
33167c478bd9Sstevel@tonic-gate 	    sizeof (umem_cache_t));
33177c478bd9Sstevel@tonic-gate 
33187c478bd9Sstevel@tonic-gate 	for (idx = 0; idx < (UMEM_MAXBUF >> UMEM_ALIGN_SHIFT); idx++)
33197c478bd9Sstevel@tonic-gate 		umem_alloc_table[idx] = &umem_null_cache;
33207c478bd9Sstevel@tonic-gate #endif
33217c478bd9Sstevel@tonic-gate 
33227c478bd9Sstevel@tonic-gate 	/*
33237c478bd9Sstevel@tonic-gate 	 * Perform initialization specific to the way we've been compiled
33247c478bd9Sstevel@tonic-gate 	 * (library or standalone)
33257c478bd9Sstevel@tonic-gate 	 */
33267c478bd9Sstevel@tonic-gate 	umem_type_init(start, len, pagesize);
33277c478bd9Sstevel@tonic-gate 
33287c478bd9Sstevel@tonic-gate 	vmem_startup();
33297c478bd9Sstevel@tonic-gate }
33307c478bd9Sstevel@tonic-gate 
33317c478bd9Sstevel@tonic-gate int
umem_init(void)33327c478bd9Sstevel@tonic-gate umem_init(void)
33337c478bd9Sstevel@tonic-gate {
33347c478bd9Sstevel@tonic-gate 	size_t maxverify, minfirewall;
33357c478bd9Sstevel@tonic-gate 	size_t size;
33367c478bd9Sstevel@tonic-gate 	int idx;
33377c478bd9Sstevel@tonic-gate 	umem_cpu_t *new_cpus;
33387c478bd9Sstevel@tonic-gate 
33397c478bd9Sstevel@tonic-gate 	vmem_t *memalign_arena, *oversize_arena;
33407c478bd9Sstevel@tonic-gate 
33417c478bd9Sstevel@tonic-gate 	if (thr_self() != umem_init_thr) {
33427c478bd9Sstevel@tonic-gate 		/*
33437c478bd9Sstevel@tonic-gate 		 * The usual case -- non-recursive invocation of umem_init().
33447c478bd9Sstevel@tonic-gate 		 */
33457c478bd9Sstevel@tonic-gate 		(void) mutex_lock(&umem_init_lock);
33467c478bd9Sstevel@tonic-gate 		if (umem_ready != UMEM_READY_STARTUP) {
33477c478bd9Sstevel@tonic-gate 			/*
33487c478bd9Sstevel@tonic-gate 			 * someone else beat us to initializing umem.  Wait
33497c478bd9Sstevel@tonic-gate 			 * for them to complete, then return.
33507c478bd9Sstevel@tonic-gate 			 */
3351a574db85Sraf 			while (umem_ready == UMEM_READY_INITING) {
3352a574db85Sraf 				int cancel_state;
3353a574db85Sraf 
3354a574db85Sraf 				(void) pthread_setcancelstate(
3355a574db85Sraf 				    PTHREAD_CANCEL_DISABLE, &cancel_state);
3356a574db85Sraf 				(void) cond_wait(&umem_init_cv,
33577c478bd9Sstevel@tonic-gate 				    &umem_init_lock);
3358a574db85Sraf 				(void) pthread_setcancelstate(
3359a574db85Sraf 				    cancel_state, NULL);
3360a574db85Sraf 			}
33617c478bd9Sstevel@tonic-gate 			ASSERT(umem_ready == UMEM_READY ||
33627c478bd9Sstevel@tonic-gate 			    umem_ready == UMEM_READY_INIT_FAILED);
33637c478bd9Sstevel@tonic-gate 			(void) mutex_unlock(&umem_init_lock);
33647c478bd9Sstevel@tonic-gate 			return (umem_ready == UMEM_READY);
33657c478bd9Sstevel@tonic-gate 		}
33667c478bd9Sstevel@tonic-gate 
33677c478bd9Sstevel@tonic-gate 		ASSERT(umem_ready == UMEM_READY_STARTUP);
33687c478bd9Sstevel@tonic-gate 		ASSERT(umem_init_env_ready == 0);
33697c478bd9Sstevel@tonic-gate 
33707c478bd9Sstevel@tonic-gate 		umem_ready = UMEM_READY_INITING;
33717c478bd9Sstevel@tonic-gate 		umem_init_thr = thr_self();
33727c478bd9Sstevel@tonic-gate 
33737c478bd9Sstevel@tonic-gate 		(void) mutex_unlock(&umem_init_lock);
33747c478bd9Sstevel@tonic-gate 		umem_setup_envvars(0);		/* can recurse -- see below */
33757c478bd9Sstevel@tonic-gate 		if (umem_init_env_ready) {
33767c478bd9Sstevel@tonic-gate 			/*
33777c478bd9Sstevel@tonic-gate 			 * initialization was completed already
33787c478bd9Sstevel@tonic-gate 			 */
33797c478bd9Sstevel@tonic-gate 			ASSERT(umem_ready == UMEM_READY ||
33807c478bd9Sstevel@tonic-gate 			    umem_ready == UMEM_READY_INIT_FAILED);
33817c478bd9Sstevel@tonic-gate 			ASSERT(umem_init_thr == 0);
33827c478bd9Sstevel@tonic-gate 			return (umem_ready == UMEM_READY);
33837c478bd9Sstevel@tonic-gate 		}
33847c478bd9Sstevel@tonic-gate 	} else if (!umem_init_env_ready) {
33857c478bd9Sstevel@tonic-gate 		/*
33867c478bd9Sstevel@tonic-gate 		 * The umem_setup_envvars() call (above) makes calls into
33877c478bd9Sstevel@tonic-gate 		 * the dynamic linker and directly into user-supplied code.
33887c478bd9Sstevel@tonic-gate 		 * Since we cannot know what that code will do, we could be
33897c478bd9Sstevel@tonic-gate 		 * recursively invoked (by, say, a malloc() call in the code
33907c478bd9Sstevel@tonic-gate 		 * itself, or in a (C++) _init section it causes to be fired).
33917c478bd9Sstevel@tonic-gate 		 *
33927c478bd9Sstevel@tonic-gate 		 * This code is where we end up if such recursion occurs.  We
33937c478bd9Sstevel@tonic-gate 		 * first clean up any partial results in the envvar code, then
33947c478bd9Sstevel@tonic-gate 		 * proceed to finish initialization processing in the recursive
33957c478bd9Sstevel@tonic-gate 		 * call.  The original call will notice this, and return
33967c478bd9Sstevel@tonic-gate 		 * immediately.
33977c478bd9Sstevel@tonic-gate 		 */
33987c478bd9Sstevel@tonic-gate 		umem_setup_envvars(1);		/* clean up any partial state */
33997c478bd9Sstevel@tonic-gate 	} else {
34007c478bd9Sstevel@tonic-gate 		umem_panic(
34017c478bd9Sstevel@tonic-gate 		    "recursive allocation while initializing umem\n");
34027c478bd9Sstevel@tonic-gate 	}
34037c478bd9Sstevel@tonic-gate 	umem_init_env_ready = 1;
34047c478bd9Sstevel@tonic-gate 
34057c478bd9Sstevel@tonic-gate 	/*
34067c478bd9Sstevel@tonic-gate 	 * From this point until we finish, recursion into umem_init() will
34077c478bd9Sstevel@tonic-gate 	 * cause a umem_panic().
34087c478bd9Sstevel@tonic-gate 	 */
34097c478bd9Sstevel@tonic-gate 	maxverify = minfirewall = ULONG_MAX;
34107c478bd9Sstevel@tonic-gate 
34117c478bd9Sstevel@tonic-gate 	/* LINTED constant condition */
34127c478bd9Sstevel@tonic-gate 	if (sizeof (umem_cpu_cache_t) != UMEM_CPU_CACHE_SIZE) {
34137c478bd9Sstevel@tonic-gate 		umem_panic("sizeof (umem_cpu_cache_t) = %d, should be %d\n",
34147c478bd9Sstevel@tonic-gate 		    sizeof (umem_cpu_cache_t), UMEM_CPU_CACHE_SIZE);
34157c478bd9Sstevel@tonic-gate 	}
34167c478bd9Sstevel@tonic-gate 
34177c478bd9Sstevel@tonic-gate 	umem_max_ncpus = umem_get_max_ncpus();
34187c478bd9Sstevel@tonic-gate 
34197c478bd9Sstevel@tonic-gate 	/*
34207c478bd9Sstevel@tonic-gate 	 * load tunables from environment
34217c478bd9Sstevel@tonic-gate 	 */
34227c478bd9Sstevel@tonic-gate 	umem_process_envvars();
34237c478bd9Sstevel@tonic-gate 
34247c478bd9Sstevel@tonic-gate 	if (issetugid())
34257c478bd9Sstevel@tonic-gate 		umem_mtbf = 0;
34267c478bd9Sstevel@tonic-gate 
34277c478bd9Sstevel@tonic-gate 	/*
34287c478bd9Sstevel@tonic-gate 	 * set up vmem
34297c478bd9Sstevel@tonic-gate 	 */
34307c478bd9Sstevel@tonic-gate 	if (!(umem_flags & UMF_AUDIT))
34317c478bd9Sstevel@tonic-gate 		vmem_no_debug();
34327c478bd9Sstevel@tonic-gate 
34337c478bd9Sstevel@tonic-gate 	heap_arena = vmem_heap_arena(&heap_alloc, &heap_free);
34347c478bd9Sstevel@tonic-gate 
34357c478bd9Sstevel@tonic-gate 	pagesize = heap_arena->vm_quantum;
34367c478bd9Sstevel@tonic-gate 
34377c478bd9Sstevel@tonic-gate 	umem_internal_arena = vmem_create("umem_internal", NULL, 0, pagesize,
34387c478bd9Sstevel@tonic-gate 	    heap_alloc, heap_free, heap_arena, 0, VM_NOSLEEP);
34397c478bd9Sstevel@tonic-gate 
34407c478bd9Sstevel@tonic-gate 	umem_default_arena = umem_internal_arena;
34417c478bd9Sstevel@tonic-gate 
34427c478bd9Sstevel@tonic-gate 	if (umem_internal_arena == NULL)
34437c478bd9Sstevel@tonic-gate 		goto fail;
34447c478bd9Sstevel@tonic-gate 
34457c478bd9Sstevel@tonic-gate 	umem_cache_arena = vmem_create("umem_cache", NULL, 0, UMEM_ALIGN,
34467c478bd9Sstevel@tonic-gate 	    vmem_alloc, vmem_free, umem_internal_arena, 0, VM_NOSLEEP);
34477c478bd9Sstevel@tonic-gate 
34487c478bd9Sstevel@tonic-gate 	umem_hash_arena = vmem_create("umem_hash", NULL, 0, UMEM_ALIGN,
34497c478bd9Sstevel@tonic-gate 	    vmem_alloc, vmem_free, umem_internal_arena, 0, VM_NOSLEEP);
34507c478bd9Sstevel@tonic-gate 
34517c478bd9Sstevel@tonic-gate 	umem_log_arena = vmem_create("umem_log", NULL, 0, UMEM_ALIGN,
34527c478bd9Sstevel@tonic-gate 	    heap_alloc, heap_free, heap_arena, 0, VM_NOSLEEP);
34537c478bd9Sstevel@tonic-gate 
34547c478bd9Sstevel@tonic-gate 	umem_firewall_va_arena = vmem_create("umem_firewall_va",
34557c478bd9Sstevel@tonic-gate 	    NULL, 0, pagesize,
34567c478bd9Sstevel@tonic-gate 	    umem_firewall_va_alloc, umem_firewall_va_free, heap_arena,
34577c478bd9Sstevel@tonic-gate 	    0, VM_NOSLEEP);
34587c478bd9Sstevel@tonic-gate 
34597c478bd9Sstevel@tonic-gate 	if (umem_cache_arena == NULL || umem_hash_arena == NULL ||
34607c478bd9Sstevel@tonic-gate 	    umem_log_arena == NULL || umem_firewall_va_arena == NULL)
34617c478bd9Sstevel@tonic-gate 		goto fail;
34627c478bd9Sstevel@tonic-gate 
34637c478bd9Sstevel@tonic-gate 	umem_firewall_arena = vmem_create("umem_firewall", NULL, 0, pagesize,
34647c478bd9Sstevel@tonic-gate 	    heap_alloc, heap_free, umem_firewall_va_arena, 0,
34657c478bd9Sstevel@tonic-gate 	    VM_NOSLEEP);
34667c478bd9Sstevel@tonic-gate 
34677c478bd9Sstevel@tonic-gate 	if (umem_firewall_arena == NULL)
34687c478bd9Sstevel@tonic-gate 		goto fail;
34697c478bd9Sstevel@tonic-gate 
34707c478bd9Sstevel@tonic-gate 	oversize_arena = vmem_create("umem_oversize", NULL, 0, pagesize,
34717c478bd9Sstevel@tonic-gate 	    heap_alloc, heap_free, minfirewall < ULONG_MAX ?
34727c478bd9Sstevel@tonic-gate 	    umem_firewall_va_arena : heap_arena, 0, VM_NOSLEEP);
34737c478bd9Sstevel@tonic-gate 
34747c478bd9Sstevel@tonic-gate 	memalign_arena = vmem_create("umem_memalign", NULL, 0, UMEM_ALIGN,
34757c478bd9Sstevel@tonic-gate 	    heap_alloc, heap_free, minfirewall < ULONG_MAX ?
34767c478bd9Sstevel@tonic-gate 	    umem_firewall_va_arena : heap_arena, 0, VM_NOSLEEP);
34777c478bd9Sstevel@tonic-gate 
34787c478bd9Sstevel@tonic-gate 	if (oversize_arena == NULL || memalign_arena == NULL)
34797c478bd9Sstevel@tonic-gate 		goto fail;
34807c478bd9Sstevel@tonic-gate 
34817c478bd9Sstevel@tonic-gate 	if (umem_max_ncpus > CPUHINT_MAX())
34827c478bd9Sstevel@tonic-gate 		umem_max_ncpus = CPUHINT_MAX();
34837c478bd9Sstevel@tonic-gate 
34847c478bd9Sstevel@tonic-gate 	while ((umem_max_ncpus & (umem_max_ncpus - 1)) != 0)
34857c478bd9Sstevel@tonic-gate 		umem_max_ncpus++;
34867c478bd9Sstevel@tonic-gate 
34877c478bd9Sstevel@tonic-gate 	if (umem_max_ncpus == 0)
34887c478bd9Sstevel@tonic-gate 		umem_max_ncpus = 1;
34897c478bd9Sstevel@tonic-gate 
34907c478bd9Sstevel@tonic-gate 	size = umem_max_ncpus * sizeof (umem_cpu_t);
34917c478bd9Sstevel@tonic-gate 	new_cpus = vmem_alloc(umem_internal_arena, size, VM_NOSLEEP);
34927c478bd9Sstevel@tonic-gate 	if (new_cpus == NULL)
34937c478bd9Sstevel@tonic-gate 		goto fail;
34947c478bd9Sstevel@tonic-gate 
34957c478bd9Sstevel@tonic-gate 	bzero(new_cpus, size);
34967c478bd9Sstevel@tonic-gate 	for (idx = 0; idx < umem_max_ncpus; idx++) {
34977c478bd9Sstevel@tonic-gate 		new_cpus[idx].cpu_number = idx;
34987c478bd9Sstevel@tonic-gate 		new_cpus[idx].cpu_cache_offset = UMEM_CACHE_SIZE(idx);
34997c478bd9Sstevel@tonic-gate 	}
35007c478bd9Sstevel@tonic-gate 	umem_cpus = new_cpus;
35017c478bd9Sstevel@tonic-gate 	umem_cpu_mask = (umem_max_ncpus - 1);
35027c478bd9Sstevel@tonic-gate 
35037c478bd9Sstevel@tonic-gate 	if (umem_maxverify == 0)
35047c478bd9Sstevel@tonic-gate 		umem_maxverify = maxverify;
35057c478bd9Sstevel@tonic-gate 
35067c478bd9Sstevel@tonic-gate 	if (umem_minfirewall == 0)
35077c478bd9Sstevel@tonic-gate 		umem_minfirewall = minfirewall;
35087c478bd9Sstevel@tonic-gate 
35097c478bd9Sstevel@tonic-gate 	/*
35107c478bd9Sstevel@tonic-gate 	 * Set up updating and reaping
35117c478bd9Sstevel@tonic-gate 	 */
35127c478bd9Sstevel@tonic-gate 	umem_reap_next = gethrtime() + NANOSEC;
35137c478bd9Sstevel@tonic-gate 
35147c478bd9Sstevel@tonic-gate #ifndef UMEM_STANDALONE
35157c478bd9Sstevel@tonic-gate 	(void) gettimeofday(&umem_update_next, NULL);
35167c478bd9Sstevel@tonic-gate #endif
35177c478bd9Sstevel@tonic-gate 
35187c478bd9Sstevel@tonic-gate 	/*
35197c478bd9Sstevel@tonic-gate 	 * Set up logging -- failure here is okay, since it will just disable
35207c478bd9Sstevel@tonic-gate 	 * the logs
35217c478bd9Sstevel@tonic-gate 	 */
35227c478bd9Sstevel@tonic-gate 	if (umem_logging) {
35237c478bd9Sstevel@tonic-gate 		umem_transaction_log = umem_log_init(umem_transaction_log_size);
35247c478bd9Sstevel@tonic-gate 		umem_content_log = umem_log_init(umem_content_log_size);
35257c478bd9Sstevel@tonic-gate 		umem_failure_log = umem_log_init(umem_failure_log_size);
35267c478bd9Sstevel@tonic-gate 		umem_slab_log = umem_log_init(umem_slab_log_size);
35277c478bd9Sstevel@tonic-gate 	}
35287c478bd9Sstevel@tonic-gate 
35297c478bd9Sstevel@tonic-gate 	/*
35307c478bd9Sstevel@tonic-gate 	 * Set up caches -- if successful, initialization cannot fail, since
35317c478bd9Sstevel@tonic-gate 	 * allocations from other threads can now succeed.
35327c478bd9Sstevel@tonic-gate 	 */
35337c478bd9Sstevel@tonic-gate 	if (umem_cache_init() == 0) {
35347c478bd9Sstevel@tonic-gate 		log_message("unable to create initial caches\n");
35357c478bd9Sstevel@tonic-gate 		goto fail;
35367c478bd9Sstevel@tonic-gate 	}
35377c478bd9Sstevel@tonic-gate 	umem_oversize_arena = oversize_arena;
35387c478bd9Sstevel@tonic-gate 	umem_memalign_arena = memalign_arena;
35397c478bd9Sstevel@tonic-gate 
35407c478bd9Sstevel@tonic-gate 	umem_cache_applyall(umem_cache_magazine_enable);
35417c478bd9Sstevel@tonic-gate 
35427c478bd9Sstevel@tonic-gate 	/*
35437c478bd9Sstevel@tonic-gate 	 * initialization done, ready to go
35447c478bd9Sstevel@tonic-gate 	 */
35457c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&umem_init_lock);
35467c478bd9Sstevel@tonic-gate 	umem_ready = UMEM_READY;
35477c478bd9Sstevel@tonic-gate 	umem_init_thr = 0;
35487c478bd9Sstevel@tonic-gate 	(void) cond_broadcast(&umem_init_cv);
35497c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&umem_init_lock);
35507c478bd9Sstevel@tonic-gate 	return (1);
35517c478bd9Sstevel@tonic-gate 
35527c478bd9Sstevel@tonic-gate fail:
35537c478bd9Sstevel@tonic-gate 	log_message("umem initialization failed\n");
35547c478bd9Sstevel@tonic-gate 
35557c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&umem_init_lock);
35567c478bd9Sstevel@tonic-gate 	umem_ready = UMEM_READY_INIT_FAILED;
35577c478bd9Sstevel@tonic-gate 	umem_init_thr = 0;
35587c478bd9Sstevel@tonic-gate 	(void) cond_broadcast(&umem_init_cv);
35597c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&umem_init_lock);
35607c478bd9Sstevel@tonic-gate 	return (0);
35617c478bd9Sstevel@tonic-gate }
356214702342SRobert Mustacchi 
356314702342SRobert Mustacchi void
umem_setmtbf(uint32_t mtbf)356414702342SRobert Mustacchi umem_setmtbf(uint32_t mtbf)
356514702342SRobert Mustacchi {
356614702342SRobert Mustacchi 	umem_mtbf = mtbf;
356714702342SRobert Mustacchi }
3568