xref: /illumos-gate/usr/src/lib/libumem/common/umem.c (revision 9f160f41)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5789d94c2Sjwadams  * Common Development and Distribution License (the "License").
6789d94c2Sjwadams  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
227c478bd9Sstevel@tonic-gate /*
23a574db85Sraf  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
247c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
257c478bd9Sstevel@tonic-gate  */
274f364e7cSRobert Mustacchi /*
28b1e2e3fbSRobert Mustacchi  * Copyright (c) 2019 Joyent, Inc.
29831abf2cSDan Kimmel  * Copyright (c) 2015 by Delphix. All rights reserved.
304f364e7cSRobert Mustacchi  */
314f364e7cSRobert Mustacchi 
327c478bd9Sstevel@tonic-gate /*
337c478bd9Sstevel@tonic-gate  * based on usr/src/uts/common/os/kmem.c r1.64 from 2001/12/18
347c478bd9Sstevel@tonic-gate  *
357c478bd9Sstevel@tonic-gate  * The slab allocator, as described in the following two papers:
367c478bd9Sstevel@tonic-gate  *
377c478bd9Sstevel@tonic-gate  *	Jeff Bonwick,
387c478bd9Sstevel@tonic-gate  *	The Slab Allocator: An Object-Caching Kernel Memory Allocator.
397c478bd9Sstevel@tonic-gate  *	Proceedings of the Summer 1994 Usenix Conference.
407c478bd9Sstevel@tonic-gate  *	Available as /shared/sac/PSARC/1994/028/materials/kmem.pdf.
417c478bd9Sstevel@tonic-gate  *
427c478bd9Sstevel@tonic-gate  *	Jeff Bonwick and Jonathan Adams,
437c478bd9Sstevel@tonic-gate  *	Magazines and vmem: Extending the Slab Allocator to Many CPUs and
447c478bd9Sstevel@tonic-gate  *	Arbitrary Resources.
457c478bd9Sstevel@tonic-gate  *	Proceedings of the 2001 Usenix Conference.
467c478bd9Sstevel@tonic-gate  *	Available as /shared/sac/PSARC/2000/550/materials/vmem.pdf.
477c478bd9Sstevel@tonic-gate  *
487c478bd9Sstevel@tonic-gate  * 1. Overview
497c478bd9Sstevel@tonic-gate  * -----------
504f364e7cSRobert Mustacchi  * umem is very close to kmem in implementation.  There are seven major
517c478bd9Sstevel@tonic-gate  * areas of divergence:
527c478bd9Sstevel@tonic-gate  *
537c478bd9Sstevel@tonic-gate  *	* Initialization
547c478bd9Sstevel@tonic-gate  *
557c478bd9Sstevel@tonic-gate  *	* CPU handling
567c478bd9Sstevel@tonic-gate  *
577c478bd9Sstevel@tonic-gate  *	* umem_update()
587c478bd9Sstevel@tonic-gate  *
597c478bd9Sstevel@tonic-gate  *	* KM_SLEEP v.s. UMEM_NOFAIL
607c478bd9Sstevel@tonic-gate  *
611c326e94Sjwadams  *	* lock ordering
627c478bd9Sstevel@tonic-gate  *
634f364e7cSRobert Mustacchi  *	* changing UMEM_MAXBUF
644f364e7cSRobert Mustacchi  *
654f364e7cSRobert Mustacchi  *	* Per-thread caching for malloc/free
664f364e7cSRobert Mustacchi  *
677c478bd9Sstevel@tonic-gate  * 2. Initialization
687c478bd9Sstevel@tonic-gate  * -----------------
697c478bd9Sstevel@tonic-gate  * kmem is initialized early on in boot, and knows that no one will call
707c478bd9Sstevel@tonic-gate  * into it before it is ready.  umem does not have these luxuries. Instead,
717c478bd9Sstevel@tonic-gate  * initialization is divided into two phases:
727c478bd9Sstevel@tonic-gate  *
737c478bd9Sstevel@tonic-gate  *	* library initialization, and
747c478bd9Sstevel@tonic-gate  *
757c478bd9Sstevel@tonic-gate  *	* first use
767c478bd9Sstevel@tonic-gate  *
777c478bd9Sstevel@tonic-gate  * umem's full initialization happens at the time of the first allocation
787c478bd9Sstevel@tonic-gate  * request (via malloc() and friends, umem_alloc(), or umem_zalloc()),
797c478bd9Sstevel@tonic-gate  * or the first call to umem_cache_create().
807c478bd9Sstevel@tonic-gate  *
817c478bd9Sstevel@tonic-gate  * umem_free(), and umem_cache_alloc() do not require special handling,
827c478bd9Sstevel@tonic-gate  * since the only way to get valid arguments for them is to successfully
837c478bd9Sstevel@tonic-gate  * call a function from the first group.
847c478bd9Sstevel@tonic-gate  *
857c478bd9Sstevel@tonic-gate  * 2.1. Library Initialization: umem_startup()
867c478bd9Sstevel@tonic-gate  * -------------------------------------------
877c478bd9Sstevel@tonic-gate  * umem_startup() is libumem.so's .init section.  It calls pthread_atfork()
887c478bd9Sstevel@tonic-gate  * to install the handlers necessary for umem's Fork1-Safety.  Because of
897c478bd9Sstevel@tonic-gate  * race condition issues, all other pre-umem_init() initialization is done
907c478bd9Sstevel@tonic-gate  * statically (i.e. by the dynamic linker).
917c478bd9Sstevel@tonic-gate  *
927c478bd9Sstevel@tonic-gate  * For standalone use, umem_startup() returns everything to its initial
937c478bd9Sstevel@tonic-gate  * state.
947c478bd9Sstevel@tonic-gate  *
957c478bd9Sstevel@tonic-gate  * 2.2. First use: umem_init()
967c478bd9Sstevel@tonic-gate  * ------------------------------
977c478bd9Sstevel@tonic-gate  * The first time any memory allocation function is used, we have to
987c478bd9Sstevel@tonic-gate  * create the backing caches and vmem arenas which are needed for it.
997c478bd9Sstevel@tonic-gate  * umem_init() is the central point for that task.  When it completes,
1007c478bd9Sstevel@tonic-gate  * umem_ready is either UMEM_READY (all set) or UMEM_READY_INIT_FAILED (unable
1017c478bd9Sstevel@tonic-gate  * to initialize, probably due to lack of memory).
1027c478bd9Sstevel@tonic-gate  *
1037c478bd9Sstevel@tonic-gate  * There are four different paths from which umem_init() is called:
1047c478bd9Sstevel@tonic-gate  *
1057c478bd9Sstevel@tonic-gate  *	* from umem_alloc() or umem_zalloc(), with 0 < size < UMEM_MAXBUF,
1067c478bd9Sstevel@tonic-gate  *
1077c478bd9Sstevel@tonic-gate  *	* from umem_alloc() or umem_zalloc(), with size > UMEM_MAXBUF,
1087c478bd9Sstevel@tonic-gate  *
1097c478bd9Sstevel@tonic-gate  *	* from umem_cache_create(), and
1107c478bd9Sstevel@tonic-gate  *
1117c478bd9Sstevel@tonic-gate  *	* from memalign(), with align > UMEM_ALIGN.
1127c478bd9Sstevel@tonic-gate  *
1137c478bd9Sstevel@tonic-gate  * The last three just check if umem is initialized, and call umem_init()
1147c478bd9Sstevel@tonic-gate  * if it is not.  For performance reasons, the first case is more complicated.
1157c478bd9Sstevel@tonic-gate  *
1167c478bd9Sstevel@tonic-gate  * 2.2.1. umem_alloc()/umem_zalloc(), with 0 < size < UMEM_MAXBUF
1177c478bd9Sstevel@tonic-gate  * -----------------------------------------------------------------
1187c478bd9Sstevel@tonic-gate  * In this case, umem_cache_alloc(&umem_null_cache, ...) is called.
1197c478bd9Sstevel@tonic-gate  * There is special case code in which causes any allocation on
1207c478bd9Sstevel@tonic-gate  * &umem_null_cache to fail by returning (NULL), regardless of the
1217c478bd9Sstevel@tonic-gate  * flags argument.
1227c478bd9Sstevel@tonic-gate  *
1237c478bd9Sstevel@tonic-gate  * So umem_cache_alloc() returns NULL, and umem_alloc()/umem_zalloc() call
1247c478bd9Sstevel@tonic-gate  * umem_alloc_retry().  umem_alloc_retry() sees that the allocation
1257c478bd9Sstevel@tonic-gate  * was agains &umem_null_cache, and calls umem_init().
1267c478bd9Sstevel@tonic-gate  *
1277c478bd9Sstevel@tonic-gate  * If initialization is successful, umem_alloc_retry() returns 1, which
1287c478bd9Sstevel@tonic-gate  * causes umem_alloc()/umem_zalloc() to start over, which causes it to load
1297c478bd9Sstevel@tonic-gate  * the (now valid) cache pointer from umem_alloc_table.
1307c478bd9Sstevel@tonic-gate  *
1317c478bd9Sstevel@tonic-gate  * 2.2.2. Dealing with race conditions
1327c478bd9Sstevel@tonic-gate  * -----------------------------------
1337c478bd9Sstevel@tonic-gate  * There are a couple race conditions resulting from the initialization
1347c478bd9Sstevel@tonic-gate  * code that we have to guard against:
1357c478bd9Sstevel@tonic-gate  *
1367c478bd9Sstevel@tonic-gate  *	* In umem_cache_create(), there is a special UMC_INTERNAL cflag
1377c478bd9Sstevel@tonic-gate  *	that is passed for caches created during initialization.  It
1387c478bd9Sstevel@tonic-gate  *	is illegal for a user to try to create a UMC_INTERNAL cache.
1397c478bd9Sstevel@tonic-gate  *	This allows initialization to proceed, but any other
1407c478bd9Sstevel@tonic-gate  *	umem_cache_create()s will block by calling umem_init().
1417c478bd9Sstevel@tonic-gate  *
1427c478bd9Sstevel@tonic-gate  *	* Since umem_null_cache has a 1-element cache_cpu, it's cache_cpu_mask
1437c478bd9Sstevel@tonic-gate  *	is always zero.  umem_cache_alloc uses cp->cache_cpu_mask to
1447c478bd9Sstevel@tonic-gate  *	mask the cpu number.  This prevents a race between grabbing a
1457c478bd9Sstevel@tonic-gate  *	cache pointer out of umem_alloc_table and growing the cpu array.
1467c478bd9Sstevel@tonic-gate  *
1477c478bd9Sstevel@tonic-gate  *
1487c478bd9Sstevel@tonic-gate  * 3. CPU handling
1497c478bd9Sstevel@tonic-gate  * ---------------
1507c478bd9Sstevel@tonic-gate  * kmem uses the CPU's sequence number to determine which "cpu cache" to
1517c478bd9Sstevel@tonic-gate  * use for an allocation.  Currently, there is no way to get the sequence
1527c478bd9Sstevel@tonic-gate  * number in userspace.
1537c478bd9Sstevel@tonic-gate  *
1547c478bd9Sstevel@tonic-gate  * umem keeps track of cpu information in umem_cpus, an array of umem_max_ncpus
1557c478bd9Sstevel@tonic-gate  * umem_cpu_t structures.  CURCPU() is a a "hint" function, which we then mask
1567c478bd9Sstevel@tonic-gate  * with either umem_cpu_mask or cp->cache_cpu_mask to find the actual "cpu" id.
1577c478bd9Sstevel@tonic-gate  * The mechanics of this is all in the CPU(mask) macro.
1587c478bd9Sstevel@tonic-gate  *
1597c478bd9Sstevel@tonic-gate  * Currently, umem uses _lwp_self() as its hint.
1607c478bd9Sstevel@tonic-gate  *
1617c478bd9Sstevel@tonic-gate  *
1627c478bd9Sstevel@tonic-gate  * 4. The update thread
1637c478bd9Sstevel@tonic-gate  * --------------------
1647c478bd9Sstevel@tonic-gate  * kmem uses a task queue, kmem_taskq, to do periodic maintenance on
1657c478bd9Sstevel@tonic-gate  * every kmem cache.  vmem has a periodic timeout for hash table resizing.
1667c478bd9Sstevel@tonic-gate  * The kmem_taskq also provides a separate context for kmem_cache_reap()'s
1677c478bd9Sstevel@tonic-gate  * to be done in, avoiding issues of the context of kmem_reap() callers.
1687c478bd9Sstevel@tonic-gate  *
1697c478bd9Sstevel@tonic-gate  * Instead, umem has the concept of "updates", which are asynchronous requests
1707c478bd9Sstevel@tonic-gate  * for work attached to single caches.  All caches with pending work are
1717c478bd9Sstevel@tonic-gate  * on a doubly linked list rooted at the umem_null_cache.  All update state
1727c478bd9Sstevel@tonic-gate  * is protected by the umem_update_lock mutex, and the umem_update_cv is used
1737c478bd9Sstevel@tonic-gate  * for notification between threads.
1747c478bd9Sstevel@tonic-gate  *
1757c478bd9Sstevel@tonic-gate  * 4.1. Cache states with regards to updates
1767c478bd9Sstevel@tonic-gate  * -----------------------------------------
1777c478bd9Sstevel@tonic-gate  * A given cache is in one of three states:
1787c478bd9Sstevel@tonic-gate  *
1797c478bd9Sstevel@tonic-gate  * Inactive		cache_uflags is zero, cache_u{next,prev} are NULL
1807c478bd9Sstevel@tonic-gate  *
1817c478bd9Sstevel@tonic-gate  * Work Requested	cache_uflags is non-zero (but UMU_ACTIVE is not set),
1827c478bd9Sstevel@tonic-gate  *			cache_u{next,prev} link the cache onto the global
1837c478bd9Sstevel@tonic-gate  *			update list
1847c478bd9Sstevel@tonic-gate  *
1857c478bd9Sstevel@tonic-gate  * Active		cache_uflags has UMU_ACTIVE set, cache_u{next,prev}
1867c478bd9Sstevel@tonic-gate  *			are NULL, and either umem_update_thr or
1877c478bd9Sstevel@tonic-gate  *			umem_st_update_thr are actively doing work on the
1887c478bd9Sstevel@tonic-gate  *			cache.
1897c478bd9Sstevel@tonic-gate  *
1907c478bd9Sstevel@tonic-gate  * An update can be added to any cache in any state -- if the cache is
1917c478bd9Sstevel@tonic-gate  * Inactive, it transitions to being Work Requested.  If the cache is
1927c478bd9Sstevel@tonic-gate  * Active, the worker will notice the new update and act on it before
1937c478bd9Sstevel@tonic-gate  * transitioning the cache to the Inactive state.
1947c478bd9Sstevel@tonic-gate  *
1957c478bd9Sstevel@tonic-gate  * If a cache is in the Active state, UMU_NOTIFY can be set, which asks
1967c478bd9Sstevel@tonic-gate  * the worker to broadcast the umem_update_cv when it has finished.
1977c478bd9Sstevel@tonic-gate  *
1987c478bd9Sstevel@tonic-gate  * 4.2. Update interface
1997c478bd9Sstevel@tonic-gate  * ---------------------
2007c478bd9Sstevel@tonic-gate  * umem_add_update() adds an update to a particular cache.
2017c478bd9Sstevel@tonic-gate  * umem_updateall() adds an update to all caches.
2027c478bd9Sstevel@tonic-gate  * umem_remove_updates() returns a cache to the Inactive state.
2037c478bd9Sstevel@tonic-gate  *
2047c478bd9Sstevel@tonic-gate  * umem_process_updates() process all caches in the Work Requested state.
2057c478bd9Sstevel@tonic-gate  *
2067c478bd9Sstevel@tonic-gate  * 4.3. Reaping
2077c478bd9Sstevel@tonic-gate  * ------------
2087c478bd9Sstevel@tonic-gate  * When umem_reap() is called (at the time of heap growth), it schedule
2097c478bd9Sstevel@tonic-gate  * UMU_REAP updates on every cache.  It then checks to see if the update
2107c478bd9Sstevel@tonic-gate  * thread exists (umem_update_thr != 0).  If it is, it broadcasts
2117c478bd9Sstevel@tonic-gate  * the umem_update_cv to wake the update thread up, and returns.
2127c478bd9Sstevel@tonic-gate  *
2137c478bd9Sstevel@tonic-gate  * If the update thread does not exist (umem_update_thr == 0), and the
2147c478bd9Sstevel@tonic-gate  * program currently has multiple threads, umem_reap() attempts to create
2157c478bd9Sstevel@tonic-gate  * a new update thread.
2167c478bd9Sstevel@tonic-gate  *
2177c478bd9Sstevel@tonic-gate  * If the process is not multithreaded, or the creation fails, umem_reap()
2187c478bd9Sstevel@tonic-gate  * calls umem_st_update() to do an inline update.
2197c478bd9Sstevel@tonic-gate  *
2207c478bd9Sstevel@tonic-gate  * 4.4. The update thread
2217c478bd9Sstevel@tonic-gate  * ----------------------
2227c478bd9Sstevel@tonic-gate  * The update thread spends most of its time in cond_timedwait() on the
2237c478bd9Sstevel@tonic-gate  * umem_update_cv.  It wakes up under two conditions:
2247c478bd9Sstevel@tonic-gate  *
2257c478bd9Sstevel@tonic-gate  *	* The timedwait times out, in which case it needs to run a global
2267c478bd9Sstevel@tonic-gate  *	update, or
2277c478bd9Sstevel@tonic-gate  *
2287c478bd9Sstevel@tonic-gate  *	* someone cond_broadcast(3THR)s the umem_update_cv, in which case
2297c478bd9Sstevel@tonic-gate  *	it needs to check if there are any caches in the Work Requested
2307c478bd9Sstevel@tonic-gate  *	state.
2317c478bd9Sstevel@tonic-gate  *
2327c478bd9Sstevel@tonic-gate  * When it is time for another global update, umem calls umem_cache_update()
2337c478bd9Sstevel@tonic-gate  * on every cache, then calls vmem_update(), which tunes the vmem structures.
2347c478bd9Sstevel@tonic-gate  * umem_cache_update() can request further work using umem_add_update().
2357c478bd9Sstevel@tonic-gate  *
2367c478bd9Sstevel@tonic-gate  * After any work from the global update completes, the update timer is
2377c478bd9Sstevel@tonic-gate  * reset to umem_reap_interval seconds in the future.  This makes the
2387c478bd9Sstevel@tonic-gate  * updates self-throttling.
2397c478bd9Sstevel@tonic-gate  *
2407c478bd9Sstevel@tonic-gate  * Reaps are similarly self-throttling.  After a UMU_REAP update has
2417c478bd9Sstevel@tonic-gate  * been scheduled on all caches, umem_reap() sets a flag and wakes up the
2427c478bd9Sstevel@tonic-gate  * update thread.  The update thread notices the flag, and resets the
2437c478bd9Sstevel@tonic-gate  * reap state.
2447c478bd9Sstevel@tonic-gate  *
2457c478bd9Sstevel@tonic-gate  * 4.5. Inline updates
2467c478bd9Sstevel@tonic-gate  * -------------------
2477c478bd9Sstevel@tonic-gate  * If the update thread is not running, umem_st_update() is used instead.  It
2487c478bd9Sstevel@tonic-gate  * immediately does a global update (as above), then calls
2497c478bd9Sstevel@tonic-gate  * umem_process_updates() to process both the reaps that umem_reap() added and
2507c478bd9Sstevel@tonic-gate  * any work generated by the global update.  Afterwards, it resets the reap
2517c478bd9Sstevel@tonic-gate  * state.
2527c478bd9Sstevel@tonic-gate  *
2537c478bd9Sstevel@tonic-gate  * While the umem_st_update() is running, umem_st_update_thr holds the thread
2547c478bd9Sstevel@tonic-gate  * id of the thread performing the update.
2557c478bd9Sstevel@tonic-gate  *
2567c478bd9Sstevel@tonic-gate  * 4.6. Updates and fork1()
2577c478bd9Sstevel@tonic-gate  * ------------------------
2587c478bd9Sstevel@tonic-gate  * umem has fork1() pre- and post-handlers which lock up (and release) every
2597c478bd9Sstevel@tonic-gate  * mutex in every cache.  They also lock up the umem_update_lock.  Since
2607c478bd9Sstevel@tonic-gate  * fork1() only copies over a single lwp, other threads (including the update
2617c478bd9Sstevel@tonic-gate  * thread) could have been actively using a cache in the parent.  This
2627c478bd9Sstevel@tonic-gate  * can lead to inconsistencies in the child process.
2637c478bd9Sstevel@tonic-gate  *
2647c478bd9Sstevel@tonic-gate  * Because we locked all of the mutexes, the only possible inconsistancies are:
2657c478bd9Sstevel@tonic-gate  *
2667c478bd9Sstevel@tonic-gate  *	* a umem_cache_alloc() could leak its buffer.
2677c478bd9Sstevel@tonic-gate  *
2687c478bd9Sstevel@tonic-gate  *	* a caller of umem_depot_alloc() could leak a magazine, and all the
2697c478bd9Sstevel@tonic-gate  *	buffers contained in it.
2707c478bd9Sstevel@tonic-gate  *
2717c478bd9Sstevel@tonic-gate  *	* a cache could be in the Active update state.  In the child, there
2727c478bd9Sstevel@tonic-gate  *	would be no thread actually working on it.
2737c478bd9Sstevel@tonic-gate  *
2747c478bd9Sstevel@tonic-gate  *	* a umem_hash_rescale() could leak the new hash table.
2757c478bd9Sstevel@tonic-gate  *
2767c478bd9Sstevel@tonic-gate  *	* a umem_magazine_resize() could be in progress.
2777c478bd9Sstevel@tonic-gate  *
2787c478bd9Sstevel@tonic-gate  *	* a umem_reap() could be in progress.
2797c478bd9Sstevel@tonic-gate  *
2807c478bd9Sstevel@tonic-gate  * The memory leaks we can't do anything about.  umem_release_child() resets
2817c478bd9Sstevel@tonic-gate  * the update state, moves any caches in the Active state to the Work Requested
2827c478bd9Sstevel@tonic-gate  * state.  This might cause some updates to be re-run, but UMU_REAP and
2837c478bd9Sstevel@tonic-gate  * UMU_HASH_RESCALE are effectively idempotent, and the worst that can
2847c478bd9Sstevel@tonic-gate  * happen from umem_magazine_resize() is resizing the magazine twice in close
2857c478bd9Sstevel@tonic-gate  * succession.
2867c478bd9Sstevel@tonic-gate  *
2877c478bd9Sstevel@tonic-gate  * Much of the cleanup in umem_release_child() is skipped if
2887c478bd9Sstevel@tonic-gate  * umem_st_update_thr == thr_self().  This is so that applications which call
2897c478bd9Sstevel@tonic-gate  * fork1() from a cache callback does not break.  Needless to say, any such
2907c478bd9Sstevel@tonic-gate  * application is tremendously broken.
2917c478bd9Sstevel@tonic-gate  *
2927c478bd9Sstevel@tonic-gate  *
2937c478bd9Sstevel@tonic-gate  * 5. KM_SLEEP v.s. UMEM_NOFAIL
2947c478bd9Sstevel@tonic-gate  * ----------------------------
2957c478bd9Sstevel@tonic-gate  * Allocations against kmem and vmem have two basic modes:  SLEEP and
2967c478bd9Sstevel@tonic-gate  * NOSLEEP.  A sleeping allocation is will go to sleep (waiting for
2977c478bd9Sstevel@tonic-gate  * more memory) instead of failing (returning NULL).
2987c478bd9Sstevel@tonic-gate  *
2997c478bd9Sstevel@tonic-gate  * SLEEP allocations presume an extremely multithreaded model, with
3007c478bd9Sstevel@tonic-gate  * a lot of allocation and deallocation activity.  umem cannot presume
3017c478bd9Sstevel@tonic-gate  * that its clients have any particular type of behavior.  Instead,
3027c478bd9Sstevel@tonic-gate  * it provides two types of allocations:
3037c478bd9Sstevel@tonic-gate  *
3047c478bd9Sstevel@tonic-gate  *	* UMEM_DEFAULT, equivalent to KM_NOSLEEP (i.e. return NULL on
3057c478bd9Sstevel@tonic-gate  *	failure)
3067c478bd9Sstevel@tonic-gate  *
3077c478bd9Sstevel@tonic-gate  *	* UMEM_NOFAIL, which, on failure, calls an optional callback
3087c478bd9Sstevel@tonic-gate  *	(registered with umem_nofail_callback()).
3097c478bd9Sstevel@tonic-gate  *
3107c478bd9Sstevel@tonic-gate  * The callback is invoked with no locks held, and can do an arbitrary
3117c478bd9Sstevel@tonic-gate  * amount of work.  It then has a choice between:
3127c478bd9Sstevel@tonic-gate  *
3137c478bd9Sstevel@tonic-gate  *	* Returning UMEM_CALLBACK_RETRY, which will cause the allocation
3147c478bd9Sstevel@tonic-gate  *	to be restarted.
3157c478bd9Sstevel@tonic-gate  *
3167c478bd9Sstevel@tonic-gate  *	* Returning UMEM_CALLBACK_EXIT(status), which will cause exit(2)
3177c478bd9Sstevel@tonic-gate  *	to be invoked with status.  If multiple threads attempt to do
3187c478bd9Sstevel@tonic-gate  *	this simultaneously, only one will call exit(2).
3197c478bd9Sstevel@tonic-gate  *
3207c478bd9Sstevel@tonic-gate  *	* Doing some kind of non-local exit (thr_exit(3thr), longjmp(3C),
3217c478bd9Sstevel@tonic-gate  *	etc.)
3227c478bd9Sstevel@tonic-gate  *
3237c478bd9Sstevel@tonic-gate  * The default callback returns UMEM_CALLBACK_EXIT(255).
3247c478bd9Sstevel@tonic-gate  *
3257c478bd9Sstevel@tonic-gate  * To have these callbacks without risk of state corruption (in the case of
3267c478bd9Sstevel@tonic-gate  * a non-local exit), we have to ensure that the callbacks get invoked
3277c478bd9Sstevel@tonic-gate  * close to the original allocation, with no inconsistent state or held
3287c478bd9Sstevel@tonic-gate  * locks.  The following steps are taken:
3297c478bd9Sstevel@tonic-gate  *
3307c478bd9Sstevel@tonic-gate  *	* All invocations of vmem are VM_NOSLEEP.
3317c478bd9Sstevel@tonic-gate  *
3327c478bd9Sstevel@tonic-gate  *	* All constructor callbacks (which can themselves to allocations)
3337c478bd9Sstevel@tonic-gate  *	are passed UMEM_DEFAULT as their required allocation argument.  This
3347c478bd9Sstevel@tonic-gate  *	way, the constructor will fail, allowing the highest-level allocation
3357c478bd9Sstevel@tonic-gate  *	invoke the nofail callback.
3367c478bd9Sstevel@tonic-gate  *
3377c478bd9Sstevel@tonic-gate  *	If a constructor callback _does_ do a UMEM_NOFAIL allocation, and
3387c478bd9Sstevel@tonic-gate  *	the nofail callback does a non-local exit, we will leak the
3397c478bd9Sstevel@tonic-gate  *	partially-constructed buffer.
3401c326e94Sjwadams  *
3411c326e94Sjwadams  *
3421c326e94Sjwadams  * 6. Lock Ordering
3431c326e94Sjwadams  * ----------------
3441c326e94Sjwadams  * umem has a few more locks than kmem does, mostly in the update path.  The
3451c326e94Sjwadams  * overall lock ordering (earlier locks must be acquired first) is:
3461c326e94Sjwadams  *
3471c326e94Sjwadams  *	umem_init_lock
3481c326e94Sjwadams  *
3491c326e94Sjwadams  *	vmem_list_lock
3501c326e94Sjwadams  *	vmem_nosleep_lock.vmpl_mutex
3511c326e94Sjwadams  *	vmem_t's:
3521c326e94Sjwadams  *		vm_lock
353789d94c2Sjwadams  *	sbrk_lock
3541c326e94Sjwadams  *
3551c326e94Sjwadams  *	umem_cache_lock
3561c326e94Sjwadams  *	umem_update_lock
3571c326e94Sjwadams  *	umem_flags_lock
3581c326e94Sjwadams  *	umem_cache_t's:
3591c326e94Sjwadams  *		cache_cpu[*].cc_lock
3601c326e94Sjwadams  *		cache_depot_lock
3611c326e94Sjwadams  *		cache_lock
3621c326e94Sjwadams  *	umem_log_header_t's:
3631c326e94Sjwadams  *		lh_cpu[*].clh_lock
3641c326e94Sjwadams  *		lh_lock
36538849194SRobert Mustacchi  *
36638849194SRobert Mustacchi  * 7. Changing UMEM_MAXBUF
36738849194SRobert Mustacchi  * -----------------------
36838849194SRobert Mustacchi  *
36938849194SRobert Mustacchi  * When changing UMEM_MAXBUF extra care has to be taken. It is not sufficient to
37038849194SRobert Mustacchi  * simply increase this number. First, one must update the umem_alloc_table to
37138849194SRobert Mustacchi  * have the appropriate number of entires based upon the new size. If this is
37238849194SRobert Mustacchi  * not done, this will lead to libumem blowing an assertion.
37338849194SRobert Mustacchi  *
37438849194SRobert Mustacchi  * The second place to update, which is not required, is the umem_alloc_sizes.
37538849194SRobert Mustacchi  * These determine the default cache sizes that we're going to support.
3764f364e7cSRobert Mustacchi  *
3774f364e7cSRobert Mustacchi  * 8. Per-thread caching for malloc/free
3784f364e7cSRobert Mustacchi  * -------------------------------------
3794f364e7cSRobert Mustacchi  *
3804f364e7cSRobert Mustacchi  * "Time is an illusion. Lunchtime doubly so." -- Douglas Adams
3814f364e7cSRobert Mustacchi  *
3824f364e7cSRobert Mustacchi  * Time may be an illusion, but CPU cycles aren't.  While libumem is designed
3834f364e7cSRobert Mustacchi  * to be a highly scalable allocator, that scalability comes with a fixed cycle
3844f364e7cSRobert Mustacchi  * penalty even in the absence of contention: libumem must acquire (and release
3854f364e7cSRobert Mustacchi  * a per-CPU lock for each allocation.  When contention is low and malloc(3C)
3864f364e7cSRobert Mustacchi  * frequency is high, this overhead can dominate execution time.  To alleviate
3874f364e7cSRobert Mustacchi  * this, we allow for per-thread caching, a lock-free means of caching recent
3884f364e7cSRobert Mustacchi  * deallocations on a per-thread basis for use in satisfying subsequent calls
3894f364e7cSRobert Mustacchi  *
3904f364e7cSRobert Mustacchi  * In addition to improving performance, we also want to:
3914f364e7cSRobert Mustacchi  *	* Minimize fragmentation
3924f364e7cSRobert Mustacchi  *	* Not add additional memory overhead (no larger malloc tags)
3934f364e7cSRobert Mustacchi  *
3944f364e7cSRobert Mustacchi  * In the ulwp_t of each thread there is a private data structure called a
3954f364e7cSRobert Mustacchi  * umem_t that looks like:
3964f364e7cSRobert Mustacchi  *
3974f364e7cSRobert Mustacchi  * typedef struct {
39814702342SRobert Mustacchi  *	size_t	tm_size;
39914702342SRobert Mustacchi  *	void	*tm_roots[NTMEMBASE];  (Currently 16)
4004f364e7cSRobert Mustacchi  * } tmem_t;
4014f364e7cSRobert Mustacchi  *
4024f364e7cSRobert Mustacchi  * Each of the roots is treated as the head of a linked list. Each entry in the
4034f364e7cSRobert Mustacchi  * list can be thought of as a void ** which points to the next entry, until one
4044f364e7cSRobert Mustacchi  * of them points to NULL. If the head points to NULL, the list is empty.
4054f364e7cSRobert Mustacchi  *
4064f364e7cSRobert Mustacchi  * Each head corresponds to a umem_cache. Currently there is a linear mapping
4074f364e7cSRobert Mustacchi  * where the first root corresponds to the first cache, second root to the
4084f364e7cSRobert Mustacchi  * second cache, etc. This works because every allocation that malloc makes to
4094f364e7cSRobert Mustacchi  * umem_alloc that can be satisified by a umem_cache will actually return a
4104f364e7cSRobert Mustacchi  * number of bytes equal to the size of that cache. Because of this property and
4114f364e7cSRobert Mustacchi  * a one to one mapping between caches and roots we can guarantee that every
4124f364e7cSRobert Mustacchi  * entry in a given root's list will be able to satisfy the same requests as the
4134f364e7cSRobert Mustacchi  * corresponding cache.
4144f364e7cSRobert Mustacchi  *
4154f364e7cSRobert Mustacchi  * The choice of sixteen roots is based on where we believe we get the biggest
4164f364e7cSRobert Mustacchi  * bang for our buck. The per-thread caches will cache up to 256 byte and 448
4174f364e7cSRobert Mustacchi  * byte allocations on ILP32 and LP64 respectively. Generally applications plan
4184f364e7cSRobert Mustacchi  * more carefully how they do larger allocations than smaller ones. Therefore
4194f364e7cSRobert Mustacchi  * sixteen roots is a reasonable compromise between the amount of additional
4204f364e7cSRobert Mustacchi  * overhead per thread, and the likelihood of a program to benefit from it.
4214f364e7cSRobert Mustacchi  *
4224f364e7cSRobert Mustacchi  * The maximum amount of memory that can be cached in each thread is determined
4234f364e7cSRobert Mustacchi  * by the perthread_cache UMEM_OPTION. It corresponds to the umem_ptc_size
4244f364e7cSRobert Mustacchi  * value. The default value for this is currently 1 MB. Once umem_init() has
4254f364e7cSRobert Mustacchi  * finished this cannot be directly tuned without directly modifying the
4264f364e7cSRobert Mustacchi  * instruction text. If, upon calling free(3C), the amount cached would exceed
4274f364e7cSRobert Mustacchi  * this maximum, we instead actually return the buffer to the umem_cache instead
4284f364e7cSRobert Mustacchi  * of holding onto it in the thread.
4294f364e7cSRobert Mustacchi  *
4304f364e7cSRobert Mustacchi  * When a thread calls malloc(3C) it first determines which umem_cache it
4314f364e7cSRobert Mustacchi  * would be serviced by. If the allocation is not covered by ptcumem it goes to
4324f364e7cSRobert Mustacchi  * the normal malloc instead.  Next, it checks if the tmem_root's list is empty
4334f364e7cSRobert Mustacchi  * or not. If it is empty, we instead go and allocate the memory from
4344f364e7cSRobert Mustacchi  * umem_alloc. If it is not empty, we remove the head of the list, set the
4354f364e7cSRobert Mustacchi  * appropriate malloc tags, and return that buffer.
4364f364e7cSRobert Mustacchi  *
4374f364e7cSRobert Mustacchi  * When a thread calls free(3C) it first looks at the malloc tag and if it is
4384f364e7cSRobert Mustacchi  * invalid or the allocation exceeds the largest cache in ptcumem and sends it
4394f364e7cSRobert Mustacchi  * off to the original free() to handle and clean up appropriately. Next, it
4404f364e7cSRobert Mustacchi  * checks if the allocation size is covered by one of the per-thread roots and
4414f364e7cSRobert Mustacchi  * if it isn't, it passes it off to the original free() to be released. Finally,
4424f364e7cSRobert Mustacchi  * before it inserts this buffer as the head, it checks if adding this buffer
4434f364e7cSRobert Mustacchi  * would put the thread over its maximum cache size. If it would, it frees the
4444f364e7cSRobert Mustacchi  * buffer back to the umem_cache. Otherwise it increments the threads total
4454f364e7cSRobert Mustacchi  * cached amount and makes the buffer the new head of the appropriate tm_root.
4464f364e7cSRobert Mustacchi  *
4474f364e7cSRobert Mustacchi  * When a thread exits, all of the buffers that it has in its per-thread cache
4484f364e7cSRobert Mustacchi  * will be passed to umem_free() and returned to the appropriate umem_cache.
4494f364e7cSRobert Mustacchi  *
4504f364e7cSRobert Mustacchi  * 8.1 Handling addition and removal of umem_caches
4514f364e7cSRobert Mustacchi  * ------------------------------------------------
4524f364e7cSRobert Mustacchi  *
4534f364e7cSRobert Mustacchi  * The set of umem_caches that are used to back calls to umem_alloc() and
4544f364e7cSRobert Mustacchi  * ultimately malloc() are determined at program execution time. The default set
4554f364e7cSRobert Mustacchi  * of caches is defined below in umem_alloc_sizes[]. Various umem_options exist
4564f364e7cSRobert Mustacchi  * that modify the set of caches: size_add, size_clear, and size_remove. Because
4574f364e7cSRobert Mustacchi  * the set of caches can only be determined once umem_init() has been called and
4584f364e7cSRobert Mustacchi  * we have the additional goals of minimizing additional fragmentation and
4594f364e7cSRobert Mustacchi  * metadata space overhead in the malloc tags, this forces our hand to go down a
4604f364e7cSRobert Mustacchi  * slightly different path: the one tread by fasttrap and trapstat.
4614f364e7cSRobert Mustacchi  *
4624f364e7cSRobert Mustacchi  * During umem_init we're going to dynamically construct a new version of
4634f364e7cSRobert Mustacchi  * malloc(3C) and free(3C) that utilizes the known cache sizes and then ensure
4644f364e7cSRobert Mustacchi  * that ptcmalloc and ptcfree replace malloc and free as entries in the plt. If
4654f364e7cSRobert Mustacchi  * ptcmalloc and ptcfree cannot handle a request, they simply jump to the
4664f364e7cSRobert Mustacchi  * original libumem implementations.
4674f364e7cSRobert Mustacchi  *
4684f364e7cSRobert Mustacchi  * After creating all of the umem_caches, but before making them visible,
4694f364e7cSRobert Mustacchi  * umem_cache_init checks that umem_genasm_supported is non-zero. This value is
4704f364e7cSRobert Mustacchi  * set by each architecture in $ARCH/umem_genasm.c to indicate whether or not
4714f364e7cSRobert Mustacchi  * they support this. If the value is zero, then this process is skipped.
4724f364e7cSRobert Mustacchi  * Similarly, if the cache size has been tuned to zero by UMEM_OPTIONS, then
4734f364e7cSRobert Mustacchi  * this is also skipped.
4744f364e7cSRobert Mustacchi  *
4754f364e7cSRobert Mustacchi  * In umem_genasm.c, each architecture's implementation implements a single
4764f364e7cSRobert Mustacchi  * function called umem_genasm() that is responsible for generating the
4774f364e7cSRobert Mustacchi  * appropriate versions of ptcmalloc() and ptcfree(), placing them in the
4784f364e7cSRobert Mustacchi  * appropriate memory location, and finally doing the switch from malloc() and
4794f364e7cSRobert Mustacchi  * free() to ptcmalloc() and ptcfree().  Once the change has been made, there is
4804f364e7cSRobert Mustacchi  * no way to switch back, short of restarting the program or modifying program
4814f364e7cSRobert Mustacchi  * text with mdb.
4824f364e7cSRobert Mustacchi  *
4834f364e7cSRobert Mustacchi  * 8.2 Modifying the Procedure Linkage Table (PLT)
4844f364e7cSRobert Mustacchi  * -----------------------------------------------
4854f364e7cSRobert Mustacchi  *
4864f364e7cSRobert Mustacchi  * The last piece of this puzzle is how we actually jam ptcmalloc() into the
487*9f160f41SRichard Lowe  * PLT.  To handle this, we have defined two functions, _malloc and _free, we
488*9f160f41SRichard Lowe  * use a standard #pragma weak for malloc and free and direct them to those
489*9f160f41SRichard Lowe  * symbols. By default, those symbols have text defined as nops for our
490*9f160f41SRichard Lowe  * generated functions and when they're invoked, they jump to the default
491*9f160f41SRichard Lowe  * malloc and free functions.
492*9f160f41SRichard Lowe  *
493*9f160f41SRichard Lowe  * When umem_genasm() is called, it makes _malloc and _free writeable and goes
494*9f160f41SRichard Lowe  * through and updates the text provided for by _malloc and _free just after
495*9f160f41SRichard Lowe  * the jump. Once both have been successfully generated, umem_genasm() nops
496*9f160f41SRichard Lowe  * over the original jump so that we now call into the genasm versions of
497*9f160f41SRichard Lowe  * these functions, and makes the functions read-only once again.
4984f364e7cSRobert Mustacchi  *
4994f364e7cSRobert Mustacchi  * 8.3 umem_genasm()
5004f364e7cSRobert Mustacchi  * -----------------
5014f364e7cSRobert Mustacchi  *
5024f364e7cSRobert Mustacchi  * umem_genasm() is currently implemented for i386 and amd64. This section
5034f364e7cSRobert Mustacchi  * describes the theory behind the construction. For specific byte code to
5044f364e7cSRobert Mustacchi  * assembly instructions and niceish C and asm versions of ptcmalloc and
5054f364e7cSRobert Mustacchi  * ptcfree, see the individual umem_genasm.c files. The layout consists of the
5064f364e7cSRobert Mustacchi  * following sections:
5074f364e7cSRobert Mustacchi  *
5084f364e7cSRobert Mustacchi  *	o. function-specfic prologue
5094f364e7cSRobert Mustacchi  *	o. function-generic cache-selecting elements
5104f364e7cSRobert Mustacchi  *	o. function-specific epilogue
5114f364e7cSRobert Mustacchi  *
5124f364e7cSRobert Mustacchi  * There are three different generic cache elements that exist:
5134f364e7cSRobert Mustacchi  *
5144f364e7cSRobert Mustacchi  *	o. the last or only cache
5154f364e7cSRobert Mustacchi  *	o. the intermediary caches if more than two
5164f364e7cSRobert Mustacchi  *	o. the first one if more than one cache
5174f364e7cSRobert Mustacchi  *
5184f364e7cSRobert Mustacchi  * The malloc and free prologues and epilogues mimic the necessary portions of
5194f364e7cSRobert Mustacchi  * libumem's malloc and free. This includes things like checking for size
5204f364e7cSRobert Mustacchi  * overflow, setting and verifying the malloc tags.
5214f364e7cSRobert Mustacchi  *
5224f364e7cSRobert Mustacchi  * It is an important constraint that these functions do not make use of the
5234f364e7cSRobert Mustacchi  * call instruction. The only jmp outside of the individual functions is to the
5244f364e7cSRobert Mustacchi  * original libumem malloc and free respectively. Because doing things like
5254f364e7cSRobert Mustacchi  * setting errno or raising an internal umem error on improper malloc tags would
5264f364e7cSRobert Mustacchi  * require using calls into the PLT, whenever we encounter one of those cases we
5274f364e7cSRobert Mustacchi  * just jump to the original malloc and free functions reusing the same stack
5284f364e7cSRobert Mustacchi  * frame.
5294f364e7cSRobert Mustacchi  *
5304f364e7cSRobert Mustacchi  * Each of the above sections, the three caches, and the malloc and free
5314f364e7cSRobert Mustacchi  * prologue and epilogue are implemented as blocks of machine code with the
5324f364e7cSRobert Mustacchi  * corresponding assembly in comments. There are known offsets into each block
5334f364e7cSRobert Mustacchi  * that corresponds to locations of data and addresses that we only know at run
5344f364e7cSRobert Mustacchi  * time. These blocks are copied as necessary and the blanks filled in
5354f364e7cSRobert Mustacchi  * appropriately.
5364f364e7cSRobert Mustacchi  *
5374f364e7cSRobert Mustacchi  * As mentioned in section 8.2, the trampoline library uses specifically named
5384f364e7cSRobert Mustacchi  * variables to communicate the buffers and size to use. These variables are:
5394f364e7cSRobert Mustacchi  *
5404f364e7cSRobert Mustacchi  *	o. umem_genasm_mptr: The buffer for ptcmalloc
5414f364e7cSRobert Mustacchi  *	o. umem_genasm_msize: The size in bytes of the above buffer
5424f364e7cSRobert Mustacchi  *	o. umem_genasm_fptr: The buffer for ptcfree
5434f364e7cSRobert Mustacchi  *	o. umem_genasm_fsize: The size in bytes of the above buffer
5444f364e7cSRobert Mustacchi  *
5454f364e7cSRobert Mustacchi  * Finally, to enable the generated assembly we need to remove the previous jump
5464f364e7cSRobert Mustacchi  * to the actual malloc that exists at the start of these buffers. On x86, this
5474f364e7cSRobert Mustacchi  * is a five byte region. We could zero out the jump offset to be a jmp +0, but
5484f364e7cSRobert Mustacchi  * using nops can be faster. We specifically use a single five byte nop on x86
5494f364e7cSRobert Mustacchi  * as it is faster. When porting ptcumem to other architectures, the various
5504f364e7cSRobert Mustacchi  * opcode changes and options should be analyzed.
5514f364e7cSRobert Mustacchi  *
5524f364e7cSRobert Mustacchi  * 8.4 Interface with libc.so
5534f364e7cSRobert Mustacchi  * --------------------------
5544f364e7cSRobert Mustacchi  *
5554f364e7cSRobert Mustacchi  * The tmem_t structure as described in the beginning of section 8, is part of a
5564f364e7cSRobert Mustacchi  * private interface with libc. There are three functions that exist to cover
5574f364e7cSRobert Mustacchi  * this. They are not documented in man pages or header files. They are in the
5584f364e7cSRobert Mustacchi  * SUNWprivate part of libc's mapfile.
5594f364e7cSRobert Mustacchi  *
5604f364e7cSRobert Mustacchi  *	o. _tmem_get_base(void)
5614f364e7cSRobert Mustacchi  *
56214702342SRobert Mustacchi  *	Returns the offset from the ulwp_t (curthread) to the tmem_t structure.
56314702342SRobert Mustacchi  *	This is a constant for all threads and is effectively a way to to do
56414702342SRobert Mustacchi  *	::offsetof ulwp_t ul_tmem without having to know the specifics of the
56514702342SRobert Mustacchi  *	structure outside of libc.
5664f364e7cSRobert Mustacchi  *
5674f364e7cSRobert Mustacchi  *	o. _tmem_get_nentries(void)
5684f364e7cSRobert Mustacchi  *
5694f364e7cSRobert Mustacchi  *	Returns the number of roots that exist in the tmem_t. This is one part
5704f364e7cSRobert Mustacchi  *	of the cap on the number of umem_caches that we can back with tmem.
5714f364e7cSRobert Mustacchi  *
5724f364e7cSRobert Mustacchi  *	o. _tmem_set_cleanup(void (*)(void *, int))
5734f364e7cSRobert Mustacchi  *
5744f364e7cSRobert Mustacchi  *	This sets a clean up handler that gets called back when a thread exits.
5754f364e7cSRobert Mustacchi  *	There is one call per buffer, the void * is a pointer to the buffer on
5764f364e7cSRobert Mustacchi  *	the list, the int is the index into the roots array for this buffer.
5774f364e7cSRobert Mustacchi  *
5784f364e7cSRobert Mustacchi  * 8.5 Tuning and disabling per-thread caching
5794f364e7cSRobert Mustacchi  * -------------------------------------------
5804f364e7cSRobert Mustacchi  *
5814f364e7cSRobert Mustacchi  * There is only one tunable for per-thread caching:  the amount of memory each
5824f364e7cSRobert Mustacchi  * thread should be able to cache.  This is specified via the perthread_cache
5834f364e7cSRobert Mustacchi  * UMEM_OPTION option.  No attempt is made to to sanity check the specified
5844f364e7cSRobert Mustacchi  * value; the limit is simply the maximum value of a size_t.
5854f364e7cSRobert Mustacchi  *
5864f364e7cSRobert Mustacchi  * If the perthread_cache UMEM_OPTION is set to zero, nomagazines was requested,
5874f364e7cSRobert Mustacchi  * or UMEM_DEBUG has been turned on then we will never call into umem_genasm;
5884f364e7cSRobert Mustacchi  * however, the trampoline audit library and jump will still be in place.
5894f364e7cSRobert Mustacchi  *
5904f364e7cSRobert Mustacchi  * 8.6 Observing efficacy of per-thread caching
5914f364e7cSRobert Mustacchi  * --------------------------------------------
5924f364e7cSRobert Mustacchi  *
5934f364e7cSRobert Mustacchi  * To understand the efficacy of per-thread caching, use the ::umastat dcmd
5944f364e7cSRobert Mustacchi  * to see the percentage of capacity consumed on a per-thread basis, the
5954f364e7cSRobert Mustacchi  * degree to which each umem cache contributes to per-thread cache consumption,
5964f364e7cSRobert Mustacchi  * and the number of buffers in per-thread caches on a per-umem cache basis.
5974f364e7cSRobert Mustacchi  * If more detail is required, the specific buffers in a per-thread cache can
5984f364e7cSRobert Mustacchi  * be iterated over with the umem_ptc_* walkers. (These walkers allow an
5994f364e7cSRobert Mustacchi  * optional ulwp_t to be specified to iterate only over a particular thread's
6004f364e7cSRobert Mustacchi  * cache.)
6017c478bd9Sstevel@tonic-gate  */
6037c478bd9Sstevel@tonic-gate #include <umem_impl.h>
6047c478bd9Sstevel@tonic-gate #include <sys/vmem_impl_user.h>
6057c478bd9Sstevel@tonic-gate #include "umem_base.h"
6067c478bd9Sstevel@tonic-gate #include "vmem_base.h"
6087c478bd9Sstevel@tonic-gate #include <sys/processor.h>
6097c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
6117c478bd9Sstevel@tonic-gate #include <alloca.h>
6127c478bd9Sstevel@tonic-gate #include <errno.h>
6137c478bd9Sstevel@tonic-gate #include <limits.h>
6147c478bd9Sstevel@tonic-gate #include <stdio.h>
6157c478bd9Sstevel@tonic-gate #include <stdlib.h>
6167c478bd9Sstevel@tonic-gate #include <string.h>
6177c478bd9Sstevel@tonic-gate #include <strings.h>
6187c478bd9Sstevel@tonic-gate #include <signal.h>
6197c478bd9Sstevel@tonic-gate #include <unistd.h>
6207c478bd9Sstevel@tonic-gate #include <atomic.h>
6227c478bd9Sstevel@tonic-gate #include "misc.h"
6247c478bd9Sstevel@tonic-gate #define	UMEM_VMFLAGS(umflag)	(VM_NOSLEEP)
6267c478bd9Sstevel@tonic-gate size_t pagesize;
6287c478bd9Sstevel@tonic-gate /*
6297c478bd9Sstevel@tonic-gate  * The default set of caches to back umem_alloc().
6307c478bd9Sstevel@tonic-gate  * These sizes should be reevaluated periodically.
6317c478bd9Sstevel@tonic-gate  *
6327c478bd9Sstevel@tonic-gate  * We want allocations that are multiples of the coherency granularity
6337c478bd9Sstevel@tonic-gate  * (64 bytes) to be satisfied from a cache which is a multiple of 64
6347c478bd9Sstevel@tonic-gate  * bytes, so that it will be 64-byte aligned.  For all multiples of 64,
6357c478bd9Sstevel@tonic-gate  * the next kmem_cache_size greater than or equal to it must be a
6367c478bd9Sstevel@tonic-gate  * multiple of 64.
637789d94c2Sjwadams  *
638789d94c2Sjwadams  * This table must be in sorted order, from smallest to highest.  The
639789d94c2Sjwadams  * highest slot must be UMEM_MAXBUF, and every slot afterwards must be
640789d94c2Sjwadams  * zero.
6417c478bd9Sstevel@tonic-gate  */
642789d94c2Sjwadams static int umem_alloc_sizes[] = {
6437c478bd9Sstevel@tonic-gate #ifdef _LP64
6447c478bd9Sstevel@tonic-gate 	1 * 8,
6457c478bd9Sstevel@tonic-gate 	1 * 16,
6467c478bd9Sstevel@tonic-gate 	2 * 16,
6477c478bd9Sstevel@tonic-gate 	3 * 16,
6487c478bd9Sstevel@tonic-gate #else
6497c478bd9Sstevel@tonic-gate 	1 * 8,
6507c478bd9Sstevel@tonic-gate 	2 * 8,
6517c478bd9Sstevel@tonic-gate 	3 * 8,
6527c478bd9Sstevel@tonic-gate 	4 * 8,		5 * 8,		6 * 8,		7 * 8,
6537c478bd9Sstevel@tonic-gate #endif
6547c478bd9Sstevel@tonic-gate 	4 * 16,		5 * 16,		6 * 16,		7 * 16,
6557c478bd9Sstevel@tonic-gate 	4 * 32,		5 * 32,		6 * 32,		7 * 32,
6567c478bd9Sstevel@tonic-gate 	4 * 64,		5 * 64,		6 * 64,		7 * 64,
6577c478bd9Sstevel@tonic-gate 	4 * 128,	5 * 128,	6 * 128,	7 * 128,
6587c478bd9Sstevel@tonic-gate 	P2ALIGN(8192 / 7, 64),
6597c478bd9Sstevel@tonic-gate 	P2ALIGN(8192 / 6, 64),
6607c478bd9Sstevel@tonic-gate 	P2ALIGN(8192 / 5, 64),
661789d94c2Sjwadams 	P2ALIGN(8192 / 4, 64), 2304,
6627c478bd9Sstevel@tonic-gate 	P2ALIGN(8192 / 3, 64),
663789d94c2Sjwadams 	P2ALIGN(8192 / 2, 64), 4544,
664789d94c2Sjwadams 	P2ALIGN(8192 / 1, 64), 9216,
6657c478bd9Sstevel@tonic-gate 	4096 * 3,
66638849194SRobert Mustacchi 	8192 * 2,				/* = 8192 * 2 */
66738849194SRobert Mustacchi 	24576, 32768, 40960, 49152, 57344, 65536, 73728, 81920,
66838849194SRobert Mustacchi 	90112, 98304, 106496, 114688, 122880, UMEM_MAXBUF, /* 128k */
669789d94c2Sjwadams 	/* 24 slots for user expansion */
670789d94c2Sjwadams 	0, 0, 0, 0, 0, 0, 0, 0,
671789d94c2Sjwadams 	0, 0, 0, 0, 0, 0, 0, 0,
672789d94c2Sjwadams 	0, 0, 0, 0, 0, 0, 0, 0,
6737c478bd9Sstevel@tonic-gate };
6747c478bd9Sstevel@tonic-gate #define	NUM_ALLOC_SIZES (sizeof (umem_alloc_sizes) / sizeof (*umem_alloc_sizes))
6767c478bd9Sstevel@tonic-gate static umem_magtype_t umem_magtype[] = {
6777c478bd9Sstevel@tonic-gate 	{ 1,	8,	3200,	65536	},
6787c478bd9Sstevel@tonic-gate 	{ 3,	16,	256,	32768	},
6797c478bd9Sstevel@tonic-gate 	{ 7,	32,	64,	16384	},
6807c478bd9Sstevel@tonic-gate 	{ 15,	64,	0,	8192	},
6817c478bd9Sstevel@tonic-gate 	{ 31,	64,	0,	4096	},
6827c478bd9Sstevel@tonic-gate 	{ 47,	64,	0,	2048	},
6837c478bd9Sstevel@tonic-gate 	{ 63,	64,	0,	1024	},
6847c478bd9Sstevel@tonic-gate 	{ 95,	64,	0,	512	},
6857c478bd9Sstevel@tonic-gate 	{ 143,	64,	0,	0	},
6867c478bd9Sstevel@tonic-gate };
6887c478bd9Sstevel@tonic-gate /*
6897c478bd9Sstevel@tonic-gate  * umem tunables
6907c478bd9Sstevel@tonic-gate  */
6917c478bd9Sstevel@tonic-gate uint32_t umem_max_ncpus;	/* # of CPU caches. */
6937c478bd9Sstevel@tonic-gate uint32_t umem_stack_depth = 15; /* # stack frames in a bufctl_audit */
6947c478bd9Sstevel@tonic-gate uint32_t umem_reap_interval = 10; /* max reaping rate (seconds) */
6957c478bd9Sstevel@tonic-gate uint_t umem_depot_contention = 2; /* max failed trylocks per real interval */
6967c478bd9Sstevel@tonic-gate uint_t umem_abort = 1;		/* whether to abort on error */
6977c478bd9Sstevel@tonic-gate uint_t umem_output = 0;		/* whether to write to standard error */
6987c478bd9Sstevel@tonic-gate uint_t umem_logging = 0;	/* umem_log_enter() override */
6997c478bd9Sstevel@tonic-gate uint32_t umem_mtbf = 0;		/* mean time between failures [default: off] */
7007c478bd9Sstevel@tonic-gate size_t umem_transaction_log_size; /* size of transaction log */
7017c478bd9Sstevel@tonic-gate size_t umem_content_log_size;	/* size of content log */
7027c478bd9Sstevel@tonic-gate size_t umem_failure_log_size;	/* failure log [4 pages per CPU] */
7037c478bd9Sstevel@tonic-gate size_t umem_slab_log_size;	/* slab create log [4 pages per CPU] */
7047c478bd9Sstevel@tonic-gate size_t umem_content_maxsave = 256; /* UMF_CONTENTS max bytes to log */
7057c478bd9Sstevel@tonic-gate size_t umem_lite_minsize = 0;	/* minimum buffer size for UMF_LITE */
7067c478bd9Sstevel@tonic-gate size_t umem_lite_maxalign = 1024; /* maximum buffer alignment for UMF_LITE */
7077c478bd9Sstevel@tonic-gate size_t umem_maxverify;		/* maximum bytes to inspect in debug routines */
7087c478bd9Sstevel@tonic-gate size_t umem_minfirewall;	/* hardware-enforced redzone threshold */
7094f364e7cSRobert Mustacchi size_t umem_ptc_size = 1048576;	/* size of per-thread cache (in bytes) */
7117c478bd9Sstevel@tonic-gate uint_t umem_flags = 0;
7124f364e7cSRobert Mustacchi uintptr_t umem_tmem_off;
7147c478bd9Sstevel@tonic-gate mutex_t			umem_init_lock;		/* locks initialization */
7157c478bd9Sstevel@tonic-gate cond_t			umem_init_cv;		/* initialization CV */
7167c478bd9Sstevel@tonic-gate thread_t		umem_init_thr;		/* thread initializing */
7177c478bd9Sstevel@tonic-gate int			umem_init_env_ready;	/* environ pre-initted */
7187c478bd9Sstevel@tonic-gate int			umem_ready = UMEM_READY_STARTUP;
7204f364e7cSRobert Mustacchi int			umem_ptc_enabled;	/* per-thread caching enabled */
7214f364e7cSRobert Mustacchi 
7227c478bd9Sstevel@tonic-gate static umem_nofail_callback_t *nofail_callback;
7237c478bd9Sstevel@tonic-gate static mutex_t		umem_nofail_exit_lock;
7247c478bd9Sstevel@tonic-gate static thread_t		umem_nofail_exit_thr;
7267c478bd9Sstevel@tonic-gate static umem_cache_t	*umem_slab_cache;
7277c478bd9Sstevel@tonic-gate static umem_cache_t	*umem_bufctl_cache;
7287c478bd9Sstevel@tonic-gate static umem_cache_t	*umem_bufctl_audit_cache;
7307c478bd9Sstevel@tonic-gate mutex_t			umem_flags_lock;
7327c478bd9Sstevel@tonic-gate static vmem_t		*heap_arena;
7337c478bd9Sstevel@tonic-gate static vmem_alloc_t	*heap_alloc;
7347c478bd9Sstevel@tonic-gate static vmem_free_t	*heap_free;
7367c478bd9Sstevel@tonic-gate static vmem_t		*umem_internal_arena;
7377c478bd9Sstevel@tonic-gate static vmem_t		*umem_cache_arena;
7387c478bd9Sstevel@tonic-gate static vmem_t		*umem_hash_arena;
7397c478bd9Sstevel@tonic-gate static vmem_t		*umem_log_arena;
7407c478bd9Sstevel@tonic-gate static vmem_t		*umem_oversize_arena;
7417c478bd9Sstevel@tonic-gate static vmem_t		*umem_va_arena;
7427c478bd9Sstevel@tonic-gate static vmem_t		*umem_default_arena;
7437c478bd9Sstevel@tonic-gate static vmem_t		*umem_firewall_va_arena;
7447c478bd9Sstevel@tonic-gate static vmem_t		*umem_firewall_arena;
7467c478bd9Sstevel@tonic-gate vmem_t			*umem_memalign_arena;
7487c478bd9Sstevel@tonic-gate umem_log_header_t *umem_transaction_log;
7497c478bd9Sstevel@tonic-gate umem_log_header_t *umem_content_log;
7507c478bd9Sstevel@tonic-gate umem_log_header_t *umem_failure_log;
7517c478bd9Sstevel@tonic-gate umem_log_header_t *umem_slab_log;
7537257d1b4Sraf #define	CPUHINT()		(thr_self())
7547c478bd9Sstevel@tonic-gate #define	CPUHINT_MAX()		INT_MAX
7567c478bd9Sstevel@tonic-gate #define	CPU(mask)		(umem_cpus + (CPUHINT() & (mask)))
7577c478bd9Sstevel@tonic-gate static umem_cpu_t umem_startup_cpu = {	/* initial, single, cpu */
7587c478bd9Sstevel@tonic-gate 	UMEM_CACHE_SIZE(0),
7597c478bd9Sstevel@tonic-gate 	0
7607c478bd9Sstevel@tonic-gate };
7627c478bd9Sstevel@tonic-gate static uint32_t umem_cpu_mask = 0;			/* global cpu mask */
7637c478bd9Sstevel@tonic-gate static umem_cpu_t *umem_cpus = &umem_startup_cpu;	/* cpu list */
7657c478bd9Sstevel@tonic-gate volatile uint32_t umem_reaping;
7677c478bd9Sstevel@tonic-gate thread_t		umem_update_thr;
7687c478bd9Sstevel@tonic-gate struct timeval		umem_update_next;	/* timeofday of next update */
7697c478bd9Sstevel@tonic-gate volatile thread_t	umem_st_update_thr;	/* only used when single-thd */
7717c478bd9Sstevel@tonic-gate #define	IN_UPDATE()	(thr_self() == umem_update_thr || \
7727c478bd9Sstevel@tonic-gate 			    thr_self() == umem_st_update_thr)
7737c478bd9Sstevel@tonic-gate #define	IN_REAP()	IN_UPDATE()
7757c478bd9Sstevel@tonic-gate mutex_t			umem_update_lock;	/* cache_u{next,prev,flags} */
7767c478bd9Sstevel@tonic-gate cond_t			umem_update_cv;
7787c478bd9Sstevel@tonic-gate volatile hrtime_t umem_reap_next;	/* min hrtime of next reap */
7807c478bd9Sstevel@tonic-gate mutex_t			umem_cache_lock;	/* inter-cache linkage only */
7827c478bd9Sstevel@tonic-gate #ifdef UMEM_STANDALONE
7837c478bd9Sstevel@tonic-gate umem_cache_t		umem_null_cache;
7847c478bd9Sstevel@tonic-gate static const umem_cache_t umem_null_cache_template = {
7857c478bd9Sstevel@tonic-gate #else
7867c478bd9Sstevel@tonic-gate umem_cache_t		umem_null_cache = {
7877c478bd9Sstevel@tonic-gate #endif
7887c478bd9Sstevel@tonic-gate 	0, 0, 0, 0, 0,
7897c478bd9Sstevel@tonic-gate 	0, 0,
7907c478bd9Sstevel@tonic-gate 	0, 0,
7917c478bd9Sstevel@tonic-gate 	0, 0,
7927c478bd9Sstevel@tonic-gate 	"invalid_cache",
7937c478bd9Sstevel@tonic-gate 	0, 0,
7947c478bd9Sstevel@tonic-gate 	NULL, NULL, NULL, NULL,
7957c478bd9Sstevel@tonic-gate 	NULL,
7967c478bd9Sstevel@tonic-gate 	0, 0, 0, 0,
7977c478bd9Sstevel@tonic-gate 	&umem_null_cache, &umem_null_cache,
7987c478bd9Sstevel@tonic-gate 	&umem_null_cache, &umem_null_cache,
7997c478bd9Sstevel@tonic-gate 	0,
8007c478bd9Sstevel@tonic-gate 	DEFAULTMUTEX,				/* start of slab layer */
8017c478bd9Sstevel@tonic-gate 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
8027c478bd9Sstevel@tonic-gate 	&umem_null_cache.cache_nullslab,
8037c478bd9Sstevel@tonic-gate 	{
8047c478bd9Sstevel@tonic-gate 		&umem_null_cache,
8057c478bd9Sstevel@tonic-gate 		NULL,
8067c478bd9Sstevel@tonic-gate 		&umem_null_cache.cache_nullslab,
8077c478bd9Sstevel@tonic-gate 		&umem_null_cache.cache_nullslab,
8087c478bd9Sstevel@tonic-gate 		NULL,
8097c478bd9Sstevel@tonic-gate 		-1,
8107c478bd9Sstevel@tonic-gate 		0
8117c478bd9Sstevel@tonic-gate 	},
8127c478bd9Sstevel@tonic-gate 	NULL,
8137c478bd9Sstevel@tonic-gate 	NULL,
8147c478bd9Sstevel@tonic-gate 	DEFAULTMUTEX,				/* start of depot layer */
8157c478bd9Sstevel@tonic-gate 	NULL, {
8167c478bd9Sstevel@tonic-gate 		NULL, 0, 0, 0, 0
8177c478bd9Sstevel@tonic-gate 	}, {
8187c478bd9Sstevel@tonic-gate 		NULL, 0, 0, 0, 0
8197c478bd9Sstevel@tonic-gate 	}, {
8207c478bd9Sstevel@tonic-gate 		{
8217c478bd9Sstevel@tonic-gate 			DEFAULTMUTEX,		/* start of CPU cache */
8227c478bd9Sstevel@tonic-gate 			0, 0, NULL, NULL, -1, -1, 0
8237c478bd9Sstevel@tonic-gate 		}
8247c478bd9Sstevel@tonic-gate 	}
8257c478bd9Sstevel@tonic-gate };
8277c478bd9Sstevel@tonic-gate #define	ALLOC_TABLE_4 \
8287c478bd9Sstevel@tonic-gate 	&umem_null_cache, &umem_null_cache, &umem_null_cache, &umem_null_cache
8307c478bd9Sstevel@tonic-gate #define	ALLOC_TABLE_64 \
8317c478bd9Sstevel@tonic-gate 	ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, \
8327c478bd9Sstevel@tonic-gate 	ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, \
8337c478bd9Sstevel@tonic-gate 	ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, \
8347c478bd9Sstevel@tonic-gate 	ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4
8367c478bd9Sstevel@tonic-gate #define	ALLOC_TABLE_1024 \
8377c478bd9Sstevel@tonic-gate 	ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, \
8387c478bd9Sstevel@tonic-gate 	ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, \
8397c478bd9Sstevel@tonic-gate 	ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, \
8407c478bd9Sstevel@tonic-gate 	ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64
8427c478bd9Sstevel@tonic-gate static umem_cache_t *umem_alloc_table[UMEM_MAXBUF >> UMEM_ALIGN_SHIFT] = {
84338849194SRobert Mustacchi 	ALLOC_TABLE_1024,
84438849194SRobert Mustacchi 	ALLOC_TABLE_1024,
84538849194SRobert Mustacchi 	ALLOC_TABLE_1024,
84638849194SRobert Mustacchi 	ALLOC_TABLE_1024,
84738849194SRobert Mustacchi 	ALLOC_TABLE_1024,
84838849194SRobert Mustacchi 	ALLOC_TABLE_1024,
84938849194SRobert Mustacchi 	ALLOC_TABLE_1024,
85038849194SRobert Mustacchi 	ALLOC_TABLE_1024,
85138849194SRobert Mustacchi 	ALLOC_TABLE_1024,
85238849194SRobert Mustacchi 	ALLOC_TABLE_1024,
85338849194SRobert Mustacchi 	ALLOC_TABLE_1024,
85438849194SRobert Mustacchi 	ALLOC_TABLE_1024,
85538849194SRobert Mustacchi 	ALLOC_TABLE_1024,
85638849194SRobert Mustacchi 	ALLOC_TABLE_1024,
8577c478bd9Sstevel@tonic-gate 	ALLOC_TABLE_1024,
8587c478bd9Sstevel@tonic-gate 	ALLOC_TABLE_1024
8597c478bd9Sstevel@tonic-gate };
8627c478bd9Sstevel@tonic-gate /* Used to constrain audit-log stack traces */
8637c478bd9Sstevel@tonic-gate caddr_t			umem_min_stack;
8647c478bd9Sstevel@tonic-gate caddr_t			umem_max_stack;
8677c478bd9Sstevel@tonic-gate #define	UMERR_MODIFIED	0	/* buffer modified while on freelist */
8687c478bd9Sstevel@tonic-gate #define	UMERR_REDZONE	1	/* redzone violation (write past end of buf) */
8697c478bd9Sstevel@tonic-gate #define	UMERR_DUPFREE	2	/* freed a buffer twice */
8707c478bd9Sstevel@tonic-gate #define	UMERR_BADADDR	3	/* freed a bad (unallocated) address */
8717c478bd9Sstevel@tonic-gate #define	UMERR_BADBUFTAG	4	/* buftag corrupted */
8727c478bd9Sstevel@tonic-gate #define	UMERR_BADBUFCTL	5	/* bufctl corrupted */
8737c478bd9Sstevel@tonic-gate #define	UMERR_BADCACHE	6	/* freed a buffer to the wrong cache */
8747c478bd9Sstevel@tonic-gate #define	UMERR_BADSIZE	7	/* alloc size != free size */
8757c478bd9Sstevel@tonic-gate #define	UMERR_BADBASE	8	/* buffer base address wrong */
8777c478bd9Sstevel@tonic-gate struct {
8787c478bd9Sstevel@tonic-gate 	hrtime_t	ump_timestamp;	/* timestamp of error */
8797c478bd9Sstevel@tonic-gate 	int		ump_error;	/* type of umem error (UMERR_*) */
8807c478bd9Sstevel@tonic-gate 	void		*ump_buffer;	/* buffer that induced abort */
8817c478bd9Sstevel@tonic-gate 	void		*ump_realbuf;	/* real start address for buffer */
8827c478bd9Sstevel@tonic-gate 	umem_cache_t	*ump_cache;	/* buffer's cache according to client */
8837c478bd9Sstevel@tonic-gate 	umem_cache_t	*ump_realcache;	/* actual cache containing buffer */
8847c478bd9Sstevel@tonic-gate 	umem_slab_t	*ump_slab;	/* slab accoring to umem_findslab() */
8857c478bd9Sstevel@tonic-gate 	umem_bufctl_t	*ump_bufctl;	/* bufctl */
8867c478bd9Sstevel@tonic-gate } umem_abort_info;
8887c478bd9Sstevel@tonic-gate static void
copy_pattern(uint64_t pattern,void * buf_arg,size_t size)8897c478bd9Sstevel@tonic-gate copy_pattern(uint64_t pattern, void *buf_arg, size_t size)
8907c478bd9Sstevel@tonic-gate {
8917c478bd9Sstevel@tonic-gate 	uint64_t *bufend = (uint64_t *)((char *)buf_arg + size);
8927c478bd9Sstevel@tonic-gate 	uint64_t *buf = buf_arg;
8947c478bd9Sstevel@tonic-gate 	while (buf < bufend)
8957c478bd9Sstevel@tonic-gate 		*buf++ = pattern;
8967c478bd9Sstevel@tonic-gate }
8987c478bd9Sstevel@tonic-gate static void *
verify_pattern(uint64_t pattern,void * buf_arg,size_t size)8997c478bd9Sstevel@tonic-gate verify_pattern(uint64_t pattern, void *buf_arg, size_t size)
9007c478bd9Sstevel@tonic-gate {
9017c478bd9Sstevel@tonic-gate 	uint64_t *bufend = (uint64_t *)((char *)buf_arg + size);
9027c478bd9Sstevel@tonic-gate 	uint64_t *buf;
9047c478bd9Sstevel@tonic-gate 	for (buf = buf_arg; buf < bufend; buf++)
9057c478bd9Sstevel@tonic-gate 		if (*buf != pattern)
9067c478bd9Sstevel@tonic-gate 			return (buf);
9077c478bd9Sstevel@tonic-gate 	return (NULL);
9087c478bd9Sstevel@tonic-gate }
9107c478bd9Sstevel@tonic-gate static void *
verify_and_copy_pattern(uint64_t old,uint64_t new,void * buf_arg,size_t size)9117c478bd9Sstevel@tonic-gate verify_and_copy_pattern(uint64_t old, uint64_t new, void *buf_arg, size_t size)
9127c478bd9Sstevel@tonic-gate {
9137c478bd9Sstevel@tonic-gate 	uint64_t *bufend = (uint64_t *)((char *)buf_arg + size);
9147c478bd9Sstevel@tonic-gate 	uint64_t *buf;
9167c478bd9Sstevel@tonic-gate 	for (buf = buf_arg; buf < bufend; buf++) {
9177c478bd9Sstevel@tonic-gate 		if (*buf != old) {
9187c478bd9Sstevel@tonic-gate 			copy_pattern(old, buf_arg,
9197c478bd9Sstevel@tonic-gate 			    (char *)buf - (char *)buf_arg);
9207c478bd9Sstevel@tonic-gate 			return (buf);
9217c478bd9Sstevel@tonic-gate 		}
9227c478bd9Sstevel@tonic-gate 		*buf = new;
9237c478bd9Sstevel@tonic-gate 	}
9257c478bd9Sstevel@tonic-gate 	return (NULL);
9267c478bd9Sstevel@tonic-gate }
9287c478bd9Sstevel@tonic-gate void
umem_cache_applyall(void (* func)(umem_cache_t *))9297c478bd9Sstevel@tonic-gate umem_cache_applyall(void (*func)(umem_cache_t *))
9307c478bd9Sstevel@tonic-gate {
9317c478bd9Sstevel@tonic-gate 	umem_cache_t *cp;
9337c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&umem_cache_lock);
9347c478bd9Sstevel@tonic-gate 	for (cp = umem_null_cache.cache_next; cp != &umem_null_cache;
9357c478bd9Sstevel@tonic-gate 	    cp = cp->cache_next)
9367c478bd9Sstevel@tonic-gate 		func(cp);
9377c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&umem_cache_lock);
9387c478bd9Sstevel@tonic-gate }
9407c478bd9Sstevel@tonic-gate static void
umem_add_update_unlocked(umem_cache_t * cp,int flags)9417c478bd9Sstevel@tonic-gate umem_add_update_unlocked(umem_cache_t *cp, int flags)
9427c478bd9Sstevel@tonic-gate {
9437c478bd9Sstevel@tonic-gate 	umem_cache_t *cnext, *cprev;
9457c478bd9Sstevel@tonic-gate 	flags &= ~UMU_ACTIVE;
9477c478bd9Sstevel@tonic-gate 	if (!flags)
9487c478bd9Sstevel@tonic-gate 		return;
9507c478bd9Sstevel@tonic-gate 	if (cp->cache_uflags & UMU_ACTIVE) {
9517c478bd9Sstevel@tonic-gate 		cp->cache_uflags |= flags;
9527c478bd9Sstevel@tonic-gate 	} else {
9537c478bd9Sstevel@tonic-gate 		if (cp->cache_unext != NULL) {
9547c478bd9Sstevel@tonic-gate 			ASSERT(cp->cache_uflags != 0);
9557c478bd9Sstevel@tonic-gate 			cp->cache_uflags |= flags;
9567c478bd9Sstevel@tonic-gate 		} else {
9577c478bd9Sstevel@tonic-gate 			ASSERT(cp->cache_uflags == 0);
9587c478bd9Sstevel@tonic-gate 			cp->cache_uflags = flags;
9597c478bd9Sstevel@tonic-gate 			cp->cache_unext = cnext = &umem_null_cache;
9607c478bd9Sstevel@tonic-gate 			cp->cache_uprev = cprev = umem_null_cache.cache_uprev;
9617c478bd9Sstevel@tonic-gate 			cnext->cache_uprev = cp;
9627c478bd9Sstevel@tonic-gate 			cprev->cache_unext = cp;
9637c478bd9Sstevel@tonic-gate 		}
9647c478bd9Sstevel@tonic-gate 	}
9657c478bd9Sstevel@tonic-gate }
9677c478bd9Sstevel@tonic-gate static void
umem_add_update(umem_cache_t * cp,int flags)9687c478bd9Sstevel@tonic-gate umem_add_update(umem_cache_t *cp, int flags)
9697c478bd9Sstevel@tonic-gate {
9707c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&umem_update_lock);
9727c478bd9Sstevel@tonic-gate 	umem_add_update_unlocked(cp, flags);
9747c478bd9Sstevel@tonic-gate 	if (!IN_UPDATE())
9757c478bd9Sstevel@tonic-gate 		(void) cond_broadcast(&umem_update_cv);
9777c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&umem_update_lock);
9787c478bd9Sstevel@tonic-gate }
9807c478bd9Sstevel@tonic-gate /*
9817c478bd9Sstevel@tonic-gate  * Remove a cache from the update list, waiting for any in-progress work to
9827c478bd9Sstevel@tonic-gate  * complete first.
9837c478bd9Sstevel@tonic-gate  */
9847c478bd9Sstevel@tonic-gate static void
umem_remove_updates(umem_cache_t * cp)9857c478bd9Sstevel@tonic-gate umem_remove_updates(umem_cache_t *cp)
9867c478bd9Sstevel@tonic-gate {
9877c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&umem_update_lock);
9897c478bd9Sstevel@tonic-gate 	/*
9907c478bd9Sstevel@tonic-gate 	 * Get it out of the active state
9917c478bd9Sstevel@tonic-gate 	 */
9927c478bd9Sstevel@tonic-gate 	while (cp->cache_uflags & UMU_ACTIVE) {
993a574db85Sraf 		int cancel_state;
9957c478bd9Sstevel@tonic-gate 		ASSERT(cp->cache_unext == NULL);
9977c478bd9Sstevel@tonic-gate 		cp->cache_uflags |= UMU_NOTIFY;
9997c478bd9Sstevel@tonic-gate 		/*
10007c478bd9Sstevel@tonic-gate 		 * Make sure the update state is sane, before we wait
10017c478bd9Sstevel@tonic-gate 		 */
10027c478bd9Sstevel@tonic-gate 		ASSERT(umem_update_thr != 0 || umem_st_update_thr != 0);
10037c478bd9Sstevel@tonic-gate 		ASSERT(umem_update_thr != thr_self() &&
10047c478bd9Sstevel@tonic-gate 		    umem_st_update_thr != thr_self());
1006a574db85Sraf 		(void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE,
1007a574db85Sraf 		    &cancel_state);
1008a574db85Sraf 		(void) cond_wait(&umem_update_cv, &umem_update_lock);
1009a574db85Sraf 		(void) pthread_setcancelstate(cancel_state, NULL);
10107c478bd9Sstevel@tonic-gate 	}
10117c478bd9Sstevel@tonic-gate 	/*
10127c478bd9Sstevel@tonic-gate 	 * Get it out of the Work Requested state
10137c478bd9Sstevel@tonic-gate 	 */
10147c478bd9Sstevel@tonic-gate 	if (cp->cache_unext != NULL) {
10157c478bd9Sstevel@tonic-gate 		cp->cache_uprev->cache_unext = cp->cache_unext;
10167c478bd9Sstevel@tonic-gate 		cp->cache_unext->cache_uprev = cp->cache_uprev;
10177c478bd9Sstevel@tonic-gate 		cp->cache_uprev = cp->cache_unext = NULL;
10187c478bd9Sstevel@tonic-gate 		cp->cache_uflags = 0;
10197c478bd9Sstevel@tonic-gate 	}
10207c478bd9Sstevel@tonic-gate 	/*
10217c478bd9Sstevel@tonic-gate 	 * Make sure it is in the Inactive state
10227c478bd9Sstevel@tonic-gate 	 */
10237c478bd9Sstevel@tonic-gate 	ASSERT(cp->cache_unext == NULL && cp->cache_uflags == 0);
10247c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&umem_update_lock);
10257c478bd9Sstevel@tonic-gate }
10277c478bd9Sstevel@tonic-gate static void
umem_updateall(int flags)10287c478bd9Sstevel@tonic-gate umem_updateall(int flags)
10297c478bd9Sstevel@tonic-gate {
10307c478bd9Sstevel@tonic-gate 	umem_cache_t *cp;
10327c478bd9Sstevel@tonic-gate 	/*
10337c478bd9Sstevel@tonic-gate 	 * NOTE:  To prevent deadlock, umem_cache_lock is always acquired first.
10347c478bd9Sstevel@tonic-gate 	 *
10357c478bd9Sstevel@tonic-gate 	 * (umem_add_update is called from things run via umem_cache_applyall)
10367c478bd9Sstevel@tonic-gate 	 */
10377c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&umem_cache_lock);
10387c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&umem_update_lock);
10407c478bd9Sstevel@tonic-gate 	for (cp = umem_null_cache.cache_next; cp != &umem_null_cache;
10417c478bd9Sstevel@tonic-gate 	    cp = cp->cache_next)
10427c478bd9Sstevel@tonic-gate 		umem_add_update_unlocked(cp, flags);
10447c478bd9Sstevel@tonic-gate 	if (!IN_UPDATE())
10457c478bd9Sstevel@tonic-gate 		(void) cond_broadcast(&umem_update_cv);
10477c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&umem_update_lock);
10487c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&umem_cache_lock);
10497c478bd9Sstevel@tonic-gate }
10517c478bd9Sstevel@tonic-gate /*
10527c478bd9Sstevel@tonic-gate  * Debugging support.  Given a buffer address, find its slab.
10537c478bd9Sstevel@tonic-gate  */
10547c478bd9Sstevel@tonic-gate static umem_slab_t *
umem_findslab(umem_cache_t * cp,void * buf)10557c478bd9Sstevel@tonic-gate umem_findslab(umem_cache_t *cp, void *buf)
10567c478bd9Sstevel@tonic-gate {