17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 57c478bd9Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 67c478bd9Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 77c478bd9Sstevel@tonic-gate * with the License. 87c478bd9Sstevel@tonic-gate * 97c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 107c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 117c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 127c478bd9Sstevel@tonic-gate * and limitations under the License. 137c478bd9Sstevel@tonic-gate * 147c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 157c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 167c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 177c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 187c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 197c478bd9Sstevel@tonic-gate * 207c478bd9Sstevel@tonic-gate * CDDL HEADER END 217c478bd9Sstevel@tonic-gate */ 227c478bd9Sstevel@tonic-gate /* 23*1c326e94Sjwadams * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 247c478bd9Sstevel@tonic-gate * Use is subject to license terms. 257c478bd9Sstevel@tonic-gate */ 267c478bd9Sstevel@tonic-gate 277c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate /* 307c478bd9Sstevel@tonic-gate * based on usr/src/uts/common/os/kmem.c r1.64 from 2001/12/18 317c478bd9Sstevel@tonic-gate * 327c478bd9Sstevel@tonic-gate * The slab allocator, as described in the following two papers: 337c478bd9Sstevel@tonic-gate * 347c478bd9Sstevel@tonic-gate * Jeff Bonwick, 357c478bd9Sstevel@tonic-gate * The Slab Allocator: An Object-Caching Kernel Memory Allocator. 367c478bd9Sstevel@tonic-gate * Proceedings of the Summer 1994 Usenix Conference. 377c478bd9Sstevel@tonic-gate * Available as /shared/sac/PSARC/1994/028/materials/kmem.pdf. 387c478bd9Sstevel@tonic-gate * 397c478bd9Sstevel@tonic-gate * Jeff Bonwick and Jonathan Adams, 407c478bd9Sstevel@tonic-gate * Magazines and vmem: Extending the Slab Allocator to Many CPUs and 417c478bd9Sstevel@tonic-gate * Arbitrary Resources. 427c478bd9Sstevel@tonic-gate * Proceedings of the 2001 Usenix Conference. 437c478bd9Sstevel@tonic-gate * Available as /shared/sac/PSARC/2000/550/materials/vmem.pdf. 447c478bd9Sstevel@tonic-gate * 457c478bd9Sstevel@tonic-gate * 1. Overview 467c478bd9Sstevel@tonic-gate * ----------- 477c478bd9Sstevel@tonic-gate * umem is very close to kmem in implementation. There are four major 487c478bd9Sstevel@tonic-gate * areas of divergence: 497c478bd9Sstevel@tonic-gate * 507c478bd9Sstevel@tonic-gate * * Initialization 517c478bd9Sstevel@tonic-gate * 527c478bd9Sstevel@tonic-gate * * CPU handling 537c478bd9Sstevel@tonic-gate * 547c478bd9Sstevel@tonic-gate * * umem_update() 557c478bd9Sstevel@tonic-gate * 567c478bd9Sstevel@tonic-gate * * KM_SLEEP v.s. UMEM_NOFAIL 577c478bd9Sstevel@tonic-gate * 58*1c326e94Sjwadams * * lock ordering 597c478bd9Sstevel@tonic-gate * 607c478bd9Sstevel@tonic-gate * 2. Initialization 617c478bd9Sstevel@tonic-gate * ----------------- 627c478bd9Sstevel@tonic-gate * kmem is initialized early on in boot, and knows that no one will call 637c478bd9Sstevel@tonic-gate * into it before it is ready. umem does not have these luxuries. Instead, 647c478bd9Sstevel@tonic-gate * initialization is divided into two phases: 657c478bd9Sstevel@tonic-gate * 667c478bd9Sstevel@tonic-gate * * library initialization, and 677c478bd9Sstevel@tonic-gate * 687c478bd9Sstevel@tonic-gate * * first use 697c478bd9Sstevel@tonic-gate * 707c478bd9Sstevel@tonic-gate * umem's full initialization happens at the time of the first allocation 717c478bd9Sstevel@tonic-gate * request (via malloc() and friends, umem_alloc(), or umem_zalloc()), 727c478bd9Sstevel@tonic-gate * or the first call to umem_cache_create(). 737c478bd9Sstevel@tonic-gate * 747c478bd9Sstevel@tonic-gate * umem_free(), and umem_cache_alloc() do not require special handling, 757c478bd9Sstevel@tonic-gate * since the only way to get valid arguments for them is to successfully 767c478bd9Sstevel@tonic-gate * call a function from the first group. 777c478bd9Sstevel@tonic-gate * 787c478bd9Sstevel@tonic-gate * 2.1. Library Initialization: umem_startup() 797c478bd9Sstevel@tonic-gate * ------------------------------------------- 807c478bd9Sstevel@tonic-gate * umem_startup() is libumem.so's .init section. It calls pthread_atfork() 817c478bd9Sstevel@tonic-gate * to install the handlers necessary for umem's Fork1-Safety. Because of 827c478bd9Sstevel@tonic-gate * race condition issues, all other pre-umem_init() initialization is done 837c478bd9Sstevel@tonic-gate * statically (i.e. by the dynamic linker). 847c478bd9Sstevel@tonic-gate * 857c478bd9Sstevel@tonic-gate * For standalone use, umem_startup() returns everything to its initial 867c478bd9Sstevel@tonic-gate * state. 877c478bd9Sstevel@tonic-gate * 887c478bd9Sstevel@tonic-gate * 2.2. First use: umem_init() 897c478bd9Sstevel@tonic-gate * ------------------------------ 907c478bd9Sstevel@tonic-gate * The first time any memory allocation function is used, we have to 917c478bd9Sstevel@tonic-gate * create the backing caches and vmem arenas which are needed for it. 927c478bd9Sstevel@tonic-gate * umem_init() is the central point for that task. When it completes, 937c478bd9Sstevel@tonic-gate * umem_ready is either UMEM_READY (all set) or UMEM_READY_INIT_FAILED (unable 947c478bd9Sstevel@tonic-gate * to initialize, probably due to lack of memory). 957c478bd9Sstevel@tonic-gate * 967c478bd9Sstevel@tonic-gate * There are four different paths from which umem_init() is called: 977c478bd9Sstevel@tonic-gate * 987c478bd9Sstevel@tonic-gate * * from umem_alloc() or umem_zalloc(), with 0 < size < UMEM_MAXBUF, 997c478bd9Sstevel@tonic-gate * 1007c478bd9Sstevel@tonic-gate * * from umem_alloc() or umem_zalloc(), with size > UMEM_MAXBUF, 1017c478bd9Sstevel@tonic-gate * 1027c478bd9Sstevel@tonic-gate * * from umem_cache_create(), and 1037c478bd9Sstevel@tonic-gate * 1047c478bd9Sstevel@tonic-gate * * from memalign(), with align > UMEM_ALIGN. 1057c478bd9Sstevel@tonic-gate * 1067c478bd9Sstevel@tonic-gate * The last three just check if umem is initialized, and call umem_init() 1077c478bd9Sstevel@tonic-gate * if it is not. For performance reasons, the first case is more complicated. 1087c478bd9Sstevel@tonic-gate * 1097c478bd9Sstevel@tonic-gate * 2.2.1. umem_alloc()/umem_zalloc(), with 0 < size < UMEM_MAXBUF 1107c478bd9Sstevel@tonic-gate * ----------------------------------------------------------------- 1117c478bd9Sstevel@tonic-gate * In this case, umem_cache_alloc(&umem_null_cache, ...) is called. 1127c478bd9Sstevel@tonic-gate * There is special case code in which causes any allocation on 1137c478bd9Sstevel@tonic-gate * &umem_null_cache to fail by returning (NULL), regardless of the 1147c478bd9Sstevel@tonic-gate * flags argument. 1157c478bd9Sstevel@tonic-gate * 1167c478bd9Sstevel@tonic-gate * So umem_cache_alloc() returns NULL, and umem_alloc()/umem_zalloc() call 1177c478bd9Sstevel@tonic-gate * umem_alloc_retry(). umem_alloc_retry() sees that the allocation 1187c478bd9Sstevel@tonic-gate * was agains &umem_null_cache, and calls umem_init(). 1197c478bd9Sstevel@tonic-gate * 1207c478bd9Sstevel@tonic-gate * If initialization is successful, umem_alloc_retry() returns 1, which 1217c478bd9Sstevel@tonic-gate * causes umem_alloc()/umem_zalloc() to start over, which causes it to load 1227c478bd9Sstevel@tonic-gate * the (now valid) cache pointer from umem_alloc_table. 1237c478bd9Sstevel@tonic-gate * 1247c478bd9Sstevel@tonic-gate * 2.2.2. Dealing with race conditions 1257c478bd9Sstevel@tonic-gate * ----------------------------------- 1267c478bd9Sstevel@tonic-gate * There are a couple race conditions resulting from the initialization 1277c478bd9Sstevel@tonic-gate * code that we have to guard against: 1287c478bd9Sstevel@tonic-gate * 1297c478bd9Sstevel@tonic-gate * * In umem_cache_create(), there is a special UMC_INTERNAL cflag 1307c478bd9Sstevel@tonic-gate * that is passed for caches created during initialization. It 1317c478bd9Sstevel@tonic-gate * is illegal for a user to try to create a UMC_INTERNAL cache. 1327c478bd9Sstevel@tonic-gate * This allows initialization to proceed, but any other 1337c478bd9Sstevel@tonic-gate * umem_cache_create()s will block by calling umem_init(). 1347c478bd9Sstevel@tonic-gate * 1357c478bd9Sstevel@tonic-gate * * Since umem_null_cache has a 1-element cache_cpu, it's cache_cpu_mask 1367c478bd9Sstevel@tonic-gate * is always zero. umem_cache_alloc uses cp->cache_cpu_mask to 1377c478bd9Sstevel@tonic-gate * mask the cpu number. This prevents a race between grabbing a 1387c478bd9Sstevel@tonic-gate * cache pointer out of umem_alloc_table and growing the cpu array. 1397c478bd9Sstevel@tonic-gate * 1407c478bd9Sstevel@tonic-gate * 1417c478bd9Sstevel@tonic-gate * 3. CPU handling 1427c478bd9Sstevel@tonic-gate * --------------- 1437c478bd9Sstevel@tonic-gate * kmem uses the CPU's sequence number to determine which "cpu cache" to 1447c478bd9Sstevel@tonic-gate * use for an allocation. Currently, there is no way to get the sequence 1457c478bd9Sstevel@tonic-gate * number in userspace. 1467c478bd9Sstevel@tonic-gate * 1477c478bd9Sstevel@tonic-gate * umem keeps track of cpu information in umem_cpus, an array of umem_max_ncpus 1487c478bd9Sstevel@tonic-gate * umem_cpu_t structures. CURCPU() is a a "hint" function, which we then mask 1497c478bd9Sstevel@tonic-gate * with either umem_cpu_mask or cp->cache_cpu_mask to find the actual "cpu" id. 1507c478bd9Sstevel@tonic-gate * The mechanics of this is all in the CPU(mask) macro. 1517c478bd9Sstevel@tonic-gate * 1527c478bd9Sstevel@tonic-gate * Currently, umem uses _lwp_self() as its hint. 1537c478bd9Sstevel@tonic-gate * 1547c478bd9Sstevel@tonic-gate * 1557c478bd9Sstevel@tonic-gate * 4. The update thread 1567c478bd9Sstevel@tonic-gate * -------------------- 1577c478bd9Sstevel@tonic-gate * kmem uses a task queue, kmem_taskq, to do periodic maintenance on 1587c478bd9Sstevel@tonic-gate * every kmem cache. vmem has a periodic timeout for hash table resizing. 1597c478bd9Sstevel@tonic-gate * The kmem_taskq also provides a separate context for kmem_cache_reap()'s 1607c478bd9Sstevel@tonic-gate * to be done in, avoiding issues of the context of kmem_reap() callers. 1617c478bd9Sstevel@tonic-gate * 1627c478bd9Sstevel@tonic-gate * Instead, umem has the concept of "updates", which are asynchronous requests 1637c478bd9Sstevel@tonic-gate * for work attached to single caches. All caches with pending work are 1647c478bd9Sstevel@tonic-gate * on a doubly linked list rooted at the umem_null_cache. All update state 1657c478bd9Sstevel@tonic-gate * is protected by the umem_update_lock mutex, and the umem_update_cv is used 1667c478bd9Sstevel@tonic-gate * for notification between threads. 1677c478bd9Sstevel@tonic-gate * 1687c478bd9Sstevel@tonic-gate * 4.1. Cache states with regards to updates 1697c478bd9Sstevel@tonic-gate * ----------------------------------------- 1707c478bd9Sstevel@tonic-gate * A given cache is in one of three states: 1717c478bd9Sstevel@tonic-gate * 1727c478bd9Sstevel@tonic-gate * Inactive cache_uflags is zero, cache_u{next,prev} are NULL 1737c478bd9Sstevel@tonic-gate * 1747c478bd9Sstevel@tonic-gate * Work Requested cache_uflags is non-zero (but UMU_ACTIVE is not set), 1757c478bd9Sstevel@tonic-gate * cache_u{next,prev} link the cache onto the global 1767c478bd9Sstevel@tonic-gate * update list 1777c478bd9Sstevel@tonic-gate * 1787c478bd9Sstevel@tonic-gate * Active cache_uflags has UMU_ACTIVE set, cache_u{next,prev} 1797c478bd9Sstevel@tonic-gate * are NULL, and either umem_update_thr or 1807c478bd9Sstevel@tonic-gate * umem_st_update_thr are actively doing work on the 1817c478bd9Sstevel@tonic-gate * cache. 1827c478bd9Sstevel@tonic-gate * 1837c478bd9Sstevel@tonic-gate * An update can be added to any cache in any state -- if the cache is 1847c478bd9Sstevel@tonic-gate * Inactive, it transitions to being Work Requested. If the cache is 1857c478bd9Sstevel@tonic-gate * Active, the worker will notice the new update and act on it before 1867c478bd9Sstevel@tonic-gate * transitioning the cache to the Inactive state. 1877c478bd9Sstevel@tonic-gate * 1887c478bd9Sstevel@tonic-gate * If a cache is in the Active state, UMU_NOTIFY can be set, which asks 1897c478bd9Sstevel@tonic-gate * the worker to broadcast the umem_update_cv when it has finished. 1907c478bd9Sstevel@tonic-gate * 1917c478bd9Sstevel@tonic-gate * 4.2. Update interface 1927c478bd9Sstevel@tonic-gate * --------------------- 1937c478bd9Sstevel@tonic-gate * umem_add_update() adds an update to a particular cache. 1947c478bd9Sstevel@tonic-gate * umem_updateall() adds an update to all caches. 1957c478bd9Sstevel@tonic-gate * umem_remove_updates() returns a cache to the Inactive state. 1967c478bd9Sstevel@tonic-gate * 1977c478bd9Sstevel@tonic-gate * umem_process_updates() process all caches in the Work Requested state. 1987c478bd9Sstevel@tonic-gate * 1997c478bd9Sstevel@tonic-gate * 4.3. Reaping 2007c478bd9Sstevel@tonic-gate * ------------ 2017c478bd9Sstevel@tonic-gate * When umem_reap() is called (at the time of heap growth), it schedule 2027c478bd9Sstevel@tonic-gate * UMU_REAP updates on every cache. It then checks to see if the update 2037c478bd9Sstevel@tonic-gate * thread exists (umem_update_thr != 0). If it is, it broadcasts 2047c478bd9Sstevel@tonic-gate * the umem_update_cv to wake the update thread up, and returns. 2057c478bd9Sstevel@tonic-gate * 2067c478bd9Sstevel@tonic-gate * If the update thread does not exist (umem_update_thr == 0), and the 2077c478bd9Sstevel@tonic-gate * program currently has multiple threads, umem_reap() attempts to create 2087c478bd9Sstevel@tonic-gate * a new update thread. 2097c478bd9Sstevel@tonic-gate * 2107c478bd9Sstevel@tonic-gate * If the process is not multithreaded, or the creation fails, umem_reap() 2117c478bd9Sstevel@tonic-gate * calls umem_st_update() to do an inline update. 2127c478bd9Sstevel@tonic-gate * 2137c478bd9Sstevel@tonic-gate * 4.4. The update thread 2147c478bd9Sstevel@tonic-gate * ---------------------- 2157c478bd9Sstevel@tonic-gate * The update thread spends most of its time in cond_timedwait() on the 2167c478bd9Sstevel@tonic-gate * umem_update_cv. It wakes up under two conditions: 2177c478bd9Sstevel@tonic-gate * 2187c478bd9Sstevel@tonic-gate * * The timedwait times out, in which case it needs to run a global 2197c478bd9Sstevel@tonic-gate * update, or 2207c478bd9Sstevel@tonic-gate * 2217c478bd9Sstevel@tonic-gate * * someone cond_broadcast(3THR)s the umem_update_cv, in which case 2227c478bd9Sstevel@tonic-gate * it needs to check if there are any caches in the Work Requested 2237c478bd9Sstevel@tonic-gate * state. 2247c478bd9Sstevel@tonic-gate * 2257c478bd9Sstevel@tonic-gate * When it is time for another global update, umem calls umem_cache_update() 2267c478bd9Sstevel@tonic-gate * on every cache, then calls vmem_update(), which tunes the vmem structures. 2277c478bd9Sstevel@tonic-gate * umem_cache_update() can request further work using umem_add_update(). 2287c478bd9Sstevel@tonic-gate * 2297c478bd9Sstevel@tonic-gate * After any work from the global update completes, the update timer is 2307c478bd9Sstevel@tonic-gate * reset to umem_reap_interval seconds in the future. This makes the 2317c478bd9Sstevel@tonic-gate * updates self-throttling. 2327c478bd9Sstevel@tonic-gate * 2337c478bd9Sstevel@tonic-gate * Reaps are similarly self-throttling. After a UMU_REAP update has 2347c478bd9Sstevel@tonic-gate * been scheduled on all caches, umem_reap() sets a flag and wakes up the 2357c478bd9Sstevel@tonic-gate * update thread. The update thread notices the flag, and resets the 2367c478bd9Sstevel@tonic-gate * reap state. 2377c478bd9Sstevel@tonic-gate * 2387c478bd9Sstevel@tonic-gate * 4.5. Inline updates 2397c478bd9Sstevel@tonic-gate * ------------------- 2407c478bd9Sstevel@tonic-gate * If the update thread is not running, umem_st_update() is used instead. It 2417c478bd9Sstevel@tonic-gate * immediately does a global update (as above), then calls 2427c478bd9Sstevel@tonic-gate * umem_process_updates() to process both the reaps that umem_reap() added and 2437c478bd9Sstevel@tonic-gate * any work generated by the global update. Afterwards, it resets the reap 2447c478bd9Sstevel@tonic-gate * state. 2457c478bd9Sstevel@tonic-gate * 2467c478bd9Sstevel@tonic-gate * While the umem_st_update() is running, umem_st_update_thr holds the thread 2477c478bd9Sstevel@tonic-gate * id of the thread performing the update. 2487c478bd9Sstevel@tonic-gate * 2497c478bd9Sstevel@tonic-gate * 4.6. Updates and fork1() 2507c478bd9Sstevel@tonic-gate * ------------------------ 2517c478bd9Sstevel@tonic-gate * umem has fork1() pre- and post-handlers which lock up (and release) every 2527c478bd9Sstevel@tonic-gate * mutex in every cache. They also lock up the umem_update_lock. Since 2537c478bd9Sstevel@tonic-gate * fork1() only copies over a single lwp, other threads (including the update 2547c478bd9Sstevel@tonic-gate * thread) could have been actively using a cache in the parent. This 2557c478bd9Sstevel@tonic-gate * can lead to inconsistencies in the child process. 2567c478bd9Sstevel@tonic-gate * 2577c478bd9Sstevel@tonic-gate * Because we locked all of the mutexes, the only possible inconsistancies are: 2587c478bd9Sstevel@tonic-gate * 2597c478bd9Sstevel@tonic-gate * * a umem_cache_alloc() could leak its buffer. 2607c478bd9Sstevel@tonic-gate * 2617c478bd9Sstevel@tonic-gate * * a caller of umem_depot_alloc() could leak a magazine, and all the 2627c478bd9Sstevel@tonic-gate * buffers contained in it. 2637c478bd9Sstevel@tonic-gate * 2647c478bd9Sstevel@tonic-gate * * a cache could be in the Active update state. In the child, there 2657c478bd9Sstevel@tonic-gate * would be no thread actually working on it. 2667c478bd9Sstevel@tonic-gate * 2677c478bd9Sstevel@tonic-gate * * a umem_hash_rescale() could leak the new hash table. 2687c478bd9Sstevel@tonic-gate * 2697c478bd9Sstevel@tonic-gate * * a umem_magazine_resize() could be in progress. 2707c478bd9Sstevel@tonic-gate * 2717c478bd9Sstevel@tonic-gate * * a umem_reap() could be in progress. 2727c478bd9Sstevel@tonic-gate * 2737c478bd9Sstevel@tonic-gate * The memory leaks we can't do anything about. umem_release_child() resets 2747c478bd9Sstevel@tonic-gate * the update state, moves any caches in the Active state to the Work Requested 2757c478bd9Sstevel@tonic-gate * state. This might cause some updates to be re-run, but UMU_REAP and 2767c478bd9Sstevel@tonic-gate * UMU_HASH_RESCALE are effectively idempotent, and the worst that can 2777c478bd9Sstevel@tonic-gate * happen from umem_magazine_resize() is resizing the magazine twice in close 2787c478bd9Sstevel@tonic-gate * succession. 2797c478bd9Sstevel@tonic-gate * 2807c478bd9Sstevel@tonic-gate * Much of the cleanup in umem_release_child() is skipped if 2817c478bd9Sstevel@tonic-gate * umem_st_update_thr == thr_self(). This is so that applications which call 2827c478bd9Sstevel@tonic-gate * fork1() from a cache callback does not break. Needless to say, any such 2837c478bd9Sstevel@tonic-gate * application is tremendously broken. 2847c478bd9Sstevel@tonic-gate * 2857c478bd9Sstevel@tonic-gate * 2867c478bd9Sstevel@tonic-gate * 5. KM_SLEEP v.s. UMEM_NOFAIL 2877c478bd9Sstevel@tonic-gate * ---------------------------- 2887c478bd9Sstevel@tonic-gate * Allocations against kmem and vmem have two basic modes: SLEEP and 2897c478bd9Sstevel@tonic-gate * NOSLEEP. A sleeping allocation is will go to sleep (waiting for 2907c478bd9Sstevel@tonic-gate * more memory) instead of failing (returning NULL). 2917c478bd9Sstevel@tonic-gate * 2927c478bd9Sstevel@tonic-gate * SLEEP allocations presume an extremely multithreaded model, with 2937c478bd9Sstevel@tonic-gate * a lot of allocation and deallocation activity. umem cannot presume 2947c478bd9Sstevel@tonic-gate * that its clients have any particular type of behavior. Instead, 2957c478bd9Sstevel@tonic-gate * it provides two types of allocations: 2967c478bd9Sstevel@tonic-gate * 2977c478bd9Sstevel@tonic-gate * * UMEM_DEFAULT, equivalent to KM_NOSLEEP (i.e. return NULL on 2987c478bd9Sstevel@tonic-gate * failure) 2997c478bd9Sstevel@tonic-gate * 3007c478bd9Sstevel@tonic-gate * * UMEM_NOFAIL, which, on failure, calls an optional callback 3017c478bd9Sstevel@tonic-gate * (registered with umem_nofail_callback()). 3027c478bd9Sstevel@tonic-gate * 3037c478bd9Sstevel@tonic-gate * The callback is invoked with no locks held, and can do an arbitrary 3047c478bd9Sstevel@tonic-gate * amount of work. It then has a choice between: 3057c478bd9Sstevel@tonic-gate * 3067c478bd9Sstevel@tonic-gate * * Returning UMEM_CALLBACK_RETRY, which will cause the allocation 3077c478bd9Sstevel@tonic-gate * to be restarted. 3087c478bd9Sstevel@tonic-gate * 3097c478bd9Sstevel@tonic-gate * * Returning UMEM_CALLBACK_EXIT(status), which will cause exit(2) 3107c478bd9Sstevel@tonic-gate * to be invoked with status. If multiple threads attempt to do 3117c478bd9Sstevel@tonic-gate * this simultaneously, only one will call exit(2). 3127c478bd9Sstevel@tonic-gate * 3137c478bd9Sstevel@tonic-gate * * Doing some kind of non-local exit (thr_exit(3thr), longjmp(3C), 3147c478bd9Sstevel@tonic-gate * etc.) 3157c478bd9Sstevel@tonic-gate * 3167c478bd9Sstevel@tonic-gate * The default callback returns UMEM_CALLBACK_EXIT(255). 3177c478bd9Sstevel@tonic-gate * 3187c478bd9Sstevel@tonic-gate * To have these callbacks without risk of state corruption (in the case of 3197c478bd9Sstevel@tonic-gate * a non-local exit), we have to ensure that the callbacks get invoked 3207c478bd9Sstevel@tonic-gate * close to the original allocation, with no inconsistent state or held 3217c478bd9Sstevel@tonic-gate * locks. The following steps are taken: 3227c478bd9Sstevel@tonic-gate * 3237c478bd9Sstevel@tonic-gate * * All invocations of vmem are VM_NOSLEEP. 3247c478bd9Sstevel@tonic-gate * 3257c478bd9Sstevel@tonic-gate * * All constructor callbacks (which can themselves to allocations) 3267c478bd9Sstevel@tonic-gate * are passed UMEM_DEFAULT as their required allocation argument. This 3277c478bd9Sstevel@tonic-gate * way, the constructor will fail, allowing the highest-level allocation 3287c478bd9Sstevel@tonic-gate * invoke the nofail callback. 3297c478bd9Sstevel@tonic-gate * 3307c478bd9Sstevel@tonic-gate * If a constructor callback _does_ do a UMEM_NOFAIL allocation, and 3317c478bd9Sstevel@tonic-gate * the nofail callback does a non-local exit, we will leak the 3327c478bd9Sstevel@tonic-gate * partially-constructed buffer. 333*1c326e94Sjwadams * 334*1c326e94Sjwadams * 335*1c326e94Sjwadams * 6. Lock Ordering 336*1c326e94Sjwadams * ---------------- 337*1c326e94Sjwadams * umem has a few more locks than kmem does, mostly in the update path. The 338*1c326e94Sjwadams * overall lock ordering (earlier locks must be acquired first) is: 339*1c326e94Sjwadams * 340*1c326e94Sjwadams * umem_init_lock 341*1c326e94Sjwadams * 342*1c326e94Sjwadams * vmem_list_lock 343*1c326e94Sjwadams * vmem_nosleep_lock.vmpl_mutex 344*1c326e94Sjwadams * vmem_t's: 345*1c326e94Sjwadams * vm_lock 346*1c326e94Sjwadams * sbrk_faillock 347*1c326e94Sjwadams * 348*1c326e94Sjwadams * umem_cache_lock 349*1c326e94Sjwadams * umem_update_lock 350*1c326e94Sjwadams * umem_flags_lock 351*1c326e94Sjwadams * umem_cache_t's: 352*1c326e94Sjwadams * cache_cpu[*].cc_lock 353*1c326e94Sjwadams * cache_depot_lock 354*1c326e94Sjwadams * cache_lock 355*1c326e94Sjwadams * umem_log_header_t's: 356*1c326e94Sjwadams * lh_cpu[*].clh_lock 357*1c326e94Sjwadams * lh_lock 3587c478bd9Sstevel@tonic-gate */ 3597c478bd9Sstevel@tonic-gate 3607c478bd9Sstevel@tonic-gate #include "mtlib.h" 3617c478bd9Sstevel@tonic-gate #include <umem_impl.h> 3627c478bd9Sstevel@tonic-gate #include <sys/vmem_impl_user.h> 3637c478bd9Sstevel@tonic-gate #include "umem_base.h" 3647c478bd9Sstevel@tonic-gate #include "vmem_base.h" 3657c478bd9Sstevel@tonic-gate 3667c478bd9Sstevel@tonic-gate #include <sys/processor.h> 3677c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 3687c478bd9Sstevel@tonic-gate 3697c478bd9Sstevel@tonic-gate #include <alloca.h> 3707c478bd9Sstevel@tonic-gate #include <errno.h> 3717c478bd9Sstevel@tonic-gate #include <limits.h> 3727c478bd9Sstevel@tonic-gate #include <stdio.h> 3737c478bd9Sstevel@tonic-gate #include <stdlib.h> 3747c478bd9Sstevel@tonic-gate #include <string.h> 3757c478bd9Sstevel@tonic-gate #include <strings.h> 3767c478bd9Sstevel@tonic-gate #include <signal.h> 3777c478bd9Sstevel@tonic-gate #include <unistd.h> 3787c478bd9Sstevel@tonic-gate #include <atomic.h> 3797c478bd9Sstevel@tonic-gate 3807c478bd9Sstevel@tonic-gate #include "misc.h" 3817c478bd9Sstevel@tonic-gate 3827c478bd9Sstevel@tonic-gate #define UMEM_VMFLAGS(umflag) (VM_NOSLEEP) 3837c478bd9Sstevel@tonic-gate 3847c478bd9Sstevel@tonic-gate size_t pagesize; 3857c478bd9Sstevel@tonic-gate 3867c478bd9Sstevel@tonic-gate /* 3877c478bd9Sstevel@tonic-gate * The default set of caches to back umem_alloc(). 3887c478bd9Sstevel@tonic-gate * These sizes should be reevaluated periodically. 3897c478bd9Sstevel@tonic-gate * 3907c478bd9Sstevel@tonic-gate * We want allocations that are multiples of the coherency granularity 3917c478bd9Sstevel@tonic-gate * (64 bytes) to be satisfied from a cache which is a multiple of 64 3927c478bd9Sstevel@tonic-gate * bytes, so that it will be 64-byte aligned. For all multiples of 64, 3937c478bd9Sstevel@tonic-gate * the next kmem_cache_size greater than or equal to it must be a 3947c478bd9Sstevel@tonic-gate * multiple of 64. 3957c478bd9Sstevel@tonic-gate */ 3967c478bd9Sstevel@tonic-gate static const int umem_alloc_sizes[] = { 3977c478bd9Sstevel@tonic-gate #ifdef _LP64 3987c478bd9Sstevel@tonic-gate 1 * 8, 3997c478bd9Sstevel@tonic-gate 1 * 16, 4007c478bd9Sstevel@tonic-gate 2 * 16, 4017c478bd9Sstevel@tonic-gate 3 * 16, 4027c478bd9Sstevel@tonic-gate #else 4037c478bd9Sstevel@tonic-gate 1 * 8, 4047c478bd9Sstevel@tonic-gate 2 * 8, 4057c478bd9Sstevel@tonic-gate 3 * 8, 4067c478bd9Sstevel@tonic-gate 4 * 8, 5 * 8, 6 * 8, 7 * 8, 4077c478bd9Sstevel@tonic-gate #endif 4087c478bd9Sstevel@tonic-gate 4 * 16, 5 * 16, 6 * 16, 7 * 16, 4097c478bd9Sstevel@tonic-gate 4 * 32, 5 * 32, 6 * 32, 7 * 32, 4107c478bd9Sstevel@tonic-gate 4 * 64, 5 * 64, 6 * 64, 7 * 64, 4117c478bd9Sstevel@tonic-gate 4 * 128, 5 * 128, 6 * 128, 7 * 128, 4127c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 7, 64), 4137c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 6, 64), 4147c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 5, 64), 4157c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 4, 64), 4167c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 3, 64), 4177c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 2, 64), 4187c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 1, 64), 4197c478bd9Sstevel@tonic-gate 4096 * 3, 4207c478bd9Sstevel@tonic-gate 8192 * 2, 4217c478bd9Sstevel@tonic-gate }; 4227c478bd9Sstevel@tonic-gate #define NUM_ALLOC_SIZES (sizeof (umem_alloc_sizes) / sizeof (*umem_alloc_sizes)) 4237c478bd9Sstevel@tonic-gate 4247c478bd9Sstevel@tonic-gate #define UMEM_MAXBUF 16384 4257c478bd9Sstevel@tonic-gate 4267c478bd9Sstevel@tonic-gate static umem_magtype_t umem_magtype[] = { 4277c478bd9Sstevel@tonic-gate { 1, 8, 3200, 65536 }, 4287c478bd9Sstevel@tonic-gate { 3, 16, 256, 32768 }, 4297c478bd9Sstevel@tonic-gate { 7, 32, 64, 16384 }, 4307c478bd9Sstevel@tonic-gate { 15, 64, 0, 8192 }, 4317c478bd9Sstevel@tonic-gate { 31, 64, 0, 4096 }, 4327c478bd9Sstevel@tonic-gate { 47, 64, 0, 2048 }, 4337c478bd9Sstevel@tonic-gate { 63, 64, 0, 1024 }, 4347c478bd9Sstevel@tonic-gate { 95, 64, 0, 512 }, 4357c478bd9Sstevel@tonic-gate { 143, 64, 0, 0 }, 4367c478bd9Sstevel@tonic-gate }; 4377c478bd9Sstevel@tonic-gate 4387c478bd9Sstevel@tonic-gate /* 4397c478bd9Sstevel@tonic-gate * umem tunables 4407c478bd9Sstevel@tonic-gate */ 4417c478bd9Sstevel@tonic-gate uint32_t umem_max_ncpus; /* # of CPU caches. */ 4427c478bd9Sstevel@tonic-gate 4437c478bd9Sstevel@tonic-gate uint32_t umem_stack_depth = 15; /* # stack frames in a bufctl_audit */ 4447c478bd9Sstevel@tonic-gate uint32_t umem_reap_interval = 10; /* max reaping rate (seconds) */ 4457c478bd9Sstevel@tonic-gate uint_t umem_depot_contention = 2; /* max failed trylocks per real interval */ 4467c478bd9Sstevel@tonic-gate uint_t umem_abort = 1; /* whether to abort on error */ 4477c478bd9Sstevel@tonic-gate uint_t umem_output = 0; /* whether to write to standard error */ 4487c478bd9Sstevel@tonic-gate uint_t umem_logging = 0; /* umem_log_enter() override */ 4497c478bd9Sstevel@tonic-gate uint32_t umem_mtbf = 0; /* mean time between failures [default: off] */ 4507c478bd9Sstevel@tonic-gate size_t umem_transaction_log_size; /* size of transaction log */ 4517c478bd9Sstevel@tonic-gate size_t umem_content_log_size; /* size of content log */ 4527c478bd9Sstevel@tonic-gate size_t umem_failure_log_size; /* failure log [4 pages per CPU] */ 4537c478bd9Sstevel@tonic-gate size_t umem_slab_log_size; /* slab create log [4 pages per CPU] */ 4547c478bd9Sstevel@tonic-gate size_t umem_content_maxsave = 256; /* UMF_CONTENTS max bytes to log */ 4557c478bd9Sstevel@tonic-gate size_t umem_lite_minsize = 0; /* minimum buffer size for UMF_LITE */ 4567c478bd9Sstevel@tonic-gate size_t umem_lite_maxalign = 1024; /* maximum buffer alignment for UMF_LITE */ 4577c478bd9Sstevel@tonic-gate size_t umem_maxverify; /* maximum bytes to inspect in debug routines */ 4587c478bd9Sstevel@tonic-gate size_t umem_minfirewall; /* hardware-enforced redzone threshold */ 4597c478bd9Sstevel@tonic-gate 4607c478bd9Sstevel@tonic-gate uint_t umem_flags = 0; 4617c478bd9Sstevel@tonic-gate 4627c478bd9Sstevel@tonic-gate mutex_t umem_init_lock; /* locks initialization */ 4637c478bd9Sstevel@tonic-gate cond_t umem_init_cv; /* initialization CV */ 4647c478bd9Sstevel@tonic-gate thread_t umem_init_thr; /* thread initializing */ 4657c478bd9Sstevel@tonic-gate int umem_init_env_ready; /* environ pre-initted */ 4667c478bd9Sstevel@tonic-gate int umem_ready = UMEM_READY_STARTUP; 4677c478bd9Sstevel@tonic-gate 4687c478bd9Sstevel@tonic-gate static umem_nofail_callback_t *nofail_callback; 4697c478bd9Sstevel@tonic-gate static mutex_t umem_nofail_exit_lock; 4707c478bd9Sstevel@tonic-gate static thread_t umem_nofail_exit_thr; 4717c478bd9Sstevel@tonic-gate 4727c478bd9Sstevel@tonic-gate static umem_cache_t *umem_slab_cache; 4737c478bd9Sstevel@tonic-gate static umem_cache_t *umem_bufctl_cache; 4747c478bd9Sstevel@tonic-gate static umem_cache_t *umem_bufctl_audit_cache; 4757c478bd9Sstevel@tonic-gate 4767c478bd9Sstevel@tonic-gate mutex_t umem_flags_lock; 4777c478bd9Sstevel@tonic-gate 4787c478bd9Sstevel@tonic-gate static vmem_t *heap_arena; 4797c478bd9Sstevel@tonic-gate static vmem_alloc_t *heap_alloc; 4807c478bd9Sstevel@tonic-gate static vmem_free_t *heap_free; 4817c478bd9Sstevel@tonic-gate 4827c478bd9Sstevel@tonic-gate static vmem_t *umem_internal_arena; 4837c478bd9Sstevel@tonic-gate static vmem_t *umem_cache_arena; 4847c478bd9Sstevel@tonic-gate static vmem_t *umem_hash_arena; 4857c478bd9Sstevel@tonic-gate static vmem_t *umem_log_arena; 4867c478bd9Sstevel@tonic-gate static vmem_t *umem_oversize_arena; 4877c478bd9Sstevel@tonic-gate static vmem_t *umem_va_arena; 4887c478bd9Sstevel@tonic-gate static vmem_t *umem_default_arena; 4897c478bd9Sstevel@tonic-gate static vmem_t *umem_firewall_va_arena; 4907c478bd9Sstevel@tonic-gate static vmem_t *umem_firewall_arena; 4917c478bd9Sstevel@tonic-gate 4927c478bd9Sstevel@tonic-gate vmem_t *umem_memalign_arena; 4937c478bd9Sstevel@tonic-gate 4947c478bd9Sstevel@tonic-gate umem_log_header_t *umem_transaction_log; 4957c478bd9Sstevel@tonic-gate umem_log_header_t *umem_content_log; 4967c478bd9Sstevel@tonic-gate umem_log_header_t *umem_failure_log; 4977c478bd9Sstevel@tonic-gate umem_log_header_t *umem_slab_log; 4987c478bd9Sstevel@tonic-gate 4997c478bd9Sstevel@tonic-gate extern thread_t _thr_self(void); 5007c478bd9Sstevel@tonic-gate #define CPUHINT() (_thr_self()) 5017c478bd9Sstevel@tonic-gate #define CPUHINT_MAX() INT_MAX 5027c478bd9Sstevel@tonic-gate 5037c478bd9Sstevel@tonic-gate #define CPU(mask) (umem_cpus + (CPUHINT() & (mask))) 5047c478bd9Sstevel@tonic-gate static umem_cpu_t umem_startup_cpu = { /* initial, single, cpu */ 5057c478bd9Sstevel@tonic-gate UMEM_CACHE_SIZE(0), 5067c478bd9Sstevel@tonic-gate 0 5077c478bd9Sstevel@tonic-gate }; 5087c478bd9Sstevel@tonic-gate 5097c478bd9Sstevel@tonic-gate static uint32_t umem_cpu_mask = 0; /* global cpu mask */ 5107c478bd9Sstevel@tonic-gate static umem_cpu_t *umem_cpus = &umem_startup_cpu; /* cpu list */ 5117c478bd9Sstevel@tonic-gate 5127c478bd9Sstevel@tonic-gate volatile uint32_t umem_reaping; 5137c478bd9Sstevel@tonic-gate 5147c478bd9Sstevel@tonic-gate thread_t umem_update_thr; 5157c478bd9Sstevel@tonic-gate struct timeval umem_update_next; /* timeofday of next update */ 5167c478bd9Sstevel@tonic-gate volatile thread_t umem_st_update_thr; /* only used when single-thd */ 5177c478bd9Sstevel@tonic-gate 5187c478bd9Sstevel@tonic-gate #define IN_UPDATE() (thr_self() == umem_update_thr || \ 5197c478bd9Sstevel@tonic-gate thr_self() == umem_st_update_thr) 5207c478bd9Sstevel@tonic-gate #define IN_REAP() IN_UPDATE() 5217c478bd9Sstevel@tonic-gate 5227c478bd9Sstevel@tonic-gate mutex_t umem_update_lock; /* cache_u{next,prev,flags} */ 5237c478bd9Sstevel@tonic-gate cond_t umem_update_cv; 5247c478bd9Sstevel@tonic-gate 5257c478bd9Sstevel@tonic-gate volatile hrtime_t umem_reap_next; /* min hrtime of next reap */ 5267c478bd9Sstevel@tonic-gate 5277c478bd9Sstevel@tonic-gate mutex_t umem_cache_lock; /* inter-cache linkage only */ 5287c478bd9Sstevel@tonic-gate 5297c478bd9Sstevel@tonic-gate #ifdef UMEM_STANDALONE 5307c478bd9Sstevel@tonic-gate umem_cache_t umem_null_cache; 5317c478bd9Sstevel@tonic-gate static const umem_cache_t umem_null_cache_template = { 5327c478bd9Sstevel@tonic-gate #else 5337c478bd9Sstevel@tonic-gate umem_cache_t umem_null_cache = { 5347c478bd9Sstevel@tonic-gate #endif 5357c478bd9Sstevel@tonic-gate 0, 0, 0, 0, 0, 5367c478bd9Sstevel@tonic-gate 0, 0, 5377c478bd9Sstevel@tonic-gate 0, 0, 5387c478bd9Sstevel@tonic-gate 0, 0, 5397c478bd9Sstevel@tonic-gate "invalid_cache", 5407c478bd9Sstevel@tonic-gate 0, 0, 5417c478bd9Sstevel@tonic-gate NULL, NULL, NULL, NULL, 5427c478bd9Sstevel@tonic-gate NULL, 5437c478bd9Sstevel@tonic-gate 0, 0, 0, 0, 5447c478bd9Sstevel@tonic-gate &umem_null_cache, &umem_null_cache, 5457c478bd9Sstevel@tonic-gate &umem_null_cache, &umem_null_cache, 5467c478bd9Sstevel@tonic-gate 0, 5477c478bd9Sstevel@tonic-gate DEFAULTMUTEX, /* start of slab layer */ 5487c478bd9Sstevel@tonic-gate 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5497c478bd9Sstevel@tonic-gate &umem_null_cache.cache_nullslab, 5507c478bd9Sstevel@tonic-gate { 5517c478bd9Sstevel@tonic-gate &umem_null_cache, 5527c478bd9Sstevel@tonic-gate NULL, 5537c478bd9Sstevel@tonic-gate &umem_null_cache.cache_nullslab, 5547c478bd9Sstevel@tonic-gate &umem_null_cache.cache_nullslab, 5557c478bd9Sstevel@tonic-gate NULL, 5567c478bd9Sstevel@tonic-gate -1, 5577c478bd9Sstevel@tonic-gate 0 5587c478bd9Sstevel@tonic-gate }, 5597c478bd9Sstevel@tonic-gate NULL, 5607c478bd9Sstevel@tonic-gate NULL, 5617c478bd9Sstevel@tonic-gate DEFAULTMUTEX, /* start of depot layer */ 5627c478bd9Sstevel@tonic-gate NULL, { 5637c478bd9Sstevel@tonic-gate NULL, 0, 0, 0, 0 5647c478bd9Sstevel@tonic-gate }, { 5657c478bd9Sstevel@tonic-gate NULL, 0, 0, 0, 0 5667c478bd9Sstevel@tonic-gate }, { 5677c478bd9Sstevel@tonic-gate { 5687c478bd9Sstevel@tonic-gate DEFAULTMUTEX, /* start of CPU cache */ 5697c478bd9Sstevel@tonic-gate 0, 0, NULL, NULL, -1, -1, 0 5707c478bd9Sstevel@tonic-gate } 5717c478bd9Sstevel@tonic-gate } 5727c478bd9Sstevel@tonic-gate }; 5737c478bd9Sstevel@tonic-gate 5747c478bd9Sstevel@tonic-gate #define ALLOC_TABLE_4 \ 5757c478bd9Sstevel@tonic-gate &umem_null_cache, &umem_null_cache, &umem_null_cache, &umem_null_cache 5767c478bd9Sstevel@tonic-gate 5777c478bd9Sstevel@tonic-gate #define ALLOC_TABLE_64 \ 5787c478bd9Sstevel@tonic-gate ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, \ 5797c478bd9Sstevel@tonic-gate ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, \ 5807c478bd9Sstevel@tonic-gate ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, \ 5817c478bd9Sstevel@tonic-gate ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4 5827c478bd9Sstevel@tonic-gate 5837c478bd9Sstevel@tonic-gate #define ALLOC_TABLE_1024 \ 5847c478bd9Sstevel@tonic-gate ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, \ 5857c478bd9Sstevel@tonic-gate ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, \ 5867c478bd9Sstevel@tonic-gate ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, \ 5877c478bd9Sstevel@tonic-gate ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64 5887c478bd9Sstevel@tonic-gate 5897c478bd9Sstevel@tonic-gate static umem_cache_t *umem_alloc_table[UMEM_MAXBUF >> UMEM_ALIGN_SHIFT] = { 5907c478bd9Sstevel@tonic-gate ALLOC_TABLE_1024, 5917c478bd9Sstevel@tonic-gate ALLOC_TABLE_1024 5927c478bd9Sstevel@tonic-gate }; 5937c478bd9Sstevel@tonic-gate 5947c478bd9Sstevel@tonic-gate 5957c478bd9Sstevel@tonic-gate /* Used to constrain audit-log stack traces */ 5967c478bd9Sstevel@tonic-gate caddr_t umem_min_stack; 5977c478bd9Sstevel@tonic-gate caddr_t umem_max_stack; 5987c478bd9Sstevel@tonic-gate 5997c478bd9Sstevel@tonic-gate 6007c478bd9Sstevel@tonic-gate /* 6017c478bd9Sstevel@tonic-gate * we use the _ versions, since we don't want to be cancelled. 6027c478bd9Sstevel@tonic-gate * Actually, this is automatically taken care of by including "mtlib.h". 6037c478bd9Sstevel@tonic-gate */ 6047c478bd9Sstevel@tonic-gate extern int _cond_wait(cond_t *cv, mutex_t *mutex); 6057c478bd9Sstevel@tonic-gate 6067c478bd9Sstevel@tonic-gate #define UMERR_MODIFIED 0 /* buffer modified while on freelist */ 6077c478bd9Sstevel@tonic-gate #define UMERR_REDZONE 1 /* redzone violation (write past end of buf) */ 6087c478bd9Sstevel@tonic-gate #define UMERR_DUPFREE 2 /* freed a buffer twice */ 6097c478bd9Sstevel@tonic-gate #define UMERR_BADADDR 3 /* freed a bad (unallocated) address */ 6107c478bd9Sstevel@tonic-gate #define UMERR_BADBUFTAG 4 /* buftag corrupted */ 6117c478bd9Sstevel@tonic-gate #define UMERR_BADBUFCTL 5 /* bufctl corrupted */ 6127c478bd9Sstevel@tonic-gate #define UMERR_BADCACHE 6 /* freed a buffer to the wrong cache */ 6137c478bd9Sstevel@tonic-gate #define UMERR_BADSIZE 7 /* alloc size != free size */ 6147c478bd9Sstevel@tonic-gate #define UMERR_BADBASE 8 /* buffer base address wrong */ 6157c478bd9Sstevel@tonic-gate 6167c478bd9Sstevel@tonic-gate struct { 6177c478bd9Sstevel@tonic-gate hrtime_t ump_timestamp; /* timestamp of error */ 6187c478bd9Sstevel@tonic-gate int ump_error; /* type of umem error (UMERR_*) */ 6197c478bd9Sstevel@tonic-gate void *ump_buffer; /* buffer that induced abort */ 6207c478bd9Sstevel@tonic-gate void *ump_realbuf; /* real start address for buffer */ 6217c478bd9Sstevel@tonic-gate umem_cache_t *ump_cache; /* buffer's cache according to client */ 6227c478bd9Sstevel@tonic-gate umem_cache_t *ump_realcache; /* actual cache containing buffer */ 6237c478bd9Sstevel@tonic-gate umem_slab_t *ump_slab; /* slab accoring to umem_findslab() */ 6247c478bd9Sstevel@tonic-gate umem_bufctl_t *ump_bufctl; /* bufctl */ 6257c478bd9Sstevel@tonic-gate } umem_abort_info; 6267c478bd9Sstevel@tonic-gate 6277c478bd9Sstevel@tonic-gate static void 6287c478bd9Sstevel@tonic-gate copy_pattern(uint64_t pattern, void *buf_arg, size_t size) 6297c478bd9Sstevel@tonic-gate { 6307c478bd9Sstevel@tonic-gate uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 6317c478bd9Sstevel@tonic-gate uint64_t *buf = buf_arg; 6327c478bd9Sstevel@tonic-gate 6337c478bd9Sstevel@tonic-gate while (buf < bufend) 6347c478bd9Sstevel@tonic-gate *buf++ = pattern; 6357c478bd9Sstevel@tonic-gate } 6367c478bd9Sstevel@tonic-gate 6377c478bd9Sstevel@tonic-gate static void * 6387c478bd9Sstevel@tonic-gate verify_pattern(uint64_t pattern, void *buf_arg, size_t size) 6397c478bd9Sstevel@tonic-gate { 6407c478bd9Sstevel@tonic-gate uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 6417c478bd9Sstevel@tonic-gate uint64_t *buf; 6427c478bd9Sstevel@tonic-gate 6437c478bd9Sstevel@tonic-gate for (buf = buf_arg; buf < bufend; buf++) 6447c478bd9Sstevel@tonic-gate if (*buf != pattern) 6457c478bd9Sstevel@tonic-gate return (buf); 6467c478bd9Sstevel@tonic-gate return (NULL); 6477c478bd9Sstevel@tonic-gate } 6487c478bd9Sstevel@tonic-gate 6497c478bd9Sstevel@tonic-gate static void * 6507c478bd9Sstevel@tonic-gate verify_and_copy_pattern(uint64_t old, uint64_t new, void *buf_arg, size_t size) 6517c478bd9Sstevel@tonic-gate { 6527c478bd9Sstevel@tonic-gate uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 6537c478bd9Sstevel@tonic-gate uint64_t *buf; 6547c478bd9Sstevel@tonic-gate 6557c478bd9Sstevel@tonic-gate for (buf = buf_arg; buf < bufend; buf++) { 6567c478bd9Sstevel@tonic-gate if (*buf != old) { 6577c478bd9Sstevel@tonic-gate copy_pattern(old, buf_arg, 6587c478bd9Sstevel@tonic-gate (char *)buf - (char *)buf_arg); 6597c478bd9Sstevel@tonic-gate return (buf); 6607c478bd9Sstevel@tonic-gate } 6617c478bd9Sstevel@tonic-gate *buf = new; 6627c478bd9Sstevel@tonic-gate } 6637c478bd9Sstevel@tonic-gate 6647c478bd9Sstevel@tonic-gate return (NULL); 6657c478bd9Sstevel@tonic-gate } 6667c478bd9Sstevel@tonic-gate 6677c478bd9Sstevel@tonic-gate void 6687c478bd9Sstevel@tonic-gate umem_cache_applyall(void (*func)(umem_cache_t *)) 6697c478bd9Sstevel@tonic-gate { 6707c478bd9Sstevel@tonic-gate umem_cache_t *cp; 6717c478bd9Sstevel@tonic-gate 6727c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_cache_lock); 6737c478bd9Sstevel@tonic-gate for (cp = umem_null_cache.cache_next; cp != &umem_null_cache; 6747c478bd9Sstevel@tonic-gate cp = cp->cache_next) 6757c478bd9Sstevel@tonic-gate func(cp); 6767c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_cache_lock); 6777c478bd9Sstevel@tonic-gate } 6787c478bd9Sstevel@tonic-gate 6797c478bd9Sstevel@tonic-gate static void 6807c478bd9Sstevel@tonic-gate umem_add_update_unlocked(umem_cache_t *cp, int flags) 6817c478bd9Sstevel@tonic-gate { 6827c478bd9Sstevel@tonic-gate umem_cache_t *cnext, *cprev; 6837c478bd9Sstevel@tonic-gate 6847c478bd9Sstevel@tonic-gate flags &= ~UMU_ACTIVE; 6857c478bd9Sstevel@tonic-gate 6867c478bd9Sstevel@tonic-gate if (!flags) 6877c478bd9Sstevel@tonic-gate return; 6887c478bd9Sstevel@tonic-gate 6897c478bd9Sstevel@tonic-gate if (cp->cache_uflags & UMU_ACTIVE) { 6907c478bd9Sstevel@tonic-gate cp->cache_uflags |= flags; 6917c478bd9Sstevel@tonic-gate } else { 6927c478bd9Sstevel@tonic-gate if (cp->cache_unext != NULL) { 6937c478bd9Sstevel@tonic-gate ASSERT(cp->cache_uflags != 0); 6947c478bd9Sstevel@tonic-gate cp->cache_uflags |= flags; 6957c478bd9Sstevel@tonic-gate } else { 6967c478bd9Sstevel@tonic-gate ASSERT(cp->cache_uflags == 0); 6977c478bd9Sstevel@tonic-gate cp->cache_uflags = flags; 6987c478bd9Sstevel@tonic-gate cp->cache_unext = cnext = &umem_null_cache; 6997c478bd9Sstevel@tonic-gate cp->cache_uprev = cprev = umem_null_cache.cache_uprev; 7007c478bd9Sstevel@tonic-gate cnext->cache_uprev = cp; 7017c478bd9Sstevel@tonic-gate cprev->cache_unext = cp; 7027c478bd9Sstevel@tonic-gate } 7037c478bd9Sstevel@tonic-gate } 7047c478bd9Sstevel@tonic-gate } 7057c478bd9Sstevel@tonic-gate 7067c478bd9Sstevel@tonic-gate static void 7077c478bd9Sstevel@tonic-gate umem_add_update(umem_cache_t *cp, int flags) 7087c478bd9Sstevel@tonic-gate { 7097c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_update_lock); 7107c478bd9Sstevel@tonic-gate 7117c478bd9Sstevel@tonic-gate umem_add_update_unlocked(cp, flags); 7127c478bd9Sstevel@tonic-gate 7137c478bd9Sstevel@tonic-gate if (!IN_UPDATE()) 7147c478bd9Sstevel@tonic-gate (void) cond_broadcast(&umem_update_cv); 7157c478bd9Sstevel@tonic-gate 7167c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_update_lock); 7177c478bd9Sstevel@tonic-gate } 7187c478bd9Sstevel@tonic-gate 7197c478bd9Sstevel@tonic-gate /* 7207c478bd9Sstevel@tonic-gate * Remove a cache from the update list, waiting for any in-progress work to 7217c478bd9Sstevel@tonic-gate * complete first. 7227c478bd9Sstevel@tonic-gate */ 7237c478bd9Sstevel@tonic-gate static void 7247c478bd9Sstevel@tonic-gate umem_remove_updates(umem_cache_t *cp) 7257c478bd9Sstevel@tonic-gate { 7267c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_update_lock); 7277c478bd9Sstevel@tonic-gate 7287c478bd9Sstevel@tonic-gate /* 7297c478bd9Sstevel@tonic-gate * Get it out of the active state 7307c478bd9Sstevel@tonic-gate */ 7317c478bd9Sstevel@tonic-gate while (cp->cache_uflags & UMU_ACTIVE) { 7327c478bd9Sstevel@tonic-gate ASSERT(cp->cache_unext == NULL); 7337c478bd9Sstevel@tonic-gate 7347c478bd9Sstevel@tonic-gate cp->cache_uflags |= UMU_NOTIFY; 7357c478bd9Sstevel@tonic-gate 7367c478bd9Sstevel@tonic-gate /* 7377c478bd9Sstevel@tonic-gate * Make sure the update state is sane, before we wait 7387c478bd9Sstevel@tonic-gate */ 7397c478bd9Sstevel@tonic-gate ASSERT(umem_update_thr != 0 || umem_st_update_thr != 0); 7407c478bd9Sstevel@tonic-gate ASSERT(umem_update_thr != thr_self() && 7417c478bd9Sstevel@tonic-gate umem_st_update_thr != thr_self()); 7427c478bd9Sstevel@tonic-gate 7437c478bd9Sstevel@tonic-gate (void) _cond_wait(&umem_update_cv, &umem_update_lock); 7447c478bd9Sstevel@tonic-gate } 7457c478bd9Sstevel@tonic-gate /* 7467c478bd9Sstevel@tonic-gate * Get it out of the Work Requested state 7477c478bd9Sstevel@tonic-gate */ 7487c478bd9Sstevel@tonic-gate if (cp->cache_unext != NULL) { 7497c478bd9Sstevel@tonic-gate cp->cache_uprev->cache_unext = cp->cache_unext; 7507c478bd9Sstevel@tonic-gate cp->cache_unext->cache_uprev = cp->cache_uprev; 7517c478bd9Sstevel@tonic-gate cp->cache_uprev = cp->cache_unext = NULL; 7527c478bd9Sstevel@tonic-gate cp->cache_uflags = 0; 7537c478bd9Sstevel@tonic-gate } 7547c478bd9Sstevel@tonic-gate /* 7557c478bd9Sstevel@tonic-gate * Make sure it is in the Inactive state 7567c478bd9Sstevel@tonic-gate */ 7577c478bd9Sstevel@tonic-gate ASSERT(cp->cache_unext == NULL && cp->cache_uflags == 0); 7587c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_update_lock); 7597c478bd9Sstevel@tonic-gate } 7607c478bd9Sstevel@tonic-gate 7617c478bd9Sstevel@tonic-gate static void 7627c478bd9Sstevel@tonic-gate umem_updateall(int flags) 7637c478bd9Sstevel@tonic-gate { 7647c478bd9Sstevel@tonic-gate umem_cache_t *cp; 7657c478bd9Sstevel@tonic-gate 7667c478bd9Sstevel@tonic-gate /* 7677c478bd9Sstevel@tonic-gate * NOTE: To prevent deadlock, umem_cache_lock is always acquired first. 7687c478bd9Sstevel@tonic-gate * 7697c478bd9Sstevel@tonic-gate * (umem_add_update is called from things run via umem_cache_applyall) 7707c478bd9Sstevel@tonic-gate */ 7717c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_cache_lock); 7727c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_update_lock); 7737c478bd9Sstevel@tonic-gate 7747c478bd9Sstevel@tonic-gate for (cp = umem_null_cache.cache_next; cp != &umem_null_cache; 7757c478bd9Sstevel@tonic-gate cp = cp->cache_next) 7767c478bd9Sstevel@tonic-gate umem_add_update_unlocked(cp, flags); 7777c478bd9Sstevel@tonic-gate 7787c478bd9Sstevel@tonic-gate if (!IN_UPDATE()) 7797c478bd9Sstevel@tonic-gate (void) cond_broadcast(&umem_update_cv); 7807c478bd9Sstevel@tonic-gate 7817c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_update_lock); 7827c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_cache_lock); 7837c478bd9Sstevel@tonic-gate } 7847c478bd9Sstevel@tonic-gate 7857c478bd9Sstevel@tonic-gate /* 7867c478bd9Sstevel@tonic-gate * Debugging support. Given a buffer address, find its slab. 7877c478bd9Sstevel@tonic-gate */ 7887c478bd9Sstevel@tonic-gate static umem_slab_t * 7897c478bd9Sstevel@tonic-gate umem_findslab(umem_cache_t *cp, void *buf) 7907c478bd9Sstevel@tonic-gate { 7917c478bd9Sstevel@tonic-gate umem_slab_t *sp; 7927c478bd9Sstevel@tonic-gate 7937c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_lock); 7947c478bd9Sstevel@tonic-gate for (sp = cp->cache_nullslab.slab_next; 7957c478bd9Sstevel@tonic-gate sp != &cp->cache_nullslab; sp = sp->slab_next) { 7967c478bd9Sstevel@tonic-gate if (UMEM_SLAB_MEMBER(sp, buf)) { 7977c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 7987c478bd9Sstevel@tonic-gate return (sp); 7997c478bd9Sstevel@tonic-gate } 8007c478bd9Sstevel@tonic-gate } 8017c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 8027c478bd9Sstevel@tonic-gate 8037c478bd9Sstevel@tonic-gate return (NULL); 8047c478bd9Sstevel@tonic-gate } 8057c478bd9Sstevel@tonic-gate 8067c478bd9Sstevel@tonic-gate static void 8077c478bd9Sstevel@tonic-gate umem_error(int error, umem_cache_t *cparg, void *bufarg) 8087c478bd9Sstevel@tonic-gate { 8097c478bd9Sstevel@tonic-gate umem_buftag_t *btp = NULL; 8107c478bd9Sstevel@tonic-gate umem_bufctl_t *bcp = NULL; 8117c478bd9Sstevel@tonic-gate umem_cache_t *cp = cparg; 8127c478bd9Sstevel@tonic-gate umem_slab_t *sp; 8137c478bd9Sstevel@tonic-gate uint64_t *off; 8147c478bd9Sstevel@tonic-gate void *buf = bufarg; 8157c478bd9Sstevel@tonic-gate 8167c478bd9Sstevel@tonic-gate int old_logging = umem_logging; 8177c478bd9Sstevel@tonic-gate 8187c478bd9Sstevel@tonic-gate umem_logging = 0; /* stop logging when a bad thing happens */ 8197c478bd9Sstevel@tonic-gate 8207c478bd9Sstevel@tonic-gate umem_abort_info.ump_timestamp = gethrtime(); 8217c478bd9Sstevel@tonic-gate 8227c478bd9Sstevel@tonic-gate sp = umem_findslab(cp, buf); 8237c478bd9Sstevel@tonic-gate if (sp == NULL) { 8247c478bd9Sstevel@tonic-gate for (cp = umem_null_cache.cache_prev; cp != &umem_null_cache; 8257c478bd9Sstevel@tonic-gate cp = cp->cache_prev) { 8267c478bd9Sstevel@tonic-gate if ((sp = umem_findslab(cp, buf)) != NULL) 8277c478bd9Sstevel@tonic-gate break; 8287c478bd9Sstevel@tonic-gate } 8297c478bd9Sstevel@tonic-gate } 8307c478bd9Sstevel@tonic-gate 8317c478bd9Sstevel@tonic-gate if (sp == NULL) { 8327c478bd9Sstevel@tonic-gate cp = NULL; 8337c478bd9Sstevel@tonic-gate error = UMERR_BADADDR; 8347c478bd9Sstevel@tonic-gate } else { 8357c478bd9Sstevel@tonic-gate if (cp != cparg) 8367c478bd9Sstevel@tonic-gate error = UMERR_BADCACHE; 8377c478bd9Sstevel@tonic-gate else 8387c478bd9Sstevel@tonic-gate buf = (char *)bufarg - ((uintptr_t)bufarg - 8397c478bd9Sstevel@tonic-gate (uintptr_t)sp->slab_base) % cp->cache_chunksize; 8407c478bd9Sstevel@tonic-gate if (buf != bufarg) 8417c478bd9Sstevel@tonic-gate error = UMERR_BADBASE; 8427c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_BUFTAG) 8437c478bd9Sstevel@tonic-gate btp = UMEM_BUFTAG(cp, buf); 8447c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_HASH) { 8457c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_lock); 8467c478bd9Sstevel@tonic-gate for (bcp = *UMEM_HASH(cp, buf); bcp; bcp = bcp->bc_next) 8477c478bd9Sstevel@tonic-gate if (bcp->bc_addr == buf) 8487c478bd9Sstevel@tonic-gate break; 8497c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 8507c478bd9Sstevel@tonic-gate if (bcp == NULL && btp != NULL) 8517c478bd9Sstevel@tonic-gate bcp = btp->bt_bufctl; 8527c478bd9Sstevel@tonic-gate if (umem_findslab(cp->cache_bufctl_cache, bcp) == 8537c478bd9Sstevel@tonic-gate NULL || P2PHASE((uintptr_t)bcp, UMEM_ALIGN) || 8547c478bd9Sstevel@tonic-gate bcp->bc_addr != buf) { 8557c478bd9Sstevel@tonic-gate error = UMERR_BADBUFCTL; 8567c478bd9Sstevel@tonic-gate bcp = NULL; 8577c478bd9Sstevel@tonic-gate } 8587c478bd9Sstevel@tonic-gate } 8597c478bd9Sstevel@tonic-gate } 8607c478bd9Sstevel@tonic-gate 8617c478bd9Sstevel@tonic-gate umem_abort_info.ump_error = error; 8627c478bd9Sstevel@tonic-gate umem_abort_info.ump_buffer = bufarg; 8637c478bd9Sstevel@tonic-gate umem_abort_info.ump_realbuf = buf; 8647c478bd9Sstevel@tonic-gate umem_abort_info.ump_cache = cparg; 8657c478bd9Sstevel@tonic-gate umem_abort_info.ump_realcache = cp; 8667c478bd9Sstevel@tonic-gate umem_abort_info.ump_slab = sp; 8677c478bd9Sstevel@tonic-gate umem_abort_info.ump_bufctl = bcp; 8687c478bd9Sstevel@tonic-gate 8697c478bd9Sstevel@tonic-gate umem_printf("umem allocator: "); 8707c478bd9Sstevel@tonic-gate 8717c478bd9Sstevel@tonic-gate switch (error) { 8727c478bd9Sstevel@tonic-gate 8737c478bd9Sstevel@tonic-gate case UMERR_MODIFIED: 8747c478bd9Sstevel@tonic-gate umem_printf("buffer modified after being freed\n"); 8757c478bd9Sstevel@tonic-gate off = verify_pattern(UMEM_FREE_PATTERN, buf, cp->cache_verify); 8767c478bd9Sstevel@tonic-gate if (off == NULL) /* shouldn't happen */ 8777c478bd9Sstevel@tonic-gate off = buf; 8787c478bd9Sstevel@tonic-gate umem_printf("modification occurred at offset 0x%lx " 8797c478bd9Sstevel@tonic-gate "(0x%llx replaced by 0x%llx)\n", 8807c478bd9Sstevel@tonic-gate (uintptr_t)off - (uintptr_t)buf, 8817c478bd9Sstevel@tonic-gate (longlong_t)UMEM_FREE_PATTERN, (longlong_t)*off); 8827c478bd9Sstevel@tonic-gate break; 8837c478bd9Sstevel@tonic-gate 8847c478bd9Sstevel@tonic-gate case UMERR_REDZONE: 8857c478bd9Sstevel@tonic-gate umem_printf("redzone violation: write past end of buffer\n"); 8867c478bd9Sstevel@tonic-gate break; 8877c478bd9Sstevel@tonic-gate 8887c478bd9Sstevel@tonic-gate case UMERR_BADADDR: 8897c478bd9Sstevel@tonic-gate umem_printf("invalid free: buffer not in cache\n"); 8907c478bd9Sstevel@tonic-gate break; 8917c478bd9Sstevel@tonic-gate 8927c478bd9Sstevel@tonic-gate case UMERR_DUPFREE: 8937c478bd9Sstevel@tonic-gate umem_printf("duplicate free: buffer freed twice\n"); 8947c478bd9Sstevel@tonic-gate break; 8957c478bd9Sstevel@tonic-gate 8967c478bd9Sstevel@tonic-gate case UMERR_BADBUFTAG: 8977c478bd9Sstevel@tonic-gate umem_printf("boundary tag corrupted\n"); 8987c478bd9Sstevel@tonic-gate umem_printf("bcp ^ bxstat = %lx, should be %lx\n", 8997c478bd9Sstevel@tonic-gate (intptr_t)btp->bt_bufctl ^ btp->bt_bxstat, 9007c478bd9Sstevel@tonic-gate UMEM_BUFTAG_FREE); 9017c478bd9Sstevel@tonic-gate break; 9027c478bd9Sstevel@tonic-gate 9037c478bd9Sstevel@tonic-gate case UMERR_BADBUFCTL: 9047c478bd9Sstevel@tonic-gate umem_printf("bufctl corrupted\n"); 9057c478bd9Sstevel@tonic-gate break; 9067c478bd9Sstevel@tonic-gate 9077c478bd9Sstevel@tonic-gate case UMERR_BADCACHE: 9087c478bd9Sstevel@tonic-gate umem_printf("buffer freed to wrong cache\n"); 9097c478bd9Sstevel@tonic-gate umem_printf("buffer was allocated from %s,\n", cp->cache_name); 9107c478bd9Sstevel@tonic-gate umem_printf("caller attempting free to %s.\n", 9117c478bd9Sstevel@tonic-gate cparg->cache_name); 9127c478bd9Sstevel@tonic-gate break; 9137c478bd9Sstevel@tonic-gate 9147c478bd9Sstevel@tonic-gate case UMERR_BADSIZE: 9157c478bd9Sstevel@tonic-gate umem_printf("bad free: free size (%u) != alloc size (%u)\n", 9167c478bd9Sstevel@tonic-gate UMEM_SIZE_DECODE(((uint32_t *)btp)[0]), 9177c478bd9Sstevel@tonic-gate UMEM_SIZE_DECODE(((uint32_t *)btp)[1])); 9187c478bd9Sstevel@tonic-gate break; 9197c478bd9Sstevel@tonic-gate 9207c478bd9Sstevel@tonic-gate case UMERR_BADBASE: 9217c478bd9Sstevel@tonic-gate umem_printf("bad free: free address (%p) != alloc address " 9227c478bd9Sstevel@tonic-gate "(%p)\n", bufarg, buf); 9237c478bd9Sstevel@tonic-gate break; 9247c478bd9Sstevel@tonic-gate } 9257c478bd9Sstevel@tonic-gate 9267c478bd9Sstevel@tonic-gate umem_printf("buffer=%p bufctl=%p cache: %s\n", 9277c478bd9Sstevel@tonic-gate bufarg, (void *)bcp, cparg->cache_name); 9287c478bd9Sstevel@tonic-gate 9297c478bd9Sstevel@tonic-gate if (bcp != NULL && (cp->cache_flags & UMF_AUDIT) && 9307c478bd9Sstevel@tonic-gate error != UMERR_BADBUFCTL) { 9317c478bd9Sstevel@tonic-gate int d; 9327c478bd9Sstevel@tonic-gate timespec_t ts; 9337c478bd9Sstevel@tonic-gate hrtime_t diff; 9347c478bd9Sstevel@tonic-gate umem_bufctl_audit_t *bcap = (umem_bufctl_audit_t *)bcp; 9357c478bd9Sstevel@tonic-gate 9367c478bd9Sstevel@tonic-gate diff = umem_abort_info.ump_timestamp - bcap->bc_timestamp; 9377c478bd9Sstevel@tonic-gate ts.tv_sec = diff / NANOSEC; 9387c478bd9Sstevel@tonic-gate ts.tv_nsec = diff % NANOSEC; 9397c478bd9Sstevel@tonic-gate 9407c478bd9Sstevel@tonic-gate umem_printf("previous transaction on buffer %p:\n", buf); 9417c478bd9Sstevel@tonic-gate umem_printf("thread=%p time=T-%ld.%09ld slab=%p cache: %s\n", 9427c478bd9Sstevel@tonic-gate (void *)(intptr_t)bcap->bc_thread, ts.tv_sec, ts.tv_nsec, 9437c478bd9Sstevel@tonic-gate (void *)sp, cp->cache_name); 9447c478bd9Sstevel@tonic-gate for (d = 0; d < MIN(bcap->bc_depth, umem_stack_depth); d++) { 9457c478bd9Sstevel@tonic-gate (void) print_sym((void *)bcap->bc_stack[d]); 9467c478bd9Sstevel@tonic-gate umem_printf("\n"); 9477c478bd9Sstevel@tonic-gate } 9487c478bd9Sstevel@tonic-gate } 9497c478bd9Sstevel@tonic-gate 9507c478bd9Sstevel@tonic-gate umem_err_recoverable("umem: heap corruption detected"); 9517c478bd9Sstevel@tonic-gate 9527c478bd9Sstevel@tonic-gate umem_logging = old_logging; /* resume logging */ 9537c478bd9Sstevel@tonic-gate } 9547c478bd9Sstevel@tonic-gate 9557c478bd9Sstevel@tonic-gate void 9567c478bd9Sstevel@tonic-gate umem_nofail_callback(umem_nofail_callback_t *cb) 9577c478bd9Sstevel@tonic-gate { 9587c478bd9Sstevel@tonic-gate nofail_callback = cb; 9597c478bd9Sstevel@tonic-gate } 9607c478bd9Sstevel@tonic-gate 9617c478bd9Sstevel@tonic-gate static int 9627c478bd9Sstevel@tonic-gate umem_alloc_retry(umem_cache_t *cp, int umflag) 9637c478bd9Sstevel@tonic-gate { 9647c478bd9Sstevel@tonic-gate if (cp == &umem_null_cache) { 9657c478bd9Sstevel@tonic-gate if (umem_init()) 9667c478bd9Sstevel@tonic-gate return (1); /* retry */ 9677c478bd9Sstevel@tonic-gate /* 9687c478bd9Sstevel@tonic-gate * Initialization failed. Do normal failure processing. 9697c478bd9Sstevel@tonic-gate */ 9707c478bd9Sstevel@tonic-gate } 9717c478bd9Sstevel@tonic-gate if (umflag & UMEM_NOFAIL) { 9727c478bd9Sstevel@tonic-gate int def_result = UMEM_CALLBACK_EXIT(255); 9737c478bd9Sstevel@tonic-gate int result = def_result; 9747c478bd9Sstevel@tonic-gate umem_nofail_callback_t *callback = nofail_callback; 9757c478bd9Sstevel@tonic-gate 9767c478bd9Sstevel@tonic-gate if (callback != NULL) 9777c478bd9Sstevel@tonic-gate result = callback(); 9787c478bd9Sstevel@tonic-gate 9797c478bd9Sstevel@tonic-gate if (result == UMEM_CALLBACK_RETRY) 9807c478bd9Sstevel@tonic-gate return (1); 9817c478bd9Sstevel@tonic-gate 9827c478bd9Sstevel@tonic-gate if ((result & ~0xFF) != UMEM_CALLBACK_EXIT(0)) { 9837c478bd9Sstevel@tonic-gate log_message("nofail callback returned %x\n", result); 9847c478bd9Sstevel@tonic-gate result = def_result; 9857c478bd9Sstevel@tonic-gate } 9867c478bd9Sstevel@tonic-gate 9877c478bd9Sstevel@tonic-gate /* 9887c478bd9Sstevel@tonic-gate * only one thread will call exit 9897c478bd9Sstevel@tonic-gate */ 9907c478bd9Sstevel@tonic-gate if (umem_nofail_exit_thr == thr_self()) 9917c478bd9Sstevel@tonic-gate umem_panic("recursive UMEM_CALLBACK_EXIT()\n"); 9927c478bd9Sstevel@tonic-gate 9937c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_nofail_exit_lock); 9947c478bd9Sstevel@tonic-gate umem_nofail_exit_thr = thr_self(); 9957c478bd9Sstevel@tonic-gate exit(result & 0xFF); 9967c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 9977c478bd9Sstevel@tonic-gate } 9987c478bd9Sstevel@tonic-gate return (0); 9997c478bd9Sstevel@tonic-gate } 10007c478bd9Sstevel@tonic-gate 10017c478bd9Sstevel@tonic-gate static umem_log_header_t * 10027c478bd9Sstevel@tonic-gate umem_log_init(size_t logsize) 10037c478bd9Sstevel@tonic-gate { 10047c478bd9Sstevel@tonic-gate umem_log_header_t *lhp; 10057c478bd9Sstevel@tonic-gate int nchunks = 4 * umem_max_ncpus; 10067c478bd9Sstevel@tonic-gate size_t lhsize = offsetof(umem_log_header_t, lh_cpu[umem_max_ncpus]); 10077c478bd9Sstevel@tonic-gate int i; 10087c478bd9Sstevel@tonic-gate 10097c478bd9Sstevel@tonic-gate if (logsize == 0) 10107c478bd9Sstevel@tonic-gate return (NULL); 10117c478bd9Sstevel@tonic-gate 10127c478bd9Sstevel@tonic-gate /* 10137c478bd9Sstevel@tonic-gate * Make sure that lhp->lh_cpu[] is nicely aligned 10147c478bd9Sstevel@tonic-gate * to prevent false sharing of cache lines. 10157c478bd9Sstevel@tonic-gate */ 10167c478bd9Sstevel@tonic-gate lhsize = P2ROUNDUP(lhsize, UMEM_ALIGN); 10177c478bd9Sstevel@tonic-gate lhp = vmem_xalloc(umem_log_arena, lhsize, 64, P2NPHASE(lhsize, 64), 0, 10187c478bd9Sstevel@tonic-gate NULL, NULL, VM_NOSLEEP); 10197c478bd9Sstevel@tonic-gate if (lhp == NULL) 10207c478bd9Sstevel@tonic-gate goto fail; 10217c478bd9Sstevel@tonic-gate 10227c478bd9Sstevel@tonic-gate bzero(lhp, lhsize); 10237c478bd9Sstevel@tonic-gate 10247c478bd9Sstevel@tonic-gate (void) mutex_init(&lhp->lh_lock, USYNC_THREAD, NULL); 10257c478bd9Sstevel@tonic-gate lhp->lh_nchunks = nchunks; 10267c478bd9Sstevel@tonic-gate lhp->lh_chunksize = P2ROUNDUP(logsize / nchunks, PAGESIZE); 10277c478bd9Sstevel@tonic-gate if (lhp->lh_chunksize == 0) 10287c478bd9Sstevel@tonic-gate lhp->lh_chunksize = PAGESIZE; 10297c478bd9Sstevel@tonic-gate 10307c478bd9Sstevel@tonic-gate lhp->lh_base = vmem_alloc(umem_log_arena, 10317c478bd9Sstevel@tonic-gate lhp->lh_chunksize * nchunks, VM_NOSLEEP); 10327c478bd9Sstevel@tonic-gate if (lhp->lh_base == NULL) 10337c478bd9Sstevel@tonic-gate goto fail; 10347c478bd9Sstevel@tonic-gate 10357c478bd9Sstevel@tonic-gate lhp->lh_free = vmem_alloc(umem_log_arena, 10367c478bd9Sstevel@tonic-gate nchunks * sizeof (int), VM_NOSLEEP); 10377c478bd9Sstevel@tonic-gate if (lhp->lh_free == NULL) 10387c478bd9Sstevel@tonic-gate goto fail; 10397c478bd9Sstevel@tonic-gate 10407c478bd9Sstevel@tonic-gate bzero(lhp->lh_base, lhp->lh_chunksize * nchunks); 10417c478bd9Sstevel@tonic-gate 10427c478bd9Sstevel@tonic-gate for (i = 0; i < umem_max_ncpus; i++) { 10437c478bd9Sstevel@tonic-gate umem_cpu_log_header_t *clhp = &lhp->lh_cpu[i]; 10447c478bd9Sstevel@tonic-gate (void) mutex_init(&clhp->clh_lock, USYNC_THREAD, NULL); 10457c478bd9Sstevel@tonic-gate clhp->clh_chunk = i; 10467c478bd9Sstevel@tonic-gate } 10477c478bd9Sstevel@tonic-gate 10487c478bd9Sstevel@tonic-gate for (i = umem_max_ncpus; i < nchunks; i++) 10497c478bd9Sstevel@tonic-gate lhp->lh_free[i] = i; 10507c478bd9Sstevel@tonic-gate 10517c478bd9Sstevel@tonic-gate lhp->lh_head = umem_max_ncpus; 10527c478bd9Sstevel@tonic-gate lhp->lh_tail = 0; 10537c478bd9Sstevel@tonic-gate 10547c478bd9Sstevel@tonic-gate return (lhp); 10557c478bd9Sstevel@tonic-gate 10567c478bd9Sstevel@tonic-gate fail: 10577c478bd9Sstevel@tonic-gate if (lhp != NULL) { 10587c478bd9Sstevel@tonic-gate if (lhp->lh_base != NULL) 10597c478bd9Sstevel@tonic-gate vmem_free(umem_log_arena, lhp->lh_base, 10607c478bd9Sstevel@tonic-gate lhp->lh_chunksize * nchunks); 10617c478bd9Sstevel@tonic-gate 10627c478bd9Sstevel@tonic-gate vmem_xfree(umem_log_arena, lhp, lhsize); 10637c478bd9Sstevel@tonic-gate } 10647c478bd9Sstevel@tonic-gate return (NULL); 10657c478bd9Sstevel@tonic-gate } 10667c478bd9Sstevel@tonic-gate 10677c478bd9Sstevel@tonic-gate static void * 10687c478bd9Sstevel@tonic-gate umem_log_enter(umem_log_header_t *lhp, void *data, size_t size) 10697c478bd9Sstevel@tonic-gate { 10707c478bd9Sstevel@tonic-gate void *logspace; 10717c478bd9Sstevel@tonic-gate umem_cpu_log_header_t *clhp = 10727c478bd9Sstevel@tonic-gate &lhp->lh_cpu[CPU(umem_cpu_mask)->cpu_number]; 10737c478bd9Sstevel@tonic-gate 10747c478bd9Sstevel@tonic-gate if (lhp == NULL || umem_logging == 0) 10757c478bd9Sstevel@tonic-gate return (NULL); 10767c478bd9Sstevel@tonic-gate 10777c478bd9Sstevel@tonic-gate (void) mutex_lock(&clhp->clh_lock); 10787c478bd9Sstevel@tonic-gate clhp->clh_hits++; 10797c478bd9Sstevel@tonic-gate if (size > clhp->clh_avail) { 10807c478bd9Sstevel@tonic-gate (void) mutex_lock(&lhp->lh_lock); 10817c478bd9Sstevel@tonic-gate lhp->lh_hits++; 10827c478bd9Sstevel@tonic-gate lhp->lh_free[lhp->lh_tail] = clhp->clh_chunk; 10837c478bd9Sstevel@tonic-gate lhp->lh_tail = (lhp->lh_tail + 1) % lhp->lh_nchunks; 10847c478bd9Sstevel@tonic-gate clhp->clh_chunk = lhp->lh_free[lhp->lh_head]; 10857c478bd9Sstevel@tonic-gate lhp->lh_head = (lhp->lh_head + 1) % lhp->lh_nchunks; 10867c478bd9Sstevel@tonic-gate clhp->clh_current = lhp->lh_base + 10877c478bd9Sstevel@tonic-gate clhp->clh_chunk * lhp->lh_chunksize; 10887c478bd9Sstevel@tonic-gate clhp->clh_avail = lhp->lh_chunksize; 10897c478bd9Sstevel@tonic-gate if (size > lhp->lh_chunksize) 10907c478bd9Sstevel@tonic-gate size = lhp->lh_chunksize; 10917c478bd9Sstevel@tonic-gate (void) mutex_unlock(&lhp->lh_lock); 10927c478bd9Sstevel@tonic-gate } 10937c478bd9Sstevel@tonic-gate logspace = clhp->clh_current; 10947c478bd9Sstevel@tonic-gate clhp->clh_current += size; 10957c478bd9Sstevel@tonic-gate clhp->clh_avail -= size; 10967c478bd9Sstevel@tonic-gate bcopy(data, logspace, size); 10977c478bd9Sstevel@tonic-gate (void) mutex_unlock(&clhp->clh_lock); 10987c478bd9Sstevel@tonic-gate return (logspace); 10997c478bd9Sstevel@tonic-gate } 11007c478bd9Sstevel@tonic-gate 11017c478bd9Sstevel@tonic-gate #define UMEM_AUDIT(lp, cp, bcp) \ 11027c478bd9Sstevel@tonic-gate { \ 11037c478bd9Sstevel@tonic-gate umem_bufctl_audit_t *_bcp = (umem_bufctl_audit_t *)(bcp); \ 11047c478bd9Sstevel@tonic-gate _bcp->bc_timestamp = gethrtime(); \ 11057c478bd9Sstevel@tonic-gate _bcp->bc_thread = thr_self(); \ 11067c478bd9Sstevel@tonic-gate _bcp->bc_depth = getpcstack(_bcp->bc_stack, umem_stack_depth, \ 11077c478bd9Sstevel@tonic-gate (cp != NULL) && (cp->cache_flags & UMF_CHECKSIGNAL)); \ 11087c478bd9Sstevel@tonic-gate _bcp->bc_lastlog = umem_log_enter((lp), _bcp, \ 11097c478bd9Sstevel@tonic-gate UMEM_BUFCTL_AUDIT_SIZE); \ 11107c478bd9Sstevel@tonic-gate } 11117c478bd9Sstevel@tonic-gate 11127c478bd9Sstevel@tonic-gate static void 11137c478bd9Sstevel@tonic-gate umem_log_event(umem_log_header_t *lp, umem_cache_t *cp, 11147c478bd9Sstevel@tonic-gate umem_slab_t *sp, void *addr) 11157c478bd9Sstevel@tonic-gate { 11167c478bd9Sstevel@tonic-gate umem_bufctl_audit_t *bcp; 11177c478bd9Sstevel@tonic-gate UMEM_LOCAL_BUFCTL_AUDIT(&bcp); 11187c478bd9Sstevel@tonic-gate 11197c478bd9Sstevel@tonic-gate bzero(bcp, UMEM_BUFCTL_AUDIT_SIZE); 11207c478bd9Sstevel@tonic-gate bcp->bc_addr = addr; 11217c478bd9Sstevel@tonic-gate bcp->bc_slab = sp; 11227c478bd9Sstevel@tonic-gate bcp->bc_cache = cp; 11237c478bd9Sstevel@tonic-gate UMEM_AUDIT(lp, cp, bcp); 11247c478bd9Sstevel@tonic-gate } 11257c478bd9Sstevel@tonic-gate 11267c478bd9Sstevel@tonic-gate /* 11277c478bd9Sstevel@tonic-gate * Create a new slab for cache cp. 11287c478bd9Sstevel@tonic-gate */ 11297c478bd9Sstevel@tonic-gate static umem_slab_t * 11307c478bd9Sstevel@tonic-gate umem_slab_create(umem_cache_t *cp, int umflag) 11317c478bd9Sstevel@tonic-gate { 11327c478bd9Sstevel@tonic-gate size_t slabsize = cp->cache_slabsize; 11337c478bd9Sstevel@tonic-gate size_t chunksize = cp->cache_chunksize; 11347c478bd9Sstevel@tonic-gate int cache_flags = cp->cache_flags; 11357c478bd9Sstevel@tonic-gate size_t color, chunks; 11367c478bd9Sstevel@tonic-gate char *buf, *slab; 11377c478bd9Sstevel@tonic-gate umem_slab_t *sp; 11387c478bd9Sstevel@tonic-gate umem_bufctl_t *bcp; 11397c478bd9Sstevel@tonic-gate vmem_t *vmp = cp->cache_arena; 11407c478bd9Sstevel@tonic-gate 11417c478bd9Sstevel@tonic-gate color = cp->cache_color + cp->cache_align; 11427c478bd9Sstevel@tonic-gate if (color > cp->cache_maxcolor) 11437c478bd9Sstevel@tonic-gate color = cp->cache_mincolor; 11447c478bd9Sstevel@tonic-gate cp->cache_color = color; 11457c478bd9Sstevel@tonic-gate 11467c478bd9Sstevel@tonic-gate slab = vmem_alloc(vmp, slabsize, UMEM_VMFLAGS(umflag)); 11477c478bd9Sstevel@tonic-gate 11487c478bd9Sstevel@tonic-gate if (slab == NULL) 11497c478bd9Sstevel@tonic-gate goto vmem_alloc_failure; 11507c478bd9Sstevel@tonic-gate 11517c478bd9Sstevel@tonic-gate ASSERT(P2PHASE((uintptr_t)slab, vmp->vm_quantum) == 0); 11527c478bd9Sstevel@tonic-gate 11537c478bd9Sstevel@tonic-gate if (!(cp->cache_cflags & UMC_NOTOUCH) && 11547c478bd9Sstevel@tonic-gate (cp->cache_flags & UMF_DEADBEEF)) 11557c478bd9Sstevel@tonic-gate copy_pattern(UMEM_UNINITIALIZED_PATTERN, slab, slabsize); 11567c478bd9Sstevel@tonic-gate 11577c478bd9Sstevel@tonic-gate if (cache_flags & UMF_HASH) { 11587c478bd9Sstevel@tonic-gate if ((sp = _umem_cache_alloc(umem_slab_cache, umflag)) == NULL) 11597c478bd9Sstevel@tonic-gate goto slab_alloc_failure; 11607c478bd9Sstevel@tonic-gate chunks = (slabsize - color) / chunksize; 11617c478bd9Sstevel@tonic-gate } else { 11627c478bd9Sstevel@tonic-gate sp = UMEM_SLAB(cp, slab); 11637c478bd9Sstevel@tonic-gate chunks = (slabsize - sizeof (umem_slab_t) - color) / chunksize; 11647c478bd9Sstevel@tonic-gate } 11657c478bd9Sstevel@tonic-gate 11667c478bd9Sstevel@tonic-gate sp->slab_cache = cp; 11677c478bd9Sstevel@tonic-gate sp->slab_head = NULL; 11687c478bd9Sstevel@tonic-gate sp->slab_refcnt = 0; 11697c478bd9Sstevel@tonic-gate sp->slab_base = buf = slab + color; 11707c478bd9Sstevel@tonic-gate sp->slab_chunks = chunks; 11717c478bd9Sstevel@tonic-gate 11727c478bd9Sstevel@tonic-gate ASSERT(chunks > 0); 11737c478bd9Sstevel@tonic-gate while (chunks-- != 0) { 11747c478bd9Sstevel@tonic-gate if (cache_flags & UMF_HASH) { 11757c478bd9Sstevel@tonic-gate bcp = _umem_cache_alloc(cp->cache_bufctl_cache, umflag); 11767c478bd9Sstevel@tonic-gate if (bcp == NULL) 11777c478bd9Sstevel@tonic-gate goto bufctl_alloc_failure; 11787c478bd9Sstevel@tonic-gate if (cache_flags & UMF_AUDIT) { 11797c478bd9Sstevel@tonic-gate umem_bufctl_audit_t *bcap = 11807c478bd9Sstevel@tonic-gate (umem_bufctl_audit_t *)bcp; 11817c478bd9Sstevel@tonic-gate bzero(bcap, UMEM_BUFCTL_AUDIT_SIZE); 11827c478bd9Sstevel@tonic-gate bcap->bc_cache = cp; 11837c478bd9Sstevel@tonic-gate } 11847c478bd9Sstevel@tonic-gate bcp->bc_addr = buf; 11857c478bd9Sstevel@tonic-gate bcp->bc_slab = sp; 11867c478bd9Sstevel@tonic-gate } else { 11877c478bd9Sstevel@tonic-gate bcp = UMEM_BUFCTL(cp, buf); 11887c478bd9Sstevel@tonic-gate } 11897c478bd9Sstevel@tonic-gate if (cache_flags & UMF_BUFTAG) { 11907c478bd9Sstevel@tonic-gate umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); 11917c478bd9Sstevel@tonic-gate btp->bt_redzone = UMEM_REDZONE_PATTERN; 11927c478bd9Sstevel@tonic-gate btp->bt_bufctl = bcp; 11937c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ UMEM_BUFTAG_FREE; 11947c478bd9Sstevel@tonic-gate if (cache_flags & UMF_DEADBEEF) { 11957c478bd9Sstevel@tonic-gate copy_pattern(UMEM_FREE_PATTERN, buf, 11967c478bd9Sstevel@tonic-gate cp->cache_verify); 11977c478bd9Sstevel@tonic-gate } 11987c478bd9Sstevel@tonic-gate } 11997c478bd9Sstevel@tonic-gate bcp->bc_next = sp->slab_head; 12007c478bd9Sstevel@tonic-gate sp->slab_head = bcp; 12017c478bd9Sstevel@tonic-gate buf += chunksize; 12027c478bd9Sstevel@tonic-gate } 12037c478bd9Sstevel@tonic-gate 12047c478bd9Sstevel@tonic-gate umem_log_event(umem_slab_log, cp, sp, slab); 12057c478bd9Sstevel@tonic-gate 12067c478bd9Sstevel@tonic-gate return (sp); 12077c478bd9Sstevel@tonic-gate 12087c478bd9Sstevel@tonic-gate bufctl_alloc_failure: 12097c478bd9Sstevel@tonic-gate 12107c478bd9Sstevel@tonic-gate while ((bcp = sp->slab_head) != NULL) { 12117c478bd9Sstevel@tonic-gate sp->slab_head = bcp->bc_next; 12127c478bd9Sstevel@tonic-gate _umem_cache_free(cp->cache_bufctl_cache, bcp); 12137c478bd9Sstevel@tonic-gate } 12147c478bd9Sstevel@tonic-gate _umem_cache_free(umem_slab_cache, sp); 12157c478bd9Sstevel@tonic-gate 12167c478bd9Sstevel@tonic-gate slab_alloc_failure: 12177c478bd9Sstevel@tonic-gate 12187c478bd9Sstevel@tonic-gate vmem_free(vmp, slab, slabsize); 12197c478bd9Sstevel@tonic-gate 12207c478bd9Sstevel@tonic-gate vmem_alloc_failure: 12217c478bd9Sstevel@tonic-gate 12227c478bd9Sstevel@tonic-gate umem_log_event(umem_failure_log, cp, NULL, NULL); 12237c478bd9Sstevel@tonic-gate atomic_add_64(&cp->cache_alloc_fail, 1); 12247c478bd9Sstevel@tonic-gate 12257c478bd9Sstevel@tonic-gate return (NULL); 12267c478bd9Sstevel@tonic-gate } 12277c478bd9Sstevel@tonic-gate 12287c478bd9Sstevel@tonic-gate /* 12297c478bd9Sstevel@tonic-gate * Destroy a slab. 12307c478bd9Sstevel@tonic-gate */ 12317c478bd9Sstevel@tonic-gate static void 12327c478bd9Sstevel@tonic-gate umem_slab_destroy(umem_cache_t *cp, umem_slab_t *sp) 12337c478bd9Sstevel@tonic-gate { 12347c478bd9Sstevel@tonic-gate vmem_t *vmp = cp->cache_arena; 12357c478bd9Sstevel@tonic-gate void *slab = (void *)P2ALIGN((uintptr_t)sp->slab_base, vmp->vm_quantum); 12367c478bd9Sstevel@tonic-gate 12377c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_HASH) { 12387c478bd9Sstevel@tonic-gate umem_bufctl_t *bcp; 12397c478bd9Sstevel@tonic-gate while ((bcp = sp->slab_head) != NULL) { 12407c478bd9Sstevel@tonic-gate sp->slab_head = bcp->bc_next; 12417c478bd9Sstevel@tonic-gate _umem_cache_free(cp->cache_bufctl_cache, bcp); 12427c478bd9Sstevel@tonic-gate } 12437c478bd9Sstevel@tonic-gate _umem_cache_free(umem_slab_cache, sp); 12447c478bd9Sstevel@tonic-gate } 12457c478bd9Sstevel@tonic-gate vmem_free(vmp, slab, cp->cache_slabsize); 12467c478bd9Sstevel@tonic-gate } 12477c478bd9Sstevel@tonic-gate 12487c478bd9Sstevel@tonic-gate /* 12497c478bd9Sstevel@tonic-gate * Allocate a raw (unconstructed) buffer from cp's slab layer. 12507c478bd9Sstevel@tonic-gate */ 12517c478bd9Sstevel@tonic-gate static void * 12527c478bd9Sstevel@tonic-gate umem_slab_alloc(umem_cache_t *cp, int umflag) 12537c478bd9Sstevel@tonic-gate { 12547c478bd9Sstevel@tonic-gate umem_bufctl_t *bcp, **hash_bucket; 12557c478bd9Sstevel@tonic-gate umem_slab_t *sp; 12567c478bd9Sstevel@tonic-gate void *buf; 12577c478bd9Sstevel@tonic-gate 12587c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_lock); 12597c478bd9Sstevel@tonic-gate cp->cache_slab_alloc++; 12607c478bd9Sstevel@tonic-gate sp = cp->cache_freelist; 12617c478bd9Sstevel@tonic-gate ASSERT(sp->slab_cache == cp); 12627c478bd9Sstevel@tonic-gate if (sp->slab_head == NULL) { 12637c478bd9Sstevel@tonic-gate /* 12647c478bd9Sstevel@tonic-gate * The freelist is empty. Create a new slab. 12657c478bd9Sstevel@tonic-gate */ 12667c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 12677c478bd9Sstevel@tonic-gate if (cp == &umem_null_cache) 12687c478bd9Sstevel@tonic-gate return (NULL); 12697c478bd9Sstevel@tonic-gate if ((sp = umem_slab_create(cp, umflag)) == NULL) 12707c478bd9Sstevel@tonic-gate return (NULL); 12717c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_lock); 12727c478bd9Sstevel@tonic-gate cp->cache_slab_create++; 12737c478bd9Sstevel@tonic-gate if ((cp->cache_buftotal += sp->slab_chunks) > cp->cache_bufmax) 12747c478bd9Sstevel@tonic-gate cp->cache_bufmax = cp->cache_buftotal; 12757c478bd9Sstevel@tonic-gate sp->slab_next = cp->cache_freelist; 12767c478bd9Sstevel@tonic-gate sp->slab_prev = cp->cache_freelist->slab_prev; 12777c478bd9Sstevel@tonic-gate sp->slab_next->slab_prev = sp; 12787c478bd9Sstevel@tonic-gate sp->slab_prev->slab_next = sp; 12797c478bd9Sstevel@tonic-gate cp->cache_freelist = sp; 12807c478bd9Sstevel@tonic-gate } 12817c478bd9Sstevel@tonic-gate 12827c478bd9Sstevel@tonic-gate sp->slab_refcnt++; 12837c478bd9Sstevel@tonic-gate ASSERT(sp->slab_refcnt <= sp->slab_chunks); 12847c478bd9Sstevel@tonic-gate 12857c478bd9Sstevel@tonic-gate /* 12867c478bd9Sstevel@tonic-gate * If we're taking the last buffer in the slab, 12877c478bd9Sstevel@tonic-gate * remove the slab from the cache's freelist. 12887c478bd9Sstevel@tonic-gate */ 12897c478bd9Sstevel@tonic-gate bcp = sp->slab_head; 12907c478bd9Sstevel@tonic-gate if ((sp->slab_head = bcp->bc_next) == NULL) { 12917c478bd9Sstevel@tonic-gate cp->cache_freelist = sp->slab_next; 12927c478bd9Sstevel@tonic-gate ASSERT(sp->slab_refcnt == sp->slab_chunks); 12937c478bd9Sstevel@tonic-gate } 12947c478bd9Sstevel@tonic-gate 12957c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_HASH) { 12967c478bd9Sstevel@tonic-gate /* 12977c478bd9Sstevel@tonic-gate * Add buffer to allocated-address hash table. 12987c478bd9Sstevel@tonic-gate */ 12997c478bd9Sstevel@tonic-gate buf = bcp->bc_addr; 13007c478bd9Sstevel@tonic-gate hash_bucket = UMEM_HASH(cp, buf); 13017c478bd9Sstevel@tonic-gate bcp->bc_next = *hash_bucket; 13027c478bd9Sstevel@tonic-gate *hash_bucket = bcp; 13037c478bd9Sstevel@tonic-gate if ((cp->cache_flags & (UMF_AUDIT | UMF_BUFTAG)) == UMF_AUDIT) { 13047c478bd9Sstevel@tonic-gate UMEM_AUDIT(umem_transaction_log, cp, bcp); 13057c478bd9Sstevel@tonic-gate } 13067c478bd9Sstevel@tonic-gate } else { 13077c478bd9Sstevel@tonic-gate buf = UMEM_BUF(cp, bcp); 13087c478bd9Sstevel@tonic-gate } 13097c478bd9Sstevel@tonic-gate 13107c478bd9Sstevel@tonic-gate ASSERT(UMEM_SLAB_MEMBER(sp, buf)); 13117c478bd9Sstevel@tonic-gate 13127c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 13137c478bd9Sstevel@tonic-gate 13147c478bd9Sstevel@tonic-gate return (buf); 13157c478bd9Sstevel@tonic-gate } 13167c478bd9Sstevel@tonic-gate 13177c478bd9Sstevel@tonic-gate /* 13187c478bd9Sstevel@tonic-gate * Free a raw (unconstructed) buffer to cp's slab layer. 13197c478bd9Sstevel@tonic-gate */ 13207c478bd9Sstevel@tonic-gate static void 13217c478bd9Sstevel@tonic-gate umem_slab_free(umem_cache_t *cp, void *buf) 13227c478bd9Sstevel@tonic-gate { 13237c478bd9Sstevel@tonic-gate umem_slab_t *sp; 13247c478bd9Sstevel@tonic-gate umem_bufctl_t *bcp, **prev_bcpp; 13257c478bd9Sstevel@tonic-gate 13267c478bd9Sstevel@tonic-gate ASSERT(buf != NULL); 13277c478bd9Sstevel@tonic-gate 13287c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_lock); 13297c478bd9Sstevel@tonic-gate cp->cache_slab_free++; 13307c478bd9Sstevel@tonic-gate 13317c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_HASH) { 13327c478bd9Sstevel@tonic-gate /* 13337c478bd9Sstevel@tonic-gate * Look up buffer in allocated-address hash table. 13347c478bd9Sstevel@tonic-gate */ 13357c478bd9Sstevel@tonic-gate prev_bcpp = UMEM_HASH(cp, buf); 13367c478bd9Sstevel@tonic-gate while ((bcp = *prev_bcpp) != NULL) { 13377c478bd9Sstevel@tonic-gate if (bcp->bc_addr == buf) { 13387c478bd9Sstevel@tonic-gate *prev_bcpp = bcp->bc_next; 13397c478bd9Sstevel@tonic-gate sp = bcp->bc_slab; 13407c478bd9Sstevel@tonic-gate break; 13417c478bd9Sstevel@tonic-gate } 13427c478bd9Sstevel@tonic-gate cp->cache_lookup_depth++; 13437c478bd9Sstevel@tonic-gate prev_bcpp = &bcp->bc_next; 13447c478bd9Sstevel@tonic-gate } 13457c478bd9Sstevel@tonic-gate } else { 13467c478bd9Sstevel@tonic-gate bcp = UMEM_BUFCTL(cp, buf); 13477c478bd9Sstevel@tonic-gate sp = UMEM_SLAB(cp, buf); 13487c478bd9Sstevel@tonic-gate } 13497c478bd9Sstevel@tonic-gate 13507c478bd9Sstevel@tonic-gate if (bcp == NULL || sp->slab_cache != cp || !UMEM_SLAB_MEMBER(sp, buf)) { 13517c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 13527c478bd9Sstevel@tonic-gate umem_error(UMERR_BADADDR, cp, buf); 13537c478bd9Sstevel@tonic-gate return; 13547c478bd9Sstevel@tonic-gate } 13557c478bd9Sstevel@tonic-gate 13567c478bd9Sstevel@tonic-gate if ((cp->cache_flags & (UMF_AUDIT | UMF_BUFTAG)) == UMF_AUDIT) { 13577c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_CONTENTS) 13587c478bd9Sstevel@tonic-gate ((umem_bufctl_audit_t *)bcp)->bc_contents = 13597c478bd9Sstevel@tonic-gate umem_log_enter(umem_content_log, buf, 13607c478bd9Sstevel@tonic-gate cp->cache_contents); 13617c478bd9Sstevel@tonic-gate UMEM_AUDIT(umem_transaction_log, cp, bcp); 13627c478bd9Sstevel@tonic-gate } 13637c478bd9Sstevel@tonic-gate 13647c478bd9Sstevel@tonic-gate /* 13657c478bd9Sstevel@tonic-gate * If this slab isn't currently on the freelist, put it there. 13667c478bd9Sstevel@tonic-gate */ 13677c478bd9Sstevel@tonic-gate if (sp->slab_head == NULL) { 13687c478bd9Sstevel@tonic-gate ASSERT(sp->slab_refcnt == sp->slab_chunks); 13697c478bd9Sstevel@tonic-gate ASSERT(cp->cache_freelist != sp); 13707c478bd9Sstevel@tonic-gate sp->slab_next->slab_prev = sp->slab_prev; 13717c478bd9Sstevel@tonic-gate sp->slab_prev->slab_next = sp->slab_next; 13727c478bd9Sstevel@tonic-gate sp->slab_next = cp->cache_freelist; 13737c478bd9Sstevel@tonic-gate sp->slab_prev = cp->cache_freelist->slab_prev; 13747c478bd9Sstevel@tonic-gate sp->slab_next->slab_prev = sp; 13757c478bd9Sstevel@tonic-gate sp->slab_prev->slab_next = sp; 13767c478bd9Sstevel@tonic-gate cp->cache_freelist = sp; 13777c478bd9Sstevel@tonic-gate } 13787c478bd9Sstevel@tonic-gate 13797c478bd9Sstevel@tonic-gate bcp->bc_next = sp->slab_head; 13807c478bd9Sstevel@tonic-gate sp->slab_head = bcp; 13817c478bd9Sstevel@tonic-gate 13827c478bd9Sstevel@tonic-gate ASSERT(sp->slab_refcnt >= 1); 13837c478bd9Sstevel@tonic-gate if (--sp->slab_refcnt == 0) { 13847c478bd9Sstevel@tonic-gate /* 13857c478bd9Sstevel@tonic-gate * There are no outstanding allocations from this slab, 13867c478bd9Sstevel@tonic-gate * so we can reclaim the memory. 13877c478bd9Sstevel@tonic-gate */ 13887c478bd9Sstevel@tonic-gate sp->slab_next->slab_prev = sp->slab_prev; 13897c478bd9Sstevel@tonic-gate sp->slab_prev->slab_next = sp->slab_next; 13907c478bd9Sstevel@tonic-gate if (sp == cp->cache_freelist) 13917c478bd9Sstevel@tonic-gate cp->cache_freelist = sp->slab_next; 13927c478bd9Sstevel@tonic-gate cp->cache_slab_destroy++; 13937c478bd9Sstevel@tonic-gate cp->cache_buftotal -= sp->slab_chunks; 13947c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 13957c478bd9Sstevel@tonic-gate umem_slab_destroy(cp, sp); 13967c478bd9Sstevel@tonic-gate return; 13977c478bd9Sstevel@tonic-gate } 13987c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 13997c478bd9Sstevel@tonic-gate } 14007c478bd9Sstevel@tonic-gate 14017c478bd9Sstevel@tonic-gate static int 14027c478bd9Sstevel@tonic-gate umem_cache_alloc_debug(umem_cache_t *cp, void *buf, int umflag) 14037c478bd9Sstevel@tonic-gate { 14047c478bd9Sstevel@tonic-gate umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); 14057c478bd9Sstevel@tonic-gate umem_bufctl_audit_t *bcp = (umem_bufctl_audit_t *)btp->bt_bufctl; 14067c478bd9Sstevel@tonic-gate uint32_t mtbf; 14077c478bd9Sstevel@tonic-gate int flags_nfatal; 14087c478bd9Sstevel@tonic-gate 14097c478bd9Sstevel@tonic-gate if (btp->bt_bxstat != ((intptr_t)bcp ^ UMEM_BUFTAG_FREE)) { 14107c478bd9Sstevel@tonic-gate umem_error(UMERR_BADBUFTAG, cp, buf); 14117c478bd9Sstevel@tonic-gate return (-1); 14127c478bd9Sstevel@tonic-gate } 14137c478bd9Sstevel@tonic-gate 14147c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ UMEM_BUFTAG_ALLOC; 14157c478bd9Sstevel@tonic-gate 14167c478bd9Sstevel@tonic-gate if ((cp->cache_flags & UMF_HASH) && bcp->bc_addr != buf) { 14177c478bd9Sstevel@tonic-gate umem_error(UMERR_BADBUFCTL, cp, buf); 14187c478bd9Sstevel@tonic-gate return (-1); 14197c478bd9Sstevel@tonic-gate } 14207c478bd9Sstevel@tonic-gate 14217c478bd9Sstevel@tonic-gate btp->bt_redzone = UMEM_REDZONE_PATTERN; 14227c478bd9Sstevel@tonic-gate 14237c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_DEADBEEF) { 14247c478bd9Sstevel@tonic-gate if (verify_and_copy_pattern(UMEM_FREE_PATTERN, 14257c478bd9Sstevel@tonic-gate UMEM_UNINITIALIZED_PATTERN, buf, cp->cache_verify)) { 14267c478bd9Sstevel@tonic-gate umem_error(UMERR_MODIFIED, cp, buf); 14277c478bd9Sstevel@tonic-gate return (-1); 14287c478bd9Sstevel@tonic-gate } 14297c478bd9Sstevel@tonic-gate } 14307c478bd9Sstevel@tonic-gate 14317c478bd9Sstevel@tonic-gate if ((mtbf = umem_mtbf | cp->cache_mtbf) != 0 && 14327c478bd9Sstevel@tonic-gate gethrtime() % mtbf == 0 && 14337c478bd9Sstevel@tonic-gate (umflag & (UMEM_FATAL_FLAGS)) == 0) { 14347c478bd9Sstevel@tonic-gate umem_log_event(umem_failure_log, cp, NULL, NULL); 14357c478bd9Sstevel@tonic-gate } else { 14367c478bd9Sstevel@tonic-gate mtbf = 0; 14377c478bd9Sstevel@tonic-gate } 14387c478bd9Sstevel@tonic-gate 14397c478bd9Sstevel@tonic-gate /* 14407c478bd9Sstevel@tonic-gate * We do not pass fatal flags on to the constructor. This prevents 14417c478bd9Sstevel@tonic-gate * leaking buffers in the event of a subordinate constructor failing. 14427c478bd9Sstevel@tonic-gate */ 14437c478bd9Sstevel@tonic-gate flags_nfatal = UMEM_DEFAULT; 14447c478bd9Sstevel@tonic-gate if (mtbf || (cp->cache_constructor != NULL && 14457c478bd9Sstevel@tonic-gate cp->cache_constructor(buf, cp->cache_private, flags_nfatal) != 0)) { 14467c478bd9Sstevel@tonic-gate atomic_add_64(&cp->cache_alloc_fail, 1); 14477c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ UMEM_BUFTAG_FREE; 14487c478bd9Sstevel@tonic-gate copy_pattern(UMEM_FREE_PATTERN, buf, cp->cache_verify); 14497c478bd9Sstevel@tonic-gate umem_slab_free(cp, buf); 14507c478bd9Sstevel@tonic-gate return (-1); 14517c478bd9Sstevel@tonic-gate } 14527c478bd9Sstevel@tonic-gate 14537c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_AUDIT) { 14547c478bd9Sstevel@tonic-gate UMEM_AUDIT(umem_transaction_log, cp, bcp); 14557c478bd9Sstevel@tonic-gate } 14567c478bd9Sstevel@tonic-gate 14577c478bd9Sstevel@tonic-gate return (0); 14587c478bd9Sstevel@tonic-gate } 14597c478bd9Sstevel@tonic-gate 14607c478bd9Sstevel@tonic-gate static int 14617c478bd9Sstevel@tonic-gate umem_cache_free_debug(umem_cache_t *cp, void *buf) 14627c478bd9Sstevel@tonic-gate { 14637c478bd9Sstevel@tonic-gate umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); 14647c478bd9Sstevel@tonic-gate umem_bufctl_audit_t *bcp = (umem_bufctl_audit_t *)btp->bt_bufctl; 14657c478bd9Sstevel@tonic-gate umem_slab_t *sp; 14667c478bd9Sstevel@tonic-gate 14677c478bd9Sstevel@tonic-gate if (btp->bt_bxstat != ((intptr_t)bcp ^ UMEM_BUFTAG_ALLOC)) { 14687c478bd9Sstevel@tonic-gate if (btp->bt_bxstat == ((intptr_t)bcp ^ UMEM_BUFTAG_FREE)) { 14697c478bd9Sstevel@tonic-gate umem_error(UMERR_DUPFREE, cp, buf); 14707c478bd9Sstevel@tonic-gate return (-1); 14717c478bd9Sstevel@tonic-gate } 14727c478bd9Sstevel@tonic-gate sp = umem_findslab(cp, buf); 14737c478bd9Sstevel@tonic-gate if (sp == NULL || sp->slab_cache != cp) 14747c478bd9Sstevel@tonic-gate umem_error(UMERR_BADADDR, cp, buf); 14757c478bd9Sstevel@tonic-gate else 14767c478bd9Sstevel@tonic-gate umem_error(UMERR_REDZONE, cp, buf); 14777c478bd9Sstevel@tonic-gate return (-1); 14787c478bd9Sstevel@tonic-gate } 14797c478bd9Sstevel@tonic-gate 14807c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ UMEM_BUFTAG_FREE; 14817c478bd9Sstevel@tonic-gate 14827c478bd9Sstevel@tonic-gate if ((cp->cache_flags & UMF_HASH) && bcp->bc_addr != buf) { 14837c478bd9Sstevel@tonic-gate umem_error(UMERR_BADBUFCTL, cp, buf); 14847c478bd9Sstevel@tonic-gate return (-1); 14857c478bd9Sstevel@tonic-gate } 14867c478bd9Sstevel@tonic-gate 14877c478bd9Sstevel@tonic-gate if (btp->bt_redzone != UMEM_REDZONE_PATTERN) { 14887c478bd9Sstevel@tonic-gate umem_error(UMERR_REDZONE, cp, buf); 14897c478bd9Sstevel@tonic-gate return (-1); 14907c478bd9Sstevel@tonic-gate } 14917c478bd9Sstevel@tonic-gate 14927c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_AUDIT) { 14937c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_CONTENTS) 14947c478bd9Sstevel@tonic-gate bcp->bc_contents = umem_log_enter(umem_content_log, 14957c478bd9Sstevel@tonic-gate buf, cp->cache_contents); 14967c478bd9Sstevel@tonic-gate UMEM_AUDIT(umem_transaction_log, cp, bcp); 14977c478bd9Sstevel@tonic-gate } 14987c478bd9Sstevel@tonic-gate 14997c478bd9Sstevel@tonic-gate if (cp->cache_destructor != NULL) 15007c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 15017c478bd9Sstevel@tonic-gate 15027c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_DEADBEEF) 15037c478bd9Sstevel@tonic-gate copy_pattern(UMEM_FREE_PATTERN, buf, cp->cache_verify); 15047c478bd9Sstevel@tonic-gate 15057c478bd9Sstevel@tonic-gate return (0); 15067c478bd9Sstevel@tonic-gate } 15077c478bd9Sstevel@tonic-gate 15087c478bd9Sstevel@tonic-gate /* 15097c478bd9Sstevel@tonic-gate * Free each object in magazine mp to cp's slab layer, and free mp itself. 15107c478bd9Sstevel@tonic-gate */ 15117c478bd9Sstevel@tonic-gate static void 15127c478bd9Sstevel@tonic-gate umem_magazine_destroy(umem_cache_t *cp, umem_magazine_t *mp, int nrounds) 15137c478bd9Sstevel@tonic-gate { 15147c478bd9Sstevel@tonic-gate int round; 15157c478bd9Sstevel@tonic-gate 15167c478bd9Sstevel@tonic-gate ASSERT(cp->cache_next == NULL || IN_UPDATE()); 15177c478bd9Sstevel@tonic-gate 15187c478bd9Sstevel@tonic-gate for (round = 0; round < nrounds; round++) { 15197c478bd9Sstevel@tonic-gate void *buf = mp->mag_round[round]; 15207c478bd9Sstevel@tonic-gate 15217c478bd9Sstevel@tonic-gate if ((cp->cache_flags & UMF_DEADBEEF) && 15227c478bd9Sstevel@tonic-gate verify_pattern(UMEM_FREE_PATTERN, buf, 15237c478bd9Sstevel@tonic-gate cp->cache_verify) != NULL) { 15247c478bd9Sstevel@tonic-gate umem_error(UMERR_MODIFIED, cp, buf); 15257c478bd9Sstevel@tonic-gate continue; 15267c478bd9Sstevel@tonic-gate } 15277c478bd9Sstevel@tonic-gate 15287c478bd9Sstevel@tonic-gate if (!(cp->cache_flags & UMF_BUFTAG) && 15297c478bd9Sstevel@tonic-gate cp->cache_destructor != NULL) 15307c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 15317c478bd9Sstevel@tonic-gate 15327c478bd9Sstevel@tonic-gate umem_slab_free(cp, buf); 15337c478bd9Sstevel@tonic-gate } 15347c478bd9Sstevel@tonic-gate ASSERT(UMEM_MAGAZINE_VALID(cp, mp)); 15357c478bd9Sstevel@tonic-gate _umem_cache_free(cp->cache_magtype->mt_cache, mp); 15367c478bd9Sstevel@tonic-gate } 15377c478bd9Sstevel@tonic-gate 15387c478bd9Sstevel@tonic-gate /* 15397c478bd9Sstevel@tonic-gate * Allocate a magazine from the depot. 15407c478bd9Sstevel@tonic-gate */ 15417c478bd9Sstevel@tonic-gate static umem_magazine_t * 15427c478bd9Sstevel@tonic-gate umem_depot_alloc(umem_cache_t *cp, umem_maglist_t *mlp) 15437c478bd9Sstevel@tonic-gate { 15447c478bd9Sstevel@tonic-gate umem_magazine_t *mp; 15457c478bd9Sstevel@tonic-gate 15467c478bd9Sstevel@tonic-gate /* 15477c478bd9Sstevel@tonic-gate * If we can't get the depot lock without contention, 15487c478bd9Sstevel@tonic-gate * update our contention count. We use the depot 15497c478bd9Sstevel@tonic-gate * contention rate to determine whether we need to 15507c478bd9Sstevel@tonic-gate * increase the magazine size for better scalability. 15517c478bd9Sstevel@tonic-gate */ 15527c478bd9Sstevel@tonic-gate if (mutex_trylock(&cp->cache_depot_lock) != 0) { 15537c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_depot_lock); 15547c478bd9Sstevel@tonic-gate cp->cache_depot_contention++; 15557c478bd9Sstevel@tonic-gate } 15567c478bd9Sstevel@tonic-gate 15577c478bd9Sstevel@tonic-gate if ((mp = mlp->ml_list) != NULL) { 15587c478bd9Sstevel@tonic-gate ASSERT(UMEM_MAGAZINE_VALID(cp, mp)); 15597c478bd9Sstevel@tonic-gate mlp->ml_list = mp->mag_next; 15607c478bd9Sstevel@tonic-gate if (--mlp->ml_total < mlp->ml_min) 15617c478bd9Sstevel@tonic-gate mlp->ml_min = mlp->ml_total; 15627c478bd9Sstevel@tonic-gate mlp->ml_alloc++; 15637c478bd9Sstevel@tonic-gate } 15647c478bd9Sstevel@tonic-gate 15657c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_depot_lock); 15667c478bd9Sstevel@tonic-gate 15677c478bd9Sstevel@tonic-gate return (mp); 15687c478bd9Sstevel@tonic-gate } 15697c478bd9Sstevel@tonic-gate 15707c478bd9Sstevel@tonic-gate /* 15717c478bd9Sstevel@tonic-gate * Free a magazine to the depot. 15727c478bd9Sstevel@tonic-gate */ 15737c478bd9Sstevel@tonic-gate static void 15747c478bd9Sstevel@tonic-gate umem_depot_free(umem_cache_t *cp, umem_maglist_t *mlp, umem_magazine_t *mp) 15757c478bd9Sstevel@tonic-gate { 15767c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_depot_lock); 15777c478bd9Sstevel@tonic-gate ASSERT(UMEM_MAGAZINE_VALID(cp, mp)); 15787c478bd9Sstevel@tonic-gate mp->mag_next = mlp->ml_list; 15797c478bd9Sstevel@tonic-gate mlp->ml_list = mp; 15807c478bd9Sstevel@tonic-gate mlp->ml_total++; 15817c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_depot_lock); 15827c478bd9Sstevel@tonic-gate } 15837c478bd9Sstevel@tonic-gate 15847c478bd9Sstevel@tonic-gate /* 15857c478bd9Sstevel@tonic-gate * Update the working set statistics for cp's depot. 15867c478bd9Sstevel@tonic-gate */ 15877c478bd9Sstevel@tonic-gate static void 15887c478bd9Sstevel@tonic-gate umem_depot_ws_update(umem_cache_t *cp) 15897c478bd9Sstevel@tonic-gate { 15907c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_depot_lock); 15917c478bd9Sstevel@tonic-gate cp->cache_full.ml_reaplimit = cp->cache_full.ml_min; 15927c478bd9Sstevel@tonic-gate cp->cache_full.ml_min = cp->cache_full.ml_total; 15937c478bd9Sstevel@tonic-gate cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_min; 15947c478bd9Sstevel@tonic-gate cp->cache_empty.ml_min = cp->cache_empty.ml_total; 15957c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_depot_lock); 15967c478bd9Sstevel@tonic-gate } 15977c478bd9Sstevel@tonic-gate 15987c478bd9Sstevel@tonic-gate /* 15997c478bd9Sstevel@tonic-gate * Reap all magazines that have fallen out of the depot's working set. 16007c478bd9Sstevel@tonic-gate */ 16017c478bd9Sstevel@tonic-gate static void 16027c478bd9Sstevel@tonic-gate umem_depot_ws_reap(umem_cache_t *cp) 16037c478bd9Sstevel@tonic-gate { 16047c478bd9Sstevel@tonic-gate long reap; 16057c478bd9Sstevel@tonic-gate umem_magazine_t *mp; 16067c478bd9Sstevel@tonic-gate 16077c478bd9Sstevel@tonic-gate ASSERT(cp->cache_next == NULL || IN_REAP()); 16087c478bd9Sstevel@tonic-gate 16097c478bd9Sstevel@tonic-gate reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min); 16107c478bd9Sstevel@tonic-gate while (reap-- && (mp = umem_depot_alloc(cp, &cp->cache_full)) != NULL) 16117c478bd9Sstevel@tonic-gate umem_magazine_destroy(cp, mp, cp->cache_magtype->mt_magsize); 16127c478bd9Sstevel@tonic-gate 16137c478bd9Sstevel@tonic-gate reap = MIN(cp->cache_empty.ml_reaplimit, cp->cache_empty.ml_min); 16147c478bd9Sstevel@tonic-gate while (reap-- && (mp = umem_depot_alloc(cp, &cp->cache_empty)) != NULL) 16157c478bd9Sstevel@tonic-gate umem_magazine_destroy(cp, mp, 0); 16167c478bd9Sstevel@tonic-gate } 16177c478bd9Sstevel@tonic-gate 16187c478bd9Sstevel@tonic-gate static void 16197c478bd9Sstevel@tonic-gate umem_cpu_reload(umem_cpu_cache_t *ccp, umem_magazine_t *mp, int rounds) 16207c478bd9Sstevel@tonic-gate { 16217c478bd9Sstevel@tonic-gate ASSERT((ccp->cc_loaded == NULL && ccp->cc_rounds == -1) || 16227c478bd9Sstevel@tonic-gate (ccp->cc_loaded && ccp->cc_rounds + rounds == ccp->cc_magsize)); 16237c478bd9Sstevel@tonic-gate ASSERT(ccp->cc_magsize > 0); 16247c478bd9Sstevel@tonic-gate 16257c478bd9Sstevel@tonic-gate ccp->cc_ploaded = ccp->cc_loaded; 16267c478bd9Sstevel@tonic-gate ccp->cc_prounds = ccp->cc_rounds; 16277c478bd9Sstevel@tonic-gate ccp->cc_loaded = mp; 16287c478bd9Sstevel@tonic-gate ccp->cc_rounds = rounds; 16297c478bd9Sstevel@tonic-gate } 16307c478bd9Sstevel@tonic-gate 16317c478bd9Sstevel@tonic-gate /* 16327c478bd9Sstevel@tonic-gate * Allocate a constructed object from cache cp. 16337c478bd9Sstevel@tonic-gate */ 16347c478bd9Sstevel@tonic-gate #pragma weak umem_cache_alloc = _umem_cache_alloc 16357c478bd9Sstevel@tonic-gate void * 16367c478bd9Sstevel@tonic-gate _umem_cache_alloc(umem_cache_t *cp, int umflag) 16377c478bd9Sstevel@tonic-gate { 16387c478bd9Sstevel@tonic-gate umem_cpu_cache_t *ccp; 16397c478bd9Sstevel@tonic-gate umem_magazine_t *fmp; 16407c478bd9Sstevel@tonic-gate void *buf; 16417c478bd9Sstevel@tonic-gate int flags_nfatal; 16427c478bd9Sstevel@tonic-gate 16437c478bd9Sstevel@tonic-gate retry: 16447c478bd9Sstevel@tonic-gate ccp = UMEM_CPU_CACHE(cp, CPU(cp->cache_cpu_mask)); 16457c478bd9Sstevel@tonic-gate (void) mutex_lock(&ccp->cc_lock); 16467c478bd9Sstevel@tonic-gate for (;;) { 16477c478bd9Sstevel@tonic-gate /* 16487c478bd9Sstevel@tonic-gate * If there's an object available in the current CPU's 16497c478bd9Sstevel@tonic-gate * loaded magazine, just take it and return. 16507c478bd9Sstevel@tonic-gate */ 16517c478bd9Sstevel@tonic-gate if (ccp->cc_rounds > 0) { 16527c478bd9Sstevel@tonic-gate buf = ccp->cc_loaded->mag_round[--ccp->cc_rounds]; 16537c478bd9Sstevel@tonic-gate ccp->cc_alloc++; 16547c478bd9Sstevel@tonic-gate (void) mutex_unlock(&ccp->cc_lock); 16557c478bd9Sstevel@tonic-gate if ((ccp->cc_flags & UMF_BUFTAG) && 16567c478bd9Sstevel@tonic-gate umem_cache_alloc_debug(cp, buf, umflag) == -1) { 16577c478bd9Sstevel@tonic-gate if (umem_alloc_retry(cp, umflag)) { 16587c478bd9Sstevel@tonic-gate goto retry; 16597c478bd9Sstevel@tonic-gate } 16607c478bd9Sstevel@tonic-gate 16617c478bd9Sstevel@tonic-gate return (NULL); 16627c478bd9Sstevel@tonic-gate } 16637c478bd9Sstevel@tonic-gate return (buf); 16647c478bd9Sstevel@tonic-gate } 16657c478bd9Sstevel@tonic-gate 16667c478bd9Sstevel@tonic-gate /* 16677c478bd9Sstevel@tonic-gate * The loaded magazine is empty. If the previously loaded 16687c478bd9Sstevel@tonic-gate * magazine was full, exchange them and try again. 16697c478bd9Sstevel@tonic-gate */ 16707c478bd9Sstevel@tonic-gate if (ccp->cc_prounds > 0) { 16717c478bd9Sstevel@tonic-gate umem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds); 16727c478bd9Sstevel@tonic-gate continue; 16737c478bd9Sstevel@tonic-gate } 16747c478bd9Sstevel@tonic-gate 16757c478bd9Sstevel@tonic-gate /* 16767c478bd9Sstevel@tonic-gate * If the magazine layer is disabled, break out now. 16777c478bd9Sstevel@tonic-gate */ 16787c478bd9Sstevel@tonic-gate if (ccp->cc_magsize == 0) 16797c478bd9Sstevel@tonic-gate break; 16807c478bd9Sstevel@tonic-gate 16817c478bd9Sstevel@tonic-gate /* 16827c478bd9Sstevel@tonic-gate * Try to get a full magazine from the depot. 16837c478bd9Sstevel@tonic-gate */ 16847c478bd9Sstevel@tonic-gate fmp = umem_depot_alloc(cp, &cp->cache_full); 16857c478bd9Sstevel@tonic-gate if (fmp != NULL) { 16867c478bd9Sstevel@tonic-gate if (ccp->cc_ploaded != NULL) 16877c478bd9Sstevel@tonic-gate umem_depot_free(cp, &cp->cache_empty, 16887c478bd9Sstevel@tonic-gate ccp->cc_ploaded); 16897c478bd9Sstevel@tonic-gate umem_cpu_reload(ccp, fmp, ccp->cc_magsize); 16907c478bd9Sstevel@tonic-gate continue; 16917c478bd9Sstevel@tonic-gate } 16927c478bd9Sstevel@tonic-gate 16937c478bd9Sstevel@tonic-gate /* 16947c478bd9Sstevel@tonic-gate * There are no full magazines in the depot, 16957c478bd9Sstevel@tonic-gate * so fall through to the slab layer. 16967c478bd9Sstevel@tonic-gate */ 16977c478bd9Sstevel@tonic-gate break; 16987c478bd9Sstevel@tonic-gate } 16997c478bd9Sstevel@tonic-gate (void) mutex_unlock(&ccp->cc_lock); 17007c478bd9Sstevel@tonic-gate 17017c478bd9Sstevel@tonic-gate /* 17027c478bd9Sstevel@tonic-gate * We couldn't allocate a constructed object from the magazine layer, 17037c478bd9Sstevel@tonic-gate * so get a raw buffer from the slab layer and apply its constructor. 17047c478bd9Sstevel@tonic-gate */ 17057c478bd9Sstevel@tonic-gate buf = umem_slab_alloc(cp, umflag); 17067c478bd9Sstevel@tonic-gate 17077c478bd9Sstevel@tonic-gate if (buf == NULL) { 17087c478bd9Sstevel@tonic-gate if (cp == &umem_null_cache) 17097c478bd9Sstevel@tonic-gate return (NULL); 17107c478bd9Sstevel@tonic-gate if (umem_alloc_retry(cp, umflag)) { 17117c478bd9Sstevel@tonic-gate goto retry; 17127c478bd9Sstevel@tonic-gate } 17137c478bd9Sstevel@tonic-gate 17147c478bd9Sstevel@tonic-gate return (NULL); 17157c478bd9Sstevel@tonic-gate } 17167c478bd9Sstevel@tonic-gate 17177c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_BUFTAG) { 17187c478bd9Sstevel@tonic-gate /* 17197c478bd9Sstevel@tonic-gate * Let umem_cache_alloc_debug() apply the constructor for us. 17207c478bd9Sstevel@tonic-gate */ 17217c478bd9Sstevel@tonic-gate if (umem_cache_alloc_debug(cp, buf, umflag) == -1) { 17227c478bd9Sstevel@tonic-gate if (umem_alloc_retry(cp, umflag)) { 17237c478bd9Sstevel@tonic-gate goto retry; 17247c478bd9Sstevel@tonic-gate } 17257c478bd9Sstevel@tonic-gate return (NULL); 17267c478bd9Sstevel@tonic-gate } 17277c478bd9Sstevel@tonic-gate return (buf); 17287c478bd9Sstevel@tonic-gate } 17297c478bd9Sstevel@tonic-gate 17307c478bd9Sstevel@tonic-gate /* 17317c478bd9Sstevel@tonic-gate * We do not pass fatal flags on to the constructor. This prevents 17327c478bd9Sstevel@tonic-gate * leaking buffers in the event of a subordinate constructor failing. 17337c478bd9Sstevel@tonic-gate */ 17347c478bd9Sstevel@tonic-gate flags_nfatal = UMEM_DEFAULT; 17357c478bd9Sstevel@tonic-gate if (cp->cache_constructor != NULL && 17367c478bd9Sstevel@tonic-gate cp->cache_constructor(buf, cp->cache_private, flags_nfatal) != 0) { 17377c478bd9Sstevel@tonic-gate atomic_add_64(&cp->cache_alloc_fail, 1); 17387c478bd9Sstevel@tonic-gate umem_slab_free(cp, buf); 17397c478bd9Sstevel@tonic-gate 17407c478bd9Sstevel@tonic-gate if (umem_alloc_retry(cp, umflag)) { 17417c478bd9Sstevel@tonic-gate goto retry; 17427c478bd9Sstevel@tonic-gate } 17437c478bd9Sstevel@tonic-gate return (NULL); 17447c478bd9Sstevel@tonic-gate } 17457c478bd9Sstevel@tonic-gate 17467c478bd9Sstevel@tonic-gate return (buf); 17477c478bd9Sstevel@tonic-gate } 17487c478bd9Sstevel@tonic-gate 17497c478bd9Sstevel@tonic-gate /* 17507c478bd9Sstevel@tonic-gate * Free a constructed object to cache cp. 17517c478bd9Sstevel@tonic-gate */ 17527c478bd9Sstevel@tonic-gate #pragma weak umem_cache_free = _umem_cache_free 17537c478bd9Sstevel@tonic-gate void 17547c478bd9Sstevel@tonic-gate _umem_cache_free(umem_cache_t *cp, void *buf) 17557c478bd9Sstevel@tonic-gate { 17567c478bd9Sstevel@tonic-gate umem_cpu_cache_t *ccp = UMEM_CPU_CACHE(cp, CPU(cp->cache_cpu_mask)); 17577c478bd9Sstevel@tonic-gate umem_magazine_t *emp; 17587c478bd9Sstevel@tonic-gate umem_magtype_t *mtp; 17597c478bd9Sstevel@tonic-gate 17607c478bd9Sstevel@tonic-gate if (ccp->cc_flags & UMF_BUFTAG) 17617c478bd9Sstevel@tonic-gate if (umem_cache_free_debug(cp, buf) == -1) 17627c478bd9Sstevel@tonic-gate return; 17637c478bd9Sstevel@tonic-gate 17647c478bd9Sstevel@tonic-gate (void) mutex_lock(&ccp->cc_lock); 17657c478bd9Sstevel@tonic-gate for (;;) { 17667c478bd9Sstevel@tonic-gate /* 17677c478bd9Sstevel@tonic-gate * If there's a slot available in the current CPU's 17687c478bd9Sstevel@tonic-gate * loaded magazine, just put the object there and return. 17697c478bd9Sstevel@tonic-gate */ 17707c478bd9Sstevel@tonic-gate if ((uint_t)ccp->cc_rounds < ccp->cc_magsize) { 17717c478bd9Sstevel@tonic-gate ccp->cc_loaded->mag_round[ccp->cc_rounds++] = buf; 17727c478bd9Sstevel@tonic-gate ccp->cc_free++; 17737c478bd9Sstevel@tonic-gate (void) mutex_unlock(&ccp->cc_lock); 17747c478bd9Sstevel@tonic-gate return; 17757c478bd9Sstevel@tonic-gate } 17767c478bd9Sstevel@tonic-gate 17777c478bd9Sstevel@tonic-gate /* 17787c478bd9Sstevel@tonic-gate * The loaded magazine is full. If the previously loaded 17797c478bd9Sstevel@tonic-gate * magazine was empty, exchange them and try again. 17807c478bd9Sstevel@tonic-gate */ 17817c478bd9Sstevel@tonic-gate if (ccp->cc_prounds == 0) { 17827c478bd9Sstevel@tonic-gate umem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds); 17837c478bd9Sstevel@tonic-gate continue; 17847c478bd9Sstevel@tonic-gate } 17857c478bd9Sstevel@tonic-gate 17867c478bd9Sstevel@tonic-gate /* 17877c478bd9Sstevel@tonic-gate * If the magazine layer is disabled, break out now. 17887c478bd9Sstevel@tonic-gate */ 17897c478bd9Sstevel@tonic-gate if (ccp->cc_magsize == 0) 17907c478bd9Sstevel@tonic-gate break; 17917c478bd9Sstevel@tonic-gate 17927c478bd9Sstevel@tonic-gate /* 17937c478bd9Sstevel@tonic-gate * Try to get an empty magazine from the depot. 17947c478bd9Sstevel@tonic-gate */ 17957c478bd9Sstevel@tonic-gate emp = umem_depot_alloc(cp, &cp->cache_empty); 17967c478bd9Sstevel@tonic-gate if (emp != NULL) { 17977c478bd9Sstevel@tonic-gate if (ccp->cc_ploaded != NULL) 17987c478bd9Sstevel@tonic-gate umem_depot_free(cp, &cp->cache_full, 17997c478bd9Sstevel@tonic-gate ccp->cc_ploaded); 18007c478bd9Sstevel@tonic-gate umem_cpu_reload(ccp, emp, 0); 18017c478bd9Sstevel@tonic-gate continue; 18027c478bd9Sstevel@tonic-gate } 18037c478bd9Sstevel@tonic-gate 18047c478bd9Sstevel@tonic-gate /* 18057c478bd9Sstevel@tonic-gate * There are no empty magazines in the depot, 18067c478bd9Sstevel@tonic-gate * so try to allocate a new one. We must drop all locks 18077c478bd9Sstevel@tonic-gate * across umem_cache_alloc() because lower layers may 18087c478bd9Sstevel@tonic-gate * attempt to allocate from this cache. 18097c478bd9Sstevel@tonic-gate */ 18107c478bd9Sstevel@tonic-gate mtp = cp->cache_magtype; 18117c478bd9Sstevel@tonic-gate (void) mutex_unlock(&ccp->cc_lock); 18127c478bd9Sstevel@tonic-gate emp = _umem_cache_alloc(mtp->mt_cache, UMEM_DEFAULT); 18137c478bd9Sstevel@tonic-gate (void) mutex_lock(&ccp->cc_lock); 18147c478bd9Sstevel@tonic-gate 18157c478bd9Sstevel@tonic-gate if (emp != NULL) { 18167c478bd9Sstevel@tonic-gate /* 18177c478bd9Sstevel@tonic-gate * We successfully allocated an empty magazine. 18187c478bd9Sstevel@tonic-gate * However, we had to drop ccp->cc_lock to do it, 18197c478bd9Sstevel@tonic-gate * so the cache's magazine size may have changed. 18207c478bd9Sstevel@tonic-gate * If so, free the magazine and try again. 18217c478bd9Sstevel@tonic-gate */ 18227c478bd9Sstevel@tonic-gate if (ccp->cc_magsize != mtp->mt_magsize) { 18237c478bd9Sstevel@tonic-gate (void) mutex_unlock(&ccp->cc_lock); 18247c478bd9Sstevel@tonic-gate _umem_cache_free(mtp->mt_cache, emp); 18257c478bd9Sstevel@tonic-gate (void) mutex_lock(&ccp->cc_lock); 18267c478bd9Sstevel@tonic-gate continue; 18277c478bd9Sstevel@tonic-gate } 18287c478bd9Sstevel@tonic-gate 18297c478bd9Sstevel@tonic-gate /* 18307c478bd9Sstevel@tonic-gate * We got a magazine of the right size. Add it to 18317c478bd9Sstevel@tonic-gate * the depot and try the whole dance again. 18327c478bd9Sstevel@tonic-gate */ 18337c478bd9Sstevel@tonic-gate umem_depot_free(cp, &cp->cache_empty, emp); 18347c478bd9Sstevel@tonic-gate continue; 18357c478bd9Sstevel@tonic-gate } 18367c478bd9Sstevel@tonic-gate 18377c478bd9Sstevel@tonic-gate /* 18387c478bd9Sstevel@tonic-gate * We couldn't allocate an empty magazine, 18397c478bd9Sstevel@tonic-gate * so fall through to the slab layer. 18407c478bd9Sstevel@tonic-gate */ 18417c478bd9Sstevel@tonic-gate break; 18427c478bd9Sstevel@tonic-gate } 18437c478bd9Sstevel@tonic-gate (void) mutex_unlock(&ccp->cc_lock); 18447c478bd9Sstevel@tonic-gate 18457c478bd9Sstevel@tonic-gate /* 18467c478bd9Sstevel@tonic-gate * We couldn't free our constructed object to the magazine layer, 18477c478bd9Sstevel@tonic-gate * so apply its destructor and free it to the slab layer. 18487c478bd9Sstevel@tonic-gate * Note that if UMF_BUFTAG is in effect, umem_cache_free_debug() 18497c478bd9Sstevel@tonic-gate * will have already applied the destructor. 18507c478bd9Sstevel@tonic-gate */ 18517c478bd9Sstevel@tonic-gate if (!(cp->cache_flags & UMF_BUFTAG) && cp->cache_destructor != NULL) 18527c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 18537c478bd9Sstevel@tonic-gate 18547c478bd9Sstevel@tonic-gate umem_slab_free(cp, buf); 18557c478bd9Sstevel@tonic-gate } 18567c478bd9Sstevel@tonic-gate 18577c478bd9Sstevel@tonic-gate #pragma weak umem_zalloc = _umem_zalloc 18587c478bd9Sstevel@tonic-gate void * 18597c478bd9Sstevel@tonic-gate _umem_zalloc(size_t size, int umflag) 18607c478bd9Sstevel@tonic-gate { 18617c478bd9Sstevel@tonic-gate size_t index = (size - 1) >> UMEM_ALIGN_SHIFT; 18627c478bd9Sstevel@tonic-gate void *buf; 18637c478bd9Sstevel@tonic-gate 18647c478bd9Sstevel@tonic-gate retry: 18657c478bd9Sstevel@tonic-gate if (index < UMEM_MAXBUF >> UMEM_ALIGN_SHIFT) { 18667c478bd9Sstevel@tonic-gate umem_cache_t *cp = umem_alloc_table[index]; 18677c478bd9Sstevel@tonic-gate buf = _umem_cache_alloc(cp, umflag); 18687c478bd9Sstevel@tonic-gate if (buf != NULL) { 18697c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_BUFTAG) { 18707c478bd9Sstevel@tonic-gate umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); 18717c478bd9Sstevel@tonic-gate ((uint8_t *)buf)[size] = UMEM_REDZONE_BYTE; 18727c478bd9Sstevel@tonic-gate ((uint32_t *)btp)[1] = UMEM_SIZE_ENCODE(size); 18737c478bd9Sstevel@tonic-gate } 18747c478bd9Sstevel@tonic-gate bzero(buf, size); 18757c478bd9Sstevel@tonic-gate } else if (umem_alloc_retry(cp, umflag)) 18767c478bd9Sstevel@tonic-gate goto retry; 18777c478bd9Sstevel@tonic-gate } else { 18787c478bd9Sstevel@tonic-gate buf = _umem_alloc(size, umflag); /* handles failure */ 18797c478bd9Sstevel@tonic-gate if (buf != NULL) 18807c478bd9Sstevel@tonic-gate bzero(buf, size); 18817c478bd9Sstevel@tonic-gate } 18827c478bd9Sstevel@tonic-gate return (buf); 18837c478bd9Sstevel@tonic-gate } 18847c478bd9Sstevel@tonic-gate 18857c478bd9Sstevel@tonic-gate #pragma weak umem_alloc = _umem_alloc 18867c478bd9Sstevel@tonic-gate void * 18877c478bd9Sstevel@tonic-gate _umem_alloc(size_t size, int umflag) 18887c478bd9Sstevel@tonic-gate { 18897c478bd9Sstevel@tonic-gate size_t index = (size - 1) >> UMEM_ALIGN_SHIFT; 18907c478bd9Sstevel@tonic-gate void *buf; 18917c478bd9Sstevel@tonic-gate umem_alloc_retry: 18927c478bd9Sstevel@tonic-gate if (index < UMEM_MAXBUF >> UMEM_ALIGN_SHIFT) { 18937c478bd9Sstevel@tonic-gate umem_cache_t *cp = umem_alloc_table[index]; 18947c478bd9Sstevel@tonic-gate buf = _umem_cache_alloc(cp, umflag); 18957c478bd9Sstevel@tonic-gate if ((cp->cache_flags & UMF_BUFTAG) && buf != NULL) { 18967c478bd9Sstevel@tonic-gate umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); 18977c478bd9Sstevel@tonic-gate ((uint8_t *)buf)[size] = UMEM_REDZONE_BYTE; 18987c478bd9Sstevel@tonic-gate ((uint32_t *)btp)[1] = UMEM_SIZE_ENCODE(size); 18997c478bd9Sstevel@tonic-gate } 19007c478bd9Sstevel@tonic-gate if (buf == NULL && umem_alloc_retry(cp, umflag)) 19017c478bd9Sstevel@tonic-gate goto umem_alloc_retry; 19027c478bd9Sstevel@tonic-gate return (buf); 19037c478bd9Sstevel@tonic-gate } 19047c478bd9Sstevel@tonic-gate if (size == 0) 19057c478bd9Sstevel@tonic-gate return (NULL); 19067c478bd9Sstevel@tonic-gate if (umem_oversize_arena == NULL) { 19077c478bd9Sstevel@tonic-gate if (umem_init()) 19087c478bd9Sstevel@tonic-gate ASSERT(umem_oversize_arena != NULL); 19097c478bd9Sstevel@tonic-gate else 19107c478bd9Sstevel@tonic-gate return (NULL); 19117c478bd9Sstevel@tonic-gate } 19127c478bd9Sstevel@tonic-gate buf = vmem_alloc(umem_oversize_arena, size, UMEM_VMFLAGS(umflag)); 19137c478bd9Sstevel@tonic-gate if (buf == NULL) { 19147c478bd9Sstevel@tonic-gate umem_log_event(umem_failure_log, NULL, NULL, (void *)size); 19157c478bd9Sstevel@tonic-gate if (umem_alloc_retry(NULL, umflag)) 19167c478bd9Sstevel@tonic-gate goto umem_alloc_retry; 19177c478bd9Sstevel@tonic-gate } 19187c478bd9Sstevel@tonic-gate return (buf); 19197c478bd9Sstevel@tonic-gate } 19207c478bd9Sstevel@tonic-gate 19217c478bd9Sstevel@tonic-gate #pragma weak umem_alloc_align = _umem_alloc_align 19227c478bd9Sstevel@tonic-gate void * 19237c478bd9Sstevel@tonic-gate _umem_alloc_align(size_t size, size_t align, int umflag) 19247c478bd9Sstevel@tonic-gate { 19257c478bd9Sstevel@tonic-gate void *buf; 19267c478bd9Sstevel@tonic-gate 19277c478bd9Sstevel@tonic-gate if (size == 0) 19287c478bd9Sstevel@tonic-gate return (NULL); 19297c478bd9Sstevel@tonic-gate if ((align & (align - 1)) != 0) 19307c478bd9Sstevel@tonic-gate return (NULL); 19317c478bd9Sstevel@tonic-gate if (align < UMEM_ALIGN) 19327c478bd9Sstevel@tonic-gate align = UMEM_ALIGN; 19337c478bd9Sstevel@tonic-gate 19347c478bd9Sstevel@tonic-gate umem_alloc_align_retry: 19357c478bd9Sstevel@tonic-gate if (umem_memalign_arena == NULL) { 19367c478bd9Sstevel@tonic-gate if (umem_init()) 19377c478bd9Sstevel@tonic-gate ASSERT(umem_oversize_arena != NULL); 19387c478bd9Sstevel@tonic-gate else 19397c478bd9Sstevel@tonic-gate return (NULL); 19407c478bd9Sstevel@tonic-gate } 19417c478bd9Sstevel@tonic-gate buf = vmem_xalloc(umem_memalign_arena, size, align, 0, 0, NULL, NULL, 19427c478bd9Sstevel@tonic-gate UMEM_VMFLAGS(umflag)); 19437c478bd9Sstevel@tonic-gate if (buf == NULL) { 19447c478bd9Sstevel@tonic-gate umem_log_event(umem_failure_log, NULL, NULL, (void *)size); 19457c478bd9Sstevel@tonic-gate if (umem_alloc_retry(NULL, umflag)) 19467c478bd9Sstevel@tonic-gate goto umem_alloc_align_retry; 19477c478bd9Sstevel@tonic-gate } 19487c478bd9Sstevel@tonic-gate return (buf); 19497c478bd9Sstevel@tonic-gate } 19507c478bd9Sstevel@tonic-gate 19517c478bd9Sstevel@tonic-gate #pragma weak umem_free = _umem_free 19527c478bd9Sstevel@tonic-gate void 19537c478bd9Sstevel@tonic-gate _umem_free(void *buf, size_t size) 19547c478bd9Sstevel@tonic-gate { 19557c478bd9Sstevel@tonic-gate size_t index = (size - 1) >> UMEM_ALIGN_SHIFT; 19567c478bd9Sstevel@tonic-gate 19577c478bd9Sstevel@tonic-gate if (index < UMEM_MAXBUF >> UMEM_ALIGN_SHIFT) { 19587c478bd9Sstevel@tonic-gate umem_cache_t *cp = umem_alloc_table[index]; 19597c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_BUFTAG) { 19607c478bd9Sstevel@tonic-gate umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); 19617c478bd9Sstevel@tonic-gate uint32_t *ip = (uint32_t *)btp; 19627c478bd9Sstevel@tonic-gate if (ip[1] != UMEM_SIZE_ENCODE(size)) { 19637c478bd9Sstevel@tonic-gate if (*(uint64_t *)buf == UMEM_FREE_PATTERN) { 19647c478bd9Sstevel@tonic-gate umem_error(UMERR_DUPFREE, cp, buf); 19657c478bd9Sstevel@tonic-gate return; 19667c478bd9Sstevel@tonic-gate } 19677c478bd9Sstevel@tonic-gate if (UMEM_SIZE_VALID(ip[1])) { 19687c478bd9Sstevel@tonic-gate ip[0] = UMEM_SIZE_ENCODE(size); 19697c478bd9Sstevel@tonic-gate umem_error(UMERR_BADSIZE, cp, buf); 19707c478bd9Sstevel@tonic-gate } else { 19717c478bd9Sstevel@tonic-gate umem_error(UMERR_REDZONE, cp, buf); 19727c478bd9Sstevel@tonic-gate } 19737c478bd9Sstevel@tonic-gate return; 19747c478bd9Sstevel@tonic-gate } 19757c478bd9Sstevel@tonic-gate if (((uint8_t *)buf)[size] != UMEM_REDZONE_BYTE) { 19767c478bd9Sstevel@tonic-gate umem_error(UMERR_REDZONE, cp, buf); 19777c478bd9Sstevel@tonic-gate return; 19787c478bd9Sstevel@tonic-gate } 19797c478bd9Sstevel@tonic-gate btp->bt_redzone = UMEM_REDZONE_PATTERN; 19807c478bd9Sstevel@tonic-gate } 19817c478bd9Sstevel@tonic-gate _umem_cache_free(cp, buf); 19827c478bd9Sstevel@tonic-gate } else { 19837c478bd9Sstevel@tonic-gate if (buf == NULL && size == 0) 19847c478bd9Sstevel@tonic-gate return; 19857c478bd9Sstevel@tonic-gate vmem_free(umem_oversize_arena, buf, size); 19867c478bd9Sstevel@tonic-gate } 19877c478bd9Sstevel@tonic-gate } 19887c478bd9Sstevel@tonic-gate 19897c478bd9Sstevel@tonic-gate #pragma weak umem_free_align = _umem_free_align 19907c478bd9Sstevel@tonic-gate void 19917c478bd9Sstevel@tonic-gate _umem_free_align(void *buf, size_t size) 19927c478bd9Sstevel@tonic-gate { 19937c478bd9Sstevel@tonic-gate if (buf == NULL && size == 0) 19947c478bd9Sstevel@tonic-gate return; 19957c478bd9Sstevel@tonic-gate vmem_xfree(umem_memalign_arena, buf, size); 19967c478bd9Sstevel@tonic-gate } 19977c478bd9Sstevel@tonic-gate 19987c478bd9Sstevel@tonic-gate static void * 19997c478bd9Sstevel@tonic-gate umem_firewall_va_alloc(vmem_t *vmp, size_t size, int vmflag) 20007c478bd9Sstevel@tonic-gate { 20017c478bd9Sstevel@tonic-gate size_t realsize = size + vmp->vm_quantum; 20027c478bd9Sstevel@tonic-gate 20037c478bd9Sstevel@tonic-gate /* 20047c478bd9Sstevel@tonic-gate * Annoying edge case: if 'size' is just shy of ULONG_MAX, adding 20057c478bd9Sstevel@tonic-gate * vm_quantum will cause integer wraparound. Check for this, and 20067c478bd9Sstevel@tonic-gate * blow off the firewall page in this case. Note that such a 20077c478bd9Sstevel@tonic-gate * giant allocation (the entire address space) can never be 20087c478bd9Sstevel@tonic-gate * satisfied, so it will either fail immediately (VM_NOSLEEP) 20097c478bd9Sstevel@tonic-gate * or sleep forever (VM_SLEEP). Thus, there is no need for a 20107c478bd9Sstevel@tonic-gate * corresponding check in umem_firewall_va_free(). 20117c478bd9Sstevel@tonic-gate */ 20127c478bd9Sstevel@tonic-gate if (realsize < size) 20137c478bd9Sstevel@tonic-gate realsize = size; 20147c478bd9Sstevel@tonic-gate 20157c478bd9Sstevel@tonic-gate return (vmem_alloc(vmp, realsize, vmflag | VM_NEXTFIT)); 20167c478bd9Sstevel@tonic-gate } 20177c478bd9Sstevel@tonic-gate 20187c478bd9Sstevel@tonic-gate static void 20197c478bd9Sstevel@tonic-gate umem_firewall_va_free(vmem_t *vmp, void *addr, size_t size) 20207c478bd9Sstevel@tonic-gate { 20217c478bd9Sstevel@tonic-gate vmem_free(vmp, addr, size + vmp->vm_quantum); 20227c478bd9Sstevel@tonic-gate } 20237c478bd9Sstevel@tonic-gate 20247c478bd9Sstevel@tonic-gate /* 20257c478bd9Sstevel@tonic-gate * Reclaim all unused memory from a cache. 20267c478bd9Sstevel@tonic-gate */ 20277c478bd9Sstevel@tonic-gate static void 20287c478bd9Sstevel@tonic-gate umem_cache_reap(umem_cache_t *cp) 20297c478bd9Sstevel@tonic-gate { 20307c478bd9Sstevel@tonic-gate /* 20317c478bd9Sstevel@tonic-gate * Ask the cache's owner to free some memory if possible. 20327c478bd9Sstevel@tonic-gate * The idea is to handle things like the inode cache, which 20337c478bd9Sstevel@tonic-gate * typically sits on a bunch of memory that it doesn't truly 20347c478bd9Sstevel@tonic-gate * *need*. Reclaim policy is entirely up to the owner; this 20357c478bd9Sstevel@tonic-gate * callback is just an advisory plea for help. 20367c478bd9Sstevel@tonic-gate */ 20377c478bd9Sstevel@tonic-gate if (cp->cache_reclaim != NULL) 20387c478bd9Sstevel@tonic-gate cp->cache_reclaim(cp->cache_private); 20397c478bd9Sstevel@tonic-gate 20407c478bd9Sstevel@tonic-gate umem_depot_ws_reap(cp); 20417c478bd9Sstevel@tonic-gate } 20427c478bd9Sstevel@tonic-gate 20437c478bd9Sstevel@tonic-gate /* 20447c478bd9Sstevel@tonic-gate * Purge all magazines from a cache and set its magazine limit to zero. 20457c478bd9Sstevel@tonic-gate * All calls are serialized by being done by the update thread, except for 20467c478bd9Sstevel@tonic-gate * the final call from umem_cache_destroy(). 20477c478bd9Sstevel@tonic-gate */ 20487c478bd9Sstevel@tonic-gate static void 20497c478bd9Sstevel@tonic-gate umem_cache_magazine_purge(umem_cache_t *cp) 20507c478bd9Sstevel@tonic-gate { 20517c478bd9Sstevel@tonic-gate umem_cpu_cache_t *ccp; 20527c478bd9Sstevel@tonic-gate umem_magazine_t *mp, *pmp; 20537c478bd9Sstevel@tonic-gate int rounds, prounds, cpu_seqid; 20547c478bd9Sstevel@tonic-gate 20557c478bd9Sstevel@tonic-gate ASSERT(cp->cache_next == NULL || IN_UPDATE()); 20567c478bd9Sstevel@tonic-gate 20577c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < umem_max_ncpus; cpu_seqid++) { 20587c478bd9Sstevel@tonic-gate ccp = &cp->cache_cpu[cpu_seqid]; 20597c478bd9Sstevel@tonic-gate 20607c478bd9Sstevel@tonic-gate (void) mutex_lock(&ccp->cc_lock); 20617c478bd9Sstevel@tonic-gate mp = ccp->cc_loaded; 20627c478bd9Sstevel@tonic-gate pmp = ccp->cc_ploaded; 20637c478bd9Sstevel@tonic-gate rounds = ccp->cc_rounds; 20647c478bd9Sstevel@tonic-gate prounds = ccp->cc_prounds; 20657c478bd9Sstevel@tonic-gate ccp->cc_loaded = NULL; 20667c478bd9Sstevel@tonic-gate ccp->cc_ploaded = NULL; 20677c478bd9Sstevel@tonic-gate ccp->cc_rounds = -1; 20687c478bd9Sstevel@tonic-gate ccp->cc_prounds = -1; 20697c478bd9Sstevel@tonic-gate ccp->cc_magsize = 0; 20707c478bd9Sstevel@tonic-gate (void) mutex_unlock(&ccp->cc_lock); 20717c478bd9Sstevel@tonic-gate 20727c478bd9Sstevel@tonic-gate if (mp) 20737c478bd9Sstevel@tonic-gate umem_magazine_destroy(cp, mp, rounds); 20747c478bd9Sstevel@tonic-gate if (pmp) 20757c478bd9Sstevel@tonic-gate umem_magazine_destroy(cp, pmp, prounds); 20767c478bd9Sstevel@tonic-gate } 20777c478bd9Sstevel@tonic-gate 20787c478bd9Sstevel@tonic-gate /* 20797c478bd9Sstevel@tonic-gate * Updating the working set statistics twice in a row has the 20807c478bd9Sstevel@tonic-gate * effect of setting the working set size to zero, so everything 20817c478bd9Sstevel@tonic-gate * is eligible for reaping. 20827c478bd9Sstevel@tonic-gate */ 20837c478bd9Sstevel@tonic-gate umem_depot_ws_update(cp); 20847c478bd9Sstevel@tonic-gate umem_depot_ws_update(cp); 20857c478bd9Sstevel@tonic-gate 20867c478bd9Sstevel@tonic-gate umem_depot_ws_reap(cp); 20877c478bd9Sstevel@tonic-gate } 20887c478bd9Sstevel@tonic-gate 20897c478bd9Sstevel@tonic-gate /* 20907c478bd9Sstevel@tonic-gate * Enable per-cpu magazines on a cache. 20917c478bd9Sstevel@tonic-gate */ 20927c478bd9Sstevel@tonic-gate static void 20937c478bd9Sstevel@tonic-gate umem_cache_magazine_enable(umem_cache_t *cp) 20947c478bd9Sstevel@tonic-gate { 20957c478bd9Sstevel@tonic-gate int cpu_seqid; 20967c478bd9Sstevel@tonic-gate 20977c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_NOMAGAZINE) 20987c478bd9Sstevel@tonic-gate return; 20997c478bd9Sstevel@tonic-gate 21007c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < umem_max_ncpus; cpu_seqid++) { 21017c478bd9Sstevel@tonic-gate umem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 21027c478bd9Sstevel@tonic-gate (void) mutex_lock(&ccp->cc_lock); 21037c478bd9Sstevel@tonic-gate ccp->cc_magsize = cp->cache_magtype->mt_magsize; 21047c478bd9Sstevel@tonic-gate (void) mutex_unlock(&ccp->cc_lock); 21057c478bd9Sstevel@tonic-gate } 21067c478bd9Sstevel@tonic-gate 21077c478bd9Sstevel@tonic-gate } 21087c478bd9Sstevel@tonic-gate 21097c478bd9Sstevel@tonic-gate /* 21107c478bd9Sstevel@tonic-gate * Recompute a cache's magazine size. The trade-off is that larger magazines 21117c478bd9Sstevel@tonic-gate * provide a higher transfer rate with the depot, while smaller magazines 21127c478bd9Sstevel@tonic-gate * reduce memory consumption. Magazine resizing is an expensive operation; 21137c478bd9Sstevel@tonic-gate * it should not be done frequently. 21147c478bd9Sstevel@tonic-gate * 21157c478bd9Sstevel@tonic-gate * Changes to the magazine size are serialized by only having one thread 21167c478bd9Sstevel@tonic-gate * doing updates. (the update thread) 21177c478bd9Sstevel@tonic-gate * 21187c478bd9Sstevel@tonic-gate * Note: at present this only grows the magazine size. It might be useful 21197c478bd9Sstevel@tonic-gate * to allow shrinkage too. 21207c478bd9Sstevel@tonic-gate */ 21217c478bd9Sstevel@tonic-gate static void 21227c478bd9Sstevel@tonic-gate umem_cache_magazine_resize(umem_cache_t *cp) 21237c478bd9Sstevel@tonic-gate { 21247c478bd9Sstevel@tonic-gate umem_magtype_t *mtp = cp->cache_magtype; 21257c478bd9Sstevel@tonic-gate 21267c478bd9Sstevel@tonic-gate ASSERT(IN_UPDATE()); 21277c478bd9Sstevel@tonic-gate 21287c478bd9Sstevel@tonic-gate if (cp->cache_chunksize < mtp->mt_maxbuf) { 21297c478bd9Sstevel@tonic-gate umem_cache_magazine_purge(cp); 21307c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_depot_lock); 21317c478bd9Sstevel@tonic-gate cp->cache_magtype = ++mtp; 21327c478bd9Sstevel@tonic-gate cp->cache_depot_contention_prev = 21337c478bd9Sstevel@tonic-gate cp->cache_depot_contention + INT_MAX; 21347c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_depot_lock); 21357c478bd9Sstevel@tonic-gate umem_cache_magazine_enable(cp); 21367c478bd9Sstevel@tonic-gate } 21377c478bd9Sstevel@tonic-gate } 21387c478bd9Sstevel@tonic-gate 21397c478bd9Sstevel@tonic-gate /* 21407c478bd9Sstevel@tonic-gate * Rescale a cache's hash table, so that the table size is roughly the 21417c478bd9Sstevel@tonic-gate * cache size. We want the average lookup time to be extremely small. 21427c478bd9Sstevel@tonic-gate */ 21437c478bd9Sstevel@tonic-gate static void 21447c478bd9Sstevel@tonic-gate umem_hash_rescale(umem_cache_t *cp) 21457c478bd9Sstevel@tonic-gate { 21467c478bd9Sstevel@tonic-gate umem_bufctl_t **old_table, **new_table, *bcp; 21477c478bd9Sstevel@tonic-gate size_t old_size, new_size, h; 21487c478bd9Sstevel@tonic-gate 21497c478bd9Sstevel@tonic-gate ASSERT(IN_UPDATE()); 21507c478bd9Sstevel@tonic-gate 21517c478bd9Sstevel@tonic-gate new_size = MAX(UMEM_HASH_INITIAL, 21527c478bd9Sstevel@tonic-gate 1 << (highbit(3 * cp->cache_buftotal + 4) - 2)); 21537c478bd9Sstevel@tonic-gate old_size = cp->cache_hash_mask + 1; 21547c478bd9Sstevel@tonic-gate 21557c478bd9Sstevel@tonic-gate if ((old_size >> 1) <= new_size && new_size <= (old_size << 1)) 21567c478bd9Sstevel@tonic-gate return; 21577c478bd9Sstevel@tonic-gate 21587c478bd9Sstevel@tonic-gate new_table = vmem_alloc(umem_hash_arena, new_size * sizeof (void *), 21597c478bd9Sstevel@tonic-gate VM_NOSLEEP); 21607c478bd9Sstevel@tonic-gate if (new_table == NULL) 21617c478bd9Sstevel@tonic-gate return; 21627c478bd9Sstevel@tonic-gate bzero(new_table, new_size * sizeof (void *)); 21637c478bd9Sstevel@tonic-gate 21647c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_lock); 21657c478bd9Sstevel@tonic-gate 21667c478bd9Sstevel@tonic-gate old_size = cp->cache_hash_mask + 1; 21677c478bd9Sstevel@tonic-gate old_table = cp->cache_hash_table; 21687c478bd9Sstevel@tonic-gate 21697c478bd9Sstevel@tonic-gate cp->cache_hash_mask = new_size - 1; 21707c478bd9Sstevel@tonic-gate cp->cache_hash_table = new_table; 21717c478bd9Sstevel@tonic-gate cp->cache_rescale++; 21727c478bd9Sstevel@tonic-gate 21737c478bd9Sstevel@tonic-gate for (h = 0; h < old_size; h++) { 21747c478bd9Sstevel@tonic-gate bcp = old_table[h]; 21757c478bd9Sstevel@tonic-gate while (bcp != NULL) { 21767c478bd9Sstevel@tonic-gate void *addr = bcp->bc_addr; 21777c478bd9Sstevel@tonic-gate umem_bufctl_t *next_bcp = bcp->bc_next; 21787c478bd9Sstevel@tonic-gate umem_bufctl_t **hash_bucket = UMEM_HASH(cp, addr); 21797c478bd9Sstevel@tonic-gate bcp->bc_next = *hash_bucket; 21807c478bd9Sstevel@tonic-gate *hash_bucket = bcp; 21817c478bd9Sstevel@tonic-gate bcp = next_bcp; 21827c478bd9Sstevel@tonic-gate } 21837c478bd9Sstevel@tonic-gate } 21847c478bd9Sstevel@tonic-gate 21857c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 21867c478bd9Sstevel@tonic-gate 21877c478bd9Sstevel@tonic-gate vmem_free(umem_hash_arena, old_table, old_size * sizeof (void *)); 21887c478bd9Sstevel@tonic-gate } 21897c478bd9Sstevel@tonic-gate 21907c478bd9Sstevel@tonic-gate /* 21917c478bd9Sstevel@tonic-gate * Perform periodic maintenance on a cache: hash rescaling, 21927c478bd9Sstevel@tonic-gate * depot working-set update, and magazine resizing. 21937c478bd9Sstevel@tonic-gate */ 21947c478bd9Sstevel@tonic-gate void 21957c478bd9Sstevel@tonic-gate umem_cache_update(umem_cache_t *cp) 21967c478bd9Sstevel@tonic-gate { 21977c478bd9Sstevel@tonic-gate int update_flags = 0; 21987c478bd9Sstevel@tonic-gate 21997c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&umem_cache_lock)); 22007c478bd9Sstevel@tonic-gate 22017c478bd9Sstevel@tonic-gate /* 22027c478bd9Sstevel@tonic-gate * If the cache has become much larger or smaller than its hash table, 22037c478bd9Sstevel@tonic-gate * fire off a request to rescale the hash table. 22047c478bd9Sstevel@tonic-gate */ 22057c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_lock); 22067c478bd9Sstevel@tonic-gate 22077c478bd9Sstevel@tonic-gate if ((cp->cache_flags & UMF_HASH) && 22087c478bd9Sstevel@tonic-gate (cp->cache_buftotal > (cp->cache_hash_mask << 1) || 22097c478bd9Sstevel@tonic-gate (cp->cache_buftotal < (cp->cache_hash_mask >> 1) && 22107c478bd9Sstevel@tonic-gate cp->cache_hash_mask > UMEM_HASH_INITIAL))) 22117c478bd9Sstevel@tonic-gate update_flags |= UMU_HASH_RESCALE; 22127c478bd9Sstevel@tonic-gate 22137c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 22147c478bd9Sstevel@tonic-gate 22157c478bd9Sstevel@tonic-gate /* 22167c478bd9Sstevel@tonic-gate * Update the depot working set statistics. 22177c478bd9Sstevel@tonic-gate */ 22187c478bd9Sstevel@tonic-gate umem_depot_ws_update(cp); 22197c478bd9Sstevel@tonic-gate 22207c478bd9Sstevel@tonic-gate /* 22217c478bd9Sstevel@tonic-gate * If there's a lot of contention in the depot, 22227c478bd9Sstevel@tonic-gate * increase the magazine size. 22237c478bd9Sstevel@tonic-gate */ 22247c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_depot_lock); 22257c478bd9Sstevel@tonic-gate 22267c478bd9Sstevel@tonic-gate if (cp->cache_chunksize < cp->cache_magtype->mt_maxbuf && 22277c478bd9Sstevel@tonic-gate (int)(cp->cache_depot_contention - 22287c478bd9Sstevel@tonic-gate cp->cache_depot_contention_prev) > umem_depot_contention) 22297c478bd9Sstevel@tonic-gate update_flags |= UMU_MAGAZINE_RESIZE; 22307c478bd9Sstevel@tonic-gate 22317c478bd9Sstevel@tonic-gate cp->cache_depot_contention_prev = cp->cache_depot_contention; 22327c478bd9Sstevel@tonic-gate 22337c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_depot_lock); 22347c478bd9Sstevel@tonic-gate 22357c478bd9Sstevel@tonic-gate if (update_flags) 22367c478bd9Sstevel@tonic-gate umem_add_update(cp, update_flags); 22377c478bd9Sstevel@tonic-gate } 22387c478bd9Sstevel@tonic-gate 22397c478bd9Sstevel@tonic-gate /* 22407c478bd9Sstevel@tonic-gate * Runs all pending updates. 22417c478bd9Sstevel@tonic-gate * 22427c478bd9Sstevel@tonic-gate * The update lock must be held on entrance, and will be held on exit. 22437c478bd9Sstevel@tonic-gate */ 22447c478bd9Sstevel@tonic-gate void 22457c478bd9Sstevel@tonic-gate umem_process_updates(void) 22467c478bd9Sstevel@tonic-gate { 22477c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&umem_update_lock)); 22487c478bd9Sstevel@tonic-gate 22497c478bd9Sstevel@tonic-gate while (umem_null_cache.cache_unext != &umem_null_cache) { 22507c478bd9Sstevel@tonic-gate int notify = 0; 22517c478bd9Sstevel@tonic-gate umem_cache_t *cp = umem_null_cache.cache_unext; 22527c478bd9Sstevel@tonic-gate 22537c478bd9Sstevel@tonic-gate cp->cache_uprev->cache_unext = cp->cache_unext; 22547c478bd9Sstevel@tonic-gate cp->cache_unext->cache_uprev = cp->cache_uprev; 22557c478bd9Sstevel@tonic-gate cp->cache_uprev = cp->cache_unext = NULL; 22567c478bd9Sstevel@tonic-gate 22577c478bd9Sstevel@tonic-gate ASSERT(!(cp->cache_uflags & UMU_ACTIVE)); 22587c478bd9Sstevel@tonic-gate 22597c478bd9Sstevel@tonic-gate while (cp->cache_uflags) { 22607c478bd9Sstevel@tonic-gate int uflags = (cp->cache_uflags |= UMU_ACTIVE); 22617c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_update_lock); 22627c478bd9Sstevel@tonic-gate 22637c478bd9Sstevel@tonic-gate /* 22647c478bd9Sstevel@tonic-gate * The order here is important. Each step can speed up 22657c478bd9Sstevel@tonic-gate * later steps. 22667c478bd9Sstevel@tonic-gate */ 22677c478bd9Sstevel@tonic-gate 22687c478bd9Sstevel@tonic-gate if (uflags & UMU_HASH_RESCALE) 22697c478bd9Sstevel@tonic-gate umem_hash_rescale(cp); 22707c478bd9Sstevel@tonic-gate 22717c478bd9Sstevel@tonic-gate if (uflags & UMU_MAGAZINE_RESIZE) 22727c478bd9Sstevel@tonic-gate umem_cache_magazine_resize(cp); 22737c478bd9Sstevel@tonic-gate 22747c478bd9Sstevel@tonic-gate if (uflags & UMU_REAP) 22757c478bd9Sstevel@tonic-gate umem_cache_reap(cp); 22767c478bd9Sstevel@tonic-gate 22777c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_update_lock); 22787c478bd9Sstevel@tonic-gate 22797c478bd9Sstevel@tonic-gate /* 22807c478bd9Sstevel@tonic-gate * check if anyone has requested notification 22817c478bd9Sstevel@tonic-gate */ 22827c478bd9Sstevel@tonic-gate if (cp->cache_uflags & UMU_NOTIFY) { 22837c478bd9Sstevel@tonic-gate uflags |= UMU_NOTIFY; 22847c478bd9Sstevel@tonic-gate notify = 1; 22857c478bd9Sstevel@tonic-gate } 22867c478bd9Sstevel@tonic-gate cp->cache_uflags &= ~uflags; 22877c478bd9Sstevel@tonic-gate } 22887c478bd9Sstevel@tonic-gate if (notify) 22897c478bd9Sstevel@tonic-gate (void) cond_broadcast(&umem_update_cv); 22907c478bd9Sstevel@tonic-gate } 22917c478bd9Sstevel@tonic-gate } 22927c478bd9Sstevel@tonic-gate 22937c478bd9Sstevel@tonic-gate #ifndef UMEM_STANDALONE 22947c478bd9Sstevel@tonic-gate static void 22957c478bd9Sstevel@tonic-gate umem_st_update(void) 22967c478bd9Sstevel@tonic-gate { 22977c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&umem_update_lock)); 22987c478bd9Sstevel@tonic-gate ASSERT(umem_update_thr == 0 && umem_st_update_thr == 0); 22997c478bd9Sstevel@tonic-gate 23007c478bd9Sstevel@tonic-gate umem_st_update_thr = thr_self(); 23017c478bd9Sstevel@tonic-gate 23027c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_update_lock); 23037c478bd9Sstevel@tonic-gate 23047c478bd9Sstevel@tonic-gate vmem_update(NULL); 23057c478bd9Sstevel@tonic-gate umem_cache_applyall(umem_cache_update); 23067c478bd9Sstevel@tonic-gate 23077c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_update_lock); 23087c478bd9Sstevel@tonic-gate 23097c478bd9Sstevel@tonic-gate umem_process_updates(); /* does all of the requested work */ 23107c478bd9Sstevel@tonic-gate 23117c478bd9Sstevel@tonic-gate umem_reap_next = gethrtime() + 23127c478bd9Sstevel@tonic-gate (hrtime_t)umem_reap_interval * NANOSEC; 23137c478bd9Sstevel@tonic-gate 23147c478bd9Sstevel@tonic-gate umem_reaping = UMEM_REAP_DONE; 23157c478bd9Sstevel@tonic-gate 23167c478bd9Sstevel@tonic-gate umem_st_update_thr = 0; 23177c478bd9Sstevel@tonic-gate } 23187c478bd9Sstevel@tonic-gate #endif 23197c478bd9Sstevel@tonic-gate 23207c478bd9Sstevel@tonic-gate /* 23217c478bd9Sstevel@tonic-gate * Reclaim all unused memory from all caches. Called from vmem when memory 23227c478bd9Sstevel@tonic-gate * gets tight. Must be called with no locks held. 23237c478bd9Sstevel@tonic-gate * 23247c478bd9Sstevel@tonic-gate * This just requests a reap on all caches, and notifies the update thread. 23257c478bd9Sstevel@tonic-gate */ 23267c478bd9Sstevel@tonic-gate void 23277c478bd9Sstevel@tonic-gate umem_reap(void) 23287c478bd9Sstevel@tonic-gate { 23297c478bd9Sstevel@tonic-gate #ifndef UMEM_STANDALONE 23307c478bd9Sstevel@tonic-gate extern int __nthreads(void); 23317c478bd9Sstevel@tonic-gate #endif 23327c478bd9Sstevel@tonic-gate 23337c478bd9Sstevel@tonic-gate if (umem_ready != UMEM_READY || umem_reaping != UMEM_REAP_DONE || 23347c478bd9Sstevel@tonic-gate gethrtime() < umem_reap_next) 23357c478bd9Sstevel@tonic-gate return; 23367c478bd9Sstevel@tonic-gate 23377c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_update_lock); 23387c478bd9Sstevel@tonic-gate 23397c478bd9Sstevel@tonic-gate if (umem_reaping != UMEM_REAP_DONE || gethrtime() < umem_reap_next) { 23407c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_update_lock); 23417c478bd9Sstevel@tonic-gate return; 23427c478bd9Sstevel@tonic-gate } 23437c478bd9Sstevel@tonic-gate umem_reaping = UMEM_REAP_ADDING; /* lock out other reaps */ 23447c478bd9Sstevel@tonic-gate 23457c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_update_lock); 23467c478bd9Sstevel@tonic-gate 23477c478bd9Sstevel@tonic-gate umem_updateall(UMU_REAP); 23487c478bd9Sstevel@tonic-gate 23497c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_update_lock); 23507c478bd9Sstevel@tonic-gate 23517c478bd9Sstevel@tonic-gate umem_reaping = UMEM_REAP_ACTIVE; 23527c478bd9Sstevel@tonic-gate 23537c478bd9Sstevel@tonic-gate /* Standalone is single-threaded */ 23547c478bd9Sstevel@tonic-gate #ifndef UMEM_STANDALONE 23557c478bd9Sstevel@tonic-gate if (umem_update_thr == 0) { 23567c478bd9Sstevel@tonic-gate /* 23577c478bd9Sstevel@tonic-gate * The update thread does not exist. If the process is 23587c478bd9Sstevel@tonic-gate * multi-threaded, create it. If not, or the creation fails, 23597c478bd9Sstevel@tonic-gate * do the update processing inline. 23607c478bd9Sstevel@tonic-gate */ 23617c478bd9Sstevel@tonic-gate ASSERT(umem_st_update_thr == 0); 23627c478bd9Sstevel@tonic-gate 23637c478bd9Sstevel@tonic-gate if (__nthreads() <= 1 || umem_create_update_thread() == 0) 23647c478bd9Sstevel@tonic-gate umem_st_update(); 23657c478bd9Sstevel@tonic-gate } 23667c478bd9Sstevel@tonic-gate 23677c478bd9Sstevel@tonic-gate (void) cond_broadcast(&umem_update_cv); /* wake up the update thread */ 23687c478bd9Sstevel@tonic-gate #endif 23697c478bd9Sstevel@tonic-gate 23707c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_update_lock); 23717c478bd9Sstevel@tonic-gate } 23727c478bd9Sstevel@tonic-gate 23737c478bd9Sstevel@tonic-gate umem_cache_t * 23747c478bd9Sstevel@tonic-gate umem_cache_create( 23757c478bd9Sstevel@tonic-gate char *name, /* descriptive name for this cache */ 23767c478bd9Sstevel@tonic-gate size_t bufsize, /* size of the objects it manages */ 23777c478bd9Sstevel@tonic-gate size_t align, /* required object alignment */ 23787c478bd9Sstevel@tonic-gate umem_constructor_t *constructor, /* object constructor */ 23797c478bd9Sstevel@tonic-gate umem_destructor_t *destructor, /* object destructor */ 23807c478bd9Sstevel@tonic-gate umem_reclaim_t *reclaim, /* memory reclaim callback */ 23817c478bd9Sstevel@tonic-gate void *private, /* pass-thru arg for constr/destr/reclaim */ 23827c478bd9Sstevel@tonic-gate vmem_t *vmp, /* vmem source for slab allocation */ 23837c478bd9Sstevel@tonic-gate int cflags) /* cache creation flags */ 23847c478bd9Sstevel@tonic-gate { 23857c478bd9Sstevel@tonic-gate int cpu_seqid; 23867c478bd9Sstevel@tonic-gate size_t chunksize; 23877c478bd9Sstevel@tonic-gate umem_cache_t *cp, *cnext, *cprev; 23887c478bd9Sstevel@tonic-gate umem_magtype_t *mtp; 23897c478bd9Sstevel@tonic-gate size_t csize; 23907c478bd9Sstevel@tonic-gate size_t phase; 23917c478bd9Sstevel@tonic-gate 23927c478bd9Sstevel@tonic-gate /* 23937c478bd9Sstevel@tonic-gate * The init thread is allowed to create internal and quantum caches. 23947c478bd9Sstevel@tonic-gate * 23957c478bd9Sstevel@tonic-gate * Other threads must wait until until initialization is complete. 23967c478bd9Sstevel@tonic-gate */ 23977c478bd9Sstevel@tonic-gate if (umem_init_thr == thr_self()) 23987c478bd9Sstevel@tonic-gate ASSERT((cflags & (UMC_INTERNAL | UMC_QCACHE)) != 0); 23997c478bd9Sstevel@tonic-gate else { 24007c478bd9Sstevel@tonic-gate ASSERT(!(cflags & UMC_INTERNAL)); 24017c478bd9Sstevel@tonic-gate if (umem_ready != UMEM_READY && umem_init() == 0) { 24027c478bd9Sstevel@tonic-gate errno = EAGAIN; 24037c478bd9Sstevel@tonic-gate return (NULL); 24047c478bd9Sstevel@tonic-gate } 24057c478bd9Sstevel@tonic-gate } 24067c478bd9Sstevel@tonic-gate 24077c478bd9Sstevel@tonic-gate csize = UMEM_CACHE_SIZE(umem_max_ncpus); 24087c478bd9Sstevel@tonic-gate phase = P2NPHASE(csize, UMEM_CPU_CACHE_SIZE); 24097c478bd9Sstevel@tonic-gate 24107c478bd9Sstevel@tonic-gate if (vmp == NULL) 24117c478bd9Sstevel@tonic-gate vmp = umem_default_arena; 24127c478bd9Sstevel@tonic-gate 24137c478bd9Sstevel@tonic-gate ASSERT(P2PHASE(phase, UMEM_ALIGN) == 0); 24147c478bd9Sstevel@tonic-gate 24157c478bd9Sstevel@tonic-gate /* 24167c478bd9Sstevel@tonic-gate * Check that the arguments are reasonable 24177c478bd9Sstevel@tonic-gate */ 24187c478bd9Sstevel@tonic-gate if ((align & (align - 1)) != 0 || align > vmp->vm_quantum || 24197c478bd9Sstevel@tonic-gate ((cflags & UMC_NOHASH) && (cflags & UMC_NOTOUCH)) || 24207c478bd9Sstevel@tonic-gate name == NULL || bufsize == 0) { 24217c478bd9Sstevel@tonic-gate errno = EINVAL; 24227c478bd9Sstevel@tonic-gate return (NULL); 24237c478bd9Sstevel@tonic-gate } 24247c478bd9Sstevel@tonic-gate 24257c478bd9Sstevel@tonic-gate /* 24267c478bd9Sstevel@tonic-gate * If align == 0, we set it to the minimum required alignment. 24277c478bd9Sstevel@tonic-gate * 24287c478bd9Sstevel@tonic-gate * If align < UMEM_ALIGN, we round it up to UMEM_ALIGN, unless 24297c478bd9Sstevel@tonic-gate * UMC_NOTOUCH was passed. 24307c478bd9Sstevel@tonic-gate */ 24317c478bd9Sstevel@tonic-gate if (align == 0) { 24327c478bd9Sstevel@tonic-gate if (P2ROUNDUP(bufsize, UMEM_ALIGN) >= UMEM_SECOND_ALIGN) 24337c478bd9Sstevel@tonic-gate align = UMEM_SECOND_ALIGN; 24347c478bd9Sstevel@tonic-gate else 24357c478bd9Sstevel@tonic-gate align = UMEM_ALIGN; 24367c478bd9Sstevel@tonic-gate } else if (align < UMEM_ALIGN && (cflags & UMC_NOTOUCH) == 0) 24377c478bd9Sstevel@tonic-gate align = UMEM_ALIGN; 24387c478bd9Sstevel@tonic-gate 24397c478bd9Sstevel@tonic-gate 24407c478bd9Sstevel@tonic-gate /* 24417c478bd9Sstevel@tonic-gate * Get a umem_cache structure. We arrange that cp->cache_cpu[] 24427c478bd9Sstevel@tonic-gate * is aligned on a UMEM_CPU_CACHE_SIZE boundary to prevent 24437c478bd9Sstevel@tonic-gate * false sharing of per-CPU data. 24447c478bd9Sstevel@tonic-gate */ 24457c478bd9Sstevel@tonic-gate cp = vmem_xalloc(umem_cache_arena, csize, UMEM_CPU_CACHE_SIZE, phase, 24467c478bd9Sstevel@tonic-gate 0, NULL, NULL, VM_NOSLEEP); 24477c478bd9Sstevel@tonic-gate 24487c478bd9Sstevel@tonic-gate if (cp == NULL) { 24497c478bd9Sstevel@tonic-gate errno = EAGAIN; 24507c478bd9Sstevel@tonic-gate return (NULL); 24517c478bd9Sstevel@tonic-gate } 24527c478bd9Sstevel@tonic-gate 24537c478bd9Sstevel@tonic-gate bzero(cp, csize); 24547c478bd9Sstevel@tonic-gate 24557c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_flags_lock); 24567c478bd9Sstevel@tonic-gate if (umem_flags & UMF_RANDOMIZE) 24577c478bd9Sstevel@tonic-gate umem_flags = (((umem_flags | ~UMF_RANDOM) + 1) & UMF_RANDOM) | 24587c478bd9Sstevel@tonic-gate UMF_RANDOMIZE; 24597c478bd9Sstevel@tonic-gate cp->cache_flags = umem_flags | (cflags & UMF_DEBUG); 24607c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_flags_lock); 24617c478bd9Sstevel@tonic-gate 24627c478bd9Sstevel@tonic-gate /* 24637c478bd9Sstevel@tonic-gate * Make sure all the various flags are reasonable. 24647c478bd9Sstevel@tonic-gate */ 24657c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_LITE) { 24667c478bd9Sstevel@tonic-gate if (bufsize >= umem_lite_minsize && 24677c478bd9Sstevel@tonic-gate align <= umem_lite_maxalign && 24687c478bd9Sstevel@tonic-gate P2PHASE(bufsize, umem_lite_maxalign) != 0) { 24697c478bd9Sstevel@tonic-gate cp->cache_flags |= UMF_BUFTAG; 24707c478bd9Sstevel@tonic-gate cp->cache_flags &= ~(UMF_AUDIT | UMF_FIREWALL); 24717c478bd9Sstevel@tonic-gate } else { 24727c478bd9Sstevel@tonic-gate cp->cache_flags &= ~UMF_DEBUG; 24737c478bd9Sstevel@tonic-gate } 24747c478bd9Sstevel@tonic-gate } 24757c478bd9Sstevel@tonic-gate 24767c478bd9Sstevel@tonic-gate if ((cflags & UMC_QCACHE) && (cp->cache_flags & UMF_AUDIT)) 24777c478bd9Sstevel@tonic-gate cp->cache_flags |= UMF_NOMAGAZINE; 24787c478bd9Sstevel@tonic-gate 24797c478bd9Sstevel@tonic-gate if (cflags & UMC_NODEBUG) 24807c478bd9Sstevel@tonic-gate cp->cache_flags &= ~UMF_DEBUG; 24817c478bd9Sstevel@tonic-gate 24827c478bd9Sstevel@tonic-gate if (cflags & UMC_NOTOUCH) 24837c478bd9Sstevel@tonic-gate cp->cache_flags &= ~UMF_TOUCH; 24847c478bd9Sstevel@tonic-gate 24857c478bd9Sstevel@tonic-gate if (cflags & UMC_NOHASH) 24867c478bd9Sstevel@tonic-gate cp->cache_flags &= ~(UMF_AUDIT | UMF_FIREWALL); 24877c478bd9Sstevel@tonic-gate 24887c478bd9Sstevel@tonic-gate if (cflags & UMC_NOMAGAZINE) 24897c478bd9Sstevel@tonic-gate cp->cache_flags |= UMF_NOMAGAZINE; 24907c478bd9Sstevel@tonic-gate 24917c478bd9Sstevel@tonic-gate if ((cp->cache_flags & UMF_AUDIT) && !(cflags & UMC_NOTOUCH)) 24927c478bd9Sstevel@tonic-gate cp->cache_flags |= UMF_REDZONE; 24937c478bd9Sstevel@tonic-gate 24947c478bd9Sstevel@tonic-gate if ((cp->cache_flags & UMF_BUFTAG) && bufsize >= umem_minfirewall && 24957c478bd9Sstevel@tonic-gate !(cp->cache_flags & UMF_LITE) && !(cflags & UMC_NOHASH)) 24967c478bd9Sstevel@tonic-gate cp->cache_flags |= UMF_FIREWALL; 24977c478bd9Sstevel@tonic-gate 24987c478bd9Sstevel@tonic-gate if (vmp != umem_default_arena || umem_firewall_arena == NULL) 24997c478bd9Sstevel@tonic-gate cp->cache_flags &= ~UMF_FIREWALL; 25007c478bd9Sstevel@tonic-gate 25017c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_FIREWALL) { 25027c478bd9Sstevel@tonic-gate cp->cache_flags &= ~UMF_BUFTAG; 25037c478bd9Sstevel@tonic-gate cp->cache_flags |= UMF_NOMAGAZINE; 25047c478bd9Sstevel@tonic-gate ASSERT(vmp == umem_default_arena); 25057c478bd9Sstevel@tonic-gate vmp = umem_firewall_arena; 25067c478bd9Sstevel@tonic-gate } 25077c478bd9Sstevel@tonic-gate 25087c478bd9Sstevel@tonic-gate /* 25097c478bd9Sstevel@tonic-gate * Set cache properties. 25107c478bd9Sstevel@tonic-gate */ 25117c478bd9Sstevel@tonic-gate (void) strncpy(cp->cache_name, name, sizeof (cp->cache_name) - 1); 25127c478bd9Sstevel@tonic-gate cp->cache_bufsize = bufsize; 25137c478bd9Sstevel@tonic-gate cp->cache_align = align; 25147c478bd9Sstevel@tonic-gate cp->cache_constructor = constructor; 25157c478bd9Sstevel@tonic-gate cp->cache_destructor = destructor; 25167c478bd9Sstevel@tonic-gate cp->cache_reclaim = reclaim; 25177c478bd9Sstevel@tonic-gate cp->cache_private = private; 25187c478bd9Sstevel@tonic-gate cp->cache_arena = vmp; 25197c478bd9Sstevel@tonic-gate cp->cache_cflags = cflags; 25207c478bd9Sstevel@tonic-gate cp->cache_cpu_mask = umem_cpu_mask; 25217c478bd9Sstevel@tonic-gate 25227c478bd9Sstevel@tonic-gate /* 25237c478bd9Sstevel@tonic-gate * Determine the chunk size. 25247c478bd9Sstevel@tonic-gate */ 25257c478bd9Sstevel@tonic-gate chunksize = bufsize; 25267c478bd9Sstevel@tonic-gate 25277c478bd9Sstevel@tonic-gate if (align >= UMEM_ALIGN) { 25287c478bd9Sstevel@tonic-gate chunksize = P2ROUNDUP(chunksize, UMEM_ALIGN); 25297c478bd9Sstevel@tonic-gate cp->cache_bufctl = chunksize - UMEM_ALIGN; 25307c478bd9Sstevel@tonic-gate } 25317c478bd9Sstevel@tonic-gate 25327c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_BUFTAG) { 25337c478bd9Sstevel@tonic-gate cp->cache_bufctl = chunksize; 25347c478bd9Sstevel@tonic-gate cp->cache_buftag = chunksize; 25357c478bd9Sstevel@tonic-gate chunksize += sizeof (umem_buftag_t); 25367c478bd9Sstevel@tonic-gate } 25377c478bd9Sstevel@tonic-gate 25387c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_DEADBEEF) { 25397c478bd9Sstevel@tonic-gate cp->cache_verify = MIN(cp->cache_buftag, umem_maxverify); 25407c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_LITE) 25417c478bd9Sstevel@tonic-gate cp->cache_verify = MIN(cp->cache_verify, UMEM_ALIGN); 25427c478bd9Sstevel@tonic-gate } 25437c478bd9Sstevel@tonic-gate 25447c478bd9Sstevel@tonic-gate cp->cache_contents = MIN(cp->cache_bufctl, umem_content_maxsave); 25457c478bd9Sstevel@tonic-gate 25467c478bd9Sstevel@tonic-gate cp->cache_chunksize = chunksize = P2ROUNDUP(chunksize, align); 25477c478bd9Sstevel@tonic-gate 25487c478bd9Sstevel@tonic-gate if (chunksize < bufsize) { 25497c478bd9Sstevel@tonic-gate errno = ENOMEM; 25507c478bd9Sstevel@tonic-gate goto fail; 25517c478bd9Sstevel@tonic-gate } 25527c478bd9Sstevel@tonic-gate 25537c478bd9Sstevel@tonic-gate /* 25547c478bd9Sstevel@tonic-gate * Now that we know the chunk size, determine the optimal slab size. 25557c478bd9Sstevel@tonic-gate */ 25567c478bd9Sstevel@tonic-gate if (vmp == umem_firewall_arena) { 25577c478bd9Sstevel@tonic-gate cp->cache_slabsize = P2ROUNDUP(chunksize, vmp->vm_quantum); 25587c478bd9Sstevel@tonic-gate cp->cache_mincolor = cp->cache_slabsize - chunksize; 25597c478bd9Sstevel@tonic-gate cp->cache_maxcolor = cp->cache_mincolor; 25607c478bd9Sstevel@tonic-gate cp->cache_flags |= UMF_HASH; 25617c478bd9Sstevel@tonic-gate ASSERT(!(cp->cache_flags & UMF_BUFTAG)); 25627c478bd9Sstevel@tonic-gate } else if ((cflags & UMC_NOHASH) || (!(cflags & UMC_NOTOUCH) && 25637c478bd9Sstevel@tonic-gate !(cp->cache_flags & UMF_AUDIT) && 25647c478bd9Sstevel@tonic-gate chunksize < vmp->vm_quantum / UMEM_VOID_FRACTION)) { 25657c478bd9Sstevel@tonic-gate cp->cache_slabsize = vmp->vm_quantum; 25667c478bd9Sstevel@tonic-gate cp->cache_mincolor = 0; 25677c478bd9Sstevel@tonic-gate cp->cache_maxcolor = 25687c478bd9Sstevel@tonic-gate (cp->cache_slabsize - sizeof (umem_slab_t)) % chunksize; 25697c478bd9Sstevel@tonic-gate 25707c478bd9Sstevel@tonic-gate if (chunksize + sizeof (umem_slab_t) > cp->cache_slabsize) { 25717c478bd9Sstevel@tonic-gate errno = EINVAL; 25727c478bd9Sstevel@tonic-gate goto fail; 25737c478bd9Sstevel@tonic-gate } 25747c478bd9Sstevel@tonic-gate ASSERT(!(cp->cache_flags & UMF_AUDIT)); 25757c478bd9Sstevel@tonic-gate } else { 25767c478bd9Sstevel@tonic-gate size_t chunks, bestfit, waste, slabsize; 25777c478bd9Sstevel@tonic-gate size_t minwaste = LONG_MAX; 25787c478bd9Sstevel@tonic-gate 25797c478bd9Sstevel@tonic-gate for (chunks = 1; chunks <= UMEM_VOID_FRACTION; chunks++) { 25807c478bd9Sstevel@tonic-gate slabsize = P2ROUNDUP(chunksize * chunks, 25817c478bd9Sstevel@tonic-gate vmp->vm_quantum); 25827c478bd9Sstevel@tonic-gate /* 25837c478bd9Sstevel@tonic-gate * check for overflow 25847c478bd9Sstevel@tonic-gate */ 25857c478bd9Sstevel@tonic-gate if ((slabsize / chunks) < chunksize) { 25867c478bd9Sstevel@tonic-gate errno = ENOMEM; 25877c478bd9Sstevel@tonic-gate goto fail; 25887c478bd9Sstevel@tonic-gate } 25897c478bd9Sstevel@tonic-gate chunks = slabsize / chunksize; 25907c478bd9Sstevel@tonic-gate waste = (slabsize % chunksize) / chunks; 25917c478bd9Sstevel@tonic-gate if (waste < minwaste) { 25927c478bd9Sstevel@tonic-gate minwaste = waste; 25937c478bd9Sstevel@tonic-gate bestfit = slabsize; 25947c478bd9Sstevel@tonic-gate } 25957c478bd9Sstevel@tonic-gate } 25967c478bd9Sstevel@tonic-gate if (cflags & UMC_QCACHE) 25977c478bd9Sstevel@tonic-gate bestfit = MAX(1 << highbit(3 * vmp->vm_qcache_max), 64); 25987c478bd9Sstevel@tonic-gate cp->cache_slabsize = bestfit; 25997c478bd9Sstevel@tonic-gate cp->cache_mincolor = 0; 26007c478bd9Sstevel@tonic-gate cp->cache_maxcolor = bestfit % chunksize; 26017c478bd9Sstevel@tonic-gate cp->cache_flags |= UMF_HASH; 26027c478bd9Sstevel@tonic-gate } 26037c478bd9Sstevel@tonic-gate 26047c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_HASH) { 26057c478bd9Sstevel@tonic-gate ASSERT(!(cflags & UMC_NOHASH)); 26067c478bd9Sstevel@tonic-gate cp->cache_bufctl_cache = (cp->cache_flags & UMF_AUDIT) ? 26077c478bd9Sstevel@tonic-gate umem_bufctl_audit_cache : umem_bufctl_cache; 26087c478bd9Sstevel@tonic-gate } 26097c478bd9Sstevel@tonic-gate 26107c478bd9Sstevel@tonic-gate if (cp->cache_maxcolor >= vmp->vm_quantum) 26117c478bd9Sstevel@tonic-gate cp->cache_maxcolor = vmp->vm_quantum - 1; 26127c478bd9Sstevel@tonic-gate 26137c478bd9Sstevel@tonic-gate cp->cache_color = cp->cache_mincolor; 26147c478bd9Sstevel@tonic-gate 26157c478bd9Sstevel@tonic-gate /* 26167c478bd9Sstevel@tonic-gate * Initialize the rest of the slab layer. 26177c478bd9Sstevel@tonic-gate */ 26187c478bd9Sstevel@tonic-gate (void) mutex_init(&cp->cache_lock, USYNC_THREAD, NULL); 26197c478bd9Sstevel@tonic-gate 26207c478bd9Sstevel@tonic-gate cp->cache_freelist = &cp->cache_nullslab; 26217c478bd9Sstevel@tonic-gate cp->cache_nullslab.slab_cache = cp; 26227c478bd9Sstevel@tonic-gate cp->cache_nullslab.slab_refcnt = -1; 26237c478bd9Sstevel@tonic-gate cp->cache_nullslab.slab_next = &cp->cache_nullslab; 26247c478bd9Sstevel@tonic-gate cp->cache_nullslab.slab_prev = &cp->cache_nullslab; 26257c478bd9Sstevel@tonic-gate 26267c478bd9Sstevel@tonic-gate if (cp->cache_flags & UMF_HASH) { 26277c478bd9Sstevel@tonic-gate cp->cache_hash_table = vmem_alloc(umem_hash_arena, 26287c478bd9Sstevel@tonic-gate UMEM_HASH_INITIAL * sizeof (void *), VM_NOSLEEP); 26297c478bd9Sstevel@tonic-gate if (cp->cache_hash_table == NULL) { 26307c478bd9Sstevel@tonic-gate errno = EAGAIN; 26317c478bd9Sstevel@tonic-gate goto fail_lock; 26327c478bd9Sstevel@tonic-gate } 26337c478bd9Sstevel@tonic-gate bzero(cp->cache_hash_table, 26347c478bd9Sstevel@tonic-gate UMEM_HASH_INITIAL * sizeof (void *)); 26357c478bd9Sstevel@tonic-gate cp->cache_hash_mask = UMEM_HASH_INITIAL - 1; 26367c478bd9Sstevel@tonic-gate cp->cache_hash_shift = highbit((ulong_t)chunksize) - 1; 26377c478bd9Sstevel@tonic-gate } 26387c478bd9Sstevel@tonic-gate 26397c478bd9Sstevel@tonic-gate /* 26407c478bd9Sstevel@tonic-gate * Initialize the depot. 26417c478bd9Sstevel@tonic-gate */ 26427c478bd9Sstevel@tonic-gate (void) mutex_init(&cp->cache_depot_lock, USYNC_THREAD, NULL); 26437c478bd9Sstevel@tonic-gate 26447c478bd9Sstevel@tonic-gate for (mtp = umem_magtype; chunksize <= mtp->mt_minbuf; mtp++) 26457c478bd9Sstevel@tonic-gate continue; 26467c478bd9Sstevel@tonic-gate 26477c478bd9Sstevel@tonic-gate cp->cache_magtype = mtp; 26487c478bd9Sstevel@tonic-gate 26497c478bd9Sstevel@tonic-gate /* 26507c478bd9Sstevel@tonic-gate * Initialize the CPU layer. 26517c478bd9Sstevel@tonic-gate */ 26527c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < umem_max_ncpus; cpu_seqid++) { 26537c478bd9Sstevel@tonic-gate umem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 26547c478bd9Sstevel@tonic-gate (void) mutex_init(&ccp->cc_lock, USYNC_THREAD, NULL); 26557c478bd9Sstevel@tonic-gate ccp->cc_flags = cp->cache_flags; 26567c478bd9Sstevel@tonic-gate ccp->cc_rounds = -1; 26577c478bd9Sstevel@tonic-gate ccp->cc_prounds = -1; 26587c478bd9Sstevel@tonic-gate } 26597c478bd9Sstevel@tonic-gate 26607c478bd9Sstevel@tonic-gate /* 26617c478bd9Sstevel@tonic-gate * Add the cache to the global list. This makes it visible 26627c478bd9Sstevel@tonic-gate * to umem_update(), so the cache must be ready for business. 26637c478bd9Sstevel@tonic-gate */ 26647c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_cache_lock); 26657c478bd9Sstevel@tonic-gate cp->cache_next = cnext = &umem_null_cache; 26667c478bd9Sstevel@tonic-gate cp->cache_prev = cprev = umem_null_cache.cache_prev; 26677c478bd9Sstevel@tonic-gate cnext->cache_prev = cp; 26687c478bd9Sstevel@tonic-gate cprev->cache_next = cp; 26697c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_cache_lock); 26707c478bd9Sstevel@tonic-gate 26717c478bd9Sstevel@tonic-gate if (umem_ready == UMEM_READY) 26727c478bd9Sstevel@tonic-gate umem_cache_magazine_enable(cp); 26737c478bd9Sstevel@tonic-gate 26747c478bd9Sstevel@tonic-gate return (cp); 26757c478bd9Sstevel@tonic-gate 26767c478bd9Sstevel@tonic-gate fail_lock: 26777c478bd9Sstevel@tonic-gate (void) mutex_destroy(&cp->cache_lock); 26787c478bd9Sstevel@tonic-gate fail: 26797c478bd9Sstevel@tonic-gate vmem_xfree(umem_cache_arena, cp, csize); 26807c478bd9Sstevel@tonic-gate return (NULL); 26817c478bd9Sstevel@tonic-gate } 26827c478bd9Sstevel@tonic-gate 26837c478bd9Sstevel@tonic-gate void 26847c478bd9Sstevel@tonic-gate umem_cache_destroy(umem_cache_t *cp) 26857c478bd9Sstevel@tonic-gate { 26867c478bd9Sstevel@tonic-gate int cpu_seqid; 26877c478bd9Sstevel@tonic-gate 26887c478bd9Sstevel@tonic-gate /* 26897c478bd9Sstevel@tonic-gate * Remove the cache from the global cache list so that no new updates 26907c478bd9Sstevel@tonic-gate * will be scheduled on its behalf, wait for any pending tasks to 26917c478bd9Sstevel@tonic-gate * complete, purge the cache, and then destroy it. 26927c478bd9Sstevel@tonic-gate */ 26937c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_cache_lock); 26947c478bd9Sstevel@tonic-gate cp->cache_prev->cache_next = cp->cache_next; 26957c478bd9Sstevel@tonic-gate cp->cache_next->cache_prev = cp->cache_prev; 26967c478bd9Sstevel@tonic-gate cp->cache_prev = cp->cache_next = NULL; 26977c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_cache_lock); 26987c478bd9Sstevel@tonic-gate 26997c478bd9Sstevel@tonic-gate umem_remove_updates(cp); 27007c478bd9Sstevel@tonic-gate 27017c478bd9Sstevel@tonic-gate umem_cache_magazine_purge(cp); 27027c478bd9Sstevel@tonic-gate 27037c478bd9Sstevel@tonic-gate (void) mutex_lock(&cp->cache_lock); 27047c478bd9Sstevel@tonic-gate if (cp->cache_buftotal != 0) 27057c478bd9Sstevel@tonic-gate log_message("umem_cache_destroy: '%s' (%p) not empty\n", 27067c478bd9Sstevel@tonic-gate cp->cache_name, (void *)cp); 27077c478bd9Sstevel@tonic-gate cp->cache_reclaim = NULL; 27087c478bd9Sstevel@tonic-gate /* 27097c478bd9Sstevel@tonic-gate * The cache is now dead. There should be no further activity. 27107c478bd9Sstevel@tonic-gate * We enforce this by setting land mines in the constructor and 27117c478bd9Sstevel@tonic-gate * destructor routines that induce a segmentation fault if invoked. 27127c478bd9Sstevel@tonic-gate */ 27137c478bd9Sstevel@tonic-gate cp->cache_constructor = (umem_constructor_t *)1; 27147c478bd9Sstevel@tonic-gate cp->cache_destructor = (umem_destructor_t *)2; 27157c478bd9Sstevel@tonic-gate (void) mutex_unlock(&cp->cache_lock); 27167c478bd9Sstevel@tonic-gate 27177c478bd9Sstevel@tonic-gate if (cp->cache_hash_table != NULL) 27187c478bd9Sstevel@tonic-gate vmem_free(umem_hash_arena, cp->cache_hash_table, 27197c478bd9Sstevel@tonic-gate (cp->cache_hash_mask + 1) * sizeof (void *)); 27207c478bd9Sstevel@tonic-gate 27217c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < umem_max_ncpus; cpu_seqid++) 27227c478bd9Sstevel@tonic-gate (void) mutex_destroy(&cp->cache_cpu[cpu_seqid].cc_lock); 27237c478bd9Sstevel@tonic-gate 27247c478bd9Sstevel@tonic-gate (void) mutex_destroy(&cp->cache_depot_lock); 27257c478bd9Sstevel@tonic-gate (void) mutex_destroy(&cp->cache_lock); 27267c478bd9Sstevel@tonic-gate 27277c478bd9Sstevel@tonic-gate vmem_free(umem_cache_arena, cp, UMEM_CACHE_SIZE(umem_max_ncpus)); 27287c478bd9Sstevel@tonic-gate } 27297c478bd9Sstevel@tonic-gate 27307c478bd9Sstevel@tonic-gate static int 27317c478bd9Sstevel@tonic-gate umem_cache_init(void) 27327c478bd9Sstevel@tonic-gate { 27337c478bd9Sstevel@tonic-gate int i; 27347c478bd9Sstevel@tonic-gate size_t size, max_size; 27357c478bd9Sstevel@tonic-gate umem_cache_t *cp; 27367c478bd9Sstevel@tonic-gate umem_magtype_t *mtp; 27377c478bd9Sstevel@tonic-gate char name[UMEM_CACHE_NAMELEN + 1]; 27387c478bd9Sstevel@tonic-gate umem_cache_t *umem_alloc_caches[NUM_ALLOC_SIZES]; 27397c478bd9Sstevel@tonic-gate 27407c478bd9Sstevel@tonic-gate for (i = 0; i < sizeof (umem_magtype) / sizeof (*mtp); i++) { 27417c478bd9Sstevel@tonic-gate mtp = &umem_magtype[i]; 27427c478bd9Sstevel@tonic-gate (void) snprintf(name, sizeof (name), "umem_magazine_%d", 27437c478bd9Sstevel@tonic-gate mtp->mt_magsize); 27447c478bd9Sstevel@tonic-gate mtp->mt_cache = umem_cache_create(name, 27457c478bd9Sstevel@tonic-gate (mtp->mt_magsize + 1) * sizeof (void *), 27467c478bd9Sstevel@tonic-gate mtp->mt_align, NULL, NULL, NULL, NULL, 27477c478bd9Sstevel@tonic-gate umem_internal_arena, UMC_NOHASH | UMC_INTERNAL); 27487c478bd9Sstevel@tonic-gate if (mtp->mt_cache == NULL) 27497c478bd9Sstevel@tonic-gate return (0); 27507c478bd9Sstevel@tonic-gate } 27517c478bd9Sstevel@tonic-gate 27527c478bd9Sstevel@tonic-gate umem_slab_cache = umem_cache_create("umem_slab_cache", 27537c478bd9Sstevel@tonic-gate sizeof (umem_slab_t), 0, NULL, NULL, NULL, NULL, 27547c478bd9Sstevel@tonic-gate umem_internal_arena, UMC_NOHASH | UMC_INTERNAL); 27557c478bd9Sstevel@tonic-gate 27567c478bd9Sstevel@tonic-gate if (umem_slab_cache == NULL) 27577c478bd9Sstevel@tonic-gate return (0); 27587c478bd9Sstevel@tonic-gate 27597c478bd9Sstevel@tonic-gate umem_bufctl_cache = umem_cache_create("umem_bufctl_cache", 27607c478bd9Sstevel@tonic-gate sizeof (umem_bufctl_t), 0, NULL, NULL, NULL, NULL, 27617c478bd9Sstevel@tonic-gate umem_internal_arena, UMC_NOHASH | UMC_INTERNAL); 27627c478bd9Sstevel@tonic-gate 27637c478bd9Sstevel@tonic-gate if (umem_bufctl_cache == NULL) 27647c478bd9Sstevel@tonic-gate return (0); 27657c478bd9Sstevel@tonic-gate 27667c478bd9Sstevel@tonic-gate /* 27677c478bd9Sstevel@tonic-gate * The size of the umem_bufctl_audit structure depends upon 27687c478bd9Sstevel@tonic-gate * umem_stack_depth. See umem_impl.h for details on the size 27697c478bd9Sstevel@tonic-gate * restrictions. 27707c478bd9Sstevel@tonic-gate */ 27717c478bd9Sstevel@tonic-gate 27727c478bd9Sstevel@tonic-gate size = UMEM_BUFCTL_AUDIT_SIZE_DEPTH(umem_stack_depth); 27737c478bd9Sstevel@tonic-gate max_size = UMEM_BUFCTL_AUDIT_MAX_SIZE; 27747c478bd9Sstevel@tonic-gate 27757c478bd9Sstevel@tonic-gate if (size > max_size) { /* too large -- truncate */ 27767c478bd9Sstevel@tonic-gate int max_frames = UMEM_MAX_STACK_DEPTH; 27777c478bd9Sstevel@tonic-gate 27787c478bd9Sstevel@tonic-gate ASSERT(UMEM_BUFCTL_AUDIT_SIZE_DEPTH(max_frames) <= max_size); 27797c478bd9Sstevel@tonic-gate 27807c478bd9Sstevel@tonic-gate umem_stack_depth = max_frames; 27817c478bd9Sstevel@tonic-gate size = UMEM_BUFCTL_AUDIT_SIZE_DEPTH(umem_stack_depth); 27827c478bd9Sstevel@tonic-gate } 27837c478bd9Sstevel@tonic-gate 27847c478bd9Sstevel@tonic-gate umem_bufctl_audit_cache = umem_cache_create("umem_bufctl_audit_cache", 27857c478bd9Sstevel@tonic-gate size, 0, NULL, NULL, NULL, NULL, umem_internal_arena, 27867c478bd9Sstevel@tonic-gate UMC_NOHASH | UMC_INTERNAL); 27877c478bd9Sstevel@tonic-gate 27887c478bd9Sstevel@tonic-gate if (umem_bufctl_audit_cache == NULL) 27897c478bd9Sstevel@tonic-gate return (0); 27907c478bd9Sstevel@tonic-gate 27917c478bd9Sstevel@tonic-gate if (vmem_backend & VMEM_BACKEND_MMAP) 27927c478bd9Sstevel@tonic-gate umem_va_arena = vmem_create("umem_va", 27937c478bd9Sstevel@tonic-gate NULL, 0, pagesize, 27947c478bd9Sstevel@tonic-gate vmem_alloc, vmem_free, heap_arena, 27957c478bd9Sstevel@tonic-gate 8 * pagesize, VM_NOSLEEP); 27967c478bd9Sstevel@tonic-gate else 27977c478bd9Sstevel@tonic-gate umem_va_arena = heap_arena; 27987c478bd9Sstevel@tonic-gate 27997c478bd9Sstevel@tonic-gate if (umem_va_arena == NULL) 28007c478bd9Sstevel@tonic-gate return (0); 28017c478bd9Sstevel@tonic-gate 28027c478bd9Sstevel@tonic-gate umem_default_arena = vmem_create("umem_default", 28037c478bd9Sstevel@tonic-gate NULL, 0, pagesize, 28047c478bd9Sstevel@tonic-gate heap_alloc, heap_free, umem_va_arena, 28057c478bd9Sstevel@tonic-gate 0, VM_NOSLEEP); 28067c478bd9Sstevel@tonic-gate 28077c478bd9Sstevel@tonic-gate if (umem_default_arena == NULL) 28087c478bd9Sstevel@tonic-gate return (0); 28097c478bd9Sstevel@tonic-gate 28107c478bd9Sstevel@tonic-gate /* 28117c478bd9Sstevel@tonic-gate * make sure the umem_alloc table initializer is correct 28127c478bd9Sstevel@tonic-gate */ 28137c478bd9Sstevel@tonic-gate i = sizeof (umem_alloc_table) / sizeof (*umem_alloc_table); 28147c478bd9Sstevel@tonic-gate ASSERT(umem_alloc_table[i - 1] == &umem_null_cache); 28157c478bd9Sstevel@tonic-gate 28167c478bd9Sstevel@tonic-gate /* 28177c478bd9Sstevel@tonic-gate * Create the default caches to back umem_alloc() 28187c478bd9Sstevel@tonic-gate */ 28197c478bd9Sstevel@tonic-gate for (i = 0; i < NUM_ALLOC_SIZES; i++) { 28207c478bd9Sstevel@tonic-gate size_t cache_size = umem_alloc_sizes[i]; 28217c478bd9Sstevel@tonic-gate size_t align = 0; 28227c478bd9Sstevel@tonic-gate /* 28237c478bd9Sstevel@tonic-gate * If they allocate a multiple of the coherency granularity, 28247c478bd9Sstevel@tonic-gate * they get a coherency-granularity-aligned address. 28257c478bd9Sstevel@tonic-gate */ 28267c478bd9Sstevel@tonic-gate if (IS_P2ALIGNED(cache_size, 64)) 28277c478bd9Sstevel@tonic-gate align = 64; 28287c478bd9Sstevel@tonic-gate if (IS_P2ALIGNED(cache_size, pagesize)) 28297c478bd9Sstevel@tonic-gate align = pagesize; 28307c478bd9Sstevel@tonic-gate (void) snprintf(name, sizeof (name), "umem_alloc_%lu", 28317c478bd9Sstevel@tonic-gate (long)cache_size); 28327c478bd9Sstevel@tonic-gate 28337c478bd9Sstevel@tonic-gate cp = umem_cache_create(name, cache_size, align, 28347c478bd9Sstevel@tonic-gate NULL, NULL, NULL, NULL, NULL, UMC_INTERNAL); 28357c478bd9Sstevel@tonic-gate if (cp == NULL) 28367c478bd9Sstevel@tonic-gate return (0); 28377c478bd9Sstevel@tonic-gate 28387c478bd9Sstevel@tonic-gate umem_alloc_caches[i] = cp; 28397c478bd9Sstevel@tonic-gate } 28407c478bd9Sstevel@tonic-gate 28417c478bd9Sstevel@tonic-gate /* 28427c478bd9Sstevel@tonic-gate * Initialization cannot fail at this point. Make the caches 28437c478bd9Sstevel@tonic-gate * visible to umem_alloc() and friends. 28447c478bd9Sstevel@tonic-gate */ 28457c478bd9Sstevel@tonic-gate size = UMEM_ALIGN; 28467c478bd9Sstevel@tonic-gate for (i = 0; i < NUM_ALLOC_SIZES; i++) { 28477c478bd9Sstevel@tonic-gate size_t cache_size = umem_alloc_sizes[i]; 28487c478bd9Sstevel@tonic-gate 28497c478bd9Sstevel@tonic-gate cp = umem_alloc_caches[i]; 28507c478bd9Sstevel@tonic-gate 28517c478bd9Sstevel@tonic-gate while (size <= cache_size) { 28527c478bd9Sstevel@tonic-gate umem_alloc_table[(size - 1) >> UMEM_ALIGN_SHIFT] = cp; 28537c478bd9Sstevel@tonic-gate size += UMEM_ALIGN; 28547c478bd9Sstevel@tonic-gate } 28557c478bd9Sstevel@tonic-gate } 28567c478bd9Sstevel@tonic-gate return (1); 28577c478bd9Sstevel@tonic-gate } 28587c478bd9Sstevel@tonic-gate 28597c478bd9Sstevel@tonic-gate /* 28607c478bd9Sstevel@tonic-gate * umem_startup() is called early on, and must be called explicitly if we're 28617c478bd9Sstevel@tonic-gate * the standalone version. 28627c478bd9Sstevel@tonic-gate */ 28637c478bd9Sstevel@tonic-gate #ifdef UMEM_STANDALONE 28647c478bd9Sstevel@tonic-gate void 28657c478bd9Sstevel@tonic-gate #else 28667c478bd9Sstevel@tonic-gate #pragma init(umem_startup) 28677c478bd9Sstevel@tonic-gate static void 28687c478bd9Sstevel@tonic-gate #endif 28697c478bd9Sstevel@tonic-gate umem_startup(caddr_t start, size_t len, size_t pagesize, caddr_t minstack, 28707c478bd9Sstevel@tonic-gate caddr_t maxstack) 28717c478bd9Sstevel@tonic-gate { 28727c478bd9Sstevel@tonic-gate #ifdef UMEM_STANDALONE 28737c478bd9Sstevel@tonic-gate int idx; 28747c478bd9Sstevel@tonic-gate /* Standalone doesn't fork */ 28757c478bd9Sstevel@tonic-gate #else 28767c478bd9Sstevel@tonic-gate umem_forkhandler_init(); /* register the fork handler */ 28777c478bd9Sstevel@tonic-gate #endif 28787c478bd9Sstevel@tonic-gate 28797c478bd9Sstevel@tonic-gate #ifdef __lint 28807c478bd9Sstevel@tonic-gate /* make lint happy */ 28817c478bd9Sstevel@tonic-gate minstack = maxstack; 28827c478bd9Sstevel@tonic-gate #endif 28837c478bd9Sstevel@tonic-gate 28847c478bd9Sstevel@tonic-gate #ifdef UMEM_STANDALONE 28857c478bd9Sstevel@tonic-gate umem_ready = UMEM_READY_STARTUP; 28867c478bd9Sstevel@tonic-gate umem_init_env_ready = 0; 28877c478bd9Sstevel@tonic-gate 28887c478bd9Sstevel@tonic-gate umem_min_stack = minstack; 28897c478bd9Sstevel@tonic-gate umem_max_stack = maxstack; 28907c478bd9Sstevel@tonic-gate 28917c478bd9Sstevel@tonic-gate nofail_callback = NULL; 28927c478bd9Sstevel@tonic-gate umem_slab_cache = NULL; 28937c478bd9Sstevel@tonic-gate umem_bufctl_cache = NULL; 28947c478bd9Sstevel@tonic-gate umem_bufctl_audit_cache = NULL; 28957c478bd9Sstevel@tonic-gate heap_arena = NULL; 28967c478bd9Sstevel@tonic-gate heap_alloc = NULL; 28977c478bd9Sstevel@tonic-gate heap_free = NULL; 28987c478bd9Sstevel@tonic-gate umem_internal_arena = NULL; 28997c478bd9Sstevel@tonic-gate umem_cache_arena = NULL; 29007c478bd9Sstevel@tonic-gate umem_hash_arena = NULL; 29017c478bd9Sstevel@tonic-gate umem_log_arena = NULL; 29027c478bd9Sstevel@tonic-gate umem_oversize_arena = NULL; 29037c478bd9Sstevel@tonic-gate umem_va_arena = NULL; 29047c478bd9Sstevel@tonic-gate umem_default_arena = NULL; 29057c478bd9Sstevel@tonic-gate umem_firewall_va_arena = NULL; 29067c478bd9Sstevel@tonic-gate umem_firewall_arena = NULL; 29077c478bd9Sstevel@tonic-gate umem_memalign_arena = NULL; 29087c478bd9Sstevel@tonic-gate umem_transaction_log = NULL; 29097c478bd9Sstevel@tonic-gate umem_content_log = NULL; 29107c478bd9Sstevel@tonic-gate umem_failure_log = NULL; 29117c478bd9Sstevel@tonic-gate umem_slab_log = NULL; 29127c478bd9Sstevel@tonic-gate umem_cpu_mask = 0; 29137c478bd9Sstevel@tonic-gate 29147c478bd9Sstevel@tonic-gate umem_cpus = &umem_startup_cpu; 29157c478bd9Sstevel@tonic-gate umem_startup_cpu.cpu_cache_offset = UMEM_CACHE_SIZE(0); 29167c478bd9Sstevel@tonic-gate umem_startup_cpu.cpu_number = 0; 29177c478bd9Sstevel@tonic-gate 29187c478bd9Sstevel@tonic-gate bcopy(&umem_null_cache_template, &umem_null_cache, 29197c478bd9Sstevel@tonic-gate sizeof (umem_cache_t)); 29207c478bd9Sstevel@tonic-gate 29217c478bd9Sstevel@tonic-gate for (idx = 0; idx < (UMEM_MAXBUF >> UMEM_ALIGN_SHIFT); idx++) 29227c478bd9Sstevel@tonic-gate umem_alloc_table[idx] = &umem_null_cache; 29237c478bd9Sstevel@tonic-gate #endif 29247c478bd9Sstevel@tonic-gate 29257c478bd9Sstevel@tonic-gate /* 29267c478bd9Sstevel@tonic-gate * Perform initialization specific to the way we've been compiled 29277c478bd9Sstevel@tonic-gate * (library or standalone) 29287c478bd9Sstevel@tonic-gate */ 29297c478bd9Sstevel@tonic-gate umem_type_init(start, len, pagesize); 29307c478bd9Sstevel@tonic-gate 29317c478bd9Sstevel@tonic-gate vmem_startup(); 29327c478bd9Sstevel@tonic-gate } 29337c478bd9Sstevel@tonic-gate 29347c478bd9Sstevel@tonic-gate int 29357c478bd9Sstevel@tonic-gate umem_init(void) 29367c478bd9Sstevel@tonic-gate { 29377c478bd9Sstevel@tonic-gate size_t maxverify, minfirewall; 29387c478bd9Sstevel@tonic-gate size_t size; 29397c478bd9Sstevel@tonic-gate int idx; 29407c478bd9Sstevel@tonic-gate umem_cpu_t *new_cpus; 29417c478bd9Sstevel@tonic-gate 29427c478bd9Sstevel@tonic-gate vmem_t *memalign_arena, *oversize_arena; 29437c478bd9Sstevel@tonic-gate 29447c478bd9Sstevel@tonic-gate if (thr_self() != umem_init_thr) { 29457c478bd9Sstevel@tonic-gate /* 29467c478bd9Sstevel@tonic-gate * The usual case -- non-recursive invocation of umem_init(). 29477c478bd9Sstevel@tonic-gate */ 29487c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_init_lock); 29497c478bd9Sstevel@tonic-gate if (umem_ready != UMEM_READY_STARTUP) { 29507c478bd9Sstevel@tonic-gate /* 29517c478bd9Sstevel@tonic-gate * someone else beat us to initializing umem. Wait 29527c478bd9Sstevel@tonic-gate * for them to complete, then return. 29537c478bd9Sstevel@tonic-gate */ 29547c478bd9Sstevel@tonic-gate while (umem_ready == UMEM_READY_INITING) 29557c478bd9Sstevel@tonic-gate (void) _cond_wait(&umem_init_cv, 29567c478bd9Sstevel@tonic-gate &umem_init_lock); 29577c478bd9Sstevel@tonic-gate ASSERT(umem_ready == UMEM_READY || 29587c478bd9Sstevel@tonic-gate umem_ready == UMEM_READY_INIT_FAILED); 29597c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_init_lock); 29607c478bd9Sstevel@tonic-gate return (umem_ready == UMEM_READY); 29617c478bd9Sstevel@tonic-gate } 29627c478bd9Sstevel@tonic-gate 29637c478bd9Sstevel@tonic-gate ASSERT(umem_ready == UMEM_READY_STARTUP); 29647c478bd9Sstevel@tonic-gate ASSERT(umem_init_env_ready == 0); 29657c478bd9Sstevel@tonic-gate 29667c478bd9Sstevel@tonic-gate umem_ready = UMEM_READY_INITING; 29677c478bd9Sstevel@tonic-gate umem_init_thr = thr_self(); 29687c478bd9Sstevel@tonic-gate 29697c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_init_lock); 29707c478bd9Sstevel@tonic-gate umem_setup_envvars(0); /* can recurse -- see below */ 29717c478bd9Sstevel@tonic-gate if (umem_init_env_ready) { 29727c478bd9Sstevel@tonic-gate /* 29737c478bd9Sstevel@tonic-gate * initialization was completed already 29747c478bd9Sstevel@tonic-gate */ 29757c478bd9Sstevel@tonic-gate ASSERT(umem_ready == UMEM_READY || 29767c478bd9Sstevel@tonic-gate umem_ready == UMEM_READY_INIT_FAILED); 29777c478bd9Sstevel@tonic-gate ASSERT(umem_init_thr == 0); 29787c478bd9Sstevel@tonic-gate return (umem_ready == UMEM_READY); 29797c478bd9Sstevel@tonic-gate } 29807c478bd9Sstevel@tonic-gate } else if (!umem_init_env_ready) { 29817c478bd9Sstevel@tonic-gate /* 29827c478bd9Sstevel@tonic-gate * The umem_setup_envvars() call (above) makes calls into 29837c478bd9Sstevel@tonic-gate * the dynamic linker and directly into user-supplied code. 29847c478bd9Sstevel@tonic-gate * Since we cannot know what that code will do, we could be 29857c478bd9Sstevel@tonic-gate * recursively invoked (by, say, a malloc() call in the code 29867c478bd9Sstevel@tonic-gate * itself, or in a (C++) _init section it causes to be fired). 29877c478bd9Sstevel@tonic-gate * 29887c478bd9Sstevel@tonic-gate * This code is where we end up if such recursion occurs. We 29897c478bd9Sstevel@tonic-gate * first clean up any partial results in the envvar code, then 29907c478bd9Sstevel@tonic-gate * proceed to finish initialization processing in the recursive 29917c478bd9Sstevel@tonic-gate * call. The original call will notice this, and return 29927c478bd9Sstevel@tonic-gate * immediately. 29937c478bd9Sstevel@tonic-gate */ 29947c478bd9Sstevel@tonic-gate umem_setup_envvars(1); /* clean up any partial state */ 29957c478bd9Sstevel@tonic-gate } else { 29967c478bd9Sstevel@tonic-gate umem_panic( 29977c478bd9Sstevel@tonic-gate "recursive allocation while initializing umem\n"); 29987c478bd9Sstevel@tonic-gate } 29997c478bd9Sstevel@tonic-gate umem_init_env_ready = 1; 30007c478bd9Sstevel@tonic-gate 30017c478bd9Sstevel@tonic-gate /* 30027c478bd9Sstevel@tonic-gate * From this point until we finish, recursion into umem_init() will 30037c478bd9Sstevel@tonic-gate * cause a umem_panic(). 30047c478bd9Sstevel@tonic-gate */ 30057c478bd9Sstevel@tonic-gate maxverify = minfirewall = ULONG_MAX; 30067c478bd9Sstevel@tonic-gate 30077c478bd9Sstevel@tonic-gate /* LINTED constant condition */ 30087c478bd9Sstevel@tonic-gate if (sizeof (umem_cpu_cache_t) != UMEM_CPU_CACHE_SIZE) { 30097c478bd9Sstevel@tonic-gate umem_panic("sizeof (umem_cpu_cache_t) = %d, should be %d\n", 30107c478bd9Sstevel@tonic-gate sizeof (umem_cpu_cache_t), UMEM_CPU_CACHE_SIZE); 30117c478bd9Sstevel@tonic-gate } 30127c478bd9Sstevel@tonic-gate 30137c478bd9Sstevel@tonic-gate umem_max_ncpus = umem_get_max_ncpus(); 30147c478bd9Sstevel@tonic-gate 30157c478bd9Sstevel@tonic-gate /* 30167c478bd9Sstevel@tonic-gate * load tunables from environment 30177c478bd9Sstevel@tonic-gate */ 30187c478bd9Sstevel@tonic-gate umem_process_envvars(); 30197c478bd9Sstevel@tonic-gate 30207c478bd9Sstevel@tonic-gate if (issetugid()) 30217c478bd9Sstevel@tonic-gate umem_mtbf = 0; 30227c478bd9Sstevel@tonic-gate 30237c478bd9Sstevel@tonic-gate /* 30247c478bd9Sstevel@tonic-gate * set up vmem 30257c478bd9Sstevel@tonic-gate */ 30267c478bd9Sstevel@tonic-gate if (!(umem_flags & UMF_AUDIT)) 30277c478bd9Sstevel@tonic-gate vmem_no_debug(); 30287c478bd9Sstevel@tonic-gate 30297c478bd9Sstevel@tonic-gate heap_arena = vmem_heap_arena(&heap_alloc, &heap_free); 30307c478bd9Sstevel@tonic-gate 30317c478bd9Sstevel@tonic-gate pagesize = heap_arena->vm_quantum; 30327c478bd9Sstevel@tonic-gate 30337c478bd9Sstevel@tonic-gate umem_internal_arena = vmem_create("umem_internal", NULL, 0, pagesize, 30347c478bd9Sstevel@tonic-gate heap_alloc, heap_free, heap_arena, 0, VM_NOSLEEP); 30357c478bd9Sstevel@tonic-gate 30367c478bd9Sstevel@tonic-gate umem_default_arena = umem_internal_arena; 30377c478bd9Sstevel@tonic-gate 30387c478bd9Sstevel@tonic-gate if (umem_internal_arena == NULL) 30397c478bd9Sstevel@tonic-gate goto fail; 30407c478bd9Sstevel@tonic-gate 30417c478bd9Sstevel@tonic-gate umem_cache_arena = vmem_create("umem_cache", NULL, 0, UMEM_ALIGN, 30427c478bd9Sstevel@tonic-gate vmem_alloc, vmem_free, umem_internal_arena, 0, VM_NOSLEEP); 30437c478bd9Sstevel@tonic-gate 30447c478bd9Sstevel@tonic-gate umem_hash_arena = vmem_create("umem_hash", NULL, 0, UMEM_ALIGN, 30457c478bd9Sstevel@tonic-gate vmem_alloc, vmem_free, umem_internal_arena, 0, VM_NOSLEEP); 30467c478bd9Sstevel@tonic-gate 30477c478bd9Sstevel@tonic-gate umem_log_arena = vmem_create("umem_log", NULL, 0, UMEM_ALIGN, 30487c478bd9Sstevel@tonic-gate heap_alloc, heap_free, heap_arena, 0, VM_NOSLEEP); 30497c478bd9Sstevel@tonic-gate 30507c478bd9Sstevel@tonic-gate umem_firewall_va_arena = vmem_create("umem_firewall_va", 30517c478bd9Sstevel@tonic-gate NULL, 0, pagesize, 30527c478bd9Sstevel@tonic-gate umem_firewall_va_alloc, umem_firewall_va_free, heap_arena, 30537c478bd9Sstevel@tonic-gate 0, VM_NOSLEEP); 30547c478bd9Sstevel@tonic-gate 30557c478bd9Sstevel@tonic-gate if (umem_cache_arena == NULL || umem_hash_arena == NULL || 30567c478bd9Sstevel@tonic-gate umem_log_arena == NULL || umem_firewall_va_arena == NULL) 30577c478bd9Sstevel@tonic-gate goto fail; 30587c478bd9Sstevel@tonic-gate 30597c478bd9Sstevel@tonic-gate umem_firewall_arena = vmem_create("umem_firewall", NULL, 0, pagesize, 30607c478bd9Sstevel@tonic-gate heap_alloc, heap_free, umem_firewall_va_arena, 0, 30617c478bd9Sstevel@tonic-gate VM_NOSLEEP); 30627c478bd9Sstevel@tonic-gate 30637c478bd9Sstevel@tonic-gate if (umem_firewall_arena == NULL) 30647c478bd9Sstevel@tonic-gate goto fail; 30657c478bd9Sstevel@tonic-gate 30667c478bd9Sstevel@tonic-gate oversize_arena = vmem_create("umem_oversize", NULL, 0, pagesize, 30677c478bd9Sstevel@tonic-gate heap_alloc, heap_free, minfirewall < ULONG_MAX ? 30687c478bd9Sstevel@tonic-gate umem_firewall_va_arena : heap_arena, 0, VM_NOSLEEP); 30697c478bd9Sstevel@tonic-gate 30707c478bd9Sstevel@tonic-gate memalign_arena = vmem_create("umem_memalign", NULL, 0, UMEM_ALIGN, 30717c478bd9Sstevel@tonic-gate heap_alloc, heap_free, minfirewall < ULONG_MAX ? 30727c478bd9Sstevel@tonic-gate umem_firewall_va_arena : heap_arena, 0, VM_NOSLEEP); 30737c478bd9Sstevel@tonic-gate 30747c478bd9Sstevel@tonic-gate if (oversize_arena == NULL || memalign_arena == NULL) 30757c478bd9Sstevel@tonic-gate goto fail; 30767c478bd9Sstevel@tonic-gate 30777c478bd9Sstevel@tonic-gate if (umem_max_ncpus > CPUHINT_MAX()) 30787c478bd9Sstevel@tonic-gate umem_max_ncpus = CPUHINT_MAX(); 30797c478bd9Sstevel@tonic-gate 30807c478bd9Sstevel@tonic-gate while ((umem_max_ncpus & (umem_max_ncpus - 1)) != 0) 30817c478bd9Sstevel@tonic-gate umem_max_ncpus++; 30827c478bd9Sstevel@tonic-gate 30837c478bd9Sstevel@tonic-gate if (umem_max_ncpus == 0) 30847c478bd9Sstevel@tonic-gate umem_max_ncpus = 1; 30857c478bd9Sstevel@tonic-gate 30867c478bd9Sstevel@tonic-gate size = umem_max_ncpus * sizeof (umem_cpu_t); 30877c478bd9Sstevel@tonic-gate new_cpus = vmem_alloc(umem_internal_arena, size, VM_NOSLEEP); 30887c478bd9Sstevel@tonic-gate if (new_cpus == NULL) 30897c478bd9Sstevel@tonic-gate goto fail; 30907c478bd9Sstevel@tonic-gate 30917c478bd9Sstevel@tonic-gate bzero(new_cpus, size); 30927c478bd9Sstevel@tonic-gate for (idx = 0; idx < umem_max_ncpus; idx++) { 30937c478bd9Sstevel@tonic-gate new_cpus[idx].cpu_number = idx; 30947c478bd9Sstevel@tonic-gate new_cpus[idx].cpu_cache_offset = UMEM_CACHE_SIZE(idx); 30957c478bd9Sstevel@tonic-gate } 30967c478bd9Sstevel@tonic-gate umem_cpus = new_cpus; 30977c478bd9Sstevel@tonic-gate umem_cpu_mask = (umem_max_ncpus - 1); 30987c478bd9Sstevel@tonic-gate 30997c478bd9Sstevel@tonic-gate if (umem_maxverify == 0) 31007c478bd9Sstevel@tonic-gate umem_maxverify = maxverify; 31017c478bd9Sstevel@tonic-gate 31027c478bd9Sstevel@tonic-gate if (umem_minfirewall == 0) 31037c478bd9Sstevel@tonic-gate umem_minfirewall = minfirewall; 31047c478bd9Sstevel@tonic-gate 31057c478bd9Sstevel@tonic-gate /* 31067c478bd9Sstevel@tonic-gate * Set up updating and reaping 31077c478bd9Sstevel@tonic-gate */ 31087c478bd9Sstevel@tonic-gate umem_reap_next = gethrtime() + NANOSEC; 31097c478bd9Sstevel@tonic-gate 31107c478bd9Sstevel@tonic-gate #ifndef UMEM_STANDALONE 31117c478bd9Sstevel@tonic-gate (void) gettimeofday(&umem_update_next, NULL); 31127c478bd9Sstevel@tonic-gate #endif 31137c478bd9Sstevel@tonic-gate 31147c478bd9Sstevel@tonic-gate /* 31157c478bd9Sstevel@tonic-gate * Set up logging -- failure here is okay, since it will just disable 31167c478bd9Sstevel@tonic-gate * the logs 31177c478bd9Sstevel@tonic-gate */ 31187c478bd9Sstevel@tonic-gate if (umem_logging) { 31197c478bd9Sstevel@tonic-gate umem_transaction_log = umem_log_init(umem_transaction_log_size); 31207c478bd9Sstevel@tonic-gate umem_content_log = umem_log_init(umem_content_log_size); 31217c478bd9Sstevel@tonic-gate umem_failure_log = umem_log_init(umem_failure_log_size); 31227c478bd9Sstevel@tonic-gate umem_slab_log = umem_log_init(umem_slab_log_size); 31237c478bd9Sstevel@tonic-gate } 31247c478bd9Sstevel@tonic-gate 31257c478bd9Sstevel@tonic-gate /* 31267c478bd9Sstevel@tonic-gate * Set up caches -- if successful, initialization cannot fail, since 31277c478bd9Sstevel@tonic-gate * allocations from other threads can now succeed. 31287c478bd9Sstevel@tonic-gate */ 31297c478bd9Sstevel@tonic-gate if (umem_cache_init() == 0) { 31307c478bd9Sstevel@tonic-gate log_message("unable to create initial caches\n"); 31317c478bd9Sstevel@tonic-gate goto fail; 31327c478bd9Sstevel@tonic-gate } 31337c478bd9Sstevel@tonic-gate umem_oversize_arena = oversize_arena; 31347c478bd9Sstevel@tonic-gate umem_memalign_arena = memalign_arena; 31357c478bd9Sstevel@tonic-gate 31367c478bd9Sstevel@tonic-gate umem_cache_applyall(umem_cache_magazine_enable); 31377c478bd9Sstevel@tonic-gate 31387c478bd9Sstevel@tonic-gate /* 31397c478bd9Sstevel@tonic-gate * initialization done, ready to go 31407c478bd9Sstevel@tonic-gate */ 31417c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_init_lock); 31427c478bd9Sstevel@tonic-gate umem_ready = UMEM_READY; 31437c478bd9Sstevel@tonic-gate umem_init_thr = 0; 31447c478bd9Sstevel@tonic-gate (void) cond_broadcast(&umem_init_cv); 31457c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_init_lock); 31467c478bd9Sstevel@tonic-gate return (1); 31477c478bd9Sstevel@tonic-gate 31487c478bd9Sstevel@tonic-gate fail: 31497c478bd9Sstevel@tonic-gate log_message("umem initialization failed\n"); 31507c478bd9Sstevel@tonic-gate 31517c478bd9Sstevel@tonic-gate (void) mutex_lock(&umem_init_lock); 31527c478bd9Sstevel@tonic-gate umem_ready = UMEM_READY_INIT_FAILED; 31537c478bd9Sstevel@tonic-gate umem_init_thr = 0; 31547c478bd9Sstevel@tonic-gate (void) cond_broadcast(&umem_init_cv); 31557c478bd9Sstevel@tonic-gate (void) mutex_unlock(&umem_init_lock); 31567c478bd9Sstevel@tonic-gate return (0); 31577c478bd9Sstevel@tonic-gate } 3158