17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 57d692464Sdp * Common Development and Distribution License (the "License"). 67d692464Sdp * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22b942e89bSDavid Valin * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved. 23929d5b43SMatthew Ahrens * Copyright (c) 2012, 2017 by Delphix. All rights reserved. 240c833d64SJosef 'Jeff' Sipek * Copyright 2015 Nexenta Systems, Inc. All rights reserved. 257c478bd9Sstevel@tonic-gate */ 267c478bd9Sstevel@tonic-gate 277c478bd9Sstevel@tonic-gate /* 28b5fca8f8Stomee * Kernel memory allocator, as described in the following two papers and a 29b5fca8f8Stomee * statement about the consolidator: 307c478bd9Sstevel@tonic-gate * 317c478bd9Sstevel@tonic-gate * Jeff Bonwick, 327c478bd9Sstevel@tonic-gate * The Slab Allocator: An Object-Caching Kernel Memory Allocator. 337c478bd9Sstevel@tonic-gate * Proceedings of the Summer 1994 Usenix Conference. 347c478bd9Sstevel@tonic-gate * Available as /shared/sac/PSARC/1994/028/materials/kmem.pdf. 357c478bd9Sstevel@tonic-gate * 367c478bd9Sstevel@tonic-gate * Jeff Bonwick and Jonathan Adams, 377c478bd9Sstevel@tonic-gate * Magazines and vmem: Extending the Slab Allocator to Many CPUs and 387c478bd9Sstevel@tonic-gate * Arbitrary Resources. 397c478bd9Sstevel@tonic-gate * Proceedings of the 2001 Usenix Conference. 407c478bd9Sstevel@tonic-gate * Available as /shared/sac/PSARC/2000/550/materials/vmem.pdf. 41b5fca8f8Stomee * 42b5fca8f8Stomee * kmem Slab Consolidator Big Theory Statement: 43b5fca8f8Stomee * 44b5fca8f8Stomee * 1. Motivation 45b5fca8f8Stomee * 46b5fca8f8Stomee * As stated in Bonwick94, slabs provide the following advantages over other 47b5fca8f8Stomee * allocation structures in terms of memory fragmentation: 48b5fca8f8Stomee * 49b5fca8f8Stomee * - Internal fragmentation (per-buffer wasted space) is minimal. 50b5fca8f8Stomee * - Severe external fragmentation (unused buffers on the free list) is 51b5fca8f8Stomee * unlikely. 52b5fca8f8Stomee * 53b5fca8f8Stomee * Segregating objects by size eliminates one source of external fragmentation, 54b5fca8f8Stomee * and according to Bonwick: 55b5fca8f8Stomee * 56b5fca8f8Stomee * The other reason that slabs reduce external fragmentation is that all 57b5fca8f8Stomee * objects in a slab are of the same type, so they have the same lifetime 58b5fca8f8Stomee * distribution. The resulting segregation of short-lived and long-lived 59b5fca8f8Stomee * objects at slab granularity reduces the likelihood of an entire page being 60b5fca8f8Stomee * held hostage due to a single long-lived allocation [Barrett93, Hanson90]. 61b5fca8f8Stomee * 62b5fca8f8Stomee * While unlikely, severe external fragmentation remains possible. Clients that 63b5fca8f8Stomee * allocate both short- and long-lived objects from the same cache cannot 64b5fca8f8Stomee * anticipate the distribution of long-lived objects within the allocator's slab 65b5fca8f8Stomee * implementation. Even a small percentage of long-lived objects distributed 66b5fca8f8Stomee * randomly across many slabs can lead to a worst case scenario where the client 67b5fca8f8Stomee * frees the majority of its objects and the system gets back almost none of the 68b5fca8f8Stomee * slabs. Despite the client doing what it reasonably can to help the system 69b5fca8f8Stomee * reclaim memory, the allocator cannot shake free enough slabs because of 70b5fca8f8Stomee * lonely allocations stubbornly hanging on. Although the allocator is in a 71b5fca8f8Stomee * position to diagnose the fragmentation, there is nothing that the allocator 72b5fca8f8Stomee * by itself can do about it. It only takes a single allocated object to prevent 73b5fca8f8Stomee * an entire slab from being reclaimed, and any object handed out by 74b5fca8f8Stomee * kmem_cache_alloc() is by definition in the client's control. Conversely, 75b5fca8f8Stomee * although the client is in a position to move a long-lived object, it has no 76b5fca8f8Stomee * way of knowing if the object is causing fragmentation, and if so, where to 77b5fca8f8Stomee * move it. A solution necessarily requires further cooperation between the 78b5fca8f8Stomee * allocator and the client. 79b5fca8f8Stomee * 80b5fca8f8Stomee * 2. Move Callback 81b5fca8f8Stomee * 82b5fca8f8Stomee * The kmem slab consolidator therefore adds a move callback to the 83b5fca8f8Stomee * allocator/client interface, improving worst-case external fragmentation in 84b5fca8f8Stomee * kmem caches that supply a function to move objects from one memory location 85b5fca8f8Stomee * to another. In a situation of low memory kmem attempts to consolidate all of 86b5fca8f8Stomee * a cache's slabs at once; otherwise it works slowly to bring external 87b5fca8f8Stomee * fragmentation within the 1/8 limit guaranteed for internal fragmentation, 88b5fca8f8Stomee * thereby helping to avoid a low memory situation in the future. 89b5fca8f8Stomee * 90b5fca8f8Stomee * The callback has the following signature: 91b5fca8f8Stomee * 92b5fca8f8Stomee * kmem_cbrc_t move(void *old, void *new, size_t size, void *user_arg) 93b5fca8f8Stomee * 94b5fca8f8Stomee * It supplies the kmem client with two addresses: the allocated object that 95b5fca8f8Stomee * kmem wants to move and a buffer selected by kmem for the client to use as the 96b5fca8f8Stomee * copy destination. The callback is kmem's way of saying "Please get off of 97b5fca8f8Stomee * this buffer and use this one instead." kmem knows where it wants to move the 98b5fca8f8Stomee * object in order to best reduce fragmentation. All the client needs to know 99b5fca8f8Stomee * about the second argument (void *new) is that it is an allocated, constructed 100b5fca8f8Stomee * object ready to take the contents of the old object. When the move function 101b5fca8f8Stomee * is called, the system is likely to be low on memory, and the new object 102b5fca8f8Stomee * spares the client from having to worry about allocating memory for the 103b5fca8f8Stomee * requested move. The third argument supplies the size of the object, in case a 104b5fca8f8Stomee * single move function handles multiple caches whose objects differ only in 105b5fca8f8Stomee * size (such as zio_buf_512, zio_buf_1024, etc). Finally, the same optional 106b5fca8f8Stomee * user argument passed to the constructor, destructor, and reclaim functions is 107b5fca8f8Stomee * also passed to the move callback. 108b5fca8f8Stomee * 109b5fca8f8Stomee * 2.1 Setting the Move Callback 110b5fca8f8Stomee * 111b5fca8f8Stomee * The client sets the move callback after creating the cache and before 112b5fca8f8Stomee * allocating from it: 113b5fca8f8Stomee * 114b5fca8f8Stomee * object_cache = kmem_cache_create(...); 115b5fca8f8Stomee * kmem_cache_set_move(object_cache, object_move); 116b5fca8f8Stomee * 117b5fca8f8Stomee * 2.2 Move Callback Return Values 118b5fca8f8Stomee * 119b5fca8f8Stomee * Only the client knows about its own data and when is a good time to move it. 120b5fca8f8Stomee * The client is cooperating with kmem to return unused memory to the system, 121b5fca8f8Stomee * and kmem respectfully accepts this help at the client's convenience. When 122b5fca8f8Stomee * asked to move an object, the client can respond with any of the following: 123b5fca8f8Stomee * 124b5fca8f8Stomee * typedef enum kmem_cbrc { 125b5fca8f8Stomee * KMEM_CBRC_YES, 126b5fca8f8Stomee * KMEM_CBRC_NO, 127b5fca8f8Stomee * KMEM_CBRC_LATER, 128b5fca8f8Stomee * KMEM_CBRC_DONT_NEED, 129b5fca8f8Stomee * KMEM_CBRC_DONT_KNOW 130b5fca8f8Stomee * } kmem_cbrc_t; 131b5fca8f8Stomee * 132b5fca8f8Stomee * The client must not explicitly kmem_cache_free() either of the objects passed 133b5fca8f8Stomee * to the callback, since kmem wants to free them directly to the slab layer 134b5fca8f8Stomee * (bypassing the per-CPU magazine layer). The response tells kmem which of the 135b5fca8f8Stomee * objects to free: 136b5fca8f8Stomee * 137b5fca8f8Stomee * YES: (Did it) The client moved the object, so kmem frees the old one. 138b5fca8f8Stomee * NO: (Never) The client refused, so kmem frees the new object (the 139b5fca8f8Stomee * unused copy destination). kmem also marks the slab of the old 140b5fca8f8Stomee * object so as not to bother the client with further callbacks for 141b5fca8f8Stomee * that object as long as the slab remains on the partial slab list. 142b5fca8f8Stomee * (The system won't be getting the slab back as long as the 143b5fca8f8Stomee * immovable object holds it hostage, so there's no point in moving 144b5fca8f8Stomee * any of its objects.) 145b5fca8f8Stomee * LATER: The client is using the object and cannot move it now, so kmem 146b5fca8f8Stomee * frees the new object (the unused copy destination). kmem still 147b5fca8f8Stomee * attempts to move other objects off the slab, since it expects to 148b5fca8f8Stomee * succeed in clearing the slab in a later callback. The client 149b5fca8f8Stomee * should use LATER instead of NO if the object is likely to become 150b5fca8f8Stomee * movable very soon. 151b5fca8f8Stomee * DONT_NEED: The client no longer needs the object, so kmem frees the old along 152b5fca8f8Stomee * with the new object (the unused copy destination). This response 153b5fca8f8Stomee * is the client's opportunity to be a model citizen and give back as 154b5fca8f8Stomee * much as it can. 155b5fca8f8Stomee * DONT_KNOW: The client does not know about the object because 156b5fca8f8Stomee * a) the client has just allocated the object and not yet put it 157b5fca8f8Stomee * wherever it expects to find known objects 158b5fca8f8Stomee * b) the client has removed the object from wherever it expects to 159b5fca8f8Stomee * find known objects and is about to free it, or 160b5fca8f8Stomee * c) the client has freed the object. 161b5fca8f8Stomee * In all these cases (a, b, and c) kmem frees the new object (the 162*d7db73d1SBryan Cantrill * unused copy destination). In the first case, the object is in 163*d7db73d1SBryan Cantrill * use and the correct action is that for LATER; in the latter two 164*d7db73d1SBryan Cantrill * cases, we know that the object is either freed or about to be 165*d7db73d1SBryan Cantrill * freed, in which case it is either already in a magazine or about 166*d7db73d1SBryan Cantrill * to be in one. In these cases, we know that the object will either 167*d7db73d1SBryan Cantrill * be reallocated and reused, or it will end up in a full magazine 168*d7db73d1SBryan Cantrill * that will be reaped (thereby liberating the slab). Because it 169*d7db73d1SBryan Cantrill * is prohibitively expensive to differentiate these cases, and 170*d7db73d1SBryan Cantrill * because the defrag code is executed when we're low on memory 171*d7db73d1SBryan Cantrill * (thereby biasing the system to reclaim full magazines) we treat 172*d7db73d1SBryan Cantrill * all DONT_KNOW cases as LATER and rely on cache reaping to 173*d7db73d1SBryan Cantrill * generally clean up full magazines. While we take the same action 174*d7db73d1SBryan Cantrill * for these cases, we maintain their semantic distinction: if 175*d7db73d1SBryan Cantrill * defragmentation is not occurring, it is useful to know if this 176*d7db73d1SBryan Cantrill * is due to objects in use (LATER) or objects in an unknown state 177*d7db73d1SBryan Cantrill * of transition (DONT_KNOW). 178b5fca8f8Stomee * 179b5fca8f8Stomee * 2.3 Object States 180b5fca8f8Stomee * 181b5fca8f8Stomee * Neither kmem nor the client can be assumed to know the object's whereabouts 182b5fca8f8Stomee * at the time of the callback. An object belonging to a kmem cache may be in 183b5fca8f8Stomee * any of the following states: 184b5fca8f8Stomee * 185b5fca8f8Stomee * 1. Uninitialized on the slab 186b5fca8f8Stomee * 2. Allocated from the slab but not constructed (still uninitialized) 187b5fca8f8Stomee * 3. Allocated from the slab, constructed, but not yet ready for business 188b5fca8f8Stomee * (not in a valid state for the move callback) 189b5fca8f8Stomee * 4. In use (valid and known to the client) 190b5fca8f8Stomee * 5. About to be freed (no longer in a valid state for the move callback) 191b5fca8f8Stomee * 6. Freed to a magazine (still constructed) 192b5fca8f8Stomee * 7. Allocated from a magazine, not yet ready for business (not in a valid 193b5fca8f8Stomee * state for the move callback), and about to return to state #4 194b5fca8f8Stomee * 8. Deconstructed on a magazine that is about to be freed 195b5fca8f8Stomee * 9. Freed to the slab 196b5fca8f8Stomee * 197b5fca8f8Stomee * Since the move callback may be called at any time while the object is in any 198b5fca8f8Stomee * of the above states (except state #1), the client needs a safe way to 199b5fca8f8Stomee * determine whether or not it knows about the object. Specifically, the client 200b5fca8f8Stomee * needs to know whether or not the object is in state #4, the only state in 201b5fca8f8Stomee * which a move is valid. If the object is in any other state, the client should 202b5fca8f8Stomee * immediately return KMEM_CBRC_DONT_KNOW, since it is unsafe to access any of 203b5fca8f8Stomee * the object's fields. 204b5fca8f8Stomee * 205b5fca8f8Stomee * Note that although an object may be in state #4 when kmem initiates the move 206b5fca8f8Stomee * request, the object may no longer be in that state by the time kmem actually 207b5fca8f8Stomee * calls the move function. Not only does the client free objects 208b5fca8f8Stomee * asynchronously, kmem itself puts move requests on a queue where thay are 209b5fca8f8Stomee * pending until kmem processes them from another context. Also, objects freed 210b5fca8f8Stomee * to a magazine appear allocated from the point of view of the slab layer, so 211b5fca8f8Stomee * kmem may even initiate requests for objects in a state other than state #4. 212b5fca8f8Stomee * 213b5fca8f8Stomee * 2.3.1 Magazine Layer 214b5fca8f8Stomee * 215b5fca8f8Stomee * An important insight revealed by the states listed above is that the magazine 216b5fca8f8Stomee * layer is populated only by kmem_cache_free(). Magazines of constructed 217b5fca8f8Stomee * objects are never populated directly from the slab layer (which contains raw, 218b5fca8f8Stomee * unconstructed objects). Whenever an allocation request cannot be satisfied 219b5fca8f8Stomee * from the magazine layer, the magazines are bypassed and the request is 220b5fca8f8Stomee * satisfied from the slab layer (creating a new slab if necessary). kmem calls 221b5fca8f8Stomee * the object constructor only when allocating from the slab layer, and only in 222b5fca8f8Stomee * response to kmem_cache_alloc() or to prepare the destination buffer passed in 223b5fca8f8Stomee * the move callback. kmem does not preconstruct objects in anticipation of 224b5fca8f8Stomee * kmem_cache_alloc(). 225b5fca8f8Stomee * 226b5fca8f8Stomee * 2.3.2 Object Constructor and Destructor 227b5fca8f8Stomee * 228b5fca8f8Stomee * If the client supplies a destructor, it must be valid to call the destructor 229b5fca8f8Stomee * on a newly created object (immediately after the constructor). 230b5fca8f8Stomee * 231b5fca8f8Stomee * 2.4 Recognizing Known Objects 232b5fca8f8Stomee * 233b5fca8f8Stomee * There is a simple test to determine safely whether or not the client knows 234b5fca8f8Stomee * about a given object in the move callback. It relies on the fact that kmem 235b5fca8f8Stomee * guarantees that the object of the move callback has only been touched by the 236b5fca8f8Stomee * client itself or else by kmem. kmem does this by ensuring that none of the 237b5fca8f8Stomee * cache's slabs are freed to the virtual memory (VM) subsystem while a move 238b5fca8f8Stomee * callback is pending. When the last object on a slab is freed, if there is a 239b5fca8f8Stomee * pending move, kmem puts the slab on a per-cache dead list and defers freeing 240b5fca8f8Stomee * slabs on that list until all pending callbacks are completed. That way, 241b5fca8f8Stomee * clients can be certain that the object of a move callback is in one of the 242b5fca8f8Stomee * states listed above, making it possible to distinguish known objects (in 243b5fca8f8Stomee * state #4) using the two low order bits of any pointer member (with the 244b5fca8f8Stomee * exception of 'char *' or 'short *' which may not be 4-byte aligned on some 245b5fca8f8Stomee * platforms). 246b5fca8f8Stomee * 247b5fca8f8Stomee * The test works as long as the client always transitions objects from state #4 248b5fca8f8Stomee * (known, in use) to state #5 (about to be freed, invalid) by setting the low 249b5fca8f8Stomee * order bit of the client-designated pointer member. Since kmem only writes 250b5fca8f8Stomee * invalid memory patterns, such as 0xbaddcafe to uninitialized memory and 251b5fca8f8Stomee * 0xdeadbeef to freed memory, any scribbling on the object done by kmem is 252b5fca8f8Stomee * guaranteed to set at least one of the two low order bits. Therefore, given an 253b5fca8f8Stomee * object with a back pointer to a 'container_t *o_container', the client can 254b5fca8f8Stomee * test 255b5fca8f8Stomee * 256b5fca8f8Stomee * container_t *container = object->o_container; 257b5fca8f8Stomee * if ((uintptr_t)container & 0x3) { 258b5fca8f8Stomee * return (KMEM_CBRC_DONT_KNOW); 259b5fca8f8Stomee * } 260b5fca8f8Stomee * 261b5fca8f8Stomee * Typically, an object will have a pointer to some structure with a list or 262b5fca8f8Stomee * hash where objects from the cache are kept while in use. Assuming that the 263b5fca8f8Stomee * client has some way of knowing that the container structure is valid and will 264b5fca8f8Stomee * not go away during the move, and assuming that the structure includes a lock 265b5fca8f8Stomee * to protect whatever collection is used, then the client would continue as 266b5fca8f8Stomee * follows: 267b5fca8f8Stomee * 268b5fca8f8Stomee * // Ensure that the container structure does not go away. 269b5fca8f8Stomee * if (container_hold(container) == 0) { 270b5fca8f8Stomee * return (KMEM_CBRC_DONT_KNOW); 271b5fca8f8Stomee * } 272b5fca8f8Stomee * mutex_enter(&container->c_objects_lock); 273b5fca8f8Stomee * if (container != object->o_container) { 274b5fca8f8Stomee * mutex_exit(&container->c_objects_lock); 275b5fca8f8Stomee * container_rele(container); 276b5fca8f8Stomee * return (KMEM_CBRC_DONT_KNOW); 277b5fca8f8Stomee * } 278b5fca8f8Stomee * 279b5fca8f8Stomee * At this point the client knows that the object cannot be freed as long as 280b5fca8f8Stomee * c_objects_lock is held. Note that after acquiring the lock, the client must 281b5fca8f8Stomee * recheck the o_container pointer in case the object was removed just before 282b5fca8f8Stomee * acquiring the lock. 283b5fca8f8Stomee * 284b5fca8f8Stomee * When the client is about to free an object, it must first remove that object 285b5fca8f8Stomee * from the list, hash, or other structure where it is kept. At that time, to 286b5fca8f8Stomee * mark the object so it can be distinguished from the remaining, known objects, 287b5fca8f8Stomee * the client sets the designated low order bit: 288b5fca8f8Stomee * 289b5fca8f8Stomee * mutex_enter(&container->c_objects_lock); 290b5fca8f8Stomee * object->o_container = (void *)((uintptr_t)object->o_container | 0x1); 291b5fca8f8Stomee * list_remove(&container->c_objects, object); 292b5fca8f8Stomee * mutex_exit(&container->c_objects_lock); 293b5fca8f8Stomee * 294b5fca8f8Stomee * In the common case, the object is freed to the magazine layer, where it may 295b5fca8f8Stomee * be reused on a subsequent allocation without the overhead of calling the 296b5fca8f8Stomee * constructor. While in the magazine it appears allocated from the point of 297b5fca8f8Stomee * view of the slab layer, making it a candidate for the move callback. Most 298b5fca8f8Stomee * objects unrecognized by the client in the move callback fall into this 299b5fca8f8Stomee * category and are cheaply distinguished from known objects by the test 300*d7db73d1SBryan Cantrill * described earlier. Because searching magazines is prohibitively expensive 301*d7db73d1SBryan Cantrill * for kmem, clients that do not mark freed objects (and therefore return 302*d7db73d1SBryan Cantrill * KMEM_CBRC_DONT_KNOW for large numbers of objects) may find defragmentation 303*d7db73d1SBryan Cantrill * efficacy reduced. 304b5fca8f8Stomee * 305b5fca8f8Stomee * Invalidating the designated pointer member before freeing the object marks 306b5fca8f8Stomee * the object to be avoided in the callback, and conversely, assigning a valid 307b5fca8f8Stomee * value to the designated pointer member after allocating the object makes the 308b5fca8f8Stomee * object fair game for the callback: 309b5fca8f8Stomee * 310b5fca8f8Stomee * ... allocate object ... 311b5fca8f8Stomee * ... set any initial state not set by the constructor ... 312b5fca8f8Stomee * 313b5fca8f8Stomee * mutex_enter(&container->c_objects_lock); 314b5fca8f8Stomee * list_insert_tail(&container->c_objects, object); 315b5fca8f8Stomee * membar_producer(); 316b5fca8f8Stomee * object->o_container = container; 317b5fca8f8Stomee * mutex_exit(&container->c_objects_lock); 318b5fca8f8Stomee * 319b5fca8f8Stomee * Note that everything else must be valid before setting o_container makes the 320b5fca8f8Stomee * object fair game for the move callback. The membar_producer() call ensures 321b5fca8f8Stomee * that all the object's state is written to memory before setting the pointer 322b5fca8f8Stomee * that transitions the object from state #3 or #7 (allocated, constructed, not 323b5fca8f8Stomee * yet in use) to state #4 (in use, valid). That's important because the move 324b5fca8f8Stomee * function has to check the validity of the pointer before it can safely 325b5fca8f8Stomee * acquire the lock protecting the collection where it expects to find known 326b5fca8f8Stomee * objects. 327b5fca8f8Stomee * 328b5fca8f8Stomee * This method of distinguishing known objects observes the usual symmetry: 329b5fca8f8Stomee * invalidating the designated pointer is the first thing the client does before 330b5fca8f8Stomee * freeing the object, and setting the designated pointer is the last thing the 331b5fca8f8Stomee * client does after allocating the object. Of course, the client is not 332b5fca8f8Stomee * required to use this method. Fundamentally, how the client recognizes known 333b5fca8f8Stomee * objects is completely up to the client, but this method is recommended as an 334b5fca8f8Stomee * efficient and safe way to take advantage of the guarantees made by kmem. If 335b5fca8f8Stomee * the entire object is arbitrary data without any markable bits from a suitable 336b5fca8f8Stomee * pointer member, then the client must find some other method, such as 337b5fca8f8Stomee * searching a hash table of known objects. 338b5fca8f8Stomee * 339b5fca8f8Stomee * 2.5 Preventing Objects From Moving 340b5fca8f8Stomee * 341b5fca8f8Stomee * Besides a way to distinguish known objects, the other thing that the client 342b5fca8f8Stomee * needs is a strategy to ensure that an object will not move while the client 343b5fca8f8Stomee * is actively using it. The details of satisfying this requirement tend to be 344b5fca8f8Stomee * highly cache-specific. It might seem that the same rules that let a client 345b5fca8f8Stomee * remove an object safely should also decide when an object can be moved 346b5fca8f8Stomee * safely. However, any object state that makes a removal attempt invalid is 347b5fca8f8Stomee * likely to be long-lasting for objects that the client does not expect to 348b5fca8f8Stomee * remove. kmem knows nothing about the object state and is equally likely (from 349b5fca8f8Stomee * the client's point of view) to request a move for any object in the cache, 350b5fca8f8Stomee * whether prepared for removal or not. Even a low percentage of objects stuck 351b5fca8f8Stomee * in place by unremovability will defeat the consolidator if the stuck objects 352b5fca8f8Stomee * are the same long-lived allocations likely to hold slabs hostage. 353b5fca8f8Stomee * Fundamentally, the consolidator is not aimed at common cases. Severe external 354b5fca8f8Stomee * fragmentation is a worst case scenario manifested as sparsely allocated 355b5fca8f8Stomee * slabs, by definition a low percentage of the cache's objects. When deciding 356b5fca8f8Stomee * what makes an object movable, keep in mind the goal of the consolidator: to 357b5fca8f8Stomee * bring worst-case external fragmentation within the limits guaranteed for 358b5fca8f8Stomee * internal fragmentation. Removability is a poor criterion if it is likely to 359b5fca8f8Stomee * exclude more than an insignificant percentage of objects for long periods of 360b5fca8f8Stomee * time. 361b5fca8f8Stomee * 362b5fca8f8Stomee * A tricky general solution exists, and it has the advantage of letting you 363b5fca8f8Stomee * move any object at almost any moment, practically eliminating the likelihood 364b5fca8f8Stomee * that an object can hold a slab hostage. However, if there is a cache-specific 365b5fca8f8Stomee * way to ensure that an object is not actively in use in the vast majority of 366b5fca8f8Stomee * cases, a simpler solution that leverages this cache-specific knowledge is 367b5fca8f8Stomee * preferred. 368b5fca8f8Stomee * 369b5fca8f8Stomee * 2.5.1 Cache-Specific Solution 370b5fca8f8Stomee * 371b5fca8f8Stomee * As an example of a cache-specific solution, the ZFS znode cache takes 372b5fca8f8Stomee * advantage of the fact that the vast majority of znodes are only being 373b5fca8f8Stomee * referenced from the DNLC. (A typical case might be a few hundred in active 374b5fca8f8Stomee * use and a hundred thousand in the DNLC.) In the move callback, after the ZFS 375b5fca8f8Stomee * client has established that it recognizes the znode and can access its fields 376b5fca8f8Stomee * safely (using the method described earlier), it then tests whether the znode 377b5fca8f8Stomee * is referenced by anything other than the DNLC. If so, it assumes that the 378b5fca8f8Stomee * znode may be in active use and is unsafe to move, so it drops its locks and 379b5fca8f8Stomee * returns KMEM_CBRC_LATER. The advantage of this strategy is that everywhere 380b5fca8f8Stomee * else znodes are used, no change is needed to protect against the possibility 381b5fca8f8Stomee * of the znode moving. The disadvantage is that it remains possible for an 382b5fca8f8Stomee * application to hold a znode slab hostage with an open file descriptor. 383b5fca8f8Stomee * However, this case ought to be rare and the consolidator has a way to deal 384b5fca8f8Stomee * with it: If the client responds KMEM_CBRC_LATER repeatedly for the same 385b5fca8f8Stomee * object, kmem eventually stops believing it and treats the slab as if the 386b5fca8f8Stomee * client had responded KMEM_CBRC_NO. Having marked the hostage slab, kmem can 387b5fca8f8Stomee * then focus on getting it off of the partial slab list by allocating rather 388b5fca8f8Stomee * than freeing all of its objects. (Either way of getting a slab off the 389b5fca8f8Stomee * free list reduces fragmentation.) 390b5fca8f8Stomee * 391b5fca8f8Stomee * 2.5.2 General Solution 392b5fca8f8Stomee * 393b5fca8f8Stomee * The general solution, on the other hand, requires an explicit hold everywhere 394b5fca8f8Stomee * the object is used to prevent it from moving. To keep the client locking 395b5fca8f8Stomee * strategy as uncomplicated as possible, kmem guarantees the simplifying 396b5fca8f8Stomee * assumption that move callbacks are sequential, even across multiple caches. 397b5fca8f8Stomee * Internally, a global queue processed by a single thread supports all caches 398b5fca8f8Stomee * implementing the callback function. No matter how many caches supply a move 399b5fca8f8Stomee * function, the consolidator never moves more than one object at a time, so the 400b5fca8f8Stomee * client does not have to worry about tricky lock ordering involving several 401b5fca8f8Stomee * related objects from different kmem caches. 402b5fca8f8Stomee * 403b5fca8f8Stomee * The general solution implements the explicit hold as a read-write lock, which 404b5fca8f8Stomee * allows multiple readers to access an object from the cache simultaneously 405b5fca8f8Stomee * while a single writer is excluded from moving it. A single rwlock for the 406b5fca8f8Stomee * entire cache would lock out all threads from using any of the cache's objects 407b5fca8f8Stomee * even though only a single object is being moved, so to reduce contention, 408b5fca8f8Stomee * the client can fan out the single rwlock into an array of rwlocks hashed by 409b5fca8f8Stomee * the object address, making it probable that moving one object will not 410b5fca8f8Stomee * prevent other threads from using a different object. The rwlock cannot be a 411b5fca8f8Stomee * member of the object itself, because the possibility of the object moving 412b5fca8f8Stomee * makes it unsafe to access any of the object's fields until the lock is 413b5fca8f8Stomee * acquired. 414b5fca8f8Stomee * 415b5fca8f8Stomee * Assuming a small, fixed number of locks, it's possible that multiple objects 416b5fca8f8Stomee * will hash to the same lock. A thread that needs to use multiple objects in 417b5fca8f8Stomee * the same function may acquire the same lock multiple times. Since rwlocks are 418b5fca8f8Stomee * reentrant for readers, and since there is never more than a single writer at 419b5fca8f8Stomee * a time (assuming that the client acquires the lock as a writer only when 420b5fca8f8Stomee * moving an object inside the callback), there would seem to be no problem. 421b5fca8f8Stomee * However, a client locking multiple objects in the same function must handle 422b5fca8f8Stomee * one case of potential deadlock: Assume that thread A needs to prevent both 423b5fca8f8Stomee * object 1 and object 2 from moving, and thread B, the callback, meanwhile 424b5fca8f8Stomee * tries to move object 3. It's possible, if objects 1, 2, and 3 all hash to the 425b5fca8f8Stomee * same lock, that thread A will acquire the lock for object 1 as a reader 426b5fca8f8Stomee * before thread B sets the lock's write-wanted bit, preventing thread A from 427b5fca8f8Stomee * reacquiring the lock for object 2 as a reader. Unable to make forward 428b5fca8f8Stomee * progress, thread A will never release the lock for object 1, resulting in 429b5fca8f8Stomee * deadlock. 430b5fca8f8Stomee * 431b5fca8f8Stomee * There are two ways of avoiding the deadlock just described. The first is to 432b5fca8f8Stomee * use rw_tryenter() rather than rw_enter() in the callback function when 433b5fca8f8Stomee * attempting to acquire the lock as a writer. If tryenter discovers that the 434b5fca8f8Stomee * same object (or another object hashed to the same lock) is already in use, it 435b5fca8f8Stomee * aborts the callback and returns KMEM_CBRC_LATER. The second way is to use 436b5fca8f8Stomee * rprwlock_t (declared in common/fs/zfs/sys/rprwlock.h) instead of rwlock_t, 437b5fca8f8Stomee * since it allows a thread to acquire the lock as a reader in spite of a 438b5fca8f8Stomee * waiting writer. This second approach insists on moving the object now, no 439b5fca8f8Stomee * matter how many readers the move function must wait for in order to do so, 440b5fca8f8Stomee * and could delay the completion of the callback indefinitely (blocking 441b5fca8f8Stomee * callbacks to other clients). In practice, a less insistent callback using 442b5fca8f8Stomee * rw_tryenter() returns KMEM_CBRC_LATER infrequently enough that there seems 443b5fca8f8Stomee * little reason to use anything else. 444b5fca8f8Stomee * 445b5fca8f8Stomee * Avoiding deadlock is not the only problem that an implementation using an 446b5fca8f8Stomee * explicit hold needs to solve. Locking the object in the first place (to 447b5fca8f8Stomee * prevent it from moving) remains a problem, since the object could move 448b5fca8f8Stomee * between the time you obtain a pointer to the object and the time you acquire 449b5fca8f8Stomee * the rwlock hashed to that pointer value. Therefore the client needs to 450b5fca8f8Stomee * recheck the value of the pointer after acquiring the lock, drop the lock if 451b5fca8f8Stomee * the value has changed, and try again. This requires a level of indirection: 452b5fca8f8Stomee * something that points to the object rather than the object itself, that the 453b5fca8f8Stomee * client can access safely while attempting to acquire the lock. (The object 454b5fca8f8Stomee * itself cannot be referenced safely because it can move at any time.) 455b5fca8f8Stomee * The following lock-acquisition function takes whatever is safe to reference 456b5fca8f8Stomee * (arg), follows its pointer to the object (using function f), and tries as 457b5fca8f8Stomee * often as necessary to acquire the hashed lock and verify that the object 458b5fca8f8Stomee * still has not moved: 459b5fca8f8Stomee * 460b5fca8f8Stomee * object_t * 461b5fca8f8Stomee * object_hold(object_f f, void *arg) 462b5fca8f8Stomee * { 463b5fca8f8Stomee * object_t *op; 464b5fca8f8Stomee * 465b5fca8f8Stomee * op = f(arg); 466b5fca8f8Stomee * if (op == NULL) { 467b5fca8f8Stomee * return (NULL); 468b5fca8f8Stomee * } 469b5fca8f8Stomee * 470b5fca8f8Stomee * rw_enter(OBJECT_RWLOCK(op), RW_READER); 471b5fca8f8Stomee * while (op != f(arg)) { 472b5fca8f8Stomee * rw_exit(OBJECT_RWLOCK(op)); 473b5fca8f8Stomee * op = f(arg); 474b5fca8f8Stomee * if (op == NULL) { 475b5fca8f8Stomee * break; 476b5fca8f8Stomee * } 477b5fca8f8Stomee * rw_enter(OBJECT_RWLOCK(op), RW_READER); 478b5fca8f8Stomee * } 479b5fca8f8Stomee * 480b5fca8f8Stomee * return (op); 481b5fca8f8Stomee * } 482b5fca8f8Stomee * 483b5fca8f8Stomee * The OBJECT_RWLOCK macro hashes the object address to obtain the rwlock. The 484b5fca8f8Stomee * lock reacquisition loop, while necessary, almost never executes. The function 485b5fca8f8Stomee * pointer f (used to obtain the object pointer from arg) has the following type 486b5fca8f8Stomee * definition: 487b5fca8f8Stomee * 488b5fca8f8Stomee * typedef object_t *(*object_f)(void *arg); 489b5fca8f8Stomee * 490b5fca8f8Stomee * An object_f implementation is likely to be as simple as accessing a structure 491b5fca8f8Stomee * member: 492b5fca8f8Stomee * 493b5fca8f8Stomee * object_t * 494b5fca8f8Stomee * s_object(void *arg) 495b5fca8f8Stomee * { 496b5fca8f8Stomee * something_t *sp = arg; 497b5fca8f8Stomee * return (sp->s_object); 498b5fca8f8Stomee * } 499b5fca8f8Stomee * 500b5fca8f8Stomee * The flexibility of a function pointer allows the path to the object to be 501b5fca8f8Stomee * arbitrarily complex and also supports the notion that depending on where you 502b5fca8f8Stomee * are using the object, you may need to get it from someplace different. 503b5fca8f8Stomee * 504b5fca8f8Stomee * The function that releases the explicit hold is simpler because it does not 505b5fca8f8Stomee * have to worry about the object moving: 506b5fca8f8Stomee * 507b5fca8f8Stomee * void 508b5fca8f8Stomee * object_rele(object_t *op) 509b5fca8f8Stomee * { 510b5fca8f8Stomee * rw_exit(OBJECT_RWLOCK(op)); 511b5fca8f8Stomee * } 512b5fca8f8Stomee * 513b5fca8f8Stomee * The caller is spared these details so that obtaining and releasing an 514b5fca8f8Stomee * explicit hold feels like a simple mutex_enter()/mutex_exit() pair. The caller 515b5fca8f8Stomee * of object_hold() only needs to know that the returned object pointer is valid 516b5fca8f8Stomee * if not NULL and that the object will not move until released. 517b5fca8f8Stomee * 518b5fca8f8Stomee * Although object_hold() prevents an object from moving, it does not prevent it 519b5fca8f8Stomee * from being freed. The caller must take measures before calling object_hold() 520b5fca8f8Stomee * (afterwards is too late) to ensure that the held object cannot be freed. The 521b5fca8f8Stomee * caller must do so without accessing the unsafe object reference, so any lock 522b5fca8f8Stomee * or reference count used to ensure the continued existence of the object must 523b5fca8f8Stomee * live outside the object itself. 524b5fca8f8Stomee * 525b5fca8f8Stomee * Obtaining a new object is a special case where an explicit hold is impossible 526b5fca8f8Stomee * for the caller. Any function that returns a newly allocated object (either as 527b5fca8f8Stomee * a return value, or as an in-out paramter) must return it already held; after 528b5fca8f8Stomee * the caller gets it is too late, since the object cannot be safely accessed 529b5fca8f8Stomee * without the level of indirection described earlier. The following 530b5fca8f8Stomee * object_alloc() example uses the same code shown earlier to transition a new 531b5fca8f8Stomee * object into the state of being recognized (by the client) as a known object. 532b5fca8f8Stomee * The function must acquire the hold (rw_enter) before that state transition 533b5fca8f8Stomee * makes the object movable: 534b5fca8f8Stomee * 535b5fca8f8Stomee * static object_t * 536b5fca8f8Stomee * object_alloc(container_t *container) 537b5fca8f8Stomee * { 5384d4c4c43STom Erickson * object_t *object = kmem_cache_alloc(object_cache, 0); 539b5fca8f8Stomee * ... set any initial state not set by the constructor ... 540b5fca8f8Stomee * rw_enter(OBJECT_RWLOCK(object), RW_READER); 541b5fca8f8Stomee * mutex_enter(&container->c_objects_lock); 542b5fca8f8Stomee * list_insert_tail(&container->c_objects, object); 543b5fca8f8Stomee * membar_producer(); 544b5fca8f8Stomee * object->o_container = container; 545b5fca8f8Stomee * mutex_exit(&container->c_objects_lock); 546b5fca8f8Stomee * return (object); 547b5fca8f8Stomee * } 548b5fca8f8Stomee * 549b5fca8f8Stomee * Functions that implicitly acquire an object hold (any function that calls 550b5fca8f8Stomee * object_alloc() to supply an object for the caller) need to be carefully noted 551b5fca8f8Stomee * so that the matching object_rele() is not neglected. Otherwise, leaked holds 552b5fca8f8Stomee * prevent all objects hashed to the affected rwlocks from ever being moved. 553b5fca8f8Stomee * 554b5fca8f8Stomee * The pointer to a held object can be hashed to the holding rwlock even after 555b5fca8f8Stomee * the object has been freed. Although it is possible to release the hold 556b5fca8f8Stomee * after freeing the object, you may decide to release the hold implicitly in 557b5fca8f8Stomee * whatever function frees the object, so as to release the hold as soon as 558b5fca8f8Stomee * possible, and for the sake of symmetry with the function that implicitly 559b5fca8f8Stomee * acquires the hold when it allocates the object. Here, object_free() releases 560b5fca8f8Stomee * the hold acquired by object_alloc(). Its implicit object_rele() forms a 561b5fca8f8Stomee * matching pair with object_hold(): 562b5fca8f8Stomee * 563b5fca8f8Stomee * void 564b5fca8f8Stomee * object_free(object_t *object) 565b5fca8f8Stomee * { 566b5fca8f8Stomee * container_t *container; 567b5fca8f8Stomee * 568b5fca8f8Stomee * ASSERT(object_held(object)); 569b5fca8f8Stomee * container = object->o_container; 570b5fca8f8Stomee * mutex_enter(&container->c_objects_lock); 571b5fca8f8Stomee * object->o_container = 572b5fca8f8Stomee * (void *)((uintptr_t)object->o_container | 0x1); 573b5fca8f8Stomee * list_remove(&container->c_objects, object); 574b5fca8f8Stomee * mutex_exit(&container->c_objects_lock); 575b5fca8f8Stomee * object_rele(object); 576b5fca8f8Stomee * kmem_cache_free(object_cache, object); 577b5fca8f8Stomee * } 578b5fca8f8Stomee * 579b5fca8f8Stomee * Note that object_free() cannot safely accept an object pointer as an argument 580b5fca8f8Stomee * unless the object is already held. Any function that calls object_free() 581b5fca8f8Stomee * needs to be carefully noted since it similarly forms a matching pair with 582b5fca8f8Stomee * object_hold(). 583b5fca8f8Stomee * 584b5fca8f8Stomee * To complete the picture, the following callback function implements the 585b5fca8f8Stomee * general solution by moving objects only if they are currently unheld: 586b5fca8f8Stomee * 587b5fca8f8Stomee * static kmem_cbrc_t 588b5fca8f8Stomee * object_move(void *buf, void *newbuf, size_t size, void *arg) 589b5fca8f8Stomee * { 590b5fca8f8Stomee * object_t *op = buf, *np = newbuf; 591b5fca8f8Stomee * container_t *container; 592b5fca8f8Stomee * 593b5fca8f8Stomee * container = op->o_container; 594b5fca8f8Stomee * if ((uintptr_t)container & 0x3) { 595b5fca8f8Stomee * return (KMEM_CBRC_DONT_KNOW); 596b5fca8f8Stomee * } 597b5fca8f8Stomee * 598b5fca8f8Stomee * // Ensure that the container structure does not go away. 599b5fca8f8Stomee * if (container_hold(container) == 0) { 600b5fca8f8Stomee * return (KMEM_CBRC_DONT_KNOW); 601b5fca8f8Stomee * } 602b5fca8f8Stomee * 603b5fca8f8Stomee * mutex_enter(&container->c_objects_lock); 604b5fca8f8Stomee * if (container != op->o_container) { 605b5fca8f8Stomee * mutex_exit(&container->c_objects_lock); 606b5fca8f8Stomee * container_rele(container); 607b5fca8f8Stomee * return (KMEM_CBRC_DONT_KNOW); 608b5fca8f8Stomee * } 609b5fca8f8Stomee * 610b5fca8f8Stomee * if (rw_tryenter(OBJECT_RWLOCK(op), RW_WRITER) == 0) { 611b5fca8f8Stomee * mutex_exit(&container->c_objects_lock); 612b5fca8f8Stomee * container_rele(container); 613b5fca8f8Stomee * return (KMEM_CBRC_LATER); 614b5fca8f8Stomee * } 615b5fca8f8Stomee * 616b5fca8f8Stomee * object_move_impl(op, np); // critical section 617b5fca8f8Stomee * rw_exit(OBJECT_RWLOCK(op)); 618b5fca8f8Stomee * 619b5fca8f8Stomee * op->o_container = (void *)((uintptr_t)op->o_container | 0x1); 620b5fca8f8Stomee * list_link_replace(&op->o_link_node, &np->o_link_node); 621b5fca8f8Stomee * mutex_exit(&container->c_objects_lock); 622b5fca8f8Stomee * container_rele(container); 623b5fca8f8Stomee * return (KMEM_CBRC_YES); 624b5fca8f8Stomee * } 625b5fca8f8Stomee * 626b5fca8f8Stomee * Note that object_move() must invalidate the designated o_container pointer of 627b5fca8f8Stomee * the old object in the same way that object_free() does, since kmem will free 628b5fca8f8Stomee * the object in response to the KMEM_CBRC_YES return value. 629b5fca8f8Stomee * 630b5fca8f8Stomee * The lock order in object_move() differs from object_alloc(), which locks 631b5fca8f8Stomee * OBJECT_RWLOCK first and &container->c_objects_lock second, but as long as the 632b5fca8f8Stomee * callback uses rw_tryenter() (preventing the deadlock described earlier), it's 633b5fca8f8Stomee * not a problem. Holding the lock on the object list in the example above 634b5fca8f8Stomee * through the entire callback not only prevents the object from going away, it 635b5fca8f8Stomee * also allows you to lock the list elsewhere and know that none of its elements 636b5fca8f8Stomee * will move during iteration. 637b5fca8f8Stomee * 638b5fca8f8Stomee * Adding an explicit hold everywhere an object from the cache is used is tricky 639b5fca8f8Stomee * and involves much more change to client code than a cache-specific solution 640b5fca8f8Stomee * that leverages existing state to decide whether or not an object is 641b5fca8f8Stomee * movable. However, this approach has the advantage that no object remains 642b5fca8f8Stomee * immovable for any significant length of time, making it extremely unlikely 643b5fca8f8Stomee * that long-lived allocations can continue holding slabs hostage; and it works 644b5fca8f8Stomee * for any cache. 645b5fca8f8Stomee * 646b5fca8f8Stomee * 3. Consolidator Implementation 647b5fca8f8Stomee * 648b5fca8f8Stomee * Once the client supplies a move function that a) recognizes known objects and 649b5fca8f8Stomee * b) avoids moving objects that are actively in use, the remaining work is up 650b5fca8f8Stomee * to the consolidator to decide which objects to move and when to issue 651b5fca8f8Stomee * callbacks. 652b5fca8f8Stomee * 653b5fca8f8Stomee * The consolidator relies on the fact that a cache's slabs are ordered by 654b5fca8f8Stomee * usage. Each slab has a fixed number of objects. Depending on the slab's 655b5fca8f8Stomee * "color" (the offset of the first object from the beginning of the slab; 656b5fca8f8Stomee * offsets are staggered to mitigate false sharing of cache lines) it is either 657b5fca8f8Stomee * the maximum number of objects per slab determined at cache creation time or 658b5fca8f8Stomee * else the number closest to the maximum that fits within the space remaining 659b5fca8f8Stomee * after the initial offset. A completely allocated slab may contribute some 660b5fca8f8Stomee * internal fragmentation (per-slab overhead) but no external fragmentation, so 661b5fca8f8Stomee * it is of no interest to the consolidator. At the other extreme, slabs whose 662b5fca8f8Stomee * objects have all been freed to the slab are released to the virtual memory 663b5fca8f8Stomee * (VM) subsystem (objects freed to magazines are still allocated as far as the 664b5fca8f8Stomee * slab is concerned). External fragmentation exists when there are slabs 665b5fca8f8Stomee * somewhere between these extremes. A partial slab has at least one but not all 666b5fca8f8Stomee * of its objects allocated. The more partial slabs, and the fewer allocated 667b5fca8f8Stomee * objects on each of them, the higher the fragmentation. Hence the 668b5fca8f8Stomee * consolidator's overall strategy is to reduce the number of partial slabs by 669b5fca8f8Stomee * moving allocated objects from the least allocated slabs to the most allocated 670b5fca8f8Stomee * slabs. 671b5fca8f8Stomee * 672b5fca8f8Stomee * Partial slabs are kept in an AVL tree ordered by usage. Completely allocated 673b5fca8f8Stomee * slabs are kept separately in an unordered list. Since the majority of slabs 674b5fca8f8Stomee * tend to be completely allocated (a typical unfragmented cache may have 675b5fca8f8Stomee * thousands of complete slabs and only a single partial slab), separating 676b5fca8f8Stomee * complete slabs improves the efficiency of partial slab ordering, since the 677b5fca8f8Stomee * complete slabs do not affect the depth or balance of the AVL tree. This 678b5fca8f8Stomee * ordered sequence of partial slabs acts as a "free list" supplying objects for 679b5fca8f8Stomee * allocation requests. 680b5fca8f8Stomee * 681b5fca8f8Stomee * Objects are always allocated from the first partial slab in the free list, 682b5fca8f8Stomee * where the allocation is most likely to eliminate a partial slab (by 683b5fca8f8Stomee * completely allocating it). Conversely, when a single object from a completely 684b5fca8f8Stomee * allocated slab is freed to the slab, that slab is added to the front of the 685b5fca8f8Stomee * free list. Since most free list activity involves highly allocated slabs 686b5fca8f8Stomee * coming and going at the front of the list, slabs tend naturally toward the 687b5fca8f8Stomee * ideal order: highly allocated at the front, sparsely allocated at the back. 688b5fca8f8Stomee * Slabs with few allocated objects are likely to become completely free if they 689b5fca8f8Stomee * keep a safe distance away from the front of the free list. Slab misorders 690b5fca8f8Stomee * interfere with the natural tendency of slabs to become completely free or 691b5fca8f8Stomee * completely allocated. For example, a slab with a single allocated object 692b5fca8f8Stomee * needs only a single free to escape the cache; its natural desire is 693b5fca8f8Stomee * frustrated when it finds itself at the front of the list where a second 694b5fca8f8Stomee * allocation happens just before the free could have released it. Another slab 695b5fca8f8Stomee * with all but one object allocated might have supplied the buffer instead, so 696b5fca8f8Stomee * that both (as opposed to neither) of the slabs would have been taken off the 697b5fca8f8Stomee * free list. 698b5fca8f8Stomee * 699b5fca8f8Stomee * Although slabs tend naturally toward the ideal order, misorders allowed by a 700b5fca8f8Stomee * simple list implementation defeat the consolidator's strategy of merging 701b5fca8f8Stomee * least- and most-allocated slabs. Without an AVL tree to guarantee order, kmem 702b5fca8f8Stomee * needs another way to fix misorders to optimize its callback strategy. One 703b5fca8f8Stomee * approach is to periodically scan a limited number of slabs, advancing a 704b5fca8f8Stomee * marker to hold the current scan position, and to move extreme misorders to 705b5fca8f8Stomee * the front or back of the free list and to the front or back of the current 706b5fca8f8Stomee * scan range. By making consecutive scan ranges overlap by one slab, the least 707b5fca8f8Stomee * allocated slab in the current range can be carried along from the end of one 708b5fca8f8Stomee * scan to the start of the next. 709b5fca8f8Stomee * 710b5fca8f8Stomee * Maintaining partial slabs in an AVL tree relieves kmem of this additional 711b5fca8f8Stomee * task, however. Since most of the cache's activity is in the magazine layer, 712b5fca8f8Stomee * and allocations from the slab layer represent only a startup cost, the 713b5fca8f8Stomee * overhead of maintaining a balanced tree is not a significant concern compared 714b5fca8f8Stomee * to the opportunity of reducing complexity by eliminating the partial slab 715b5fca8f8Stomee * scanner just described. The overhead of an AVL tree is minimized by 716b5fca8f8Stomee * maintaining only partial slabs in the tree and keeping completely allocated 717b5fca8f8Stomee * slabs separately in a list. To avoid increasing the size of the slab 718b5fca8f8Stomee * structure the AVL linkage pointers are reused for the slab's list linkage, 719b5fca8f8Stomee * since the slab will always be either partial or complete, never stored both 720b5fca8f8Stomee * ways at the same time. To further minimize the overhead of the AVL tree the 721b5fca8f8Stomee * compare function that orders partial slabs by usage divides the range of 722b5fca8f8Stomee * allocated object counts into bins such that counts within the same bin are 723b5fca8f8Stomee * considered equal. Binning partial slabs makes it less likely that allocating 724b5fca8f8Stomee * or freeing a single object will change the slab's order, requiring a tree 725b5fca8f8Stomee * reinsertion (an avl_remove() followed by an avl_add(), both potentially 726b5fca8f8Stomee * requiring some rebalancing of the tree). Allocation counts closest to 727b5fca8f8Stomee * completely free and completely allocated are left unbinned (finely sorted) to 728b5fca8f8Stomee * better support the consolidator's strategy of merging slabs at either 729b5fca8f8Stomee * extreme. 730b5fca8f8Stomee * 731b5fca8f8Stomee * 3.1 Assessing Fragmentation and Selecting Candidate Slabs 732b5fca8f8Stomee * 733b5fca8f8Stomee * The consolidator piggybacks on the kmem maintenance thread and is called on 734b5fca8f8Stomee * the same interval as kmem_cache_update(), once per cache every fifteen 735b5fca8f8Stomee * seconds. kmem maintains a running count of unallocated objects in the slab 736b5fca8f8Stomee * layer (cache_bufslab). The consolidator checks whether that number exceeds 737b5fca8f8Stomee * 12.5% (1/8) of the total objects in the cache (cache_buftotal), and whether 738b5fca8f8Stomee * there is a significant number of slabs in the cache (arbitrarily a minimum 739b5fca8f8Stomee * 101 total slabs). Unused objects that have fallen out of the magazine layer's 740b5fca8f8Stomee * working set are included in the assessment, and magazines in the depot are 741b5fca8f8Stomee * reaped if those objects would lift cache_bufslab above the fragmentation 742b5fca8f8Stomee * threshold. Once the consolidator decides that a cache is fragmented, it looks 743b5fca8f8Stomee * for a candidate slab to reclaim, starting at the end of the partial slab free 744b5fca8f8Stomee * list and scanning backwards. At first the consolidator is choosy: only a slab 745b5fca8f8Stomee * with fewer than 12.5% (1/8) of its objects allocated qualifies (or else a 746b5fca8f8Stomee * single allocated object, regardless of percentage). If there is difficulty 747b5fca8f8Stomee * finding a candidate slab, kmem raises the allocation threshold incrementally, 748b5fca8f8Stomee * up to a maximum 87.5% (7/8), so that eventually the consolidator will reduce 749b5fca8f8Stomee * external fragmentation (unused objects on the free list) below 12.5% (1/8), 750b5fca8f8Stomee * even in the worst case of every slab in the cache being almost 7/8 allocated. 751b5fca8f8Stomee * The threshold can also be lowered incrementally when candidate slabs are easy 752b5fca8f8Stomee * to find, and the threshold is reset to the minimum 1/8 as soon as the cache 753b5fca8f8Stomee * is no longer fragmented. 754b5fca8f8Stomee * 755b5fca8f8Stomee * 3.2 Generating Callbacks 756b5fca8f8Stomee * 757b5fca8f8Stomee * Once an eligible slab is chosen, a callback is generated for every allocated 758b5fca8f8Stomee * object on the slab, in the hope that the client will move everything off the 759b5fca8f8Stomee * slab and make it reclaimable. Objects selected as move destinations are 760b5fca8f8Stomee * chosen from slabs at the front of the free list. Assuming slabs in the ideal 761b5fca8f8Stomee * order (most allocated at the front, least allocated at the back) and a 762b5fca8f8Stomee * cooperative client, the consolidator will succeed in removing slabs from both 763b5fca8f8Stomee * ends of the free list, completely allocating on the one hand and completely 764b5fca8f8Stomee * freeing on the other. Objects selected as move destinations are allocated in 765b5fca8f8Stomee * the kmem maintenance thread where move requests are enqueued. A separate 766b5fca8f8Stomee * callback thread removes pending callbacks from the queue and calls the 767b5fca8f8Stomee * client. The separate thread ensures that client code (the move function) does 768b5fca8f8Stomee * not interfere with internal kmem maintenance tasks. A map of pending 769b5fca8f8Stomee * callbacks keyed by object address (the object to be moved) is checked to 770b5fca8f8Stomee * ensure that duplicate callbacks are not generated for the same object. 771b5fca8f8Stomee * Allocating the move destination (the object to move to) prevents subsequent 772b5fca8f8Stomee * callbacks from selecting the same destination as an earlier pending callback. 773b5fca8f8Stomee * 774b5fca8f8Stomee * Move requests can also be generated by kmem_cache_reap() when the system is 775b5fca8f8Stomee * desperate for memory and by kmem_cache_move_notify(), called by the client to 776b5fca8f8Stomee * notify kmem that a move refused earlier with KMEM_CBRC_LATER is now possible. 777b5fca8f8Stomee * The map of pending callbacks is protected by the same lock that protects the 778b5fca8f8Stomee * slab layer. 779b5fca8f8Stomee * 780b5fca8f8Stomee * When the system is desperate for memory, kmem does not bother to determine 781b5fca8f8Stomee * whether or not the cache exceeds the fragmentation threshold, but tries to 782b5fca8f8Stomee * consolidate as many slabs as possible. Normally, the consolidator chews 783b5fca8f8Stomee * slowly, one sparsely allocated slab at a time during each maintenance 784b5fca8f8Stomee * interval that the cache is fragmented. When desperate, the consolidator 785b5fca8f8Stomee * starts at the last partial slab and enqueues callbacks for every allocated 786b5fca8f8Stomee * object on every partial slab, working backwards until it reaches the first 787b5fca8f8Stomee * partial slab. The first partial slab, meanwhile, advances in pace with the 788b5fca8f8Stomee * consolidator as allocations to supply move destinations for the enqueued 789b5fca8f8Stomee * callbacks use up the highly allocated slabs at the front of the free list. 790b5fca8f8Stomee * Ideally, the overgrown free list collapses like an accordion, starting at 791b5fca8f8Stomee * both ends and ending at the center with a single partial slab. 792b5fca8f8Stomee * 793b5fca8f8Stomee * 3.3 Client Responses 794b5fca8f8Stomee * 795b5fca8f8Stomee * When the client returns KMEM_CBRC_NO in response to the move callback, kmem 796b5fca8f8Stomee * marks the slab that supplied the stuck object non-reclaimable and moves it to 797b5fca8f8Stomee * front of the free list. The slab remains marked as long as it remains on the 798b5fca8f8Stomee * free list, and it appears more allocated to the partial slab compare function 799b5fca8f8Stomee * than any unmarked slab, no matter how many of its objects are allocated. 800b5fca8f8Stomee * Since even one immovable object ties up the entire slab, the goal is to 801b5fca8f8Stomee * completely allocate any slab that cannot be completely freed. kmem does not 802b5fca8f8Stomee * bother generating callbacks to move objects from a marked slab unless the 803b5fca8f8Stomee * system is desperate. 804b5fca8f8Stomee * 805b5fca8f8Stomee * When the client responds KMEM_CBRC_LATER, kmem increments a count for the 806b5fca8f8Stomee * slab. If the client responds LATER too many times, kmem disbelieves and 807b5fca8f8Stomee * treats the response as a NO. The count is cleared when the slab is taken off 808b5fca8f8Stomee * the partial slab list or when the client moves one of the slab's objects. 809b5fca8f8Stomee * 810b5fca8f8Stomee * 4. Observability 811b5fca8f8Stomee * 812b5fca8f8Stomee * A kmem cache's external fragmentation is best observed with 'mdb -k' using 813b5fca8f8Stomee * the ::kmem_slabs dcmd. For a complete description of the command, enter 814b5fca8f8Stomee * '::help kmem_slabs' at the mdb prompt. 8157c478bd9Sstevel@tonic-gate */ 8167c478bd9Sstevel@tonic-gate 8177c478bd9Sstevel@tonic-gate #include <sys/kmem_impl.h> 8187c478bd9Sstevel@tonic-gate #include <sys/vmem_impl.h> 8197c478bd9Sstevel@tonic-gate #include <sys/param.h> 8207c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 8217c478bd9Sstevel@tonic-gate #include <sys/vm.h> 8227c478bd9Sstevel@tonic-gate #include <sys/proc.h> 8237c478bd9Sstevel@tonic-gate #include <sys/tuneable.h> 8247c478bd9Sstevel@tonic-gate #include <sys/systm.h> 8257c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 8267c478bd9Sstevel@tonic-gate #include <sys/debug.h> 827b5fca8f8Stomee #include <sys/sdt.h> 8287c478bd9Sstevel@tonic-gate #include <sys/mutex.h> 8297c478bd9Sstevel@tonic-gate #include <sys/bitmap.h> 8307c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 8317c478bd9Sstevel@tonic-gate #include <sys/kobj.h> 8327c478bd9Sstevel@tonic-gate #include <sys/disp.h> 8337c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h> 8347c478bd9Sstevel@tonic-gate #include <sys/log.h> 8357c478bd9Sstevel@tonic-gate #include <sys/callb.h> 8367c478bd9Sstevel@tonic-gate #include <sys/taskq.h> 8377c478bd9Sstevel@tonic-gate #include <sys/modctl.h> 8387c478bd9Sstevel@tonic-gate #include <sys/reboot.h> 8397c478bd9Sstevel@tonic-gate #include <sys/id32.h> 8407c478bd9Sstevel@tonic-gate #include <sys/zone.h> 841f4b3ec61Sdh #include <sys/netstack.h> 842b5fca8f8Stomee #ifdef DEBUG 843b5fca8f8Stomee #include <sys/random.h> 844b5fca8f8Stomee #endif 8457c478bd9Sstevel@tonic-gate 8467c478bd9Sstevel@tonic-gate extern void streams_msg_init(void); 8477c478bd9Sstevel@tonic-gate extern int segkp_fromheap; 8487c478bd9Sstevel@tonic-gate extern void segkp_cache_free(void); 8496e00b116SPeter Telford extern int callout_init_done; 8507c478bd9Sstevel@tonic-gate 8517c478bd9Sstevel@tonic-gate struct kmem_cache_kstat { 8527c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_size; 8537c478bd9Sstevel@tonic-gate kstat_named_t kmc_align; 8547c478bd9Sstevel@tonic-gate kstat_named_t kmc_chunk_size; 8557c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_size; 8567c478bd9Sstevel@tonic-gate kstat_named_t kmc_alloc; 8577c478bd9Sstevel@tonic-gate kstat_named_t kmc_alloc_fail; 8587c478bd9Sstevel@tonic-gate kstat_named_t kmc_free; 8597c478bd9Sstevel@tonic-gate kstat_named_t kmc_depot_alloc; 8607c478bd9Sstevel@tonic-gate kstat_named_t kmc_depot_free; 8617c478bd9Sstevel@tonic-gate kstat_named_t kmc_depot_contention; 8627c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_alloc; 8637c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_free; 8647c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_constructed; 8657c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_avail; 8667c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_inuse; 8677c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_total; 8687c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_max; 8697c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_create; 8707c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_destroy; 8717c478bd9Sstevel@tonic-gate kstat_named_t kmc_vmem_source; 8727c478bd9Sstevel@tonic-gate kstat_named_t kmc_hash_size; 8737c478bd9Sstevel@tonic-gate kstat_named_t kmc_hash_lookup_depth; 8747c478bd9Sstevel@tonic-gate kstat_named_t kmc_hash_rescale; 8757c478bd9Sstevel@tonic-gate kstat_named_t kmc_full_magazines; 8767c478bd9Sstevel@tonic-gate kstat_named_t kmc_empty_magazines; 8777c478bd9Sstevel@tonic-gate kstat_named_t kmc_magazine_size; 878686031edSTom Erickson kstat_named_t kmc_reap; /* number of kmem_cache_reap() calls */ 879686031edSTom Erickson kstat_named_t kmc_defrag; /* attempts to defrag all partial slabs */ 880686031edSTom Erickson kstat_named_t kmc_scan; /* attempts to defrag one partial slab */ 881686031edSTom Erickson kstat_named_t kmc_move_callbacks; /* sum of yes, no, later, dn, dk */ 882b5fca8f8Stomee kstat_named_t kmc_move_yes; 883b5fca8f8Stomee kstat_named_t kmc_move_no; 884b5fca8f8Stomee kstat_named_t kmc_move_later; 885b5fca8f8Stomee kstat_named_t kmc_move_dont_need; 886686031edSTom Erickson kstat_named_t kmc_move_dont_know; /* obj unrecognized by client ... */ 887686031edSTom Erickson kstat_named_t kmc_move_hunt_found; /* ... but found in mag layer */ 888686031edSTom Erickson kstat_named_t kmc_move_slabs_freed; /* slabs freed by consolidator */ 889686031edSTom Erickson kstat_named_t kmc_move_reclaimable; /* buffers, if consolidator ran */ 8907c478bd9Sstevel@tonic-gate } kmem_cache_kstat = { 8917c478bd9Sstevel@tonic-gate { "buf_size", KSTAT_DATA_UINT64 }, 8927c478bd9Sstevel@tonic-gate { "align", KSTAT_DATA_UINT64 }, 8937c478bd9Sstevel@tonic-gate { "chunk_size", KSTAT_DATA_UINT64 }, 8947c478bd9Sstevel@tonic-gate { "slab_size", KSTAT_DATA_UINT64 }, 8957c478bd9Sstevel@tonic-gate { "alloc", KSTAT_DATA_UINT64 }, 8967c478bd9Sstevel@tonic-gate { "alloc_fail", KSTAT_DATA_UINT64 }, 8977c478bd9Sstevel@tonic-gate { "free", KSTAT_DATA_UINT64 }, 8987c478bd9Sstevel@tonic-gate { "depot_alloc", KSTAT_DATA_UINT64 }, 8997c478bd9Sstevel@tonic-gate { "depot_free", KSTAT_DATA_UINT64 }, 9007c478bd9Sstevel@tonic-gate { "depot_contention", KSTAT_DATA_UINT64 }, 9017c478bd9Sstevel@tonic-gate { "slab_alloc", KSTAT_DATA_UINT64 }, 9027c478bd9Sstevel@tonic-gate { "slab_free", KSTAT_DATA_UINT64 }, 9037c478bd9Sstevel@tonic-gate { "buf_constructed", KSTAT_DATA_UINT64 }, 9047c478bd9Sstevel@tonic-gate { "buf_avail", KSTAT_DATA_UINT64 }, 9057c478bd9Sstevel@tonic-gate { "buf_inuse", KSTAT_DATA_UINT64 }, 9067c478bd9Sstevel@tonic-gate { "buf_total", KSTAT_DATA_UINT64 }, 9077c478bd9Sstevel@tonic-gate { "buf_max", KSTAT_DATA_UINT64 }, 9087c478bd9Sstevel@tonic-gate { "slab_create", KSTAT_DATA_UINT64 }, 9097c478bd9Sstevel@tonic-gate { "slab_destroy", KSTAT_DATA_UINT64 }, 9107c478bd9Sstevel@tonic-gate { "vmem_source", KSTAT_DATA_UINT64 }, 9117c478bd9Sstevel@tonic-gate { "hash_size", KSTAT_DATA_UINT64 }, 9127c478bd9Sstevel@tonic-gate { "hash_lookup_depth", KSTAT_DATA_UINT64 }, 9137c478bd9Sstevel@tonic-gate { "hash_rescale", KSTAT_DATA_UINT64 }, 9147c478bd9Sstevel@tonic-gate { "full_magazines", KSTAT_DATA_UINT64 }, 9157c478bd9Sstevel@tonic-gate { "empty_magazines", KSTAT_DATA_UINT64 }, 9167c478bd9Sstevel@tonic-gate { "magazine_size", KSTAT_DATA_UINT64 }, 917686031edSTom Erickson { "reap", KSTAT_DATA_UINT64 }, 918686031edSTom Erickson { "defrag", KSTAT_DATA_UINT64 }, 919686031edSTom Erickson { "scan", KSTAT_DATA_UINT64 }, 920b5fca8f8Stomee { "move_callbacks", KSTAT_DATA_UINT64 }, 921b5fca8f8Stomee { "move_yes", KSTAT_DATA_UINT64 }, 922b5fca8f8Stomee { "move_no", KSTAT_DATA_UINT64 }, 923b5fca8f8Stomee { "move_later", KSTAT_DATA_UINT64 }, 924b5fca8f8Stomee { "move_dont_need", KSTAT_DATA_UINT64 }, 925b5fca8f8Stomee { "move_dont_know", KSTAT_DATA_UINT64 }, 926b5fca8f8Stomee { "move_hunt_found", KSTAT_DATA_UINT64 }, 927686031edSTom Erickson { "move_slabs_freed", KSTAT_DATA_UINT64 }, 928686031edSTom Erickson { "move_reclaimable", KSTAT_DATA_UINT64 }, 9297c478bd9Sstevel@tonic-gate }; 9307c478bd9Sstevel@tonic-gate 9317c478bd9Sstevel@tonic-gate static kmutex_t kmem_cache_kstat_lock; 9327c478bd9Sstevel@tonic-gate 9337c478bd9Sstevel@tonic-gate /* 9347c478bd9Sstevel@tonic-gate * The default set of caches to back kmem_alloc(). 9357c478bd9Sstevel@tonic-gate * These sizes should be reevaluated periodically. 9367c478bd9Sstevel@tonic-gate * 9377c478bd9Sstevel@tonic-gate * We want allocations that are multiples of the coherency granularity 9387c478bd9Sstevel@tonic-gate * (64 bytes) to be satisfied from a cache which is a multiple of 64 9397c478bd9Sstevel@tonic-gate * bytes, so that it will be 64-byte aligned. For all multiples of 64, 9407c478bd9Sstevel@tonic-gate * the next kmem_cache_size greater than or equal to it must be a 9417c478bd9Sstevel@tonic-gate * multiple of 64. 942dce01e3fSJonathan W Adams * 943dce01e3fSJonathan W Adams * We split the table into two sections: size <= 4k and size > 4k. This 944dce01e3fSJonathan W Adams * saves a lot of space and cache footprint in our cache tables. 9457c478bd9Sstevel@tonic-gate */ 9467c478bd9Sstevel@tonic-gate static const int kmem_alloc_sizes[] = { 9477c478bd9Sstevel@tonic-gate 1 * 8, 9487c478bd9Sstevel@tonic-gate 2 * 8, 9497c478bd9Sstevel@tonic-gate 3 * 8, 9507c478bd9Sstevel@tonic-gate 4 * 8, 5 * 8, 6 * 8, 7 * 8, 9517c478bd9Sstevel@tonic-gate 4 * 16, 5 * 16, 6 * 16, 7 * 16, 9527c478bd9Sstevel@tonic-gate 4 * 32, 5 * 32, 6 * 32, 7 * 32, 9537c478bd9Sstevel@tonic-gate 4 * 64, 5 * 64, 6 * 64, 7 * 64, 9547c478bd9Sstevel@tonic-gate 4 * 128, 5 * 128, 6 * 128, 7 * 128, 9557c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 7, 64), 9567c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 6, 64), 9577c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 5, 64), 9587c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 4, 64), 9597c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 3, 64), 9607c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 2, 64), 9617c478bd9Sstevel@tonic-gate }; 9627c478bd9Sstevel@tonic-gate 963dce01e3fSJonathan W Adams static const int kmem_big_alloc_sizes[] = { 964dce01e3fSJonathan W Adams 2 * 4096, 3 * 4096, 965dce01e3fSJonathan W Adams 2 * 8192, 3 * 8192, 966dce01e3fSJonathan W Adams 4 * 8192, 5 * 8192, 6 * 8192, 7 * 8192, 967dce01e3fSJonathan W Adams 8 * 8192, 9 * 8192, 10 * 8192, 11 * 8192, 968dce01e3fSJonathan W Adams 12 * 8192, 13 * 8192, 14 * 8192, 15 * 8192, 969dce01e3fSJonathan W Adams 16 * 8192 970dce01e3fSJonathan W Adams }; 971dce01e3fSJonathan W Adams 972dce01e3fSJonathan W Adams #define KMEM_MAXBUF 4096 973dce01e3fSJonathan W Adams #define KMEM_BIG_MAXBUF_32BIT 32768 974dce01e3fSJonathan W Adams #define KMEM_BIG_MAXBUF 131072 975dce01e3fSJonathan W Adams 976dce01e3fSJonathan W Adams #define KMEM_BIG_MULTIPLE 4096 /* big_alloc_sizes must be a multiple */ 977dce01e3fSJonathan W Adams #define KMEM_BIG_SHIFT 12 /* lg(KMEM_BIG_MULTIPLE) */ 9787c478bd9Sstevel@tonic-gate 9797c478bd9Sstevel@tonic-gate static kmem_cache_t *kmem_alloc_table[KMEM_MAXBUF >> KMEM_ALIGN_SHIFT]; 980dce01e3fSJonathan W Adams static kmem_cache_t *kmem_big_alloc_table[KMEM_BIG_MAXBUF >> KMEM_BIG_SHIFT]; 981dce01e3fSJonathan W Adams 982dce01e3fSJonathan W Adams #define KMEM_ALLOC_TABLE_MAX (KMEM_MAXBUF >> KMEM_ALIGN_SHIFT) 983dce01e3fSJonathan W Adams static size_t kmem_big_alloc_table_max = 0; /* # of filled elements */ 9847c478bd9Sstevel@tonic-gate 9857c478bd9Sstevel@tonic-gate static kmem_magtype_t kmem_magtype[] = { 9867c478bd9Sstevel@tonic-gate { 1, 8, 3200, 65536 }, 9877c478bd9Sstevel@tonic-gate { 3, 16, 256, 32768 }, 9887c478bd9Sstevel@tonic-gate { 7, 32, 64, 16384 }, 9897c478bd9Sstevel@tonic-gate { 15, 64, 0, 8192 }, 9907c478bd9Sstevel@tonic-gate { 31, 64, 0, 4096 }, 9917c478bd9Sstevel@tonic-gate { 47, 64, 0, 2048 }, 9927c478bd9Sstevel@tonic-gate { 63, 64, 0, 1024 }, 9937c478bd9Sstevel@tonic-gate { 95, 64, 0, 512 }, 9947c478bd9Sstevel@tonic-gate { 143, 64, 0, 0 }, 9957c478bd9Sstevel@tonic-gate }; 9967c478bd9Sstevel@tonic-gate 9977c478bd9Sstevel@tonic-gate static uint32_t kmem_reaping; 9987c478bd9Sstevel@tonic-gate static uint32_t kmem_reaping_idspace; 9997c478bd9Sstevel@tonic-gate 10007c478bd9Sstevel@tonic-gate /* 10017c478bd9Sstevel@tonic-gate * kmem tunables 10027c478bd9Sstevel@tonic-gate */ 10037c478bd9Sstevel@tonic-gate clock_t kmem_reap_interval; /* cache reaping rate [15 * HZ ticks] */ 10047c478bd9Sstevel@tonic-gate int kmem_depot_contention = 3; /* max failed tryenters per real interval */ 10057c478bd9Sstevel@tonic-gate pgcnt_t kmem_reapahead = 0; /* start reaping N pages before pageout */ 10067c478bd9Sstevel@tonic-gate int kmem_panic = 1; /* whether to panic on error */ 10077c478bd9Sstevel@tonic-gate int kmem_logging = 1; /* kmem_log_enter() override */ 10087c478bd9Sstevel@tonic-gate uint32_t kmem_mtbf = 0; /* mean time between failures [default: off] */ 10097c478bd9Sstevel@tonic-gate size_t kmem_transaction_log_size; /* transaction log size [2% of memory] */ 10107c478bd9Sstevel@tonic-gate size_t kmem_content_log_size; /* content log size [2% of memory] */ 10117c478bd9Sstevel@tonic-gate size_t kmem_failure_log_size; /* failure log [4 pages per CPU] */ 10127c478bd9Sstevel@tonic-gate size_t kmem_slab_log_size; /* slab create log [4 pages per CPU] */ 10137c478bd9Sstevel@tonic-gate size_t kmem_content_maxsave = 256; /* KMF_CONTENTS max bytes to log */ 10147c478bd9Sstevel@tonic-gate size_t kmem_lite_minsize = 0; /* minimum buffer size for KMF_LITE */ 10157c478bd9Sstevel@tonic-gate size_t kmem_lite_maxalign = 1024; /* maximum buffer alignment for KMF_LITE */ 10167c478bd9Sstevel@tonic-gate int kmem_lite_pcs = 4; /* number of PCs to store in KMF_LITE mode */ 10177c478bd9Sstevel@tonic-gate size_t kmem_maxverify; /* maximum bytes to inspect in debug routines */ 10187c478bd9Sstevel@tonic-gate size_t kmem_minfirewall; /* hardware-enforced redzone threshold */ 10197c478bd9Sstevel@tonic-gate 1020dce01e3fSJonathan W Adams #ifdef _LP64 1021dce01e3fSJonathan W Adams size_t kmem_max_cached = KMEM_BIG_MAXBUF; /* maximum kmem_alloc cache */ 1022dce01e3fSJonathan W Adams #else 1023dce01e3fSJonathan W Adams size_t kmem_max_cached = KMEM_BIG_MAXBUF_32BIT; /* maximum kmem_alloc cache */ 1024dce01e3fSJonathan W Adams #endif 1025dce01e3fSJonathan W Adams 10267c478bd9Sstevel@tonic-gate #ifdef DEBUG 10277c478bd9Sstevel@tonic-gate int kmem_flags = KMF_AUDIT | KMF_DEADBEEF | KMF_REDZONE | KMF_CONTENTS; 10287c478bd9Sstevel@tonic-gate #else 10297c478bd9Sstevel@tonic-gate int kmem_flags = 0; 10307c478bd9Sstevel@tonic-gate #endif 10317c478bd9Sstevel@tonic-gate int kmem_ready; 10327c478bd9Sstevel@tonic-gate 10337c478bd9Sstevel@tonic-gate static kmem_cache_t *kmem_slab_cache; 10347c478bd9Sstevel@tonic-gate static kmem_cache_t *kmem_bufctl_cache; 10357c478bd9Sstevel@tonic-gate static kmem_cache_t *kmem_bufctl_audit_cache; 10367c478bd9Sstevel@tonic-gate 10377c478bd9Sstevel@tonic-gate static kmutex_t kmem_cache_lock; /* inter-cache linkage only */ 1038b5fca8f8Stomee static list_t kmem_caches; 10397c478bd9Sstevel@tonic-gate 10407c478bd9Sstevel@tonic-gate static taskq_t *kmem_taskq; 10417c478bd9Sstevel@tonic-gate static kmutex_t kmem_flags_lock; 10427c478bd9Sstevel@tonic-gate static vmem_t *kmem_metadata_arena; 10437c478bd9Sstevel@tonic-gate static vmem_t *kmem_msb_arena; /* arena for metadata caches */ 10447c478bd9Sstevel@tonic-gate static vmem_t *kmem_cache_arena; 10457c478bd9Sstevel@tonic-gate static vmem_t *kmem_hash_arena; 10467c478bd9Sstevel@tonic-gate static vmem_t *kmem_log_arena; 10477c478bd9Sstevel@tonic-gate static vmem_t *kmem_oversize_arena; 10487c478bd9Sstevel@tonic-gate static vmem_t *kmem_va_arena; 10497c478bd9Sstevel@tonic-gate static vmem_t *kmem_default_arena; 10507c478bd9Sstevel@tonic-gate static vmem_t *kmem_firewall_va_arena; 10517c478bd9Sstevel@tonic-gate static vmem_t *kmem_firewall_arena; 10527c478bd9Sstevel@tonic-gate 1053b5fca8f8Stomee /* 1054b5fca8f8Stomee * kmem slab consolidator thresholds (tunables) 1055b5fca8f8Stomee */ 1056686031edSTom Erickson size_t kmem_frag_minslabs = 101; /* minimum total slabs */ 1057686031edSTom Erickson size_t kmem_frag_numer = 1; /* free buffers (numerator) */ 1058686031edSTom Erickson size_t kmem_frag_denom = KMEM_VOID_FRACTION; /* buffers (denominator) */ 1059b5fca8f8Stomee /* 1060b5fca8f8Stomee * Maximum number of slabs from which to move buffers during a single 1061b5fca8f8Stomee * maintenance interval while the system is not low on memory. 1062b5fca8f8Stomee */ 1063686031edSTom Erickson size_t kmem_reclaim_max_slabs = 1; 1064b5fca8f8Stomee /* 1065b5fca8f8Stomee * Number of slabs to scan backwards from the end of the partial slab list 1066b5fca8f8Stomee * when searching for buffers to relocate. 1067b5fca8f8Stomee */ 1068686031edSTom Erickson size_t kmem_reclaim_scan_range = 12; 1069b5fca8f8Stomee 1070b5fca8f8Stomee /* consolidator knobs */ 1071929d5b43SMatthew Ahrens boolean_t kmem_move_noreap; 1072929d5b43SMatthew Ahrens boolean_t kmem_move_blocked; 1073929d5b43SMatthew Ahrens boolean_t kmem_move_fulltilt; 1074929d5b43SMatthew Ahrens boolean_t kmem_move_any_partial; 1075b5fca8f8Stomee 1076b5fca8f8Stomee #ifdef DEBUG 1077b5fca8f8Stomee /* 1078686031edSTom Erickson * kmem consolidator debug tunables: 1079b5fca8f8Stomee * Ensure code coverage by occasionally running the consolidator even when the 1080b5fca8f8Stomee * caches are not fragmented (they may never be). These intervals are mean time 1081b5fca8f8Stomee * in cache maintenance intervals (kmem_cache_update). 1082b5fca8f8Stomee */ 1083686031edSTom Erickson uint32_t kmem_mtb_move = 60; /* defrag 1 slab (~15min) */ 1084686031edSTom Erickson uint32_t kmem_mtb_reap = 1800; /* defrag all slabs (~7.5hrs) */ 1085b5fca8f8Stomee #endif /* DEBUG */ 1086b5fca8f8Stomee 1087b5fca8f8Stomee static kmem_cache_t *kmem_defrag_cache; 1088b5fca8f8Stomee static kmem_cache_t *kmem_move_cache; 1089b5fca8f8Stomee static taskq_t *kmem_move_taskq; 1090b5fca8f8Stomee 1091b5fca8f8Stomee static void kmem_cache_scan(kmem_cache_t *); 1092b5fca8f8Stomee static void kmem_cache_defrag(kmem_cache_t *); 1093b942e89bSDavid Valin static void kmem_slab_prefill(kmem_cache_t *, kmem_slab_t *); 1094b5fca8f8Stomee 1095b5fca8f8Stomee 10967c478bd9Sstevel@tonic-gate kmem_log_header_t *kmem_transaction_log; 10977c478bd9Sstevel@tonic-gate kmem_log_header_t *kmem_content_log; 10987c478bd9Sstevel@tonic-gate kmem_log_header_t *kmem_failure_log; 10997c478bd9Sstevel@tonic-gate kmem_log_header_t *kmem_slab_log; 11007c478bd9Sstevel@tonic-gate 11017c478bd9Sstevel@tonic-gate static int kmem_lite_count; /* # of PCs in kmem_buftag_lite_t */ 11027c478bd9Sstevel@tonic-gate 11037c478bd9Sstevel@tonic-gate #define KMEM_BUFTAG_LITE_ENTER(bt, count, caller) \ 11047c478bd9Sstevel@tonic-gate if ((count) > 0) { \ 11057c478bd9Sstevel@tonic-gate pc_t *_s = ((kmem_buftag_lite_t *)(bt))->bt_history; \ 11067c478bd9Sstevel@tonic-gate pc_t *_e; \ 11077c478bd9Sstevel@tonic-gate /* memmove() the old entries down one notch */ \ 11087c478bd9Sstevel@tonic-gate for (_e = &_s[(count) - 1]; _e > _s; _e--) \ 11097c478bd9Sstevel@tonic-gate *_e = *(_e - 1); \ 11107c478bd9Sstevel@tonic-gate *_s = (uintptr_t)(caller); \ 11117c478bd9Sstevel@tonic-gate } 11127c478bd9Sstevel@tonic-gate 11137c478bd9Sstevel@tonic-gate #define KMERR_MODIFIED 0 /* buffer modified while on freelist */ 11147c478bd9Sstevel@tonic-gate #define KMERR_REDZONE 1 /* redzone violation (write past end of buf) */ 11157c478bd9Sstevel@tonic-gate #define KMERR_DUPFREE 2 /* freed a buffer twice */ 11167c478bd9Sstevel@tonic-gate #define KMERR_BADADDR 3 /* freed a bad (unallocated) address */ 11177c478bd9Sstevel@tonic-gate #define KMERR_BADBUFTAG 4 /* buftag corrupted */ 11187c478bd9Sstevel@tonic-gate #define KMERR_BADBUFCTL 5 /* bufctl corrupted */ 11197c478bd9Sstevel@tonic-gate #define KMERR_BADCACHE 6 /* freed a buffer to the wrong cache */ 11207c478bd9Sstevel@tonic-gate #define KMERR_BADSIZE 7 /* alloc size != free size */ 11217c478bd9Sstevel@tonic-gate #define KMERR_BADBASE 8 /* buffer base address wrong */ 11227c478bd9Sstevel@tonic-gate 11237c478bd9Sstevel@tonic-gate struct { 11247c478bd9Sstevel@tonic-gate hrtime_t kmp_timestamp; /* timestamp of panic */ 11257c478bd9Sstevel@tonic-gate int kmp_error; /* type of kmem error */ 11267c478bd9Sstevel@tonic-gate void *kmp_buffer; /* buffer that induced panic */ 11277c478bd9Sstevel@tonic-gate void *kmp_realbuf; /* real start address for buffer */ 11287c478bd9Sstevel@tonic-gate kmem_cache_t *kmp_cache; /* buffer's cache according to client */ 11297c478bd9Sstevel@tonic-gate kmem_cache_t *kmp_realcache; /* actual cache containing buffer */ 11307c478bd9Sstevel@tonic-gate kmem_slab_t *kmp_slab; /* slab accoring to kmem_findslab() */ 11317c478bd9Sstevel@tonic-gate kmem_bufctl_t *kmp_bufctl; /* bufctl */ 11327c478bd9Sstevel@tonic-gate } kmem_panic_info; 11337c478bd9Sstevel@tonic-gate 11347c478bd9Sstevel@tonic-gate 11357c478bd9Sstevel@tonic-gate static void 11367c478bd9Sstevel@tonic-gate copy_pattern(uint64_t pattern, void *buf_arg, size_t size) 11377c478bd9Sstevel@tonic-gate { 11387c478bd9Sstevel@tonic-gate uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 11397c478bd9Sstevel@tonic-gate uint64_t *buf = buf_arg; 11407c478bd9Sstevel@tonic-gate 11417c478bd9Sstevel@tonic-gate while (buf < bufend) 11427c478bd9Sstevel@tonic-gate *buf++ = pattern; 11437c478bd9Sstevel@tonic-gate } 11447c478bd9Sstevel@tonic-gate 11457c478bd9Sstevel@tonic-gate static void * 11467c478bd9Sstevel@tonic-gate verify_pattern(uint64_t pattern, void *buf_arg, size_t size) 11477c478bd9Sstevel@tonic-gate { 11487c478bd9Sstevel@tonic-gate uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 11497c478bd9Sstevel@tonic-gate uint64_t *buf; 11507c478bd9Sstevel@tonic-gate 11517c478bd9Sstevel@tonic-gate for (buf = buf_arg; buf < bufend; buf++) 11527c478bd9Sstevel@tonic-gate if (*buf != pattern) 11537c478bd9Sstevel@tonic-gate return (buf); 11547c478bd9Sstevel@tonic-gate return (NULL); 11557c478bd9Sstevel@tonic-gate } 11567c478bd9Sstevel@tonic-gate 11577c478bd9Sstevel@tonic-gate static void * 11587c478bd9Sstevel@tonic-gate verify_and_copy_pattern(uint64_t old, uint64_t new, void *buf_arg, size_t size) 11597c478bd9Sstevel@tonic-gate { 11607c478bd9Sstevel@tonic-gate uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 11617c478bd9Sstevel@tonic-gate uint64_t *buf; 11627c478bd9Sstevel@tonic-gate 11637c478bd9Sstevel@tonic-gate for (buf = buf_arg; buf < bufend; buf++) { 11647c478bd9Sstevel@tonic-gate if (*buf != old) { 11657c478bd9Sstevel@tonic-gate copy_pattern(old, buf_arg, 11669f1b636aStomee (char *)buf - (char *)buf_arg); 11677c478bd9Sstevel@tonic-gate return (buf); 11687c478bd9Sstevel@tonic-gate } 11697c478bd9Sstevel@tonic-gate *buf = new; 11707c478bd9Sstevel@tonic-gate } 11717c478bd9Sstevel@tonic-gate 11727c478bd9Sstevel@tonic-gate return (NULL); 11737c478bd9Sstevel@tonic-gate } 11747c478bd9Sstevel@tonic-gate 11757c478bd9Sstevel@tonic-gate static void 11767c478bd9Sstevel@tonic-gate kmem_cache_applyall(void (*func)(kmem_cache_t *), taskq_t *tq, int tqflag) 11777c478bd9Sstevel@tonic-gate { 11787c478bd9Sstevel@tonic-gate kmem_cache_t *cp; 11797c478bd9Sstevel@tonic-gate 11807c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_lock); 1181b5fca8f8Stomee for (cp = list_head(&kmem_caches); cp != NULL; 1182b5fca8f8Stomee cp = list_next(&kmem_caches, cp)) 11837c478bd9Sstevel@tonic-gate if (tq != NULL) 11847c478bd9Sstevel@tonic-gate (void) taskq_dispatch(tq, (task_func_t *)func, cp, 11857c478bd9Sstevel@tonic-gate tqflag); 11867c478bd9Sstevel@tonic-gate else 11877c478bd9Sstevel@tonic-gate func(cp); 11887c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_lock); 11897c478bd9Sstevel@tonic-gate } 11907c478bd9Sstevel@tonic-gate 11917c478bd9Sstevel@tonic-gate static void 11927c478bd9Sstevel@tonic-gate kmem_cache_applyall_id(void (*func)(kmem_cache_t *), taskq_t *tq, int tqflag) 11937c478bd9Sstevel@tonic-gate { 11947c478bd9Sstevel@tonic-gate kmem_cache_t *cp; 11957c478bd9Sstevel@tonic-gate 11967c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_lock); 1197b5fca8f8Stomee for (cp = list_head(&kmem_caches); cp != NULL; 1198b5fca8f8Stomee cp = list_next(&kmem_caches, cp)) { 11997c478bd9Sstevel@tonic-gate if (!(cp->cache_cflags & KMC_IDENTIFIER)) 12007c478bd9Sstevel@tonic-gate continue; 12017c478bd9Sstevel@tonic-gate if (tq != NULL) 12027c478bd9Sstevel@tonic-gate (void) taskq_dispatch(tq, (task_func_t *)func, cp, 12037c478bd9Sstevel@tonic-gate tqflag); 12047c478bd9Sstevel@tonic-gate else 12057c478bd9Sstevel@tonic-gate func(cp); 12067c478bd9Sstevel@tonic-gate } 12077c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_lock); 12087c478bd9Sstevel@tonic-gate } 12097c478bd9Sstevel@tonic-gate 12107c478bd9Sstevel@tonic-gate /* 12117c478bd9Sstevel@tonic-gate * Debugging support. Given a buffer address, find its slab. 12127c478bd9Sstevel@tonic-gate */ 12137c478bd9Sstevel@tonic-gate static kmem_slab_t * 12147c478bd9Sstevel@tonic-gate kmem_findslab(kmem_cache_t *cp, void *buf) 12157c478bd9Sstevel@tonic-gate { 12167c478bd9Sstevel@tonic-gate kmem_slab_t *sp; 12177c478bd9Sstevel@tonic-gate 12187c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 1219b5fca8f8Stomee for (sp = list_head(&cp->cache_complete_slabs); sp != NULL; 1220b5fca8f8Stomee sp = list_next(&cp->cache_complete_slabs, sp)) { 1221b5fca8f8Stomee if (KMEM_SLAB_MEMBER(sp, buf)) { 1222b5fca8f8Stomee mutex_exit(&cp->cache_lock); 1223b5fca8f8Stomee return (sp); 1224b5fca8f8Stomee } 1225b5fca8f8Stomee } 1226b5fca8f8Stomee for (sp = avl_first(&cp->cache_partial_slabs); sp != NULL; 1227b5fca8f8Stomee sp = AVL_NEXT(&cp->cache_partial_slabs, sp)) { 12287c478bd9Sstevel@tonic-gate if (KMEM_SLAB_MEMBER(sp, buf)) { 12297c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 12307c478bd9Sstevel@tonic-gate return (sp); 12317c478bd9Sstevel@tonic-gate } 12327c478bd9Sstevel@tonic-gate } 12337c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 12347c478bd9Sstevel@tonic-gate 12357c478bd9Sstevel@tonic-gate return (NULL); 12367c478bd9Sstevel@tonic-gate } 12377c478bd9Sstevel@tonic-gate 12387c478bd9Sstevel@tonic-gate static void 12397c478bd9Sstevel@tonic-gate kmem_error(int error, kmem_cache_t *cparg, void *bufarg) 12407c478bd9Sstevel@tonic-gate { 12417c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = NULL; 12427c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp = NULL; 12437c478bd9Sstevel@tonic-gate kmem_cache_t *cp = cparg; 12447c478bd9Sstevel@tonic-gate kmem_slab_t *sp; 12457c478bd9Sstevel@tonic-gate uint64_t *off; 12467c478bd9Sstevel@tonic-gate void *buf = bufarg; 12477c478bd9Sstevel@tonic-gate 12487c478bd9Sstevel@tonic-gate kmem_logging = 0; /* stop logging when a bad thing happens */ 12497c478bd9Sstevel@tonic-gate 12507c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_timestamp = gethrtime(); 12517c478bd9Sstevel@tonic-gate 12527c478bd9Sstevel@tonic-gate sp = kmem_findslab(cp, buf); 12537c478bd9Sstevel@tonic-gate if (sp == NULL) { 1254b5fca8f8Stomee for (cp = list_tail(&kmem_caches); cp != NULL; 1255b5fca8f8Stomee cp = list_prev(&kmem_caches, cp)) { 12567c478bd9Sstevel@tonic-gate if ((sp = kmem_findslab(cp, buf)) != NULL) 12577c478bd9Sstevel@tonic-gate break; 12587c478bd9Sstevel@tonic-gate } 12597c478bd9Sstevel@tonic-gate } 12607c478bd9Sstevel@tonic-gate 12617c478bd9Sstevel@tonic-gate if (sp == NULL) { 12627c478bd9Sstevel@tonic-gate cp = NULL; 12637c478bd9Sstevel@tonic-gate error = KMERR_BADADDR; 12647c478bd9Sstevel@tonic-gate } else { 12657c478bd9Sstevel@tonic-gate if (cp != cparg) 12667c478bd9Sstevel@tonic-gate error = KMERR_BADCACHE; 12677c478bd9Sstevel@tonic-gate else 12687c478bd9Sstevel@tonic-gate buf = (char *)bufarg - ((uintptr_t)bufarg - 12697c478bd9Sstevel@tonic-gate (uintptr_t)sp->slab_base) % cp->cache_chunksize; 12707c478bd9Sstevel@tonic-gate if (buf != bufarg) 12717c478bd9Sstevel@tonic-gate error = KMERR_BADBASE; 12727c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_BUFTAG) 12737c478bd9Sstevel@tonic-gate btp = KMEM_BUFTAG(cp, buf); 12747c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) { 12757c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 12767c478bd9Sstevel@tonic-gate for (bcp = *KMEM_HASH(cp, buf); bcp; bcp = bcp->bc_next) 12777c478bd9Sstevel@tonic-gate if (bcp->bc_addr == buf) 12787c478bd9Sstevel@tonic-gate break; 12797c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 12807c478bd9Sstevel@tonic-gate if (bcp == NULL && btp != NULL) 12817c478bd9Sstevel@tonic-gate bcp = btp->bt_bufctl; 12827c478bd9Sstevel@tonic-gate if (kmem_findslab(cp->cache_bufctl_cache, bcp) == 12837c478bd9Sstevel@tonic-gate NULL || P2PHASE((uintptr_t)bcp, KMEM_ALIGN) || 12847c478bd9Sstevel@tonic-gate bcp->bc_addr != buf) { 12857c478bd9Sstevel@tonic-gate error = KMERR_BADBUFCTL; 12867c478bd9Sstevel@tonic-gate bcp = NULL; 12877c478bd9Sstevel@tonic-gate } 12887c478bd9Sstevel@tonic-gate } 12897c478bd9Sstevel@tonic-gate } 12907c478bd9Sstevel@tonic-gate 12917c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_error = error; 12927c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_buffer = bufarg; 12937c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_realbuf = buf; 12947c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_cache = cparg; 12957c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_realcache = cp; 12967c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_slab = sp; 12977c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_bufctl = bcp; 12987c478bd9Sstevel@tonic-gate 12997c478bd9Sstevel@tonic-gate printf("kernel memory allocator: "); 13007c478bd9Sstevel@tonic-gate 13017c478bd9Sstevel@tonic-gate switch (error) { 13027c478bd9Sstevel@tonic-gate 13037c478bd9Sstevel@tonic-gate case KMERR_MODIFIED: 13047c478bd9Sstevel@tonic-gate printf("buffer modified after being freed\n"); 13057c478bd9Sstevel@tonic-gate off = verify_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify); 13067c478bd9Sstevel@tonic-gate if (off == NULL) /* shouldn't happen */ 13077c478bd9Sstevel@tonic-gate off = buf; 13087c478bd9Sstevel@tonic-gate printf("modification occurred at offset 0x%lx " 13097c478bd9Sstevel@tonic-gate "(0x%llx replaced by 0x%llx)\n", 13107c478bd9Sstevel@tonic-gate (uintptr_t)off - (uintptr_t)buf, 13117c478bd9Sstevel@tonic-gate (longlong_t)KMEM_FREE_PATTERN, (longlong_t)*off); 13127c478bd9Sstevel@tonic-gate break; 13137c478bd9Sstevel@tonic-gate 13147c478bd9Sstevel@tonic-gate case KMERR_REDZONE: 13157c478bd9Sstevel@tonic-gate printf("redzone violation: write past end of buffer\n"); 13167c478bd9Sstevel@tonic-gate break; 13177c478bd9Sstevel@tonic-gate 13187c478bd9Sstevel@tonic-gate case KMERR_BADADDR: 13197c478bd9Sstevel@tonic-gate printf("invalid free: buffer not in cache\n"); 13207c478bd9Sstevel@tonic-gate break; 13217c478bd9Sstevel@tonic-gate 13227c478bd9Sstevel@tonic-gate case KMERR_DUPFREE: 13237c478bd9Sstevel@tonic-gate printf("duplicate free: buffer freed twice\n"); 13247c478bd9Sstevel@tonic-gate break; 13257c478bd9Sstevel@tonic-gate 13267c478bd9Sstevel@tonic-gate case KMERR_BADBUFTAG: 13277c478bd9Sstevel@tonic-gate printf("boundary tag corrupted\n"); 13287c478bd9Sstevel@tonic-gate printf("bcp ^ bxstat = %lx, should be %lx\n", 13297c478bd9Sstevel@tonic-gate (intptr_t)btp->bt_bufctl ^ btp->bt_bxstat, 13307c478bd9Sstevel@tonic-gate KMEM_BUFTAG_FREE); 13317c478bd9Sstevel@tonic-gate break; 13327c478bd9Sstevel@tonic-gate 13337c478bd9Sstevel@tonic-gate case KMERR_BADBUFCTL: 13347c478bd9Sstevel@tonic-gate printf("bufctl corrupted\n"); 13357c478bd9Sstevel@tonic-gate break; 13367c478bd9Sstevel@tonic-gate 13377c478bd9Sstevel@tonic-gate case KMERR_BADCACHE: 13387c478bd9Sstevel@tonic-gate printf("buffer freed to wrong cache\n"); 13397c478bd9Sstevel@tonic-gate printf("buffer was allocated from %s,\n", cp->cache_name); 13407c478bd9Sstevel@tonic-gate printf("caller attempting free to %s.\n", cparg->cache_name); 13417c478bd9Sstevel@tonic-gate break; 13427c478bd9Sstevel@tonic-gate 13437c478bd9Sstevel@tonic-gate case KMERR_BADSIZE: 13447c478bd9Sstevel@tonic-gate printf("bad free: free size (%u) != alloc size (%u)\n", 13457c478bd9Sstevel@tonic-gate KMEM_SIZE_DECODE(((uint32_t *)btp)[0]), 13467c478bd9Sstevel@tonic-gate KMEM_SIZE_DECODE(((uint32_t *)btp)[1])); 13477c478bd9Sstevel@tonic-gate break; 13487c478bd9Sstevel@tonic-gate 13497c478bd9Sstevel@tonic-gate case KMERR_BADBASE: 13507c478bd9Sstevel@tonic-gate printf("bad free: free address (%p) != alloc address (%p)\n", 13517c478bd9Sstevel@tonic-gate bufarg, buf); 13527c478bd9Sstevel@tonic-gate break; 13537c478bd9Sstevel@tonic-gate } 13547c478bd9Sstevel@tonic-gate 13557c478bd9Sstevel@tonic-gate printf("buffer=%p bufctl=%p cache: %s\n", 13567c478bd9Sstevel@tonic-gate bufarg, (void *)bcp, cparg->cache_name); 13577c478bd9Sstevel@tonic-gate 13587c478bd9Sstevel@tonic-gate if (bcp != NULL && (cp->cache_flags & KMF_AUDIT) && 13597c478bd9Sstevel@tonic-gate error != KMERR_BADBUFCTL) { 13607c478bd9Sstevel@tonic-gate int d; 13617c478bd9Sstevel@tonic-gate timestruc_t ts; 13627c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *bcap = (kmem_bufctl_audit_t *)bcp; 13637c478bd9Sstevel@tonic-gate 13647c478bd9Sstevel@tonic-gate hrt2ts(kmem_panic_info.kmp_timestamp - bcap->bc_timestamp, &ts); 13657c478bd9Sstevel@tonic-gate printf("previous transaction on buffer %p:\n", buf); 13667c478bd9Sstevel@tonic-gate printf("thread=%p time=T-%ld.%09ld slab=%p cache: %s\n", 13677c478bd9Sstevel@tonic-gate (void *)bcap->bc_thread, ts.tv_sec, ts.tv_nsec, 13687c478bd9Sstevel@tonic-gate (void *)sp, cp->cache_name); 13697c478bd9Sstevel@tonic-gate for (d = 0; d < MIN(bcap->bc_depth, KMEM_STACK_DEPTH); d++) { 13707c478bd9Sstevel@tonic-gate ulong_t off; 13717c478bd9Sstevel@tonic-gate char *sym = kobj_getsymname(bcap->bc_stack[d], &off); 13727c478bd9Sstevel@tonic-gate printf("%s+%lx\n", sym ? sym : "?", off); 13737c478bd9Sstevel@tonic-gate } 13747c478bd9Sstevel@tonic-gate } 13757c478bd9Sstevel@tonic-gate if (kmem_panic > 0) 13767c478bd9Sstevel@tonic-gate panic("kernel heap corruption detected"); 13777c478bd9Sstevel@tonic-gate if (kmem_panic == 0) 13787c478bd9Sstevel@tonic-gate debug_enter(NULL); 13797c478bd9Sstevel@tonic-gate kmem_logging = 1; /* resume logging */ 13807c478bd9Sstevel@tonic-gate } 13817c478bd9Sstevel@tonic-gate 13827c478bd9Sstevel@tonic-gate static kmem_log_header_t * 13837c478bd9Sstevel@tonic-gate kmem_log_init(size_t logsize) 13847c478bd9Sstevel@tonic-gate { 13857c478bd9Sstevel@tonic-gate kmem_log_header_t *lhp; 13867c478bd9Sstevel@tonic-gate int nchunks = 4 * max_ncpus; 13877c478bd9Sstevel@tonic-gate size_t lhsize = (size_t)&((kmem_log_header_t *)0)->lh_cpu[max_ncpus]; 13887c478bd9Sstevel@tonic-gate int i; 13897c478bd9Sstevel@tonic-gate 13907c478bd9Sstevel@tonic-gate /* 13917c478bd9Sstevel@tonic-gate * Make sure that lhp->lh_cpu[] is nicely aligned 13927c478bd9Sstevel@tonic-gate * to prevent false sharing of cache lines. 13937c478bd9Sstevel@tonic-gate */ 13947c478bd9Sstevel@tonic-gate lhsize = P2ROUNDUP(lhsize, KMEM_ALIGN); 13957c478bd9Sstevel@tonic-gate lhp = vmem_xalloc(kmem_log_arena, lhsize, 64, P2NPHASE(lhsize, 64), 0, 13967c478bd9Sstevel@tonic-gate NULL, NULL, VM_SLEEP); 13977c478bd9Sstevel@tonic-gate bzero(lhp, lhsize); 13987c478bd9Sstevel@tonic-gate 13997c478bd9Sstevel@tonic-gate mutex_init(&lhp->lh_lock, NULL, MUTEX_DEFAULT, NULL); 14007c478bd9Sstevel@tonic-gate lhp->lh_nchunks = nchunks; 14017c478bd9Sstevel@tonic-gate lhp->lh_chunksize = P2ROUNDUP(logsize / nchunks + 1, PAGESIZE); 14027c478bd9Sstevel@tonic-gate lhp->lh_base = vmem_alloc(kmem_log_arena, 14037c478bd9Sstevel@tonic-gate lhp->lh_chunksize * nchunks, VM_SLEEP); 14047c478bd9Sstevel@tonic-gate lhp->lh_free = vmem_alloc(kmem_log_arena, 14057c478bd9Sstevel@tonic-gate nchunks * sizeof (int), VM_SLEEP); 14067c478bd9Sstevel@tonic-gate bzero(lhp->lh_base, lhp->lh_chunksize * nchunks); 14077c478bd9Sstevel@tonic-gate 14087c478bd9Sstevel@tonic-gate for (i = 0; i < max_ncpus; i++) { 14097c478bd9Sstevel@tonic-gate kmem_cpu_log_header_t *clhp = &lhp->lh_cpu[i]; 14107c478bd9Sstevel@tonic-gate mutex_init(&clhp->clh_lock, NULL, MUTEX_DEFAULT, NULL); 14117c478bd9Sstevel@tonic-gate clhp->clh_chunk = i; 14127c478bd9Sstevel@tonic-gate } 14137c478bd9Sstevel@tonic-gate 14147c478bd9Sstevel@tonic-gate for (i = max_ncpus; i < nchunks; i++) 14157c478bd9Sstevel@tonic-gate lhp->lh_free[i] = i; 14167c478bd9Sstevel@tonic-gate 14177c478bd9Sstevel@tonic-gate lhp->lh_head = max_ncpus; 14187c478bd9Sstevel@tonic-gate lhp->lh_tail = 0; 14197c478bd9Sstevel@tonic-gate 14207c478bd9Sstevel@tonic-gate return (lhp); 14217c478bd9Sstevel@tonic-gate } 14227c478bd9Sstevel@tonic-gate 14237c478bd9Sstevel@tonic-gate static void * 14247c478bd9Sstevel@tonic-gate kmem_log_enter(kmem_log_header_t *lhp, void *data, size_t size) 14257c478bd9Sstevel@tonic-gate { 14267c478bd9Sstevel@tonic-gate void *logspace; 14277c478bd9Sstevel@tonic-gate kmem_cpu_log_header_t *clhp = &lhp->lh_cpu[CPU->cpu_seqid]; 14287c478bd9Sstevel@tonic-gate 14297c478bd9Sstevel@tonic-gate if (lhp == NULL || kmem_logging == 0 || panicstr) 14307c478bd9Sstevel@tonic-gate return (NULL); 14317c478bd9Sstevel@tonic-gate 14327c478bd9Sstevel@tonic-gate mutex_enter(&clhp->clh_lock); 14337c478bd9Sstevel@tonic-gate clhp->clh_hits++; 14347c478bd9Sstevel@tonic-gate if (size > clhp->clh_avail) { 14357c478bd9Sstevel@tonic-gate mutex_enter(&lhp->lh_lock); 14367c478bd9Sstevel@tonic-gate lhp->lh_hits++; 14377c478bd9Sstevel@tonic-gate lhp->lh_free[lhp->lh_tail] = clhp->clh_chunk; 14387c478bd9Sstevel@tonic-gate lhp->lh_tail = (lhp->lh_tail + 1) % lhp->lh_nchunks; 14397c478bd9Sstevel@tonic-gate clhp->clh_chunk = lhp->lh_free[lhp->lh_head]; 14407c478bd9Sstevel@tonic-gate lhp->lh_head = (lhp->lh_head + 1) % lhp->lh_nchunks; 14417c478bd9Sstevel@tonic-gate clhp->clh_current = lhp->lh_base + 14429f1b636aStomee clhp->clh_chunk * lhp->lh_chunksize; 14437c478bd9Sstevel@tonic-gate clhp->clh_avail = lhp->lh_chunksize; 14447c478bd9Sstevel@tonic-gate if (size > lhp->lh_chunksize) 14457c478bd9Sstevel@tonic-gate size = lhp->lh_chunksize; 14467c478bd9Sstevel@tonic-gate mutex_exit(&lhp->lh_lock); 14477c478bd9Sstevel@tonic-gate } 14487c478bd9Sstevel@tonic-gate logspace = clhp->clh_current; 14497c478bd9Sstevel@tonic-gate clhp->clh_current += size; 14507c478bd9Sstevel@tonic-gate clhp->clh_avail -= size; 14517c478bd9Sstevel@tonic-gate bcopy(data, logspace, size); 14527c478bd9Sstevel@tonic-gate mutex_exit(&clhp->clh_lock); 14537c478bd9Sstevel@tonic-gate return (logspace); 14547c478bd9Sstevel@tonic-gate } 14557c478bd9Sstevel@tonic-gate 14567c478bd9Sstevel@tonic-gate #define KMEM_AUDIT(lp, cp, bcp) \ 14577c478bd9Sstevel@tonic-gate { \ 14587c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *_bcp = (kmem_bufctl_audit_t *)(bcp); \ 14597c478bd9Sstevel@tonic-gate _bcp->bc_timestamp = gethrtime(); \ 14607c478bd9Sstevel@tonic-gate _bcp->bc_thread = curthread; \ 14617c478bd9Sstevel@tonic-gate _bcp->bc_depth = getpcstack(_bcp->bc_stack, KMEM_STACK_DEPTH); \ 14627c478bd9Sstevel@tonic-gate _bcp->bc_lastlog = kmem_log_enter((lp), _bcp, sizeof (*_bcp)); \ 14637c478bd9Sstevel@tonic-gate } 14647c478bd9Sstevel@tonic-gate 14657c478bd9Sstevel@tonic-gate static void 14667c478bd9Sstevel@tonic-gate kmem_log_event(kmem_log_header_t *lp, kmem_cache_t *cp, 14671c207ae9SMatthew Ahrens kmem_slab_t *sp, void *addr) 14687c478bd9Sstevel@tonic-gate { 14697c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t bca; 14707c478bd9Sstevel@tonic-gate 14717c478bd9Sstevel@tonic-gate bzero(&bca, sizeof (kmem_bufctl_audit_t)); 14727c478bd9Sstevel@tonic-gate bca.bc_addr = addr; 14737c478bd9Sstevel@tonic-gate bca.bc_slab = sp; 14747c478bd9Sstevel@tonic-gate bca.bc_cache = cp; 14757c478bd9Sstevel@tonic-gate KMEM_AUDIT(lp, cp, &bca); 14767c478bd9Sstevel@tonic-gate } 14777c478bd9Sstevel@tonic-gate 14787c478bd9Sstevel@tonic-gate /* 14797c478bd9Sstevel@tonic-gate * Create a new slab for cache cp. 14807c478bd9Sstevel@tonic-gate */ 14817c478bd9Sstevel@tonic-gate static kmem_slab_t * 14827c478bd9Sstevel@tonic-gate kmem_slab_create(kmem_cache_t *cp, int kmflag) 14837c478bd9Sstevel@tonic-gate { 14847c478bd9Sstevel@tonic-gate size_t slabsize = cp->cache_slabsize; 14857c478bd9Sstevel@tonic-gate size_t chunksize = cp->cache_chunksize; 14867c478bd9Sstevel@tonic-gate int cache_flags = cp->cache_flags; 14877c478bd9Sstevel@tonic-gate size_t color, chunks; 14887c478bd9Sstevel@tonic-gate char *buf, *slab; 14897c478bd9Sstevel@tonic-gate kmem_slab_t *sp; 14907c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp; 14917c478bd9Sstevel@tonic-gate vmem_t *vmp = cp->cache_arena; 14927c478bd9Sstevel@tonic-gate 1493b5fca8f8Stomee ASSERT(MUTEX_NOT_HELD(&cp->cache_lock)); 1494b5fca8f8Stomee 14957c478bd9Sstevel@tonic-gate color = cp->cache_color + cp->cache_align; 14967c478bd9Sstevel@tonic-gate if (color > cp->cache_maxcolor) 14977c478bd9Sstevel@tonic-gate color = cp->cache_mincolor; 14987c478bd9Sstevel@tonic-gate cp->cache_color = color; 14997c478bd9Sstevel@tonic-gate 15007c478bd9Sstevel@tonic-gate slab = vmem_alloc(vmp, slabsize, kmflag & KM_VMFLAGS); 15017c478bd9Sstevel@tonic-gate 15027c478bd9Sstevel@tonic-gate if (slab == NULL) 15037c478bd9Sstevel@tonic-gate goto vmem_alloc_failure; 15047c478bd9Sstevel@tonic-gate 15057c478bd9Sstevel@tonic-gate ASSERT(P2PHASE((uintptr_t)slab, vmp->vm_quantum) == 0); 15067c478bd9Sstevel@tonic-gate 1507b5fca8f8Stomee /* 1508b5fca8f8Stomee * Reverify what was already checked in kmem_cache_set_move(), since the 1509b5fca8f8Stomee * consolidator depends (for correctness) on slabs being initialized 1510b5fca8f8Stomee * with the 0xbaddcafe memory pattern (setting a low order bit usable by 1511b5fca8f8Stomee * clients to distinguish uninitialized memory from known objects). 1512b5fca8f8Stomee */ 1513b5fca8f8Stomee ASSERT((cp->cache_move == NULL) || !(cp->cache_cflags & KMC_NOTOUCH)); 15147c478bd9Sstevel@tonic-gate if (!(cp->cache_cflags & KMC_NOTOUCH)) 15157c478bd9Sstevel@tonic-gate copy_pattern(KMEM_UNINITIALIZED_PATTERN, slab, slabsize); 15167c478bd9Sstevel@tonic-gate 15177c478bd9Sstevel@tonic-gate if (cache_flags & KMF_HASH) { 15187c478bd9Sstevel@tonic-gate if ((sp = kmem_cache_alloc(kmem_slab_cache, kmflag)) == NULL) 15197c478bd9Sstevel@tonic-gate goto slab_alloc_failure; 15207c478bd9Sstevel@tonic-gate chunks = (slabsize - color) / chunksize; 15217c478bd9Sstevel@tonic-gate } else { 15227c478bd9Sstevel@tonic-gate sp = KMEM_SLAB(cp, slab); 15237c478bd9Sstevel@tonic-gate chunks = (slabsize - sizeof (kmem_slab_t) - color) / chunksize; 15247c478bd9Sstevel@tonic-gate } 15257c478bd9Sstevel@tonic-gate 15267c478bd9Sstevel@tonic-gate sp->slab_cache = cp; 15277c478bd9Sstevel@tonic-gate sp->slab_head = NULL; 15287c478bd9Sstevel@tonic-gate sp->slab_refcnt = 0; 15297c478bd9Sstevel@tonic-gate sp->slab_base = buf = slab + color; 15307c478bd9Sstevel@tonic-gate sp->slab_chunks = chunks; 1531b5fca8f8Stomee sp->slab_stuck_offset = (uint32_t)-1; 1532b5fca8f8Stomee sp->slab_later_count = 0; 1533b5fca8f8Stomee sp->slab_flags = 0; 15347c478bd9Sstevel@tonic-gate 15357c478bd9Sstevel@tonic-gate ASSERT(chunks > 0); 15367c478bd9Sstevel@tonic-gate while (chunks-- != 0) { 15377c478bd9Sstevel@tonic-gate if (cache_flags & KMF_HASH) { 15387c478bd9Sstevel@tonic-gate bcp = kmem_cache_alloc(cp->cache_bufctl_cache, kmflag); 15397c478bd9Sstevel@tonic-gate if (bcp == NULL) 15407c478bd9Sstevel@tonic-gate goto bufctl_alloc_failure; 15417c478bd9Sstevel@tonic-gate if (cache_flags & KMF_AUDIT) { 15427c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *bcap = 15437c478bd9Sstevel@tonic-gate (kmem_bufctl_audit_t *)bcp; 15447c478bd9Sstevel@tonic-gate bzero(bcap, sizeof (kmem_bufctl_audit_t)); 15457c478bd9Sstevel@tonic-gate bcap->bc_cache = cp; 15467c478bd9Sstevel@tonic-gate } 15477c478bd9Sstevel@tonic-gate bcp->bc_addr = buf; 15487c478bd9Sstevel@tonic-gate bcp->bc_slab = sp; 15497c478bd9Sstevel@tonic-gate } else { 15507c478bd9Sstevel@tonic-gate bcp = KMEM_BUFCTL(cp, buf); 15517c478bd9Sstevel@tonic-gate } 15527c478bd9Sstevel@tonic-gate if (cache_flags & KMF_BUFTAG) { 15537c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 15547c478bd9Sstevel@tonic-gate btp->bt_redzone = KMEM_REDZONE_PATTERN; 15557c478bd9Sstevel@tonic-gate btp->bt_bufctl = bcp; 15567c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE; 15577c478bd9Sstevel@tonic-gate if (cache_flags & KMF_DEADBEEF) { 15587c478bd9Sstevel@tonic-gate copy_pattern(KMEM_FREE_PATTERN, buf, 15597c478bd9Sstevel@tonic-gate cp->cache_verify); 15607c478bd9Sstevel@tonic-gate } 15617c478bd9Sstevel@tonic-gate } 15627c478bd9Sstevel@tonic-gate bcp->bc_next = sp->slab_head; 15637c478bd9Sstevel@tonic-gate sp->slab_head = bcp; 15647c478bd9Sstevel@tonic-gate buf += chunksize; 15657c478bd9Sstevel@tonic-gate } 15667c478bd9Sstevel@tonic-gate 15677c478bd9Sstevel@tonic-gate kmem_log_event(kmem_slab_log, cp, sp, slab); 15687c478bd9Sstevel@tonic-gate 15697c478bd9Sstevel@tonic-gate return (sp); 15707c478bd9Sstevel@tonic-gate 15717c478bd9Sstevel@tonic-gate bufctl_alloc_failure: 15727c478bd9Sstevel@tonic-gate 15737c478bd9Sstevel@tonic-gate while ((bcp = sp->slab_head) != NULL) { 15747c478bd9Sstevel@tonic-gate sp->slab_head = bcp->bc_next; 15757c478bd9Sstevel@tonic-gate kmem_cache_free(cp->cache_bufctl_cache, bcp); 15767c478bd9Sstevel@tonic-gate } 15777c478bd9Sstevel@tonic-gate kmem_cache_free(kmem_slab_cache, sp); 15787c478bd9Sstevel@tonic-gate 15797c478bd9Sstevel@tonic-gate slab_alloc_failure: 15807c478bd9Sstevel@tonic-gate 15817c478bd9Sstevel@tonic-gate vmem_free(vmp, slab, slabsize); 15827c478bd9Sstevel@tonic-gate 15837c478bd9Sstevel@tonic-gate vmem_alloc_failure: 15847c478bd9Sstevel@tonic-gate 15857c478bd9Sstevel@tonic-gate kmem_log_event(kmem_failure_log, cp, NULL, NULL); 15861a5e258fSJosef 'Jeff' Sipek atomic_inc_64(&cp->cache_alloc_fail); 15877c478bd9Sstevel@tonic-gate 15887c478bd9Sstevel@tonic-gate return (NULL); 15897c478bd9Sstevel@tonic-gate } 15907c478bd9Sstevel@tonic-gate 15917c478bd9Sstevel@tonic-gate /* 15927c478bd9Sstevel@tonic-gate * Destroy a slab. 15937c478bd9Sstevel@tonic-gate */ 15947c478bd9Sstevel@tonic-gate static void 15957c478bd9Sstevel@tonic-gate kmem_slab_destroy(kmem_cache_t *cp, kmem_slab_t *sp) 15967c478bd9Sstevel@tonic-gate { 15977c478bd9Sstevel@tonic-gate vmem_t *vmp = cp->cache_arena; 15987c478bd9Sstevel@tonic-gate void *slab = (void *)P2ALIGN((uintptr_t)sp->slab_base, vmp->vm_quantum); 15997c478bd9Sstevel@tonic-gate 1600b5fca8f8Stomee ASSERT(MUTEX_NOT_HELD(&cp->cache_lock)); 1601b5fca8f8Stomee ASSERT(sp->slab_refcnt == 0); 1602b5fca8f8Stomee 16037c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) { 16047c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp; 16057c478bd9Sstevel@tonic-gate while ((bcp = sp->slab_head) != NULL) { 16067c478bd9Sstevel@tonic-gate sp->slab_head = bcp->bc_next; 16077c478bd9Sstevel@tonic-gate kmem_cache_free(cp->cache_bufctl_cache, bcp); 16087c478bd9Sstevel@tonic-gate } 16097c478bd9Sstevel@tonic-gate kmem_cache_free(kmem_slab_cache, sp); 16107c478bd9Sstevel@tonic-gate } 16117c478bd9Sstevel@tonic-gate vmem_free(vmp, slab, cp->cache_slabsize); 16127c478bd9Sstevel@tonic-gate } 16137c478bd9Sstevel@tonic-gate 16147c478bd9Sstevel@tonic-gate static void * 1615b942e89bSDavid Valin kmem_slab_alloc_impl(kmem_cache_t *cp, kmem_slab_t *sp, boolean_t prefill) 16167c478bd9Sstevel@tonic-gate { 16177c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp, **hash_bucket; 16187c478bd9Sstevel@tonic-gate void *buf; 1619b942e89bSDavid Valin boolean_t new_slab = (sp->slab_refcnt == 0); 16207c478bd9Sstevel@tonic-gate 1621b5fca8f8Stomee ASSERT(MUTEX_HELD(&cp->cache_lock)); 1622b5fca8f8Stomee /* 1623b5fca8f8Stomee * kmem_slab_alloc() drops cache_lock when it creates a new slab, so we 1624b5fca8f8Stomee * can't ASSERT(avl_is_empty(&cp->cache_partial_slabs)) here when the 1625b942e89bSDavid Valin * slab is newly created. 1626b5fca8f8Stomee */ 1627b942e89bSDavid Valin ASSERT(new_slab || (KMEM_SLAB_IS_PARTIAL(sp) && 1628b5fca8f8Stomee (sp == avl_first(&cp->cache_partial_slabs)))); 16297c478bd9Sstevel@tonic-gate ASSERT(sp->slab_cache == cp); 16307c478bd9Sstevel@tonic-gate 1631b5fca8f8Stomee cp->cache_slab_alloc++; 16329f1b636aStomee cp->cache_bufslab--; 16337c478bd9Sstevel@tonic-gate sp->slab_refcnt++; 16347c478bd9Sstevel@tonic-gate 16357c478bd9Sstevel@tonic-gate bcp = sp->slab_head; 1636b942e89bSDavid Valin sp->slab_head = bcp->bc_next; 16377c478bd9Sstevel@tonic-gate 16387c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) { 16397c478bd9Sstevel@tonic-gate /* 16407c478bd9Sstevel@tonic-gate * Add buffer to allocated-address hash table. 16417c478bd9Sstevel@tonic-gate */ 16427c478bd9Sstevel@tonic-gate buf = bcp->bc_addr; 16437c478bd9Sstevel@tonic-gate hash_bucket = KMEM_HASH(cp, buf); 16447c478bd9Sstevel@tonic-gate bcp->bc_next = *hash_bucket; 16457c478bd9Sstevel@tonic-gate *hash_bucket = bcp; 16467c478bd9Sstevel@tonic-gate if ((cp->cache_flags & (KMF_AUDIT | KMF_BUFTAG)) == KMF_AUDIT) { 16477c478bd9Sstevel@tonic-gate KMEM_AUDIT(kmem_transaction_log, cp, bcp); 16487c478bd9Sstevel@tonic-gate } 16497c478bd9Sstevel@tonic-gate } else { 16507c478bd9Sstevel@tonic-gate buf = KMEM_BUF(cp, bcp); 16517c478bd9Sstevel@tonic-gate } 16527c478bd9Sstevel@tonic-gate 16537c478bd9Sstevel@tonic-gate ASSERT(KMEM_SLAB_MEMBER(sp, buf)); 1654b942e89bSDavid Valin 1655b942e89bSDavid Valin if (sp->slab_head == NULL) { 1656b942e89bSDavid Valin ASSERT(KMEM_SLAB_IS_ALL_USED(sp)); 1657b942e89bSDavid Valin if (new_slab) { 1658b942e89bSDavid Valin ASSERT(sp->slab_chunks == 1); 1659b942e89bSDavid Valin } else { 1660b942e89bSDavid Valin ASSERT(sp->slab_chunks > 1); /* the slab was partial */ 1661b942e89bSDavid Valin avl_remove(&cp->cache_partial_slabs, sp); 1662b942e89bSDavid Valin sp->slab_later_count = 0; /* clear history */ 1663b942e89bSDavid Valin sp->slab_flags &= ~KMEM_SLAB_NOMOVE; 1664b942e89bSDavid Valin sp->slab_stuck_offset = (uint32_t)-1; 1665b942e89bSDavid Valin } 1666b942e89bSDavid Valin list_insert_head(&cp->cache_complete_slabs, sp); 1667b942e89bSDavid Valin cp->cache_complete_slab_count++; 1668b942e89bSDavid Valin return (buf); 1669b942e89bSDavid Valin } 1670b942e89bSDavid Valin 1671b942e89bSDavid Valin ASSERT(KMEM_SLAB_IS_PARTIAL(sp)); 1672b942e89bSDavid Valin /* 1673b942e89bSDavid Valin * Peek to see if the magazine layer is enabled before 1674b942e89bSDavid Valin * we prefill. We're not holding the cpu cache lock, 1675b942e89bSDavid Valin * so the peek could be wrong, but there's no harm in it. 1676b942e89bSDavid Valin */ 1677b942e89bSDavid Valin if (new_slab && prefill && (cp->cache_flags & KMF_PREFILL) && 1678b942e89bSDavid Valin (KMEM_CPU_CACHE(cp)->cc_magsize != 0)) { 1679b942e89bSDavid Valin kmem_slab_prefill(cp, sp); 1680b942e89bSDavid Valin return (buf); 1681b942e89bSDavid Valin } 1682b942e89bSDavid Valin 1683b942e89bSDavid Valin if (new_slab) { 1684b942e89bSDavid Valin avl_add(&cp->cache_partial_slabs, sp); 1685b942e89bSDavid Valin return (buf); 1686b942e89bSDavid Valin } 1687b942e89bSDavid Valin 1688b942e89bSDavid Valin /* 1689b942e89bSDavid Valin * The slab is now more allocated than it was, so the 1690b942e89bSDavid Valin * order remains unchanged. 1691b942e89bSDavid Valin */ 1692b942e89bSDavid Valin ASSERT(!avl_update(&cp->cache_partial_slabs, sp)); 1693b5fca8f8Stomee return (buf); 1694b5fca8f8Stomee } 1695b5fca8f8Stomee 1696b5fca8f8Stomee /* 1697b5fca8f8Stomee * Allocate a raw (unconstructed) buffer from cp's slab layer. 1698b5fca8f8Stomee */ 1699b5fca8f8Stomee static void * 1700b5fca8f8Stomee kmem_slab_alloc(kmem_cache_t *cp, int kmflag) 1701b5fca8f8Stomee { 1702b5fca8f8Stomee kmem_slab_t *sp; 1703b5fca8f8Stomee void *buf; 17044d4c4c43STom Erickson boolean_t test_destructor; 1705b5fca8f8Stomee 1706b5fca8f8Stomee mutex_enter(&cp->cache_lock); 17074d4c4c43STom Erickson test_destructor = (cp->cache_slab_alloc == 0); 1708b5fca8f8Stomee sp = avl_first(&cp->cache_partial_slabs); 1709b5fca8f8Stomee if (sp == NULL) { 1710b5fca8f8Stomee ASSERT(cp->cache_bufslab == 0); 1711b5fca8f8Stomee 1712b5fca8f8Stomee /* 1713b5fca8f8Stomee * The freelist is empty. Create a new slab. 1714b5fca8f8Stomee */ 1715b5fca8f8Stomee mutex_exit(&cp->cache_lock); 1716b5fca8f8Stomee if ((sp = kmem_slab_create(cp, kmflag)) == NULL) { 1717b5fca8f8Stomee return (NULL); 1718b5fca8f8Stomee } 1719b5fca8f8Stomee mutex_enter(&cp->cache_lock); 1720b5fca8f8Stomee cp->cache_slab_create++; 1721b5fca8f8Stomee if ((cp->cache_buftotal += sp->slab_chunks) > cp->cache_bufmax) 1722b5fca8f8Stomee cp->cache_bufmax = cp->cache_buftotal; 1723b5fca8f8Stomee cp->cache_bufslab += sp->slab_chunks; 1724b5fca8f8Stomee } 17257c478bd9Sstevel@tonic-gate 1726b942e89bSDavid Valin buf = kmem_slab_alloc_impl(cp, sp, B_TRUE); 1727b5fca8f8Stomee ASSERT((cp->cache_slab_create - cp->cache_slab_destroy) == 1728b5fca8f8Stomee (cp->cache_complete_slab_count + 1729b5fca8f8Stomee avl_numnodes(&cp->cache_partial_slabs) + 1730b5fca8f8Stomee (cp->cache_defrag == NULL ? 0 : cp->cache_defrag->kmd_deadcount))); 17317c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 17327c478bd9Sstevel@tonic-gate 17334d4c4c43STom Erickson if (test_destructor && cp->cache_destructor != NULL) { 17344d4c4c43STom Erickson /* 17354d4c4c43STom Erickson * On the first kmem_slab_alloc(), assert that it is valid to 17364d4c4c43STom Erickson * call the destructor on a newly constructed object without any 17374d4c4c43STom Erickson * client involvement. 17384d4c4c43STom Erickson */ 17394d4c4c43STom Erickson if ((cp->cache_constructor == NULL) || 17404d4c4c43STom Erickson cp->cache_constructor(buf, cp->cache_private, 17414d4c4c43STom Erickson kmflag) == 0) { 17424d4c4c43STom Erickson cp->cache_destructor(buf, cp->cache_private); 17434d4c4c43STom Erickson } 17444d4c4c43STom Erickson copy_pattern(KMEM_UNINITIALIZED_PATTERN, buf, 17454d4c4c43STom Erickson cp->cache_bufsize); 17464d4c4c43STom Erickson if (cp->cache_flags & KMF_DEADBEEF) { 17474d4c4c43STom Erickson copy_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify); 17484d4c4c43STom Erickson } 17494d4c4c43STom Erickson } 17504d4c4c43STom Erickson 17517c478bd9Sstevel@tonic-gate return (buf); 17527c478bd9Sstevel@tonic-gate } 17537c478bd9Sstevel@tonic-gate 1754b5fca8f8Stomee static void kmem_slab_move_yes(kmem_cache_t *, kmem_slab_t *, void *); 1755b5fca8f8Stomee 17567c478bd9Sstevel@tonic-gate /* 17577c478bd9Sstevel@tonic-gate * Free a raw (unconstructed) buffer to cp's slab layer. 17587c478bd9Sstevel@tonic-gate */ 17597c478bd9Sstevel@tonic-gate static void 17607c478bd9Sstevel@tonic-gate kmem_slab_free(kmem_cache_t *cp, void *buf) 17617c478bd9Sstevel@tonic-gate { 17627c478bd9Sstevel@tonic-gate kmem_slab_t *sp; 17637c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp, **prev_bcpp; 17647c478bd9Sstevel@tonic-gate 17657c478bd9Sstevel@tonic-gate ASSERT(buf != NULL); 17667c478bd9Sstevel@tonic-gate 17677c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 17687c478bd9Sstevel@tonic-gate cp->cache_slab_free++; 17697c478bd9Sstevel@tonic-gate 17707c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) { 17717c478bd9Sstevel@tonic-gate /* 17727c478bd9Sstevel@tonic-gate * Look up buffer in allocated-address hash table. 17737c478bd9Sstevel@tonic-gate */ 17747c478bd9Sstevel@tonic-gate prev_bcpp = KMEM_HASH(cp, buf); 17757c478bd9Sstevel@tonic-gate while ((bcp = *prev_bcpp) != NULL) { 17767c478bd9Sstevel@tonic-gate if (bcp->bc_addr == buf) { 17777c478bd9Sstevel@tonic-gate *prev_bcpp = bcp->bc_next; 17787c478bd9Sstevel@tonic-gate sp = bcp->bc_slab; 17797c478bd9Sstevel@tonic-gate break; 17807c478bd9Sstevel@tonic-gate } 17817c478bd9Sstevel@tonic-gate cp->cache_lookup_depth++; 17827c478bd9Sstevel@tonic-gate prev_bcpp = &bcp->bc_next; 17837c478bd9Sstevel@tonic-gate } 17847c478bd9Sstevel@tonic-gate } else { 17857c478bd9Sstevel@tonic-gate bcp = KMEM_BUFCTL(cp, buf); 17867c478bd9Sstevel@tonic-gate sp = KMEM_SLAB(cp, buf); 17877c478bd9Sstevel@tonic-gate } 17887c478bd9Sstevel@tonic-gate 17897c478bd9Sstevel@tonic-gate if (bcp == NULL || sp->slab_cache != cp || !KMEM_SLAB_MEMBER(sp, buf)) { 17907c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 17917c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADADDR, cp, buf); 17927c478bd9Sstevel@tonic-gate return; 17937c478bd9Sstevel@tonic-gate } 17947c478bd9Sstevel@tonic-gate 1795b5fca8f8Stomee if (KMEM_SLAB_OFFSET(sp, buf) == sp->slab_stuck_offset) { 1796b5fca8f8Stomee /* 1797b5fca8f8Stomee * If this is the buffer that prevented the consolidator from 1798b5fca8f8Stomee * clearing the slab, we can reset the slab flags now that the 1799b5fca8f8Stomee * buffer is freed. (It makes sense to do this in 1800b5fca8f8Stomee * kmem_cache_free(), where the client gives up ownership of the 1801b5fca8f8Stomee * buffer, but on the hot path the test is too expensive.) 1802b5fca8f8Stomee */ 1803b5fca8f8Stomee kmem_slab_move_yes(cp, sp, buf); 1804b5fca8f8Stomee } 1805b5fca8f8Stomee 18067c478bd9Sstevel@tonic-gate if ((cp->cache_flags & (KMF_AUDIT | KMF_BUFTAG)) == KMF_AUDIT) { 18077c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_CONTENTS) 18087c478bd9Sstevel@tonic-gate ((kmem_bufctl_audit_t *)bcp)->bc_contents = 18097c478bd9Sstevel@tonic-gate kmem_log_enter(kmem_content_log, buf, 18109f1b636aStomee cp->cache_contents); 18117c478bd9Sstevel@tonic-gate KMEM_AUDIT(kmem_transaction_log, cp, bcp); 18127c478bd9Sstevel@tonic-gate } 18137c478bd9Sstevel@tonic-gate 18147c478bd9Sstevel@tonic-gate bcp->bc_next = sp->slab_head; 18157c478bd9Sstevel@tonic-gate sp->slab_head = bcp; 18167c478bd9Sstevel@tonic-gate 18179f1b636aStomee cp->cache_bufslab++; 18187c478bd9Sstevel@tonic-gate ASSERT(sp->slab_refcnt >= 1); 1819b5fca8f8Stomee 18207c478bd9Sstevel@tonic-gate if (--sp->slab_refcnt == 0) { 18217c478bd9Sstevel@tonic-gate /* 18227c478bd9Sstevel@tonic-gate * There are no outstanding allocations from this slab, 18237c478bd9Sstevel@tonic-gate * so we can reclaim the memory. 18247c478bd9Sstevel@tonic-gate */ 1825b5fca8f8Stomee if (sp->slab_chunks == 1) { 1826b5fca8f8Stomee list_remove(&cp->cache_complete_slabs, sp); 1827b5fca8f8Stomee cp->cache_complete_slab_count--; 1828b5fca8f8Stomee } else { 1829b5fca8f8Stomee avl_remove(&cp->cache_partial_slabs, sp); 1830b5fca8f8Stomee } 1831b5fca8f8Stomee 18327c478bd9Sstevel@tonic-gate cp->cache_buftotal -= sp->slab_chunks; 18339f1b636aStomee cp->cache_bufslab -= sp->slab_chunks; 1834b5fca8f8Stomee /* 1835b5fca8f8Stomee * Defer releasing the slab to the virtual memory subsystem 1836b5fca8f8Stomee * while there is a pending move callback, since we guarantee 1837b5fca8f8Stomee * that buffers passed to the move callback have only been 1838b5fca8f8Stomee * touched by kmem or by the client itself. Since the memory 1839b5fca8f8Stomee * patterns baddcafe (uninitialized) and deadbeef (freed) both 1840b5fca8f8Stomee * set at least one of the two lowest order bits, the client can 1841b5fca8f8Stomee * test those bits in the move callback to determine whether or 1842b5fca8f8Stomee * not it knows about the buffer (assuming that the client also 1843b5fca8f8Stomee * sets one of those low order bits whenever it frees a buffer). 1844b5fca8f8Stomee */ 1845b5fca8f8Stomee if (cp->cache_defrag == NULL || 1846b5fca8f8Stomee (avl_is_empty(&cp->cache_defrag->kmd_moves_pending) && 1847b5fca8f8Stomee !(sp->slab_flags & KMEM_SLAB_MOVE_PENDING))) { 1848b5fca8f8Stomee cp->cache_slab_destroy++; 1849b5fca8f8Stomee mutex_exit(&cp->cache_lock); 1850b5fca8f8Stomee kmem_slab_destroy(cp, sp); 1851b5fca8f8Stomee } else { 1852b5fca8f8Stomee list_t *deadlist = &cp->cache_defrag->kmd_deadlist; 1853b5fca8f8Stomee /* 1854b5fca8f8Stomee * Slabs are inserted at both ends of the deadlist to 1855b5fca8f8Stomee * distinguish between slabs freed while move callbacks 1856b5fca8f8Stomee * are pending (list head) and a slab freed while the 1857b5fca8f8Stomee * lock is dropped in kmem_move_buffers() (list tail) so 1858b5fca8f8Stomee * that in both cases slab_destroy() is called from the 1859b5fca8f8Stomee * right context. 1860b5fca8f8Stomee */ 1861b5fca8f8Stomee if (sp->slab_flags & KMEM_SLAB_MOVE_PENDING) { 1862b5fca8f8Stomee list_insert_tail(deadlist, sp); 1863b5fca8f8Stomee } else { 1864b5fca8f8Stomee list_insert_head(deadlist, sp); 1865b5fca8f8Stomee } 1866b5fca8f8Stomee cp->cache_defrag->kmd_deadcount++; 1867b5fca8f8Stomee mutex_exit(&cp->cache_lock); 1868b5fca8f8Stomee } 18697c478bd9Sstevel@tonic-gate return; 18707c478bd9Sstevel@tonic-gate } 1871b5fca8f8Stomee 1872b5fca8f8Stomee if (bcp->bc_next == NULL) { 1873b5fca8f8Stomee /* Transition the slab from completely allocated to partial. */ 1874b5fca8f8Stomee ASSERT(sp->slab_refcnt == (sp->slab_chunks - 1)); 1875b5fca8f8Stomee ASSERT(sp->slab_chunks > 1); 1876b5fca8f8Stomee list_remove(&cp->cache_complete_slabs, sp); 1877b5fca8f8Stomee cp->cache_complete_slab_count--; 1878b5fca8f8Stomee avl_add(&cp->cache_partial_slabs, sp); 1879b5fca8f8Stomee } else { 1880b5fca8f8Stomee (void) avl_update_gt(&cp->cache_partial_slabs, sp); 1881b5fca8f8Stomee } 1882b5fca8f8Stomee 1883b5fca8f8Stomee ASSERT((cp->cache_slab_create - cp->cache_slab_destroy) == 1884b5fca8f8Stomee (cp->cache_complete_slab_count + 1885b5fca8f8Stomee avl_numnodes(&cp->cache_partial_slabs) + 1886b5fca8f8Stomee (cp->cache_defrag == NULL ? 0 : cp->cache_defrag->kmd_deadcount))); 18877c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 18887c478bd9Sstevel@tonic-gate } 18897c478bd9Sstevel@tonic-gate 1890b5fca8f8Stomee /* 1891b5fca8f8Stomee * Return -1 if kmem_error, 1 if constructor fails, 0 if successful. 1892b5fca8f8Stomee */ 18937c478bd9Sstevel@tonic-gate static int 18947c478bd9Sstevel@tonic-gate kmem_cache_alloc_debug(kmem_cache_t *cp, void *buf, int kmflag, int construct, 18957c478bd9Sstevel@tonic-gate caddr_t caller) 18967c478bd9Sstevel@tonic-gate { 18977c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 18987c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *bcp = (kmem_bufctl_audit_t *)btp->bt_bufctl; 18997c478bd9Sstevel@tonic-gate uint32_t mtbf; 19007c478bd9Sstevel@tonic-gate 19017c478bd9Sstevel@tonic-gate if (btp->bt_bxstat != ((intptr_t)bcp ^ KMEM_BUFTAG_FREE)) { 19027c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADBUFTAG, cp, buf); 19037c478bd9Sstevel@tonic-gate return (-1); 19047c478bd9Sstevel@tonic-gate } 19057c478bd9Sstevel@tonic-gate 19067c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_ALLOC; 19077c478bd9Sstevel@tonic-gate 19087c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_HASH) && bcp->bc_addr != buf) { 19097c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADBUFCTL, cp, buf); 19107c478bd9Sstevel@tonic-gate return (-1); 19117c478bd9Sstevel@tonic-gate } 19127c478bd9Sstevel@tonic-gate 19137c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) { 19147c478bd9Sstevel@tonic-gate if (!construct && (cp->cache_flags & KMF_LITE)) { 19157c478bd9Sstevel@tonic-gate if (*(uint64_t *)buf != KMEM_FREE_PATTERN) { 19167c478bd9Sstevel@tonic-gate kmem_error(KMERR_MODIFIED, cp, buf); 19177c478bd9Sstevel@tonic-gate return (-1); 19187c478bd9Sstevel@tonic-gate } 19197c478bd9Sstevel@tonic-gate if (cp->cache_constructor != NULL) 19207c478bd9Sstevel@tonic-gate *(uint64_t *)buf = btp->bt_redzone; 19217c478bd9Sstevel@tonic-gate else 19227c478bd9Sstevel@tonic-gate *(uint64_t *)buf = KMEM_UNINITIALIZED_PATTERN; 19237c478bd9Sstevel@tonic-gate } else { 19247c478bd9Sstevel@tonic-gate construct = 1; 19257c478bd9Sstevel@tonic-gate if (verify_and_copy_pattern(KMEM_FREE_PATTERN, 19267c478bd9Sstevel@tonic-gate KMEM_UNINITIALIZED_PATTERN, buf, 19277c478bd9Sstevel@tonic-gate cp->cache_verify)) { 19287c478bd9Sstevel@tonic-gate kmem_error(KMERR_MODIFIED, cp, buf); 19297c478bd9Sstevel@tonic-gate return (-1); 19307c478bd9Sstevel@tonic-gate } 19317c478bd9Sstevel@tonic-gate } 19327c478bd9Sstevel@tonic-gate } 19337c478bd9Sstevel@tonic-gate btp->bt_redzone = KMEM_REDZONE_PATTERN; 19347c478bd9Sstevel@tonic-gate 19357c478bd9Sstevel@tonic-gate if ((mtbf = kmem_mtbf | cp->cache_mtbf) != 0 && 19367c478bd9Sstevel@tonic-gate gethrtime() % mtbf == 0 && 19377c478bd9Sstevel@tonic-gate (kmflag & (KM_NOSLEEP | KM_PANIC)) == KM_NOSLEEP) { 19387c478bd9Sstevel@tonic-gate kmem_log_event(kmem_failure_log, cp, NULL, NULL); 19397c478bd9Sstevel@tonic-gate if (!construct && cp->cache_destructor != NULL) 19407c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 19417c478bd9Sstevel@tonic-gate } else { 19427c478bd9Sstevel@tonic-gate mtbf = 0; 19437c478bd9Sstevel@tonic-gate } 19447c478bd9Sstevel@tonic-gate 19457c478bd9Sstevel@tonic-gate if (mtbf || (construct && cp->cache_constructor != NULL && 19467c478bd9Sstevel@tonic-gate cp->cache_constructor(buf, cp->cache_private, kmflag) != 0)) { 19471a5e258fSJosef 'Jeff' Sipek atomic_inc_64(&cp->cache_alloc_fail); 19487c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE; 19497c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) 19507c478bd9Sstevel@tonic-gate copy_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify); 19517c478bd9Sstevel@tonic-gate kmem_slab_free(cp, buf); 1952b5fca8f8Stomee return (1); 19537c478bd9Sstevel@tonic-gate } 19547c478bd9Sstevel@tonic-gate 19557c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_AUDIT) { 19567c478bd9Sstevel@tonic-gate KMEM_AUDIT(kmem_transaction_log, cp, bcp); 19577c478bd9Sstevel@tonic-gate } 19587c478bd9Sstevel@tonic-gate 19597c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_LITE) && 19607c478bd9Sstevel@tonic-gate !(cp->cache_cflags & KMC_KMEM_ALLOC)) { 19617c478bd9Sstevel@tonic-gate KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, caller); 19627c478bd9Sstevel@tonic-gate } 19637c478bd9Sstevel@tonic-gate 19647c478bd9Sstevel@tonic-gate return (0); 19657c478bd9Sstevel@tonic-gate } 19667c478bd9Sstevel@tonic-gate 19677c478bd9Sstevel@tonic-gate static int 19687c478bd9Sstevel@tonic-gate kmem_cache_free_debug(kmem_cache_t *cp, void *buf, caddr_t caller) 19697c478bd9Sstevel@tonic-gate { 19707c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 19717c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *bcp = (kmem_bufctl_audit_t *)btp->bt_bufctl; 19727c478bd9Sstevel@tonic-gate kmem_slab_t *sp; 19737c478bd9Sstevel@tonic-gate 19747c478bd9Sstevel@tonic-gate if (btp->bt_bxstat != ((intptr_t)bcp ^ KMEM_BUFTAG_ALLOC)) { 19757c478bd9Sstevel@tonic-gate if (btp->bt_bxstat == ((intptr_t)bcp ^ KMEM_BUFTAG_FREE)) { 19767c478bd9Sstevel@tonic-gate kmem_error(KMERR_DUPFREE, cp, buf); 19777c478bd9Sstevel@tonic-gate return (-1); 19787c478bd9Sstevel@tonic-gate } 19797c478bd9Sstevel@tonic-gate sp = kmem_findslab(cp, buf); 19807c478bd9Sstevel@tonic-gate if (sp == NULL || sp->slab_cache != cp) 19817c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADADDR, cp, buf); 19827c478bd9Sstevel@tonic-gate else 19837c478bd9Sstevel@tonic-gate kmem_error(KMERR_REDZONE, cp, buf); 19847c478bd9Sstevel@tonic-gate return (-1); 19857c478bd9Sstevel@tonic-gate } 19867c478bd9Sstevel@tonic-gate 19877c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE; 19887c478bd9Sstevel@tonic-gate 19897c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_HASH) && bcp->bc_addr != buf) { 19907c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADBUFCTL, cp, buf); 19917c478bd9Sstevel@tonic-gate return (-1); 19927c478bd9Sstevel@tonic-gate } 19937c478bd9Sstevel@tonic-gate 19947c478bd9Sstevel@tonic-gate if (btp->bt_redzone != KMEM_REDZONE_PATTERN) { 19957c478bd9Sstevel@tonic-gate kmem_error(KMERR_REDZONE, cp, buf); 19967c478bd9Sstevel@tonic-gate return (-1); 19977c478bd9Sstevel@tonic-gate } 19987c478bd9Sstevel@tonic-gate 19997c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_AUDIT) { 20007c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_CONTENTS) 20017c478bd9Sstevel@tonic-gate bcp->bc_contents = kmem_log_enter(kmem_content_log, 20027c478bd9Sstevel@tonic-gate buf, cp->cache_contents); 20037c478bd9Sstevel@tonic-gate KMEM_AUDIT(kmem_transaction_log, cp, bcp); 20047c478bd9Sstevel@tonic-gate } 20057c478bd9Sstevel@tonic-gate 20067c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_LITE) && 20077c478bd9Sstevel@tonic-gate !(cp->cache_cflags & KMC_KMEM_ALLOC)) { 20087c478bd9Sstevel@tonic-gate KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, caller); 20097c478bd9Sstevel@tonic-gate } 20107c478bd9Sstevel@tonic-gate 20117c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) { 20127c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) 20137c478bd9Sstevel@tonic-gate btp->bt_redzone = *(uint64_t *)buf; 20147c478bd9Sstevel@tonic-gate else if (cp->cache_destructor != NULL) 20157c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 20167c478bd9Sstevel@tonic-gate 20177c478bd9Sstevel@tonic-gate copy_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify); 20187c478bd9Sstevel@tonic-gate } 20197c478bd9Sstevel@tonic-gate 20207c478bd9Sstevel@tonic-gate return (0); 20217c478bd9Sstevel@tonic-gate } 20227c478bd9Sstevel@tonic-gate 20237c478bd9Sstevel@tonic-gate /* 20247c478bd9Sstevel@tonic-gate * Free each object in magazine mp to cp's slab layer, and free mp itself. 20257c478bd9Sstevel@tonic-gate */ 20267c478bd9Sstevel@tonic-gate static void 20277c478bd9Sstevel@tonic-gate kmem_magazine_destroy(kmem_cache_t *cp, kmem_magazine_t *mp, int nrounds) 20287c478bd9Sstevel@tonic-gate { 20297c478bd9Sstevel@tonic-gate int round; 20307c478bd9Sstevel@tonic-gate 2031b5fca8f8Stomee ASSERT(!list_link_active(&cp->cache_link) || 2032b5fca8f8Stomee taskq_member(kmem_taskq, curthread)); 20337c478bd9Sstevel@tonic-gate 20347c478bd9Sstevel@tonic-gate for (round = 0; round < nrounds; round++) { 20357c478bd9Sstevel@tonic-gate void *buf = mp->mag_round[round]; 20367c478bd9Sstevel@tonic-gate 20377c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) { 20387c478bd9Sstevel@tonic-gate if (verify_pattern(KMEM_FREE_PATTERN, buf, 20397c478bd9Sstevel@tonic-gate cp->cache_verify) != NULL) { 20407c478bd9Sstevel@tonic-gate kmem_error(KMERR_MODIFIED, cp, buf); 20417c478bd9Sstevel@tonic-gate continue; 20427c478bd9Sstevel@tonic-gate } 20437c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_LITE) && 20447c478bd9Sstevel@tonic-gate cp->cache_destructor != NULL) { 20457c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 20467c478bd9Sstevel@tonic-gate *(uint64_t *)buf = btp->bt_redzone; 20477c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 20487c478bd9Sstevel@tonic-gate *(uint64_t *)buf = KMEM_FREE_PATTERN; 20497c478bd9Sstevel@tonic-gate } 20507c478bd9Sstevel@tonic-gate } else if (cp->cache_destructor != NULL) { 20517c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 20527c478bd9Sstevel@tonic-gate } 20537c478bd9Sstevel@tonic-gate 20547c478bd9Sstevel@tonic-gate kmem_slab_free(cp, buf); 20557c478bd9Sstevel@tonic-gate } 20567c478bd9Sstevel@tonic-gate ASSERT(KMEM_MAGAZINE_VALID(cp, mp)); 20577c478bd9Sstevel@tonic-gate kmem_cache_free(cp->cache_magtype->mt_cache, mp); 20587c478bd9Sstevel@tonic-gate } 20597c478bd9Sstevel@tonic-gate 20607c478bd9Sstevel@tonic-gate /* 20617c478bd9Sstevel@tonic-gate * Allocate a magazine from the depot. 20627c478bd9Sstevel@tonic-gate */ 20637c478bd9Sstevel@tonic-gate static kmem_magazine_t * 20647c478bd9Sstevel@tonic-gate kmem_depot_alloc(kmem_cache_t *cp, kmem_maglist_t *mlp) 20657c478bd9Sstevel@tonic-gate { 20667c478bd9Sstevel@tonic-gate kmem_magazine_t *mp; 20677c478bd9Sstevel@tonic-gate 20687c478bd9Sstevel@tonic-gate /* 20697c478bd9Sstevel@tonic-gate * If we can't get the depot lock without contention, 20707c478bd9Sstevel@tonic-gate * update our contention count. We use the depot 20717c478bd9Sstevel@tonic-gate * contention rate to determine whether we need to 20727c478bd9Sstevel@tonic-gate * increase the magazine size for better scalability. 20737c478bd9Sstevel@tonic-gate */ 20747c478bd9Sstevel@tonic-gate if (!mutex_tryenter(&cp->cache_depot_lock)) { 20757c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock); 20767c478bd9Sstevel@tonic-gate cp->cache_depot_contention++; 20777c478bd9Sstevel@tonic-gate } 20787c478bd9Sstevel@tonic-gate 20797c478bd9Sstevel@tonic-gate if ((mp = mlp->ml_list) != NULL) { 20807c478bd9Sstevel@tonic-gate ASSERT(KMEM_MAGAZINE_VALID(cp, mp)); 20817c478bd9Sstevel@tonic-gate mlp->ml_list = mp->mag_next; 20827c478bd9Sstevel@tonic-gate if (--mlp->ml_total < mlp->ml_min) 20837c478bd9Sstevel@tonic-gate mlp->ml_min = mlp->ml_total; 20847c478bd9Sstevel@tonic-gate mlp->ml_alloc++; 20857c478bd9Sstevel@tonic-gate } 20867c478bd9Sstevel@tonic-gate 20877c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock); 20887c478bd9Sstevel@tonic-gate 20897c478bd9Sstevel@tonic-gate return (mp); 20907c478bd9Sstevel@tonic-gate } 20917c478bd9Sstevel@tonic-gate 20927c478bd9Sstevel@tonic-gate /* 20937c478bd9Sstevel@tonic-gate * Free a magazine to the depot. 20947c478bd9Sstevel@tonic-gate */ 20957c478bd9Sstevel@tonic-gate static void 20967c478bd9Sstevel@tonic-gate kmem_depot_free(kmem_cache_t *cp, kmem_maglist_t *mlp, kmem_magazine_t *mp) 20977c478bd9Sstevel@tonic-gate { 20987c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock); 20997c478bd9Sstevel@tonic-gate ASSERT(KMEM_MAGAZINE_VALID(cp, mp)); 21007c478bd9Sstevel@tonic-gate mp->mag_next = mlp->ml_list; 21017c478bd9Sstevel@tonic-gate mlp->ml_list = mp; 21027c478bd9Sstevel@tonic-gate mlp->ml_total++; 21037c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock); 21047c478bd9Sstevel@tonic-gate } 21057c478bd9Sstevel@tonic-gate 21067c478bd9Sstevel@tonic-gate /* 21077c478bd9Sstevel@tonic-gate * Update the working set statistics for cp's depot. 21087c478bd9Sstevel@tonic-gate */ 21097c478bd9Sstevel@tonic-gate static void 21107c478bd9Sstevel@tonic-gate kmem_depot_ws_update(kmem_cache_t *cp) 21117c478bd9Sstevel@tonic-gate { 21127c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock); 21137c478bd9Sstevel@tonic-gate cp->cache_full.ml_reaplimit = cp->cache_full.ml_min; 21147c478bd9Sstevel@tonic-gate cp->cache_full.ml_min = cp->cache_full.ml_total; 21157c478bd9Sstevel@tonic-gate cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_min; 21167c478bd9Sstevel@tonic-gate cp->cache_empty.ml_min = cp->cache_empty.ml_total; 21177c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock); 21187c478bd9Sstevel@tonic-gate } 21197c478bd9Sstevel@tonic-gate 21200c833d64SJosef 'Jeff' Sipek /* 21210c833d64SJosef 'Jeff' Sipek * Set the working set statistics for cp's depot to zero. (Everything is 21220c833d64SJosef 'Jeff' Sipek * eligible for reaping.) 21230c833d64SJosef 'Jeff' Sipek */ 21240c833d64SJosef 'Jeff' Sipek static void 21250c833d64SJosef 'Jeff' Sipek kmem_depot_ws_zero(kmem_cache_t *cp) 21260c833d64SJosef 'Jeff' Sipek { 21270c833d64SJosef 'Jeff' Sipek mutex_enter(&cp->cache_depot_lock); 21280c833d64SJosef 'Jeff' Sipek cp->cache_full.ml_reaplimit = cp->cache_full.ml_total; 21290c833d64SJosef 'Jeff' Sipek cp->cache_full.ml_min = cp->cache_full.ml_total; 21300c833d64SJosef 'Jeff' Sipek cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_total; 21310c833d64SJosef 'Jeff' Sipek cp->cache_empty.ml_min = cp->cache_empty.ml_total; 21320c833d64SJosef 'Jeff' Sipek mutex_exit(&cp->cache_depot_lock); 21330c833d64SJosef 'Jeff' Sipek } 21340c833d64SJosef 'Jeff' Sipek 21351c207ae9SMatthew Ahrens /* 21361c207ae9SMatthew Ahrens * The number of bytes to reap before we call kpreempt(). The default (1MB) 21371c207ae9SMatthew Ahrens * causes us to preempt reaping up to hundreds of times per second. Using a 21381c207ae9SMatthew Ahrens * larger value (1GB) causes this to have virtually no effect. 21391c207ae9SMatthew Ahrens */ 21401c207ae9SMatthew Ahrens size_t kmem_reap_preempt_bytes = 1024 * 1024; 21411c207ae9SMatthew Ahrens 21427c478bd9Sstevel@tonic-gate /* 21437c478bd9Sstevel@tonic-gate * Reap all magazines that have fallen out of the depot's working set. 21447c478bd9Sstevel@tonic-gate */ 21457c478bd9Sstevel@tonic-gate static void 21467c478bd9Sstevel@tonic-gate kmem_depot_ws_reap(kmem_cache_t *cp) 21477c478bd9Sstevel@tonic-gate { 21481c207ae9SMatthew Ahrens size_t bytes = 0; 21497c478bd9Sstevel@tonic-gate long reap; 21507c478bd9Sstevel@tonic-gate kmem_magazine_t *mp; 21517c478bd9Sstevel@tonic-gate 2152b5fca8f8Stomee ASSERT(!list_link_active(&cp->cache_link) || 2153b5fca8f8Stomee taskq_member(kmem_taskq, curthread)); 21547c478bd9Sstevel@tonic-gate 21557c478bd9Sstevel@tonic-gate reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min); 21561c207ae9SMatthew Ahrens while (reap-- && 21571c207ae9SMatthew Ahrens (mp = kmem_depot_alloc(cp, &cp->cache_full)) != NULL) { 21587c478bd9Sstevel@tonic-gate kmem_magazine_destroy(cp, mp, cp->cache_magtype->mt_magsize); 21591c207ae9SMatthew Ahrens bytes += cp->cache_magtype->mt_magsize * cp->cache_bufsize; 21601c207ae9SMatthew Ahrens if (bytes > kmem_reap_preempt_bytes) { 21611c207ae9SMatthew Ahrens kpreempt(KPREEMPT_SYNC); 21621c207ae9SMatthew Ahrens bytes = 0; 21631c207ae9SMatthew Ahrens } 21641c207ae9SMatthew Ahrens } 21657c478bd9Sstevel@tonic-gate 21667c478bd9Sstevel@tonic-gate reap = MIN(cp->cache_empty.ml_reaplimit, cp->cache_empty.ml_min); 21671c207ae9SMatthew Ahrens while (reap-- && 21681c207ae9SMatthew Ahrens (mp = kmem_depot_alloc(cp, &cp->cache_empty)) != NULL) { 21697c478bd9Sstevel@tonic-gate kmem_magazine_destroy(cp, mp, 0); 21701c207ae9SMatthew Ahrens bytes += cp->cache_magtype->mt_magsize * cp->cache_bufsize; 21711c207ae9SMatthew Ahrens if (bytes > kmem_reap_preempt_bytes) { 21721c207ae9SMatthew Ahrens kpreempt(KPREEMPT_SYNC); 21731c207ae9SMatthew Ahrens bytes = 0; 21741c207ae9SMatthew Ahrens } 21751c207ae9SMatthew Ahrens } 21767c478bd9Sstevel@tonic-gate } 21777c478bd9Sstevel@tonic-gate 21787c478bd9Sstevel@tonic-gate static void 21797c478bd9Sstevel@tonic-gate kmem_cpu_reload(kmem_cpu_cache_t *ccp, kmem_magazine_t *mp, int rounds) 21807c478bd9Sstevel@tonic-gate { 21817c478bd9Sstevel@tonic-gate ASSERT((ccp->cc_loaded == NULL && ccp->cc_rounds == -1) || 21827c478bd9Sstevel@tonic-gate (ccp->cc_loaded && ccp->cc_rounds + rounds == ccp->cc_magsize)); 21837c478bd9Sstevel@tonic-gate ASSERT(ccp->cc_magsize > 0); 21847c478bd9Sstevel@tonic-gate 21857c478bd9Sstevel@tonic-gate ccp->cc_ploaded = ccp->cc_loaded; 21867c478bd9Sstevel@tonic-gate ccp->cc_prounds = ccp->cc_rounds; 21877c478bd9Sstevel@tonic-gate ccp->cc_loaded = mp; 21887c478bd9Sstevel@tonic-gate ccp->cc_rounds = rounds; 21897c478bd9Sstevel@tonic-gate } 21907c478bd9Sstevel@tonic-gate 21919dd77bc8SDave Plauger /* 21929dd77bc8SDave Plauger * Intercept kmem alloc/free calls during crash dump in order to avoid 21939dd77bc8SDave Plauger * changing kmem state while memory is being saved to the dump device. 21949dd77bc8SDave Plauger * Otherwise, ::kmem_verify will report "corrupt buffers". Note that 21959dd77bc8SDave Plauger * there are no locks because only one CPU calls kmem during a crash 21969dd77bc8SDave Plauger * dump. To enable this feature, first create the associated vmem 21979dd77bc8SDave Plauger * arena with VMC_DUMPSAFE. 21989dd77bc8SDave Plauger */ 21999dd77bc8SDave Plauger static void *kmem_dump_start; /* start of pre-reserved heap */ 22009dd77bc8SDave Plauger static void *kmem_dump_end; /* end of heap area */ 22019dd77bc8SDave Plauger static void *kmem_dump_curr; /* current free heap pointer */ 22029dd77bc8SDave Plauger static size_t kmem_dump_size; /* size of heap area */ 22039dd77bc8SDave Plauger 22049dd77bc8SDave Plauger /* append to each buf created in the pre-reserved heap */ 22059dd77bc8SDave Plauger typedef struct kmem_dumpctl { 22069dd77bc8SDave Plauger void *kdc_next; /* cache dump free list linkage */ 22079dd77bc8SDave Plauger } kmem_dumpctl_t; 22089dd77bc8SDave Plauger 22099dd77bc8SDave Plauger #define KMEM_DUMPCTL(cp, buf) \ 22109dd77bc8SDave Plauger ((kmem_dumpctl_t *)P2ROUNDUP((uintptr_t)(buf) + (cp)->cache_bufsize, \ 22119dd77bc8SDave Plauger sizeof (void *))) 22129dd77bc8SDave Plauger 22139dd77bc8SDave Plauger /* Keep some simple stats. */ 22149dd77bc8SDave Plauger #define KMEM_DUMP_LOGS (100) 22159dd77bc8SDave Plauger 22169dd77bc8SDave Plauger typedef struct kmem_dump_log { 22179dd77bc8SDave Plauger kmem_cache_t *kdl_cache; 22189dd77bc8SDave Plauger uint_t kdl_allocs; /* # of dump allocations */ 22199dd77bc8SDave Plauger uint_t kdl_frees; /* # of dump frees */ 22209dd77bc8SDave Plauger uint_t kdl_alloc_fails; /* # of allocation failures */ 22219dd77bc8SDave Plauger uint_t kdl_free_nondump; /* # of non-dump frees */ 22229dd77bc8SDave Plauger uint_t kdl_unsafe; /* cache was used, but unsafe */ 22239dd77bc8SDave Plauger } kmem_dump_log_t; 22249dd77bc8SDave Plauger 22259dd77bc8SDave Plauger static kmem_dump_log_t *kmem_dump_log; 22269dd77bc8SDave Plauger static int kmem_dump_log_idx; 22279dd77bc8SDave Plauger 22289dd77bc8SDave Plauger #define KDI_LOG(cp, stat) { \ 22299dd77bc8SDave Plauger kmem_dump_log_t *kdl; \ 22309dd77bc8SDave Plauger if ((kdl = (kmem_dump_log_t *)((cp)->cache_dumplog)) != NULL) { \ 22319dd77bc8SDave Plauger kdl->stat++; \ 22329dd77bc8SDave Plauger } else if (kmem_dump_log_idx < KMEM_DUMP_LOGS) { \ 22339dd77bc8SDave Plauger kdl = &kmem_dump_log[kmem_dump_log_idx++]; \ 22349dd77bc8SDave Plauger kdl->stat++; \ 22359dd77bc8SDave Plauger kdl->kdl_cache = (cp); \ 22369dd77bc8SDave Plauger (cp)->cache_dumplog = kdl; \ 22379dd77bc8SDave Plauger } \ 22389dd77bc8SDave Plauger } 22399dd77bc8SDave Plauger 22409dd77bc8SDave Plauger /* set non zero for full report */ 22419dd77bc8SDave Plauger uint_t kmem_dump_verbose = 0; 22429dd77bc8SDave Plauger 22439dd77bc8SDave Plauger /* stats for overize heap */ 22449dd77bc8SDave Plauger uint_t kmem_dump_oversize_allocs = 0; 22459dd77bc8SDave Plauger uint_t kmem_dump_oversize_max = 0; 22469dd77bc8SDave Plauger 22479dd77bc8SDave Plauger static void 22489dd77bc8SDave Plauger kmem_dumppr(char **pp, char *e, const char *format, ...) 22499dd77bc8SDave Plauger { 22509dd77bc8SDave Plauger char *p = *pp; 22519dd77bc8SDave Plauger 22529dd77bc8SDave Plauger if (p < e) { 22539dd77bc8SDave Plauger int n; 22549dd77bc8SDave Plauger va_list ap; 22559dd77bc8SDave Plauger 22569dd77bc8SDave Plauger va_start(ap, format); 22579dd77bc8SDave Plauger n = vsnprintf(p, e - p, format, ap); 22589dd77bc8SDave Plauger va_end(ap); 22599dd77bc8SDave Plauger *pp = p + n; 22609dd77bc8SDave Plauger } 22619dd77bc8SDave Plauger } 22629dd77bc8SDave Plauger 22639dd77bc8SDave Plauger /* 22649dd77bc8SDave Plauger * Called when dumpadm(1M) configures dump parameters. 22659dd77bc8SDave Plauger */ 22669dd77bc8SDave Plauger void 22679dd77bc8SDave Plauger kmem_dump_init(size_t size) 22689dd77bc8SDave Plauger { 22699dd77bc8SDave Plauger if (kmem_dump_start != NULL) 22709dd77bc8SDave Plauger kmem_free(kmem_dump_start, kmem_dump_size); 22719dd77bc8SDave Plauger 22729dd77bc8SDave Plauger if (kmem_dump_log == NULL) 22739dd77bc8SDave Plauger kmem_dump_log = (kmem_dump_log_t *)kmem_zalloc(KMEM_DUMP_LOGS * 22749dd77bc8SDave Plauger sizeof (kmem_dump_log_t), KM_SLEEP); 22759dd77bc8SDave Plauger 22769dd77bc8SDave Plauger kmem_dump_start = kmem_alloc(size, KM_SLEEP); 22779dd77bc8SDave Plauger 22789dd77bc8SDave Plauger if (kmem_dump_start != NULL) { 22799dd77bc8SDave Plauger kmem_dump_size = size; 22809dd77bc8SDave Plauger kmem_dump_curr = kmem_dump_start; 22819dd77bc8SDave Plauger kmem_dump_end = (void *)((char *)kmem_dump_start + size); 22829dd77bc8SDave Plauger copy_pattern(KMEM_UNINITIALIZED_PATTERN, kmem_dump_start, size); 22839dd77bc8SDave Plauger } else { 22849dd77bc8SDave Plauger kmem_dump_size = 0; 22859dd77bc8SDave Plauger kmem_dump_curr = NULL; 22869dd77bc8SDave Plauger kmem_dump_end = NULL; 22879dd77bc8SDave Plauger } 22889dd77bc8SDave Plauger } 22899dd77bc8SDave Plauger 22909dd77bc8SDave Plauger /* 22919dd77bc8SDave Plauger * Set flag for each kmem_cache_t if is safe to use alternate dump 22929dd77bc8SDave Plauger * memory. Called just before panic crash dump starts. Set the flag 22939dd77bc8SDave Plauger * for the calling CPU. 22949dd77bc8SDave Plauger */ 22959dd77bc8SDave Plauger void 22969dd77bc8SDave Plauger kmem_dump_begin(void) 22979dd77bc8SDave Plauger { 22989dd77bc8SDave Plauger ASSERT(panicstr != NULL); 22999dd77bc8SDave Plauger if (kmem_dump_start != NULL) { 23009dd77bc8SDave Plauger kmem_cache_t *cp; 23019dd77bc8SDave Plauger 23029dd77bc8SDave Plauger for (cp = list_head(&kmem_caches); cp != NULL; 23039dd77bc8SDave Plauger cp = list_next(&kmem_caches, cp)) { 23049dd77bc8SDave Plauger kmem_cpu_cache_t *ccp = KMEM_CPU_CACHE(cp); 23059dd77bc8SDave Plauger 23069dd77bc8SDave Plauger if (cp->cache_arena->vm_cflags & VMC_DUMPSAFE) { 23079dd77bc8SDave Plauger cp->cache_flags |= KMF_DUMPDIVERT; 23089dd77bc8SDave Plauger ccp->cc_flags |= KMF_DUMPDIVERT; 23099dd77bc8SDave Plauger ccp->cc_dump_rounds = ccp->cc_rounds; 23109dd77bc8SDave Plauger ccp->cc_dump_prounds = ccp->cc_prounds; 23119dd77bc8SDave Plauger ccp->cc_rounds = ccp->cc_prounds = -1; 23129dd77bc8SDave Plauger } else { 23139dd77bc8SDave Plauger cp->cache_flags |= KMF_DUMPUNSAFE; 23149dd77bc8SDave Plauger ccp->cc_flags |= KMF_DUMPUNSAFE; 23159dd77bc8SDave Plauger } 23169dd77bc8SDave Plauger } 23179dd77bc8SDave Plauger } 23189dd77bc8SDave Plauger } 23199dd77bc8SDave Plauger 23209dd77bc8SDave Plauger /* 23219dd77bc8SDave Plauger * finished dump intercept 23229dd77bc8SDave Plauger * print any warnings on the console 23239dd77bc8SDave Plauger * return verbose information to dumpsys() in the given buffer 23249dd77bc8SDave Plauger */ 23259dd77bc8SDave Plauger size_t 23269dd77bc8SDave Plauger kmem_dump_finish(char *buf, size_t size) 23279dd77bc8SDave Plauger { 23289dd77bc8SDave Plauger int kdi_idx; 23299dd77bc8SDave Plauger int kdi_end = kmem_dump_log_idx; 23309dd77bc8SDave Plauger int percent = 0; 23319dd77bc8SDave Plauger int header = 0; 23329dd77bc8SDave Plauger int warn = 0; 23339dd77bc8SDave Plauger size_t used; 23349dd77bc8SDave Plauger kmem_cache_t *cp; 23359dd77bc8SDave Plauger kmem_dump_log_t *kdl; 23369dd77bc8SDave Plauger char *e = buf + size; 23379dd77bc8SDave Plauger char *p = buf; 23389dd77bc8SDave Plauger 23399dd77bc8SDave Plauger if (kmem_dump_size == 0 || kmem_dump_verbose == 0) 23409dd77bc8SDave Plauger return (0); 23419dd77bc8SDave Plauger 23429dd77bc8SDave Plauger used = (char *)kmem_dump_curr - (char *)kmem_dump_start; 23439dd77bc8SDave Plauger percent = (used * 100) / kmem_dump_size; 23449dd77bc8SDave Plauger 23459dd77bc8SDave Plauger kmem_dumppr(&p, e, "%% heap used,%d\n", percent); 23469dd77bc8SDave Plauger kmem_dumppr(&p, e, "used bytes,%ld\n", used); 23479dd77bc8SDave Plauger kmem_dumppr(&p, e, "heap size,%ld\n", kmem_dump_size); 23489dd77bc8SDave Plauger kmem_dumppr(&p, e, "Oversize allocs,%d\n", 23499dd77bc8SDave Plauger kmem_dump_oversize_allocs); 23509dd77bc8SDave Plauger kmem_dumppr(&p, e, "Oversize max size,%ld\n", 23519dd77bc8SDave Plauger kmem_dump_oversize_max); 23529dd77bc8SDave Plauger 23539dd77bc8SDave Plauger for (kdi_idx = 0; kdi_idx < kdi_end; kdi_idx++) { 23549dd77bc8SDave Plauger kdl = &kmem_dump_log[kdi_idx]; 23559dd77bc8SDave Plauger cp = kdl->kdl_cache; 23569dd77bc8SDave Plauger if (cp == NULL) 23579dd77bc8SDave Plauger break; 23589dd77bc8SDave Plauger if (kdl->kdl_alloc_fails) 23599dd77bc8SDave Plauger ++warn; 23609dd77bc8SDave Plauger if (header == 0) { 23619dd77bc8SDave Plauger kmem_dumppr(&p, e, 23629dd77bc8SDave Plauger "Cache Name,Allocs,Frees,Alloc Fails," 23639dd77bc8SDave Plauger "Nondump Frees,Unsafe Allocs/Frees\n"); 23649dd77bc8SDave Plauger header = 1; 23659dd77bc8SDave Plauger } 23669dd77bc8SDave Plauger kmem_dumppr(&p, e, "%s,%d,%d,%d,%d,%d\n", 23679dd77bc8SDave Plauger cp->cache_name, kdl->kdl_allocs, kdl->kdl_frees, 23689dd77bc8SDave Plauger kdl->kdl_alloc_fails, kdl->kdl_free_nondump, 23699dd77bc8SDave Plauger kdl->kdl_unsafe); 23709dd77bc8SDave Plauger } 23719dd77bc8SDave Plauger 23729dd77bc8SDave Plauger /* return buffer size used */ 23739dd77bc8SDave Plauger if (p < e) 23749dd77bc8SDave Plauger bzero(p, e - p); 23759dd77bc8SDave Plauger return (p - buf); 23769dd77bc8SDave Plauger } 23779dd77bc8SDave Plauger 23789dd77bc8SDave Plauger /* 23799dd77bc8SDave Plauger * Allocate a constructed object from alternate dump memory. 23809dd77bc8SDave Plauger */ 23819dd77bc8SDave Plauger void * 23829dd77bc8SDave Plauger kmem_cache_alloc_dump(kmem_cache_t *cp, int kmflag) 23839dd77bc8SDave Plauger { 23849dd77bc8SDave Plauger void *buf; 23859dd77bc8SDave Plauger void *curr; 23869dd77bc8SDave Plauger char *bufend; 23879dd77bc8SDave Plauger 23889dd77bc8SDave Plauger /* return a constructed object */ 23899dd77bc8SDave Plauger if ((buf = cp->cache_dumpfreelist) != NULL) { 23909dd77bc8SDave Plauger cp->cache_dumpfreelist = KMEM_DUMPCTL(cp, buf)->kdc_next; 23919dd77bc8SDave Plauger KDI_LOG(cp, kdl_allocs); 23929dd77bc8SDave Plauger return (buf); 23939dd77bc8SDave Plauger } 23949dd77bc8SDave Plauger 23959dd77bc8SDave Plauger /* create a new constructed object */ 23969dd77bc8SDave Plauger curr = kmem_dump_curr; 23979dd77bc8SDave Plauger buf = (void *)P2ROUNDUP((uintptr_t)curr, cp->cache_align); 23989dd77bc8SDave Plauger bufend = (char *)KMEM_DUMPCTL(cp, buf) + sizeof (kmem_dumpctl_t); 23999dd77bc8SDave Plauger 24009dd77bc8SDave Plauger /* hat layer objects cannot cross a page boundary */ 24019dd77bc8SDave Plauger if (cp->cache_align < PAGESIZE) { 24029dd77bc8SDave Plauger char *page = (char *)P2ROUNDUP((uintptr_t)buf, PAGESIZE); 24039dd77bc8SDave Plauger if (bufend > page) { 24049dd77bc8SDave Plauger bufend += page - (char *)buf; 24059dd77bc8SDave Plauger buf = (void *)page; 24069dd77bc8SDave Plauger } 24079dd77bc8SDave Plauger } 24089dd77bc8SDave Plauger 24099dd77bc8SDave Plauger /* fall back to normal alloc if reserved area is used up */ 24109dd77bc8SDave Plauger if (bufend > (char *)kmem_dump_end) { 24119dd77bc8SDave Plauger kmem_dump_curr = kmem_dump_end; 24129dd77bc8SDave Plauger KDI_LOG(cp, kdl_alloc_fails); 24139dd77bc8SDave Plauger return (NULL); 24149dd77bc8SDave Plauger } 24159dd77bc8SDave Plauger 24169dd77bc8SDave Plauger /* 24179dd77bc8SDave Plauger * Must advance curr pointer before calling a constructor that 24189dd77bc8SDave Plauger * may also allocate memory. 24199dd77bc8SDave Plauger */ 24209dd77bc8SDave Plauger kmem_dump_curr = bufend; 24219dd77bc8SDave Plauger 24229dd77bc8SDave Plauger /* run constructor */ 24239dd77bc8SDave Plauger if (cp->cache_constructor != NULL && 24249dd77bc8SDave Plauger cp->cache_constructor(buf, cp->cache_private, kmflag) 24259dd77bc8SDave Plauger != 0) { 24269dd77bc8SDave Plauger #ifdef DEBUG 24279dd77bc8SDave Plauger printf("name='%s' cache=0x%p: kmem cache constructor failed\n", 24289dd77bc8SDave Plauger cp->cache_name, (void *)cp); 24299dd77bc8SDave Plauger #endif 24309dd77bc8SDave Plauger /* reset curr pointer iff no allocs were done */ 24319dd77bc8SDave Plauger if (kmem_dump_curr == bufend) 24329dd77bc8SDave Plauger kmem_dump_curr = curr; 24339dd77bc8SDave Plauger 24349dd77bc8SDave Plauger /* fall back to normal alloc if the constructor fails */ 24359dd77bc8SDave Plauger KDI_LOG(cp, kdl_alloc_fails); 24369dd77bc8SDave Plauger return (NULL); 24379dd77bc8SDave Plauger } 24389dd77bc8SDave Plauger 24399dd77bc8SDave Plauger KDI_LOG(cp, kdl_allocs); 24409dd77bc8SDave Plauger return (buf); 24419dd77bc8SDave Plauger } 24429dd77bc8SDave Plauger 24439dd77bc8SDave Plauger /* 24449dd77bc8SDave Plauger * Free a constructed object in alternate dump memory. 24459dd77bc8SDave Plauger */ 24469dd77bc8SDave Plauger int 24479dd77bc8SDave Plauger kmem_cache_free_dump(kmem_cache_t *cp, void *buf) 24489dd77bc8SDave Plauger { 24499dd77bc8SDave Plauger /* save constructed buffers for next time */ 24509dd77bc8SDave Plauger if ((char *)buf >= (char *)kmem_dump_start && 24519dd77bc8SDave Plauger (char *)buf < (char *)kmem_dump_end) { 24529dd77bc8SDave Plauger KMEM_DUMPCTL(cp, buf)->kdc_next = cp->cache_dumpfreelist; 24539dd77bc8SDave Plauger cp->cache_dumpfreelist = buf; 24549dd77bc8SDave Plauger KDI_LOG(cp, kdl_frees); 24559dd77bc8SDave Plauger return (0); 24569dd77bc8SDave Plauger } 24579dd77bc8SDave Plauger 24589dd77bc8SDave Plauger /* count all non-dump buf frees */ 24599dd77bc8SDave Plauger KDI_LOG(cp, kdl_free_nondump); 24609dd77bc8SDave Plauger 24619dd77bc8SDave Plauger /* just drop buffers that were allocated before dump started */ 24629dd77bc8SDave Plauger if (kmem_dump_curr < kmem_dump_end) 24639dd77bc8SDave Plauger return (0); 24649dd77bc8SDave Plauger 24659dd77bc8SDave Plauger /* fall back to normal free if reserved area is used up */ 24669dd77bc8SDave Plauger return (1); 24679dd77bc8SDave Plauger } 24689dd77bc8SDave Plauger 24697c478bd9Sstevel@tonic-gate /* 24707c478bd9Sstevel@tonic-gate * Allocate a constructed object from cache cp. 24717c478bd9Sstevel@tonic-gate */ 24727c478bd9Sstevel@tonic-gate void * 24737c478bd9Sstevel@tonic-gate kmem_cache_alloc(kmem_cache_t *cp, int kmflag) 24747c478bd9Sstevel@tonic-gate { 24757c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp = KMEM_CPU_CACHE(cp); 24767c478bd9Sstevel@tonic-gate kmem_magazine_t *fmp; 24777c478bd9Sstevel@tonic-gate void *buf; 24787c478bd9Sstevel@tonic-gate 24797c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 24807c478bd9Sstevel@tonic-gate for (;;) { 24817c478bd9Sstevel@tonic-gate /* 24827c478bd9Sstevel@tonic-gate * If there's an object available in the current CPU's 24837c478bd9Sstevel@tonic-gate * loaded magazine, just take it and return. 24847c478bd9Sstevel@tonic-gate */ 24857c478bd9Sstevel@tonic-gate if (ccp->cc_rounds > 0) { 24867c478bd9Sstevel@tonic-gate buf = ccp->cc_loaded->mag_round[--ccp->cc_rounds]; 24877c478bd9Sstevel@tonic-gate ccp->cc_alloc++; 24887c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 24899dd77bc8SDave Plauger if (ccp->cc_flags & (KMF_BUFTAG | KMF_DUMPUNSAFE)) { 24909dd77bc8SDave Plauger if (ccp->cc_flags & KMF_DUMPUNSAFE) { 24919dd77bc8SDave Plauger ASSERT(!(ccp->cc_flags & 24929dd77bc8SDave Plauger KMF_DUMPDIVERT)); 24939dd77bc8SDave Plauger KDI_LOG(cp, kdl_unsafe); 24949dd77bc8SDave Plauger } 24959dd77bc8SDave Plauger if ((ccp->cc_flags & KMF_BUFTAG) && 24969dd77bc8SDave Plauger kmem_cache_alloc_debug(cp, buf, kmflag, 0, 24979dd77bc8SDave Plauger caller()) != 0) { 24989dd77bc8SDave Plauger if (kmflag & KM_NOSLEEP) 24999dd77bc8SDave Plauger return (NULL); 25009dd77bc8SDave Plauger mutex_enter(&ccp->cc_lock); 25019dd77bc8SDave Plauger continue; 25029dd77bc8SDave Plauger } 25037c478bd9Sstevel@tonic-gate } 25047c478bd9Sstevel@tonic-gate return (buf); 25057c478bd9Sstevel@tonic-gate } 25067c478bd9Sstevel@tonic-gate 25077c478bd9Sstevel@tonic-gate /* 25087c478bd9Sstevel@tonic-gate * The loaded magazine is empty. If the previously loaded 25097c478bd9Sstevel@tonic-gate * magazine was full, exchange them and try again. 25107c478bd9Sstevel@tonic-gate */ 25117c478bd9Sstevel@tonic-gate if (ccp->cc_prounds > 0) { 25127c478bd9Sstevel@tonic-gate kmem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds); 25137c478bd9Sstevel@tonic-gate continue; 25147c478bd9Sstevel@tonic-gate } 25157c478bd9Sstevel@tonic-gate 25169dd77bc8SDave Plauger /* 25179dd77bc8SDave Plauger * Return an alternate buffer at dump time to preserve 25189dd77bc8SDave Plauger * the heap. 25199dd77bc8SDave Plauger */ 25209dd77bc8SDave Plauger if (ccp->cc_flags & (KMF_DUMPDIVERT | KMF_DUMPUNSAFE)) { 25219dd77bc8SDave Plauger if (ccp->cc_flags & KMF_DUMPUNSAFE) { 25229dd77bc8SDave Plauger ASSERT(!(ccp->cc_flags & KMF_DUMPDIVERT)); 25239dd77bc8SDave Plauger /* log it so that we can warn about it */ 25249dd77bc8SDave Plauger KDI_LOG(cp, kdl_unsafe); 25259dd77bc8SDave Plauger } else { 25269dd77bc8SDave Plauger if ((buf = kmem_cache_alloc_dump(cp, kmflag)) != 25279dd77bc8SDave Plauger NULL) { 25289dd77bc8SDave Plauger mutex_exit(&ccp->cc_lock); 25299dd77bc8SDave Plauger return (buf); 25309dd77bc8SDave Plauger } 25319dd77bc8SDave Plauger break; /* fall back to slab layer */ 25329dd77bc8SDave Plauger } 25339dd77bc8SDave Plauger } 25349dd77bc8SDave Plauger 25357c478bd9Sstevel@tonic-gate /* 25367c478bd9Sstevel@tonic-gate * If the magazine layer is disabled, break out now. 25377c478bd9Sstevel@tonic-gate */ 25387c478bd9Sstevel@tonic-gate if (ccp->cc_magsize == 0) 25397c478bd9Sstevel@tonic-gate break; 25407c478bd9Sstevel@tonic-gate 25417c478bd9Sstevel@tonic-gate /* 25427c478bd9Sstevel@tonic-gate * Try to get a full magazine from the depot. 25437c478bd9Sstevel@tonic-gate */ 25447c478bd9Sstevel@tonic-gate fmp = kmem_depot_alloc(cp, &cp->cache_full); 25457c478bd9Sstevel@tonic-gate if (fmp != NULL) { 25467c478bd9Sstevel@tonic-gate if (ccp->cc_ploaded != NULL) 25477c478bd9Sstevel@tonic-gate kmem_depot_free(cp, &cp->cache_empty, 25487c478bd9Sstevel@tonic-gate ccp->cc_ploaded); 25497c478bd9Sstevel@tonic-gate kmem_cpu_reload(ccp, fmp, ccp->cc_magsize); 25507c478bd9Sstevel@tonic-gate continue; 25517c478bd9Sstevel@tonic-gate } 25527c478bd9Sstevel@tonic-gate 25537c478bd9Sstevel@tonic-gate /* 25547c478bd9Sstevel@tonic-gate * There are no full magazines in the depot, 25557c478bd9Sstevel@tonic-gate * so fall through to the slab layer. 25567c478bd9Sstevel@tonic-gate */ 25577c478bd9Sstevel@tonic-gate break; 25587c478bd9Sstevel@tonic-gate } 25597c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 25607c478bd9Sstevel@tonic-gate 25617c478bd9Sstevel@tonic-gate /* 25627c478bd9Sstevel@tonic-gate * We couldn't allocate a constructed object from the magazine layer, 25637c478bd9Sstevel@tonic-gate * so get a raw buffer from the slab layer and apply its constructor. 25647c478bd9Sstevel@tonic-gate */ 25657c478bd9Sstevel@tonic-gate buf = kmem_slab_alloc(cp, kmflag); 25667c478bd9Sstevel@tonic-gate 25677c478bd9Sstevel@tonic-gate if (buf == NULL) 25687c478bd9Sstevel@tonic-gate return (NULL); 25697c478bd9Sstevel@tonic-gate 25707c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_BUFTAG) { 25717c478bd9Sstevel@tonic-gate /* 25727c478bd9Sstevel@tonic-gate * Make kmem_cache_alloc_debug() apply the constructor for us. 25737c478bd9Sstevel@tonic-gate */ 2574b5fca8f8Stomee int rc = kmem_cache_alloc_debug(cp, buf, kmflag, 1, caller()); 2575b5fca8f8Stomee if (rc != 0) { 25767c478bd9Sstevel@tonic-gate if (kmflag & KM_NOSLEEP) 25777c478bd9Sstevel@tonic-gate return (NULL); 25787c478bd9Sstevel@tonic-gate /* 25797c478bd9Sstevel@tonic-gate * kmem_cache_alloc_debug() detected corruption 2580b5fca8f8Stomee * but didn't panic (kmem_panic <= 0). We should not be 2581b5fca8f8Stomee * here because the constructor failed (indicated by a 2582b5fca8f8Stomee * return code of 1). Try again. 25837c478bd9Sstevel@tonic-gate */ 2584b5fca8f8Stomee ASSERT(rc == -1); 25857c478bd9Sstevel@tonic-gate return (kmem_cache_alloc(cp, kmflag)); 25867c478bd9Sstevel@tonic-gate } 25877c478bd9Sstevel@tonic-gate return (buf); 25887c478bd9Sstevel@tonic-gate } 25897c478bd9Sstevel@tonic-gate 25907c478bd9Sstevel@tonic-gate if (cp->cache_constructor != NULL && 25917c478bd9Sstevel@tonic-gate cp->cache_constructor(buf, cp->cache_private, kmflag) != 0) { 25921a5e258fSJosef 'Jeff' Sipek atomic_inc_64(&cp->cache_alloc_fail); 25937c478bd9Sstevel@tonic-gate kmem_slab_free(cp, buf); 25947c478bd9Sstevel@tonic-gate return (NULL); 25957c478bd9Sstevel@tonic-gate } 25967c478bd9Sstevel@tonic-gate 25977c478bd9Sstevel@tonic-gate return (buf); 25987c478bd9Sstevel@tonic-gate } 25997c478bd9Sstevel@tonic-gate 26007c478bd9Sstevel@tonic-gate /* 2601b5fca8f8Stomee * The freed argument tells whether or not kmem_cache_free_debug() has already 2602b5fca8f8Stomee * been called so that we can avoid the duplicate free error. For example, a 2603b5fca8f8Stomee * buffer on a magazine has already been freed by the client but is still 2604b5fca8f8Stomee * constructed. 26057c478bd9Sstevel@tonic-gate */ 2606b5fca8f8Stomee static void 2607b5fca8f8Stomee kmem_slab_free_constructed(kmem_cache_t *cp, void *buf, boolean_t freed) 26087c478bd9Sstevel@tonic-gate { 2609b5fca8f8Stomee if (!freed && (cp->cache_flags & KMF_BUFTAG)) 26107c478bd9Sstevel@tonic-gate if (kmem_cache_free_debug(cp, buf, caller()) == -1) 26117c478bd9Sstevel@tonic-gate return; 26127c478bd9Sstevel@tonic-gate 2613b5fca8f8Stomee /* 2614b5fca8f8Stomee * Note that if KMF_DEADBEEF is in effect and KMF_LITE is not, 2615b5fca8f8Stomee * kmem_cache_free_debug() will have already applied the destructor. 2616b5fca8f8Stomee */ 2617b5fca8f8Stomee if ((cp->cache_flags & (KMF_DEADBEEF | KMF_LITE)) != KMF_DEADBEEF && 2618b5fca8f8Stomee cp->cache_destructor != NULL) { 2619b5fca8f8Stomee if (cp->cache_flags & KMF_DEADBEEF) { /* KMF_LITE implied */ 2620b5fca8f8Stomee kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 2621b5fca8f8Stomee *(uint64_t *)buf = btp->bt_redzone; 2622b5fca8f8Stomee cp->cache_destructor(buf, cp->cache_private); 2623b5fca8f8Stomee *(uint64_t *)buf = KMEM_FREE_PATTERN; 2624b5fca8f8Stomee } else { 2625b5fca8f8Stomee cp->cache_destructor(buf, cp->cache_private); 2626b5fca8f8Stomee } 2627b5fca8f8Stomee } 2628b5fca8f8Stomee 2629b5fca8f8Stomee kmem_slab_free(cp, buf); 2630b5fca8f8Stomee } 2631b5fca8f8Stomee 2632b942e89bSDavid Valin /* 2633b942e89bSDavid Valin * Used when there's no room to free a buffer to the per-CPU cache. 2634b942e89bSDavid Valin * Drops and re-acquires &ccp->cc_lock, and returns non-zero if the 2635b942e89bSDavid Valin * caller should try freeing to the per-CPU cache again. 2636b942e89bSDavid Valin * Note that we don't directly install the magazine in the cpu cache, 2637b942e89bSDavid Valin * since its state may have changed wildly while the lock was dropped. 2638b942e89bSDavid Valin */ 2639b942e89bSDavid Valin static int 2640b942e89bSDavid Valin kmem_cpucache_magazine_alloc(kmem_cpu_cache_t *ccp, kmem_cache_t *cp) 2641b942e89bSDavid Valin { 2642b942e89bSDavid Valin kmem_magazine_t *emp; 2643b942e89bSDavid Valin kmem_magtype_t *mtp; 2644b942e89bSDavid Valin 2645b942e89bSDavid Valin ASSERT(MUTEX_HELD(&ccp->cc_lock)); 2646b942e89bSDavid Valin ASSERT(((uint_t)ccp->cc_rounds == ccp->cc_magsize || 2647b942e89bSDavid Valin ((uint_t)ccp->cc_rounds == -1)) && 2648b942e89bSDavid Valin ((uint_t)ccp->cc_prounds == ccp->cc_magsize || 2649b942e89bSDavid Valin ((uint_t)ccp->cc_prounds == -1))); 2650b942e89bSDavid Valin 2651b942e89bSDavid Valin emp = kmem_depot_alloc(cp, &cp->cache_empty); 2652b942e89bSDavid Valin if (emp != NULL) { 2653b942e89bSDavid Valin if (ccp->cc_ploaded != NULL) 2654b942e89bSDavid Valin kmem_depot_free(cp, &cp->cache_full, 2655b942e89bSDavid Valin ccp->cc_ploaded); 2656b942e89bSDavid Valin kmem_cpu_reload(ccp, emp, 0); 2657b942e89bSDavid Valin return (1); 2658b942e89bSDavid Valin } 2659b942e89bSDavid Valin /* 2660b942e89bSDavid Valin * There are no empty magazines in the depot, 2661b942e89bSDavid Valin * so try to allocate a new one. We must drop all locks 2662b942e89bSDavid Valin * across kmem_cache_alloc() because lower layers may 2663b942e89bSDavid Valin * attempt to allocate from this cache. 2664b942e89bSDavid Valin */ 2665b942e89bSDavid Valin mtp = cp->cache_magtype; 2666b942e89bSDavid Valin mutex_exit(&ccp->cc_lock); 2667b942e89bSDavid Valin emp = kmem_cache_alloc(mtp->mt_cache, KM_NOSLEEP); 2668b942e89bSDavid Valin mutex_enter(&ccp->cc_lock); 2669b942e89bSDavid Valin 2670b942e89bSDavid Valin if (emp != NULL) { 2671b942e89bSDavid Valin /* 2672b942e89bSDavid Valin * We successfully allocated an empty magazine. 2673b942e89bSDavid Valin * However, we had to drop ccp->cc_lock to do it, 2674b942e89bSDavid Valin * so the cache's magazine size may have changed. 2675b942e89bSDavid Valin * If so, free the magazine and try again. 2676b942e89bSDavid Valin */ 2677b942e89bSDavid Valin if (ccp->cc_magsize != mtp->mt_magsize) { 2678b942e89bSDavid Valin mutex_exit(&ccp->cc_lock); 2679b942e89bSDavid Valin kmem_cache_free(mtp->mt_cache, emp); 2680b942e89bSDavid Valin mutex_enter(&ccp->cc_lock); 2681b942e89bSDavid Valin return (1); 2682b942e89bSDavid Valin } 2683b942e89bSDavid Valin 2684b942e89bSDavid Valin /* 2685b942e89bSDavid Valin * We got a magazine of the right size. Add it to 2686b942e89bSDavid Valin * the depot and try the whole dance again. 2687b942e89bSDavid Valin */ 2688b942e89bSDavid Valin kmem_depot_free(cp, &cp->cache_empty, emp); 2689b942e89bSDavid Valin return (1); 2690b942e89bSDavid Valin } 2691b942e89bSDavid Valin 2692b942e89bSDavid Valin /* 2693b942e89bSDavid Valin * We couldn't allocate an empty magazine, 2694b942e89bSDavid Valin * so fall through to the slab layer. 2695b942e89bSDavid Valin */ 2696b942e89bSDavid Valin return (0); 2697b942e89bSDavid Valin } 2698b942e89bSDavid Valin 2699b5fca8f8Stomee /* 2700b5fca8f8Stomee * Free a constructed object to cache cp. 2701b5fca8f8Stomee */ 2702b5fca8f8Stomee void 2703b5fca8f8Stomee kmem_cache_free(kmem_cache_t *cp, void *buf) 2704b5fca8f8Stomee { 2705b5fca8f8Stomee kmem_cpu_cache_t *ccp = KMEM_CPU_CACHE(cp); 2706b5fca8f8Stomee 2707b5fca8f8Stomee /* 2708b5fca8f8Stomee * The client must not free either of the buffers passed to the move 2709b5fca8f8Stomee * callback function. 2710b5fca8f8Stomee */ 2711b5fca8f8Stomee ASSERT(cp->cache_defrag == NULL || 2712b5fca8f8Stomee cp->cache_defrag->kmd_thread != curthread || 2713b5fca8f8Stomee (buf != cp->cache_defrag->kmd_from_buf && 2714b5fca8f8Stomee buf != cp->cache_defrag->kmd_to_buf)); 2715b5fca8f8Stomee 27169dd77bc8SDave Plauger if (ccp->cc_flags & (KMF_BUFTAG | KMF_DUMPDIVERT | KMF_DUMPUNSAFE)) { 27179dd77bc8SDave Plauger if (ccp->cc_flags & KMF_DUMPUNSAFE) { 27189dd77bc8SDave Plauger ASSERT(!(ccp->cc_flags & KMF_DUMPDIVERT)); 27199dd77bc8SDave Plauger /* log it so that we can warn about it */ 27209dd77bc8SDave Plauger KDI_LOG(cp, kdl_unsafe); 27219dd77bc8SDave Plauger } else if (KMEM_DUMPCC(ccp) && !kmem_cache_free_dump(cp, buf)) { 2722b5fca8f8Stomee return; 27239dd77bc8SDave Plauger } 27249dd77bc8SDave Plauger if (ccp->cc_flags & KMF_BUFTAG) { 27259dd77bc8SDave Plauger if (kmem_cache_free_debug(cp, buf, caller()) == -1) 27269dd77bc8SDave Plauger return; 27279dd77bc8SDave Plauger } 27289dd77bc8SDave Plauger } 2729b5fca8f8Stomee 2730b5fca8f8Stomee mutex_enter(&ccp->cc_lock); 2731b942e89bSDavid Valin /* 2732b942e89bSDavid Valin * Any changes to this logic should be reflected in kmem_slab_prefill() 2733b942e89bSDavid Valin */ 2734b5fca8f8Stomee for (;;) { 2735b5fca8f8Stomee /* 2736b5fca8f8Stomee * If there's a slot available in the current CPU's 2737b5fca8f8Stomee * loaded magazine, just put the object there and return. 2738b5fca8f8Stomee */ 2739b5fca8f8Stomee if ((uint_t)ccp->cc_rounds < ccp->cc_magsize) { 2740b5fca8f8Stomee ccp->cc_loaded->mag_round[ccp->cc_rounds++] = buf; 2741b5fca8f8Stomee ccp->cc_free++; 2742b5fca8f8Stomee mutex_exit(&ccp->cc_lock); 2743b5fca8f8Stomee return; 2744b5fca8f8Stomee } 2745b5fca8f8Stomee 27467c478bd9Sstevel@tonic-gate /* 27477c478bd9Sstevel@tonic-gate * The loaded magazine is full. If the previously loaded 27487c478bd9Sstevel@tonic-gate * magazine was empty, exchange them and try again. 27497c478bd9Sstevel@tonic-gate */ 27507c478bd9Sstevel@tonic-gate if (ccp->cc_prounds == 0) { 27517c478bd9Sstevel@tonic-gate kmem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds); 27527c478bd9Sstevel@tonic-gate continue; 27537c478bd9Sstevel@tonic-gate } 27547c478bd9Sstevel@tonic-gate 27557c478bd9Sstevel@tonic-gate /* 27567c478bd9Sstevel@tonic-gate * If the magazine layer is disabled, break out now. 27577c478bd9Sstevel@tonic-gate */ 27587c478bd9Sstevel@tonic-gate if (ccp->cc_magsize == 0) 27597c478bd9Sstevel@tonic-gate break; 27607c478bd9Sstevel@tonic-gate 2761b942e89bSDavid Valin if (!kmem_cpucache_magazine_alloc(ccp, cp)) { 2762b942e89bSDavid Valin /* 2763b942e89bSDavid Valin * We couldn't free our constructed object to the 2764b942e89bSDavid Valin * magazine layer, so apply its destructor and free it 2765b942e89bSDavid Valin * to the slab layer. 2766b942e89bSDavid Valin */ 2767b942e89bSDavid Valin break; 2768b942e89bSDavid Valin } 2769b942e89bSDavid Valin } 2770b942e89bSDavid Valin mutex_exit(&ccp->cc_lock); 2771b942e89bSDavid Valin kmem_slab_free_constructed(cp, buf, B_TRUE); 2772b942e89bSDavid Valin } 2773b942e89bSDavid Valin 2774b942e89bSDavid Valin static void 2775b942e89bSDavid Valin kmem_slab_prefill(kmem_cache_t *cp, kmem_slab_t *sp) 2776b942e89bSDavid Valin { 2777b942e89bSDavid Valin kmem_cpu_cache_t *ccp = KMEM_CPU_CACHE(cp); 2778b942e89bSDavid Valin int cache_flags = cp->cache_flags; 2779b942e89bSDavid Valin 2780b942e89bSDavid Valin kmem_bufctl_t *next, *head; 2781b942e89bSDavid Valin size_t nbufs; 2782b942e89bSDavid Valin 2783b942e89bSDavid Valin /* 2784b942e89bSDavid Valin * Completely allocate the newly created slab and put the pre-allocated 2785b942e89bSDavid Valin * buffers in magazines. Any of the buffers that cannot be put in 2786b942e89bSDavid Valin * magazines must be returned to the slab. 2787b942e89bSDavid Valin */ 2788b942e89bSDavid Valin ASSERT(MUTEX_HELD(&cp->cache_lock)); 2789b942e89bSDavid Valin ASSERT((cache_flags & (KMF_PREFILL|KMF_BUFTAG)) == KMF_PREFILL); 2790b942e89bSDavid Valin ASSERT(cp->cache_constructor == NULL); 2791b942e89bSDavid Valin ASSERT(sp->slab_cache == cp); 2792b942e89bSDavid Valin ASSERT(sp->slab_refcnt == 1); 2793b942e89bSDavid Valin ASSERT(sp->slab_head != NULL && sp->slab_chunks > sp->slab_refcnt); 2794b942e89bSDavid Valin ASSERT(avl_find(&cp->cache_partial_slabs, sp, NULL) == NULL); 2795b942e89bSDavid Valin 2796b942e89bSDavid Valin head = sp->slab_head; 2797b942e89bSDavid Valin nbufs = (sp->slab_chunks - sp->slab_refcnt); 2798b942e89bSDavid Valin sp->slab_head = NULL; 2799b942e89bSDavid Valin sp->slab_refcnt += nbufs; 2800b942e89bSDavid Valin cp->cache_bufslab -= nbufs; 2801b942e89bSDavid Valin cp->cache_slab_alloc += nbufs; 2802b942e89bSDavid Valin list_insert_head(&cp->cache_complete_slabs, sp); 2803b942e89bSDavid Valin cp->cache_complete_slab_count++; 2804b942e89bSDavid Valin mutex_exit(&cp->cache_lock); 2805b942e89bSDavid Valin mutex_enter(&ccp->cc_lock); 2806b942e89bSDavid Valin 2807b942e89bSDavid Valin while (head != NULL) { 2808b942e89bSDavid Valin void *buf = KMEM_BUF(cp, head); 28097c478bd9Sstevel@tonic-gate /* 2810b942e89bSDavid Valin * If there's a slot available in the current CPU's 2811b942e89bSDavid Valin * loaded magazine, just put the object there and 2812b942e89bSDavid Valin * continue. 28137c478bd9Sstevel@tonic-gate */ 2814b942e89bSDavid Valin if ((uint_t)ccp->cc_rounds < ccp->cc_magsize) { 2815b942e89bSDavid Valin ccp->cc_loaded->mag_round[ccp->cc_rounds++] = 2816b942e89bSDavid Valin buf; 2817b942e89bSDavid Valin ccp->cc_free++; 2818b942e89bSDavid Valin nbufs--; 2819b942e89bSDavid Valin head = head->bc_next; 28207c478bd9Sstevel@tonic-gate continue; 28217c478bd9Sstevel@tonic-gate } 28227c478bd9Sstevel@tonic-gate 28237c478bd9Sstevel@tonic-gate /* 2824b942e89bSDavid Valin * The loaded magazine is full. If the previously 2825b942e89bSDavid Valin * loaded magazine was empty, exchange them and try 2826b942e89bSDavid Valin * again. 28277c478bd9Sstevel@tonic-gate */ 2828b942e89bSDavid Valin if (ccp->cc_prounds == 0) { 2829b942e89bSDavid Valin kmem_cpu_reload(ccp, ccp->cc_ploaded, 2830b942e89bSDavid Valin ccp->cc_prounds); 28317c478bd9Sstevel@tonic-gate continue; 28327c478bd9Sstevel@tonic-gate } 28337c478bd9Sstevel@tonic-gate 28347c478bd9Sstevel@tonic-gate /* 2835b942e89bSDavid Valin * If the magazine layer is disabled, break out now. 28367c478bd9Sstevel@tonic-gate */ 2837b942e89bSDavid Valin 2838b942e89bSDavid Valin if (ccp->cc_magsize == 0) { 2839b942e89bSDavid Valin break; 2840b942e89bSDavid Valin } 2841b942e89bSDavid Valin 2842b942e89bSDavid Valin if (!kmem_cpucache_magazine_alloc(ccp, cp)) 2843b942e89bSDavid Valin break; 28447c478bd9Sstevel@tonic-gate } 28457c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 2846b942e89bSDavid Valin if (nbufs != 0) { 2847b942e89bSDavid Valin ASSERT(head != NULL); 28487c478bd9Sstevel@tonic-gate 2849b942e89bSDavid Valin /* 2850b942e89bSDavid Valin * If there was a failure, return remaining objects to 2851b942e89bSDavid Valin * the slab 2852b942e89bSDavid Valin */ 2853b942e89bSDavid Valin while (head != NULL) { 2854b942e89bSDavid Valin ASSERT(nbufs != 0); 2855b942e89bSDavid Valin next = head->bc_next; 2856b942e89bSDavid Valin head->bc_next = NULL; 2857b942e89bSDavid Valin kmem_slab_free(cp, KMEM_BUF(cp, head)); 2858b942e89bSDavid Valin head = next; 2859b942e89bSDavid Valin nbufs--; 2860b942e89bSDavid Valin } 2861b942e89bSDavid Valin } 2862b942e89bSDavid Valin ASSERT(head == NULL); 2863b942e89bSDavid Valin ASSERT(nbufs == 0); 2864b942e89bSDavid Valin mutex_enter(&cp->cache_lock); 28657c478bd9Sstevel@tonic-gate } 28667c478bd9Sstevel@tonic-gate 28677c478bd9Sstevel@tonic-gate void * 28687c478bd9Sstevel@tonic-gate kmem_zalloc(size_t size, int kmflag) 28697c478bd9Sstevel@tonic-gate { 2870dce01e3fSJonathan W Adams size_t index; 28717c478bd9Sstevel@tonic-gate void *buf; 28727c478bd9Sstevel@tonic-gate 2873dce01e3fSJonathan W Adams if ((index = ((size - 1) >> KMEM_ALIGN_SHIFT)) < KMEM_ALLOC_TABLE_MAX) { 28747c478bd9Sstevel@tonic-gate kmem_cache_t *cp = kmem_alloc_table[index]; 28757c478bd9Sstevel@tonic-gate buf = kmem_cache_alloc(cp, kmflag); 28767c478bd9Sstevel@tonic-gate if (buf != NULL) { 28779dd77bc8SDave Plauger if ((cp->cache_flags & KMF_BUFTAG) && !KMEM_DUMP(cp)) { 28787c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 28797c478bd9Sstevel@tonic-gate ((uint8_t *)buf)[size] = KMEM_REDZONE_BYTE; 28807c478bd9Sstevel@tonic-gate ((uint32_t *)btp)[1] = KMEM_SIZE_ENCODE(size); 28817c478bd9Sstevel@tonic-gate 28827c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) { 28837c478bd9Sstevel@tonic-gate KMEM_BUFTAG_LITE_ENTER(btp, 28847c478bd9Sstevel@tonic-gate kmem_lite_count, caller()); 28857c478bd9Sstevel@tonic-gate } 28867c478bd9Sstevel@tonic-gate } 28877c478bd9Sstevel@tonic-gate bzero(buf, size); 28887c478bd9Sstevel@tonic-gate } 28897c478bd9Sstevel@tonic-gate } else { 28907c478bd9Sstevel@tonic-gate buf = kmem_alloc(size, kmflag); 28917c478bd9Sstevel@tonic-gate if (buf != NULL) 28927c478bd9Sstevel@tonic-gate bzero(buf, size); 28937c478bd9Sstevel@tonic-gate } 28947c478bd9Sstevel@tonic-gate return (buf); 28957c478bd9Sstevel@tonic-gate } 28967c478bd9Sstevel@tonic-gate 28977c478bd9Sstevel@tonic-gate void * 28987c478bd9Sstevel@tonic-gate kmem_alloc(size_t size, int kmflag) 28997c478bd9Sstevel@tonic-gate { 2900dce01e3fSJonathan W Adams size_t index; 2901dce01e3fSJonathan W Adams kmem_cache_t *cp; 29027c478bd9Sstevel@tonic-gate void *buf; 29037c478bd9Sstevel@tonic-gate 2904dce01e3fSJonathan W Adams if ((index = ((size - 1) >> KMEM_ALIGN_SHIFT)) < KMEM_ALLOC_TABLE_MAX) { 2905dce01e3fSJonathan W Adams cp = kmem_alloc_table[index]; 2906dce01e3fSJonathan W Adams /* fall through to kmem_cache_alloc() */ 29077c478bd9Sstevel@tonic-gate 2908dce01e3fSJonathan W Adams } else if ((index = ((size - 1) >> KMEM_BIG_SHIFT)) < 2909dce01e3fSJonathan W Adams kmem_big_alloc_table_max) { 2910dce01e3fSJonathan W Adams cp = kmem_big_alloc_table[index]; 2911dce01e3fSJonathan W Adams /* fall through to kmem_cache_alloc() */ 2912dce01e3fSJonathan W Adams 2913dce01e3fSJonathan W Adams } else { 2914dce01e3fSJonathan W Adams if (size == 0) 2915dce01e3fSJonathan W Adams return (NULL); 2916dce01e3fSJonathan W Adams 2917dce01e3fSJonathan W Adams buf = vmem_alloc(kmem_oversize_arena, size, 2918dce01e3fSJonathan W Adams kmflag & KM_VMFLAGS); 2919dce01e3fSJonathan W Adams if (buf == NULL) 2920dce01e3fSJonathan W Adams kmem_log_event(kmem_failure_log, NULL, NULL, 2921dce01e3fSJonathan W Adams (void *)size); 29229dd77bc8SDave Plauger else if (KMEM_DUMP(kmem_slab_cache)) { 29239dd77bc8SDave Plauger /* stats for dump intercept */ 29249dd77bc8SDave Plauger kmem_dump_oversize_allocs++; 29259dd77bc8SDave Plauger if (size > kmem_dump_oversize_max) 29269dd77bc8SDave Plauger kmem_dump_oversize_max = size; 29279dd77bc8SDave Plauger } 29287c478bd9Sstevel@tonic-gate return (buf); 29297c478bd9Sstevel@tonic-gate } 2930dce01e3fSJonathan W Adams 2931dce01e3fSJonathan W Adams buf = kmem_cache_alloc(cp, kmflag); 29329dd77bc8SDave Plauger if ((cp->cache_flags & KMF_BUFTAG) && !KMEM_DUMP(cp) && buf != NULL) { 2933dce01e3fSJonathan W Adams kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 2934dce01e3fSJonathan W Adams ((uint8_t *)buf)[size] = KMEM_REDZONE_BYTE; 2935dce01e3fSJonathan W Adams ((uint32_t *)btp)[1] = KMEM_SIZE_ENCODE(size); 2936dce01e3fSJonathan W Adams 2937dce01e3fSJonathan W Adams if (cp->cache_flags & KMF_LITE) { 2938dce01e3fSJonathan W Adams KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, caller()); 2939dce01e3fSJonathan W Adams } 2940dce01e3fSJonathan W Adams } 29417c478bd9Sstevel@tonic-gate return (buf); 29427c478bd9Sstevel@tonic-gate } 29437c478bd9Sstevel@tonic-gate 29447c478bd9Sstevel@tonic-gate void 29457c478bd9Sstevel@tonic-gate kmem_free(void *buf, size_t size) 29467c478bd9Sstevel@tonic-gate { 2947dce01e3fSJonathan W Adams size_t index; 2948dce01e3fSJonathan W Adams kmem_cache_t *cp; 29497c478bd9Sstevel@tonic-gate 2950dce01e3fSJonathan W Adams if ((index = (size - 1) >> KMEM_ALIGN_SHIFT) < KMEM_ALLOC_TABLE_MAX) { 2951dce01e3fSJonathan W Adams cp = kmem_alloc_table[index]; 2952dce01e3fSJonathan W Adams /* fall through to kmem_cache_free() */ 2953dce01e3fSJonathan W Adams 2954dce01e3fSJonathan W Adams } else if ((index = ((size - 1) >> KMEM_BIG_SHIFT)) < 2955dce01e3fSJonathan W Adams kmem_big_alloc_table_max) { 2956dce01e3fSJonathan W Adams cp = kmem_big_alloc_table[index]; 2957dce01e3fSJonathan W Adams /* fall through to kmem_cache_free() */ 2958dce01e3fSJonathan W Adams 2959dce01e3fSJonathan W Adams } else { 296096992ee7SEthindra Ramamurthy EQUIV(buf == NULL, size == 0); 2961dce01e3fSJonathan W Adams if (buf == NULL && size == 0) 2962dce01e3fSJonathan W Adams return; 2963dce01e3fSJonathan W Adams vmem_free(kmem_oversize_arena, buf, size); 2964dce01e3fSJonathan W Adams return; 2965dce01e3fSJonathan W Adams } 2966dce01e3fSJonathan W Adams 29679dd77bc8SDave Plauger if ((cp->cache_flags & KMF_BUFTAG) && !KMEM_DUMP(cp)) { 2968dce01e3fSJonathan W Adams kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 2969dce01e3fSJonathan W Adams uint32_t *ip = (uint32_t *)btp; 2970dce01e3fSJonathan W Adams if (ip[1] != KMEM_SIZE_ENCODE(size)) { 2971dce01e3fSJonathan W Adams if (*(uint64_t *)buf == KMEM_FREE_PATTERN) { 2972dce01e3fSJonathan W Adams kmem_error(KMERR_DUPFREE, cp, buf); 29737c478bd9Sstevel@tonic-gate return; 29747c478bd9Sstevel@tonic-gate } 2975dce01e3fSJonathan W Adams if (KMEM_SIZE_VALID(ip[1])) { 2976dce01e3fSJonathan W Adams ip[0] = KMEM_SIZE_ENCODE(size); 2977dce01e3fSJonathan W Adams kmem_error(KMERR_BADSIZE, cp, buf); 2978dce01e3fSJonathan W Adams } else { 29797c478bd9Sstevel@tonic-gate kmem_error(KMERR_REDZONE, cp, buf); 29807c478bd9Sstevel@tonic-gate } 2981dce01e3fSJonathan W Adams return; 29827c478bd9Sstevel@tonic-gate } 2983dce01e3fSJonathan W Adams if (((uint8_t *)buf)[size] != KMEM_REDZONE_BYTE) { 2984dce01e3fSJonathan W Adams kmem_error(KMERR_REDZONE, cp, buf); 29857c478bd9Sstevel@tonic-gate return; 2986dce01e3fSJonathan W Adams } 2987dce01e3fSJonathan W Adams btp->bt_redzone = KMEM_REDZONE_PATTERN; 2988dce01e3fSJonathan W Adams if (cp->cache_flags & KMF_LITE) { 2989dce01e3fSJonathan W Adams KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, 2990dce01e3fSJonathan W Adams caller()); 2991dce01e3fSJonathan W Adams } 29927c478bd9Sstevel@tonic-gate } 2993dce01e3fSJonathan W Adams kmem_cache_free(cp, buf); 29947c478bd9Sstevel@tonic-gate } 29957c478bd9Sstevel@tonic-gate 29967c478bd9Sstevel@tonic-gate void * 29977c478bd9Sstevel@tonic-gate kmem_firewall_va_alloc(vmem_t *vmp, size_t size, int vmflag) 29987c478bd9Sstevel@tonic-gate { 29997c478bd9Sstevel@tonic-gate size_t realsize = size + vmp->vm_quantum; 30007c478bd9Sstevel@tonic-gate void *addr; 30017c478bd9Sstevel@tonic-gate 30027c478bd9Sstevel@tonic-gate /* 30037c478bd9Sstevel@tonic-gate * Annoying edge case: if 'size' is just shy of ULONG_MAX, adding 30047c478bd9Sstevel@tonic-gate * vm_quantum will cause integer wraparound. Check for this, and 30057c478bd9Sstevel@tonic-gate * blow off the firewall page in this case. Note that such a 30067c478bd9Sstevel@tonic-gate * giant allocation (the entire kernel address space) can never 30077c478bd9Sstevel@tonic-gate * be satisfied, so it will either fail immediately (VM_NOSLEEP) 30087c478bd9Sstevel@tonic-gate * or sleep forever (VM_SLEEP). Thus, there is no need for a 30097c478bd9Sstevel@tonic-gate * corresponding check in kmem_firewall_va_free(). 30107c478bd9Sstevel@tonic-gate */ 30117c478bd9Sstevel@tonic-gate if (realsize < size) 30127c478bd9Sstevel@tonic-gate realsize = size; 30137c478bd9Sstevel@tonic-gate 30147c478bd9Sstevel@tonic-gate /* 30157c478bd9Sstevel@tonic-gate * While boot still owns resource management, make sure that this 30167c478bd9Sstevel@tonic-gate * redzone virtual address allocation is properly accounted for in 30177c478bd9Sstevel@tonic-gate * OBPs "virtual-memory" "available" lists because we're 30187c478bd9Sstevel@tonic-gate * effectively claiming them for a red zone. If we don't do this, 30197c478bd9Sstevel@tonic-gate * the available lists become too fragmented and too large for the 30207c478bd9Sstevel@tonic-gate * current boot/kernel memory list interface. 30217c478bd9Sstevel@tonic-gate */ 30227c478bd9Sstevel@tonic-gate addr = vmem_alloc(vmp, realsize, vmflag | VM_NEXTFIT); 30237c478bd9Sstevel@tonic-gate 30247c478bd9Sstevel@tonic-gate if (addr != NULL && kvseg.s_base == NULL && realsize != size) 30257c478bd9Sstevel@tonic-gate (void) boot_virt_alloc((char *)addr + size, vmp->vm_quantum); 30267c478bd9Sstevel@tonic-gate 30277c478bd9Sstevel@tonic-gate return (addr); 30287c478bd9Sstevel@tonic-gate } 30297c478bd9Sstevel@tonic-gate 30307c478bd9Sstevel@tonic-gate void 30317c478bd9Sstevel@tonic-gate kmem_firewall_va_free(vmem_t *vmp, void *addr, size_t size) 30327c478bd9Sstevel@tonic-gate { 30337c478bd9Sstevel@tonic-gate ASSERT((kvseg.s_base == NULL ? 30347c478bd9Sstevel@tonic-gate va_to_pfn((char *)addr + size) : 30357c478bd9Sstevel@tonic-gate hat_getpfnum(kas.a_hat, (caddr_t)addr + size)) == PFN_INVALID); 30367c478bd9Sstevel@tonic-gate 30377c478bd9Sstevel@tonic-gate vmem_free(vmp, addr, size + vmp->vm_quantum); 30387c478bd9Sstevel@tonic-gate } 30397c478bd9Sstevel@tonic-gate 30407c478bd9Sstevel@tonic-gate /* 30417c478bd9Sstevel@tonic-gate * Try to allocate at least `size' bytes of memory without sleeping or 30427c478bd9Sstevel@tonic-gate * panicking. Return actual allocated size in `asize'. If allocation failed, 30437c478bd9Sstevel@tonic-gate * try final allocation with sleep or panic allowed. 30447c478bd9Sstevel@tonic-gate */ 30457c478bd9Sstevel@tonic-gate void * 30467c478bd9Sstevel@tonic-gate kmem_alloc_tryhard(size_t size, size_t *asize, int kmflag) 30477c478bd9Sstevel@tonic-gate { 30487c478bd9Sstevel@tonic-gate void *p; 30497c478bd9Sstevel@tonic-gate 30507c478bd9Sstevel@tonic-gate *asize = P2ROUNDUP(size, KMEM_ALIGN); 30517c478bd9Sstevel@tonic-gate do { 30527c478bd9Sstevel@tonic-gate p = kmem_alloc(*asize, (kmflag | KM_NOSLEEP) & ~KM_PANIC); 30537c478bd9Sstevel@tonic-gate if (p != NULL) 30547c478bd9Sstevel@tonic-gate return (p); 30557c478bd9Sstevel@tonic-gate *asize += KMEM_ALIGN; 30567c478bd9Sstevel@tonic-gate } while (*asize <= PAGESIZE); 30577c478bd9Sstevel@tonic-gate 30587c478bd9Sstevel@tonic-gate *asize = P2ROUNDUP(size, KMEM_ALIGN); 30597c478bd9Sstevel@tonic-gate return (kmem_alloc(*asize, kmflag)); 30607c478bd9Sstevel@tonic-gate } 30617c478bd9Sstevel@tonic-gate 30627c478bd9Sstevel@tonic-gate /* 30637c478bd9Sstevel@tonic-gate * Reclaim all unused memory from a cache. 30647c478bd9Sstevel@tonic-gate */ 30657c478bd9Sstevel@tonic-gate static void 30667c478bd9Sstevel@tonic-gate kmem_cache_reap(kmem_cache_t *cp) 30677c478bd9Sstevel@tonic-gate { 3068b5fca8f8Stomee ASSERT(taskq_member(kmem_taskq, curthread)); 3069686031edSTom Erickson cp->cache_reap++; 3070b5fca8f8Stomee 30717c478bd9Sstevel@tonic-gate /* 30727c478bd9Sstevel@tonic-gate * Ask the cache's owner to free some memory if possible. 30737c478bd9Sstevel@tonic-gate * The idea is to handle things like the inode cache, which 30747c478bd9Sstevel@tonic-gate * typically sits on a bunch of memory that it doesn't truly 30757c478bd9Sstevel@tonic-gate * *need*. Reclaim policy is entirely up to the owner; this 30767c478bd9Sstevel@tonic-gate * callback is just an advisory plea for help. 30777c478bd9Sstevel@tonic-gate */ 3078b5fca8f8Stomee if (cp->cache_reclaim != NULL) { 3079b5fca8f8Stomee long delta; 3080b5fca8f8Stomee 3081b5fca8f8Stomee /* 3082b5fca8f8Stomee * Reclaimed memory should be reapable (not included in the 3083b5fca8f8Stomee * depot's working set). 3084b5fca8f8Stomee */ 3085b5fca8f8Stomee delta = cp->cache_full.ml_total; 30867c478bd9Sstevel@tonic-gate cp->cache_reclaim(cp->cache_private); 3087b5fca8f8Stomee delta = cp->cache_full.ml_total - delta; 3088b5fca8f8Stomee if (delta > 0) { 3089b5fca8f8Stomee mutex_enter(&cp->cache_depot_lock); 3090b5fca8f8Stomee cp->cache_full.ml_reaplimit += delta; 3091b5fca8f8Stomee cp->cache_full.ml_min += delta; 3092b5fca8f8Stomee mutex_exit(&cp->cache_depot_lock); 3093b5fca8f8Stomee } 3094b5fca8f8Stomee } 30957c478bd9Sstevel@tonic-gate 30967c478bd9Sstevel@tonic-gate kmem_depot_ws_reap(cp); 3097b5fca8f8Stomee 3098b5fca8f8Stomee if (cp->cache_defrag != NULL && !kmem_move_noreap) { 3099b5fca8f8Stomee kmem_cache_defrag(cp); 3100b5fca8f8Stomee } 31017c478bd9Sstevel@tonic-gate } 31027c478bd9Sstevel@tonic-gate 31037c478bd9Sstevel@tonic-gate static void 31047c478bd9Sstevel@tonic-gate kmem_reap_timeout(void *flag_arg) 31057c478bd9Sstevel@tonic-gate { 31067c478bd9Sstevel@tonic-gate uint32_t *flag = (uint32_t *)flag_arg; 31077c478bd9Sstevel@tonic-gate 31087c478bd9Sstevel@tonic-gate ASSERT(flag == &kmem_reaping || flag == &kmem_reaping_idspace); 31097c478bd9Sstevel@tonic-gate *flag = 0; 31107c478bd9Sstevel@tonic-gate } 31117c478bd9Sstevel@tonic-gate 31127c478bd9Sstevel@tonic-gate static void 31137c478bd9Sstevel@tonic-gate kmem_reap_done(void *flag) 31147c478bd9Sstevel@tonic-gate { 31156e00b116SPeter Telford if (!callout_init_done) { 31166e00b116SPeter Telford /* can't schedule a timeout at this point */ 31176e00b116SPeter Telford kmem_reap_timeout(flag); 31186e00b116SPeter Telford } else { 31196e00b116SPeter Telford (void) timeout(kmem_reap_timeout, flag, kmem_reap_interval); 31206e00b116SPeter Telford } 31217c478bd9Sstevel@tonic-gate } 31227c478bd9Sstevel@tonic-gate 31237c478bd9Sstevel@tonic-gate static void 31247c478bd9Sstevel@tonic-gate kmem_reap_start(void *flag) 31257c478bd9Sstevel@tonic-gate { 31267c478bd9Sstevel@tonic-gate ASSERT(flag == &kmem_reaping || flag == &kmem_reaping_idspace); 31277c478bd9Sstevel@tonic-gate 31287c478bd9Sstevel@tonic-gate if (flag == &kmem_reaping) { 31297c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_reap, kmem_taskq, TQ_NOSLEEP); 31307c478bd9Sstevel@tonic-gate /* 31317c478bd9Sstevel@tonic-gate * if we have segkp under heap, reap segkp cache. 31327c478bd9Sstevel@tonic-gate */ 31337c478bd9Sstevel@tonic-gate if (segkp_fromheap) 31347c478bd9Sstevel@tonic-gate segkp_cache_free(); 31357c478bd9Sstevel@tonic-gate } 31367c478bd9Sstevel@tonic-gate else 31377c478bd9Sstevel@tonic-gate kmem_cache_applyall_id(kmem_cache_reap, kmem_taskq, TQ_NOSLEEP); 31387c478bd9Sstevel@tonic-gate 31397c478bd9Sstevel@tonic-gate /* 31407c478bd9Sstevel@tonic-gate * We use taskq_dispatch() to schedule a timeout to clear 31417c478bd9Sstevel@tonic-gate * the flag so that kmem_reap() becomes self-throttling: 31427c478bd9Sstevel@tonic-gate * we won't reap again until the current reap completes *and* 31437c478bd9Sstevel@tonic-gate * at least kmem_reap_interval ticks have elapsed. 31447c478bd9Sstevel@tonic-gate */ 31457c478bd9Sstevel@tonic-gate if (!taskq_dispatch(kmem_taskq, kmem_reap_done, flag, TQ_NOSLEEP)) 31467c478bd9Sstevel@tonic-gate kmem_reap_done(flag); 31477c478bd9Sstevel@tonic-gate } 31487c478bd9Sstevel@tonic-gate 31497c478bd9Sstevel@tonic-gate static void 31507c478bd9Sstevel@tonic-gate kmem_reap_common(void *flag_arg) 31517c478bd9Sstevel@tonic-gate { 31527c478bd9Sstevel@tonic-gate uint32_t *flag = (uint32_t *)flag_arg; 31537c478bd9Sstevel@tonic-gate 31547c478bd9Sstevel@tonic-gate if (MUTEX_HELD(&kmem_cache_lock) || kmem_taskq == NULL || 315575d94465SJosef 'Jeff' Sipek atomic_cas_32(flag, 0, 1) != 0) 31567c478bd9Sstevel@tonic-gate return; 31577c478bd9Sstevel@tonic-gate 31587c478bd9Sstevel@tonic-gate /* 31597c478bd9Sstevel@tonic-gate * It may not be kosher to do memory allocation when a reap is called 31609321cd04SJosef 'Jeff' Sipek * (for example, if vmem_populate() is in the call chain). So we 31619321cd04SJosef 'Jeff' Sipek * start the reap going with a TQ_NOALLOC dispatch. If the dispatch 31629321cd04SJosef 'Jeff' Sipek * fails, we reset the flag, and the next reap will try again. 31637c478bd9Sstevel@tonic-gate */ 31647c478bd9Sstevel@tonic-gate if (!taskq_dispatch(kmem_taskq, kmem_reap_start, flag, TQ_NOALLOC)) 31657c478bd9Sstevel@tonic-gate *flag = 0; 31667c478bd9Sstevel@tonic-gate } 31677c478bd9Sstevel@tonic-gate 31687c478bd9Sstevel@tonic-gate /* 31697c478bd9Sstevel@tonic-gate * Reclaim all unused memory from all caches. Called from the VM system 31707c478bd9Sstevel@tonic-gate * when memory gets tight. 31717c478bd9Sstevel@tonic-gate */ 31727c478bd9Sstevel@tonic-gate void 31737c478bd9Sstevel@tonic-gate kmem_reap(void) 31747c478bd9Sstevel@tonic-gate { 31757c478bd9Sstevel@tonic-gate kmem_reap_common(&kmem_reaping); 31767c478bd9Sstevel@tonic-gate } 31777c478bd9Sstevel@tonic-gate 31787c478bd9Sstevel@tonic-gate /* 31797c478bd9Sstevel@tonic-gate * Reclaim all unused memory from identifier arenas, called when a vmem 31807c478bd9Sstevel@tonic-gate * arena not back by memory is exhausted. Since reaping memory-backed caches 31817c478bd9Sstevel@tonic-gate * cannot help with identifier exhaustion, we avoid both a large amount of 31827c478bd9Sstevel@tonic-gate * work and unwanted side-effects from reclaim callbacks. 31837c478bd9Sstevel@tonic-gate */ 31847c478bd9Sstevel@tonic-gate void 31857c478bd9Sstevel@tonic-gate kmem_reap_idspace(void) 31867c478bd9Sstevel@tonic-gate { 31877c478bd9Sstevel@tonic-gate kmem_reap_common(&kmem_reaping_idspace); 31887c478bd9Sstevel@tonic-gate } 31897c478bd9Sstevel@tonic-gate 31907c478bd9Sstevel@tonic-gate /* 31917c478bd9Sstevel@tonic-gate * Purge all magazines from a cache and set its magazine limit to zero. 31927c478bd9Sstevel@tonic-gate * All calls are serialized by the kmem_taskq lock, except for the final 31937c478bd9Sstevel@tonic-gate * call from kmem_cache_destroy(). 31947c478bd9Sstevel@tonic-gate */ 31957c478bd9Sstevel@tonic-gate static void 31967c478bd9Sstevel@tonic-gate kmem_cache_magazine_purge(kmem_cache_t *cp) 31977c478bd9Sstevel@tonic-gate { 31987c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp; 31997c478bd9Sstevel@tonic-gate kmem_magazine_t *mp, *pmp; 32007c478bd9Sstevel@tonic-gate int rounds, prounds, cpu_seqid; 32017c478bd9Sstevel@tonic-gate 3202b5fca8f8Stomee ASSERT(!list_link_active(&cp->cache_link) || 3203b5fca8f8Stomee taskq_member(kmem_taskq, curthread)); 32047c478bd9Sstevel@tonic-gate ASSERT(MUTEX_NOT_HELD(&cp->cache_lock)); 32057c478bd9Sstevel@tonic-gate 32067c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 32077c478bd9Sstevel@tonic-gate ccp = &cp->cache_cpu[cpu_seqid]; 32087c478bd9Sstevel@tonic-gate 32097c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 32107c478bd9Sstevel@tonic-gate mp = ccp->cc_loaded; 32117c478bd9Sstevel@tonic-gate pmp = ccp->cc_ploaded; 32127c478bd9Sstevel@tonic-gate rounds = ccp->cc_rounds; 32137c478bd9Sstevel@tonic-gate prounds = ccp->cc_prounds; 32147c478bd9Sstevel@tonic-gate ccp->cc_loaded = NULL; 32157c478bd9Sstevel@tonic-gate ccp->cc_ploaded = NULL; 32167c478bd9Sstevel@tonic-gate ccp->cc_rounds = -1; 32177c478bd9Sstevel@tonic-gate ccp->cc_prounds = -1; 32187c478bd9Sstevel@tonic-gate ccp->cc_magsize = 0; 32197c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 32207c478bd9Sstevel@tonic-gate 32217c478bd9Sstevel@tonic-gate if (mp) 32227c478bd9Sstevel@tonic-gate kmem_magazine_destroy(cp, mp, rounds); 32237c478bd9Sstevel@tonic-gate if (pmp) 32247c478bd9Sstevel@tonic-gate kmem_magazine_destroy(cp, pmp, prounds); 32257c478bd9Sstevel@tonic-gate } 32267c478bd9Sstevel@tonic-gate 32270c833d64SJosef 'Jeff' Sipek kmem_depot_ws_zero(cp); 32287c478bd9Sstevel@tonic-gate kmem_depot_ws_reap(cp); 32297c478bd9Sstevel@tonic-gate } 32307c478bd9Sstevel@tonic-gate 32317c478bd9Sstevel@tonic-gate /* 32327c478bd9Sstevel@tonic-gate * Enable per-cpu magazines on a cache. 32337c478bd9Sstevel@tonic-gate */ 32347c478bd9Sstevel@tonic-gate static void 32357c478bd9Sstevel@tonic-gate kmem_cache_magazine_enable(kmem_cache_t *cp) 32367c478bd9Sstevel@tonic-gate { 32377c478bd9Sstevel@tonic-gate int cpu_seqid; 32387c478bd9Sstevel@tonic-gate 32397c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_NOMAGAZINE) 32407c478bd9Sstevel@tonic-gate return; 32417c478bd9Sstevel@tonic-gate 32427c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 32437c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 32447c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 32457c478bd9Sstevel@tonic-gate ccp->cc_magsize = cp->cache_magtype->mt_magsize; 32467c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 32477c478bd9Sstevel@tonic-gate } 32487c478bd9Sstevel@tonic-gate 32497c478bd9Sstevel@tonic-gate } 32507c478bd9Sstevel@tonic-gate 3251fa9e4066Sahrens /* 32520c833d64SJosef 'Jeff' Sipek * Reap (almost) everything right now. 3253fa9e4066Sahrens */ 3254fa9e4066Sahrens void 3255fa9e4066Sahrens kmem_cache_reap_now(kmem_cache_t *cp) 3256fa9e4066Sahrens { 3257b5fca8f8Stomee ASSERT(list_link_active(&cp->cache_link)); 3258b5fca8f8Stomee 32590c833d64SJosef 'Jeff' Sipek kmem_depot_ws_zero(cp); 3260fa9e4066Sahrens 3261fa9e4066Sahrens (void) taskq_dispatch(kmem_taskq, 3262fa9e4066Sahrens (task_func_t *)kmem_depot_ws_reap, cp, TQ_SLEEP); 3263fa9e4066Sahrens taskq_wait(kmem_taskq); 3264fa9e4066Sahrens } 3265fa9e4066Sahrens 32667c478bd9Sstevel@tonic-gate /* 32677c478bd9Sstevel@tonic-gate * Recompute a cache's magazine size. The trade-off is that larger magazines 32687c478bd9Sstevel@tonic-gate * provide a higher transfer rate with the depot, while smaller magazines 32697c478bd9Sstevel@tonic-gate * reduce memory consumption. Magazine resizing is an expensive operation; 32707c478bd9Sstevel@tonic-gate * it should not be done frequently. 32717c478bd9Sstevel@tonic-gate * 32727c478bd9Sstevel@tonic-gate * Changes to the magazine size are serialized by the kmem_taskq lock. 32737c478bd9Sstevel@tonic-gate * 32747c478bd9Sstevel@tonic-gate * Note: at present this only grows the magazine size. It might be useful 32757c478bd9Sstevel@tonic-gate * to allow shrinkage too. 32767c478bd9Sstevel@tonic-gate */ 32777c478bd9Sstevel@tonic-gate static void 32787c478bd9Sstevel@tonic-gate kmem_cache_magazine_resize(kmem_cache_t *cp) 32797c478bd9Sstevel@tonic-gate { 32807c478bd9Sstevel@tonic-gate kmem_magtype_t *mtp = cp->cache_magtype; 32817c478bd9Sstevel@tonic-gate 32827c478bd9Sstevel@tonic-gate ASSERT(taskq_member(kmem_taskq, curthread)); 32837c478bd9Sstevel@tonic-gate 32847c478bd9Sstevel@tonic-gate if (cp->cache_chunksize < mtp->mt_maxbuf) { 32857c478bd9Sstevel@tonic-gate kmem_cache_magazine_purge(cp); 32867c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock); 32877c478bd9Sstevel@tonic-gate cp->cache_magtype = ++mtp; 32887c478bd9Sstevel@tonic-gate cp->cache_depot_contention_prev = 32897c478bd9Sstevel@tonic-gate cp->cache_depot_contention + INT_MAX; 32907c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock); 32917c478bd9Sstevel@tonic-gate kmem_cache_magazine_enable(cp); 32927c478bd9Sstevel@tonic-gate } 32937c478bd9Sstevel@tonic-gate } 32947c478bd9Sstevel@tonic-gate 32957c478bd9Sstevel@tonic-gate /* 32967c478bd9Sstevel@tonic-gate * Rescale a cache's hash table, so that the table size is roughly the 32977c478bd9Sstevel@tonic-gate * cache size. We want the average lookup time to be extremely small. 32987c478bd9Sstevel@tonic-gate */ 32997c478bd9Sstevel@tonic-gate static void 33007c478bd9Sstevel@tonic-gate kmem_hash_rescale(kmem_cache_t *cp) 33017c478bd9Sstevel@tonic-gate { 33027c478bd9Sstevel@tonic-gate kmem_bufctl_t **old_table, **new_table, *bcp; 33037c478bd9Sstevel@tonic-gate size_t old_size, new_size, h; 33047c478bd9Sstevel@tonic-gate 33057c478bd9Sstevel@tonic-gate ASSERT(taskq_member(kmem_taskq, curthread)); 33067c478bd9Sstevel@tonic-gate 33077c478bd9Sstevel@tonic-gate new_size = MAX(KMEM_HASH_INITIAL, 33087c478bd9Sstevel@tonic-gate 1 << (highbit(3 * cp->cache_buftotal + 4) - 2)); 33097c478bd9Sstevel@tonic-gate old_size = cp->cache_hash_mask + 1; 33107c478bd9Sstevel@tonic-gate 33117c478bd9Sstevel@tonic-gate if ((old_size >> 1) <= new_size && new_size <= (old_size << 1)) 33127c478bd9Sstevel@tonic-gate return; 33137c478bd9Sstevel@tonic-gate 33147c478bd9Sstevel@tonic-gate new_table = vmem_alloc(kmem_hash_arena, new_size * sizeof (void *), 33157c478bd9Sstevel@tonic-gate VM_NOSLEEP); 33167c478bd9Sstevel@tonic-gate if (new_table == NULL) 33177c478bd9Sstevel@tonic-gate return; 33187c478bd9Sstevel@tonic-gate bzero(new_table, new_size * sizeof (void *)); 33197c478bd9Sstevel@tonic-gate 33207c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 33217c478bd9Sstevel@tonic-gate 33227c478bd9Sstevel@tonic-gate old_size = cp->cache_hash_mask + 1; 33237c478bd9Sstevel@tonic-gate old_table = cp->cache_hash_table; 33247c478bd9Sstevel@tonic-gate 33257c478bd9Sstevel@tonic-gate cp->cache_hash_mask = new_size - 1; 33267c478bd9Sstevel@tonic-gate cp->cache_hash_table = new_table; 33277c478bd9Sstevel@tonic-gate cp->cache_rescale++; 33287c478bd9Sstevel@tonic-gate 33297c478bd9Sstevel@tonic-gate for (h = 0; h < old_size; h++) { 33307c478bd9Sstevel@tonic-gate bcp = old_table[h]; 33317c478bd9Sstevel@tonic-gate while (bcp != NULL) { 33327c478bd9Sstevel@tonic-gate void *addr = bcp->bc_addr; 33337c478bd9Sstevel@tonic-gate kmem_bufctl_t *next_bcp = bcp->bc_next; 33347c478bd9Sstevel@tonic-gate kmem_bufctl_t **hash_bucket = KMEM_HASH(cp, addr); 33357c478bd9Sstevel@tonic-gate bcp->bc_next = *hash_bucket; 33367c478bd9Sstevel@tonic-gate *hash_bucket = bcp; 33377c478bd9Sstevel@tonic-gate bcp = next_bcp; 33387c478bd9Sstevel@tonic-gate } 33397c478bd9Sstevel@tonic-gate } 33407c478bd9Sstevel@tonic-gate 33417c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 33427c478bd9Sstevel@tonic-gate 33437c478bd9Sstevel@tonic-gate vmem_free(kmem_hash_arena, old_table, old_size * sizeof (void *)); 33447c478bd9Sstevel@tonic-gate } 33457c478bd9Sstevel@tonic-gate 33467c478bd9Sstevel@tonic-gate /* 3347b5fca8f8Stomee * Perform periodic maintenance on a cache: hash rescaling, depot working-set 3348b5fca8f8Stomee * update, magazine resizing, and slab consolidation. 33497c478bd9Sstevel@tonic-gate */ 33507c478bd9Sstevel@tonic-gate static void 33517c478bd9Sstevel@tonic-gate kmem_cache_update(kmem_cache_t *cp) 33527c478bd9Sstevel@tonic-gate { 33537c478bd9Sstevel@tonic-gate int need_hash_rescale = 0; 33547c478bd9Sstevel@tonic-gate int need_magazine_resize = 0; 33557c478bd9Sstevel@tonic-gate 33567c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&kmem_cache_lock)); 33577c478bd9Sstevel@tonic-gate 33587c478bd9Sstevel@tonic-gate /* 33597c478bd9Sstevel@tonic-gate * If the cache has become much larger or smaller than its hash table, 33607c478bd9Sstevel@tonic-gate * fire off a request to rescale the hash table. 33617c478bd9Sstevel@tonic-gate */ 33627c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 33637c478bd9Sstevel@tonic-gate 33647c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_HASH) && 33657c478bd9Sstevel@tonic-gate (cp->cache_buftotal > (cp->cache_hash_mask << 1) || 33667c478bd9Sstevel@tonic-gate (cp->cache_buftotal < (cp->cache_hash_mask >> 1) && 33677c478bd9Sstevel@tonic-gate cp->cache_hash_mask > KMEM_HASH_INITIAL))) 33687c478bd9Sstevel@tonic-gate need_hash_rescale = 1; 33697c478bd9Sstevel@tonic-gate 33707c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 33717c478bd9Sstevel@tonic-gate 33727c478bd9Sstevel@tonic-gate /* 33737c478bd9Sstevel@tonic-gate * Update the depot working set statistics. 33747c478bd9Sstevel@tonic-gate */ 33757c478bd9Sstevel@tonic-gate kmem_depot_ws_update(cp); 33767c478bd9Sstevel@tonic-gate 33777c478bd9Sstevel@tonic-gate /* 33787c478bd9Sstevel@tonic-gate * If there's a lot of contention in the depot, 33797c478bd9Sstevel@tonic-gate * increase the magazine size. 33807c478bd9Sstevel@tonic-gate */ 33817c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock); 33827c478bd9Sstevel@tonic-gate 33837c478bd9Sstevel@tonic-gate if (cp->cache_chunksize < cp->cache_magtype->mt_maxbuf && 33847c478bd9Sstevel@tonic-gate (int)(cp->cache_depot_contention - 33857c478bd9Sstevel@tonic-gate cp->cache_depot_contention_prev) > kmem_depot_contention) 33867c478bd9Sstevel@tonic-gate need_magazine_resize = 1; 33877c478bd9Sstevel@tonic-gate 33887c478bd9Sstevel@tonic-gate cp->cache_depot_contention_prev = cp->cache_depot_contention; 33897c478bd9Sstevel@tonic-gate 33907c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock); 33917c478bd9Sstevel@tonic-gate 33927c478bd9Sstevel@tonic-gate if (need_hash_rescale) 33937c478bd9Sstevel@tonic-gate (void) taskq_dispatch(kmem_taskq, 33947c478bd9Sstevel@tonic-gate (task_func_t *)kmem_hash_rescale, cp, TQ_NOSLEEP); 33957c478bd9Sstevel@tonic-gate 33967c478bd9Sstevel@tonic-gate if (need_magazine_resize) 33977c478bd9Sstevel@tonic-gate (void) taskq_dispatch(kmem_taskq, 33987c478bd9Sstevel@tonic-gate (task_func_t *)kmem_cache_magazine_resize, cp, TQ_NOSLEEP); 3399b5fca8f8Stomee 3400b5fca8f8Stomee if (cp->cache_defrag != NULL) 3401b5fca8f8Stomee (void) taskq_dispatch(kmem_taskq, 3402b5fca8f8Stomee (task_func_t *)kmem_cache_scan, cp, TQ_NOSLEEP); 34037c478bd9Sstevel@tonic-gate } 34047c478bd9Sstevel@tonic-gate 3405d67944fbSScott Rotondo static void kmem_update(void *); 3406d67944fbSScott Rotondo 34077c478bd9Sstevel@tonic-gate static void 34087c478bd9Sstevel@tonic-gate kmem_update_timeout(void *dummy) 34097c478bd9Sstevel@tonic-gate { 34107c478bd9Sstevel@tonic-gate (void) timeout(kmem_update, dummy, kmem_reap_interval); 34117c478bd9Sstevel@tonic-gate } 34127c478bd9Sstevel@tonic-gate 34137c478bd9Sstevel@tonic-gate static void 34147c478bd9Sstevel@tonic-gate kmem_update(void *dummy) 34157c478bd9Sstevel@tonic-gate { 34167c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_update, NULL, TQ_NOSLEEP); 34177c478bd9Sstevel@tonic-gate 34187c478bd9Sstevel@tonic-gate /* 34197c478bd9Sstevel@tonic-gate * We use taskq_dispatch() to reschedule the timeout so that 34207c478bd9Sstevel@tonic-gate * kmem_update() becomes self-throttling: it won't schedule 34217c478bd9Sstevel@tonic-gate * new tasks until all previous tasks have completed. 34227c478bd9Sstevel@tonic-gate */ 34237c478bd9Sstevel@tonic-gate if (!taskq_dispatch(kmem_taskq, kmem_update_timeout, dummy, TQ_NOSLEEP)) 34247c478bd9Sstevel@tonic-gate kmem_update_timeout(NULL); 34257c478bd9Sstevel@tonic-gate } 34267c478bd9Sstevel@tonic-gate 34277c478bd9Sstevel@tonic-gate static int 34287c478bd9Sstevel@tonic-gate kmem_cache_kstat_update(kstat_t *ksp, int rw) 34297c478bd9Sstevel@tonic-gate { 34307c478bd9Sstevel@tonic-gate struct kmem_cache_kstat *kmcp = &kmem_cache_kstat; 34317c478bd9Sstevel@tonic-gate kmem_cache_t *cp = ksp->ks_private; 34327c478bd9Sstevel@tonic-gate uint64_t cpu_buf_avail; 34337c478bd9Sstevel@tonic-gate uint64_t buf_avail = 0; 34347c478bd9Sstevel@tonic-gate int cpu_seqid; 3435686031edSTom Erickson long reap; 34367c478bd9Sstevel@tonic-gate 34377c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&kmem_cache_kstat_lock)); 34387c478bd9Sstevel@tonic-gate 34397c478bd9Sstevel@tonic-gate if (rw == KSTAT_WRITE) 34407c478bd9Sstevel@tonic-gate return (EACCES); 34417c478bd9Sstevel@tonic-gate 34427c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 34437c478bd9Sstevel@tonic-gate 34447c478bd9Sstevel@tonic-gate kmcp->kmc_alloc_fail.value.ui64 = cp->cache_alloc_fail; 34457c478bd9Sstevel@tonic-gate kmcp->kmc_alloc.value.ui64 = cp->cache_slab_alloc; 34467c478bd9Sstevel@tonic-gate kmcp->kmc_free.value.ui64 = cp->cache_slab_free; 34477c478bd9Sstevel@tonic-gate kmcp->kmc_slab_alloc.value.ui64 = cp->cache_slab_alloc; 34487c478bd9Sstevel@tonic-gate kmcp->kmc_slab_free.value.ui64 = cp->cache_slab_free; 34497c478bd9Sstevel@tonic-gate 34507c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 34517c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 34527c478bd9Sstevel@tonic-gate 34537c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 34547c478bd9Sstevel@tonic-gate 34557c478bd9Sstevel@tonic-gate cpu_buf_avail = 0; 34567c478bd9Sstevel@tonic-gate if (ccp->cc_rounds > 0) 34577c478bd9Sstevel@tonic-gate cpu_buf_avail += ccp->cc_rounds; 34587c478bd9Sstevel@tonic-gate if (ccp->cc_prounds > 0) 34597c478bd9Sstevel@tonic-gate cpu_buf_avail += ccp->cc_prounds; 34607c478bd9Sstevel@tonic-gate 34617c478bd9Sstevel@tonic-gate kmcp->kmc_alloc.value.ui64 += ccp->cc_alloc; 34627c478bd9Sstevel@tonic-gate kmcp->kmc_free.value.ui64 += ccp->cc_free; 34637c478bd9Sstevel@tonic-gate buf_avail += cpu_buf_avail; 34647c478bd9Sstevel@tonic-gate 34657c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 34667c478bd9Sstevel@tonic-gate } 34677c478bd9Sstevel@tonic-gate 34687c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock); 34697c478bd9Sstevel@tonic-gate 34707c478bd9Sstevel@tonic-gate kmcp->kmc_depot_alloc.value.ui64 = cp->cache_full.ml_alloc; 34717c478bd9Sstevel@tonic-gate kmcp->kmc_depot_free.value.ui64 = cp->cache_empty.ml_alloc; 34727c478bd9Sstevel@tonic-gate kmcp->kmc_depot_contention.value.ui64 = cp->cache_depot_contention; 34737c478bd9Sstevel@tonic-gate kmcp->kmc_full_magazines.value.ui64 = cp->cache_full.ml_total; 34747c478bd9Sstevel@tonic-gate kmcp->kmc_empty_magazines.value.ui64 = cp->cache_empty.ml_total; 34757c478bd9Sstevel@tonic-gate kmcp->kmc_magazine_size.value.ui64 = 34767c478bd9Sstevel@tonic-gate (cp->cache_flags & KMF_NOMAGAZINE) ? 34777c478bd9Sstevel@tonic-gate 0 : cp->cache_magtype->mt_magsize; 34787c478bd9Sstevel@tonic-gate 34797c478bd9Sstevel@tonic-gate kmcp->kmc_alloc.value.ui64 += cp->cache_full.ml_alloc; 34807c478bd9Sstevel@tonic-gate kmcp->kmc_free.value.ui64 += cp->cache_empty.ml_alloc; 34817c478bd9Sstevel@tonic-gate buf_avail += cp->cache_full.ml_total * cp->cache_magtype->mt_magsize; 34827c478bd9Sstevel@tonic-gate 3483686031edSTom Erickson reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min); 3484686031edSTom Erickson reap = MIN(reap, cp->cache_full.ml_total); 3485686031edSTom Erickson 34867c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock); 34877c478bd9Sstevel@tonic-gate 34887c478bd9Sstevel@tonic-gate kmcp->kmc_buf_size.value.ui64 = cp->cache_bufsize; 34897c478bd9Sstevel@tonic-gate kmcp->kmc_align.value.ui64 = cp->cache_align; 34907c478bd9Sstevel@tonic-gate kmcp->kmc_chunk_size.value.ui64 = cp->cache_chunksize; 34917c478bd9Sstevel@tonic-gate kmcp->kmc_slab_size.value.ui64 = cp->cache_slabsize; 34927c478bd9Sstevel@tonic-gate kmcp->kmc_buf_constructed.value.ui64 = buf_avail; 34939f1b636aStomee buf_avail += cp->cache_bufslab; 34947c478bd9Sstevel@tonic-gate kmcp->kmc_buf_avail.value.ui64 = buf_avail; 34957c478bd9Sstevel@tonic-gate kmcp->kmc_buf_inuse.value.ui64 = cp->cache_buftotal - buf_avail; 34967c478bd9Sstevel@tonic-gate kmcp->kmc_buf_total.value.ui64 = cp->cache_buftotal; 34977c478bd9Sstevel@tonic-gate kmcp->kmc_buf_max.value.ui64 = cp->cache_bufmax; 34987c478bd9Sstevel@tonic-gate kmcp->kmc_slab_create.value.ui64 = cp->cache_slab_create; 34997c478bd9Sstevel@tonic-gate kmcp->kmc_slab_destroy.value.ui64 = cp->cache_slab_destroy; 35007c478bd9Sstevel@tonic-gate kmcp->kmc_hash_size.value.ui64 = (cp->cache_flags & KMF_HASH) ? 35017c478bd9Sstevel@tonic-gate cp->cache_hash_mask + 1 : 0; 35027c478bd9Sstevel@tonic-gate kmcp->kmc_hash_lookup_depth.value.ui64 = cp->cache_lookup_depth; 35037c478bd9Sstevel@tonic-gate kmcp->kmc_hash_rescale.value.ui64 = cp->cache_rescale; 35047c478bd9Sstevel@tonic-gate kmcp->kmc_vmem_source.value.ui64 = cp->cache_arena->vm_id; 3505686031edSTom Erickson kmcp->kmc_reap.value.ui64 = cp->cache_reap; 35067c478bd9Sstevel@tonic-gate 3507b5fca8f8Stomee if (cp->cache_defrag == NULL) { 3508b5fca8f8Stomee kmcp->kmc_move_callbacks.value.ui64 = 0; 3509b5fca8f8Stomee kmcp->kmc_move_yes.value.ui64 = 0; 3510b5fca8f8Stomee kmcp->kmc_move_no.value.ui64 = 0; 3511b5fca8f8Stomee kmcp->kmc_move_later.value.ui64 = 0; 3512b5fca8f8Stomee kmcp->kmc_move_dont_need.value.ui64 = 0; 3513b5fca8f8Stomee kmcp->kmc_move_dont_know.value.ui64 = 0; 3514b5fca8f8Stomee kmcp->kmc_move_hunt_found.value.ui64 = 0; 3515686031edSTom Erickson kmcp->kmc_move_slabs_freed.value.ui64 = 0; 3516686031edSTom Erickson kmcp->kmc_defrag.value.ui64 = 0; 3517686031edSTom Erickson kmcp->kmc_scan.value.ui64 = 0; 3518686031edSTom Erickson kmcp->kmc_move_reclaimable.value.ui64 = 0; 3519b5fca8f8Stomee } else { 3520686031edSTom Erickson int64_t reclaimable; 3521686031edSTom Erickson 3522b5fca8f8Stomee kmem_defrag_t *kd = cp->cache_defrag; 3523b5fca8f8Stomee kmcp->kmc_move_callbacks.value.ui64 = kd->kmd_callbacks; 3524b5fca8f8Stomee kmcp->kmc_move_yes.value.ui64 = kd->kmd_yes; 3525b5fca8f8Stomee kmcp->kmc_move_no.value.ui64 = kd->kmd_no; 3526b5fca8f8Stomee kmcp->kmc_move_later.value.ui64 = kd->kmd_later; 3527b5fca8f8Stomee kmcp->kmc_move_dont_need.value.ui64 = kd->kmd_dont_need; 3528b5fca8f8Stomee kmcp->kmc_move_dont_know.value.ui64 = kd->kmd_dont_know; 3529*d7db73d1SBryan Cantrill kmcp->kmc_move_hunt_found.value.ui64 = 0; 3530686031edSTom Erickson kmcp->kmc_move_slabs_freed.value.ui64 = kd->kmd_slabs_freed; 3531686031edSTom Erickson kmcp->kmc_defrag.value.ui64 = kd->kmd_defrags; 3532686031edSTom Erickson kmcp->kmc_scan.value.ui64 = kd->kmd_scans; 3533686031edSTom Erickson 3534686031edSTom Erickson reclaimable = cp->cache_bufslab - (cp->cache_maxchunks - 1); 3535686031edSTom Erickson reclaimable = MAX(reclaimable, 0); 3536686031edSTom Erickson reclaimable += ((uint64_t)reap * cp->cache_magtype->mt_magsize); 3537686031edSTom Erickson kmcp->kmc_move_reclaimable.value.ui64 = reclaimable; 3538b5fca8f8Stomee } 3539b5fca8f8Stomee 35407c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 35417c478bd9Sstevel@tonic-gate return (0); 35427c478bd9Sstevel@tonic-gate } 35437c478bd9Sstevel@tonic-gate 35447c478bd9Sstevel@tonic-gate /* 35457c478bd9Sstevel@tonic-gate * Return a named statistic about a particular cache. 35467c478bd9Sstevel@tonic-gate * This shouldn't be called very often, so it's currently designed for 35477c478bd9Sstevel@tonic-gate * simplicity (leverages existing kstat support) rather than efficiency. 35487c478bd9Sstevel@tonic-gate */ 35497c478bd9Sstevel@tonic-gate uint64_t 35507c478bd9Sstevel@tonic-gate kmem_cache_stat(kmem_cache_t *cp, char *name) 35517c478bd9Sstevel@tonic-gate { 35527c478bd9Sstevel@tonic-gate int i; 35537c478bd9Sstevel@tonic-gate kstat_t *ksp = cp->cache_kstat; 35547c478bd9Sstevel@tonic-gate kstat_named_t *knp = (kstat_named_t *)&kmem_cache_kstat; 35557c478bd9Sstevel@tonic-gate uint64_t value = 0; 35567c478bd9Sstevel@tonic-gate 35577c478bd9Sstevel@tonic-gate if (ksp != NULL) { 35587c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_kstat_lock); 35597c478bd9Sstevel@tonic-gate (void) kmem_cache_kstat_update(ksp, KSTAT_READ); 35607c478bd9Sstevel@tonic-gate for (i = 0; i < ksp->ks_ndata; i++) { 35617c478bd9Sstevel@tonic-gate if (strcmp(knp[i].name, name) == 0) { 35627c478bd9Sstevel@tonic-gate value = knp[i].value.ui64; 35637c478bd9Sstevel@tonic-gate break; 35647c478bd9Sstevel@tonic-gate } 35657c478bd9Sstevel@tonic-gate } 35667c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_kstat_lock); 35677c478bd9Sstevel@tonic-gate } 35687c478bd9Sstevel@tonic-gate return (value); 35697c478bd9Sstevel@tonic-gate } 35707c478bd9Sstevel@tonic-gate 35717c478bd9Sstevel@tonic-gate /* 35727c478bd9Sstevel@tonic-gate * Return an estimate of currently available kernel heap memory. 35737c478bd9Sstevel@tonic-gate * On 32-bit systems, physical memory may exceed virtual memory, 35747c478bd9Sstevel@tonic-gate * we just truncate the result at 1GB. 35757c478bd9Sstevel@tonic-gate */ 35767c478bd9Sstevel@tonic-gate size_t 35777c478bd9Sstevel@tonic-gate kmem_avail(void) 35787c478bd9Sstevel@tonic-gate { 35797c478bd9Sstevel@tonic-gate spgcnt_t rmem = availrmem - tune.t_minarmem; 35807c478bd9Sstevel@tonic-gate spgcnt_t fmem = freemem - minfree; 35817c478bd9Sstevel@tonic-gate 35827c478bd9Sstevel@tonic-gate return ((size_t)ptob(MIN(MAX(MIN(rmem, fmem), 0), 35837c478bd9Sstevel@tonic-gate 1 << (30 - PAGESHIFT)))); 35847c478bd9Sstevel@tonic-gate } 35857c478bd9Sstevel@tonic-gate 35867c478bd9Sstevel@tonic-gate /* 35877c478bd9Sstevel@tonic-gate * Return the maximum amount of memory that is (in theory) allocatable 35887c478bd9Sstevel@tonic-gate * from the heap. This may be used as an estimate only since there 35897c478bd9Sstevel@tonic-gate * is no guarentee this space will still be available when an allocation 35907c478bd9Sstevel@tonic-gate * request is made, nor that the space may be allocated in one big request 35917c478bd9Sstevel@tonic-gate * due to kernel heap fragmentation. 35927c478bd9Sstevel@tonic-gate */ 35937c478bd9Sstevel@tonic-gate size_t 35947c478bd9Sstevel@tonic-gate kmem_maxavail(void) 35957c478bd9Sstevel@tonic-gate { 35967c478bd9Sstevel@tonic-gate spgcnt_t pmem = availrmem - tune.t_minarmem; 35977c478bd9Sstevel@tonic-gate spgcnt_t vmem = btop(vmem_size(heap_arena, VMEM_FREE)); 35987c478bd9Sstevel@tonic-gate 35997c478bd9Sstevel@tonic-gate return ((size_t)ptob(MAX(MIN(pmem, vmem), 0))); 36007c478bd9Sstevel@tonic-gate } 36017c478bd9Sstevel@tonic-gate 3602fa9e4066Sahrens /* 3603fa9e4066Sahrens * Indicate whether memory-intensive kmem debugging is enabled. 3604fa9e4066Sahrens */ 3605fa9e4066Sahrens int 3606fa9e4066Sahrens kmem_debugging(void) 3607fa9e4066Sahrens { 3608fa9e4066Sahrens return (kmem_flags & (KMF_AUDIT | KMF_REDZONE)); 3609fa9e4066Sahrens } 3610fa9e4066Sahrens 3611b5fca8f8Stomee /* binning function, sorts finely at the two extremes */ 3612b5fca8f8Stomee #define KMEM_PARTIAL_SLAB_WEIGHT(sp, binshift) \ 3613b5fca8f8Stomee ((((sp)->slab_refcnt <= (binshift)) || \ 3614b5fca8f8Stomee (((sp)->slab_chunks - (sp)->slab_refcnt) <= (binshift))) \ 3615b5fca8f8Stomee ? -(sp)->slab_refcnt \ 3616b5fca8f8Stomee : -((binshift) + ((sp)->slab_refcnt >> (binshift)))) 3617b5fca8f8Stomee 3618b5fca8f8Stomee /* 3619b5fca8f8Stomee * Minimizing the number of partial slabs on the freelist minimizes 3620b5fca8f8Stomee * fragmentation (the ratio of unused buffers held by the slab layer). There are 3621b5fca8f8Stomee * two ways to get a slab off of the freelist: 1) free all the buffers on the 3622b5fca8f8Stomee * slab, and 2) allocate all the buffers on the slab. It follows that we want 3623b5fca8f8Stomee * the most-used slabs at the front of the list where they have the best chance 3624b5fca8f8Stomee * of being completely allocated, and the least-used slabs at a safe distance 3625b5fca8f8Stomee * from the front to improve the odds that the few remaining buffers will all be 3626b5fca8f8Stomee * freed before another allocation can tie up the slab. For that reason a slab 3627b5fca8f8Stomee * with a higher slab_refcnt sorts less than than a slab with a lower 3628b5fca8f8Stomee * slab_refcnt. 3629b5fca8f8Stomee * 3630b5fca8f8Stomee * However, if a slab has at least one buffer that is deemed unfreeable, we 3631b5fca8f8Stomee * would rather have that slab at the front of the list regardless of 3632b5fca8f8Stomee * slab_refcnt, since even one unfreeable buffer makes the entire slab 3633b5fca8f8Stomee * unfreeable. If the client returns KMEM_CBRC_NO in response to a cache_move() 3634b5fca8f8Stomee * callback, the slab is marked unfreeable for as long as it remains on the 3635b5fca8f8Stomee * freelist. 3636b5fca8f8Stomee */ 3637b5fca8f8Stomee static int 3638b5fca8f8Stomee kmem_partial_slab_cmp(const void *p0, const void *p1) 3639b5fca8f8Stomee { 3640b5fca8f8Stomee const kmem_cache_t *cp; 3641b5fca8f8Stomee const kmem_slab_t *s0 = p0; 3642b5fca8f8Stomee const kmem_slab_t *s1 = p1; 3643b5fca8f8Stomee int w0, w1; 3644b5fca8f8Stomee size_t binshift; 3645b5fca8f8Stomee 3646b5fca8f8Stomee ASSERT(KMEM_SLAB_IS_PARTIAL(s0)); 3647b5fca8f8Stomee ASSERT(KMEM_SLAB_IS_PARTIAL(s1)); 3648b5fca8f8Stomee ASSERT(s0->slab_cache == s1->slab_cache); 3649b5fca8f8Stomee cp = s1->slab_cache; 3650b5fca8f8Stomee ASSERT(MUTEX_HELD(&cp->cache_lock)); 3651b5fca8f8Stomee binshift = cp->cache_partial_binshift; 3652b5fca8f8Stomee 3653b5fca8f8Stomee /* weight of first slab */ 3654b5fca8f8Stomee w0 = KMEM_PARTIAL_SLAB_WEIGHT(s0, binshift); 3655b5fca8f8Stomee if (s0->slab_flags & KMEM_SLAB_NOMOVE) { 3656b5fca8f8Stomee w0 -= cp->cache_maxchunks; 3657b5fca8f8Stomee } 3658b5fca8f8Stomee 3659b5fca8f8Stomee /* weight of second slab */ 3660b5fca8f8Stomee w1 = KMEM_PARTIAL_SLAB_WEIGHT(s1, binshift); 3661b5fca8f8Stomee if (s1->slab_flags & KMEM_SLAB_NOMOVE) { 3662b5fca8f8Stomee w1 -= cp->cache_maxchunks; 3663b5fca8f8Stomee } 3664b5fca8f8Stomee 3665b5fca8f8Stomee if (w0 < w1) 3666b5fca8f8Stomee return (-1); 3667b5fca8f8Stomee if (w0 > w1) 3668b5fca8f8Stomee return (1); 3669b5fca8f8Stomee 3670b5fca8f8Stomee /* compare pointer values */ 3671b5fca8f8Stomee if ((uintptr_t)s0 < (uintptr_t)s1) 3672b5fca8f8Stomee return (-1); 3673b5fca8f8Stomee if ((uintptr_t)s0 > (uintptr_t)s1) 3674b5fca8f8Stomee return (1); 3675b5fca8f8Stomee 3676b5fca8f8Stomee return (0); 3677b5fca8f8Stomee } 3678b5fca8f8Stomee 3679b5fca8f8Stomee /* 3680b5fca8f8Stomee * It must be valid to call the destructor (if any) on a newly created object. 3681b5fca8f8Stomee * That is, the constructor (if any) must leave the object in a valid state for 3682b5fca8f8Stomee * the destructor. 3683b5fca8f8Stomee */ 36847c478bd9Sstevel@tonic-gate kmem_cache_t * 36857c478bd9Sstevel@tonic-gate kmem_cache_create( 36867c478bd9Sstevel@tonic-gate char *name, /* descriptive name for this cache */ 36877c478bd9Sstevel@tonic-gate size_t bufsize, /* size of the objects it manages */ 36887c478bd9Sstevel@tonic-gate size_t align, /* required object alignment */ 36897c478bd9Sstevel@tonic-gate int (*constructor)(void *, void *, int), /* object constructor */ 36907c478bd9Sstevel@tonic-gate void (*destructor)(void *, void *), /* object destructor */ 36917c478bd9Sstevel@tonic-gate void (*reclaim)(void *), /* memory reclaim callback */ 36927c478bd9Sstevel@tonic-gate void *private, /* pass-thru arg for constr/destr/reclaim */ 36937c478bd9Sstevel@tonic-gate vmem_t *vmp, /* vmem source for slab allocation */ 36947c478bd9Sstevel@tonic-gate int cflags) /* cache creation flags */ 36957c478bd9Sstevel@tonic-gate { 36967c478bd9Sstevel@tonic-gate int cpu_seqid; 36977c478bd9Sstevel@tonic-gate size_t chunksize; 3698b5fca8f8Stomee kmem_cache_t *cp; 36997c478bd9Sstevel@tonic-gate kmem_magtype_t *mtp; 37007c478bd9Sstevel@tonic-gate size_t csize = KMEM_CACHE_SIZE(max_ncpus); 37017c478bd9Sstevel@tonic-gate 37027c478bd9Sstevel@tonic-gate #ifdef DEBUG 37037c478bd9Sstevel@tonic-gate /* 37047c478bd9Sstevel@tonic-gate * Cache names should conform to the rules for valid C identifiers 37057c478bd9Sstevel@tonic-gate */ 37067c478bd9Sstevel@tonic-gate if (!strident_valid(name)) { 37077c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, 37087c478bd9Sstevel@tonic-gate "kmem_cache_create: '%s' is an invalid cache name\n" 37097c478bd9Sstevel@tonic-gate "cache names must conform to the rules for " 37107c478bd9Sstevel@tonic-gate "C identifiers\n", name); 37117c478bd9Sstevel@tonic-gate } 37127c478bd9Sstevel@tonic-gate #endif /* DEBUG */ 37137c478bd9Sstevel@tonic-gate 37147c478bd9Sstevel@tonic-gate if (vmp == NULL) 37157c478bd9Sstevel@tonic-gate vmp = kmem_default_arena; 37167c478bd9Sstevel@tonic-gate 37177c478bd9Sstevel@tonic-gate /* 37187c478bd9Sstevel@tonic-gate * If this kmem cache has an identifier vmem arena as its source, mark 37197c478bd9Sstevel@tonic-gate * it such to allow kmem_reap_idspace(). 37207c478bd9Sstevel@tonic-gate */ 37217c478bd9Sstevel@tonic-gate ASSERT(!(cflags & KMC_IDENTIFIER)); /* consumer should not set this */ 37227c478bd9Sstevel@tonic-gate if (vmp->vm_cflags & VMC_IDENTIFIER) 37237c478bd9Sstevel@tonic-gate cflags |= KMC_IDENTIFIER; 37247c478bd9Sstevel@tonic-gate 37257c478bd9Sstevel@tonic-gate /* 37267c478bd9Sstevel@tonic-gate * Get a kmem_cache structure. We arrange that cp->cache_cpu[] 37277c478bd9Sstevel@tonic-gate * is aligned on a KMEM_CPU_CACHE_SIZE boundary to prevent 37287c478bd9Sstevel@tonic-gate * false sharing of per-CPU data. 37297c478bd9Sstevel@tonic-gate */ 37307c478bd9Sstevel@tonic-gate cp = vmem_xalloc(kmem_cache_arena, csize, KMEM_CPU_CACHE_SIZE, 37317c478bd9Sstevel@tonic-gate P2NPHASE(csize, KMEM_CPU_CACHE_SIZE), 0, NULL, NULL, VM_SLEEP); 37327c478bd9Sstevel@tonic-gate bzero(cp, csize); 3733b5fca8f8Stomee list_link_init(&cp->cache_link); 37347c478bd9Sstevel@tonic-gate 37357c478bd9Sstevel@tonic-gate if (align == 0) 37367c478bd9Sstevel@tonic-gate align = KMEM_ALIGN; 37377c478bd9Sstevel@tonic-gate 37387c478bd9Sstevel@tonic-gate /* 37397c478bd9Sstevel@tonic-gate * If we're not at least KMEM_ALIGN aligned, we can't use free 37407c478bd9Sstevel@tonic-gate * memory to hold bufctl information (because we can't safely 37417c478bd9Sstevel@tonic-gate * perform word loads and stores on it). 37427c478bd9Sstevel@tonic-gate */ 37437c478bd9Sstevel@tonic-gate if (align < KMEM_ALIGN) 37447c478bd9Sstevel@tonic-gate cflags |= KMC_NOTOUCH; 37457c478bd9Sstevel@tonic-gate 3746de710d24SJosef 'Jeff' Sipek if (!ISP2(align) || align > vmp->vm_quantum) 37477c478bd9Sstevel@tonic-gate panic("kmem_cache_create: bad alignment %lu", align); 37487c478bd9Sstevel@tonic-gate 37497c478bd9Sstevel@tonic-gate mutex_enter(&kmem_flags_lock); 37507c478bd9Sstevel@tonic-gate if (kmem_flags & KMF_RANDOMIZE) 37517c478bd9Sstevel@tonic-gate kmem_flags = (((kmem_flags | ~KMF_RANDOM) + 1) & KMF_RANDOM) | 37527c478bd9Sstevel@tonic-gate KMF_RANDOMIZE; 37537c478bd9Sstevel@tonic-gate cp->cache_flags = (kmem_flags | cflags) & KMF_DEBUG; 37547c478bd9Sstevel@tonic-gate mutex_exit(&kmem_flags_lock); 37557c478bd9Sstevel@tonic-gate 37567c478bd9Sstevel@tonic-gate /* 37577c478bd9Sstevel@tonic-gate * Make sure all the various flags are reasonable. 37587c478bd9Sstevel@tonic-gate */ 37597c478bd9Sstevel@tonic-gate ASSERT(!(cflags & KMC_NOHASH) || !(cflags & KMC_NOTOUCH)); 37607c478bd9Sstevel@tonic-gate 37617c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) { 37627c478bd9Sstevel@tonic-gate if (bufsize >= kmem_lite_minsize && 37637c478bd9Sstevel@tonic-gate align <= kmem_lite_maxalign && 37647c478bd9Sstevel@tonic-gate P2PHASE(bufsize, kmem_lite_maxalign) != 0) { 37657c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_BUFTAG; 37667c478bd9Sstevel@tonic-gate cp->cache_flags &= ~(KMF_AUDIT | KMF_FIREWALL); 37677c478bd9Sstevel@tonic-gate } else { 37687c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_DEBUG; 37697c478bd9Sstevel@tonic-gate } 37707c478bd9Sstevel@tonic-gate } 37717c478bd9Sstevel@tonic-gate 37727c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) 37737c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_REDZONE; 37747c478bd9Sstevel@tonic-gate 37757c478bd9Sstevel@tonic-gate if ((cflags & KMC_QCACHE) && (cp->cache_flags & KMF_AUDIT)) 37767c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_NOMAGAZINE; 37777c478bd9Sstevel@tonic-gate 37787c478bd9Sstevel@tonic-gate if (cflags & KMC_NODEBUG) 37797c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_DEBUG; 37807c478bd9Sstevel@tonic-gate 37817c478bd9Sstevel@tonic-gate if (cflags & KMC_NOTOUCH) 37827c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_TOUCH; 37837c478bd9Sstevel@tonic-gate 3784b942e89bSDavid Valin if (cflags & KMC_PREFILL) 3785b942e89bSDavid Valin cp->cache_flags |= KMF_PREFILL; 3786b942e89bSDavid Valin 37877c478bd9Sstevel@tonic-gate if (cflags & KMC_NOHASH) 37887c478bd9Sstevel@tonic-gate cp->cache_flags &= ~(KMF_AUDIT | KMF_FIREWALL); 37897c478bd9Sstevel@tonic-gate 37907c478bd9Sstevel@tonic-gate if (cflags & KMC_NOMAGAZINE) 37917c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_NOMAGAZINE; 37927c478bd9Sstevel@tonic-gate 37937c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_AUDIT) && !(cflags & KMC_NOTOUCH)) 37947c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_REDZONE; 37957c478bd9Sstevel@tonic-gate 37967c478bd9Sstevel@tonic-gate if (!(cp->cache_flags & KMF_AUDIT)) 37977c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_CONTENTS; 37987c478bd9Sstevel@tonic-gate 37997c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_BUFTAG) && bufsize >= kmem_minfirewall && 38007c478bd9Sstevel@tonic-gate !(cp->cache_flags & KMF_LITE) && !(cflags & KMC_NOHASH)) 38017c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_FIREWALL; 38027c478bd9Sstevel@tonic-gate 38037c478bd9Sstevel@tonic-gate if (vmp != kmem_default_arena || kmem_firewall_arena == NULL) 38047c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_FIREWALL; 38057c478bd9Sstevel@tonic-gate 38067c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_FIREWALL) { 38077c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_BUFTAG; 38087c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_NOMAGAZINE; 38097c478bd9Sstevel@tonic-gate ASSERT(vmp == kmem_default_arena); 38107c478bd9Sstevel@tonic-gate vmp = kmem_firewall_arena; 38117c478bd9Sstevel@tonic-gate } 38127c478bd9Sstevel@tonic-gate 38137c478bd9Sstevel@tonic-gate /* 38147c478bd9Sstevel@tonic-gate * Set cache properties. 38157c478bd9Sstevel@tonic-gate */ 38167c478bd9Sstevel@tonic-gate (void) strncpy(cp->cache_name, name, KMEM_CACHE_NAMELEN); 3817b5fca8f8Stomee strident_canon(cp->cache_name, KMEM_CACHE_NAMELEN + 1); 38187c478bd9Sstevel@tonic-gate cp->cache_bufsize = bufsize; 38197c478bd9Sstevel@tonic-gate cp->cache_align = align; 38207c478bd9Sstevel@tonic-gate cp->cache_constructor = constructor; 38217c478bd9Sstevel@tonic-gate cp->cache_destructor = destructor; 38227c478bd9Sstevel@tonic-gate cp->cache_reclaim = reclaim; 38237c478bd9Sstevel@tonic-gate cp->cache_private = private; 38247c478bd9Sstevel@tonic-gate cp->cache_arena = vmp; 38257c478bd9Sstevel@tonic-gate cp->cache_cflags = cflags; 38267c478bd9Sstevel@tonic-gate 38277c478bd9Sstevel@tonic-gate /* 38287c478bd9Sstevel@tonic-gate * Determine the chunk size. 38297c478bd9Sstevel@tonic-gate */ 38307c478bd9Sstevel@tonic-gate chunksize = bufsize; 38317c478bd9Sstevel@tonic-gate 38327c478bd9Sstevel@tonic-gate if (align >= KMEM_ALIGN) { 38337c478bd9Sstevel@tonic-gate chunksize = P2ROUNDUP(chunksize, KMEM_ALIGN); 38347c478bd9Sstevel@tonic-gate cp->cache_bufctl = chunksize - KMEM_ALIGN; 38357c478bd9Sstevel@tonic-gate } 38367c478bd9Sstevel@tonic-gate 38377c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_BUFTAG) { 38387c478bd9Sstevel@tonic-gate cp->cache_bufctl = chunksize; 38397c478bd9Sstevel@tonic-gate cp->cache_buftag = chunksize; 38407c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) 38417c478bd9Sstevel@tonic-gate chunksize += KMEM_BUFTAG_LITE_SIZE(kmem_lite_count); 38427c478bd9Sstevel@tonic-gate else 38437c478bd9Sstevel@tonic-gate chunksize += sizeof (kmem_buftag_t); 38447c478bd9Sstevel@tonic-gate } 38457c478bd9Sstevel@tonic-gate 38467c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) { 38477c478bd9Sstevel@tonic-gate cp->cache_verify = MIN(cp->cache_buftag, kmem_maxverify); 38487c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) 38497c478bd9Sstevel@tonic-gate cp->cache_verify = sizeof (uint64_t); 38507c478bd9Sstevel@tonic-gate } 38517c478bd9Sstevel@tonic-gate 38527c478bd9Sstevel@tonic-gate cp->cache_contents = MIN(cp->cache_bufctl, kmem_content_maxsave); 38537c478bd9Sstevel@tonic-gate 38547c478bd9Sstevel@tonic-gate cp->cache_chunksize = chunksize = P2ROUNDUP(chunksize, align); 38557c478bd9Sstevel@tonic-gate 38567c478bd9Sstevel@tonic-gate /* 38577c478bd9Sstevel@tonic-gate * Now that we know the chunk size, determine the optimal slab size. 38587c478bd9Sstevel@tonic-gate */ 38597c478bd9Sstevel@tonic-gate if (vmp == kmem_firewall_arena) { 38607c478bd9Sstevel@tonic-gate cp->cache_slabsize = P2ROUNDUP(chunksize, vmp->vm_quantum); 38617c478bd9Sstevel@tonic-gate cp->cache_mincolor = cp->cache_slabsize - chunksize; 38627c478bd9Sstevel@tonic-gate cp->cache_maxcolor = cp->cache_mincolor; 38637c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_HASH; 38647c478bd9Sstevel@tonic-gate ASSERT(!(cp->cache_flags & KMF_BUFTAG)); 38657c478bd9Sstevel@tonic-gate } else if ((cflags & KMC_NOHASH) || (!(cflags & KMC_NOTOUCH) && 38667c478bd9Sstevel@tonic-gate !(cp->cache_flags & KMF_AUDIT) && 38677c478bd9Sstevel@tonic-gate chunksize < vmp->vm_quantum / KMEM_VOID_FRACTION)) { 38687c478bd9Sstevel@tonic-gate cp->cache_slabsize = vmp->vm_quantum; 38697c478bd9Sstevel@tonic-gate cp->cache_mincolor = 0; 38707c478bd9Sstevel@tonic-gate cp->cache_maxcolor = 38717c478bd9Sstevel@tonic-gate (cp->cache_slabsize - sizeof (kmem_slab_t)) % chunksize; 38727c478bd9Sstevel@tonic-gate ASSERT(chunksize + sizeof (kmem_slab_t) <= cp->cache_slabsize); 38737c478bd9Sstevel@tonic-gate ASSERT(!(cp->cache_flags & KMF_AUDIT)); 38747c478bd9Sstevel@tonic-gate } else { 38757c478bd9Sstevel@tonic-gate size_t chunks, bestfit, waste, slabsize; 38767c478bd9Sstevel@tonic-gate size_t minwaste = LONG_MAX; 38777c478bd9Sstevel@tonic-gate 38787c478bd9Sstevel@tonic-gate for (chunks = 1; chunks <= KMEM_VOID_FRACTION; chunks++) { 38797c478bd9Sstevel@tonic-gate slabsize = P2ROUNDUP(chunksize * chunks, 38807c478bd9Sstevel@tonic-gate vmp->vm_quantum); 38817c478bd9Sstevel@tonic-gate chunks = slabsize / chunksize; 38827c478bd9Sstevel@tonic-gate waste = (slabsize % chunksize) / chunks; 38837c478bd9Sstevel@tonic-gate if (waste < minwaste) { 38847c478bd9Sstevel@tonic-gate minwaste = waste; 38857c478bd9Sstevel@tonic-gate bestfit = slabsize; 38867c478bd9Sstevel@tonic-gate } 38877c478bd9Sstevel@tonic-gate } 38887c478bd9Sstevel@tonic-gate if (cflags & KMC_QCACHE) 38897c478bd9Sstevel@tonic-gate bestfit = VMEM_QCACHE_SLABSIZE(vmp->vm_qcache_max); 38907c478bd9Sstevel@tonic-gate cp->cache_slabsize = bestfit; 38917c478bd9Sstevel@tonic-gate cp->cache_mincolor = 0; 38927c478bd9Sstevel@tonic-gate cp->cache_maxcolor = bestfit % chunksize; 38937c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_HASH; 38947c478bd9Sstevel@tonic-gate } 38957c478bd9Sstevel@tonic-gate 3896b5fca8f8Stomee cp->cache_maxchunks = (cp->cache_slabsize / cp->cache_chunksize); 3897b5fca8f8Stomee cp->cache_partial_binshift = highbit(cp->cache_maxchunks / 16) + 1; 3898b5fca8f8Stomee 3899b942e89bSDavid Valin /* 3900b942e89bSDavid Valin * Disallowing prefill when either the DEBUG or HASH flag is set or when 3901b942e89bSDavid Valin * there is a constructor avoids some tricky issues with debug setup 3902b942e89bSDavid Valin * that may be revisited later. We cannot allow prefill in a 3903b942e89bSDavid Valin * metadata cache because of potential recursion. 3904b942e89bSDavid Valin */ 3905b942e89bSDavid Valin if (vmp == kmem_msb_arena || 3906b942e89bSDavid Valin cp->cache_flags & (KMF_HASH | KMF_BUFTAG) || 3907b942e89bSDavid Valin cp->cache_constructor != NULL) 3908b942e89bSDavid Valin cp->cache_flags &= ~KMF_PREFILL; 3909b942e89bSDavid Valin 39107c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) { 39117c478bd9Sstevel@tonic-gate ASSERT(!(cflags & KMC_NOHASH)); 39127c478bd9Sstevel@tonic-gate cp->cache_bufctl_cache = (cp->cache_flags & KMF_AUDIT) ? 39137c478bd9Sstevel@tonic-gate kmem_bufctl_audit_cache : kmem_bufctl_cache; 39147c478bd9Sstevel@tonic-gate } 39157c478bd9Sstevel@tonic-gate 39167c478bd9Sstevel@tonic-gate if (cp->cache_maxcolor >= vmp->vm_quantum) 39177c478bd9Sstevel@tonic-gate cp->cache_maxcolor = vmp->vm_quantum - 1; 39187c478bd9Sstevel@tonic-gate 39197c478bd9Sstevel@tonic-gate cp->cache_color = cp->cache_mincolor; 39207c478bd9Sstevel@tonic-gate 39217c478bd9Sstevel@tonic-gate /* 39227c478bd9Sstevel@tonic-gate * Initialize the rest of the slab layer. 39237c478bd9Sstevel@tonic-gate */ 39247c478bd9Sstevel@tonic-gate mutex_init(&cp->cache_lock, NULL, MUTEX_DEFAULT, NULL); 39257c478bd9Sstevel@tonic-gate 3926b5fca8f8Stomee avl_create(&cp->cache_partial_slabs, kmem_partial_slab_cmp, 3927b5fca8f8Stomee sizeof (kmem_slab_t), offsetof(kmem_slab_t, slab_link)); 3928b5fca8f8Stomee /* LINTED: E_TRUE_LOGICAL_EXPR */ 3929b5fca8f8Stomee ASSERT(sizeof (list_node_t) <= sizeof (avl_node_t)); 3930b5fca8f8Stomee /* reuse partial slab AVL linkage for complete slab list linkage */ 3931b5fca8f8Stomee list_create(&cp->cache_complete_slabs, 3932b5fca8f8Stomee sizeof (kmem_slab_t), offsetof(kmem_slab_t, slab_link)); 39337c478bd9Sstevel@tonic-gate 39347c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) { 39357c478bd9Sstevel@tonic-gate cp->cache_hash_table = vmem_alloc(kmem_hash_arena, 39367c478bd9Sstevel@tonic-gate KMEM_HASH_INITIAL * sizeof (void *), VM_SLEEP); 39377c478bd9Sstevel@tonic-gate bzero(cp->cache_hash_table, 39387c478bd9Sstevel@tonic-gate KMEM_HASH_INITIAL * sizeof (void *)); 39397c478bd9Sstevel@tonic-gate cp->cache_hash_mask = KMEM_HASH_INITIAL - 1; 39407c478bd9Sstevel@tonic-gate cp->cache_hash_shift = highbit((ulong_t)chunksize) - 1; 39417c478bd9Sstevel@tonic-gate } 39427c478bd9Sstevel@tonic-gate 39437c478bd9Sstevel@tonic-gate /* 39447c478bd9Sstevel@tonic-gate * Initialize the depot. 39457c478bd9Sstevel@tonic-gate */ 39467c478bd9Sstevel@tonic-gate mutex_init(&cp->cache_depot_lock, NULL, MUTEX_DEFAULT, NULL); 39477c478bd9Sstevel@tonic-gate 39487c478bd9Sstevel@tonic-gate for (mtp = kmem_magtype; chunksize <= mtp->mt_minbuf; mtp++) 39497c478bd9Sstevel@tonic-gate continue; 39507c478bd9Sstevel@tonic-gate 39517c478bd9Sstevel@tonic-gate cp->cache_magtype = mtp; 39527c478bd9Sstevel@tonic-gate 39537c478bd9Sstevel@tonic-gate /* 39547c478bd9Sstevel@tonic-gate * Initialize the CPU layer. 39557c478bd9Sstevel@tonic-gate */ 39567c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 39577c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 39587c478bd9Sstevel@tonic-gate mutex_init(&ccp->cc_lock, NULL, MUTEX_DEFAULT, NULL); 39597c478bd9Sstevel@tonic-gate ccp->cc_flags = cp->cache_flags; 39607c478bd9Sstevel@tonic-gate ccp->cc_rounds = -1; 39617c478bd9Sstevel@tonic-gate ccp->cc_prounds = -1; 39627c478bd9Sstevel@tonic-gate } 39637c478bd9Sstevel@tonic-gate 39647c478bd9Sstevel@tonic-gate /* 39657c478bd9Sstevel@tonic-gate * Create the cache's kstats. 39667c478bd9Sstevel@tonic-gate */ 39677c478bd9Sstevel@tonic-gate if ((cp->cache_kstat = kstat_create("unix", 0, cp->cache_name, 39687c478bd9Sstevel@tonic-gate "kmem_cache", KSTAT_TYPE_NAMED, 39697c478bd9Sstevel@tonic-gate sizeof (kmem_cache_kstat) / sizeof (kstat_named_t), 39707c478bd9Sstevel@tonic-gate KSTAT_FLAG_VIRTUAL)) != NULL) { 39717c478bd9Sstevel@tonic-gate cp->cache_kstat->ks_data = &kmem_cache_kstat; 39727c478bd9Sstevel@tonic-gate cp->cache_kstat->ks_update = kmem_cache_kstat_update; 39737c478bd9Sstevel@tonic-gate cp->cache_kstat->ks_private = cp; 39747c478bd9Sstevel@tonic-gate cp->cache_kstat->ks_lock = &kmem_cache_kstat_lock; 39757c478bd9Sstevel@tonic-gate kstat_install(cp->cache_kstat); 39767c478bd9Sstevel@tonic-gate } 39777c478bd9Sstevel@tonic-gate 39787c478bd9Sstevel@tonic-gate /* 39797c478bd9Sstevel@tonic-gate * Add the cache to the global list. This makes it visible 39807c478bd9Sstevel@tonic-gate * to kmem_update(), so the cache must be ready for business. 39817c478bd9Sstevel@tonic-gate */ 39827c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_lock); 3983b5fca8f8Stomee list_insert_tail(&kmem_caches, cp); 39847c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_lock); 39857c478bd9Sstevel@tonic-gate 39867c478bd9Sstevel@tonic-gate if (kmem_ready) 39877c478bd9Sstevel@tonic-gate kmem_cache_magazine_enable(cp); 39887c478bd9Sstevel@tonic-gate 39897c478bd9Sstevel@tonic-gate return (cp); 39907c478bd9Sstevel@tonic-gate } 39917c478bd9Sstevel@tonic-gate 3992b5fca8f8Stomee static int 3993b5fca8f8Stomee kmem_move_cmp(const void *buf, const void *p) 3994b5fca8f8Stomee { 3995b5fca8f8Stomee const kmem_move_t *kmm = p; 3996b5fca8f8Stomee uintptr_t v1 = (uintptr_t)buf; 3997b5fca8f8Stomee uintptr_t v2 = (uintptr_t)kmm->kmm_from_buf; 3998b5fca8f8Stomee return (v1 < v2 ? -1 : (v1 > v2 ? 1 : 0)); 3999b5fca8f8Stomee } 4000b5fca8f8Stomee 4001b5fca8f8Stomee static void 4002b5fca8f8Stomee kmem_reset_reclaim_threshold(kmem_defrag_t *kmd) 4003b5fca8f8Stomee { 4004b5fca8f8Stomee kmd->kmd_reclaim_numer = 1; 4005b5fca8f8Stomee } 4006b5fca8f8Stomee 4007b5fca8f8Stomee /* 4008b5fca8f8Stomee * Initially, when choosing candidate slabs for buffers to move, we want to be 4009b5fca8f8Stomee * very selective and take only slabs that are less than 4010b5fca8f8Stomee * (1 / KMEM_VOID_FRACTION) allocated. If we have difficulty finding candidate 4011b5fca8f8Stomee * slabs, then we raise the allocation ceiling incrementally. The reclaim 4012b5fca8f8Stomee * threshold is reset to (1 / KMEM_VOID_FRACTION) as soon as the cache is no 4013b5fca8f8Stomee * longer fragmented. 4014b5fca8f8Stomee */ 4015b5fca8f8Stomee static void 4016b5fca8f8Stomee kmem_adjust_reclaim_threshold(kmem_defrag_t *kmd, int direction) 4017b5fca8f8Stomee { 4018b5fca8f8Stomee if (direction > 0) { 4019b5fca8f8Stomee /* make it easier to find a candidate slab */ 4020b5fca8f8Stomee if (kmd->kmd_reclaim_numer < (KMEM_VOID_FRACTION - 1)) { 4021b5fca8f8Stomee kmd->kmd_reclaim_numer++; 4022b5fca8f8Stomee } 4023b5fca8f8Stomee } else { 4024b5fca8f8Stomee /* be more selective */ 4025b5fca8f8Stomee if (kmd->kmd_reclaim_numer > 1) { 4026b5fca8f8Stomee kmd->kmd_reclaim_numer--; 4027b5fca8f8Stomee } 4028b5fca8f8Stomee } 4029b5fca8f8Stomee } 4030b5fca8f8Stomee 4031b5fca8f8Stomee void 4032b5fca8f8Stomee kmem_cache_set_move(kmem_cache_t *cp, 4033b5fca8f8Stomee kmem_cbrc_t (*move)(void *, void *, size_t, void *)) 4034b5fca8f8Stomee { 4035b5fca8f8Stomee kmem_defrag_t *defrag; 4036b5fca8f8Stomee 4037b5fca8f8Stomee ASSERT(move != NULL); 4038b5fca8f8Stomee /* 4039b5fca8f8Stomee * The consolidator does not support NOTOUCH caches because kmem cannot 4040b5fca8f8Stomee * initialize their slabs with the 0xbaddcafe memory pattern, which sets 4041b5fca8f8Stomee * a low order bit usable by clients to distinguish uninitialized memory 4042b5fca8f8Stomee * from known objects (see kmem_slab_create). 4043b5fca8f8Stomee */ 4044b5fca8f8Stomee ASSERT(!(cp->cache_cflags & KMC_NOTOUCH)); 4045b5fca8f8Stomee ASSERT(!(cp->cache_cflags & KMC_IDENTIFIER)); 4046b5fca8f8Stomee 4047b5fca8f8Stomee /* 4048b5fca8f8Stomee * We should not be holding anyone's cache lock when calling 4049b5fca8f8Stomee * kmem_cache_alloc(), so allocate in all cases before acquiring the 4050b5fca8f8Stomee * lock. 4051b5fca8f8Stomee */ 4052b5fca8f8Stomee defrag = kmem_cache_alloc(kmem_defrag_cache, KM_SLEEP); 4053b5fca8f8Stomee 4054b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4055b5fca8f8Stomee 4056b5fca8f8Stomee if (KMEM_IS_MOVABLE(cp)) { 4057b5fca8f8Stomee if (cp->cache_move == NULL) { 40584d4c4c43STom Erickson ASSERT(cp->cache_slab_alloc == 0); 4059b5fca8f8Stomee 4060b5fca8f8Stomee cp->cache_defrag = defrag; 4061b5fca8f8Stomee defrag = NULL; /* nothing to free */ 4062b5fca8f8Stomee bzero(cp->cache_defrag, sizeof (kmem_defrag_t)); 4063b5fca8f8Stomee avl_create(&cp->cache_defrag->kmd_moves_pending, 4064b5fca8f8Stomee kmem_move_cmp, sizeof (kmem_move_t), 4065b5fca8f8Stomee offsetof(kmem_move_t, kmm_entry)); 4066b5fca8f8Stomee /* LINTED: E_TRUE_LOGICAL_EXPR */ 4067b5fca8f8Stomee ASSERT(sizeof (list_node_t) <= sizeof (avl_node_t)); 4068b5fca8f8Stomee /* reuse the slab's AVL linkage for deadlist linkage */ 4069b5fca8f8Stomee list_create(&cp->cache_defrag->kmd_deadlist, 4070b5fca8f8Stomee sizeof (kmem_slab_t), 4071b5fca8f8Stomee offsetof(kmem_slab_t, slab_link)); 4072b5fca8f8Stomee kmem_reset_reclaim_threshold(cp->cache_defrag); 4073b5fca8f8Stomee } 4074b5fca8f8Stomee cp->cache_move = move; 4075b5fca8f8Stomee } 4076b5fca8f8Stomee 4077b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4078b5fca8f8Stomee 4079b5fca8f8Stomee if (defrag != NULL) { 4080b5fca8f8Stomee kmem_cache_free(kmem_defrag_cache, defrag); /* unused */ 4081b5fca8f8Stomee } 4082b5fca8f8Stomee } 4083b5fca8f8Stomee 40847c478bd9Sstevel@tonic-gate void 40857c478bd9Sstevel@tonic-gate kmem_cache_destroy(kmem_cache_t *cp) 40867c478bd9Sstevel@tonic-gate { 40877c478bd9Sstevel@tonic-gate int cpu_seqid; 40887c478bd9Sstevel@tonic-gate 40897c478bd9Sstevel@tonic-gate /* 40907c478bd9Sstevel@tonic-gate * Remove the cache from the global cache list so that no one else 40917c478bd9Sstevel@tonic-gate * can schedule tasks on its behalf, wait for any pending tasks to 40927c478bd9Sstevel@tonic-gate * complete, purge the cache, and then destroy it. 40937c478bd9Sstevel@tonic-gate */ 40947c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_lock); 4095b5fca8f8Stomee list_remove(&kmem_caches, cp); 40967c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_lock); 40977c478bd9Sstevel@tonic-gate 40987c478bd9Sstevel@tonic-gate if (kmem_taskq != NULL) 40997c478bd9Sstevel@tonic-gate taskq_wait(kmem_taskq); 4100*d7db73d1SBryan Cantrill 4101*d7db73d1SBryan Cantrill if (kmem_move_taskq != NULL && cp->cache_defrag != NULL) 4102b5fca8f8Stomee taskq_wait(kmem_move_taskq); 41037c478bd9Sstevel@tonic-gate 41047c478bd9Sstevel@tonic-gate kmem_cache_magazine_purge(cp); 41057c478bd9Sstevel@tonic-gate 41067c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 41077c478bd9Sstevel@tonic-gate if (cp->cache_buftotal != 0) 41087c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "kmem_cache_destroy: '%s' (%p) not empty", 41097c478bd9Sstevel@tonic-gate cp->cache_name, (void *)cp); 4110b5fca8f8Stomee if (cp->cache_defrag != NULL) { 4111b5fca8f8Stomee avl_destroy(&cp->cache_defrag->kmd_moves_pending); 4112b5fca8f8Stomee list_destroy(&cp->cache_defrag->kmd_deadlist); 4113b5fca8f8Stomee kmem_cache_free(kmem_defrag_cache, cp->cache_defrag); 4114b5fca8f8Stomee cp->cache_defrag = NULL; 4115b5fca8f8Stomee } 41167c478bd9Sstevel@tonic-gate /* 4117b5fca8f8Stomee * The cache is now dead. There should be no further activity. We 4118b5fca8f8Stomee * enforce this by setting land mines in the constructor, destructor, 4119b5fca8f8Stomee * reclaim, and move routines that induce a kernel text fault if 4120b5fca8f8Stomee * invoked. 41217c478bd9Sstevel@tonic-gate */ 41227c478bd9Sstevel@tonic-gate cp->cache_constructor = (int (*)(void *, void *, int))1; 41237c478bd9Sstevel@tonic-gate cp->cache_destructor = (void (*)(void *, void *))2; 4124b5fca8f8Stomee cp->cache_reclaim = (void (*)(void *))3; 4125b5fca8f8Stomee cp->cache_move = (kmem_cbrc_t (*)(void *, void *, size_t, void *))4; 41267c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 41277c478bd9Sstevel@tonic-gate 41287c478bd9Sstevel@tonic-gate kstat_delete(cp->cache_kstat); 41297c478bd9Sstevel@tonic-gate 41307c478bd9Sstevel@tonic-gate if (cp->cache_hash_table != NULL) 41317c478bd9Sstevel@tonic-gate vmem_free(kmem_hash_arena, cp->cache_hash_table, 41327c478bd9Sstevel@tonic-gate (cp->cache_hash_mask + 1) * sizeof (void *)); 41337c478bd9Sstevel@tonic-gate 41347c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) 41357c478bd9Sstevel@tonic-gate mutex_destroy(&cp->cache_cpu[cpu_seqid].cc_lock); 41367c478bd9Sstevel@tonic-gate 41377c478bd9Sstevel@tonic-gate mutex_destroy(&cp->cache_depot_lock); 41387c478bd9Sstevel@tonic-gate mutex_destroy(&cp->cache_lock); 41397c478bd9Sstevel@tonic-gate 41407c478bd9Sstevel@tonic-gate vmem_free(kmem_cache_arena, cp, KMEM_CACHE_SIZE(max_ncpus)); 41417c478bd9Sstevel@tonic-gate } 41427c478bd9Sstevel@tonic-gate 41437c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 41447c478bd9Sstevel@tonic-gate static int 41457c478bd9Sstevel@tonic-gate kmem_cpu_setup(cpu_setup_t what, int id, void *arg) 41467c478bd9Sstevel@tonic-gate { 41477c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 41487c478bd9Sstevel@tonic-gate if (what == CPU_UNCONFIG) { 41497c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_magazine_purge, 41507c478bd9Sstevel@tonic-gate kmem_taskq, TQ_SLEEP); 41517c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_magazine_enable, 41527c478bd9Sstevel@tonic-gate kmem_taskq, TQ_SLEEP); 41537c478bd9Sstevel@tonic-gate } 41547c478bd9Sstevel@tonic-gate return (0); 41557c478bd9Sstevel@tonic-gate } 41567c478bd9Sstevel@tonic-gate 4157dce01e3fSJonathan W Adams static void 4158dce01e3fSJonathan W Adams kmem_alloc_caches_create(const int *array, size_t count, 4159dce01e3fSJonathan W Adams kmem_cache_t **alloc_table, size_t maxbuf, uint_t shift) 4160dce01e3fSJonathan W Adams { 4161dce01e3fSJonathan W Adams char name[KMEM_CACHE_NAMELEN + 1]; 4162dce01e3fSJonathan W Adams size_t table_unit = (1 << shift); /* range of one alloc_table entry */ 4163dce01e3fSJonathan W Adams size_t size = table_unit; 4164dce01e3fSJonathan W Adams int i; 4165dce01e3fSJonathan W Adams 4166dce01e3fSJonathan W Adams for (i = 0; i < count; i++) { 4167dce01e3fSJonathan W Adams size_t cache_size = array[i]; 4168dce01e3fSJonathan W Adams size_t align = KMEM_ALIGN; 4169dce01e3fSJonathan W Adams kmem_cache_t *cp; 4170dce01e3fSJonathan W Adams 4171dce01e3fSJonathan W Adams /* if the table has an entry for maxbuf, we're done */ 4172dce01e3fSJonathan W Adams if (size > maxbuf) 4173dce01e3fSJonathan W Adams break; 4174dce01e3fSJonathan W Adams 4175dce01e3fSJonathan W Adams /* cache size must be a multiple of the table unit */ 4176dce01e3fSJonathan W Adams ASSERT(P2PHASE(cache_size, table_unit) == 0); 4177dce01e3fSJonathan W Adams 4178dce01e3fSJonathan W Adams /* 4179dce01e3fSJonathan W Adams * If they allocate a multiple of the coherency granularity, 4180dce01e3fSJonathan W Adams * they get a coherency-granularity-aligned address. 4181dce01e3fSJonathan W Adams */ 4182dce01e3fSJonathan W Adams if (IS_P2ALIGNED(cache_size, 64)) 4183dce01e3fSJonathan W Adams align = 64; 4184dce01e3fSJonathan W Adams if (IS_P2ALIGNED(cache_size, PAGESIZE)) 4185dce01e3fSJonathan W Adams align = PAGESIZE; 4186dce01e3fSJonathan W Adams (void) snprintf(name, sizeof (name), 4187dce01e3fSJonathan W Adams "kmem_alloc_%lu", cache_size); 4188dce01e3fSJonathan W Adams cp = kmem_cache_create(name, cache_size, align, 4189dce01e3fSJonathan W Adams NULL, NULL, NULL, NULL, NULL, KMC_KMEM_ALLOC); 4190dce01e3fSJonathan W Adams 4191dce01e3fSJonathan W Adams while (size <= cache_size) { 4192dce01e3fSJonathan W Adams alloc_table[(size - 1) >> shift] = cp; 4193dce01e3fSJonathan W Adams size += table_unit; 4194dce01e3fSJonathan W Adams } 4195dce01e3fSJonathan W Adams } 4196dce01e3fSJonathan W Adams 4197dce01e3fSJonathan W Adams ASSERT(size > maxbuf); /* i.e. maxbuf <= max(cache_size) */ 4198dce01e3fSJonathan W Adams } 4199dce01e3fSJonathan W Adams 42007c478bd9Sstevel@tonic-gate static void 42017c478bd9Sstevel@tonic-gate kmem_cache_init(int pass, int use_large_pages) 42027c478bd9Sstevel@tonic-gate { 42037c478bd9Sstevel@tonic-gate int i; 4204dce01e3fSJonathan W Adams size_t maxbuf; 42057c478bd9Sstevel@tonic-gate kmem_magtype_t *mtp; 42067c478bd9Sstevel@tonic-gate 42077c478bd9Sstevel@tonic-gate for (i = 0; i < sizeof (kmem_magtype) / sizeof (*mtp); i++) { 4208dce01e3fSJonathan W Adams char name[KMEM_CACHE_NAMELEN + 1]; 4209dce01e3fSJonathan W Adams 42107c478bd9Sstevel@tonic-gate mtp = &kmem_magtype[i]; 42117c478bd9Sstevel@tonic-gate (void) sprintf(name, "kmem_magazine_%d", mtp->mt_magsize); 42127c478bd9Sstevel@tonic-gate mtp->mt_cache = kmem_cache_create(name, 42137c478bd9Sstevel@tonic-gate (mtp->mt_magsize + 1) * sizeof (void *), 42147c478bd9Sstevel@tonic-gate mtp->mt_align, NULL, NULL, NULL, NULL, 42157c478bd9Sstevel@tonic-gate kmem_msb_arena, KMC_NOHASH); 42167c478bd9Sstevel@tonic-gate } 42177c478bd9Sstevel@tonic-gate 42187c478bd9Sstevel@tonic-gate kmem_slab_cache = kmem_cache_create("kmem_slab_cache", 42197c478bd9Sstevel@tonic-gate sizeof (kmem_slab_t), 0, NULL, NULL, NULL, NULL, 42207c478bd9Sstevel@tonic-gate kmem_msb_arena, KMC_NOHASH); 42217c478bd9Sstevel@tonic-gate 42227c478bd9Sstevel@tonic-gate kmem_bufctl_cache = kmem_cache_create("kmem_bufctl_cache", 42237c478bd9Sstevel@tonic-gate sizeof (kmem_bufctl_t), 0, NULL, NULL, NULL, NULL, 42247c478bd9Sstevel@tonic-gate kmem_msb_arena, KMC_NOHASH); 42257c478bd9Sstevel@tonic-gate 42267c478bd9Sstevel@tonic-gate kmem_bufctl_audit_cache = kmem_cache_create("kmem_bufctl_audit_cache", 42277c478bd9Sstevel@tonic-gate sizeof (kmem_bufctl_audit_t), 0, NULL, NULL, NULL, NULL, 42287c478bd9Sstevel@tonic-gate kmem_msb_arena, KMC_NOHASH); 42297c478bd9Sstevel@tonic-gate 42307c478bd9Sstevel@tonic-gate if (pass == 2) { 42317c478bd9Sstevel@tonic-gate kmem_va_arena = vmem_create("kmem_va", 42327c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE, 42337c478bd9Sstevel@tonic-gate vmem_alloc, vmem_free, heap_arena, 42347c478bd9Sstevel@tonic-gate 8 * PAGESIZE, VM_SLEEP); 42357c478bd9Sstevel@tonic-gate 42367c478bd9Sstevel@tonic-gate if (use_large_pages) { 42377c478bd9Sstevel@tonic-gate kmem_default_arena = vmem_xcreate("kmem_default", 42387c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE, 42397c478bd9Sstevel@tonic-gate segkmem_alloc_lp, segkmem_free_lp, kmem_va_arena, 42409dd77bc8SDave Plauger 0, VMC_DUMPSAFE | VM_SLEEP); 42417c478bd9Sstevel@tonic-gate } else { 42427c478bd9Sstevel@tonic-gate kmem_default_arena = vmem_create("kmem_default", 42437c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE, 42447c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, kmem_va_arena, 42459dd77bc8SDave Plauger 0, VMC_DUMPSAFE | VM_SLEEP); 42467c478bd9Sstevel@tonic-gate } 4247dce01e3fSJonathan W Adams 4248dce01e3fSJonathan W Adams /* Figure out what our maximum cache size is */ 4249dce01e3fSJonathan W Adams maxbuf = kmem_max_cached; 4250dce01e3fSJonathan W Adams if (maxbuf <= KMEM_MAXBUF) { 4251dce01e3fSJonathan W Adams maxbuf = 0; 4252dce01e3fSJonathan W Adams kmem_max_cached = KMEM_MAXBUF; 4253dce01e3fSJonathan W Adams } else { 4254dce01e3fSJonathan W Adams size_t size = 0; 4255dce01e3fSJonathan W Adams size_t max = 4256dce01e3fSJonathan W Adams sizeof (kmem_big_alloc_sizes) / sizeof (int); 4257dce01e3fSJonathan W Adams /* 4258dce01e3fSJonathan W Adams * Round maxbuf up to an existing cache size. If maxbuf 4259dce01e3fSJonathan W Adams * is larger than the largest cache, we truncate it to 4260dce01e3fSJonathan W Adams * the largest cache's size. 4261dce01e3fSJonathan W Adams */ 4262dce01e3fSJonathan W Adams for (i = 0; i < max; i++) { 4263dce01e3fSJonathan W Adams size = kmem_big_alloc_sizes[i]; 4264dce01e3fSJonathan W Adams if (maxbuf <= size) 4265dce01e3fSJonathan W Adams break; 4266dce01e3fSJonathan W Adams } 4267dce01e3fSJonathan W Adams kmem_max_cached = maxbuf = size; 4268dce01e3fSJonathan W Adams } 4269dce01e3fSJonathan W Adams 4270dce01e3fSJonathan W Adams /* 4271dce01e3fSJonathan W Adams * The big alloc table may not be completely overwritten, so 4272dce01e3fSJonathan W Adams * we clear out any stale cache pointers from the first pass. 4273dce01e3fSJonathan W Adams */ 4274dce01e3fSJonathan W Adams bzero(kmem_big_alloc_table, sizeof (kmem_big_alloc_table)); 42757c478bd9Sstevel@tonic-gate } else { 42767c478bd9Sstevel@tonic-gate /* 42777c478bd9Sstevel@tonic-gate * During the first pass, the kmem_alloc_* caches 42787c478bd9Sstevel@tonic-gate * are treated as metadata. 42797c478bd9Sstevel@tonic-gate */ 42807c478bd9Sstevel@tonic-gate kmem_default_arena = kmem_msb_arena; 4281dce01e3fSJonathan W Adams maxbuf = KMEM_BIG_MAXBUF_32BIT; 42827c478bd9Sstevel@tonic-gate } 42837c478bd9Sstevel@tonic-gate 42847c478bd9Sstevel@tonic-gate /* 42857c478bd9Sstevel@tonic-gate * Set up the default caches to back kmem_alloc() 42867c478bd9Sstevel@tonic-gate */ 4287dce01e3fSJonathan W Adams kmem_alloc_caches_create( 4288dce01e3fSJonathan W Adams kmem_alloc_sizes, sizeof (kmem_alloc_sizes) / sizeof (int), 4289dce01e3fSJonathan W Adams kmem_alloc_table, KMEM_MAXBUF, KMEM_ALIGN_SHIFT); 4290dce01e3fSJonathan W Adams 4291dce01e3fSJonathan W Adams kmem_alloc_caches_create( 4292dce01e3fSJonathan W Adams kmem_big_alloc_sizes, sizeof (kmem_big_alloc_sizes) / sizeof (int), 4293dce01e3fSJonathan W Adams kmem_big_alloc_table, maxbuf, KMEM_BIG_SHIFT); 4294dce01e3fSJonathan W Adams 4295dce01e3fSJonathan W Adams kmem_big_alloc_table_max = maxbuf >> KMEM_BIG_SHIFT; 42967c478bd9Sstevel@tonic-gate } 42977c478bd9Sstevel@tonic-gate 42987c478bd9Sstevel@tonic-gate void 42997c478bd9Sstevel@tonic-gate kmem_init(void) 43007c478bd9Sstevel@tonic-gate { 43017c478bd9Sstevel@tonic-gate kmem_cache_t *cp; 43027c478bd9Sstevel@tonic-gate int old_kmem_flags = kmem_flags; 43037c478bd9Sstevel@tonic-gate int use_large_pages = 0; 43047c478bd9Sstevel@tonic-gate size_t maxverify, minfirewall; 43057c478bd9Sstevel@tonic-gate 43067c478bd9Sstevel@tonic-gate kstat_init(); 43077c478bd9Sstevel@tonic-gate 43087c478bd9Sstevel@tonic-gate /* 43097c478bd9Sstevel@tonic-gate * Don't do firewalled allocations if the heap is less than 1TB 43107c478bd9Sstevel@tonic-gate * (i.e. on a 32-bit kernel) 43117c478bd9Sstevel@tonic-gate * The resulting VM_NEXTFIT allocations would create too much 43127c478bd9Sstevel@tonic-gate * fragmentation in a small heap. 43137c478bd9Sstevel@tonic-gate */ 43147c478bd9Sstevel@tonic-gate #if defined(_LP64) 43157c478bd9Sstevel@tonic-gate maxverify = minfirewall = PAGESIZE / 2; 43167c478bd9Sstevel@tonic-gate #else 43177c478bd9Sstevel@tonic-gate maxverify = minfirewall = ULONG_MAX; 43187c478bd9Sstevel@tonic-gate #endif 43197c478bd9Sstevel@tonic-gate 43207c478bd9Sstevel@tonic-gate /* LINTED */ 43217c478bd9Sstevel@tonic-gate ASSERT(sizeof (kmem_cpu_cache_t) == KMEM_CPU_CACHE_SIZE); 43227c478bd9Sstevel@tonic-gate 4323b5fca8f8Stomee list_create(&kmem_caches, sizeof (kmem_cache_t), 4324b5fca8f8Stomee offsetof(kmem_cache_t, cache_link)); 43257c478bd9Sstevel@tonic-gate 43267c478bd9Sstevel@tonic-gate kmem_metadata_arena = vmem_create("kmem_metadata", NULL, 0, PAGESIZE, 43277c478bd9Sstevel@tonic-gate vmem_alloc, vmem_free, heap_arena, 8 * PAGESIZE, 43287c478bd9Sstevel@tonic-gate VM_SLEEP | VMC_NO_QCACHE); 43297c478bd9Sstevel@tonic-gate 43307c478bd9Sstevel@tonic-gate kmem_msb_arena = vmem_create("kmem_msb", NULL, 0, 43317c478bd9Sstevel@tonic-gate PAGESIZE, segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, 43329dd77bc8SDave Plauger VMC_DUMPSAFE | VM_SLEEP); 43337c478bd9Sstevel@tonic-gate 43347c478bd9Sstevel@tonic-gate kmem_cache_arena = vmem_create("kmem_cache", NULL, 0, KMEM_ALIGN, 43357c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, VM_SLEEP); 43367c478bd9Sstevel@tonic-gate 43377c478bd9Sstevel@tonic-gate kmem_hash_arena = vmem_create("kmem_hash", NULL, 0, KMEM_ALIGN, 43387c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, VM_SLEEP); 43397c478bd9Sstevel@tonic-gate 43407c478bd9Sstevel@tonic-gate kmem_log_arena = vmem_create("kmem_log", NULL, 0, KMEM_ALIGN, 43417c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP); 43427c478bd9Sstevel@tonic-gate 43437c478bd9Sstevel@tonic-gate kmem_firewall_va_arena = vmem_create("kmem_firewall_va", 43447c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE, 43457c478bd9Sstevel@tonic-gate kmem_firewall_va_alloc, kmem_firewall_va_free, heap_arena, 43467c478bd9Sstevel@tonic-gate 0, VM_SLEEP); 43477c478bd9Sstevel@tonic-gate 43487c478bd9Sstevel@tonic-gate kmem_firewall_arena = vmem_create("kmem_firewall", NULL, 0, PAGESIZE, 43499dd77bc8SDave Plauger segkmem_alloc, segkmem_free, kmem_firewall_va_arena, 0, 43509dd77bc8SDave Plauger VMC_DUMPSAFE | VM_SLEEP); 43517c478bd9Sstevel@tonic-gate 43527c478bd9Sstevel@tonic-gate /* temporary oversize arena for mod_read_system_file */ 43537c478bd9Sstevel@tonic-gate kmem_oversize_arena = vmem_create("kmem_oversize", NULL, 0, PAGESIZE, 43547c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP); 43557c478bd9Sstevel@tonic-gate 43567c478bd9Sstevel@tonic-gate kmem_reap_interval = 15 * hz; 43577c478bd9Sstevel@tonic-gate 43587c478bd9Sstevel@tonic-gate /* 43597c478bd9Sstevel@tonic-gate * Read /etc/system. This is a chicken-and-egg problem because 43607c478bd9Sstevel@tonic-gate * kmem_flags may be set in /etc/system, but mod_read_system_file() 43617c478bd9Sstevel@tonic-gate * needs to use the allocator. The simplest solution is to create 43627c478bd9Sstevel@tonic-gate * all the standard kmem caches, read /etc/system, destroy all the 43637c478bd9Sstevel@tonic-gate * caches we just created, and then create them all again in light 43647c478bd9Sstevel@tonic-gate * of the (possibly) new kmem_flags and other kmem tunables. 43657c478bd9Sstevel@tonic-gate */ 43667c478bd9Sstevel@tonic-gate kmem_cache_init(1, 0); 43677c478bd9Sstevel@tonic-gate 43687c478bd9Sstevel@tonic-gate mod_read_system_file(boothowto & RB_ASKNAME); 43697c478bd9Sstevel@tonic-gate 4370b5fca8f8Stomee while ((cp = list_tail(&kmem_caches)) != NULL) 43717c478bd9Sstevel@tonic-gate kmem_cache_destroy(cp); 43727c478bd9Sstevel@tonic-gate 43737c478bd9Sstevel@tonic-gate vmem_destroy(kmem_oversize_arena); 43747c478bd9Sstevel@tonic-gate 43757c478bd9Sstevel@tonic-gate if (old_kmem_flags & KMF_STICKY) 43767c478bd9Sstevel@tonic-gate kmem_flags = old_kmem_flags; 43777c478bd9Sstevel@tonic-gate 43787c478bd9Sstevel@tonic-gate if (!(kmem_flags & KMF_AUDIT)) 43797c478bd9Sstevel@tonic-gate vmem_seg_size = offsetof(vmem_seg_t, vs_thread); 43807c478bd9Sstevel@tonic-gate 43817c478bd9Sstevel@tonic-gate if (kmem_maxverify == 0) 43827c478bd9Sstevel@tonic-gate kmem_maxverify = maxverify; 43837c478bd9Sstevel@tonic-gate 43847c478bd9Sstevel@tonic-gate if (kmem_minfirewall == 0) 43857c478bd9Sstevel@tonic-gate kmem_minfirewall = minfirewall; 43867c478bd9Sstevel@tonic-gate 43877c478bd9Sstevel@tonic-gate /* 43887c478bd9Sstevel@tonic-gate * give segkmem a chance to figure out if we are using large pages 43897c478bd9Sstevel@tonic-gate * for the kernel heap 43907c478bd9Sstevel@tonic-gate */ 43917c478bd9Sstevel@tonic-gate use_large_pages = segkmem_lpsetup(); 43927c478bd9Sstevel@tonic-gate 43937c478bd9Sstevel@tonic-gate /* 43947c478bd9Sstevel@tonic-gate * To protect against corruption, we keep the actual number of callers 43957c478bd9Sstevel@tonic-gate * KMF_LITE records seperate from the tunable. We arbitrarily clamp 43967c478bd9Sstevel@tonic-gate * to 16, since the overhead for small buffers quickly gets out of 43977c478bd9Sstevel@tonic-gate * hand. 43987c478bd9Sstevel@tonic-gate * 43997c478bd9Sstevel@tonic-gate * The real limit would depend on the needs of the largest KMC_NOHASH 44007c478bd9Sstevel@tonic-gate * cache. 44017c478bd9Sstevel@tonic-gate */ 44027c478bd9Sstevel@tonic-gate kmem_lite_count = MIN(MAX(0, kmem_lite_pcs), 16); 44037c478bd9Sstevel@tonic-gate kmem_lite_pcs = kmem_lite_count; 44047c478bd9Sstevel@tonic-gate 44057c478bd9Sstevel@tonic-gate /* 44067c478bd9Sstevel@tonic-gate * Normally, we firewall oversized allocations when possible, but 44077c478bd9Sstevel@tonic-gate * if we are using large pages for kernel memory, and we don't have 44087c478bd9Sstevel@tonic-gate * any non-LITE debugging flags set, we want to allocate oversized 44097c478bd9Sstevel@tonic-gate * buffers from large pages, and so skip the firewalling. 44107c478bd9Sstevel@tonic-gate */ 44117c478bd9Sstevel@tonic-gate if (use_large_pages && 44127c478bd9Sstevel@tonic-gate ((kmem_flags & KMF_LITE) || !(kmem_flags & KMF_DEBUG))) { 44137c478bd9Sstevel@tonic-gate kmem_oversize_arena = vmem_xcreate("kmem_oversize", NULL, 0, 44147c478bd9Sstevel@tonic-gate PAGESIZE, segkmem_alloc_lp, segkmem_free_lp, heap_arena, 44159dd77bc8SDave Plauger 0, VMC_DUMPSAFE | VM_SLEEP); 44167c478bd9Sstevel@tonic-gate } else { 44177c478bd9Sstevel@tonic-gate kmem_oversize_arena = vmem_create("kmem_oversize", 44187c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE, 44197c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, kmem_minfirewall < ULONG_MAX? 44209dd77bc8SDave Plauger kmem_firewall_va_arena : heap_arena, 0, VMC_DUMPSAFE | 44219dd77bc8SDave Plauger VM_SLEEP); 44227c478bd9Sstevel@tonic-gate } 44237c478bd9Sstevel@tonic-gate 44247c478bd9Sstevel@tonic-gate kmem_cache_init(2, use_large_pages); 44257c478bd9Sstevel@tonic-gate 44267c478bd9Sstevel@tonic-gate if (kmem_flags & (KMF_AUDIT | KMF_RANDOMIZE)) { 44277c478bd9Sstevel@tonic-gate if (kmem_transaction_log_size == 0) 44287c478bd9Sstevel@tonic-gate kmem_transaction_log_size = kmem_maxavail() / 50; 44297c478bd9Sstevel@tonic-gate kmem_transaction_log = kmem_log_init(kmem_transaction_log_size); 44307c478bd9Sstevel@tonic-gate } 44317c478bd9Sstevel@tonic-gate 44327c478bd9Sstevel@tonic-gate if (kmem_flags & (KMF_CONTENTS | KMF_RANDOMIZE)) { 44337c478bd9Sstevel@tonic-gate if (kmem_content_log_size == 0) 44347c478bd9Sstevel@tonic-gate kmem_content_log_size = kmem_maxavail() / 50; 44357c478bd9Sstevel@tonic-gate kmem_content_log = kmem_log_init(kmem_content_log_size); 44367c478bd9Sstevel@tonic-gate } 44377c478bd9Sstevel@tonic-gate 44387c478bd9Sstevel@tonic-gate kmem_failure_log = kmem_log_init(kmem_failure_log_size); 44397c478bd9Sstevel@tonic-gate 44407c478bd9Sstevel@tonic-gate kmem_slab_log = kmem_log_init(kmem_slab_log_size); 44417c478bd9Sstevel@tonic-gate 44427c478bd9Sstevel@tonic-gate /* 44437c478bd9Sstevel@tonic-gate * Initialize STREAMS message caches so allocb() is available. 44447c478bd9Sstevel@tonic-gate * This allows us to initialize the logging framework (cmn_err(9F), 44457c478bd9Sstevel@tonic-gate * strlog(9F), etc) so we can start recording messages. 44467c478bd9Sstevel@tonic-gate */ 44477c478bd9Sstevel@tonic-gate streams_msg_init(); 44487d692464Sdp 44497c478bd9Sstevel@tonic-gate /* 44507c478bd9Sstevel@tonic-gate * Initialize the ZSD framework in Zones so modules loaded henceforth 44517c478bd9Sstevel@tonic-gate * can register their callbacks. 44527c478bd9Sstevel@tonic-gate */ 44537c478bd9Sstevel@tonic-gate zone_zsd_init(); 4454f4b3ec61Sdh 44557c478bd9Sstevel@tonic-gate log_init(); 44567c478bd9Sstevel@tonic-gate taskq_init(); 44577c478bd9Sstevel@tonic-gate 44587d692464Sdp /* 44597d692464Sdp * Warn about invalid or dangerous values of kmem_flags. 44607d692464Sdp * Always warn about unsupported values. 44617d692464Sdp */ 44627d692464Sdp if (((kmem_flags & ~(KMF_AUDIT | KMF_DEADBEEF | KMF_REDZONE | 44637d692464Sdp KMF_CONTENTS | KMF_LITE)) != 0) || 44647d692464Sdp ((kmem_flags & KMF_LITE) && kmem_flags != KMF_LITE)) 44657d692464Sdp cmn_err(CE_WARN, "kmem_flags set to unsupported value 0x%x. " 44667d692464Sdp "See the Solaris Tunable Parameters Reference Manual.", 44677d692464Sdp kmem_flags); 44687d692464Sdp 44697d692464Sdp #ifdef DEBUG 44707d692464Sdp if ((kmem_flags & KMF_DEBUG) == 0) 44717d692464Sdp cmn_err(CE_NOTE, "kmem debugging disabled."); 44727d692464Sdp #else 44737d692464Sdp /* 44747d692464Sdp * For non-debug kernels, the only "normal" flags are 0, KMF_LITE, 44757d692464Sdp * KMF_REDZONE, and KMF_CONTENTS (the last because it is only enabled 44767d692464Sdp * if KMF_AUDIT is set). We should warn the user about the performance 44777d692464Sdp * penalty of KMF_AUDIT or KMF_DEADBEEF if they are set and KMF_LITE 44787d692464Sdp * isn't set (since that disables AUDIT). 44797d692464Sdp */ 44807d692464Sdp if (!(kmem_flags & KMF_LITE) && 44817d692464Sdp (kmem_flags & (KMF_AUDIT | KMF_DEADBEEF)) != 0) 44827d692464Sdp cmn_err(CE_WARN, "High-overhead kmem debugging features " 44837d692464Sdp "enabled (kmem_flags = 0x%x). Performance degradation " 44847d692464Sdp "and large memory overhead possible. See the Solaris " 44857d692464Sdp "Tunable Parameters Reference Manual.", kmem_flags); 44867d692464Sdp #endif /* not DEBUG */ 44877d692464Sdp 44887c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_magazine_enable, NULL, TQ_SLEEP); 44897c478bd9Sstevel@tonic-gate 44907c478bd9Sstevel@tonic-gate kmem_ready = 1; 44917c478bd9Sstevel@tonic-gate 44927c478bd9Sstevel@tonic-gate /* 44937c478bd9Sstevel@tonic-gate * Initialize the platform-specific aligned/DMA memory allocator. 44947c478bd9Sstevel@tonic-gate */ 44957c478bd9Sstevel@tonic-gate ka_init(); 44967c478bd9Sstevel@tonic-gate 44977c478bd9Sstevel@tonic-gate /* 44987c478bd9Sstevel@tonic-gate * Initialize 32-bit ID cache. 44997c478bd9Sstevel@tonic-gate */ 45007c478bd9Sstevel@tonic-gate id32_init(); 4501f4b3ec61Sdh 4502f4b3ec61Sdh /* 4503f4b3ec61Sdh * Initialize the networking stack so modules loaded can 4504f4b3ec61Sdh * register their callbacks. 4505f4b3ec61Sdh */ 4506f4b3ec61Sdh netstack_init(); 45077c478bd9Sstevel@tonic-gate } 45087c478bd9Sstevel@tonic-gate 4509b5fca8f8Stomee static void 4510b5fca8f8Stomee kmem_move_init(void) 4511b5fca8f8Stomee { 4512b5fca8f8Stomee kmem_defrag_cache = kmem_cache_create("kmem_defrag_cache", 4513b5fca8f8Stomee sizeof (kmem_defrag_t), 0, NULL, NULL, NULL, NULL, 4514b5fca8f8Stomee kmem_msb_arena, KMC_NOHASH); 4515b5fca8f8Stomee kmem_move_cache = kmem_cache_create("kmem_move_cache", 4516b5fca8f8Stomee sizeof (kmem_move_t), 0, NULL, NULL, NULL, NULL, 4517b5fca8f8Stomee kmem_msb_arena, KMC_NOHASH); 4518b5fca8f8Stomee 4519b5fca8f8Stomee /* 4520b5fca8f8Stomee * kmem guarantees that move callbacks are sequential and that even 4521b5fca8f8Stomee * across multiple caches no two moves ever execute simultaneously. 4522b5fca8f8Stomee * Move callbacks are processed on a separate taskq so that client code 4523b5fca8f8Stomee * does not interfere with internal maintenance tasks. 4524b5fca8f8Stomee */ 4525b5fca8f8Stomee kmem_move_taskq = taskq_create_instance("kmem_move_taskq", 0, 1, 4526b5fca8f8Stomee minclsyspri, 100, INT_MAX, TASKQ_PREPOPULATE); 4527b5fca8f8Stomee } 4528b5fca8f8Stomee 45297c478bd9Sstevel@tonic-gate void 45307c478bd9Sstevel@tonic-gate kmem_thread_init(void) 45317c478bd9Sstevel@tonic-gate { 4532b5fca8f8Stomee kmem_move_init(); 45337c478bd9Sstevel@tonic-gate kmem_taskq = taskq_create_instance("kmem_taskq", 0, 1, minclsyspri, 45347c478bd9Sstevel@tonic-gate 300, INT_MAX, TASKQ_PREPOPULATE); 45357c478bd9Sstevel@tonic-gate } 45367c478bd9Sstevel@tonic-gate 45377c478bd9Sstevel@tonic-gate void 45387c478bd9Sstevel@tonic-gate kmem_mp_init(void) 45397c478bd9Sstevel@tonic-gate { 45407c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 45417c478bd9Sstevel@tonic-gate register_cpu_setup_func(kmem_cpu_setup, NULL); 45427c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 45437c478bd9Sstevel@tonic-gate 45447c478bd9Sstevel@tonic-gate kmem_update_timeout(NULL); 45452e0c549eSJonathan Adams 45462e0c549eSJonathan Adams taskq_mp_init(); 45477c478bd9Sstevel@tonic-gate } 4548b5fca8f8Stomee 4549b5fca8f8Stomee /* 4550b5fca8f8Stomee * Return the slab of the allocated buffer, or NULL if the buffer is not 4551b5fca8f8Stomee * allocated. This function may be called with a known slab address to determine 4552b5fca8f8Stomee * whether or not the buffer is allocated, or with a NULL slab address to obtain 4553b5fca8f8Stomee * an allocated buffer's slab. 4554b5fca8f8Stomee */ 4555b5fca8f8Stomee static kmem_slab_t * 4556b5fca8f8Stomee kmem_slab_allocated(kmem_cache_t *cp, kmem_slab_t *sp, void *buf) 4557b5fca8f8Stomee { 4558b5fca8f8Stomee kmem_bufctl_t *bcp, *bufbcp; 4559b5fca8f8Stomee 4560b5fca8f8Stomee ASSERT(MUTEX_HELD(&cp->cache_lock)); 4561b5fca8f8Stomee ASSERT(sp == NULL || KMEM_SLAB_MEMBER(sp, buf)); 4562b5fca8f8Stomee 4563b5fca8f8Stomee if (cp->cache_flags & KMF_HASH) { 4564b5fca8f8Stomee for (bcp = *KMEM_HASH(cp, buf); 4565b5fca8f8Stomee (bcp != NULL) && (bcp->bc_addr != buf); 4566b5fca8f8Stomee bcp = bcp->bc_next) { 4567b5fca8f8Stomee continue; 4568b5fca8f8Stomee } 4569b5fca8f8Stomee ASSERT(sp != NULL && bcp != NULL ? sp == bcp->bc_slab : 1); 4570b5fca8f8Stomee return (bcp == NULL ? NULL : bcp->bc_slab); 4571b5fca8f8Stomee } 4572b5fca8f8Stomee 4573b5fca8f8Stomee if (sp == NULL) { 4574b5fca8f8Stomee sp = KMEM_SLAB(cp, buf); 4575b5fca8f8Stomee } 4576b5fca8f8Stomee bufbcp = KMEM_BUFCTL(cp, buf); 4577b5fca8f8Stomee for (bcp = sp->slab_head; 4578b5fca8f8Stomee (bcp != NULL) && (bcp != bufbcp); 4579b5fca8f8Stomee bcp = bcp->bc_next) { 4580b5fca8f8Stomee continue; 4581b5fca8f8Stomee } 4582b5fca8f8Stomee return (bcp == NULL ? sp : NULL); 4583b5fca8f8Stomee } 4584b5fca8f8Stomee 4585b5fca8f8Stomee static boolean_t 4586b5fca8f8Stomee kmem_slab_is_reclaimable(kmem_cache_t *cp, kmem_slab_t *sp, int flags) 4587b5fca8f8Stomee { 4588686031edSTom Erickson long refcnt = sp->slab_refcnt; 4589b5fca8f8Stomee 4590b5fca8f8Stomee ASSERT(cp->cache_defrag != NULL); 4591b5fca8f8Stomee 4592686031edSTom Erickson /* 4593686031edSTom Erickson * For code coverage we want to be able to move an object within the 4594686031edSTom Erickson * same slab (the only partial slab) even if allocating the destination 4595686031edSTom Erickson * buffer resulted in a completely allocated slab. 4596686031edSTom Erickson */ 4597686031edSTom Erickson if (flags & KMM_DEBUG) { 4598686031edSTom Erickson return ((flags & KMM_DESPERATE) || 4599686031edSTom Erickson ((sp->slab_flags & KMEM_SLAB_NOMOVE) == 0)); 4600686031edSTom Erickson } 4601686031edSTom Erickson 4602b5fca8f8Stomee /* If we're desperate, we don't care if the client said NO. */ 4603b5fca8f8Stomee if (flags & KMM_DESPERATE) { 4604b5fca8f8Stomee return (refcnt < sp->slab_chunks); /* any partial */ 4605b5fca8f8Stomee } 4606b5fca8f8Stomee 4607b5fca8f8Stomee if (sp->slab_flags & KMEM_SLAB_NOMOVE) { 4608b5fca8f8Stomee return (B_FALSE); 4609b5fca8f8Stomee } 4610b5fca8f8Stomee 4611686031edSTom Erickson if ((refcnt == 1) || kmem_move_any_partial) { 4612b5fca8f8Stomee return (refcnt < sp->slab_chunks); 4613b5fca8f8Stomee } 4614b5fca8f8Stomee 4615b5fca8f8Stomee /* 4616b5fca8f8Stomee * The reclaim threshold is adjusted at each kmem_cache_scan() so that 4617b5fca8f8Stomee * slabs with a progressively higher percentage of used buffers can be 4618b5fca8f8Stomee * reclaimed until the cache as a whole is no longer fragmented. 4619b5fca8f8Stomee * 4620b5fca8f8Stomee * sp->slab_refcnt kmd_reclaim_numer 4621b5fca8f8Stomee * --------------- < ------------------ 4622b5fca8f8Stomee * sp->slab_chunks KMEM_VOID_FRACTION 4623b5fca8f8Stomee */ 4624b5fca8f8Stomee return ((refcnt * KMEM_VOID_FRACTION) < 4625b5fca8f8Stomee (sp->slab_chunks * cp->cache_defrag->kmd_reclaim_numer)); 4626b5fca8f8Stomee } 4627b5fca8f8Stomee 4628b5fca8f8Stomee /* 4629b5fca8f8Stomee * May be called from the kmem_move_taskq, from kmem_cache_move_notify_task(), 4630b5fca8f8Stomee * or when the buffer is freed. 4631b5fca8f8Stomee */ 4632b5fca8f8Stomee static void 4633b5fca8f8Stomee kmem_slab_move_yes(kmem_cache_t *cp, kmem_slab_t *sp, void *from_buf) 4634b5fca8f8Stomee { 4635b5fca8f8Stomee ASSERT(MUTEX_HELD(&cp->cache_lock)); 4636b5fca8f8Stomee ASSERT(KMEM_SLAB_MEMBER(sp, from_buf)); 4637b5fca8f8Stomee 4638b5fca8f8Stomee if (!KMEM_SLAB_IS_PARTIAL(sp)) { 4639b5fca8f8Stomee return; 4640b5fca8f8Stomee } 4641b5fca8f8Stomee 4642b5fca8f8Stomee if (sp->slab_flags & KMEM_SLAB_NOMOVE) { 4643b5fca8f8Stomee if (KMEM_SLAB_OFFSET(sp, from_buf) == sp->slab_stuck_offset) { 4644b5fca8f8Stomee avl_remove(&cp->cache_partial_slabs, sp); 4645b5fca8f8Stomee sp->slab_flags &= ~KMEM_SLAB_NOMOVE; 4646b5fca8f8Stomee sp->slab_stuck_offset = (uint32_t)-1; 4647b5fca8f8Stomee avl_add(&cp->cache_partial_slabs, sp); 4648b5fca8f8Stomee } 4649b5fca8f8Stomee } else { 4650b5fca8f8Stomee sp->slab_later_count = 0; 4651b5fca8f8Stomee sp->slab_stuck_offset = (uint32_t)-1; 4652b5fca8f8Stomee } 4653b5fca8f8Stomee } 4654b5fca8f8Stomee 4655b5fca8f8Stomee static void 4656b5fca8f8Stomee kmem_slab_move_no(kmem_cache_t *cp, kmem_slab_t *sp, void *from_buf) 4657b5fca8f8Stomee { 4658b5fca8f8Stomee ASSERT(taskq_member(kmem_move_taskq, curthread)); 4659b5fca8f8Stomee ASSERT(MUTEX_HELD(&cp->cache_lock)); 4660b5fca8f8Stomee ASSERT(KMEM_SLAB_MEMBER(sp, from_buf)); 4661b5fca8f8Stomee 4662b5fca8f8Stomee if (!KMEM_SLAB_IS_PARTIAL(sp)) { 4663b5fca8f8Stomee return; 4664b5fca8f8Stomee } 4665b5fca8f8Stomee 4666b5fca8f8Stomee avl_remove(&cp->cache_partial_slabs, sp); 4667b5fca8f8Stomee sp->slab_later_count = 0; 4668b5fca8f8Stomee sp->slab_flags |= KMEM_SLAB_NOMOVE; 4669b5fca8f8Stomee sp->slab_stuck_offset = KMEM_SLAB_OFFSET(sp, from_buf); 4670b5fca8f8Stomee avl_add(&cp->cache_partial_slabs, sp); 4671b5fca8f8Stomee } 4672b5fca8f8Stomee 4673b5fca8f8Stomee static void kmem_move_end(kmem_cache_t *, kmem_move_t *); 4674b5fca8f8Stomee 4675b5fca8f8Stomee /* 4676b5fca8f8Stomee * The move callback takes two buffer addresses, the buffer to be moved, and a 4677b5fca8f8Stomee * newly allocated and constructed buffer selected by kmem as the destination. 4678b5fca8f8Stomee * It also takes the size of the buffer and an optional user argument specified 4679b5fca8f8Stomee * at cache creation time. kmem guarantees that the buffer to be moved has not 4680b5fca8f8Stomee * been unmapped by the virtual memory subsystem. Beyond that, it cannot 4681b5fca8f8Stomee * guarantee the present whereabouts of the buffer to be moved, so it is up to 4682b5fca8f8Stomee * the client to safely determine whether or not it is still using the buffer. 4683b5fca8f8Stomee * The client must not free either of the buffers passed to the move callback, 4684b5fca8f8Stomee * since kmem wants to free them directly to the slab layer. The client response 4685b5fca8f8Stomee * tells kmem which of the two buffers to free: 4686b5fca8f8Stomee * 4687b5fca8f8Stomee * YES kmem frees the old buffer (the move was successful) 4688b5fca8f8Stomee * NO kmem frees the new buffer, marks the slab of the old buffer 4689b5fca8f8Stomee * non-reclaimable to avoid bothering the client again 4690b5fca8f8Stomee * LATER kmem frees the new buffer, increments slab_later_count 4691*d7db73d1SBryan Cantrill * DONT_KNOW kmem frees the new buffer 4692b5fca8f8Stomee * DONT_NEED kmem frees both the old buffer and the new buffer 4693b5fca8f8Stomee * 4694b5fca8f8Stomee * The pending callback argument now being processed contains both of the 4695b5fca8f8Stomee * buffers (old and new) passed to the move callback function, the slab of the 4696b5fca8f8Stomee * old buffer, and flags related to the move request, such as whether or not the 4697b5fca8f8Stomee * system was desperate for memory. 4698686031edSTom Erickson * 4699686031edSTom Erickson * Slabs are not freed while there is a pending callback, but instead are kept 4700686031edSTom Erickson * on a deadlist, which is drained after the last callback completes. This means 4701686031edSTom Erickson * that slabs are safe to access until kmem_move_end(), no matter how many of 4702686031edSTom Erickson * their buffers have been freed. Once slab_refcnt reaches zero, it stays at 4703686031edSTom Erickson * zero for as long as the slab remains on the deadlist and until the slab is 4704686031edSTom Erickson * freed. 4705b5fca8f8Stomee */ 4706b5fca8f8Stomee static void 4707b5fca8f8Stomee kmem_move_buffer(kmem_move_t *callback) 4708b5fca8f8Stomee { 4709b5fca8f8Stomee kmem_cbrc_t response; 4710b5fca8f8Stomee kmem_slab_t *sp = callback->kmm_from_slab; 4711b5fca8f8Stomee kmem_cache_t *cp = sp->slab_cache; 4712b5fca8f8Stomee boolean_t free_on_slab; 4713b5fca8f8Stomee 4714b5fca8f8Stomee ASSERT(taskq_member(kmem_move_taskq, curthread)); 4715b5fca8f8Stomee ASSERT(MUTEX_NOT_HELD(&cp->cache_lock)); 4716b5fca8f8Stomee ASSERT(KMEM_SLAB_MEMBER(sp, callback->kmm_from_buf)); 4717b5fca8f8Stomee 4718b5fca8f8Stomee /* 4719b5fca8f8Stomee * The number of allocated buffers on the slab may have changed since we 4720b5fca8f8Stomee * last checked the slab's reclaimability (when the pending move was 4721b5fca8f8Stomee * enqueued), or the client may have responded NO when asked to move 4722b5fca8f8Stomee * another buffer on the same slab. 4723b5fca8f8Stomee */ 4724b5fca8f8Stomee if (!kmem_slab_is_reclaimable(cp, sp, callback->kmm_flags)) { 4725b5fca8f8Stomee kmem_slab_free(cp, callback->kmm_to_buf); 4726b5fca8f8Stomee kmem_move_end(cp, callback); 4727b5fca8f8Stomee return; 4728b5fca8f8Stomee } 4729b5fca8f8Stomee 4730b5fca8f8Stomee /* 4731*d7db73d1SBryan Cantrill * Checking the slab layer is easy, so we might as well do that here 4732*d7db73d1SBryan Cantrill * in case we can avoid bothering the client. 4733b5fca8f8Stomee */ 4734b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4735b5fca8f8Stomee free_on_slab = (kmem_slab_allocated(cp, sp, 4736b5fca8f8Stomee callback->kmm_from_buf) == NULL); 4737b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4738b5fca8f8Stomee 4739b5fca8f8Stomee if (free_on_slab) { 4740b5fca8f8Stomee kmem_slab_free(cp, callback->kmm_to_buf); 4741b5fca8f8Stomee kmem_move_end(cp, callback); 4742b5fca8f8Stomee return; 4743b5fca8f8Stomee } 4744b5fca8f8Stomee 4745b5fca8f8Stomee if (cp->cache_flags & KMF_BUFTAG) { 4746b5fca8f8Stomee /* 4747b5fca8f8Stomee * Make kmem_cache_alloc_debug() apply the constructor for us. 4748b5fca8f8Stomee */ 4749b5fca8f8Stomee if (kmem_cache_alloc_debug(cp, callback->kmm_to_buf, 4750b5fca8f8Stomee KM_NOSLEEP, 1, caller()) != 0) { 4751b5fca8f8Stomee kmem_move_end(cp, callback); 4752b5fca8f8Stomee return; 4753b5fca8f8Stomee } 4754b5fca8f8Stomee } else if (cp->cache_constructor != NULL && 4755b5fca8f8Stomee cp->cache_constructor(callback->kmm_to_buf, cp->cache_private, 4756b5fca8f8Stomee KM_NOSLEEP) != 0) { 47571a5e258fSJosef 'Jeff' Sipek atomic_inc_64(&cp->cache_alloc_fail); 4758b5fca8f8Stomee kmem_slab_free(cp, callback->kmm_to_buf); 4759b5fca8f8Stomee kmem_move_end(cp, callback); 4760b5fca8f8Stomee return; 4761b5fca8f8Stomee } 4762b5fca8f8Stomee 4763b5fca8f8Stomee cp->cache_defrag->kmd_callbacks++; 4764b5fca8f8Stomee cp->cache_defrag->kmd_thread = curthread; 4765b5fca8f8Stomee cp->cache_defrag->kmd_from_buf = callback->kmm_from_buf; 4766b5fca8f8Stomee cp->cache_defrag->kmd_to_buf = callback->kmm_to_buf; 4767b5fca8f8Stomee DTRACE_PROBE2(kmem__move__start, kmem_cache_t *, cp, kmem_move_t *, 4768b5fca8f8Stomee callback); 4769b5fca8f8Stomee 4770b5fca8f8Stomee response = cp->cache_move(callback->kmm_from_buf, 4771b5fca8f8Stomee callback->kmm_to_buf, cp->cache_bufsize, cp->cache_private); 4772b5fca8f8Stomee 4773b5fca8f8Stomee DTRACE_PROBE3(kmem__move__end, kmem_cache_t *, cp, kmem_move_t *, 4774b5fca8f8Stomee callback, kmem_cbrc_t, response); 4775b5fca8f8Stomee cp->cache_defrag->kmd_thread = NULL; 4776b5fca8f8Stomee cp->cache_defrag->kmd_from_buf = NULL; 4777b5fca8f8Stomee cp->cache_defrag->kmd_to_buf = NULL; 4778b5fca8f8Stomee 4779b5fca8f8Stomee if (response == KMEM_CBRC_YES) { 4780b5fca8f8Stomee cp->cache_defrag->kmd_yes++; 4781b5fca8f8Stomee kmem_slab_free_constructed(cp, callback->kmm_from_buf, B_FALSE); 4782686031edSTom Erickson /* slab safe to access until kmem_move_end() */ 4783686031edSTom Erickson if (sp->slab_refcnt == 0) 4784686031edSTom Erickson cp->cache_defrag->kmd_slabs_freed++; 4785b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4786b5fca8f8Stomee kmem_slab_move_yes(cp, sp, callback->kmm_from_buf); 4787b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4788b5fca8f8Stomee kmem_move_end(cp, callback); 4789b5fca8f8Stomee return; 4790b5fca8f8Stomee } 4791b5fca8f8Stomee 4792b5fca8f8Stomee switch (response) { 4793b5fca8f8Stomee case KMEM_CBRC_NO: 4794b5fca8f8Stomee cp->cache_defrag->kmd_no++; 4795b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4796b5fca8f8Stomee kmem_slab_move_no(cp, sp, callback->kmm_from_buf); 4797b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4798b5fca8f8Stomee break; 4799b5fca8f8Stomee case KMEM_CBRC_LATER: 4800b5fca8f8Stomee cp->cache_defrag->kmd_later++; 4801b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4802b5fca8f8Stomee if (!KMEM_SLAB_IS_PARTIAL(sp)) { 4803b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4804b5fca8f8Stomee break; 4805b5fca8f8Stomee } 4806b5fca8f8Stomee 4807b5fca8f8Stomee if (++sp->slab_later_count >= KMEM_DISBELIEF) { 4808b5fca8f8Stomee kmem_slab_move_no(cp, sp, callback->kmm_from_buf); 4809b5fca8f8Stomee } else if (!(sp->slab_flags & KMEM_SLAB_NOMOVE)) { 4810b5fca8f8Stomee sp->slab_stuck_offset = KMEM_SLAB_OFFSET(sp, 4811b5fca8f8Stomee callback->kmm_from_buf); 4812b5fca8f8Stomee } 4813b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4814b5fca8f8Stomee break; 4815b5fca8f8Stomee case KMEM_CBRC_DONT_NEED: 4816b5fca8f8Stomee cp->cache_defrag->kmd_dont_need++; 4817b5fca8f8Stomee kmem_slab_free_constructed(cp, callback->kmm_from_buf, B_FALSE); 4818686031edSTom Erickson if (sp->slab_refcnt == 0) 4819686031edSTom Erickson cp->cache_defrag->kmd_slabs_freed++; 4820b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4821b5fca8f8Stomee kmem_slab_move_yes(cp, sp, callback->kmm_from_buf); 4822b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4823b5fca8f8Stomee break; 4824b5fca8f8Stomee case KMEM_CBRC_DONT_KNOW: 4825*d7db73d1SBryan Cantrill /* 4826*d7db73d1SBryan Cantrill * If we don't know if we can move this buffer or not, we'll 4827*d7db73d1SBryan Cantrill * just assume that we can't: if the buffer is in fact free, 4828*d7db73d1SBryan Cantrill * then it is sitting in one of the per-CPU magazines or in 4829*d7db73d1SBryan Cantrill * a full magazine in the depot layer. Either way, because 4830*d7db73d1SBryan Cantrill * defrag is induced in the same logic that reaps a cache, 4831*d7db73d1SBryan Cantrill * it's likely that full magazines will be returned to the 4832*d7db73d1SBryan Cantrill * system soon (thereby accomplishing what we're trying to 4833*d7db73d1SBryan Cantrill * accomplish here: return those magazines to their slabs). 4834*d7db73d1SBryan Cantrill * Given this, any work that we might do now to locate a buffer 4835*d7db73d1SBryan Cantrill * in a magazine is wasted (and expensive!) work; we bump 4836*d7db73d1SBryan Cantrill * a counter in this case and otherwise assume that we can't 4837*d7db73d1SBryan Cantrill * move it. 4838*d7db73d1SBryan Cantrill */ 4839b5fca8f8Stomee cp->cache_defrag->kmd_dont_know++; 4840b5fca8f8Stomee break; 4841b5fca8f8Stomee default: 4842b5fca8f8Stomee panic("'%s' (%p) unexpected move callback response %d\n", 4843b5fca8f8Stomee cp->cache_name, (void *)cp, response); 4844b5fca8f8Stomee } 4845b5fca8f8Stomee 4846b5fca8f8Stomee kmem_slab_free_constructed(cp, callback->kmm_to_buf, B_FALSE); 4847b5fca8f8Stomee kmem_move_end(cp, callback); 4848b5fca8f8Stomee } 4849b5fca8f8Stomee 4850b5fca8f8Stomee /* Return B_FALSE if there is insufficient memory for the move request. */ 4851b5fca8f8Stomee static boolean_t 4852b5fca8f8Stomee kmem_move_begin(kmem_cache_t *cp, kmem_slab_t *sp, void *buf, int flags) 4853b5fca8f8Stomee { 4854b5fca8f8Stomee void *to_buf; 4855b5fca8f8Stomee avl_index_t index; 4856b5fca8f8Stomee kmem_move_t *callback, *pending; 4857686031edSTom Erickson ulong_t n; 4858b5fca8f8Stomee 4859b5fca8f8Stomee ASSERT(taskq_member(kmem_taskq, curthread)); 4860b5fca8f8Stomee ASSERT(MUTEX_NOT_HELD(&cp->cache_lock)); 4861b5fca8f8Stomee ASSERT(sp->slab_flags & KMEM_SLAB_MOVE_PENDING); 4862b5fca8f8Stomee 4863b5fca8f8Stomee callback = kmem_cache_alloc(kmem_move_cache, KM_NOSLEEP); 4864*d7db73d1SBryan Cantrill 4865*d7db73d1SBryan Cantrill if (callback == NULL) 4866b5fca8f8Stomee return (B_FALSE); 4867b5fca8f8Stomee 4868b5fca8f8Stomee callback->kmm_from_slab = sp; 4869b5fca8f8Stomee callback->kmm_from_buf = buf; 4870b5fca8f8Stomee callback->kmm_flags = flags; 4871b5fca8f8Stomee 4872b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4873b5fca8f8Stomee 4874686031edSTom Erickson n = avl_numnodes(&cp->cache_partial_slabs); 4875686031edSTom Erickson if ((n == 0) || ((n == 1) && !(flags & KMM_DEBUG))) { 4876b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4877b5fca8f8Stomee kmem_cache_free(kmem_move_cache, callback); 4878b5fca8f8Stomee return (B_TRUE); /* there is no need for the move request */ 4879b5fca8f8Stomee } 4880b5fca8f8Stomee 4881b5fca8f8Stomee pending = avl_find(&cp->cache_defrag->kmd_moves_pending, buf, &index); 4882b5fca8f8Stomee if (pending != NULL) { 4883b5fca8f8Stomee /* 4884b5fca8f8Stomee * If the move is already pending and we're desperate now, 4885b5fca8f8Stomee * update the move flags. 4886b5fca8f8Stomee */ 4887b5fca8f8Stomee if (flags & KMM_DESPERATE) { 4888b5fca8f8Stomee pending->kmm_flags |= KMM_DESPERATE; 4889b5fca8f8Stomee } 4890b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4891b5fca8f8Stomee kmem_cache_free(kmem_move_cache, callback); 4892b5fca8f8Stomee return (B_TRUE); 4893b5fca8f8Stomee } 4894b5fca8f8Stomee 4895b942e89bSDavid Valin to_buf = kmem_slab_alloc_impl(cp, avl_first(&cp->cache_partial_slabs), 4896b942e89bSDavid Valin B_FALSE); 4897b5fca8f8Stomee callback->kmm_to_buf = to_buf; 4898b5fca8f8Stomee avl_insert(&cp->cache_defrag->kmd_moves_pending, callback, index); 4899b5fca8f8Stomee 4900b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4901b5fca8f8Stomee 4902b5fca8f8Stomee if (!taskq_dispatch(kmem_move_taskq, (task_func_t *)kmem_move_buffer, 4903b5fca8f8Stomee callback, TQ_NOSLEEP)) { 4904b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4905b5fca8f8Stomee avl_remove(&cp->cache_defrag->kmd_moves_pending, callback); 4906b5fca8f8Stomee mutex_exit(&cp->cache_lock); 490725e2c9cfStomee kmem_slab_free(cp, to_buf); 4908b5fca8f8Stomee kmem_cache_free(kmem_move_cache, callback); 4909b5fca8f8Stomee return (B_FALSE); 4910b5fca8f8Stomee } 4911b5fca8f8Stomee 4912b5fca8f8Stomee return (B_TRUE); 4913b5fca8f8Stomee } 4914b5fca8f8Stomee 4915b5fca8f8Stomee static void 4916b5fca8f8Stomee kmem_move_end(kmem_cache_t *cp, kmem_move_t *callback) 4917b5fca8f8Stomee { 4918b5fca8f8Stomee avl_index_t index; 4919b5fca8f8Stomee 4920b5fca8f8Stomee ASSERT(cp->cache_defrag != NULL); 4921b5fca8f8Stomee ASSERT(taskq_member(kmem_move_taskq, curthread)); 4922b5fca8f8Stomee ASSERT(MUTEX_NOT_HELD(&cp->cache_lock)); 4923b5fca8f8Stomee 4924b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4925b5fca8f8Stomee VERIFY(avl_find(&cp->cache_defrag->kmd_moves_pending, 4926b5fca8f8Stomee callback->kmm_from_buf, &index) != NULL); 4927b5fca8f8Stomee avl_remove(&cp->cache_defrag->kmd_moves_pending, callback); 4928b5fca8f8Stomee if (avl_is_empty(&cp->cache_defrag->kmd_moves_pending)) { 4929b5fca8f8Stomee list_t *deadlist = &cp->cache_defrag->kmd_deadlist; 4930b5fca8f8Stomee kmem_slab_t *sp; 4931b5fca8f8Stomee 4932b5fca8f8Stomee /* 4933b5fca8f8Stomee * The last pending move completed. Release all slabs from the 4934b5fca8f8Stomee * front of the dead list except for any slab at the tail that 4935b5fca8f8Stomee * needs to be released from the context of kmem_move_buffers(). 4936b5fca8f8Stomee * kmem deferred unmapping the buffers on these slabs in order 4937b5fca8f8Stomee * to guarantee that buffers passed to the move callback have 4938b5fca8f8Stomee * been touched only by kmem or by the client itself. 4939b5fca8f8Stomee */ 4940b5fca8f8Stomee while ((sp = list_remove_head(deadlist)) != NULL) { 4941b5fca8f8Stomee if (sp->slab_flags & KMEM_SLAB_MOVE_PENDING) { 4942b5fca8f8Stomee list_insert_tail(deadlist, sp); 4943b5fca8f8Stomee break; 4944b5fca8f8Stomee } 4945b5fca8f8Stomee cp->cache_defrag->kmd_deadcount--; 4946b5fca8f8Stomee cp->cache_slab_destroy++; 4947b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4948b5fca8f8Stomee kmem_slab_destroy(cp, sp); 4949b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4950b5fca8f8Stomee } 4951b5fca8f8Stomee } 4952b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4953b5fca8f8Stomee kmem_cache_free(kmem_move_cache, callback); 4954b5fca8f8Stomee } 4955b5fca8f8Stomee 4956b5fca8f8Stomee /* 4957b5fca8f8Stomee * Move buffers from least used slabs first by scanning backwards from the end 4958b5fca8f8Stomee * of the partial slab list. Scan at most max_scan candidate slabs and move 4959b5fca8f8Stomee * buffers from at most max_slabs slabs (0 for all partial slabs in both cases). 4960b5fca8f8Stomee * If desperate to reclaim memory, move buffers from any partial slab, otherwise 4961b5fca8f8Stomee * skip slabs with a ratio of allocated buffers at or above the current 4962b5fca8f8Stomee * threshold. Return the number of unskipped slabs (at most max_slabs, -1 if the 4963b5fca8f8Stomee * scan is aborted) so that the caller can adjust the reclaimability threshold 4964b5fca8f8Stomee * depending on how many reclaimable slabs it finds. 4965b5fca8f8Stomee * 4966b5fca8f8Stomee * kmem_move_buffers() drops and reacquires cache_lock every time it issues a 4967b5fca8f8Stomee * move request, since it is not valid for kmem_move_begin() to call 4968b5fca8f8Stomee * kmem_cache_alloc() or taskq_dispatch() with cache_lock held. 4969b5fca8f8Stomee */ 4970b5fca8f8Stomee static int 4971b5fca8f8Stomee kmem_move_buffers(kmem_cache_t *cp, size_t max_scan, size_t max_slabs, 4972b5fca8f8Stomee int flags) 4973b5fca8f8Stomee { 4974b5fca8f8Stomee kmem_slab_t *sp; 4975b5fca8f8Stomee void *buf; 4976b5fca8f8Stomee int i, j; /* slab index, buffer index */ 4977b5fca8f8Stomee int s; /* reclaimable slabs */ 4978b5fca8f8Stomee int b; /* allocated (movable) buffers on reclaimable slab */ 4979b5fca8f8Stomee boolean_t success; 4980b5fca8f8Stomee int refcnt; 4981b5fca8f8Stomee int nomove; 4982b5fca8f8Stomee 4983b5fca8f8Stomee ASSERT(taskq_member(kmem_taskq, curthread)); 4984b5fca8f8Stomee ASSERT(MUTEX_HELD(&cp->cache_lock)); 4985b5fca8f8Stomee ASSERT(kmem_move_cache != NULL); 4986b5fca8f8Stomee ASSERT(cp->cache_move != NULL && cp->cache_defrag != NULL); 4987686031edSTom Erickson ASSERT((flags & KMM_DEBUG) ? !avl_is_empty(&cp->cache_partial_slabs) : 4988686031edSTom Erickson avl_numnodes(&cp->cache_partial_slabs) > 1); 4989b5fca8f8Stomee 4990b5fca8f8Stomee if (kmem_move_blocked) { 4991b5fca8f8Stomee return (0); 4992b5fca8f8Stomee } 4993b5fca8f8Stomee 4994b5fca8f8Stomee if (kmem_move_fulltilt) { 4995b5fca8f8Stomee flags |= KMM_DESPERATE; 4996b5fca8f8Stomee } 4997b5fca8f8Stomee 4998b5fca8f8Stomee if (max_scan == 0 || (flags & KMM_DESPERATE)) { 4999b5fca8f8Stomee /* 5000b5fca8f8Stomee * Scan as many slabs as needed to find the desired number of 5001b5fca8f8Stomee * candidate slabs. 5002b5fca8f8Stomee */ 5003b5fca8f8Stomee max_scan = (size_t)-1; 5004b5fca8f8Stomee } 5005b5fca8f8Stomee 5006b5fca8f8Stomee if (max_slabs == 0 || (flags & KMM_DESPERATE)) { 5007b5fca8f8Stomee /* Find as many candidate slabs as possible. */ 5008b5fca8f8Stomee max_slabs = (size_t)-1; 5009b5fca8f8Stomee } 5010b5fca8f8Stomee 5011b5fca8f8Stomee sp = avl_last(&cp->cache_partial_slabs); 5012686031edSTom Erickson ASSERT(KMEM_SLAB_IS_PARTIAL(sp)); 5013686031edSTom Erickson for (i = 0, s = 0; (i < max_scan) && (s < max_slabs) && (sp != NULL) && 5014686031edSTom Erickson ((sp != avl_first(&cp->cache_partial_slabs)) || 5015686031edSTom Erickson (flags & KMM_DEBUG)); 5016b5fca8f8Stomee sp = AVL_PREV(&cp->cache_partial_slabs, sp), i++) { 5017b5fca8f8Stomee 5018b5fca8f8Stomee if (!kmem_slab_is_reclaimable(cp, sp, flags)) { 5019b5fca8f8Stomee continue; 5020b5fca8f8Stomee } 5021b5fca8f8Stomee s++; 5022b5fca8f8Stomee 5023b5fca8f8Stomee /* Look for allocated buffers to move. */ 5024b5fca8f8Stomee for (j = 0, b = 0, buf = sp->slab_base; 5025b5fca8f8Stomee (j < sp->slab_chunks) && (b < sp->slab_refcnt); 5026b5fca8f8Stomee buf = (((char *)buf) + cp->cache_chunksize), j++) { 5027b5fca8f8Stomee 5028b5fca8f8Stomee if (kmem_slab_allocated(cp, sp, buf) == NULL) { 5029b5fca8f8Stomee continue; 5030b5fca8f8Stomee } 5031b5fca8f8Stomee 5032b5fca8f8Stomee b++; 5033b5fca8f8Stomee 5034b5fca8f8Stomee /* 5035b5fca8f8Stomee * Prevent the slab from being destroyed while we drop 5036b5fca8f8Stomee * cache_lock and while the pending move is not yet 5037b5fca8f8Stomee * registered. Flag the pending move while 5038b5fca8f8Stomee * kmd_moves_pending may still be empty, since we can't 5039b5fca8f8Stomee * yet rely on a non-zero pending move count to prevent 5040b5fca8f8Stomee * the slab from being destroyed. 5041b5fca8f8Stomee */ 5042b5fca8f8Stomee ASSERT(!(sp->slab_flags & KMEM_SLAB_MOVE_PENDING)); 5043b5fca8f8Stomee sp->slab_flags |= KMEM_SLAB_MOVE_PENDING; 5044b5fca8f8Stomee /* 5045b5fca8f8Stomee * Recheck refcnt and nomove after reacquiring the lock, 5046b5fca8f8Stomee * since these control the order of partial slabs, and 5047b5fca8f8Stomee * we want to know if we can pick up the scan where we 5048b5fca8f8Stomee * left off. 5049b5fca8f8Stomee */ 5050b5fca8f8Stomee refcnt = sp->slab_refcnt; 5051b5fca8f8Stomee nomove = (sp->slab_flags & KMEM_SLAB_NOMOVE); 5052b5fca8f8Stomee mutex_exit(&cp->cache_lock); 5053b5fca8f8Stomee 5054b5fca8f8Stomee success = kmem_move_begin(cp, sp, buf, flags); 5055b5fca8f8Stomee 5056b5fca8f8Stomee /* 5057b5fca8f8Stomee * Now, before the lock is reacquired, kmem could 5058b5fca8f8Stomee * process all pending move requests and purge the 5059b5fca8f8Stomee * deadlist, so that upon reacquiring the lock, sp has 5060686031edSTom Erickson * been remapped. Or, the client may free all the 5061686031edSTom Erickson * objects on the slab while the pending moves are still 5062686031edSTom Erickson * on the taskq. Therefore, the KMEM_SLAB_MOVE_PENDING 5063b5fca8f8Stomee * flag causes the slab to be put at the end of the 5064686031edSTom Erickson * deadlist and prevents it from being destroyed, since 5065686031edSTom Erickson * we plan to destroy it here after reacquiring the 5066686031edSTom Erickson * lock. 5067b5fca8f8Stomee */ 5068b5fca8f8Stomee mutex_enter(&cp->cache_lock); 5069b5fca8f8Stomee ASSERT(sp->slab_flags & KMEM_SLAB_MOVE_PENDING); 5070b5fca8f8Stomee sp->slab_flags &= ~KMEM_SLAB_MOVE_PENDING; 5071b5fca8f8Stomee 5072b5fca8f8Stomee if (sp->slab_refcnt == 0) { 5073b5fca8f8Stomee list_t *deadlist = 5074b5fca8f8Stomee &cp->cache_defrag->kmd_deadlist; 5075686031edSTom Erickson list_remove(deadlist, sp); 5076b5fca8f8Stomee 5077686031edSTom Erickson if (!avl_is_empty( 5078686031edSTom Erickson &cp->cache_defrag->kmd_moves_pending)) { 5079686031edSTom Erickson /* 5080686031edSTom Erickson * A pending move makes it unsafe to 5081686031edSTom Erickson * destroy the slab, because even though 5082686031edSTom Erickson * the move is no longer needed, the 5083686031edSTom Erickson * context where that is determined 5084686031edSTom Erickson * requires the slab to exist. 5085686031edSTom Erickson * Fortunately, a pending move also 5086686031edSTom Erickson * means we don't need to destroy the 5087686031edSTom Erickson * slab here, since it will get 5088686031edSTom Erickson * destroyed along with any other slabs 5089686031edSTom Erickson * on the deadlist after the last 5090686031edSTom Erickson * pending move completes. 5091686031edSTom Erickson */ 5092686031edSTom Erickson list_insert_head(deadlist, sp); 5093686031edSTom Erickson return (-1); 5094686031edSTom Erickson } 5095b5fca8f8Stomee 5096686031edSTom Erickson /* 5097686031edSTom Erickson * Destroy the slab now if it was completely 5098686031edSTom Erickson * freed while we dropped cache_lock and there 5099686031edSTom Erickson * are no pending moves. Since slab_refcnt 5100686031edSTom Erickson * cannot change once it reaches zero, no new 5101686031edSTom Erickson * pending moves from that slab are possible. 5102686031edSTom Erickson */ 5103b5fca8f8Stomee cp->cache_defrag->kmd_deadcount--; 5104b5fca8f8Stomee cp->cache_slab_destroy++; 5105b5fca8f8Stomee mutex_exit(&cp->cache_lock); 5106b5fca8f8Stomee kmem_slab_destroy(cp, sp); 5107b5fca8f8Stomee mutex_enter(&cp->cache_lock); 5108b5fca8f8Stomee /* 5109b5fca8f8Stomee * Since we can't pick up the scan where we left 5110b5fca8f8Stomee * off, abort the scan and say nothing about the 5111b5fca8f8Stomee * number of reclaimable slabs. 5112b5fca8f8Stomee */ 5113b5fca8f8Stomee return (-1); 5114b5fca8f8Stomee } 5115b5fca8f8Stomee 5116b5fca8f8Stomee if (!success) { 5117b5fca8f8Stomee /* 5118b5fca8f8Stomee * Abort the scan if there is not enough memory 5119b5fca8f8Stomee * for the request and say nothing about the 5120b5fca8f8Stomee * number of reclaimable slabs. 5121b5fca8f8Stomee */ 5122b5fca8f8Stomee return (-1); 5123b5fca8f8Stomee } 5124b5fca8f8Stomee 5125b5fca8f8Stomee /* 5126b5fca8f8Stomee * The slab's position changed while the lock was 5127b5fca8f8Stomee * dropped, so we don't know where we are in the 5128b5fca8f8Stomee * sequence any more. 5129b5fca8f8Stomee */ 5130b5fca8f8Stomee if (sp->slab_refcnt != refcnt) { 5131686031edSTom Erickson /* 5132686031edSTom Erickson * If this is a KMM_DEBUG move, the slab_refcnt 5133686031edSTom Erickson * may have changed because we allocated a 5134686031edSTom Erickson * destination buffer on the same slab. In that 5135686031edSTom Erickson * case, we're not interested in counting it. 5136686031edSTom Erickson */ 5137b5fca8f8Stomee return (-1); 5138b5fca8f8Stomee } 5139*d7db73d1SBryan Cantrill if ((sp->slab_flags & KMEM_SLAB_NOMOVE) != nomove) 5140b5fca8f8Stomee return (-1); 5141b5fca8f8Stomee 5142b5fca8f8Stomee /* 5143b5fca8f8Stomee * Generating a move request allocates a destination 5144686031edSTom Erickson * buffer from the slab layer, bumping the first partial 5145686031edSTom Erickson * slab if it is completely allocated. If the current 5146686031edSTom Erickson * slab becomes the first partial slab as a result, we 5147686031edSTom Erickson * can't continue to scan backwards. 5148686031edSTom Erickson * 5149686031edSTom Erickson * If this is a KMM_DEBUG move and we allocated the 5150686031edSTom Erickson * destination buffer from the last partial slab, then 5151686031edSTom Erickson * the buffer we're moving is on the same slab and our 5152686031edSTom Erickson * slab_refcnt has changed, causing us to return before 5153686031edSTom Erickson * reaching here if there are no partial slabs left. 5154b5fca8f8Stomee */ 5155b5fca8f8Stomee ASSERT(!avl_is_empty(&cp->cache_partial_slabs)); 5156b5fca8f8Stomee if (sp == avl_first(&cp->cache_partial_slabs)) { 5157686031edSTom Erickson /* 5158686031edSTom Erickson * We're not interested in a second KMM_DEBUG 5159686031edSTom Erickson * move. 5160686031edSTom Erickson */ 5161b5fca8f8Stomee goto end_scan; 5162b5fca8f8Stomee } 5163b5fca8f8Stomee } 5164b5fca8f8Stomee } 5165b5fca8f8Stomee end_scan: 5166b5fca8f8Stomee 5167b5fca8f8Stomee return (s); 5168b5fca8f8Stomee } 5169b5fca8f8Stomee 5170b5fca8f8Stomee typedef struct kmem_move_notify_args { 5171b5fca8f8Stomee kmem_cache_t *kmna_cache; 5172b5fca8f8Stomee void *kmna_buf; 5173b5fca8f8Stomee } kmem_move_notify_args_t; 5174b5fca8f8Stomee 5175b5fca8f8Stomee static void 5176b5fca8f8Stomee kmem_cache_move_notify_task(void *arg) 5177b5fca8f8Stomee { 5178b5fca8f8Stomee kmem_move_notify_args_t *args = arg; 5179b5fca8f8Stomee kmem_cache_t *cp = args->kmna_cache; 5180b5fca8f8Stomee void *buf = args->kmna_buf; 5181b5fca8f8Stomee kmem_slab_t *sp; 5182b5fca8f8Stomee 5183b5fca8f8Stomee ASSERT(taskq_member(kmem_taskq, curthread)); 5184b5fca8f8Stomee ASSERT(list_link_active(&cp->cache_link)); 5185b5fca8f8Stomee 5186b5fca8f8Stomee kmem_free(args, sizeof (kmem_move_notify_args_t)); 5187b5fca8f8Stomee mutex_enter(&cp->cache_lock); 5188b5fca8f8Stomee sp = kmem_slab_allocated(cp, NULL, buf); 5189b5fca8f8Stomee 5190b5fca8f8Stomee /* Ignore the notification if the buffer is no longer allocated. */ 5191b5fca8f8Stomee if (sp == NULL) { 5192b5fca8f8Stomee mutex_exit(&cp->cache_lock); 5193b5fca8f8Stomee return; 5194b5fca8f8Stomee } 5195b5fca8f8Stomee 5196b5fca8f8Stomee /* Ignore the notification if there's no reason to move the buffer. */ 5197b5fca8f8Stomee if (avl_numnodes(&cp->cache_partial_slabs) > 1) { 5198b5fca8f8Stomee /* 5199b5fca8f8Stomee * So far the notification is not ignored. Ignore the 5200b5fca8f8Stomee * notification if the slab is not marked by an earlier refusal 5201b5fca8f8Stomee * to move a buffer. 5202b5fca8f8Stomee */ 5203b5fca8f8Stomee if (!(sp->slab_flags & KMEM_SLAB_NOMOVE) && 5204b5fca8f8Stomee (sp->slab_later_count == 0)) { 5205b5fca8f8Stomee mutex_exit(&cp->cache_lock); 5206b5fca8f8Stomee return; 5207b5fca8f8Stomee } 5208b5fca8f8Stomee 5209b5fca8f8Stomee kmem_slab_move_yes(cp, sp, buf); 5210b5fca8f8Stomee ASSERT(!(sp->slab_flags & KMEM_SLAB_MOVE_PENDING)); 5211b5fca8f8Stomee sp->slab_flags |= KMEM_SLAB_MOVE_PENDING; 5212b5fca8f8Stomee mutex_exit(&cp->cache_lock); 5213b5fca8f8Stomee /* see kmem_move_buffers() about dropping the lock */ 5214b5fca8f8Stomee (void) kmem_move_begin(cp, sp, buf, KMM_NOTIFY); 5215b5fca8f8Stomee mutex_enter(&cp->cache_lock); 5216b5fca8f8Stomee ASSERT(sp->slab_flags & KMEM_SLAB_MOVE_PENDING); 5217b5fca8f8Stomee sp->slab_flags &= ~KMEM_SLAB_MOVE_PENDING; 5218b5fca8f8Stomee if (sp->slab_refcnt == 0) { 5219b5fca8f8Stomee list_t *deadlist = &cp->cache_defrag->kmd_deadlist; 5220686031edSTom Erickson list_remove(deadlist, sp); 5221b5fca8f8Stomee 5222686031edSTom Erickson if (!avl_is_empty( 5223686031edSTom Erickson &cp->cache_defrag->kmd_moves_pending)) { 5224686031edSTom Erickson list_insert_head(deadlist, sp); 5225686031edSTom Erickson mutex_exit(&cp->cache_lock); 5226686031edSTom Erickson return; 5227686031edSTom Erickson } 5228b5fca8f8Stomee 5229b5fca8f8Stomee cp->cache_defrag->kmd_deadcount--; 5230b5fca8f8Stomee cp->cache_slab_destroy++; 5231b5fca8f8Stomee mutex_exit(&cp->cache_lock); 5232b5fca8f8Stomee kmem_slab_destroy(cp, sp); 5233b5fca8f8Stomee return; 5234b5fca8f8Stomee } 5235b5fca8f8Stomee } else { 5236b5fca8f8Stomee kmem_slab_move_yes(cp, sp, buf); 5237b5fca8f8Stomee } 5238b5fca8f8Stomee mutex_exit(&cp->cache_lock); 5239b5fca8f8Stomee } 5240b5fca8f8Stomee 5241b5fca8f8Stomee void 5242b5fca8f8Stomee kmem_cache_move_notify(kmem_cache_t *cp, void *buf) 5243b5fca8f8Stomee { 5244b5fca8f8Stomee kmem_move_notify_args_t *args; 5245b5fca8f8Stomee 5246b5fca8f8Stomee args = kmem_alloc(sizeof (kmem_move_notify_args_t), KM_NOSLEEP); 5247b5fca8f8Stomee if (args != NULL) { 5248b5fca8f8Stomee args->kmna_cache = cp; 5249b5fca8f8Stomee args->kmna_buf = buf; 5250eb697d4eStomee if (!taskq_dispatch(kmem_taskq, 5251b5fca8f8Stomee (task_func_t *)kmem_cache_move_notify_task, args, 5252eb697d4eStomee TQ_NOSLEEP)) 5253eb697d4eStomee kmem_free(args, sizeof (kmem_move_notify_args_t)); 5254b5fca8f8Stomee } 5255b5fca8f8Stomee } 5256b5fca8f8Stomee 5257b5fca8f8Stomee static void 5258b5fca8f8Stomee kmem_cache_defrag(kmem_cache_t *cp) 5259b5fca8f8Stomee { 5260b5fca8f8Stomee size_t n; 5261b5fca8f8Stomee 5262b5fca8f8Stomee ASSERT(cp->cache_defrag != NULL); 5263b5fca8f8Stomee 5264b5fca8f8Stomee mutex_enter(&cp->cache_lock); 5265b5fca8f8Stomee n = avl_numnodes(&cp->cache_partial_slabs); 5266b5fca8f8Stomee if (n > 1) { 5267b5fca8f8Stomee /* kmem_move_buffers() drops and reacquires cache_lock */ 5268686031edSTom Erickson cp->cache_defrag->kmd_defrags++; 5269686031edSTom Erickson (void) kmem_move_buffers(cp, n, 0, KMM_DESPERATE); 5270b5fca8f8Stomee } 5271b5fca8f8Stomee mutex_exit(&cp->cache_lock); 5272b5fca8f8Stomee } 5273b5fca8f8Stomee 5274b5fca8f8Stomee /* Is this cache above the fragmentation threshold? */ 5275b5fca8f8Stomee static boolean_t 5276b5fca8f8Stomee kmem_cache_frag_threshold(kmem_cache_t *cp, uint64_t nfree) 5277b5fca8f8Stomee { 5278b5fca8f8Stomee /* 5279b5fca8f8Stomee * nfree kmem_frag_numer 5280b5fca8f8Stomee * ------------------ > --------------- 5281b5fca8f8Stomee * cp->cache_buftotal kmem_frag_denom 5282b5fca8f8Stomee */ 5283b5fca8f8Stomee return ((nfree * kmem_frag_denom) > 5284b5fca8f8Stomee (cp->cache_buftotal * kmem_frag_numer)); 5285b5fca8f8Stomee } 5286b5fca8f8Stomee 5287b5fca8f8Stomee static boolean_t 5288b5fca8f8Stomee kmem_cache_is_fragmented(kmem_cache_t *cp, boolean_t *doreap) 5289b5fca8f8Stomee { 5290b5fca8f8Stomee boolean_t fragmented; 5291b5fca8f8Stomee uint64_t nfree; 5292b5fca8f8Stomee 5293b5fca8f8Stomee ASSERT(MUTEX_HELD(&cp->cache_lock)); 5294b5fca8f8Stomee *doreap = B_FALSE; 5295b5fca8f8Stomee 5296686031edSTom Erickson if (kmem_move_fulltilt) { 5297686031edSTom Erickson if (avl_numnodes(&cp->cache_partial_slabs) > 1) { 5298686031edSTom Erickson return (B_TRUE); 5299686031edSTom Erickson } 5300686031edSTom Erickson } else { 5301686031edSTom Erickson if ((cp->cache_complete_slab_count + avl_numnodes( 5302686031edSTom Erickson &cp->cache_partial_slabs)) < kmem_frag_minslabs) { 5303686031edSTom Erickson return (B_FALSE); 5304686031edSTom Erickson } 5305686031edSTom Erickson } 5306b5fca8f8Stomee 5307b5fca8f8Stomee nfree = cp->cache_bufslab; 5308686031edSTom Erickson fragmented = ((avl_numnodes(&cp->cache_partial_slabs) > 1) && 5309686031edSTom Erickson kmem_cache_frag_threshold(cp, nfree)); 5310686031edSTom Erickson 5311b5fca8f8Stomee /* 5312b5fca8f8Stomee * Free buffers in the magazine layer appear allocated from the point of 5313b5fca8f8Stomee * view of the slab layer. We want to know if the slab layer would 5314b5fca8f8Stomee * appear fragmented if we included free buffers from magazines that 5315b5fca8f8Stomee * have fallen out of the working set. 5316b5fca8f8Stomee */ 5317b5fca8f8Stomee if (!fragmented) { 5318b5fca8f8Stomee long reap; 5319b5fca8f8Stomee 5320b5fca8f8Stomee mutex_enter(&cp->cache_depot_lock); 5321b5fca8f8Stomee reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min); 5322b5fca8f8Stomee reap = MIN(reap, cp->cache_full.ml_total); 5323b5fca8f8Stomee mutex_exit(&cp->cache_depot_lock); 5324b5fca8f8Stomee 5325b5fca8f8Stomee nfree += ((uint64_t)reap * cp->cache_magtype->mt_magsize); 5326b5fca8f8Stomee if (kmem_cache_frag_threshold(cp, nfree)) { 5327b5fca8f8Stomee *doreap = B_TRUE; 5328b5fca8f8Stomee } 5329b5fca8f8Stomee } 5330b5fca8f8Stomee 5331b5fca8f8Stomee return (fragmented); 5332b5fca8f8Stomee } 5333b5fca8f8Stomee 5334b5fca8f8Stomee /* Called periodically from kmem_taskq */ 5335b5fca8f8Stomee static void 5336b5fca8f8Stomee kmem_cache_scan(kmem_cache_t *cp) 5337b5fca8f8Stomee { 5338b5fca8f8Stomee boolean_t reap = B_FALSE; 5339686031edSTom Erickson kmem_defrag_t *kmd; 5340b5fca8f8Stomee 5341b5fca8f8Stomee ASSERT(taskq_member(kmem_taskq, curthread)); 5342b5fca8f8Stomee 5343b5fca8f8Stomee mutex_enter(&cp->cache_lock); 5344b5fca8f8Stomee 5345686031edSTom Erickson kmd = cp->cache_defrag; 5346686031edSTom Erickson if (kmd->kmd_consolidate > 0) { 5347686031edSTom Erickson kmd->kmd_consolidate--; 5348686031edSTom Erickson mutex_exit(&cp->cache_lock); 5349686031edSTom Erickson kmem_cache_reap(cp); 5350686031edSTom Erickson return; 5351686031edSTom Erickson } 5352686031edSTom Erickson 5353b5fca8f8Stomee if (kmem_cache_is_fragmented(cp, &reap)) { 5354b5fca8f8Stomee size_t slabs_found; 5355b5fca8f8Stomee 5356b5fca8f8Stomee /* 5357b5fca8f8Stomee * Consolidate reclaimable slabs from the end of the partial 5358b5fca8f8Stomee * slab list (scan at most kmem_reclaim_scan_range slabs to find 5359b5fca8f8Stomee * reclaimable slabs). Keep track of how many candidate slabs we 5360b5fca8f8Stomee * looked for and how many we actually found so we can adjust 5361b5fca8f8Stomee * the definition of a candidate slab if we're having trouble 5362b5fca8f8Stomee * finding them. 5363b5fca8f8Stomee * 5364b5fca8f8Stomee * kmem_move_buffers() drops and reacquires cache_lock. 5365b5fca8f8Stomee */ 5366686031edSTom Erickson kmd->kmd_scans++; 5367b5fca8f8Stomee slabs_found = kmem_move_buffers(cp, kmem_reclaim_scan_range, 5368b5fca8f8Stomee kmem_reclaim_max_slabs, 0); 5369b5fca8f8Stomee if (slabs_found >= 0) { 5370b5fca8f8Stomee kmd->kmd_slabs_sought += kmem_reclaim_max_slabs; 5371b5fca8f8Stomee kmd->kmd_slabs_found += slabs_found; 5372b5fca8f8Stomee } 5373b5fca8f8Stomee 5374686031edSTom Erickson if (++kmd->kmd_tries >= kmem_reclaim_scan_range) { 5375686031edSTom Erickson kmd->kmd_tries = 0; 5376b5fca8f8Stomee 5377b5fca8f8Stomee /* 5378b5fca8f8Stomee * If we had difficulty finding candidate slabs in 5379b5fca8f8Stomee * previous scans, adjust the threshold so that 5380b5fca8f8Stomee * candidates are easier to find. 5381b5fca8f8Stomee */ 5382b5fca8f8Stomee if (kmd->kmd_slabs_found == kmd->kmd_slabs_sought) { 5383b5fca8f8Stomee kmem_adjust_reclaim_threshold(kmd, -1); 5384b5fca8f8Stomee } else if ((kmd->kmd_slabs_found * 2) < 5385b5fca8f8Stomee kmd->kmd_slabs_sought) { 5386b5fca8f8Stomee kmem_adjust_reclaim_threshold(kmd, 1); 5387b5fca8f8Stomee } 5388b5fca8f8Stomee kmd->kmd_slabs_sought = 0; 5389b5fca8f8Stomee kmd->kmd_slabs_found = 0; 5390b5fca8f8Stomee } 5391b5fca8f8Stomee } else { 5392b5fca8f8Stomee kmem_reset_reclaim_threshold(cp->cache_defrag); 5393b5fca8f8Stomee #ifdef DEBUG 5394686031edSTom Erickson if (!avl_is_empty(&cp->cache_partial_slabs)) { 5395b5fca8f8Stomee /* 5396b5fca8f8Stomee * In a debug kernel we want the consolidator to 5397b5fca8f8Stomee * run occasionally even when there is plenty of 5398b5fca8f8Stomee * memory. 5399b5fca8f8Stomee */ 5400686031edSTom Erickson uint16_t debug_rand; 5401b5fca8f8Stomee 5402686031edSTom Erickson (void) random_get_bytes((uint8_t *)&debug_rand, 2); 5403b5fca8f8Stomee if (!kmem_move_noreap && 5404b5fca8f8Stomee ((debug_rand % kmem_mtb_reap) == 0)) { 5405b5fca8f8Stomee mutex_exit(&cp->cache_lock); 5406686031edSTom Erickson kmem_cache_reap(cp); 5407b5fca8f8Stomee return; 5408b5fca8f8Stomee } else if ((debug_rand % kmem_mtb_move) == 0) { 5409686031edSTom Erickson kmd->kmd_scans++; 5410b5fca8f8Stomee (void) kmem_move_buffers(cp, 5411686031edSTom Erickson kmem_reclaim_scan_range, 1, KMM_DEBUG); 5412b5fca8f8Stomee } 5413b5fca8f8Stomee } 5414b5fca8f8Stomee #endif /* DEBUG */ 5415b5fca8f8Stomee } 5416b5fca8f8Stomee 5417b5fca8f8Stomee mutex_exit(&cp->cache_lock); 5418b5fca8f8Stomee 5419*d7db73d1SBryan Cantrill if (reap) 5420b5fca8f8Stomee kmem_depot_ws_reap(cp); 5421b5fca8f8Stomee } 5422