17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 57d692464Sdp * Common Development and Distribution License (the "License"). 67d692464Sdp * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22*dce01e3fSJonathan W Adams * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate /* 27b5fca8f8Stomee * Kernel memory allocator, as described in the following two papers and a 28b5fca8f8Stomee * statement about the consolidator: 297c478bd9Sstevel@tonic-gate * 307c478bd9Sstevel@tonic-gate * Jeff Bonwick, 317c478bd9Sstevel@tonic-gate * The Slab Allocator: An Object-Caching Kernel Memory Allocator. 327c478bd9Sstevel@tonic-gate * Proceedings of the Summer 1994 Usenix Conference. 337c478bd9Sstevel@tonic-gate * Available as /shared/sac/PSARC/1994/028/materials/kmem.pdf. 347c478bd9Sstevel@tonic-gate * 357c478bd9Sstevel@tonic-gate * Jeff Bonwick and Jonathan Adams, 367c478bd9Sstevel@tonic-gate * Magazines and vmem: Extending the Slab Allocator to Many CPUs and 377c478bd9Sstevel@tonic-gate * Arbitrary Resources. 387c478bd9Sstevel@tonic-gate * Proceedings of the 2001 Usenix Conference. 397c478bd9Sstevel@tonic-gate * Available as /shared/sac/PSARC/2000/550/materials/vmem.pdf. 40b5fca8f8Stomee * 41b5fca8f8Stomee * kmem Slab Consolidator Big Theory Statement: 42b5fca8f8Stomee * 43b5fca8f8Stomee * 1. Motivation 44b5fca8f8Stomee * 45b5fca8f8Stomee * As stated in Bonwick94, slabs provide the following advantages over other 46b5fca8f8Stomee * allocation structures in terms of memory fragmentation: 47b5fca8f8Stomee * 48b5fca8f8Stomee * - Internal fragmentation (per-buffer wasted space) is minimal. 49b5fca8f8Stomee * - Severe external fragmentation (unused buffers on the free list) is 50b5fca8f8Stomee * unlikely. 51b5fca8f8Stomee * 52b5fca8f8Stomee * Segregating objects by size eliminates one source of external fragmentation, 53b5fca8f8Stomee * and according to Bonwick: 54b5fca8f8Stomee * 55b5fca8f8Stomee * The other reason that slabs reduce external fragmentation is that all 56b5fca8f8Stomee * objects in a slab are of the same type, so they have the same lifetime 57b5fca8f8Stomee * distribution. The resulting segregation of short-lived and long-lived 58b5fca8f8Stomee * objects at slab granularity reduces the likelihood of an entire page being 59b5fca8f8Stomee * held hostage due to a single long-lived allocation [Barrett93, Hanson90]. 60b5fca8f8Stomee * 61b5fca8f8Stomee * While unlikely, severe external fragmentation remains possible. Clients that 62b5fca8f8Stomee * allocate both short- and long-lived objects from the same cache cannot 63b5fca8f8Stomee * anticipate the distribution of long-lived objects within the allocator's slab 64b5fca8f8Stomee * implementation. Even a small percentage of long-lived objects distributed 65b5fca8f8Stomee * randomly across many slabs can lead to a worst case scenario where the client 66b5fca8f8Stomee * frees the majority of its objects and the system gets back almost none of the 67b5fca8f8Stomee * slabs. Despite the client doing what it reasonably can to help the system 68b5fca8f8Stomee * reclaim memory, the allocator cannot shake free enough slabs because of 69b5fca8f8Stomee * lonely allocations stubbornly hanging on. Although the allocator is in a 70b5fca8f8Stomee * position to diagnose the fragmentation, there is nothing that the allocator 71b5fca8f8Stomee * by itself can do about it. It only takes a single allocated object to prevent 72b5fca8f8Stomee * an entire slab from being reclaimed, and any object handed out by 73b5fca8f8Stomee * kmem_cache_alloc() is by definition in the client's control. Conversely, 74b5fca8f8Stomee * although the client is in a position to move a long-lived object, it has no 75b5fca8f8Stomee * way of knowing if the object is causing fragmentation, and if so, where to 76b5fca8f8Stomee * move it. A solution necessarily requires further cooperation between the 77b5fca8f8Stomee * allocator and the client. 78b5fca8f8Stomee * 79b5fca8f8Stomee * 2. Move Callback 80b5fca8f8Stomee * 81b5fca8f8Stomee * The kmem slab consolidator therefore adds a move callback to the 82b5fca8f8Stomee * allocator/client interface, improving worst-case external fragmentation in 83b5fca8f8Stomee * kmem caches that supply a function to move objects from one memory location 84b5fca8f8Stomee * to another. In a situation of low memory kmem attempts to consolidate all of 85b5fca8f8Stomee * a cache's slabs at once; otherwise it works slowly to bring external 86b5fca8f8Stomee * fragmentation within the 1/8 limit guaranteed for internal fragmentation, 87b5fca8f8Stomee * thereby helping to avoid a low memory situation in the future. 88b5fca8f8Stomee * 89b5fca8f8Stomee * The callback has the following signature: 90b5fca8f8Stomee * 91b5fca8f8Stomee * kmem_cbrc_t move(void *old, void *new, size_t size, void *user_arg) 92b5fca8f8Stomee * 93b5fca8f8Stomee * It supplies the kmem client with two addresses: the allocated object that 94b5fca8f8Stomee * kmem wants to move and a buffer selected by kmem for the client to use as the 95b5fca8f8Stomee * copy destination. The callback is kmem's way of saying "Please get off of 96b5fca8f8Stomee * this buffer and use this one instead." kmem knows where it wants to move the 97b5fca8f8Stomee * object in order to best reduce fragmentation. All the client needs to know 98b5fca8f8Stomee * about the second argument (void *new) is that it is an allocated, constructed 99b5fca8f8Stomee * object ready to take the contents of the old object. When the move function 100b5fca8f8Stomee * is called, the system is likely to be low on memory, and the new object 101b5fca8f8Stomee * spares the client from having to worry about allocating memory for the 102b5fca8f8Stomee * requested move. The third argument supplies the size of the object, in case a 103b5fca8f8Stomee * single move function handles multiple caches whose objects differ only in 104b5fca8f8Stomee * size (such as zio_buf_512, zio_buf_1024, etc). Finally, the same optional 105b5fca8f8Stomee * user argument passed to the constructor, destructor, and reclaim functions is 106b5fca8f8Stomee * also passed to the move callback. 107b5fca8f8Stomee * 108b5fca8f8Stomee * 2.1 Setting the Move Callback 109b5fca8f8Stomee * 110b5fca8f8Stomee * The client sets the move callback after creating the cache and before 111b5fca8f8Stomee * allocating from it: 112b5fca8f8Stomee * 113b5fca8f8Stomee * object_cache = kmem_cache_create(...); 114b5fca8f8Stomee * kmem_cache_set_move(object_cache, object_move); 115b5fca8f8Stomee * 116b5fca8f8Stomee * 2.2 Move Callback Return Values 117b5fca8f8Stomee * 118b5fca8f8Stomee * Only the client knows about its own data and when is a good time to move it. 119b5fca8f8Stomee * The client is cooperating with kmem to return unused memory to the system, 120b5fca8f8Stomee * and kmem respectfully accepts this help at the client's convenience. When 121b5fca8f8Stomee * asked to move an object, the client can respond with any of the following: 122b5fca8f8Stomee * 123b5fca8f8Stomee * typedef enum kmem_cbrc { 124b5fca8f8Stomee * KMEM_CBRC_YES, 125b5fca8f8Stomee * KMEM_CBRC_NO, 126b5fca8f8Stomee * KMEM_CBRC_LATER, 127b5fca8f8Stomee * KMEM_CBRC_DONT_NEED, 128b5fca8f8Stomee * KMEM_CBRC_DONT_KNOW 129b5fca8f8Stomee * } kmem_cbrc_t; 130b5fca8f8Stomee * 131b5fca8f8Stomee * The client must not explicitly kmem_cache_free() either of the objects passed 132b5fca8f8Stomee * to the callback, since kmem wants to free them directly to the slab layer 133b5fca8f8Stomee * (bypassing the per-CPU magazine layer). The response tells kmem which of the 134b5fca8f8Stomee * objects to free: 135b5fca8f8Stomee * 136b5fca8f8Stomee * YES: (Did it) The client moved the object, so kmem frees the old one. 137b5fca8f8Stomee * NO: (Never) The client refused, so kmem frees the new object (the 138b5fca8f8Stomee * unused copy destination). kmem also marks the slab of the old 139b5fca8f8Stomee * object so as not to bother the client with further callbacks for 140b5fca8f8Stomee * that object as long as the slab remains on the partial slab list. 141b5fca8f8Stomee * (The system won't be getting the slab back as long as the 142b5fca8f8Stomee * immovable object holds it hostage, so there's no point in moving 143b5fca8f8Stomee * any of its objects.) 144b5fca8f8Stomee * LATER: The client is using the object and cannot move it now, so kmem 145b5fca8f8Stomee * frees the new object (the unused copy destination). kmem still 146b5fca8f8Stomee * attempts to move other objects off the slab, since it expects to 147b5fca8f8Stomee * succeed in clearing the slab in a later callback. The client 148b5fca8f8Stomee * should use LATER instead of NO if the object is likely to become 149b5fca8f8Stomee * movable very soon. 150b5fca8f8Stomee * DONT_NEED: The client no longer needs the object, so kmem frees the old along 151b5fca8f8Stomee * with the new object (the unused copy destination). This response 152b5fca8f8Stomee * is the client's opportunity to be a model citizen and give back as 153b5fca8f8Stomee * much as it can. 154b5fca8f8Stomee * DONT_KNOW: The client does not know about the object because 155b5fca8f8Stomee * a) the client has just allocated the object and not yet put it 156b5fca8f8Stomee * wherever it expects to find known objects 157b5fca8f8Stomee * b) the client has removed the object from wherever it expects to 158b5fca8f8Stomee * find known objects and is about to free it, or 159b5fca8f8Stomee * c) the client has freed the object. 160b5fca8f8Stomee * In all these cases (a, b, and c) kmem frees the new object (the 161b5fca8f8Stomee * unused copy destination) and searches for the old object in the 162b5fca8f8Stomee * magazine layer. If found, the object is removed from the magazine 163b5fca8f8Stomee * layer and freed to the slab layer so it will no longer hold the 164b5fca8f8Stomee * slab hostage. 165b5fca8f8Stomee * 166b5fca8f8Stomee * 2.3 Object States 167b5fca8f8Stomee * 168b5fca8f8Stomee * Neither kmem nor the client can be assumed to know the object's whereabouts 169b5fca8f8Stomee * at the time of the callback. An object belonging to a kmem cache may be in 170b5fca8f8Stomee * any of the following states: 171b5fca8f8Stomee * 172b5fca8f8Stomee * 1. Uninitialized on the slab 173b5fca8f8Stomee * 2. Allocated from the slab but not constructed (still uninitialized) 174b5fca8f8Stomee * 3. Allocated from the slab, constructed, but not yet ready for business 175b5fca8f8Stomee * (not in a valid state for the move callback) 176b5fca8f8Stomee * 4. In use (valid and known to the client) 177b5fca8f8Stomee * 5. About to be freed (no longer in a valid state for the move callback) 178b5fca8f8Stomee * 6. Freed to a magazine (still constructed) 179b5fca8f8Stomee * 7. Allocated from a magazine, not yet ready for business (not in a valid 180b5fca8f8Stomee * state for the move callback), and about to return to state #4 181b5fca8f8Stomee * 8. Deconstructed on a magazine that is about to be freed 182b5fca8f8Stomee * 9. Freed to the slab 183b5fca8f8Stomee * 184b5fca8f8Stomee * Since the move callback may be called at any time while the object is in any 185b5fca8f8Stomee * of the above states (except state #1), the client needs a safe way to 186b5fca8f8Stomee * determine whether or not it knows about the object. Specifically, the client 187b5fca8f8Stomee * needs to know whether or not the object is in state #4, the only state in 188b5fca8f8Stomee * which a move is valid. If the object is in any other state, the client should 189b5fca8f8Stomee * immediately return KMEM_CBRC_DONT_KNOW, since it is unsafe to access any of 190b5fca8f8Stomee * the object's fields. 191b5fca8f8Stomee * 192b5fca8f8Stomee * Note that although an object may be in state #4 when kmem initiates the move 193b5fca8f8Stomee * request, the object may no longer be in that state by the time kmem actually 194b5fca8f8Stomee * calls the move function. Not only does the client free objects 195b5fca8f8Stomee * asynchronously, kmem itself puts move requests on a queue where thay are 196b5fca8f8Stomee * pending until kmem processes them from another context. Also, objects freed 197b5fca8f8Stomee * to a magazine appear allocated from the point of view of the slab layer, so 198b5fca8f8Stomee * kmem may even initiate requests for objects in a state other than state #4. 199b5fca8f8Stomee * 200b5fca8f8Stomee * 2.3.1 Magazine Layer 201b5fca8f8Stomee * 202b5fca8f8Stomee * An important insight revealed by the states listed above is that the magazine 203b5fca8f8Stomee * layer is populated only by kmem_cache_free(). Magazines of constructed 204b5fca8f8Stomee * objects are never populated directly from the slab layer (which contains raw, 205b5fca8f8Stomee * unconstructed objects). Whenever an allocation request cannot be satisfied 206b5fca8f8Stomee * from the magazine layer, the magazines are bypassed and the request is 207b5fca8f8Stomee * satisfied from the slab layer (creating a new slab if necessary). kmem calls 208b5fca8f8Stomee * the object constructor only when allocating from the slab layer, and only in 209b5fca8f8Stomee * response to kmem_cache_alloc() or to prepare the destination buffer passed in 210b5fca8f8Stomee * the move callback. kmem does not preconstruct objects in anticipation of 211b5fca8f8Stomee * kmem_cache_alloc(). 212b5fca8f8Stomee * 213b5fca8f8Stomee * 2.3.2 Object Constructor and Destructor 214b5fca8f8Stomee * 215b5fca8f8Stomee * If the client supplies a destructor, it must be valid to call the destructor 216b5fca8f8Stomee * on a newly created object (immediately after the constructor). 217b5fca8f8Stomee * 218b5fca8f8Stomee * 2.4 Recognizing Known Objects 219b5fca8f8Stomee * 220b5fca8f8Stomee * There is a simple test to determine safely whether or not the client knows 221b5fca8f8Stomee * about a given object in the move callback. It relies on the fact that kmem 222b5fca8f8Stomee * guarantees that the object of the move callback has only been touched by the 223b5fca8f8Stomee * client itself or else by kmem. kmem does this by ensuring that none of the 224b5fca8f8Stomee * cache's slabs are freed to the virtual memory (VM) subsystem while a move 225b5fca8f8Stomee * callback is pending. When the last object on a slab is freed, if there is a 226b5fca8f8Stomee * pending move, kmem puts the slab on a per-cache dead list and defers freeing 227b5fca8f8Stomee * slabs on that list until all pending callbacks are completed. That way, 228b5fca8f8Stomee * clients can be certain that the object of a move callback is in one of the 229b5fca8f8Stomee * states listed above, making it possible to distinguish known objects (in 230b5fca8f8Stomee * state #4) using the two low order bits of any pointer member (with the 231b5fca8f8Stomee * exception of 'char *' or 'short *' which may not be 4-byte aligned on some 232b5fca8f8Stomee * platforms). 233b5fca8f8Stomee * 234b5fca8f8Stomee * The test works as long as the client always transitions objects from state #4 235b5fca8f8Stomee * (known, in use) to state #5 (about to be freed, invalid) by setting the low 236b5fca8f8Stomee * order bit of the client-designated pointer member. Since kmem only writes 237b5fca8f8Stomee * invalid memory patterns, such as 0xbaddcafe to uninitialized memory and 238b5fca8f8Stomee * 0xdeadbeef to freed memory, any scribbling on the object done by kmem is 239b5fca8f8Stomee * guaranteed to set at least one of the two low order bits. Therefore, given an 240b5fca8f8Stomee * object with a back pointer to a 'container_t *o_container', the client can 241b5fca8f8Stomee * test 242b5fca8f8Stomee * 243b5fca8f8Stomee * container_t *container = object->o_container; 244b5fca8f8Stomee * if ((uintptr_t)container & 0x3) { 245b5fca8f8Stomee * return (KMEM_CBRC_DONT_KNOW); 246b5fca8f8Stomee * } 247b5fca8f8Stomee * 248b5fca8f8Stomee * Typically, an object will have a pointer to some structure with a list or 249b5fca8f8Stomee * hash where objects from the cache are kept while in use. Assuming that the 250b5fca8f8Stomee * client has some way of knowing that the container structure is valid and will 251b5fca8f8Stomee * not go away during the move, and assuming that the structure includes a lock 252b5fca8f8Stomee * to protect whatever collection is used, then the client would continue as 253b5fca8f8Stomee * follows: 254b5fca8f8Stomee * 255b5fca8f8Stomee * // Ensure that the container structure does not go away. 256b5fca8f8Stomee * if (container_hold(container) == 0) { 257b5fca8f8Stomee * return (KMEM_CBRC_DONT_KNOW); 258b5fca8f8Stomee * } 259b5fca8f8Stomee * mutex_enter(&container->c_objects_lock); 260b5fca8f8Stomee * if (container != object->o_container) { 261b5fca8f8Stomee * mutex_exit(&container->c_objects_lock); 262b5fca8f8Stomee * container_rele(container); 263b5fca8f8Stomee * return (KMEM_CBRC_DONT_KNOW); 264b5fca8f8Stomee * } 265b5fca8f8Stomee * 266b5fca8f8Stomee * At this point the client knows that the object cannot be freed as long as 267b5fca8f8Stomee * c_objects_lock is held. Note that after acquiring the lock, the client must 268b5fca8f8Stomee * recheck the o_container pointer in case the object was removed just before 269b5fca8f8Stomee * acquiring the lock. 270b5fca8f8Stomee * 271b5fca8f8Stomee * When the client is about to free an object, it must first remove that object 272b5fca8f8Stomee * from the list, hash, or other structure where it is kept. At that time, to 273b5fca8f8Stomee * mark the object so it can be distinguished from the remaining, known objects, 274b5fca8f8Stomee * the client sets the designated low order bit: 275b5fca8f8Stomee * 276b5fca8f8Stomee * mutex_enter(&container->c_objects_lock); 277b5fca8f8Stomee * object->o_container = (void *)((uintptr_t)object->o_container | 0x1); 278b5fca8f8Stomee * list_remove(&container->c_objects, object); 279b5fca8f8Stomee * mutex_exit(&container->c_objects_lock); 280b5fca8f8Stomee * 281b5fca8f8Stomee * In the common case, the object is freed to the magazine layer, where it may 282b5fca8f8Stomee * be reused on a subsequent allocation without the overhead of calling the 283b5fca8f8Stomee * constructor. While in the magazine it appears allocated from the point of 284b5fca8f8Stomee * view of the slab layer, making it a candidate for the move callback. Most 285b5fca8f8Stomee * objects unrecognized by the client in the move callback fall into this 286b5fca8f8Stomee * category and are cheaply distinguished from known objects by the test 287b5fca8f8Stomee * described earlier. Since recognition is cheap for the client, and searching 288b5fca8f8Stomee * magazines is expensive for kmem, kmem defers searching until the client first 289b5fca8f8Stomee * returns KMEM_CBRC_DONT_KNOW. As long as the needed effort is reasonable, kmem 290b5fca8f8Stomee * elsewhere does what it can to avoid bothering the client unnecessarily. 291b5fca8f8Stomee * 292b5fca8f8Stomee * Invalidating the designated pointer member before freeing the object marks 293b5fca8f8Stomee * the object to be avoided in the callback, and conversely, assigning a valid 294b5fca8f8Stomee * value to the designated pointer member after allocating the object makes the 295b5fca8f8Stomee * object fair game for the callback: 296b5fca8f8Stomee * 297b5fca8f8Stomee * ... allocate object ... 298b5fca8f8Stomee * ... set any initial state not set by the constructor ... 299b5fca8f8Stomee * 300b5fca8f8Stomee * mutex_enter(&container->c_objects_lock); 301b5fca8f8Stomee * list_insert_tail(&container->c_objects, object); 302b5fca8f8Stomee * membar_producer(); 303b5fca8f8Stomee * object->o_container = container; 304b5fca8f8Stomee * mutex_exit(&container->c_objects_lock); 305b5fca8f8Stomee * 306b5fca8f8Stomee * Note that everything else must be valid before setting o_container makes the 307b5fca8f8Stomee * object fair game for the move callback. The membar_producer() call ensures 308b5fca8f8Stomee * that all the object's state is written to memory before setting the pointer 309b5fca8f8Stomee * that transitions the object from state #3 or #7 (allocated, constructed, not 310b5fca8f8Stomee * yet in use) to state #4 (in use, valid). That's important because the move 311b5fca8f8Stomee * function has to check the validity of the pointer before it can safely 312b5fca8f8Stomee * acquire the lock protecting the collection where it expects to find known 313b5fca8f8Stomee * objects. 314b5fca8f8Stomee * 315b5fca8f8Stomee * This method of distinguishing known objects observes the usual symmetry: 316b5fca8f8Stomee * invalidating the designated pointer is the first thing the client does before 317b5fca8f8Stomee * freeing the object, and setting the designated pointer is the last thing the 318b5fca8f8Stomee * client does after allocating the object. Of course, the client is not 319b5fca8f8Stomee * required to use this method. Fundamentally, how the client recognizes known 320b5fca8f8Stomee * objects is completely up to the client, but this method is recommended as an 321b5fca8f8Stomee * efficient and safe way to take advantage of the guarantees made by kmem. If 322b5fca8f8Stomee * the entire object is arbitrary data without any markable bits from a suitable 323b5fca8f8Stomee * pointer member, then the client must find some other method, such as 324b5fca8f8Stomee * searching a hash table of known objects. 325b5fca8f8Stomee * 326b5fca8f8Stomee * 2.5 Preventing Objects From Moving 327b5fca8f8Stomee * 328b5fca8f8Stomee * Besides a way to distinguish known objects, the other thing that the client 329b5fca8f8Stomee * needs is a strategy to ensure that an object will not move while the client 330b5fca8f8Stomee * is actively using it. The details of satisfying this requirement tend to be 331b5fca8f8Stomee * highly cache-specific. It might seem that the same rules that let a client 332b5fca8f8Stomee * remove an object safely should also decide when an object can be moved 333b5fca8f8Stomee * safely. However, any object state that makes a removal attempt invalid is 334b5fca8f8Stomee * likely to be long-lasting for objects that the client does not expect to 335b5fca8f8Stomee * remove. kmem knows nothing about the object state and is equally likely (from 336b5fca8f8Stomee * the client's point of view) to request a move for any object in the cache, 337b5fca8f8Stomee * whether prepared for removal or not. Even a low percentage of objects stuck 338b5fca8f8Stomee * in place by unremovability will defeat the consolidator if the stuck objects 339b5fca8f8Stomee * are the same long-lived allocations likely to hold slabs hostage. 340b5fca8f8Stomee * Fundamentally, the consolidator is not aimed at common cases. Severe external 341b5fca8f8Stomee * fragmentation is a worst case scenario manifested as sparsely allocated 342b5fca8f8Stomee * slabs, by definition a low percentage of the cache's objects. When deciding 343b5fca8f8Stomee * what makes an object movable, keep in mind the goal of the consolidator: to 344b5fca8f8Stomee * bring worst-case external fragmentation within the limits guaranteed for 345b5fca8f8Stomee * internal fragmentation. Removability is a poor criterion if it is likely to 346b5fca8f8Stomee * exclude more than an insignificant percentage of objects for long periods of 347b5fca8f8Stomee * time. 348b5fca8f8Stomee * 349b5fca8f8Stomee * A tricky general solution exists, and it has the advantage of letting you 350b5fca8f8Stomee * move any object at almost any moment, practically eliminating the likelihood 351b5fca8f8Stomee * that an object can hold a slab hostage. However, if there is a cache-specific 352b5fca8f8Stomee * way to ensure that an object is not actively in use in the vast majority of 353b5fca8f8Stomee * cases, a simpler solution that leverages this cache-specific knowledge is 354b5fca8f8Stomee * preferred. 355b5fca8f8Stomee * 356b5fca8f8Stomee * 2.5.1 Cache-Specific Solution 357b5fca8f8Stomee * 358b5fca8f8Stomee * As an example of a cache-specific solution, the ZFS znode cache takes 359b5fca8f8Stomee * advantage of the fact that the vast majority of znodes are only being 360b5fca8f8Stomee * referenced from the DNLC. (A typical case might be a few hundred in active 361b5fca8f8Stomee * use and a hundred thousand in the DNLC.) In the move callback, after the ZFS 362b5fca8f8Stomee * client has established that it recognizes the znode and can access its fields 363b5fca8f8Stomee * safely (using the method described earlier), it then tests whether the znode 364b5fca8f8Stomee * is referenced by anything other than the DNLC. If so, it assumes that the 365b5fca8f8Stomee * znode may be in active use and is unsafe to move, so it drops its locks and 366b5fca8f8Stomee * returns KMEM_CBRC_LATER. The advantage of this strategy is that everywhere 367b5fca8f8Stomee * else znodes are used, no change is needed to protect against the possibility 368b5fca8f8Stomee * of the znode moving. The disadvantage is that it remains possible for an 369b5fca8f8Stomee * application to hold a znode slab hostage with an open file descriptor. 370b5fca8f8Stomee * However, this case ought to be rare and the consolidator has a way to deal 371b5fca8f8Stomee * with it: If the client responds KMEM_CBRC_LATER repeatedly for the same 372b5fca8f8Stomee * object, kmem eventually stops believing it and treats the slab as if the 373b5fca8f8Stomee * client had responded KMEM_CBRC_NO. Having marked the hostage slab, kmem can 374b5fca8f8Stomee * then focus on getting it off of the partial slab list by allocating rather 375b5fca8f8Stomee * than freeing all of its objects. (Either way of getting a slab off the 376b5fca8f8Stomee * free list reduces fragmentation.) 377b5fca8f8Stomee * 378b5fca8f8Stomee * 2.5.2 General Solution 379b5fca8f8Stomee * 380b5fca8f8Stomee * The general solution, on the other hand, requires an explicit hold everywhere 381b5fca8f8Stomee * the object is used to prevent it from moving. To keep the client locking 382b5fca8f8Stomee * strategy as uncomplicated as possible, kmem guarantees the simplifying 383b5fca8f8Stomee * assumption that move callbacks are sequential, even across multiple caches. 384b5fca8f8Stomee * Internally, a global queue processed by a single thread supports all caches 385b5fca8f8Stomee * implementing the callback function. No matter how many caches supply a move 386b5fca8f8Stomee * function, the consolidator never moves more than one object at a time, so the 387b5fca8f8Stomee * client does not have to worry about tricky lock ordering involving several 388b5fca8f8Stomee * related objects from different kmem caches. 389b5fca8f8Stomee * 390b5fca8f8Stomee * The general solution implements the explicit hold as a read-write lock, which 391b5fca8f8Stomee * allows multiple readers to access an object from the cache simultaneously 392b5fca8f8Stomee * while a single writer is excluded from moving it. A single rwlock for the 393b5fca8f8Stomee * entire cache would lock out all threads from using any of the cache's objects 394b5fca8f8Stomee * even though only a single object is being moved, so to reduce contention, 395b5fca8f8Stomee * the client can fan out the single rwlock into an array of rwlocks hashed by 396b5fca8f8Stomee * the object address, making it probable that moving one object will not 397b5fca8f8Stomee * prevent other threads from using a different object. The rwlock cannot be a 398b5fca8f8Stomee * member of the object itself, because the possibility of the object moving 399b5fca8f8Stomee * makes it unsafe to access any of the object's fields until the lock is 400b5fca8f8Stomee * acquired. 401b5fca8f8Stomee * 402b5fca8f8Stomee * Assuming a small, fixed number of locks, it's possible that multiple objects 403b5fca8f8Stomee * will hash to the same lock. A thread that needs to use multiple objects in 404b5fca8f8Stomee * the same function may acquire the same lock multiple times. Since rwlocks are 405b5fca8f8Stomee * reentrant for readers, and since there is never more than a single writer at 406b5fca8f8Stomee * a time (assuming that the client acquires the lock as a writer only when 407b5fca8f8Stomee * moving an object inside the callback), there would seem to be no problem. 408b5fca8f8Stomee * However, a client locking multiple objects in the same function must handle 409b5fca8f8Stomee * one case of potential deadlock: Assume that thread A needs to prevent both 410b5fca8f8Stomee * object 1 and object 2 from moving, and thread B, the callback, meanwhile 411b5fca8f8Stomee * tries to move object 3. It's possible, if objects 1, 2, and 3 all hash to the 412b5fca8f8Stomee * same lock, that thread A will acquire the lock for object 1 as a reader 413b5fca8f8Stomee * before thread B sets the lock's write-wanted bit, preventing thread A from 414b5fca8f8Stomee * reacquiring the lock for object 2 as a reader. Unable to make forward 415b5fca8f8Stomee * progress, thread A will never release the lock for object 1, resulting in 416b5fca8f8Stomee * deadlock. 417b5fca8f8Stomee * 418b5fca8f8Stomee * There are two ways of avoiding the deadlock just described. The first is to 419b5fca8f8Stomee * use rw_tryenter() rather than rw_enter() in the callback function when 420b5fca8f8Stomee * attempting to acquire the lock as a writer. If tryenter discovers that the 421b5fca8f8Stomee * same object (or another object hashed to the same lock) is already in use, it 422b5fca8f8Stomee * aborts the callback and returns KMEM_CBRC_LATER. The second way is to use 423b5fca8f8Stomee * rprwlock_t (declared in common/fs/zfs/sys/rprwlock.h) instead of rwlock_t, 424b5fca8f8Stomee * since it allows a thread to acquire the lock as a reader in spite of a 425b5fca8f8Stomee * waiting writer. This second approach insists on moving the object now, no 426b5fca8f8Stomee * matter how many readers the move function must wait for in order to do so, 427b5fca8f8Stomee * and could delay the completion of the callback indefinitely (blocking 428b5fca8f8Stomee * callbacks to other clients). In practice, a less insistent callback using 429b5fca8f8Stomee * rw_tryenter() returns KMEM_CBRC_LATER infrequently enough that there seems 430b5fca8f8Stomee * little reason to use anything else. 431b5fca8f8Stomee * 432b5fca8f8Stomee * Avoiding deadlock is not the only problem that an implementation using an 433b5fca8f8Stomee * explicit hold needs to solve. Locking the object in the first place (to 434b5fca8f8Stomee * prevent it from moving) remains a problem, since the object could move 435b5fca8f8Stomee * between the time you obtain a pointer to the object and the time you acquire 436b5fca8f8Stomee * the rwlock hashed to that pointer value. Therefore the client needs to 437b5fca8f8Stomee * recheck the value of the pointer after acquiring the lock, drop the lock if 438b5fca8f8Stomee * the value has changed, and try again. This requires a level of indirection: 439b5fca8f8Stomee * something that points to the object rather than the object itself, that the 440b5fca8f8Stomee * client can access safely while attempting to acquire the lock. (The object 441b5fca8f8Stomee * itself cannot be referenced safely because it can move at any time.) 442b5fca8f8Stomee * The following lock-acquisition function takes whatever is safe to reference 443b5fca8f8Stomee * (arg), follows its pointer to the object (using function f), and tries as 444b5fca8f8Stomee * often as necessary to acquire the hashed lock and verify that the object 445b5fca8f8Stomee * still has not moved: 446b5fca8f8Stomee * 447b5fca8f8Stomee * object_t * 448b5fca8f8Stomee * object_hold(object_f f, void *arg) 449b5fca8f8Stomee * { 450b5fca8f8Stomee * object_t *op; 451b5fca8f8Stomee * 452b5fca8f8Stomee * op = f(arg); 453b5fca8f8Stomee * if (op == NULL) { 454b5fca8f8Stomee * return (NULL); 455b5fca8f8Stomee * } 456b5fca8f8Stomee * 457b5fca8f8Stomee * rw_enter(OBJECT_RWLOCK(op), RW_READER); 458b5fca8f8Stomee * while (op != f(arg)) { 459b5fca8f8Stomee * rw_exit(OBJECT_RWLOCK(op)); 460b5fca8f8Stomee * op = f(arg); 461b5fca8f8Stomee * if (op == NULL) { 462b5fca8f8Stomee * break; 463b5fca8f8Stomee * } 464b5fca8f8Stomee * rw_enter(OBJECT_RWLOCK(op), RW_READER); 465b5fca8f8Stomee * } 466b5fca8f8Stomee * 467b5fca8f8Stomee * return (op); 468b5fca8f8Stomee * } 469b5fca8f8Stomee * 470b5fca8f8Stomee * The OBJECT_RWLOCK macro hashes the object address to obtain the rwlock. The 471b5fca8f8Stomee * lock reacquisition loop, while necessary, almost never executes. The function 472b5fca8f8Stomee * pointer f (used to obtain the object pointer from arg) has the following type 473b5fca8f8Stomee * definition: 474b5fca8f8Stomee * 475b5fca8f8Stomee * typedef object_t *(*object_f)(void *arg); 476b5fca8f8Stomee * 477b5fca8f8Stomee * An object_f implementation is likely to be as simple as accessing a structure 478b5fca8f8Stomee * member: 479b5fca8f8Stomee * 480b5fca8f8Stomee * object_t * 481b5fca8f8Stomee * s_object(void *arg) 482b5fca8f8Stomee * { 483b5fca8f8Stomee * something_t *sp = arg; 484b5fca8f8Stomee * return (sp->s_object); 485b5fca8f8Stomee * } 486b5fca8f8Stomee * 487b5fca8f8Stomee * The flexibility of a function pointer allows the path to the object to be 488b5fca8f8Stomee * arbitrarily complex and also supports the notion that depending on where you 489b5fca8f8Stomee * are using the object, you may need to get it from someplace different. 490b5fca8f8Stomee * 491b5fca8f8Stomee * The function that releases the explicit hold is simpler because it does not 492b5fca8f8Stomee * have to worry about the object moving: 493b5fca8f8Stomee * 494b5fca8f8Stomee * void 495b5fca8f8Stomee * object_rele(object_t *op) 496b5fca8f8Stomee * { 497b5fca8f8Stomee * rw_exit(OBJECT_RWLOCK(op)); 498b5fca8f8Stomee * } 499b5fca8f8Stomee * 500b5fca8f8Stomee * The caller is spared these details so that obtaining and releasing an 501b5fca8f8Stomee * explicit hold feels like a simple mutex_enter()/mutex_exit() pair. The caller 502b5fca8f8Stomee * of object_hold() only needs to know that the returned object pointer is valid 503b5fca8f8Stomee * if not NULL and that the object will not move until released. 504b5fca8f8Stomee * 505b5fca8f8Stomee * Although object_hold() prevents an object from moving, it does not prevent it 506b5fca8f8Stomee * from being freed. The caller must take measures before calling object_hold() 507b5fca8f8Stomee * (afterwards is too late) to ensure that the held object cannot be freed. The 508b5fca8f8Stomee * caller must do so without accessing the unsafe object reference, so any lock 509b5fca8f8Stomee * or reference count used to ensure the continued existence of the object must 510b5fca8f8Stomee * live outside the object itself. 511b5fca8f8Stomee * 512b5fca8f8Stomee * Obtaining a new object is a special case where an explicit hold is impossible 513b5fca8f8Stomee * for the caller. Any function that returns a newly allocated object (either as 514b5fca8f8Stomee * a return value, or as an in-out paramter) must return it already held; after 515b5fca8f8Stomee * the caller gets it is too late, since the object cannot be safely accessed 516b5fca8f8Stomee * without the level of indirection described earlier. The following 517b5fca8f8Stomee * object_alloc() example uses the same code shown earlier to transition a new 518b5fca8f8Stomee * object into the state of being recognized (by the client) as a known object. 519b5fca8f8Stomee * The function must acquire the hold (rw_enter) before that state transition 520b5fca8f8Stomee * makes the object movable: 521b5fca8f8Stomee * 522b5fca8f8Stomee * static object_t * 523b5fca8f8Stomee * object_alloc(container_t *container) 524b5fca8f8Stomee * { 5254d4c4c43STom Erickson * object_t *object = kmem_cache_alloc(object_cache, 0); 526b5fca8f8Stomee * ... set any initial state not set by the constructor ... 527b5fca8f8Stomee * rw_enter(OBJECT_RWLOCK(object), RW_READER); 528b5fca8f8Stomee * mutex_enter(&container->c_objects_lock); 529b5fca8f8Stomee * list_insert_tail(&container->c_objects, object); 530b5fca8f8Stomee * membar_producer(); 531b5fca8f8Stomee * object->o_container = container; 532b5fca8f8Stomee * mutex_exit(&container->c_objects_lock); 533b5fca8f8Stomee * return (object); 534b5fca8f8Stomee * } 535b5fca8f8Stomee * 536b5fca8f8Stomee * Functions that implicitly acquire an object hold (any function that calls 537b5fca8f8Stomee * object_alloc() to supply an object for the caller) need to be carefully noted 538b5fca8f8Stomee * so that the matching object_rele() is not neglected. Otherwise, leaked holds 539b5fca8f8Stomee * prevent all objects hashed to the affected rwlocks from ever being moved. 540b5fca8f8Stomee * 541b5fca8f8Stomee * The pointer to a held object can be hashed to the holding rwlock even after 542b5fca8f8Stomee * the object has been freed. Although it is possible to release the hold 543b5fca8f8Stomee * after freeing the object, you may decide to release the hold implicitly in 544b5fca8f8Stomee * whatever function frees the object, so as to release the hold as soon as 545b5fca8f8Stomee * possible, and for the sake of symmetry with the function that implicitly 546b5fca8f8Stomee * acquires the hold when it allocates the object. Here, object_free() releases 547b5fca8f8Stomee * the hold acquired by object_alloc(). Its implicit object_rele() forms a 548b5fca8f8Stomee * matching pair with object_hold(): 549b5fca8f8Stomee * 550b5fca8f8Stomee * void 551b5fca8f8Stomee * object_free(object_t *object) 552b5fca8f8Stomee * { 553b5fca8f8Stomee * container_t *container; 554b5fca8f8Stomee * 555b5fca8f8Stomee * ASSERT(object_held(object)); 556b5fca8f8Stomee * container = object->o_container; 557b5fca8f8Stomee * mutex_enter(&container->c_objects_lock); 558b5fca8f8Stomee * object->o_container = 559b5fca8f8Stomee * (void *)((uintptr_t)object->o_container | 0x1); 560b5fca8f8Stomee * list_remove(&container->c_objects, object); 561b5fca8f8Stomee * mutex_exit(&container->c_objects_lock); 562b5fca8f8Stomee * object_rele(object); 563b5fca8f8Stomee * kmem_cache_free(object_cache, object); 564b5fca8f8Stomee * } 565b5fca8f8Stomee * 566b5fca8f8Stomee * Note that object_free() cannot safely accept an object pointer as an argument 567b5fca8f8Stomee * unless the object is already held. Any function that calls object_free() 568b5fca8f8Stomee * needs to be carefully noted since it similarly forms a matching pair with 569b5fca8f8Stomee * object_hold(). 570b5fca8f8Stomee * 571b5fca8f8Stomee * To complete the picture, the following callback function implements the 572b5fca8f8Stomee * general solution by moving objects only if they are currently unheld: 573b5fca8f8Stomee * 574b5fca8f8Stomee * static kmem_cbrc_t 575b5fca8f8Stomee * object_move(void *buf, void *newbuf, size_t size, void *arg) 576b5fca8f8Stomee * { 577b5fca8f8Stomee * object_t *op = buf, *np = newbuf; 578b5fca8f8Stomee * container_t *container; 579b5fca8f8Stomee * 580b5fca8f8Stomee * container = op->o_container; 581b5fca8f8Stomee * if ((uintptr_t)container & 0x3) { 582b5fca8f8Stomee * return (KMEM_CBRC_DONT_KNOW); 583b5fca8f8Stomee * } 584b5fca8f8Stomee * 585b5fca8f8Stomee * // Ensure that the container structure does not go away. 586b5fca8f8Stomee * if (container_hold(container) == 0) { 587b5fca8f8Stomee * return (KMEM_CBRC_DONT_KNOW); 588b5fca8f8Stomee * } 589b5fca8f8Stomee * 590b5fca8f8Stomee * mutex_enter(&container->c_objects_lock); 591b5fca8f8Stomee * if (container != op->o_container) { 592b5fca8f8Stomee * mutex_exit(&container->c_objects_lock); 593b5fca8f8Stomee * container_rele(container); 594b5fca8f8Stomee * return (KMEM_CBRC_DONT_KNOW); 595b5fca8f8Stomee * } 596b5fca8f8Stomee * 597b5fca8f8Stomee * if (rw_tryenter(OBJECT_RWLOCK(op), RW_WRITER) == 0) { 598b5fca8f8Stomee * mutex_exit(&container->c_objects_lock); 599b5fca8f8Stomee * container_rele(container); 600b5fca8f8Stomee * return (KMEM_CBRC_LATER); 601b5fca8f8Stomee * } 602b5fca8f8Stomee * 603b5fca8f8Stomee * object_move_impl(op, np); // critical section 604b5fca8f8Stomee * rw_exit(OBJECT_RWLOCK(op)); 605b5fca8f8Stomee * 606b5fca8f8Stomee * op->o_container = (void *)((uintptr_t)op->o_container | 0x1); 607b5fca8f8Stomee * list_link_replace(&op->o_link_node, &np->o_link_node); 608b5fca8f8Stomee * mutex_exit(&container->c_objects_lock); 609b5fca8f8Stomee * container_rele(container); 610b5fca8f8Stomee * return (KMEM_CBRC_YES); 611b5fca8f8Stomee * } 612b5fca8f8Stomee * 613b5fca8f8Stomee * Note that object_move() must invalidate the designated o_container pointer of 614b5fca8f8Stomee * the old object in the same way that object_free() does, since kmem will free 615b5fca8f8Stomee * the object in response to the KMEM_CBRC_YES return value. 616b5fca8f8Stomee * 617b5fca8f8Stomee * The lock order in object_move() differs from object_alloc(), which locks 618b5fca8f8Stomee * OBJECT_RWLOCK first and &container->c_objects_lock second, but as long as the 619b5fca8f8Stomee * callback uses rw_tryenter() (preventing the deadlock described earlier), it's 620b5fca8f8Stomee * not a problem. Holding the lock on the object list in the example above 621b5fca8f8Stomee * through the entire callback not only prevents the object from going away, it 622b5fca8f8Stomee * also allows you to lock the list elsewhere and know that none of its elements 623b5fca8f8Stomee * will move during iteration. 624b5fca8f8Stomee * 625b5fca8f8Stomee * Adding an explicit hold everywhere an object from the cache is used is tricky 626b5fca8f8Stomee * and involves much more change to client code than a cache-specific solution 627b5fca8f8Stomee * that leverages existing state to decide whether or not an object is 628b5fca8f8Stomee * movable. However, this approach has the advantage that no object remains 629b5fca8f8Stomee * immovable for any significant length of time, making it extremely unlikely 630b5fca8f8Stomee * that long-lived allocations can continue holding slabs hostage; and it works 631b5fca8f8Stomee * for any cache. 632b5fca8f8Stomee * 633b5fca8f8Stomee * 3. Consolidator Implementation 634b5fca8f8Stomee * 635b5fca8f8Stomee * Once the client supplies a move function that a) recognizes known objects and 636b5fca8f8Stomee * b) avoids moving objects that are actively in use, the remaining work is up 637b5fca8f8Stomee * to the consolidator to decide which objects to move and when to issue 638b5fca8f8Stomee * callbacks. 639b5fca8f8Stomee * 640b5fca8f8Stomee * The consolidator relies on the fact that a cache's slabs are ordered by 641b5fca8f8Stomee * usage. Each slab has a fixed number of objects. Depending on the slab's 642b5fca8f8Stomee * "color" (the offset of the first object from the beginning of the slab; 643b5fca8f8Stomee * offsets are staggered to mitigate false sharing of cache lines) it is either 644b5fca8f8Stomee * the maximum number of objects per slab determined at cache creation time or 645b5fca8f8Stomee * else the number closest to the maximum that fits within the space remaining 646b5fca8f8Stomee * after the initial offset. A completely allocated slab may contribute some 647b5fca8f8Stomee * internal fragmentation (per-slab overhead) but no external fragmentation, so 648b5fca8f8Stomee * it is of no interest to the consolidator. At the other extreme, slabs whose 649b5fca8f8Stomee * objects have all been freed to the slab are released to the virtual memory 650b5fca8f8Stomee * (VM) subsystem (objects freed to magazines are still allocated as far as the 651b5fca8f8Stomee * slab is concerned). External fragmentation exists when there are slabs 652b5fca8f8Stomee * somewhere between these extremes. A partial slab has at least one but not all 653b5fca8f8Stomee * of its objects allocated. The more partial slabs, and the fewer allocated 654b5fca8f8Stomee * objects on each of them, the higher the fragmentation. Hence the 655b5fca8f8Stomee * consolidator's overall strategy is to reduce the number of partial slabs by 656b5fca8f8Stomee * moving allocated objects from the least allocated slabs to the most allocated 657b5fca8f8Stomee * slabs. 658b5fca8f8Stomee * 659b5fca8f8Stomee * Partial slabs are kept in an AVL tree ordered by usage. Completely allocated 660b5fca8f8Stomee * slabs are kept separately in an unordered list. Since the majority of slabs 661b5fca8f8Stomee * tend to be completely allocated (a typical unfragmented cache may have 662b5fca8f8Stomee * thousands of complete slabs and only a single partial slab), separating 663b5fca8f8Stomee * complete slabs improves the efficiency of partial slab ordering, since the 664b5fca8f8Stomee * complete slabs do not affect the depth or balance of the AVL tree. This 665b5fca8f8Stomee * ordered sequence of partial slabs acts as a "free list" supplying objects for 666b5fca8f8Stomee * allocation requests. 667b5fca8f8Stomee * 668b5fca8f8Stomee * Objects are always allocated from the first partial slab in the free list, 669b5fca8f8Stomee * where the allocation is most likely to eliminate a partial slab (by 670b5fca8f8Stomee * completely allocating it). Conversely, when a single object from a completely 671b5fca8f8Stomee * allocated slab is freed to the slab, that slab is added to the front of the 672b5fca8f8Stomee * free list. Since most free list activity involves highly allocated slabs 673b5fca8f8Stomee * coming and going at the front of the list, slabs tend naturally toward the 674b5fca8f8Stomee * ideal order: highly allocated at the front, sparsely allocated at the back. 675b5fca8f8Stomee * Slabs with few allocated objects are likely to become completely free if they 676b5fca8f8Stomee * keep a safe distance away from the front of the free list. Slab misorders 677b5fca8f8Stomee * interfere with the natural tendency of slabs to become completely free or 678b5fca8f8Stomee * completely allocated. For example, a slab with a single allocated object 679b5fca8f8Stomee * needs only a single free to escape the cache; its natural desire is 680b5fca8f8Stomee * frustrated when it finds itself at the front of the list where a second 681b5fca8f8Stomee * allocation happens just before the free could have released it. Another slab 682b5fca8f8Stomee * with all but one object allocated might have supplied the buffer instead, so 683b5fca8f8Stomee * that both (as opposed to neither) of the slabs would have been taken off the 684b5fca8f8Stomee * free list. 685b5fca8f8Stomee * 686b5fca8f8Stomee * Although slabs tend naturally toward the ideal order, misorders allowed by a 687b5fca8f8Stomee * simple list implementation defeat the consolidator's strategy of merging 688b5fca8f8Stomee * least- and most-allocated slabs. Without an AVL tree to guarantee order, kmem 689b5fca8f8Stomee * needs another way to fix misorders to optimize its callback strategy. One 690b5fca8f8Stomee * approach is to periodically scan a limited number of slabs, advancing a 691b5fca8f8Stomee * marker to hold the current scan position, and to move extreme misorders to 692b5fca8f8Stomee * the front or back of the free list and to the front or back of the current 693b5fca8f8Stomee * scan range. By making consecutive scan ranges overlap by one slab, the least 694b5fca8f8Stomee * allocated slab in the current range can be carried along from the end of one 695b5fca8f8Stomee * scan to the start of the next. 696b5fca8f8Stomee * 697b5fca8f8Stomee * Maintaining partial slabs in an AVL tree relieves kmem of this additional 698b5fca8f8Stomee * task, however. Since most of the cache's activity is in the magazine layer, 699b5fca8f8Stomee * and allocations from the slab layer represent only a startup cost, the 700b5fca8f8Stomee * overhead of maintaining a balanced tree is not a significant concern compared 701b5fca8f8Stomee * to the opportunity of reducing complexity by eliminating the partial slab 702b5fca8f8Stomee * scanner just described. The overhead of an AVL tree is minimized by 703b5fca8f8Stomee * maintaining only partial slabs in the tree and keeping completely allocated 704b5fca8f8Stomee * slabs separately in a list. To avoid increasing the size of the slab 705b5fca8f8Stomee * structure the AVL linkage pointers are reused for the slab's list linkage, 706b5fca8f8Stomee * since the slab will always be either partial or complete, never stored both 707b5fca8f8Stomee * ways at the same time. To further minimize the overhead of the AVL tree the 708b5fca8f8Stomee * compare function that orders partial slabs by usage divides the range of 709b5fca8f8Stomee * allocated object counts into bins such that counts within the same bin are 710b5fca8f8Stomee * considered equal. Binning partial slabs makes it less likely that allocating 711b5fca8f8Stomee * or freeing a single object will change the slab's order, requiring a tree 712b5fca8f8Stomee * reinsertion (an avl_remove() followed by an avl_add(), both potentially 713b5fca8f8Stomee * requiring some rebalancing of the tree). Allocation counts closest to 714b5fca8f8Stomee * completely free and completely allocated are left unbinned (finely sorted) to 715b5fca8f8Stomee * better support the consolidator's strategy of merging slabs at either 716b5fca8f8Stomee * extreme. 717b5fca8f8Stomee * 718b5fca8f8Stomee * 3.1 Assessing Fragmentation and Selecting Candidate Slabs 719b5fca8f8Stomee * 720b5fca8f8Stomee * The consolidator piggybacks on the kmem maintenance thread and is called on 721b5fca8f8Stomee * the same interval as kmem_cache_update(), once per cache every fifteen 722b5fca8f8Stomee * seconds. kmem maintains a running count of unallocated objects in the slab 723b5fca8f8Stomee * layer (cache_bufslab). The consolidator checks whether that number exceeds 724b5fca8f8Stomee * 12.5% (1/8) of the total objects in the cache (cache_buftotal), and whether 725b5fca8f8Stomee * there is a significant number of slabs in the cache (arbitrarily a minimum 726b5fca8f8Stomee * 101 total slabs). Unused objects that have fallen out of the magazine layer's 727b5fca8f8Stomee * working set are included in the assessment, and magazines in the depot are 728b5fca8f8Stomee * reaped if those objects would lift cache_bufslab above the fragmentation 729b5fca8f8Stomee * threshold. Once the consolidator decides that a cache is fragmented, it looks 730b5fca8f8Stomee * for a candidate slab to reclaim, starting at the end of the partial slab free 731b5fca8f8Stomee * list and scanning backwards. At first the consolidator is choosy: only a slab 732b5fca8f8Stomee * with fewer than 12.5% (1/8) of its objects allocated qualifies (or else a 733b5fca8f8Stomee * single allocated object, regardless of percentage). If there is difficulty 734b5fca8f8Stomee * finding a candidate slab, kmem raises the allocation threshold incrementally, 735b5fca8f8Stomee * up to a maximum 87.5% (7/8), so that eventually the consolidator will reduce 736b5fca8f8Stomee * external fragmentation (unused objects on the free list) below 12.5% (1/8), 737b5fca8f8Stomee * even in the worst case of every slab in the cache being almost 7/8 allocated. 738b5fca8f8Stomee * The threshold can also be lowered incrementally when candidate slabs are easy 739b5fca8f8Stomee * to find, and the threshold is reset to the minimum 1/8 as soon as the cache 740b5fca8f8Stomee * is no longer fragmented. 741b5fca8f8Stomee * 742b5fca8f8Stomee * 3.2 Generating Callbacks 743b5fca8f8Stomee * 744b5fca8f8Stomee * Once an eligible slab is chosen, a callback is generated for every allocated 745b5fca8f8Stomee * object on the slab, in the hope that the client will move everything off the 746b5fca8f8Stomee * slab and make it reclaimable. Objects selected as move destinations are 747b5fca8f8Stomee * chosen from slabs at the front of the free list. Assuming slabs in the ideal 748b5fca8f8Stomee * order (most allocated at the front, least allocated at the back) and a 749b5fca8f8Stomee * cooperative client, the consolidator will succeed in removing slabs from both 750b5fca8f8Stomee * ends of the free list, completely allocating on the one hand and completely 751b5fca8f8Stomee * freeing on the other. Objects selected as move destinations are allocated in 752b5fca8f8Stomee * the kmem maintenance thread where move requests are enqueued. A separate 753b5fca8f8Stomee * callback thread removes pending callbacks from the queue and calls the 754b5fca8f8Stomee * client. The separate thread ensures that client code (the move function) does 755b5fca8f8Stomee * not interfere with internal kmem maintenance tasks. A map of pending 756b5fca8f8Stomee * callbacks keyed by object address (the object to be moved) is checked to 757b5fca8f8Stomee * ensure that duplicate callbacks are not generated for the same object. 758b5fca8f8Stomee * Allocating the move destination (the object to move to) prevents subsequent 759b5fca8f8Stomee * callbacks from selecting the same destination as an earlier pending callback. 760b5fca8f8Stomee * 761b5fca8f8Stomee * Move requests can also be generated by kmem_cache_reap() when the system is 762b5fca8f8Stomee * desperate for memory and by kmem_cache_move_notify(), called by the client to 763b5fca8f8Stomee * notify kmem that a move refused earlier with KMEM_CBRC_LATER is now possible. 764b5fca8f8Stomee * The map of pending callbacks is protected by the same lock that protects the 765b5fca8f8Stomee * slab layer. 766b5fca8f8Stomee * 767b5fca8f8Stomee * When the system is desperate for memory, kmem does not bother to determine 768b5fca8f8Stomee * whether or not the cache exceeds the fragmentation threshold, but tries to 769b5fca8f8Stomee * consolidate as many slabs as possible. Normally, the consolidator chews 770b5fca8f8Stomee * slowly, one sparsely allocated slab at a time during each maintenance 771b5fca8f8Stomee * interval that the cache is fragmented. When desperate, the consolidator 772b5fca8f8Stomee * starts at the last partial slab and enqueues callbacks for every allocated 773b5fca8f8Stomee * object on every partial slab, working backwards until it reaches the first 774b5fca8f8Stomee * partial slab. The first partial slab, meanwhile, advances in pace with the 775b5fca8f8Stomee * consolidator as allocations to supply move destinations for the enqueued 776b5fca8f8Stomee * callbacks use up the highly allocated slabs at the front of the free list. 777b5fca8f8Stomee * Ideally, the overgrown free list collapses like an accordion, starting at 778b5fca8f8Stomee * both ends and ending at the center with a single partial slab. 779b5fca8f8Stomee * 780b5fca8f8Stomee * 3.3 Client Responses 781b5fca8f8Stomee * 782b5fca8f8Stomee * When the client returns KMEM_CBRC_NO in response to the move callback, kmem 783b5fca8f8Stomee * marks the slab that supplied the stuck object non-reclaimable and moves it to 784b5fca8f8Stomee * front of the free list. The slab remains marked as long as it remains on the 785b5fca8f8Stomee * free list, and it appears more allocated to the partial slab compare function 786b5fca8f8Stomee * than any unmarked slab, no matter how many of its objects are allocated. 787b5fca8f8Stomee * Since even one immovable object ties up the entire slab, the goal is to 788b5fca8f8Stomee * completely allocate any slab that cannot be completely freed. kmem does not 789b5fca8f8Stomee * bother generating callbacks to move objects from a marked slab unless the 790b5fca8f8Stomee * system is desperate. 791b5fca8f8Stomee * 792b5fca8f8Stomee * When the client responds KMEM_CBRC_LATER, kmem increments a count for the 793b5fca8f8Stomee * slab. If the client responds LATER too many times, kmem disbelieves and 794b5fca8f8Stomee * treats the response as a NO. The count is cleared when the slab is taken off 795b5fca8f8Stomee * the partial slab list or when the client moves one of the slab's objects. 796b5fca8f8Stomee * 797b5fca8f8Stomee * 4. Observability 798b5fca8f8Stomee * 799b5fca8f8Stomee * A kmem cache's external fragmentation is best observed with 'mdb -k' using 800b5fca8f8Stomee * the ::kmem_slabs dcmd. For a complete description of the command, enter 801b5fca8f8Stomee * '::help kmem_slabs' at the mdb prompt. 8027c478bd9Sstevel@tonic-gate */ 8037c478bd9Sstevel@tonic-gate 8047c478bd9Sstevel@tonic-gate #include <sys/kmem_impl.h> 8057c478bd9Sstevel@tonic-gate #include <sys/vmem_impl.h> 8067c478bd9Sstevel@tonic-gate #include <sys/param.h> 8077c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 8087c478bd9Sstevel@tonic-gate #include <sys/vm.h> 8097c478bd9Sstevel@tonic-gate #include <sys/proc.h> 8107c478bd9Sstevel@tonic-gate #include <sys/tuneable.h> 8117c478bd9Sstevel@tonic-gate #include <sys/systm.h> 8127c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 8137c478bd9Sstevel@tonic-gate #include <sys/debug.h> 814b5fca8f8Stomee #include <sys/sdt.h> 8157c478bd9Sstevel@tonic-gate #include <sys/mutex.h> 8167c478bd9Sstevel@tonic-gate #include <sys/bitmap.h> 8177c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 8187c478bd9Sstevel@tonic-gate #include <sys/kobj.h> 8197c478bd9Sstevel@tonic-gate #include <sys/disp.h> 8207c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h> 8217c478bd9Sstevel@tonic-gate #include <sys/log.h> 8227c478bd9Sstevel@tonic-gate #include <sys/callb.h> 8237c478bd9Sstevel@tonic-gate #include <sys/taskq.h> 8247c478bd9Sstevel@tonic-gate #include <sys/modctl.h> 8257c478bd9Sstevel@tonic-gate #include <sys/reboot.h> 8267c478bd9Sstevel@tonic-gate #include <sys/id32.h> 8277c478bd9Sstevel@tonic-gate #include <sys/zone.h> 828f4b3ec61Sdh #include <sys/netstack.h> 829b5fca8f8Stomee #ifdef DEBUG 830b5fca8f8Stomee #include <sys/random.h> 831b5fca8f8Stomee #endif 8327c478bd9Sstevel@tonic-gate 8337c478bd9Sstevel@tonic-gate extern void streams_msg_init(void); 8347c478bd9Sstevel@tonic-gate extern int segkp_fromheap; 8357c478bd9Sstevel@tonic-gate extern void segkp_cache_free(void); 8367c478bd9Sstevel@tonic-gate 8377c478bd9Sstevel@tonic-gate struct kmem_cache_kstat { 8387c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_size; 8397c478bd9Sstevel@tonic-gate kstat_named_t kmc_align; 8407c478bd9Sstevel@tonic-gate kstat_named_t kmc_chunk_size; 8417c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_size; 8427c478bd9Sstevel@tonic-gate kstat_named_t kmc_alloc; 8437c478bd9Sstevel@tonic-gate kstat_named_t kmc_alloc_fail; 8447c478bd9Sstevel@tonic-gate kstat_named_t kmc_free; 8457c478bd9Sstevel@tonic-gate kstat_named_t kmc_depot_alloc; 8467c478bd9Sstevel@tonic-gate kstat_named_t kmc_depot_free; 8477c478bd9Sstevel@tonic-gate kstat_named_t kmc_depot_contention; 8487c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_alloc; 8497c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_free; 8507c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_constructed; 8517c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_avail; 8527c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_inuse; 8537c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_total; 8547c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_max; 8557c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_create; 8567c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_destroy; 8577c478bd9Sstevel@tonic-gate kstat_named_t kmc_vmem_source; 8587c478bd9Sstevel@tonic-gate kstat_named_t kmc_hash_size; 8597c478bd9Sstevel@tonic-gate kstat_named_t kmc_hash_lookup_depth; 8607c478bd9Sstevel@tonic-gate kstat_named_t kmc_hash_rescale; 8617c478bd9Sstevel@tonic-gate kstat_named_t kmc_full_magazines; 8627c478bd9Sstevel@tonic-gate kstat_named_t kmc_empty_magazines; 8637c478bd9Sstevel@tonic-gate kstat_named_t kmc_magazine_size; 864b5fca8f8Stomee kstat_named_t kmc_move_callbacks; 865b5fca8f8Stomee kstat_named_t kmc_move_yes; 866b5fca8f8Stomee kstat_named_t kmc_move_no; 867b5fca8f8Stomee kstat_named_t kmc_move_later; 868b5fca8f8Stomee kstat_named_t kmc_move_dont_need; 869b5fca8f8Stomee kstat_named_t kmc_move_dont_know; 870b5fca8f8Stomee kstat_named_t kmc_move_hunt_found; 8717c478bd9Sstevel@tonic-gate } kmem_cache_kstat = { 8727c478bd9Sstevel@tonic-gate { "buf_size", KSTAT_DATA_UINT64 }, 8737c478bd9Sstevel@tonic-gate { "align", KSTAT_DATA_UINT64 }, 8747c478bd9Sstevel@tonic-gate { "chunk_size", KSTAT_DATA_UINT64 }, 8757c478bd9Sstevel@tonic-gate { "slab_size", KSTAT_DATA_UINT64 }, 8767c478bd9Sstevel@tonic-gate { "alloc", KSTAT_DATA_UINT64 }, 8777c478bd9Sstevel@tonic-gate { "alloc_fail", KSTAT_DATA_UINT64 }, 8787c478bd9Sstevel@tonic-gate { "free", KSTAT_DATA_UINT64 }, 8797c478bd9Sstevel@tonic-gate { "depot_alloc", KSTAT_DATA_UINT64 }, 8807c478bd9Sstevel@tonic-gate { "depot_free", KSTAT_DATA_UINT64 }, 8817c478bd9Sstevel@tonic-gate { "depot_contention", KSTAT_DATA_UINT64 }, 8827c478bd9Sstevel@tonic-gate { "slab_alloc", KSTAT_DATA_UINT64 }, 8837c478bd9Sstevel@tonic-gate { "slab_free", KSTAT_DATA_UINT64 }, 8847c478bd9Sstevel@tonic-gate { "buf_constructed", KSTAT_DATA_UINT64 }, 8857c478bd9Sstevel@tonic-gate { "buf_avail", KSTAT_DATA_UINT64 }, 8867c478bd9Sstevel@tonic-gate { "buf_inuse", KSTAT_DATA_UINT64 }, 8877c478bd9Sstevel@tonic-gate { "buf_total", KSTAT_DATA_UINT64 }, 8887c478bd9Sstevel@tonic-gate { "buf_max", KSTAT_DATA_UINT64 }, 8897c478bd9Sstevel@tonic-gate { "slab_create", KSTAT_DATA_UINT64 }, 8907c478bd9Sstevel@tonic-gate { "slab_destroy", KSTAT_DATA_UINT64 }, 8917c478bd9Sstevel@tonic-gate { "vmem_source", KSTAT_DATA_UINT64 }, 8927c478bd9Sstevel@tonic-gate { "hash_size", KSTAT_DATA_UINT64 }, 8937c478bd9Sstevel@tonic-gate { "hash_lookup_depth", KSTAT_DATA_UINT64 }, 8947c478bd9Sstevel@tonic-gate { "hash_rescale", KSTAT_DATA_UINT64 }, 8957c478bd9Sstevel@tonic-gate { "full_magazines", KSTAT_DATA_UINT64 }, 8967c478bd9Sstevel@tonic-gate { "empty_magazines", KSTAT_DATA_UINT64 }, 8977c478bd9Sstevel@tonic-gate { "magazine_size", KSTAT_DATA_UINT64 }, 898b5fca8f8Stomee { "move_callbacks", KSTAT_DATA_UINT64 }, 899b5fca8f8Stomee { "move_yes", KSTAT_DATA_UINT64 }, 900b5fca8f8Stomee { "move_no", KSTAT_DATA_UINT64 }, 901b5fca8f8Stomee { "move_later", KSTAT_DATA_UINT64 }, 902b5fca8f8Stomee { "move_dont_need", KSTAT_DATA_UINT64 }, 903b5fca8f8Stomee { "move_dont_know", KSTAT_DATA_UINT64 }, 904b5fca8f8Stomee { "move_hunt_found", KSTAT_DATA_UINT64 }, 9057c478bd9Sstevel@tonic-gate }; 9067c478bd9Sstevel@tonic-gate 9077c478bd9Sstevel@tonic-gate static kmutex_t kmem_cache_kstat_lock; 9087c478bd9Sstevel@tonic-gate 9097c478bd9Sstevel@tonic-gate /* 9107c478bd9Sstevel@tonic-gate * The default set of caches to back kmem_alloc(). 9117c478bd9Sstevel@tonic-gate * These sizes should be reevaluated periodically. 9127c478bd9Sstevel@tonic-gate * 9137c478bd9Sstevel@tonic-gate * We want allocations that are multiples of the coherency granularity 9147c478bd9Sstevel@tonic-gate * (64 bytes) to be satisfied from a cache which is a multiple of 64 9157c478bd9Sstevel@tonic-gate * bytes, so that it will be 64-byte aligned. For all multiples of 64, 9167c478bd9Sstevel@tonic-gate * the next kmem_cache_size greater than or equal to it must be a 9177c478bd9Sstevel@tonic-gate * multiple of 64. 918*dce01e3fSJonathan W Adams * 919*dce01e3fSJonathan W Adams * We split the table into two sections: size <= 4k and size > 4k. This 920*dce01e3fSJonathan W Adams * saves a lot of space and cache footprint in our cache tables. 9217c478bd9Sstevel@tonic-gate */ 9227c478bd9Sstevel@tonic-gate static const int kmem_alloc_sizes[] = { 9237c478bd9Sstevel@tonic-gate 1 * 8, 9247c478bd9Sstevel@tonic-gate 2 * 8, 9257c478bd9Sstevel@tonic-gate 3 * 8, 9267c478bd9Sstevel@tonic-gate 4 * 8, 5 * 8, 6 * 8, 7 * 8, 9277c478bd9Sstevel@tonic-gate 4 * 16, 5 * 16, 6 * 16, 7 * 16, 9287c478bd9Sstevel@tonic-gate 4 * 32, 5 * 32, 6 * 32, 7 * 32, 9297c478bd9Sstevel@tonic-gate 4 * 64, 5 * 64, 6 * 64, 7 * 64, 9307c478bd9Sstevel@tonic-gate 4 * 128, 5 * 128, 6 * 128, 7 * 128, 9317c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 7, 64), 9327c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 6, 64), 9337c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 5, 64), 9347c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 4, 64), 9357c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 3, 64), 9367c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 2, 64), 9377c478bd9Sstevel@tonic-gate }; 9387c478bd9Sstevel@tonic-gate 939*dce01e3fSJonathan W Adams static const int kmem_big_alloc_sizes[] = { 940*dce01e3fSJonathan W Adams 2 * 4096, 3 * 4096, 941*dce01e3fSJonathan W Adams 2 * 8192, 3 * 8192, 942*dce01e3fSJonathan W Adams 4 * 8192, 5 * 8192, 6 * 8192, 7 * 8192, 943*dce01e3fSJonathan W Adams 8 * 8192, 9 * 8192, 10 * 8192, 11 * 8192, 944*dce01e3fSJonathan W Adams 12 * 8192, 13 * 8192, 14 * 8192, 15 * 8192, 945*dce01e3fSJonathan W Adams 16 * 8192 946*dce01e3fSJonathan W Adams }; 947*dce01e3fSJonathan W Adams 948*dce01e3fSJonathan W Adams #define KMEM_MAXBUF 4096 949*dce01e3fSJonathan W Adams #define KMEM_BIG_MAXBUF_32BIT 32768 950*dce01e3fSJonathan W Adams #define KMEM_BIG_MAXBUF 131072 951*dce01e3fSJonathan W Adams 952*dce01e3fSJonathan W Adams #define KMEM_BIG_MULTIPLE 4096 /* big_alloc_sizes must be a multiple */ 953*dce01e3fSJonathan W Adams #define KMEM_BIG_SHIFT 12 /* lg(KMEM_BIG_MULTIPLE) */ 9547c478bd9Sstevel@tonic-gate 9557c478bd9Sstevel@tonic-gate static kmem_cache_t *kmem_alloc_table[KMEM_MAXBUF >> KMEM_ALIGN_SHIFT]; 956*dce01e3fSJonathan W Adams static kmem_cache_t *kmem_big_alloc_table[KMEM_BIG_MAXBUF >> KMEM_BIG_SHIFT]; 957*dce01e3fSJonathan W Adams 958*dce01e3fSJonathan W Adams #define KMEM_ALLOC_TABLE_MAX (KMEM_MAXBUF >> KMEM_ALIGN_SHIFT) 959*dce01e3fSJonathan W Adams static size_t kmem_big_alloc_table_max = 0; /* # of filled elements */ 9607c478bd9Sstevel@tonic-gate 9617c478bd9Sstevel@tonic-gate static kmem_magtype_t kmem_magtype[] = { 9627c478bd9Sstevel@tonic-gate { 1, 8, 3200, 65536 }, 9637c478bd9Sstevel@tonic-gate { 3, 16, 256, 32768 }, 9647c478bd9Sstevel@tonic-gate { 7, 32, 64, 16384 }, 9657c478bd9Sstevel@tonic-gate { 15, 64, 0, 8192 }, 9667c478bd9Sstevel@tonic-gate { 31, 64, 0, 4096 }, 9677c478bd9Sstevel@tonic-gate { 47, 64, 0, 2048 }, 9687c478bd9Sstevel@tonic-gate { 63, 64, 0, 1024 }, 9697c478bd9Sstevel@tonic-gate { 95, 64, 0, 512 }, 9707c478bd9Sstevel@tonic-gate { 143, 64, 0, 0 }, 9717c478bd9Sstevel@tonic-gate }; 9727c478bd9Sstevel@tonic-gate 9737c478bd9Sstevel@tonic-gate static uint32_t kmem_reaping; 9747c478bd9Sstevel@tonic-gate static uint32_t kmem_reaping_idspace; 9757c478bd9Sstevel@tonic-gate 9767c478bd9Sstevel@tonic-gate /* 9777c478bd9Sstevel@tonic-gate * kmem tunables 9787c478bd9Sstevel@tonic-gate */ 9797c478bd9Sstevel@tonic-gate clock_t kmem_reap_interval; /* cache reaping rate [15 * HZ ticks] */ 9807c478bd9Sstevel@tonic-gate int kmem_depot_contention = 3; /* max failed tryenters per real interval */ 9817c478bd9Sstevel@tonic-gate pgcnt_t kmem_reapahead = 0; /* start reaping N pages before pageout */ 9827c478bd9Sstevel@tonic-gate int kmem_panic = 1; /* whether to panic on error */ 9837c478bd9Sstevel@tonic-gate int kmem_logging = 1; /* kmem_log_enter() override */ 9847c478bd9Sstevel@tonic-gate uint32_t kmem_mtbf = 0; /* mean time between failures [default: off] */ 9857c478bd9Sstevel@tonic-gate size_t kmem_transaction_log_size; /* transaction log size [2% of memory] */ 9867c478bd9Sstevel@tonic-gate size_t kmem_content_log_size; /* content log size [2% of memory] */ 9877c478bd9Sstevel@tonic-gate size_t kmem_failure_log_size; /* failure log [4 pages per CPU] */ 9887c478bd9Sstevel@tonic-gate size_t kmem_slab_log_size; /* slab create log [4 pages per CPU] */ 9897c478bd9Sstevel@tonic-gate size_t kmem_content_maxsave = 256; /* KMF_CONTENTS max bytes to log */ 9907c478bd9Sstevel@tonic-gate size_t kmem_lite_minsize = 0; /* minimum buffer size for KMF_LITE */ 9917c478bd9Sstevel@tonic-gate size_t kmem_lite_maxalign = 1024; /* maximum buffer alignment for KMF_LITE */ 9927c478bd9Sstevel@tonic-gate int kmem_lite_pcs = 4; /* number of PCs to store in KMF_LITE mode */ 9937c478bd9Sstevel@tonic-gate size_t kmem_maxverify; /* maximum bytes to inspect in debug routines */ 9947c478bd9Sstevel@tonic-gate size_t kmem_minfirewall; /* hardware-enforced redzone threshold */ 9957c478bd9Sstevel@tonic-gate 996*dce01e3fSJonathan W Adams #ifdef _LP64 997*dce01e3fSJonathan W Adams size_t kmem_max_cached = KMEM_BIG_MAXBUF; /* maximum kmem_alloc cache */ 998*dce01e3fSJonathan W Adams #else 999*dce01e3fSJonathan W Adams size_t kmem_max_cached = KMEM_BIG_MAXBUF_32BIT; /* maximum kmem_alloc cache */ 1000*dce01e3fSJonathan W Adams #endif 1001*dce01e3fSJonathan W Adams 10027c478bd9Sstevel@tonic-gate #ifdef DEBUG 10037c478bd9Sstevel@tonic-gate int kmem_flags = KMF_AUDIT | KMF_DEADBEEF | KMF_REDZONE | KMF_CONTENTS; 10047c478bd9Sstevel@tonic-gate #else 10057c478bd9Sstevel@tonic-gate int kmem_flags = 0; 10067c478bd9Sstevel@tonic-gate #endif 10077c478bd9Sstevel@tonic-gate int kmem_ready; 10087c478bd9Sstevel@tonic-gate 10097c478bd9Sstevel@tonic-gate static kmem_cache_t *kmem_slab_cache; 10107c478bd9Sstevel@tonic-gate static kmem_cache_t *kmem_bufctl_cache; 10117c478bd9Sstevel@tonic-gate static kmem_cache_t *kmem_bufctl_audit_cache; 10127c478bd9Sstevel@tonic-gate 10137c478bd9Sstevel@tonic-gate static kmutex_t kmem_cache_lock; /* inter-cache linkage only */ 1014b5fca8f8Stomee static list_t kmem_caches; 10157c478bd9Sstevel@tonic-gate 10167c478bd9Sstevel@tonic-gate static taskq_t *kmem_taskq; 10177c478bd9Sstevel@tonic-gate static kmutex_t kmem_flags_lock; 10187c478bd9Sstevel@tonic-gate static vmem_t *kmem_metadata_arena; 10197c478bd9Sstevel@tonic-gate static vmem_t *kmem_msb_arena; /* arena for metadata caches */ 10207c478bd9Sstevel@tonic-gate static vmem_t *kmem_cache_arena; 10217c478bd9Sstevel@tonic-gate static vmem_t *kmem_hash_arena; 10227c478bd9Sstevel@tonic-gate static vmem_t *kmem_log_arena; 10237c478bd9Sstevel@tonic-gate static vmem_t *kmem_oversize_arena; 10247c478bd9Sstevel@tonic-gate static vmem_t *kmem_va_arena; 10257c478bd9Sstevel@tonic-gate static vmem_t *kmem_default_arena; 10267c478bd9Sstevel@tonic-gate static vmem_t *kmem_firewall_va_arena; 10277c478bd9Sstevel@tonic-gate static vmem_t *kmem_firewall_arena; 10287c478bd9Sstevel@tonic-gate 1029b5fca8f8Stomee /* 1030b5fca8f8Stomee * Define KMEM_STATS to turn on statistic gathering. By default, it is only 1031b5fca8f8Stomee * turned on when DEBUG is also defined. 1032b5fca8f8Stomee */ 1033b5fca8f8Stomee #ifdef DEBUG 1034b5fca8f8Stomee #define KMEM_STATS 1035b5fca8f8Stomee #endif /* DEBUG */ 1036b5fca8f8Stomee 1037b5fca8f8Stomee #ifdef KMEM_STATS 1038b5fca8f8Stomee #define KMEM_STAT_ADD(stat) ((stat)++) 1039b5fca8f8Stomee #define KMEM_STAT_COND_ADD(cond, stat) ((void) (!(cond) || (stat)++)) 1040b5fca8f8Stomee #else 1041b5fca8f8Stomee #define KMEM_STAT_ADD(stat) /* nothing */ 1042b5fca8f8Stomee #define KMEM_STAT_COND_ADD(cond, stat) /* nothing */ 1043b5fca8f8Stomee #endif /* KMEM_STATS */ 1044b5fca8f8Stomee 1045b5fca8f8Stomee /* 1046b5fca8f8Stomee * kmem slab consolidator thresholds (tunables) 1047b5fca8f8Stomee */ 1048b5fca8f8Stomee static size_t kmem_frag_minslabs = 101; /* minimum total slabs */ 1049b5fca8f8Stomee static size_t kmem_frag_numer = 1; /* free buffers (numerator) */ 1050b5fca8f8Stomee static size_t kmem_frag_denom = KMEM_VOID_FRACTION; /* buffers (denominator) */ 1051b5fca8f8Stomee /* 1052b5fca8f8Stomee * Maximum number of slabs from which to move buffers during a single 1053b5fca8f8Stomee * maintenance interval while the system is not low on memory. 1054b5fca8f8Stomee */ 1055b5fca8f8Stomee static size_t kmem_reclaim_max_slabs = 1; 1056b5fca8f8Stomee /* 1057b5fca8f8Stomee * Number of slabs to scan backwards from the end of the partial slab list 1058b5fca8f8Stomee * when searching for buffers to relocate. 1059b5fca8f8Stomee */ 1060b5fca8f8Stomee static size_t kmem_reclaim_scan_range = 12; 1061b5fca8f8Stomee 1062b5fca8f8Stomee #ifdef KMEM_STATS 1063b5fca8f8Stomee static struct { 1064b5fca8f8Stomee uint64_t kms_callbacks; 1065b5fca8f8Stomee uint64_t kms_yes; 1066b5fca8f8Stomee uint64_t kms_no; 1067b5fca8f8Stomee uint64_t kms_later; 1068b5fca8f8Stomee uint64_t kms_dont_need; 1069b5fca8f8Stomee uint64_t kms_dont_know; 1070b5fca8f8Stomee uint64_t kms_hunt_found_slab; 1071b5fca8f8Stomee uint64_t kms_hunt_found_mag; 1072b5fca8f8Stomee uint64_t kms_hunt_alloc_fail; 1073b5fca8f8Stomee uint64_t kms_hunt_lucky; 1074b5fca8f8Stomee uint64_t kms_notify; 1075b5fca8f8Stomee uint64_t kms_notify_callbacks; 1076b5fca8f8Stomee uint64_t kms_disbelief; 1077b5fca8f8Stomee uint64_t kms_already_pending; 1078b5fca8f8Stomee uint64_t kms_callback_alloc_fail; 107925e2c9cfStomee uint64_t kms_callback_taskq_fail; 1080b5fca8f8Stomee uint64_t kms_endscan_slab_destroyed; 1081b5fca8f8Stomee uint64_t kms_endscan_nomem; 1082b5fca8f8Stomee uint64_t kms_endscan_slab_all_used; 1083b5fca8f8Stomee uint64_t kms_endscan_refcnt_changed; 1084b5fca8f8Stomee uint64_t kms_endscan_nomove_changed; 1085b5fca8f8Stomee uint64_t kms_endscan_freelist; 1086b5fca8f8Stomee uint64_t kms_avl_update; 1087b5fca8f8Stomee uint64_t kms_avl_noupdate; 1088b5fca8f8Stomee uint64_t kms_no_longer_reclaimable; 1089b5fca8f8Stomee uint64_t kms_notify_no_longer_reclaimable; 1090b5fca8f8Stomee uint64_t kms_alloc_fail; 1091b5fca8f8Stomee uint64_t kms_constructor_fail; 1092b5fca8f8Stomee uint64_t kms_dead_slabs_freed; 1093b5fca8f8Stomee uint64_t kms_defrags; 1094b5fca8f8Stomee uint64_t kms_scan_depot_ws_reaps; 1095b5fca8f8Stomee uint64_t kms_debug_reaps; 1096b5fca8f8Stomee uint64_t kms_debug_move_scans; 1097b5fca8f8Stomee } kmem_move_stats; 1098b5fca8f8Stomee #endif /* KMEM_STATS */ 1099b5fca8f8Stomee 1100b5fca8f8Stomee /* consolidator knobs */ 1101b5fca8f8Stomee static boolean_t kmem_move_noreap; 1102b5fca8f8Stomee static boolean_t kmem_move_blocked; 1103b5fca8f8Stomee static boolean_t kmem_move_fulltilt; 1104b5fca8f8Stomee static boolean_t kmem_move_any_partial; 1105b5fca8f8Stomee 1106b5fca8f8Stomee #ifdef DEBUG 1107b5fca8f8Stomee /* 1108b5fca8f8Stomee * Ensure code coverage by occasionally running the consolidator even when the 1109b5fca8f8Stomee * caches are not fragmented (they may never be). These intervals are mean time 1110b5fca8f8Stomee * in cache maintenance intervals (kmem_cache_update). 1111b5fca8f8Stomee */ 1112b5fca8f8Stomee static int kmem_mtb_move = 60; /* defrag 1 slab (~15min) */ 1113b5fca8f8Stomee static int kmem_mtb_reap = 1800; /* defrag all slabs (~7.5hrs) */ 1114b5fca8f8Stomee #endif /* DEBUG */ 1115b5fca8f8Stomee 1116b5fca8f8Stomee static kmem_cache_t *kmem_defrag_cache; 1117b5fca8f8Stomee static kmem_cache_t *kmem_move_cache; 1118b5fca8f8Stomee static taskq_t *kmem_move_taskq; 1119b5fca8f8Stomee 1120b5fca8f8Stomee static void kmem_cache_scan(kmem_cache_t *); 1121b5fca8f8Stomee static void kmem_cache_defrag(kmem_cache_t *); 1122b5fca8f8Stomee 1123b5fca8f8Stomee 11247c478bd9Sstevel@tonic-gate kmem_log_header_t *kmem_transaction_log; 11257c478bd9Sstevel@tonic-gate kmem_log_header_t *kmem_content_log; 11267c478bd9Sstevel@tonic-gate kmem_log_header_t *kmem_failure_log; 11277c478bd9Sstevel@tonic-gate kmem_log_header_t *kmem_slab_log; 11287c478bd9Sstevel@tonic-gate 11297c478bd9Sstevel@tonic-gate static int kmem_lite_count; /* # of PCs in kmem_buftag_lite_t */ 11307c478bd9Sstevel@tonic-gate 11317c478bd9Sstevel@tonic-gate #define KMEM_BUFTAG_LITE_ENTER(bt, count, caller) \ 11327c478bd9Sstevel@tonic-gate if ((count) > 0) { \ 11337c478bd9Sstevel@tonic-gate pc_t *_s = ((kmem_buftag_lite_t *)(bt))->bt_history; \ 11347c478bd9Sstevel@tonic-gate pc_t *_e; \ 11357c478bd9Sstevel@tonic-gate /* memmove() the old entries down one notch */ \ 11367c478bd9Sstevel@tonic-gate for (_e = &_s[(count) - 1]; _e > _s; _e--) \ 11377c478bd9Sstevel@tonic-gate *_e = *(_e - 1); \ 11387c478bd9Sstevel@tonic-gate *_s = (uintptr_t)(caller); \ 11397c478bd9Sstevel@tonic-gate } 11407c478bd9Sstevel@tonic-gate 11417c478bd9Sstevel@tonic-gate #define KMERR_MODIFIED 0 /* buffer modified while on freelist */ 11427c478bd9Sstevel@tonic-gate #define KMERR_REDZONE 1 /* redzone violation (write past end of buf) */ 11437c478bd9Sstevel@tonic-gate #define KMERR_DUPFREE 2 /* freed a buffer twice */ 11447c478bd9Sstevel@tonic-gate #define KMERR_BADADDR 3 /* freed a bad (unallocated) address */ 11457c478bd9Sstevel@tonic-gate #define KMERR_BADBUFTAG 4 /* buftag corrupted */ 11467c478bd9Sstevel@tonic-gate #define KMERR_BADBUFCTL 5 /* bufctl corrupted */ 11477c478bd9Sstevel@tonic-gate #define KMERR_BADCACHE 6 /* freed a buffer to the wrong cache */ 11487c478bd9Sstevel@tonic-gate #define KMERR_BADSIZE 7 /* alloc size != free size */ 11497c478bd9Sstevel@tonic-gate #define KMERR_BADBASE 8 /* buffer base address wrong */ 11507c478bd9Sstevel@tonic-gate 11517c478bd9Sstevel@tonic-gate struct { 11527c478bd9Sstevel@tonic-gate hrtime_t kmp_timestamp; /* timestamp of panic */ 11537c478bd9Sstevel@tonic-gate int kmp_error; /* type of kmem error */ 11547c478bd9Sstevel@tonic-gate void *kmp_buffer; /* buffer that induced panic */ 11557c478bd9Sstevel@tonic-gate void *kmp_realbuf; /* real start address for buffer */ 11567c478bd9Sstevel@tonic-gate kmem_cache_t *kmp_cache; /* buffer's cache according to client */ 11577c478bd9Sstevel@tonic-gate kmem_cache_t *kmp_realcache; /* actual cache containing buffer */ 11587c478bd9Sstevel@tonic-gate kmem_slab_t *kmp_slab; /* slab accoring to kmem_findslab() */ 11597c478bd9Sstevel@tonic-gate kmem_bufctl_t *kmp_bufctl; /* bufctl */ 11607c478bd9Sstevel@tonic-gate } kmem_panic_info; 11617c478bd9Sstevel@tonic-gate 11627c478bd9Sstevel@tonic-gate 11637c478bd9Sstevel@tonic-gate static void 11647c478bd9Sstevel@tonic-gate copy_pattern(uint64_t pattern, void *buf_arg, size_t size) 11657c478bd9Sstevel@tonic-gate { 11667c478bd9Sstevel@tonic-gate uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 11677c478bd9Sstevel@tonic-gate uint64_t *buf = buf_arg; 11687c478bd9Sstevel@tonic-gate 11697c478bd9Sstevel@tonic-gate while (buf < bufend) 11707c478bd9Sstevel@tonic-gate *buf++ = pattern; 11717c478bd9Sstevel@tonic-gate } 11727c478bd9Sstevel@tonic-gate 11737c478bd9Sstevel@tonic-gate static void * 11747c478bd9Sstevel@tonic-gate verify_pattern(uint64_t pattern, void *buf_arg, size_t size) 11757c478bd9Sstevel@tonic-gate { 11767c478bd9Sstevel@tonic-gate uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 11777c478bd9Sstevel@tonic-gate uint64_t *buf; 11787c478bd9Sstevel@tonic-gate 11797c478bd9Sstevel@tonic-gate for (buf = buf_arg; buf < bufend; buf++) 11807c478bd9Sstevel@tonic-gate if (*buf != pattern) 11817c478bd9Sstevel@tonic-gate return (buf); 11827c478bd9Sstevel@tonic-gate return (NULL); 11837c478bd9Sstevel@tonic-gate } 11847c478bd9Sstevel@tonic-gate 11857c478bd9Sstevel@tonic-gate static void * 11867c478bd9Sstevel@tonic-gate verify_and_copy_pattern(uint64_t old, uint64_t new, void *buf_arg, size_t size) 11877c478bd9Sstevel@tonic-gate { 11887c478bd9Sstevel@tonic-gate uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 11897c478bd9Sstevel@tonic-gate uint64_t *buf; 11907c478bd9Sstevel@tonic-gate 11917c478bd9Sstevel@tonic-gate for (buf = buf_arg; buf < bufend; buf++) { 11927c478bd9Sstevel@tonic-gate if (*buf != old) { 11937c478bd9Sstevel@tonic-gate copy_pattern(old, buf_arg, 11949f1b636aStomee (char *)buf - (char *)buf_arg); 11957c478bd9Sstevel@tonic-gate return (buf); 11967c478bd9Sstevel@tonic-gate } 11977c478bd9Sstevel@tonic-gate *buf = new; 11987c478bd9Sstevel@tonic-gate } 11997c478bd9Sstevel@tonic-gate 12007c478bd9Sstevel@tonic-gate return (NULL); 12017c478bd9Sstevel@tonic-gate } 12027c478bd9Sstevel@tonic-gate 12037c478bd9Sstevel@tonic-gate static void 12047c478bd9Sstevel@tonic-gate kmem_cache_applyall(void (*func)(kmem_cache_t *), taskq_t *tq, int tqflag) 12057c478bd9Sstevel@tonic-gate { 12067c478bd9Sstevel@tonic-gate kmem_cache_t *cp; 12077c478bd9Sstevel@tonic-gate 12087c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_lock); 1209b5fca8f8Stomee for (cp = list_head(&kmem_caches); cp != NULL; 1210b5fca8f8Stomee cp = list_next(&kmem_caches, cp)) 12117c478bd9Sstevel@tonic-gate if (tq != NULL) 12127c478bd9Sstevel@tonic-gate (void) taskq_dispatch(tq, (task_func_t *)func, cp, 12137c478bd9Sstevel@tonic-gate tqflag); 12147c478bd9Sstevel@tonic-gate else 12157c478bd9Sstevel@tonic-gate func(cp); 12167c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_lock); 12177c478bd9Sstevel@tonic-gate } 12187c478bd9Sstevel@tonic-gate 12197c478bd9Sstevel@tonic-gate static void 12207c478bd9Sstevel@tonic-gate kmem_cache_applyall_id(void (*func)(kmem_cache_t *), taskq_t *tq, int tqflag) 12217c478bd9Sstevel@tonic-gate { 12227c478bd9Sstevel@tonic-gate kmem_cache_t *cp; 12237c478bd9Sstevel@tonic-gate 12247c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_lock); 1225b5fca8f8Stomee for (cp = list_head(&kmem_caches); cp != NULL; 1226b5fca8f8Stomee cp = list_next(&kmem_caches, cp)) { 12277c478bd9Sstevel@tonic-gate if (!(cp->cache_cflags & KMC_IDENTIFIER)) 12287c478bd9Sstevel@tonic-gate continue; 12297c478bd9Sstevel@tonic-gate if (tq != NULL) 12307c478bd9Sstevel@tonic-gate (void) taskq_dispatch(tq, (task_func_t *)func, cp, 12317c478bd9Sstevel@tonic-gate tqflag); 12327c478bd9Sstevel@tonic-gate else 12337c478bd9Sstevel@tonic-gate func(cp); 12347c478bd9Sstevel@tonic-gate } 12357c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_lock); 12367c478bd9Sstevel@tonic-gate } 12377c478bd9Sstevel@tonic-gate 12387c478bd9Sstevel@tonic-gate /* 12397c478bd9Sstevel@tonic-gate * Debugging support. Given a buffer address, find its slab. 12407c478bd9Sstevel@tonic-gate */ 12417c478bd9Sstevel@tonic-gate static kmem_slab_t * 12427c478bd9Sstevel@tonic-gate kmem_findslab(kmem_cache_t *cp, void *buf) 12437c478bd9Sstevel@tonic-gate { 12447c478bd9Sstevel@tonic-gate kmem_slab_t *sp; 12457c478bd9Sstevel@tonic-gate 12467c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 1247b5fca8f8Stomee for (sp = list_head(&cp->cache_complete_slabs); sp != NULL; 1248b5fca8f8Stomee sp = list_next(&cp->cache_complete_slabs, sp)) { 1249b5fca8f8Stomee if (KMEM_SLAB_MEMBER(sp, buf)) { 1250b5fca8f8Stomee mutex_exit(&cp->cache_lock); 1251b5fca8f8Stomee return (sp); 1252b5fca8f8Stomee } 1253b5fca8f8Stomee } 1254b5fca8f8Stomee for (sp = avl_first(&cp->cache_partial_slabs); sp != NULL; 1255b5fca8f8Stomee sp = AVL_NEXT(&cp->cache_partial_slabs, sp)) { 12567c478bd9Sstevel@tonic-gate if (KMEM_SLAB_MEMBER(sp, buf)) { 12577c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 12587c478bd9Sstevel@tonic-gate return (sp); 12597c478bd9Sstevel@tonic-gate } 12607c478bd9Sstevel@tonic-gate } 12617c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 12627c478bd9Sstevel@tonic-gate 12637c478bd9Sstevel@tonic-gate return (NULL); 12647c478bd9Sstevel@tonic-gate } 12657c478bd9Sstevel@tonic-gate 12667c478bd9Sstevel@tonic-gate static void 12677c478bd9Sstevel@tonic-gate kmem_error(int error, kmem_cache_t *cparg, void *bufarg) 12687c478bd9Sstevel@tonic-gate { 12697c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = NULL; 12707c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp = NULL; 12717c478bd9Sstevel@tonic-gate kmem_cache_t *cp = cparg; 12727c478bd9Sstevel@tonic-gate kmem_slab_t *sp; 12737c478bd9Sstevel@tonic-gate uint64_t *off; 12747c478bd9Sstevel@tonic-gate void *buf = bufarg; 12757c478bd9Sstevel@tonic-gate 12767c478bd9Sstevel@tonic-gate kmem_logging = 0; /* stop logging when a bad thing happens */ 12777c478bd9Sstevel@tonic-gate 12787c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_timestamp = gethrtime(); 12797c478bd9Sstevel@tonic-gate 12807c478bd9Sstevel@tonic-gate sp = kmem_findslab(cp, buf); 12817c478bd9Sstevel@tonic-gate if (sp == NULL) { 1282b5fca8f8Stomee for (cp = list_tail(&kmem_caches); cp != NULL; 1283b5fca8f8Stomee cp = list_prev(&kmem_caches, cp)) { 12847c478bd9Sstevel@tonic-gate if ((sp = kmem_findslab(cp, buf)) != NULL) 12857c478bd9Sstevel@tonic-gate break; 12867c478bd9Sstevel@tonic-gate } 12877c478bd9Sstevel@tonic-gate } 12887c478bd9Sstevel@tonic-gate 12897c478bd9Sstevel@tonic-gate if (sp == NULL) { 12907c478bd9Sstevel@tonic-gate cp = NULL; 12917c478bd9Sstevel@tonic-gate error = KMERR_BADADDR; 12927c478bd9Sstevel@tonic-gate } else { 12937c478bd9Sstevel@tonic-gate if (cp != cparg) 12947c478bd9Sstevel@tonic-gate error = KMERR_BADCACHE; 12957c478bd9Sstevel@tonic-gate else 12967c478bd9Sstevel@tonic-gate buf = (char *)bufarg - ((uintptr_t)bufarg - 12977c478bd9Sstevel@tonic-gate (uintptr_t)sp->slab_base) % cp->cache_chunksize; 12987c478bd9Sstevel@tonic-gate if (buf != bufarg) 12997c478bd9Sstevel@tonic-gate error = KMERR_BADBASE; 13007c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_BUFTAG) 13017c478bd9Sstevel@tonic-gate btp = KMEM_BUFTAG(cp, buf); 13027c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) { 13037c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 13047c478bd9Sstevel@tonic-gate for (bcp = *KMEM_HASH(cp, buf); bcp; bcp = bcp->bc_next) 13057c478bd9Sstevel@tonic-gate if (bcp->bc_addr == buf) 13067c478bd9Sstevel@tonic-gate break; 13077c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 13087c478bd9Sstevel@tonic-gate if (bcp == NULL && btp != NULL) 13097c478bd9Sstevel@tonic-gate bcp = btp->bt_bufctl; 13107c478bd9Sstevel@tonic-gate if (kmem_findslab(cp->cache_bufctl_cache, bcp) == 13117c478bd9Sstevel@tonic-gate NULL || P2PHASE((uintptr_t)bcp, KMEM_ALIGN) || 13127c478bd9Sstevel@tonic-gate bcp->bc_addr != buf) { 13137c478bd9Sstevel@tonic-gate error = KMERR_BADBUFCTL; 13147c478bd9Sstevel@tonic-gate bcp = NULL; 13157c478bd9Sstevel@tonic-gate } 13167c478bd9Sstevel@tonic-gate } 13177c478bd9Sstevel@tonic-gate } 13187c478bd9Sstevel@tonic-gate 13197c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_error = error; 13207c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_buffer = bufarg; 13217c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_realbuf = buf; 13227c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_cache = cparg; 13237c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_realcache = cp; 13247c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_slab = sp; 13257c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_bufctl = bcp; 13267c478bd9Sstevel@tonic-gate 13277c478bd9Sstevel@tonic-gate printf("kernel memory allocator: "); 13287c478bd9Sstevel@tonic-gate 13297c478bd9Sstevel@tonic-gate switch (error) { 13307c478bd9Sstevel@tonic-gate 13317c478bd9Sstevel@tonic-gate case KMERR_MODIFIED: 13327c478bd9Sstevel@tonic-gate printf("buffer modified after being freed\n"); 13337c478bd9Sstevel@tonic-gate off = verify_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify); 13347c478bd9Sstevel@tonic-gate if (off == NULL) /* shouldn't happen */ 13357c478bd9Sstevel@tonic-gate off = buf; 13367c478bd9Sstevel@tonic-gate printf("modification occurred at offset 0x%lx " 13377c478bd9Sstevel@tonic-gate "(0x%llx replaced by 0x%llx)\n", 13387c478bd9Sstevel@tonic-gate (uintptr_t)off - (uintptr_t)buf, 13397c478bd9Sstevel@tonic-gate (longlong_t)KMEM_FREE_PATTERN, (longlong_t)*off); 13407c478bd9Sstevel@tonic-gate break; 13417c478bd9Sstevel@tonic-gate 13427c478bd9Sstevel@tonic-gate case KMERR_REDZONE: 13437c478bd9Sstevel@tonic-gate printf("redzone violation: write past end of buffer\n"); 13447c478bd9Sstevel@tonic-gate break; 13457c478bd9Sstevel@tonic-gate 13467c478bd9Sstevel@tonic-gate case KMERR_BADADDR: 13477c478bd9Sstevel@tonic-gate printf("invalid free: buffer not in cache\n"); 13487c478bd9Sstevel@tonic-gate break; 13497c478bd9Sstevel@tonic-gate 13507c478bd9Sstevel@tonic-gate case KMERR_DUPFREE: 13517c478bd9Sstevel@tonic-gate printf("duplicate free: buffer freed twice\n"); 13527c478bd9Sstevel@tonic-gate break; 13537c478bd9Sstevel@tonic-gate 13547c478bd9Sstevel@tonic-gate case KMERR_BADBUFTAG: 13557c478bd9Sstevel@tonic-gate printf("boundary tag corrupted\n"); 13567c478bd9Sstevel@tonic-gate printf("bcp ^ bxstat = %lx, should be %lx\n", 13577c478bd9Sstevel@tonic-gate (intptr_t)btp->bt_bufctl ^ btp->bt_bxstat, 13587c478bd9Sstevel@tonic-gate KMEM_BUFTAG_FREE); 13597c478bd9Sstevel@tonic-gate break; 13607c478bd9Sstevel@tonic-gate 13617c478bd9Sstevel@tonic-gate case KMERR_BADBUFCTL: 13627c478bd9Sstevel@tonic-gate printf("bufctl corrupted\n"); 13637c478bd9Sstevel@tonic-gate break; 13647c478bd9Sstevel@tonic-gate 13657c478bd9Sstevel@tonic-gate case KMERR_BADCACHE: 13667c478bd9Sstevel@tonic-gate printf("buffer freed to wrong cache\n"); 13677c478bd9Sstevel@tonic-gate printf("buffer was allocated from %s,\n", cp->cache_name); 13687c478bd9Sstevel@tonic-gate printf("caller attempting free to %s.\n", cparg->cache_name); 13697c478bd9Sstevel@tonic-gate break; 13707c478bd9Sstevel@tonic-gate 13717c478bd9Sstevel@tonic-gate case KMERR_BADSIZE: 13727c478bd9Sstevel@tonic-gate printf("bad free: free size (%u) != alloc size (%u)\n", 13737c478bd9Sstevel@tonic-gate KMEM_SIZE_DECODE(((uint32_t *)btp)[0]), 13747c478bd9Sstevel@tonic-gate KMEM_SIZE_DECODE(((uint32_t *)btp)[1])); 13757c478bd9Sstevel@tonic-gate break; 13767c478bd9Sstevel@tonic-gate 13777c478bd9Sstevel@tonic-gate case KMERR_BADBASE: 13787c478bd9Sstevel@tonic-gate printf("bad free: free address (%p) != alloc address (%p)\n", 13797c478bd9Sstevel@tonic-gate bufarg, buf); 13807c478bd9Sstevel@tonic-gate break; 13817c478bd9Sstevel@tonic-gate } 13827c478bd9Sstevel@tonic-gate 13837c478bd9Sstevel@tonic-gate printf("buffer=%p bufctl=%p cache: %s\n", 13847c478bd9Sstevel@tonic-gate bufarg, (void *)bcp, cparg->cache_name); 13857c478bd9Sstevel@tonic-gate 13867c478bd9Sstevel@tonic-gate if (bcp != NULL && (cp->cache_flags & KMF_AUDIT) && 13877c478bd9Sstevel@tonic-gate error != KMERR_BADBUFCTL) { 13887c478bd9Sstevel@tonic-gate int d; 13897c478bd9Sstevel@tonic-gate timestruc_t ts; 13907c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *bcap = (kmem_bufctl_audit_t *)bcp; 13917c478bd9Sstevel@tonic-gate 13927c478bd9Sstevel@tonic-gate hrt2ts(kmem_panic_info.kmp_timestamp - bcap->bc_timestamp, &ts); 13937c478bd9Sstevel@tonic-gate printf("previous transaction on buffer %p:\n", buf); 13947c478bd9Sstevel@tonic-gate printf("thread=%p time=T-%ld.%09ld slab=%p cache: %s\n", 13957c478bd9Sstevel@tonic-gate (void *)bcap->bc_thread, ts.tv_sec, ts.tv_nsec, 13967c478bd9Sstevel@tonic-gate (void *)sp, cp->cache_name); 13977c478bd9Sstevel@tonic-gate for (d = 0; d < MIN(bcap->bc_depth, KMEM_STACK_DEPTH); d++) { 13987c478bd9Sstevel@tonic-gate ulong_t off; 13997c478bd9Sstevel@tonic-gate char *sym = kobj_getsymname(bcap->bc_stack[d], &off); 14007c478bd9Sstevel@tonic-gate printf("%s+%lx\n", sym ? sym : "?", off); 14017c478bd9Sstevel@tonic-gate } 14027c478bd9Sstevel@tonic-gate } 14037c478bd9Sstevel@tonic-gate if (kmem_panic > 0) 14047c478bd9Sstevel@tonic-gate panic("kernel heap corruption detected"); 14057c478bd9Sstevel@tonic-gate if (kmem_panic == 0) 14067c478bd9Sstevel@tonic-gate debug_enter(NULL); 14077c478bd9Sstevel@tonic-gate kmem_logging = 1; /* resume logging */ 14087c478bd9Sstevel@tonic-gate } 14097c478bd9Sstevel@tonic-gate 14107c478bd9Sstevel@tonic-gate static kmem_log_header_t * 14117c478bd9Sstevel@tonic-gate kmem_log_init(size_t logsize) 14127c478bd9Sstevel@tonic-gate { 14137c478bd9Sstevel@tonic-gate kmem_log_header_t *lhp; 14147c478bd9Sstevel@tonic-gate int nchunks = 4 * max_ncpus; 14157c478bd9Sstevel@tonic-gate size_t lhsize = (size_t)&((kmem_log_header_t *)0)->lh_cpu[max_ncpus]; 14167c478bd9Sstevel@tonic-gate int i; 14177c478bd9Sstevel@tonic-gate 14187c478bd9Sstevel@tonic-gate /* 14197c478bd9Sstevel@tonic-gate * Make sure that lhp->lh_cpu[] is nicely aligned 14207c478bd9Sstevel@tonic-gate * to prevent false sharing of cache lines. 14217c478bd9Sstevel@tonic-gate */ 14227c478bd9Sstevel@tonic-gate lhsize = P2ROUNDUP(lhsize, KMEM_ALIGN); 14237c478bd9Sstevel@tonic-gate lhp = vmem_xalloc(kmem_log_arena, lhsize, 64, P2NPHASE(lhsize, 64), 0, 14247c478bd9Sstevel@tonic-gate NULL, NULL, VM_SLEEP); 14257c478bd9Sstevel@tonic-gate bzero(lhp, lhsize); 14267c478bd9Sstevel@tonic-gate 14277c478bd9Sstevel@tonic-gate mutex_init(&lhp->lh_lock, NULL, MUTEX_DEFAULT, NULL); 14287c478bd9Sstevel@tonic-gate lhp->lh_nchunks = nchunks; 14297c478bd9Sstevel@tonic-gate lhp->lh_chunksize = P2ROUNDUP(logsize / nchunks + 1, PAGESIZE); 14307c478bd9Sstevel@tonic-gate lhp->lh_base = vmem_alloc(kmem_log_arena, 14317c478bd9Sstevel@tonic-gate lhp->lh_chunksize * nchunks, VM_SLEEP); 14327c478bd9Sstevel@tonic-gate lhp->lh_free = vmem_alloc(kmem_log_arena, 14337c478bd9Sstevel@tonic-gate nchunks * sizeof (int), VM_SLEEP); 14347c478bd9Sstevel@tonic-gate bzero(lhp->lh_base, lhp->lh_chunksize * nchunks); 14357c478bd9Sstevel@tonic-gate 14367c478bd9Sstevel@tonic-gate for (i = 0; i < max_ncpus; i++) { 14377c478bd9Sstevel@tonic-gate kmem_cpu_log_header_t *clhp = &lhp->lh_cpu[i]; 14387c478bd9Sstevel@tonic-gate mutex_init(&clhp->clh_lock, NULL, MUTEX_DEFAULT, NULL); 14397c478bd9Sstevel@tonic-gate clhp->clh_chunk = i; 14407c478bd9Sstevel@tonic-gate } 14417c478bd9Sstevel@tonic-gate 14427c478bd9Sstevel@tonic-gate for (i = max_ncpus; i < nchunks; i++) 14437c478bd9Sstevel@tonic-gate lhp->lh_free[i] = i; 14447c478bd9Sstevel@tonic-gate 14457c478bd9Sstevel@tonic-gate lhp->lh_head = max_ncpus; 14467c478bd9Sstevel@tonic-gate lhp->lh_tail = 0; 14477c478bd9Sstevel@tonic-gate 14487c478bd9Sstevel@tonic-gate return (lhp); 14497c478bd9Sstevel@tonic-gate } 14507c478bd9Sstevel@tonic-gate 14517c478bd9Sstevel@tonic-gate static void * 14527c478bd9Sstevel@tonic-gate kmem_log_enter(kmem_log_header_t *lhp, void *data, size_t size) 14537c478bd9Sstevel@tonic-gate { 14547c478bd9Sstevel@tonic-gate void *logspace; 14557c478bd9Sstevel@tonic-gate kmem_cpu_log_header_t *clhp = &lhp->lh_cpu[CPU->cpu_seqid]; 14567c478bd9Sstevel@tonic-gate 14577c478bd9Sstevel@tonic-gate if (lhp == NULL || kmem_logging == 0 || panicstr) 14587c478bd9Sstevel@tonic-gate return (NULL); 14597c478bd9Sstevel@tonic-gate 14607c478bd9Sstevel@tonic-gate mutex_enter(&clhp->clh_lock); 14617c478bd9Sstevel@tonic-gate clhp->clh_hits++; 14627c478bd9Sstevel@tonic-gate if (size > clhp->clh_avail) { 14637c478bd9Sstevel@tonic-gate mutex_enter(&lhp->lh_lock); 14647c478bd9Sstevel@tonic-gate lhp->lh_hits++; 14657c478bd9Sstevel@tonic-gate lhp->lh_free[lhp->lh_tail] = clhp->clh_chunk; 14667c478bd9Sstevel@tonic-gate lhp->lh_tail = (lhp->lh_tail + 1) % lhp->lh_nchunks; 14677c478bd9Sstevel@tonic-gate clhp->clh_chunk = lhp->lh_free[lhp->lh_head]; 14687c478bd9Sstevel@tonic-gate lhp->lh_head = (lhp->lh_head + 1) % lhp->lh_nchunks; 14697c478bd9Sstevel@tonic-gate clhp->clh_current = lhp->lh_base + 14709f1b636aStomee clhp->clh_chunk * lhp->lh_chunksize; 14717c478bd9Sstevel@tonic-gate clhp->clh_avail = lhp->lh_chunksize; 14727c478bd9Sstevel@tonic-gate if (size > lhp->lh_chunksize) 14737c478bd9Sstevel@tonic-gate size = lhp->lh_chunksize; 14747c478bd9Sstevel@tonic-gate mutex_exit(&lhp->lh_lock); 14757c478bd9Sstevel@tonic-gate } 14767c478bd9Sstevel@tonic-gate logspace = clhp->clh_current; 14777c478bd9Sstevel@tonic-gate clhp->clh_current += size; 14787c478bd9Sstevel@tonic-gate clhp->clh_avail -= size; 14797c478bd9Sstevel@tonic-gate bcopy(data, logspace, size); 14807c478bd9Sstevel@tonic-gate mutex_exit(&clhp->clh_lock); 14817c478bd9Sstevel@tonic-gate return (logspace); 14827c478bd9Sstevel@tonic-gate } 14837c478bd9Sstevel@tonic-gate 14847c478bd9Sstevel@tonic-gate #define KMEM_AUDIT(lp, cp, bcp) \ 14857c478bd9Sstevel@tonic-gate { \ 14867c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *_bcp = (kmem_bufctl_audit_t *)(bcp); \ 14877c478bd9Sstevel@tonic-gate _bcp->bc_timestamp = gethrtime(); \ 14887c478bd9Sstevel@tonic-gate _bcp->bc_thread = curthread; \ 14897c478bd9Sstevel@tonic-gate _bcp->bc_depth = getpcstack(_bcp->bc_stack, KMEM_STACK_DEPTH); \ 14907c478bd9Sstevel@tonic-gate _bcp->bc_lastlog = kmem_log_enter((lp), _bcp, sizeof (*_bcp)); \ 14917c478bd9Sstevel@tonic-gate } 14927c478bd9Sstevel@tonic-gate 14937c478bd9Sstevel@tonic-gate static void 14947c478bd9Sstevel@tonic-gate kmem_log_event(kmem_log_header_t *lp, kmem_cache_t *cp, 14957c478bd9Sstevel@tonic-gate kmem_slab_t *sp, void *addr) 14967c478bd9Sstevel@tonic-gate { 14977c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t bca; 14987c478bd9Sstevel@tonic-gate 14997c478bd9Sstevel@tonic-gate bzero(&bca, sizeof (kmem_bufctl_audit_t)); 15007c478bd9Sstevel@tonic-gate bca.bc_addr = addr; 15017c478bd9Sstevel@tonic-gate bca.bc_slab = sp; 15027c478bd9Sstevel@tonic-gate bca.bc_cache = cp; 15037c478bd9Sstevel@tonic-gate KMEM_AUDIT(lp, cp, &bca); 15047c478bd9Sstevel@tonic-gate } 15057c478bd9Sstevel@tonic-gate 15067c478bd9Sstevel@tonic-gate /* 15077c478bd9Sstevel@tonic-gate * Create a new slab for cache cp. 15087c478bd9Sstevel@tonic-gate */ 15097c478bd9Sstevel@tonic-gate static kmem_slab_t * 15107c478bd9Sstevel@tonic-gate kmem_slab_create(kmem_cache_t *cp, int kmflag) 15117c478bd9Sstevel@tonic-gate { 15127c478bd9Sstevel@tonic-gate size_t slabsize = cp->cache_slabsize; 15137c478bd9Sstevel@tonic-gate size_t chunksize = cp->cache_chunksize; 15147c478bd9Sstevel@tonic-gate int cache_flags = cp->cache_flags; 15157c478bd9Sstevel@tonic-gate size_t color, chunks; 15167c478bd9Sstevel@tonic-gate char *buf, *slab; 15177c478bd9Sstevel@tonic-gate kmem_slab_t *sp; 15187c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp; 15197c478bd9Sstevel@tonic-gate vmem_t *vmp = cp->cache_arena; 15207c478bd9Sstevel@tonic-gate 1521b5fca8f8Stomee ASSERT(MUTEX_NOT_HELD(&cp->cache_lock)); 1522b5fca8f8Stomee 15237c478bd9Sstevel@tonic-gate color = cp->cache_color + cp->cache_align; 15247c478bd9Sstevel@tonic-gate if (color > cp->cache_maxcolor) 15257c478bd9Sstevel@tonic-gate color = cp->cache_mincolor; 15267c478bd9Sstevel@tonic-gate cp->cache_color = color; 15277c478bd9Sstevel@tonic-gate 15287c478bd9Sstevel@tonic-gate slab = vmem_alloc(vmp, slabsize, kmflag & KM_VMFLAGS); 15297c478bd9Sstevel@tonic-gate 15307c478bd9Sstevel@tonic-gate if (slab == NULL) 15317c478bd9Sstevel@tonic-gate goto vmem_alloc_failure; 15327c478bd9Sstevel@tonic-gate 15337c478bd9Sstevel@tonic-gate ASSERT(P2PHASE((uintptr_t)slab, vmp->vm_quantum) == 0); 15347c478bd9Sstevel@tonic-gate 1535b5fca8f8Stomee /* 1536b5fca8f8Stomee * Reverify what was already checked in kmem_cache_set_move(), since the 1537b5fca8f8Stomee * consolidator depends (for correctness) on slabs being initialized 1538b5fca8f8Stomee * with the 0xbaddcafe memory pattern (setting a low order bit usable by 1539b5fca8f8Stomee * clients to distinguish uninitialized memory from known objects). 1540b5fca8f8Stomee */ 1541b5fca8f8Stomee ASSERT((cp->cache_move == NULL) || !(cp->cache_cflags & KMC_NOTOUCH)); 15427c478bd9Sstevel@tonic-gate if (!(cp->cache_cflags & KMC_NOTOUCH)) 15437c478bd9Sstevel@tonic-gate copy_pattern(KMEM_UNINITIALIZED_PATTERN, slab, slabsize); 15447c478bd9Sstevel@tonic-gate 15457c478bd9Sstevel@tonic-gate if (cache_flags & KMF_HASH) { 15467c478bd9Sstevel@tonic-gate if ((sp = kmem_cache_alloc(kmem_slab_cache, kmflag)) == NULL) 15477c478bd9Sstevel@tonic-gate goto slab_alloc_failure; 15487c478bd9Sstevel@tonic-gate chunks = (slabsize - color) / chunksize; 15497c478bd9Sstevel@tonic-gate } else { 15507c478bd9Sstevel@tonic-gate sp = KMEM_SLAB(cp, slab); 15517c478bd9Sstevel@tonic-gate chunks = (slabsize - sizeof (kmem_slab_t) - color) / chunksize; 15527c478bd9Sstevel@tonic-gate } 15537c478bd9Sstevel@tonic-gate 15547c478bd9Sstevel@tonic-gate sp->slab_cache = cp; 15557c478bd9Sstevel@tonic-gate sp->slab_head = NULL; 15567c478bd9Sstevel@tonic-gate sp->slab_refcnt = 0; 15577c478bd9Sstevel@tonic-gate sp->slab_base = buf = slab + color; 15587c478bd9Sstevel@tonic-gate sp->slab_chunks = chunks; 1559b5fca8f8Stomee sp->slab_stuck_offset = (uint32_t)-1; 1560b5fca8f8Stomee sp->slab_later_count = 0; 1561b5fca8f8Stomee sp->slab_flags = 0; 15627c478bd9Sstevel@tonic-gate 15637c478bd9Sstevel@tonic-gate ASSERT(chunks > 0); 15647c478bd9Sstevel@tonic-gate while (chunks-- != 0) { 15657c478bd9Sstevel@tonic-gate if (cache_flags & KMF_HASH) { 15667c478bd9Sstevel@tonic-gate bcp = kmem_cache_alloc(cp->cache_bufctl_cache, kmflag); 15677c478bd9Sstevel@tonic-gate if (bcp == NULL) 15687c478bd9Sstevel@tonic-gate goto bufctl_alloc_failure; 15697c478bd9Sstevel@tonic-gate if (cache_flags & KMF_AUDIT) { 15707c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *bcap = 15717c478bd9Sstevel@tonic-gate (kmem_bufctl_audit_t *)bcp; 15727c478bd9Sstevel@tonic-gate bzero(bcap, sizeof (kmem_bufctl_audit_t)); 15737c478bd9Sstevel@tonic-gate bcap->bc_cache = cp; 15747c478bd9Sstevel@tonic-gate } 15757c478bd9Sstevel@tonic-gate bcp->bc_addr = buf; 15767c478bd9Sstevel@tonic-gate bcp->bc_slab = sp; 15777c478bd9Sstevel@tonic-gate } else { 15787c478bd9Sstevel@tonic-gate bcp = KMEM_BUFCTL(cp, buf); 15797c478bd9Sstevel@tonic-gate } 15807c478bd9Sstevel@tonic-gate if (cache_flags & KMF_BUFTAG) { 15817c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 15827c478bd9Sstevel@tonic-gate btp->bt_redzone = KMEM_REDZONE_PATTERN; 15837c478bd9Sstevel@tonic-gate btp->bt_bufctl = bcp; 15847c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE; 15857c478bd9Sstevel@tonic-gate if (cache_flags & KMF_DEADBEEF) { 15867c478bd9Sstevel@tonic-gate copy_pattern(KMEM_FREE_PATTERN, buf, 15877c478bd9Sstevel@tonic-gate cp->cache_verify); 15887c478bd9Sstevel@tonic-gate } 15897c478bd9Sstevel@tonic-gate } 15907c478bd9Sstevel@tonic-gate bcp->bc_next = sp->slab_head; 15917c478bd9Sstevel@tonic-gate sp->slab_head = bcp; 15927c478bd9Sstevel@tonic-gate buf += chunksize; 15937c478bd9Sstevel@tonic-gate } 15947c478bd9Sstevel@tonic-gate 15957c478bd9Sstevel@tonic-gate kmem_log_event(kmem_slab_log, cp, sp, slab); 15967c478bd9Sstevel@tonic-gate 15977c478bd9Sstevel@tonic-gate return (sp); 15987c478bd9Sstevel@tonic-gate 15997c478bd9Sstevel@tonic-gate bufctl_alloc_failure: 16007c478bd9Sstevel@tonic-gate 16017c478bd9Sstevel@tonic-gate while ((bcp = sp->slab_head) != NULL) { 16027c478bd9Sstevel@tonic-gate sp->slab_head = bcp->bc_next; 16037c478bd9Sstevel@tonic-gate kmem_cache_free(cp->cache_bufctl_cache, bcp); 16047c478bd9Sstevel@tonic-gate } 16057c478bd9Sstevel@tonic-gate kmem_cache_free(kmem_slab_cache, sp); 16067c478bd9Sstevel@tonic-gate 16077c478bd9Sstevel@tonic-gate slab_alloc_failure: 16087c478bd9Sstevel@tonic-gate 16097c478bd9Sstevel@tonic-gate vmem_free(vmp, slab, slabsize); 16107c478bd9Sstevel@tonic-gate 16117c478bd9Sstevel@tonic-gate vmem_alloc_failure: 16127c478bd9Sstevel@tonic-gate 16137c478bd9Sstevel@tonic-gate kmem_log_event(kmem_failure_log, cp, NULL, NULL); 16147c478bd9Sstevel@tonic-gate atomic_add_64(&cp->cache_alloc_fail, 1); 16157c478bd9Sstevel@tonic-gate 16167c478bd9Sstevel@tonic-gate return (NULL); 16177c478bd9Sstevel@tonic-gate } 16187c478bd9Sstevel@tonic-gate 16197c478bd9Sstevel@tonic-gate /* 16207c478bd9Sstevel@tonic-gate * Destroy a slab. 16217c478bd9Sstevel@tonic-gate */ 16227c478bd9Sstevel@tonic-gate static void 16237c478bd9Sstevel@tonic-gate kmem_slab_destroy(kmem_cache_t *cp, kmem_slab_t *sp) 16247c478bd9Sstevel@tonic-gate { 16257c478bd9Sstevel@tonic-gate vmem_t *vmp = cp->cache_arena; 16267c478bd9Sstevel@tonic-gate void *slab = (void *)P2ALIGN((uintptr_t)sp->slab_base, vmp->vm_quantum); 16277c478bd9Sstevel@tonic-gate 1628b5fca8f8Stomee ASSERT(MUTEX_NOT_HELD(&cp->cache_lock)); 1629b5fca8f8Stomee ASSERT(sp->slab_refcnt == 0); 1630b5fca8f8Stomee 16317c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) { 16327c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp; 16337c478bd9Sstevel@tonic-gate while ((bcp = sp->slab_head) != NULL) { 16347c478bd9Sstevel@tonic-gate sp->slab_head = bcp->bc_next; 16357c478bd9Sstevel@tonic-gate kmem_cache_free(cp->cache_bufctl_cache, bcp); 16367c478bd9Sstevel@tonic-gate } 16377c478bd9Sstevel@tonic-gate kmem_cache_free(kmem_slab_cache, sp); 16387c478bd9Sstevel@tonic-gate } 16397c478bd9Sstevel@tonic-gate vmem_free(vmp, slab, cp->cache_slabsize); 16407c478bd9Sstevel@tonic-gate } 16417c478bd9Sstevel@tonic-gate 16427c478bd9Sstevel@tonic-gate static void * 1643b5fca8f8Stomee kmem_slab_alloc_impl(kmem_cache_t *cp, kmem_slab_t *sp) 16447c478bd9Sstevel@tonic-gate { 16457c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp, **hash_bucket; 16467c478bd9Sstevel@tonic-gate void *buf; 16477c478bd9Sstevel@tonic-gate 1648b5fca8f8Stomee ASSERT(MUTEX_HELD(&cp->cache_lock)); 1649b5fca8f8Stomee /* 1650b5fca8f8Stomee * kmem_slab_alloc() drops cache_lock when it creates a new slab, so we 1651b5fca8f8Stomee * can't ASSERT(avl_is_empty(&cp->cache_partial_slabs)) here when the 1652b5fca8f8Stomee * slab is newly created (sp->slab_refcnt == 0). 1653b5fca8f8Stomee */ 1654b5fca8f8Stomee ASSERT((sp->slab_refcnt == 0) || (KMEM_SLAB_IS_PARTIAL(sp) && 1655b5fca8f8Stomee (sp == avl_first(&cp->cache_partial_slabs)))); 16567c478bd9Sstevel@tonic-gate ASSERT(sp->slab_cache == cp); 16577c478bd9Sstevel@tonic-gate 1658b5fca8f8Stomee cp->cache_slab_alloc++; 16599f1b636aStomee cp->cache_bufslab--; 16607c478bd9Sstevel@tonic-gate sp->slab_refcnt++; 16617c478bd9Sstevel@tonic-gate 16627c478bd9Sstevel@tonic-gate bcp = sp->slab_head; 16637c478bd9Sstevel@tonic-gate if ((sp->slab_head = bcp->bc_next) == NULL) { 1664b5fca8f8Stomee ASSERT(KMEM_SLAB_IS_ALL_USED(sp)); 1665b5fca8f8Stomee if (sp->slab_refcnt == 1) { 1666b5fca8f8Stomee ASSERT(sp->slab_chunks == 1); 1667b5fca8f8Stomee } else { 1668b5fca8f8Stomee ASSERT(sp->slab_chunks > 1); /* the slab was partial */ 1669b5fca8f8Stomee avl_remove(&cp->cache_partial_slabs, sp); 1670b5fca8f8Stomee sp->slab_later_count = 0; /* clear history */ 1671b5fca8f8Stomee sp->slab_flags &= ~KMEM_SLAB_NOMOVE; 1672b5fca8f8Stomee sp->slab_stuck_offset = (uint32_t)-1; 1673b5fca8f8Stomee } 1674b5fca8f8Stomee list_insert_head(&cp->cache_complete_slabs, sp); 1675b5fca8f8Stomee cp->cache_complete_slab_count++; 1676b5fca8f8Stomee } else { 1677b5fca8f8Stomee ASSERT(KMEM_SLAB_IS_PARTIAL(sp)); 1678b5fca8f8Stomee if (sp->slab_refcnt == 1) { 1679b5fca8f8Stomee avl_add(&cp->cache_partial_slabs, sp); 1680b5fca8f8Stomee } else { 1681b5fca8f8Stomee /* 1682b5fca8f8Stomee * The slab is now more allocated than it was, so the 1683b5fca8f8Stomee * order remains unchanged. 1684b5fca8f8Stomee */ 1685b5fca8f8Stomee ASSERT(!avl_update(&cp->cache_partial_slabs, sp)); 1686b5fca8f8Stomee } 16877c478bd9Sstevel@tonic-gate } 16887c478bd9Sstevel@tonic-gate 16897c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) { 16907c478bd9Sstevel@tonic-gate /* 16917c478bd9Sstevel@tonic-gate * Add buffer to allocated-address hash table. 16927c478bd9Sstevel@tonic-gate */ 16937c478bd9Sstevel@tonic-gate buf = bcp->bc_addr; 16947c478bd9Sstevel@tonic-gate hash_bucket = KMEM_HASH(cp, buf); 16957c478bd9Sstevel@tonic-gate bcp->bc_next = *hash_bucket; 16967c478bd9Sstevel@tonic-gate *hash_bucket = bcp; 16977c478bd9Sstevel@tonic-gate if ((cp->cache_flags & (KMF_AUDIT | KMF_BUFTAG)) == KMF_AUDIT) { 16987c478bd9Sstevel@tonic-gate KMEM_AUDIT(kmem_transaction_log, cp, bcp); 16997c478bd9Sstevel@tonic-gate } 17007c478bd9Sstevel@tonic-gate } else { 17017c478bd9Sstevel@tonic-gate buf = KMEM_BUF(cp, bcp); 17027c478bd9Sstevel@tonic-gate } 17037c478bd9Sstevel@tonic-gate 17047c478bd9Sstevel@tonic-gate ASSERT(KMEM_SLAB_MEMBER(sp, buf)); 1705b5fca8f8Stomee return (buf); 1706b5fca8f8Stomee } 1707b5fca8f8Stomee 1708b5fca8f8Stomee /* 1709b5fca8f8Stomee * Allocate a raw (unconstructed) buffer from cp's slab layer. 1710b5fca8f8Stomee */ 1711b5fca8f8Stomee static void * 1712b5fca8f8Stomee kmem_slab_alloc(kmem_cache_t *cp, int kmflag) 1713b5fca8f8Stomee { 1714b5fca8f8Stomee kmem_slab_t *sp; 1715b5fca8f8Stomee void *buf; 17164d4c4c43STom Erickson boolean_t test_destructor; 1717b5fca8f8Stomee 1718b5fca8f8Stomee mutex_enter(&cp->cache_lock); 17194d4c4c43STom Erickson test_destructor = (cp->cache_slab_alloc == 0); 1720b5fca8f8Stomee sp = avl_first(&cp->cache_partial_slabs); 1721b5fca8f8Stomee if (sp == NULL) { 1722b5fca8f8Stomee ASSERT(cp->cache_bufslab == 0); 1723b5fca8f8Stomee 1724b5fca8f8Stomee /* 1725b5fca8f8Stomee * The freelist is empty. Create a new slab. 1726b5fca8f8Stomee */ 1727b5fca8f8Stomee mutex_exit(&cp->cache_lock); 1728b5fca8f8Stomee if ((sp = kmem_slab_create(cp, kmflag)) == NULL) { 1729b5fca8f8Stomee return (NULL); 1730b5fca8f8Stomee } 1731b5fca8f8Stomee mutex_enter(&cp->cache_lock); 1732b5fca8f8Stomee cp->cache_slab_create++; 1733b5fca8f8Stomee if ((cp->cache_buftotal += sp->slab_chunks) > cp->cache_bufmax) 1734b5fca8f8Stomee cp->cache_bufmax = cp->cache_buftotal; 1735b5fca8f8Stomee cp->cache_bufslab += sp->slab_chunks; 1736b5fca8f8Stomee } 17377c478bd9Sstevel@tonic-gate 1738b5fca8f8Stomee buf = kmem_slab_alloc_impl(cp, sp); 1739b5fca8f8Stomee ASSERT((cp->cache_slab_create - cp->cache_slab_destroy) == 1740b5fca8f8Stomee (cp->cache_complete_slab_count + 1741b5fca8f8Stomee avl_numnodes(&cp->cache_partial_slabs) + 1742b5fca8f8Stomee (cp->cache_defrag == NULL ? 0 : cp->cache_defrag->kmd_deadcount))); 17437c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 17447c478bd9Sstevel@tonic-gate 17454d4c4c43STom Erickson if (test_destructor && cp->cache_destructor != NULL) { 17464d4c4c43STom Erickson /* 17474d4c4c43STom Erickson * On the first kmem_slab_alloc(), assert that it is valid to 17484d4c4c43STom Erickson * call the destructor on a newly constructed object without any 17494d4c4c43STom Erickson * client involvement. 17504d4c4c43STom Erickson */ 17514d4c4c43STom Erickson if ((cp->cache_constructor == NULL) || 17524d4c4c43STom Erickson cp->cache_constructor(buf, cp->cache_private, 17534d4c4c43STom Erickson kmflag) == 0) { 17544d4c4c43STom Erickson cp->cache_destructor(buf, cp->cache_private); 17554d4c4c43STom Erickson } 17564d4c4c43STom Erickson copy_pattern(KMEM_UNINITIALIZED_PATTERN, buf, 17574d4c4c43STom Erickson cp->cache_bufsize); 17584d4c4c43STom Erickson if (cp->cache_flags & KMF_DEADBEEF) { 17594d4c4c43STom Erickson copy_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify); 17604d4c4c43STom Erickson } 17614d4c4c43STom Erickson } 17624d4c4c43STom Erickson 17637c478bd9Sstevel@tonic-gate return (buf); 17647c478bd9Sstevel@tonic-gate } 17657c478bd9Sstevel@tonic-gate 1766b5fca8f8Stomee static void kmem_slab_move_yes(kmem_cache_t *, kmem_slab_t *, void *); 1767b5fca8f8Stomee 17687c478bd9Sstevel@tonic-gate /* 17697c478bd9Sstevel@tonic-gate * Free a raw (unconstructed) buffer to cp's slab layer. 17707c478bd9Sstevel@tonic-gate */ 17717c478bd9Sstevel@tonic-gate static void 17727c478bd9Sstevel@tonic-gate kmem_slab_free(kmem_cache_t *cp, void *buf) 17737c478bd9Sstevel@tonic-gate { 17747c478bd9Sstevel@tonic-gate kmem_slab_t *sp; 17757c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp, **prev_bcpp; 17767c478bd9Sstevel@tonic-gate 17777c478bd9Sstevel@tonic-gate ASSERT(buf != NULL); 17787c478bd9Sstevel@tonic-gate 17797c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 17807c478bd9Sstevel@tonic-gate cp->cache_slab_free++; 17817c478bd9Sstevel@tonic-gate 17827c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) { 17837c478bd9Sstevel@tonic-gate /* 17847c478bd9Sstevel@tonic-gate * Look up buffer in allocated-address hash table. 17857c478bd9Sstevel@tonic-gate */ 17867c478bd9Sstevel@tonic-gate prev_bcpp = KMEM_HASH(cp, buf); 17877c478bd9Sstevel@tonic-gate while ((bcp = *prev_bcpp) != NULL) { 17887c478bd9Sstevel@tonic-gate if (bcp->bc_addr == buf) { 17897c478bd9Sstevel@tonic-gate *prev_bcpp = bcp->bc_next; 17907c478bd9Sstevel@tonic-gate sp = bcp->bc_slab; 17917c478bd9Sstevel@tonic-gate break; 17927c478bd9Sstevel@tonic-gate } 17937c478bd9Sstevel@tonic-gate cp->cache_lookup_depth++; 17947c478bd9Sstevel@tonic-gate prev_bcpp = &bcp->bc_next; 17957c478bd9Sstevel@tonic-gate } 17967c478bd9Sstevel@tonic-gate } else { 17977c478bd9Sstevel@tonic-gate bcp = KMEM_BUFCTL(cp, buf); 17987c478bd9Sstevel@tonic-gate sp = KMEM_SLAB(cp, buf); 17997c478bd9Sstevel@tonic-gate } 18007c478bd9Sstevel@tonic-gate 18017c478bd9Sstevel@tonic-gate if (bcp == NULL || sp->slab_cache != cp || !KMEM_SLAB_MEMBER(sp, buf)) { 18027c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 18037c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADADDR, cp, buf); 18047c478bd9Sstevel@tonic-gate return; 18057c478bd9Sstevel@tonic-gate } 18067c478bd9Sstevel@tonic-gate 1807b5fca8f8Stomee if (KMEM_SLAB_OFFSET(sp, buf) == sp->slab_stuck_offset) { 1808b5fca8f8Stomee /* 1809b5fca8f8Stomee * If this is the buffer that prevented the consolidator from 1810b5fca8f8Stomee * clearing the slab, we can reset the slab flags now that the 1811b5fca8f8Stomee * buffer is freed. (It makes sense to do this in 1812b5fca8f8Stomee * kmem_cache_free(), where the client gives up ownership of the 1813b5fca8f8Stomee * buffer, but on the hot path the test is too expensive.) 1814b5fca8f8Stomee */ 1815b5fca8f8Stomee kmem_slab_move_yes(cp, sp, buf); 1816b5fca8f8Stomee } 1817b5fca8f8Stomee 18187c478bd9Sstevel@tonic-gate if ((cp->cache_flags & (KMF_AUDIT | KMF_BUFTAG)) == KMF_AUDIT) { 18197c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_CONTENTS) 18207c478bd9Sstevel@tonic-gate ((kmem_bufctl_audit_t *)bcp)->bc_contents = 18217c478bd9Sstevel@tonic-gate kmem_log_enter(kmem_content_log, buf, 18229f1b636aStomee cp->cache_contents); 18237c478bd9Sstevel@tonic-gate KMEM_AUDIT(kmem_transaction_log, cp, bcp); 18247c478bd9Sstevel@tonic-gate } 18257c478bd9Sstevel@tonic-gate 18267c478bd9Sstevel@tonic-gate bcp->bc_next = sp->slab_head; 18277c478bd9Sstevel@tonic-gate sp->slab_head = bcp; 18287c478bd9Sstevel@tonic-gate 18299f1b636aStomee cp->cache_bufslab++; 18307c478bd9Sstevel@tonic-gate ASSERT(sp->slab_refcnt >= 1); 1831b5fca8f8Stomee 18327c478bd9Sstevel@tonic-gate if (--sp->slab_refcnt == 0) { 18337c478bd9Sstevel@tonic-gate /* 18347c478bd9Sstevel@tonic-gate * There are no outstanding allocations from this slab, 18357c478bd9Sstevel@tonic-gate * so we can reclaim the memory. 18367c478bd9Sstevel@tonic-gate */ 1837b5fca8f8Stomee if (sp->slab_chunks == 1) { 1838b5fca8f8Stomee list_remove(&cp->cache_complete_slabs, sp); 1839b5fca8f8Stomee cp->cache_complete_slab_count--; 1840b5fca8f8Stomee } else { 1841b5fca8f8Stomee avl_remove(&cp->cache_partial_slabs, sp); 1842b5fca8f8Stomee } 1843b5fca8f8Stomee 18447c478bd9Sstevel@tonic-gate cp->cache_buftotal -= sp->slab_chunks; 18459f1b636aStomee cp->cache_bufslab -= sp->slab_chunks; 1846b5fca8f8Stomee /* 1847b5fca8f8Stomee * Defer releasing the slab to the virtual memory subsystem 1848b5fca8f8Stomee * while there is a pending move callback, since we guarantee 1849b5fca8f8Stomee * that buffers passed to the move callback have only been 1850b5fca8f8Stomee * touched by kmem or by the client itself. Since the memory 1851b5fca8f8Stomee * patterns baddcafe (uninitialized) and deadbeef (freed) both 1852b5fca8f8Stomee * set at least one of the two lowest order bits, the client can 1853b5fca8f8Stomee * test those bits in the move callback to determine whether or 1854b5fca8f8Stomee * not it knows about the buffer (assuming that the client also 1855b5fca8f8Stomee * sets one of those low order bits whenever it frees a buffer). 1856b5fca8f8Stomee */ 1857b5fca8f8Stomee if (cp->cache_defrag == NULL || 1858b5fca8f8Stomee (avl_is_empty(&cp->cache_defrag->kmd_moves_pending) && 1859b5fca8f8Stomee !(sp->slab_flags & KMEM_SLAB_MOVE_PENDING))) { 1860b5fca8f8Stomee cp->cache_slab_destroy++; 1861b5fca8f8Stomee mutex_exit(&cp->cache_lock); 1862b5fca8f8Stomee kmem_slab_destroy(cp, sp); 1863b5fca8f8Stomee } else { 1864b5fca8f8Stomee list_t *deadlist = &cp->cache_defrag->kmd_deadlist; 1865b5fca8f8Stomee /* 1866b5fca8f8Stomee * Slabs are inserted at both ends of the deadlist to 1867b5fca8f8Stomee * distinguish between slabs freed while move callbacks 1868b5fca8f8Stomee * are pending (list head) and a slab freed while the 1869b5fca8f8Stomee * lock is dropped in kmem_move_buffers() (list tail) so 1870b5fca8f8Stomee * that in both cases slab_destroy() is called from the 1871b5fca8f8Stomee * right context. 1872b5fca8f8Stomee */ 1873b5fca8f8Stomee if (sp->slab_flags & KMEM_SLAB_MOVE_PENDING) { 1874b5fca8f8Stomee list_insert_tail(deadlist, sp); 1875b5fca8f8Stomee } else { 1876b5fca8f8Stomee list_insert_head(deadlist, sp); 1877b5fca8f8Stomee } 1878b5fca8f8Stomee cp->cache_defrag->kmd_deadcount++; 1879b5fca8f8Stomee mutex_exit(&cp->cache_lock); 1880b5fca8f8Stomee } 18817c478bd9Sstevel@tonic-gate return; 18827c478bd9Sstevel@tonic-gate } 1883b5fca8f8Stomee 1884b5fca8f8Stomee if (bcp->bc_next == NULL) { 1885b5fca8f8Stomee /* Transition the slab from completely allocated to partial. */ 1886b5fca8f8Stomee ASSERT(sp->slab_refcnt == (sp->slab_chunks - 1)); 1887b5fca8f8Stomee ASSERT(sp->slab_chunks > 1); 1888b5fca8f8Stomee list_remove(&cp->cache_complete_slabs, sp); 1889b5fca8f8Stomee cp->cache_complete_slab_count--; 1890b5fca8f8Stomee avl_add(&cp->cache_partial_slabs, sp); 1891b5fca8f8Stomee } else { 1892b5fca8f8Stomee #ifdef DEBUG 1893b5fca8f8Stomee if (avl_update_gt(&cp->cache_partial_slabs, sp)) { 1894b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats.kms_avl_update); 1895b5fca8f8Stomee } else { 1896b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats.kms_avl_noupdate); 1897b5fca8f8Stomee } 1898b5fca8f8Stomee #else 1899b5fca8f8Stomee (void) avl_update_gt(&cp->cache_partial_slabs, sp); 1900b5fca8f8Stomee #endif 1901b5fca8f8Stomee } 1902b5fca8f8Stomee 1903b5fca8f8Stomee ASSERT((cp->cache_slab_create - cp->cache_slab_destroy) == 1904b5fca8f8Stomee (cp->cache_complete_slab_count + 1905b5fca8f8Stomee avl_numnodes(&cp->cache_partial_slabs) + 1906b5fca8f8Stomee (cp->cache_defrag == NULL ? 0 : cp->cache_defrag->kmd_deadcount))); 19077c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 19087c478bd9Sstevel@tonic-gate } 19097c478bd9Sstevel@tonic-gate 1910b5fca8f8Stomee /* 1911b5fca8f8Stomee * Return -1 if kmem_error, 1 if constructor fails, 0 if successful. 1912b5fca8f8Stomee */ 19137c478bd9Sstevel@tonic-gate static int 19147c478bd9Sstevel@tonic-gate kmem_cache_alloc_debug(kmem_cache_t *cp, void *buf, int kmflag, int construct, 19157c478bd9Sstevel@tonic-gate caddr_t caller) 19167c478bd9Sstevel@tonic-gate { 19177c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 19187c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *bcp = (kmem_bufctl_audit_t *)btp->bt_bufctl; 19197c478bd9Sstevel@tonic-gate uint32_t mtbf; 19207c478bd9Sstevel@tonic-gate 19217c478bd9Sstevel@tonic-gate if (btp->bt_bxstat != ((intptr_t)bcp ^ KMEM_BUFTAG_FREE)) { 19227c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADBUFTAG, cp, buf); 19237c478bd9Sstevel@tonic-gate return (-1); 19247c478bd9Sstevel@tonic-gate } 19257c478bd9Sstevel@tonic-gate 19267c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_ALLOC; 19277c478bd9Sstevel@tonic-gate 19287c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_HASH) && bcp->bc_addr != buf) { 19297c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADBUFCTL, cp, buf); 19307c478bd9Sstevel@tonic-gate return (-1); 19317c478bd9Sstevel@tonic-gate } 19327c478bd9Sstevel@tonic-gate 19337c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) { 19347c478bd9Sstevel@tonic-gate if (!construct && (cp->cache_flags & KMF_LITE)) { 19357c478bd9Sstevel@tonic-gate if (*(uint64_t *)buf != KMEM_FREE_PATTERN) { 19367c478bd9Sstevel@tonic-gate kmem_error(KMERR_MODIFIED, cp, buf); 19377c478bd9Sstevel@tonic-gate return (-1); 19387c478bd9Sstevel@tonic-gate } 19397c478bd9Sstevel@tonic-gate if (cp->cache_constructor != NULL) 19407c478bd9Sstevel@tonic-gate *(uint64_t *)buf = btp->bt_redzone; 19417c478bd9Sstevel@tonic-gate else 19427c478bd9Sstevel@tonic-gate *(uint64_t *)buf = KMEM_UNINITIALIZED_PATTERN; 19437c478bd9Sstevel@tonic-gate } else { 19447c478bd9Sstevel@tonic-gate construct = 1; 19457c478bd9Sstevel@tonic-gate if (verify_and_copy_pattern(KMEM_FREE_PATTERN, 19467c478bd9Sstevel@tonic-gate KMEM_UNINITIALIZED_PATTERN, buf, 19477c478bd9Sstevel@tonic-gate cp->cache_verify)) { 19487c478bd9Sstevel@tonic-gate kmem_error(KMERR_MODIFIED, cp, buf); 19497c478bd9Sstevel@tonic-gate return (-1); 19507c478bd9Sstevel@tonic-gate } 19517c478bd9Sstevel@tonic-gate } 19527c478bd9Sstevel@tonic-gate } 19537c478bd9Sstevel@tonic-gate btp->bt_redzone = KMEM_REDZONE_PATTERN; 19547c478bd9Sstevel@tonic-gate 19557c478bd9Sstevel@tonic-gate if ((mtbf = kmem_mtbf | cp->cache_mtbf) != 0 && 19567c478bd9Sstevel@tonic-gate gethrtime() % mtbf == 0 && 19577c478bd9Sstevel@tonic-gate (kmflag & (KM_NOSLEEP | KM_PANIC)) == KM_NOSLEEP) { 19587c478bd9Sstevel@tonic-gate kmem_log_event(kmem_failure_log, cp, NULL, NULL); 19597c478bd9Sstevel@tonic-gate if (!construct && cp->cache_destructor != NULL) 19607c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 19617c478bd9Sstevel@tonic-gate } else { 19627c478bd9Sstevel@tonic-gate mtbf = 0; 19637c478bd9Sstevel@tonic-gate } 19647c478bd9Sstevel@tonic-gate 19657c478bd9Sstevel@tonic-gate if (mtbf || (construct && cp->cache_constructor != NULL && 19667c478bd9Sstevel@tonic-gate cp->cache_constructor(buf, cp->cache_private, kmflag) != 0)) { 19677c478bd9Sstevel@tonic-gate atomic_add_64(&cp->cache_alloc_fail, 1); 19687c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE; 19697c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) 19707c478bd9Sstevel@tonic-gate copy_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify); 19717c478bd9Sstevel@tonic-gate kmem_slab_free(cp, buf); 1972b5fca8f8Stomee return (1); 19737c478bd9Sstevel@tonic-gate } 19747c478bd9Sstevel@tonic-gate 19757c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_AUDIT) { 19767c478bd9Sstevel@tonic-gate KMEM_AUDIT(kmem_transaction_log, cp, bcp); 19777c478bd9Sstevel@tonic-gate } 19787c478bd9Sstevel@tonic-gate 19797c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_LITE) && 19807c478bd9Sstevel@tonic-gate !(cp->cache_cflags & KMC_KMEM_ALLOC)) { 19817c478bd9Sstevel@tonic-gate KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, caller); 19827c478bd9Sstevel@tonic-gate } 19837c478bd9Sstevel@tonic-gate 19847c478bd9Sstevel@tonic-gate return (0); 19857c478bd9Sstevel@tonic-gate } 19867c478bd9Sstevel@tonic-gate 19877c478bd9Sstevel@tonic-gate static int 19887c478bd9Sstevel@tonic-gate kmem_cache_free_debug(kmem_cache_t *cp, void *buf, caddr_t caller) 19897c478bd9Sstevel@tonic-gate { 19907c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 19917c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *bcp = (kmem_bufctl_audit_t *)btp->bt_bufctl; 19927c478bd9Sstevel@tonic-gate kmem_slab_t *sp; 19937c478bd9Sstevel@tonic-gate 19947c478bd9Sstevel@tonic-gate if (btp->bt_bxstat != ((intptr_t)bcp ^ KMEM_BUFTAG_ALLOC)) { 19957c478bd9Sstevel@tonic-gate if (btp->bt_bxstat == ((intptr_t)bcp ^ KMEM_BUFTAG_FREE)) { 19967c478bd9Sstevel@tonic-gate kmem_error(KMERR_DUPFREE, cp, buf); 19977c478bd9Sstevel@tonic-gate return (-1); 19987c478bd9Sstevel@tonic-gate } 19997c478bd9Sstevel@tonic-gate sp = kmem_findslab(cp, buf); 20007c478bd9Sstevel@tonic-gate if (sp == NULL || sp->slab_cache != cp) 20017c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADADDR, cp, buf); 20027c478bd9Sstevel@tonic-gate else 20037c478bd9Sstevel@tonic-gate kmem_error(KMERR_REDZONE, cp, buf); 20047c478bd9Sstevel@tonic-gate return (-1); 20057c478bd9Sstevel@tonic-gate } 20067c478bd9Sstevel@tonic-gate 20077c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE; 20087c478bd9Sstevel@tonic-gate 20097c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_HASH) && bcp->bc_addr != buf) { 20107c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADBUFCTL, cp, buf); 20117c478bd9Sstevel@tonic-gate return (-1); 20127c478bd9Sstevel@tonic-gate } 20137c478bd9Sstevel@tonic-gate 20147c478bd9Sstevel@tonic-gate if (btp->bt_redzone != KMEM_REDZONE_PATTERN) { 20157c478bd9Sstevel@tonic-gate kmem_error(KMERR_REDZONE, cp, buf); 20167c478bd9Sstevel@tonic-gate return (-1); 20177c478bd9Sstevel@tonic-gate } 20187c478bd9Sstevel@tonic-gate 20197c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_AUDIT) { 20207c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_CONTENTS) 20217c478bd9Sstevel@tonic-gate bcp->bc_contents = kmem_log_enter(kmem_content_log, 20227c478bd9Sstevel@tonic-gate buf, cp->cache_contents); 20237c478bd9Sstevel@tonic-gate KMEM_AUDIT(kmem_transaction_log, cp, bcp); 20247c478bd9Sstevel@tonic-gate } 20257c478bd9Sstevel@tonic-gate 20267c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_LITE) && 20277c478bd9Sstevel@tonic-gate !(cp->cache_cflags & KMC_KMEM_ALLOC)) { 20287c478bd9Sstevel@tonic-gate KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, caller); 20297c478bd9Sstevel@tonic-gate } 20307c478bd9Sstevel@tonic-gate 20317c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) { 20327c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) 20337c478bd9Sstevel@tonic-gate btp->bt_redzone = *(uint64_t *)buf; 20347c478bd9Sstevel@tonic-gate else if (cp->cache_destructor != NULL) 20357c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 20367c478bd9Sstevel@tonic-gate 20377c478bd9Sstevel@tonic-gate copy_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify); 20387c478bd9Sstevel@tonic-gate } 20397c478bd9Sstevel@tonic-gate 20407c478bd9Sstevel@tonic-gate return (0); 20417c478bd9Sstevel@tonic-gate } 20427c478bd9Sstevel@tonic-gate 20437c478bd9Sstevel@tonic-gate /* 20447c478bd9Sstevel@tonic-gate * Free each object in magazine mp to cp's slab layer, and free mp itself. 20457c478bd9Sstevel@tonic-gate */ 20467c478bd9Sstevel@tonic-gate static void 20477c478bd9Sstevel@tonic-gate kmem_magazine_destroy(kmem_cache_t *cp, kmem_magazine_t *mp, int nrounds) 20487c478bd9Sstevel@tonic-gate { 20497c478bd9Sstevel@tonic-gate int round; 20507c478bd9Sstevel@tonic-gate 2051b5fca8f8Stomee ASSERT(!list_link_active(&cp->cache_link) || 2052b5fca8f8Stomee taskq_member(kmem_taskq, curthread)); 20537c478bd9Sstevel@tonic-gate 20547c478bd9Sstevel@tonic-gate for (round = 0; round < nrounds; round++) { 20557c478bd9Sstevel@tonic-gate void *buf = mp->mag_round[round]; 20567c478bd9Sstevel@tonic-gate 20577c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) { 20587c478bd9Sstevel@tonic-gate if (verify_pattern(KMEM_FREE_PATTERN, buf, 20597c478bd9Sstevel@tonic-gate cp->cache_verify) != NULL) { 20607c478bd9Sstevel@tonic-gate kmem_error(KMERR_MODIFIED, cp, buf); 20617c478bd9Sstevel@tonic-gate continue; 20627c478bd9Sstevel@tonic-gate } 20637c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_LITE) && 20647c478bd9Sstevel@tonic-gate cp->cache_destructor != NULL) { 20657c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 20667c478bd9Sstevel@tonic-gate *(uint64_t *)buf = btp->bt_redzone; 20677c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 20687c478bd9Sstevel@tonic-gate *(uint64_t *)buf = KMEM_FREE_PATTERN; 20697c478bd9Sstevel@tonic-gate } 20707c478bd9Sstevel@tonic-gate } else if (cp->cache_destructor != NULL) { 20717c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private); 20727c478bd9Sstevel@tonic-gate } 20737c478bd9Sstevel@tonic-gate 20747c478bd9Sstevel@tonic-gate kmem_slab_free(cp, buf); 20757c478bd9Sstevel@tonic-gate } 20767c478bd9Sstevel@tonic-gate ASSERT(KMEM_MAGAZINE_VALID(cp, mp)); 20777c478bd9Sstevel@tonic-gate kmem_cache_free(cp->cache_magtype->mt_cache, mp); 20787c478bd9Sstevel@tonic-gate } 20797c478bd9Sstevel@tonic-gate 20807c478bd9Sstevel@tonic-gate /* 20817c478bd9Sstevel@tonic-gate * Allocate a magazine from the depot. 20827c478bd9Sstevel@tonic-gate */ 20837c478bd9Sstevel@tonic-gate static kmem_magazine_t * 20847c478bd9Sstevel@tonic-gate kmem_depot_alloc(kmem_cache_t *cp, kmem_maglist_t *mlp) 20857c478bd9Sstevel@tonic-gate { 20867c478bd9Sstevel@tonic-gate kmem_magazine_t *mp; 20877c478bd9Sstevel@tonic-gate 20887c478bd9Sstevel@tonic-gate /* 20897c478bd9Sstevel@tonic-gate * If we can't get the depot lock without contention, 20907c478bd9Sstevel@tonic-gate * update our contention count. We use the depot 20917c478bd9Sstevel@tonic-gate * contention rate to determine whether we need to 20927c478bd9Sstevel@tonic-gate * increase the magazine size for better scalability. 20937c478bd9Sstevel@tonic-gate */ 20947c478bd9Sstevel@tonic-gate if (!mutex_tryenter(&cp->cache_depot_lock)) { 20957c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock); 20967c478bd9Sstevel@tonic-gate cp->cache_depot_contention++; 20977c478bd9Sstevel@tonic-gate } 20987c478bd9Sstevel@tonic-gate 20997c478bd9Sstevel@tonic-gate if ((mp = mlp->ml_list) != NULL) { 21007c478bd9Sstevel@tonic-gate ASSERT(KMEM_MAGAZINE_VALID(cp, mp)); 21017c478bd9Sstevel@tonic-gate mlp->ml_list = mp->mag_next; 21027c478bd9Sstevel@tonic-gate if (--mlp->ml_total < mlp->ml_min) 21037c478bd9Sstevel@tonic-gate mlp->ml_min = mlp->ml_total; 21047c478bd9Sstevel@tonic-gate mlp->ml_alloc++; 21057c478bd9Sstevel@tonic-gate } 21067c478bd9Sstevel@tonic-gate 21077c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock); 21087c478bd9Sstevel@tonic-gate 21097c478bd9Sstevel@tonic-gate return (mp); 21107c478bd9Sstevel@tonic-gate } 21117c478bd9Sstevel@tonic-gate 21127c478bd9Sstevel@tonic-gate /* 21137c478bd9Sstevel@tonic-gate * Free a magazine to the depot. 21147c478bd9Sstevel@tonic-gate */ 21157c478bd9Sstevel@tonic-gate static void 21167c478bd9Sstevel@tonic-gate kmem_depot_free(kmem_cache_t *cp, kmem_maglist_t *mlp, kmem_magazine_t *mp) 21177c478bd9Sstevel@tonic-gate { 21187c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock); 21197c478bd9Sstevel@tonic-gate ASSERT(KMEM_MAGAZINE_VALID(cp, mp)); 21207c478bd9Sstevel@tonic-gate mp->mag_next = mlp->ml_list; 21217c478bd9Sstevel@tonic-gate mlp->ml_list = mp; 21227c478bd9Sstevel@tonic-gate mlp->ml_total++; 21237c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock); 21247c478bd9Sstevel@tonic-gate } 21257c478bd9Sstevel@tonic-gate 21267c478bd9Sstevel@tonic-gate /* 21277c478bd9Sstevel@tonic-gate * Update the working set statistics for cp's depot. 21287c478bd9Sstevel@tonic-gate */ 21297c478bd9Sstevel@tonic-gate static void 21307c478bd9Sstevel@tonic-gate kmem_depot_ws_update(kmem_cache_t *cp) 21317c478bd9Sstevel@tonic-gate { 21327c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock); 21337c478bd9Sstevel@tonic-gate cp->cache_full.ml_reaplimit = cp->cache_full.ml_min; 21347c478bd9Sstevel@tonic-gate cp->cache_full.ml_min = cp->cache_full.ml_total; 21357c478bd9Sstevel@tonic-gate cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_min; 21367c478bd9Sstevel@tonic-gate cp->cache_empty.ml_min = cp->cache_empty.ml_total; 21377c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock); 21387c478bd9Sstevel@tonic-gate } 21397c478bd9Sstevel@tonic-gate 21407c478bd9Sstevel@tonic-gate /* 21417c478bd9Sstevel@tonic-gate * Reap all magazines that have fallen out of the depot's working set. 21427c478bd9Sstevel@tonic-gate */ 21437c478bd9Sstevel@tonic-gate static void 21447c478bd9Sstevel@tonic-gate kmem_depot_ws_reap(kmem_cache_t *cp) 21457c478bd9Sstevel@tonic-gate { 21467c478bd9Sstevel@tonic-gate long reap; 21477c478bd9Sstevel@tonic-gate kmem_magazine_t *mp; 21487c478bd9Sstevel@tonic-gate 2149b5fca8f8Stomee ASSERT(!list_link_active(&cp->cache_link) || 2150b5fca8f8Stomee taskq_member(kmem_taskq, curthread)); 21517c478bd9Sstevel@tonic-gate 21527c478bd9Sstevel@tonic-gate reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min); 21537c478bd9Sstevel@tonic-gate while (reap-- && (mp = kmem_depot_alloc(cp, &cp->cache_full)) != NULL) 21547c478bd9Sstevel@tonic-gate kmem_magazine_destroy(cp, mp, cp->cache_magtype->mt_magsize); 21557c478bd9Sstevel@tonic-gate 21567c478bd9Sstevel@tonic-gate reap = MIN(cp->cache_empty.ml_reaplimit, cp->cache_empty.ml_min); 21577c478bd9Sstevel@tonic-gate while (reap-- && (mp = kmem_depot_alloc(cp, &cp->cache_empty)) != NULL) 21587c478bd9Sstevel@tonic-gate kmem_magazine_destroy(cp, mp, 0); 21597c478bd9Sstevel@tonic-gate } 21607c478bd9Sstevel@tonic-gate 21617c478bd9Sstevel@tonic-gate static void 21627c478bd9Sstevel@tonic-gate kmem_cpu_reload(kmem_cpu_cache_t *ccp, kmem_magazine_t *mp, int rounds) 21637c478bd9Sstevel@tonic-gate { 21647c478bd9Sstevel@tonic-gate ASSERT((ccp->cc_loaded == NULL && ccp->cc_rounds == -1) || 21657c478bd9Sstevel@tonic-gate (ccp->cc_loaded && ccp->cc_rounds + rounds == ccp->cc_magsize)); 21667c478bd9Sstevel@tonic-gate ASSERT(ccp->cc_magsize > 0); 21677c478bd9Sstevel@tonic-gate 21687c478bd9Sstevel@tonic-gate ccp->cc_ploaded = ccp->cc_loaded; 21697c478bd9Sstevel@tonic-gate ccp->cc_prounds = ccp->cc_rounds; 21707c478bd9Sstevel@tonic-gate ccp->cc_loaded = mp; 21717c478bd9Sstevel@tonic-gate ccp->cc_rounds = rounds; 21727c478bd9Sstevel@tonic-gate } 21737c478bd9Sstevel@tonic-gate 21747c478bd9Sstevel@tonic-gate /* 21757c478bd9Sstevel@tonic-gate * Allocate a constructed object from cache cp. 21767c478bd9Sstevel@tonic-gate */ 21777c478bd9Sstevel@tonic-gate void * 21787c478bd9Sstevel@tonic-gate kmem_cache_alloc(kmem_cache_t *cp, int kmflag) 21797c478bd9Sstevel@tonic-gate { 21807c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp = KMEM_CPU_CACHE(cp); 21817c478bd9Sstevel@tonic-gate kmem_magazine_t *fmp; 21827c478bd9Sstevel@tonic-gate void *buf; 21837c478bd9Sstevel@tonic-gate 21847c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 21857c478bd9Sstevel@tonic-gate for (;;) { 21867c478bd9Sstevel@tonic-gate /* 21877c478bd9Sstevel@tonic-gate * If there's an object available in the current CPU's 21887c478bd9Sstevel@tonic-gate * loaded magazine, just take it and return. 21897c478bd9Sstevel@tonic-gate */ 21907c478bd9Sstevel@tonic-gate if (ccp->cc_rounds > 0) { 21917c478bd9Sstevel@tonic-gate buf = ccp->cc_loaded->mag_round[--ccp->cc_rounds]; 21927c478bd9Sstevel@tonic-gate ccp->cc_alloc++; 21937c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 21947c478bd9Sstevel@tonic-gate if ((ccp->cc_flags & KMF_BUFTAG) && 21957c478bd9Sstevel@tonic-gate kmem_cache_alloc_debug(cp, buf, kmflag, 0, 2196b5fca8f8Stomee caller()) != 0) { 21977c478bd9Sstevel@tonic-gate if (kmflag & KM_NOSLEEP) 21987c478bd9Sstevel@tonic-gate return (NULL); 21997c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 22007c478bd9Sstevel@tonic-gate continue; 22017c478bd9Sstevel@tonic-gate } 22027c478bd9Sstevel@tonic-gate return (buf); 22037c478bd9Sstevel@tonic-gate } 22047c478bd9Sstevel@tonic-gate 22057c478bd9Sstevel@tonic-gate /* 22067c478bd9Sstevel@tonic-gate * The loaded magazine is empty. If the previously loaded 22077c478bd9Sstevel@tonic-gate * magazine was full, exchange them and try again. 22087c478bd9Sstevel@tonic-gate */ 22097c478bd9Sstevel@tonic-gate if (ccp->cc_prounds > 0) { 22107c478bd9Sstevel@tonic-gate kmem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds); 22117c478bd9Sstevel@tonic-gate continue; 22127c478bd9Sstevel@tonic-gate } 22137c478bd9Sstevel@tonic-gate 22147c478bd9Sstevel@tonic-gate /* 22157c478bd9Sstevel@tonic-gate * If the magazine layer is disabled, break out now. 22167c478bd9Sstevel@tonic-gate */ 22177c478bd9Sstevel@tonic-gate if (ccp->cc_magsize == 0) 22187c478bd9Sstevel@tonic-gate break; 22197c478bd9Sstevel@tonic-gate 22207c478bd9Sstevel@tonic-gate /* 22217c478bd9Sstevel@tonic-gate * Try to get a full magazine from the depot. 22227c478bd9Sstevel@tonic-gate */ 22237c478bd9Sstevel@tonic-gate fmp = kmem_depot_alloc(cp, &cp->cache_full); 22247c478bd9Sstevel@tonic-gate if (fmp != NULL) { 22257c478bd9Sstevel@tonic-gate if (ccp->cc_ploaded != NULL) 22267c478bd9Sstevel@tonic-gate kmem_depot_free(cp, &cp->cache_empty, 22277c478bd9Sstevel@tonic-gate ccp->cc_ploaded); 22287c478bd9Sstevel@tonic-gate kmem_cpu_reload(ccp, fmp, ccp->cc_magsize); 22297c478bd9Sstevel@tonic-gate continue; 22307c478bd9Sstevel@tonic-gate } 22317c478bd9Sstevel@tonic-gate 22327c478bd9Sstevel@tonic-gate /* 22337c478bd9Sstevel@tonic-gate * There are no full magazines in the depot, 22347c478bd9Sstevel@tonic-gate * so fall through to the slab layer. 22357c478bd9Sstevel@tonic-gate */ 22367c478bd9Sstevel@tonic-gate break; 22377c478bd9Sstevel@tonic-gate } 22387c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 22397c478bd9Sstevel@tonic-gate 22407c478bd9Sstevel@tonic-gate /* 22417c478bd9Sstevel@tonic-gate * We couldn't allocate a constructed object from the magazine layer, 22427c478bd9Sstevel@tonic-gate * so get a raw buffer from the slab layer and apply its constructor. 22437c478bd9Sstevel@tonic-gate */ 22447c478bd9Sstevel@tonic-gate buf = kmem_slab_alloc(cp, kmflag); 22457c478bd9Sstevel@tonic-gate 22467c478bd9Sstevel@tonic-gate if (buf == NULL) 22477c478bd9Sstevel@tonic-gate return (NULL); 22487c478bd9Sstevel@tonic-gate 22497c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_BUFTAG) { 22507c478bd9Sstevel@tonic-gate /* 22517c478bd9Sstevel@tonic-gate * Make kmem_cache_alloc_debug() apply the constructor for us. 22527c478bd9Sstevel@tonic-gate */ 2253b5fca8f8Stomee int rc = kmem_cache_alloc_debug(cp, buf, kmflag, 1, caller()); 2254b5fca8f8Stomee if (rc != 0) { 22557c478bd9Sstevel@tonic-gate if (kmflag & KM_NOSLEEP) 22567c478bd9Sstevel@tonic-gate return (NULL); 22577c478bd9Sstevel@tonic-gate /* 22587c478bd9Sstevel@tonic-gate * kmem_cache_alloc_debug() detected corruption 2259b5fca8f8Stomee * but didn't panic (kmem_panic <= 0). We should not be 2260b5fca8f8Stomee * here because the constructor failed (indicated by a 2261b5fca8f8Stomee * return code of 1). Try again. 22627c478bd9Sstevel@tonic-gate */ 2263b5fca8f8Stomee ASSERT(rc == -1); 22647c478bd9Sstevel@tonic-gate return (kmem_cache_alloc(cp, kmflag)); 22657c478bd9Sstevel@tonic-gate } 22667c478bd9Sstevel@tonic-gate return (buf); 22677c478bd9Sstevel@tonic-gate } 22687c478bd9Sstevel@tonic-gate 22697c478bd9Sstevel@tonic-gate if (cp->cache_constructor != NULL && 22707c478bd9Sstevel@tonic-gate cp->cache_constructor(buf, cp->cache_private, kmflag) != 0) { 22717c478bd9Sstevel@tonic-gate atomic_add_64(&cp->cache_alloc_fail, 1); 22727c478bd9Sstevel@tonic-gate kmem_slab_free(cp, buf); 22737c478bd9Sstevel@tonic-gate return (NULL); 22747c478bd9Sstevel@tonic-gate } 22757c478bd9Sstevel@tonic-gate 22767c478bd9Sstevel@tonic-gate return (buf); 22777c478bd9Sstevel@tonic-gate } 22787c478bd9Sstevel@tonic-gate 22797c478bd9Sstevel@tonic-gate /* 2280b5fca8f8Stomee * The freed argument tells whether or not kmem_cache_free_debug() has already 2281b5fca8f8Stomee * been called so that we can avoid the duplicate free error. For example, a 2282b5fca8f8Stomee * buffer on a magazine has already been freed by the client but is still 2283b5fca8f8Stomee * constructed. 22847c478bd9Sstevel@tonic-gate */ 2285b5fca8f8Stomee static void 2286b5fca8f8Stomee kmem_slab_free_constructed(kmem_cache_t *cp, void *buf, boolean_t freed) 22877c478bd9Sstevel@tonic-gate { 2288b5fca8f8Stomee if (!freed && (cp->cache_flags & KMF_BUFTAG)) 22897c478bd9Sstevel@tonic-gate if (kmem_cache_free_debug(cp, buf, caller()) == -1) 22907c478bd9Sstevel@tonic-gate return; 22917c478bd9Sstevel@tonic-gate 2292b5fca8f8Stomee /* 2293b5fca8f8Stomee * Note that if KMF_DEADBEEF is in effect and KMF_LITE is not, 2294b5fca8f8Stomee * kmem_cache_free_debug() will have already applied the destructor. 2295b5fca8f8Stomee */ 2296b5fca8f8Stomee if ((cp->cache_flags & (KMF_DEADBEEF | KMF_LITE)) != KMF_DEADBEEF && 2297b5fca8f8Stomee cp->cache_destructor != NULL) { 2298b5fca8f8Stomee if (cp->cache_flags & KMF_DEADBEEF) { /* KMF_LITE implied */ 2299b5fca8f8Stomee kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 2300b5fca8f8Stomee *(uint64_t *)buf = btp->bt_redzone; 2301b5fca8f8Stomee cp->cache_destructor(buf, cp->cache_private); 2302b5fca8f8Stomee *(uint64_t *)buf = KMEM_FREE_PATTERN; 2303b5fca8f8Stomee } else { 2304b5fca8f8Stomee cp->cache_destructor(buf, cp->cache_private); 2305b5fca8f8Stomee } 2306b5fca8f8Stomee } 2307b5fca8f8Stomee 2308b5fca8f8Stomee kmem_slab_free(cp, buf); 2309b5fca8f8Stomee } 2310b5fca8f8Stomee 2311b5fca8f8Stomee /* 2312b5fca8f8Stomee * Free a constructed object to cache cp. 2313b5fca8f8Stomee */ 2314b5fca8f8Stomee void 2315b5fca8f8Stomee kmem_cache_free(kmem_cache_t *cp, void *buf) 2316b5fca8f8Stomee { 2317b5fca8f8Stomee kmem_cpu_cache_t *ccp = KMEM_CPU_CACHE(cp); 2318b5fca8f8Stomee kmem_magazine_t *emp; 2319b5fca8f8Stomee kmem_magtype_t *mtp; 2320b5fca8f8Stomee 2321b5fca8f8Stomee /* 2322b5fca8f8Stomee * The client must not free either of the buffers passed to the move 2323b5fca8f8Stomee * callback function. 2324b5fca8f8Stomee */ 2325b5fca8f8Stomee ASSERT(cp->cache_defrag == NULL || 2326b5fca8f8Stomee cp->cache_defrag->kmd_thread != curthread || 2327b5fca8f8Stomee (buf != cp->cache_defrag->kmd_from_buf && 2328b5fca8f8Stomee buf != cp->cache_defrag->kmd_to_buf)); 2329b5fca8f8Stomee 2330b5fca8f8Stomee if (ccp->cc_flags & KMF_BUFTAG) 2331b5fca8f8Stomee if (kmem_cache_free_debug(cp, buf, caller()) == -1) 2332b5fca8f8Stomee return; 2333b5fca8f8Stomee 2334b5fca8f8Stomee mutex_enter(&ccp->cc_lock); 2335b5fca8f8Stomee for (;;) { 2336b5fca8f8Stomee /* 2337b5fca8f8Stomee * If there's a slot available in the current CPU's 2338b5fca8f8Stomee * loaded magazine, just put the object there and return. 2339b5fca8f8Stomee */ 2340b5fca8f8Stomee if ((uint_t)ccp->cc_rounds < ccp->cc_magsize) { 2341b5fca8f8Stomee ccp->cc_loaded->mag_round[ccp->cc_rounds++] = buf; 2342b5fca8f8Stomee ccp->cc_free++; 2343b5fca8f8Stomee mutex_exit(&ccp->cc_lock); 2344b5fca8f8Stomee return; 2345b5fca8f8Stomee } 2346b5fca8f8Stomee 23477c478bd9Sstevel@tonic-gate /* 23487c478bd9Sstevel@tonic-gate * The loaded magazine is full. If the previously loaded 23497c478bd9Sstevel@tonic-gate * magazine was empty, exchange them and try again. 23507c478bd9Sstevel@tonic-gate */ 23517c478bd9Sstevel@tonic-gate if (ccp->cc_prounds == 0) { 23527c478bd9Sstevel@tonic-gate kmem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds); 23537c478bd9Sstevel@tonic-gate continue; 23547c478bd9Sstevel@tonic-gate } 23557c478bd9Sstevel@tonic-gate 23567c478bd9Sstevel@tonic-gate /* 23577c478bd9Sstevel@tonic-gate * If the magazine layer is disabled, break out now. 23587c478bd9Sstevel@tonic-gate */ 23597c478bd9Sstevel@tonic-gate if (ccp->cc_magsize == 0) 23607c478bd9Sstevel@tonic-gate break; 23617c478bd9Sstevel@tonic-gate 23627c478bd9Sstevel@tonic-gate /* 23637c478bd9Sstevel@tonic-gate * Try to get an empty magazine from the depot. 23647c478bd9Sstevel@tonic-gate */ 23657c478bd9Sstevel@tonic-gate emp = kmem_depot_alloc(cp, &cp->cache_empty); 23667c478bd9Sstevel@tonic-gate if (emp != NULL) { 23677c478bd9Sstevel@tonic-gate if (ccp->cc_ploaded != NULL) 23687c478bd9Sstevel@tonic-gate kmem_depot_free(cp, &cp->cache_full, 23697c478bd9Sstevel@tonic-gate ccp->cc_ploaded); 23707c478bd9Sstevel@tonic-gate kmem_cpu_reload(ccp, emp, 0); 23717c478bd9Sstevel@tonic-gate continue; 23727c478bd9Sstevel@tonic-gate } 23737c478bd9Sstevel@tonic-gate 23747c478bd9Sstevel@tonic-gate /* 23757c478bd9Sstevel@tonic-gate * There are no empty magazines in the depot, 23767c478bd9Sstevel@tonic-gate * so try to allocate a new one. We must drop all locks 23777c478bd9Sstevel@tonic-gate * across kmem_cache_alloc() because lower layers may 23787c478bd9Sstevel@tonic-gate * attempt to allocate from this cache. 23797c478bd9Sstevel@tonic-gate */ 23807c478bd9Sstevel@tonic-gate mtp = cp->cache_magtype; 23817c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 23827c478bd9Sstevel@tonic-gate emp = kmem_cache_alloc(mtp->mt_cache, KM_NOSLEEP); 23837c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 23847c478bd9Sstevel@tonic-gate 23857c478bd9Sstevel@tonic-gate if (emp != NULL) { 23867c478bd9Sstevel@tonic-gate /* 23877c478bd9Sstevel@tonic-gate * We successfully allocated an empty magazine. 23887c478bd9Sstevel@tonic-gate * However, we had to drop ccp->cc_lock to do it, 23897c478bd9Sstevel@tonic-gate * so the cache's magazine size may have changed. 23907c478bd9Sstevel@tonic-gate * If so, free the magazine and try again. 23917c478bd9Sstevel@tonic-gate */ 23927c478bd9Sstevel@tonic-gate if (ccp->cc_magsize != mtp->mt_magsize) { 23937c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 23947c478bd9Sstevel@tonic-gate kmem_cache_free(mtp->mt_cache, emp); 23957c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 23967c478bd9Sstevel@tonic-gate continue; 23977c478bd9Sstevel@tonic-gate } 23987c478bd9Sstevel@tonic-gate 23997c478bd9Sstevel@tonic-gate /* 24007c478bd9Sstevel@tonic-gate * We got a magazine of the right size. Add it to 24017c478bd9Sstevel@tonic-gate * the depot and try the whole dance again. 24027c478bd9Sstevel@tonic-gate */ 24037c478bd9Sstevel@tonic-gate kmem_depot_free(cp, &cp->cache_empty, emp); 24047c478bd9Sstevel@tonic-gate continue; 24057c478bd9Sstevel@tonic-gate } 24067c478bd9Sstevel@tonic-gate 24077c478bd9Sstevel@tonic-gate /* 24087c478bd9Sstevel@tonic-gate * We couldn't allocate an empty magazine, 24097c478bd9Sstevel@tonic-gate * so fall through to the slab layer. 24107c478bd9Sstevel@tonic-gate */ 24117c478bd9Sstevel@tonic-gate break; 24127c478bd9Sstevel@tonic-gate } 24137c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 24147c478bd9Sstevel@tonic-gate 24157c478bd9Sstevel@tonic-gate /* 24167c478bd9Sstevel@tonic-gate * We couldn't free our constructed object to the magazine layer, 24177c478bd9Sstevel@tonic-gate * so apply its destructor and free it to the slab layer. 24187c478bd9Sstevel@tonic-gate */ 2419b5fca8f8Stomee kmem_slab_free_constructed(cp, buf, B_TRUE); 24207c478bd9Sstevel@tonic-gate } 24217c478bd9Sstevel@tonic-gate 24227c478bd9Sstevel@tonic-gate void * 24237c478bd9Sstevel@tonic-gate kmem_zalloc(size_t size, int kmflag) 24247c478bd9Sstevel@tonic-gate { 2425*dce01e3fSJonathan W Adams size_t index; 24267c478bd9Sstevel@tonic-gate void *buf; 24277c478bd9Sstevel@tonic-gate 2428*dce01e3fSJonathan W Adams if ((index = ((size - 1) >> KMEM_ALIGN_SHIFT)) < KMEM_ALLOC_TABLE_MAX) { 24297c478bd9Sstevel@tonic-gate kmem_cache_t *cp = kmem_alloc_table[index]; 24307c478bd9Sstevel@tonic-gate buf = kmem_cache_alloc(cp, kmflag); 24317c478bd9Sstevel@tonic-gate if (buf != NULL) { 24327c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_BUFTAG) { 24337c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 24347c478bd9Sstevel@tonic-gate ((uint8_t *)buf)[size] = KMEM_REDZONE_BYTE; 24357c478bd9Sstevel@tonic-gate ((uint32_t *)btp)[1] = KMEM_SIZE_ENCODE(size); 24367c478bd9Sstevel@tonic-gate 24377c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) { 24387c478bd9Sstevel@tonic-gate KMEM_BUFTAG_LITE_ENTER(btp, 24397c478bd9Sstevel@tonic-gate kmem_lite_count, caller()); 24407c478bd9Sstevel@tonic-gate } 24417c478bd9Sstevel@tonic-gate } 24427c478bd9Sstevel@tonic-gate bzero(buf, size); 24437c478bd9Sstevel@tonic-gate } 24447c478bd9Sstevel@tonic-gate } else { 24457c478bd9Sstevel@tonic-gate buf = kmem_alloc(size, kmflag); 24467c478bd9Sstevel@tonic-gate if (buf != NULL) 24477c478bd9Sstevel@tonic-gate bzero(buf, size); 24487c478bd9Sstevel@tonic-gate } 24497c478bd9Sstevel@tonic-gate return (buf); 24507c478bd9Sstevel@tonic-gate } 24517c478bd9Sstevel@tonic-gate 24527c478bd9Sstevel@tonic-gate void * 24537c478bd9Sstevel@tonic-gate kmem_alloc(size_t size, int kmflag) 24547c478bd9Sstevel@tonic-gate { 2455*dce01e3fSJonathan W Adams size_t index; 2456*dce01e3fSJonathan W Adams kmem_cache_t *cp; 24577c478bd9Sstevel@tonic-gate void *buf; 24587c478bd9Sstevel@tonic-gate 2459*dce01e3fSJonathan W Adams if ((index = ((size - 1) >> KMEM_ALIGN_SHIFT)) < KMEM_ALLOC_TABLE_MAX) { 2460*dce01e3fSJonathan W Adams cp = kmem_alloc_table[index]; 2461*dce01e3fSJonathan W Adams /* fall through to kmem_cache_alloc() */ 24627c478bd9Sstevel@tonic-gate 2463*dce01e3fSJonathan W Adams } else if ((index = ((size - 1) >> KMEM_BIG_SHIFT)) < 2464*dce01e3fSJonathan W Adams kmem_big_alloc_table_max) { 2465*dce01e3fSJonathan W Adams cp = kmem_big_alloc_table[index]; 2466*dce01e3fSJonathan W Adams /* fall through to kmem_cache_alloc() */ 2467*dce01e3fSJonathan W Adams 2468*dce01e3fSJonathan W Adams } else { 2469*dce01e3fSJonathan W Adams if (size == 0) 2470*dce01e3fSJonathan W Adams return (NULL); 2471*dce01e3fSJonathan W Adams 2472*dce01e3fSJonathan W Adams buf = vmem_alloc(kmem_oversize_arena, size, 2473*dce01e3fSJonathan W Adams kmflag & KM_VMFLAGS); 2474*dce01e3fSJonathan W Adams if (buf == NULL) 2475*dce01e3fSJonathan W Adams kmem_log_event(kmem_failure_log, NULL, NULL, 2476*dce01e3fSJonathan W Adams (void *)size); 24777c478bd9Sstevel@tonic-gate return (buf); 24787c478bd9Sstevel@tonic-gate } 2479*dce01e3fSJonathan W Adams 2480*dce01e3fSJonathan W Adams buf = kmem_cache_alloc(cp, kmflag); 2481*dce01e3fSJonathan W Adams if ((cp->cache_flags & KMF_BUFTAG) && buf != NULL) { 2482*dce01e3fSJonathan W Adams kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 2483*dce01e3fSJonathan W Adams ((uint8_t *)buf)[size] = KMEM_REDZONE_BYTE; 2484*dce01e3fSJonathan W Adams ((uint32_t *)btp)[1] = KMEM_SIZE_ENCODE(size); 2485*dce01e3fSJonathan W Adams 2486*dce01e3fSJonathan W Adams if (cp->cache_flags & KMF_LITE) { 2487*dce01e3fSJonathan W Adams KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, caller()); 2488*dce01e3fSJonathan W Adams } 2489*dce01e3fSJonathan W Adams } 24907c478bd9Sstevel@tonic-gate return (buf); 24917c478bd9Sstevel@tonic-gate } 24927c478bd9Sstevel@tonic-gate 24937c478bd9Sstevel@tonic-gate void 24947c478bd9Sstevel@tonic-gate kmem_free(void *buf, size_t size) 24957c478bd9Sstevel@tonic-gate { 2496*dce01e3fSJonathan W Adams size_t index; 2497*dce01e3fSJonathan W Adams kmem_cache_t *cp; 24987c478bd9Sstevel@tonic-gate 2499*dce01e3fSJonathan W Adams if ((index = (size - 1) >> KMEM_ALIGN_SHIFT) < KMEM_ALLOC_TABLE_MAX) { 2500*dce01e3fSJonathan W Adams cp = kmem_alloc_table[index]; 2501*dce01e3fSJonathan W Adams /* fall through to kmem_cache_free() */ 2502*dce01e3fSJonathan W Adams 2503*dce01e3fSJonathan W Adams } else if ((index = ((size - 1) >> KMEM_BIG_SHIFT)) < 2504*dce01e3fSJonathan W Adams kmem_big_alloc_table_max) { 2505*dce01e3fSJonathan W Adams cp = kmem_big_alloc_table[index]; 2506*dce01e3fSJonathan W Adams /* fall through to kmem_cache_free() */ 2507*dce01e3fSJonathan W Adams 2508*dce01e3fSJonathan W Adams } else { 2509*dce01e3fSJonathan W Adams if (buf == NULL && size == 0) 2510*dce01e3fSJonathan W Adams return; 2511*dce01e3fSJonathan W Adams vmem_free(kmem_oversize_arena, buf, size); 2512*dce01e3fSJonathan W Adams return; 2513*dce01e3fSJonathan W Adams } 2514*dce01e3fSJonathan W Adams 2515*dce01e3fSJonathan W Adams if (cp->cache_flags & KMF_BUFTAG) { 2516*dce01e3fSJonathan W Adams kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 2517*dce01e3fSJonathan W Adams uint32_t *ip = (uint32_t *)btp; 2518*dce01e3fSJonathan W Adams if (ip[1] != KMEM_SIZE_ENCODE(size)) { 2519*dce01e3fSJonathan W Adams if (*(uint64_t *)buf == KMEM_FREE_PATTERN) { 2520*dce01e3fSJonathan W Adams kmem_error(KMERR_DUPFREE, cp, buf); 25217c478bd9Sstevel@tonic-gate return; 25227c478bd9Sstevel@tonic-gate } 2523*dce01e3fSJonathan W Adams if (KMEM_SIZE_VALID(ip[1])) { 2524*dce01e3fSJonathan W Adams ip[0] = KMEM_SIZE_ENCODE(size); 2525*dce01e3fSJonathan W Adams kmem_error(KMERR_BADSIZE, cp, buf); 2526*dce01e3fSJonathan W Adams } else { 25277c478bd9Sstevel@tonic-gate kmem_error(KMERR_REDZONE, cp, buf); 25287c478bd9Sstevel@tonic-gate } 2529*dce01e3fSJonathan W Adams return; 25307c478bd9Sstevel@tonic-gate } 2531*dce01e3fSJonathan W Adams if (((uint8_t *)buf)[size] != KMEM_REDZONE_BYTE) { 2532*dce01e3fSJonathan W Adams kmem_error(KMERR_REDZONE, cp, buf); 25337c478bd9Sstevel@tonic-gate return; 2534*dce01e3fSJonathan W Adams } 2535*dce01e3fSJonathan W Adams btp->bt_redzone = KMEM_REDZONE_PATTERN; 2536*dce01e3fSJonathan W Adams if (cp->cache_flags & KMF_LITE) { 2537*dce01e3fSJonathan W Adams KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, 2538*dce01e3fSJonathan W Adams caller()); 2539*dce01e3fSJonathan W Adams } 25407c478bd9Sstevel@tonic-gate } 2541*dce01e3fSJonathan W Adams kmem_cache_free(cp, buf); 25427c478bd9Sstevel@tonic-gate } 25437c478bd9Sstevel@tonic-gate 25447c478bd9Sstevel@tonic-gate void * 25457c478bd9Sstevel@tonic-gate kmem_firewall_va_alloc(vmem_t *vmp, size_t size, int vmflag) 25467c478bd9Sstevel@tonic-gate { 25477c478bd9Sstevel@tonic-gate size_t realsize = size + vmp->vm_quantum; 25487c478bd9Sstevel@tonic-gate void *addr; 25497c478bd9Sstevel@tonic-gate 25507c478bd9Sstevel@tonic-gate /* 25517c478bd9Sstevel@tonic-gate * Annoying edge case: if 'size' is just shy of ULONG_MAX, adding 25527c478bd9Sstevel@tonic-gate * vm_quantum will cause integer wraparound. Check for this, and 25537c478bd9Sstevel@tonic-gate * blow off the firewall page in this case. Note that such a 25547c478bd9Sstevel@tonic-gate * giant allocation (the entire kernel address space) can never 25557c478bd9Sstevel@tonic-gate * be satisfied, so it will either fail immediately (VM_NOSLEEP) 25567c478bd9Sstevel@tonic-gate * or sleep forever (VM_SLEEP). Thus, there is no need for a 25577c478bd9Sstevel@tonic-gate * corresponding check in kmem_firewall_va_free(). 25587c478bd9Sstevel@tonic-gate */ 25597c478bd9Sstevel@tonic-gate if (realsize < size) 25607c478bd9Sstevel@tonic-gate realsize = size; 25617c478bd9Sstevel@tonic-gate 25627c478bd9Sstevel@tonic-gate /* 25637c478bd9Sstevel@tonic-gate * While boot still owns resource management, make sure that this 25647c478bd9Sstevel@tonic-gate * redzone virtual address allocation is properly accounted for in 25657c478bd9Sstevel@tonic-gate * OBPs "virtual-memory" "available" lists because we're 25667c478bd9Sstevel@tonic-gate * effectively claiming them for a red zone. If we don't do this, 25677c478bd9Sstevel@tonic-gate * the available lists become too fragmented and too large for the 25687c478bd9Sstevel@tonic-gate * current boot/kernel memory list interface. 25697c478bd9Sstevel@tonic-gate */ 25707c478bd9Sstevel@tonic-gate addr = vmem_alloc(vmp, realsize, vmflag | VM_NEXTFIT); 25717c478bd9Sstevel@tonic-gate 25727c478bd9Sstevel@tonic-gate if (addr != NULL && kvseg.s_base == NULL && realsize != size) 25737c478bd9Sstevel@tonic-gate (void) boot_virt_alloc((char *)addr + size, vmp->vm_quantum); 25747c478bd9Sstevel@tonic-gate 25757c478bd9Sstevel@tonic-gate return (addr); 25767c478bd9Sstevel@tonic-gate } 25777c478bd9Sstevel@tonic-gate 25787c478bd9Sstevel@tonic-gate void 25797c478bd9Sstevel@tonic-gate kmem_firewall_va_free(vmem_t *vmp, void *addr, size_t size) 25807c478bd9Sstevel@tonic-gate { 25817c478bd9Sstevel@tonic-gate ASSERT((kvseg.s_base == NULL ? 25827c478bd9Sstevel@tonic-gate va_to_pfn((char *)addr + size) : 25837c478bd9Sstevel@tonic-gate hat_getpfnum(kas.a_hat, (caddr_t)addr + size)) == PFN_INVALID); 25847c478bd9Sstevel@tonic-gate 25857c478bd9Sstevel@tonic-gate vmem_free(vmp, addr, size + vmp->vm_quantum); 25867c478bd9Sstevel@tonic-gate } 25877c478bd9Sstevel@tonic-gate 25887c478bd9Sstevel@tonic-gate /* 25897c478bd9Sstevel@tonic-gate * Try to allocate at least `size' bytes of memory without sleeping or 25907c478bd9Sstevel@tonic-gate * panicking. Return actual allocated size in `asize'. If allocation failed, 25917c478bd9Sstevel@tonic-gate * try final allocation with sleep or panic allowed. 25927c478bd9Sstevel@tonic-gate */ 25937c478bd9Sstevel@tonic-gate void * 25947c478bd9Sstevel@tonic-gate kmem_alloc_tryhard(size_t size, size_t *asize, int kmflag) 25957c478bd9Sstevel@tonic-gate { 25967c478bd9Sstevel@tonic-gate void *p; 25977c478bd9Sstevel@tonic-gate 25987c478bd9Sstevel@tonic-gate *asize = P2ROUNDUP(size, KMEM_ALIGN); 25997c478bd9Sstevel@tonic-gate do { 26007c478bd9Sstevel@tonic-gate p = kmem_alloc(*asize, (kmflag | KM_NOSLEEP) & ~KM_PANIC); 26017c478bd9Sstevel@tonic-gate if (p != NULL) 26027c478bd9Sstevel@tonic-gate return (p); 26037c478bd9Sstevel@tonic-gate *asize += KMEM_ALIGN; 26047c478bd9Sstevel@tonic-gate } while (*asize <= PAGESIZE); 26057c478bd9Sstevel@tonic-gate 26067c478bd9Sstevel@tonic-gate *asize = P2ROUNDUP(size, KMEM_ALIGN); 26077c478bd9Sstevel@tonic-gate return (kmem_alloc(*asize, kmflag)); 26087c478bd9Sstevel@tonic-gate } 26097c478bd9Sstevel@tonic-gate 26107c478bd9Sstevel@tonic-gate /* 26117c478bd9Sstevel@tonic-gate * Reclaim all unused memory from a cache. 26127c478bd9Sstevel@tonic-gate */ 26137c478bd9Sstevel@tonic-gate static void 26147c478bd9Sstevel@tonic-gate kmem_cache_reap(kmem_cache_t *cp) 26157c478bd9Sstevel@tonic-gate { 2616b5fca8f8Stomee ASSERT(taskq_member(kmem_taskq, curthread)); 2617b5fca8f8Stomee 26187c478bd9Sstevel@tonic-gate /* 26197c478bd9Sstevel@tonic-gate * Ask the cache's owner to free some memory if possible. 26207c478bd9Sstevel@tonic-gate * The idea is to handle things like the inode cache, which 26217c478bd9Sstevel@tonic-gate * typically sits on a bunch of memory that it doesn't truly 26227c478bd9Sstevel@tonic-gate * *need*. Reclaim policy is entirely up to the owner; this 26237c478bd9Sstevel@tonic-gate * callback is just an advisory plea for help. 26247c478bd9Sstevel@tonic-gate */ 2625b5fca8f8Stomee if (cp->cache_reclaim != NULL) { 2626b5fca8f8Stomee long delta; 2627b5fca8f8Stomee 2628b5fca8f8Stomee /* 2629b5fca8f8Stomee * Reclaimed memory should be reapable (not included in the 2630b5fca8f8Stomee * depot's working set). 2631b5fca8f8Stomee */ 2632b5fca8f8Stomee delta = cp->cache_full.ml_total; 26337c478bd9Sstevel@tonic-gate cp->cache_reclaim(cp->cache_private); 2634b5fca8f8Stomee delta = cp->cache_full.ml_total - delta; 2635b5fca8f8Stomee if (delta > 0) { 2636b5fca8f8Stomee mutex_enter(&cp->cache_depot_lock); 2637b5fca8f8Stomee cp->cache_full.ml_reaplimit += delta; 2638b5fca8f8Stomee cp->cache_full.ml_min += delta; 2639b5fca8f8Stomee mutex_exit(&cp->cache_depot_lock); 2640b5fca8f8Stomee } 2641b5fca8f8Stomee } 26427c478bd9Sstevel@tonic-gate 26437c478bd9Sstevel@tonic-gate kmem_depot_ws_reap(cp); 2644b5fca8f8Stomee 2645b5fca8f8Stomee if (cp->cache_defrag != NULL && !kmem_move_noreap) { 2646b5fca8f8Stomee kmem_cache_defrag(cp); 2647b5fca8f8Stomee } 26487c478bd9Sstevel@tonic-gate } 26497c478bd9Sstevel@tonic-gate 26507c478bd9Sstevel@tonic-gate static void 26517c478bd9Sstevel@tonic-gate kmem_reap_timeout(void *flag_arg) 26527c478bd9Sstevel@tonic-gate { 26537c478bd9Sstevel@tonic-gate uint32_t *flag = (uint32_t *)flag_arg; 26547c478bd9Sstevel@tonic-gate 26557c478bd9Sstevel@tonic-gate ASSERT(flag == &kmem_reaping || flag == &kmem_reaping_idspace); 26567c478bd9Sstevel@tonic-gate *flag = 0; 26577c478bd9Sstevel@tonic-gate } 26587c478bd9Sstevel@tonic-gate 26597c478bd9Sstevel@tonic-gate static void 26607c478bd9Sstevel@tonic-gate kmem_reap_done(void *flag) 26617c478bd9Sstevel@tonic-gate { 26627c478bd9Sstevel@tonic-gate (void) timeout(kmem_reap_timeout, flag, kmem_reap_interval); 26637c478bd9Sstevel@tonic-gate } 26647c478bd9Sstevel@tonic-gate 26657c478bd9Sstevel@tonic-gate static void 26667c478bd9Sstevel@tonic-gate kmem_reap_start(void *flag) 26677c478bd9Sstevel@tonic-gate { 26687c478bd9Sstevel@tonic-gate ASSERT(flag == &kmem_reaping || flag == &kmem_reaping_idspace); 26697c478bd9Sstevel@tonic-gate 26707c478bd9Sstevel@tonic-gate if (flag == &kmem_reaping) { 26717c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_reap, kmem_taskq, TQ_NOSLEEP); 26727c478bd9Sstevel@tonic-gate /* 26737c478bd9Sstevel@tonic-gate * if we have segkp under heap, reap segkp cache. 26747c478bd9Sstevel@tonic-gate */ 26757c478bd9Sstevel@tonic-gate if (segkp_fromheap) 26767c478bd9Sstevel@tonic-gate segkp_cache_free(); 26777c478bd9Sstevel@tonic-gate } 26787c478bd9Sstevel@tonic-gate else 26797c478bd9Sstevel@tonic-gate kmem_cache_applyall_id(kmem_cache_reap, kmem_taskq, TQ_NOSLEEP); 26807c478bd9Sstevel@tonic-gate 26817c478bd9Sstevel@tonic-gate /* 26827c478bd9Sstevel@tonic-gate * We use taskq_dispatch() to schedule a timeout to clear 26837c478bd9Sstevel@tonic-gate * the flag so that kmem_reap() becomes self-throttling: 26847c478bd9Sstevel@tonic-gate * we won't reap again until the current reap completes *and* 26857c478bd9Sstevel@tonic-gate * at least kmem_reap_interval ticks have elapsed. 26867c478bd9Sstevel@tonic-gate */ 26877c478bd9Sstevel@tonic-gate if (!taskq_dispatch(kmem_taskq, kmem_reap_done, flag, TQ_NOSLEEP)) 26887c478bd9Sstevel@tonic-gate kmem_reap_done(flag); 26897c478bd9Sstevel@tonic-gate } 26907c478bd9Sstevel@tonic-gate 26917c478bd9Sstevel@tonic-gate static void 26927c478bd9Sstevel@tonic-gate kmem_reap_common(void *flag_arg) 26937c478bd9Sstevel@tonic-gate { 26947c478bd9Sstevel@tonic-gate uint32_t *flag = (uint32_t *)flag_arg; 26957c478bd9Sstevel@tonic-gate 26967c478bd9Sstevel@tonic-gate if (MUTEX_HELD(&kmem_cache_lock) || kmem_taskq == NULL || 26977c478bd9Sstevel@tonic-gate cas32(flag, 0, 1) != 0) 26987c478bd9Sstevel@tonic-gate return; 26997c478bd9Sstevel@tonic-gate 27007c478bd9Sstevel@tonic-gate /* 27017c478bd9Sstevel@tonic-gate * It may not be kosher to do memory allocation when a reap is called 27027c478bd9Sstevel@tonic-gate * is called (for example, if vmem_populate() is in the call chain). 27037c478bd9Sstevel@tonic-gate * So we start the reap going with a TQ_NOALLOC dispatch. If the 27047c478bd9Sstevel@tonic-gate * dispatch fails, we reset the flag, and the next reap will try again. 27057c478bd9Sstevel@tonic-gate */ 27067c478bd9Sstevel@tonic-gate if (!taskq_dispatch(kmem_taskq, kmem_reap_start, flag, TQ_NOALLOC)) 27077c478bd9Sstevel@tonic-gate *flag = 0; 27087c478bd9Sstevel@tonic-gate } 27097c478bd9Sstevel@tonic-gate 27107c478bd9Sstevel@tonic-gate /* 27117c478bd9Sstevel@tonic-gate * Reclaim all unused memory from all caches. Called from the VM system 27127c478bd9Sstevel@tonic-gate * when memory gets tight. 27137c478bd9Sstevel@tonic-gate */ 27147c478bd9Sstevel@tonic-gate void 27157c478bd9Sstevel@tonic-gate kmem_reap(void) 27167c478bd9Sstevel@tonic-gate { 27177c478bd9Sstevel@tonic-gate kmem_reap_common(&kmem_reaping); 27187c478bd9Sstevel@tonic-gate } 27197c478bd9Sstevel@tonic-gate 27207c478bd9Sstevel@tonic-gate /* 27217c478bd9Sstevel@tonic-gate * Reclaim all unused memory from identifier arenas, called when a vmem 27227c478bd9Sstevel@tonic-gate * arena not back by memory is exhausted. Since reaping memory-backed caches 27237c478bd9Sstevel@tonic-gate * cannot help with identifier exhaustion, we avoid both a large amount of 27247c478bd9Sstevel@tonic-gate * work and unwanted side-effects from reclaim callbacks. 27257c478bd9Sstevel@tonic-gate */ 27267c478bd9Sstevel@tonic-gate void 27277c478bd9Sstevel@tonic-gate kmem_reap_idspace(void) 27287c478bd9Sstevel@tonic-gate { 27297c478bd9Sstevel@tonic-gate kmem_reap_common(&kmem_reaping_idspace); 27307c478bd9Sstevel@tonic-gate } 27317c478bd9Sstevel@tonic-gate 27327c478bd9Sstevel@tonic-gate /* 27337c478bd9Sstevel@tonic-gate * Purge all magazines from a cache and set its magazine limit to zero. 27347c478bd9Sstevel@tonic-gate * All calls are serialized by the kmem_taskq lock, except for the final 27357c478bd9Sstevel@tonic-gate * call from kmem_cache_destroy(). 27367c478bd9Sstevel@tonic-gate */ 27377c478bd9Sstevel@tonic-gate static void 27387c478bd9Sstevel@tonic-gate kmem_cache_magazine_purge(kmem_cache_t *cp) 27397c478bd9Sstevel@tonic-gate { 27407c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp; 27417c478bd9Sstevel@tonic-gate kmem_magazine_t *mp, *pmp; 27427c478bd9Sstevel@tonic-gate int rounds, prounds, cpu_seqid; 27437c478bd9Sstevel@tonic-gate 2744b5fca8f8Stomee ASSERT(!list_link_active(&cp->cache_link) || 2745b5fca8f8Stomee taskq_member(kmem_taskq, curthread)); 27467c478bd9Sstevel@tonic-gate ASSERT(MUTEX_NOT_HELD(&cp->cache_lock)); 27477c478bd9Sstevel@tonic-gate 27487c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 27497c478bd9Sstevel@tonic-gate ccp = &cp->cache_cpu[cpu_seqid]; 27507c478bd9Sstevel@tonic-gate 27517c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 27527c478bd9Sstevel@tonic-gate mp = ccp->cc_loaded; 27537c478bd9Sstevel@tonic-gate pmp = ccp->cc_ploaded; 27547c478bd9Sstevel@tonic-gate rounds = ccp->cc_rounds; 27557c478bd9Sstevel@tonic-gate prounds = ccp->cc_prounds; 27567c478bd9Sstevel@tonic-gate ccp->cc_loaded = NULL; 27577c478bd9Sstevel@tonic-gate ccp->cc_ploaded = NULL; 27587c478bd9Sstevel@tonic-gate ccp->cc_rounds = -1; 27597c478bd9Sstevel@tonic-gate ccp->cc_prounds = -1; 27607c478bd9Sstevel@tonic-gate ccp->cc_magsize = 0; 27617c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 27627c478bd9Sstevel@tonic-gate 27637c478bd9Sstevel@tonic-gate if (mp) 27647c478bd9Sstevel@tonic-gate kmem_magazine_destroy(cp, mp, rounds); 27657c478bd9Sstevel@tonic-gate if (pmp) 27667c478bd9Sstevel@tonic-gate kmem_magazine_destroy(cp, pmp, prounds); 27677c478bd9Sstevel@tonic-gate } 27687c478bd9Sstevel@tonic-gate 27697c478bd9Sstevel@tonic-gate /* 27707c478bd9Sstevel@tonic-gate * Updating the working set statistics twice in a row has the 27717c478bd9Sstevel@tonic-gate * effect of setting the working set size to zero, so everything 27727c478bd9Sstevel@tonic-gate * is eligible for reaping. 27737c478bd9Sstevel@tonic-gate */ 27747c478bd9Sstevel@tonic-gate kmem_depot_ws_update(cp); 27757c478bd9Sstevel@tonic-gate kmem_depot_ws_update(cp); 27767c478bd9Sstevel@tonic-gate 27777c478bd9Sstevel@tonic-gate kmem_depot_ws_reap(cp); 27787c478bd9Sstevel@tonic-gate } 27797c478bd9Sstevel@tonic-gate 27807c478bd9Sstevel@tonic-gate /* 27817c478bd9Sstevel@tonic-gate * Enable per-cpu magazines on a cache. 27827c478bd9Sstevel@tonic-gate */ 27837c478bd9Sstevel@tonic-gate static void 27847c478bd9Sstevel@tonic-gate kmem_cache_magazine_enable(kmem_cache_t *cp) 27857c478bd9Sstevel@tonic-gate { 27867c478bd9Sstevel@tonic-gate int cpu_seqid; 27877c478bd9Sstevel@tonic-gate 27887c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_NOMAGAZINE) 27897c478bd9Sstevel@tonic-gate return; 27907c478bd9Sstevel@tonic-gate 27917c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 27927c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 27937c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 27947c478bd9Sstevel@tonic-gate ccp->cc_magsize = cp->cache_magtype->mt_magsize; 27957c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 27967c478bd9Sstevel@tonic-gate } 27977c478bd9Sstevel@tonic-gate 27987c478bd9Sstevel@tonic-gate } 27997c478bd9Sstevel@tonic-gate 2800fa9e4066Sahrens /* 2801fa9e4066Sahrens * Reap (almost) everything right now. See kmem_cache_magazine_purge() 2802fa9e4066Sahrens * for explanation of the back-to-back kmem_depot_ws_update() calls. 2803fa9e4066Sahrens */ 2804fa9e4066Sahrens void 2805fa9e4066Sahrens kmem_cache_reap_now(kmem_cache_t *cp) 2806fa9e4066Sahrens { 2807b5fca8f8Stomee ASSERT(list_link_active(&cp->cache_link)); 2808b5fca8f8Stomee 2809fa9e4066Sahrens kmem_depot_ws_update(cp); 2810fa9e4066Sahrens kmem_depot_ws_update(cp); 2811fa9e4066Sahrens 2812fa9e4066Sahrens (void) taskq_dispatch(kmem_taskq, 2813fa9e4066Sahrens (task_func_t *)kmem_depot_ws_reap, cp, TQ_SLEEP); 2814fa9e4066Sahrens taskq_wait(kmem_taskq); 2815fa9e4066Sahrens } 2816fa9e4066Sahrens 28177c478bd9Sstevel@tonic-gate /* 28187c478bd9Sstevel@tonic-gate * Recompute a cache's magazine size. The trade-off is that larger magazines 28197c478bd9Sstevel@tonic-gate * provide a higher transfer rate with the depot, while smaller magazines 28207c478bd9Sstevel@tonic-gate * reduce memory consumption. Magazine resizing is an expensive operation; 28217c478bd9Sstevel@tonic-gate * it should not be done frequently. 28227c478bd9Sstevel@tonic-gate * 28237c478bd9Sstevel@tonic-gate * Changes to the magazine size are serialized by the kmem_taskq lock. 28247c478bd9Sstevel@tonic-gate * 28257c478bd9Sstevel@tonic-gate * Note: at present this only grows the magazine size. It might be useful 28267c478bd9Sstevel@tonic-gate * to allow shrinkage too. 28277c478bd9Sstevel@tonic-gate */ 28287c478bd9Sstevel@tonic-gate static void 28297c478bd9Sstevel@tonic-gate kmem_cache_magazine_resize(kmem_cache_t *cp) 28307c478bd9Sstevel@tonic-gate { 28317c478bd9Sstevel@tonic-gate kmem_magtype_t *mtp = cp->cache_magtype; 28327c478bd9Sstevel@tonic-gate 28337c478bd9Sstevel@tonic-gate ASSERT(taskq_member(kmem_taskq, curthread)); 28347c478bd9Sstevel@tonic-gate 28357c478bd9Sstevel@tonic-gate if (cp->cache_chunksize < mtp->mt_maxbuf) { 28367c478bd9Sstevel@tonic-gate kmem_cache_magazine_purge(cp); 28377c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock); 28387c478bd9Sstevel@tonic-gate cp->cache_magtype = ++mtp; 28397c478bd9Sstevel@tonic-gate cp->cache_depot_contention_prev = 28407c478bd9Sstevel@tonic-gate cp->cache_depot_contention + INT_MAX; 28417c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock); 28427c478bd9Sstevel@tonic-gate kmem_cache_magazine_enable(cp); 28437c478bd9Sstevel@tonic-gate } 28447c478bd9Sstevel@tonic-gate } 28457c478bd9Sstevel@tonic-gate 28467c478bd9Sstevel@tonic-gate /* 28477c478bd9Sstevel@tonic-gate * Rescale a cache's hash table, so that the table size is roughly the 28487c478bd9Sstevel@tonic-gate * cache size. We want the average lookup time to be extremely small. 28497c478bd9Sstevel@tonic-gate */ 28507c478bd9Sstevel@tonic-gate static void 28517c478bd9Sstevel@tonic-gate kmem_hash_rescale(kmem_cache_t *cp) 28527c478bd9Sstevel@tonic-gate { 28537c478bd9Sstevel@tonic-gate kmem_bufctl_t **old_table, **new_table, *bcp; 28547c478bd9Sstevel@tonic-gate size_t old_size, new_size, h; 28557c478bd9Sstevel@tonic-gate 28567c478bd9Sstevel@tonic-gate ASSERT(taskq_member(kmem_taskq, curthread)); 28577c478bd9Sstevel@tonic-gate 28587c478bd9Sstevel@tonic-gate new_size = MAX(KMEM_HASH_INITIAL, 28597c478bd9Sstevel@tonic-gate 1 << (highbit(3 * cp->cache_buftotal + 4) - 2)); 28607c478bd9Sstevel@tonic-gate old_size = cp->cache_hash_mask + 1; 28617c478bd9Sstevel@tonic-gate 28627c478bd9Sstevel@tonic-gate if ((old_size >> 1) <= new_size && new_size <= (old_size << 1)) 28637c478bd9Sstevel@tonic-gate return; 28647c478bd9Sstevel@tonic-gate 28657c478bd9Sstevel@tonic-gate new_table = vmem_alloc(kmem_hash_arena, new_size * sizeof (void *), 28667c478bd9Sstevel@tonic-gate VM_NOSLEEP); 28677c478bd9Sstevel@tonic-gate if (new_table == NULL) 28687c478bd9Sstevel@tonic-gate return; 28697c478bd9Sstevel@tonic-gate bzero(new_table, new_size * sizeof (void *)); 28707c478bd9Sstevel@tonic-gate 28717c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 28727c478bd9Sstevel@tonic-gate 28737c478bd9Sstevel@tonic-gate old_size = cp->cache_hash_mask + 1; 28747c478bd9Sstevel@tonic-gate old_table = cp->cache_hash_table; 28757c478bd9Sstevel@tonic-gate 28767c478bd9Sstevel@tonic-gate cp->cache_hash_mask = new_size - 1; 28777c478bd9Sstevel@tonic-gate cp->cache_hash_table = new_table; 28787c478bd9Sstevel@tonic-gate cp->cache_rescale++; 28797c478bd9Sstevel@tonic-gate 28807c478bd9Sstevel@tonic-gate for (h = 0; h < old_size; h++) { 28817c478bd9Sstevel@tonic-gate bcp = old_table[h]; 28827c478bd9Sstevel@tonic-gate while (bcp != NULL) { 28837c478bd9Sstevel@tonic-gate void *addr = bcp->bc_addr; 28847c478bd9Sstevel@tonic-gate kmem_bufctl_t *next_bcp = bcp->bc_next; 28857c478bd9Sstevel@tonic-gate kmem_bufctl_t **hash_bucket = KMEM_HASH(cp, addr); 28867c478bd9Sstevel@tonic-gate bcp->bc_next = *hash_bucket; 28877c478bd9Sstevel@tonic-gate *hash_bucket = bcp; 28887c478bd9Sstevel@tonic-gate bcp = next_bcp; 28897c478bd9Sstevel@tonic-gate } 28907c478bd9Sstevel@tonic-gate } 28917c478bd9Sstevel@tonic-gate 28927c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 28937c478bd9Sstevel@tonic-gate 28947c478bd9Sstevel@tonic-gate vmem_free(kmem_hash_arena, old_table, old_size * sizeof (void *)); 28957c478bd9Sstevel@tonic-gate } 28967c478bd9Sstevel@tonic-gate 28977c478bd9Sstevel@tonic-gate /* 2898b5fca8f8Stomee * Perform periodic maintenance on a cache: hash rescaling, depot working-set 2899b5fca8f8Stomee * update, magazine resizing, and slab consolidation. 29007c478bd9Sstevel@tonic-gate */ 29017c478bd9Sstevel@tonic-gate static void 29027c478bd9Sstevel@tonic-gate kmem_cache_update(kmem_cache_t *cp) 29037c478bd9Sstevel@tonic-gate { 29047c478bd9Sstevel@tonic-gate int need_hash_rescale = 0; 29057c478bd9Sstevel@tonic-gate int need_magazine_resize = 0; 29067c478bd9Sstevel@tonic-gate 29077c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&kmem_cache_lock)); 29087c478bd9Sstevel@tonic-gate 29097c478bd9Sstevel@tonic-gate /* 29107c478bd9Sstevel@tonic-gate * If the cache has become much larger or smaller than its hash table, 29117c478bd9Sstevel@tonic-gate * fire off a request to rescale the hash table. 29127c478bd9Sstevel@tonic-gate */ 29137c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 29147c478bd9Sstevel@tonic-gate 29157c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_HASH) && 29167c478bd9Sstevel@tonic-gate (cp->cache_buftotal > (cp->cache_hash_mask << 1) || 29177c478bd9Sstevel@tonic-gate (cp->cache_buftotal < (cp->cache_hash_mask >> 1) && 29187c478bd9Sstevel@tonic-gate cp->cache_hash_mask > KMEM_HASH_INITIAL))) 29197c478bd9Sstevel@tonic-gate need_hash_rescale = 1; 29207c478bd9Sstevel@tonic-gate 29217c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 29227c478bd9Sstevel@tonic-gate 29237c478bd9Sstevel@tonic-gate /* 29247c478bd9Sstevel@tonic-gate * Update the depot working set statistics. 29257c478bd9Sstevel@tonic-gate */ 29267c478bd9Sstevel@tonic-gate kmem_depot_ws_update(cp); 29277c478bd9Sstevel@tonic-gate 29287c478bd9Sstevel@tonic-gate /* 29297c478bd9Sstevel@tonic-gate * If there's a lot of contention in the depot, 29307c478bd9Sstevel@tonic-gate * increase the magazine size. 29317c478bd9Sstevel@tonic-gate */ 29327c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock); 29337c478bd9Sstevel@tonic-gate 29347c478bd9Sstevel@tonic-gate if (cp->cache_chunksize < cp->cache_magtype->mt_maxbuf && 29357c478bd9Sstevel@tonic-gate (int)(cp->cache_depot_contention - 29367c478bd9Sstevel@tonic-gate cp->cache_depot_contention_prev) > kmem_depot_contention) 29377c478bd9Sstevel@tonic-gate need_magazine_resize = 1; 29387c478bd9Sstevel@tonic-gate 29397c478bd9Sstevel@tonic-gate cp->cache_depot_contention_prev = cp->cache_depot_contention; 29407c478bd9Sstevel@tonic-gate 29417c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock); 29427c478bd9Sstevel@tonic-gate 29437c478bd9Sstevel@tonic-gate if (need_hash_rescale) 29447c478bd9Sstevel@tonic-gate (void) taskq_dispatch(kmem_taskq, 29457c478bd9Sstevel@tonic-gate (task_func_t *)kmem_hash_rescale, cp, TQ_NOSLEEP); 29467c478bd9Sstevel@tonic-gate 29477c478bd9Sstevel@tonic-gate if (need_magazine_resize) 29487c478bd9Sstevel@tonic-gate (void) taskq_dispatch(kmem_taskq, 29497c478bd9Sstevel@tonic-gate (task_func_t *)kmem_cache_magazine_resize, cp, TQ_NOSLEEP); 2950b5fca8f8Stomee 2951b5fca8f8Stomee if (cp->cache_defrag != NULL) 2952b5fca8f8Stomee (void) taskq_dispatch(kmem_taskq, 2953b5fca8f8Stomee (task_func_t *)kmem_cache_scan, cp, TQ_NOSLEEP); 29547c478bd9Sstevel@tonic-gate } 29557c478bd9Sstevel@tonic-gate 29567c478bd9Sstevel@tonic-gate static void 29577c478bd9Sstevel@tonic-gate kmem_update_timeout(void *dummy) 29587c478bd9Sstevel@tonic-gate { 29597c478bd9Sstevel@tonic-gate static void kmem_update(void *); 29607c478bd9Sstevel@tonic-gate 29617c478bd9Sstevel@tonic-gate (void) timeout(kmem_update, dummy, kmem_reap_interval); 29627c478bd9Sstevel@tonic-gate } 29637c478bd9Sstevel@tonic-gate 29647c478bd9Sstevel@tonic-gate static void 29657c478bd9Sstevel@tonic-gate kmem_update(void *dummy) 29667c478bd9Sstevel@tonic-gate { 29677c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_update, NULL, TQ_NOSLEEP); 29687c478bd9Sstevel@tonic-gate 29697c478bd9Sstevel@tonic-gate /* 29707c478bd9Sstevel@tonic-gate * We use taskq_dispatch() to reschedule the timeout so that 29717c478bd9Sstevel@tonic-gate * kmem_update() becomes self-throttling: it won't schedule 29727c478bd9Sstevel@tonic-gate * new tasks until all previous tasks have completed. 29737c478bd9Sstevel@tonic-gate */ 29747c478bd9Sstevel@tonic-gate if (!taskq_dispatch(kmem_taskq, kmem_update_timeout, dummy, TQ_NOSLEEP)) 29757c478bd9Sstevel@tonic-gate kmem_update_timeout(NULL); 29767c478bd9Sstevel@tonic-gate } 29777c478bd9Sstevel@tonic-gate 29787c478bd9Sstevel@tonic-gate static int 29797c478bd9Sstevel@tonic-gate kmem_cache_kstat_update(kstat_t *ksp, int rw) 29807c478bd9Sstevel@tonic-gate { 29817c478bd9Sstevel@tonic-gate struct kmem_cache_kstat *kmcp = &kmem_cache_kstat; 29827c478bd9Sstevel@tonic-gate kmem_cache_t *cp = ksp->ks_private; 29837c478bd9Sstevel@tonic-gate uint64_t cpu_buf_avail; 29847c478bd9Sstevel@tonic-gate uint64_t buf_avail = 0; 29857c478bd9Sstevel@tonic-gate int cpu_seqid; 29867c478bd9Sstevel@tonic-gate 29877c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&kmem_cache_kstat_lock)); 29887c478bd9Sstevel@tonic-gate 29897c478bd9Sstevel@tonic-gate if (rw == KSTAT_WRITE) 29907c478bd9Sstevel@tonic-gate return (EACCES); 29917c478bd9Sstevel@tonic-gate 29927c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 29937c478bd9Sstevel@tonic-gate 29947c478bd9Sstevel@tonic-gate kmcp->kmc_alloc_fail.value.ui64 = cp->cache_alloc_fail; 29957c478bd9Sstevel@tonic-gate kmcp->kmc_alloc.value.ui64 = cp->cache_slab_alloc; 29967c478bd9Sstevel@tonic-gate kmcp->kmc_free.value.ui64 = cp->cache_slab_free; 29977c478bd9Sstevel@tonic-gate kmcp->kmc_slab_alloc.value.ui64 = cp->cache_slab_alloc; 29987c478bd9Sstevel@tonic-gate kmcp->kmc_slab_free.value.ui64 = cp->cache_slab_free; 29997c478bd9Sstevel@tonic-gate 30007c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 30017c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 30027c478bd9Sstevel@tonic-gate 30037c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock); 30047c478bd9Sstevel@tonic-gate 30057c478bd9Sstevel@tonic-gate cpu_buf_avail = 0; 30067c478bd9Sstevel@tonic-gate if (ccp->cc_rounds > 0) 30077c478bd9Sstevel@tonic-gate cpu_buf_avail += ccp->cc_rounds; 30087c478bd9Sstevel@tonic-gate if (ccp->cc_prounds > 0) 30097c478bd9Sstevel@tonic-gate cpu_buf_avail += ccp->cc_prounds; 30107c478bd9Sstevel@tonic-gate 30117c478bd9Sstevel@tonic-gate kmcp->kmc_alloc.value.ui64 += ccp->cc_alloc; 30127c478bd9Sstevel@tonic-gate kmcp->kmc_free.value.ui64 += ccp->cc_free; 30137c478bd9Sstevel@tonic-gate buf_avail += cpu_buf_avail; 30147c478bd9Sstevel@tonic-gate 30157c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock); 30167c478bd9Sstevel@tonic-gate } 30177c478bd9Sstevel@tonic-gate 30187c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock); 30197c478bd9Sstevel@tonic-gate 30207c478bd9Sstevel@tonic-gate kmcp->kmc_depot_alloc.value.ui64 = cp->cache_full.ml_alloc; 30217c478bd9Sstevel@tonic-gate kmcp->kmc_depot_free.value.ui64 = cp->cache_empty.ml_alloc; 30227c478bd9Sstevel@tonic-gate kmcp->kmc_depot_contention.value.ui64 = cp->cache_depot_contention; 30237c478bd9Sstevel@tonic-gate kmcp->kmc_full_magazines.value.ui64 = cp->cache_full.ml_total; 30247c478bd9Sstevel@tonic-gate kmcp->kmc_empty_magazines.value.ui64 = cp->cache_empty.ml_total; 30257c478bd9Sstevel@tonic-gate kmcp->kmc_magazine_size.value.ui64 = 30267c478bd9Sstevel@tonic-gate (cp->cache_flags & KMF_NOMAGAZINE) ? 30277c478bd9Sstevel@tonic-gate 0 : cp->cache_magtype->mt_magsize; 30287c478bd9Sstevel@tonic-gate 30297c478bd9Sstevel@tonic-gate kmcp->kmc_alloc.value.ui64 += cp->cache_full.ml_alloc; 30307c478bd9Sstevel@tonic-gate kmcp->kmc_free.value.ui64 += cp->cache_empty.ml_alloc; 30317c478bd9Sstevel@tonic-gate buf_avail += cp->cache_full.ml_total * cp->cache_magtype->mt_magsize; 30327c478bd9Sstevel@tonic-gate 30337c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock); 30347c478bd9Sstevel@tonic-gate 30357c478bd9Sstevel@tonic-gate kmcp->kmc_buf_size.value.ui64 = cp->cache_bufsize; 30367c478bd9Sstevel@tonic-gate kmcp->kmc_align.value.ui64 = cp->cache_align; 30377c478bd9Sstevel@tonic-gate kmcp->kmc_chunk_size.value.ui64 = cp->cache_chunksize; 30387c478bd9Sstevel@tonic-gate kmcp->kmc_slab_size.value.ui64 = cp->cache_slabsize; 30397c478bd9Sstevel@tonic-gate kmcp->kmc_buf_constructed.value.ui64 = buf_avail; 30409f1b636aStomee buf_avail += cp->cache_bufslab; 30417c478bd9Sstevel@tonic-gate kmcp->kmc_buf_avail.value.ui64 = buf_avail; 30427c478bd9Sstevel@tonic-gate kmcp->kmc_buf_inuse.value.ui64 = cp->cache_buftotal - buf_avail; 30437c478bd9Sstevel@tonic-gate kmcp->kmc_buf_total.value.ui64 = cp->cache_buftotal; 30447c478bd9Sstevel@tonic-gate kmcp->kmc_buf_max.value.ui64 = cp->cache_bufmax; 30457c478bd9Sstevel@tonic-gate kmcp->kmc_slab_create.value.ui64 = cp->cache_slab_create; 30467c478bd9Sstevel@tonic-gate kmcp->kmc_slab_destroy.value.ui64 = cp->cache_slab_destroy; 30477c478bd9Sstevel@tonic-gate kmcp->kmc_hash_size.value.ui64 = (cp->cache_flags & KMF_HASH) ? 30487c478bd9Sstevel@tonic-gate cp->cache_hash_mask + 1 : 0; 30497c478bd9Sstevel@tonic-gate kmcp->kmc_hash_lookup_depth.value.ui64 = cp->cache_lookup_depth; 30507c478bd9Sstevel@tonic-gate kmcp->kmc_hash_rescale.value.ui64 = cp->cache_rescale; 30517c478bd9Sstevel@tonic-gate kmcp->kmc_vmem_source.value.ui64 = cp->cache_arena->vm_id; 30527c478bd9Sstevel@tonic-gate 3053b5fca8f8Stomee if (cp->cache_defrag == NULL) { 3054b5fca8f8Stomee kmcp->kmc_move_callbacks.value.ui64 = 0; 3055b5fca8f8Stomee kmcp->kmc_move_yes.value.ui64 = 0; 3056b5fca8f8Stomee kmcp->kmc_move_no.value.ui64 = 0; 3057b5fca8f8Stomee kmcp->kmc_move_later.value.ui64 = 0; 3058b5fca8f8Stomee kmcp->kmc_move_dont_need.value.ui64 = 0; 3059b5fca8f8Stomee kmcp->kmc_move_dont_know.value.ui64 = 0; 3060b5fca8f8Stomee kmcp->kmc_move_hunt_found.value.ui64 = 0; 3061b5fca8f8Stomee } else { 3062b5fca8f8Stomee kmem_defrag_t *kd = cp->cache_defrag; 3063b5fca8f8Stomee kmcp->kmc_move_callbacks.value.ui64 = kd->kmd_callbacks; 3064b5fca8f8Stomee kmcp->kmc_move_yes.value.ui64 = kd->kmd_yes; 3065b5fca8f8Stomee kmcp->kmc_move_no.value.ui64 = kd->kmd_no; 3066b5fca8f8Stomee kmcp->kmc_move_later.value.ui64 = kd->kmd_later; 3067b5fca8f8Stomee kmcp->kmc_move_dont_need.value.ui64 = kd->kmd_dont_need; 3068b5fca8f8Stomee kmcp->kmc_move_dont_know.value.ui64 = kd->kmd_dont_know; 3069b5fca8f8Stomee kmcp->kmc_move_hunt_found.value.ui64 = kd->kmd_hunt_found; 3070b5fca8f8Stomee } 3071b5fca8f8Stomee 30727c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 30737c478bd9Sstevel@tonic-gate return (0); 30747c478bd9Sstevel@tonic-gate } 30757c478bd9Sstevel@tonic-gate 30767c478bd9Sstevel@tonic-gate /* 30777c478bd9Sstevel@tonic-gate * Return a named statistic about a particular cache. 30787c478bd9Sstevel@tonic-gate * This shouldn't be called very often, so it's currently designed for 30797c478bd9Sstevel@tonic-gate * simplicity (leverages existing kstat support) rather than efficiency. 30807c478bd9Sstevel@tonic-gate */ 30817c478bd9Sstevel@tonic-gate uint64_t 30827c478bd9Sstevel@tonic-gate kmem_cache_stat(kmem_cache_t *cp, char *name) 30837c478bd9Sstevel@tonic-gate { 30847c478bd9Sstevel@tonic-gate int i; 30857c478bd9Sstevel@tonic-gate kstat_t *ksp = cp->cache_kstat; 30867c478bd9Sstevel@tonic-gate kstat_named_t *knp = (kstat_named_t *)&kmem_cache_kstat; 30877c478bd9Sstevel@tonic-gate uint64_t value = 0; 30887c478bd9Sstevel@tonic-gate 30897c478bd9Sstevel@tonic-gate if (ksp != NULL) { 30907c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_kstat_lock); 30917c478bd9Sstevel@tonic-gate (void) kmem_cache_kstat_update(ksp, KSTAT_READ); 30927c478bd9Sstevel@tonic-gate for (i = 0; i < ksp->ks_ndata; i++) { 30937c478bd9Sstevel@tonic-gate if (strcmp(knp[i].name, name) == 0) { 30947c478bd9Sstevel@tonic-gate value = knp[i].value.ui64; 30957c478bd9Sstevel@tonic-gate break; 30967c478bd9Sstevel@tonic-gate } 30977c478bd9Sstevel@tonic-gate } 30987c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_kstat_lock); 30997c478bd9Sstevel@tonic-gate } 31007c478bd9Sstevel@tonic-gate return (value); 31017c478bd9Sstevel@tonic-gate } 31027c478bd9Sstevel@tonic-gate 31037c478bd9Sstevel@tonic-gate /* 31047c478bd9Sstevel@tonic-gate * Return an estimate of currently available kernel heap memory. 31057c478bd9Sstevel@tonic-gate * On 32-bit systems, physical memory may exceed virtual memory, 31067c478bd9Sstevel@tonic-gate * we just truncate the result at 1GB. 31077c478bd9Sstevel@tonic-gate */ 31087c478bd9Sstevel@tonic-gate size_t 31097c478bd9Sstevel@tonic-gate kmem_avail(void) 31107c478bd9Sstevel@tonic-gate { 31117c478bd9Sstevel@tonic-gate spgcnt_t rmem = availrmem - tune.t_minarmem; 31127c478bd9Sstevel@tonic-gate spgcnt_t fmem = freemem - minfree; 31137c478bd9Sstevel@tonic-gate 31147c478bd9Sstevel@tonic-gate return ((size_t)ptob(MIN(MAX(MIN(rmem, fmem), 0), 31157c478bd9Sstevel@tonic-gate 1 << (30 - PAGESHIFT)))); 31167c478bd9Sstevel@tonic-gate } 31177c478bd9Sstevel@tonic-gate 31187c478bd9Sstevel@tonic-gate /* 31197c478bd9Sstevel@tonic-gate * Return the maximum amount of memory that is (in theory) allocatable 31207c478bd9Sstevel@tonic-gate * from the heap. This may be used as an estimate only since there 31217c478bd9Sstevel@tonic-gate * is no guarentee this space will still be available when an allocation 31227c478bd9Sstevel@tonic-gate * request is made, nor that the space may be allocated in one big request 31237c478bd9Sstevel@tonic-gate * due to kernel heap fragmentation. 31247c478bd9Sstevel@tonic-gate */ 31257c478bd9Sstevel@tonic-gate size_t 31267c478bd9Sstevel@tonic-gate kmem_maxavail(void) 31277c478bd9Sstevel@tonic-gate { 31287c478bd9Sstevel@tonic-gate spgcnt_t pmem = availrmem - tune.t_minarmem; 31297c478bd9Sstevel@tonic-gate spgcnt_t vmem = btop(vmem_size(heap_arena, VMEM_FREE)); 31307c478bd9Sstevel@tonic-gate 31317c478bd9Sstevel@tonic-gate return ((size_t)ptob(MAX(MIN(pmem, vmem), 0))); 31327c478bd9Sstevel@tonic-gate } 31337c478bd9Sstevel@tonic-gate 3134fa9e4066Sahrens /* 3135fa9e4066Sahrens * Indicate whether memory-intensive kmem debugging is enabled. 3136fa9e4066Sahrens */ 3137fa9e4066Sahrens int 3138fa9e4066Sahrens kmem_debugging(void) 3139fa9e4066Sahrens { 3140fa9e4066Sahrens return (kmem_flags & (KMF_AUDIT | KMF_REDZONE)); 3141fa9e4066Sahrens } 3142fa9e4066Sahrens 3143b5fca8f8Stomee /* binning function, sorts finely at the two extremes */ 3144b5fca8f8Stomee #define KMEM_PARTIAL_SLAB_WEIGHT(sp, binshift) \ 3145b5fca8f8Stomee ((((sp)->slab_refcnt <= (binshift)) || \ 3146b5fca8f8Stomee (((sp)->slab_chunks - (sp)->slab_refcnt) <= (binshift))) \ 3147b5fca8f8Stomee ? -(sp)->slab_refcnt \ 3148b5fca8f8Stomee : -((binshift) + ((sp)->slab_refcnt >> (binshift)))) 3149b5fca8f8Stomee 3150b5fca8f8Stomee /* 3151b5fca8f8Stomee * Minimizing the number of partial slabs on the freelist minimizes 3152b5fca8f8Stomee * fragmentation (the ratio of unused buffers held by the slab layer). There are 3153b5fca8f8Stomee * two ways to get a slab off of the freelist: 1) free all the buffers on the 3154b5fca8f8Stomee * slab, and 2) allocate all the buffers on the slab. It follows that we want 3155b5fca8f8Stomee * the most-used slabs at the front of the list where they have the best chance 3156b5fca8f8Stomee * of being completely allocated, and the least-used slabs at a safe distance 3157b5fca8f8Stomee * from the front to improve the odds that the few remaining buffers will all be 3158b5fca8f8Stomee * freed before another allocation can tie up the slab. For that reason a slab 3159b5fca8f8Stomee * with a higher slab_refcnt sorts less than than a slab with a lower 3160b5fca8f8Stomee * slab_refcnt. 3161b5fca8f8Stomee * 3162b5fca8f8Stomee * However, if a slab has at least one buffer that is deemed unfreeable, we 3163b5fca8f8Stomee * would rather have that slab at the front of the list regardless of 3164b5fca8f8Stomee * slab_refcnt, since even one unfreeable buffer makes the entire slab 3165b5fca8f8Stomee * unfreeable. If the client returns KMEM_CBRC_NO in response to a cache_move() 3166b5fca8f8Stomee * callback, the slab is marked unfreeable for as long as it remains on the 3167b5fca8f8Stomee * freelist. 3168b5fca8f8Stomee */ 3169b5fca8f8Stomee static int 3170b5fca8f8Stomee kmem_partial_slab_cmp(const void *p0, const void *p1) 3171b5fca8f8Stomee { 3172b5fca8f8Stomee const kmem_cache_t *cp; 3173b5fca8f8Stomee const kmem_slab_t *s0 = p0; 3174b5fca8f8Stomee const kmem_slab_t *s1 = p1; 3175b5fca8f8Stomee int w0, w1; 3176b5fca8f8Stomee size_t binshift; 3177b5fca8f8Stomee 3178b5fca8f8Stomee ASSERT(KMEM_SLAB_IS_PARTIAL(s0)); 3179b5fca8f8Stomee ASSERT(KMEM_SLAB_IS_PARTIAL(s1)); 3180b5fca8f8Stomee ASSERT(s0->slab_cache == s1->slab_cache); 3181b5fca8f8Stomee cp = s1->slab_cache; 3182b5fca8f8Stomee ASSERT(MUTEX_HELD(&cp->cache_lock)); 3183b5fca8f8Stomee binshift = cp->cache_partial_binshift; 3184b5fca8f8Stomee 3185b5fca8f8Stomee /* weight of first slab */ 3186b5fca8f8Stomee w0 = KMEM_PARTIAL_SLAB_WEIGHT(s0, binshift); 3187b5fca8f8Stomee if (s0->slab_flags & KMEM_SLAB_NOMOVE) { 3188b5fca8f8Stomee w0 -= cp->cache_maxchunks; 3189b5fca8f8Stomee } 3190b5fca8f8Stomee 3191b5fca8f8Stomee /* weight of second slab */ 3192b5fca8f8Stomee w1 = KMEM_PARTIAL_SLAB_WEIGHT(s1, binshift); 3193b5fca8f8Stomee if (s1->slab_flags & KMEM_SLAB_NOMOVE) { 3194b5fca8f8Stomee w1 -= cp->cache_maxchunks; 3195b5fca8f8Stomee } 3196b5fca8f8Stomee 3197b5fca8f8Stomee if (w0 < w1) 3198b5fca8f8Stomee return (-1); 3199b5fca8f8Stomee if (w0 > w1) 3200b5fca8f8Stomee return (1); 3201b5fca8f8Stomee 3202b5fca8f8Stomee /* compare pointer values */ 3203b5fca8f8Stomee if ((uintptr_t)s0 < (uintptr_t)s1) 3204b5fca8f8Stomee return (-1); 3205b5fca8f8Stomee if ((uintptr_t)s0 > (uintptr_t)s1) 3206b5fca8f8Stomee return (1); 3207b5fca8f8Stomee 3208b5fca8f8Stomee return (0); 3209b5fca8f8Stomee } 3210b5fca8f8Stomee 3211b5fca8f8Stomee /* 3212b5fca8f8Stomee * It must be valid to call the destructor (if any) on a newly created object. 3213b5fca8f8Stomee * That is, the constructor (if any) must leave the object in a valid state for 3214b5fca8f8Stomee * the destructor. 3215b5fca8f8Stomee */ 32167c478bd9Sstevel@tonic-gate kmem_cache_t * 32177c478bd9Sstevel@tonic-gate kmem_cache_create( 32187c478bd9Sstevel@tonic-gate char *name, /* descriptive name for this cache */ 32197c478bd9Sstevel@tonic-gate size_t bufsize, /* size of the objects it manages */ 32207c478bd9Sstevel@tonic-gate size_t align, /* required object alignment */ 32217c478bd9Sstevel@tonic-gate int (*constructor)(void *, void *, int), /* object constructor */ 32227c478bd9Sstevel@tonic-gate void (*destructor)(void *, void *), /* object destructor */ 32237c478bd9Sstevel@tonic-gate void (*reclaim)(void *), /* memory reclaim callback */ 32247c478bd9Sstevel@tonic-gate void *private, /* pass-thru arg for constr/destr/reclaim */ 32257c478bd9Sstevel@tonic-gate vmem_t *vmp, /* vmem source for slab allocation */ 32267c478bd9Sstevel@tonic-gate int cflags) /* cache creation flags */ 32277c478bd9Sstevel@tonic-gate { 32287c478bd9Sstevel@tonic-gate int cpu_seqid; 32297c478bd9Sstevel@tonic-gate size_t chunksize; 3230b5fca8f8Stomee kmem_cache_t *cp; 32317c478bd9Sstevel@tonic-gate kmem_magtype_t *mtp; 32327c478bd9Sstevel@tonic-gate size_t csize = KMEM_CACHE_SIZE(max_ncpus); 32337c478bd9Sstevel@tonic-gate 32347c478bd9Sstevel@tonic-gate #ifdef DEBUG 32357c478bd9Sstevel@tonic-gate /* 32367c478bd9Sstevel@tonic-gate * Cache names should conform to the rules for valid C identifiers 32377c478bd9Sstevel@tonic-gate */ 32387c478bd9Sstevel@tonic-gate if (!strident_valid(name)) { 32397c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, 32407c478bd9Sstevel@tonic-gate "kmem_cache_create: '%s' is an invalid cache name\n" 32417c478bd9Sstevel@tonic-gate "cache names must conform to the rules for " 32427c478bd9Sstevel@tonic-gate "C identifiers\n", name); 32437c478bd9Sstevel@tonic-gate } 32447c478bd9Sstevel@tonic-gate #endif /* DEBUG */ 32457c478bd9Sstevel@tonic-gate 32467c478bd9Sstevel@tonic-gate if (vmp == NULL) 32477c478bd9Sstevel@tonic-gate vmp = kmem_default_arena; 32487c478bd9Sstevel@tonic-gate 32497c478bd9Sstevel@tonic-gate /* 32507c478bd9Sstevel@tonic-gate * If this kmem cache has an identifier vmem arena as its source, mark 32517c478bd9Sstevel@tonic-gate * it such to allow kmem_reap_idspace(). 32527c478bd9Sstevel@tonic-gate */ 32537c478bd9Sstevel@tonic-gate ASSERT(!(cflags & KMC_IDENTIFIER)); /* consumer should not set this */ 32547c478bd9Sstevel@tonic-gate if (vmp->vm_cflags & VMC_IDENTIFIER) 32557c478bd9Sstevel@tonic-gate cflags |= KMC_IDENTIFIER; 32567c478bd9Sstevel@tonic-gate 32577c478bd9Sstevel@tonic-gate /* 32587c478bd9Sstevel@tonic-gate * Get a kmem_cache structure. We arrange that cp->cache_cpu[] 32597c478bd9Sstevel@tonic-gate * is aligned on a KMEM_CPU_CACHE_SIZE boundary to prevent 32607c478bd9Sstevel@tonic-gate * false sharing of per-CPU data. 32617c478bd9Sstevel@tonic-gate */ 32627c478bd9Sstevel@tonic-gate cp = vmem_xalloc(kmem_cache_arena, csize, KMEM_CPU_CACHE_SIZE, 32637c478bd9Sstevel@tonic-gate P2NPHASE(csize, KMEM_CPU_CACHE_SIZE), 0, NULL, NULL, VM_SLEEP); 32647c478bd9Sstevel@tonic-gate bzero(cp, csize); 3265b5fca8f8Stomee list_link_init(&cp->cache_link); 32667c478bd9Sstevel@tonic-gate 32677c478bd9Sstevel@tonic-gate if (align == 0) 32687c478bd9Sstevel@tonic-gate align = KMEM_ALIGN; 32697c478bd9Sstevel@tonic-gate 32707c478bd9Sstevel@tonic-gate /* 32717c478bd9Sstevel@tonic-gate * If we're not at least KMEM_ALIGN aligned, we can't use free 32727c478bd9Sstevel@tonic-gate * memory to hold bufctl information (because we can't safely 32737c478bd9Sstevel@tonic-gate * perform word loads and stores on it). 32747c478bd9Sstevel@tonic-gate */ 32757c478bd9Sstevel@tonic-gate if (align < KMEM_ALIGN) 32767c478bd9Sstevel@tonic-gate cflags |= KMC_NOTOUCH; 32777c478bd9Sstevel@tonic-gate 32787c478bd9Sstevel@tonic-gate if ((align & (align - 1)) != 0 || align > vmp->vm_quantum) 32797c478bd9Sstevel@tonic-gate panic("kmem_cache_create: bad alignment %lu", align); 32807c478bd9Sstevel@tonic-gate 32817c478bd9Sstevel@tonic-gate mutex_enter(&kmem_flags_lock); 32827c478bd9Sstevel@tonic-gate if (kmem_flags & KMF_RANDOMIZE) 32837c478bd9Sstevel@tonic-gate kmem_flags = (((kmem_flags | ~KMF_RANDOM) + 1) & KMF_RANDOM) | 32847c478bd9Sstevel@tonic-gate KMF_RANDOMIZE; 32857c478bd9Sstevel@tonic-gate cp->cache_flags = (kmem_flags | cflags) & KMF_DEBUG; 32867c478bd9Sstevel@tonic-gate mutex_exit(&kmem_flags_lock); 32877c478bd9Sstevel@tonic-gate 32887c478bd9Sstevel@tonic-gate /* 32897c478bd9Sstevel@tonic-gate * Make sure all the various flags are reasonable. 32907c478bd9Sstevel@tonic-gate */ 32917c478bd9Sstevel@tonic-gate ASSERT(!(cflags & KMC_NOHASH) || !(cflags & KMC_NOTOUCH)); 32927c478bd9Sstevel@tonic-gate 32937c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) { 32947c478bd9Sstevel@tonic-gate if (bufsize >= kmem_lite_minsize && 32957c478bd9Sstevel@tonic-gate align <= kmem_lite_maxalign && 32967c478bd9Sstevel@tonic-gate P2PHASE(bufsize, kmem_lite_maxalign) != 0) { 32977c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_BUFTAG; 32987c478bd9Sstevel@tonic-gate cp->cache_flags &= ~(KMF_AUDIT | KMF_FIREWALL); 32997c478bd9Sstevel@tonic-gate } else { 33007c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_DEBUG; 33017c478bd9Sstevel@tonic-gate } 33027c478bd9Sstevel@tonic-gate } 33037c478bd9Sstevel@tonic-gate 33047c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) 33057c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_REDZONE; 33067c478bd9Sstevel@tonic-gate 33077c478bd9Sstevel@tonic-gate if ((cflags & KMC_QCACHE) && (cp->cache_flags & KMF_AUDIT)) 33087c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_NOMAGAZINE; 33097c478bd9Sstevel@tonic-gate 33107c478bd9Sstevel@tonic-gate if (cflags & KMC_NODEBUG) 33117c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_DEBUG; 33127c478bd9Sstevel@tonic-gate 33137c478bd9Sstevel@tonic-gate if (cflags & KMC_NOTOUCH) 33147c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_TOUCH; 33157c478bd9Sstevel@tonic-gate 33167c478bd9Sstevel@tonic-gate if (cflags & KMC_NOHASH) 33177c478bd9Sstevel@tonic-gate cp->cache_flags &= ~(KMF_AUDIT | KMF_FIREWALL); 33187c478bd9Sstevel@tonic-gate 33197c478bd9Sstevel@tonic-gate if (cflags & KMC_NOMAGAZINE) 33207c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_NOMAGAZINE; 33217c478bd9Sstevel@tonic-gate 33227c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_AUDIT) && !(cflags & KMC_NOTOUCH)) 33237c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_REDZONE; 33247c478bd9Sstevel@tonic-gate 33257c478bd9Sstevel@tonic-gate if (!(cp->cache_flags & KMF_AUDIT)) 33267c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_CONTENTS; 33277c478bd9Sstevel@tonic-gate 33287c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_BUFTAG) && bufsize >= kmem_minfirewall && 33297c478bd9Sstevel@tonic-gate !(cp->cache_flags & KMF_LITE) && !(cflags & KMC_NOHASH)) 33307c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_FIREWALL; 33317c478bd9Sstevel@tonic-gate 33327c478bd9Sstevel@tonic-gate if (vmp != kmem_default_arena || kmem_firewall_arena == NULL) 33337c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_FIREWALL; 33347c478bd9Sstevel@tonic-gate 33357c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_FIREWALL) { 33367c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_BUFTAG; 33377c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_NOMAGAZINE; 33387c478bd9Sstevel@tonic-gate ASSERT(vmp == kmem_default_arena); 33397c478bd9Sstevel@tonic-gate vmp = kmem_firewall_arena; 33407c478bd9Sstevel@tonic-gate } 33417c478bd9Sstevel@tonic-gate 33427c478bd9Sstevel@tonic-gate /* 33437c478bd9Sstevel@tonic-gate * Set cache properties. 33447c478bd9Sstevel@tonic-gate */ 33457c478bd9Sstevel@tonic-gate (void) strncpy(cp->cache_name, name, KMEM_CACHE_NAMELEN); 3346b5fca8f8Stomee strident_canon(cp->cache_name, KMEM_CACHE_NAMELEN + 1); 33477c478bd9Sstevel@tonic-gate cp->cache_bufsize = bufsize; 33487c478bd9Sstevel@tonic-gate cp->cache_align = align; 33497c478bd9Sstevel@tonic-gate cp->cache_constructor = constructor; 33507c478bd9Sstevel@tonic-gate cp->cache_destructor = destructor; 33517c478bd9Sstevel@tonic-gate cp->cache_reclaim = reclaim; 33527c478bd9Sstevel@tonic-gate cp->cache_private = private; 33537c478bd9Sstevel@tonic-gate cp->cache_arena = vmp; 33547c478bd9Sstevel@tonic-gate cp->cache_cflags = cflags; 33557c478bd9Sstevel@tonic-gate 33567c478bd9Sstevel@tonic-gate /* 33577c478bd9Sstevel@tonic-gate * Determine the chunk size. 33587c478bd9Sstevel@tonic-gate */ 33597c478bd9Sstevel@tonic-gate chunksize = bufsize; 33607c478bd9Sstevel@tonic-gate 33617c478bd9Sstevel@tonic-gate if (align >= KMEM_ALIGN) { 33627c478bd9Sstevel@tonic-gate chunksize = P2ROUNDUP(chunksize, KMEM_ALIGN); 33637c478bd9Sstevel@tonic-gate cp->cache_bufctl = chunksize - KMEM_ALIGN; 33647c478bd9Sstevel@tonic-gate } 33657c478bd9Sstevel@tonic-gate 33667c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_BUFTAG) { 33677c478bd9Sstevel@tonic-gate cp->cache_bufctl = chunksize; 33687c478bd9Sstevel@tonic-gate cp->cache_buftag = chunksize; 33697c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) 33707c478bd9Sstevel@tonic-gate chunksize += KMEM_BUFTAG_LITE_SIZE(kmem_lite_count); 33717c478bd9Sstevel@tonic-gate else 33727c478bd9Sstevel@tonic-gate chunksize += sizeof (kmem_buftag_t); 33737c478bd9Sstevel@tonic-gate } 33747c478bd9Sstevel@tonic-gate 33757c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) { 33767c478bd9Sstevel@tonic-gate cp->cache_verify = MIN(cp->cache_buftag, kmem_maxverify); 33777c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) 33787c478bd9Sstevel@tonic-gate cp->cache_verify = sizeof (uint64_t); 33797c478bd9Sstevel@tonic-gate } 33807c478bd9Sstevel@tonic-gate 33817c478bd9Sstevel@tonic-gate cp->cache_contents = MIN(cp->cache_bufctl, kmem_content_maxsave); 33827c478bd9Sstevel@tonic-gate 33837c478bd9Sstevel@tonic-gate cp->cache_chunksize = chunksize = P2ROUNDUP(chunksize, align); 33847c478bd9Sstevel@tonic-gate 33857c478bd9Sstevel@tonic-gate /* 33867c478bd9Sstevel@tonic-gate * Now that we know the chunk size, determine the optimal slab size. 33877c478bd9Sstevel@tonic-gate */ 33887c478bd9Sstevel@tonic-gate if (vmp == kmem_firewall_arena) { 33897c478bd9Sstevel@tonic-gate cp->cache_slabsize = P2ROUNDUP(chunksize, vmp->vm_quantum); 33907c478bd9Sstevel@tonic-gate cp->cache_mincolor = cp->cache_slabsize - chunksize; 33917c478bd9Sstevel@tonic-gate cp->cache_maxcolor = cp->cache_mincolor; 33927c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_HASH; 33937c478bd9Sstevel@tonic-gate ASSERT(!(cp->cache_flags & KMF_BUFTAG)); 33947c478bd9Sstevel@tonic-gate } else if ((cflags & KMC_NOHASH) || (!(cflags & KMC_NOTOUCH) && 33957c478bd9Sstevel@tonic-gate !(cp->cache_flags & KMF_AUDIT) && 33967c478bd9Sstevel@tonic-gate chunksize < vmp->vm_quantum / KMEM_VOID_FRACTION)) { 33977c478bd9Sstevel@tonic-gate cp->cache_slabsize = vmp->vm_quantum; 33987c478bd9Sstevel@tonic-gate cp->cache_mincolor = 0; 33997c478bd9Sstevel@tonic-gate cp->cache_maxcolor = 34007c478bd9Sstevel@tonic-gate (cp->cache_slabsize - sizeof (kmem_slab_t)) % chunksize; 34017c478bd9Sstevel@tonic-gate ASSERT(chunksize + sizeof (kmem_slab_t) <= cp->cache_slabsize); 34027c478bd9Sstevel@tonic-gate ASSERT(!(cp->cache_flags & KMF_AUDIT)); 34037c478bd9Sstevel@tonic-gate } else { 34047c478bd9Sstevel@tonic-gate size_t chunks, bestfit, waste, slabsize; 34057c478bd9Sstevel@tonic-gate size_t minwaste = LONG_MAX; 34067c478bd9Sstevel@tonic-gate 34077c478bd9Sstevel@tonic-gate for (chunks = 1; chunks <= KMEM_VOID_FRACTION; chunks++) { 34087c478bd9Sstevel@tonic-gate slabsize = P2ROUNDUP(chunksize * chunks, 34097c478bd9Sstevel@tonic-gate vmp->vm_quantum); 34107c478bd9Sstevel@tonic-gate chunks = slabsize / chunksize; 34117c478bd9Sstevel@tonic-gate waste = (slabsize % chunksize) / chunks; 34127c478bd9Sstevel@tonic-gate if (waste < minwaste) { 34137c478bd9Sstevel@tonic-gate minwaste = waste; 34147c478bd9Sstevel@tonic-gate bestfit = slabsize; 34157c478bd9Sstevel@tonic-gate } 34167c478bd9Sstevel@tonic-gate } 34177c478bd9Sstevel@tonic-gate if (cflags & KMC_QCACHE) 34187c478bd9Sstevel@tonic-gate bestfit = VMEM_QCACHE_SLABSIZE(vmp->vm_qcache_max); 34197c478bd9Sstevel@tonic-gate cp->cache_slabsize = bestfit; 34207c478bd9Sstevel@tonic-gate cp->cache_mincolor = 0; 34217c478bd9Sstevel@tonic-gate cp->cache_maxcolor = bestfit % chunksize; 34227c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_HASH; 34237c478bd9Sstevel@tonic-gate } 34247c478bd9Sstevel@tonic-gate 3425b5fca8f8Stomee cp->cache_maxchunks = (cp->cache_slabsize / cp->cache_chunksize); 3426b5fca8f8Stomee cp->cache_partial_binshift = highbit(cp->cache_maxchunks / 16) + 1; 3427b5fca8f8Stomee 34287c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) { 34297c478bd9Sstevel@tonic-gate ASSERT(!(cflags & KMC_NOHASH)); 34307c478bd9Sstevel@tonic-gate cp->cache_bufctl_cache = (cp->cache_flags & KMF_AUDIT) ? 34317c478bd9Sstevel@tonic-gate kmem_bufctl_audit_cache : kmem_bufctl_cache; 34327c478bd9Sstevel@tonic-gate } 34337c478bd9Sstevel@tonic-gate 34347c478bd9Sstevel@tonic-gate if (cp->cache_maxcolor >= vmp->vm_quantum) 34357c478bd9Sstevel@tonic-gate cp->cache_maxcolor = vmp->vm_quantum - 1; 34367c478bd9Sstevel@tonic-gate 34377c478bd9Sstevel@tonic-gate cp->cache_color = cp->cache_mincolor; 34387c478bd9Sstevel@tonic-gate 34397c478bd9Sstevel@tonic-gate /* 34407c478bd9Sstevel@tonic-gate * Initialize the rest of the slab layer. 34417c478bd9Sstevel@tonic-gate */ 34427c478bd9Sstevel@tonic-gate mutex_init(&cp->cache_lock, NULL, MUTEX_DEFAULT, NULL); 34437c478bd9Sstevel@tonic-gate 3444b5fca8f8Stomee avl_create(&cp->cache_partial_slabs, kmem_partial_slab_cmp, 3445b5fca8f8Stomee sizeof (kmem_slab_t), offsetof(kmem_slab_t, slab_link)); 3446b5fca8f8Stomee /* LINTED: E_TRUE_LOGICAL_EXPR */ 3447b5fca8f8Stomee ASSERT(sizeof (list_node_t) <= sizeof (avl_node_t)); 3448b5fca8f8Stomee /* reuse partial slab AVL linkage for complete slab list linkage */ 3449b5fca8f8Stomee list_create(&cp->cache_complete_slabs, 3450b5fca8f8Stomee sizeof (kmem_slab_t), offsetof(kmem_slab_t, slab_link)); 34517c478bd9Sstevel@tonic-gate 34527c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) { 34537c478bd9Sstevel@tonic-gate cp->cache_hash_table = vmem_alloc(kmem_hash_arena, 34547c478bd9Sstevel@tonic-gate KMEM_HASH_INITIAL * sizeof (void *), VM_SLEEP); 34557c478bd9Sstevel@tonic-gate bzero(cp->cache_hash_table, 34567c478bd9Sstevel@tonic-gate KMEM_HASH_INITIAL * sizeof (void *)); 34577c478bd9Sstevel@tonic-gate cp->cache_hash_mask = KMEM_HASH_INITIAL - 1; 34587c478bd9Sstevel@tonic-gate cp->cache_hash_shift = highbit((ulong_t)chunksize) - 1; 34597c478bd9Sstevel@tonic-gate } 34607c478bd9Sstevel@tonic-gate 34617c478bd9Sstevel@tonic-gate /* 34627c478bd9Sstevel@tonic-gate * Initialize the depot. 34637c478bd9Sstevel@tonic-gate */ 34647c478bd9Sstevel@tonic-gate mutex_init(&cp->cache_depot_lock, NULL, MUTEX_DEFAULT, NULL); 34657c478bd9Sstevel@tonic-gate 34667c478bd9Sstevel@tonic-gate for (mtp = kmem_magtype; chunksize <= mtp->mt_minbuf; mtp++) 34677c478bd9Sstevel@tonic-gate continue; 34687c478bd9Sstevel@tonic-gate 34697c478bd9Sstevel@tonic-gate cp->cache_magtype = mtp; 34707c478bd9Sstevel@tonic-gate 34717c478bd9Sstevel@tonic-gate /* 34727c478bd9Sstevel@tonic-gate * Initialize the CPU layer. 34737c478bd9Sstevel@tonic-gate */ 34747c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 34757c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 34767c478bd9Sstevel@tonic-gate mutex_init(&ccp->cc_lock, NULL, MUTEX_DEFAULT, NULL); 34777c478bd9Sstevel@tonic-gate ccp->cc_flags = cp->cache_flags; 34787c478bd9Sstevel@tonic-gate ccp->cc_rounds = -1; 34797c478bd9Sstevel@tonic-gate ccp->cc_prounds = -1; 34807c478bd9Sstevel@tonic-gate } 34817c478bd9Sstevel@tonic-gate 34827c478bd9Sstevel@tonic-gate /* 34837c478bd9Sstevel@tonic-gate * Create the cache's kstats. 34847c478bd9Sstevel@tonic-gate */ 34857c478bd9Sstevel@tonic-gate if ((cp->cache_kstat = kstat_create("unix", 0, cp->cache_name, 34867c478bd9Sstevel@tonic-gate "kmem_cache", KSTAT_TYPE_NAMED, 34877c478bd9Sstevel@tonic-gate sizeof (kmem_cache_kstat) / sizeof (kstat_named_t), 34887c478bd9Sstevel@tonic-gate KSTAT_FLAG_VIRTUAL)) != NULL) { 34897c478bd9Sstevel@tonic-gate cp->cache_kstat->ks_data = &kmem_cache_kstat; 34907c478bd9Sstevel@tonic-gate cp->cache_kstat->ks_update = kmem_cache_kstat_update; 34917c478bd9Sstevel@tonic-gate cp->cache_kstat->ks_private = cp; 34927c478bd9Sstevel@tonic-gate cp->cache_kstat->ks_lock = &kmem_cache_kstat_lock; 34937c478bd9Sstevel@tonic-gate kstat_install(cp->cache_kstat); 34947c478bd9Sstevel@tonic-gate } 34957c478bd9Sstevel@tonic-gate 34967c478bd9Sstevel@tonic-gate /* 34977c478bd9Sstevel@tonic-gate * Add the cache to the global list. This makes it visible 34987c478bd9Sstevel@tonic-gate * to kmem_update(), so the cache must be ready for business. 34997c478bd9Sstevel@tonic-gate */ 35007c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_lock); 3501b5fca8f8Stomee list_insert_tail(&kmem_caches, cp); 35027c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_lock); 35037c478bd9Sstevel@tonic-gate 35047c478bd9Sstevel@tonic-gate if (kmem_ready) 35057c478bd9Sstevel@tonic-gate kmem_cache_magazine_enable(cp); 35067c478bd9Sstevel@tonic-gate 35077c478bd9Sstevel@tonic-gate return (cp); 35087c478bd9Sstevel@tonic-gate } 35097c478bd9Sstevel@tonic-gate 3510b5fca8f8Stomee static int 3511b5fca8f8Stomee kmem_move_cmp(const void *buf, const void *p) 3512b5fca8f8Stomee { 3513b5fca8f8Stomee const kmem_move_t *kmm = p; 3514b5fca8f8Stomee uintptr_t v1 = (uintptr_t)buf; 3515b5fca8f8Stomee uintptr_t v2 = (uintptr_t)kmm->kmm_from_buf; 3516b5fca8f8Stomee return (v1 < v2 ? -1 : (v1 > v2 ? 1 : 0)); 3517b5fca8f8Stomee } 3518b5fca8f8Stomee 3519b5fca8f8Stomee static void 3520b5fca8f8Stomee kmem_reset_reclaim_threshold(kmem_defrag_t *kmd) 3521b5fca8f8Stomee { 3522b5fca8f8Stomee kmd->kmd_reclaim_numer = 1; 3523b5fca8f8Stomee } 3524b5fca8f8Stomee 3525b5fca8f8Stomee /* 3526b5fca8f8Stomee * Initially, when choosing candidate slabs for buffers to move, we want to be 3527b5fca8f8Stomee * very selective and take only slabs that are less than 3528b5fca8f8Stomee * (1 / KMEM_VOID_FRACTION) allocated. If we have difficulty finding candidate 3529b5fca8f8Stomee * slabs, then we raise the allocation ceiling incrementally. The reclaim 3530b5fca8f8Stomee * threshold is reset to (1 / KMEM_VOID_FRACTION) as soon as the cache is no 3531b5fca8f8Stomee * longer fragmented. 3532b5fca8f8Stomee */ 3533b5fca8f8Stomee static void 3534b5fca8f8Stomee kmem_adjust_reclaim_threshold(kmem_defrag_t *kmd, int direction) 3535b5fca8f8Stomee { 3536b5fca8f8Stomee if (direction > 0) { 3537b5fca8f8Stomee /* make it easier to find a candidate slab */ 3538b5fca8f8Stomee if (kmd->kmd_reclaim_numer < (KMEM_VOID_FRACTION - 1)) { 3539b5fca8f8Stomee kmd->kmd_reclaim_numer++; 3540b5fca8f8Stomee } 3541b5fca8f8Stomee } else { 3542b5fca8f8Stomee /* be more selective */ 3543b5fca8f8Stomee if (kmd->kmd_reclaim_numer > 1) { 3544b5fca8f8Stomee kmd->kmd_reclaim_numer--; 3545b5fca8f8Stomee } 3546b5fca8f8Stomee } 3547b5fca8f8Stomee } 3548b5fca8f8Stomee 3549b5fca8f8Stomee void 3550b5fca8f8Stomee kmem_cache_set_move(kmem_cache_t *cp, 3551b5fca8f8Stomee kmem_cbrc_t (*move)(void *, void *, size_t, void *)) 3552b5fca8f8Stomee { 3553b5fca8f8Stomee kmem_defrag_t *defrag; 3554b5fca8f8Stomee 3555b5fca8f8Stomee ASSERT(move != NULL); 3556b5fca8f8Stomee /* 3557b5fca8f8Stomee * The consolidator does not support NOTOUCH caches because kmem cannot 3558b5fca8f8Stomee * initialize their slabs with the 0xbaddcafe memory pattern, which sets 3559b5fca8f8Stomee * a low order bit usable by clients to distinguish uninitialized memory 3560b5fca8f8Stomee * from known objects (see kmem_slab_create). 3561b5fca8f8Stomee */ 3562b5fca8f8Stomee ASSERT(!(cp->cache_cflags & KMC_NOTOUCH)); 3563b5fca8f8Stomee ASSERT(!(cp->cache_cflags & KMC_IDENTIFIER)); 3564b5fca8f8Stomee 3565b5fca8f8Stomee /* 3566b5fca8f8Stomee * We should not be holding anyone's cache lock when calling 3567b5fca8f8Stomee * kmem_cache_alloc(), so allocate in all cases before acquiring the 3568b5fca8f8Stomee * lock. 3569b5fca8f8Stomee */ 3570b5fca8f8Stomee defrag = kmem_cache_alloc(kmem_defrag_cache, KM_SLEEP); 3571b5fca8f8Stomee 3572b5fca8f8Stomee mutex_enter(&cp->cache_lock); 3573b5fca8f8Stomee 3574b5fca8f8Stomee if (KMEM_IS_MOVABLE(cp)) { 3575b5fca8f8Stomee if (cp->cache_move == NULL) { 35764d4c4c43STom Erickson ASSERT(cp->cache_slab_alloc == 0); 3577b5fca8f8Stomee 3578b5fca8f8Stomee cp->cache_defrag = defrag; 3579b5fca8f8Stomee defrag = NULL; /* nothing to free */ 3580b5fca8f8Stomee bzero(cp->cache_defrag, sizeof (kmem_defrag_t)); 3581b5fca8f8Stomee avl_create(&cp->cache_defrag->kmd_moves_pending, 3582b5fca8f8Stomee kmem_move_cmp, sizeof (kmem_move_t), 3583b5fca8f8Stomee offsetof(kmem_move_t, kmm_entry)); 3584b5fca8f8Stomee /* LINTED: E_TRUE_LOGICAL_EXPR */ 3585b5fca8f8Stomee ASSERT(sizeof (list_node_t) <= sizeof (avl_node_t)); 3586b5fca8f8Stomee /* reuse the slab's AVL linkage for deadlist linkage */ 3587b5fca8f8Stomee list_create(&cp->cache_defrag->kmd_deadlist, 3588b5fca8f8Stomee sizeof (kmem_slab_t), 3589b5fca8f8Stomee offsetof(kmem_slab_t, slab_link)); 3590b5fca8f8Stomee kmem_reset_reclaim_threshold(cp->cache_defrag); 3591b5fca8f8Stomee } 3592b5fca8f8Stomee cp->cache_move = move; 3593b5fca8f8Stomee } 3594b5fca8f8Stomee 3595b5fca8f8Stomee mutex_exit(&cp->cache_lock); 3596b5fca8f8Stomee 3597b5fca8f8Stomee if (defrag != NULL) { 3598b5fca8f8Stomee kmem_cache_free(kmem_defrag_cache, defrag); /* unused */ 3599b5fca8f8Stomee } 3600b5fca8f8Stomee } 3601b5fca8f8Stomee 36027c478bd9Sstevel@tonic-gate void 36037c478bd9Sstevel@tonic-gate kmem_cache_destroy(kmem_cache_t *cp) 36047c478bd9Sstevel@tonic-gate { 36057c478bd9Sstevel@tonic-gate int cpu_seqid; 36067c478bd9Sstevel@tonic-gate 36077c478bd9Sstevel@tonic-gate /* 36087c478bd9Sstevel@tonic-gate * Remove the cache from the global cache list so that no one else 36097c478bd9Sstevel@tonic-gate * can schedule tasks on its behalf, wait for any pending tasks to 36107c478bd9Sstevel@tonic-gate * complete, purge the cache, and then destroy it. 36117c478bd9Sstevel@tonic-gate */ 36127c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_lock); 3613b5fca8f8Stomee list_remove(&kmem_caches, cp); 36147c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_lock); 36157c478bd9Sstevel@tonic-gate 36167c478bd9Sstevel@tonic-gate if (kmem_taskq != NULL) 36177c478bd9Sstevel@tonic-gate taskq_wait(kmem_taskq); 3618b5fca8f8Stomee if (kmem_move_taskq != NULL) 3619b5fca8f8Stomee taskq_wait(kmem_move_taskq); 36207c478bd9Sstevel@tonic-gate 36217c478bd9Sstevel@tonic-gate kmem_cache_magazine_purge(cp); 36227c478bd9Sstevel@tonic-gate 36237c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock); 36247c478bd9Sstevel@tonic-gate if (cp->cache_buftotal != 0) 36257c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "kmem_cache_destroy: '%s' (%p) not empty", 36267c478bd9Sstevel@tonic-gate cp->cache_name, (void *)cp); 3627b5fca8f8Stomee if (cp->cache_defrag != NULL) { 3628b5fca8f8Stomee avl_destroy(&cp->cache_defrag->kmd_moves_pending); 3629b5fca8f8Stomee list_destroy(&cp->cache_defrag->kmd_deadlist); 3630b5fca8f8Stomee kmem_cache_free(kmem_defrag_cache, cp->cache_defrag); 3631b5fca8f8Stomee cp->cache_defrag = NULL; 3632b5fca8f8Stomee } 36337c478bd9Sstevel@tonic-gate /* 3634b5fca8f8Stomee * The cache is now dead. There should be no further activity. We 3635b5fca8f8Stomee * enforce this by setting land mines in the constructor, destructor, 3636b5fca8f8Stomee * reclaim, and move routines that induce a kernel text fault if 3637b5fca8f8Stomee * invoked. 36387c478bd9Sstevel@tonic-gate */ 36397c478bd9Sstevel@tonic-gate cp->cache_constructor = (int (*)(void *, void *, int))1; 36407c478bd9Sstevel@tonic-gate cp->cache_destructor = (void (*)(void *, void *))2; 3641b5fca8f8Stomee cp->cache_reclaim = (void (*)(void *))3; 3642b5fca8f8Stomee cp->cache_move = (kmem_cbrc_t (*)(void *, void *, size_t, void *))4; 36437c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock); 36447c478bd9Sstevel@tonic-gate 36457c478bd9Sstevel@tonic-gate kstat_delete(cp->cache_kstat); 36467c478bd9Sstevel@tonic-gate 36477c478bd9Sstevel@tonic-gate if (cp->cache_hash_table != NULL) 36487c478bd9Sstevel@tonic-gate vmem_free(kmem_hash_arena, cp->cache_hash_table, 36497c478bd9Sstevel@tonic-gate (cp->cache_hash_mask + 1) * sizeof (void *)); 36507c478bd9Sstevel@tonic-gate 36517c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) 36527c478bd9Sstevel@tonic-gate mutex_destroy(&cp->cache_cpu[cpu_seqid].cc_lock); 36537c478bd9Sstevel@tonic-gate 36547c478bd9Sstevel@tonic-gate mutex_destroy(&cp->cache_depot_lock); 36557c478bd9Sstevel@tonic-gate mutex_destroy(&cp->cache_lock); 36567c478bd9Sstevel@tonic-gate 36577c478bd9Sstevel@tonic-gate vmem_free(kmem_cache_arena, cp, KMEM_CACHE_SIZE(max_ncpus)); 36587c478bd9Sstevel@tonic-gate } 36597c478bd9Sstevel@tonic-gate 36607c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 36617c478bd9Sstevel@tonic-gate static int 36627c478bd9Sstevel@tonic-gate kmem_cpu_setup(cpu_setup_t what, int id, void *arg) 36637c478bd9Sstevel@tonic-gate { 36647c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 36657c478bd9Sstevel@tonic-gate if (what == CPU_UNCONFIG) { 36667c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_magazine_purge, 36677c478bd9Sstevel@tonic-gate kmem_taskq, TQ_SLEEP); 36687c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_magazine_enable, 36697c478bd9Sstevel@tonic-gate kmem_taskq, TQ_SLEEP); 36707c478bd9Sstevel@tonic-gate } 36717c478bd9Sstevel@tonic-gate return (0); 36727c478bd9Sstevel@tonic-gate } 36737c478bd9Sstevel@tonic-gate 3674*dce01e3fSJonathan W Adams static void 3675*dce01e3fSJonathan W Adams kmem_alloc_caches_create(const int *array, size_t count, 3676*dce01e3fSJonathan W Adams kmem_cache_t **alloc_table, size_t maxbuf, uint_t shift) 3677*dce01e3fSJonathan W Adams { 3678*dce01e3fSJonathan W Adams char name[KMEM_CACHE_NAMELEN + 1]; 3679*dce01e3fSJonathan W Adams size_t table_unit = (1 << shift); /* range of one alloc_table entry */ 3680*dce01e3fSJonathan W Adams size_t size = table_unit; 3681*dce01e3fSJonathan W Adams int i; 3682*dce01e3fSJonathan W Adams 3683*dce01e3fSJonathan W Adams for (i = 0; i < count; i++) { 3684*dce01e3fSJonathan W Adams size_t cache_size = array[i]; 3685*dce01e3fSJonathan W Adams size_t align = KMEM_ALIGN; 3686*dce01e3fSJonathan W Adams kmem_cache_t *cp; 3687*dce01e3fSJonathan W Adams 3688*dce01e3fSJonathan W Adams /* if the table has an entry for maxbuf, we're done */ 3689*dce01e3fSJonathan W Adams if (size > maxbuf) 3690*dce01e3fSJonathan W Adams break; 3691*dce01e3fSJonathan W Adams 3692*dce01e3fSJonathan W Adams /* cache size must be a multiple of the table unit */ 3693*dce01e3fSJonathan W Adams ASSERT(P2PHASE(cache_size, table_unit) == 0); 3694*dce01e3fSJonathan W Adams 3695*dce01e3fSJonathan W Adams /* 3696*dce01e3fSJonathan W Adams * If they allocate a multiple of the coherency granularity, 3697*dce01e3fSJonathan W Adams * they get a coherency-granularity-aligned address. 3698*dce01e3fSJonathan W Adams */ 3699*dce01e3fSJonathan W Adams if (IS_P2ALIGNED(cache_size, 64)) 3700*dce01e3fSJonathan W Adams align = 64; 3701*dce01e3fSJonathan W Adams if (IS_P2ALIGNED(cache_size, PAGESIZE)) 3702*dce01e3fSJonathan W Adams align = PAGESIZE; 3703*dce01e3fSJonathan W Adams (void) snprintf(name, sizeof (name), 3704*dce01e3fSJonathan W Adams "kmem_alloc_%lu", cache_size); 3705*dce01e3fSJonathan W Adams cp = kmem_cache_create(name, cache_size, align, 3706*dce01e3fSJonathan W Adams NULL, NULL, NULL, NULL, NULL, KMC_KMEM_ALLOC); 3707*dce01e3fSJonathan W Adams 3708*dce01e3fSJonathan W Adams while (size <= cache_size) { 3709*dce01e3fSJonathan W Adams alloc_table[(size - 1) >> shift] = cp; 3710*dce01e3fSJonathan W Adams size += table_unit; 3711*dce01e3fSJonathan W Adams } 3712*dce01e3fSJonathan W Adams } 3713*dce01e3fSJonathan W Adams 3714*dce01e3fSJonathan W Adams ASSERT(size > maxbuf); /* i.e. maxbuf <= max(cache_size) */ 3715*dce01e3fSJonathan W Adams } 3716*dce01e3fSJonathan W Adams 37177c478bd9Sstevel@tonic-gate static void 37187c478bd9Sstevel@tonic-gate kmem_cache_init(int pass, int use_large_pages) 37197c478bd9Sstevel@tonic-gate { 37207c478bd9Sstevel@tonic-gate int i; 3721*dce01e3fSJonathan W Adams size_t maxbuf; 37227c478bd9Sstevel@tonic-gate kmem_magtype_t *mtp; 37237c478bd9Sstevel@tonic-gate 37247c478bd9Sstevel@tonic-gate for (i = 0; i < sizeof (kmem_magtype) / sizeof (*mtp); i++) { 3725*dce01e3fSJonathan W Adams char name[KMEM_CACHE_NAMELEN + 1]; 3726*dce01e3fSJonathan W Adams 37277c478bd9Sstevel@tonic-gate mtp = &kmem_magtype[i]; 37287c478bd9Sstevel@tonic-gate (void) sprintf(name, "kmem_magazine_%d", mtp->mt_magsize); 37297c478bd9Sstevel@tonic-gate mtp->mt_cache = kmem_cache_create(name, 37307c478bd9Sstevel@tonic-gate (mtp->mt_magsize + 1) * sizeof (void *), 37317c478bd9Sstevel@tonic-gate mtp->mt_align, NULL, NULL, NULL, NULL, 37327c478bd9Sstevel@tonic-gate kmem_msb_arena, KMC_NOHASH); 37337c478bd9Sstevel@tonic-gate } 37347c478bd9Sstevel@tonic-gate 37357c478bd9Sstevel@tonic-gate kmem_slab_cache = kmem_cache_create("kmem_slab_cache", 37367c478bd9Sstevel@tonic-gate sizeof (kmem_slab_t), 0, NULL, NULL, NULL, NULL, 37377c478bd9Sstevel@tonic-gate kmem_msb_arena, KMC_NOHASH); 37387c478bd9Sstevel@tonic-gate 37397c478bd9Sstevel@tonic-gate kmem_bufctl_cache = kmem_cache_create("kmem_bufctl_cache", 37407c478bd9Sstevel@tonic-gate sizeof (kmem_bufctl_t), 0, NULL, NULL, NULL, NULL, 37417c478bd9Sstevel@tonic-gate kmem_msb_arena, KMC_NOHASH); 37427c478bd9Sstevel@tonic-gate 37437c478bd9Sstevel@tonic-gate kmem_bufctl_audit_cache = kmem_cache_create("kmem_bufctl_audit_cache", 37447c478bd9Sstevel@tonic-gate sizeof (kmem_bufctl_audit_t), 0, NULL, NULL, NULL, NULL, 37457c478bd9Sstevel@tonic-gate kmem_msb_arena, KMC_NOHASH); 37467c478bd9Sstevel@tonic-gate 37477c478bd9Sstevel@tonic-gate if (pass == 2) { 37487c478bd9Sstevel@tonic-gate kmem_va_arena = vmem_create("kmem_va", 37497c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE, 37507c478bd9Sstevel@tonic-gate vmem_alloc, vmem_free, heap_arena, 37517c478bd9Sstevel@tonic-gate 8 * PAGESIZE, VM_SLEEP); 37527c478bd9Sstevel@tonic-gate 37537c478bd9Sstevel@tonic-gate if (use_large_pages) { 37547c478bd9Sstevel@tonic-gate kmem_default_arena = vmem_xcreate("kmem_default", 37557c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE, 37567c478bd9Sstevel@tonic-gate segkmem_alloc_lp, segkmem_free_lp, kmem_va_arena, 37577c478bd9Sstevel@tonic-gate 0, VM_SLEEP); 37587c478bd9Sstevel@tonic-gate } else { 37597c478bd9Sstevel@tonic-gate kmem_default_arena = vmem_create("kmem_default", 37607c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE, 37617c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, kmem_va_arena, 37627c478bd9Sstevel@tonic-gate 0, VM_SLEEP); 37637c478bd9Sstevel@tonic-gate } 3764*dce01e3fSJonathan W Adams 3765*dce01e3fSJonathan W Adams /* Figure out what our maximum cache size is */ 3766*dce01e3fSJonathan W Adams maxbuf = kmem_max_cached; 3767*dce01e3fSJonathan W Adams if (maxbuf <= KMEM_MAXBUF) { 3768*dce01e3fSJonathan W Adams maxbuf = 0; 3769*dce01e3fSJonathan W Adams kmem_max_cached = KMEM_MAXBUF; 3770*dce01e3fSJonathan W Adams } else { 3771*dce01e3fSJonathan W Adams size_t size = 0; 3772*dce01e3fSJonathan W Adams size_t max = 3773*dce01e3fSJonathan W Adams sizeof (kmem_big_alloc_sizes) / sizeof (int); 3774*dce01e3fSJonathan W Adams /* 3775*dce01e3fSJonathan W Adams * Round maxbuf up to an existing cache size. If maxbuf 3776*dce01e3fSJonathan W Adams * is larger than the largest cache, we truncate it to 3777*dce01e3fSJonathan W Adams * the largest cache's size. 3778*dce01e3fSJonathan W Adams */ 3779*dce01e3fSJonathan W Adams for (i = 0; i < max; i++) { 3780*dce01e3fSJonathan W Adams size = kmem_big_alloc_sizes[i]; 3781*dce01e3fSJonathan W Adams if (maxbuf <= size) 3782*dce01e3fSJonathan W Adams break; 3783*dce01e3fSJonathan W Adams } 3784*dce01e3fSJonathan W Adams kmem_max_cached = maxbuf = size; 3785*dce01e3fSJonathan W Adams } 3786*dce01e3fSJonathan W Adams 3787*dce01e3fSJonathan W Adams /* 3788*dce01e3fSJonathan W Adams * The big alloc table may not be completely overwritten, so 3789*dce01e3fSJonathan W Adams * we clear out any stale cache pointers from the first pass. 3790*dce01e3fSJonathan W Adams */ 3791*dce01e3fSJonathan W Adams bzero(kmem_big_alloc_table, sizeof (kmem_big_alloc_table)); 37927c478bd9Sstevel@tonic-gate } else { 37937c478bd9Sstevel@tonic-gate /* 37947c478bd9Sstevel@tonic-gate * During the first pass, the kmem_alloc_* caches 37957c478bd9Sstevel@tonic-gate * are treated as metadata. 37967c478bd9Sstevel@tonic-gate */ 37977c478bd9Sstevel@tonic-gate kmem_default_arena = kmem_msb_arena; 3798*dce01e3fSJonathan W Adams maxbuf = KMEM_BIG_MAXBUF_32BIT; 37997c478bd9Sstevel@tonic-gate } 38007c478bd9Sstevel@tonic-gate 38017c478bd9Sstevel@tonic-gate /* 38027c478bd9Sstevel@tonic-gate * Set up the default caches to back kmem_alloc() 38037c478bd9Sstevel@tonic-gate */ 3804*dce01e3fSJonathan W Adams kmem_alloc_caches_create( 3805*dce01e3fSJonathan W Adams kmem_alloc_sizes, sizeof (kmem_alloc_sizes) / sizeof (int), 3806*dce01e3fSJonathan W Adams kmem_alloc_table, KMEM_MAXBUF, KMEM_ALIGN_SHIFT); 3807*dce01e3fSJonathan W Adams 3808*dce01e3fSJonathan W Adams kmem_alloc_caches_create( 3809*dce01e3fSJonathan W Adams kmem_big_alloc_sizes, sizeof (kmem_big_alloc_sizes) / sizeof (int), 3810*dce01e3fSJonathan W Adams kmem_big_alloc_table, maxbuf, KMEM_BIG_SHIFT); 3811*dce01e3fSJonathan W Adams 3812*dce01e3fSJonathan W Adams kmem_big_alloc_table_max = maxbuf >> KMEM_BIG_SHIFT; 38137c478bd9Sstevel@tonic-gate } 38147c478bd9Sstevel@tonic-gate 38157c478bd9Sstevel@tonic-gate void 38167c478bd9Sstevel@tonic-gate kmem_init(void) 38177c478bd9Sstevel@tonic-gate { 38187c478bd9Sstevel@tonic-gate kmem_cache_t *cp; 38197c478bd9Sstevel@tonic-gate int old_kmem_flags = kmem_flags; 38207c478bd9Sstevel@tonic-gate int use_large_pages = 0; 38217c478bd9Sstevel@tonic-gate size_t maxverify, minfirewall; 38227c478bd9Sstevel@tonic-gate 38237c478bd9Sstevel@tonic-gate kstat_init(); 38247c478bd9Sstevel@tonic-gate 38257c478bd9Sstevel@tonic-gate /* 38267c478bd9Sstevel@tonic-gate * Small-memory systems (< 24 MB) can't handle kmem_flags overhead. 38277c478bd9Sstevel@tonic-gate */ 38287c478bd9Sstevel@tonic-gate if (physmem < btop(24 << 20) && !(old_kmem_flags & KMF_STICKY)) 38297c478bd9Sstevel@tonic-gate kmem_flags = 0; 38307c478bd9Sstevel@tonic-gate 38317c478bd9Sstevel@tonic-gate /* 38327c478bd9Sstevel@tonic-gate * Don't do firewalled allocations if the heap is less than 1TB 38337c478bd9Sstevel@tonic-gate * (i.e. on a 32-bit kernel) 38347c478bd9Sstevel@tonic-gate * The resulting VM_NEXTFIT allocations would create too much 38357c478bd9Sstevel@tonic-gate * fragmentation in a small heap. 38367c478bd9Sstevel@tonic-gate */ 38377c478bd9Sstevel@tonic-gate #if defined(_LP64) 38387c478bd9Sstevel@tonic-gate maxverify = minfirewall = PAGESIZE / 2; 38397c478bd9Sstevel@tonic-gate #else 38407c478bd9Sstevel@tonic-gate maxverify = minfirewall = ULONG_MAX; 38417c478bd9Sstevel@tonic-gate #endif 38427c478bd9Sstevel@tonic-gate 38437c478bd9Sstevel@tonic-gate /* LINTED */ 38447c478bd9Sstevel@tonic-gate ASSERT(sizeof (kmem_cpu_cache_t) == KMEM_CPU_CACHE_SIZE); 38457c478bd9Sstevel@tonic-gate 3846b5fca8f8Stomee list_create(&kmem_caches, sizeof (kmem_cache_t), 3847b5fca8f8Stomee offsetof(kmem_cache_t, cache_link)); 38487c478bd9Sstevel@tonic-gate 38497c478bd9Sstevel@tonic-gate kmem_metadata_arena = vmem_create("kmem_metadata", NULL, 0, PAGESIZE, 38507c478bd9Sstevel@tonic-gate vmem_alloc, vmem_free, heap_arena, 8 * PAGESIZE, 38517c478bd9Sstevel@tonic-gate VM_SLEEP | VMC_NO_QCACHE); 38527c478bd9Sstevel@tonic-gate 38537c478bd9Sstevel@tonic-gate kmem_msb_arena = vmem_create("kmem_msb", NULL, 0, 38547c478bd9Sstevel@tonic-gate PAGESIZE, segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, 38557c478bd9Sstevel@tonic-gate VM_SLEEP); 38567c478bd9Sstevel@tonic-gate 38577c478bd9Sstevel@tonic-gate kmem_cache_arena = vmem_create("kmem_cache", NULL, 0, KMEM_ALIGN, 38587c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, VM_SLEEP); 38597c478bd9Sstevel@tonic-gate 38607c478bd9Sstevel@tonic-gate kmem_hash_arena = vmem_create("kmem_hash", NULL, 0, KMEM_ALIGN, 38617c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, VM_SLEEP); 38627c478bd9Sstevel@tonic-gate 38637c478bd9Sstevel@tonic-gate kmem_log_arena = vmem_create("kmem_log", NULL, 0, KMEM_ALIGN, 38647c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP); 38657c478bd9Sstevel@tonic-gate 38667c478bd9Sstevel@tonic-gate kmem_firewall_va_arena = vmem_create("kmem_firewall_va", 38677c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE, 38687c478bd9Sstevel@tonic-gate kmem_firewall_va_alloc, kmem_firewall_va_free, heap_arena, 38697c478bd9Sstevel@tonic-gate 0, VM_SLEEP); 38707c478bd9Sstevel@tonic-gate 38717c478bd9Sstevel@tonic-gate kmem_firewall_arena = vmem_create("kmem_firewall", NULL, 0, PAGESIZE, 38727c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, kmem_firewall_va_arena, 0, VM_SLEEP); 38737c478bd9Sstevel@tonic-gate 38747c478bd9Sstevel@tonic-gate /* temporary oversize arena for mod_read_system_file */ 38757c478bd9Sstevel@tonic-gate kmem_oversize_arena = vmem_create("kmem_oversize", NULL, 0, PAGESIZE, 38767c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP); 38777c478bd9Sstevel@tonic-gate 38787c478bd9Sstevel@tonic-gate kmem_reap_interval = 15 * hz; 38797c478bd9Sstevel@tonic-gate 38807c478bd9Sstevel@tonic-gate /* 38817c478bd9Sstevel@tonic-gate * Read /etc/system. This is a chicken-and-egg problem because 38827c478bd9Sstevel@tonic-gate * kmem_flags may be set in /etc/system, but mod_read_system_file() 38837c478bd9Sstevel@tonic-gate * needs to use the allocator. The simplest solution is to create 38847c478bd9Sstevel@tonic-gate * all the standard kmem caches, read /etc/system, destroy all the 38857c478bd9Sstevel@tonic-gate * caches we just created, and then create them all again in light 38867c478bd9Sstevel@tonic-gate * of the (possibly) new kmem_flags and other kmem tunables. 38877c478bd9Sstevel@tonic-gate */ 38887c478bd9Sstevel@tonic-gate kmem_cache_init(1, 0); 38897c478bd9Sstevel@tonic-gate 38907c478bd9Sstevel@tonic-gate mod_read_system_file(boothowto & RB_ASKNAME); 38917c478bd9Sstevel@tonic-gate 3892b5fca8f8Stomee while ((cp = list_tail(&kmem_caches)) != NULL) 38937c478bd9Sstevel@tonic-gate kmem_cache_destroy(cp); 38947c478bd9Sstevel@tonic-gate 38957c478bd9Sstevel@tonic-gate vmem_destroy(kmem_oversize_arena); 38967c478bd9Sstevel@tonic-gate 38977c478bd9Sstevel@tonic-gate if (old_kmem_flags & KMF_STICKY) 38987c478bd9Sstevel@tonic-gate kmem_flags = old_kmem_flags; 38997c478bd9Sstevel@tonic-gate 39007c478bd9Sstevel@tonic-gate if (!(kmem_flags & KMF_AUDIT)) 39017c478bd9Sstevel@tonic-gate vmem_seg_size = offsetof(vmem_seg_t, vs_thread); 39027c478bd9Sstevel@tonic-gate 39037c478bd9Sstevel@tonic-gate if (kmem_maxverify == 0) 39047c478bd9Sstevel@tonic-gate kmem_maxverify = maxverify; 39057c478bd9Sstevel@tonic-gate 39067c478bd9Sstevel@tonic-gate if (kmem_minfirewall == 0) 39077c478bd9Sstevel@tonic-gate kmem_minfirewall = minfirewall; 39087c478bd9Sstevel@tonic-gate 39097c478bd9Sstevel@tonic-gate /* 39107c478bd9Sstevel@tonic-gate * give segkmem a chance to figure out if we are using large pages 39117c478bd9Sstevel@tonic-gate * for the kernel heap 39127c478bd9Sstevel@tonic-gate */ 39137c478bd9Sstevel@tonic-gate use_large_pages = segkmem_lpsetup(); 39147c478bd9Sstevel@tonic-gate 39157c478bd9Sstevel@tonic-gate /* 39167c478bd9Sstevel@tonic-gate * To protect against corruption, we keep the actual number of callers 39177c478bd9Sstevel@tonic-gate * KMF_LITE records seperate from the tunable. We arbitrarily clamp 39187c478bd9Sstevel@tonic-gate * to 16, since the overhead for small buffers quickly gets out of 39197c478bd9Sstevel@tonic-gate * hand. 39207c478bd9Sstevel@tonic-gate * 39217c478bd9Sstevel@tonic-gate * The real limit would depend on the needs of the largest KMC_NOHASH 39227c478bd9Sstevel@tonic-gate * cache. 39237c478bd9Sstevel@tonic-gate */ 39247c478bd9Sstevel@tonic-gate kmem_lite_count = MIN(MAX(0, kmem_lite_pcs), 16); 39257c478bd9Sstevel@tonic-gate kmem_lite_pcs = kmem_lite_count; 39267c478bd9Sstevel@tonic-gate 39277c478bd9Sstevel@tonic-gate /* 39287c478bd9Sstevel@tonic-gate * Normally, we firewall oversized allocations when possible, but 39297c478bd9Sstevel@tonic-gate * if we are using large pages for kernel memory, and we don't have 39307c478bd9Sstevel@tonic-gate * any non-LITE debugging flags set, we want to allocate oversized 39317c478bd9Sstevel@tonic-gate * buffers from large pages, and so skip the firewalling. 39327c478bd9Sstevel@tonic-gate */ 39337c478bd9Sstevel@tonic-gate if (use_large_pages && 39347c478bd9Sstevel@tonic-gate ((kmem_flags & KMF_LITE) || !(kmem_flags & KMF_DEBUG))) { 39357c478bd9Sstevel@tonic-gate kmem_oversize_arena = vmem_xcreate("kmem_oversize", NULL, 0, 39367c478bd9Sstevel@tonic-gate PAGESIZE, segkmem_alloc_lp, segkmem_free_lp, heap_arena, 39377c478bd9Sstevel@tonic-gate 0, VM_SLEEP); 39387c478bd9Sstevel@tonic-gate } else { 39397c478bd9Sstevel@tonic-gate kmem_oversize_arena = vmem_create("kmem_oversize", 39407c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE, 39417c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, kmem_minfirewall < ULONG_MAX? 39427c478bd9Sstevel@tonic-gate kmem_firewall_va_arena : heap_arena, 0, VM_SLEEP); 39437c478bd9Sstevel@tonic-gate } 39447c478bd9Sstevel@tonic-gate 39457c478bd9Sstevel@tonic-gate kmem_cache_init(2, use_large_pages); 39467c478bd9Sstevel@tonic-gate 39477c478bd9Sstevel@tonic-gate if (kmem_flags & (KMF_AUDIT | KMF_RANDOMIZE)) { 39487c478bd9Sstevel@tonic-gate if (kmem_transaction_log_size == 0) 39497c478bd9Sstevel@tonic-gate kmem_transaction_log_size = kmem_maxavail() / 50; 39507c478bd9Sstevel@tonic-gate kmem_transaction_log = kmem_log_init(kmem_transaction_log_size); 39517c478bd9Sstevel@tonic-gate } 39527c478bd9Sstevel@tonic-gate 39537c478bd9Sstevel@tonic-gate if (kmem_flags & (KMF_CONTENTS | KMF_RANDOMIZE)) { 39547c478bd9Sstevel@tonic-gate if (kmem_content_log_size == 0) 39557c478bd9Sstevel@tonic-gate kmem_content_log_size = kmem_maxavail() / 50; 39567c478bd9Sstevel@tonic-gate kmem_content_log = kmem_log_init(kmem_content_log_size); 39577c478bd9Sstevel@tonic-gate } 39587c478bd9Sstevel@tonic-gate 39597c478bd9Sstevel@tonic-gate kmem_failure_log = kmem_log_init(kmem_failure_log_size); 39607c478bd9Sstevel@tonic-gate 39617c478bd9Sstevel@tonic-gate kmem_slab_log = kmem_log_init(kmem_slab_log_size); 39627c478bd9Sstevel@tonic-gate 39637c478bd9Sstevel@tonic-gate /* 39647c478bd9Sstevel@tonic-gate * Initialize STREAMS message caches so allocb() is available. 39657c478bd9Sstevel@tonic-gate * This allows us to initialize the logging framework (cmn_err(9F), 39667c478bd9Sstevel@tonic-gate * strlog(9F), etc) so we can start recording messages. 39677c478bd9Sstevel@tonic-gate */ 39687c478bd9Sstevel@tonic-gate streams_msg_init(); 39697d692464Sdp 39707c478bd9Sstevel@tonic-gate /* 39717c478bd9Sstevel@tonic-gate * Initialize the ZSD framework in Zones so modules loaded henceforth 39727c478bd9Sstevel@tonic-gate * can register their callbacks. 39737c478bd9Sstevel@tonic-gate */ 39747c478bd9Sstevel@tonic-gate zone_zsd_init(); 3975f4b3ec61Sdh 39767c478bd9Sstevel@tonic-gate log_init(); 39777c478bd9Sstevel@tonic-gate taskq_init(); 39787c478bd9Sstevel@tonic-gate 39797d692464Sdp /* 39807d692464Sdp * Warn about invalid or dangerous values of kmem_flags. 39817d692464Sdp * Always warn about unsupported values. 39827d692464Sdp */ 39837d692464Sdp if (((kmem_flags & ~(KMF_AUDIT | KMF_DEADBEEF | KMF_REDZONE | 39847d692464Sdp KMF_CONTENTS | KMF_LITE)) != 0) || 39857d692464Sdp ((kmem_flags & KMF_LITE) && kmem_flags != KMF_LITE)) 39867d692464Sdp cmn_err(CE_WARN, "kmem_flags set to unsupported value 0x%x. " 39877d692464Sdp "See the Solaris Tunable Parameters Reference Manual.", 39887d692464Sdp kmem_flags); 39897d692464Sdp 39907d692464Sdp #ifdef DEBUG 39917d692464Sdp if ((kmem_flags & KMF_DEBUG) == 0) 39927d692464Sdp cmn_err(CE_NOTE, "kmem debugging disabled."); 39937d692464Sdp #else 39947d692464Sdp /* 39957d692464Sdp * For non-debug kernels, the only "normal" flags are 0, KMF_LITE, 39967d692464Sdp * KMF_REDZONE, and KMF_CONTENTS (the last because it is only enabled 39977d692464Sdp * if KMF_AUDIT is set). We should warn the user about the performance 39987d692464Sdp * penalty of KMF_AUDIT or KMF_DEADBEEF if they are set and KMF_LITE 39997d692464Sdp * isn't set (since that disables AUDIT). 40007d692464Sdp */ 40017d692464Sdp if (!(kmem_flags & KMF_LITE) && 40027d692464Sdp (kmem_flags & (KMF_AUDIT | KMF_DEADBEEF)) != 0) 40037d692464Sdp cmn_err(CE_WARN, "High-overhead kmem debugging features " 40047d692464Sdp "enabled (kmem_flags = 0x%x). Performance degradation " 40057d692464Sdp "and large memory overhead possible. See the Solaris " 40067d692464Sdp "Tunable Parameters Reference Manual.", kmem_flags); 40077d692464Sdp #endif /* not DEBUG */ 40087d692464Sdp 40097c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_magazine_enable, NULL, TQ_SLEEP); 40107c478bd9Sstevel@tonic-gate 40117c478bd9Sstevel@tonic-gate kmem_ready = 1; 40127c478bd9Sstevel@tonic-gate 40137c478bd9Sstevel@tonic-gate /* 40147c478bd9Sstevel@tonic-gate * Initialize the platform-specific aligned/DMA memory allocator. 40157c478bd9Sstevel@tonic-gate */ 40167c478bd9Sstevel@tonic-gate ka_init(); 40177c478bd9Sstevel@tonic-gate 40187c478bd9Sstevel@tonic-gate /* 40197c478bd9Sstevel@tonic-gate * Initialize 32-bit ID cache. 40207c478bd9Sstevel@tonic-gate */ 40217c478bd9Sstevel@tonic-gate id32_init(); 4022f4b3ec61Sdh 4023f4b3ec61Sdh /* 4024f4b3ec61Sdh * Initialize the networking stack so modules loaded can 4025f4b3ec61Sdh * register their callbacks. 4026f4b3ec61Sdh */ 4027f4b3ec61Sdh netstack_init(); 40287c478bd9Sstevel@tonic-gate } 40297c478bd9Sstevel@tonic-gate 4030b5fca8f8Stomee static void 4031b5fca8f8Stomee kmem_move_init(void) 4032b5fca8f8Stomee { 4033b5fca8f8Stomee kmem_defrag_cache = kmem_cache_create("kmem_defrag_cache", 4034b5fca8f8Stomee sizeof (kmem_defrag_t), 0, NULL, NULL, NULL, NULL, 4035b5fca8f8Stomee kmem_msb_arena, KMC_NOHASH); 4036b5fca8f8Stomee kmem_move_cache = kmem_cache_create("kmem_move_cache", 4037b5fca8f8Stomee sizeof (kmem_move_t), 0, NULL, NULL, NULL, NULL, 4038b5fca8f8Stomee kmem_msb_arena, KMC_NOHASH); 4039b5fca8f8Stomee 4040b5fca8f8Stomee /* 4041b5fca8f8Stomee * kmem guarantees that move callbacks are sequential and that even 4042b5fca8f8Stomee * across multiple caches no two moves ever execute simultaneously. 4043b5fca8f8Stomee * Move callbacks are processed on a separate taskq so that client code 4044b5fca8f8Stomee * does not interfere with internal maintenance tasks. 4045b5fca8f8Stomee */ 4046b5fca8f8Stomee kmem_move_taskq = taskq_create_instance("kmem_move_taskq", 0, 1, 4047b5fca8f8Stomee minclsyspri, 100, INT_MAX, TASKQ_PREPOPULATE); 4048b5fca8f8Stomee } 4049b5fca8f8Stomee 40507c478bd9Sstevel@tonic-gate void 40517c478bd9Sstevel@tonic-gate kmem_thread_init(void) 40527c478bd9Sstevel@tonic-gate { 4053b5fca8f8Stomee kmem_move_init(); 40547c478bd9Sstevel@tonic-gate kmem_taskq = taskq_create_instance("kmem_taskq", 0, 1, minclsyspri, 40557c478bd9Sstevel@tonic-gate 300, INT_MAX, TASKQ_PREPOPULATE); 40567c478bd9Sstevel@tonic-gate } 40577c478bd9Sstevel@tonic-gate 40587c478bd9Sstevel@tonic-gate void 40597c478bd9Sstevel@tonic-gate kmem_mp_init(void) 40607c478bd9Sstevel@tonic-gate { 40617c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 40627c478bd9Sstevel@tonic-gate register_cpu_setup_func(kmem_cpu_setup, NULL); 40637c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 40647c478bd9Sstevel@tonic-gate 40657c478bd9Sstevel@tonic-gate kmem_update_timeout(NULL); 40667c478bd9Sstevel@tonic-gate } 4067b5fca8f8Stomee 4068b5fca8f8Stomee /* 4069b5fca8f8Stomee * Return the slab of the allocated buffer, or NULL if the buffer is not 4070b5fca8f8Stomee * allocated. This function may be called with a known slab address to determine 4071b5fca8f8Stomee * whether or not the buffer is allocated, or with a NULL slab address to obtain 4072b5fca8f8Stomee * an allocated buffer's slab. 4073b5fca8f8Stomee */ 4074b5fca8f8Stomee static kmem_slab_t * 4075b5fca8f8Stomee kmem_slab_allocated(kmem_cache_t *cp, kmem_slab_t *sp, void *buf) 4076b5fca8f8Stomee { 4077b5fca8f8Stomee kmem_bufctl_t *bcp, *bufbcp; 4078b5fca8f8Stomee 4079b5fca8f8Stomee ASSERT(MUTEX_HELD(&cp->cache_lock)); 4080b5fca8f8Stomee ASSERT(sp == NULL || KMEM_SLAB_MEMBER(sp, buf)); 4081b5fca8f8Stomee 4082b5fca8f8Stomee if (cp->cache_flags & KMF_HASH) { 4083b5fca8f8Stomee for (bcp = *KMEM_HASH(cp, buf); 4084b5fca8f8Stomee (bcp != NULL) && (bcp->bc_addr != buf); 4085b5fca8f8Stomee bcp = bcp->bc_next) { 4086b5fca8f8Stomee continue; 4087b5fca8f8Stomee } 4088b5fca8f8Stomee ASSERT(sp != NULL && bcp != NULL ? sp == bcp->bc_slab : 1); 4089b5fca8f8Stomee return (bcp == NULL ? NULL : bcp->bc_slab); 4090b5fca8f8Stomee } 4091b5fca8f8Stomee 4092b5fca8f8Stomee if (sp == NULL) { 4093b5fca8f8Stomee sp = KMEM_SLAB(cp, buf); 4094b5fca8f8Stomee } 4095b5fca8f8Stomee bufbcp = KMEM_BUFCTL(cp, buf); 4096b5fca8f8Stomee for (bcp = sp->slab_head; 4097b5fca8f8Stomee (bcp != NULL) && (bcp != bufbcp); 4098b5fca8f8Stomee bcp = bcp->bc_next) { 4099b5fca8f8Stomee continue; 4100b5fca8f8Stomee } 4101b5fca8f8Stomee return (bcp == NULL ? sp : NULL); 4102b5fca8f8Stomee } 4103b5fca8f8Stomee 4104b5fca8f8Stomee static boolean_t 4105b5fca8f8Stomee kmem_slab_is_reclaimable(kmem_cache_t *cp, kmem_slab_t *sp, int flags) 4106b5fca8f8Stomee { 4107b5fca8f8Stomee long refcnt; 4108b5fca8f8Stomee 4109b5fca8f8Stomee ASSERT(cp->cache_defrag != NULL); 4110b5fca8f8Stomee 4111b5fca8f8Stomee /* If we're desperate, we don't care if the client said NO. */ 4112b5fca8f8Stomee refcnt = sp->slab_refcnt; 4113b5fca8f8Stomee if (flags & KMM_DESPERATE) { 4114b5fca8f8Stomee return (refcnt < sp->slab_chunks); /* any partial */ 4115b5fca8f8Stomee } 4116b5fca8f8Stomee 4117b5fca8f8Stomee if (sp->slab_flags & KMEM_SLAB_NOMOVE) { 4118b5fca8f8Stomee return (B_FALSE); 4119b5fca8f8Stomee } 4120b5fca8f8Stomee 4121b5fca8f8Stomee if (kmem_move_any_partial) { 4122b5fca8f8Stomee return (refcnt < sp->slab_chunks); 4123b5fca8f8Stomee } 4124b5fca8f8Stomee 4125b5fca8f8Stomee if ((refcnt == 1) && (refcnt < sp->slab_chunks)) { 4126b5fca8f8Stomee return (B_TRUE); 4127b5fca8f8Stomee } 4128b5fca8f8Stomee 4129b5fca8f8Stomee /* 4130b5fca8f8Stomee * The reclaim threshold is adjusted at each kmem_cache_scan() so that 4131b5fca8f8Stomee * slabs with a progressively higher percentage of used buffers can be 4132b5fca8f8Stomee * reclaimed until the cache as a whole is no longer fragmented. 4133b5fca8f8Stomee * 4134b5fca8f8Stomee * sp->slab_refcnt kmd_reclaim_numer 4135b5fca8f8Stomee * --------------- < ------------------ 4136b5fca8f8Stomee * sp->slab_chunks KMEM_VOID_FRACTION 4137b5fca8f8Stomee */ 4138b5fca8f8Stomee return ((refcnt * KMEM_VOID_FRACTION) < 4139b5fca8f8Stomee (sp->slab_chunks * cp->cache_defrag->kmd_reclaim_numer)); 4140b5fca8f8Stomee } 4141b5fca8f8Stomee 4142b5fca8f8Stomee static void * 4143b5fca8f8Stomee kmem_hunt_mag(kmem_cache_t *cp, kmem_magazine_t *m, int n, void *buf, 4144b5fca8f8Stomee void *tbuf) 4145b5fca8f8Stomee { 4146b5fca8f8Stomee int i; /* magazine round index */ 4147b5fca8f8Stomee 4148b5fca8f8Stomee for (i = 0; i < n; i++) { 4149b5fca8f8Stomee if (buf == m->mag_round[i]) { 4150b5fca8f8Stomee if (cp->cache_flags & KMF_BUFTAG) { 4151b5fca8f8Stomee (void) kmem_cache_free_debug(cp, tbuf, 4152b5fca8f8Stomee caller()); 4153b5fca8f8Stomee } 4154b5fca8f8Stomee m->mag_round[i] = tbuf; 4155b5fca8f8Stomee return (buf); 4156b5fca8f8Stomee } 4157b5fca8f8Stomee } 4158b5fca8f8Stomee 4159b5fca8f8Stomee return (NULL); 4160b5fca8f8Stomee } 4161b5fca8f8Stomee 4162b5fca8f8Stomee /* 4163b5fca8f8Stomee * Hunt the magazine layer for the given buffer. If found, the buffer is 4164b5fca8f8Stomee * removed from the magazine layer and returned, otherwise NULL is returned. 4165b5fca8f8Stomee * The state of the returned buffer is freed and constructed. 4166b5fca8f8Stomee */ 4167b5fca8f8Stomee static void * 4168b5fca8f8Stomee kmem_hunt_mags(kmem_cache_t *cp, void *buf) 4169b5fca8f8Stomee { 4170b5fca8f8Stomee kmem_cpu_cache_t *ccp; 4171b5fca8f8Stomee kmem_magazine_t *m; 4172b5fca8f8Stomee int cpu_seqid; 4173b5fca8f8Stomee int n; /* magazine rounds */ 4174b5fca8f8Stomee void *tbuf; /* temporary swap buffer */ 4175b5fca8f8Stomee 4176b5fca8f8Stomee ASSERT(MUTEX_NOT_HELD(&cp->cache_lock)); 4177b5fca8f8Stomee 4178b5fca8f8Stomee /* 4179b5fca8f8Stomee * Allocated a buffer to swap with the one we hope to pull out of a 4180b5fca8f8Stomee * magazine when found. 4181b5fca8f8Stomee */ 4182b5fca8f8Stomee tbuf = kmem_cache_alloc(cp, KM_NOSLEEP); 4183b5fca8f8Stomee if (tbuf == NULL) { 4184b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats.kms_hunt_alloc_fail); 4185b5fca8f8Stomee return (NULL); 4186b5fca8f8Stomee } 4187b5fca8f8Stomee if (tbuf == buf) { 4188b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats.kms_hunt_lucky); 4189b5fca8f8Stomee if (cp->cache_flags & KMF_BUFTAG) { 4190b5fca8f8Stomee (void) kmem_cache_free_debug(cp, buf, caller()); 4191b5fca8f8Stomee } 4192b5fca8f8Stomee return (buf); 4193b5fca8f8Stomee } 4194b5fca8f8Stomee 4195b5fca8f8Stomee /* Hunt the depot. */ 4196b5fca8f8Stomee mutex_enter(&cp->cache_depot_lock); 4197b5fca8f8Stomee n = cp->cache_magtype->mt_magsize; 4198b5fca8f8Stomee for (m = cp->cache_full.ml_list; m != NULL; m = m->mag_next) { 4199b5fca8f8Stomee if (kmem_hunt_mag(cp, m, n, buf, tbuf) != NULL) { 4200b5fca8f8Stomee mutex_exit(&cp->cache_depot_lock); 4201b5fca8f8Stomee return (buf); 4202b5fca8f8Stomee } 4203b5fca8f8Stomee } 4204b5fca8f8Stomee mutex_exit(&cp->cache_depot_lock); 4205b5fca8f8Stomee 4206b5fca8f8Stomee /* Hunt the per-CPU magazines. */ 4207b5fca8f8Stomee for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 4208b5fca8f8Stomee ccp = &cp->cache_cpu[cpu_seqid]; 4209b5fca8f8Stomee 4210b5fca8f8Stomee mutex_enter(&ccp->cc_lock); 4211b5fca8f8Stomee m = ccp->cc_loaded; 4212b5fca8f8Stomee n = ccp->cc_rounds; 4213b5fca8f8Stomee if (kmem_hunt_mag(cp, m, n, buf, tbuf) != NULL) { 4214b5fca8f8Stomee mutex_exit(&ccp->cc_lock); 4215b5fca8f8Stomee return (buf); 4216b5fca8f8Stomee } 4217b5fca8f8Stomee m = ccp->cc_ploaded; 4218b5fca8f8Stomee n = ccp->cc_prounds; 4219b5fca8f8Stomee if (kmem_hunt_mag(cp, m, n, buf, tbuf) != NULL) { 4220b5fca8f8Stomee mutex_exit(&ccp->cc_lock); 4221b5fca8f8Stomee return (buf); 4222b5fca8f8Stomee } 4223b5fca8f8Stomee mutex_exit(&ccp->cc_lock); 4224b5fca8f8Stomee } 4225b5fca8f8Stomee 4226b5fca8f8Stomee kmem_cache_free(cp, tbuf); 4227b5fca8f8Stomee return (NULL); 4228b5fca8f8Stomee } 4229b5fca8f8Stomee 4230b5fca8f8Stomee /* 4231b5fca8f8Stomee * May be called from the kmem_move_taskq, from kmem_cache_move_notify_task(), 4232b5fca8f8Stomee * or when the buffer is freed. 4233b5fca8f8Stomee */ 4234b5fca8f8Stomee static void 4235b5fca8f8Stomee kmem_slab_move_yes(kmem_cache_t *cp, kmem_slab_t *sp, void *from_buf) 4236b5fca8f8Stomee { 4237b5fca8f8Stomee ASSERT(MUTEX_HELD(&cp->cache_lock)); 4238b5fca8f8Stomee ASSERT(KMEM_SLAB_MEMBER(sp, from_buf)); 4239b5fca8f8Stomee 4240b5fca8f8Stomee if (!KMEM_SLAB_IS_PARTIAL(sp)) { 4241b5fca8f8Stomee return; 4242b5fca8f8Stomee } 4243b5fca8f8Stomee 4244b5fca8f8Stomee if (sp->slab_flags & KMEM_SLAB_NOMOVE) { 4245b5fca8f8Stomee if (KMEM_SLAB_OFFSET(sp, from_buf) == sp->slab_stuck_offset) { 4246b5fca8f8Stomee avl_remove(&cp->cache_partial_slabs, sp); 4247b5fca8f8Stomee sp->slab_flags &= ~KMEM_SLAB_NOMOVE; 4248b5fca8f8Stomee sp->slab_stuck_offset = (uint32_t)-1; 4249b5fca8f8Stomee avl_add(&cp->cache_partial_slabs, sp); 4250b5fca8f8Stomee } 4251b5fca8f8Stomee } else { 4252b5fca8f8Stomee sp->slab_later_count = 0; 4253b5fca8f8Stomee sp->slab_stuck_offset = (uint32_t)-1; 4254b5fca8f8Stomee } 4255b5fca8f8Stomee } 4256b5fca8f8Stomee 4257b5fca8f8Stomee static void 4258b5fca8f8Stomee kmem_slab_move_no(kmem_cache_t *cp, kmem_slab_t *sp, void *from_buf) 4259b5fca8f8Stomee { 4260b5fca8f8Stomee ASSERT(taskq_member(kmem_move_taskq, curthread)); 4261b5fca8f8Stomee ASSERT(MUTEX_HELD(&cp->cache_lock)); 4262b5fca8f8Stomee ASSERT(KMEM_SLAB_MEMBER(sp, from_buf)); 4263b5fca8f8Stomee 4264b5fca8f8Stomee if (!KMEM_SLAB_IS_PARTIAL(sp)) { 4265b5fca8f8Stomee return; 4266b5fca8f8Stomee } 4267b5fca8f8Stomee 4268b5fca8f8Stomee avl_remove(&cp->cache_partial_slabs, sp); 4269b5fca8f8Stomee sp->slab_later_count = 0; 4270b5fca8f8Stomee sp->slab_flags |= KMEM_SLAB_NOMOVE; 4271b5fca8f8Stomee sp->slab_stuck_offset = KMEM_SLAB_OFFSET(sp, from_buf); 4272b5fca8f8Stomee avl_add(&cp->cache_partial_slabs, sp); 4273b5fca8f8Stomee } 4274b5fca8f8Stomee 4275b5fca8f8Stomee static void kmem_move_end(kmem_cache_t *, kmem_move_t *); 4276b5fca8f8Stomee 4277b5fca8f8Stomee /* 4278b5fca8f8Stomee * The move callback takes two buffer addresses, the buffer to be moved, and a 4279b5fca8f8Stomee * newly allocated and constructed buffer selected by kmem as the destination. 4280b5fca8f8Stomee * It also takes the size of the buffer and an optional user argument specified 4281b5fca8f8Stomee * at cache creation time. kmem guarantees that the buffer to be moved has not 4282b5fca8f8Stomee * been unmapped by the virtual memory subsystem. Beyond that, it cannot 4283b5fca8f8Stomee * guarantee the present whereabouts of the buffer to be moved, so it is up to 4284b5fca8f8Stomee * the client to safely determine whether or not it is still using the buffer. 4285b5fca8f8Stomee * The client must not free either of the buffers passed to the move callback, 4286b5fca8f8Stomee * since kmem wants to free them directly to the slab layer. The client response 4287b5fca8f8Stomee * tells kmem which of the two buffers to free: 4288b5fca8f8Stomee * 4289b5fca8f8Stomee * YES kmem frees the old buffer (the move was successful) 4290b5fca8f8Stomee * NO kmem frees the new buffer, marks the slab of the old buffer 4291b5fca8f8Stomee * non-reclaimable to avoid bothering the client again 4292b5fca8f8Stomee * LATER kmem frees the new buffer, increments slab_later_count 4293b5fca8f8Stomee * DONT_KNOW kmem frees the new buffer, searches mags for the old buffer 4294b5fca8f8Stomee * DONT_NEED kmem frees both the old buffer and the new buffer 4295b5fca8f8Stomee * 4296b5fca8f8Stomee * The pending callback argument now being processed contains both of the 4297b5fca8f8Stomee * buffers (old and new) passed to the move callback function, the slab of the 4298b5fca8f8Stomee * old buffer, and flags related to the move request, such as whether or not the 4299b5fca8f8Stomee * system was desperate for memory. 4300b5fca8f8Stomee */ 4301b5fca8f8Stomee static void 4302b5fca8f8Stomee kmem_move_buffer(kmem_move_t *callback) 4303b5fca8f8Stomee { 4304b5fca8f8Stomee kmem_cbrc_t response; 4305b5fca8f8Stomee kmem_slab_t *sp = callback->kmm_from_slab; 4306b5fca8f8Stomee kmem_cache_t *cp = sp->slab_cache; 4307b5fca8f8Stomee boolean_t free_on_slab; 4308b5fca8f8Stomee 4309b5fca8f8Stomee ASSERT(taskq_member(kmem_move_taskq, curthread)); 4310b5fca8f8Stomee ASSERT(MUTEX_NOT_HELD(&cp->cache_lock)); 4311b5fca8f8Stomee ASSERT(KMEM_SLAB_MEMBER(sp, callback->kmm_from_buf)); 4312b5fca8f8Stomee 4313b5fca8f8Stomee /* 4314b5fca8f8Stomee * The number of allocated buffers on the slab may have changed since we 4315b5fca8f8Stomee * last checked the slab's reclaimability (when the pending move was 4316b5fca8f8Stomee * enqueued), or the client may have responded NO when asked to move 4317b5fca8f8Stomee * another buffer on the same slab. 4318b5fca8f8Stomee */ 4319b5fca8f8Stomee if (!kmem_slab_is_reclaimable(cp, sp, callback->kmm_flags)) { 4320b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats.kms_no_longer_reclaimable); 4321b5fca8f8Stomee KMEM_STAT_COND_ADD((callback->kmm_flags & KMM_NOTIFY), 4322b5fca8f8Stomee kmem_move_stats.kms_notify_no_longer_reclaimable); 4323b5fca8f8Stomee kmem_slab_free(cp, callback->kmm_to_buf); 4324b5fca8f8Stomee kmem_move_end(cp, callback); 4325b5fca8f8Stomee return; 4326b5fca8f8Stomee } 4327b5fca8f8Stomee 4328b5fca8f8Stomee /* 4329b5fca8f8Stomee * Hunting magazines is expensive, so we'll wait to do that until the 4330b5fca8f8Stomee * client responds KMEM_CBRC_DONT_KNOW. However, checking the slab layer 4331b5fca8f8Stomee * is cheap, so we might as well do that here in case we can avoid 4332b5fca8f8Stomee * bothering the client. 4333b5fca8f8Stomee */ 4334b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4335b5fca8f8Stomee free_on_slab = (kmem_slab_allocated(cp, sp, 4336b5fca8f8Stomee callback->kmm_from_buf) == NULL); 4337b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4338b5fca8f8Stomee 4339b5fca8f8Stomee if (free_on_slab) { 4340b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats.kms_hunt_found_slab); 4341b5fca8f8Stomee kmem_slab_free(cp, callback->kmm_to_buf); 4342b5fca8f8Stomee kmem_move_end(cp, callback); 4343b5fca8f8Stomee return; 4344b5fca8f8Stomee } 4345b5fca8f8Stomee 4346b5fca8f8Stomee if (cp->cache_flags & KMF_BUFTAG) { 4347b5fca8f8Stomee /* 4348b5fca8f8Stomee * Make kmem_cache_alloc_debug() apply the constructor for us. 4349b5fca8f8Stomee */ 4350b5fca8f8Stomee if (kmem_cache_alloc_debug(cp, callback->kmm_to_buf, 4351b5fca8f8Stomee KM_NOSLEEP, 1, caller()) != 0) { 4352b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats.kms_alloc_fail); 4353b5fca8f8Stomee kmem_move_end(cp, callback); 4354b5fca8f8Stomee return; 4355b5fca8f8Stomee } 4356b5fca8f8Stomee } else if (cp->cache_constructor != NULL && 4357b5fca8f8Stomee cp->cache_constructor(callback->kmm_to_buf, cp->cache_private, 4358b5fca8f8Stomee KM_NOSLEEP) != 0) { 4359b5fca8f8Stomee atomic_add_64(&cp->cache_alloc_fail, 1); 4360b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats.kms_constructor_fail); 4361b5fca8f8Stomee kmem_slab_free(cp, callback->kmm_to_buf); 4362b5fca8f8Stomee kmem_move_end(cp, callback); 4363b5fca8f8Stomee return; 4364b5fca8f8Stomee } 4365b5fca8f8Stomee 4366b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats.kms_callbacks); 4367b5fca8f8Stomee KMEM_STAT_COND_ADD((callback->kmm_flags & KMM_NOTIFY), 4368b5fca8f8Stomee kmem_move_stats.kms_notify_callbacks); 4369b5fca8f8Stomee cp->cache_defrag->kmd_callbacks++; 4370b5fca8f8Stomee cp->cache_defrag->kmd_thread = curthread; 4371b5fca8f8Stomee cp->cache_defrag->kmd_from_buf = callback->kmm_from_buf; 4372b5fca8f8Stomee cp->cache_defrag->kmd_to_buf = callback->kmm_to_buf; 4373b5fca8f8Stomee DTRACE_PROBE2(kmem__move__start, kmem_cache_t *, cp, kmem_move_t *, 4374b5fca8f8Stomee callback); 4375b5fca8f8Stomee 4376b5fca8f8Stomee response = cp->cache_move(callback->kmm_from_buf, 4377b5fca8f8Stomee callback->kmm_to_buf, cp->cache_bufsize, cp->cache_private); 4378b5fca8f8Stomee 4379b5fca8f8Stomee DTRACE_PROBE3(kmem__move__end, kmem_cache_t *, cp, kmem_move_t *, 4380b5fca8f8Stomee callback, kmem_cbrc_t, response); 4381b5fca8f8Stomee cp->cache_defrag->kmd_thread = NULL; 4382b5fca8f8Stomee cp->cache_defrag->kmd_from_buf = NULL; 4383b5fca8f8Stomee cp->cache_defrag->kmd_to_buf = NULL; 4384b5fca8f8Stomee 4385b5fca8f8Stomee if (response == KMEM_CBRC_YES) { 4386b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats.kms_yes); 4387b5fca8f8Stomee cp->cache_defrag->kmd_yes++; 4388b5fca8f8Stomee kmem_slab_free_constructed(cp, callback->kmm_from_buf, B_FALSE); 4389b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4390b5fca8f8Stomee kmem_slab_move_yes(cp, sp, callback->kmm_from_buf); 4391b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4392b5fca8f8Stomee kmem_move_end(cp, callback); 4393b5fca8f8Stomee return; 4394b5fca8f8Stomee } 4395b5fca8f8Stomee 4396b5fca8f8Stomee switch (response) { 4397b5fca8f8Stomee case KMEM_CBRC_NO: 4398b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats.kms_no); 4399b5fca8f8Stomee cp->cache_defrag->kmd_no++; 4400b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4401b5fca8f8Stomee kmem_slab_move_no(cp, sp, callback->kmm_from_buf); 4402b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4403b5fca8f8Stomee break; 4404b5fca8f8Stomee case KMEM_CBRC_LATER: 4405b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats.kms_later); 4406b5fca8f8Stomee cp->cache_defrag->kmd_later++; 4407b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4408b5fca8f8Stomee if (!KMEM_SLAB_IS_PARTIAL(sp)) { 4409b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4410b5fca8f8Stomee break; 4411b5fca8f8Stomee } 4412b5fca8f8Stomee 4413b5fca8f8Stomee if (++sp->slab_later_count >= KMEM_DISBELIEF) { 4414b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats.kms_disbelief); 4415b5fca8f8Stomee kmem_slab_move_no(cp, sp, callback->kmm_from_buf); 4416b5fca8f8Stomee } else if (!(sp->slab_flags & KMEM_SLAB_NOMOVE)) { 4417b5fca8f8Stomee sp->slab_stuck_offset = KMEM_SLAB_OFFSET(sp, 4418b5fca8f8Stomee callback->kmm_from_buf); 4419b5fca8f8Stomee } 4420b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4421b5fca8f8Stomee break; 4422b5fca8f8Stomee case KMEM_CBRC_DONT_NEED: 4423b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats.kms_dont_need); 4424b5fca8f8Stomee cp->cache_defrag->kmd_dont_need++; 4425b5fca8f8Stomee kmem_slab_free_constructed(cp, callback->kmm_from_buf, B_FALSE); 4426b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4427b5fca8f8Stomee kmem_slab_move_yes(cp, sp, callback->kmm_from_buf); 4428b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4429b5fca8f8Stomee break; 4430b5fca8f8Stomee case KMEM_CBRC_DONT_KNOW: 4431b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats.kms_dont_know); 4432b5fca8f8Stomee cp->cache_defrag->kmd_dont_know++; 4433b5fca8f8Stomee if (kmem_hunt_mags(cp, callback->kmm_from_buf) != NULL) { 4434b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats.kms_hunt_found_mag); 4435b5fca8f8Stomee cp->cache_defrag->kmd_hunt_found++; 4436b5fca8f8Stomee kmem_slab_free_constructed(cp, callback->kmm_from_buf, 4437b5fca8f8Stomee B_TRUE); 4438b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4439b5fca8f8Stomee kmem_slab_move_yes(cp, sp, callback->kmm_from_buf); 4440b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4441b5fca8f8Stomee } 4442b5fca8f8Stomee break; 4443b5fca8f8Stomee default: 4444b5fca8f8Stomee panic("'%s' (%p) unexpected move callback response %d\n", 4445b5fca8f8Stomee cp->cache_name, (void *)cp, response); 4446b5fca8f8Stomee } 4447b5fca8f8Stomee 4448b5fca8f8Stomee kmem_slab_free_constructed(cp, callback->kmm_to_buf, B_FALSE); 4449b5fca8f8Stomee kmem_move_end(cp, callback); 4450b5fca8f8Stomee } 4451b5fca8f8Stomee 4452b5fca8f8Stomee /* Return B_FALSE if there is insufficient memory for the move request. */ 4453b5fca8f8Stomee static boolean_t 4454b5fca8f8Stomee kmem_move_begin(kmem_cache_t *cp, kmem_slab_t *sp, void *buf, int flags) 4455b5fca8f8Stomee { 4456b5fca8f8Stomee void *to_buf; 4457b5fca8f8Stomee avl_index_t index; 4458b5fca8f8Stomee kmem_move_t *callback, *pending; 4459b5fca8f8Stomee 4460b5fca8f8Stomee ASSERT(taskq_member(kmem_taskq, curthread)); 4461b5fca8f8Stomee ASSERT(MUTEX_NOT_HELD(&cp->cache_lock)); 4462b5fca8f8Stomee ASSERT(sp->slab_flags & KMEM_SLAB_MOVE_PENDING); 4463b5fca8f8Stomee 4464b5fca8f8Stomee callback = kmem_cache_alloc(kmem_move_cache, KM_NOSLEEP); 4465b5fca8f8Stomee if (callback == NULL) { 4466b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats.kms_callback_alloc_fail); 4467b5fca8f8Stomee return (B_FALSE); 4468b5fca8f8Stomee } 4469b5fca8f8Stomee 4470b5fca8f8Stomee callback->kmm_from_slab = sp; 4471b5fca8f8Stomee callback->kmm_from_buf = buf; 4472b5fca8f8Stomee callback->kmm_flags = flags; 4473b5fca8f8Stomee 4474b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4475b5fca8f8Stomee 4476b5fca8f8Stomee if (avl_numnodes(&cp->cache_partial_slabs) <= 1) { 4477b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4478b5fca8f8Stomee kmem_cache_free(kmem_move_cache, callback); 4479b5fca8f8Stomee return (B_TRUE); /* there is no need for the move request */ 4480b5fca8f8Stomee } 4481b5fca8f8Stomee 4482b5fca8f8Stomee pending = avl_find(&cp->cache_defrag->kmd_moves_pending, buf, &index); 4483b5fca8f8Stomee if (pending != NULL) { 4484b5fca8f8Stomee /* 4485b5fca8f8Stomee * If the move is already pending and we're desperate now, 4486b5fca8f8Stomee * update the move flags. 4487b5fca8f8Stomee */ 4488b5fca8f8Stomee if (flags & KMM_DESPERATE) { 4489b5fca8f8Stomee pending->kmm_flags |= KMM_DESPERATE; 4490b5fca8f8Stomee } 4491b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4492b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats.kms_already_pending); 4493b5fca8f8Stomee kmem_cache_free(kmem_move_cache, callback); 4494b5fca8f8Stomee return (B_TRUE); 4495b5fca8f8Stomee } 4496b5fca8f8Stomee 4497b5fca8f8Stomee to_buf = kmem_slab_alloc_impl(cp, avl_first(&cp->cache_partial_slabs)); 4498b5fca8f8Stomee callback->kmm_to_buf = to_buf; 4499b5fca8f8Stomee avl_insert(&cp->cache_defrag->kmd_moves_pending, callback, index); 4500b5fca8f8Stomee 4501b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4502b5fca8f8Stomee 4503b5fca8f8Stomee if (!taskq_dispatch(kmem_move_taskq, (task_func_t *)kmem_move_buffer, 4504b5fca8f8Stomee callback, TQ_NOSLEEP)) { 450525e2c9cfStomee KMEM_STAT_ADD(kmem_move_stats.kms_callback_taskq_fail); 4506b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4507b5fca8f8Stomee avl_remove(&cp->cache_defrag->kmd_moves_pending, callback); 4508b5fca8f8Stomee mutex_exit(&cp->cache_lock); 450925e2c9cfStomee kmem_slab_free(cp, to_buf); 4510b5fca8f8Stomee kmem_cache_free(kmem_move_cache, callback); 4511b5fca8f8Stomee return (B_FALSE); 4512b5fca8f8Stomee } 4513b5fca8f8Stomee 4514b5fca8f8Stomee return (B_TRUE); 4515b5fca8f8Stomee } 4516b5fca8f8Stomee 4517b5fca8f8Stomee static void 4518b5fca8f8Stomee kmem_move_end(kmem_cache_t *cp, kmem_move_t *callback) 4519b5fca8f8Stomee { 4520b5fca8f8Stomee avl_index_t index; 4521b5fca8f8Stomee 4522b5fca8f8Stomee ASSERT(cp->cache_defrag != NULL); 4523b5fca8f8Stomee ASSERT(taskq_member(kmem_move_taskq, curthread)); 4524b5fca8f8Stomee ASSERT(MUTEX_NOT_HELD(&cp->cache_lock)); 4525b5fca8f8Stomee 4526b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4527b5fca8f8Stomee VERIFY(avl_find(&cp->cache_defrag->kmd_moves_pending, 4528b5fca8f8Stomee callback->kmm_from_buf, &index) != NULL); 4529b5fca8f8Stomee avl_remove(&cp->cache_defrag->kmd_moves_pending, callback); 4530b5fca8f8Stomee if (avl_is_empty(&cp->cache_defrag->kmd_moves_pending)) { 4531b5fca8f8Stomee list_t *deadlist = &cp->cache_defrag->kmd_deadlist; 4532b5fca8f8Stomee kmem_slab_t *sp; 4533b5fca8f8Stomee 4534b5fca8f8Stomee /* 4535b5fca8f8Stomee * The last pending move completed. Release all slabs from the 4536b5fca8f8Stomee * front of the dead list except for any slab at the tail that 4537b5fca8f8Stomee * needs to be released from the context of kmem_move_buffers(). 4538b5fca8f8Stomee * kmem deferred unmapping the buffers on these slabs in order 4539b5fca8f8Stomee * to guarantee that buffers passed to the move callback have 4540b5fca8f8Stomee * been touched only by kmem or by the client itself. 4541b5fca8f8Stomee */ 4542b5fca8f8Stomee while ((sp = list_remove_head(deadlist)) != NULL) { 4543b5fca8f8Stomee if (sp->slab_flags & KMEM_SLAB_MOVE_PENDING) { 4544b5fca8f8Stomee list_insert_tail(deadlist, sp); 4545b5fca8f8Stomee break; 4546b5fca8f8Stomee } 4547b5fca8f8Stomee cp->cache_defrag->kmd_deadcount--; 4548b5fca8f8Stomee cp->cache_slab_destroy++; 4549b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4550b5fca8f8Stomee kmem_slab_destroy(cp, sp); 4551b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats.kms_dead_slabs_freed); 4552b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4553b5fca8f8Stomee } 4554b5fca8f8Stomee } 4555b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4556b5fca8f8Stomee kmem_cache_free(kmem_move_cache, callback); 4557b5fca8f8Stomee } 4558b5fca8f8Stomee 4559b5fca8f8Stomee /* 4560b5fca8f8Stomee * Move buffers from least used slabs first by scanning backwards from the end 4561b5fca8f8Stomee * of the partial slab list. Scan at most max_scan candidate slabs and move 4562b5fca8f8Stomee * buffers from at most max_slabs slabs (0 for all partial slabs in both cases). 4563b5fca8f8Stomee * If desperate to reclaim memory, move buffers from any partial slab, otherwise 4564b5fca8f8Stomee * skip slabs with a ratio of allocated buffers at or above the current 4565b5fca8f8Stomee * threshold. Return the number of unskipped slabs (at most max_slabs, -1 if the 4566b5fca8f8Stomee * scan is aborted) so that the caller can adjust the reclaimability threshold 4567b5fca8f8Stomee * depending on how many reclaimable slabs it finds. 4568b5fca8f8Stomee * 4569b5fca8f8Stomee * kmem_move_buffers() drops and reacquires cache_lock every time it issues a 4570b5fca8f8Stomee * move request, since it is not valid for kmem_move_begin() to call 4571b5fca8f8Stomee * kmem_cache_alloc() or taskq_dispatch() with cache_lock held. 4572b5fca8f8Stomee */ 4573b5fca8f8Stomee static int 4574b5fca8f8Stomee kmem_move_buffers(kmem_cache_t *cp, size_t max_scan, size_t max_slabs, 4575b5fca8f8Stomee int flags) 4576b5fca8f8Stomee { 4577b5fca8f8Stomee kmem_slab_t *sp; 4578b5fca8f8Stomee void *buf; 4579b5fca8f8Stomee int i, j; /* slab index, buffer index */ 4580b5fca8f8Stomee int s; /* reclaimable slabs */ 4581b5fca8f8Stomee int b; /* allocated (movable) buffers on reclaimable slab */ 4582b5fca8f8Stomee boolean_t success; 4583b5fca8f8Stomee int refcnt; 4584b5fca8f8Stomee int nomove; 4585b5fca8f8Stomee 4586b5fca8f8Stomee ASSERT(taskq_member(kmem_taskq, curthread)); 4587b5fca8f8Stomee ASSERT(MUTEX_HELD(&cp->cache_lock)); 4588b5fca8f8Stomee ASSERT(kmem_move_cache != NULL); 4589b5fca8f8Stomee ASSERT(cp->cache_move != NULL && cp->cache_defrag != NULL); 4590b5fca8f8Stomee ASSERT(avl_numnodes(&cp->cache_partial_slabs) > 1); 4591b5fca8f8Stomee 4592b5fca8f8Stomee if (kmem_move_blocked) { 4593b5fca8f8Stomee return (0); 4594b5fca8f8Stomee } 4595b5fca8f8Stomee 4596b5fca8f8Stomee if (kmem_move_fulltilt) { 4597b5fca8f8Stomee max_slabs = 0; 4598b5fca8f8Stomee flags |= KMM_DESPERATE; 4599b5fca8f8Stomee } 4600b5fca8f8Stomee 4601b5fca8f8Stomee if (max_scan == 0 || (flags & KMM_DESPERATE)) { 4602b5fca8f8Stomee /* 4603b5fca8f8Stomee * Scan as many slabs as needed to find the desired number of 4604b5fca8f8Stomee * candidate slabs. 4605b5fca8f8Stomee */ 4606b5fca8f8Stomee max_scan = (size_t)-1; 4607b5fca8f8Stomee } 4608b5fca8f8Stomee 4609b5fca8f8Stomee if (max_slabs == 0 || (flags & KMM_DESPERATE)) { 4610b5fca8f8Stomee /* Find as many candidate slabs as possible. */ 4611b5fca8f8Stomee max_slabs = (size_t)-1; 4612b5fca8f8Stomee } 4613b5fca8f8Stomee 4614b5fca8f8Stomee sp = avl_last(&cp->cache_partial_slabs); 4615b5fca8f8Stomee ASSERT(sp != NULL && KMEM_SLAB_IS_PARTIAL(sp)); 4616b5fca8f8Stomee for (i = 0, s = 0; (i < max_scan) && (s < max_slabs) && 4617b5fca8f8Stomee (sp != avl_first(&cp->cache_partial_slabs)); 4618b5fca8f8Stomee sp = AVL_PREV(&cp->cache_partial_slabs, sp), i++) { 4619b5fca8f8Stomee 4620b5fca8f8Stomee if (!kmem_slab_is_reclaimable(cp, sp, flags)) { 4621b5fca8f8Stomee continue; 4622b5fca8f8Stomee } 4623b5fca8f8Stomee s++; 4624b5fca8f8Stomee 4625b5fca8f8Stomee /* Look for allocated buffers to move. */ 4626b5fca8f8Stomee for (j = 0, b = 0, buf = sp->slab_base; 4627b5fca8f8Stomee (j < sp->slab_chunks) && (b < sp->slab_refcnt); 4628b5fca8f8Stomee buf = (((char *)buf) + cp->cache_chunksize), j++) { 4629b5fca8f8Stomee 4630b5fca8f8Stomee if (kmem_slab_allocated(cp, sp, buf) == NULL) { 4631b5fca8f8Stomee continue; 4632b5fca8f8Stomee } 4633b5fca8f8Stomee 4634b5fca8f8Stomee b++; 4635b5fca8f8Stomee 4636b5fca8f8Stomee /* 4637b5fca8f8Stomee * Prevent the slab from being destroyed while we drop 4638b5fca8f8Stomee * cache_lock and while the pending move is not yet 4639b5fca8f8Stomee * registered. Flag the pending move while 4640b5fca8f8Stomee * kmd_moves_pending may still be empty, since we can't 4641b5fca8f8Stomee * yet rely on a non-zero pending move count to prevent 4642b5fca8f8Stomee * the slab from being destroyed. 4643b5fca8f8Stomee */ 4644b5fca8f8Stomee ASSERT(!(sp->slab_flags & KMEM_SLAB_MOVE_PENDING)); 4645b5fca8f8Stomee sp->slab_flags |= KMEM_SLAB_MOVE_PENDING; 4646b5fca8f8Stomee /* 4647b5fca8f8Stomee * Recheck refcnt and nomove after reacquiring the lock, 4648b5fca8f8Stomee * since these control the order of partial slabs, and 4649b5fca8f8Stomee * we want to know if we can pick up the scan where we 4650b5fca8f8Stomee * left off. 4651b5fca8f8Stomee */ 4652b5fca8f8Stomee refcnt = sp->slab_refcnt; 4653b5fca8f8Stomee nomove = (sp->slab_flags & KMEM_SLAB_NOMOVE); 4654b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4655b5fca8f8Stomee 4656b5fca8f8Stomee success = kmem_move_begin(cp, sp, buf, flags); 4657b5fca8f8Stomee 4658b5fca8f8Stomee /* 4659b5fca8f8Stomee * Now, before the lock is reacquired, kmem could 4660b5fca8f8Stomee * process all pending move requests and purge the 4661b5fca8f8Stomee * deadlist, so that upon reacquiring the lock, sp has 4662b5fca8f8Stomee * been remapped. Therefore, the KMEM_SLAB_MOVE_PENDING 4663b5fca8f8Stomee * flag causes the slab to be put at the end of the 4664b5fca8f8Stomee * deadlist and prevents it from being purged, since we 4665b5fca8f8Stomee * plan to destroy it here after reacquiring the lock. 4666b5fca8f8Stomee */ 4667b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4668b5fca8f8Stomee ASSERT(sp->slab_flags & KMEM_SLAB_MOVE_PENDING); 4669b5fca8f8Stomee sp->slab_flags &= ~KMEM_SLAB_MOVE_PENDING; 4670b5fca8f8Stomee 4671b5fca8f8Stomee /* 4672b5fca8f8Stomee * Destroy the slab now if it was completely freed while 4673b5fca8f8Stomee * we dropped cache_lock. 4674b5fca8f8Stomee */ 4675b5fca8f8Stomee if (sp->slab_refcnt == 0) { 4676b5fca8f8Stomee list_t *deadlist = 4677b5fca8f8Stomee &cp->cache_defrag->kmd_deadlist; 4678b5fca8f8Stomee 4679b5fca8f8Stomee ASSERT(!list_is_empty(deadlist)); 4680b5fca8f8Stomee ASSERT(list_link_active((list_node_t *) 4681b5fca8f8Stomee &sp->slab_link)); 4682b5fca8f8Stomee 4683b5fca8f8Stomee list_remove(deadlist, sp); 4684b5fca8f8Stomee cp->cache_defrag->kmd_deadcount--; 4685b5fca8f8Stomee cp->cache_slab_destroy++; 4686b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4687b5fca8f8Stomee kmem_slab_destroy(cp, sp); 4688b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats. 4689b5fca8f8Stomee kms_dead_slabs_freed); 4690b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats. 4691b5fca8f8Stomee kms_endscan_slab_destroyed); 4692b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4693b5fca8f8Stomee /* 4694b5fca8f8Stomee * Since we can't pick up the scan where we left 4695b5fca8f8Stomee * off, abort the scan and say nothing about the 4696b5fca8f8Stomee * number of reclaimable slabs. 4697b5fca8f8Stomee */ 4698b5fca8f8Stomee return (-1); 4699b5fca8f8Stomee } 4700b5fca8f8Stomee 4701b5fca8f8Stomee if (!success) { 4702b5fca8f8Stomee /* 4703b5fca8f8Stomee * Abort the scan if there is not enough memory 4704b5fca8f8Stomee * for the request and say nothing about the 4705b5fca8f8Stomee * number of reclaimable slabs. 4706b5fca8f8Stomee */ 4707b5fca8f8Stomee KMEM_STAT_ADD( 4708b5fca8f8Stomee kmem_move_stats.kms_endscan_nomem); 4709b5fca8f8Stomee return (-1); 4710b5fca8f8Stomee } 4711b5fca8f8Stomee 4712b5fca8f8Stomee /* 4713b5fca8f8Stomee * The slab may have been completely allocated while the 4714b5fca8f8Stomee * lock was dropped. 4715b5fca8f8Stomee */ 4716b5fca8f8Stomee if (KMEM_SLAB_IS_ALL_USED(sp)) { 4717b5fca8f8Stomee KMEM_STAT_ADD( 4718b5fca8f8Stomee kmem_move_stats.kms_endscan_slab_all_used); 4719b5fca8f8Stomee return (-1); 4720b5fca8f8Stomee } 4721b5fca8f8Stomee 4722b5fca8f8Stomee /* 4723b5fca8f8Stomee * The slab's position changed while the lock was 4724b5fca8f8Stomee * dropped, so we don't know where we are in the 4725b5fca8f8Stomee * sequence any more. 4726b5fca8f8Stomee */ 4727b5fca8f8Stomee if (sp->slab_refcnt != refcnt) { 4728b5fca8f8Stomee KMEM_STAT_ADD( 4729b5fca8f8Stomee kmem_move_stats.kms_endscan_refcnt_changed); 4730b5fca8f8Stomee return (-1); 4731b5fca8f8Stomee } 4732b5fca8f8Stomee if ((sp->slab_flags & KMEM_SLAB_NOMOVE) != nomove) { 4733b5fca8f8Stomee KMEM_STAT_ADD( 4734b5fca8f8Stomee kmem_move_stats.kms_endscan_nomove_changed); 4735b5fca8f8Stomee return (-1); 4736b5fca8f8Stomee } 4737b5fca8f8Stomee 4738b5fca8f8Stomee /* 4739b5fca8f8Stomee * Generating a move request allocates a destination 4740b5fca8f8Stomee * buffer from the slab layer, bumping the first slab if 4741b5fca8f8Stomee * it is completely allocated. 4742b5fca8f8Stomee */ 4743b5fca8f8Stomee ASSERT(!avl_is_empty(&cp->cache_partial_slabs)); 4744b5fca8f8Stomee if (sp == avl_first(&cp->cache_partial_slabs)) { 4745b5fca8f8Stomee goto end_scan; 4746b5fca8f8Stomee } 4747b5fca8f8Stomee } 4748b5fca8f8Stomee } 4749b5fca8f8Stomee end_scan: 4750b5fca8f8Stomee 4751b5fca8f8Stomee KMEM_STAT_COND_ADD(sp == avl_first(&cp->cache_partial_slabs), 4752b5fca8f8Stomee kmem_move_stats.kms_endscan_freelist); 4753b5fca8f8Stomee 4754b5fca8f8Stomee return (s); 4755b5fca8f8Stomee } 4756b5fca8f8Stomee 4757b5fca8f8Stomee typedef struct kmem_move_notify_args { 4758b5fca8f8Stomee kmem_cache_t *kmna_cache; 4759b5fca8f8Stomee void *kmna_buf; 4760b5fca8f8Stomee } kmem_move_notify_args_t; 4761b5fca8f8Stomee 4762b5fca8f8Stomee static void 4763b5fca8f8Stomee kmem_cache_move_notify_task(void *arg) 4764b5fca8f8Stomee { 4765b5fca8f8Stomee kmem_move_notify_args_t *args = arg; 4766b5fca8f8Stomee kmem_cache_t *cp = args->kmna_cache; 4767b5fca8f8Stomee void *buf = args->kmna_buf; 4768b5fca8f8Stomee kmem_slab_t *sp; 4769b5fca8f8Stomee 4770b5fca8f8Stomee ASSERT(taskq_member(kmem_taskq, curthread)); 4771b5fca8f8Stomee ASSERT(list_link_active(&cp->cache_link)); 4772b5fca8f8Stomee 4773b5fca8f8Stomee kmem_free(args, sizeof (kmem_move_notify_args_t)); 4774b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4775b5fca8f8Stomee sp = kmem_slab_allocated(cp, NULL, buf); 4776b5fca8f8Stomee 4777b5fca8f8Stomee /* Ignore the notification if the buffer is no longer allocated. */ 4778b5fca8f8Stomee if (sp == NULL) { 4779b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4780b5fca8f8Stomee return; 4781b5fca8f8Stomee } 4782b5fca8f8Stomee 4783b5fca8f8Stomee /* Ignore the notification if there's no reason to move the buffer. */ 4784b5fca8f8Stomee if (avl_numnodes(&cp->cache_partial_slabs) > 1) { 4785b5fca8f8Stomee /* 4786b5fca8f8Stomee * So far the notification is not ignored. Ignore the 4787b5fca8f8Stomee * notification if the slab is not marked by an earlier refusal 4788b5fca8f8Stomee * to move a buffer. 4789b5fca8f8Stomee */ 4790b5fca8f8Stomee if (!(sp->slab_flags & KMEM_SLAB_NOMOVE) && 4791b5fca8f8Stomee (sp->slab_later_count == 0)) { 4792b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4793b5fca8f8Stomee return; 4794b5fca8f8Stomee } 4795b5fca8f8Stomee 4796b5fca8f8Stomee kmem_slab_move_yes(cp, sp, buf); 4797b5fca8f8Stomee ASSERT(!(sp->slab_flags & KMEM_SLAB_MOVE_PENDING)); 4798b5fca8f8Stomee sp->slab_flags |= KMEM_SLAB_MOVE_PENDING; 4799b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4800b5fca8f8Stomee /* see kmem_move_buffers() about dropping the lock */ 4801b5fca8f8Stomee (void) kmem_move_begin(cp, sp, buf, KMM_NOTIFY); 4802b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4803b5fca8f8Stomee ASSERT(sp->slab_flags & KMEM_SLAB_MOVE_PENDING); 4804b5fca8f8Stomee sp->slab_flags &= ~KMEM_SLAB_MOVE_PENDING; 4805b5fca8f8Stomee if (sp->slab_refcnt == 0) { 4806b5fca8f8Stomee list_t *deadlist = &cp->cache_defrag->kmd_deadlist; 4807b5fca8f8Stomee 4808b5fca8f8Stomee ASSERT(!list_is_empty(deadlist)); 4809b5fca8f8Stomee ASSERT(list_link_active((list_node_t *) 4810b5fca8f8Stomee &sp->slab_link)); 4811b5fca8f8Stomee 4812b5fca8f8Stomee list_remove(deadlist, sp); 4813b5fca8f8Stomee cp->cache_defrag->kmd_deadcount--; 4814b5fca8f8Stomee cp->cache_slab_destroy++; 4815b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4816b5fca8f8Stomee kmem_slab_destroy(cp, sp); 4817b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats.kms_dead_slabs_freed); 4818b5fca8f8Stomee return; 4819b5fca8f8Stomee } 4820b5fca8f8Stomee } else { 4821b5fca8f8Stomee kmem_slab_move_yes(cp, sp, buf); 4822b5fca8f8Stomee } 4823b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4824b5fca8f8Stomee } 4825b5fca8f8Stomee 4826b5fca8f8Stomee void 4827b5fca8f8Stomee kmem_cache_move_notify(kmem_cache_t *cp, void *buf) 4828b5fca8f8Stomee { 4829b5fca8f8Stomee kmem_move_notify_args_t *args; 4830b5fca8f8Stomee 4831b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats.kms_notify); 4832b5fca8f8Stomee args = kmem_alloc(sizeof (kmem_move_notify_args_t), KM_NOSLEEP); 4833b5fca8f8Stomee if (args != NULL) { 4834b5fca8f8Stomee args->kmna_cache = cp; 4835b5fca8f8Stomee args->kmna_buf = buf; 4836eb697d4eStomee if (!taskq_dispatch(kmem_taskq, 4837b5fca8f8Stomee (task_func_t *)kmem_cache_move_notify_task, args, 4838eb697d4eStomee TQ_NOSLEEP)) 4839eb697d4eStomee kmem_free(args, sizeof (kmem_move_notify_args_t)); 4840b5fca8f8Stomee } 4841b5fca8f8Stomee } 4842b5fca8f8Stomee 4843b5fca8f8Stomee static void 4844b5fca8f8Stomee kmem_cache_defrag(kmem_cache_t *cp) 4845b5fca8f8Stomee { 4846b5fca8f8Stomee size_t n; 4847b5fca8f8Stomee 4848b5fca8f8Stomee ASSERT(cp->cache_defrag != NULL); 4849b5fca8f8Stomee 4850b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4851b5fca8f8Stomee n = avl_numnodes(&cp->cache_partial_slabs); 4852b5fca8f8Stomee if (n > 1) { 4853b5fca8f8Stomee /* kmem_move_buffers() drops and reacquires cache_lock */ 4854b5fca8f8Stomee (void) kmem_move_buffers(cp, n, 0, KMM_DESPERATE); 4855b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats.kms_defrags); 4856b5fca8f8Stomee } 4857b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4858b5fca8f8Stomee } 4859b5fca8f8Stomee 4860b5fca8f8Stomee /* Is this cache above the fragmentation threshold? */ 4861b5fca8f8Stomee static boolean_t 4862b5fca8f8Stomee kmem_cache_frag_threshold(kmem_cache_t *cp, uint64_t nfree) 4863b5fca8f8Stomee { 4864b5fca8f8Stomee if (avl_numnodes(&cp->cache_partial_slabs) <= 1) 4865b5fca8f8Stomee return (B_FALSE); 4866b5fca8f8Stomee 4867b5fca8f8Stomee /* 4868b5fca8f8Stomee * nfree kmem_frag_numer 4869b5fca8f8Stomee * ------------------ > --------------- 4870b5fca8f8Stomee * cp->cache_buftotal kmem_frag_denom 4871b5fca8f8Stomee */ 4872b5fca8f8Stomee return ((nfree * kmem_frag_denom) > 4873b5fca8f8Stomee (cp->cache_buftotal * kmem_frag_numer)); 4874b5fca8f8Stomee } 4875b5fca8f8Stomee 4876b5fca8f8Stomee static boolean_t 4877b5fca8f8Stomee kmem_cache_is_fragmented(kmem_cache_t *cp, boolean_t *doreap) 4878b5fca8f8Stomee { 4879b5fca8f8Stomee boolean_t fragmented; 4880b5fca8f8Stomee uint64_t nfree; 4881b5fca8f8Stomee 4882b5fca8f8Stomee ASSERT(MUTEX_HELD(&cp->cache_lock)); 4883b5fca8f8Stomee *doreap = B_FALSE; 4884b5fca8f8Stomee 4885b5fca8f8Stomee if (!kmem_move_fulltilt && ((cp->cache_complete_slab_count + 4886b5fca8f8Stomee avl_numnodes(&cp->cache_partial_slabs)) < kmem_frag_minslabs)) 4887b5fca8f8Stomee return (B_FALSE); 4888b5fca8f8Stomee 4889b5fca8f8Stomee nfree = cp->cache_bufslab; 4890b5fca8f8Stomee fragmented = kmem_cache_frag_threshold(cp, nfree); 4891b5fca8f8Stomee /* 4892b5fca8f8Stomee * Free buffers in the magazine layer appear allocated from the point of 4893b5fca8f8Stomee * view of the slab layer. We want to know if the slab layer would 4894b5fca8f8Stomee * appear fragmented if we included free buffers from magazines that 4895b5fca8f8Stomee * have fallen out of the working set. 4896b5fca8f8Stomee */ 4897b5fca8f8Stomee if (!fragmented) { 4898b5fca8f8Stomee long reap; 4899b5fca8f8Stomee 4900b5fca8f8Stomee mutex_enter(&cp->cache_depot_lock); 4901b5fca8f8Stomee reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min); 4902b5fca8f8Stomee reap = MIN(reap, cp->cache_full.ml_total); 4903b5fca8f8Stomee mutex_exit(&cp->cache_depot_lock); 4904b5fca8f8Stomee 4905b5fca8f8Stomee nfree += ((uint64_t)reap * cp->cache_magtype->mt_magsize); 4906b5fca8f8Stomee if (kmem_cache_frag_threshold(cp, nfree)) { 4907b5fca8f8Stomee *doreap = B_TRUE; 4908b5fca8f8Stomee } 4909b5fca8f8Stomee } 4910b5fca8f8Stomee 4911b5fca8f8Stomee return (fragmented); 4912b5fca8f8Stomee } 4913b5fca8f8Stomee 4914b5fca8f8Stomee /* Called periodically from kmem_taskq */ 4915b5fca8f8Stomee static void 4916b5fca8f8Stomee kmem_cache_scan(kmem_cache_t *cp) 4917b5fca8f8Stomee { 4918b5fca8f8Stomee boolean_t reap = B_FALSE; 4919b5fca8f8Stomee 4920b5fca8f8Stomee ASSERT(taskq_member(kmem_taskq, curthread)); 4921b5fca8f8Stomee ASSERT(cp->cache_defrag != NULL); 4922b5fca8f8Stomee 4923b5fca8f8Stomee mutex_enter(&cp->cache_lock); 4924b5fca8f8Stomee 4925b5fca8f8Stomee if (kmem_cache_is_fragmented(cp, &reap)) { 4926b5fca8f8Stomee kmem_defrag_t *kmd = cp->cache_defrag; 4927b5fca8f8Stomee size_t slabs_found; 4928b5fca8f8Stomee 4929b5fca8f8Stomee /* 4930b5fca8f8Stomee * Consolidate reclaimable slabs from the end of the partial 4931b5fca8f8Stomee * slab list (scan at most kmem_reclaim_scan_range slabs to find 4932b5fca8f8Stomee * reclaimable slabs). Keep track of how many candidate slabs we 4933b5fca8f8Stomee * looked for and how many we actually found so we can adjust 4934b5fca8f8Stomee * the definition of a candidate slab if we're having trouble 4935b5fca8f8Stomee * finding them. 4936b5fca8f8Stomee * 4937b5fca8f8Stomee * kmem_move_buffers() drops and reacquires cache_lock. 4938b5fca8f8Stomee */ 4939b5fca8f8Stomee slabs_found = kmem_move_buffers(cp, kmem_reclaim_scan_range, 4940b5fca8f8Stomee kmem_reclaim_max_slabs, 0); 4941b5fca8f8Stomee if (slabs_found >= 0) { 4942b5fca8f8Stomee kmd->kmd_slabs_sought += kmem_reclaim_max_slabs; 4943b5fca8f8Stomee kmd->kmd_slabs_found += slabs_found; 4944b5fca8f8Stomee } 4945b5fca8f8Stomee 4946b5fca8f8Stomee if (++kmd->kmd_scans >= kmem_reclaim_scan_range) { 4947b5fca8f8Stomee kmd->kmd_scans = 0; 4948b5fca8f8Stomee 4949b5fca8f8Stomee /* 4950b5fca8f8Stomee * If we had difficulty finding candidate slabs in 4951b5fca8f8Stomee * previous scans, adjust the threshold so that 4952b5fca8f8Stomee * candidates are easier to find. 4953b5fca8f8Stomee */ 4954b5fca8f8Stomee if (kmd->kmd_slabs_found == kmd->kmd_slabs_sought) { 4955b5fca8f8Stomee kmem_adjust_reclaim_threshold(kmd, -1); 4956b5fca8f8Stomee } else if ((kmd->kmd_slabs_found * 2) < 4957b5fca8f8Stomee kmd->kmd_slabs_sought) { 4958b5fca8f8Stomee kmem_adjust_reclaim_threshold(kmd, 1); 4959b5fca8f8Stomee } 4960b5fca8f8Stomee kmd->kmd_slabs_sought = 0; 4961b5fca8f8Stomee kmd->kmd_slabs_found = 0; 4962b5fca8f8Stomee } 4963b5fca8f8Stomee } else { 4964b5fca8f8Stomee kmem_reset_reclaim_threshold(cp->cache_defrag); 4965b5fca8f8Stomee #ifdef DEBUG 4966b5fca8f8Stomee if (avl_numnodes(&cp->cache_partial_slabs) > 1) { 4967b5fca8f8Stomee /* 4968b5fca8f8Stomee * In a debug kernel we want the consolidator to 4969b5fca8f8Stomee * run occasionally even when there is plenty of 4970b5fca8f8Stomee * memory. 4971b5fca8f8Stomee */ 4972b5fca8f8Stomee uint32_t debug_rand; 4973b5fca8f8Stomee 4974b5fca8f8Stomee (void) random_get_bytes((uint8_t *)&debug_rand, 4); 4975b5fca8f8Stomee if (!kmem_move_noreap && 4976b5fca8f8Stomee ((debug_rand % kmem_mtb_reap) == 0)) { 4977b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4978b5fca8f8Stomee kmem_cache_reap(cp); 4979b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats.kms_debug_reaps); 4980b5fca8f8Stomee return; 4981b5fca8f8Stomee } else if ((debug_rand % kmem_mtb_move) == 0) { 4982b5fca8f8Stomee (void) kmem_move_buffers(cp, 4983b5fca8f8Stomee kmem_reclaim_scan_range, 1, 0); 4984b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats. 4985b5fca8f8Stomee kms_debug_move_scans); 4986b5fca8f8Stomee } 4987b5fca8f8Stomee } 4988b5fca8f8Stomee #endif /* DEBUG */ 4989b5fca8f8Stomee } 4990b5fca8f8Stomee 4991b5fca8f8Stomee mutex_exit(&cp->cache_lock); 4992b5fca8f8Stomee 4993b5fca8f8Stomee if (reap) { 4994b5fca8f8Stomee KMEM_STAT_ADD(kmem_move_stats.kms_scan_depot_ws_reaps); 4995b5fca8f8Stomee kmem_depot_ws_reap(cp); 4996b5fca8f8Stomee } 4997b5fca8f8Stomee } 4998