17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate * CDDL HEADER START
37c478bd9Sstevel@tonic-gate *
47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the
57d692464Sdp * Common Development and Distribution License (the "License").
67d692464Sdp * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate *
87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate * and limitations under the License.
127c478bd9Sstevel@tonic-gate *
137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate *
197c478bd9Sstevel@tonic-gate * CDDL HEADER END
207c478bd9Sstevel@tonic-gate */
217c478bd9Sstevel@tonic-gate /*
22b942e89bSDavid Valin * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.
23929d5b43SMatthew Ahrens * Copyright (c) 2012, 2017 by Delphix. All rights reserved.
240c833d64SJosef 'Jeff' Sipek * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
2536a64e62STim Kordas * Copyright 2018, Joyent, Inc.
26*ed093b41SRobert Mustacchi * Copyright 2023 Oxide Computer Company
277c478bd9Sstevel@tonic-gate */
287c478bd9Sstevel@tonic-gate
297c478bd9Sstevel@tonic-gate /*
30b5fca8f8Stomee * Kernel memory allocator, as described in the following two papers and a
31b5fca8f8Stomee * statement about the consolidator:
327c478bd9Sstevel@tonic-gate *
337c478bd9Sstevel@tonic-gate * Jeff Bonwick,
347c478bd9Sstevel@tonic-gate * The Slab Allocator: An Object-Caching Kernel Memory Allocator.
357c478bd9Sstevel@tonic-gate * Proceedings of the Summer 1994 Usenix Conference.
367c478bd9Sstevel@tonic-gate * Available as /shared/sac/PSARC/1994/028/materials/kmem.pdf.
377c478bd9Sstevel@tonic-gate *
387c478bd9Sstevel@tonic-gate * Jeff Bonwick and Jonathan Adams,
397c478bd9Sstevel@tonic-gate * Magazines and vmem: Extending the Slab Allocator to Many CPUs and
407c478bd9Sstevel@tonic-gate * Arbitrary Resources.
417c478bd9Sstevel@tonic-gate * Proceedings of the 2001 Usenix Conference.
427c478bd9Sstevel@tonic-gate * Available as /shared/sac/PSARC/2000/550/materials/vmem.pdf.
43b5fca8f8Stomee *
44b5fca8f8Stomee * kmem Slab Consolidator Big Theory Statement:
45b5fca8f8Stomee *
46b5fca8f8Stomee * 1. Motivation
47b5fca8f8Stomee *
48b5fca8f8Stomee * As stated in Bonwick94, slabs provide the following advantages over other
49b5fca8f8Stomee * allocation structures in terms of memory fragmentation:
50b5fca8f8Stomee *
51b5fca8f8Stomee * - Internal fragmentation (per-buffer wasted space) is minimal.
52b5fca8f8Stomee * - Severe external fragmentation (unused buffers on the free list) is
53b5fca8f8Stomee * unlikely.
54b5fca8f8Stomee *
55b5fca8f8Stomee * Segregating objects by size eliminates one source of external fragmentation,
56b5fca8f8Stomee * and according to Bonwick:
57b5fca8f8Stomee *
58b5fca8f8Stomee * The other reason that slabs reduce external fragmentation is that all
59b5fca8f8Stomee * objects in a slab are of the same type, so they have the same lifetime
60b5fca8f8Stomee * distribution. The resulting segregation of short-lived and long-lived
61b5fca8f8Stomee * objects at slab granularity reduces the likelihood of an entire page being
62b5fca8f8Stomee * held hostage due to a single long-lived allocation [Barrett93, Hanson90].
63b5fca8f8Stomee *
64b5fca8f8Stomee * While unlikely, severe external fragmentation remains possible. Clients that
65b5fca8f8Stomee * allocate both short- and long-lived objects from the same cache cannot
66b5fca8f8Stomee * anticipate the distribution of long-lived objects within the allocator's slab
67b5fca8f8Stomee * implementation. Even a small percentage of long-lived objects distributed
68b5fca8f8Stomee * randomly across many slabs can lead to a worst case scenario where the client
69b5fca8f8Stomee * frees the majority of its objects and the system gets back almost none of the
70b5fca8f8Stomee * slabs. Despite the client doing what it reasonably can to help the system
71b5fca8f8Stomee * reclaim memory, the allocator cannot shake free enough slabs because of
72b5fca8f8Stomee * lonely allocations stubbornly hanging on. Although the allocator is in a
73b5fca8f8Stomee * position to diagnose the fragmentation, there is nothing that the allocator
74b5fca8f8Stomee * by itself can do about it. It only takes a single allocated object to prevent
75b5fca8f8Stomee * an entire slab from being reclaimed, and any object handed out by
76b5fca8f8Stomee * kmem_cache_alloc() is by definition in the client's control. Conversely,
77b5fca8f8Stomee * although the client is in a position to move a long-lived object, it has no
78b5fca8f8Stomee * way of knowing if the object is causing fragmentation, and if so, where to
79b5fca8f8Stomee * move it. A solution necessarily requires further cooperation between the
80b5fca8f8Stomee * allocator and the client.
81b5fca8f8Stomee *
82b5fca8f8Stomee * 2. Move Callback
83b5fca8f8Stomee *
84b5fca8f8Stomee * The kmem slab consolidator therefore adds a move callback to the
85b5fca8f8Stomee * allocator/client interface, improving worst-case external fragmentation in
86b5fca8f8Stomee * kmem caches that supply a function to move objects from one memory location
87b5fca8f8Stomee * to another. In a situation of low memory kmem attempts to consolidate all of
88b5fca8f8Stomee * a cache's slabs at once; otherwise it works slowly to bring external
89b5fca8f8Stomee * fragmentation within the 1/8 limit guaranteed for internal fragmentation,
90b5fca8f8Stomee * thereby helping to avoid a low memory situation in the future.
91b5fca8f8Stomee *
92b5fca8f8Stomee * The callback has the following signature:
93b5fca8f8Stomee *
94b5fca8f8Stomee * kmem_cbrc_t move(void *old, void *new, size_t size, void *user_arg)
95b5fca8f8Stomee *
96b5fca8f8Stomee * It supplies the kmem client with two addresses: the allocated object that
97b5fca8f8Stomee * kmem wants to move and a buffer selected by kmem for the client to use as the
98b5fca8f8Stomee * copy destination. The callback is kmem's way of saying "Please get off of
99b5fca8f8Stomee * this buffer and use this one instead." kmem knows where it wants to move the
100b5fca8f8Stomee * object in order to best reduce fragmentation. All the client needs to know
101b5fca8f8Stomee * about the second argument (void *new) is that it is an allocated, constructed
102b5fca8f8Stomee * object ready to take the contents of the old object. When the move function
103b5fca8f8Stomee * is called, the system is likely to be low on memory, and the new object
104b5fca8f8Stomee * spares the client from having to worry about allocating memory for the
105b5fca8f8Stomee * requested move. The third argument supplies the size of the object, in case a
106b5fca8f8Stomee * single move function handles multiple caches whose objects differ only in
107b5fca8f8Stomee * size (such as zio_buf_512, zio_buf_1024, etc). Finally, the same optional
108b5fca8f8Stomee * user argument passed to the constructor, destructor, and reclaim functions is
109b5fca8f8Stomee * also passed to the move callback.
110b5fca8f8Stomee *
111b5fca8f8Stomee * 2.1 Setting the Move Callback
112b5fca8f8Stomee *
113b5fca8f8Stomee * The client sets the move callback after creating the cache and before
114b5fca8f8Stomee * allocating from it:
115b5fca8f8Stomee *
116b5fca8f8Stomee * object_cache = kmem_cache_create(...);
117b5fca8f8Stomee * kmem_cache_set_move(object_cache, object_move);
118b5fca8f8Stomee *
119b5fca8f8Stomee * 2.2 Move Callback Return Values
120b5fca8f8Stomee *
121b5fca8f8Stomee * Only the client knows about its own data and when is a good time to move it.
122b5fca8f8Stomee * The client is cooperating with kmem to return unused memory to the system,
123b5fca8f8Stomee * and kmem respectfully accepts this help at the client's convenience. When
124b5fca8f8Stomee * asked to move an object, the client can respond with any of the following:
125b5fca8f8Stomee *
126b5fca8f8Stomee * typedef enum kmem_cbrc {
127b5fca8f8Stomee * KMEM_CBRC_YES,
128b5fca8f8Stomee * KMEM_CBRC_NO,
129b5fca8f8Stomee * KMEM_CBRC_LATER,
130b5fca8f8Stomee * KMEM_CBRC_DONT_NEED,
131b5fca8f8Stomee * KMEM_CBRC_DONT_KNOW
132b5fca8f8Stomee * } kmem_cbrc_t;
133b5fca8f8Stomee *
134b5fca8f8Stomee * The client must not explicitly kmem_cache_free() either of the objects passed
135b5fca8f8Stomee * to the callback, since kmem wants to free them directly to the slab layer
136b5fca8f8Stomee * (bypassing the per-CPU magazine layer). The response tells kmem which of the
137b5fca8f8Stomee * objects to free:
138b5fca8f8Stomee *
139b5fca8f8Stomee * YES: (Did it) The client moved the object, so kmem frees the old one.
140b5fca8f8Stomee * NO: (Never) The client refused, so kmem frees the new object (the
141b5fca8f8Stomee * unused copy destination). kmem also marks the slab of the old
142b5fca8f8Stomee * object so as not to bother the client with further callbacks for
143b5fca8f8Stomee * that object as long as the slab remains on the partial slab list.
144b5fca8f8Stomee * (The system won't be getting the slab back as long as the
145b5fca8f8Stomee * immovable object holds it hostage, so there's no point in moving
146b5fca8f8Stomee * any of its objects.)
147b5fca8f8Stomee * LATER: The client is using the object and cannot move it now, so kmem
148b5fca8f8Stomee * frees the new object (the unused copy destination). kmem still
149b5fca8f8Stomee * attempts to move other objects off the slab, since it expects to
150b5fca8f8Stomee * succeed in clearing the slab in a later callback. The client
151b5fca8f8Stomee * should use LATER instead of NO if the object is likely to become
152b5fca8f8Stomee * movable very soon.
153b5fca8f8Stomee * DONT_NEED: The client no longer needs the object, so kmem frees the old along
154b5fca8f8Stomee * with the new object (the unused copy destination). This response
155b5fca8f8Stomee * is the client's opportunity to be a model citizen and give back as
156b5fca8f8Stomee * much as it can.
157b5fca8f8Stomee * DONT_KNOW: The client does not know about the object because
158b5fca8f8Stomee * a) the client has just allocated the object and not yet put it
159b5fca8f8Stomee * wherever it expects to find known objects
160b5fca8f8Stomee * b) the client has removed the object from wherever it expects to
161b5fca8f8Stomee * find known objects and is about to free it, or
162b5fca8f8Stomee * c) the client has freed the object.
163b5fca8f8Stomee * In all these cases (a, b, and c) kmem frees the new object (the
164d7db73d1SBryan Cantrill * unused copy destination). In the first case, the object is in
165d7db73d1SBryan Cantrill * use and the correct action is that for LATER; in the latter two
166d7db73d1SBryan Cantrill * cases, we know that the object is either freed or about to be
167d7db73d1SBryan Cantrill * freed, in which case it is either already in a magazine or about
168d7db73d1SBryan Cantrill * to be in one. In these cases, we know that the object will either
169d7db73d1SBryan Cantrill * be reallocated and reused, or it will end up in a full magazine
170d7db73d1SBryan Cantrill * that will be reaped (thereby liberating the slab). Because it
171d7db73d1SBryan Cantrill * is prohibitively expensive to differentiate these cases, and
172d7db73d1SBryan Cantrill * because the defrag code is executed when we're low on memory
173d7db73d1SBryan Cantrill * (thereby biasing the system to reclaim full magazines) we treat
174d7db73d1SBryan Cantrill * all DONT_KNOW cases as LATER and rely on cache reaping to
175d7db73d1SBryan Cantrill * generally clean up full magazines. While we take the same action
176d7db73d1SBryan Cantrill * for these cases, we maintain their semantic distinction: if
177d7db73d1SBryan Cantrill * defragmentation is not occurring, it is useful to know if this
178d7db73d1SBryan Cantrill * is due to objects in use (LATER) or objects in an unknown state
179d7db73d1SBryan Cantrill * of transition (DONT_KNOW).
180b5fca8f8Stomee *
181b5fca8f8Stomee * 2.3 Object States
182b5fca8f8Stomee *
183b5fca8f8Stomee * Neither kmem nor the client can be assumed to know the object's whereabouts
184b5fca8f8Stomee * at the time of the callback. An object belonging to a kmem cache may be in
185b5fca8f8Stomee * any of the following states:
186b5fca8f8Stomee *
187b5fca8f8Stomee * 1. Uninitialized on the slab
188b5fca8f8Stomee * 2. Allocated from the slab but not constructed (still uninitialized)
189b5fca8f8Stomee * 3. Allocated from the slab, constructed, but not yet ready for business
190b5fca8f8Stomee * (not in a valid state for the move callback)
191b5fca8f8Stomee * 4. In use (valid and known to the client)
192b5fca8f8Stomee * 5. About to be freed (no longer in a valid state for the move callback)
193b5fca8f8Stomee * 6. Freed to a magazine (still constructed)
194b5fca8f8Stomee * 7. Allocated from a magazine, not yet ready for business (not in a valid
195b5fca8f8Stomee * state for the move callback), and about to return to state #4
196b5fca8f8Stomee * 8. Deconstructed on a magazine that is about to be freed
197b5fca8f8Stomee * 9. Freed to the slab
198b5fca8f8Stomee *
199b5fca8f8Stomee * Since the move callback may be called at any time while the object is in any
200b5fca8f8Stomee * of the above states (except state #1), the client needs a safe way to
201b5fca8f8Stomee * determine whether or not it knows about the object. Specifically, the client
202b5fca8f8Stomee * needs to know whether or not the object is in state #4, the only state in
203b5fca8f8Stomee * which a move is valid. If the object is in any other state, the client should
204b5fca8f8Stomee * immediately return KMEM_CBRC_DONT_KNOW, since it is unsafe to access any of
205b5fca8f8Stomee * the object's fields.
206b5fca8f8Stomee *
207b5fca8f8Stomee * Note that although an object may be in state #4 when kmem initiates the move
208b5fca8f8Stomee * request, the object may no longer be in that state by the time kmem actually
209b5fca8f8Stomee * calls the move function. Not only does the client free objects
210b5fca8f8Stomee * asynchronously, kmem itself puts move requests on a queue where thay are
211b5fca8f8Stomee * pending until kmem processes them from another context. Also, objects freed
212b5fca8f8Stomee * to a magazine appear allocated from the point of view of the slab layer, so
213b5fca8f8Stomee * kmem may even initiate requests for objects in a state other than state #4.
214b5fca8f8Stomee *
215b5fca8f8Stomee * 2.3.1 Magazine Layer
216b5fca8f8Stomee *
217b5fca8f8Stomee * An important insight revealed by the states listed above is that the magazine
218b5fca8f8Stomee * layer is populated only by kmem_cache_free(). Magazines of constructed
219b5fca8f8Stomee * objects are never populated directly from the slab layer (which contains raw,
220b5fca8f8Stomee * unconstructed objects). Whenever an allocation request cannot be satisfied
221b5fca8f8Stomee * from the magazine layer, the magazines are bypassed and the request is
222b5fca8f8Stomee * satisfied from the slab layer (creating a new slab if necessary). kmem calls
223b5fca8f8Stomee * the object constructor only when allocating from the slab layer, and only in
224b5fca8f8Stomee * response to kmem_cache_alloc() or to prepare the destination buffer passed in
225b5fca8f8Stomee * the move callback. kmem does not preconstruct objects in anticipation of
226b5fca8f8Stomee * kmem_cache_alloc().
227b5fca8f8Stomee *
228b5fca8f8Stomee * 2.3.2 Object Constructor and Destructor
229b5fca8f8Stomee *
230b5fca8f8Stomee * If the client supplies a destructor, it must be valid to call the destructor
231b5fca8f8Stomee * on a newly created object (immediately after the constructor).
232b5fca8f8Stomee *
233b5fca8f8Stomee * 2.4 Recognizing Known Objects
234b5fca8f8Stomee *
235b5fca8f8Stomee * There is a simple test to determine safely whether or not the client knows
236b5fca8f8Stomee * about a given object in the move callback. It relies on the fact that kmem
237b5fca8f8Stomee * guarantees that the object of the move callback has only been touched by the
238b5fca8f8Stomee * client itself or else by kmem. kmem does this by ensuring that none of the
239b5fca8f8Stomee * cache's slabs are freed to the virtual memory (VM) subsystem while a move
240b5fca8f8Stomee * callback is pending. When the last object on a slab is freed, if there is a
241b5fca8f8Stomee * pending move, kmem puts the slab on a per-cache dead list and defers freeing
242b5fca8f8Stomee * slabs on that list until all pending callbacks are completed. That way,
243b5fca8f8Stomee * clients can be certain that the object of a move callback is in one of the
244b5fca8f8Stomee * states listed above, making it possible to distinguish known objects (in
245b5fca8f8Stomee * state #4) using the two low order bits of any pointer member (with the
246b5fca8f8Stomee * exception of 'char *' or 'short *' which may not be 4-byte aligned on some
247b5fca8f8Stomee * platforms).
248b5fca8f8Stomee *
249b5fca8f8Stomee * The test works as long as the client always transitions objects from state #4
250b5fca8f8Stomee * (known, in use) to state #5 (about to be freed, invalid) by setting the low
251b5fca8f8Stomee * order bit of the client-designated pointer member. Since kmem only writes
252b5fca8f8Stomee * invalid memory patterns, such as 0xbaddcafe to uninitialized memory and
253b5fca8f8Stomee * 0xdeadbeef to freed memory, any scribbling on the object done by kmem is
254b5fca8f8Stomee * guaranteed to set at least one of the two low order bits. Therefore, given an
255b5fca8f8Stomee * object with a back pointer to a 'container_t *o_container', the client can
256b5fca8f8Stomee * test
257b5fca8f8Stomee *
258b5fca8f8Stomee * container_t *container = object->o_container;
259b5fca8f8Stomee * if ((uintptr_t)container & 0x3) {
260b5fca8f8Stomee * return (KMEM_CBRC_DONT_KNOW);
261b5fca8f8Stomee * }
262b5fca8f8Stomee *
263b5fca8f8Stomee * Typically, an object will have a pointer to some structure with a list or
264b5fca8f8Stomee * hash where objects from the cache are kept while in use. Assuming that the
265b5fca8f8Stomee * client has some way of knowing that the container structure is valid and will
266b5fca8f8Stomee * not go away during the move, and assuming that the structure includes a lock
267b5fca8f8Stomee * to protect whatever collection is used, then the client would continue as
268b5fca8f8Stomee * follows:
269b5fca8f8Stomee *
270b5fca8f8Stomee * // Ensure that the container structure does not go away.
271b5fca8f8Stomee * if (container_hold(container) == 0) {
272b5fca8f8Stomee * return (KMEM_CBRC_DONT_KNOW);
273b5fca8f8Stomee * }
274b5fca8f8Stomee * mutex_enter(&container->c_objects_lock);
275b5fca8f8Stomee * if (container != object->o_container) {
276b5fca8f8Stomee * mutex_exit(&container->c_objects_lock);
277b5fca8f8Stomee * container_rele(container);
278b5fca8f8Stomee * return (KMEM_CBRC_DONT_KNOW);
279b5fca8f8Stomee * }
280b5fca8f8Stomee *
281b5fca8f8Stomee * At this point the client knows that the object cannot be freed as long as
282b5fca8f8Stomee * c_objects_lock is held. Note that after acquiring the lock, the client must
283b5fca8f8Stomee * recheck the o_container pointer in case the object was removed just before
284b5fca8f8Stomee * acquiring the lock.
285b5fca8f8Stomee *
286b5fca8f8Stomee * When the client is about to free an object, it must first remove that object
287b5fca8f8Stomee * from the list, hash, or other structure where it is kept. At that time, to
288b5fca8f8Stomee * mark the object so it can be distinguished from the remaining, known objects,
289b5fca8f8Stomee * the client sets the designated low order bit:
290b5fca8f8Stomee *
291b5fca8f8Stomee * mutex_enter(&container->c_objects_lock);
292b5fca8f8Stomee * object->o_container = (void *)((uintptr_t)object->o_container | 0x1);
293b5fca8f8Stomee * list_remove(&container->c_objects, object);
294b5fca8f8Stomee * mutex_exit(&container->c_objects_lock);
295b5fca8f8Stomee *
296b5fca8f8Stomee * In the common case, the object is freed to the magazine layer, where it may
297b5fca8f8Stomee * be reused on a subsequent allocation without the overhead of calling the
298b5fca8f8Stomee * constructor. While in the magazine it appears allocated from the point of
299b5fca8f8Stomee * view of the slab layer, making it a candidate for the move callback. Most
300b5fca8f8Stomee * objects unrecognized by the client in the move callback fall into this
301b5fca8f8Stomee * category and are cheaply distinguished from known objects by the test
302d7db73d1SBryan Cantrill * described earlier. Because searching magazines is prohibitively expensive
303d7db73d1SBryan Cantrill * for kmem, clients that do not mark freed objects (and therefore return
304d7db73d1SBryan Cantrill * KMEM_CBRC_DONT_KNOW for large numbers of objects) may find defragmentation
305d7db73d1SBryan Cantrill * efficacy reduced.
306b5fca8f8Stomee *
307b5fca8f8Stomee * Invalidating the designated pointer member before freeing the object marks
308b5fca8f8Stomee * the object to be avoided in the callback, and conversely, assigning a valid
309b5fca8f8Stomee * value to the designated pointer member after allocating the object makes the
310b5fca8f8Stomee * object fair game for the callback:
311b5fca8f8Stomee *
312b5fca8f8Stomee * ... allocate object ...
313b5fca8f8Stomee * ... set any initial state not set by the constructor ...
314b5fca8f8Stomee *
315b5fca8f8Stomee * mutex_enter(&container->c_objects_lock);
316b5fca8f8Stomee * list_insert_tail(&container->c_objects, object);
317b5fca8f8Stomee * membar_producer();
318b5fca8f8Stomee * object->o_container = container;
319b5fca8f8Stomee * mutex_exit(&container->c_objects_lock);
320b5fca8f8Stomee *
321b5fca8f8Stomee * Note that everything else must be valid before setting o_container makes the
322b5fca8f8Stomee * object fair game for the move callback. The membar_producer() call ensures
323b5fca8f8Stomee * that all the object's state is written to memory before setting the pointer
324b5fca8f8Stomee * that transitions the object from state #3 or #7 (allocated, constructed, not
325b5fca8f8Stomee * yet in use) to state #4 (in use, valid). That's important because the move
326b5fca8f8Stomee * function has to check the validity of the pointer before it can safely
327b5fca8f8Stomee * acquire the lock protecting the collection where it expects to find known
328b5fca8f8Stomee * objects.
329b5fca8f8Stomee *
330b5fca8f8Stomee * This method of distinguishing known objects observes the usual symmetry:
331b5fca8f8Stomee * invalidating the designated pointer is the first thing the client does before
332b5fca8f8Stomee * freeing the object, and setting the designated pointer is the last thing the
333b5fca8f8Stomee * client does after allocating the object. Of course, the client is not
334b5fca8f8Stomee * required to use this method. Fundamentally, how the client recognizes known
335b5fca8f8Stomee * objects is completely up to the client, but this method is recommended as an
336b5fca8f8Stomee * efficient and safe way to take advantage of the guarantees made by kmem. If
337b5fca8f8Stomee * the entire object is arbitrary data without any markable bits from a suitable
338b5fca8f8Stomee * pointer member, then the client must find some other method, such as
339b5fca8f8Stomee * searching a hash table of known objects.
340b5fca8f8Stomee *
341b5fca8f8Stomee * 2.5 Preventing Objects From Moving
342b5fca8f8Stomee *
343b5fca8f8Stomee * Besides a way to distinguish known objects, the other thing that the client
344b5fca8f8Stomee * needs is a strategy to ensure that an object will not move while the client
345b5fca8f8Stomee * is actively using it. The details of satisfying this requirement tend to be
346b5fca8f8Stomee * highly cache-specific. It might seem that the same rules that let a client
347b5fca8f8Stomee * remove an object safely should also decide when an object can be moved
348b5fca8f8Stomee * safely. However, any object state that makes a removal attempt invalid is
349b5fca8f8Stomee * likely to be long-lasting for objects that the client does not expect to
350b5fca8f8Stomee * remove. kmem knows nothing about the object state and is equally likely (from
351b5fca8f8Stomee * the client's point of view) to request a move for any object in the cache,
352b5fca8f8Stomee * whether prepared for removal or not. Even a low percentage of objects stuck
353b5fca8f8Stomee * in place by unremovability will defeat the consolidator if the stuck objects
354b5fca8f8Stomee * are the same long-lived allocations likely to hold slabs hostage.
355b5fca8f8Stomee * Fundamentally, the consolidator is not aimed at common cases. Severe external
356b5fca8f8Stomee * fragmentation is a worst case scenario manifested as sparsely allocated
357b5fca8f8Stomee * slabs, by definition a low percentage of the cache's objects. When deciding
358b5fca8f8Stomee * what makes an object movable, keep in mind the goal of the consolidator: to
359b5fca8f8Stomee * bring worst-case external fragmentation within the limits guaranteed for
360b5fca8f8Stomee * internal fragmentation. Removability is a poor criterion if it is likely to
361b5fca8f8Stomee * exclude more than an insignificant percentage of objects for long periods of
362b5fca8f8Stomee * time.
363b5fca8f8Stomee *
364b5fca8f8Stomee * A tricky general solution exists, and it has the advantage of letting you
365b5fca8f8Stomee * move any object at almost any moment, practically eliminating the likelihood
366b5fca8f8Stomee * that an object can hold a slab hostage. However, if there is a cache-specific
367b5fca8f8Stomee * way to ensure that an object is not actively in use in the vast majority of
368b5fca8f8Stomee * cases, a simpler solution that leverages this cache-specific knowledge is
369b5fca8f8Stomee * preferred.
370b5fca8f8Stomee *
371b5fca8f8Stomee * 2.5.1 Cache-Specific Solution
372b5fca8f8Stomee *
373b5fca8f8Stomee * As an example of a cache-specific solution, the ZFS znode cache takes
374b5fca8f8Stomee * advantage of the fact that the vast majority of znodes are only being
375b5fca8f8Stomee * referenced from the DNLC. (A typical case might be a few hundred in active
376b5fca8f8Stomee * use and a hundred thousand in the DNLC.) In the move callback, after the ZFS
377b5fca8f8Stomee * client has established that it recognizes the znode and can access its fields
378b5fca8f8Stomee * safely (using the method described earlier), it then tests whether the znode
379b5fca8f8Stomee * is referenced by anything other than the DNLC. If so, it assumes that the
380b5fca8f8Stomee * znode may be in active use and is unsafe to move, so it drops its locks and
381b5fca8f8Stomee * returns KMEM_CBRC_LATER. The advantage of this strategy is that everywhere
382b5fca8f8Stomee * else znodes are used, no change is needed to protect against the possibility
383b5fca8f8Stomee * of the znode moving. The disadvantage is that it remains possible for an
384b5fca8f8Stomee * application to hold a znode slab hostage with an open file descriptor.
385b5fca8f8Stomee * However, this case ought to be rare and the consolidator has a way to deal
386b5fca8f8Stomee * with it: If the client responds KMEM_CBRC_LATER repeatedly for the same
387b5fca8f8Stomee * object, kmem eventually stops believing it and treats the slab as if the
388b5fca8f8Stomee * client had responded KMEM_CBRC_NO. Having marked the hostage slab, kmem can
389b5fca8f8Stomee * then focus on getting it off of the partial slab list by allocating rather
390b5fca8f8Stomee * than freeing all of its objects. (Either way of getting a slab off the
391b5fca8f8Stomee * free list reduces fragmentation.)
392b5fca8f8Stomee *
393b5fca8f8Stomee * 2.5.2 General Solution
394b5fca8f8Stomee *
395b5fca8f8Stomee * The general solution, on the other hand, requires an explicit hold everywhere
396b5fca8f8Stomee * the object is used to prevent it from moving. To keep the client locking
397b5fca8f8Stomee * strategy as uncomplicated as possible, kmem guarantees the simplifying
398b5fca8f8Stomee * assumption that move callbacks are sequential, even across multiple caches.
399b5fca8f8Stomee * Internally, a global queue processed by a single thread supports all caches
400b5fca8f8Stomee * implementing the callback function. No matter how many caches supply a move
401b5fca8f8Stomee * function, the consolidator never moves more than one object at a time, so the
402b5fca8f8Stomee * client does not have to worry about tricky lock ordering involving several
403b5fca8f8Stomee * related objects from different kmem caches.
404b5fca8f8Stomee *
405b5fca8f8Stomee * The general solution implements the explicit hold as a read-write lock, which
406b5fca8f8Stomee * allows multiple readers to access an object from the cache simultaneously
407b5fca8f8Stomee * while a single writer is excluded from moving it. A single rwlock for the
408b5fca8f8Stomee * entire cache would lock out all threads from using any of the cache's objects
409b5fca8f8Stomee * even though only a single object is being moved, so to reduce contention,
410b5fca8f8Stomee * the client can fan out the single rwlock into an array of rwlocks hashed by
411b5fca8f8Stomee * the object address, making it probable that moving one object will not
412b5fca8f8Stomee * prevent other threads from using a different object. The rwlock cannot be a
413b5fca8f8Stomee * member of the object itself, because the possibility of the object moving
414b5fca8f8Stomee * makes it unsafe to access any of the object's fields until the lock is
415b5fca8f8Stomee * acquired.
416b5fca8f8Stomee *
417b5fca8f8Stomee * Assuming a small, fixed number of locks, it's possible that multiple objects
418b5fca8f8Stomee * will hash to the same lock. A thread that needs to use multiple objects in
419b5fca8f8Stomee * the same function may acquire the same lock multiple times. Since rwlocks are
420b5fca8f8Stomee * reentrant for readers, and since there is never more than a single writer at
421b5fca8f8Stomee * a time (assuming that the client acquires the lock as a writer only when
422b5fca8f8Stomee * moving an object inside the callback), there would seem to be no problem.
423b5fca8f8Stomee * However, a client locking multiple objects in the same function must handle
424b5fca8f8Stomee * one case of potential deadlock: Assume that thread A needs to prevent both
425b5fca8f8Stomee * object 1 and object 2 from moving, and thread B, the callback, meanwhile
426b5fca8f8Stomee * tries to move object 3. It's possible, if objects 1, 2, and 3 all hash to the
427b5fca8f8Stomee * same lock, that thread A will acquire the lock for object 1 as a reader
428b5fca8f8Stomee * before thread B sets the lock's write-wanted bit, preventing thread A from
429b5fca8f8Stomee * reacquiring the lock for object 2 as a reader. Unable to make forward
430b5fca8f8Stomee * progress, thread A will never release the lock for object 1, resulting in
431b5fca8f8Stomee * deadlock.
432b5fca8f8Stomee *
433b5fca8f8Stomee * There are two ways of avoiding the deadlock just described. The first is to
434b5fca8f8Stomee * use rw_tryenter() rather than rw_enter() in the callback function when
435b5fca8f8Stomee * attempting to acquire the lock as a writer. If tryenter discovers that the
436b5fca8f8Stomee * same object (or another object hashed to the same lock) is already in use, it
437b5fca8f8Stomee * aborts the callback and returns KMEM_CBRC_LATER. The second way is to use
438b5fca8f8Stomee * rprwlock_t (declared in common/fs/zfs/sys/rprwlock.h) instead of rwlock_t,
439b5fca8f8Stomee * since it allows a thread to acquire the lock as a reader in spite of a
440b5fca8f8Stomee * waiting writer. This second approach insists on moving the object now, no
441b5fca8f8Stomee * matter how many readers the move function must wait for in order to do so,
442b5fca8f8Stomee * and could delay the completion of the callback indefinitely (blocking
443b5fca8f8Stomee * callbacks to other clients). In practice, a less insistent callback using
444b5fca8f8Stomee * rw_tryenter() returns KMEM_CBRC_LATER infrequently enough that there seems
445b5fca8f8Stomee * little reason to use anything else.
446b5fca8f8Stomee *
447b5fca8f8Stomee * Avoiding deadlock is not the only problem that an implementation using an
448b5fca8f8Stomee * explicit hold needs to solve. Locking the object in the first place (to
449b5fca8f8Stomee * prevent it from moving) remains a problem, since the object could move
450b5fca8f8Stomee * between the time you obtain a pointer to the object and the time you acquire
451b5fca8f8Stomee * the rwlock hashed to that pointer value. Therefore the client needs to
452b5fca8f8Stomee * recheck the value of the pointer after acquiring the lock, drop the lock if
453b5fca8f8Stomee * the value has changed, and try again. This requires a level of indirection:
454b5fca8f8Stomee * something that points to the object rather than the object itself, that the
455b5fca8f8Stomee * client can access safely while attempting to acquire the lock. (The object
456b5fca8f8Stomee * itself cannot be referenced safely because it can move at any time.)
457b5fca8f8Stomee * The following lock-acquisition function takes whatever is safe to reference
458b5fca8f8Stomee * (arg), follows its pointer to the object (using function f), and tries as
459b5fca8f8Stomee * often as necessary to acquire the hashed lock and verify that the object
460b5fca8f8Stomee * still has not moved:
461b5fca8f8Stomee *
462b5fca8f8Stomee * object_t *
463b5fca8f8Stomee * object_hold(object_f f, void *arg)
464b5fca8f8Stomee * {
465b5fca8f8Stomee * object_t *op;
466b5fca8f8Stomee *
467b5fca8f8Stomee * op = f(arg);
468b5fca8f8Stomee * if (op == NULL) {
469b5fca8f8Stomee * return (NULL);
470b5fca8f8Stomee * }
471b5fca8f8Stomee *
472b5fca8f8Stomee * rw_enter(OBJECT_RWLOCK(op), RW_READER);
473b5fca8f8Stomee * while (op != f(arg)) {
474b5fca8f8Stomee * rw_exit(OBJECT_RWLOCK(op));
475b5fca8f8Stomee * op = f(arg);
476b5fca8f8Stomee * if (op == NULL) {
477b5fca8f8Stomee * break;
478b5fca8f8Stomee * }
479b5fca8f8Stomee * rw_enter(OBJECT_RWLOCK(op), RW_READER);
480b5fca8f8Stomee * }
481b5fca8f8Stomee *
482b5fca8f8Stomee * return (op);
483b5fca8f8Stomee * }
484b5fca8f8Stomee *
485b5fca8f8Stomee * The OBJECT_RWLOCK macro hashes the object address to obtain the rwlock. The
486b5fca8f8Stomee * lock reacquisition loop, while necessary, almost never executes. The function
487b5fca8f8Stomee * pointer f (used to obtain the object pointer from arg) has the following type
488b5fca8f8Stomee * definition:
489b5fca8f8Stomee *
490b5fca8f8Stomee * typedef object_t *(*object_f)(void *arg);
491b5fca8f8Stomee *
492b5fca8f8Stomee * An object_f implementation is likely to be as simple as accessing a structure
493b5fca8f8Stomee * member:
494b5fca8f8Stomee *
495b5fca8f8Stomee * object_t *
496b5fca8f8Stomee * s_object(void *arg)
497b5fca8f8Stomee * {
498b5fca8f8Stomee * something_t *sp = arg;
499b5fca8f8Stomee * return (sp->s_object);
500b5fca8f8Stomee * }
501b5fca8f8Stomee *
502b5fca8f8Stomee * The flexibility of a function pointer allows the path to the object to be
503b5fca8f8Stomee * arbitrarily complex and also supports the notion that depending on where you
504b5fca8f8Stomee * are using the object, you may need to get it from someplace different.
505b5fca8f8Stomee *
506b5fca8f8Stomee * The function that releases the explicit hold is simpler because it does not
507b5fca8f8Stomee * have to worry about the object moving:
508b5fca8f8Stomee *
509b5fca8f8Stomee * void
510b5fca8f8Stomee * object_rele(object_t *op)
511b5fca8f8Stomee * {
512b5fca8f8Stomee * rw_exit(OBJECT_RWLOCK(op));
513b5fca8f8Stomee * }
514b5fca8f8Stomee *
515b5fca8f8Stomee * The caller is spared these details so that obtaining and releasing an
516b5fca8f8Stomee * explicit hold feels like a simple mutex_enter()/mutex_exit() pair. The caller
517b5fca8f8Stomee * of object_hold() only needs to know that the returned object pointer is valid
518b5fca8f8Stomee * if not NULL and that the object will not move until released.
519b5fca8f8Stomee *
520b5fca8f8Stomee * Although object_hold() prevents an object from moving, it does not prevent it
521b5fca8f8Stomee * from being freed. The caller must take measures before calling object_hold()
522b5fca8f8Stomee * (afterwards is too late) to ensure that the held object cannot be freed. The
523b5fca8f8Stomee * caller must do so without accessing the unsafe object reference, so any lock
524b5fca8f8Stomee * or reference count used to ensure the continued existence of the object must
525b5fca8f8Stomee * live outside the object itself.
526b5fca8f8Stomee *
527b5fca8f8Stomee * Obtaining a new object is a special case where an explicit hold is impossible
528b5fca8f8Stomee * for the caller. Any function that returns a newly allocated object (either as
529b5fca8f8Stomee * a return value, or as an in-out paramter) must return it already held; after
530b5fca8f8Stomee * the caller gets it is too late, since the object cannot be safely accessed
531b5fca8f8Stomee * without the level of indirection described earlier. The following
532b5fca8f8Stomee * object_alloc() example uses the same code shown earlier to transition a new
533b5fca8f8Stomee * object into the state of being recognized (by the client) as a known object.
534b5fca8f8Stomee * The function must acquire the hold (rw_enter) before that state transition
535b5fca8f8Stomee * makes the object movable:
536b5fca8f8Stomee *
537b5fca8f8Stomee * static object_t *
538b5fca8f8Stomee * object_alloc(container_t *container)
539b5fca8f8Stomee * {
5404d4c4c43STom Erickson * object_t *object = kmem_cache_alloc(object_cache, 0);
541b5fca8f8Stomee * ... set any initial state not set by the constructor ...
542b5fca8f8Stomee * rw_enter(OBJECT_RWLOCK(object), RW_READER);
543b5fca8f8Stomee * mutex_enter(&container->c_objects_lock);
544b5fca8f8Stomee * list_insert_tail(&container->c_objects, object);
545b5fca8f8Stomee * membar_producer();
546b5fca8f8Stomee * object->o_container = container;
547b5fca8f8Stomee * mutex_exit(&container->c_objects_lock);
548b5fca8f8Stomee * return (object);
549b5fca8f8Stomee * }
550b5fca8f8Stomee *
551b5fca8f8Stomee * Functions that implicitly acquire an object hold (any function that calls
552b5fca8f8Stomee * object_alloc() to supply an object for the caller) need to be carefully noted
553b5fca8f8Stomee * so that the matching object_rele() is not neglected. Otherwise, leaked holds
554b5fca8f8Stomee * prevent all objects hashed to the affected rwlocks from ever being moved.
555b5fca8f8Stomee *
556b5fca8f8Stomee * The pointer to a held object can be hashed to the holding rwlock even after
557b5fca8f8Stomee * the object has been freed. Although it is possible to release the hold
558b5fca8f8Stomee * after freeing the object, you may decide to release the hold implicitly in
559b5fca8f8Stomee * whatever function frees the object, so as to release the hold as soon as
560b5fca8f8Stomee * possible, and for the sake of symmetry with the function that implicitly
561b5fca8f8Stomee * acquires the hold when it allocates the object. Here, object_free() releases
562b5fca8f8Stomee * the hold acquired by object_alloc(). Its implicit object_rele() forms a
563b5fca8f8Stomee * matching pair with object_hold():
564b5fca8f8Stomee *
565b5fca8f8Stomee * void
566b5fca8f8Stomee * object_free(object_t *object)
567b5fca8f8Stomee * {
568b5fca8f8Stomee * container_t *container;
569b5fca8f8Stomee *
570b5fca8f8Stomee * ASSERT(object_held(object));
571b5fca8f8Stomee * container = object->o_container;
572b5fca8f8Stomee * mutex_enter(&container->c_objects_lock);
573b5fca8f8Stomee * object->o_container =
574b5fca8f8Stomee * (void *)((uintptr_t)object->o_container | 0x1);
575b5fca8f8Stomee * list_remove(&container->c_objects, object);
576b5fca8f8Stomee * mutex_exit(&container->c_objects_lock);
577b5fca8f8Stomee * object_rele(object);
578b5fca8f8Stomee * kmem_cache_free(object_cache, object);
579b5fca8f8Stomee * }
580b5fca8f8Stomee *
581b5fca8f8Stomee * Note that object_free() cannot safely accept an object pointer as an argument
582b5fca8f8Stomee * unless the object is already held. Any function that calls object_free()
583b5fca8f8Stomee * needs to be carefully noted since it similarly forms a matching pair with
584b5fca8f8Stomee * object_hold().
585b5fca8f8Stomee *
586b5fca8f8Stomee * To complete the picture, the following callback function implements the
587b5fca8f8Stomee * general solution by moving objects only if they are currently unheld:
588b5fca8f8Stomee *
589b5fca8f8Stomee * static kmem_cbrc_t
590b5fca8f8Stomee * object_move(void *buf, void *newbuf, size_t size, void *arg)
591b5fca8f8Stomee * {
592b5fca8f8Stomee * object_t *op = buf, *np = newbuf;
593b5fca8f8Stomee * container_t *container;
594b5fca8f8Stomee *
595b5fca8f8Stomee * container = op->o_container;
596b5fca8f8Stomee * if ((uintptr_t)container & 0x3) {
597b5fca8f8Stomee * return (KMEM_CBRC_DONT_KNOW);
598b5fca8f8Stomee * }
599b5fca8f8Stomee *
600b5fca8f8Stomee * // Ensure that the container structure does not go away.
601b5fca8f8Stomee * if (container_hold(container) == 0) {
602b5fca8f8Stomee * return (KMEM_CBRC_DONT_KNOW);
603b5fca8f8Stomee * }
604b5fca8f8Stomee *
605b5fca8f8Stomee * mutex_enter(&container->c_objects_lock);
606b5fca8f8Stomee * if (container != op->o_container) {
607b5fca8f8Stomee * mutex_exit(&container->c_objects_lock);
608b5fca8f8Stomee * container_rele(container);
609b5fca8f8Stomee * return (KMEM_CBRC_DONT_KNOW);
610b5fca8f8Stomee * }
611b5fca8f8Stomee *
612b5fca8f8Stomee * if (rw_tryenter(OBJECT_RWLOCK(op), RW_WRITER) == 0) {
613b5fca8f8Stomee * mutex_exit(&container->c_objects_lock);
614b5fca8f8Stomee * container_rele(container);
615b5fca8f8Stomee * return (KMEM_CBRC_LATER);
616b5fca8f8Stomee * }
617b5fca8f8Stomee *
618b5fca8f8Stomee * object_move_impl(op, np); // critical section
619b5fca8f8Stomee * rw_exit(OBJECT_RWLOCK(op));
620b5fca8f8Stomee *
621b5fca8f8Stomee * op->o_container = (void *)((uintptr_t)op->o_container | 0x1);
622b5fca8f8Stomee * list_link_replace(&op->o_link_node, &np->o_link_node);
623b5fca8f8Stomee * mutex_exit(&container->c_objects_lock);
624b5fca8f8Stomee * container_rele(container);
625b5fca8f8Stomee * return (KMEM_CBRC_YES);
626b5fca8f8Stomee * }
627b5fca8f8Stomee *
628b5fca8f8Stomee * Note that object_move() must invalidate the designated o_container pointer of
629b5fca8f8Stomee * the old object in the same way that object_free() does, since kmem will free
630b5fca8f8Stomee * the object in response to the KMEM_CBRC_YES return value.
631b5fca8f8Stomee *
632b5fca8f8Stomee * The lock order in object_move() differs from object_alloc(), which locks
633b5fca8f8Stomee * OBJECT_RWLOCK first and &container->c_objects_lock second, but as long as the
634b5fca8f8Stomee * callback uses rw_tryenter() (preventing the deadlock described earlier), it's
635b5fca8f8Stomee * not a problem. Holding the lock on the object list in the example above
636b5fca8f8Stomee * through the entire callback not only prevents the object from going away, it
637b5fca8f8Stomee * also allows you to lock the list elsewhere and know that none of its elements
638b5fca8f8Stomee * will move during iteration.
639b5fca8f8Stomee *
640b5fca8f8Stomee * Adding an explicit hold everywhere an object from the cache is used is tricky
641b5fca8f8Stomee * and involves much more change to client code than a cache-specific solution
642b5fca8f8Stomee * that leverages existing state to decide whether or not an object is
643b5fca8f8Stomee * movable. However, this approach has the advantage that no object remains
644b5fca8f8Stomee * immovable for any significant length of time, making it extremely unlikely
645b5fca8f8Stomee * that long-lived allocations can continue holding slabs hostage; and it works
646b5fca8f8Stomee * for any cache.
647b5fca8f8Stomee *
648b5fca8f8Stomee * 3. Consolidator Implementation
649b5fca8f8Stomee *
650b5fca8f8Stomee * Once the client supplies a move function that a) recognizes known objects and
651b5fca8f8Stomee * b) avoids moving objects that are actively in use, the remaining work is up
652b5fca8f8Stomee * to the consolidator to decide which objects to move and when to issue
653b5fca8f8Stomee * callbacks.
654b5fca8f8Stomee *
655b5fca8f8Stomee * The consolidator relies on the fact that a cache's slabs are ordered by
656b5fca8f8Stomee * usage. Each slab has a fixed number of objects. Depending on the slab's
657b5fca8f8Stomee * "color" (the offset of the first object from the beginning of the slab;
658b5fca8f8Stomee * offsets are staggered to mitigate false sharing of cache lines) it is either
659b5fca8f8Stomee * the maximum number of objects per slab determined at cache creation time or
660b5fca8f8Stomee * else the number closest to the maximum that fits within the space remaining
661b5fca8f8Stomee * after the initial offset. A completely allocated slab may contribute some
662b5fca8f8Stomee * internal fragmentation (per-slab overhead) but no external fragmentation, so
663b5fca8f8Stomee * it is of no interest to the consolidator. At the other extreme, slabs whose
664b5fca8f8Stomee * objects have all been freed to the slab are released to the virtual memory
665b5fca8f8Stomee * (VM) subsystem (objects freed to magazines are still allocated as far as the
666b5fca8f8Stomee * slab is concerned). External fragmentation exists when there are slabs
667b5fca8f8Stomee * somewhere between these extremes. A partial slab has at least one but not all
668b5fca8f8Stomee * of its objects allocated. The more partial slabs, and the fewer allocated
669b5fca8f8Stomee * objects on each of them, the higher the fragmentation. Hence the
670b5fca8f8Stomee * consolidator's overall strategy is to reduce the number of partial slabs by
671b5fca8f8Stomee * moving allocated objects from the least allocated slabs to the most allocated
672b5fca8f8Stomee * slabs.
673b5fca8f8Stomee *
674b5fca8f8Stomee * Partial slabs are kept in an AVL tree ordered by usage. Completely allocated
675b5fca8f8Stomee * slabs are kept separately in an unordered list. Since the majority of slabs
676b5fca8f8Stomee * tend to be completely allocated (a typical unfragmented cache may have
677b5fca8f8Stomee * thousands of complete slabs and only a single partial slab), separating
678b5fca8f8Stomee * complete slabs improves the efficiency of partial slab ordering, since the
679b5fca8f8Stomee * complete slabs do not affect the depth or balance of the AVL tree. This
680b5fca8f8Stomee * ordered sequence of partial slabs acts as a "free list" supplying objects for
681b5fca8f8Stomee * allocation requests.
682b5fca8f8Stomee *
683b5fca8f8Stomee * Objects are always allocated from the first partial slab in the free list,
684b5fca8f8Stomee * where the allocation is most likely to eliminate a partial slab (by
685b5fca8f8Stomee * completely allocating it). Conversely, when a single object from a completely
686b5fca8f8Stomee * allocated slab is freed to the slab, that slab is added to the front of the
687b5fca8f8Stomee * free list. Since most free list activity involves highly allocated slabs
688b5fca8f8Stomee * coming and going at the front of the list, slabs tend naturally toward the
689b5fca8f8Stomee * ideal order: highly allocated at the front, sparsely allocated at the back.
690b5fca8f8Stomee * Slabs with few allocated objects are likely to become completely free if they
691b5fca8f8Stomee * keep a safe distance away from the front of the free list. Slab misorders
692b5fca8f8Stomee * interfere with the natural tendency of slabs to become completely free or
693b5fca8f8Stomee * completely allocated. For example, a slab with a single allocated object
694b5fca8f8Stomee * needs only a single free to escape the cache; its natural desire is
695b5fca8f8Stomee * frustrated when it finds itself at the front of the list where a second
696b5fca8f8Stomee * allocation happens just before the free could have released it. Another slab
697b5fca8f8Stomee * with all but one object allocated might have supplied the buffer instead, so
698b5fca8f8Stomee * that both (as opposed to neither) of the slabs would have been taken off the
699b5fca8f8Stomee * free list.
700b5fca8f8Stomee *
701b5fca8f8Stomee * Although slabs tend naturally toward the ideal order, misorders allowed by a
702b5fca8f8Stomee * simple list implementation defeat the consolidator's strategy of merging
703b5fca8f8Stomee * least- and most-allocated slabs. Without an AVL tree to guarantee order, kmem
704b5fca8f8Stomee * needs another way to fix misorders to optimize its callback strategy. One
705b5fca8f8Stomee * approach is to periodically scan a limited number of slabs, advancing a
706b5fca8f8Stomee * marker to hold the current scan position, and to move extreme misorders to
707b5fca8f8Stomee * the front or back of the free list and to the front or back of the current
708b5fca8f8Stomee * scan range. By making consecutive scan ranges overlap by one slab, the least
709b5fca8f8Stomee * allocated slab in the current range can be carried along from the end of one
710b5fca8f8Stomee * scan to the start of the next.
711b5fca8f8Stomee *
712b5fca8f8Stomee * Maintaining partial slabs in an AVL tree relieves kmem of this additional
713b5fca8f8Stomee * task, however. Since most of the cache's activity is in the magazine layer,
714b5fca8f8Stomee * and allocations from the slab layer represent only a startup cost, the
715b5fca8f8Stomee * overhead of maintaining a balanced tree is not a significant concern compared
716b5fca8f8Stomee * to the opportunity of reducing complexity by eliminating the partial slab
717b5fca8f8Stomee * scanner just described. The overhead of an AVL tree is minimized by
718b5fca8f8Stomee * maintaining only partial slabs in the tree and keeping completely allocated
719b5fca8f8Stomee * slabs separately in a list. To avoid increasing the size of the slab
720b5fca8f8Stomee * structure the AVL linkage pointers are reused for the slab's list linkage,
721b5fca8f8Stomee * since the slab will always be either partial or complete, never stored both
722b5fca8f8Stomee * ways at the same time. To further minimize the overhead of the AVL tree the
723b5fca8f8Stomee * compare function that orders partial slabs by usage divides the range of
724b5fca8f8Stomee * allocated object counts into bins such that counts within the same bin are
725b5fca8f8Stomee * considered equal. Binning partial slabs makes it less likely that allocating
726b5fca8f8Stomee * or freeing a single object will change the slab's order, requiring a tree
727b5fca8f8Stomee * reinsertion (an avl_remove() followed by an avl_add(), both potentially
728b5fca8f8Stomee * requiring some rebalancing of the tree). Allocation counts closest to
729b5fca8f8Stomee * completely free and completely allocated are left unbinned (finely sorted) to
730b5fca8f8Stomee * better support the consolidator's strategy of merging slabs at either
731b5fca8f8Stomee * extreme.
732b5fca8f8Stomee *
733b5fca8f8Stomee * 3.1 Assessing Fragmentation and Selecting Candidate Slabs
734b5fca8f8Stomee *
735b5fca8f8Stomee * The consolidator piggybacks on the kmem maintenance thread and is called on
736b5fca8f8Stomee * the same interval as kmem_cache_update(), once per cache every fifteen
737b5fca8f8Stomee * seconds. kmem maintains a running count of unallocated objects in the slab
738b5fca8f8Stomee * layer (cache_bufslab). The consolidator checks whether that number exceeds
739b5fca8f8Stomee * 12.5% (1/8) of the total objects in the cache (cache_buftotal), and whether
740b5fca8f8Stomee * there is a significant number of slabs in the cache (arbitrarily a minimum
741b5fca8f8Stomee * 101 total slabs). Unused objects that have fallen out of the magazine layer's
742b5fca8f8Stomee * working set are included in the assessment, and magazines in the depot are
743b5fca8f8Stomee * reaped if those objects would lift cache_bufslab above the fragmentation
744b5fca8f8Stomee * threshold. Once the consolidator decides that a cache is fragmented, it looks
745b5fca8f8Stomee * for a candidate slab to reclaim, starting at the end of the partial slab free
746b5fca8f8Stomee * list and scanning backwards. At first the consolidator is choosy: only a slab
747b5fca8f8Stomee * with fewer than 12.5% (1/8) of its objects allocated qualifies (or else a
748b5fca8f8Stomee * single allocated object, regardless of percentage). If there is difficulty
749b5fca8f8Stomee * finding a candidate slab, kmem raises the allocation threshold incrementally,
750b5fca8f8Stomee * up to a maximum 87.5% (7/8), so that eventually the consolidator will reduce
751b5fca8f8Stomee * external fragmentation (unused objects on the free list) below 12.5% (1/8),
752b5fca8f8Stomee * even in the worst case of every slab in the cache being almost 7/8 allocated.
753b5fca8f8Stomee * The threshold can also be lowered incrementally when candidate slabs are easy
754b5fca8f8Stomee * to find, and the threshold is reset to the minimum 1/8 as soon as the cache
755b5fca8f8Stomee * is no longer fragmented.
756b5fca8f8Stomee *
757b5fca8f8Stomee * 3.2 Generating Callbacks
758b5fca8f8Stomee *
759b5fca8f8Stomee * Once an eligible slab is chosen, a callback is generated for every allocated
760b5fca8f8Stomee * object on the slab, in the hope that the client will move everything off the
761b5fca8f8Stomee * slab and make it reclaimable. Objects selected as move destinations are
762b5fca8f8Stomee * chosen from slabs at the front of the free list. Assuming slabs in the ideal
763b5fca8f8Stomee * order (most allocated at the front, least allocated at the back) and a
764b5fca8f8Stomee * cooperative client, the consolidator will succeed in removing slabs from both
765b5fca8f8Stomee * ends of the free list, completely allocating on the one hand and completely
766b5fca8f8Stomee * freeing on the other. Objects selected as move destinations are allocated in
767b5fca8f8Stomee * the kmem maintenance thread where move requests are enqueued. A separate
768b5fca8f8Stomee * callback thread removes pending callbacks from the queue and calls the
769b5fca8f8Stomee * client. The separate thread ensures that client code (the move function) does
770b5fca8f8Stomee * not interfere with internal kmem maintenance tasks. A map of pending
771b5fca8f8Stomee * callbacks keyed by object address (the object to be moved) is checked to
772b5fca8f8Stomee * ensure that duplicate callbacks are not generated for the same object.
773b5fca8f8Stomee * Allocating the move destination (the object to move to) prevents subsequent
774b5fca8f8Stomee * callbacks from selecting the same destination as an earlier pending callback.
775b5fca8f8Stomee *
776b5fca8f8Stomee * Move requests can also be generated by kmem_cache_reap() when the system is
777b5fca8f8Stomee * desperate for memory and by kmem_cache_move_notify(), called by the client to
778b5fca8f8Stomee * notify kmem that a move refused earlier with KMEM_CBRC_LATER is now possible.
779b5fca8f8Stomee * The map of pending callbacks is protected by the same lock that protects the
780b5fca8f8Stomee * slab layer.
781b5fca8f8Stomee *
782b5fca8f8Stomee * When the system is desperate for memory, kmem does not bother to determine
783b5fca8f8Stomee * whether or not the cache exceeds the fragmentation threshold, but tries to
784b5fca8f8Stomee * consolidate as many slabs as possible. Normally, the consolidator chews
785b5fca8f8Stomee * slowly, one sparsely allocated slab at a time during each maintenance
786b5fca8f8Stomee * interval that the cache is fragmented. When desperate, the consolidator
787b5fca8f8Stomee * starts at the last partial slab and enqueues callbacks for every allocated
788b5fca8f8Stomee * object on every partial slab, working backwards until it reaches the first
789b5fca8f8Stomee * partial slab. The first partial slab, meanwhile, advances in pace with the
790b5fca8f8Stomee * consolidator as allocations to supply move destinations for the enqueued
791b5fca8f8Stomee * callbacks use up the highly allocated slabs at the front of the free list.
792b5fca8f8Stomee * Ideally, the overgrown free list collapses like an accordion, starting at
793b5fca8f8Stomee * both ends and ending at the center with a single partial slab.
794b5fca8f8Stomee *
795b5fca8f8Stomee * 3.3 Client Responses
796b5fca8f8Stomee *
797b5fca8f8Stomee * When the client returns KMEM_CBRC_NO in response to the move callback, kmem
798b5fca8f8Stomee * marks the slab that supplied the stuck object non-reclaimable and moves it to
799b5fca8f8Stomee * front of the free list. The slab remains marked as long as it remains on the
800b5fca8f8Stomee * free list, and it appears more allocated to the partial slab compare function
801b5fca8f8Stomee * than any unmarked slab, no matter how many of its objects are allocated.
802b5fca8f8Stomee * Since even one immovable object ties up the entire slab, the goal is to
803b5fca8f8Stomee * completely allocate any slab that cannot be completely freed. kmem does not
804b5fca8f8Stomee * bother generating callbacks to move objects from a marked slab unless the
805b5fca8f8Stomee * system is desperate.
806b5fca8f8Stomee *
807b5fca8f8Stomee * When the client responds KMEM_CBRC_LATER, kmem increments a count for the
808b5fca8f8Stomee * slab. If the client responds LATER too many times, kmem disbelieves and
809b5fca8f8Stomee * treats the response as a NO. The count is cleared when the slab is taken off
810b5fca8f8Stomee * the partial slab list or when the client moves one of the slab's objects.
811b5fca8f8Stomee *
812b5fca8f8Stomee * 4. Observability
813b5fca8f8Stomee *
814b5fca8f8Stomee * A kmem cache's external fragmentation is best observed with 'mdb -k' using
815b5fca8f8Stomee * the ::kmem_slabs dcmd. For a complete description of the command, enter
816b5fca8f8Stomee * '::help kmem_slabs' at the mdb prompt.
8177c478bd9Sstevel@tonic-gate */
8187c478bd9Sstevel@tonic-gate
8197c478bd9Sstevel@tonic-gate #include <sys/kmem_impl.h>
8207c478bd9Sstevel@tonic-gate #include <sys/vmem_impl.h>
8217c478bd9Sstevel@tonic-gate #include <sys/param.h>
8227c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
8237c478bd9Sstevel@tonic-gate #include <sys/vm.h>
8247c478bd9Sstevel@tonic-gate #include <sys/proc.h>
8257c478bd9Sstevel@tonic-gate #include <sys/tuneable.h>
8267c478bd9Sstevel@tonic-gate #include <sys/systm.h>
8277c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
8287c478bd9Sstevel@tonic-gate #include <sys/debug.h>
829b5fca8f8Stomee #include <sys/sdt.h>
8307c478bd9Sstevel@tonic-gate #include <sys/mutex.h>
8317c478bd9Sstevel@tonic-gate #include <sys/bitmap.h>
8327c478bd9Sstevel@tonic-gate #include <sys/atomic.h>
8337c478bd9Sstevel@tonic-gate #include <sys/kobj.h>
8347c478bd9Sstevel@tonic-gate #include <sys/disp.h>
8357c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
8367c478bd9Sstevel@tonic-gate #include <sys/log.h>
8377c478bd9Sstevel@tonic-gate #include <sys/callb.h>
8387c478bd9Sstevel@tonic-gate #include <sys/taskq.h>
8397c478bd9Sstevel@tonic-gate #include <sys/modctl.h>
8407c478bd9Sstevel@tonic-gate #include <sys/reboot.h>
8417c478bd9Sstevel@tonic-gate #include <sys/id32.h>
8427c478bd9Sstevel@tonic-gate #include <sys/zone.h>
843f4b3ec61Sdh #include <sys/netstack.h>
844b5fca8f8Stomee #ifdef DEBUG
845b5fca8f8Stomee #include <sys/random.h>
846b5fca8f8Stomee #endif
8477c478bd9Sstevel@tonic-gate
8487c478bd9Sstevel@tonic-gate extern void streams_msg_init(void);
8497c478bd9Sstevel@tonic-gate extern int segkp_fromheap;
8507c478bd9Sstevel@tonic-gate extern void segkp_cache_free(void);
8516e00b116SPeter Telford extern int callout_init_done;
8527c478bd9Sstevel@tonic-gate
8537c478bd9Sstevel@tonic-gate struct kmem_cache_kstat {
8547c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_size;
8557c478bd9Sstevel@tonic-gate kstat_named_t kmc_align;
8567c478bd9Sstevel@tonic-gate kstat_named_t kmc_chunk_size;
8577c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_size;
8587c478bd9Sstevel@tonic-gate kstat_named_t kmc_alloc;
8597c478bd9Sstevel@tonic-gate kstat_named_t kmc_alloc_fail;
8607c478bd9Sstevel@tonic-gate kstat_named_t kmc_free;
8617c478bd9Sstevel@tonic-gate kstat_named_t kmc_depot_alloc;
8627c478bd9Sstevel@tonic-gate kstat_named_t kmc_depot_free;
8637c478bd9Sstevel@tonic-gate kstat_named_t kmc_depot_contention;
8647c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_alloc;
8657c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_free;
8667c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_constructed;
8677c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_avail;
8687c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_inuse;
8697c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_total;
8707c478bd9Sstevel@tonic-gate kstat_named_t kmc_buf_max;
8717c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_create;
8727c478bd9Sstevel@tonic-gate kstat_named_t kmc_slab_destroy;
8737c478bd9Sstevel@tonic-gate kstat_named_t kmc_vmem_source;
8747c478bd9Sstevel@tonic-gate kstat_named_t kmc_hash_size;
8757c478bd9Sstevel@tonic-gate kstat_named_t kmc_hash_lookup_depth;
8767c478bd9Sstevel@tonic-gate kstat_named_t kmc_hash_rescale;
8777c478bd9Sstevel@tonic-gate kstat_named_t kmc_full_magazines;
8787c478bd9Sstevel@tonic-gate kstat_named_t kmc_empty_magazines;
8797c478bd9Sstevel@tonic-gate kstat_named_t kmc_magazine_size;
880686031edSTom Erickson kstat_named_t kmc_reap; /* number of kmem_cache_reap() calls */
881686031edSTom Erickson kstat_named_t kmc_defrag; /* attempts to defrag all partial slabs */
882686031edSTom Erickson kstat_named_t kmc_scan; /* attempts to defrag one partial slab */
883686031edSTom Erickson kstat_named_t kmc_move_callbacks; /* sum of yes, no, later, dn, dk */
884b5fca8f8Stomee kstat_named_t kmc_move_yes;
885b5fca8f8Stomee kstat_named_t kmc_move_no;
886b5fca8f8Stomee kstat_named_t kmc_move_later;
887b5fca8f8Stomee kstat_named_t kmc_move_dont_need;
888686031edSTom Erickson kstat_named_t kmc_move_dont_know; /* obj unrecognized by client ... */
889686031edSTom Erickson kstat_named_t kmc_move_hunt_found; /* ... but found in mag layer */
890686031edSTom Erickson kstat_named_t kmc_move_slabs_freed; /* slabs freed by consolidator */
891686031edSTom Erickson kstat_named_t kmc_move_reclaimable; /* buffers, if consolidator ran */
8927c478bd9Sstevel@tonic-gate } kmem_cache_kstat = {
8937c478bd9Sstevel@tonic-gate { "buf_size", KSTAT_DATA_UINT64 },
8947c478bd9Sstevel@tonic-gate { "align", KSTAT_DATA_UINT64 },
8957c478bd9Sstevel@tonic-gate { "chunk_size", KSTAT_DATA_UINT64 },
8967c478bd9Sstevel@tonic-gate { "slab_size", KSTAT_DATA_UINT64 },
8977c478bd9Sstevel@tonic-gate { "alloc", KSTAT_DATA_UINT64 },
8987c478bd9Sstevel@tonic-gate { "alloc_fail", KSTAT_DATA_UINT64 },
8997c478bd9Sstevel@tonic-gate { "free", KSTAT_DATA_UINT64 },
9007c478bd9Sstevel@tonic-gate { "depot_alloc", KSTAT_DATA_UINT64 },
9017c478bd9Sstevel@tonic-gate { "depot_free", KSTAT_DATA_UINT64 },
9027c478bd9Sstevel@tonic-gate { "depot_contention", KSTAT_DATA_UINT64 },
9037c478bd9Sstevel@tonic-gate { "slab_alloc", KSTAT_DATA_UINT64 },
9047c478bd9Sstevel@tonic-gate { "slab_free", KSTAT_DATA_UINT64 },
9057c478bd9Sstevel@tonic-gate { "buf_constructed", KSTAT_DATA_UINT64 },
9067c478bd9Sstevel@tonic-gate { "buf_avail", KSTAT_DATA_UINT64 },
9077c478bd9Sstevel@tonic-gate { "buf_inuse", KSTAT_DATA_UINT64 },
9087c478bd9Sstevel@tonic-gate { "buf_total", KSTAT_DATA_UINT64 },
9097c478bd9Sstevel@tonic-gate { "buf_max", KSTAT_DATA_UINT64 },
9107c478bd9Sstevel@tonic-gate { "slab_create", KSTAT_DATA_UINT64 },
9117c478bd9Sstevel@tonic-gate { "slab_destroy", KSTAT_DATA_UINT64 },
9127c478bd9Sstevel@tonic-gate { "vmem_source", KSTAT_DATA_UINT64 },
9137c478bd9Sstevel@tonic-gate { "hash_size", KSTAT_DATA_UINT64 },
9147c478bd9Sstevel@tonic-gate { "hash_lookup_depth", KSTAT_DATA_UINT64 },
9157c478bd9Sstevel@tonic-gate { "hash_rescale", KSTAT_DATA_UINT64 },
9167c478bd9Sstevel@tonic-gate { "full_magazines", KSTAT_DATA_UINT64 },
9177c478bd9Sstevel@tonic-gate { "empty_magazines", KSTAT_DATA_UINT64 },
9187c478bd9Sstevel@tonic-gate { "magazine_size", KSTAT_DATA_UINT64 },
919686031edSTom Erickson { "reap", KSTAT_DATA_UINT64 },
920686031edSTom Erickson { "defrag", KSTAT_DATA_UINT64 },
921686031edSTom Erickson { "scan", KSTAT_DATA_UINT64 },
922b5fca8f8Stomee { "move_callbacks", KSTAT_DATA_UINT64 },
923b5fca8f8Stomee { "move_yes", KSTAT_DATA_UINT64 },
924b5fca8f8Stomee { "move_no", KSTAT_DATA_UINT64 },
925b5fca8f8Stomee { "move_later", KSTAT_DATA_UINT64 },
926b5fca8f8Stomee { "move_dont_need", KSTAT_DATA_UINT64 },
927b5fca8f8Stomee { "move_dont_know", KSTAT_DATA_UINT64 },
928b5fca8f8Stomee { "move_hunt_found", KSTAT_DATA_UINT64 },
929686031edSTom Erickson { "move_slabs_freed", KSTAT_DATA_UINT64 },
930686031edSTom Erickson { "move_reclaimable", KSTAT_DATA_UINT64 },
9317c478bd9Sstevel@tonic-gate };
9327c478bd9Sstevel@tonic-gate
9337c478bd9Sstevel@tonic-gate static kmutex_t kmem_cache_kstat_lock;
9347c478bd9Sstevel@tonic-gate
9357c478bd9Sstevel@tonic-gate /*
9367c478bd9Sstevel@tonic-gate * The default set of caches to back kmem_alloc().
9377c478bd9Sstevel@tonic-gate * These sizes should be reevaluated periodically.
9387c478bd9Sstevel@tonic-gate *
9397c478bd9Sstevel@tonic-gate * We want allocations that are multiples of the coherency granularity
9407c478bd9Sstevel@tonic-gate * (64 bytes) to be satisfied from a cache which is a multiple of 64
9417c478bd9Sstevel@tonic-gate * bytes, so that it will be 64-byte aligned. For all multiples of 64,
9427c478bd9Sstevel@tonic-gate * the next kmem_cache_size greater than or equal to it must be a
9437c478bd9Sstevel@tonic-gate * multiple of 64.
944dce01e3fSJonathan W Adams *
945dce01e3fSJonathan W Adams * We split the table into two sections: size <= 4k and size > 4k. This
946dce01e3fSJonathan W Adams * saves a lot of space and cache footprint in our cache tables.
9477c478bd9Sstevel@tonic-gate */
9487c478bd9Sstevel@tonic-gate static const int kmem_alloc_sizes[] = {
9497c478bd9Sstevel@tonic-gate 1 * 8,
9507c478bd9Sstevel@tonic-gate 2 * 8,
9517c478bd9Sstevel@tonic-gate 3 * 8,
9527c478bd9Sstevel@tonic-gate 4 * 8, 5 * 8, 6 * 8, 7 * 8,
9537c478bd9Sstevel@tonic-gate 4 * 16, 5 * 16, 6 * 16, 7 * 16,
9547c478bd9Sstevel@tonic-gate 4 * 32, 5 * 32, 6 * 32, 7 * 32,
9557c478bd9Sstevel@tonic-gate 4 * 64, 5 * 64, 6 * 64, 7 * 64,
9567c478bd9Sstevel@tonic-gate 4 * 128, 5 * 128, 6 * 128, 7 * 128,
9577c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 7, 64),
9587c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 6, 64),
9597c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 5, 64),
9607c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 4, 64),
9617c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 3, 64),
9627c478bd9Sstevel@tonic-gate P2ALIGN(8192 / 2, 64),
9637c478bd9Sstevel@tonic-gate };
9647c478bd9Sstevel@tonic-gate
965dce01e3fSJonathan W Adams static const int kmem_big_alloc_sizes[] = {
966dce01e3fSJonathan W Adams 2 * 4096, 3 * 4096,
967dce01e3fSJonathan W Adams 2 * 8192, 3 * 8192,
968dce01e3fSJonathan W Adams 4 * 8192, 5 * 8192, 6 * 8192, 7 * 8192,
969dce01e3fSJonathan W Adams 8 * 8192, 9 * 8192, 10 * 8192, 11 * 8192,
970dce01e3fSJonathan W Adams 12 * 8192, 13 * 8192, 14 * 8192, 15 * 8192,
971dce01e3fSJonathan W Adams 16 * 8192
972dce01e3fSJonathan W Adams };
973dce01e3fSJonathan W Adams
974dce01e3fSJonathan W Adams #define KMEM_MAXBUF 4096
975dce01e3fSJonathan W Adams #define KMEM_BIG_MAXBUF_32BIT 32768
976dce01e3fSJonathan W Adams #define KMEM_BIG_MAXBUF 131072
977dce01e3fSJonathan W Adams
978dce01e3fSJonathan W Adams #define KMEM_BIG_MULTIPLE 4096 /* big_alloc_sizes must be a multiple */
979dce01e3fSJonathan W Adams #define KMEM_BIG_SHIFT 12 /* lg(KMEM_BIG_MULTIPLE) */
9807c478bd9Sstevel@tonic-gate
9817c478bd9Sstevel@tonic-gate static kmem_cache_t *kmem_alloc_table[KMEM_MAXBUF >> KMEM_ALIGN_SHIFT];
982dce01e3fSJonathan W Adams static kmem_cache_t *kmem_big_alloc_table[KMEM_BIG_MAXBUF >> KMEM_BIG_SHIFT];
983dce01e3fSJonathan W Adams
984dce01e3fSJonathan W Adams #define KMEM_ALLOC_TABLE_MAX (KMEM_MAXBUF >> KMEM_ALIGN_SHIFT)
985dce01e3fSJonathan W Adams static size_t kmem_big_alloc_table_max = 0; /* # of filled elements */
9867c478bd9Sstevel@tonic-gate
9877c478bd9Sstevel@tonic-gate static kmem_magtype_t kmem_magtype[] = {
9887c478bd9Sstevel@tonic-gate { 1, 8, 3200, 65536 },
9897c478bd9Sstevel@tonic-gate { 3, 16, 256, 32768 },
9907c478bd9Sstevel@tonic-gate { 7, 32, 64, 16384 },
9917c478bd9Sstevel@tonic-gate { 15, 64, 0, 8192 },
9927c478bd9Sstevel@tonic-gate { 31, 64, 0, 4096 },
9937c478bd9Sstevel@tonic-gate { 47, 64, 0, 2048 },
9947c478bd9Sstevel@tonic-gate { 63, 64, 0, 1024 },
9957c478bd9Sstevel@tonic-gate { 95, 64, 0, 512 },
9967c478bd9Sstevel@tonic-gate { 143, 64, 0, 0 },
9977c478bd9Sstevel@tonic-gate };
9987c478bd9Sstevel@tonic-gate
9997c478bd9Sstevel@tonic-gate static uint32_t kmem_reaping;
10007c478bd9Sstevel@tonic-gate static uint32_t kmem_reaping_idspace;
10017c478bd9Sstevel@tonic-gate
10027c478bd9Sstevel@tonic-gate /*
10037c478bd9Sstevel@tonic-gate * kmem tunables
10047c478bd9Sstevel@tonic-gate */
10057c478bd9Sstevel@tonic-gate clock_t kmem_reap_interval; /* cache reaping rate [15 * HZ ticks] */
10067c478bd9Sstevel@tonic-gate int kmem_depot_contention = 3; /* max failed tryenters per real interval */
10077c478bd9Sstevel@tonic-gate pgcnt_t kmem_reapahead = 0; /* start reaping N pages before pageout */
10087c478bd9Sstevel@tonic-gate int kmem_panic = 1; /* whether to panic on error */
10097c478bd9Sstevel@tonic-gate int kmem_logging = 1; /* kmem_log_enter() override */
10107c478bd9Sstevel@tonic-gate uint32_t kmem_mtbf = 0; /* mean time between failures [default: off] */
10117c478bd9Sstevel@tonic-gate size_t kmem_transaction_log_size; /* transaction log size [2% of memory] */
10127c478bd9Sstevel@tonic-gate size_t kmem_content_log_size; /* content log size [2% of memory] */
10137c478bd9Sstevel@tonic-gate size_t kmem_failure_log_size; /* failure log [4 pages per CPU] */
10147c478bd9Sstevel@tonic-gate size_t kmem_slab_log_size; /* slab create log [4 pages per CPU] */
1015d1580181SBryan Cantrill size_t kmem_zerosized_log_size; /* zero-sized log [4 pages per CPU] */
10167c478bd9Sstevel@tonic-gate size_t kmem_content_maxsave = 256; /* KMF_CONTENTS max bytes to log */
10177c478bd9Sstevel@tonic-gate size_t kmem_lite_minsize = 0; /* minimum buffer size for KMF_LITE */
10187c478bd9Sstevel@tonic-gate size_t kmem_lite_maxalign = 1024; /* maximum buffer alignment for KMF_LITE */
10197c478bd9Sstevel@tonic-gate int kmem_lite_pcs = 4; /* number of PCs to store in KMF_LITE mode */
10207c478bd9Sstevel@tonic-gate size_t kmem_maxverify; /* maximum bytes to inspect in debug routines */
10217c478bd9Sstevel@tonic-gate size_t kmem_minfirewall; /* hardware-enforced redzone threshold */
10227c478bd9Sstevel@tonic-gate
1023d1580181SBryan Cantrill #ifdef DEBUG
1024d1580181SBryan Cantrill int kmem_warn_zerosized = 1; /* whether to warn on zero-sized KM_SLEEP */
1025d1580181SBryan Cantrill #else
1026d1580181SBryan Cantrill int kmem_warn_zerosized = 0; /* whether to warn on zero-sized KM_SLEEP */
1027d1580181SBryan Cantrill #endif
1028d1580181SBryan Cantrill
1029d1580181SBryan Cantrill int kmem_panic_zerosized = 0; /* whether to panic on zero-sized KM_SLEEP */
1030d1580181SBryan Cantrill
1031dce01e3fSJonathan W Adams #ifdef _LP64
1032dce01e3fSJonathan W Adams size_t kmem_max_cached = KMEM_BIG_MAXBUF; /* maximum kmem_alloc cache */
1033dce01e3fSJonathan W Adams #else
1034dce01e3fSJonathan W Adams size_t kmem_max_cached = KMEM_BIG_MAXBUF_32BIT; /* maximum kmem_alloc cache */
1035dce01e3fSJonathan W Adams #endif
1036dce01e3fSJonathan W Adams
10377c478bd9Sstevel@tonic-gate #ifdef DEBUG
10387c478bd9Sstevel@tonic-gate int kmem_flags = KMF_AUDIT | KMF_DEADBEEF | KMF_REDZONE | KMF_CONTENTS;
10397c478bd9Sstevel@tonic-gate #else
10407c478bd9Sstevel@tonic-gate int kmem_flags = 0;
10417c478bd9Sstevel@tonic-gate #endif
10427c478bd9Sstevel@tonic-gate int kmem_ready;
10437c478bd9Sstevel@tonic-gate
10447c478bd9Sstevel@tonic-gate static kmem_cache_t *kmem_slab_cache;
10457c478bd9Sstevel@tonic-gate static kmem_cache_t *kmem_bufctl_cache;
10467c478bd9Sstevel@tonic-gate static kmem_cache_t *kmem_bufctl_audit_cache;
10477c478bd9Sstevel@tonic-gate
10487c478bd9Sstevel@tonic-gate static kmutex_t kmem_cache_lock; /* inter-cache linkage only */
1049b5fca8f8Stomee static list_t kmem_caches;
10507c478bd9Sstevel@tonic-gate
10517c478bd9Sstevel@tonic-gate static taskq_t *kmem_taskq;
10527c478bd9Sstevel@tonic-gate static kmutex_t kmem_flags_lock;
10537c478bd9Sstevel@tonic-gate static vmem_t *kmem_metadata_arena;
10547c478bd9Sstevel@tonic-gate static vmem_t *kmem_msb_arena; /* arena for metadata caches */
10557c478bd9Sstevel@tonic-gate static vmem_t *kmem_cache_arena;
10567c478bd9Sstevel@tonic-gate static vmem_t *kmem_hash_arena;
10577c478bd9Sstevel@tonic-gate static vmem_t *kmem_log_arena;
10587c478bd9Sstevel@tonic-gate static vmem_t *kmem_oversize_arena;
10597c478bd9Sstevel@tonic-gate static vmem_t *kmem_va_arena;
10607c478bd9Sstevel@tonic-gate static vmem_t *kmem_default_arena;
10617c478bd9Sstevel@tonic-gate static vmem_t *kmem_firewall_va_arena;
10627c478bd9Sstevel@tonic-gate static vmem_t *kmem_firewall_arena;
10637c478bd9Sstevel@tonic-gate
1064d1580181SBryan Cantrill static int kmem_zerosized; /* # of zero-sized allocs */
1065d1580181SBryan Cantrill
1066b5fca8f8Stomee /*
1067b5fca8f8Stomee * kmem slab consolidator thresholds (tunables)
1068b5fca8f8Stomee */
1069686031edSTom Erickson size_t kmem_frag_minslabs = 101; /* minimum total slabs */
1070686031edSTom Erickson size_t kmem_frag_numer = 1; /* free buffers (numerator) */
1071686031edSTom Erickson size_t kmem_frag_denom = KMEM_VOID_FRACTION; /* buffers (denominator) */
1072b5fca8f8Stomee /*
1073b5fca8f8Stomee * Maximum number of slabs from which to move buffers during a single
1074b5fca8f8Stomee * maintenance interval while the system is not low on memory.
1075b5fca8f8Stomee */
1076686031edSTom Erickson size_t kmem_reclaim_max_slabs = 1;
1077b5fca8f8Stomee /*
1078b5fca8f8Stomee * Number of slabs to scan backwards from the end of the partial slab list
1079b5fca8f8Stomee * when searching for buffers to relocate.
1080b5fca8f8Stomee */
1081686031edSTom Erickson size_t kmem_reclaim_scan_range = 12;
1082b5fca8f8Stomee
1083b5fca8f8Stomee /* consolidator knobs */
1084929d5b43SMatthew Ahrens boolean_t kmem_move_noreap;
1085929d5b43SMatthew Ahrens boolean_t kmem_move_blocked;
1086929d5b43SMatthew Ahrens boolean_t kmem_move_fulltilt;
1087929d5b43SMatthew Ahrens boolean_t kmem_move_any_partial;
1088b5fca8f8Stomee
1089b5fca8f8Stomee #ifdef DEBUG
1090b5fca8f8Stomee /*
1091686031edSTom Erickson * kmem consolidator debug tunables:
1092b5fca8f8Stomee * Ensure code coverage by occasionally running the consolidator even when the
1093b5fca8f8Stomee * caches are not fragmented (they may never be). These intervals are mean time
1094b5fca8f8Stomee * in cache maintenance intervals (kmem_cache_update).
1095b5fca8f8Stomee */
1096686031edSTom Erickson uint32_t kmem_mtb_move = 60; /* defrag 1 slab (~15min) */
1097686031edSTom Erickson uint32_t kmem_mtb_reap = 1800; /* defrag all slabs (~7.5hrs) */
1098b5fca8f8Stomee #endif /* DEBUG */
1099b5fca8f8Stomee
1100b5fca8f8Stomee static kmem_cache_t *kmem_defrag_cache;
1101b5fca8f8Stomee static kmem_cache_t *kmem_move_cache;
1102b5fca8f8Stomee static taskq_t *kmem_move_taskq;
1103b5fca8f8Stomee
1104b5fca8f8Stomee static void kmem_cache_scan(kmem_cache_t *);
1105b5fca8f8Stomee static void kmem_cache_defrag(kmem_cache_t *);
1106b942e89bSDavid Valin static void kmem_slab_prefill(kmem_cache_t *, kmem_slab_t *);
1107b5fca8f8Stomee
1108b5fca8f8Stomee
11097c478bd9Sstevel@tonic-gate kmem_log_header_t *kmem_transaction_log;
11107c478bd9Sstevel@tonic-gate kmem_log_header_t *kmem_content_log;
11117c478bd9Sstevel@tonic-gate kmem_log_header_t *kmem_failure_log;
11127c478bd9Sstevel@tonic-gate kmem_log_header_t *kmem_slab_log;
1113d1580181SBryan Cantrill kmem_log_header_t *kmem_zerosized_log;
11147c478bd9Sstevel@tonic-gate
11157c478bd9Sstevel@tonic-gate static int kmem_lite_count; /* # of PCs in kmem_buftag_lite_t */
11167c478bd9Sstevel@tonic-gate
11177c478bd9Sstevel@tonic-gate #define KMEM_BUFTAG_LITE_ENTER(bt, count, caller) \
11187c478bd9Sstevel@tonic-gate if ((count) > 0) { \
11197c478bd9Sstevel@tonic-gate pc_t *_s = ((kmem_buftag_lite_t *)(bt))->bt_history; \
11207c478bd9Sstevel@tonic-gate pc_t *_e; \
11217c478bd9Sstevel@tonic-gate /* memmove() the old entries down one notch */ \
11227c478bd9Sstevel@tonic-gate for (_e = &_s[(count) - 1]; _e > _s; _e--) \
11237c478bd9Sstevel@tonic-gate *_e = *(_e - 1); \
11247c478bd9Sstevel@tonic-gate *_s = (uintptr_t)(caller); \
11257c478bd9Sstevel@tonic-gate }
11267c478bd9Sstevel@tonic-gate
11277c478bd9Sstevel@tonic-gate #define KMERR_MODIFIED 0 /* buffer modified while on freelist */
11287c478bd9Sstevel@tonic-gate #define KMERR_REDZONE 1 /* redzone violation (write past end of buf) */
11297c478bd9Sstevel@tonic-gate #define KMERR_DUPFREE 2 /* freed a buffer twice */
11307c478bd9Sstevel@tonic-gate #define KMERR_BADADDR 3 /* freed a bad (unallocated) address */
11317c478bd9Sstevel@tonic-gate #define KMERR_BADBUFTAG 4 /* buftag corrupted */
11327c478bd9Sstevel@tonic-gate #define KMERR_BADBUFCTL 5 /* bufctl corrupted */
11337c478bd9Sstevel@tonic-gate #define KMERR_BADCACHE 6 /* freed a buffer to the wrong cache */
11347c478bd9Sstevel@tonic-gate #define KMERR_BADSIZE 7 /* alloc size != free size */
11357c478bd9Sstevel@tonic-gate #define KMERR_BADBASE 8 /* buffer base address wrong */
11367c478bd9Sstevel@tonic-gate
11377c478bd9Sstevel@tonic-gate struct {
11387c478bd9Sstevel@tonic-gate hrtime_t kmp_timestamp; /* timestamp of panic */
11397c478bd9Sstevel@tonic-gate int kmp_error; /* type of kmem error */
11407c478bd9Sstevel@tonic-gate void *kmp_buffer; /* buffer that induced panic */
11417c478bd9Sstevel@tonic-gate void *kmp_realbuf; /* real start address for buffer */
11427c478bd9Sstevel@tonic-gate kmem_cache_t *kmp_cache; /* buffer's cache according to client */
11437c478bd9Sstevel@tonic-gate kmem_cache_t *kmp_realcache; /* actual cache containing buffer */
11447c478bd9Sstevel@tonic-gate kmem_slab_t *kmp_slab; /* slab accoring to kmem_findslab() */
11457c478bd9Sstevel@tonic-gate kmem_bufctl_t *kmp_bufctl; /* bufctl */
11467c478bd9Sstevel@tonic-gate } kmem_panic_info;
11477c478bd9Sstevel@tonic-gate
11487c478bd9Sstevel@tonic-gate
11497c478bd9Sstevel@tonic-gate static void
copy_pattern(uint64_t pattern,void * buf_arg,size_t size)11507c478bd9Sstevel@tonic-gate copy_pattern(uint64_t pattern, void *buf_arg, size_t size)
11517c478bd9Sstevel@tonic-gate {
11527c478bd9Sstevel@tonic-gate uint64_t *bufend = (uint64_t *)((char *)buf_arg + size);
11537c478bd9Sstevel@tonic-gate uint64_t *buf = buf_arg;
11547c478bd9Sstevel@tonic-gate
11557c478bd9Sstevel@tonic-gate while (buf < bufend)
11567c478bd9Sstevel@tonic-gate *buf++ = pattern;
11577c478bd9Sstevel@tonic-gate }
11587c478bd9Sstevel@tonic-gate
11597c478bd9Sstevel@tonic-gate static void *
verify_pattern(uint64_t pattern,void * buf_arg,size_t size)11607c478bd9Sstevel@tonic-gate verify_pattern(uint64_t pattern, void *buf_arg, size_t size)
11617c478bd9Sstevel@tonic-gate {
11627c478bd9Sstevel@tonic-gate uint64_t *bufend = (uint64_t *)((char *)buf_arg + size);
11637c478bd9Sstevel@tonic-gate uint64_t *buf;
11647c478bd9Sstevel@tonic-gate
11657c478bd9Sstevel@tonic-gate for (buf = buf_arg; buf < bufend; buf++)
11667c478bd9Sstevel@tonic-gate if (*buf != pattern)
11677c478bd9Sstevel@tonic-gate return (buf);
11687c478bd9Sstevel@tonic-gate return (NULL);
11697c478bd9Sstevel@tonic-gate }
11707c478bd9Sstevel@tonic-gate
11717c478bd9Sstevel@tonic-gate static void *
verify_and_copy_pattern(uint64_t old,uint64_t new,void * buf_arg,size_t size)11727c478bd9Sstevel@tonic-gate verify_and_copy_pattern(uint64_t old, uint64_t new, void *buf_arg, size_t size)
11737c478bd9Sstevel@tonic-gate {
11747c478bd9Sstevel@tonic-gate uint64_t *bufend = (uint64_t *)((char *)buf_arg + size);
11757c478bd9Sstevel@tonic-gate uint64_t *buf;
11767c478bd9Sstevel@tonic-gate
11777c478bd9Sstevel@tonic-gate for (buf = buf_arg; buf < bufend; buf++) {
11787c478bd9Sstevel@tonic-gate if (*buf != old) {
11797c478bd9Sstevel@tonic-gate copy_pattern(old, buf_arg,
11809f1b636aStomee (char *)buf - (char *)buf_arg);
11817c478bd9Sstevel@tonic-gate return (buf);
11827c478bd9Sstevel@tonic-gate }
11837c478bd9Sstevel@tonic-gate *buf = new;
11847c478bd9Sstevel@tonic-gate }
11857c478bd9Sstevel@tonic-gate
11867c478bd9Sstevel@tonic-gate return (NULL);
11877c478bd9Sstevel@tonic-gate }
11887c478bd9Sstevel@tonic-gate
11897c478bd9Sstevel@tonic-gate static void
kmem_cache_applyall(void (* func)(kmem_cache_t *),taskq_t * tq,int tqflag)11907c478bd9Sstevel@tonic-gate kmem_cache_applyall(void (*func)(kmem_cache_t *), taskq_t *tq, int tqflag)
11917c478bd9Sstevel@tonic-gate {
11927c478bd9Sstevel@tonic-gate kmem_cache_t *cp;
11937c478bd9Sstevel@tonic-gate
11947c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_lock);
1195b5fca8f8Stomee for (cp = list_head(&kmem_caches); cp != NULL;
1196b5fca8f8Stomee cp = list_next(&kmem_caches, cp))
11977c478bd9Sstevel@tonic-gate if (tq != NULL)
11987c478bd9Sstevel@tonic-gate (void) taskq_dispatch(tq, (task_func_t *)func, cp,
11997c478bd9Sstevel@tonic-gate tqflag);
12007c478bd9Sstevel@tonic-gate else
12017c478bd9Sstevel@tonic-gate func(cp);
12027c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_lock);
12037c478bd9Sstevel@tonic-gate }
12047c478bd9Sstevel@tonic-gate
12057c478bd9Sstevel@tonic-gate static void
kmem_cache_applyall_id(void (* func)(kmem_cache_t *),taskq_t * tq,int tqflag)12067c478bd9Sstevel@tonic-gate kmem_cache_applyall_id(void (*func)(kmem_cache_t *), taskq_t *tq, int tqflag)
12077c478bd9Sstevel@tonic-gate {
12087c478bd9Sstevel@tonic-gate kmem_cache_t *cp;
12097c478bd9Sstevel@tonic-gate
12107c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_lock);
1211b5fca8f8Stomee for (cp = list_head(&kmem_caches); cp != NULL;
1212b5fca8f8Stomee cp = list_next(&kmem_caches, cp)) {
12137c478bd9Sstevel@tonic-gate if (!(cp->cache_cflags & KMC_IDENTIFIER))
12147c478bd9Sstevel@tonic-gate continue;
12157c478bd9Sstevel@tonic-gate if (tq != NULL)
12167c478bd9Sstevel@tonic-gate (void) taskq_dispatch(tq, (task_func_t *)func, cp,
12177c478bd9Sstevel@tonic-gate tqflag);
12187c478bd9Sstevel@tonic-gate else
12197c478bd9Sstevel@tonic-gate func(cp);
12207c478bd9Sstevel@tonic-gate }
12217c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_lock);
12227c478bd9Sstevel@tonic-gate }
12237c478bd9Sstevel@tonic-gate
12247c478bd9Sstevel@tonic-gate /*
12257c478bd9Sstevel@tonic-gate * Debugging support. Given a buffer address, find its slab.
12267c478bd9Sstevel@tonic-gate */
12277c478bd9Sstevel@tonic-gate static kmem_slab_t *
kmem_findslab(kmem_cache_t * cp,void * buf)12287c478bd9Sstevel@tonic-gate kmem_findslab(kmem_cache_t *cp, void *buf)
12297c478bd9Sstevel@tonic-gate {
12307c478bd9Sstevel@tonic-gate kmem_slab_t *sp;
12317c478bd9Sstevel@tonic-gate
12327c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock);
1233b5fca8f8Stomee for (sp = list_head(&cp->cache_complete_slabs); sp != NULL;
1234b5fca8f8Stomee sp = list_next(&cp->cache_complete_slabs, sp)) {
1235b5fca8f8Stomee if (KMEM_SLAB_MEMBER(sp, buf)) {
1236b5fca8f8Stomee mutex_exit(&cp->cache_lock);
1237b5fca8f8Stomee return (sp);
1238b5fca8f8Stomee }
1239b5fca8f8Stomee }
1240b5fca8f8Stomee for (sp = avl_first(&cp->cache_partial_slabs); sp != NULL;
1241b5fca8f8Stomee sp = AVL_NEXT(&cp->cache_partial_slabs, sp)) {
12427c478bd9Sstevel@tonic-gate if (KMEM_SLAB_MEMBER(sp, buf)) {
12437c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock);
12447c478bd9Sstevel@tonic-gate return (sp);
12457c478bd9Sstevel@tonic-gate }
12467c478bd9Sstevel@tonic-gate }
12477c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock);
12487c478bd9Sstevel@tonic-gate
12497c478bd9Sstevel@tonic-gate return (NULL);
12507c478bd9Sstevel@tonic-gate }
12517c478bd9Sstevel@tonic-gate
12527c478bd9Sstevel@tonic-gate static void
kmem_error(int error,kmem_cache_t * cparg,void * bufarg)12537c478bd9Sstevel@tonic-gate kmem_error(int error, kmem_cache_t *cparg, void *bufarg)
12547c478bd9Sstevel@tonic-gate {
12557c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = NULL;
12567c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp = NULL;
12577c478bd9Sstevel@tonic-gate kmem_cache_t *cp = cparg;
12587c478bd9Sstevel@tonic-gate kmem_slab_t *sp;
12597c478bd9Sstevel@tonic-gate uint64_t *off;
12607c478bd9Sstevel@tonic-gate void *buf = bufarg;
12617c478bd9Sstevel@tonic-gate
12627c478bd9Sstevel@tonic-gate kmem_logging = 0; /* stop logging when a bad thing happens */
12637c478bd9Sstevel@tonic-gate
12647c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_timestamp = gethrtime();
12657c478bd9Sstevel@tonic-gate
12667c478bd9Sstevel@tonic-gate sp = kmem_findslab(cp, buf);
12677c478bd9Sstevel@tonic-gate if (sp == NULL) {
1268b5fca8f8Stomee for (cp = list_tail(&kmem_caches); cp != NULL;
1269b5fca8f8Stomee cp = list_prev(&kmem_caches, cp)) {
12707c478bd9Sstevel@tonic-gate if ((sp = kmem_findslab(cp, buf)) != NULL)
12717c478bd9Sstevel@tonic-gate break;
12727c478bd9Sstevel@tonic-gate }
12737c478bd9Sstevel@tonic-gate }
12747c478bd9Sstevel@tonic-gate
12757c478bd9Sstevel@tonic-gate if (sp == NULL) {
12767c478bd9Sstevel@tonic-gate cp = NULL;
12777c478bd9Sstevel@tonic-gate error = KMERR_BADADDR;
12787c478bd9Sstevel@tonic-gate } else {
12797c478bd9Sstevel@tonic-gate if (cp != cparg)
12807c478bd9Sstevel@tonic-gate error = KMERR_BADCACHE;
12817c478bd9Sstevel@tonic-gate else
12827c478bd9Sstevel@tonic-gate buf = (char *)bufarg - ((uintptr_t)bufarg -
12837c478bd9Sstevel@tonic-gate (uintptr_t)sp->slab_base) % cp->cache_chunksize;
12847c478bd9Sstevel@tonic-gate if (buf != bufarg)
12857c478bd9Sstevel@tonic-gate error = KMERR_BADBASE;
12867c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_BUFTAG)
12877c478bd9Sstevel@tonic-gate btp = KMEM_BUFTAG(cp, buf);
12887c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) {
12897c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock);
12907c478bd9Sstevel@tonic-gate for (bcp = *KMEM_HASH(cp, buf); bcp; bcp = bcp->bc_next)
12917c478bd9Sstevel@tonic-gate if (bcp->bc_addr == buf)
12927c478bd9Sstevel@tonic-gate break;
12937c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock);
12947c478bd9Sstevel@tonic-gate if (bcp == NULL && btp != NULL)
12957c478bd9Sstevel@tonic-gate bcp = btp->bt_bufctl;
12967c478bd9Sstevel@tonic-gate if (kmem_findslab(cp->cache_bufctl_cache, bcp) ==
12977c478bd9Sstevel@tonic-gate NULL || P2PHASE((uintptr_t)bcp, KMEM_ALIGN) ||
12987c478bd9Sstevel@tonic-gate bcp->bc_addr != buf) {
12997c478bd9Sstevel@tonic-gate error = KMERR_BADBUFCTL;
13007c478bd9Sstevel@tonic-gate bcp = NULL;
13017c478bd9Sstevel@tonic-gate }
13027c478bd9Sstevel@tonic-gate }
13037c478bd9Sstevel@tonic-gate }
13047c478bd9Sstevel@tonic-gate
13057c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_error = error;
13067c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_buffer = bufarg;
13077c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_realbuf = buf;
13087c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_cache = cparg;
13097c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_realcache = cp;
13107c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_slab = sp;
13117c478bd9Sstevel@tonic-gate kmem_panic_info.kmp_bufctl = bcp;
13127c478bd9Sstevel@tonic-gate
13137c478bd9Sstevel@tonic-gate printf("kernel memory allocator: ");
13147c478bd9Sstevel@tonic-gate
13157c478bd9Sstevel@tonic-gate switch (error) {
13167c478bd9Sstevel@tonic-gate
13177c478bd9Sstevel@tonic-gate case KMERR_MODIFIED:
13187c478bd9Sstevel@tonic-gate printf("buffer modified after being freed\n");
13197c478bd9Sstevel@tonic-gate off = verify_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify);
13207c478bd9Sstevel@tonic-gate if (off == NULL) /* shouldn't happen */
13217c478bd9Sstevel@tonic-gate off = buf;
13227c478bd9Sstevel@tonic-gate printf("modification occurred at offset 0x%lx "
13237c478bd9Sstevel@tonic-gate "(0x%llx replaced by 0x%llx)\n",
13247c478bd9Sstevel@tonic-gate (uintptr_t)off - (uintptr_t)buf,
13257c478bd9Sstevel@tonic-gate (longlong_t)KMEM_FREE_PATTERN, (longlong_t)*off);
13267c478bd9Sstevel@tonic-gate break;
13277c478bd9Sstevel@tonic-gate
13287c478bd9Sstevel@tonic-gate case KMERR_REDZONE:
13297c478bd9Sstevel@tonic-gate printf("redzone violation: write past end of buffer\n");
13307c478bd9Sstevel@tonic-gate break;
13317c478bd9Sstevel@tonic-gate
13327c478bd9Sstevel@tonic-gate case KMERR_BADADDR:
13337c478bd9Sstevel@tonic-gate printf("invalid free: buffer not in cache\n");
13347c478bd9Sstevel@tonic-gate break;
13357c478bd9Sstevel@tonic-gate
13367c478bd9Sstevel@tonic-gate case KMERR_DUPFREE:
13377c478bd9Sstevel@tonic-gate printf("duplicate free: buffer freed twice\n");
13387c478bd9Sstevel@tonic-gate break;
13397c478bd9Sstevel@tonic-gate
13407c478bd9Sstevel@tonic-gate case KMERR_BADBUFTAG:
13417c478bd9Sstevel@tonic-gate printf("boundary tag corrupted\n");
13427c478bd9Sstevel@tonic-gate printf("bcp ^ bxstat = %lx, should be %lx\n",
13437c478bd9Sstevel@tonic-gate (intptr_t)btp->bt_bufctl ^ btp->bt_bxstat,
13447c478bd9Sstevel@tonic-gate KMEM_BUFTAG_FREE);
13457c478bd9Sstevel@tonic-gate break;
13467c478bd9Sstevel@tonic-gate
13477c478bd9Sstevel@tonic-gate case KMERR_BADBUFCTL:
13487c478bd9Sstevel@tonic-gate printf("bufctl corrupted\n");
13497c478bd9Sstevel@tonic-gate break;
13507c478bd9Sstevel@tonic-gate
13517c478bd9Sstevel@tonic-gate case KMERR_BADCACHE:
13527c478bd9Sstevel@tonic-gate printf("buffer freed to wrong cache\n");
13537c478bd9Sstevel@tonic-gate printf("buffer was allocated from %s,\n", cp->cache_name);
13547c478bd9Sstevel@tonic-gate printf("caller attempting free to %s.\n", cparg->cache_name);
13557c478bd9Sstevel@tonic-gate break;
13567c478bd9Sstevel@tonic-gate
13577c478bd9Sstevel@tonic-gate case KMERR_BADSIZE:
13587c478bd9Sstevel@tonic-gate printf("bad free: free size (%u) != alloc size (%u)\n",
13597c478bd9Sstevel@tonic-gate KMEM_SIZE_DECODE(((uint32_t *)btp)[0]),
13607c478bd9Sstevel@tonic-gate KMEM_SIZE_DECODE(((uint32_t *)btp)[1]));
13617c478bd9Sstevel@tonic-gate break;
13627c478bd9Sstevel@tonic-gate
13637c478bd9Sstevel@tonic-gate case KMERR_BADBASE:
13647c478bd9Sstevel@tonic-gate printf("bad free: free address (%p) != alloc address (%p)\n",
13657c478bd9Sstevel@tonic-gate bufarg, buf);
13667c478bd9Sstevel@tonic-gate break;
13677c478bd9Sstevel@tonic-gate }
13687c478bd9Sstevel@tonic-gate
13697c478bd9Sstevel@tonic-gate printf("buffer=%p bufctl=%p cache: %s\n",
13707c478bd9Sstevel@tonic-gate bufarg, (void *)bcp, cparg->cache_name);
13717c478bd9Sstevel@tonic-gate
13727c478bd9Sstevel@tonic-gate if (bcp != NULL && (cp->cache_flags & KMF_AUDIT) &&
13737c478bd9Sstevel@tonic-gate error != KMERR_BADBUFCTL) {
13747c478bd9Sstevel@tonic-gate int d;
13757c478bd9Sstevel@tonic-gate timestruc_t ts;
13767c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *bcap = (kmem_bufctl_audit_t *)bcp;
13777c478bd9Sstevel@tonic-gate
13787c478bd9Sstevel@tonic-gate hrt2ts(kmem_panic_info.kmp_timestamp - bcap->bc_timestamp, &ts);
13797c478bd9Sstevel@tonic-gate printf("previous transaction on buffer %p:\n", buf);
13807c478bd9Sstevel@tonic-gate printf("thread=%p time=T-%ld.%09ld slab=%p cache: %s\n",
13817c478bd9Sstevel@tonic-gate (void *)bcap->bc_thread, ts.tv_sec, ts.tv_nsec,
13827c478bd9Sstevel@tonic-gate (void *)sp, cp->cache_name);
13837c478bd9Sstevel@tonic-gate for (d = 0; d < MIN(bcap->bc_depth, KMEM_STACK_DEPTH); d++) {
13847c478bd9Sstevel@tonic-gate ulong_t off;
13857c478bd9Sstevel@tonic-gate char *sym = kobj_getsymname(bcap->bc_stack[d], &off);
13867c478bd9Sstevel@tonic-gate printf("%s+%lx\n", sym ? sym : "?", off);
13877c478bd9Sstevel@tonic-gate }
13887c478bd9Sstevel@tonic-gate }
13897c478bd9Sstevel@tonic-gate if (kmem_panic > 0)
13907c478bd9Sstevel@tonic-gate panic("kernel heap corruption detected");
13917c478bd9Sstevel@tonic-gate if (kmem_panic == 0)
13927c478bd9Sstevel@tonic-gate debug_enter(NULL);
13937c478bd9Sstevel@tonic-gate kmem_logging = 1; /* resume logging */
13947c478bd9Sstevel@tonic-gate }
13957c478bd9Sstevel@tonic-gate
13967c478bd9Sstevel@tonic-gate static kmem_log_header_t *
kmem_log_init(size_t logsize)13977c478bd9Sstevel@tonic-gate kmem_log_init(size_t logsize)
13987c478bd9Sstevel@tonic-gate {
13997c478bd9Sstevel@tonic-gate kmem_log_header_t *lhp;
14007c478bd9Sstevel@tonic-gate int nchunks = 4 * max_ncpus;
14017c478bd9Sstevel@tonic-gate size_t lhsize = (size_t)&((kmem_log_header_t *)0)->lh_cpu[max_ncpus];
14027c478bd9Sstevel@tonic-gate int i;
14037c478bd9Sstevel@tonic-gate
14047c478bd9Sstevel@tonic-gate /*
14057c478bd9Sstevel@tonic-gate * Make sure that lhp->lh_cpu[] is nicely aligned
14067c478bd9Sstevel@tonic-gate * to prevent false sharing of cache lines.
14077c478bd9Sstevel@tonic-gate */
14087c478bd9Sstevel@tonic-gate lhsize = P2ROUNDUP(lhsize, KMEM_ALIGN);
14097c478bd9Sstevel@tonic-gate lhp = vmem_xalloc(kmem_log_arena, lhsize, 64, P2NPHASE(lhsize, 64), 0,
14107c478bd9Sstevel@tonic-gate NULL, NULL, VM_SLEEP);
14117c478bd9Sstevel@tonic-gate bzero(lhp, lhsize);
14127c478bd9Sstevel@tonic-gate
14137c478bd9Sstevel@tonic-gate mutex_init(&lhp->lh_lock, NULL, MUTEX_DEFAULT, NULL);
14147c478bd9Sstevel@tonic-gate lhp->lh_nchunks = nchunks;
14157c478bd9Sstevel@tonic-gate lhp->lh_chunksize = P2ROUNDUP(logsize / nchunks + 1, PAGESIZE);
14167c478bd9Sstevel@tonic-gate lhp->lh_base = vmem_alloc(kmem_log_arena,
14177c478bd9Sstevel@tonic-gate lhp->lh_chunksize * nchunks, VM_SLEEP);
14187c478bd9Sstevel@tonic-gate lhp->lh_free = vmem_alloc(kmem_log_arena,
14197c478bd9Sstevel@tonic-gate nchunks * sizeof (int), VM_SLEEP);
14207c478bd9Sstevel@tonic-gate bzero(lhp->lh_base, lhp->lh_chunksize * nchunks);
14217c478bd9Sstevel@tonic-gate
14227c478bd9Sstevel@tonic-gate for (i = 0; i < max_ncpus; i++) {
14237c478bd9Sstevel@tonic-gate kmem_cpu_log_header_t *clhp = &lhp->lh_cpu[i];
14247c478bd9Sstevel@tonic-gate mutex_init(&clhp->clh_lock, NULL, MUTEX_DEFAULT, NULL);
14257c478bd9Sstevel@tonic-gate clhp->clh_chunk = i;
14267c478bd9Sstevel@tonic-gate }
14277c478bd9Sstevel@tonic-gate
14287c478bd9Sstevel@tonic-gate for (i = max_ncpus; i < nchunks; i++)
14297c478bd9Sstevel@tonic-gate lhp->lh_free[i] = i;
14307c478bd9Sstevel@tonic-gate
14317c478bd9Sstevel@tonic-gate lhp->lh_head = max_ncpus;
14327c478bd9Sstevel@tonic-gate lhp->lh_tail = 0;
14337c478bd9Sstevel@tonic-gate
14347c478bd9Sstevel@tonic-gate return (lhp);
14357c478bd9Sstevel@tonic-gate }
14367c478bd9Sstevel@tonic-gate
14377c478bd9Sstevel@tonic-gate static void *
kmem_log_enter(kmem_log_header_t * lhp,void * data,size_t size)14387c478bd9Sstevel@tonic-gate kmem_log_enter(kmem_log_header_t *lhp, void *data, size_t size)
14397c478bd9Sstevel@tonic-gate {
14407c478bd9Sstevel@tonic-gate void *logspace;
1441066570e7SJohn Levon kmem_cpu_log_header_t *clhp;
14427c478bd9Sstevel@tonic-gate
14437c478bd9Sstevel@tonic-gate if (lhp == NULL || kmem_logging == 0 || panicstr)
14447c478bd9Sstevel@tonic-gate return (NULL);
14457c478bd9Sstevel@tonic-gate
1446066570e7SJohn Levon clhp = &lhp->lh_cpu[CPU->cpu_seqid];
1447066570e7SJohn Levon
14487c478bd9Sstevel@tonic-gate mutex_enter(&clhp->clh_lock);
14497c478bd9Sstevel@tonic-gate clhp->clh_hits++;
14507c478bd9Sstevel@tonic-gate if (size > clhp->clh_avail) {
14517c478bd9Sstevel@tonic-gate mutex_enter(&lhp->lh_lock);
14527c478bd9Sstevel@tonic-gate lhp->lh_hits++;
14537c478bd9Sstevel@tonic-gate lhp->lh_free[lhp->lh_tail] = clhp->clh_chunk;
14547c478bd9Sstevel@tonic-gate lhp->lh_tail = (lhp->lh_tail + 1) % lhp->lh_nchunks;
14557c478bd9Sstevel@tonic-gate clhp->clh_chunk = lhp->lh_free[lhp->lh_head];
14567c478bd9Sstevel@tonic-gate lhp->lh_head = (lhp->lh_head + 1) % lhp->lh_nchunks;
14577c478bd9Sstevel@tonic-gate clhp->clh_current = lhp->lh_base +
14589f1b636aStomee clhp->clh_chunk * lhp->lh_chunksize;
14597c478bd9Sstevel@tonic-gate clhp->clh_avail = lhp->lh_chunksize;
14607c478bd9Sstevel@tonic-gate if (size > lhp->lh_chunksize)
14617c478bd9Sstevel@tonic-gate size = lhp->lh_chunksize;
14627c478bd9Sstevel@tonic-gate mutex_exit(&lhp->lh_lock);
14637c478bd9Sstevel@tonic-gate }
14647c478bd9Sstevel@tonic-gate logspace = clhp->clh_current;
14657c478bd9Sstevel@tonic-gate clhp->clh_current += size;
14667c478bd9Sstevel@tonic-gate clhp->clh_avail -= size;
14677c478bd9Sstevel@tonic-gate bcopy(data, logspace, size);
14687c478bd9Sstevel@tonic-gate mutex_exit(&clhp->clh_lock);
14697c478bd9Sstevel@tonic-gate return (logspace);
14707c478bd9Sstevel@tonic-gate }
14717c478bd9Sstevel@tonic-gate
14727c478bd9Sstevel@tonic-gate #define KMEM_AUDIT(lp, cp, bcp) \
14737c478bd9Sstevel@tonic-gate { \
14747c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *_bcp = (kmem_bufctl_audit_t *)(bcp); \
14757c478bd9Sstevel@tonic-gate _bcp->bc_timestamp = gethrtime(); \
14767c478bd9Sstevel@tonic-gate _bcp->bc_thread = curthread; \
14777c478bd9Sstevel@tonic-gate _bcp->bc_depth = getpcstack(_bcp->bc_stack, KMEM_STACK_DEPTH); \
14787c478bd9Sstevel@tonic-gate _bcp->bc_lastlog = kmem_log_enter((lp), _bcp, sizeof (*_bcp)); \
14797c478bd9Sstevel@tonic-gate }
14807c478bd9Sstevel@tonic-gate
14817c478bd9Sstevel@tonic-gate static void
kmem_log_event(kmem_log_header_t * lp,kmem_cache_t * cp,kmem_slab_t * sp,void * addr)14827c478bd9Sstevel@tonic-gate kmem_log_event(kmem_log_header_t *lp, kmem_cache_t *cp,
14831c207ae9SMatthew Ahrens kmem_slab_t *sp, void *addr)
14847c478bd9Sstevel@tonic-gate {
14857c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t bca;
14867c478bd9Sstevel@tonic-gate
14877c478bd9Sstevel@tonic-gate bzero(&bca, sizeof (kmem_bufctl_audit_t));
14887c478bd9Sstevel@tonic-gate bca.bc_addr = addr;
14897c478bd9Sstevel@tonic-gate bca.bc_slab = sp;
14907c478bd9Sstevel@tonic-gate bca.bc_cache = cp;
14917c478bd9Sstevel@tonic-gate KMEM_AUDIT(lp, cp, &bca);
14927c478bd9Sstevel@tonic-gate }
14937c478bd9Sstevel@tonic-gate
14947c478bd9Sstevel@tonic-gate /*
14957c478bd9Sstevel@tonic-gate * Create a new slab for cache cp.
14967c478bd9Sstevel@tonic-gate */
14977c478bd9Sstevel@tonic-gate static kmem_slab_t *
kmem_slab_create(kmem_cache_t * cp,int kmflag)14987c478bd9Sstevel@tonic-gate kmem_slab_create(kmem_cache_t *cp, int kmflag)
14997c478bd9Sstevel@tonic-gate {
15007c478bd9Sstevel@tonic-gate size_t slabsize = cp->cache_slabsize;
15017c478bd9Sstevel@tonic-gate size_t chunksize = cp->cache_chunksize;
15027c478bd9Sstevel@tonic-gate int cache_flags = cp->cache_flags;
15037c478bd9Sstevel@tonic-gate size_t color, chunks;
15047c478bd9Sstevel@tonic-gate char *buf, *slab;
15057c478bd9Sstevel@tonic-gate kmem_slab_t *sp;
15067c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp;
15077c478bd9Sstevel@tonic-gate vmem_t *vmp = cp->cache_arena;
15087c478bd9Sstevel@tonic-gate
1509b5fca8f8Stomee ASSERT(MUTEX_NOT_HELD(&cp->cache_lock));
1510b5fca8f8Stomee
15117c478bd9Sstevel@tonic-gate color = cp->cache_color + cp->cache_align;
15127c478bd9Sstevel@tonic-gate if (color > cp->cache_maxcolor)
15137c478bd9Sstevel@tonic-gate color = cp->cache_mincolor;
15147c478bd9Sstevel@tonic-gate cp->cache_color = color;
15157c478bd9Sstevel@tonic-gate
15167c478bd9Sstevel@tonic-gate slab = vmem_alloc(vmp, slabsize, kmflag & KM_VMFLAGS);
15177c478bd9Sstevel@tonic-gate
15187c478bd9Sstevel@tonic-gate if (slab == NULL)
15197c478bd9Sstevel@tonic-gate goto vmem_alloc_failure;
15207c478bd9Sstevel@tonic-gate
15217c478bd9Sstevel@tonic-gate ASSERT(P2PHASE((uintptr_t)slab, vmp->vm_quantum) == 0);
15227c478bd9Sstevel@tonic-gate
1523b5fca8f8Stomee /*
1524b5fca8f8Stomee * Reverify what was already checked in kmem_cache_set_move(), since the
1525b5fca8f8Stomee * consolidator depends (for correctness) on slabs being initialized
1526b5fca8f8Stomee * with the 0xbaddcafe memory pattern (setting a low order bit usable by
1527b5fca8f8Stomee * clients to distinguish uninitialized memory from known objects).
1528b5fca8f8Stomee */
1529b5fca8f8Stomee ASSERT((cp->cache_move == NULL) || !(cp->cache_cflags & KMC_NOTOUCH));
15307c478bd9Sstevel@tonic-gate if (!(cp->cache_cflags & KMC_NOTOUCH))
15317c478bd9Sstevel@tonic-gate copy_pattern(KMEM_UNINITIALIZED_PATTERN, slab, slabsize);
15327c478bd9Sstevel@tonic-gate
15337c478bd9Sstevel@tonic-gate if (cache_flags & KMF_HASH) {
15347c478bd9Sstevel@tonic-gate if ((sp = kmem_cache_alloc(kmem_slab_cache, kmflag)) == NULL)
15357c478bd9Sstevel@tonic-gate goto slab_alloc_failure;
15367c478bd9Sstevel@tonic-gate chunks = (slabsize - color) / chunksize;
15377c478bd9Sstevel@tonic-gate } else {
15387c478bd9Sstevel@tonic-gate sp = KMEM_SLAB(cp, slab);
15397c478bd9Sstevel@tonic-gate chunks = (slabsize - sizeof (kmem_slab_t) - color) / chunksize;
15407c478bd9Sstevel@tonic-gate }
15417c478bd9Sstevel@tonic-gate
15427c478bd9Sstevel@tonic-gate sp->slab_cache = cp;
15437c478bd9Sstevel@tonic-gate sp->slab_head = NULL;
15447c478bd9Sstevel@tonic-gate sp->slab_refcnt = 0;
15457c478bd9Sstevel@tonic-gate sp->slab_base = buf = slab + color;
15467c478bd9Sstevel@tonic-gate sp->slab_chunks = chunks;
1547b5fca8f8Stomee sp->slab_stuck_offset = (uint32_t)-1;
1548b5fca8f8Stomee sp->slab_later_count = 0;
1549b5fca8f8Stomee sp->slab_flags = 0;
15507c478bd9Sstevel@tonic-gate
15517c478bd9Sstevel@tonic-gate ASSERT(chunks > 0);
15527c478bd9Sstevel@tonic-gate while (chunks-- != 0) {
15537c478bd9Sstevel@tonic-gate if (cache_flags & KMF_HASH) {
15547c478bd9Sstevel@tonic-gate bcp = kmem_cache_alloc(cp->cache_bufctl_cache, kmflag);
15557c478bd9Sstevel@tonic-gate if (bcp == NULL)
15567c478bd9Sstevel@tonic-gate goto bufctl_alloc_failure;
15577c478bd9Sstevel@tonic-gate if (cache_flags & KMF_AUDIT) {
15587c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *bcap =
15597c478bd9Sstevel@tonic-gate (kmem_bufctl_audit_t *)bcp;
15607c478bd9Sstevel@tonic-gate bzero(bcap, sizeof (kmem_bufctl_audit_t));
15617c478bd9Sstevel@tonic-gate bcap->bc_cache = cp;
15627c478bd9Sstevel@tonic-gate }
15637c478bd9Sstevel@tonic-gate bcp->bc_addr = buf;
15647c478bd9Sstevel@tonic-gate bcp->bc_slab = sp;
15657c478bd9Sstevel@tonic-gate } else {
15667c478bd9Sstevel@tonic-gate bcp = KMEM_BUFCTL(cp, buf);
15677c478bd9Sstevel@tonic-gate }
15687c478bd9Sstevel@tonic-gate if (cache_flags & KMF_BUFTAG) {
15697c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf);
15707c478bd9Sstevel@tonic-gate btp->bt_redzone = KMEM_REDZONE_PATTERN;
15717c478bd9Sstevel@tonic-gate btp->bt_bufctl = bcp;
15727c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE;
15737c478bd9Sstevel@tonic-gate if (cache_flags & KMF_DEADBEEF) {
15747c478bd9Sstevel@tonic-gate copy_pattern(KMEM_FREE_PATTERN, buf,
15757c478bd9Sstevel@tonic-gate cp->cache_verify);
15767c478bd9Sstevel@tonic-gate }
15777c478bd9Sstevel@tonic-gate }
15787c478bd9Sstevel@tonic-gate bcp->bc_next = sp->slab_head;
15797c478bd9Sstevel@tonic-gate sp->slab_head = bcp;
15807c478bd9Sstevel@tonic-gate buf += chunksize;
15817c478bd9Sstevel@tonic-gate }
15827c478bd9Sstevel@tonic-gate
15837c478bd9Sstevel@tonic-gate kmem_log_event(kmem_slab_log, cp, sp, slab);
15847c478bd9Sstevel@tonic-gate
15857c478bd9Sstevel@tonic-gate return (sp);
15867c478bd9Sstevel@tonic-gate
15877c478bd9Sstevel@tonic-gate bufctl_alloc_failure:
15887c478bd9Sstevel@tonic-gate
15897c478bd9Sstevel@tonic-gate while ((bcp = sp->slab_head) != NULL) {
15907c478bd9Sstevel@tonic-gate sp->slab_head = bcp->bc_next;
15917c478bd9Sstevel@tonic-gate kmem_cache_free(cp->cache_bufctl_cache, bcp);
15927c478bd9Sstevel@tonic-gate }
15937c478bd9Sstevel@tonic-gate kmem_cache_free(kmem_slab_cache, sp);
15947c478bd9Sstevel@tonic-gate
15957c478bd9Sstevel@tonic-gate slab_alloc_failure:
15967c478bd9Sstevel@tonic-gate
15977c478bd9Sstevel@tonic-gate vmem_free(vmp, slab, slabsize);
15987c478bd9Sstevel@tonic-gate
15997c478bd9Sstevel@tonic-gate vmem_alloc_failure:
16007c478bd9Sstevel@tonic-gate
16017c478bd9Sstevel@tonic-gate kmem_log_event(kmem_failure_log, cp, NULL, NULL);
16021a5e258fSJosef 'Jeff' Sipek atomic_inc_64(&cp->cache_alloc_fail);
16037c478bd9Sstevel@tonic-gate
16047c478bd9Sstevel@tonic-gate return (NULL);
16057c478bd9Sstevel@tonic-gate }
16067c478bd9Sstevel@tonic-gate
16077c478bd9Sstevel@tonic-gate /*
16087c478bd9Sstevel@tonic-gate * Destroy a slab.
16097c478bd9Sstevel@tonic-gate */
16107c478bd9Sstevel@tonic-gate static void
kmem_slab_destroy(kmem_cache_t * cp,kmem_slab_t * sp)16117c478bd9Sstevel@tonic-gate kmem_slab_destroy(kmem_cache_t *cp, kmem_slab_t *sp)
16127c478bd9Sstevel@tonic-gate {
16137c478bd9Sstevel@tonic-gate vmem_t *vmp = cp->cache_arena;
16147c478bd9Sstevel@tonic-gate void *slab = (void *)P2ALIGN((uintptr_t)sp->slab_base, vmp->vm_quantum);
16157c478bd9Sstevel@tonic-gate
1616b5fca8f8Stomee ASSERT(MUTEX_NOT_HELD(&cp->cache_lock));
1617b5fca8f8Stomee ASSERT(sp->slab_refcnt == 0);
1618b5fca8f8Stomee
16197c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) {
16207c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp;
16217c478bd9Sstevel@tonic-gate while ((bcp = sp->slab_head) != NULL) {
16227c478bd9Sstevel@tonic-gate sp->slab_head = bcp->bc_next;
16237c478bd9Sstevel@tonic-gate kmem_cache_free(cp->cache_bufctl_cache, bcp);
16247c478bd9Sstevel@tonic-gate }
16257c478bd9Sstevel@tonic-gate kmem_cache_free(kmem_slab_cache, sp);
16267c478bd9Sstevel@tonic-gate }
16277c478bd9Sstevel@tonic-gate vmem_free(vmp, slab, cp->cache_slabsize);
16287c478bd9Sstevel@tonic-gate }
16297c478bd9Sstevel@tonic-gate
16307c478bd9Sstevel@tonic-gate static void *
kmem_slab_alloc_impl(kmem_cache_t * cp,kmem_slab_t * sp,boolean_t prefill)1631b942e89bSDavid Valin kmem_slab_alloc_impl(kmem_cache_t *cp, kmem_slab_t *sp, boolean_t prefill)
16327c478bd9Sstevel@tonic-gate {
16337c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp, **hash_bucket;
16347c478bd9Sstevel@tonic-gate void *buf;
1635b942e89bSDavid Valin boolean_t new_slab = (sp->slab_refcnt == 0);
16367c478bd9Sstevel@tonic-gate
1637b5fca8f8Stomee ASSERT(MUTEX_HELD(&cp->cache_lock));
1638b5fca8f8Stomee /*
1639b5fca8f8Stomee * kmem_slab_alloc() drops cache_lock when it creates a new slab, so we
1640b5fca8f8Stomee * can't ASSERT(avl_is_empty(&cp->cache_partial_slabs)) here when the
1641b942e89bSDavid Valin * slab is newly created.
1642b5fca8f8Stomee */
1643b942e89bSDavid Valin ASSERT(new_slab || (KMEM_SLAB_IS_PARTIAL(sp) &&
1644b5fca8f8Stomee (sp == avl_first(&cp->cache_partial_slabs))));
16457c478bd9Sstevel@tonic-gate ASSERT(sp->slab_cache == cp);
16467c478bd9Sstevel@tonic-gate
1647b5fca8f8Stomee cp->cache_slab_alloc++;
16489f1b636aStomee cp->cache_bufslab--;
16497c478bd9Sstevel@tonic-gate sp->slab_refcnt++;
16507c478bd9Sstevel@tonic-gate
16517c478bd9Sstevel@tonic-gate bcp = sp->slab_head;
1652b942e89bSDavid Valin sp->slab_head = bcp->bc_next;
16537c478bd9Sstevel@tonic-gate
16547c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) {
16557c478bd9Sstevel@tonic-gate /*
16567c478bd9Sstevel@tonic-gate * Add buffer to allocated-address hash table.
16577c478bd9Sstevel@tonic-gate */
16587c478bd9Sstevel@tonic-gate buf = bcp->bc_addr;
16597c478bd9Sstevel@tonic-gate hash_bucket = KMEM_HASH(cp, buf);
16607c478bd9Sstevel@tonic-gate bcp->bc_next = *hash_bucket;
16617c478bd9Sstevel@tonic-gate *hash_bucket = bcp;
16627c478bd9Sstevel@tonic-gate if ((cp->cache_flags & (KMF_AUDIT | KMF_BUFTAG)) == KMF_AUDIT) {
16637c478bd9Sstevel@tonic-gate KMEM_AUDIT(kmem_transaction_log, cp, bcp);
16647c478bd9Sstevel@tonic-gate }
16657c478bd9Sstevel@tonic-gate } else {
16667c478bd9Sstevel@tonic-gate buf = KMEM_BUF(cp, bcp);
16677c478bd9Sstevel@tonic-gate }
16687c478bd9Sstevel@tonic-gate
16697c478bd9Sstevel@tonic-gate ASSERT(KMEM_SLAB_MEMBER(sp, buf));
1670b942e89bSDavid Valin
1671b942e89bSDavid Valin if (sp->slab_head == NULL) {
1672b942e89bSDavid Valin ASSERT(KMEM_SLAB_IS_ALL_USED(sp));
1673b942e89bSDavid Valin if (new_slab) {
1674b942e89bSDavid Valin ASSERT(sp->slab_chunks == 1);
1675b942e89bSDavid Valin } else {
1676b942e89bSDavid Valin ASSERT(sp->slab_chunks > 1); /* the slab was partial */
1677b942e89bSDavid Valin avl_remove(&cp->cache_partial_slabs, sp);
1678b942e89bSDavid Valin sp->slab_later_count = 0; /* clear history */
1679b942e89bSDavid Valin sp->slab_flags &= ~KMEM_SLAB_NOMOVE;
1680b942e89bSDavid Valin sp->slab_stuck_offset = (uint32_t)-1;
1681b942e89bSDavid Valin }
1682b942e89bSDavid Valin list_insert_head(&cp->cache_complete_slabs, sp);
1683b942e89bSDavid Valin cp->cache_complete_slab_count++;
1684b942e89bSDavid Valin return (buf);
1685b942e89bSDavid Valin }
1686b942e89bSDavid Valin
1687b942e89bSDavid Valin ASSERT(KMEM_SLAB_IS_PARTIAL(sp));
1688b942e89bSDavid Valin /*
1689b942e89bSDavid Valin * Peek to see if the magazine layer is enabled before
1690b942e89bSDavid Valin * we prefill. We're not holding the cpu cache lock,
1691b942e89bSDavid Valin * so the peek could be wrong, but there's no harm in it.
1692b942e89bSDavid Valin */
1693b942e89bSDavid Valin if (new_slab && prefill && (cp->cache_flags & KMF_PREFILL) &&
1694b942e89bSDavid Valin (KMEM_CPU_CACHE(cp)->cc_magsize != 0)) {
1695b942e89bSDavid Valin kmem_slab_prefill(cp, sp);
1696b942e89bSDavid Valin return (buf);
1697b942e89bSDavid Valin }
1698b942e89bSDavid Valin
1699b942e89bSDavid Valin if (new_slab) {
1700b942e89bSDavid Valin avl_add(&cp->cache_partial_slabs, sp);
1701b942e89bSDavid Valin return (buf);
1702b942e89bSDavid Valin }
1703b942e89bSDavid Valin
1704b942e89bSDavid Valin /*
1705b942e89bSDavid Valin * The slab is now more allocated than it was, so the
1706b942e89bSDavid Valin * order remains unchanged.
1707b942e89bSDavid Valin */
1708b942e89bSDavid Valin ASSERT(!avl_update(&cp->cache_partial_slabs, sp));
1709b5fca8f8Stomee return (buf);
1710b5fca8f8Stomee }
1711b5fca8f8Stomee
1712b5fca8f8Stomee /*
1713b5fca8f8Stomee * Allocate a raw (unconstructed) buffer from cp's slab layer.
1714b5fca8f8Stomee */
1715b5fca8f8Stomee static void *
kmem_slab_alloc(kmem_cache_t * cp,int kmflag)1716b5fca8f8Stomee kmem_slab_alloc(kmem_cache_t *cp, int kmflag)
1717b5fca8f8Stomee {
1718b5fca8f8Stomee kmem_slab_t *sp;
1719b5fca8f8Stomee void *buf;
17204d4c4c43STom Erickson boolean_t test_destructor;
1721b5fca8f8Stomee
1722b5fca8f8Stomee mutex_enter(&cp->cache_lock);
17234d4c4c43STom Erickson test_destructor = (cp->cache_slab_alloc == 0);
1724b5fca8f8Stomee sp = avl_first(&cp->cache_partial_slabs);
1725b5fca8f8Stomee if (sp == NULL) {
1726b5fca8f8Stomee ASSERT(cp->cache_bufslab == 0);
1727b5fca8f8Stomee
1728b5fca8f8Stomee /*
1729b5fca8f8Stomee * The freelist is empty. Create a new slab.
1730b5fca8f8Stomee */
1731b5fca8f8Stomee mutex_exit(&cp->cache_lock);
1732b5fca8f8Stomee if ((sp = kmem_slab_create(cp, kmflag)) == NULL) {
1733b5fca8f8Stomee return (NULL);
1734b5fca8f8Stomee }
1735b5fca8f8Stomee mutex_enter(&cp->cache_lock);
1736b5fca8f8Stomee cp->cache_slab_create++;
1737b5fca8f8Stomee if ((cp->cache_buftotal += sp->slab_chunks) > cp->cache_bufmax)
1738b5fca8f8Stomee cp->cache_bufmax = cp->cache_buftotal;
1739b5fca8f8Stomee cp->cache_bufslab += sp->slab_chunks;
1740b5fca8f8Stomee }
17417c478bd9Sstevel@tonic-gate
1742b942e89bSDavid Valin buf = kmem_slab_alloc_impl(cp, sp, B_TRUE);
1743b5fca8f8Stomee ASSERT((cp->cache_slab_create - cp->cache_slab_destroy) ==
1744b5fca8f8Stomee (cp->cache_complete_slab_count +
1745b5fca8f8Stomee avl_numnodes(&cp->cache_partial_slabs) +
1746b5fca8f8Stomee (cp->cache_defrag == NULL ? 0 : cp->cache_defrag->kmd_deadcount)));
17477c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock);
17487c478bd9Sstevel@tonic-gate
17494d4c4c43STom Erickson if (test_destructor && cp->cache_destructor != NULL) {
17504d4c4c43STom Erickson /*
17514d4c4c43STom Erickson * On the first kmem_slab_alloc(), assert that it is valid to
17524d4c4c43STom Erickson * call the destructor on a newly constructed object without any
17534d4c4c43STom Erickson * client involvement.
17544d4c4c43STom Erickson */
17554d4c4c43STom Erickson if ((cp->cache_constructor == NULL) ||
17564d4c4c43STom Erickson cp->cache_constructor(buf, cp->cache_private,
17574d4c4c43STom Erickson kmflag) == 0) {
17584d4c4c43STom Erickson cp->cache_destructor(buf, cp->cache_private);
17594d4c4c43STom Erickson }
17604d4c4c43STom Erickson copy_pattern(KMEM_UNINITIALIZED_PATTERN, buf,
17614d4c4c43STom Erickson cp->cache_bufsize);
17624d4c4c43STom Erickson if (cp->cache_flags & KMF_DEADBEEF) {
17634d4c4c43STom Erickson copy_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify);
17644d4c4c43STom Erickson }
17654d4c4c43STom Erickson }
17664d4c4c43STom Erickson
17677c478bd9Sstevel@tonic-gate return (buf);
17687c478bd9Sstevel@tonic-gate }
17697c478bd9Sstevel@tonic-gate
1770b5fca8f8Stomee static void kmem_slab_move_yes(kmem_cache_t *, kmem_slab_t *, void *);
1771b5fca8f8Stomee
17727c478bd9Sstevel@tonic-gate /*
17737c478bd9Sstevel@tonic-gate * Free a raw (unconstructed) buffer to cp's slab layer.
17747c478bd9Sstevel@tonic-gate */
17757c478bd9Sstevel@tonic-gate static void
kmem_slab_free(kmem_cache_t * cp,void * buf)17767c478bd9Sstevel@tonic-gate kmem_slab_free(kmem_cache_t *cp, void *buf)
17777c478bd9Sstevel@tonic-gate {
17787c478bd9Sstevel@tonic-gate kmem_slab_t *sp;
17797c478bd9Sstevel@tonic-gate kmem_bufctl_t *bcp, **prev_bcpp;
17807c478bd9Sstevel@tonic-gate
17817c478bd9Sstevel@tonic-gate ASSERT(buf != NULL);
17827c478bd9Sstevel@tonic-gate
17837c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock);
17847c478bd9Sstevel@tonic-gate cp->cache_slab_free++;
17857c478bd9Sstevel@tonic-gate
17867c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) {
17877c478bd9Sstevel@tonic-gate /*
17887c478bd9Sstevel@tonic-gate * Look up buffer in allocated-address hash table.
17897c478bd9Sstevel@tonic-gate */
17907c478bd9Sstevel@tonic-gate prev_bcpp = KMEM_HASH(cp, buf);
17917c478bd9Sstevel@tonic-gate while ((bcp = *prev_bcpp) != NULL) {
17927c478bd9Sstevel@tonic-gate if (bcp->bc_addr == buf) {
17937c478bd9Sstevel@tonic-gate *prev_bcpp = bcp->bc_next;
17947c478bd9Sstevel@tonic-gate sp = bcp->bc_slab;
17957c478bd9Sstevel@tonic-gate break;
17967c478bd9Sstevel@tonic-gate }
17977c478bd9Sstevel@tonic-gate cp->cache_lookup_depth++;
17987c478bd9Sstevel@tonic-gate prev_bcpp = &bcp->bc_next;
17997c478bd9Sstevel@tonic-gate }
18007c478bd9Sstevel@tonic-gate } else {
18017c478bd9Sstevel@tonic-gate bcp = KMEM_BUFCTL(cp, buf);
18027c478bd9Sstevel@tonic-gate sp = KMEM_SLAB(cp, buf);
18037c478bd9Sstevel@tonic-gate }
18047c478bd9Sstevel@tonic-gate
18057c478bd9Sstevel@tonic-gate if (bcp == NULL || sp->slab_cache != cp || !KMEM_SLAB_MEMBER(sp, buf)) {
18067c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock);
18077c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADADDR, cp, buf);
18087c478bd9Sstevel@tonic-gate return;
18097c478bd9Sstevel@tonic-gate }
18107c478bd9Sstevel@tonic-gate
1811b5fca8f8Stomee if (KMEM_SLAB_OFFSET(sp, buf) == sp->slab_stuck_offset) {
1812b5fca8f8Stomee /*
1813b5fca8f8Stomee * If this is the buffer that prevented the consolidator from
1814b5fca8f8Stomee * clearing the slab, we can reset the slab flags now that the
1815b5fca8f8Stomee * buffer is freed. (It makes sense to do this in
1816b5fca8f8Stomee * kmem_cache_free(), where the client gives up ownership of the
1817b5fca8f8Stomee * buffer, but on the hot path the test is too expensive.)
1818b5fca8f8Stomee */
1819b5fca8f8Stomee kmem_slab_move_yes(cp, sp, buf);
1820b5fca8f8Stomee }
1821b5fca8f8Stomee
18227c478bd9Sstevel@tonic-gate if ((cp->cache_flags & (KMF_AUDIT | KMF_BUFTAG)) == KMF_AUDIT) {
18237c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_CONTENTS)
18247c478bd9Sstevel@tonic-gate ((kmem_bufctl_audit_t *)bcp)->bc_contents =
18257c478bd9Sstevel@tonic-gate kmem_log_enter(kmem_content_log, buf,
18269f1b636aStomee cp->cache_contents);
18277c478bd9Sstevel@tonic-gate KMEM_AUDIT(kmem_transaction_log, cp, bcp);
18287c478bd9Sstevel@tonic-gate }
18297c478bd9Sstevel@tonic-gate
18307c478bd9Sstevel@tonic-gate bcp->bc_next = sp->slab_head;
18317c478bd9Sstevel@tonic-gate sp->slab_head = bcp;
18327c478bd9Sstevel@tonic-gate
18339f1b636aStomee cp->cache_bufslab++;
18347c478bd9Sstevel@tonic-gate ASSERT(sp->slab_refcnt >= 1);
1835b5fca8f8Stomee
18367c478bd9Sstevel@tonic-gate if (--sp->slab_refcnt == 0) {
18377c478bd9Sstevel@tonic-gate /*
18387c478bd9Sstevel@tonic-gate * There are no outstanding allocations from this slab,
18397c478bd9Sstevel@tonic-gate * so we can reclaim the memory.
18407c478bd9Sstevel@tonic-gate */
1841b5fca8f8Stomee if (sp->slab_chunks == 1) {
1842b5fca8f8Stomee list_remove(&cp->cache_complete_slabs, sp);
1843b5fca8f8Stomee cp->cache_complete_slab_count--;
1844b5fca8f8Stomee } else {
1845b5fca8f8Stomee avl_remove(&cp->cache_partial_slabs, sp);
1846b5fca8f8Stomee }
1847b5fca8f8Stomee
18487c478bd9Sstevel@tonic-gate cp->cache_buftotal -= sp->slab_chunks;
18499f1b636aStomee cp->cache_bufslab -= sp->slab_chunks;
1850b5fca8f8Stomee /*
1851b5fca8f8Stomee * Defer releasing the slab to the virtual memory subsystem
1852b5fca8f8Stomee * while there is a pending move callback, since we guarantee
1853b5fca8f8Stomee * that buffers passed to the move callback have only been
1854b5fca8f8Stomee * touched by kmem or by the client itself. Since the memory
1855b5fca8f8Stomee * patterns baddcafe (uninitialized) and deadbeef (freed) both
1856b5fca8f8Stomee * set at least one of the two lowest order bits, the client can
1857b5fca8f8Stomee * test those bits in the move callback to determine whether or
1858b5fca8f8Stomee * not it knows about the buffer (assuming that the client also
1859b5fca8f8Stomee * sets one of those low order bits whenever it frees a buffer).
1860b5fca8f8Stomee */
1861b5fca8f8Stomee if (cp->cache_defrag == NULL ||
1862b5fca8f8Stomee (avl_is_empty(&cp->cache_defrag->kmd_moves_pending) &&
1863b5fca8f8Stomee !(sp->slab_flags & KMEM_SLAB_MOVE_PENDING))) {
1864b5fca8f8Stomee cp->cache_slab_destroy++;
1865b5fca8f8Stomee mutex_exit(&cp->cache_lock);
1866b5fca8f8Stomee kmem_slab_destroy(cp, sp);
1867b5fca8f8Stomee } else {
1868b5fca8f8Stomee list_t *deadlist = &cp->cache_defrag->kmd_deadlist;
1869b5fca8f8Stomee /*
1870b5fca8f8Stomee * Slabs are inserted at both ends of the deadlist to
1871b5fca8f8Stomee * distinguish between slabs freed while move callbacks
1872b5fca8f8Stomee * are pending (list head) and a slab freed while the
1873b5fca8f8Stomee * lock is dropped in kmem_move_buffers() (list tail) so
1874b5fca8f8Stomee * that in both cases slab_destroy() is called from the
1875b5fca8f8Stomee * right context.
1876b5fca8f8Stomee */
1877b5fca8f8Stomee if (sp->slab_flags & KMEM_SLAB_MOVE_PENDING) {
1878b5fca8f8Stomee list_insert_tail(deadlist, sp);
1879b5fca8f8Stomee } else {
1880b5fca8f8Stomee list_insert_head(deadlist, sp);
1881b5fca8f8Stomee }
1882b5fca8f8Stomee cp->cache_defrag->kmd_deadcount++;
1883b5fca8f8Stomee mutex_exit(&cp->cache_lock);
1884b5fca8f8Stomee }
18857c478bd9Sstevel@tonic-gate return;
18867c478bd9Sstevel@tonic-gate }
1887b5fca8f8Stomee
1888b5fca8f8Stomee if (bcp->bc_next == NULL) {
1889b5fca8f8Stomee /* Transition the slab from completely allocated to partial. */
1890b5fca8f8Stomee ASSERT(sp->slab_refcnt == (sp->slab_chunks - 1));
1891b5fca8f8Stomee ASSERT(sp->slab_chunks > 1);
1892b5fca8f8Stomee list_remove(&cp->cache_complete_slabs, sp);
1893b5fca8f8Stomee cp->cache_complete_slab_count--;
1894b5fca8f8Stomee avl_add(&cp->cache_partial_slabs, sp);
1895b5fca8f8Stomee } else {
1896b5fca8f8Stomee (void) avl_update_gt(&cp->cache_partial_slabs, sp);
1897b5fca8f8Stomee }
1898b5fca8f8Stomee
1899b5fca8f8Stomee ASSERT((cp->cache_slab_create - cp->cache_slab_destroy) ==
1900b5fca8f8Stomee (cp->cache_complete_slab_count +
1901b5fca8f8Stomee avl_numnodes(&cp->cache_partial_slabs) +
1902b5fca8f8Stomee (cp->cache_defrag == NULL ? 0 : cp->cache_defrag->kmd_deadcount)));
19037c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock);
19047c478bd9Sstevel@tonic-gate }
19057c478bd9Sstevel@tonic-gate
1906b5fca8f8Stomee /*
1907b5fca8f8Stomee * Return -1 if kmem_error, 1 if constructor fails, 0 if successful.
1908b5fca8f8Stomee */
19097c478bd9Sstevel@tonic-gate static int
kmem_cache_alloc_debug(kmem_cache_t * cp,void * buf,int kmflag,int construct,caddr_t caller)19107c478bd9Sstevel@tonic-gate kmem_cache_alloc_debug(kmem_cache_t *cp, void *buf, int kmflag, int construct,
19117c478bd9Sstevel@tonic-gate caddr_t caller)
19127c478bd9Sstevel@tonic-gate {
19137c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf);
19147c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *bcp = (kmem_bufctl_audit_t *)btp->bt_bufctl;
19157c478bd9Sstevel@tonic-gate uint32_t mtbf;
19167c478bd9Sstevel@tonic-gate
19177c478bd9Sstevel@tonic-gate if (btp->bt_bxstat != ((intptr_t)bcp ^ KMEM_BUFTAG_FREE)) {
19187c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADBUFTAG, cp, buf);
19197c478bd9Sstevel@tonic-gate return (-1);
19207c478bd9Sstevel@tonic-gate }
19217c478bd9Sstevel@tonic-gate
19227c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_ALLOC;
19237c478bd9Sstevel@tonic-gate
19247c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_HASH) && bcp->bc_addr != buf) {
19257c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADBUFCTL, cp, buf);
19267c478bd9Sstevel@tonic-gate return (-1);
19277c478bd9Sstevel@tonic-gate }
19287c478bd9Sstevel@tonic-gate
19297c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) {
19307c478bd9Sstevel@tonic-gate if (!construct && (cp->cache_flags & KMF_LITE)) {
19317c478bd9Sstevel@tonic-gate if (*(uint64_t *)buf != KMEM_FREE_PATTERN) {
19327c478bd9Sstevel@tonic-gate kmem_error(KMERR_MODIFIED, cp, buf);
19337c478bd9Sstevel@tonic-gate return (-1);
19347c478bd9Sstevel@tonic-gate }
19357c478bd9Sstevel@tonic-gate if (cp->cache_constructor != NULL)
19367c478bd9Sstevel@tonic-gate *(uint64_t *)buf = btp->bt_redzone;
19377c478bd9Sstevel@tonic-gate else
19387c478bd9Sstevel@tonic-gate *(uint64_t *)buf = KMEM_UNINITIALIZED_PATTERN;
19397c478bd9Sstevel@tonic-gate } else {
19407c478bd9Sstevel@tonic-gate construct = 1;
19417c478bd9Sstevel@tonic-gate if (verify_and_copy_pattern(KMEM_FREE_PATTERN,
19427c478bd9Sstevel@tonic-gate KMEM_UNINITIALIZED_PATTERN, buf,
19437c478bd9Sstevel@tonic-gate cp->cache_verify)) {
19447c478bd9Sstevel@tonic-gate kmem_error(KMERR_MODIFIED, cp, buf);
19457c478bd9Sstevel@tonic-gate return (-1);
19467c478bd9Sstevel@tonic-gate }
19477c478bd9Sstevel@tonic-gate }
19487c478bd9Sstevel@tonic-gate }
19497c478bd9Sstevel@tonic-gate btp->bt_redzone = KMEM_REDZONE_PATTERN;
19507c478bd9Sstevel@tonic-gate
19517c478bd9Sstevel@tonic-gate if ((mtbf = kmem_mtbf | cp->cache_mtbf) != 0 &&
19527c478bd9Sstevel@tonic-gate gethrtime() % mtbf == 0 &&
19537c478bd9Sstevel@tonic-gate (kmflag & (KM_NOSLEEP | KM_PANIC)) == KM_NOSLEEP) {
19547c478bd9Sstevel@tonic-gate kmem_log_event(kmem_failure_log, cp, NULL, NULL);
19557c478bd9Sstevel@tonic-gate if (!construct && cp->cache_destructor != NULL)
19567c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private);
19577c478bd9Sstevel@tonic-gate } else {
19587c478bd9Sstevel@tonic-gate mtbf = 0;
19597c478bd9Sstevel@tonic-gate }
19607c478bd9Sstevel@tonic-gate
19617c478bd9Sstevel@tonic-gate if (mtbf || (construct && cp->cache_constructor != NULL &&
19627c478bd9Sstevel@tonic-gate cp->cache_constructor(buf, cp->cache_private, kmflag) != 0)) {
19631a5e258fSJosef 'Jeff' Sipek atomic_inc_64(&cp->cache_alloc_fail);
19647c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE;
19657c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF)
19667c478bd9Sstevel@tonic-gate copy_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify);
19677c478bd9Sstevel@tonic-gate kmem_slab_free(cp, buf);
1968b5fca8f8Stomee return (1);
19697c478bd9Sstevel@tonic-gate }
19707c478bd9Sstevel@tonic-gate
19717c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_AUDIT) {
19727c478bd9Sstevel@tonic-gate KMEM_AUDIT(kmem_transaction_log, cp, bcp);
19737c478bd9Sstevel@tonic-gate }
19747c478bd9Sstevel@tonic-gate
19757c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_LITE) &&
19767c478bd9Sstevel@tonic-gate !(cp->cache_cflags & KMC_KMEM_ALLOC)) {
19777c478bd9Sstevel@tonic-gate KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, caller);
19787c478bd9Sstevel@tonic-gate }
19797c478bd9Sstevel@tonic-gate
19807c478bd9Sstevel@tonic-gate return (0);
19817c478bd9Sstevel@tonic-gate }
19827c478bd9Sstevel@tonic-gate
19837c478bd9Sstevel@tonic-gate static int
kmem_cache_free_debug(kmem_cache_t * cp,void * buf,caddr_t caller)19847c478bd9Sstevel@tonic-gate kmem_cache_free_debug(kmem_cache_t *cp, void *buf, caddr_t caller)
19857c478bd9Sstevel@tonic-gate {
19867c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf);
19877c478bd9Sstevel@tonic-gate kmem_bufctl_audit_t *bcp = (kmem_bufctl_audit_t *)btp->bt_bufctl;
19887c478bd9Sstevel@tonic-gate kmem_slab_t *sp;
19897c478bd9Sstevel@tonic-gate
19907c478bd9Sstevel@tonic-gate if (btp->bt_bxstat != ((intptr_t)bcp ^ KMEM_BUFTAG_ALLOC)) {
19917c478bd9Sstevel@tonic-gate if (btp->bt_bxstat == ((intptr_t)bcp ^ KMEM_BUFTAG_FREE)) {
19927c478bd9Sstevel@tonic-gate kmem_error(KMERR_DUPFREE, cp, buf);
19937c478bd9Sstevel@tonic-gate return (-1);
19947c478bd9Sstevel@tonic-gate }
19957c478bd9Sstevel@tonic-gate sp = kmem_findslab(cp, buf);
19967c478bd9Sstevel@tonic-gate if (sp == NULL || sp->slab_cache != cp)
19977c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADADDR, cp, buf);
19987c478bd9Sstevel@tonic-gate else
19997c478bd9Sstevel@tonic-gate kmem_error(KMERR_REDZONE, cp, buf);
20007c478bd9Sstevel@tonic-gate return (-1);
20017c478bd9Sstevel@tonic-gate }
20027c478bd9Sstevel@tonic-gate
20037c478bd9Sstevel@tonic-gate btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE;
20047c478bd9Sstevel@tonic-gate
20057c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_HASH) && bcp->bc_addr != buf) {
20067c478bd9Sstevel@tonic-gate kmem_error(KMERR_BADBUFCTL, cp, buf);
20077c478bd9Sstevel@tonic-gate return (-1);
20087c478bd9Sstevel@tonic-gate }
20097c478bd9Sstevel@tonic-gate
20107c478bd9Sstevel@tonic-gate if (btp->bt_redzone != KMEM_REDZONE_PATTERN) {
20117c478bd9Sstevel@tonic-gate kmem_error(KMERR_REDZONE, cp, buf);
20127c478bd9Sstevel@tonic-gate return (-1);
20137c478bd9Sstevel@tonic-gate }
20147c478bd9Sstevel@tonic-gate
20157c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_AUDIT) {
20167c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_CONTENTS)
20177c478bd9Sstevel@tonic-gate bcp->bc_contents = kmem_log_enter(kmem_content_log,
20187c478bd9Sstevel@tonic-gate buf, cp->cache_contents);
20197c478bd9Sstevel@tonic-gate KMEM_AUDIT(kmem_transaction_log, cp, bcp);
20207c478bd9Sstevel@tonic-gate }
20217c478bd9Sstevel@tonic-gate
20227c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_LITE) &&
20237c478bd9Sstevel@tonic-gate !(cp->cache_cflags & KMC_KMEM_ALLOC)) {
20247c478bd9Sstevel@tonic-gate KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, caller);
20257c478bd9Sstevel@tonic-gate }
20267c478bd9Sstevel@tonic-gate
20277c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) {
20287c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE)
20297c478bd9Sstevel@tonic-gate btp->bt_redzone = *(uint64_t *)buf;
20307c478bd9Sstevel@tonic-gate else if (cp->cache_destructor != NULL)
20317c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private);
20327c478bd9Sstevel@tonic-gate
20337c478bd9Sstevel@tonic-gate copy_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify);
20347c478bd9Sstevel@tonic-gate }
20357c478bd9Sstevel@tonic-gate
20367c478bd9Sstevel@tonic-gate return (0);
20377c478bd9Sstevel@tonic-gate }
20387c478bd9Sstevel@tonic-gate
20397c478bd9Sstevel@tonic-gate /*
20407c478bd9Sstevel@tonic-gate * Free each object in magazine mp to cp's slab layer, and free mp itself.
20417c478bd9Sstevel@tonic-gate */
20427c478bd9Sstevel@tonic-gate static void
kmem_magazine_destroy(kmem_cache_t * cp,kmem_magazine_t * mp,int nrounds)20437c478bd9Sstevel@tonic-gate kmem_magazine_destroy(kmem_cache_t *cp, kmem_magazine_t *mp, int nrounds)
20447c478bd9Sstevel@tonic-gate {
20457c478bd9Sstevel@tonic-gate int round;
20467c478bd9Sstevel@tonic-gate
2047b5fca8f8Stomee ASSERT(!list_link_active(&cp->cache_link) ||
2048b5fca8f8Stomee taskq_member(kmem_taskq, curthread));
20497c478bd9Sstevel@tonic-gate
20507c478bd9Sstevel@tonic-gate for (round = 0; round < nrounds; round++) {
20517c478bd9Sstevel@tonic-gate void *buf = mp->mag_round[round];
20527c478bd9Sstevel@tonic-gate
20537c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) {
20547c478bd9Sstevel@tonic-gate if (verify_pattern(KMEM_FREE_PATTERN, buf,
20557c478bd9Sstevel@tonic-gate cp->cache_verify) != NULL) {
20567c478bd9Sstevel@tonic-gate kmem_error(KMERR_MODIFIED, cp, buf);
20577c478bd9Sstevel@tonic-gate continue;
20587c478bd9Sstevel@tonic-gate }
20597c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_LITE) &&
20607c478bd9Sstevel@tonic-gate cp->cache_destructor != NULL) {
20617c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf);
20627c478bd9Sstevel@tonic-gate *(uint64_t *)buf = btp->bt_redzone;
20637c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private);
20647c478bd9Sstevel@tonic-gate *(uint64_t *)buf = KMEM_FREE_PATTERN;
20657c478bd9Sstevel@tonic-gate }
20667c478bd9Sstevel@tonic-gate } else if (cp->cache_destructor != NULL) {
20677c478bd9Sstevel@tonic-gate cp->cache_destructor(buf, cp->cache_private);
20687c478bd9Sstevel@tonic-gate }
20697c478bd9Sstevel@tonic-gate
20707c478bd9Sstevel@tonic-gate kmem_slab_free(cp, buf);
20717c478bd9Sstevel@tonic-gate }
20727c478bd9Sstevel@tonic-gate ASSERT(KMEM_MAGAZINE_VALID(cp, mp));
20737c478bd9Sstevel@tonic-gate kmem_cache_free(cp->cache_magtype->mt_cache, mp);
20747c478bd9Sstevel@tonic-gate }
20757c478bd9Sstevel@tonic-gate
20767c478bd9Sstevel@tonic-gate /*
20777c478bd9Sstevel@tonic-gate * Allocate a magazine from the depot.
20787c478bd9Sstevel@tonic-gate */
20797c478bd9Sstevel@tonic-gate static kmem_magazine_t *
kmem_depot_alloc(kmem_cache_t * cp,kmem_maglist_t * mlp)20807c478bd9Sstevel@tonic-gate kmem_depot_alloc(kmem_cache_t *cp, kmem_maglist_t *mlp)
20817c478bd9Sstevel@tonic-gate {
20827c478bd9Sstevel@tonic-gate kmem_magazine_t *mp;
20837c478bd9Sstevel@tonic-gate
20847c478bd9Sstevel@tonic-gate /*
20857c478bd9Sstevel@tonic-gate * If we can't get the depot lock without contention,
20867c478bd9Sstevel@tonic-gate * update our contention count. We use the depot
20877c478bd9Sstevel@tonic-gate * contention rate to determine whether we need to
20887c478bd9Sstevel@tonic-gate * increase the magazine size for better scalability.
20897c478bd9Sstevel@tonic-gate */
20907c478bd9Sstevel@tonic-gate if (!mutex_tryenter(&cp->cache_depot_lock)) {
20917c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock);
20927c478bd9Sstevel@tonic-gate cp->cache_depot_contention++;
20937c478bd9Sstevel@tonic-gate }
20947c478bd9Sstevel@tonic-gate
20957c478bd9Sstevel@tonic-gate if ((mp = mlp->ml_list) != NULL) {
20967c478bd9Sstevel@tonic-gate ASSERT(KMEM_MAGAZINE_VALID(cp, mp));
20977c478bd9Sstevel@tonic-gate mlp->ml_list = mp->mag_next;
20987c478bd9Sstevel@tonic-gate if (--mlp->ml_total < mlp->ml_min)
20997c478bd9Sstevel@tonic-gate mlp->ml_min = mlp->ml_total;
21007c478bd9Sstevel@tonic-gate mlp->ml_alloc++;
21017c478bd9Sstevel@tonic-gate }
21027c478bd9Sstevel@tonic-gate
21037c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock);
21047c478bd9Sstevel@tonic-gate
21057c478bd9Sstevel@tonic-gate return (mp);
21067c478bd9Sstevel@tonic-gate }
21077c478bd9Sstevel@tonic-gate
21087c478bd9Sstevel@tonic-gate /*
21097c478bd9Sstevel@tonic-gate * Free a magazine to the depot.
21107c478bd9Sstevel@tonic-gate */
21117c478bd9Sstevel@tonic-gate static void
kmem_depot_free(kmem_cache_t * cp,kmem_maglist_t * mlp,kmem_magazine_t * mp)21127c478bd9Sstevel@tonic-gate kmem_depot_free(kmem_cache_t *cp, kmem_maglist_t *mlp, kmem_magazine_t *mp)
21137c478bd9Sstevel@tonic-gate {
21147c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock);
21157c478bd9Sstevel@tonic-gate ASSERT(KMEM_MAGAZINE_VALID(cp, mp));
21167c478bd9Sstevel@tonic-gate mp->mag_next = mlp->ml_list;
21177c478bd9Sstevel@tonic-gate mlp->ml_list = mp;
21187c478bd9Sstevel@tonic-gate mlp->ml_total++;
21197c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock);
21207c478bd9Sstevel@tonic-gate }
21217c478bd9Sstevel@tonic-gate
21227c478bd9Sstevel@tonic-gate /*
21237c478bd9Sstevel@tonic-gate * Update the working set statistics for cp's depot.
21247c478bd9Sstevel@tonic-gate */
21257c478bd9Sstevel@tonic-gate static void
kmem_depot_ws_update(kmem_cache_t * cp)21267c478bd9Sstevel@tonic-gate kmem_depot_ws_update(kmem_cache_t *cp)
21277c478bd9Sstevel@tonic-gate {
21287c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock);
21297c478bd9Sstevel@tonic-gate cp->cache_full.ml_reaplimit = cp->cache_full.ml_min;
21307c478bd9Sstevel@tonic-gate cp->cache_full.ml_min = cp->cache_full.ml_total;
21317c478bd9Sstevel@tonic-gate cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_min;
21327c478bd9Sstevel@tonic-gate cp->cache_empty.ml_min = cp->cache_empty.ml_total;
21337c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock);
21347c478bd9Sstevel@tonic-gate }
21357c478bd9Sstevel@tonic-gate
21360c833d64SJosef 'Jeff' Sipek /*
21370c833d64SJosef 'Jeff' Sipek * Set the working set statistics for cp's depot to zero. (Everything is
21380c833d64SJosef 'Jeff' Sipek * eligible for reaping.)
21390c833d64SJosef 'Jeff' Sipek */
21400c833d64SJosef 'Jeff' Sipek static void
kmem_depot_ws_zero(kmem_cache_t * cp)21410c833d64SJosef 'Jeff' Sipek kmem_depot_ws_zero(kmem_cache_t *cp)
21420c833d64SJosef 'Jeff' Sipek {
21430c833d64SJosef 'Jeff' Sipek mutex_enter(&cp->cache_depot_lock);
21440c833d64SJosef 'Jeff' Sipek cp->cache_full.ml_reaplimit = cp->cache_full.ml_total;
21450c833d64SJosef 'Jeff' Sipek cp->cache_full.ml_min = cp->cache_full.ml_total;
21460c833d64SJosef 'Jeff' Sipek cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_total;
21470c833d64SJosef 'Jeff' Sipek cp->cache_empty.ml_min = cp->cache_empty.ml_total;
21480c833d64SJosef 'Jeff' Sipek mutex_exit(&cp->cache_depot_lock);
21490c833d64SJosef 'Jeff' Sipek }
21500c833d64SJosef 'Jeff' Sipek
21511c207ae9SMatthew Ahrens /*
21521c207ae9SMatthew Ahrens * The number of bytes to reap before we call kpreempt(). The default (1MB)
21531c207ae9SMatthew Ahrens * causes us to preempt reaping up to hundreds of times per second. Using a
21541c207ae9SMatthew Ahrens * larger value (1GB) causes this to have virtually no effect.
21551c207ae9SMatthew Ahrens */
21561c207ae9SMatthew Ahrens size_t kmem_reap_preempt_bytes = 1024 * 1024;
21571c207ae9SMatthew Ahrens
21587c478bd9Sstevel@tonic-gate /*
21597c478bd9Sstevel@tonic-gate * Reap all magazines that have fallen out of the depot's working set.
21607c478bd9Sstevel@tonic-gate */
21617c478bd9Sstevel@tonic-gate static void
kmem_depot_ws_reap(kmem_cache_t * cp)21627c478bd9Sstevel@tonic-gate kmem_depot_ws_reap(kmem_cache_t *cp)
21637c478bd9Sstevel@tonic-gate {
21641c207ae9SMatthew Ahrens size_t bytes = 0;
21657c478bd9Sstevel@tonic-gate long reap;
21667c478bd9Sstevel@tonic-gate kmem_magazine_t *mp;
21677c478bd9Sstevel@tonic-gate
2168b5fca8f8Stomee ASSERT(!list_link_active(&cp->cache_link) ||
2169b5fca8f8Stomee taskq_member(kmem_taskq, curthread));
21707c478bd9Sstevel@tonic-gate
21717c478bd9Sstevel@tonic-gate reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min);
21721c207ae9SMatthew Ahrens while (reap-- &&
21731c207ae9SMatthew Ahrens (mp = kmem_depot_alloc(cp, &cp->cache_full)) != NULL) {
21747c478bd9Sstevel@tonic-gate kmem_magazine_destroy(cp, mp, cp->cache_magtype->mt_magsize);
21751c207ae9SMatthew Ahrens bytes += cp->cache_magtype->mt_magsize * cp->cache_bufsize;
21761c207ae9SMatthew Ahrens if (bytes > kmem_reap_preempt_bytes) {
21771c207ae9SMatthew Ahrens kpreempt(KPREEMPT_SYNC);
21781c207ae9SMatthew Ahrens bytes = 0;
21791c207ae9SMatthew Ahrens }
21801c207ae9SMatthew Ahrens }
21817c478bd9Sstevel@tonic-gate
21827c478bd9Sstevel@tonic-gate reap = MIN(cp->cache_empty.ml_reaplimit, cp->cache_empty.ml_min);
21831c207ae9SMatthew Ahrens while (reap-- &&
21841c207ae9SMatthew Ahrens (mp = kmem_depot_alloc(cp, &cp->cache_empty)) != NULL) {
21857c478bd9Sstevel@tonic-gate kmem_magazine_destroy(cp, mp, 0);
21861c207ae9SMatthew Ahrens bytes += cp->cache_magtype->mt_magsize * cp->cache_bufsize;
21871c207ae9SMatthew Ahrens if (bytes > kmem_reap_preempt_bytes) {
21881c207ae9SMatthew Ahrens kpreempt(KPREEMPT_SYNC);
21891c207ae9SMatthew Ahrens bytes = 0;
21901c207ae9SMatthew Ahrens }
21911c207ae9SMatthew Ahrens }
21927c478bd9Sstevel@tonic-gate }
21937c478bd9Sstevel@tonic-gate
21947c478bd9Sstevel@tonic-gate static void
kmem_cpu_reload(kmem_cpu_cache_t * ccp,kmem_magazine_t * mp,int rounds)21957c478bd9Sstevel@tonic-gate kmem_cpu_reload(kmem_cpu_cache_t *ccp, kmem_magazine_t *mp, int rounds)
21967c478bd9Sstevel@tonic-gate {
21977c478bd9Sstevel@tonic-gate ASSERT((ccp->cc_loaded == NULL && ccp->cc_rounds == -1) ||
21987c478bd9Sstevel@tonic-gate (ccp->cc_loaded && ccp->cc_rounds + rounds == ccp->cc_magsize));
21997c478bd9Sstevel@tonic-gate ASSERT(ccp->cc_magsize > 0);
22007c478bd9Sstevel@tonic-gate
22017c478bd9Sstevel@tonic-gate ccp->cc_ploaded = ccp->cc_loaded;
22027c478bd9Sstevel@tonic-gate ccp->cc_prounds = ccp->cc_rounds;
22037c478bd9Sstevel@tonic-gate ccp->cc_loaded = mp;
22047c478bd9Sstevel@tonic-gate ccp->cc_rounds = rounds;
22057c478bd9Sstevel@tonic-gate }
22067c478bd9Sstevel@tonic-gate
22079dd77bc8SDave Plauger /*
22089dd77bc8SDave Plauger * Intercept kmem alloc/free calls during crash dump in order to avoid
22099dd77bc8SDave Plauger * changing kmem state while memory is being saved to the dump device.
22109dd77bc8SDave Plauger * Otherwise, ::kmem_verify will report "corrupt buffers". Note that
22119dd77bc8SDave Plauger * there are no locks because only one CPU calls kmem during a crash
22129dd77bc8SDave Plauger * dump. To enable this feature, first create the associated vmem
22139dd77bc8SDave Plauger * arena with VMC_DUMPSAFE.
22149dd77bc8SDave Plauger */
22159dd77bc8SDave Plauger static void *kmem_dump_start; /* start of pre-reserved heap */
22169dd77bc8SDave Plauger static void *kmem_dump_end; /* end of heap area */
22179dd77bc8SDave Plauger static void *kmem_dump_curr; /* current free heap pointer */
22189dd77bc8SDave Plauger static size_t kmem_dump_size; /* size of heap area */
22199dd77bc8SDave Plauger
22209dd77bc8SDave Plauger /* append to each buf created in the pre-reserved heap */
22219dd77bc8SDave Plauger typedef struct kmem_dumpctl {
22229dd77bc8SDave Plauger void *kdc_next; /* cache dump free list linkage */
22239dd77bc8SDave Plauger } kmem_dumpctl_t;
22249dd77bc8SDave Plauger
22259dd77bc8SDave Plauger #define KMEM_DUMPCTL(cp, buf) \
22269dd77bc8SDave Plauger ((kmem_dumpctl_t *)P2ROUNDUP((uintptr_t)(buf) + (cp)->cache_bufsize, \
22279dd77bc8SDave Plauger sizeof (void *)))
22289dd77bc8SDave Plauger
22299dd77bc8SDave Plauger /* set non zero for full report */
22309dd77bc8SDave Plauger uint_t kmem_dump_verbose = 0;
22319dd77bc8SDave Plauger
22329dd77bc8SDave Plauger /* stats for overize heap */
22339dd77bc8SDave Plauger uint_t kmem_dump_oversize_allocs = 0;
22349dd77bc8SDave Plauger uint_t kmem_dump_oversize_max = 0;
22359dd77bc8SDave Plauger
22369dd77bc8SDave Plauger static void
kmem_dumppr(char ** pp,char * e,const char * format,...)22379dd77bc8SDave Plauger kmem_dumppr(char **pp, char *e, const char *format, ...)
22389dd77bc8SDave Plauger {
22399dd77bc8SDave Plauger char *p = *pp;
22409dd77bc8SDave Plauger
22419dd77bc8SDave Plauger if (p < e) {
22429dd77bc8SDave Plauger int n;
22439dd77bc8SDave Plauger va_list ap;
22449dd77bc8SDave Plauger
22459dd77bc8SDave Plauger va_start(ap, format);
22469dd77bc8SDave Plauger n = vsnprintf(p, e - p, format, ap);
22479dd77bc8SDave Plauger va_end(ap);
22489dd77bc8SDave Plauger *pp = p + n;
22499dd77bc8SDave Plauger }
22509dd77bc8SDave Plauger }
22519dd77bc8SDave Plauger
22529dd77bc8SDave Plauger /*
2253bbf21555SRichard Lowe * Called when dumpadm(8) configures dump parameters.
22549dd77bc8SDave Plauger */
22559dd77bc8SDave Plauger void
kmem_dump_init(size_t size)22569dd77bc8SDave Plauger kmem_dump_init(size_t size)
22579dd77bc8SDave Plauger {
2258dfec2ecfSJohn Levon /* Our caller ensures size is always set. */
2259dfec2ecfSJohn Levon ASSERT3U(size, >, 0);
2260dfec2ecfSJohn Levon
22619dd77bc8SDave Plauger if (kmem_dump_start != NULL)
22629dd77bc8SDave Plauger kmem_free(kmem_dump_start, kmem_dump_size);
22639dd77bc8SDave Plauger
22649dd77bc8SDave Plauger kmem_dump_start = kmem_alloc(size, KM_SLEEP);
2265dfec2ecfSJohn Levon kmem_dump_size = size;
2266dfec2ecfSJohn Levon kmem_dump_curr = kmem_dump_start;
2267dfec2ecfSJohn Levon kmem_dump_end = (void *)((char *)kmem_dump_start + size);
2268dfec2ecfSJohn Levon copy_pattern(KMEM_UNINITIALIZED_PATTERN, kmem_dump_start, size);
22699dd77bc8SDave Plauger }
22709dd77bc8SDave Plauger
22719dd77bc8SDave Plauger /*
22729dd77bc8SDave Plauger * Set flag for each kmem_cache_t if is safe to use alternate dump
22739dd77bc8SDave Plauger * memory. Called just before panic crash dump starts. Set the flag
22749dd77bc8SDave Plauger * for the calling CPU.
22759dd77bc8SDave Plauger */
22769dd77bc8SDave Plauger void
kmem_dump_begin(void)22779dd77bc8SDave Plauger kmem_dump_begin(void)
22789dd77bc8SDave Plauger {
2279dfec2ecfSJohn Levon kmem_cache_t *cp;
2280dfec2ecfSJohn Levon
22819dd77bc8SDave Plauger ASSERT(panicstr != NULL);
2282dfec2ecfSJohn Levon
2283dfec2ecfSJohn Levon for (cp = list_head(&kmem_caches); cp != NULL;
2284dfec2ecfSJohn Levon cp = list_next(&kmem_caches, cp)) {
2285dfec2ecfSJohn Levon kmem_cpu_cache_t *ccp = KMEM_CPU_CACHE(cp);
2286dfec2ecfSJohn Levon
2287dfec2ecfSJohn Levon if (cp->cache_arena->vm_cflags & VMC_DUMPSAFE) {
2288dfec2ecfSJohn Levon cp->cache_flags |= KMF_DUMPDIVERT;
2289dfec2ecfSJohn Levon ccp->cc_flags |= KMF_DUMPDIVERT;
2290dfec2ecfSJohn Levon ccp->cc_dump_rounds = ccp->cc_rounds;
2291dfec2ecfSJohn Levon ccp->cc_dump_prounds = ccp->cc_prounds;
2292dfec2ecfSJohn Levon ccp->cc_rounds = ccp->cc_prounds = -1;
2293dfec2ecfSJohn Levon } else {
2294dfec2ecfSJohn Levon cp->cache_flags |= KMF_DUMPUNSAFE;
2295dfec2ecfSJohn Levon ccp->cc_flags |= KMF_DUMPUNSAFE;
22969dd77bc8SDave Plauger }
22979dd77bc8SDave Plauger }
22989dd77bc8SDave Plauger }
22999dd77bc8SDave Plauger
23009dd77bc8SDave Plauger /*
23019dd77bc8SDave Plauger * finished dump intercept
23029dd77bc8SDave Plauger * print any warnings on the console
23039dd77bc8SDave Plauger * return verbose information to dumpsys() in the given buffer
23049dd77bc8SDave Plauger */
23059dd77bc8SDave Plauger size_t
kmem_dump_finish(char * buf,size_t size)23069dd77bc8SDave Plauger kmem_dump_finish(char *buf, size_t size)
23079dd77bc8SDave Plauger {
23089dd77bc8SDave Plauger int percent = 0;
23099dd77bc8SDave Plauger size_t used;
23109dd77bc8SDave Plauger char *e = buf + size;
23119dd77bc8SDave Plauger char *p = buf;
23129dd77bc8SDave Plauger
2313dfec2ecfSJohn Levon if (kmem_dump_curr == kmem_dump_end) {
2314dfec2ecfSJohn Levon cmn_err(CE_WARN, "exceeded kmem_dump space of %lu "
2315dfec2ecfSJohn Levon "bytes: kmem state in dump may be inconsistent",
2316dfec2ecfSJohn Levon kmem_dump_size);
2317dfec2ecfSJohn Levon }
2318dfec2ecfSJohn Levon
2319dfec2ecfSJohn Levon if (kmem_dump_verbose == 0)
23209dd77bc8SDave Plauger return (0);
23219dd77bc8SDave Plauger
23229dd77bc8SDave Plauger used = (char *)kmem_dump_curr - (char *)kmem_dump_start;
23239dd77bc8SDave Plauger percent = (used * 100) / kmem_dump_size;
23249dd77bc8SDave Plauger
23259dd77bc8SDave Plauger kmem_dumppr(&p, e, "%% heap used,%d\n", percent);
23269dd77bc8SDave Plauger kmem_dumppr(&p, e, "used bytes,%ld\n", used);
23279dd77bc8SDave Plauger kmem_dumppr(&p, e, "heap size,%ld\n", kmem_dump_size);
23289dd77bc8SDave Plauger kmem_dumppr(&p, e, "Oversize allocs,%d\n",
23299dd77bc8SDave Plauger kmem_dump_oversize_allocs);
23309dd77bc8SDave Plauger kmem_dumppr(&p, e, "Oversize max size,%ld\n",
23319dd77bc8SDave Plauger kmem_dump_oversize_max);
23329dd77bc8SDave Plauger
23339dd77bc8SDave Plauger /* return buffer size used */
23349dd77bc8SDave Plauger if (p < e)
23359dd77bc8SDave Plauger bzero(p, e - p);
23369dd77bc8SDave Plauger return (p - buf);
23379dd77bc8SDave Plauger }
23389dd77bc8SDave Plauger
23399dd77bc8SDave Plauger /*
23409dd77bc8SDave Plauger * Allocate a constructed object from alternate dump memory.
23419dd77bc8SDave Plauger */
23429dd77bc8SDave Plauger void *
kmem_cache_alloc_dump(kmem_cache_t * cp,int kmflag)23439dd77bc8SDave Plauger kmem_cache_alloc_dump(kmem_cache_t *cp, int kmflag)
23449dd77bc8SDave Plauger {
23459dd77bc8SDave Plauger void *buf;
23469dd77bc8SDave Plauger void *curr;
23479dd77bc8SDave Plauger char *bufend;
23489dd77bc8SDave Plauger
23499dd77bc8SDave Plauger /* return a constructed object */
2350dfec2ecfSJohn Levon if ((buf = cp->cache_dump.kd_freelist) != NULL) {
2351dfec2ecfSJohn Levon cp->cache_dump.kd_freelist = KMEM_DUMPCTL(cp, buf)->kdc_next;
23529dd77bc8SDave Plauger return (buf);
23539dd77bc8SDave Plauger }
23549dd77bc8SDave Plauger
23559dd77bc8SDave Plauger /* create a new constructed object */
23569dd77bc8SDave Plauger curr = kmem_dump_curr;
23579dd77bc8SDave Plauger buf = (void *)P2ROUNDUP((uintptr_t)curr, cp->cache_align);
23589dd77bc8SDave Plauger bufend = (char *)KMEM_DUMPCTL(cp, buf) + sizeof (kmem_dumpctl_t);
23599dd77bc8SDave Plauger
23609dd77bc8SDave Plauger /* hat layer objects cannot cross a page boundary */
23619dd77bc8SDave Plauger if (cp->cache_align < PAGESIZE) {
23629dd77bc8SDave Plauger char *page = (char *)P2ROUNDUP((uintptr_t)buf, PAGESIZE);
23639dd77bc8SDave Plauger if (bufend > page) {
23649dd77bc8SDave Plauger bufend += page - (char *)buf;
23659dd77bc8SDave Plauger buf = (void *)page;
23669dd77bc8SDave Plauger }
23679dd77bc8SDave Plauger }
23689dd77bc8SDave Plauger
23699dd77bc8SDave Plauger /* fall back to normal alloc if reserved area is used up */
23709dd77bc8SDave Plauger if (bufend > (char *)kmem_dump_end) {
23719dd77bc8SDave Plauger kmem_dump_curr = kmem_dump_end;
2372dfec2ecfSJohn Levon cp->cache_dump.kd_alloc_fails++;
23739dd77bc8SDave Plauger return (NULL);
23749dd77bc8SDave Plauger }
23759dd77bc8SDave Plauger
23769dd77bc8SDave Plauger /*
23779dd77bc8SDave Plauger * Must advance curr pointer before calling a constructor that
23789dd77bc8SDave Plauger * may also allocate memory.
23799dd77bc8SDave Plauger */
23809dd77bc8SDave Plauger kmem_dump_curr = bufend;
23819dd77bc8SDave Plauger
23829dd77bc8SDave Plauger /* run constructor */
23839dd77bc8SDave Plauger if (cp->cache_constructor != NULL &&
23849dd77bc8SDave Plauger cp->cache_constructor(buf, cp->cache_private, kmflag)
23859dd77bc8SDave Plauger != 0) {
23869dd77bc8SDave Plauger #ifdef DEBUG
23879dd77bc8SDave Plauger printf("name='%s' cache=0x%p: kmem cache constructor failed\n",
23889dd77bc8SDave Plauger cp->cache_name, (void *)cp);
23899dd77bc8SDave Plauger #endif
23909dd77bc8SDave Plauger /* reset curr pointer iff no allocs were done */
23919dd77bc8SDave Plauger if (kmem_dump_curr == bufend)
23929dd77bc8SDave Plauger kmem_dump_curr = curr;
23939dd77bc8SDave Plauger
2394dfec2ecfSJohn Levon cp->cache_dump.kd_alloc_fails++;
23959dd77bc8SDave Plauger /* fall back to normal alloc if the constructor fails */
23969dd77bc8SDave Plauger return (NULL);
23979dd77bc8SDave Plauger }
23989dd77bc8SDave Plauger
23999dd77bc8SDave Plauger return (buf);
24009dd77bc8SDave Plauger }
24019dd77bc8SDave Plauger
24029dd77bc8SDave Plauger /*
24039dd77bc8SDave Plauger * Free a constructed object in alternate dump memory.
24049dd77bc8SDave Plauger */
24059dd77bc8SDave Plauger int
kmem_cache_free_dump(kmem_cache_t * cp,void * buf)24069dd77bc8SDave Plauger kmem_cache_free_dump(kmem_cache_t *cp, void *buf)
24079dd77bc8SDave Plauger {
24089dd77bc8SDave Plauger /* save constructed buffers for next time */
24099dd77bc8SDave Plauger if ((char *)buf >= (char *)kmem_dump_start &&
24109dd77bc8SDave Plauger (char *)buf < (char *)kmem_dump_end) {
2411dfec2ecfSJohn Levon KMEM_DUMPCTL(cp, buf)->kdc_next = cp->cache_dump.kd_freelist;
2412dfec2ecfSJohn Levon cp->cache_dump.kd_freelist = buf;
24139dd77bc8SDave Plauger return (0);
24149dd77bc8SDave Plauger }
24159dd77bc8SDave Plauger
24169dd77bc8SDave Plauger /* just drop buffers that were allocated before dump started */
24179dd77bc8SDave Plauger if (kmem_dump_curr < kmem_dump_end)
24189dd77bc8SDave Plauger return (0);
24199dd77bc8SDave Plauger
24209dd77bc8SDave Plauger /* fall back to normal free if reserved area is used up */
24219dd77bc8SDave Plauger return (1);
24229dd77bc8SDave Plauger }
24239dd77bc8SDave Plauger
24247c478bd9Sstevel@tonic-gate /*
24257c478bd9Sstevel@tonic-gate * Allocate a constructed object from cache cp.
24267c478bd9Sstevel@tonic-gate */
24277c478bd9Sstevel@tonic-gate void *
kmem_cache_alloc(kmem_cache_t * cp,int kmflag)24287c478bd9Sstevel@tonic-gate kmem_cache_alloc(kmem_cache_t *cp, int kmflag)
24297c478bd9Sstevel@tonic-gate {
24307c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp = KMEM_CPU_CACHE(cp);
24317c478bd9Sstevel@tonic-gate kmem_magazine_t *fmp;
24327c478bd9Sstevel@tonic-gate void *buf;
24337c478bd9Sstevel@tonic-gate
24347c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock);
24357c478bd9Sstevel@tonic-gate for (;;) {
24367c478bd9Sstevel@tonic-gate /*
24377c478bd9Sstevel@tonic-gate * If there's an object available in the current CPU's
24387c478bd9Sstevel@tonic-gate * loaded magazine, just take it and return.
24397c478bd9Sstevel@tonic-gate */
24407c478bd9Sstevel@tonic-gate if (ccp->cc_rounds > 0) {
24417c478bd9Sstevel@tonic-gate buf = ccp->cc_loaded->mag_round[--ccp->cc_rounds];
24427c478bd9Sstevel@tonic-gate ccp->cc_alloc++;
24437c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock);
24449dd77bc8SDave Plauger if (ccp->cc_flags & (KMF_BUFTAG | KMF_DUMPUNSAFE)) {
24459dd77bc8SDave Plauger if (ccp->cc_flags & KMF_DUMPUNSAFE) {
24469dd77bc8SDave Plauger ASSERT(!(ccp->cc_flags &
24479dd77bc8SDave Plauger KMF_DUMPDIVERT));
2448dfec2ecfSJohn Levon cp->cache_dump.kd_unsafe++;
24499dd77bc8SDave Plauger }
24509dd77bc8SDave Plauger if ((ccp->cc_flags & KMF_BUFTAG) &&
24519dd77bc8SDave Plauger kmem_cache_alloc_debug(cp, buf, kmflag, 0,
24529dd77bc8SDave Plauger caller()) != 0) {
24539dd77bc8SDave Plauger if (kmflag & KM_NOSLEEP)
24549dd77bc8SDave Plauger return (NULL);
24559dd77bc8SDave Plauger mutex_enter(&ccp->cc_lock);
24569dd77bc8SDave Plauger continue;
24579dd77bc8SDave Plauger }
24587c478bd9Sstevel@tonic-gate }
24597c478bd9Sstevel@tonic-gate return (buf);
24607c478bd9Sstevel@tonic-gate }
24617c478bd9Sstevel@tonic-gate
24627c478bd9Sstevel@tonic-gate /*
24637c478bd9Sstevel@tonic-gate * The loaded magazine is empty. If the previously loaded
24647c478bd9Sstevel@tonic-gate * magazine was full, exchange them and try again.
24657c478bd9Sstevel@tonic-gate */
24667c478bd9Sstevel@tonic-gate if (ccp->cc_prounds > 0) {
24677c478bd9Sstevel@tonic-gate kmem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds);
24687c478bd9Sstevel@tonic-gate continue;
24697c478bd9Sstevel@tonic-gate }
24707c478bd9Sstevel@tonic-gate
24719dd77bc8SDave Plauger /*
24729dd77bc8SDave Plauger * Return an alternate buffer at dump time to preserve
24739dd77bc8SDave Plauger * the heap.
24749dd77bc8SDave Plauger */
24759dd77bc8SDave Plauger if (ccp->cc_flags & (KMF_DUMPDIVERT | KMF_DUMPUNSAFE)) {
24769dd77bc8SDave Plauger if (ccp->cc_flags & KMF_DUMPUNSAFE) {
24779dd77bc8SDave Plauger ASSERT(!(ccp->cc_flags & KMF_DUMPDIVERT));
24789dd77bc8SDave Plauger /* log it so that we can warn about it */
2479dfec2ecfSJohn Levon cp->cache_dump.kd_unsafe++;
24809dd77bc8SDave Plauger } else {
24819dd77bc8SDave Plauger if ((buf = kmem_cache_alloc_dump(cp, kmflag)) !=
24829dd77bc8SDave Plauger NULL) {
24839dd77bc8SDave Plauger mutex_exit(&ccp->cc_lock);
24849dd77bc8SDave Plauger return (buf);
24859dd77bc8SDave Plauger }
24869dd77bc8SDave Plauger break; /* fall back to slab layer */
24879dd77bc8SDave Plauger }
24889dd77bc8SDave Plauger }
24899dd77bc8SDave Plauger
24907c478bd9Sstevel@tonic-gate /*
24917c478bd9Sstevel@tonic-gate * If the magazine layer is disabled, break out now.
24927c478bd9Sstevel@tonic-gate */
24937c478bd9Sstevel@tonic-gate if (ccp->cc_magsize == 0)
24947c478bd9Sstevel@tonic-gate break;
24957c478bd9Sstevel@tonic-gate
24967c478bd9Sstevel@tonic-gate /*
24977c478bd9Sstevel@tonic-gate * Try to get a full magazine from the depot.
24987c478bd9Sstevel@tonic-gate */
24997c478bd9Sstevel@tonic-gate fmp = kmem_depot_alloc(cp, &cp->cache_full);
25007c478bd9Sstevel@tonic-gate if (fmp != NULL) {
25017c478bd9Sstevel@tonic-gate if (ccp->cc_ploaded != NULL)
25027c478bd9Sstevel@tonic-gate kmem_depot_free(cp, &cp->cache_empty,
25037c478bd9Sstevel@tonic-gate ccp->cc_ploaded);
25047c478bd9Sstevel@tonic-gate kmem_cpu_reload(ccp, fmp, ccp->cc_magsize);
25057c478bd9Sstevel@tonic-gate continue;
25067c478bd9Sstevel@tonic-gate }
25077c478bd9Sstevel@tonic-gate
25087c478bd9Sstevel@tonic-gate /*
25097c478bd9Sstevel@tonic-gate * There are no full magazines in the depot,
25107c478bd9Sstevel@tonic-gate * so fall through to the slab layer.
25117c478bd9Sstevel@tonic-gate */
25127c478bd9Sstevel@tonic-gate break;
25137c478bd9Sstevel@tonic-gate }
25147c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock);
25157c478bd9Sstevel@tonic-gate
25167c478bd9Sstevel@tonic-gate /*
25177c478bd9Sstevel@tonic-gate * We couldn't allocate a constructed object from the magazine layer,
25187c478bd9Sstevel@tonic-gate * so get a raw buffer from the slab layer and apply its constructor.
25197c478bd9Sstevel@tonic-gate */
25207c478bd9Sstevel@tonic-gate buf = kmem_slab_alloc(cp, kmflag);
25217c478bd9Sstevel@tonic-gate
25227c478bd9Sstevel@tonic-gate if (buf == NULL)
25237c478bd9Sstevel@tonic-gate return (NULL);
25247c478bd9Sstevel@tonic-gate
25257c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_BUFTAG) {
25267c478bd9Sstevel@tonic-gate /*
25277c478bd9Sstevel@tonic-gate * Make kmem_cache_alloc_debug() apply the constructor for us.
25287c478bd9Sstevel@tonic-gate */
2529b5fca8f8Stomee int rc = kmem_cache_alloc_debug(cp, buf, kmflag, 1, caller());
2530b5fca8f8Stomee if (rc != 0) {
25317c478bd9Sstevel@tonic-gate if (kmflag & KM_NOSLEEP)
25327c478bd9Sstevel@tonic-gate return (NULL);
25337c478bd9Sstevel@tonic-gate /*
25347c478bd9Sstevel@tonic-gate * kmem_cache_alloc_debug() detected corruption
2535b5fca8f8Stomee * but didn't panic (kmem_panic <= 0). We should not be
2536b5fca8f8Stomee * here because the constructor failed (indicated by a
2537b5fca8f8Stomee * return code of 1). Try again.
25387c478bd9Sstevel@tonic-gate */
2539b5fca8f8Stomee ASSERT(rc == -1);
25407c478bd9Sstevel@tonic-gate return (kmem_cache_alloc(cp, kmflag));
25417c478bd9Sstevel@tonic-gate }
25427c478bd9Sstevel@tonic-gate return (buf);
25437c478bd9Sstevel@tonic-gate }
25447c478bd9Sstevel@tonic-gate
25457c478bd9Sstevel@tonic-gate if (cp->cache_constructor != NULL &&
25467c478bd9Sstevel@tonic-gate cp->cache_constructor(buf, cp->cache_private, kmflag) != 0) {
25471a5e258fSJosef 'Jeff' Sipek atomic_inc_64(&cp->cache_alloc_fail);
25487c478bd9Sstevel@tonic-gate kmem_slab_free(cp, buf);
25497c478bd9Sstevel@tonic-gate return (NULL);
25507c478bd9Sstevel@tonic-gate }
25517c478bd9Sstevel@tonic-gate
25527c478bd9Sstevel@tonic-gate return (buf);
25537c478bd9Sstevel@tonic-gate }
25547c478bd9Sstevel@tonic-gate
25557c478bd9Sstevel@tonic-gate /*
2556b5fca8f8Stomee * The freed argument tells whether or not kmem_cache_free_debug() has already
2557b5fca8f8Stomee * been called so that we can avoid the duplicate free error. For example, a
2558b5fca8f8Stomee * buffer on a magazine has already been freed by the client but is still
2559b5fca8f8Stomee * constructed.
25607c478bd9Sstevel@tonic-gate */
2561b5fca8f8Stomee static void
kmem_slab_free_constructed(kmem_cache_t * cp,void * buf,boolean_t freed)2562b5fca8f8Stomee kmem_slab_free_constructed(kmem_cache_t *cp, void *buf, boolean_t freed)
25637c478bd9Sstevel@tonic-gate {
2564b5fca8f8Stomee if (!freed && (cp->cache_flags & KMF_BUFTAG))
25657c478bd9Sstevel@tonic-gate if (kmem_cache_free_debug(cp, buf, caller()) == -1)
25667c478bd9Sstevel@tonic-gate return;
25677c478bd9Sstevel@tonic-gate
2568b5fca8f8Stomee /*
2569b5fca8f8Stomee * Note that if KMF_DEADBEEF is in effect and KMF_LITE is not,
2570b5fca8f8Stomee * kmem_cache_free_debug() will have already applied the destructor.
2571b5fca8f8Stomee */
2572b5fca8f8Stomee if ((cp->cache_flags & (KMF_DEADBEEF | KMF_LITE)) != KMF_DEADBEEF &&
2573b5fca8f8Stomee cp->cache_destructor != NULL) {
2574b5fca8f8Stomee if (cp->cache_flags & KMF_DEADBEEF) { /* KMF_LITE implied */
2575b5fca8f8Stomee kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf);
2576b5fca8f8Stomee *(uint64_t *)buf = btp->bt_redzone;
2577b5fca8f8Stomee cp->cache_destructor(buf, cp->cache_private);
2578b5fca8f8Stomee *(uint64_t *)buf = KMEM_FREE_PATTERN;
2579b5fca8f8Stomee } else {
2580b5fca8f8Stomee cp->cache_destructor(buf, cp->cache_private);
2581b5fca8f8Stomee }
2582b5fca8f8Stomee }
2583b5fca8f8Stomee
2584b5fca8f8Stomee kmem_slab_free(cp, buf);
2585b5fca8f8Stomee }
2586b5fca8f8Stomee
2587b942e89bSDavid Valin /*
2588b942e89bSDavid Valin * Used when there's no room to free a buffer to the per-CPU cache.
2589b942e89bSDavid Valin * Drops and re-acquires &ccp->cc_lock, and returns non-zero if the
2590b942e89bSDavid Valin * caller should try freeing to the per-CPU cache again.
2591b942e89bSDavid Valin * Note that we don't directly install the magazine in the cpu cache,
2592b942e89bSDavid Valin * since its state may have changed wildly while the lock was dropped.
2593b942e89bSDavid Valin */
2594b942e89bSDavid Valin static int
kmem_cpucache_magazine_alloc(kmem_cpu_cache_t * ccp,kmem_cache_t * cp)2595b942e89bSDavid Valin kmem_cpucache_magazine_alloc(kmem_cpu_cache_t *ccp, kmem_cache_t *cp)
2596b942e89bSDavid Valin {
2597b942e89bSDavid Valin kmem_magazine_t *emp;
2598b942e89bSDavid Valin kmem_magtype_t *mtp;
2599b942e89bSDavid Valin
2600b942e89bSDavid Valin ASSERT(MUTEX_HELD(&ccp->cc_lock));
2601b942e89bSDavid Valin ASSERT(((uint_t)ccp->cc_rounds == ccp->cc_magsize ||
2602b942e89bSDavid Valin ((uint_t)ccp->cc_rounds == -1)) &&
2603b942e89bSDavid Valin ((uint_t)ccp->cc_prounds == ccp->cc_magsize ||
2604b942e89bSDavid Valin ((uint_t)ccp->cc_prounds == -1)));
2605b942e89bSDavid Valin
2606b942e89bSDavid Valin emp = kmem_depot_alloc(cp, &cp->cache_empty);
2607b942e89bSDavid Valin if (emp != NULL) {
2608b942e89bSDavid Valin if (ccp->cc_ploaded != NULL)
2609b942e89bSDavid Valin kmem_depot_free(cp, &cp->cache_full,
2610b942e89bSDavid Valin ccp->cc_ploaded);
2611b942e89bSDavid Valin kmem_cpu_reload(ccp, emp, 0);
2612b942e89bSDavid Valin return (1);
2613b942e89bSDavid Valin }
2614b942e89bSDavid Valin /*
2615b942e89bSDavid Valin * There are no empty magazines in the depot,
2616b942e89bSDavid Valin * so try to allocate a new one. We must drop all locks
2617b942e89bSDavid Valin * across kmem_cache_alloc() because lower layers may
2618b942e89bSDavid Valin * attempt to allocate from this cache.
2619b942e89bSDavid Valin */
2620b942e89bSDavid Valin mtp = cp->cache_magtype;
2621b942e89bSDavid Valin mutex_exit(&ccp->cc_lock);
2622b942e89bSDavid Valin emp = kmem_cache_alloc(mtp->mt_cache, KM_NOSLEEP);
2623b942e89bSDavid Valin mutex_enter(&ccp->cc_lock);
2624b942e89bSDavid Valin
2625b942e89bSDavid Valin if (emp != NULL) {
2626b942e89bSDavid Valin /*
2627b942e89bSDavid Valin * We successfully allocated an empty magazine.
2628b942e89bSDavid Valin * However, we had to drop ccp->cc_lock to do it,
2629b942e89bSDavid Valin * so the cache's magazine size may have changed.
2630b942e89bSDavid Valin * If so, free the magazine and try again.
2631b942e89bSDavid Valin */
2632b942e89bSDavid Valin if (ccp->cc_magsize != mtp->mt_magsize) {
2633b942e89bSDavid Valin mutex_exit(&ccp->cc_lock);
2634b942e89bSDavid Valin kmem_cache_free(mtp->mt_cache, emp);
2635b942e89bSDavid Valin mutex_enter(&ccp->cc_lock);
2636b942e89bSDavid Valin return (1);
2637b942e89bSDavid Valin }
2638b942e89bSDavid Valin
2639b942e89bSDavid Valin /*
2640b942e89bSDavid Valin * We got a magazine of the right size. Add it to
2641b942e89bSDavid Valin * the depot and try the whole dance again.
2642b942e89bSDavid Valin */
2643b942e89bSDavid Valin kmem_depot_free(cp, &cp->cache_empty, emp);
2644b942e89bSDavid Valin return (1);
2645b942e89bSDavid Valin }
2646b942e89bSDavid Valin
2647b942e89bSDavid Valin /*
2648b942e89bSDavid Valin * We couldn't allocate an empty magazine,
2649b942e89bSDavid Valin * so fall through to the slab layer.
2650b942e89bSDavid Valin */
2651b942e89bSDavid Valin return (0);
2652b942e89bSDavid Valin }
2653b942e89bSDavid Valin
2654b5fca8f8Stomee /*
2655b5fca8f8Stomee * Free a constructed object to cache cp.
2656b5fca8f8Stomee */
2657b5fca8f8Stomee void
kmem_cache_free(kmem_cache_t * cp,void * buf)2658b5fca8f8Stomee kmem_cache_free(kmem_cache_t *cp, void *buf)
2659b5fca8f8Stomee {
2660b5fca8f8Stomee kmem_cpu_cache_t *ccp = KMEM_CPU_CACHE(cp);
2661b5fca8f8Stomee
2662b5fca8f8Stomee /*
2663b5fca8f8Stomee * The client must not free either of the buffers passed to the move
2664b5fca8f8Stomee * callback function.
2665b5fca8f8Stomee */
2666b5fca8f8Stomee ASSERT(cp->cache_defrag == NULL ||
2667b5fca8f8Stomee cp->cache_defrag->kmd_thread != curthread ||
2668b5fca8f8Stomee (buf != cp->cache_defrag->kmd_from_buf &&
2669b5fca8f8Stomee buf != cp->cache_defrag->kmd_to_buf));
2670b5fca8f8Stomee
26719dd77bc8SDave Plauger if (ccp->cc_flags & (KMF_BUFTAG | KMF_DUMPDIVERT | KMF_DUMPUNSAFE)) {
26729dd77bc8SDave Plauger if (ccp->cc_flags & KMF_DUMPUNSAFE) {
26739dd77bc8SDave Plauger ASSERT(!(ccp->cc_flags & KMF_DUMPDIVERT));
26749dd77bc8SDave Plauger /* log it so that we can warn about it */
2675dfec2ecfSJohn Levon cp->cache_dump.kd_unsafe++;
26769dd77bc8SDave Plauger } else if (KMEM_DUMPCC(ccp) && !kmem_cache_free_dump(cp, buf)) {
2677b5fca8f8Stomee return;
26789dd77bc8SDave Plauger }
26799dd77bc8SDave Plauger if (ccp->cc_flags & KMF_BUFTAG) {
26809dd77bc8SDave Plauger if (kmem_cache_free_debug(cp, buf, caller()) == -1)
26819dd77bc8SDave Plauger return;
26829dd77bc8SDave Plauger }
26839dd77bc8SDave Plauger }
2684b5fca8f8Stomee
2685b5fca8f8Stomee mutex_enter(&ccp->cc_lock);
2686b942e89bSDavid Valin /*
2687b942e89bSDavid Valin * Any changes to this logic should be reflected in kmem_slab_prefill()
2688b942e89bSDavid Valin */
2689b5fca8f8Stomee for (;;) {
2690b5fca8f8Stomee /*
2691b5fca8f8Stomee * If there's a slot available in the current CPU's
2692b5fca8f8Stomee * loaded magazine, just put the object there and return.
2693b5fca8f8Stomee */
2694b5fca8f8Stomee if ((uint_t)ccp->cc_rounds < ccp->cc_magsize) {
2695b5fca8f8Stomee ccp->cc_loaded->mag_round[ccp->cc_rounds++] = buf;
2696b5fca8f8Stomee ccp->cc_free++;
2697b5fca8f8Stomee mutex_exit(&ccp->cc_lock);
2698b5fca8f8Stomee return;
2699b5fca8f8Stomee }
2700b5fca8f8Stomee
27017c478bd9Sstevel@tonic-gate /*
27027c478bd9Sstevel@tonic-gate * The loaded magazine is full. If the previously loaded
27037c478bd9Sstevel@tonic-gate * magazine was empty, exchange them and try again.
27047c478bd9Sstevel@tonic-gate */
27057c478bd9Sstevel@tonic-gate if (ccp->cc_prounds == 0) {
27067c478bd9Sstevel@tonic-gate kmem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds);
27077c478bd9Sstevel@tonic-gate continue;
27087c478bd9Sstevel@tonic-gate }
27097c478bd9Sstevel@tonic-gate
27107c478bd9Sstevel@tonic-gate /*
27117c478bd9Sstevel@tonic-gate * If the magazine layer is disabled, break out now.
27127c478bd9Sstevel@tonic-gate */
27137c478bd9Sstevel@tonic-gate if (ccp->cc_magsize == 0)
27147c478bd9Sstevel@tonic-gate break;
27157c478bd9Sstevel@tonic-gate
2716b942e89bSDavid Valin if (!kmem_cpucache_magazine_alloc(ccp, cp)) {
2717b942e89bSDavid Valin /*
2718b942e89bSDavid Valin * We couldn't free our constructed object to the
2719b942e89bSDavid Valin * magazine layer, so apply its destructor and free it
2720b942e89bSDavid Valin * to the slab layer.
2721b942e89bSDavid Valin */
2722b942e89bSDavid Valin break;
2723b942e89bSDavid Valin }
2724b942e89bSDavid Valin }
2725b942e89bSDavid Valin mutex_exit(&ccp->cc_lock);
2726b942e89bSDavid Valin kmem_slab_free_constructed(cp, buf, B_TRUE);
2727b942e89bSDavid Valin }
2728b942e89bSDavid Valin
2729b942e89bSDavid Valin static void
kmem_slab_prefill(kmem_cache_t * cp,kmem_slab_t * sp)2730b942e89bSDavid Valin kmem_slab_prefill(kmem_cache_t *cp, kmem_slab_t *sp)
2731b942e89bSDavid Valin {
2732b942e89bSDavid Valin kmem_cpu_cache_t *ccp = KMEM_CPU_CACHE(cp);
2733b942e89bSDavid Valin int cache_flags = cp->cache_flags;
2734b942e89bSDavid Valin
2735b942e89bSDavid Valin kmem_bufctl_t *next, *head;
2736b942e89bSDavid Valin size_t nbufs;
2737b942e89bSDavid Valin
2738b942e89bSDavid Valin /*
2739b942e89bSDavid Valin * Completely allocate the newly created slab and put the pre-allocated
2740b942e89bSDavid Valin * buffers in magazines. Any of the buffers that cannot be put in
2741b942e89bSDavid Valin * magazines must be returned to the slab.
2742b942e89bSDavid Valin */
2743b942e89bSDavid Valin ASSERT(MUTEX_HELD(&cp->cache_lock));
2744b942e89bSDavid Valin ASSERT((cache_flags & (KMF_PREFILL|KMF_BUFTAG)) == KMF_PREFILL);
2745b942e89bSDavid Valin ASSERT(cp->cache_constructor == NULL);
2746b942e89bSDavid Valin ASSERT(sp->slab_cache == cp);
2747b942e89bSDavid Valin ASSERT(sp->slab_refcnt == 1);
2748b942e89bSDavid Valin ASSERT(sp->slab_head != NULL && sp->slab_chunks > sp->slab_refcnt);
2749b942e89bSDavid Valin ASSERT(avl_find(&cp->cache_partial_slabs, sp, NULL) == NULL);
2750b942e89bSDavid Valin
2751b942e89bSDavid Valin head = sp->slab_head;
2752b942e89bSDavid Valin nbufs = (sp->slab_chunks - sp->slab_refcnt);
2753b942e89bSDavid Valin sp->slab_head = NULL;
2754b942e89bSDavid Valin sp->slab_refcnt += nbufs;
2755b942e89bSDavid Valin cp->cache_bufslab -= nbufs;
2756b942e89bSDavid Valin cp->cache_slab_alloc += nbufs;
2757b942e89bSDavid Valin list_insert_head(&cp->cache_complete_slabs, sp);
2758b942e89bSDavid Valin cp->cache_complete_slab_count++;
2759b942e89bSDavid Valin mutex_exit(&cp->cache_lock);
2760b942e89bSDavid Valin mutex_enter(&ccp->cc_lock);
2761b942e89bSDavid Valin
2762b942e89bSDavid Valin while (head != NULL) {
2763b942e89bSDavid Valin void *buf = KMEM_BUF(cp, head);
27647c478bd9Sstevel@tonic-gate /*
2765b942e89bSDavid Valin * If there's a slot available in the current CPU's
2766b942e89bSDavid Valin * loaded magazine, just put the object there and
2767b942e89bSDavid Valin * continue.
27687c478bd9Sstevel@tonic-gate */
2769b942e89bSDavid Valin if ((uint_t)ccp->cc_rounds < ccp->cc_magsize) {
2770b942e89bSDavid Valin ccp->cc_loaded->mag_round[ccp->cc_rounds++] =
2771b942e89bSDavid Valin buf;
2772b942e89bSDavid Valin ccp->cc_free++;
2773b942e89bSDavid Valin nbufs--;
2774b942e89bSDavid Valin head = head->bc_next;
27757c478bd9Sstevel@tonic-gate continue;
27767c478bd9Sstevel@tonic-gate }
27777c478bd9Sstevel@tonic-gate
27787c478bd9Sstevel@tonic-gate /*
2779b942e89bSDavid Valin * The loaded magazine is full. If the previously
2780b942e89bSDavid Valin * loaded magazine was empty, exchange them and try
2781b942e89bSDavid Valin * again.
27827c478bd9Sstevel@tonic-gate */
2783b942e89bSDavid Valin if (ccp->cc_prounds == 0) {
2784b942e89bSDavid Valin kmem_cpu_reload(ccp, ccp->cc_ploaded,
2785b942e89bSDavid Valin ccp->cc_prounds);
27867c478bd9Sstevel@tonic-gate continue;
27877c478bd9Sstevel@tonic-gate }
27887c478bd9Sstevel@tonic-gate
27897c478bd9Sstevel@tonic-gate /*
2790b942e89bSDavid Valin * If the magazine layer is disabled, break out now.
27917c478bd9Sstevel@tonic-gate */
2792b942e89bSDavid Valin
2793b942e89bSDavid Valin if (ccp->cc_magsize == 0) {
2794b942e89bSDavid Valin break;
2795b942e89bSDavid Valin }
2796b942e89bSDavid Valin
2797b942e89bSDavid Valin if (!kmem_cpucache_magazine_alloc(ccp, cp))
2798b942e89bSDavid Valin break;
27997c478bd9Sstevel@tonic-gate }
28007c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock);
2801b942e89bSDavid Valin if (nbufs != 0) {
2802b942e89bSDavid Valin ASSERT(head != NULL);
28037c478bd9Sstevel@tonic-gate
2804b942e89bSDavid Valin /*
2805b942e89bSDavid Valin * If there was a failure, return remaining objects to
2806b942e89bSDavid Valin * the slab
2807b942e89bSDavid Valin */
2808b942e89bSDavid Valin while (head != NULL) {
2809b942e89bSDavid Valin ASSERT(nbufs != 0);
2810b942e89bSDavid Valin next = head->bc_next;
2811b942e89bSDavid Valin head->bc_next = NULL;
2812b942e89bSDavid Valin kmem_slab_free(cp, KMEM_BUF(cp, head));
2813b942e89bSDavid Valin head = next;
2814b942e89bSDavid Valin nbufs--;
2815b942e89bSDavid Valin }
2816b942e89bSDavid Valin }
2817b942e89bSDavid Valin ASSERT(head == NULL);
2818b942e89bSDavid Valin ASSERT(nbufs == 0);
2819b942e89bSDavid Valin mutex_enter(&cp->cache_lock);
28207c478bd9Sstevel@tonic-gate }
28217c478bd9Sstevel@tonic-gate
2822*ed093b41SRobert Mustacchi /*
2823*ed093b41SRobert Mustacchi * kmem_rezalloc() is currently considered private and subject to change until
2824*ed093b41SRobert Mustacchi * we sort out how we want to handle realloc vs. reallocf style interfaces. We
2825*ed093b41SRobert Mustacchi * have currently chosen realloc.
2826*ed093b41SRobert Mustacchi */
2827*ed093b41SRobert Mustacchi void *
kmem_rezalloc(void * oldbuf,size_t oldsize,size_t newsize,int kmflag)2828*ed093b41SRobert Mustacchi kmem_rezalloc(void *oldbuf, size_t oldsize, size_t newsize, int kmflag)
2829*ed093b41SRobert Mustacchi {
2830*ed093b41SRobert Mustacchi void *newbuf = kmem_alloc(newsize, kmflag);
2831*ed093b41SRobert Mustacchi if (newbuf == NULL) {
2832*ed093b41SRobert Mustacchi return (NULL);
2833*ed093b41SRobert Mustacchi }
2834*ed093b41SRobert Mustacchi
2835*ed093b41SRobert Mustacchi bcopy(oldbuf, newbuf, MIN(oldsize, newsize));
2836*ed093b41SRobert Mustacchi if (newsize > oldsize) {
2837*ed093b41SRobert Mustacchi void *start = (void *)((uintptr_t)newbuf + oldsize);
2838*ed093b41SRobert Mustacchi bzero(start, newsize - oldsize);
2839*ed093b41SRobert Mustacchi }
2840*ed093b41SRobert Mustacchi
2841*ed093b41SRobert Mustacchi if (oldbuf != NULL) {
2842*ed093b41SRobert Mustacchi ASSERT3U(oldsize, !=, 0);
2843*ed093b41SRobert Mustacchi kmem_free(oldbuf, oldsize);
2844*ed093b41SRobert Mustacchi }
2845*ed093b41SRobert Mustacchi
2846*ed093b41SRobert Mustacchi return (newbuf);
2847*ed093b41SRobert Mustacchi }
2848*ed093b41SRobert Mustacchi
28497c478bd9Sstevel@tonic-gate void *
kmem_zalloc(size_t size,int kmflag)28507c478bd9Sstevel@tonic-gate kmem_zalloc(size_t size, int kmflag)
28517c478bd9Sstevel@tonic-gate {
2852dce01e3fSJonathan W Adams size_t index;
28537c478bd9Sstevel@tonic-gate void *buf;
28547c478bd9Sstevel@tonic-gate
2855dce01e3fSJonathan W Adams if ((index = ((size - 1) >> KMEM_ALIGN_SHIFT)) < KMEM_ALLOC_TABLE_MAX) {
28567c478bd9Sstevel@tonic-gate kmem_cache_t *cp = kmem_alloc_table[index];
28577c478bd9Sstevel@tonic-gate buf = kmem_cache_alloc(cp, kmflag);
28587c478bd9Sstevel@tonic-gate if (buf != NULL) {
28599dd77bc8SDave Plauger if ((cp->cache_flags & KMF_BUFTAG) && !KMEM_DUMP(cp)) {
28607c478bd9Sstevel@tonic-gate kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf);
28617c478bd9Sstevel@tonic-gate ((uint8_t *)buf)[size] = KMEM_REDZONE_BYTE;
28627c478bd9Sstevel@tonic-gate ((uint32_t *)btp)[1] = KMEM_SIZE_ENCODE(size);
28637c478bd9Sstevel@tonic-gate
28647c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) {
28657c478bd9Sstevel@tonic-gate KMEM_BUFTAG_LITE_ENTER(btp,
28667c478bd9Sstevel@tonic-gate kmem_lite_count, caller());
28677c478bd9Sstevel@tonic-gate }
28687c478bd9Sstevel@tonic-gate }
28697c478bd9Sstevel@tonic-gate bzero(buf, size);
28707c478bd9Sstevel@tonic-gate }
28717c478bd9Sstevel@tonic-gate } else {
28727c478bd9Sstevel@tonic-gate buf = kmem_alloc(size, kmflag);
28737c478bd9Sstevel@tonic-gate if (buf != NULL)
28747c478bd9Sstevel@tonic-gate bzero(buf, size);
28757c478bd9Sstevel@tonic-gate }
28767c478bd9Sstevel@tonic-gate return (buf);
28777c478bd9Sstevel@tonic-gate }
28787c478bd9Sstevel@tonic-gate
28797c478bd9Sstevel@tonic-gate void *
kmem_alloc(size_t size,int kmflag)28807c478bd9Sstevel@tonic-gate kmem_alloc(size_t size, int kmflag)
28817c478bd9Sstevel@tonic-gate {
2882dce01e3fSJonathan W Adams size_t index;
2883dce01e3fSJonathan W Adams kmem_cache_t *cp;
28847c478bd9Sstevel@tonic-gate void *buf;
28857c478bd9Sstevel@tonic-gate
2886dce01e3fSJonathan W Adams if ((index = ((size - 1) >> KMEM_ALIGN_SHIFT)) < KMEM_ALLOC_TABLE_MAX) {
2887dce01e3fSJonathan W Adams cp = kmem_alloc_table[index];
2888dce01e3fSJonathan W Adams /* fall through to kmem_cache_alloc() */
28897c478bd9Sstevel@tonic-gate
2890dce01e3fSJonathan W Adams } else if ((index = ((size - 1) >> KMEM_BIG_SHIFT)) <
2891dce01e3fSJonathan W Adams kmem_big_alloc_table_max) {
2892dce01e3fSJonathan W Adams cp = kmem_big_alloc_table[index];
2893dce01e3fSJonathan W Adams /* fall through to kmem_cache_alloc() */
2894dce01e3fSJonathan W Adams
2895dce01e3fSJonathan W Adams } else {
2896d1580181SBryan Cantrill if (size == 0) {
2897d1580181SBryan Cantrill if (kmflag != KM_SLEEP && !(kmflag & KM_PANIC))
2898d1580181SBryan Cantrill return (NULL);
2899d1580181SBryan Cantrill
2900d1580181SBryan Cantrill /*
2901d1580181SBryan Cantrill * If this is a sleeping allocation or one that has
2902d1580181SBryan Cantrill * been specified to panic on allocation failure, we
2903d1580181SBryan Cantrill * consider it to be deprecated behavior to allocate
2904d1580181SBryan Cantrill * 0 bytes. If we have been configured to panic under
2905d1580181SBryan Cantrill * this condition, we panic; if to warn, we warn -- and
2906d1580181SBryan Cantrill * regardless, we log to the kmem_zerosized_log that
2907d1580181SBryan Cantrill * that this condition has occurred (which gives us
2908d1580181SBryan Cantrill * enough information to be able to debug it).
2909d1580181SBryan Cantrill */
2910d1580181SBryan Cantrill if (kmem_panic && kmem_panic_zerosized)
2911d1580181SBryan Cantrill panic("attempted to kmem_alloc() size of 0");
2912d1580181SBryan Cantrill
2913d1580181SBryan Cantrill if (kmem_warn_zerosized) {
2914d1580181SBryan Cantrill cmn_err(CE_WARN, "kmem_alloc(): sleeping "
2915d1580181SBryan Cantrill "allocation with size of 0; "
2916d1580181SBryan Cantrill "see kmem_zerosized_log for details");
2917d1580181SBryan Cantrill }
2918d1580181SBryan Cantrill
2919d1580181SBryan Cantrill kmem_log_event(kmem_zerosized_log, NULL, NULL, NULL);
2920d1580181SBryan Cantrill
2921dce01e3fSJonathan W Adams return (NULL);
2922d1580181SBryan Cantrill }
2923dce01e3fSJonathan W Adams
2924dce01e3fSJonathan W Adams buf = vmem_alloc(kmem_oversize_arena, size,
2925dce01e3fSJonathan W Adams kmflag & KM_VMFLAGS);
2926dce01e3fSJonathan W Adams if (buf == NULL)
2927dce01e3fSJonathan W Adams kmem_log_event(kmem_failure_log, NULL, NULL,
2928dce01e3fSJonathan W Adams (void *)size);
29299dd77bc8SDave Plauger else if (KMEM_DUMP(kmem_slab_cache)) {
29309dd77bc8SDave Plauger /* stats for dump intercept */
29319dd77bc8SDave Plauger kmem_dump_oversize_allocs++;
29329dd77bc8SDave Plauger if (size > kmem_dump_oversize_max)
29339dd77bc8SDave Plauger kmem_dump_oversize_max = size;
29349dd77bc8SDave Plauger }
29357c478bd9Sstevel@tonic-gate return (buf);
29367c478bd9Sstevel@tonic-gate }
2937dce01e3fSJonathan W Adams
2938dce01e3fSJonathan W Adams buf = kmem_cache_alloc(cp, kmflag);
29399dd77bc8SDave Plauger if ((cp->cache_flags & KMF_BUFTAG) && !KMEM_DUMP(cp) && buf != NULL) {
2940dce01e3fSJonathan W Adams kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf);
2941dce01e3fSJonathan W Adams ((uint8_t *)buf)[size] = KMEM_REDZONE_BYTE;
2942dce01e3fSJonathan W Adams ((uint32_t *)btp)[1] = KMEM_SIZE_ENCODE(size);
2943dce01e3fSJonathan W Adams
2944dce01e3fSJonathan W Adams if (cp->cache_flags & KMF_LITE) {
2945dce01e3fSJonathan W Adams KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, caller());
2946dce01e3fSJonathan W Adams }
2947dce01e3fSJonathan W Adams }
29487c478bd9Sstevel@tonic-gate return (buf);
29497c478bd9Sstevel@tonic-gate }
29507c478bd9Sstevel@tonic-gate
29517c478bd9Sstevel@tonic-gate void
kmem_free(void * buf,size_t size)29527c478bd9Sstevel@tonic-gate kmem_free(void *buf, size_t size)
29537c478bd9Sstevel@tonic-gate {
2954dce01e3fSJonathan W Adams size_t index;
2955dce01e3fSJonathan W Adams kmem_cache_t *cp;
29567c478bd9Sstevel@tonic-gate
2957dce01e3fSJonathan W Adams if ((index = (size - 1) >> KMEM_ALIGN_SHIFT) < KMEM_ALLOC_TABLE_MAX) {
2958dce01e3fSJonathan W Adams cp = kmem_alloc_table[index];
2959dce01e3fSJonathan W Adams /* fall through to kmem_cache_free() */
2960dce01e3fSJonathan W Adams
2961dce01e3fSJonathan W Adams } else if ((index = ((size - 1) >> KMEM_BIG_SHIFT)) <
2962dce01e3fSJonathan W Adams kmem_big_alloc_table_max) {
2963dce01e3fSJonathan W Adams cp = kmem_big_alloc_table[index];
2964dce01e3fSJonathan W Adams /* fall through to kmem_cache_free() */
2965dce01e3fSJonathan W Adams
2966dce01e3fSJonathan W Adams } else {
296796992ee7SEthindra Ramamurthy EQUIV(buf == NULL, size == 0);
2968dce01e3fSJonathan W Adams if (buf == NULL && size == 0)
2969dce01e3fSJonathan W Adams return;
2970dce01e3fSJonathan W Adams vmem_free(kmem_oversize_arena, buf, size);
2971dce01e3fSJonathan W Adams return;
2972dce01e3fSJonathan W Adams }
2973dce01e3fSJonathan W Adams
29749dd77bc8SDave Plauger if ((cp->cache_flags & KMF_BUFTAG) && !KMEM_DUMP(cp)) {
2975dce01e3fSJonathan W Adams kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf);
2976dce01e3fSJonathan W Adams uint32_t *ip = (uint32_t *)btp;
2977dce01e3fSJonathan W Adams if (ip[1] != KMEM_SIZE_ENCODE(size)) {
2978dce01e3fSJonathan W Adams if (*(uint64_t *)buf == KMEM_FREE_PATTERN) {
2979dce01e3fSJonathan W Adams kmem_error(KMERR_DUPFREE, cp, buf);
29807c478bd9Sstevel@tonic-gate return;
29817c478bd9Sstevel@tonic-gate }
2982dce01e3fSJonathan W Adams if (KMEM_SIZE_VALID(ip[1])) {
2983dce01e3fSJonathan W Adams ip[0] = KMEM_SIZE_ENCODE(size);
2984dce01e3fSJonathan W Adams kmem_error(KMERR_BADSIZE, cp, buf);
2985dce01e3fSJonathan W Adams } else {
29867c478bd9Sstevel@tonic-gate kmem_error(KMERR_REDZONE, cp, buf);
29877c478bd9Sstevel@tonic-gate }
2988dce01e3fSJonathan W Adams return;
29897c478bd9Sstevel@tonic-gate }
2990dce01e3fSJonathan W Adams if (((uint8_t *)buf)[size] != KMEM_REDZONE_BYTE) {
2991dce01e3fSJonathan W Adams kmem_error(KMERR_REDZONE, cp, buf);
29927c478bd9Sstevel@tonic-gate return;
2993dce01e3fSJonathan W Adams }
2994dce01e3fSJonathan W Adams btp->bt_redzone = KMEM_REDZONE_PATTERN;
2995dce01e3fSJonathan W Adams if (cp->cache_flags & KMF_LITE) {
2996dce01e3fSJonathan W Adams KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count,
2997dce01e3fSJonathan W Adams caller());
2998dce01e3fSJonathan W Adams }
29997c478bd9Sstevel@tonic-gate }
3000dce01e3fSJonathan W Adams kmem_cache_free(cp, buf);
30017c478bd9Sstevel@tonic-gate }
30027c478bd9Sstevel@tonic-gate
30037c478bd9Sstevel@tonic-gate void *
kmem_firewall_va_alloc(vmem_t * vmp,size_t size,int vmflag)30047c478bd9Sstevel@tonic-gate kmem_firewall_va_alloc(vmem_t *vmp, size_t size, int vmflag)
30057c478bd9Sstevel@tonic-gate {
30067c478bd9Sstevel@tonic-gate size_t realsize = size + vmp->vm_quantum;
30077c478bd9Sstevel@tonic-gate void *addr;
30087c478bd9Sstevel@tonic-gate
30097c478bd9Sstevel@tonic-gate /*
30107c478bd9Sstevel@tonic-gate * Annoying edge case: if 'size' is just shy of ULONG_MAX, adding
30117c478bd9Sstevel@tonic-gate * vm_quantum will cause integer wraparound. Check for this, and
30127c478bd9Sstevel@tonic-gate * blow off the firewall page in this case. Note that such a
30137c478bd9Sstevel@tonic-gate * giant allocation (the entire kernel address space) can never
30147c478bd9Sstevel@tonic-gate * be satisfied, so it will either fail immediately (VM_NOSLEEP)
30157c478bd9Sstevel@tonic-gate * or sleep forever (VM_SLEEP). Thus, there is no need for a
30167c478bd9Sstevel@tonic-gate * corresponding check in kmem_firewall_va_free().
30177c478bd9Sstevel@tonic-gate */
30187c478bd9Sstevel@tonic-gate if (realsize < size)
30197c478bd9Sstevel@tonic-gate realsize = size;
30207c478bd9Sstevel@tonic-gate
30217c478bd9Sstevel@tonic-gate /*
30227c478bd9Sstevel@tonic-gate * While boot still owns resource management, make sure that this
30237c478bd9Sstevel@tonic-gate * redzone virtual address allocation is properly accounted for in
30247c478bd9Sstevel@tonic-gate * OBPs "virtual-memory" "available" lists because we're
30257c478bd9Sstevel@tonic-gate * effectively claiming them for a red zone. If we don't do this,
30267c478bd9Sstevel@tonic-gate * the available lists become too fragmented and too large for the
30277c478bd9Sstevel@tonic-gate * current boot/kernel memory list interface.
30287c478bd9Sstevel@tonic-gate */
30297c478bd9Sstevel@tonic-gate addr = vmem_alloc(vmp, realsize, vmflag | VM_NEXTFIT);
30307c478bd9Sstevel@tonic-gate
30317c478bd9Sstevel@tonic-gate if (addr != NULL && kvseg.s_base == NULL && realsize != size)
30327c478bd9Sstevel@tonic-gate (void) boot_virt_alloc((char *)addr + size, vmp->vm_quantum);
30337c478bd9Sstevel@tonic-gate
30347c478bd9Sstevel@tonic-gate return (addr);
30357c478bd9Sstevel@tonic-gate }
30367c478bd9Sstevel@tonic-gate
30377c478bd9Sstevel@tonic-gate void
kmem_firewall_va_free(vmem_t * vmp,void * addr,size_t size)30387c478bd9Sstevel@tonic-gate kmem_firewall_va_free(vmem_t *vmp, void *addr, size_t size)
30397c478bd9Sstevel@tonic-gate {
30407c478bd9Sstevel@tonic-gate ASSERT((kvseg.s_base == NULL ?
30417c478bd9Sstevel@tonic-gate va_to_pfn((char *)addr + size) :
30427c478bd9Sstevel@tonic-gate hat_getpfnum(kas.a_hat, (caddr_t)addr + size)) == PFN_INVALID);
30437c478bd9Sstevel@tonic-gate
30447c478bd9Sstevel@tonic-gate vmem_free(vmp, addr, size + vmp->vm_quantum);
30457c478bd9Sstevel@tonic-gate }
30467c478bd9Sstevel@tonic-gate
30477c478bd9Sstevel@tonic-gate /*
30487c478bd9Sstevel@tonic-gate * Try to allocate at least `size' bytes of memory without sleeping or
30497c478bd9Sstevel@tonic-gate * panicking. Return actual allocated size in `asize'. If allocation failed,
30507c478bd9Sstevel@tonic-gate * try final allocation with sleep or panic allowed.
30517c478bd9Sstevel@tonic-gate */
30527c478bd9Sstevel@tonic-gate void *
kmem_alloc_tryhard(size_t size,size_t * asize,int kmflag)30537c478bd9Sstevel@tonic-gate kmem_alloc_tryhard(size_t size, size_t *asize, int kmflag)
30547c478bd9Sstevel@tonic-gate {
30557c478bd9Sstevel@tonic-gate void *p;
30567c478bd9Sstevel@tonic-gate
30577c478bd9Sstevel@tonic-gate *asize = P2ROUNDUP(size, KMEM_ALIGN);
30587c478bd9Sstevel@tonic-gate do {
30597c478bd9Sstevel@tonic-gate p = kmem_alloc(*asize, (kmflag | KM_NOSLEEP) & ~KM_PANIC);
30607c478bd9Sstevel@tonic-gate if (p != NULL)
30617c478bd9Sstevel@tonic-gate return (p);
30627c478bd9Sstevel@tonic-gate *asize += KMEM_ALIGN;
30637c478bd9Sstevel@tonic-gate } while (*asize <= PAGESIZE);
30647c478bd9Sstevel@tonic-gate
30657c478bd9Sstevel@tonic-gate *asize = P2ROUNDUP(size, KMEM_ALIGN);
30667c478bd9Sstevel@tonic-gate return (kmem_alloc(*asize, kmflag));
30677c478bd9Sstevel@tonic-gate }
30687c478bd9Sstevel@tonic-gate
30697c478bd9Sstevel@tonic-gate /*
30707c478bd9Sstevel@tonic-gate * Reclaim all unused memory from a cache.
30717c478bd9Sstevel@tonic-gate */
30727c478bd9Sstevel@tonic-gate static void
kmem_cache_reap(kmem_cache_t * cp)30737c478bd9Sstevel@tonic-gate kmem_cache_reap(kmem_cache_t *cp)
30747c478bd9Sstevel@tonic-gate {
3075b5fca8f8Stomee ASSERT(taskq_member(kmem_taskq, curthread));
3076686031edSTom Erickson cp->cache_reap++;
3077b5fca8f8Stomee
30787c478bd9Sstevel@tonic-gate /*
30797c478bd9Sstevel@tonic-gate * Ask the cache's owner to free some memory if possible.
30807c478bd9Sstevel@tonic-gate * The idea is to handle things like the inode cache, which
30817c478bd9Sstevel@tonic-gate * typically sits on a bunch of memory that it doesn't truly
30827c478bd9Sstevel@tonic-gate * *need*. Reclaim policy is entirely up to the owner; this
30837c478bd9Sstevel@tonic-gate * callback is just an advisory plea for help.
30847c478bd9Sstevel@tonic-gate */
3085b5fca8f8Stomee if (cp->cache_reclaim != NULL) {
3086b5fca8f8Stomee long delta;
3087b5fca8f8Stomee
3088b5fca8f8Stomee /*
3089b5fca8f8Stomee * Reclaimed memory should be reapable (not included in the
3090b5fca8f8Stomee * depot's working set).
3091b5fca8f8Stomee */
3092b5fca8f8Stomee delta = cp->cache_full.ml_total;
30937c478bd9Sstevel@tonic-gate cp->cache_reclaim(cp->cache_private);
3094b5fca8f8Stomee delta = cp->cache_full.ml_total - delta;
3095b5fca8f8Stomee if (delta > 0) {
3096b5fca8f8Stomee mutex_enter(&cp->cache_depot_lock);
3097b5fca8f8Stomee cp->cache_full.ml_reaplimit += delta;
3098b5fca8f8Stomee cp->cache_full.ml_min += delta;
3099b5fca8f8Stomee mutex_exit(&cp->cache_depot_lock);
3100b5fca8f8Stomee }
3101b5fca8f8Stomee }
31027c478bd9Sstevel@tonic-gate
31037c478bd9Sstevel@tonic-gate kmem_depot_ws_reap(cp);
3104b5fca8f8Stomee
3105b5fca8f8Stomee if (cp->cache_defrag != NULL && !kmem_move_noreap) {
3106b5fca8f8Stomee kmem_cache_defrag(cp);
3107b5fca8f8Stomee }
31087c478bd9Sstevel@tonic-gate }
31097c478bd9Sstevel@tonic-gate
31107c478bd9Sstevel@tonic-gate static void
kmem_reap_timeout(void * flag_arg)31117c478bd9Sstevel@tonic-gate kmem_reap_timeout(void *flag_arg)
31127c478bd9Sstevel@tonic-gate {
31137c478bd9Sstevel@tonic-gate uint32_t *flag = (uint32_t *)flag_arg;
31147c478bd9Sstevel@tonic-gate
31157c478bd9Sstevel@tonic-gate ASSERT(flag == &kmem_reaping || flag == &kmem_reaping_idspace);
31167c478bd9Sstevel@tonic-gate *flag = 0;
31177c478bd9Sstevel@tonic-gate }
31187c478bd9Sstevel@tonic-gate
31197c478bd9Sstevel@tonic-gate static void
kmem_reap_done(void * flag)31207c478bd9Sstevel@tonic-gate kmem_reap_done(void *flag)
31217c478bd9Sstevel@tonic-gate {
31226e00b116SPeter Telford if (!callout_init_done) {
31236e00b116SPeter Telford /* can't schedule a timeout at this point */
31246e00b116SPeter Telford kmem_reap_timeout(flag);
31256e00b116SPeter Telford } else {
31266e00b116SPeter Telford (void) timeout(kmem_reap_timeout, flag, kmem_reap_interval);
31276e00b116SPeter Telford }
31287c478bd9Sstevel@tonic-gate }
31297c478bd9Sstevel@tonic-gate
31307c478bd9Sstevel@tonic-gate static void
kmem_reap_start(void * flag)31317c478bd9Sstevel@tonic-gate kmem_reap_start(void *flag)
31327c478bd9Sstevel@tonic-gate {
31337c478bd9Sstevel@tonic-gate ASSERT(flag == &kmem_reaping || flag == &kmem_reaping_idspace);
31347c478bd9Sstevel@tonic-gate
31357c478bd9Sstevel@tonic-gate if (flag == &kmem_reaping) {
31367c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_reap, kmem_taskq, TQ_NOSLEEP);
31377c478bd9Sstevel@tonic-gate /*
31387c478bd9Sstevel@tonic-gate * if we have segkp under heap, reap segkp cache.
31397c478bd9Sstevel@tonic-gate */
31407c478bd9Sstevel@tonic-gate if (segkp_fromheap)
31417c478bd9Sstevel@tonic-gate segkp_cache_free();
31427c478bd9Sstevel@tonic-gate }
31437c478bd9Sstevel@tonic-gate else
31447c478bd9Sstevel@tonic-gate kmem_cache_applyall_id(kmem_cache_reap, kmem_taskq, TQ_NOSLEEP);
31457c478bd9Sstevel@tonic-gate
31467c478bd9Sstevel@tonic-gate /*
31477c478bd9Sstevel@tonic-gate * We use taskq_dispatch() to schedule a timeout to clear
31487c478bd9Sstevel@tonic-gate * the flag so that kmem_reap() becomes self-throttling:
31497c478bd9Sstevel@tonic-gate * we won't reap again until the current reap completes *and*
31507c478bd9Sstevel@tonic-gate * at least kmem_reap_interval ticks have elapsed.
31517c478bd9Sstevel@tonic-gate */
3152fc8ae2ecSToomas Soome if (taskq_dispatch(kmem_taskq, kmem_reap_done, flag, TQ_NOSLEEP) ==
3153fc8ae2ecSToomas Soome TASKQID_INVALID)
31547c478bd9Sstevel@tonic-gate kmem_reap_done(flag);
31557c478bd9Sstevel@tonic-gate }
31567c478bd9Sstevel@tonic-gate
31577c478bd9Sstevel@tonic-gate static void
kmem_reap_common(void * flag_arg)31587c478bd9Sstevel@tonic-gate kmem_reap_common(void *flag_arg)
31597c478bd9Sstevel@tonic-gate {
31607c478bd9Sstevel@tonic-gate uint32_t *flag = (uint32_t *)flag_arg;
31617c478bd9Sstevel@tonic-gate
31627c478bd9Sstevel@tonic-gate if (MUTEX_HELD(&kmem_cache_lock) || kmem_taskq == NULL ||
316375d94465SJosef 'Jeff' Sipek atomic_cas_32(flag, 0, 1) != 0)
31647c478bd9Sstevel@tonic-gate return;
31657c478bd9Sstevel@tonic-gate
31667c478bd9Sstevel@tonic-gate /*
31677c478bd9Sstevel@tonic-gate * It may not be kosher to do memory allocation when a reap is called
31689321cd04SJosef 'Jeff' Sipek * (for example, if vmem_populate() is in the call chain). So we
31699321cd04SJosef 'Jeff' Sipek * start the reap going with a TQ_NOALLOC dispatch. If the dispatch
31709321cd04SJosef 'Jeff' Sipek * fails, we reset the flag, and the next reap will try again.
31717c478bd9Sstevel@tonic-gate */
3172fc8ae2ecSToomas Soome if (taskq_dispatch(kmem_taskq, kmem_reap_start, flag, TQ_NOALLOC) ==
3173fc8ae2ecSToomas Soome TASKQID_INVALID)
31747c478bd9Sstevel@tonic-gate *flag = 0;
31757c478bd9Sstevel@tonic-gate }
31767c478bd9Sstevel@tonic-gate
31777c478bd9Sstevel@tonic-gate /*
31787c478bd9Sstevel@tonic-gate * Reclaim all unused memory from all caches. Called from the VM system
31797c478bd9Sstevel@tonic-gate * when memory gets tight.
31807c478bd9Sstevel@tonic-gate */
31817c478bd9Sstevel@tonic-gate void
kmem_reap(void)31827c478bd9Sstevel@tonic-gate kmem_reap(void)
31837c478bd9Sstevel@tonic-gate {
31847c478bd9Sstevel@tonic-gate kmem_reap_common(&kmem_reaping);
31857c478bd9Sstevel@tonic-gate }
31867c478bd9Sstevel@tonic-gate
31877c478bd9Sstevel@tonic-gate /*
31887c478bd9Sstevel@tonic-gate * Reclaim all unused memory from identifier arenas, called when a vmem
31897c478bd9Sstevel@tonic-gate * arena not back by memory is exhausted. Since reaping memory-backed caches
31907c478bd9Sstevel@tonic-gate * cannot help with identifier exhaustion, we avoid both a large amount of
31917c478bd9Sstevel@tonic-gate * work and unwanted side-effects from reclaim callbacks.
31927c478bd9Sstevel@tonic-gate */
31937c478bd9Sstevel@tonic-gate void
kmem_reap_idspace(void)31947c478bd9Sstevel@tonic-gate kmem_reap_idspace(void)
31957c478bd9Sstevel@tonic-gate {
31967c478bd9Sstevel@tonic-gate kmem_reap_common(&kmem_reaping_idspace);
31977c478bd9Sstevel@tonic-gate }
31987c478bd9Sstevel@tonic-gate
31997c478bd9Sstevel@tonic-gate /*
32007c478bd9Sstevel@tonic-gate * Purge all magazines from a cache and set its magazine limit to zero.
32017c478bd9Sstevel@tonic-gate * All calls are serialized by the kmem_taskq lock, except for the final
32027c478bd9Sstevel@tonic-gate * call from kmem_cache_destroy().
32037c478bd9Sstevel@tonic-gate */
32047c478bd9Sstevel@tonic-gate static void
kmem_cache_magazine_purge(kmem_cache_t * cp)32057c478bd9Sstevel@tonic-gate kmem_cache_magazine_purge(kmem_cache_t *cp)
32067c478bd9Sstevel@tonic-gate {
32077c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp;
32087c478bd9Sstevel@tonic-gate kmem_magazine_t *mp, *pmp;
32097c478bd9Sstevel@tonic-gate int rounds, prounds, cpu_seqid;
32107c478bd9Sstevel@tonic-gate
3211b5fca8f8Stomee ASSERT(!list_link_active(&cp->cache_link) ||
3212b5fca8f8Stomee taskq_member(kmem_taskq, curthread));
32137c478bd9Sstevel@tonic-gate ASSERT(MUTEX_NOT_HELD(&cp->cache_lock));
32147c478bd9Sstevel@tonic-gate
32157c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) {
32167c478bd9Sstevel@tonic-gate ccp = &cp->cache_cpu[cpu_seqid];
32177c478bd9Sstevel@tonic-gate
32187c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock);
32197c478bd9Sstevel@tonic-gate mp = ccp->cc_loaded;
32207c478bd9Sstevel@tonic-gate pmp = ccp->cc_ploaded;
32217c478bd9Sstevel@tonic-gate rounds = ccp->cc_rounds;
32227c478bd9Sstevel@tonic-gate prounds = ccp->cc_prounds;
32237c478bd9Sstevel@tonic-gate ccp->cc_loaded = NULL;
32247c478bd9Sstevel@tonic-gate ccp->cc_ploaded = NULL;
32257c478bd9Sstevel@tonic-gate ccp->cc_rounds = -1;
32267c478bd9Sstevel@tonic-gate ccp->cc_prounds = -1;
32277c478bd9Sstevel@tonic-gate ccp->cc_magsize = 0;
32287c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock);
32297c478bd9Sstevel@tonic-gate
32307c478bd9Sstevel@tonic-gate if (mp)
32317c478bd9Sstevel@tonic-gate kmem_magazine_destroy(cp, mp, rounds);
32327c478bd9Sstevel@tonic-gate if (pmp)
32337c478bd9Sstevel@tonic-gate kmem_magazine_destroy(cp, pmp, prounds);
32347c478bd9Sstevel@tonic-gate }
32357c478bd9Sstevel@tonic-gate
32360c833d64SJosef 'Jeff' Sipek kmem_depot_ws_zero(cp);
32377c478bd9Sstevel@tonic-gate kmem_depot_ws_reap(cp);
32387c478bd9Sstevel@tonic-gate }
32397c478bd9Sstevel@tonic-gate
32407c478bd9Sstevel@tonic-gate /*
32417c478bd9Sstevel@tonic-gate * Enable per-cpu magazines on a cache.
32427c478bd9Sstevel@tonic-gate */
32437c478bd9Sstevel@tonic-gate static void
kmem_cache_magazine_enable(kmem_cache_t * cp)32447c478bd9Sstevel@tonic-gate kmem_cache_magazine_enable(kmem_cache_t *cp)
32457c478bd9Sstevel@tonic-gate {
32467c478bd9Sstevel@tonic-gate int cpu_seqid;
32477c478bd9Sstevel@tonic-gate
32487c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_NOMAGAZINE)
32497c478bd9Sstevel@tonic-gate return;
32507c478bd9Sstevel@tonic-gate
32517c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) {
32527c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid];
32537c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock);
32547c478bd9Sstevel@tonic-gate ccp->cc_magsize = cp->cache_magtype->mt_magsize;
32557c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock);
32567c478bd9Sstevel@tonic-gate }
32577c478bd9Sstevel@tonic-gate
32587c478bd9Sstevel@tonic-gate }
32597c478bd9Sstevel@tonic-gate
3260fa9e4066Sahrens /*
326136a64e62STim Kordas * Allow our caller to determine if there are running reaps.
326236a64e62STim Kordas *
326336a64e62STim Kordas * This call is very conservative and may return B_TRUE even when
326436a64e62STim Kordas * reaping activity isn't active. If it returns B_FALSE, then reaping
326536a64e62STim Kordas * activity is definitely inactive.
326636a64e62STim Kordas */
326736a64e62STim Kordas boolean_t
kmem_cache_reap_active(void)326836a64e62STim Kordas kmem_cache_reap_active(void)
326936a64e62STim Kordas {
327036a64e62STim Kordas return (!taskq_empty(kmem_taskq));
327136a64e62STim Kordas }
327236a64e62STim Kordas
327336a64e62STim Kordas /*
327436a64e62STim Kordas * Reap (almost) everything soon.
327536a64e62STim Kordas *
327636a64e62STim Kordas * Note: this does not wait for the reap-tasks to complete. Caller
327736a64e62STim Kordas * should use kmem_cache_reap_active() (above) and/or moderation to
327836a64e62STim Kordas * avoid scheduling too many reap-tasks.
3279fa9e4066Sahrens */
3280fa9e4066Sahrens void
kmem_cache_reap_soon(kmem_cache_t * cp)328136a64e62STim Kordas kmem_cache_reap_soon(kmem_cache_t *cp)
3282fa9e4066Sahrens {
3283b5fca8f8Stomee ASSERT(list_link_active(&cp->cache_link));
3284b5fca8f8Stomee
32850c833d64SJosef 'Jeff' Sipek kmem_depot_ws_zero(cp);
3286fa9e4066Sahrens
3287fa9e4066Sahrens (void) taskq_dispatch(kmem_taskq,
3288fa9e4066Sahrens (task_func_t *)kmem_depot_ws_reap, cp, TQ_SLEEP);
3289fa9e4066Sahrens }
3290fa9e4066Sahrens
32917c478bd9Sstevel@tonic-gate /*
32927c478bd9Sstevel@tonic-gate * Recompute a cache's magazine size. The trade-off is that larger magazines
32937c478bd9Sstevel@tonic-gate * provide a higher transfer rate with the depot, while smaller magazines
32947c478bd9Sstevel@tonic-gate * reduce memory consumption. Magazine resizing is an expensive operation;
32957c478bd9Sstevel@tonic-gate * it should not be done frequently.
32967c478bd9Sstevel@tonic-gate *
32977c478bd9Sstevel@tonic-gate * Changes to the magazine size are serialized by the kmem_taskq lock.
32987c478bd9Sstevel@tonic-gate *
32997c478bd9Sstevel@tonic-gate * Note: at present this only grows the magazine size. It might be useful
33007c478bd9Sstevel@tonic-gate * to allow shrinkage too.
33017c478bd9Sstevel@tonic-gate */
33027c478bd9Sstevel@tonic-gate static void
kmem_cache_magazine_resize(kmem_cache_t * cp)33037c478bd9Sstevel@tonic-gate kmem_cache_magazine_resize(kmem_cache_t *cp)
33047c478bd9Sstevel@tonic-gate {
33057c478bd9Sstevel@tonic-gate kmem_magtype_t *mtp = cp->cache_magtype;
33067c478bd9Sstevel@tonic-gate
33077c478bd9Sstevel@tonic-gate ASSERT(taskq_member(kmem_taskq, curthread));
33087c478bd9Sstevel@tonic-gate
33097c478bd9Sstevel@tonic-gate if (cp->cache_chunksize < mtp->mt_maxbuf) {
33107c478bd9Sstevel@tonic-gate kmem_cache_magazine_purge(cp);
33117c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock);
33127c478bd9Sstevel@tonic-gate cp->cache_magtype = ++mtp;
33137c478bd9Sstevel@tonic-gate cp->cache_depot_contention_prev =
33147c478bd9Sstevel@tonic-gate cp->cache_depot_contention + INT_MAX;
33157c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock);
33167c478bd9Sstevel@tonic-gate kmem_cache_magazine_enable(cp);
33177c478bd9Sstevel@tonic-gate }
33187c478bd9Sstevel@tonic-gate }
33197c478bd9Sstevel@tonic-gate
33207c478bd9Sstevel@tonic-gate /*
33217c478bd9Sstevel@tonic-gate * Rescale a cache's hash table, so that the table size is roughly the
33227c478bd9Sstevel@tonic-gate * cache size. We want the average lookup time to be extremely small.
33237c478bd9Sstevel@tonic-gate */
33247c478bd9Sstevel@tonic-gate static void
kmem_hash_rescale(kmem_cache_t * cp)33257c478bd9Sstevel@tonic-gate kmem_hash_rescale(kmem_cache_t *cp)
33267c478bd9Sstevel@tonic-gate {
33277c478bd9Sstevel@tonic-gate kmem_bufctl_t **old_table, **new_table, *bcp;
33287c478bd9Sstevel@tonic-gate size_t old_size, new_size, h;
33297c478bd9Sstevel@tonic-gate
33307c478bd9Sstevel@tonic-gate ASSERT(taskq_member(kmem_taskq, curthread));
33317c478bd9Sstevel@tonic-gate
33327c478bd9Sstevel@tonic-gate new_size = MAX(KMEM_HASH_INITIAL,
33337c478bd9Sstevel@tonic-gate 1 << (highbit(3 * cp->cache_buftotal + 4) - 2));
33347c478bd9Sstevel@tonic-gate old_size = cp->cache_hash_mask + 1;
33357c478bd9Sstevel@tonic-gate
33367c478bd9Sstevel@tonic-gate if ((old_size >> 1) <= new_size && new_size <= (old_size << 1))
33377c478bd9Sstevel@tonic-gate return;
33387c478bd9Sstevel@tonic-gate
33397c478bd9Sstevel@tonic-gate new_table = vmem_alloc(kmem_hash_arena, new_size * sizeof (void *),
33407c478bd9Sstevel@tonic-gate VM_NOSLEEP);
33417c478bd9Sstevel@tonic-gate if (new_table == NULL)
33427c478bd9Sstevel@tonic-gate return;
33437c478bd9Sstevel@tonic-gate bzero(new_table, new_size * sizeof (void *));
33447c478bd9Sstevel@tonic-gate
33457c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock);
33467c478bd9Sstevel@tonic-gate
33477c478bd9Sstevel@tonic-gate old_size = cp->cache_hash_mask + 1;
33487c478bd9Sstevel@tonic-gate old_table = cp->cache_hash_table;
33497c478bd9Sstevel@tonic-gate
33507c478bd9Sstevel@tonic-gate cp->cache_hash_mask = new_size - 1;
33517c478bd9Sstevel@tonic-gate cp->cache_hash_table = new_table;
33527c478bd9Sstevel@tonic-gate cp->cache_rescale++;
33537c478bd9Sstevel@tonic-gate
33547c478bd9Sstevel@tonic-gate for (h = 0; h < old_size; h++) {
33557c478bd9Sstevel@tonic-gate bcp = old_table[h];
33567c478bd9Sstevel@tonic-gate while (bcp != NULL) {
33577c478bd9Sstevel@tonic-gate void *addr = bcp->bc_addr;
33587c478bd9Sstevel@tonic-gate kmem_bufctl_t *next_bcp = bcp->bc_next;
33597c478bd9Sstevel@tonic-gate kmem_bufctl_t **hash_bucket = KMEM_HASH(cp, addr);
33607c478bd9Sstevel@tonic-gate bcp->bc_next = *hash_bucket;
33617c478bd9Sstevel@tonic-gate *hash_bucket = bcp;
33627c478bd9Sstevel@tonic-gate bcp = next_bcp;
33637c478bd9Sstevel@tonic-gate }
33647c478bd9Sstevel@tonic-gate }
33657c478bd9Sstevel@tonic-gate
33667c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock);
33677c478bd9Sstevel@tonic-gate
33687c478bd9Sstevel@tonic-gate vmem_free(kmem_hash_arena, old_table, old_size * sizeof (void *));
33697c478bd9Sstevel@tonic-gate }
33707c478bd9Sstevel@tonic-gate
33717c478bd9Sstevel@tonic-gate /*
3372b5fca8f8Stomee * Perform periodic maintenance on a cache: hash rescaling, depot working-set
3373b5fca8f8Stomee * update, magazine resizing, and slab consolidation.
33747c478bd9Sstevel@tonic-gate */
33757c478bd9Sstevel@tonic-gate static void
kmem_cache_update(kmem_cache_t * cp)33767c478bd9Sstevel@tonic-gate kmem_cache_update(kmem_cache_t *cp)
33777c478bd9Sstevel@tonic-gate {
33787c478bd9Sstevel@tonic-gate int need_hash_rescale = 0;
33797c478bd9Sstevel@tonic-gate int need_magazine_resize = 0;
33807c478bd9Sstevel@tonic-gate
33817c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&kmem_cache_lock));
33827c478bd9Sstevel@tonic-gate
33837c478bd9Sstevel@tonic-gate /*
33847c478bd9Sstevel@tonic-gate * If the cache has become much larger or smaller than its hash table,
33857c478bd9Sstevel@tonic-gate * fire off a request to rescale the hash table.
33867c478bd9Sstevel@tonic-gate */
33877c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock);
33887c478bd9Sstevel@tonic-gate
33897c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_HASH) &&
33907c478bd9Sstevel@tonic-gate (cp->cache_buftotal > (cp->cache_hash_mask << 1) ||
33917c478bd9Sstevel@tonic-gate (cp->cache_buftotal < (cp->cache_hash_mask >> 1) &&
33927c478bd9Sstevel@tonic-gate cp->cache_hash_mask > KMEM_HASH_INITIAL)))
33937c478bd9Sstevel@tonic-gate need_hash_rescale = 1;
33947c478bd9Sstevel@tonic-gate
33957c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock);
33967c478bd9Sstevel@tonic-gate
33977c478bd9Sstevel@tonic-gate /*
33987c478bd9Sstevel@tonic-gate * Update the depot working set statistics.
33997c478bd9Sstevel@tonic-gate */
34007c478bd9Sstevel@tonic-gate kmem_depot_ws_update(cp);
34017c478bd9Sstevel@tonic-gate
34027c478bd9Sstevel@tonic-gate /*
34037c478bd9Sstevel@tonic-gate * If there's a lot of contention in the depot,
34047c478bd9Sstevel@tonic-gate * increase the magazine size.
34057c478bd9Sstevel@tonic-gate */
34067c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock);
34077c478bd9Sstevel@tonic-gate
34087c478bd9Sstevel@tonic-gate if (cp->cache_chunksize < cp->cache_magtype->mt_maxbuf &&
34097c478bd9Sstevel@tonic-gate (int)(cp->cache_depot_contention -
34107c478bd9Sstevel@tonic-gate cp->cache_depot_contention_prev) > kmem_depot_contention)
34117c478bd9Sstevel@tonic-gate need_magazine_resize = 1;
34127c478bd9Sstevel@tonic-gate
34137c478bd9Sstevel@tonic-gate cp->cache_depot_contention_prev = cp->cache_depot_contention;
34147c478bd9Sstevel@tonic-gate
34157c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock);
34167c478bd9Sstevel@tonic-gate
34177c478bd9Sstevel@tonic-gate if (need_hash_rescale)
34187c478bd9Sstevel@tonic-gate (void) taskq_dispatch(kmem_taskq,
34197c478bd9Sstevel@tonic-gate (task_func_t *)kmem_hash_rescale, cp, TQ_NOSLEEP);
34207c478bd9Sstevel@tonic-gate
34217c478bd9Sstevel@tonic-gate if (need_magazine_resize)
34227c478bd9Sstevel@tonic-gate (void) taskq_dispatch(kmem_taskq,
34237c478bd9Sstevel@tonic-gate (task_func_t *)kmem_cache_magazine_resize, cp, TQ_NOSLEEP);
3424b5fca8f8Stomee
3425b5fca8f8Stomee if (cp->cache_defrag != NULL)
3426b5fca8f8Stomee (void) taskq_dispatch(kmem_taskq,
3427b5fca8f8Stomee (task_func_t *)kmem_cache_scan, cp, TQ_NOSLEEP);
34287c478bd9Sstevel@tonic-gate }
34297c478bd9Sstevel@tonic-gate
3430d67944fbSScott Rotondo static void kmem_update(void *);
3431d67944fbSScott Rotondo
34327c478bd9Sstevel@tonic-gate static void
kmem_update_timeout(void * dummy)34337c478bd9Sstevel@tonic-gate kmem_update_timeout(void *dummy)
34347c478bd9Sstevel@tonic-gate {
34357c478bd9Sstevel@tonic-gate (void) timeout(kmem_update, dummy, kmem_reap_interval);
34367c478bd9Sstevel@tonic-gate }
34377c478bd9Sstevel@tonic-gate
34387c478bd9Sstevel@tonic-gate static void
kmem_update(void * dummy)34397c478bd9Sstevel@tonic-gate kmem_update(void *dummy)
34407c478bd9Sstevel@tonic-gate {
34417c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_update, NULL, TQ_NOSLEEP);
34427c478bd9Sstevel@tonic-gate
34437c478bd9Sstevel@tonic-gate /*
34447c478bd9Sstevel@tonic-gate * We use taskq_dispatch() to reschedule the timeout so that
34457c478bd9Sstevel@tonic-gate * kmem_update() becomes self-throttling: it won't schedule
34467c478bd9Sstevel@tonic-gate * new tasks until all previous tasks have completed.
34477c478bd9Sstevel@tonic-gate */
3448fc8ae2ecSToomas Soome if (taskq_dispatch(kmem_taskq, kmem_update_timeout, dummy, TQ_NOSLEEP)
3449fc8ae2ecSToomas Soome == TASKQID_INVALID)
34507c478bd9Sstevel@tonic-gate kmem_update_timeout(NULL);
34517c478bd9Sstevel@tonic-gate }
34527c478bd9Sstevel@tonic-gate
34537c478bd9Sstevel@tonic-gate static int
kmem_cache_kstat_update(kstat_t * ksp,int rw)34547c478bd9Sstevel@tonic-gate kmem_cache_kstat_update(kstat_t *ksp, int rw)
34557c478bd9Sstevel@tonic-gate {
34567c478bd9Sstevel@tonic-gate struct kmem_cache_kstat *kmcp = &kmem_cache_kstat;
34577c478bd9Sstevel@tonic-gate kmem_cache_t *cp = ksp->ks_private;
34587c478bd9Sstevel@tonic-gate uint64_t cpu_buf_avail;
34597c478bd9Sstevel@tonic-gate uint64_t buf_avail = 0;
34607c478bd9Sstevel@tonic-gate int cpu_seqid;
3461686031edSTom Erickson long reap;
34627c478bd9Sstevel@tonic-gate
34637c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&kmem_cache_kstat_lock));
34647c478bd9Sstevel@tonic-gate
34657c478bd9Sstevel@tonic-gate if (rw == KSTAT_WRITE)
34667c478bd9Sstevel@tonic-gate return (EACCES);
34677c478bd9Sstevel@tonic-gate
34687c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock);
34697c478bd9Sstevel@tonic-gate
34707c478bd9Sstevel@tonic-gate kmcp->kmc_alloc_fail.value.ui64 = cp->cache_alloc_fail;
34717c478bd9Sstevel@tonic-gate kmcp->kmc_alloc.value.ui64 = cp->cache_slab_alloc;
34727c478bd9Sstevel@tonic-gate kmcp->kmc_free.value.ui64 = cp->cache_slab_free;
34737c478bd9Sstevel@tonic-gate kmcp->kmc_slab_alloc.value.ui64 = cp->cache_slab_alloc;
34747c478bd9Sstevel@tonic-gate kmcp->kmc_slab_free.value.ui64 = cp->cache_slab_free;
34757c478bd9Sstevel@tonic-gate
34767c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) {
34777c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid];
34787c478bd9Sstevel@tonic-gate
34797c478bd9Sstevel@tonic-gate mutex_enter(&ccp->cc_lock);
34807c478bd9Sstevel@tonic-gate
34817c478bd9Sstevel@tonic-gate cpu_buf_avail = 0;
34827c478bd9Sstevel@tonic-gate if (ccp->cc_rounds > 0)
34837c478bd9Sstevel@tonic-gate cpu_buf_avail += ccp->cc_rounds;
34847c478bd9Sstevel@tonic-gate if (ccp->cc_prounds > 0)
34857c478bd9Sstevel@tonic-gate cpu_buf_avail += ccp->cc_prounds;
34867c478bd9Sstevel@tonic-gate
34877c478bd9Sstevel@tonic-gate kmcp->kmc_alloc.value.ui64 += ccp->cc_alloc;
34887c478bd9Sstevel@tonic-gate kmcp->kmc_free.value.ui64 += ccp->cc_free;
34897c478bd9Sstevel@tonic-gate buf_avail += cpu_buf_avail;
34907c478bd9Sstevel@tonic-gate
34917c478bd9Sstevel@tonic-gate mutex_exit(&ccp->cc_lock);
34927c478bd9Sstevel@tonic-gate }
34937c478bd9Sstevel@tonic-gate
34947c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_depot_lock);
34957c478bd9Sstevel@tonic-gate
34967c478bd9Sstevel@tonic-gate kmcp->kmc_depot_alloc.value.ui64 = cp->cache_full.ml_alloc;
34977c478bd9Sstevel@tonic-gate kmcp->kmc_depot_free.value.ui64 = cp->cache_empty.ml_alloc;
34987c478bd9Sstevel@tonic-gate kmcp->kmc_depot_contention.value.ui64 = cp->cache_depot_contention;
34997c478bd9Sstevel@tonic-gate kmcp->kmc_full_magazines.value.ui64 = cp->cache_full.ml_total;
35007c478bd9Sstevel@tonic-gate kmcp->kmc_empty_magazines.value.ui64 = cp->cache_empty.ml_total;
35017c478bd9Sstevel@tonic-gate kmcp->kmc_magazine_size.value.ui64 =
35027c478bd9Sstevel@tonic-gate (cp->cache_flags & KMF_NOMAGAZINE) ?
35037c478bd9Sstevel@tonic-gate 0 : cp->cache_magtype->mt_magsize;
35047c478bd9Sstevel@tonic-gate
35057c478bd9Sstevel@tonic-gate kmcp->kmc_alloc.value.ui64 += cp->cache_full.ml_alloc;
35067c478bd9Sstevel@tonic-gate kmcp->kmc_free.value.ui64 += cp->cache_empty.ml_alloc;
35077c478bd9Sstevel@tonic-gate buf_avail += cp->cache_full.ml_total * cp->cache_magtype->mt_magsize;
35087c478bd9Sstevel@tonic-gate
3509686031edSTom Erickson reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min);
3510686031edSTom Erickson reap = MIN(reap, cp->cache_full.ml_total);
3511686031edSTom Erickson
35127c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_depot_lock);
35137c478bd9Sstevel@tonic-gate
35147c478bd9Sstevel@tonic-gate kmcp->kmc_buf_size.value.ui64 = cp->cache_bufsize;
35157c478bd9Sstevel@tonic-gate kmcp->kmc_align.value.ui64 = cp->cache_align;
35167c478bd9Sstevel@tonic-gate kmcp->kmc_chunk_size.value.ui64 = cp->cache_chunksize;
35177c478bd9Sstevel@tonic-gate kmcp->kmc_slab_size.value.ui64 = cp->cache_slabsize;
35187c478bd9Sstevel@tonic-gate kmcp->kmc_buf_constructed.value.ui64 = buf_avail;
35199f1b636aStomee buf_avail += cp->cache_bufslab;
35207c478bd9Sstevel@tonic-gate kmcp->kmc_buf_avail.value.ui64 = buf_avail;
35217c478bd9Sstevel@tonic-gate kmcp->kmc_buf_inuse.value.ui64 = cp->cache_buftotal - buf_avail;
35227c478bd9Sstevel@tonic-gate kmcp->kmc_buf_total.value.ui64 = cp->cache_buftotal;
35237c478bd9Sstevel@tonic-gate kmcp->kmc_buf_max.value.ui64 = cp->cache_bufmax;
35247c478bd9Sstevel@tonic-gate kmcp->kmc_slab_create.value.ui64 = cp->cache_slab_create;
35257c478bd9Sstevel@tonic-gate kmcp->kmc_slab_destroy.value.ui64 = cp->cache_slab_destroy;
35267c478bd9Sstevel@tonic-gate kmcp->kmc_hash_size.value.ui64 = (cp->cache_flags & KMF_HASH) ?
35277c478bd9Sstevel@tonic-gate cp->cache_hash_mask + 1 : 0;
35287c478bd9Sstevel@tonic-gate kmcp->kmc_hash_lookup_depth.value.ui64 = cp->cache_lookup_depth;
35297c478bd9Sstevel@tonic-gate kmcp->kmc_hash_rescale.value.ui64 = cp->cache_rescale;
35307c478bd9Sstevel@tonic-gate kmcp->kmc_vmem_source.value.ui64 = cp->cache_arena->vm_id;
3531686031edSTom Erickson kmcp->kmc_reap.value.ui64 = cp->cache_reap;
35327c478bd9Sstevel@tonic-gate
3533b5fca8f8Stomee if (cp->cache_defrag == NULL) {
3534b5fca8f8Stomee kmcp->kmc_move_callbacks.value.ui64 = 0;
3535b5fca8f8Stomee kmcp->kmc_move_yes.value.ui64 = 0;
3536b5fca8f8Stomee kmcp->kmc_move_no.value.ui64 = 0;
3537b5fca8f8Stomee kmcp->kmc_move_later.value.ui64 = 0;
3538b5fca8f8Stomee kmcp->kmc_move_dont_need.value.ui64 = 0;
3539b5fca8f8Stomee kmcp->kmc_move_dont_know.value.ui64 = 0;
3540b5fca8f8Stomee kmcp->kmc_move_hunt_found.value.ui64 = 0;
3541686031edSTom Erickson kmcp->kmc_move_slabs_freed.value.ui64 = 0;
3542686031edSTom Erickson kmcp->kmc_defrag.value.ui64 = 0;
3543686031edSTom Erickson kmcp->kmc_scan.value.ui64 = 0;
3544686031edSTom Erickson kmcp->kmc_move_reclaimable.value.ui64 = 0;
3545b5fca8f8Stomee } else {
3546686031edSTom Erickson int64_t reclaimable;
3547686031edSTom Erickson
3548b5fca8f8Stomee kmem_defrag_t *kd = cp->cache_defrag;
3549b5fca8f8Stomee kmcp->kmc_move_callbacks.value.ui64 = kd->kmd_callbacks;
3550b5fca8f8Stomee kmcp->kmc_move_yes.value.ui64 = kd->kmd_yes;
3551b5fca8f8Stomee kmcp->kmc_move_no.value.ui64 = kd->kmd_no;
3552b5fca8f8Stomee kmcp->kmc_move_later.value.ui64 = kd->kmd_later;
3553b5fca8f8Stomee kmcp->kmc_move_dont_need.value.ui64 = kd->kmd_dont_need;
3554b5fca8f8Stomee kmcp->kmc_move_dont_know.value.ui64 = kd->kmd_dont_know;
3555d7db73d1SBryan Cantrill kmcp->kmc_move_hunt_found.value.ui64 = 0;
3556686031edSTom Erickson kmcp->kmc_move_slabs_freed.value.ui64 = kd->kmd_slabs_freed;
3557686031edSTom Erickson kmcp->kmc_defrag.value.ui64 = kd->kmd_defrags;
3558686031edSTom Erickson kmcp->kmc_scan.value.ui64 = kd->kmd_scans;
3559686031edSTom Erickson
3560686031edSTom Erickson reclaimable = cp->cache_bufslab - (cp->cache_maxchunks - 1);
3561686031edSTom Erickson reclaimable = MAX(reclaimable, 0);
3562686031edSTom Erickson reclaimable += ((uint64_t)reap * cp->cache_magtype->mt_magsize);
3563686031edSTom Erickson kmcp->kmc_move_reclaimable.value.ui64 = reclaimable;
3564b5fca8f8Stomee }
3565b5fca8f8Stomee
35667c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock);
35677c478bd9Sstevel@tonic-gate return (0);
35687c478bd9Sstevel@tonic-gate }
35697c478bd9Sstevel@tonic-gate
35707c478bd9Sstevel@tonic-gate /*
35717c478bd9Sstevel@tonic-gate * Return a named statistic about a particular cache.
35727c478bd9Sstevel@tonic-gate * This shouldn't be called very often, so it's currently designed for
35737c478bd9Sstevel@tonic-gate * simplicity (leverages existing kstat support) rather than efficiency.
35747c478bd9Sstevel@tonic-gate */
35757c478bd9Sstevel@tonic-gate uint64_t
kmem_cache_stat(kmem_cache_t * cp,char * name)35767c478bd9Sstevel@tonic-gate kmem_cache_stat(kmem_cache_t *cp, char *name)
35777c478bd9Sstevel@tonic-gate {
35787c478bd9Sstevel@tonic-gate int i;
35797c478bd9Sstevel@tonic-gate kstat_t *ksp = cp->cache_kstat;
35807c478bd9Sstevel@tonic-gate kstat_named_t *knp = (kstat_named_t *)&kmem_cache_kstat;
35817c478bd9Sstevel@tonic-gate uint64_t value = 0;
35827c478bd9Sstevel@tonic-gate
35837c478bd9Sstevel@tonic-gate if (ksp != NULL) {
35847c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_kstat_lock);
35857c478bd9Sstevel@tonic-gate (void) kmem_cache_kstat_update(ksp, KSTAT_READ);
35867c478bd9Sstevel@tonic-gate for (i = 0; i < ksp->ks_ndata; i++) {
35877c478bd9Sstevel@tonic-gate if (strcmp(knp[i].name, name) == 0) {
35887c478bd9Sstevel@tonic-gate value = knp[i].value.ui64;
35897c478bd9Sstevel@tonic-gate break;
35907c478bd9Sstevel@tonic-gate }
35917c478bd9Sstevel@tonic-gate }
35927c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_kstat_lock);
35937c478bd9Sstevel@tonic-gate }
35947c478bd9Sstevel@tonic-gate return (value);
35957c478bd9Sstevel@tonic-gate }
35967c478bd9Sstevel@tonic-gate
35977c478bd9Sstevel@tonic-gate /*
35987c478bd9Sstevel@tonic-gate * Return an estimate of currently available kernel heap memory.
35997c478bd9Sstevel@tonic-gate * On 32-bit systems, physical memory may exceed virtual memory,
36007c478bd9Sstevel@tonic-gate * we just truncate the result at 1GB.
36017c478bd9Sstevel@tonic-gate */
36027c478bd9Sstevel@tonic-gate size_t
kmem_avail(void)36037c478bd9Sstevel@tonic-gate kmem_avail(void)
36047c478bd9Sstevel@tonic-gate {
36057c478bd9Sstevel@tonic-gate spgcnt_t rmem = availrmem - tune.t_minarmem;
36067c478bd9Sstevel@tonic-gate spgcnt_t fmem = freemem - minfree;
36077c478bd9Sstevel@tonic-gate
36087c478bd9Sstevel@tonic-gate return ((size_t)ptob(MIN(MAX(MIN(rmem, fmem), 0),
36097c478bd9Sstevel@tonic-gate 1 << (30 - PAGESHIFT))));
36107c478bd9Sstevel@tonic-gate }
36117c478bd9Sstevel@tonic-gate
36127c478bd9Sstevel@tonic-gate /*
36137c478bd9Sstevel@tonic-gate * Return the maximum amount of memory that is (in theory) allocatable
36147c478bd9Sstevel@tonic-gate * from the heap. This may be used as an estimate only since there
36157c478bd9Sstevel@tonic-gate * is no guarentee this space will still be available when an allocation
36167c478bd9Sstevel@tonic-gate * request is made, nor that the space may be allocated in one big request
36177c478bd9Sstevel@tonic-gate * due to kernel heap fragmentation.
36187c478bd9Sstevel@tonic-gate */
36197c478bd9Sstevel@tonic-gate size_t
kmem_maxavail(void)36207c478bd9Sstevel@tonic-gate kmem_maxavail(void)
36217c478bd9Sstevel@tonic-gate {
36227c478bd9Sstevel@tonic-gate spgcnt_t pmem = availrmem - tune.t_minarmem;
36237c478bd9Sstevel@tonic-gate spgcnt_t vmem = btop(vmem_size(heap_arena, VMEM_FREE));
36247c478bd9Sstevel@tonic-gate
36257c478bd9Sstevel@tonic-gate return ((size_t)ptob(MAX(MIN(pmem, vmem), 0)));
36267c478bd9Sstevel@tonic-gate }
36277c478bd9Sstevel@tonic-gate
3628fa9e4066Sahrens /*
3629fa9e4066Sahrens * Indicate whether memory-intensive kmem debugging is enabled.
3630fa9e4066Sahrens */
3631fa9e4066Sahrens int
kmem_debugging(void)3632fa9e4066Sahrens kmem_debugging(void)
3633fa9e4066Sahrens {
3634fa9e4066Sahrens return (kmem_flags & (KMF_AUDIT | KMF_REDZONE));
3635fa9e4066Sahrens }
3636fa9e4066Sahrens
3637b5fca8f8Stomee /* binning function, sorts finely at the two extremes */
3638b5fca8f8Stomee #define KMEM_PARTIAL_SLAB_WEIGHT(sp, binshift) \
3639b5fca8f8Stomee ((((sp)->slab_refcnt <= (binshift)) || \
3640b5fca8f8Stomee (((sp)->slab_chunks - (sp)->slab_refcnt) <= (binshift))) \
3641b5fca8f8Stomee ? -(sp)->slab_refcnt \
3642b5fca8f8Stomee : -((binshift) + ((sp)->slab_refcnt >> (binshift))))
3643b5fca8f8Stomee
3644b5fca8f8Stomee /*
3645b5fca8f8Stomee * Minimizing the number of partial slabs on the freelist minimizes
3646b5fca8f8Stomee * fragmentation (the ratio of unused buffers held by the slab layer). There are
3647b5fca8f8Stomee * two ways to get a slab off of the freelist: 1) free all the buffers on the
3648b5fca8f8Stomee * slab, and 2) allocate all the buffers on the slab. It follows that we want
3649b5fca8f8Stomee * the most-used slabs at the front of the list where they have the best chance
3650b5fca8f8Stomee * of being completely allocated, and the least-used slabs at a safe distance
3651b5fca8f8Stomee * from the front to improve the odds that the few remaining buffers will all be
3652b5fca8f8Stomee * freed before another allocation can tie up the slab. For that reason a slab
3653b5fca8f8Stomee * with a higher slab_refcnt sorts less than than a slab with a lower
3654b5fca8f8Stomee * slab_refcnt.
3655b5fca8f8Stomee *
3656b5fca8f8Stomee * However, if a slab has at least one buffer that is deemed unfreeable, we
3657b5fca8f8Stomee * would rather have that slab at the front of the list regardless of
3658b5fca8f8Stomee * slab_refcnt, since even one unfreeable buffer makes the entire slab
3659b5fca8f8Stomee * unfreeable. If the client returns KMEM_CBRC_NO in response to a cache_move()
3660b5fca8f8Stomee * callback, the slab is marked unfreeable for as long as it remains on the
3661b5fca8f8Stomee * freelist.
3662b5fca8f8Stomee */
3663b5fca8f8Stomee static int
kmem_partial_slab_cmp(const void * p0,const void * p1)3664b5fca8f8Stomee kmem_partial_slab_cmp(const void *p0, const void *p1)
3665b5fca8f8Stomee {
3666b5fca8f8Stomee const kmem_cache_t *cp;
3667b5fca8f8Stomee const kmem_slab_t *s0 = p0;
3668b5fca8f8Stomee const kmem_slab_t *s1 = p1;
3669b5fca8f8Stomee int w0, w1;
3670b5fca8f8Stomee size_t binshift;
3671b5fca8f8Stomee
3672b5fca8f8Stomee ASSERT(KMEM_SLAB_IS_PARTIAL(s0));
3673b5fca8f8Stomee ASSERT(KMEM_SLAB_IS_PARTIAL(s1));
3674b5fca8f8Stomee ASSERT(s0->slab_cache == s1->slab_cache);
3675b5fca8f8Stomee cp = s1->slab_cache;
3676b5fca8f8Stomee ASSERT(MUTEX_HELD(&cp->cache_lock));
3677b5fca8f8Stomee binshift = cp->cache_partial_binshift;
3678b5fca8f8Stomee
3679b5fca8f8Stomee /* weight of first slab */
3680b5fca8f8Stomee w0 = KMEM_PARTIAL_SLAB_WEIGHT(s0, binshift);
3681b5fca8f8Stomee if (s0->slab_flags & KMEM_SLAB_NOMOVE) {
3682b5fca8f8Stomee w0 -= cp->cache_maxchunks;
3683b5fca8f8Stomee }
3684b5fca8f8Stomee
3685b5fca8f8Stomee /* weight of second slab */
3686b5fca8f8Stomee w1 = KMEM_PARTIAL_SLAB_WEIGHT(s1, binshift);
3687b5fca8f8Stomee if (s1->slab_flags & KMEM_SLAB_NOMOVE) {
3688b5fca8f8Stomee w1 -= cp->cache_maxchunks;
3689b5fca8f8Stomee }
3690b5fca8f8Stomee
3691b5fca8f8Stomee if (w0 < w1)
3692b5fca8f8Stomee return (-1);
3693b5fca8f8Stomee if (w0 > w1)
3694b5fca8f8Stomee return (1);
3695b5fca8f8Stomee
3696b5fca8f8Stomee /* compare pointer values */
3697b5fca8f8Stomee if ((uintptr_t)s0 < (uintptr_t)s1)
3698b5fca8f8Stomee return (-1);
3699b5fca8f8Stomee if ((uintptr_t)s0 > (uintptr_t)s1)
3700b5fca8f8Stomee return (1);
3701b5fca8f8Stomee
3702b5fca8f8Stomee return (0);
3703b5fca8f8Stomee }
3704b5fca8f8Stomee
3705b5fca8f8Stomee /*
3706b5fca8f8Stomee * It must be valid to call the destructor (if any) on a newly created object.
3707b5fca8f8Stomee * That is, the constructor (if any) must leave the object in a valid state for
3708b5fca8f8Stomee * the destructor.
3709b5fca8f8Stomee */
37107c478bd9Sstevel@tonic-gate kmem_cache_t *
kmem_cache_create(char * name,size_t bufsize,size_t align,int (* constructor)(void *,void *,int),void (* destructor)(void *,void *),void (* reclaim)(void *),void * private,vmem_t * vmp,int cflags)37117c478bd9Sstevel@tonic-gate kmem_cache_create(
37127c478bd9Sstevel@tonic-gate char *name, /* descriptive name for this cache */
37137c478bd9Sstevel@tonic-gate size_t bufsize, /* size of the objects it manages */
37147c478bd9Sstevel@tonic-gate size_t align, /* required object alignment */
37157c478bd9Sstevel@tonic-gate int (*constructor)(void *, void *, int), /* object constructor */
37167c478bd9Sstevel@tonic-gate void (*destructor)(void *, void *), /* object destructor */
37177c478bd9Sstevel@tonic-gate void (*reclaim)(void *), /* memory reclaim callback */
37187c478bd9Sstevel@tonic-gate void *private, /* pass-thru arg for constr/destr/reclaim */
37197c478bd9Sstevel@tonic-gate vmem_t *vmp, /* vmem source for slab allocation */
37207c478bd9Sstevel@tonic-gate int cflags) /* cache creation flags */
37217c478bd9Sstevel@tonic-gate {
37227c478bd9Sstevel@tonic-gate int cpu_seqid;
37237c478bd9Sstevel@tonic-gate size_t chunksize;
3724b5fca8f8Stomee kmem_cache_t *cp;
37257c478bd9Sstevel@tonic-gate kmem_magtype_t *mtp;
37267c478bd9Sstevel@tonic-gate size_t csize = KMEM_CACHE_SIZE(max_ncpus);
37277c478bd9Sstevel@tonic-gate
37287c478bd9Sstevel@tonic-gate #ifdef DEBUG
37297c478bd9Sstevel@tonic-gate /*
37307c478bd9Sstevel@tonic-gate * Cache names should conform to the rules for valid C identifiers
37317c478bd9Sstevel@tonic-gate */
37327c478bd9Sstevel@tonic-gate if (!strident_valid(name)) {
37337c478bd9Sstevel@tonic-gate cmn_err(CE_CONT,
37347c478bd9Sstevel@tonic-gate "kmem_cache_create: '%s' is an invalid cache name\n"
37357c478bd9Sstevel@tonic-gate "cache names must conform to the rules for "
37367c478bd9Sstevel@tonic-gate "C identifiers\n", name);
37377c478bd9Sstevel@tonic-gate }
37387c478bd9Sstevel@tonic-gate #endif /* DEBUG */
37397c478bd9Sstevel@tonic-gate
37407c478bd9Sstevel@tonic-gate if (vmp == NULL)
37417c478bd9Sstevel@tonic-gate vmp = kmem_default_arena;
37427c478bd9Sstevel@tonic-gate
37437c478bd9Sstevel@tonic-gate /*
37447c478bd9Sstevel@tonic-gate * If this kmem cache has an identifier vmem arena as its source, mark
37457c478bd9Sstevel@tonic-gate * it such to allow kmem_reap_idspace().
37467c478bd9Sstevel@tonic-gate */
37477c478bd9Sstevel@tonic-gate ASSERT(!(cflags & KMC_IDENTIFIER)); /* consumer should not set this */
37487c478bd9Sstevel@tonic-gate if (vmp->vm_cflags & VMC_IDENTIFIER)
37497c478bd9Sstevel@tonic-gate cflags |= KMC_IDENTIFIER;
37507c478bd9Sstevel@tonic-gate
37517c478bd9Sstevel@tonic-gate /*
37527c478bd9Sstevel@tonic-gate * Get a kmem_cache structure. We arrange that cp->cache_cpu[]
37537c478bd9Sstevel@tonic-gate * is aligned on a KMEM_CPU_CACHE_SIZE boundary to prevent
37547c478bd9Sstevel@tonic-gate * false sharing of per-CPU data.
37557c478bd9Sstevel@tonic-gate */
37567c478bd9Sstevel@tonic-gate cp = vmem_xalloc(kmem_cache_arena, csize, KMEM_CPU_CACHE_SIZE,
37577c478bd9Sstevel@tonic-gate P2NPHASE(csize, KMEM_CPU_CACHE_SIZE), 0, NULL, NULL, VM_SLEEP);
37587c478bd9Sstevel@tonic-gate bzero(cp, csize);
3759b5fca8f8Stomee list_link_init(&cp->cache_link);
37607c478bd9Sstevel@tonic-gate
37617c478bd9Sstevel@tonic-gate if (align == 0)
37627c478bd9Sstevel@tonic-gate align = KMEM_ALIGN;
37637c478bd9Sstevel@tonic-gate
37647c478bd9Sstevel@tonic-gate /*
37657c478bd9Sstevel@tonic-gate * If we're not at least KMEM_ALIGN aligned, we can't use free
37667c478bd9Sstevel@tonic-gate * memory to hold bufctl information (because we can't safely
37677c478bd9Sstevel@tonic-gate * perform word loads and stores on it).
37687c478bd9Sstevel@tonic-gate */
37697c478bd9Sstevel@tonic-gate if (align < KMEM_ALIGN)
37707c478bd9Sstevel@tonic-gate cflags |= KMC_NOTOUCH;
37717c478bd9Sstevel@tonic-gate
3772de710d24SJosef 'Jeff' Sipek if (!ISP2(align) || align > vmp->vm_quantum)
37737c478bd9Sstevel@tonic-gate panic("kmem_cache_create: bad alignment %lu", align);
37747c478bd9Sstevel@tonic-gate
37757c478bd9Sstevel@tonic-gate mutex_enter(&kmem_flags_lock);
37767c478bd9Sstevel@tonic-gate if (kmem_flags & KMF_RANDOMIZE)
37777c478bd9Sstevel@tonic-gate kmem_flags = (((kmem_flags | ~KMF_RANDOM) + 1) & KMF_RANDOM) |
37787c478bd9Sstevel@tonic-gate KMF_RANDOMIZE;
37797c478bd9Sstevel@tonic-gate cp->cache_flags = (kmem_flags | cflags) & KMF_DEBUG;
37807c478bd9Sstevel@tonic-gate mutex_exit(&kmem_flags_lock);
37817c478bd9Sstevel@tonic-gate
37827c478bd9Sstevel@tonic-gate /*
37837c478bd9Sstevel@tonic-gate * Make sure all the various flags are reasonable.
37847c478bd9Sstevel@tonic-gate */
37857c478bd9Sstevel@tonic-gate ASSERT(!(cflags & KMC_NOHASH) || !(cflags & KMC_NOTOUCH));
37867c478bd9Sstevel@tonic-gate
37877c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE) {
37887c478bd9Sstevel@tonic-gate if (bufsize >= kmem_lite_minsize &&
37897c478bd9Sstevel@tonic-gate align <= kmem_lite_maxalign &&
37907c478bd9Sstevel@tonic-gate P2PHASE(bufsize, kmem_lite_maxalign) != 0) {
37917c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_BUFTAG;
37927c478bd9Sstevel@tonic-gate cp->cache_flags &= ~(KMF_AUDIT | KMF_FIREWALL);
37937c478bd9Sstevel@tonic-gate } else {
37947c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_DEBUG;
37957c478bd9Sstevel@tonic-gate }
37967c478bd9Sstevel@tonic-gate }
37977c478bd9Sstevel@tonic-gate
37987c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF)
37997c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_REDZONE;
38007c478bd9Sstevel@tonic-gate
38017c478bd9Sstevel@tonic-gate if ((cflags & KMC_QCACHE) && (cp->cache_flags & KMF_AUDIT))
38027c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_NOMAGAZINE;
38037c478bd9Sstevel@tonic-gate
38047c478bd9Sstevel@tonic-gate if (cflags & KMC_NODEBUG)
38057c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_DEBUG;
38067c478bd9Sstevel@tonic-gate
38077c478bd9Sstevel@tonic-gate if (cflags & KMC_NOTOUCH)
38087c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_TOUCH;
38097c478bd9Sstevel@tonic-gate
3810b942e89bSDavid Valin if (cflags & KMC_PREFILL)
3811b942e89bSDavid Valin cp->cache_flags |= KMF_PREFILL;
3812b942e89bSDavid Valin
38137c478bd9Sstevel@tonic-gate if (cflags & KMC_NOHASH)
38147c478bd9Sstevel@tonic-gate cp->cache_flags &= ~(KMF_AUDIT | KMF_FIREWALL);
38157c478bd9Sstevel@tonic-gate
38167c478bd9Sstevel@tonic-gate if (cflags & KMC_NOMAGAZINE)
38177c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_NOMAGAZINE;
38187c478bd9Sstevel@tonic-gate
38197c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_AUDIT) && !(cflags & KMC_NOTOUCH))
38207c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_REDZONE;
38217c478bd9Sstevel@tonic-gate
38227c478bd9Sstevel@tonic-gate if (!(cp->cache_flags & KMF_AUDIT))
38237c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_CONTENTS;
38247c478bd9Sstevel@tonic-gate
38257c478bd9Sstevel@tonic-gate if ((cp->cache_flags & KMF_BUFTAG) && bufsize >= kmem_minfirewall &&
38267c478bd9Sstevel@tonic-gate !(cp->cache_flags & KMF_LITE) && !(cflags & KMC_NOHASH))
38277c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_FIREWALL;
38287c478bd9Sstevel@tonic-gate
38297c478bd9Sstevel@tonic-gate if (vmp != kmem_default_arena || kmem_firewall_arena == NULL)
38307c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_FIREWALL;
38317c478bd9Sstevel@tonic-gate
38327c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_FIREWALL) {
38337c478bd9Sstevel@tonic-gate cp->cache_flags &= ~KMF_BUFTAG;
38347c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_NOMAGAZINE;
38357c478bd9Sstevel@tonic-gate ASSERT(vmp == kmem_default_arena);
38367c478bd9Sstevel@tonic-gate vmp = kmem_firewall_arena;
38377c478bd9Sstevel@tonic-gate }
38387c478bd9Sstevel@tonic-gate
38397c478bd9Sstevel@tonic-gate /*
38407c478bd9Sstevel@tonic-gate * Set cache properties.
38417c478bd9Sstevel@tonic-gate */
38427c478bd9Sstevel@tonic-gate (void) strncpy(cp->cache_name, name, KMEM_CACHE_NAMELEN);
3843b5fca8f8Stomee strident_canon(cp->cache_name, KMEM_CACHE_NAMELEN + 1);
38447c478bd9Sstevel@tonic-gate cp->cache_bufsize = bufsize;
38457c478bd9Sstevel@tonic-gate cp->cache_align = align;
38467c478bd9Sstevel@tonic-gate cp->cache_constructor = constructor;
38477c478bd9Sstevel@tonic-gate cp->cache_destructor = destructor;
38487c478bd9Sstevel@tonic-gate cp->cache_reclaim = reclaim;
38497c478bd9Sstevel@tonic-gate cp->cache_private = private;
38507c478bd9Sstevel@tonic-gate cp->cache_arena = vmp;
38517c478bd9Sstevel@tonic-gate cp->cache_cflags = cflags;
38527c478bd9Sstevel@tonic-gate
38537c478bd9Sstevel@tonic-gate /*
38547c478bd9Sstevel@tonic-gate * Determine the chunk size.
38557c478bd9Sstevel@tonic-gate */
38567c478bd9Sstevel@tonic-gate chunksize = bufsize;
38577c478bd9Sstevel@tonic-gate
38587c478bd9Sstevel@tonic-gate if (align >= KMEM_ALIGN) {
38597c478bd9Sstevel@tonic-gate chunksize = P2ROUNDUP(chunksize, KMEM_ALIGN);
38607c478bd9Sstevel@tonic-gate cp->cache_bufctl = chunksize - KMEM_ALIGN;
38617c478bd9Sstevel@tonic-gate }
38627c478bd9Sstevel@tonic-gate
38637c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_BUFTAG) {
38647c478bd9Sstevel@tonic-gate cp->cache_bufctl = chunksize;
38657c478bd9Sstevel@tonic-gate cp->cache_buftag = chunksize;
38667c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE)
38677c478bd9Sstevel@tonic-gate chunksize += KMEM_BUFTAG_LITE_SIZE(kmem_lite_count);
38687c478bd9Sstevel@tonic-gate else
38697c478bd9Sstevel@tonic-gate chunksize += sizeof (kmem_buftag_t);
38707c478bd9Sstevel@tonic-gate }
38717c478bd9Sstevel@tonic-gate
38727c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_DEADBEEF) {
38737c478bd9Sstevel@tonic-gate cp->cache_verify = MIN(cp->cache_buftag, kmem_maxverify);
38747c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_LITE)
38757c478bd9Sstevel@tonic-gate cp->cache_verify = sizeof (uint64_t);
38767c478bd9Sstevel@tonic-gate }
38777c478bd9Sstevel@tonic-gate
38787c478bd9Sstevel@tonic-gate cp->cache_contents = MIN(cp->cache_bufctl, kmem_content_maxsave);
38797c478bd9Sstevel@tonic-gate
38807c478bd9Sstevel@tonic-gate cp->cache_chunksize = chunksize = P2ROUNDUP(chunksize, align);
38817c478bd9Sstevel@tonic-gate
38827c478bd9Sstevel@tonic-gate /*
38837c478bd9Sstevel@tonic-gate * Now that we know the chunk size, determine the optimal slab size.
38847c478bd9Sstevel@tonic-gate */
38857c478bd9Sstevel@tonic-gate if (vmp == kmem_firewall_arena) {
38867c478bd9Sstevel@tonic-gate cp->cache_slabsize = P2ROUNDUP(chunksize, vmp->vm_quantum);
38877c478bd9Sstevel@tonic-gate cp->cache_mincolor = cp->cache_slabsize - chunksize;
38887c478bd9Sstevel@tonic-gate cp->cache_maxcolor = cp->cache_mincolor;
38897c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_HASH;
38907c478bd9Sstevel@tonic-gate ASSERT(!(cp->cache_flags & KMF_BUFTAG));
38917c478bd9Sstevel@tonic-gate } else if ((cflags & KMC_NOHASH) || (!(cflags & KMC_NOTOUCH) &&
38927c478bd9Sstevel@tonic-gate !(cp->cache_flags & KMF_AUDIT) &&
38937c478bd9Sstevel@tonic-gate chunksize < vmp->vm_quantum / KMEM_VOID_FRACTION)) {
38947c478bd9Sstevel@tonic-gate cp->cache_slabsize = vmp->vm_quantum;
38957c478bd9Sstevel@tonic-gate cp->cache_mincolor = 0;
38967c478bd9Sstevel@tonic-gate cp->cache_maxcolor =
38977c478bd9Sstevel@tonic-gate (cp->cache_slabsize - sizeof (kmem_slab_t)) % chunksize;
38987c478bd9Sstevel@tonic-gate ASSERT(chunksize + sizeof (kmem_slab_t) <= cp->cache_slabsize);
38997c478bd9Sstevel@tonic-gate ASSERT(!(cp->cache_flags & KMF_AUDIT));
39007c478bd9Sstevel@tonic-gate } else {
39017c478bd9Sstevel@tonic-gate size_t chunks, bestfit, waste, slabsize;
39027c478bd9Sstevel@tonic-gate size_t minwaste = LONG_MAX;
39037c478bd9Sstevel@tonic-gate
3904c6f039c7SToomas Soome bestfit = 0;
39057c478bd9Sstevel@tonic-gate for (chunks = 1; chunks <= KMEM_VOID_FRACTION; chunks++) {
39067c478bd9Sstevel@tonic-gate slabsize = P2ROUNDUP(chunksize * chunks,
39077c478bd9Sstevel@tonic-gate vmp->vm_quantum);
39087c478bd9Sstevel@tonic-gate chunks = slabsize / chunksize;
39097c478bd9Sstevel@tonic-gate waste = (slabsize % chunksize) / chunks;
39107c478bd9Sstevel@tonic-gate if (waste < minwaste) {
39117c478bd9Sstevel@tonic-gate minwaste = waste;
39127c478bd9Sstevel@tonic-gate bestfit = slabsize;
39137c478bd9Sstevel@tonic-gate }
39147c478bd9Sstevel@tonic-gate }
39157c478bd9Sstevel@tonic-gate if (cflags & KMC_QCACHE)
39167c478bd9Sstevel@tonic-gate bestfit = VMEM_QCACHE_SLABSIZE(vmp->vm_qcache_max);
39177c478bd9Sstevel@tonic-gate cp->cache_slabsize = bestfit;
39187c478bd9Sstevel@tonic-gate cp->cache_mincolor = 0;
39197c478bd9Sstevel@tonic-gate cp->cache_maxcolor = bestfit % chunksize;
39207c478bd9Sstevel@tonic-gate cp->cache_flags |= KMF_HASH;
39217c478bd9Sstevel@tonic-gate }
39227c478bd9Sstevel@tonic-gate
3923b5fca8f8Stomee cp->cache_maxchunks = (cp->cache_slabsize / cp->cache_chunksize);
3924b5fca8f8Stomee cp->cache_partial_binshift = highbit(cp->cache_maxchunks / 16) + 1;
3925b5fca8f8Stomee
3926b942e89bSDavid Valin /*
3927b942e89bSDavid Valin * Disallowing prefill when either the DEBUG or HASH flag is set or when
3928b942e89bSDavid Valin * there is a constructor avoids some tricky issues with debug setup
3929b942e89bSDavid Valin * that may be revisited later. We cannot allow prefill in a
3930b942e89bSDavid Valin * metadata cache because of potential recursion.
3931b942e89bSDavid Valin */
3932b942e89bSDavid Valin if (vmp == kmem_msb_arena ||
3933b942e89bSDavid Valin cp->cache_flags & (KMF_HASH | KMF_BUFTAG) ||
3934b942e89bSDavid Valin cp->cache_constructor != NULL)
3935b942e89bSDavid Valin cp->cache_flags &= ~KMF_PREFILL;
3936b942e89bSDavid Valin
39377c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) {
39387c478bd9Sstevel@tonic-gate ASSERT(!(cflags & KMC_NOHASH));
39397c478bd9Sstevel@tonic-gate cp->cache_bufctl_cache = (cp->cache_flags & KMF_AUDIT) ?
39407c478bd9Sstevel@tonic-gate kmem_bufctl_audit_cache : kmem_bufctl_cache;
39417c478bd9Sstevel@tonic-gate }
39427c478bd9Sstevel@tonic-gate
39437c478bd9Sstevel@tonic-gate if (cp->cache_maxcolor >= vmp->vm_quantum)
39447c478bd9Sstevel@tonic-gate cp->cache_maxcolor = vmp->vm_quantum - 1;
39457c478bd9Sstevel@tonic-gate
39467c478bd9Sstevel@tonic-gate cp->cache_color = cp->cache_mincolor;
39477c478bd9Sstevel@tonic-gate
39487c478bd9Sstevel@tonic-gate /*
39497c478bd9Sstevel@tonic-gate * Initialize the rest of the slab layer.
39507c478bd9Sstevel@tonic-gate */
39517c478bd9Sstevel@tonic-gate mutex_init(&cp->cache_lock, NULL, MUTEX_DEFAULT, NULL);
39527c478bd9Sstevel@tonic-gate
3953b5fca8f8Stomee avl_create(&cp->cache_partial_slabs, kmem_partial_slab_cmp,
3954b5fca8f8Stomee sizeof (kmem_slab_t), offsetof(kmem_slab_t, slab_link));
3955b5fca8f8Stomee /* LINTED: E_TRUE_LOGICAL_EXPR */
3956b5fca8f8Stomee ASSERT(sizeof (list_node_t) <= sizeof (avl_node_t));
3957b5fca8f8Stomee /* reuse partial slab AVL linkage for complete slab list linkage */
3958b5fca8f8Stomee list_create(&cp->cache_complete_slabs,
3959b5fca8f8Stomee sizeof (kmem_slab_t), offsetof(kmem_slab_t, slab_link));
39607c478bd9Sstevel@tonic-gate
39617c478bd9Sstevel@tonic-gate if (cp->cache_flags & KMF_HASH) {
39627c478bd9Sstevel@tonic-gate cp->cache_hash_table = vmem_alloc(kmem_hash_arena,
39637c478bd9Sstevel@tonic-gate KMEM_HASH_INITIAL * sizeof (void *), VM_SLEEP);
39647c478bd9Sstevel@tonic-gate bzero(cp->cache_hash_table,
39657c478bd9Sstevel@tonic-gate KMEM_HASH_INITIAL * sizeof (void *));
39667c478bd9Sstevel@tonic-gate cp->cache_hash_mask = KMEM_HASH_INITIAL - 1;
39677c478bd9Sstevel@tonic-gate cp->cache_hash_shift = highbit((ulong_t)chunksize) - 1;
39687c478bd9Sstevel@tonic-gate }
39697c478bd9Sstevel@tonic-gate
39707c478bd9Sstevel@tonic-gate /*
39717c478bd9Sstevel@tonic-gate * Initialize the depot.
39727c478bd9Sstevel@tonic-gate */
39737c478bd9Sstevel@tonic-gate mutex_init(&cp->cache_depot_lock, NULL, MUTEX_DEFAULT, NULL);
39747c478bd9Sstevel@tonic-gate
39757c478bd9Sstevel@tonic-gate for (mtp = kmem_magtype; chunksize <= mtp->mt_minbuf; mtp++)
39767c478bd9Sstevel@tonic-gate continue;
39777c478bd9Sstevel@tonic-gate
39787c478bd9Sstevel@tonic-gate cp->cache_magtype = mtp;
39797c478bd9Sstevel@tonic-gate
39807c478bd9Sstevel@tonic-gate /*
39817c478bd9Sstevel@tonic-gate * Initialize the CPU layer.
39827c478bd9Sstevel@tonic-gate */
39837c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) {
39847c478bd9Sstevel@tonic-gate kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid];
39857c478bd9Sstevel@tonic-gate mutex_init(&ccp->cc_lock, NULL, MUTEX_DEFAULT, NULL);
39867c478bd9Sstevel@tonic-gate ccp->cc_flags = cp->cache_flags;
39877c478bd9Sstevel@tonic-gate ccp->cc_rounds = -1;
39887c478bd9Sstevel@tonic-gate ccp->cc_prounds = -1;
39897c478bd9Sstevel@tonic-gate }
39907c478bd9Sstevel@tonic-gate
39917c478bd9Sstevel@tonic-gate /*
39927c478bd9Sstevel@tonic-gate * Create the cache's kstats.
39937c478bd9Sstevel@tonic-gate */
39947c478bd9Sstevel@tonic-gate if ((cp->cache_kstat = kstat_create("unix", 0, cp->cache_name,
39957c478bd9Sstevel@tonic-gate "kmem_cache", KSTAT_TYPE_NAMED,
39967c478bd9Sstevel@tonic-gate sizeof (kmem_cache_kstat) / sizeof (kstat_named_t),
39977c478bd9Sstevel@tonic-gate KSTAT_FLAG_VIRTUAL)) != NULL) {
39987c478bd9Sstevel@tonic-gate cp->cache_kstat->ks_data = &kmem_cache_kstat;
39997c478bd9Sstevel@tonic-gate cp->cache_kstat->ks_update = kmem_cache_kstat_update;
40007c478bd9Sstevel@tonic-gate cp->cache_kstat->ks_private = cp;
40017c478bd9Sstevel@tonic-gate cp->cache_kstat->ks_lock = &kmem_cache_kstat_lock;
40027c478bd9Sstevel@tonic-gate kstat_install(cp->cache_kstat);
40037c478bd9Sstevel@tonic-gate }
40047c478bd9Sstevel@tonic-gate
40057c478bd9Sstevel@tonic-gate /*
40067c478bd9Sstevel@tonic-gate * Add the cache to the global list. This makes it visible
40077c478bd9Sstevel@tonic-gate * to kmem_update(), so the cache must be ready for business.
40087c478bd9Sstevel@tonic-gate */
40097c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_lock);
4010b5fca8f8Stomee list_insert_tail(&kmem_caches, cp);
40117c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_lock);
40127c478bd9Sstevel@tonic-gate
40137c478bd9Sstevel@tonic-gate if (kmem_ready)
40147c478bd9Sstevel@tonic-gate kmem_cache_magazine_enable(cp);
40157c478bd9Sstevel@tonic-gate
40167c478bd9Sstevel@tonic-gate return (cp);
40177c478bd9Sstevel@tonic-gate }
40187c478bd9Sstevel@tonic-gate
4019b5fca8f8Stomee static int
kmem_move_cmp(const void * buf,const void * p)4020b5fca8f8Stomee kmem_move_cmp(const void *buf, const void *p)
4021b5fca8f8Stomee {
4022b5fca8f8Stomee const kmem_move_t *kmm = p;
4023b5fca8f8Stomee uintptr_t v1 = (uintptr_t)buf;
4024b5fca8f8Stomee uintptr_t v2 = (uintptr_t)kmm->kmm_from_buf;
4025b5fca8f8Stomee return (v1 < v2 ? -1 : (v1 > v2 ? 1 : 0));
4026b5fca8f8Stomee }
4027b5fca8f8Stomee
4028b5fca8f8Stomee static void
kmem_reset_reclaim_threshold(kmem_defrag_t * kmd)4029b5fca8f8Stomee kmem_reset_reclaim_threshold(kmem_defrag_t *kmd)
4030b5fca8f8Stomee {
4031b5fca8f8Stomee kmd->kmd_reclaim_numer = 1;
4032b5fca8f8Stomee }
4033b5fca8f8Stomee
4034b5fca8f8Stomee /*
4035b5fca8f8Stomee * Initially, when choosing candidate slabs for buffers to move, we want to be
4036b5fca8f8Stomee * very selective and take only slabs that are less than
4037b5fca8f8Stomee * (1 / KMEM_VOID_FRACTION) allocated. If we have difficulty finding candidate
4038b5fca8f8Stomee * slabs, then we raise the allocation ceiling incrementally. The reclaim
4039b5fca8f8Stomee * threshold is reset to (1 / KMEM_VOID_FRACTION) as soon as the cache is no
4040b5fca8f8Stomee * longer fragmented.
4041b5fca8f8Stomee */
4042b5fca8f8Stomee static void
kmem_adjust_reclaim_threshold(kmem_defrag_t * kmd,int direction)4043b5fca8f8Stomee kmem_adjust_reclaim_threshold(kmem_defrag_t *kmd, int direction)
4044b5fca8f8Stomee {
4045b5fca8f8Stomee if (direction > 0) {
4046b5fca8f8Stomee /* make it easier to find a candidate slab */
4047b5fca8f8Stomee if (kmd->kmd_reclaim_numer < (KMEM_VOID_FRACTION - 1)) {
4048b5fca8f8Stomee kmd->kmd_reclaim_numer++;
4049b5fca8f8Stomee }
4050b5fca8f8Stomee } else {
4051b5fca8f8Stomee /* be more selective */
4052b5fca8f8Stomee if (kmd->kmd_reclaim_numer > 1) {
4053b5fca8f8Stomee kmd->kmd_reclaim_numer--;
4054b5fca8f8Stomee }
4055b5fca8f8Stomee }
4056b5fca8f8Stomee }
4057b5fca8f8Stomee
4058b5fca8f8Stomee void
kmem_cache_set_move(kmem_cache_t * cp,kmem_cbrc_t (* move)(void *,void *,size_t,void *))4059b5fca8f8Stomee kmem_cache_set_move(kmem_cache_t *cp,
4060b5fca8f8Stomee kmem_cbrc_t (*move)(void *, void *, size_t, void *))
4061b5fca8f8Stomee {
4062b5fca8f8Stomee kmem_defrag_t *defrag;
4063b5fca8f8Stomee
4064b5fca8f8Stomee ASSERT(move != NULL);
4065b5fca8f8Stomee /*
4066b5fca8f8Stomee * The consolidator does not support NOTOUCH caches because kmem cannot
4067b5fca8f8Stomee * initialize their slabs with the 0xbaddcafe memory pattern, which sets
4068b5fca8f8Stomee * a low order bit usable by clients to distinguish uninitialized memory
4069b5fca8f8Stomee * from known objects (see kmem_slab_create).
4070b5fca8f8Stomee */
4071b5fca8f8Stomee ASSERT(!(cp->cache_cflags & KMC_NOTOUCH));
4072b5fca8f8Stomee ASSERT(!(cp->cache_cflags & KMC_IDENTIFIER));
4073b5fca8f8Stomee
4074b5fca8f8Stomee /*
4075b5fca8f8Stomee * We should not be holding anyone's cache lock when calling
4076b5fca8f8Stomee * kmem_cache_alloc(), so allocate in all cases before acquiring the
4077b5fca8f8Stomee * lock.
4078b5fca8f8Stomee */
4079b5fca8f8Stomee defrag = kmem_cache_alloc(kmem_defrag_cache, KM_SLEEP);
4080b5fca8f8Stomee
4081b5fca8f8Stomee mutex_enter(&cp->cache_lock);
4082b5fca8f8Stomee
4083b5fca8f8Stomee if (KMEM_IS_MOVABLE(cp)) {
4084b5fca8f8Stomee if (cp->cache_move == NULL) {
40854d4c4c43STom Erickson ASSERT(cp->cache_slab_alloc == 0);
4086b5fca8f8Stomee
4087b5fca8f8Stomee cp->cache_defrag = defrag;
4088b5fca8f8Stomee defrag = NULL; /* nothing to free */
4089b5fca8f8Stomee bzero(cp->cache_defrag, sizeof (kmem_defrag_t));
4090b5fca8f8Stomee avl_create(&cp->cache_defrag->kmd_moves_pending,
4091b5fca8f8Stomee kmem_move_cmp, sizeof (kmem_move_t),
4092b5fca8f8Stomee offsetof(kmem_move_t, kmm_entry));
4093b5fca8f8Stomee /* LINTED: E_TRUE_LOGICAL_EXPR */
4094b5fca8f8Stomee ASSERT(sizeof (list_node_t) <= sizeof (avl_node_t));
4095b5fca8f8Stomee /* reuse the slab's AVL linkage for deadlist linkage */
4096b5fca8f8Stomee list_create(&cp->cache_defrag->kmd_deadlist,
4097b5fca8f8Stomee sizeof (kmem_slab_t),
4098b5fca8f8Stomee offsetof(kmem_slab_t, slab_link));
4099b5fca8f8Stomee kmem_reset_reclaim_threshold(cp->cache_defrag);
4100b5fca8f8Stomee }
4101b5fca8f8Stomee cp->cache_move = move;
4102b5fca8f8Stomee }
4103b5fca8f8Stomee
4104b5fca8f8Stomee mutex_exit(&cp->cache_lock);
4105b5fca8f8Stomee
4106b5fca8f8Stomee if (defrag != NULL) {
4107b5fca8f8Stomee kmem_cache_free(kmem_defrag_cache, defrag); /* unused */
4108b5fca8f8Stomee }
4109b5fca8f8Stomee }
4110b5fca8f8Stomee
41117c478bd9Sstevel@tonic-gate void
kmem_cache_destroy(kmem_cache_t * cp)41127c478bd9Sstevel@tonic-gate kmem_cache_destroy(kmem_cache_t *cp)
41137c478bd9Sstevel@tonic-gate {
41147c478bd9Sstevel@tonic-gate int cpu_seqid;
41157c478bd9Sstevel@tonic-gate
41167c478bd9Sstevel@tonic-gate /*
41177c478bd9Sstevel@tonic-gate * Remove the cache from the global cache list so that no one else
41187c478bd9Sstevel@tonic-gate * can schedule tasks on its behalf, wait for any pending tasks to
41197c478bd9Sstevel@tonic-gate * complete, purge the cache, and then destroy it.
41207c478bd9Sstevel@tonic-gate */
41217c478bd9Sstevel@tonic-gate mutex_enter(&kmem_cache_lock);
4122b5fca8f8Stomee list_remove(&kmem_caches, cp);
41237c478bd9Sstevel@tonic-gate mutex_exit(&kmem_cache_lock);
41247c478bd9Sstevel@tonic-gate
41257c478bd9Sstevel@tonic-gate if (kmem_taskq != NULL)
41267c478bd9Sstevel@tonic-gate taskq_wait(kmem_taskq);
4127d7db73d1SBryan Cantrill
4128d7db73d1SBryan Cantrill if (kmem_move_taskq != NULL && cp->cache_defrag != NULL)
4129b5fca8f8Stomee taskq_wait(kmem_move_taskq);
41307c478bd9Sstevel@tonic-gate
41317c478bd9Sstevel@tonic-gate kmem_cache_magazine_purge(cp);
41327c478bd9Sstevel@tonic-gate
41337c478bd9Sstevel@tonic-gate mutex_enter(&cp->cache_lock);
41347c478bd9Sstevel@tonic-gate if (cp->cache_buftotal != 0)
41357c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "kmem_cache_destroy: '%s' (%p) not empty",
41367c478bd9Sstevel@tonic-gate cp->cache_name, (void *)cp);
4137b5fca8f8Stomee if (cp->cache_defrag != NULL) {
4138b5fca8f8Stomee avl_destroy(&cp->cache_defrag->kmd_moves_pending);
4139b5fca8f8Stomee list_destroy(&cp->cache_defrag->kmd_deadlist);
4140b5fca8f8Stomee kmem_cache_free(kmem_defrag_cache, cp->cache_defrag);
4141b5fca8f8Stomee cp->cache_defrag = NULL;
4142b5fca8f8Stomee }
41437c478bd9Sstevel@tonic-gate /*
4144b5fca8f8Stomee * The cache is now dead. There should be no further activity. We
4145b5fca8f8Stomee * enforce this by setting land mines in the constructor, destructor,
4146b5fca8f8Stomee * reclaim, and move routines that induce a kernel text fault if
4147b5fca8f8Stomee * invoked.
41487c478bd9Sstevel@tonic-gate */
41497c478bd9Sstevel@tonic-gate cp->cache_constructor = (int (*)(void *, void *, int))1;
41507c478bd9Sstevel@tonic-gate cp->cache_destructor = (void (*)(void *, void *))2;
4151b5fca8f8Stomee cp->cache_reclaim = (void (*)(void *))3;
4152b5fca8f8Stomee cp->cache_move = (kmem_cbrc_t (*)(void *, void *, size_t, void *))4;
41537c478bd9Sstevel@tonic-gate mutex_exit(&cp->cache_lock);
41547c478bd9Sstevel@tonic-gate
41557c478bd9Sstevel@tonic-gate kstat_delete(cp->cache_kstat);
41567c478bd9Sstevel@tonic-gate
41577c478bd9Sstevel@tonic-gate if (cp->cache_hash_table != NULL)
41587c478bd9Sstevel@tonic-gate vmem_free(kmem_hash_arena, cp->cache_hash_table,
41597c478bd9Sstevel@tonic-gate (cp->cache_hash_mask + 1) * sizeof (void *));
41607c478bd9Sstevel@tonic-gate
41617c478bd9Sstevel@tonic-gate for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++)
41627c478bd9Sstevel@tonic-gate mutex_destroy(&cp->cache_cpu[cpu_seqid].cc_lock);
41637c478bd9Sstevel@tonic-gate
41647c478bd9Sstevel@tonic-gate mutex_destroy(&cp->cache_depot_lock);
41657c478bd9Sstevel@tonic-gate mutex_destroy(&cp->cache_lock);
41667c478bd9Sstevel@tonic-gate
41677c478bd9Sstevel@tonic-gate vmem_free(kmem_cache_arena, cp, KMEM_CACHE_SIZE(max_ncpus));
41687c478bd9Sstevel@tonic-gate }
41697c478bd9Sstevel@tonic-gate
41707c478bd9Sstevel@tonic-gate /*ARGSUSED*/
41717c478bd9Sstevel@tonic-gate static int
kmem_cpu_setup(cpu_setup_t what,int id,void * arg)41727c478bd9Sstevel@tonic-gate kmem_cpu_setup(cpu_setup_t what, int id, void *arg)
41737c478bd9Sstevel@tonic-gate {
41747c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock));
41757c478bd9Sstevel@tonic-gate if (what == CPU_UNCONFIG) {
41767c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_magazine_purge,
41777c478bd9Sstevel@tonic-gate kmem_taskq, TQ_SLEEP);
41787c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_magazine_enable,
41797c478bd9Sstevel@tonic-gate kmem_taskq, TQ_SLEEP);
41807c478bd9Sstevel@tonic-gate }
41817c478bd9Sstevel@tonic-gate return (0);
41827c478bd9Sstevel@tonic-gate }
41837c478bd9Sstevel@tonic-gate
4184dce01e3fSJonathan W Adams static void
kmem_alloc_caches_create(const int * array,size_t count,kmem_cache_t ** alloc_table,size_t maxbuf,uint_t shift)4185dce01e3fSJonathan W Adams kmem_alloc_caches_create(const int *array, size_t count,
4186dce01e3fSJonathan W Adams kmem_cache_t **alloc_table, size_t maxbuf, uint_t shift)
4187dce01e3fSJonathan W Adams {
4188dce01e3fSJonathan W Adams char name[KMEM_CACHE_NAMELEN + 1];
4189dce01e3fSJonathan W Adams size_t table_unit = (1 << shift); /* range of one alloc_table entry */
4190dce01e3fSJonathan W Adams size_t size = table_unit;
4191dce01e3fSJonathan W Adams int i;
4192dce01e3fSJonathan W Adams
4193dce01e3fSJonathan W Adams for (i = 0; i < count; i++) {
4194dce01e3fSJonathan W Adams size_t cache_size = array[i];
4195dce01e3fSJonathan W Adams size_t align = KMEM_ALIGN;
4196dce01e3fSJonathan W Adams kmem_cache_t *cp;
4197dce01e3fSJonathan W Adams
4198dce01e3fSJonathan W Adams /* if the table has an entry for maxbuf, we're done */
4199dce01e3fSJonathan W Adams if (size > maxbuf)
4200dce01e3fSJonathan W Adams break;
4201dce01e3fSJonathan W Adams
4202dce01e3fSJonathan W Adams /* cache size must be a multiple of the table unit */
4203dce01e3fSJonathan W Adams ASSERT(P2PHASE(cache_size, table_unit) == 0);
4204dce01e3fSJonathan W Adams
4205dce01e3fSJonathan W Adams /*
4206dce01e3fSJonathan W Adams * If they allocate a multiple of the coherency granularity,
4207dce01e3fSJonathan W Adams * they get a coherency-granularity-aligned address.
4208dce01e3fSJonathan W Adams */
4209dce01e3fSJonathan W Adams if (IS_P2ALIGNED(cache_size, 64))
4210dce01e3fSJonathan W Adams align = 64;
4211dce01e3fSJonathan W Adams if (IS_P2ALIGNED(cache_size, PAGESIZE))
4212dce01e3fSJonathan W Adams align = PAGESIZE;
4213dce01e3fSJonathan W Adams (void) snprintf(name, sizeof (name),
4214dce01e3fSJonathan W Adams "kmem_alloc_%lu", cache_size);
4215dce01e3fSJonathan W Adams cp = kmem_cache_create(name, cache_size, align,
4216dce01e3fSJonathan W Adams NULL, NULL, NULL, NULL, NULL, KMC_KMEM_ALLOC);
4217dce01e3fSJonathan W Adams
4218dce01e3fSJonathan W Adams while (size <= cache_size) {
4219dce01e3fSJonathan W Adams alloc_table[(size - 1) >> shift] = cp;
4220dce01e3fSJonathan W Adams size += table_unit;
4221dce01e3fSJonathan W Adams }
4222dce01e3fSJonathan W Adams }
4223dce01e3fSJonathan W Adams
4224dce01e3fSJonathan W Adams ASSERT(size > maxbuf); /* i.e. maxbuf <= max(cache_size) */
4225dce01e3fSJonathan W Adams }
4226dce01e3fSJonathan W Adams
42277c478bd9Sstevel@tonic-gate static void
kmem_cache_init(int pass,int use_large_pages)42287c478bd9Sstevel@tonic-gate kmem_cache_init(int pass, int use_large_pages)
42297c478bd9Sstevel@tonic-gate {
42307c478bd9Sstevel@tonic-gate int i;
4231dce01e3fSJonathan W Adams size_t maxbuf;
42327c478bd9Sstevel@tonic-gate kmem_magtype_t *mtp;
42337c478bd9Sstevel@tonic-gate
42347c478bd9Sstevel@tonic-gate for (i = 0; i < sizeof (kmem_magtype) / sizeof (*mtp); i++) {
4235dce01e3fSJonathan W Adams char name[KMEM_CACHE_NAMELEN + 1];
4236dce01e3fSJonathan W Adams
42377c478bd9Sstevel@tonic-gate mtp = &kmem_magtype[i];
42387c478bd9Sstevel@tonic-gate (void) sprintf(name, "kmem_magazine_%d", mtp->mt_magsize);
42397c478bd9Sstevel@tonic-gate mtp->mt_cache = kmem_cache_create(name,
42407c478bd9Sstevel@tonic-gate (mtp->mt_magsize + 1) * sizeof (void *),
42417c478bd9Sstevel@tonic-gate mtp->mt_align, NULL, NULL, NULL, NULL,
42427c478bd9Sstevel@tonic-gate kmem_msb_arena, KMC_NOHASH);
42437c478bd9Sstevel@tonic-gate }
42447c478bd9Sstevel@tonic-gate
42457c478bd9Sstevel@tonic-gate kmem_slab_cache = kmem_cache_create("kmem_slab_cache",
42467c478bd9Sstevel@tonic-gate sizeof (kmem_slab_t), 0, NULL, NULL, NULL, NULL,
42477c478bd9Sstevel@tonic-gate kmem_msb_arena, KMC_NOHASH);
42487c478bd9Sstevel@tonic-gate
42497c478bd9Sstevel@tonic-gate kmem_bufctl_cache = kmem_cache_create("kmem_bufctl_cache",
42507c478bd9Sstevel@tonic-gate sizeof (kmem_bufctl_t), 0, NULL, NULL, NULL, NULL,
42517c478bd9Sstevel@tonic-gate kmem_msb_arena, KMC_NOHASH);
42527c478bd9Sstevel@tonic-gate
42537c478bd9Sstevel@tonic-gate kmem_bufctl_audit_cache = kmem_cache_create("kmem_bufctl_audit_cache",
42547c478bd9Sstevel@tonic-gate sizeof (kmem_bufctl_audit_t), 0, NULL, NULL, NULL, NULL,
42557c478bd9Sstevel@tonic-gate kmem_msb_arena, KMC_NOHASH);
42567c478bd9Sstevel@tonic-gate
42577c478bd9Sstevel@tonic-gate if (pass == 2) {
42587c478bd9Sstevel@tonic-gate kmem_va_arena = vmem_create("kmem_va",
42597c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE,
42607c478bd9Sstevel@tonic-gate vmem_alloc, vmem_free, heap_arena,
42617c478bd9Sstevel@tonic-gate 8 * PAGESIZE, VM_SLEEP);
42627c478bd9Sstevel@tonic-gate
42637c478bd9Sstevel@tonic-gate if (use_large_pages) {
42647c478bd9Sstevel@tonic-gate kmem_default_arena = vmem_xcreate("kmem_default",
42657c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE,
42667c478bd9Sstevel@tonic-gate segkmem_alloc_lp, segkmem_free_lp, kmem_va_arena,
42679dd77bc8SDave Plauger 0, VMC_DUMPSAFE | VM_SLEEP);
42687c478bd9Sstevel@tonic-gate } else {
42697c478bd9Sstevel@tonic-gate kmem_default_arena = vmem_create("kmem_default",
42707c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE,
42717c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, kmem_va_arena,
42729dd77bc8SDave Plauger 0, VMC_DUMPSAFE | VM_SLEEP);
42737c478bd9Sstevel@tonic-gate }
4274dce01e3fSJonathan W Adams
4275dce01e3fSJonathan W Adams /* Figure out what our maximum cache size is */
4276dce01e3fSJonathan W Adams maxbuf = kmem_max_cached;
4277dce01e3fSJonathan W Adams if (maxbuf <= KMEM_MAXBUF) {
4278dce01e3fSJonathan W Adams maxbuf = 0;
4279dce01e3fSJonathan W Adams kmem_max_cached = KMEM_MAXBUF;
4280dce01e3fSJonathan W Adams } else {
4281dce01e3fSJonathan W Adams size_t size = 0;
4282dce01e3fSJonathan W Adams size_t max =
4283dce01e3fSJonathan W Adams sizeof (kmem_big_alloc_sizes) / sizeof (int);
4284dce01e3fSJonathan W Adams /*
4285dce01e3fSJonathan W Adams * Round maxbuf up to an existing cache size. If maxbuf
4286dce01e3fSJonathan W Adams * is larger than the largest cache, we truncate it to
4287dce01e3fSJonathan W Adams * the largest cache's size.
4288dce01e3fSJonathan W Adams */
4289dce01e3fSJonathan W Adams for (i = 0; i < max; i++) {
4290dce01e3fSJonathan W Adams size = kmem_big_alloc_sizes[i];
4291dce01e3fSJonathan W Adams if (maxbuf <= size)
4292dce01e3fSJonathan W Adams break;
4293dce01e3fSJonathan W Adams }
4294dce01e3fSJonathan W Adams kmem_max_cached = maxbuf = size;
4295dce01e3fSJonathan W Adams }
4296dce01e3fSJonathan W Adams
4297dce01e3fSJonathan W Adams /*
4298dce01e3fSJonathan W Adams * The big alloc table may not be completely overwritten, so
4299dce01e3fSJonathan W Adams * we clear out any stale cache pointers from the first pass.
4300dce01e3fSJonathan W Adams */
4301dce01e3fSJonathan W Adams bzero(kmem_big_alloc_table, sizeof (kmem_big_alloc_table));
43027c478bd9Sstevel@tonic-gate } else {
43037c478bd9Sstevel@tonic-gate /*
43047c478bd9Sstevel@tonic-gate * During the first pass, the kmem_alloc_* caches
43057c478bd9Sstevel@tonic-gate * are treated as metadata.
43067c478bd9Sstevel@tonic-gate */
43077c478bd9Sstevel@tonic-gate kmem_default_arena = kmem_msb_arena;
4308dce01e3fSJonathan W Adams maxbuf = KMEM_BIG_MAXBUF_32BIT;
43097c478bd9Sstevel@tonic-gate }
43107c478bd9Sstevel@tonic-gate
43117c478bd9Sstevel@tonic-gate /*
43127c478bd9Sstevel@tonic-gate * Set up the default caches to back kmem_alloc()
43137c478bd9Sstevel@tonic-gate */
4314dce01e3fSJonathan W Adams kmem_alloc_caches_create(
4315dce01e3fSJonathan W Adams kmem_alloc_sizes, sizeof (kmem_alloc_sizes) / sizeof (int),
4316dce01e3fSJonathan W Adams kmem_alloc_table, KMEM_MAXBUF, KMEM_ALIGN_SHIFT);
4317dce01e3fSJonathan W Adams
4318dce01e3fSJonathan W Adams kmem_alloc_caches_create(
4319dce01e3fSJonathan W Adams kmem_big_alloc_sizes, sizeof (kmem_big_alloc_sizes) / sizeof (int),
4320dce01e3fSJonathan W Adams kmem_big_alloc_table, maxbuf, KMEM_BIG_SHIFT);
4321dce01e3fSJonathan W Adams
4322dce01e3fSJonathan W Adams kmem_big_alloc_table_max = maxbuf >> KMEM_BIG_SHIFT;
43237c478bd9Sstevel@tonic-gate }
43247c478bd9Sstevel@tonic-gate
43257c478bd9Sstevel@tonic-gate void
kmem_init(void)43267c478bd9Sstevel@tonic-gate kmem_init(void)
43277c478bd9Sstevel@tonic-gate {
43287c478bd9Sstevel@tonic-gate kmem_cache_t *cp;
43297c478bd9Sstevel@tonic-gate int old_kmem_flags = kmem_flags;
43307c478bd9Sstevel@tonic-gate int use_large_pages = 0;
43317c478bd9Sstevel@tonic-gate size_t maxverify, minfirewall;
43327c478bd9Sstevel@tonic-gate
43337c478bd9Sstevel@tonic-gate kstat_init();
43347c478bd9Sstevel@tonic-gate
43357c478bd9Sstevel@tonic-gate /*
43367c478bd9Sstevel@tonic-gate * Don't do firewalled allocations if the heap is less than 1TB
43377c478bd9Sstevel@tonic-gate * (i.e. on a 32-bit kernel)
43387c478bd9Sstevel@tonic-gate * The resulting VM_NEXTFIT allocations would create too much
43397c478bd9Sstevel@tonic-gate * fragmentation in a small heap.
43407c478bd9Sstevel@tonic-gate */
43417c478bd9Sstevel@tonic-gate #if defined(_LP64)
43427c478bd9Sstevel@tonic-gate maxverify = minfirewall = PAGESIZE / 2;
43437c478bd9Sstevel@tonic-gate #else
43447c478bd9Sstevel@tonic-gate maxverify = minfirewall = ULONG_MAX;
43457c478bd9Sstevel@tonic-gate #endif
43467c478bd9Sstevel@tonic-gate
43477c478bd9Sstevel@tonic-gate /* LINTED */
43487c478bd9Sstevel@tonic-gate ASSERT(sizeof (kmem_cpu_cache_t) == KMEM_CPU_CACHE_SIZE);
43497c478bd9Sstevel@tonic-gate
4350b5fca8f8Stomee list_create(&kmem_caches, sizeof (kmem_cache_t),
4351b5fca8f8Stomee offsetof(kmem_cache_t, cache_link));
43527c478bd9Sstevel@tonic-gate
43537c478bd9Sstevel@tonic-gate kmem_metadata_arena = vmem_create("kmem_metadata", NULL, 0, PAGESIZE,
43547c478bd9Sstevel@tonic-gate vmem_alloc, vmem_free, heap_arena, 8 * PAGESIZE,
43557c478bd9Sstevel@tonic-gate VM_SLEEP | VMC_NO_QCACHE);
43567c478bd9Sstevel@tonic-gate
43577c478bd9Sstevel@tonic-gate kmem_msb_arena = vmem_create("kmem_msb", NULL, 0,
43587c478bd9Sstevel@tonic-gate PAGESIZE, segkmem_alloc, segkmem_free, kmem_metadata_arena, 0,
43599dd77bc8SDave Plauger VMC_DUMPSAFE | VM_SLEEP);
43607c478bd9Sstevel@tonic-gate
43617c478bd9Sstevel@tonic-gate kmem_cache_arena = vmem_create("kmem_cache", NULL, 0, KMEM_ALIGN,
43627c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, VM_SLEEP);
43637c478bd9Sstevel@tonic-gate
43647c478bd9Sstevel@tonic-gate kmem_hash_arena = vmem_create("kmem_hash", NULL, 0, KMEM_ALIGN,
43657c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, VM_SLEEP);
43667c478bd9Sstevel@tonic-gate
43677c478bd9Sstevel@tonic-gate kmem_log_arena = vmem_create("kmem_log", NULL, 0, KMEM_ALIGN,
43687c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP);
43697c478bd9Sstevel@tonic-gate
43707c478bd9Sstevel@tonic-gate kmem_firewall_va_arena = vmem_create("kmem_firewall_va",
43717c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE,
43727c478bd9Sstevel@tonic-gate kmem_firewall_va_alloc, kmem_firewall_va_free, heap_arena,
43737c478bd9Sstevel@tonic-gate 0, VM_SLEEP);
43747c478bd9Sstevel@tonic-gate
43757c478bd9Sstevel@tonic-gate kmem_firewall_arena = vmem_create("kmem_firewall", NULL, 0, PAGESIZE,
43769dd77bc8SDave Plauger segkmem_alloc, segkmem_free, kmem_firewall_va_arena, 0,
43779dd77bc8SDave Plauger VMC_DUMPSAFE | VM_SLEEP);
43787c478bd9Sstevel@tonic-gate
43797c478bd9Sstevel@tonic-gate /* temporary oversize arena for mod_read_system_file */
43807c478bd9Sstevel@tonic-gate kmem_oversize_arena = vmem_create("kmem_oversize", NULL, 0, PAGESIZE,
43817c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP);
43827c478bd9Sstevel@tonic-gate
43837c478bd9Sstevel@tonic-gate kmem_reap_interval = 15 * hz;
43847c478bd9Sstevel@tonic-gate
43857c478bd9Sstevel@tonic-gate /*
43867c478bd9Sstevel@tonic-gate * Read /etc/system. This is a chicken-and-egg problem because
43877c478bd9Sstevel@tonic-gate * kmem_flags may be set in /etc/system, but mod_read_system_file()
43887c478bd9Sstevel@tonic-gate * needs to use the allocator. The simplest solution is to create
43897c478bd9Sstevel@tonic-gate * all the standard kmem caches, read /etc/system, destroy all the
43907c478bd9Sstevel@tonic-gate * caches we just created, and then create them all again in light
43917c478bd9Sstevel@tonic-gate * of the (possibly) new kmem_flags and other kmem tunables.
43927c478bd9Sstevel@tonic-gate */
43937c478bd9Sstevel@tonic-gate kmem_cache_init(1, 0);
43947c478bd9Sstevel@tonic-gate
43957c478bd9Sstevel@tonic-gate mod_read_system_file(boothowto & RB_ASKNAME);
43967c478bd9Sstevel@tonic-gate
4397b5fca8f8Stomee while ((cp = list_tail(&kmem_caches)) != NULL)
43987c478bd9Sstevel@tonic-gate kmem_cache_destroy(cp);
43997c478bd9Sstevel@tonic-gate
44007c478bd9Sstevel@tonic-gate vmem_destroy(kmem_oversize_arena);
44017c478bd9Sstevel@tonic-gate
44027c478bd9Sstevel@tonic-gate if (old_kmem_flags & KMF_STICKY)
44037c478bd9Sstevel@tonic-gate kmem_flags = old_kmem_flags;
44047c478bd9Sstevel@tonic-gate
44057c478bd9Sstevel@tonic-gate if (!(kmem_flags & KMF_AUDIT))
44067c478bd9Sstevel@tonic-gate vmem_seg_size = offsetof(vmem_seg_t, vs_thread);
44077c478bd9Sstevel@tonic-gate
44087c478bd9Sstevel@tonic-gate if (kmem_maxverify == 0)
44097c478bd9Sstevel@tonic-gate kmem_maxverify = maxverify;
44107c478bd9Sstevel@tonic-gate
44117c478bd9Sstevel@tonic-gate if (kmem_minfirewall == 0)
44127c478bd9Sstevel@tonic-gate kmem_minfirewall = minfirewall;
44137c478bd9Sstevel@tonic-gate
44147c478bd9Sstevel@tonic-gate /*
44157c478bd9Sstevel@tonic-gate * give segkmem a chance to figure out if we are using large pages
44167c478bd9Sstevel@tonic-gate * for the kernel heap
44177c478bd9Sstevel@tonic-gate */
44187c478bd9Sstevel@tonic-gate use_large_pages = segkmem_lpsetup();
44197c478bd9Sstevel@tonic-gate
44207c478bd9Sstevel@tonic-gate /*
44217c478bd9Sstevel@tonic-gate * To protect against corruption, we keep the actual number of callers
44227c478bd9Sstevel@tonic-gate * KMF_LITE records seperate from the tunable. We arbitrarily clamp
44237c478bd9Sstevel@tonic-gate * to 16, since the overhead for small buffers quickly gets out of
44247c478bd9Sstevel@tonic-gate * hand.
44257c478bd9Sstevel@tonic-gate *
44267c478bd9Sstevel@tonic-gate * The real limit would depend on the needs of the largest KMC_NOHASH
44277c478bd9Sstevel@tonic-gate * cache.
44287c478bd9Sstevel@tonic-gate */
44297c478bd9Sstevel@tonic-gate kmem_lite_count = MIN(MAX(0, kmem_lite_pcs), 16);
44307c478bd9Sstevel@tonic-gate kmem_lite_pcs = kmem_lite_count;
44317c478bd9Sstevel@tonic-gate
44327c478bd9Sstevel@tonic-gate /*
44337c478bd9Sstevel@tonic-gate * Normally, we firewall oversized allocations when possible, but
44347c478bd9Sstevel@tonic-gate * if we are using large pages for kernel memory, and we don't have
44357c478bd9Sstevel@tonic-gate * any non-LITE debugging flags set, we want to allocate oversized
44367c478bd9Sstevel@tonic-gate * buffers from large pages, and so skip the firewalling.
44377c478bd9Sstevel@tonic-gate */
44387c478bd9Sstevel@tonic-gate if (use_large_pages &&
44397c478bd9Sstevel@tonic-gate ((kmem_flags & KMF_LITE) || !(kmem_flags & KMF_DEBUG))) {
44407c478bd9Sstevel@tonic-gate kmem_oversize_arena = vmem_xcreate("kmem_oversize", NULL, 0,
44417c478bd9Sstevel@tonic-gate PAGESIZE, segkmem_alloc_lp, segkmem_free_lp, heap_arena,
44429dd77bc8SDave Plauger 0, VMC_DUMPSAFE | VM_SLEEP);
44437c478bd9Sstevel@tonic-gate } else {
44447c478bd9Sstevel@tonic-gate kmem_oversize_arena = vmem_create("kmem_oversize",
44457c478bd9Sstevel@tonic-gate NULL, 0, PAGESIZE,
44467c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free, kmem_minfirewall < ULONG_MAX?
44479dd77bc8SDave Plauger kmem_firewall_va_arena : heap_arena, 0, VMC_DUMPSAFE |
44489dd77bc8SDave Plauger VM_SLEEP);
44497c478bd9Sstevel@tonic-gate }
44507c478bd9Sstevel@tonic-gate
44517c478bd9Sstevel@tonic-gate kmem_cache_init(2, use_large_pages);
44527c478bd9Sstevel@tonic-gate
44537c478bd9Sstevel@tonic-gate if (kmem_flags & (KMF_AUDIT | KMF_RANDOMIZE)) {
44547c478bd9Sstevel@tonic-gate if (kmem_transaction_log_size == 0)
44557c478bd9Sstevel@tonic-gate kmem_transaction_log_size = kmem_maxavail() / 50;
44567c478bd9Sstevel@tonic-gate kmem_transaction_log = kmem_log_init(kmem_transaction_log_size);
44577c478bd9Sstevel@tonic-gate }
44587c478bd9Sstevel@tonic-gate
44597c478bd9Sstevel@tonic-gate if (kmem_flags & (KMF_CONTENTS | KMF_RANDOMIZE)) {
44607c478bd9Sstevel@tonic-gate if (kmem_content_log_size == 0)
44617c478bd9Sstevel@tonic-gate kmem_content_log_size = kmem_maxavail() / 50;
44627c478bd9Sstevel@tonic-gate kmem_content_log = kmem_log_init(kmem_content_log_size);
44637c478bd9Sstevel@tonic-gate }
44647c478bd9Sstevel@tonic-gate
44657c478bd9Sstevel@tonic-gate kmem_failure_log = kmem_log_init(kmem_failure_log_size);
44667c478bd9Sstevel@tonic-gate kmem_slab_log = kmem_log_init(kmem_slab_log_size);
4467d1580181SBryan Cantrill kmem_zerosized_log = kmem_log_init(kmem_zerosized_log_size);
44687c478bd9Sstevel@tonic-gate
44697c478bd9Sstevel@tonic-gate /*
44707c478bd9Sstevel@tonic-gate * Initialize STREAMS message caches so allocb() is available.
44717c478bd9Sstevel@tonic-gate * This allows us to initialize the logging framework (cmn_err(9F),
44727c478bd9Sstevel@tonic-gate * strlog(9F), etc) so we can start recording messages.
44737c478bd9Sstevel@tonic-gate */
44747c478bd9Sstevel@tonic-gate streams_msg_init();
44757d692464Sdp
44767c478bd9Sstevel@tonic-gate /*
44777c478bd9Sstevel@tonic-gate * Initialize the ZSD framework in Zones so modules loaded henceforth
44787c478bd9Sstevel@tonic-gate * can register their callbacks.
44797c478bd9Sstevel@tonic-gate */
44807c478bd9Sstevel@tonic-gate zone_zsd_init();
4481f4b3ec61Sdh
44827c478bd9Sstevel@tonic-gate log_init();
44837c478bd9Sstevel@tonic-gate taskq_init();
44847c478bd9Sstevel@tonic-gate
44857d692464Sdp /*
44867d692464Sdp * Warn about invalid or dangerous values of kmem_flags.
44877d692464Sdp * Always warn about unsupported values.
44887d692464Sdp */
44897d692464Sdp if (((kmem_flags & ~(KMF_AUDIT | KMF_DEADBEEF | KMF_REDZONE |
44907d692464Sdp KMF_CONTENTS | KMF_LITE)) != 0) ||
44917d692464Sdp ((kmem_flags & KMF_LITE) && kmem_flags != KMF_LITE))
44922b91c4a5SJason King cmn_err(CE_WARN, "kmem_flags set to unsupported value 0x%x.",
44937d692464Sdp kmem_flags);
44947d692464Sdp
44957d692464Sdp #ifdef DEBUG
44967d692464Sdp if ((kmem_flags & KMF_DEBUG) == 0)
44977d692464Sdp cmn_err(CE_NOTE, "kmem debugging disabled.");
44987d692464Sdp #else
44997d692464Sdp /*
45007d692464Sdp * For non-debug kernels, the only "normal" flags are 0, KMF_LITE,
45017d692464Sdp * KMF_REDZONE, and KMF_CONTENTS (the last because it is only enabled
45027d692464Sdp * if KMF_AUDIT is set). We should warn the user about the performance
45037d692464Sdp * penalty of KMF_AUDIT or KMF_DEADBEEF if they are set and KMF_LITE
45047d692464Sdp * isn't set (since that disables AUDIT).
45057d692464Sdp */
45067d692464Sdp if (!(kmem_flags & KMF_LITE) &&
45077d692464Sdp (kmem_flags & (KMF_AUDIT | KMF_DEADBEEF)) != 0)
45087d692464Sdp cmn_err(CE_WARN, "High-overhead kmem debugging features "
45097d692464Sdp "enabled (kmem_flags = 0x%x). Performance degradation "
45102b91c4a5SJason King "and large memory overhead possible.", kmem_flags);
45117d692464Sdp #endif /* not DEBUG */
45127d692464Sdp
45137c478bd9Sstevel@tonic-gate kmem_cache_applyall(kmem_cache_magazine_enable, NULL, TQ_SLEEP);
45147c478bd9Sstevel@tonic-gate
45157c478bd9Sstevel@tonic-gate kmem_ready = 1;
45167c478bd9Sstevel@tonic-gate
45177c478bd9Sstevel@tonic-gate /*
45187c478bd9Sstevel@tonic-gate * Initialize the platform-specific aligned/DMA memory allocator.
45197c478bd9Sstevel@tonic-gate */
45207c478bd9Sstevel@tonic-gate ka_init();
45217c478bd9Sstevel@tonic-gate
45227c478bd9Sstevel@tonic-gate /*
45237c478bd9Sstevel@tonic-gate * Initialize 32-bit ID cache.
45247c478bd9Sstevel@tonic-gate */
45257c478bd9Sstevel@tonic-gate id32_init();
4526f4b3ec61Sdh
4527f4b3ec61Sdh /*
4528f4b3ec61Sdh * Initialize the networking stack so modules loaded can
4529f4b3ec61Sdh * register their callbacks.
4530f4b3ec61Sdh */
4531f4b3ec61Sdh netstack_init();
45327c478bd9Sstevel@tonic-gate }
45337c478bd9Sstevel@tonic-gate
4534b5fca8f8Stomee static void
kmem_move_init(void)4535b5fca8f8Stomee kmem_move_init(void)
4536b5fca8f8Stomee {
4537b5fca8f8Stomee kmem_defrag_cache = kmem_cache_create("kmem_defrag_cache",
4538b5fca8f8Stomee sizeof (kmem_defrag_t), 0, NULL, NULL, NULL, NULL,
4539b5fca8f8Stomee kmem_msb_arena, KMC_NOHASH);
4540b5fca8f8Stomee kmem_move_cache = kmem_cache_create("kmem_move_cache",
4541b5fca8f8Stomee sizeof (kmem_move_t), 0, NULL, NULL, NULL, NULL,
4542b5fca8f8Stomee kmem_msb_arena, KMC_NOHASH);
4543b5fca8f8Stomee
4544b5fca8f8Stomee /*
4545b5fca8f8Stomee * kmem guarantees that move callbacks are sequential and that even
4546b5fca8f8Stomee * across multiple caches no two moves ever execute simultaneously.
4547b5fca8f8Stomee * Move callbacks are processed on a separate taskq so that client code
4548b5fca8f8Stomee * does not interfere with internal maintenance tasks.
4549b5fca8f8Stomee */
4550b5fca8f8Stomee kmem_move_taskq = taskq_create_instance("kmem_move_taskq", 0, 1,
4551b5fca8f8Stomee minclsyspri, 100, INT_MAX, TASKQ_PREPOPULATE);
4552b5fca8f8Stomee }
4553b5fca8f8Stomee
45547c478bd9Sstevel@tonic-gate void
kmem_thread_init(void)45557c478bd9Sstevel@tonic-gate kmem_thread_init(void)
45567c478bd9Sstevel@tonic-gate {
4557b5fca8f8Stomee kmem_move_init();
4558baf00aa8SJoshua M. Clulow
4559baf00aa8SJoshua M. Clulow /*
4560baf00aa8SJoshua M. Clulow * This taskq is used for various kmem maintenance functions, including
4561baf00aa8SJoshua M. Clulow * kmem_reap(). When maintenance is required on every cache,
4562baf00aa8SJoshua M. Clulow * kmem_cache_applyall() dispatches one task per cache onto this queue.
4563baf00aa8SJoshua M. Clulow *
4564baf00aa8SJoshua M. Clulow * In the case of kmem_reap(), the system may be under increasingly
4565baf00aa8SJoshua M. Clulow * dire memory pressure and may not be able to allocate a new task
4566baf00aa8SJoshua M. Clulow * entry. The count of entries to prepopulate (below) should cover at
4567baf00aa8SJoshua M. Clulow * least as many caches as we generally expect to exist on the system
4568baf00aa8SJoshua M. Clulow * so that they may all be scheduled for reaping under those
4569baf00aa8SJoshua M. Clulow * conditions.
4570baf00aa8SJoshua M. Clulow */
45717c478bd9Sstevel@tonic-gate kmem_taskq = taskq_create_instance("kmem_taskq", 0, 1, minclsyspri,
4572baf00aa8SJoshua M. Clulow 600, INT_MAX, TASKQ_PREPOPULATE);
45737c478bd9Sstevel@tonic-gate }
45747c478bd9Sstevel@tonic-gate
45757c478bd9Sstevel@tonic-gate void
kmem_mp_init(void)45767c478bd9Sstevel@tonic-gate kmem_mp_init(void)
45777c478bd9Sstevel@tonic-gate {
45787c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock);
45797c478bd9Sstevel@tonic-gate register_cpu_setup_func(kmem_cpu_setup, NULL);
45807c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock);
45817c478bd9Sstevel@tonic-gate
45827c478bd9Sstevel@tonic-gate kmem_update_timeout(NULL);
45832e0c549eSJonathan Adams
45842e0c549eSJonathan Adams taskq_mp_init();
45857c478bd9Sstevel@tonic-gate }
4586b5fca8f8Stomee
4587b5fca8f8Stomee /*
4588b5fca8f8Stomee * Return the slab of the allocated buffer, or NULL if the buffer is not
4589b5fca8f8Stomee * allocated. This function may be called with a known slab address to determine
4590b5fca8f8Stomee * whether or not the buffer is allocated, or with a NULL slab address to obtain
4591b5fca8f8Stomee * an allocated buffer's slab.
4592b5fca8f8Stomee */
4593b5fca8f8Stomee static kmem_slab_t *
kmem_slab_allocated(kmem_cache_t * cp,kmem_slab_t * sp,void * buf)4594b5fca8f8Stomee kmem_slab_allocated(kmem_cache_t *cp, kmem_slab_t *sp, void *buf)
4595b5fca8f8Stomee {
4596b5fca8f8Stomee kmem_bufctl_t *bcp, *bufbcp;
4597b5fca8f8Stomee
4598b5fca8f8Stomee ASSERT(MUTEX_HELD(&cp->cache_lock));
4599b5fca8f8Stomee ASSERT(sp == NULL || KMEM_SLAB_MEMBER(sp, buf));
4600b5fca8f8Stomee
4601b5fca8f8Stomee if (cp->cache_flags & KMF_HASH) {
4602b5fca8f8Stomee for (bcp = *KMEM_HASH(cp, buf);
4603b5fca8f8Stomee (bcp != NULL) && (bcp->bc_addr != buf);
4604b5fca8f8Stomee bcp = bcp->bc_next) {
4605b5fca8f8Stomee continue;
4606b5fca8f8Stomee }
4607b5fca8f8Stomee ASSERT(sp != NULL && bcp != NULL ? sp == bcp->bc_slab : 1);
4608b5fca8f8Stomee return (bcp == NULL ? NULL : bcp->bc_slab);
4609b5fca8f8Stomee }
4610b5fca8f8Stomee
4611b5fca8f8Stomee if (sp == NULL) {
4612b5fca8f8Stomee sp = KMEM_SLAB(cp, buf);
4613b5fca8f8Stomee }
4614b5fca8f8Stomee bufbcp = KMEM_BUFCTL(cp, buf);
4615b5fca8f8Stomee for (bcp = sp->slab_head;
4616b5fca8f8Stomee (bcp != NULL) && (bcp != bufbcp);
4617b5fca8f8Stomee bcp = bcp->bc_next) {
4618b5fca8f8Stomee continue;
4619b5fca8f8Stomee }
4620b5fca8f8Stomee return (bcp == NULL ? sp : NULL);
4621b5fca8f8Stomee }
4622b5fca8f8Stomee
4623b5fca8f8Stomee static boolean_t
kmem_slab_is_reclaimable(kmem_cache_t * cp,kmem_slab_t * sp,int flags)4624b5fca8f8Stomee kmem_slab_is_reclaimable(kmem_cache_t *cp, kmem_slab_t *sp, int flags)
4625b5fca8f8Stomee {
4626686031edSTom Erickson long refcnt = sp->slab_refcnt;
4627b5fca8f8Stomee
4628b5fca8f8Stomee ASSERT(cp->cache_defrag != NULL);
4629b5fca8f8Stomee
4630686031edSTom Erickson /*
4631686031edSTom Erickson * For code coverage we want to be able to move an object within the
4632686031edSTom Erickson * same slab (the only partial slab) even if allocating the destination
4633686031edSTom Erickson * buffer resulted in a completely allocated slab.
4634686031edSTom Erickson */
4635686031edSTom Erickson if (flags & KMM_DEBUG) {
4636686031edSTom Erickson return ((flags & KMM_DESPERATE) ||
4637686031edSTom Erickson ((sp->slab_flags & KMEM_SLAB_NOMOVE) == 0));
4638686031edSTom Erickson }
4639686031edSTom Erickson
4640b5fca8f8Stomee /* If we're desperate, we don't care if the client said NO. */
4641b5fca8f8Stomee if (flags & KMM_DESPERATE) {
4642b5fca8f8Stomee return (refcnt < sp->slab_chunks); /* any partial */
4643b5fca8f8Stomee }
4644b5fca8f8Stomee
4645b5fca8f8Stomee if (sp->slab_flags & KMEM_SLAB_NOMOVE) {
4646b5fca8f8Stomee return (B_FALSE);
4647b5fca8f8Stomee }
4648b5fca8f8Stomee
4649686031edSTom Erickson if ((refcnt == 1) || kmem_move_any_partial) {
4650b5fca8f8Stomee return (refcnt < sp->slab_chunks);
4651b5fca8f8Stomee }
4652b5fca8f8Stomee
4653b5fca8f8Stomee /*
4654b5fca8f8Stomee * The reclaim threshold is adjusted at each kmem_cache_scan() so that
4655b5fca8f8Stomee * slabs with a progressively higher percentage of used buffers can be
4656b5fca8f8Stomee * reclaimed until the cache as a whole is no longer fragmented.
4657b5fca8f8Stomee *
4658b5fca8f8Stomee * sp->slab_refcnt kmd_reclaim_numer
4659b5fca8f8Stomee * --------------- < ------------------
4660b5fca8f8Stomee * sp->slab_chunks KMEM_VOID_FRACTION
4661b5fca8f8Stomee */
4662b5fca8f8Stomee return ((refcnt * KMEM_VOID_FRACTION) <
4663b5fca8f8Stomee (sp->slab_chunks * cp->cache_defrag->kmd_reclaim_numer));
4664b5fca8f8Stomee }
4665b5fca8f8Stomee
4666b5fca8f8Stomee /*
4667b5fca8f8Stomee * May be called from the kmem_move_taskq, from kmem_cache_move_notify_task(),
4668b5fca8f8Stomee * or when the buffer is freed.
4669b5fca8f8Stomee */
4670b5fca8f8Stomee static void
kmem_slab_move_yes(kmem_cache_t * cp,kmem_slab_t * sp,void * from_buf)4671b5fca8f8Stomee kmem_slab_move_yes(kmem_cache_t *cp, kmem_slab_t *sp, void *from_buf)
4672b5fca8f8Stomee {
4673b5fca8f8Stomee ASSERT(MUTEX_HELD(&cp->cache_lock));
4674b5fca8f8Stomee ASSERT(KMEM_SLAB_MEMBER(sp, from_buf));
4675b5fca8f8Stomee
4676b5fca8f8Stomee if (!KMEM_SLAB_IS_PARTIAL(sp)) {
4677b5fca8f8Stomee return;
4678b5fca8f8Stomee }
4679b5fca8f8Stomee
4680b5fca8f8Stomee if (sp->slab_flags & KMEM_SLAB_NOMOVE) {
4681b5fca8f8Stomee if (KMEM_SLAB_OFFSET(sp, from_buf) == sp->slab_stuck_offset) {
4682b5fca8f8Stomee avl_remove(&cp->cache_partial_slabs, sp);
4683b5fca8f8Stomee sp->slab_flags &= ~KMEM_SLAB_NOMOVE;
4684b5fca8f8Stomee sp->slab_stuck_offset = (uint32_t)-1;
4685b5fca8f8Stomee avl_add(&cp->cache_partial_slabs, sp);
4686b5fca8f8Stomee }
4687b5fca8f8Stomee } else {
4688b5fca8f8Stomee sp->slab_later_count = 0;
4689b5fca8f8Stomee sp->slab_stuck_offset = (uint32_t)-1;
4690b5fca8f8Stomee }
4691b5fca8f8Stomee }
4692b5fca8f8Stomee
4693b5fca8f8Stomee static void
kmem_slab_move_no(kmem_cache_t * cp,kmem_slab_t * sp,void * from_buf)4694b5fca8f8Stomee kmem_slab_move_no(kmem_cache_t *cp, kmem_slab_t *sp, void *from_buf)
4695b5fca8f8Stomee {
4696b5fca8f8Stomee ASSERT(taskq_member(kmem_move_taskq, curthread));
4697b5fca8f8Stomee ASSERT(MUTEX_HELD(&cp->cache_lock));
4698b5fca8f8Stomee ASSERT(KMEM_SLAB_MEMBER(sp, from_buf));
4699b5fca8f8Stomee
4700b5fca8f8Stomee if (!KMEM_SLAB_IS_PARTIAL(sp)) {
4701b5fca8f8Stomee return;
4702b5fca8f8Stomee }
4703b5fca8f8Stomee
4704b5fca8f8Stomee avl_remove(&cp->cache_partial_slabs, sp);
4705b5fca8f8Stomee sp->slab_later_count = 0;
4706b5fca8f8Stomee sp->slab_flags |= KMEM_SLAB_NOMOVE;
4707b5fca8f8Stomee sp->slab_stuck_offset = KMEM_SLAB_OFFSET(sp, from_buf);
4708b5fca8f8Stomee avl_add(&cp->cache_partial_slabs, sp);
4709b5fca8f8Stomee }
4710b5fca8f8Stomee
4711b5fca8f8Stomee static void kmem_move_end(kmem_cache_t *, kmem_move_t *);
4712b5fca8f8Stomee
4713b5fca8f8Stomee /*
4714b5fca8f8Stomee * The move callback takes two buffer addresses, the buffer to be moved, and a
4715b5fca8f8Stomee * newly allocated and constructed buffer selected by kmem as the destination.
4716b5fca8f8Stomee * It also takes the size of the buffer and an optional user argument specified
4717b5fca8f8Stomee * at cache creation time. kmem guarantees that the buffer to be moved has not
4718b5fca8f8Stomee * been unmapped by the virtual memory subsystem. Beyond that, it cannot
4719b5fca8f8Stomee * guarantee the present whereabouts of the buffer to be moved, so it is up to
4720b5fca8f8Stomee * the client to safely determine whether or not it is still using the buffer.
4721b5fca8f8Stomee * The client must not free either of the buffers passed to the move callback,
4722b5fca8f8Stomee * since kmem wants to free them directly to the slab layer. The client response
4723b5fca8f8Stomee * tells kmem which of the two buffers to free:
4724b5fca8f8Stomee *
4725b5fca8f8Stomee * YES kmem frees the old buffer (the move was successful)
4726b5fca8f8Stomee * NO kmem frees the new buffer, marks the slab of the old buffer
4727b5fca8f8Stomee * non-reclaimable to avoid bothering the client again
4728b5fca8f8Stomee * LATER kmem frees the new buffer, increments slab_later_count
4729d7db73d1SBryan Cantrill * DONT_KNOW kmem frees the new buffer
4730b5fca8f8Stomee * DONT_NEED kmem frees both the old buffer and the new buffer
4731b5fca8f8Stomee *
4732b5fca8f8Stomee * The pending callback argument now being processed contains both of the
4733b5fca8f8Stomee * buffers (old and new) passed to the move callback function, the slab of the
4734b5fca8f8Stomee * old buffer, and flags related to the move request, such as whether or not the
4735b5fca8f8Stomee * system was desperate for memory.
4736686031edSTom Erickson *
4737686031edSTom Erickson * Slabs are not freed while there is a pending callback, but instead are kept
4738686031edSTom Erickson * on a deadlist, which is drained after the last callback completes. This means
4739686031edSTom Erickson * that slabs are safe to access until kmem_move_end(), no matter how many of
4740686031edSTom Erickson * their buffers have been freed. Once slab_refcnt reaches zero, it stays at
4741686031edSTom Erickson * zero for as long as the slab remains on the deadlist and until the slab is
4742686031edSTom Erickson * freed.
4743b5fca8f8Stomee */
4744b5fca8f8Stomee static void
kmem_move_buffer(kmem_move_t * callback)4745b5fca8f8Stomee kmem_move_buffer(kmem_move_t *callback)
4746b5fca8f8Stomee {
4747b5fca8f8Stomee kmem_cbrc_t response;
4748b5fca8f8Stomee kmem_slab_t *sp = callback->kmm_from_slab;
4749b5fca8f8Stomee kmem_cache_t *cp = sp->slab_cache;
4750b5fca8f8Stomee boolean_t free_on_slab;
4751b5fca8f8Stomee
4752b5fca8f8Stomee ASSERT(taskq_member(kmem_move_taskq, curthread));
4753b5fca8f8Stomee ASSERT(MUTEX_NOT_HELD(&cp->cache_lock));
4754b5fca8f8Stomee ASSERT(KMEM_SLAB_MEMBER(sp, callback->kmm_from_buf));
4755b5fca8f8Stomee
4756b5fca8f8Stomee /*
4757b5fca8f8Stomee * The number of allocated buffers on the slab may have changed since we
4758b5fca8f8Stomee * last checked the slab's reclaimability (when the pending move was
4759b5fca8f8Stomee * enqueued), or the client may have responded NO when asked to move
4760b5fca8f8Stomee * another buffer on the same slab.
4761b5fca8f8Stomee */
4762b5fca8f8Stomee if (!kmem_slab_is_reclaimable(cp, sp, callback->kmm_flags)) {
4763b5fca8f8Stomee kmem_slab_free(cp, callback->kmm_to_buf);
4764b5fca8f8Stomee kmem_move_end(cp, callback);
4765b5fca8f8Stomee return;
4766b5fca8f8Stomee }
4767b5fca8f8Stomee
4768b5fca8f8Stomee /*
4769d7db73d1SBryan Cantrill * Checking the slab layer is easy, so we might as well do that here
4770d7db73d1SBryan Cantrill * in case we can avoid bothering the client.
4771b5fca8f8Stomee */
4772b5fca8f8Stomee mutex_enter(&cp->cache_lock);
4773b5fca8f8Stomee free_on_slab = (kmem_slab_allocated(cp, sp,
4774b5fca8f8Stomee callback->kmm_from_buf) == NULL);
4775b5fca8f8Stomee mutex_exit(&cp->cache_lock);
4776b5fca8f8Stomee
4777b5fca8f8Stomee if (free_on_slab) {
4778b5fca8f8Stomee kmem_slab_free(cp, callback->kmm_to_buf);
4779b5fca8f8Stomee kmem_move_end(cp, callback);
4780b5fca8f8Stomee return;
4781b5fca8f8Stomee }
4782b5fca8f8Stomee
4783b5fca8f8Stomee if (cp->cache_flags & KMF_BUFTAG) {
4784b5fca8f8Stomee /*
4785b5fca8f8Stomee * Make kmem_cache_alloc_debug() apply the constructor for us.
4786b5fca8f8Stomee */
4787b5fca8f8Stomee if (kmem_cache_alloc_debug(cp, callback->kmm_to_buf,
4788b5fca8f8Stomee KM_NOSLEEP, 1, caller()) != 0) {
4789b5fca8f8Stomee kmem_move_end(cp, callback);
4790b5fca8f8Stomee return;
4791b5fca8f8Stomee }
4792b5fca8f8Stomee } else if (cp->cache_constructor != NULL &&
4793b5fca8f8Stomee cp->cache_constructor(callback->kmm_to_buf, cp->cache_private,
4794b5fca8f8Stomee KM_NOSLEEP) != 0) {
47951a5e258fSJosef 'Jeff' Sipek atomic_inc_64(&cp->cache_alloc_fail);
4796b5fca8f8Stomee kmem_slab_free(cp, callback->kmm_to_buf);
4797b5fca8f8Stomee kmem_move_end(cp, callback);
4798b5fca8f8Stomee return;
4799b5fca8f8Stomee }
4800b5fca8f8Stomee
4801b5fca8f8Stomee cp->cache_defrag->kmd_callbacks++;
4802b5fca8f8Stomee cp->cache_defrag->kmd_thread = curthread;
4803b5fca8f8Stomee cp->cache_defrag->kmd_from_buf = callback->kmm_from_buf;
4804b5fca8f8Stomee cp->cache_defrag->kmd_to_buf = callback->kmm_to_buf;
4805b5fca8f8Stomee DTRACE_PROBE2(kmem__move__start, kmem_cache_t *, cp, kmem_move_t *,
4806b5fca8f8Stomee callback);
4807b5fca8f8Stomee
4808b5fca8f8Stomee response = cp->cache_move(callback->kmm_from_buf,
4809b5fca8f8Stomee callback->kmm_to_buf, cp->cache_bufsize, cp->cache_private);
4810b5fca8f8Stomee
4811b5fca8f8Stomee DTRACE_PROBE3(kmem__move__end, kmem_cache_t *, cp, kmem_move_t *,
4812b5fca8f8Stomee callback, kmem_cbrc_t, response);
4813b5fca8f8Stomee cp->cache_defrag->kmd_thread = NULL;
4814b5fca8f8Stomee cp->cache_defrag->kmd_from_buf = NULL;
4815b5fca8f8Stomee cp->cache_defrag->kmd_to_buf = NULL;
4816b5fca8f8Stomee
4817b5fca8f8Stomee if (response == KMEM_CBRC_YES) {
4818b5fca8f8Stomee cp->cache_defrag->kmd_yes++;
4819b5fca8f8Stomee kmem_slab_free_constructed(cp, callback->kmm_from_buf, B_FALSE);
4820686031edSTom Erickson /* slab safe to access until kmem_move_end() */
4821686031edSTom Erickson if (sp->slab_refcnt == 0)
4822686031edSTom Erickson cp->cache_defrag->kmd_slabs_freed++;
4823b5fca8f8Stomee mutex_enter(&cp->cache_lock);
4824b5fca8f8Stomee kmem_slab_move_yes(cp, sp, callback->kmm_from_buf);
4825b5fca8f8Stomee mutex_exit(&cp->cache_lock);
4826b5fca8f8Stomee kmem_move_end(cp, callback);
4827b5fca8f8Stomee return;
4828b5fca8f8Stomee }
4829b5fca8f8Stomee
4830b5fca8f8Stomee switch (response) {
4831b5fca8f8Stomee case KMEM_CBRC_NO:
4832b5fca8f8Stomee cp->cache_defrag->kmd_no++;
4833b5fca8f8Stomee mutex_enter(&cp->cache_lock);
4834b5fca8f8Stomee kmem_slab_move_no(cp, sp, callback->kmm_from_buf);
4835b5fca8f8Stomee mutex_exit(&cp->cache_lock);
4836b5fca8f8Stomee break;
4837b5fca8f8Stomee case KMEM_CBRC_LATER:
4838b5fca8f8Stomee cp->cache_defrag->kmd_later++;
4839b5fca8f8Stomee mutex_enter(&cp->cache_lock);
4840b5fca8f8Stomee if (!KMEM_SLAB_IS_PARTIAL(sp)) {
4841b5fca8f8Stomee mutex_exit(&cp->cache_lock);
4842b5fca8f8Stomee break;
4843b5fca8f8Stomee }
4844b5fca8f8Stomee
4845b5fca8f8Stomee if (++sp->slab_later_count >= KMEM_DISBELIEF) {
4846b5fca8f8Stomee kmem_slab_move_no(cp, sp, callback->kmm_from_buf);
4847b5fca8f8Stomee } else if (!(sp->slab_flags & KMEM_SLAB_NOMOVE)) {
4848b5fca8f8Stomee sp->slab_stuck_offset = KMEM_SLAB_OFFSET(sp,
4849b5fca8f8Stomee callback->kmm_from_buf);
4850b5fca8f8Stomee }
4851b5fca8f8Stomee mutex_exit(&cp->cache_lock);
4852b5fca8f8Stomee break;
4853b5fca8f8Stomee case KMEM_CBRC_DONT_NEED:
4854b5fca8f8Stomee cp->cache_defrag->kmd_dont_need++;
4855b5fca8f8Stomee kmem_slab_free_constructed(cp, callback->kmm_from_buf, B_FALSE);
4856686031edSTom Erickson if (sp->slab_refcnt == 0)
4857686031edSTom Erickson cp->cache_defrag->kmd_slabs_freed++;
4858b5fca8f8Stomee mutex_enter(&cp->cache_lock);
4859b5fca8f8Stomee kmem_slab_move_yes(cp, sp, callback->kmm_from_buf);
4860b5fca8f8Stomee mutex_exit(&cp->cache_lock);
4861b5fca8f8Stomee break;
4862b5fca8f8Stomee case KMEM_CBRC_DONT_KNOW:
4863d7db73d1SBryan Cantrill /*
4864d7db73d1SBryan Cantrill * If we don't know if we can move this buffer or not, we'll
4865d7db73d1SBryan Cantrill * just assume that we can't: if the buffer is in fact free,
4866d7db73d1SBryan Cantrill * then it is sitting in one of the per-CPU magazines or in
4867d7db73d1SBryan Cantrill * a full magazine in the depot layer. Either way, because
4868d7db73d1SBryan Cantrill * defrag is induced in the same logic that reaps a cache,
4869d7db73d1SBryan Cantrill * it's likely that full magazines will be returned to the
4870d7db73d1SBryan Cantrill * system soon (thereby accomplishing what we're trying to
4871d7db73d1SBryan Cantrill * accomplish here: return those magazines to their slabs).
4872d7db73d1SBryan Cantrill * Given this, any work that we might do now to locate a buffer
4873d7db73d1SBryan Cantrill * in a magazine is wasted (and expensive!) work; we bump
4874d7db73d1SBryan Cantrill * a counter in this case and otherwise assume that we can't
4875d7db73d1SBryan Cantrill * move it.
4876d7db73d1SBryan Cantrill */
4877b5fca8f8Stomee cp->cache_defrag->kmd_dont_know++;
4878b5fca8f8Stomee break;
4879b5fca8f8Stomee default:
4880b5fca8f8Stomee panic("'%s' (%p) unexpected move callback response %d\n",
4881b5fca8f8Stomee cp->cache_name, (void *)cp, response);
4882b5fca8f8Stomee }
4883b5fca8f8Stomee
4884b5fca8f8Stomee kmem_slab_free_constructed(cp, callback->kmm_to_buf, B_FALSE);
4885b5fca8f8Stomee kmem_move_end(cp, callback);
4886b5fca8f8Stomee }
4887b5fca8f8Stomee
4888b5fca8f8Stomee /* Return B_FALSE if there is insufficient memory for the move request. */
4889b5fca8f8Stomee static boolean_t
kmem_move_begin(kmem_cache_t * cp,kmem_slab_t * sp,void * buf,int flags)4890b5fca8f8Stomee kmem_move_begin(kmem_cache_t *cp, kmem_slab_t *sp, void *buf, int flags)
4891b5fca8f8Stomee {
4892b5fca8f8Stomee void *to_buf;
4893b5fca8f8Stomee avl_index_t index;
4894b5fca8f8Stomee kmem_move_t *callback, *pending;
4895686031edSTom Erickson ulong_t n;
4896b5fca8f8Stomee
4897b5fca8f8Stomee ASSERT(taskq_member(kmem_taskq, curthread));
4898b5fca8f8Stomee ASSERT(MUTEX_NOT_HELD(&cp->cache_lock));
4899b5fca8f8Stomee ASSERT(sp->slab_flags & KMEM_SLAB_MOVE_PENDING);
4900b5fca8f8Stomee
4901b5fca8f8Stomee callback = kmem_cache_alloc(kmem_move_cache, KM_NOSLEEP);
4902d7db73d1SBryan Cantrill
4903d7db73d1SBryan Cantrill if (callback == NULL)
4904b5fca8f8Stomee return (B_FALSE);
4905b5fca8f8Stomee
4906b5fca8f8Stomee callback->kmm_from_slab = sp;
4907b5fca8f8Stomee callback->kmm_from_buf = buf;
4908b5fca8f8Stomee callback->kmm_flags = flags;
4909b5fca8f8Stomee
4910b5fca8f8Stomee mutex_enter(&cp->cache_lock);
4911b5fca8f8Stomee
4912686031edSTom Erickson n = avl_numnodes(&cp->cache_partial_slabs);
4913686031edSTom Erickson if ((n == 0) || ((n == 1) && !(flags & KMM_DEBUG))) {
4914b5fca8f8Stomee mutex_exit(&cp->cache_lock);
4915b5fca8f8Stomee kmem_cache_free(kmem_move_cache, callback);
4916b5fca8f8Stomee return (B_TRUE); /* there is no need for the move request */
4917b5fca8f8Stomee }
4918b5fca8f8Stomee
4919b5fca8f8Stomee pending = avl_find(&cp->cache_defrag->kmd_moves_pending, buf, &index);
4920b5fca8f8Stomee if (pending != NULL) {
4921b5fca8f8Stomee /*
4922b5fca8f8Stomee * If the move is already pending and we're desperate now,
4923b5fca8f8Stomee * update the move flags.
4924b5fca8f8Stomee */
4925b5fca8f8Stomee if (flags & KMM_DESPERATE) {
4926b5fca8f8Stomee pending->kmm_flags |= KMM_DESPERATE;
4927b5fca8f8Stomee }
4928b5fca8f8Stomee mutex_exit(&cp->cache_lock);
4929b5fca8f8Stomee kmem_cache_free(kmem_move_cache, callback);
4930b5fca8f8Stomee return (B_TRUE);
4931b5fca8f8Stomee }
4932b5fca8f8Stomee
4933b942e89bSDavid Valin to_buf = kmem_slab_alloc_impl(cp, avl_first(&cp->cache_partial_slabs),
4934b942e89bSDavid Valin B_FALSE);
4935b5fca8f8Stomee callback->kmm_to_buf = to_buf;
4936b5fca8f8Stomee avl_insert(&cp->cache_defrag->kmd_moves_pending, callback, index);
4937b5fca8f8Stomee
4938b5fca8f8Stomee mutex_exit(&cp->cache_lock);
4939b5fca8f8Stomee
4940fc8ae2ecSToomas Soome if (taskq_dispatch(kmem_move_taskq, (task_func_t *)kmem_move_buffer,
4941fc8ae2ecSToomas Soome callback, TQ_NOSLEEP) == TASKQID_INVALID) {
4942b5fca8f8Stomee mutex_enter(&cp->cache_lock);
4943b5fca8f8Stomee avl_remove(&cp->cache_defrag->kmd_moves_pending, callback);
4944b5fca8f8Stomee mutex_exit(&cp->cache_lock);
494525e2c9cfStomee kmem_slab_free(cp, to_buf);
4946b5fca8f8Stomee kmem_cache_free(kmem_move_cache, callback);
4947b5fca8f8Stomee return (B_FALSE);
4948b5fca8f8Stomee }
4949b5fca8f8Stomee
4950b5fca8f8Stomee return (B_TRUE);
4951b5fca8f8Stomee }
4952b5fca8f8Stomee
4953b5fca8f8Stomee static void
kmem_move_end(kmem_cache_t * cp,kmem_move_t * callback)4954b5fca8f8Stomee kmem_move_end(kmem_cache_t *cp, kmem_move_t *callback)
4955b5fca8f8Stomee {
4956b5fca8f8Stomee avl_index_t index;
4957b5fca8f8Stomee
4958b5fca8f8Stomee ASSERT(cp->cache_defrag != NULL);
4959b5fca8f8Stomee ASSERT(taskq_member(kmem_move_taskq, curthread));
4960b5fca8f8Stomee ASSERT(MUTEX_NOT_HELD(&cp->cache_lock));
4961b5fca8f8Stomee
4962b5fca8f8Stomee mutex_enter(&cp->cache_lock);
4963b5fca8f8Stomee VERIFY(avl_find(&cp->cache_defrag->kmd_moves_pending,
4964b5fca8f8Stomee callback->kmm_from_buf, &index) != NULL);
4965b5fca8f8Stomee avl_remove(&cp->cache_defrag->kmd_moves_pending, callback);
4966b5fca8f8Stomee if (avl_is_empty(&cp->cache_defrag->kmd_moves_pending)) {
4967b5fca8f8Stomee list_t *deadlist = &cp->cache_defrag->kmd_deadlist;
4968b5fca8f8Stomee kmem_slab_t *sp;
4969b5fca8f8Stomee
4970b5fca8f8Stomee /*
4971b5fca8f8Stomee * The last pending move completed. Release all slabs from the
4972b5fca8f8Stomee * front of the dead list except for any slab at the tail that
4973b5fca8f8Stomee * needs to be released from the context of kmem_move_buffers().
4974b5fca8f8Stomee * kmem deferred unmapping the buffers on these slabs in order
4975b5fca8f8Stomee * to guarantee that buffers passed to the move callback have
4976b5fca8f8Stomee * been touched only by kmem or by the client itself.
4977b5fca8f8Stomee */
4978b5fca8f8Stomee while ((sp = list_remove_head(deadlist)) != NULL) {
4979b5fca8f8Stomee if (sp->slab_flags & KMEM_SLAB_MOVE_PENDING) {
4980b5fca8f8Stomee list_insert_tail(deadlist, sp);
4981b5fca8f8Stomee break;
4982b5fca8f8Stomee }
4983b5fca8f8Stomee cp->cache_defrag->kmd_deadcount--;
4984b5fca8f8Stomee cp->cache_slab_destroy++;
4985b5fca8f8Stomee mutex_exit(&cp->cache_lock);
4986b5fca8f8Stomee kmem_slab_destroy(cp, sp);
4987b5fca8f8Stomee mutex_enter(&cp->cache_lock);
4988b5fca8f8Stomee }
4989b5fca8f8Stomee }
4990b5fca8f8Stomee mutex_exit(&cp->cache_lock);
4991b5fca8f8Stomee kmem_cache_free(kmem_move_cache, callback);
4992b5fca8f8Stomee }
4993b5fca8f8Stomee
4994b5fca8f8Stomee /*
4995b5fca8f8Stomee * Move buffers from least used slabs first by scanning backwards from the end
4996b5fca8f8Stomee * of the partial slab list. Scan at most max_scan candidate slabs and move
4997b5fca8f8Stomee * buffers from at most max_slabs slabs (0 for all partial slabs in both cases).
4998b5fca8f8Stomee * If desperate to reclaim memory, move buffers from any partial slab, otherwise
4999b5fca8f8Stomee * skip slabs with a ratio of allocated buffers at or above the current
5000b5fca8f8Stomee * threshold. Return the number of unskipped slabs (at most max_slabs, -1 if the
5001b5fca8f8Stomee * scan is aborted) so that the caller can adjust the reclaimability threshold
5002b5fca8f8Stomee * depending on how many reclaimable slabs it finds.
5003b5fca8f8Stomee *
5004b5fca8f8Stomee * kmem_move_buffers() drops and reacquires cache_lock every time it issues a
5005b5fca8f8Stomee * move request, since it is not valid for kmem_move_begin() to call
5006b5fca8f8Stomee * kmem_cache_alloc() or taskq_dispatch() with cache_lock held.
5007b5fca8f8Stomee */
5008b5fca8f8Stomee static int
kmem_move_buffers(kmem_cache_t * cp,size_t max_scan,size_t max_slabs,int flags)5009b5fca8f8Stomee kmem_move_buffers(kmem_cache_t *cp, size_t max_scan, size_t max_slabs,
5010b5fca8f8Stomee int flags)
5011b5fca8f8Stomee {
5012b5fca8f8Stomee kmem_slab_t *sp;
5013b5fca8f8Stomee void *buf;
5014b5fca8f8Stomee int i, j; /* slab index, buffer index */
5015b5fca8f8Stomee int s; /* reclaimable slabs */
5016b5fca8f8Stomee int b; /* allocated (movable) buffers on reclaimable slab */
5017b5fca8f8Stomee boolean_t success;
5018b5fca8f8Stomee int refcnt;
5019b5fca8f8Stomee int nomove;
5020b5fca8f8Stomee
5021b5fca8f8Stomee ASSERT(taskq_member(kmem_taskq, curthread));
5022b5fca8f8Stomee ASSERT(MUTEX_HELD(&cp->cache_lock));
5023b5fca8f8Stomee ASSERT(kmem_move_cache != NULL);
5024b5fca8f8Stomee ASSERT(cp->cache_move != NULL && cp->cache_defrag != NULL);
5025686031edSTom Erickson ASSERT((flags & KMM_DEBUG) ? !avl_is_empty(&cp->cache_partial_slabs) :
5026686031edSTom Erickson avl_numnodes(&cp->cache_partial_slabs) > 1);
5027b5fca8f8Stomee
5028b5fca8f8Stomee if (kmem_move_blocked) {
5029b5fca8f8Stomee return (0);
5030b5fca8f8Stomee }
5031b5fca8f8Stomee
5032b5fca8f8Stomee if (kmem_move_fulltilt) {
5033b5fca8f8Stomee flags |= KMM_DESPERATE;
5034b5fca8f8Stomee }
5035b5fca8f8Stomee
5036b5fca8f8Stomee if (max_scan == 0 || (flags & KMM_DESPERATE)) {
5037b5fca8f8Stomee /*
5038b5fca8f8Stomee * Scan as many slabs as needed to find the desired number of
5039b5fca8f8Stomee * candidate slabs.
5040b5fca8f8Stomee */
5041b5fca8f8Stomee max_scan = (size_t)-1;
5042b5fca8f8Stomee }
5043b5fca8f8Stomee
5044b5fca8f8Stomee if (max_slabs == 0 || (flags & KMM_DESPERATE)) {
5045b5fca8f8Stomee /* Find as many candidate slabs as possible. */
5046b5fca8f8Stomee max_slabs = (size_t)-1;
5047b5fca8f8Stomee }
5048b5fca8f8Stomee
5049b5fca8f8Stomee sp = avl_last(&cp->cache_partial_slabs);
5050686031edSTom Erickson ASSERT(KMEM_SLAB_IS_PARTIAL(sp));
5051686031edSTom Erickson for (i = 0, s = 0; (i < max_scan) && (s < max_slabs) && (sp != NULL) &&
5052686031edSTom Erickson ((sp != avl_first(&cp->cache_partial_slabs)) ||
5053686031edSTom Erickson (flags & KMM_DEBUG));
5054b5fca8f8Stomee sp = AVL_PREV(&cp->cache_partial_slabs, sp), i++) {
5055b5fca8f8Stomee
5056b5fca8f8Stomee if (!kmem_slab_is_reclaimable(cp, sp, flags)) {
5057b5fca8f8Stomee continue;
5058b5fca8f8Stomee }
5059b5fca8f8Stomee s++;
5060b5fca8f8Stomee
5061b5fca8f8Stomee /* Look for allocated buffers to move. */
5062b5fca8f8Stomee for (j = 0, b = 0, buf = sp->slab_base;
5063b5fca8f8Stomee (j < sp->slab_chunks) && (b < sp->slab_refcnt);
5064b5fca8f8Stomee buf = (((char *)buf) + cp->cache_chunksize), j++) {
5065b5fca8f8Stomee
5066b5fca8f8Stomee if (kmem_slab_allocated(cp, sp, buf) == NULL) {
5067b5fca8f8Stomee continue;
5068b5fca8f8Stomee }
5069b5fca8f8Stomee
5070b5fca8f8Stomee b++;
5071b5fca8f8Stomee
5072b5fca8f8Stomee /*
5073b5fca8f8Stomee * Prevent the slab from being destroyed while we drop
5074b5fca8f8Stomee * cache_lock and while the pending move is not yet
5075b5fca8f8Stomee * registered. Flag the pending move while
5076b5fca8f8Stomee * kmd_moves_pending may still be empty, since we can't
5077b5fca8f8Stomee * yet rely on a non-zero pending move count to prevent
5078b5fca8f8Stomee * the slab from being destroyed.
5079b5fca8f8Stomee */
5080b5fca8f8Stomee ASSERT(!(sp->slab_flags & KMEM_SLAB_MOVE_PENDING));
5081b5fca8f8Stomee sp->slab_flags |= KMEM_SLAB_MOVE_PENDING;
5082b5fca8f8Stomee /*
5083b5fca8f8Stomee * Recheck refcnt and nomove after reacquiring the lock,
5084b5fca8f8Stomee * since these control the order of partial slabs, and
5085b5fca8f8Stomee * we want to know if we can pick up the scan where we
5086b5fca8f8Stomee * left off.
5087b5fca8f8Stomee */
5088b5fca8f8Stomee refcnt = sp->slab_refcnt;
5089b5fca8f8Stomee nomove = (sp->slab_flags & KMEM_SLAB_NOMOVE);
5090b5fca8f8Stomee mutex_exit(&cp->cache_lock);
5091b5fca8f8Stomee
5092b5fca8f8Stomee success = kmem_move_begin(cp, sp, buf, flags);
5093b5fca8f8Stomee
5094b5fca8f8Stomee /*
5095b5fca8f8Stomee * Now, before the lock is reacquired, kmem could
5096b5fca8f8Stomee * process all pending move requests and purge the
5097b5fca8f8Stomee * deadlist, so that upon reacquiring the lock, sp has
5098686031edSTom Erickson * been remapped. Or, the client may free all the
5099686031edSTom Erickson * objects on the slab while the pending moves are still
5100686031edSTom Erickson * on the taskq. Therefore, the KMEM_SLAB_MOVE_PENDING
5101b5fca8f8Stomee * flag causes the slab to be put at the end of the
5102686031edSTom Erickson * deadlist and prevents it from being destroyed, since
5103686031edSTom Erickson * we plan to destroy it here after reacquiring the
5104686031edSTom Erickson * lock.
5105b5fca8f8Stomee */
5106b5fca8f8Stomee mutex_enter(&cp->cache_lock);
5107b5fca8f8Stomee ASSERT(sp->slab_flags & KMEM_SLAB_MOVE_PENDING);
5108b5fca8f8Stomee sp->slab_flags &= ~KMEM_SLAB_MOVE_PENDING;
5109b5fca8f8Stomee
5110b5fca8f8Stomee if (sp->slab_refcnt == 0) {
5111b5fca8f8Stomee list_t *deadlist =
5112b5fca8f8Stomee &cp->cache_defrag->kmd_deadlist;
5113686031edSTom Erickson list_remove(deadlist, sp);
5114b5fca8f8Stomee
5115686031edSTom Erickson if (!avl_is_empty(
5116686031edSTom Erickson &cp->cache_defrag->kmd_moves_pending)) {
5117686031edSTom Erickson /*
5118686031edSTom Erickson * A pending move makes it unsafe to
5119686031edSTom Erickson * destroy the slab, because even though
5120686031edSTom Erickson * the move is no longer needed, the
5121686031edSTom Erickson * context where that is determined
5122686031edSTom Erickson * requires the slab to exist.
5123686031edSTom Erickson * Fortunately, a pending move also
5124686031edSTom Erickson * means we don't need to destroy the
5125686031edSTom Erickson * slab here, since it will get
5126686031edSTom Erickson * destroyed along with any other slabs
5127686031edSTom Erickson * on the deadlist after the last
5128686031edSTom Erickson * pending move completes.
5129686031edSTom Erickson */
5130686031edSTom Erickson list_insert_head(deadlist, sp);
5131686031edSTom Erickson return (-1);
5132686031edSTom Erickson }
5133b5fca8f8Stomee
5134686031edSTom Erickson /*
5135686031edSTom Erickson * Destroy the slab now if it was completely
5136686031edSTom Erickson * freed while we dropped cache_lock and there
5137686031edSTom Erickson * are no pending moves. Since slab_refcnt
5138686031edSTom Erickson * cannot change once it reaches zero, no new
5139686031edSTom Erickson * pending moves from that slab are possible.
5140686031edSTom Erickson */
5141b5fca8f8Stomee cp->cache_defrag->kmd_deadcount--;
5142b5fca8f8Stomee cp->cache_slab_destroy++;
5143b5fca8f8Stomee mutex_exit(&cp->cache_lock);
5144b5fca8f8Stomee kmem_slab_destroy(cp, sp);
5145b5fca8f8Stomee mutex_enter(&cp->cache_lock);
5146b5fca8f8Stomee /*
5147b5fca8f8Stomee * Since we can't pick up the scan where we left
5148b5fca8f8Stomee * off, abort the scan and say nothing about the
5149b5fca8f8Stomee * number of reclaimable slabs.
5150b5fca8f8Stomee */
5151b5fca8f8Stomee return (-1);
5152b5fca8f8Stomee }
5153b5fca8f8Stomee
5154b5fca8f8Stomee if (!success) {
5155b5fca8f8Stomee /*
5156b5fca8f8Stomee * Abort the scan if there is not enough memory
5157b5fca8f8Stomee * for the request and say nothing about the
5158b5fca8f8Stomee * number of reclaimable slabs.
5159b5fca8f8Stomee */
5160b5fca8f8Stomee return (-1);
5161b5fca8f8Stomee }
5162b5fca8f8Stomee
5163b5fca8f8Stomee /*
5164b5fca8f8Stomee * The slab's position changed while the lock was
5165b5fca8f8Stomee * dropped, so we don't know where we are in the
5166b5fca8f8Stomee * sequence any more.
5167b5fca8f8Stomee */
5168b5fca8f8Stomee if (sp->slab_refcnt != refcnt) {
5169686031edSTom Erickson /*
5170686031edSTom Erickson * If this is a KMM_DEBUG move, the slab_refcnt
5171686031edSTom Erickson * may have changed because we allocated a
5172686031edSTom Erickson * destination buffer on the same slab. In that
5173686031edSTom Erickson * case, we're not interested in counting it.
5174686031edSTom Erickson */
5175b5fca8f8Stomee return (-1);
5176b5fca8f8Stomee }
5177d7db73d1SBryan Cantrill if ((sp->slab_flags & KMEM_SLAB_NOMOVE) != nomove)
5178b5fca8f8Stomee return (-1);
5179b5fca8f8Stomee
5180b5fca8f8Stomee /*
5181b5fca8f8Stomee * Generating a move request allocates a destination
5182686031edSTom Erickson * buffer from the slab layer, bumping the first partial
5183686031edSTom Erickson * slab if it is completely allocated. If the current
5184686031edSTom Erickson * slab becomes the first partial slab as a result, we
5185686031edSTom Erickson * can't continue to scan backwards.
5186686031edSTom Erickson *
5187686031edSTom Erickson * If this is a KMM_DEBUG move and we allocated the
5188686031edSTom Erickson * destination buffer from the last partial slab, then
5189686031edSTom Erickson * the buffer we're moving is on the same slab and our
5190686031edSTom Erickson * slab_refcnt has changed, causing us to return before
5191686031edSTom Erickson * reaching here if there are no partial slabs left.
5192b5fca8f8Stomee */
5193b5fca8f8Stomee ASSERT(!avl_is_empty(&cp->cache_partial_slabs));
5194b5fca8f8Stomee if (sp == avl_first(&cp->cache_partial_slabs)) {
5195686031edSTom Erickson /*
5196686031edSTom Erickson * We're not interested in a second KMM_DEBUG
5197686031edSTom Erickson * move.
5198686031edSTom Erickson */
5199b5fca8f8Stomee goto end_scan;
5200b5fca8f8Stomee }
5201b5fca8f8Stomee }
5202b5fca8f8Stomee }
5203b5fca8f8Stomee end_scan:
5204b5fca8f8Stomee
5205b5fca8f8Stomee return (s);
5206b5fca8f8Stomee }
5207b5fca8f8Stomee
5208b5fca8f8Stomee typedef struct kmem_move_notify_args {
5209b5fca8f8Stomee kmem_cache_t *kmna_cache;
5210b5fca8f8Stomee void *kmna_buf;
5211b5fca8f8Stomee } kmem_move_notify_args_t;
5212b5fca8f8Stomee
5213b5fca8f8Stomee static void
kmem_cache_move_notify_task(void * arg)5214b5fca8f8Stomee kmem_cache_move_notify_task(void *arg)
5215b5fca8f8Stomee {
5216b5fca8f8Stomee kmem_move_notify_args_t *args = arg;
5217b5fca8f8Stomee kmem_cache_t *cp = args->kmna_cache;
5218b5fca8f8Stomee void *buf = args->kmna_buf;
5219b5fca8f8Stomee kmem_slab_t *sp;
5220b5fca8f8Stomee
5221b5fca8f8Stomee ASSERT(taskq_member(kmem_taskq, curthread));
5222b5fca8f8Stomee ASSERT(list_link_active(&cp->cache_link));
5223b5fca8f8Stomee
5224b5fca8f8Stomee kmem_free(args, sizeof (kmem_move_notify_args_t));
5225b5fca8f8Stomee mutex_enter(&cp->cache_lock);
5226b5fca8f8Stomee sp = kmem_slab_allocated(cp, NULL, buf);
5227b5fca8f8Stomee
5228b5fca8f8Stomee /* Ignore the notification if the buffer is no longer allocated. */
5229b5fca8f8Stomee if (sp == NULL) {
5230b5fca8f8Stomee mutex_exit(&cp->cache_lock);
5231b5fca8f8Stomee return;
5232b5fca8f8Stomee }
5233b5fca8f8Stomee
5234b5fca8f8Stomee /* Ignore the notification if there's no reason to move the buffer. */
5235b5fca8f8Stomee if (avl_numnodes(&cp->cache_partial_slabs) > 1) {
5236b5fca8f8Stomee /*
5237b5fca8f8Stomee * So far the notification is not ignored. Ignore the
5238b5fca8f8Stomee * notification if the slab is not marked by an earlier refusal
5239b5fca8f8Stomee * to move a buffer.
5240b5fca8f8Stomee */
5241b5fca8f8Stomee if (!(sp->slab_flags & KMEM_SLAB_NOMOVE) &&
5242b5fca8f8Stomee (sp->slab_later_count == 0)) {
5243b5fca8f8Stomee mutex_exit(&cp->cache_lock);
5244b5fca8f8Stomee return;
5245b5fca8f8Stomee }
5246b5fca8f8Stomee
5247b5fca8f8Stomee kmem_slab_move_yes(cp, sp, buf);
5248b5fca8f8Stomee ASSERT(!(sp->slab_flags & KMEM_SLAB_MOVE_PENDING));
5249b5fca8f8Stomee sp->slab_flags |= KMEM_SLAB_MOVE_PENDING;
5250b5fca8f8Stomee mutex_exit(&cp->cache_lock);
5251b5fca8f8Stomee /* see kmem_move_buffers() about dropping the lock */
5252b5fca8f8Stomee (void) kmem_move_begin(cp, sp, buf, KMM_NOTIFY);
5253b5fca8f8Stomee mutex_enter(&cp->cache_lock);
5254b5fca8f8Stomee ASSERT(sp->slab_flags & KMEM_SLAB_MOVE_PENDING);
5255b5fca8f8Stomee sp->slab_flags &= ~KMEM_SLAB_MOVE_PENDING;
5256b5fca8f8Stomee if (sp->slab_refcnt == 0) {
5257b5fca8f8Stomee list_t *deadlist = &cp->cache_defrag->kmd_deadlist;
5258686031edSTom Erickson list_remove(deadlist, sp);
5259b5fca8f8Stomee
5260686031edSTom Erickson if (!avl_is_empty(
5261686031edSTom Erickson &cp->cache_defrag->kmd_moves_pending)) {
5262686031edSTom Erickson list_insert_head(deadlist, sp);
5263686031edSTom Erickson mutex_exit(&cp->cache_lock);
5264686031edSTom Erickson return;
5265686031edSTom Erickson }
5266b5fca8f8Stomee
5267b5fca8f8Stomee cp->cache_defrag->kmd_deadcount--;
5268b5fca8f8Stomee cp->cache_slab_destroy++;
5269b5fca8f8Stomee mutex_exit(&cp->cache_lock);
5270b5fca8f8Stomee kmem_slab_destroy(cp, sp);
5271b5fca8f8Stomee return;
5272b5fca8f8Stomee }
5273b5fca8f8Stomee } else {
5274b5fca8f8Stomee kmem_slab_move_yes(cp, sp, buf);
5275b5fca8f8Stomee }
5276b5fca8f8Stomee mutex_exit(&cp->cache_lock);
5277b5fca8f8Stomee }
5278b5fca8f8Stomee
5279b5fca8f8Stomee void
kmem_cache_move_notify(kmem_cache_t * cp,void * buf)5280b5fca8f8Stomee kmem_cache_move_notify(kmem_cache_t *cp, void *buf)
5281b5fca8f8Stomee {
5282b5fca8f8Stomee kmem_move_notify_args_t *args;
5283b5fca8f8Stomee
5284b5fca8f8Stomee args = kmem_alloc(sizeof (kmem_move_notify_args_t), KM_NOSLEEP);
5285b5fca8f8Stomee if (args != NULL) {
5286b5fca8f8Stomee args->kmna_cache = cp;
5287b5fca8f8Stomee args->kmna_buf = buf;
5288fc8ae2ecSToomas Soome if (taskq_dispatch(kmem_taskq,
5289b5fca8f8Stomee (task_func_t *)kmem_cache_move_notify_task, args,
5290fc8ae2ecSToomas Soome TQ_NOSLEEP) == TASKQID_INVALID)
5291eb697d4eStomee kmem_free(args, sizeof (kmem_move_notify_args_t));
5292b5fca8f8Stomee }
5293b5fca8f8Stomee }
5294b5fca8f8Stomee
5295b5fca8f8Stomee static void
kmem_cache_defrag(kmem_cache_t * cp)5296b5fca8f8Stomee kmem_cache_defrag(kmem_cache_t *cp)
5297b5fca8f8Stomee {
5298b5fca8f8Stomee size_t n;
5299b5fca8f8Stomee
5300b5fca8f8Stomee ASSERT(cp->cache_defrag != NULL);
5301b5fca8f8Stomee
5302b5fca8f8Stomee mutex_enter(&cp->cache_lock);
5303b5fca8f8Stomee n = avl_numnodes(&cp->cache_partial_slabs);
5304b5fca8f8Stomee if (n > 1) {
5305b5fca8f8Stomee /* kmem_move_buffers() drops and reacquires cache_lock */
5306686031edSTom Erickson cp->cache_defrag->kmd_defrags++;
5307686031edSTom Erickson (void) kmem_move_buffers(cp, n, 0, KMM_DESPERATE);
5308b5fca8f8Stomee }
5309b5fca8f8Stomee mutex_exit(&cp->cache_lock);
5310b5fca8f8Stomee }
5311b5fca8f8Stomee
5312b5fca8f8Stomee /* Is this cache above the fragmentation threshold? */
5313b5fca8f8Stomee static boolean_t
kmem_cache_frag_threshold(kmem_cache_t * cp,uint64_t nfree)5314b5fca8f8Stomee kmem_cache_frag_threshold(kmem_cache_t *cp, uint64_t nfree)
5315b5fca8f8Stomee {
5316b5fca8f8Stomee /*
5317b5fca8f8Stomee * nfree kmem_frag_numer
5318b5fca8f8Stomee * ------------------ > ---------------
5319b5fca8f8Stomee * cp->cache_buftotal kmem_frag_denom
5320b5fca8f8Stomee */
5321b5fca8f8Stomee return ((nfree * kmem_frag_denom) >
5322b5fca8f8Stomee (cp->cache_buftotal * kmem_frag_numer));
5323b5fca8f8Stomee }
5324b5fca8f8Stomee
5325b5fca8f8Stomee static boolean_t
kmem_cache_is_fragmented(kmem_cache_t * cp,boolean_t * doreap)5326b5fca8f8Stomee kmem_cache_is_fragmented(kmem_cache_t *cp, boolean_t *doreap)
5327b5fca8f8Stomee {
5328b5fca8f8Stomee boolean_t fragmented;
5329b5fca8f8Stomee uint64_t nfree;
5330b5fca8f8Stomee
5331b5fca8f8Stomee ASSERT(MUTEX_HELD(&cp->cache_lock));
5332b5fca8f8Stomee *doreap = B_FALSE;
5333b5fca8f8Stomee
5334686031edSTom Erickson if (kmem_move_fulltilt) {
5335686031edSTom Erickson if (avl_numnodes(&cp->cache_partial_slabs) > 1) {
5336686031edSTom Erickson return (B_TRUE);
5337686031edSTom Erickson }
5338686031edSTom Erickson } else {
5339686031edSTom Erickson if ((cp->cache_complete_slab_count + avl_numnodes(
5340686031edSTom Erickson &cp->cache_partial_slabs)) < kmem_frag_minslabs) {
5341686031edSTom Erickson return (B_FALSE);
5342686031edSTom Erickson }
5343686031edSTom Erickson }
5344b5fca8f8Stomee
5345b5fca8f8Stomee nfree = cp->cache_bufslab;
5346686031edSTom Erickson fragmented = ((avl_numnodes(&cp->cache_partial_slabs) > 1) &&
5347686031edSTom Erickson kmem_cache_frag_threshold(cp, nfree));
5348686031edSTom Erickson
5349b5fca8f8Stomee /*
5350b5fca8f8Stomee * Free buffers in the magazine layer appear allocated from the point of
5351b5fca8f8Stomee * view of the slab layer. We want to know if the slab layer would
5352b5fca8f8Stomee * appear fragmented if we included free buffers from magazines that
5353b5fca8f8Stomee * have fallen out of the working set.
5354b5fca8f8Stomee */
5355b5fca8f8Stomee if (!fragmented) {
5356b5fca8f8Stomee long reap;
5357b5fca8f8Stomee
5358b5fca8f8Stomee mutex_enter(&cp->cache_depot_lock);
5359b5fca8f8Stomee reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min);
5360b5fca8f8Stomee reap = MIN(reap, cp->cache_full.ml_total);
5361b5fca8f8Stomee mutex_exit(&cp->cache_depot_lock);
5362b5fca8f8Stomee
5363b5fca8f8Stomee nfree += ((uint64_t)reap * cp->cache_magtype->mt_magsize);
5364b5fca8f8Stomee if (kmem_cache_frag_threshold(cp, nfree)) {
5365b5fca8f8Stomee *doreap = B_TRUE;
5366b5fca8f8Stomee }
5367b5fca8f8Stomee }
5368b5fca8f8Stomee
5369b5fca8f8Stomee return (fragmented);
5370b5fca8f8Stomee }
5371b5fca8f8Stomee
5372b5fca8f8Stomee /* Called periodically from kmem_taskq */
5373b5fca8f8Stomee static void
kmem_cache_scan(kmem_cache_t * cp)5374b5fca8f8Stomee kmem_cache_scan(kmem_cache_t *cp)
5375b5fca8f8Stomee {
5376b5fca8f8Stomee boolean_t reap = B_FALSE;
5377686031edSTom Erickson kmem_defrag_t *kmd;
5378b5fca8f8Stomee
5379b5fca8f8Stomee ASSERT(taskq_member(kmem_taskq, curthread));
5380b5fca8f8Stomee
5381b5fca8f8Stomee mutex_enter(&cp->cache_lock);
5382b5fca8f8Stomee
5383686031edSTom Erickson kmd = cp->cache_defrag;
5384686031edSTom Erickson if (kmd->kmd_consolidate > 0) {
5385686031edSTom Erickson kmd->kmd_consolidate--;
5386686031edSTom Erickson mutex_exit(&cp->cache_lock);
5387686031edSTom Erickson kmem_cache_reap(cp);
5388686031edSTom Erickson return;
5389686031edSTom Erickson }
5390686031edSTom Erickson
5391b5fca8f8Stomee if (kmem_cache_is_fragmented(cp, &reap)) {
53923df2e8b2SRobert Mustacchi int slabs_found;
5393b5fca8f8Stomee
5394b5fca8f8Stomee /*
5395b5fca8f8Stomee * Consolidate reclaimable slabs from the end of the partial
5396b5fca8f8Stomee * slab list (scan at most kmem_reclaim_scan_range slabs to find
5397b5fca8f8Stomee * reclaimable slabs). Keep track of how many candidate slabs we
5398b5fca8f8Stomee * looked for and how many we actually found so we can adjust
5399b5fca8f8Stomee * the definition of a candidate slab if we're having trouble
5400b5fca8f8Stomee * finding them.
5401b5fca8f8Stomee *
5402b5fca8f8Stomee * kmem_move_buffers() drops and reacquires cache_lock.
5403b5fca8f8Stomee */
5404686031edSTom Erickson kmd->kmd_scans++;
5405b5fca8f8Stomee slabs_found = kmem_move_buffers(cp, kmem_reclaim_scan_range,
5406b5fca8f8Stomee kmem_reclaim_max_slabs, 0);
5407b5fca8f8Stomee if (slabs_found >= 0) {
5408b5fca8f8Stomee kmd->kmd_slabs_sought += kmem_reclaim_max_slabs;
5409b5fca8f8Stomee kmd->kmd_slabs_found += slabs_found;
5410b5fca8f8Stomee }
5411b5fca8f8Stomee
5412686031edSTom Erickson if (++kmd->kmd_tries >= kmem_reclaim_scan_range) {
5413686031edSTom Erickson kmd->kmd_tries = 0;
5414b5fca8f8Stomee
5415b5fca8f8Stomee /*
5416b5fca8f8Stomee * If we had difficulty finding candidate slabs in
5417b5fca8f8Stomee * previous scans, adjust the threshold so that
5418b5fca8f8Stomee * candidates are easier to find.
5419b5fca8f8Stomee */
5420b5fca8f8Stomee if (kmd->kmd_slabs_found == kmd->kmd_slabs_sought) {
5421b5fca8f8Stomee kmem_adjust_reclaim_threshold(kmd, -1);
5422b5fca8f8Stomee } else if ((kmd->kmd_slabs_found * 2) <
5423b5fca8f8Stomee kmd->kmd_slabs_sought) {
5424b5fca8f8Stomee kmem_adjust_reclaim_threshold(kmd, 1);
5425b5fca8f8Stomee }
5426b5fca8f8Stomee kmd->kmd_slabs_sought = 0;
5427b5fca8f8Stomee kmd->kmd_slabs_found = 0;
5428b5fca8f8Stomee }
5429b5fca8f8Stomee } else {
5430b5fca8f8Stomee kmem_reset_reclaim_threshold(cp->cache_defrag);
5431b5fca8f8Stomee #ifdef DEBUG
5432686031edSTom Erickson if (!avl_is_empty(&cp->cache_partial_slabs)) {
5433b5fca8f8Stomee /*
5434b5fca8f8Stomee * In a debug kernel we want the consolidator to
5435b5fca8f8Stomee * run occasionally even when there is plenty of
5436b5fca8f8Stomee * memory.
5437b5fca8f8Stomee */
5438686031edSTom Erickson uint16_t debug_rand;
5439b5fca8f8Stomee
5440686031edSTom Erickson (void) random_get_bytes((uint8_t *)&debug_rand, 2);
5441b5fca8f8Stomee if (!kmem_move_noreap &&
5442b5fca8f8Stomee ((debug_rand % kmem_mtb_reap) == 0)) {
5443b5fca8f8Stomee mutex_exit(&cp->cache_lock);
5444686031edSTom Erickson kmem_cache_reap(cp);
5445b5fca8f8Stomee return;
5446b5fca8f8Stomee } else if ((debug_rand % kmem_mtb_move) == 0) {
5447686031edSTom Erickson kmd->kmd_scans++;
5448b5fca8f8Stomee (void) kmem_move_buffers(cp,
5449686031edSTom Erickson kmem_reclaim_scan_range, 1, KMM_DEBUG);
5450b5fca8f8Stomee }
5451b5fca8f8Stomee }
5452b5fca8f8Stomee #endif /* DEBUG */
5453b5fca8f8Stomee }
5454b5fca8f8Stomee
5455b5fca8f8Stomee mutex_exit(&cp->cache_lock);
5456b5fca8f8Stomee
5457d7db73d1SBryan Cantrill if (reap)
5458b5fca8f8Stomee kmem_depot_ws_reap(cp);
5459b5fca8f8Stomee }
5460