xref: /illumos-gate/usr/src/uts/common/os/kmem.c (revision ad23a2db4cfc94c0ed1d58554479ce8d2e7e5768)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 /*
29  * Kernel memory allocator, as described in the following two papers:
30  *
31  * Jeff Bonwick,
32  * The Slab Allocator: An Object-Caching Kernel Memory Allocator.
33  * Proceedings of the Summer 1994 Usenix Conference.
34  * Available as /shared/sac/PSARC/1994/028/materials/kmem.pdf.
35  *
36  * Jeff Bonwick and Jonathan Adams,
37  * Magazines and vmem: Extending the Slab Allocator to Many CPUs and
38  * Arbitrary Resources.
39  * Proceedings of the 2001 Usenix Conference.
40  * Available as /shared/sac/PSARC/2000/550/materials/vmem.pdf.
41  */
42 
43 #include <sys/kmem_impl.h>
44 #include <sys/vmem_impl.h>
45 #include <sys/param.h>
46 #include <sys/sysmacros.h>
47 #include <sys/vm.h>
48 #include <sys/proc.h>
49 #include <sys/tuneable.h>
50 #include <sys/systm.h>
51 #include <sys/cmn_err.h>
52 #include <sys/debug.h>
53 #include <sys/mutex.h>
54 #include <sys/bitmap.h>
55 #include <sys/atomic.h>
56 #include <sys/kobj.h>
57 #include <sys/disp.h>
58 #include <vm/seg_kmem.h>
59 #include <sys/log.h>
60 #include <sys/callb.h>
61 #include <sys/taskq.h>
62 #include <sys/modctl.h>
63 #include <sys/reboot.h>
64 #include <sys/id32.h>
65 #include <sys/zone.h>
66 
67 extern void streams_msg_init(void);
68 extern int segkp_fromheap;
69 extern void segkp_cache_free(void);
70 
71 struct kmem_cache_kstat {
72 	kstat_named_t	kmc_buf_size;
73 	kstat_named_t	kmc_align;
74 	kstat_named_t	kmc_chunk_size;
75 	kstat_named_t	kmc_slab_size;
76 	kstat_named_t	kmc_alloc;
77 	kstat_named_t	kmc_alloc_fail;
78 	kstat_named_t	kmc_free;
79 	kstat_named_t	kmc_depot_alloc;
80 	kstat_named_t	kmc_depot_free;
81 	kstat_named_t	kmc_depot_contention;
82 	kstat_named_t	kmc_slab_alloc;
83 	kstat_named_t	kmc_slab_free;
84 	kstat_named_t	kmc_buf_constructed;
85 	kstat_named_t	kmc_buf_avail;
86 	kstat_named_t	kmc_buf_inuse;
87 	kstat_named_t	kmc_buf_total;
88 	kstat_named_t	kmc_buf_max;
89 	kstat_named_t	kmc_slab_create;
90 	kstat_named_t	kmc_slab_destroy;
91 	kstat_named_t	kmc_vmem_source;
92 	kstat_named_t	kmc_hash_size;
93 	kstat_named_t	kmc_hash_lookup_depth;
94 	kstat_named_t	kmc_hash_rescale;
95 	kstat_named_t	kmc_full_magazines;
96 	kstat_named_t	kmc_empty_magazines;
97 	kstat_named_t	kmc_magazine_size;
98 } kmem_cache_kstat = {
99 	{ "buf_size",		KSTAT_DATA_UINT64 },
100 	{ "align",		KSTAT_DATA_UINT64 },
101 	{ "chunk_size",		KSTAT_DATA_UINT64 },
102 	{ "slab_size",		KSTAT_DATA_UINT64 },
103 	{ "alloc",		KSTAT_DATA_UINT64 },
104 	{ "alloc_fail",		KSTAT_DATA_UINT64 },
105 	{ "free",		KSTAT_DATA_UINT64 },
106 	{ "depot_alloc",	KSTAT_DATA_UINT64 },
107 	{ "depot_free",		KSTAT_DATA_UINT64 },
108 	{ "depot_contention",	KSTAT_DATA_UINT64 },
109 	{ "slab_alloc",		KSTAT_DATA_UINT64 },
110 	{ "slab_free",		KSTAT_DATA_UINT64 },
111 	{ "buf_constructed",	KSTAT_DATA_UINT64 },
112 	{ "buf_avail",		KSTAT_DATA_UINT64 },
113 	{ "buf_inuse",		KSTAT_DATA_UINT64 },
114 	{ "buf_total",		KSTAT_DATA_UINT64 },
115 	{ "buf_max",		KSTAT_DATA_UINT64 },
116 	{ "slab_create",	KSTAT_DATA_UINT64 },
117 	{ "slab_destroy",	KSTAT_DATA_UINT64 },
118 	{ "vmem_source",	KSTAT_DATA_UINT64 },
119 	{ "hash_size",		KSTAT_DATA_UINT64 },
120 	{ "hash_lookup_depth",	KSTAT_DATA_UINT64 },
121 	{ "hash_rescale",	KSTAT_DATA_UINT64 },
122 	{ "full_magazines",	KSTAT_DATA_UINT64 },
123 	{ "empty_magazines",	KSTAT_DATA_UINT64 },
124 	{ "magazine_size",	KSTAT_DATA_UINT64 },
125 };
126 
127 static kmutex_t kmem_cache_kstat_lock;
128 
129 /*
130  * The default set of caches to back kmem_alloc().
131  * These sizes should be reevaluated periodically.
132  *
133  * We want allocations that are multiples of the coherency granularity
134  * (64 bytes) to be satisfied from a cache which is a multiple of 64
135  * bytes, so that it will be 64-byte aligned.  For all multiples of 64,
136  * the next kmem_cache_size greater than or equal to it must be a
137  * multiple of 64.
138  */
139 static const int kmem_alloc_sizes[] = {
140 	1 * 8,
141 	2 * 8,
142 	3 * 8,
143 	4 * 8,		5 * 8,		6 * 8,		7 * 8,
144 	4 * 16,		5 * 16,		6 * 16,		7 * 16,
145 	4 * 32,		5 * 32,		6 * 32,		7 * 32,
146 	4 * 64,		5 * 64,		6 * 64,		7 * 64,
147 	4 * 128,	5 * 128,	6 * 128,	7 * 128,
148 	P2ALIGN(8192 / 7, 64),
149 	P2ALIGN(8192 / 6, 64),
150 	P2ALIGN(8192 / 5, 64),
151 	P2ALIGN(8192 / 4, 64),
152 	P2ALIGN(8192 / 3, 64),
153 	P2ALIGN(8192 / 2, 64),
154 	P2ALIGN(8192 / 1, 64),
155 	4096 * 3,
156 	8192 * 2,
157 	8192 * 3,
158 	8192 * 4,
159 };
160 
161 #define	KMEM_MAXBUF	32768
162 
163 static kmem_cache_t *kmem_alloc_table[KMEM_MAXBUF >> KMEM_ALIGN_SHIFT];
164 
165 static kmem_magtype_t kmem_magtype[] = {
166 	{ 1,	8,	3200,	65536	},
167 	{ 3,	16,	256,	32768	},
168 	{ 7,	32,	64,	16384	},
169 	{ 15,	64,	0,	8192	},
170 	{ 31,	64,	0,	4096	},
171 	{ 47,	64,	0,	2048	},
172 	{ 63,	64,	0,	1024	},
173 	{ 95,	64,	0,	512	},
174 	{ 143,	64,	0,	0	},
175 };
176 
177 static uint32_t kmem_reaping;
178 static uint32_t kmem_reaping_idspace;
179 
180 /*
181  * kmem tunables
182  */
183 clock_t kmem_reap_interval;	/* cache reaping rate [15 * HZ ticks] */
184 int kmem_depot_contention = 3;	/* max failed tryenters per real interval */
185 pgcnt_t kmem_reapahead = 0;	/* start reaping N pages before pageout */
186 int kmem_panic = 1;		/* whether to panic on error */
187 int kmem_logging = 1;		/* kmem_log_enter() override */
188 uint32_t kmem_mtbf = 0;		/* mean time between failures [default: off] */
189 size_t kmem_transaction_log_size; /* transaction log size [2% of memory] */
190 size_t kmem_content_log_size;	/* content log size [2% of memory] */
191 size_t kmem_failure_log_size;	/* failure log [4 pages per CPU] */
192 size_t kmem_slab_log_size;	/* slab create log [4 pages per CPU] */
193 size_t kmem_content_maxsave = 256; /* KMF_CONTENTS max bytes to log */
194 size_t kmem_lite_minsize = 0;	/* minimum buffer size for KMF_LITE */
195 size_t kmem_lite_maxalign = 1024; /* maximum buffer alignment for KMF_LITE */
196 int kmem_lite_pcs = 4;		/* number of PCs to store in KMF_LITE mode */
197 size_t kmem_maxverify;		/* maximum bytes to inspect in debug routines */
198 size_t kmem_minfirewall;	/* hardware-enforced redzone threshold */
199 
200 #ifdef DEBUG
201 int kmem_flags = KMF_AUDIT | KMF_DEADBEEF | KMF_REDZONE | KMF_CONTENTS;
202 #else
203 int kmem_flags = 0;
204 #endif
205 int kmem_ready;
206 
207 static kmem_cache_t	*kmem_slab_cache;
208 static kmem_cache_t	*kmem_bufctl_cache;
209 static kmem_cache_t	*kmem_bufctl_audit_cache;
210 
211 static kmutex_t		kmem_cache_lock;	/* inter-cache linkage only */
212 kmem_cache_t		kmem_null_cache;
213 
214 static taskq_t		*kmem_taskq;
215 static kmutex_t		kmem_flags_lock;
216 static vmem_t		*kmem_metadata_arena;
217 static vmem_t		*kmem_msb_arena;	/* arena for metadata caches */
218 static vmem_t		*kmem_cache_arena;
219 static vmem_t		*kmem_hash_arena;
220 static vmem_t		*kmem_log_arena;
221 static vmem_t		*kmem_oversize_arena;
222 static vmem_t		*kmem_va_arena;
223 static vmem_t		*kmem_default_arena;
224 static vmem_t		*kmem_firewall_va_arena;
225 static vmem_t		*kmem_firewall_arena;
226 
227 kmem_log_header_t	*kmem_transaction_log;
228 kmem_log_header_t	*kmem_content_log;
229 kmem_log_header_t	*kmem_failure_log;
230 kmem_log_header_t	*kmem_slab_log;
231 
232 static int		kmem_lite_count; /* # of PCs in kmem_buftag_lite_t */
233 
234 #define	KMEM_BUFTAG_LITE_ENTER(bt, count, caller)			\
235 	if ((count) > 0) {						\
236 		pc_t *_s = ((kmem_buftag_lite_t *)(bt))->bt_history;	\
237 		pc_t *_e;						\
238 		/* memmove() the old entries down one notch */		\
239 		for (_e = &_s[(count) - 1]; _e > _s; _e--)		\
240 			*_e = *(_e - 1);				\
241 		*_s = (uintptr_t)(caller);				\
242 	}
243 
244 #define	KMERR_MODIFIED	0	/* buffer modified while on freelist */
245 #define	KMERR_REDZONE	1	/* redzone violation (write past end of buf) */
246 #define	KMERR_DUPFREE	2	/* freed a buffer twice */
247 #define	KMERR_BADADDR	3	/* freed a bad (unallocated) address */
248 #define	KMERR_BADBUFTAG	4	/* buftag corrupted */
249 #define	KMERR_BADBUFCTL	5	/* bufctl corrupted */
250 #define	KMERR_BADCACHE	6	/* freed a buffer to the wrong cache */
251 #define	KMERR_BADSIZE	7	/* alloc size != free size */
252 #define	KMERR_BADBASE	8	/* buffer base address wrong */
253 
254 struct {
255 	hrtime_t	kmp_timestamp;	/* timestamp of panic */
256 	int		kmp_error;	/* type of kmem error */
257 	void		*kmp_buffer;	/* buffer that induced panic */
258 	void		*kmp_realbuf;	/* real start address for buffer */
259 	kmem_cache_t	*kmp_cache;	/* buffer's cache according to client */
260 	kmem_cache_t	*kmp_realcache;	/* actual cache containing buffer */
261 	kmem_slab_t	*kmp_slab;	/* slab accoring to kmem_findslab() */
262 	kmem_bufctl_t	*kmp_bufctl;	/* bufctl */
263 } kmem_panic_info;
264 
265 
266 static void
267 copy_pattern(uint64_t pattern, void *buf_arg, size_t size)
268 {
269 	uint64_t *bufend = (uint64_t *)((char *)buf_arg + size);
270 	uint64_t *buf = buf_arg;
271 
272 	while (buf < bufend)
273 		*buf++ = pattern;
274 }
275 
276 static void *
277 verify_pattern(uint64_t pattern, void *buf_arg, size_t size)
278 {
279 	uint64_t *bufend = (uint64_t *)((char *)buf_arg + size);
280 	uint64_t *buf;
281 
282 	for (buf = buf_arg; buf < bufend; buf++)
283 		if (*buf != pattern)
284 			return (buf);
285 	return (NULL);
286 }
287 
288 static void *
289 verify_and_copy_pattern(uint64_t old, uint64_t new, void *buf_arg, size_t size)
290 {
291 	uint64_t *bufend = (uint64_t *)((char *)buf_arg + size);
292 	uint64_t *buf;
293 
294 	for (buf = buf_arg; buf < bufend; buf++) {
295 		if (*buf != old) {
296 			copy_pattern(old, buf_arg,
297 				(char *)buf - (char *)buf_arg);
298 			return (buf);
299 		}
300 		*buf = new;
301 	}
302 
303 	return (NULL);
304 }
305 
306 static void
307 kmem_cache_applyall(void (*func)(kmem_cache_t *), taskq_t *tq, int tqflag)
308 {
309 	kmem_cache_t *cp;
310 
311 	mutex_enter(&kmem_cache_lock);
312 	for (cp = kmem_null_cache.cache_next; cp != &kmem_null_cache;
313 	    cp = cp->cache_next)
314 		if (tq != NULL)
315 			(void) taskq_dispatch(tq, (task_func_t *)func, cp,
316 			    tqflag);
317 		else
318 			func(cp);
319 	mutex_exit(&kmem_cache_lock);
320 }
321 
322 static void
323 kmem_cache_applyall_id(void (*func)(kmem_cache_t *), taskq_t *tq, int tqflag)
324 {
325 	kmem_cache_t *cp;
326 
327 	mutex_enter(&kmem_cache_lock);
328 	for (cp = kmem_null_cache.cache_next; cp != &kmem_null_cache;
329 	    cp = cp->cache_next) {
330 		if (!(cp->cache_cflags & KMC_IDENTIFIER))
331 			continue;
332 		if (tq != NULL)
333 			(void) taskq_dispatch(tq, (task_func_t *)func, cp,
334 			    tqflag);
335 		else
336 			func(cp);
337 	}
338 	mutex_exit(&kmem_cache_lock);
339 }
340 
341 /*
342  * Debugging support.  Given a buffer address, find its slab.
343  */
344 static kmem_slab_t *
345 kmem_findslab(kmem_cache_t *cp, void *buf)
346 {
347 	kmem_slab_t *sp;
348 
349 	mutex_enter(&cp->cache_lock);
350 	for (sp = cp->cache_nullslab.slab_next;
351 	    sp != &cp->cache_nullslab; sp = sp->slab_next) {
352 		if (KMEM_SLAB_MEMBER(sp, buf)) {
353 			mutex_exit(&cp->cache_lock);
354 			return (sp);
355 		}
356 	}
357 	mutex_exit(&cp->cache_lock);
358 
359 	return (NULL);
360 }
361 
362 static void
363 kmem_error(int error, kmem_cache_t *cparg, void *bufarg)
364 {
365 	kmem_buftag_t *btp = NULL;
366 	kmem_bufctl_t *bcp = NULL;
367 	kmem_cache_t *cp = cparg;
368 	kmem_slab_t *sp;
369 	uint64_t *off;
370 	void *buf = bufarg;
371 
372 	kmem_logging = 0;	/* stop logging when a bad thing happens */
373 
374 	kmem_panic_info.kmp_timestamp = gethrtime();
375 
376 	sp = kmem_findslab(cp, buf);
377 	if (sp == NULL) {
378 		for (cp = kmem_null_cache.cache_prev; cp != &kmem_null_cache;
379 		    cp = cp->cache_prev) {
380 			if ((sp = kmem_findslab(cp, buf)) != NULL)
381 				break;
382 		}
383 	}
384 
385 	if (sp == NULL) {
386 		cp = NULL;
387 		error = KMERR_BADADDR;
388 	} else {
389 		if (cp != cparg)
390 			error = KMERR_BADCACHE;
391 		else
392 			buf = (char *)bufarg - ((uintptr_t)bufarg -
393 			    (uintptr_t)sp->slab_base) % cp->cache_chunksize;
394 		if (buf != bufarg)
395 			error = KMERR_BADBASE;
396 		if (cp->cache_flags & KMF_BUFTAG)
397 			btp = KMEM_BUFTAG(cp, buf);
398 		if (cp->cache_flags & KMF_HASH) {
399 			mutex_enter(&cp->cache_lock);
400 			for (bcp = *KMEM_HASH(cp, buf); bcp; bcp = bcp->bc_next)
401 				if (bcp->bc_addr == buf)
402 					break;
403 			mutex_exit(&cp->cache_lock);
404 			if (bcp == NULL && btp != NULL)
405 				bcp = btp->bt_bufctl;
406 			if (kmem_findslab(cp->cache_bufctl_cache, bcp) ==
407 			    NULL || P2PHASE((uintptr_t)bcp, KMEM_ALIGN) ||
408 			    bcp->bc_addr != buf) {
409 				error = KMERR_BADBUFCTL;
410 				bcp = NULL;
411 			}
412 		}
413 	}
414 
415 	kmem_panic_info.kmp_error = error;
416 	kmem_panic_info.kmp_buffer = bufarg;
417 	kmem_panic_info.kmp_realbuf = buf;
418 	kmem_panic_info.kmp_cache = cparg;
419 	kmem_panic_info.kmp_realcache = cp;
420 	kmem_panic_info.kmp_slab = sp;
421 	kmem_panic_info.kmp_bufctl = bcp;
422 
423 	printf("kernel memory allocator: ");
424 
425 	switch (error) {
426 
427 	case KMERR_MODIFIED:
428 		printf("buffer modified after being freed\n");
429 		off = verify_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify);
430 		if (off == NULL)	/* shouldn't happen */
431 			off = buf;
432 		printf("modification occurred at offset 0x%lx "
433 		    "(0x%llx replaced by 0x%llx)\n",
434 		    (uintptr_t)off - (uintptr_t)buf,
435 		    (longlong_t)KMEM_FREE_PATTERN, (longlong_t)*off);
436 		break;
437 
438 	case KMERR_REDZONE:
439 		printf("redzone violation: write past end of buffer\n");
440 		break;
441 
442 	case KMERR_BADADDR:
443 		printf("invalid free: buffer not in cache\n");
444 		break;
445 
446 	case KMERR_DUPFREE:
447 		printf("duplicate free: buffer freed twice\n");
448 		break;
449 
450 	case KMERR_BADBUFTAG:
451 		printf("boundary tag corrupted\n");
452 		printf("bcp ^ bxstat = %lx, should be %lx\n",
453 		    (intptr_t)btp->bt_bufctl ^ btp->bt_bxstat,
454 		    KMEM_BUFTAG_FREE);
455 		break;
456 
457 	case KMERR_BADBUFCTL:
458 		printf("bufctl corrupted\n");
459 		break;
460 
461 	case KMERR_BADCACHE:
462 		printf("buffer freed to wrong cache\n");
463 		printf("buffer was allocated from %s,\n", cp->cache_name);
464 		printf("caller attempting free to %s.\n", cparg->cache_name);
465 		break;
466 
467 	case KMERR_BADSIZE:
468 		printf("bad free: free size (%u) != alloc size (%u)\n",
469 		    KMEM_SIZE_DECODE(((uint32_t *)btp)[0]),
470 		    KMEM_SIZE_DECODE(((uint32_t *)btp)[1]));
471 		break;
472 
473 	case KMERR_BADBASE:
474 		printf("bad free: free address (%p) != alloc address (%p)\n",
475 		    bufarg, buf);
476 		break;
477 	}
478 
479 	printf("buffer=%p  bufctl=%p  cache: %s\n",
480 	    bufarg, (void *)bcp, cparg->cache_name);
481 
482 	if (bcp != NULL && (cp->cache_flags & KMF_AUDIT) &&
483 	    error != KMERR_BADBUFCTL) {
484 		int d;
485 		timestruc_t ts;
486 		kmem_bufctl_audit_t *bcap = (kmem_bufctl_audit_t *)bcp;
487 
488 		hrt2ts(kmem_panic_info.kmp_timestamp - bcap->bc_timestamp, &ts);
489 		printf("previous transaction on buffer %p:\n", buf);
490 		printf("thread=%p  time=T-%ld.%09ld  slab=%p  cache: %s\n",
491 		    (void *)bcap->bc_thread, ts.tv_sec, ts.tv_nsec,
492 		    (void *)sp, cp->cache_name);
493 		for (d = 0; d < MIN(bcap->bc_depth, KMEM_STACK_DEPTH); d++) {
494 			ulong_t off;
495 			char *sym = kobj_getsymname(bcap->bc_stack[d], &off);
496 			printf("%s+%lx\n", sym ? sym : "?", off);
497 		}
498 	}
499 	if (kmem_panic > 0)
500 		panic("kernel heap corruption detected");
501 	if (kmem_panic == 0)
502 		debug_enter(NULL);
503 	kmem_logging = 1;	/* resume logging */
504 }
505 
506 static kmem_log_header_t *
507 kmem_log_init(size_t logsize)
508 {
509 	kmem_log_header_t *lhp;
510 	int nchunks = 4 * max_ncpus;
511 	size_t lhsize = (size_t)&((kmem_log_header_t *)0)->lh_cpu[max_ncpus];
512 	int i;
513 
514 	/*
515 	 * Make sure that lhp->lh_cpu[] is nicely aligned
516 	 * to prevent false sharing of cache lines.
517 	 */
518 	lhsize = P2ROUNDUP(lhsize, KMEM_ALIGN);
519 	lhp = vmem_xalloc(kmem_log_arena, lhsize, 64, P2NPHASE(lhsize, 64), 0,
520 	    NULL, NULL, VM_SLEEP);
521 	bzero(lhp, lhsize);
522 
523 	mutex_init(&lhp->lh_lock, NULL, MUTEX_DEFAULT, NULL);
524 	lhp->lh_nchunks = nchunks;
525 	lhp->lh_chunksize = P2ROUNDUP(logsize / nchunks + 1, PAGESIZE);
526 	lhp->lh_base = vmem_alloc(kmem_log_arena,
527 	    lhp->lh_chunksize * nchunks, VM_SLEEP);
528 	lhp->lh_free = vmem_alloc(kmem_log_arena,
529 	    nchunks * sizeof (int), VM_SLEEP);
530 	bzero(lhp->lh_base, lhp->lh_chunksize * nchunks);
531 
532 	for (i = 0; i < max_ncpus; i++) {
533 		kmem_cpu_log_header_t *clhp = &lhp->lh_cpu[i];
534 		mutex_init(&clhp->clh_lock, NULL, MUTEX_DEFAULT, NULL);
535 		clhp->clh_chunk = i;
536 	}
537 
538 	for (i = max_ncpus; i < nchunks; i++)
539 		lhp->lh_free[i] = i;
540 
541 	lhp->lh_head = max_ncpus;
542 	lhp->lh_tail = 0;
543 
544 	return (lhp);
545 }
546 
547 static void *
548 kmem_log_enter(kmem_log_header_t *lhp, void *data, size_t size)
549 {
550 	void *logspace;
551 	kmem_cpu_log_header_t *clhp = &lhp->lh_cpu[CPU->cpu_seqid];
552 
553 	if (lhp == NULL || kmem_logging == 0 || panicstr)
554 		return (NULL);
555 
556 	mutex_enter(&clhp->clh_lock);
557 	clhp->clh_hits++;
558 	if (size > clhp->clh_avail) {
559 		mutex_enter(&lhp->lh_lock);
560 		lhp->lh_hits++;
561 		lhp->lh_free[lhp->lh_tail] = clhp->clh_chunk;
562 		lhp->lh_tail = (lhp->lh_tail + 1) % lhp->lh_nchunks;
563 		clhp->clh_chunk = lhp->lh_free[lhp->lh_head];
564 		lhp->lh_head = (lhp->lh_head + 1) % lhp->lh_nchunks;
565 		clhp->clh_current = lhp->lh_base +
566 			clhp->clh_chunk * lhp->lh_chunksize;
567 		clhp->clh_avail = lhp->lh_chunksize;
568 		if (size > lhp->lh_chunksize)
569 			size = lhp->lh_chunksize;
570 		mutex_exit(&lhp->lh_lock);
571 	}
572 	logspace = clhp->clh_current;
573 	clhp->clh_current += size;
574 	clhp->clh_avail -= size;
575 	bcopy(data, logspace, size);
576 	mutex_exit(&clhp->clh_lock);
577 	return (logspace);
578 }
579 
580 #define	KMEM_AUDIT(lp, cp, bcp)						\
581 {									\
582 	kmem_bufctl_audit_t *_bcp = (kmem_bufctl_audit_t *)(bcp);	\
583 	_bcp->bc_timestamp = gethrtime();				\
584 	_bcp->bc_thread = curthread;					\
585 	_bcp->bc_depth = getpcstack(_bcp->bc_stack, KMEM_STACK_DEPTH);	\
586 	_bcp->bc_lastlog = kmem_log_enter((lp), _bcp, sizeof (*_bcp));	\
587 }
588 
589 static void
590 kmem_log_event(kmem_log_header_t *lp, kmem_cache_t *cp,
591 	kmem_slab_t *sp, void *addr)
592 {
593 	kmem_bufctl_audit_t bca;
594 
595 	bzero(&bca, sizeof (kmem_bufctl_audit_t));
596 	bca.bc_addr = addr;
597 	bca.bc_slab = sp;
598 	bca.bc_cache = cp;
599 	KMEM_AUDIT(lp, cp, &bca);
600 }
601 
602 /*
603  * Create a new slab for cache cp.
604  */
605 static kmem_slab_t *
606 kmem_slab_create(kmem_cache_t *cp, int kmflag)
607 {
608 	size_t slabsize = cp->cache_slabsize;
609 	size_t chunksize = cp->cache_chunksize;
610 	int cache_flags = cp->cache_flags;
611 	size_t color, chunks;
612 	char *buf, *slab;
613 	kmem_slab_t *sp;
614 	kmem_bufctl_t *bcp;
615 	vmem_t *vmp = cp->cache_arena;
616 
617 	color = cp->cache_color + cp->cache_align;
618 	if (color > cp->cache_maxcolor)
619 		color = cp->cache_mincolor;
620 	cp->cache_color = color;
621 
622 	slab = vmem_alloc(vmp, slabsize, kmflag & KM_VMFLAGS);
623 
624 	if (slab == NULL)
625 		goto vmem_alloc_failure;
626 
627 	ASSERT(P2PHASE((uintptr_t)slab, vmp->vm_quantum) == 0);
628 
629 	if (!(cp->cache_cflags & KMC_NOTOUCH))
630 		copy_pattern(KMEM_UNINITIALIZED_PATTERN, slab, slabsize);
631 
632 	if (cache_flags & KMF_HASH) {
633 		if ((sp = kmem_cache_alloc(kmem_slab_cache, kmflag)) == NULL)
634 			goto slab_alloc_failure;
635 		chunks = (slabsize - color) / chunksize;
636 	} else {
637 		sp = KMEM_SLAB(cp, slab);
638 		chunks = (slabsize - sizeof (kmem_slab_t) - color) / chunksize;
639 	}
640 
641 	sp->slab_cache	= cp;
642 	sp->slab_head	= NULL;
643 	sp->slab_refcnt	= 0;
644 	sp->slab_base	= buf = slab + color;
645 	sp->slab_chunks	= chunks;
646 
647 	ASSERT(chunks > 0);
648 	while (chunks-- != 0) {
649 		if (cache_flags & KMF_HASH) {
650 			bcp = kmem_cache_alloc(cp->cache_bufctl_cache, kmflag);
651 			if (bcp == NULL)
652 				goto bufctl_alloc_failure;
653 			if (cache_flags & KMF_AUDIT) {
654 				kmem_bufctl_audit_t *bcap =
655 				    (kmem_bufctl_audit_t *)bcp;
656 				bzero(bcap, sizeof (kmem_bufctl_audit_t));
657 				bcap->bc_cache = cp;
658 			}
659 			bcp->bc_addr = buf;
660 			bcp->bc_slab = sp;
661 		} else {
662 			bcp = KMEM_BUFCTL(cp, buf);
663 		}
664 		if (cache_flags & KMF_BUFTAG) {
665 			kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf);
666 			btp->bt_redzone = KMEM_REDZONE_PATTERN;
667 			btp->bt_bufctl = bcp;
668 			btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE;
669 			if (cache_flags & KMF_DEADBEEF) {
670 				copy_pattern(KMEM_FREE_PATTERN, buf,
671 				    cp->cache_verify);
672 			}
673 		}
674 		bcp->bc_next = sp->slab_head;
675 		sp->slab_head = bcp;
676 		buf += chunksize;
677 	}
678 
679 	kmem_log_event(kmem_slab_log, cp, sp, slab);
680 
681 	return (sp);
682 
683 bufctl_alloc_failure:
684 
685 	while ((bcp = sp->slab_head) != NULL) {
686 		sp->slab_head = bcp->bc_next;
687 		kmem_cache_free(cp->cache_bufctl_cache, bcp);
688 	}
689 	kmem_cache_free(kmem_slab_cache, sp);
690 
691 slab_alloc_failure:
692 
693 	vmem_free(vmp, slab, slabsize);
694 
695 vmem_alloc_failure:
696 
697 	kmem_log_event(kmem_failure_log, cp, NULL, NULL);
698 	atomic_add_64(&cp->cache_alloc_fail, 1);
699 
700 	return (NULL);
701 }
702 
703 /*
704  * Destroy a slab.
705  */
706 static void
707 kmem_slab_destroy(kmem_cache_t *cp, kmem_slab_t *sp)
708 {
709 	vmem_t *vmp = cp->cache_arena;
710 	void *slab = (void *)P2ALIGN((uintptr_t)sp->slab_base, vmp->vm_quantum);
711 
712 	if (cp->cache_flags & KMF_HASH) {
713 		kmem_bufctl_t *bcp;
714 		while ((bcp = sp->slab_head) != NULL) {
715 			sp->slab_head = bcp->bc_next;
716 			kmem_cache_free(cp->cache_bufctl_cache, bcp);
717 		}
718 		kmem_cache_free(kmem_slab_cache, sp);
719 	}
720 	vmem_free(vmp, slab, cp->cache_slabsize);
721 }
722 
723 /*
724  * Allocate a raw (unconstructed) buffer from cp's slab layer.
725  */
726 static void *
727 kmem_slab_alloc(kmem_cache_t *cp, int kmflag)
728 {
729 	kmem_bufctl_t *bcp, **hash_bucket;
730 	kmem_slab_t *sp;
731 	void *buf;
732 
733 	mutex_enter(&cp->cache_lock);
734 	cp->cache_slab_alloc++;
735 	sp = cp->cache_freelist;
736 	ASSERT(sp->slab_cache == cp);
737 	if (sp->slab_head == NULL) {
738 		/*
739 		 * The freelist is empty.  Create a new slab.
740 		 */
741 		mutex_exit(&cp->cache_lock);
742 		if ((sp = kmem_slab_create(cp, kmflag)) == NULL)
743 			return (NULL);
744 		mutex_enter(&cp->cache_lock);
745 		cp->cache_slab_create++;
746 		if ((cp->cache_buftotal += sp->slab_chunks) > cp->cache_bufmax)
747 			cp->cache_bufmax = cp->cache_buftotal;
748 		sp->slab_next = cp->cache_freelist;
749 		sp->slab_prev = cp->cache_freelist->slab_prev;
750 		sp->slab_next->slab_prev = sp;
751 		sp->slab_prev->slab_next = sp;
752 		cp->cache_freelist = sp;
753 	}
754 
755 	sp->slab_refcnt++;
756 	ASSERT(sp->slab_refcnt <= sp->slab_chunks);
757 
758 	/*
759 	 * If we're taking the last buffer in the slab,
760 	 * remove the slab from the cache's freelist.
761 	 */
762 	bcp = sp->slab_head;
763 	if ((sp->slab_head = bcp->bc_next) == NULL) {
764 		cp->cache_freelist = sp->slab_next;
765 		ASSERT(sp->slab_refcnt == sp->slab_chunks);
766 	}
767 
768 	if (cp->cache_flags & KMF_HASH) {
769 		/*
770 		 * Add buffer to allocated-address hash table.
771 		 */
772 		buf = bcp->bc_addr;
773 		hash_bucket = KMEM_HASH(cp, buf);
774 		bcp->bc_next = *hash_bucket;
775 		*hash_bucket = bcp;
776 		if ((cp->cache_flags & (KMF_AUDIT | KMF_BUFTAG)) == KMF_AUDIT) {
777 			KMEM_AUDIT(kmem_transaction_log, cp, bcp);
778 		}
779 	} else {
780 		buf = KMEM_BUF(cp, bcp);
781 	}
782 
783 	ASSERT(KMEM_SLAB_MEMBER(sp, buf));
784 
785 	mutex_exit(&cp->cache_lock);
786 
787 	return (buf);
788 }
789 
790 /*
791  * Free a raw (unconstructed) buffer to cp's slab layer.
792  */
793 static void
794 kmem_slab_free(kmem_cache_t *cp, void *buf)
795 {
796 	kmem_slab_t *sp;
797 	kmem_bufctl_t *bcp, **prev_bcpp;
798 
799 	ASSERT(buf != NULL);
800 
801 	mutex_enter(&cp->cache_lock);
802 	cp->cache_slab_free++;
803 
804 	if (cp->cache_flags & KMF_HASH) {
805 		/*
806 		 * Look up buffer in allocated-address hash table.
807 		 */
808 		prev_bcpp = KMEM_HASH(cp, buf);
809 		while ((bcp = *prev_bcpp) != NULL) {
810 			if (bcp->bc_addr == buf) {
811 				*prev_bcpp = bcp->bc_next;
812 				sp = bcp->bc_slab;
813 				break;
814 			}
815 			cp->cache_lookup_depth++;
816 			prev_bcpp = &bcp->bc_next;
817 		}
818 	} else {
819 		bcp = KMEM_BUFCTL(cp, buf);
820 		sp = KMEM_SLAB(cp, buf);
821 	}
822 
823 	if (bcp == NULL || sp->slab_cache != cp || !KMEM_SLAB_MEMBER(sp, buf)) {
824 		mutex_exit(&cp->cache_lock);
825 		kmem_error(KMERR_BADADDR, cp, buf);
826 		return;
827 	}
828 
829 	if ((cp->cache_flags & (KMF_AUDIT | KMF_BUFTAG)) == KMF_AUDIT) {
830 		if (cp->cache_flags & KMF_CONTENTS)
831 			((kmem_bufctl_audit_t *)bcp)->bc_contents =
832 			    kmem_log_enter(kmem_content_log, buf,
833 				cp->cache_contents);
834 		KMEM_AUDIT(kmem_transaction_log, cp, bcp);
835 	}
836 
837 	/*
838 	 * If this slab isn't currently on the freelist, put it there.
839 	 */
840 	if (sp->slab_head == NULL) {
841 		ASSERT(sp->slab_refcnt == sp->slab_chunks);
842 		ASSERT(cp->cache_freelist != sp);
843 		sp->slab_next->slab_prev = sp->slab_prev;
844 		sp->slab_prev->slab_next = sp->slab_next;
845 		sp->slab_next = cp->cache_freelist;
846 		sp->slab_prev = cp->cache_freelist->slab_prev;
847 		sp->slab_next->slab_prev = sp;
848 		sp->slab_prev->slab_next = sp;
849 		cp->cache_freelist = sp;
850 	}
851 
852 	bcp->bc_next = sp->slab_head;
853 	sp->slab_head = bcp;
854 
855 	ASSERT(sp->slab_refcnt >= 1);
856 	if (--sp->slab_refcnt == 0) {
857 		/*
858 		 * There are no outstanding allocations from this slab,
859 		 * so we can reclaim the memory.
860 		 */
861 		sp->slab_next->slab_prev = sp->slab_prev;
862 		sp->slab_prev->slab_next = sp->slab_next;
863 		if (sp == cp->cache_freelist)
864 			cp->cache_freelist = sp->slab_next;
865 		cp->cache_slab_destroy++;
866 		cp->cache_buftotal -= sp->slab_chunks;
867 		mutex_exit(&cp->cache_lock);
868 		kmem_slab_destroy(cp, sp);
869 		return;
870 	}
871 	mutex_exit(&cp->cache_lock);
872 }
873 
874 static int
875 kmem_cache_alloc_debug(kmem_cache_t *cp, void *buf, int kmflag, int construct,
876     caddr_t caller)
877 {
878 	kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf);
879 	kmem_bufctl_audit_t *bcp = (kmem_bufctl_audit_t *)btp->bt_bufctl;
880 	uint32_t mtbf;
881 
882 	if (btp->bt_bxstat != ((intptr_t)bcp ^ KMEM_BUFTAG_FREE)) {
883 		kmem_error(KMERR_BADBUFTAG, cp, buf);
884 		return (-1);
885 	}
886 
887 	btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_ALLOC;
888 
889 	if ((cp->cache_flags & KMF_HASH) && bcp->bc_addr != buf) {
890 		kmem_error(KMERR_BADBUFCTL, cp, buf);
891 		return (-1);
892 	}
893 
894 	if (cp->cache_flags & KMF_DEADBEEF) {
895 		if (!construct && (cp->cache_flags & KMF_LITE)) {
896 			if (*(uint64_t *)buf != KMEM_FREE_PATTERN) {
897 				kmem_error(KMERR_MODIFIED, cp, buf);
898 				return (-1);
899 			}
900 			if (cp->cache_constructor != NULL)
901 				*(uint64_t *)buf = btp->bt_redzone;
902 			else
903 				*(uint64_t *)buf = KMEM_UNINITIALIZED_PATTERN;
904 		} else {
905 			construct = 1;
906 			if (verify_and_copy_pattern(KMEM_FREE_PATTERN,
907 			    KMEM_UNINITIALIZED_PATTERN, buf,
908 			    cp->cache_verify)) {
909 				kmem_error(KMERR_MODIFIED, cp, buf);
910 				return (-1);
911 			}
912 		}
913 	}
914 	btp->bt_redzone = KMEM_REDZONE_PATTERN;
915 
916 	if ((mtbf = kmem_mtbf | cp->cache_mtbf) != 0 &&
917 	    gethrtime() % mtbf == 0 &&
918 	    (kmflag & (KM_NOSLEEP | KM_PANIC)) == KM_NOSLEEP) {
919 		kmem_log_event(kmem_failure_log, cp, NULL, NULL);
920 		if (!construct && cp->cache_destructor != NULL)
921 			cp->cache_destructor(buf, cp->cache_private);
922 	} else {
923 		mtbf = 0;
924 	}
925 
926 	if (mtbf || (construct && cp->cache_constructor != NULL &&
927 	    cp->cache_constructor(buf, cp->cache_private, kmflag) != 0)) {
928 		atomic_add_64(&cp->cache_alloc_fail, 1);
929 		btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE;
930 		if (cp->cache_flags & KMF_DEADBEEF)
931 			copy_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify);
932 		kmem_slab_free(cp, buf);
933 		return (-1);
934 	}
935 
936 	if (cp->cache_flags & KMF_AUDIT) {
937 		KMEM_AUDIT(kmem_transaction_log, cp, bcp);
938 	}
939 
940 	if ((cp->cache_flags & KMF_LITE) &&
941 	    !(cp->cache_cflags & KMC_KMEM_ALLOC)) {
942 		KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, caller);
943 	}
944 
945 	return (0);
946 }
947 
948 static int
949 kmem_cache_free_debug(kmem_cache_t *cp, void *buf, caddr_t caller)
950 {
951 	kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf);
952 	kmem_bufctl_audit_t *bcp = (kmem_bufctl_audit_t *)btp->bt_bufctl;
953 	kmem_slab_t *sp;
954 
955 	if (btp->bt_bxstat != ((intptr_t)bcp ^ KMEM_BUFTAG_ALLOC)) {
956 		if (btp->bt_bxstat == ((intptr_t)bcp ^ KMEM_BUFTAG_FREE)) {
957 			kmem_error(KMERR_DUPFREE, cp, buf);
958 			return (-1);
959 		}
960 		sp = kmem_findslab(cp, buf);
961 		if (sp == NULL || sp->slab_cache != cp)
962 			kmem_error(KMERR_BADADDR, cp, buf);
963 		else
964 			kmem_error(KMERR_REDZONE, cp, buf);
965 		return (-1);
966 	}
967 
968 	btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE;
969 
970 	if ((cp->cache_flags & KMF_HASH) && bcp->bc_addr != buf) {
971 		kmem_error(KMERR_BADBUFCTL, cp, buf);
972 		return (-1);
973 	}
974 
975 	if (btp->bt_redzone != KMEM_REDZONE_PATTERN) {
976 		kmem_error(KMERR_REDZONE, cp, buf);
977 		return (-1);
978 	}
979 
980 	if (cp->cache_flags & KMF_AUDIT) {
981 		if (cp->cache_flags & KMF_CONTENTS)
982 			bcp->bc_contents = kmem_log_enter(kmem_content_log,
983 			    buf, cp->cache_contents);
984 		KMEM_AUDIT(kmem_transaction_log, cp, bcp);
985 	}
986 
987 	if ((cp->cache_flags & KMF_LITE) &&
988 	    !(cp->cache_cflags & KMC_KMEM_ALLOC)) {
989 		KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, caller);
990 	}
991 
992 	if (cp->cache_flags & KMF_DEADBEEF) {
993 		if (cp->cache_flags & KMF_LITE)
994 			btp->bt_redzone = *(uint64_t *)buf;
995 		else if (cp->cache_destructor != NULL)
996 			cp->cache_destructor(buf, cp->cache_private);
997 
998 		copy_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify);
999 	}
1000 
1001 	return (0);
1002 }
1003 
1004 /*
1005  * Free each object in magazine mp to cp's slab layer, and free mp itself.
1006  */
1007 static void
1008 kmem_magazine_destroy(kmem_cache_t *cp, kmem_magazine_t *mp, int nrounds)
1009 {
1010 	int round;
1011 
1012 	ASSERT(cp->cache_next == NULL || taskq_member(kmem_taskq, curthread));
1013 
1014 	for (round = 0; round < nrounds; round++) {
1015 		void *buf = mp->mag_round[round];
1016 
1017 		if (cp->cache_flags & KMF_DEADBEEF) {
1018 			if (verify_pattern(KMEM_FREE_PATTERN, buf,
1019 			    cp->cache_verify) != NULL) {
1020 				kmem_error(KMERR_MODIFIED, cp, buf);
1021 				continue;
1022 			}
1023 			if ((cp->cache_flags & KMF_LITE) &&
1024 			    cp->cache_destructor != NULL) {
1025 				kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf);
1026 				*(uint64_t *)buf = btp->bt_redzone;
1027 				cp->cache_destructor(buf, cp->cache_private);
1028 				*(uint64_t *)buf = KMEM_FREE_PATTERN;
1029 			}
1030 		} else if (cp->cache_destructor != NULL) {
1031 			cp->cache_destructor(buf, cp->cache_private);
1032 		}
1033 
1034 		kmem_slab_free(cp, buf);
1035 	}
1036 	ASSERT(KMEM_MAGAZINE_VALID(cp, mp));
1037 	kmem_cache_free(cp->cache_magtype->mt_cache, mp);
1038 }
1039 
1040 /*
1041  * Allocate a magazine from the depot.
1042  */
1043 static kmem_magazine_t *
1044 kmem_depot_alloc(kmem_cache_t *cp, kmem_maglist_t *mlp)
1045 {
1046 	kmem_magazine_t *mp;
1047 
1048 	/*
1049 	 * If we can't get the depot lock without contention,
1050 	 * update our contention count.  We use the depot
1051 	 * contention rate to determine whether we need to
1052 	 * increase the magazine size for better scalability.
1053 	 */
1054 	if (!mutex_tryenter(&cp->cache_depot_lock)) {
1055 		mutex_enter(&cp->cache_depot_lock);
1056 		cp->cache_depot_contention++;
1057 	}
1058 
1059 	if ((mp = mlp->ml_list) != NULL) {
1060 		ASSERT(KMEM_MAGAZINE_VALID(cp, mp));
1061 		mlp->ml_list = mp->mag_next;
1062 		if (--mlp->ml_total < mlp->ml_min)
1063 			mlp->ml_min = mlp->ml_total;
1064 		mlp->ml_alloc++;
1065 	}
1066 
1067 	mutex_exit(&cp->cache_depot_lock);
1068 
1069 	return (mp);
1070 }
1071 
1072 /*
1073  * Free a magazine to the depot.
1074  */
1075 static void
1076 kmem_depot_free(kmem_cache_t *cp, kmem_maglist_t *mlp, kmem_magazine_t *mp)
1077 {
1078 	mutex_enter(&cp->cache_depot_lock);
1079 	ASSERT(KMEM_MAGAZINE_VALID(cp, mp));
1080 	mp->mag_next = mlp->ml_list;
1081 	mlp->ml_list = mp;
1082 	mlp->ml_total++;
1083 	mutex_exit(&cp->cache_depot_lock);
1084 }
1085 
1086 /*
1087  * Update the working set statistics for cp's depot.
1088  */
1089 static void
1090 kmem_depot_ws_update(kmem_cache_t *cp)
1091 {
1092 	mutex_enter(&cp->cache_depot_lock);
1093 	cp->cache_full.ml_reaplimit = cp->cache_full.ml_min;
1094 	cp->cache_full.ml_min = cp->cache_full.ml_total;
1095 	cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_min;
1096 	cp->cache_empty.ml_min = cp->cache_empty.ml_total;
1097 	mutex_exit(&cp->cache_depot_lock);
1098 }
1099 
1100 /*
1101  * Reap all magazines that have fallen out of the depot's working set.
1102  */
1103 static void
1104 kmem_depot_ws_reap(kmem_cache_t *cp)
1105 {
1106 	long reap;
1107 	kmem_magazine_t *mp;
1108 
1109 	ASSERT(cp->cache_next == NULL || taskq_member(kmem_taskq, curthread));
1110 
1111 	reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min);
1112 	while (reap-- && (mp = kmem_depot_alloc(cp, &cp->cache_full)) != NULL)
1113 		kmem_magazine_destroy(cp, mp, cp->cache_magtype->mt_magsize);
1114 
1115 	reap = MIN(cp->cache_empty.ml_reaplimit, cp->cache_empty.ml_min);
1116 	while (reap-- && (mp = kmem_depot_alloc(cp, &cp->cache_empty)) != NULL)
1117 		kmem_magazine_destroy(cp, mp, 0);
1118 }
1119 
1120 static void
1121 kmem_cpu_reload(kmem_cpu_cache_t *ccp, kmem_magazine_t *mp, int rounds)
1122 {
1123 	ASSERT((ccp->cc_loaded == NULL && ccp->cc_rounds == -1) ||
1124 	    (ccp->cc_loaded && ccp->cc_rounds + rounds == ccp->cc_magsize));
1125 	ASSERT(ccp->cc_magsize > 0);
1126 
1127 	ccp->cc_ploaded = ccp->cc_loaded;
1128 	ccp->cc_prounds = ccp->cc_rounds;
1129 	ccp->cc_loaded = mp;
1130 	ccp->cc_rounds = rounds;
1131 }
1132 
1133 /*
1134  * Allocate a constructed object from cache cp.
1135  */
1136 void *
1137 kmem_cache_alloc(kmem_cache_t *cp, int kmflag)
1138 {
1139 	kmem_cpu_cache_t *ccp = KMEM_CPU_CACHE(cp);
1140 	kmem_magazine_t *fmp;
1141 	void *buf;
1142 
1143 	mutex_enter(&ccp->cc_lock);
1144 	for (;;) {
1145 		/*
1146 		 * If there's an object available in the current CPU's
1147 		 * loaded magazine, just take it and return.
1148 		 */
1149 		if (ccp->cc_rounds > 0) {
1150 			buf = ccp->cc_loaded->mag_round[--ccp->cc_rounds];
1151 			ccp->cc_alloc++;
1152 			mutex_exit(&ccp->cc_lock);
1153 			if ((ccp->cc_flags & KMF_BUFTAG) &&
1154 			    kmem_cache_alloc_debug(cp, buf, kmflag, 0,
1155 			    caller()) == -1) {
1156 				if (kmflag & KM_NOSLEEP)
1157 					return (NULL);
1158 				mutex_enter(&ccp->cc_lock);
1159 				continue;
1160 			}
1161 			return (buf);
1162 		}
1163 
1164 		/*
1165 		 * The loaded magazine is empty.  If the previously loaded
1166 		 * magazine was full, exchange them and try again.
1167 		 */
1168 		if (ccp->cc_prounds > 0) {
1169 			kmem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds);
1170 			continue;
1171 		}
1172 
1173 		/*
1174 		 * If the magazine layer is disabled, break out now.
1175 		 */
1176 		if (ccp->cc_magsize == 0)
1177 			break;
1178 
1179 		/*
1180 		 * Try to get a full magazine from the depot.
1181 		 */
1182 		fmp = kmem_depot_alloc(cp, &cp->cache_full);
1183 		if (fmp != NULL) {
1184 			if (ccp->cc_ploaded != NULL)
1185 				kmem_depot_free(cp, &cp->cache_empty,
1186 				    ccp->cc_ploaded);
1187 			kmem_cpu_reload(ccp, fmp, ccp->cc_magsize);
1188 			continue;
1189 		}
1190 
1191 		/*
1192 		 * There are no full magazines in the depot,
1193 		 * so fall through to the slab layer.
1194 		 */
1195 		break;
1196 	}
1197 	mutex_exit(&ccp->cc_lock);
1198 
1199 	/*
1200 	 * We couldn't allocate a constructed object from the magazine layer,
1201 	 * so get a raw buffer from the slab layer and apply its constructor.
1202 	 */
1203 	buf = kmem_slab_alloc(cp, kmflag);
1204 
1205 	if (buf == NULL)
1206 		return (NULL);
1207 
1208 	if (cp->cache_flags & KMF_BUFTAG) {
1209 		/*
1210 		 * Make kmem_cache_alloc_debug() apply the constructor for us.
1211 		 */
1212 		if (kmem_cache_alloc_debug(cp, buf, kmflag, 1,
1213 		    caller()) == -1) {
1214 			if (kmflag & KM_NOSLEEP)
1215 				return (NULL);
1216 			/*
1217 			 * kmem_cache_alloc_debug() detected corruption
1218 			 * but didn't panic (kmem_panic <= 0).  Try again.
1219 			 */
1220 			return (kmem_cache_alloc(cp, kmflag));
1221 		}
1222 		return (buf);
1223 	}
1224 
1225 	if (cp->cache_constructor != NULL &&
1226 	    cp->cache_constructor(buf, cp->cache_private, kmflag) != 0) {
1227 		atomic_add_64(&cp->cache_alloc_fail, 1);
1228 		kmem_slab_free(cp, buf);
1229 		return (NULL);
1230 	}
1231 
1232 	return (buf);
1233 }
1234 
1235 /*
1236  * Free a constructed object to cache cp.
1237  */
1238 void
1239 kmem_cache_free(kmem_cache_t *cp, void *buf)
1240 {
1241 	kmem_cpu_cache_t *ccp = KMEM_CPU_CACHE(cp);
1242 	kmem_magazine_t *emp;
1243 	kmem_magtype_t *mtp;
1244 
1245 	if (ccp->cc_flags & KMF_BUFTAG)
1246 		if (kmem_cache_free_debug(cp, buf, caller()) == -1)
1247 			return;
1248 
1249 	mutex_enter(&ccp->cc_lock);
1250 	for (;;) {
1251 		/*
1252 		 * If there's a slot available in the current CPU's
1253 		 * loaded magazine, just put the object there and return.
1254 		 */
1255 		if ((uint_t)ccp->cc_rounds < ccp->cc_magsize) {
1256 			ccp->cc_loaded->mag_round[ccp->cc_rounds++] = buf;
1257 			ccp->cc_free++;
1258 			mutex_exit(&ccp->cc_lock);
1259 			return;
1260 		}
1261 
1262 		/*
1263 		 * The loaded magazine is full.  If the previously loaded
1264 		 * magazine was empty, exchange them and try again.
1265 		 */
1266 		if (ccp->cc_prounds == 0) {
1267 			kmem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds);
1268 			continue;
1269 		}
1270 
1271 		/*
1272 		 * If the magazine layer is disabled, break out now.
1273 		 */
1274 		if (ccp->cc_magsize == 0)
1275 			break;
1276 
1277 		/*
1278 		 * Try to get an empty magazine from the depot.
1279 		 */
1280 		emp = kmem_depot_alloc(cp, &cp->cache_empty);
1281 		if (emp != NULL) {
1282 			if (ccp->cc_ploaded != NULL)
1283 				kmem_depot_free(cp, &cp->cache_full,
1284 				    ccp->cc_ploaded);
1285 			kmem_cpu_reload(ccp, emp, 0);
1286 			continue;
1287 		}
1288 
1289 		/*
1290 		 * There are no empty magazines in the depot,
1291 		 * so try to allocate a new one.  We must drop all locks
1292 		 * across kmem_cache_alloc() because lower layers may
1293 		 * attempt to allocate from this cache.
1294 		 */
1295 		mtp = cp->cache_magtype;
1296 		mutex_exit(&ccp->cc_lock);
1297 		emp = kmem_cache_alloc(mtp->mt_cache, KM_NOSLEEP);
1298 		mutex_enter(&ccp->cc_lock);
1299 
1300 		if (emp != NULL) {
1301 			/*
1302 			 * We successfully allocated an empty magazine.
1303 			 * However, we had to drop ccp->cc_lock to do it,
1304 			 * so the cache's magazine size may have changed.
1305 			 * If so, free the magazine and try again.
1306 			 */
1307 			if (ccp->cc_magsize != mtp->mt_magsize) {
1308 				mutex_exit(&ccp->cc_lock);
1309 				kmem_cache_free(mtp->mt_cache, emp);
1310 				mutex_enter(&ccp->cc_lock);
1311 				continue;
1312 			}
1313 
1314 			/*
1315 			 * We got a magazine of the right size.  Add it to
1316 			 * the depot and try the whole dance again.
1317 			 */
1318 			kmem_depot_free(cp, &cp->cache_empty, emp);
1319 			continue;
1320 		}
1321 
1322 		/*
1323 		 * We couldn't allocate an empty magazine,
1324 		 * so fall through to the slab layer.
1325 		 */
1326 		break;
1327 	}
1328 	mutex_exit(&ccp->cc_lock);
1329 
1330 	/*
1331 	 * We couldn't free our constructed object to the magazine layer,
1332 	 * so apply its destructor and free it to the slab layer.
1333 	 * Note that if KMF_DEADBEEF is in effect and KMF_LITE is not,
1334 	 * kmem_cache_free_debug() will have already applied the destructor.
1335 	 */
1336 	if ((cp->cache_flags & (KMF_DEADBEEF | KMF_LITE)) != KMF_DEADBEEF &&
1337 	    cp->cache_destructor != NULL) {
1338 		if (cp->cache_flags & KMF_DEADBEEF) {	/* KMF_LITE implied */
1339 			kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf);
1340 			*(uint64_t *)buf = btp->bt_redzone;
1341 			cp->cache_destructor(buf, cp->cache_private);
1342 			*(uint64_t *)buf = KMEM_FREE_PATTERN;
1343 		} else {
1344 			cp->cache_destructor(buf, cp->cache_private);
1345 		}
1346 	}
1347 
1348 	kmem_slab_free(cp, buf);
1349 }
1350 
1351 void *
1352 kmem_zalloc(size_t size, int kmflag)
1353 {
1354 	size_t index = (size - 1) >> KMEM_ALIGN_SHIFT;
1355 	void *buf;
1356 
1357 	if (index < KMEM_MAXBUF >> KMEM_ALIGN_SHIFT) {
1358 		kmem_cache_t *cp = kmem_alloc_table[index];
1359 		buf = kmem_cache_alloc(cp, kmflag);
1360 		if (buf != NULL) {
1361 			if (cp->cache_flags & KMF_BUFTAG) {
1362 				kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf);
1363 				((uint8_t *)buf)[size] = KMEM_REDZONE_BYTE;
1364 				((uint32_t *)btp)[1] = KMEM_SIZE_ENCODE(size);
1365 
1366 				if (cp->cache_flags & KMF_LITE) {
1367 					KMEM_BUFTAG_LITE_ENTER(btp,
1368 					    kmem_lite_count, caller());
1369 				}
1370 			}
1371 			bzero(buf, size);
1372 		}
1373 	} else {
1374 		buf = kmem_alloc(size, kmflag);
1375 		if (buf != NULL)
1376 			bzero(buf, size);
1377 	}
1378 	return (buf);
1379 }
1380 
1381 void *
1382 kmem_alloc(size_t size, int kmflag)
1383 {
1384 	size_t index = (size - 1) >> KMEM_ALIGN_SHIFT;
1385 	void *buf;
1386 
1387 	if (index < KMEM_MAXBUF >> KMEM_ALIGN_SHIFT) {
1388 		kmem_cache_t *cp = kmem_alloc_table[index];
1389 		buf = kmem_cache_alloc(cp, kmflag);
1390 		if ((cp->cache_flags & KMF_BUFTAG) && buf != NULL) {
1391 			kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf);
1392 			((uint8_t *)buf)[size] = KMEM_REDZONE_BYTE;
1393 			((uint32_t *)btp)[1] = KMEM_SIZE_ENCODE(size);
1394 
1395 			if (cp->cache_flags & KMF_LITE) {
1396 				KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count,
1397 				    caller());
1398 			}
1399 		}
1400 		return (buf);
1401 	}
1402 	if (size == 0)
1403 		return (NULL);
1404 	buf = vmem_alloc(kmem_oversize_arena, size, kmflag & KM_VMFLAGS);
1405 	if (buf == NULL)
1406 		kmem_log_event(kmem_failure_log, NULL, NULL, (void *)size);
1407 	return (buf);
1408 }
1409 
1410 void
1411 kmem_free(void *buf, size_t size)
1412 {
1413 	size_t index = (size - 1) >> KMEM_ALIGN_SHIFT;
1414 
1415 	if (index < KMEM_MAXBUF >> KMEM_ALIGN_SHIFT) {
1416 		kmem_cache_t *cp = kmem_alloc_table[index];
1417 		if (cp->cache_flags & KMF_BUFTAG) {
1418 			kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf);
1419 			uint32_t *ip = (uint32_t *)btp;
1420 			if (ip[1] != KMEM_SIZE_ENCODE(size)) {
1421 				if (*(uint64_t *)buf == KMEM_FREE_PATTERN) {
1422 					kmem_error(KMERR_DUPFREE, cp, buf);
1423 					return;
1424 				}
1425 				if (KMEM_SIZE_VALID(ip[1])) {
1426 					ip[0] = KMEM_SIZE_ENCODE(size);
1427 					kmem_error(KMERR_BADSIZE, cp, buf);
1428 				} else {
1429 					kmem_error(KMERR_REDZONE, cp, buf);
1430 				}
1431 				return;
1432 			}
1433 			if (((uint8_t *)buf)[size] != KMEM_REDZONE_BYTE) {
1434 				kmem_error(KMERR_REDZONE, cp, buf);
1435 				return;
1436 			}
1437 			btp->bt_redzone = KMEM_REDZONE_PATTERN;
1438 			if (cp->cache_flags & KMF_LITE) {
1439 				KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count,
1440 				    caller());
1441 			}
1442 		}
1443 		kmem_cache_free(cp, buf);
1444 	} else {
1445 		if (buf == NULL && size == 0)
1446 			return;
1447 		vmem_free(kmem_oversize_arena, buf, size);
1448 	}
1449 }
1450 
1451 void *
1452 kmem_firewall_va_alloc(vmem_t *vmp, size_t size, int vmflag)
1453 {
1454 	size_t realsize = size + vmp->vm_quantum;
1455 	void *addr;
1456 
1457 	/*
1458 	 * Annoying edge case: if 'size' is just shy of ULONG_MAX, adding
1459 	 * vm_quantum will cause integer wraparound.  Check for this, and
1460 	 * blow off the firewall page in this case.  Note that such a
1461 	 * giant allocation (the entire kernel address space) can never
1462 	 * be satisfied, so it will either fail immediately (VM_NOSLEEP)
1463 	 * or sleep forever (VM_SLEEP).  Thus, there is no need for a
1464 	 * corresponding check in kmem_firewall_va_free().
1465 	 */
1466 	if (realsize < size)
1467 		realsize = size;
1468 
1469 	/*
1470 	 * While boot still owns resource management, make sure that this
1471 	 * redzone virtual address allocation is properly accounted for in
1472 	 * OBPs "virtual-memory" "available" lists because we're
1473 	 * effectively claiming them for a red zone.  If we don't do this,
1474 	 * the available lists become too fragmented and too large for the
1475 	 * current boot/kernel memory list interface.
1476 	 */
1477 	addr = vmem_alloc(vmp, realsize, vmflag | VM_NEXTFIT);
1478 
1479 	if (addr != NULL && kvseg.s_base == NULL && realsize != size)
1480 		(void) boot_virt_alloc((char *)addr + size, vmp->vm_quantum);
1481 
1482 	return (addr);
1483 }
1484 
1485 void
1486 kmem_firewall_va_free(vmem_t *vmp, void *addr, size_t size)
1487 {
1488 	ASSERT((kvseg.s_base == NULL ?
1489 	    va_to_pfn((char *)addr + size) :
1490 	    hat_getpfnum(kas.a_hat, (caddr_t)addr + size)) == PFN_INVALID);
1491 
1492 	vmem_free(vmp, addr, size + vmp->vm_quantum);
1493 }
1494 
1495 /*
1496  * Try to allocate at least `size' bytes of memory without sleeping or
1497  * panicking. Return actual allocated size in `asize'. If allocation failed,
1498  * try final allocation with sleep or panic allowed.
1499  */
1500 void *
1501 kmem_alloc_tryhard(size_t size, size_t *asize, int kmflag)
1502 {
1503 	void *p;
1504 
1505 	*asize = P2ROUNDUP(size, KMEM_ALIGN);
1506 	do {
1507 		p = kmem_alloc(*asize, (kmflag | KM_NOSLEEP) & ~KM_PANIC);
1508 		if (p != NULL)
1509 			return (p);
1510 		*asize += KMEM_ALIGN;
1511 	} while (*asize <= PAGESIZE);
1512 
1513 	*asize = P2ROUNDUP(size, KMEM_ALIGN);
1514 	return (kmem_alloc(*asize, kmflag));
1515 }
1516 
1517 /*
1518  * Reclaim all unused memory from a cache.
1519  */
1520 static void
1521 kmem_cache_reap(kmem_cache_t *cp)
1522 {
1523 	/*
1524 	 * Ask the cache's owner to free some memory if possible.
1525 	 * The idea is to handle things like the inode cache, which
1526 	 * typically sits on a bunch of memory that it doesn't truly
1527 	 * *need*.  Reclaim policy is entirely up to the owner; this
1528 	 * callback is just an advisory plea for help.
1529 	 */
1530 	if (cp->cache_reclaim != NULL)
1531 		cp->cache_reclaim(cp->cache_private);
1532 
1533 	kmem_depot_ws_reap(cp);
1534 }
1535 
1536 static void
1537 kmem_reap_timeout(void *flag_arg)
1538 {
1539 	uint32_t *flag = (uint32_t *)flag_arg;
1540 
1541 	ASSERT(flag == &kmem_reaping || flag == &kmem_reaping_idspace);
1542 	*flag = 0;
1543 }
1544 
1545 static void
1546 kmem_reap_done(void *flag)
1547 {
1548 	(void) timeout(kmem_reap_timeout, flag, kmem_reap_interval);
1549 }
1550 
1551 static void
1552 kmem_reap_start(void *flag)
1553 {
1554 	ASSERT(flag == &kmem_reaping || flag == &kmem_reaping_idspace);
1555 
1556 	if (flag == &kmem_reaping) {
1557 		kmem_cache_applyall(kmem_cache_reap, kmem_taskq, TQ_NOSLEEP);
1558 		/*
1559 		 * if we have segkp under heap, reap segkp cache.
1560 		 */
1561 		if (segkp_fromheap)
1562 			segkp_cache_free();
1563 	}
1564 	else
1565 		kmem_cache_applyall_id(kmem_cache_reap, kmem_taskq, TQ_NOSLEEP);
1566 
1567 	/*
1568 	 * We use taskq_dispatch() to schedule a timeout to clear
1569 	 * the flag so that kmem_reap() becomes self-throttling:
1570 	 * we won't reap again until the current reap completes *and*
1571 	 * at least kmem_reap_interval ticks have elapsed.
1572 	 */
1573 	if (!taskq_dispatch(kmem_taskq, kmem_reap_done, flag, TQ_NOSLEEP))
1574 		kmem_reap_done(flag);
1575 }
1576 
1577 static void
1578 kmem_reap_common(void *flag_arg)
1579 {
1580 	uint32_t *flag = (uint32_t *)flag_arg;
1581 
1582 	if (MUTEX_HELD(&kmem_cache_lock) || kmem_taskq == NULL ||
1583 	    cas32(flag, 0, 1) != 0)
1584 		return;
1585 
1586 	/*
1587 	 * It may not be kosher to do memory allocation when a reap is called
1588 	 * is called (for example, if vmem_populate() is in the call chain).
1589 	 * So we start the reap going with a TQ_NOALLOC dispatch.  If the
1590 	 * dispatch fails, we reset the flag, and the next reap will try again.
1591 	 */
1592 	if (!taskq_dispatch(kmem_taskq, kmem_reap_start, flag, TQ_NOALLOC))
1593 		*flag = 0;
1594 }
1595 
1596 /*
1597  * Reclaim all unused memory from all caches.  Called from the VM system
1598  * when memory gets tight.
1599  */
1600 void
1601 kmem_reap(void)
1602 {
1603 	kmem_reap_common(&kmem_reaping);
1604 }
1605 
1606 /*
1607  * Reclaim all unused memory from identifier arenas, called when a vmem
1608  * arena not back by memory is exhausted.  Since reaping memory-backed caches
1609  * cannot help with identifier exhaustion, we avoid both a large amount of
1610  * work and unwanted side-effects from reclaim callbacks.
1611  */
1612 void
1613 kmem_reap_idspace(void)
1614 {
1615 	kmem_reap_common(&kmem_reaping_idspace);
1616 }
1617 
1618 /*
1619  * Purge all magazines from a cache and set its magazine limit to zero.
1620  * All calls are serialized by the kmem_taskq lock, except for the final
1621  * call from kmem_cache_destroy().
1622  */
1623 static void
1624 kmem_cache_magazine_purge(kmem_cache_t *cp)
1625 {
1626 	kmem_cpu_cache_t *ccp;
1627 	kmem_magazine_t *mp, *pmp;
1628 	int rounds, prounds, cpu_seqid;
1629 
1630 	ASSERT(cp->cache_next == NULL || taskq_member(kmem_taskq, curthread));
1631 	ASSERT(MUTEX_NOT_HELD(&cp->cache_lock));
1632 
1633 	for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) {
1634 		ccp = &cp->cache_cpu[cpu_seqid];
1635 
1636 		mutex_enter(&ccp->cc_lock);
1637 		mp = ccp->cc_loaded;
1638 		pmp = ccp->cc_ploaded;
1639 		rounds = ccp->cc_rounds;
1640 		prounds = ccp->cc_prounds;
1641 		ccp->cc_loaded = NULL;
1642 		ccp->cc_ploaded = NULL;
1643 		ccp->cc_rounds = -1;
1644 		ccp->cc_prounds = -1;
1645 		ccp->cc_magsize = 0;
1646 		mutex_exit(&ccp->cc_lock);
1647 
1648 		if (mp)
1649 			kmem_magazine_destroy(cp, mp, rounds);
1650 		if (pmp)
1651 			kmem_magazine_destroy(cp, pmp, prounds);
1652 	}
1653 
1654 	/*
1655 	 * Updating the working set statistics twice in a row has the
1656 	 * effect of setting the working set size to zero, so everything
1657 	 * is eligible for reaping.
1658 	 */
1659 	kmem_depot_ws_update(cp);
1660 	kmem_depot_ws_update(cp);
1661 
1662 	kmem_depot_ws_reap(cp);
1663 }
1664 
1665 /*
1666  * Enable per-cpu magazines on a cache.
1667  */
1668 static void
1669 kmem_cache_magazine_enable(kmem_cache_t *cp)
1670 {
1671 	int cpu_seqid;
1672 
1673 	if (cp->cache_flags & KMF_NOMAGAZINE)
1674 		return;
1675 
1676 	for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) {
1677 		kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid];
1678 		mutex_enter(&ccp->cc_lock);
1679 		ccp->cc_magsize = cp->cache_magtype->mt_magsize;
1680 		mutex_exit(&ccp->cc_lock);
1681 	}
1682 
1683 }
1684 
1685 /*
1686  * Reap (almost) everything right now.  See kmem_cache_magazine_purge()
1687  * for explanation of the back-to-back kmem_depot_ws_update() calls.
1688  */
1689 void
1690 kmem_cache_reap_now(kmem_cache_t *cp)
1691 {
1692 	kmem_depot_ws_update(cp);
1693 	kmem_depot_ws_update(cp);
1694 
1695 	(void) taskq_dispatch(kmem_taskq,
1696 	    (task_func_t *)kmem_depot_ws_reap, cp, TQ_SLEEP);
1697 	taskq_wait(kmem_taskq);
1698 }
1699 
1700 /*
1701  * Recompute a cache's magazine size.  The trade-off is that larger magazines
1702  * provide a higher transfer rate with the depot, while smaller magazines
1703  * reduce memory consumption.  Magazine resizing is an expensive operation;
1704  * it should not be done frequently.
1705  *
1706  * Changes to the magazine size are serialized by the kmem_taskq lock.
1707  *
1708  * Note: at present this only grows the magazine size.  It might be useful
1709  * to allow shrinkage too.
1710  */
1711 static void
1712 kmem_cache_magazine_resize(kmem_cache_t *cp)
1713 {
1714 	kmem_magtype_t *mtp = cp->cache_magtype;
1715 
1716 	ASSERT(taskq_member(kmem_taskq, curthread));
1717 
1718 	if (cp->cache_chunksize < mtp->mt_maxbuf) {
1719 		kmem_cache_magazine_purge(cp);
1720 		mutex_enter(&cp->cache_depot_lock);
1721 		cp->cache_magtype = ++mtp;
1722 		cp->cache_depot_contention_prev =
1723 		    cp->cache_depot_contention + INT_MAX;
1724 		mutex_exit(&cp->cache_depot_lock);
1725 		kmem_cache_magazine_enable(cp);
1726 	}
1727 }
1728 
1729 /*
1730  * Rescale a cache's hash table, so that the table size is roughly the
1731  * cache size.  We want the average lookup time to be extremely small.
1732  */
1733 static void
1734 kmem_hash_rescale(kmem_cache_t *cp)
1735 {
1736 	kmem_bufctl_t **old_table, **new_table, *bcp;
1737 	size_t old_size, new_size, h;
1738 
1739 	ASSERT(taskq_member(kmem_taskq, curthread));
1740 
1741 	new_size = MAX(KMEM_HASH_INITIAL,
1742 	    1 << (highbit(3 * cp->cache_buftotal + 4) - 2));
1743 	old_size = cp->cache_hash_mask + 1;
1744 
1745 	if ((old_size >> 1) <= new_size && new_size <= (old_size << 1))
1746 		return;
1747 
1748 	new_table = vmem_alloc(kmem_hash_arena, new_size * sizeof (void *),
1749 	    VM_NOSLEEP);
1750 	if (new_table == NULL)
1751 		return;
1752 	bzero(new_table, new_size * sizeof (void *));
1753 
1754 	mutex_enter(&cp->cache_lock);
1755 
1756 	old_size = cp->cache_hash_mask + 1;
1757 	old_table = cp->cache_hash_table;
1758 
1759 	cp->cache_hash_mask = new_size - 1;
1760 	cp->cache_hash_table = new_table;
1761 	cp->cache_rescale++;
1762 
1763 	for (h = 0; h < old_size; h++) {
1764 		bcp = old_table[h];
1765 		while (bcp != NULL) {
1766 			void *addr = bcp->bc_addr;
1767 			kmem_bufctl_t *next_bcp = bcp->bc_next;
1768 			kmem_bufctl_t **hash_bucket = KMEM_HASH(cp, addr);
1769 			bcp->bc_next = *hash_bucket;
1770 			*hash_bucket = bcp;
1771 			bcp = next_bcp;
1772 		}
1773 	}
1774 
1775 	mutex_exit(&cp->cache_lock);
1776 
1777 	vmem_free(kmem_hash_arena, old_table, old_size * sizeof (void *));
1778 }
1779 
1780 /*
1781  * Perform periodic maintenance on a cache: hash rescaling,
1782  * depot working-set update, and magazine resizing.
1783  */
1784 static void
1785 kmem_cache_update(kmem_cache_t *cp)
1786 {
1787 	int need_hash_rescale = 0;
1788 	int need_magazine_resize = 0;
1789 
1790 	ASSERT(MUTEX_HELD(&kmem_cache_lock));
1791 
1792 	/*
1793 	 * If the cache has become much larger or smaller than its hash table,
1794 	 * fire off a request to rescale the hash table.
1795 	 */
1796 	mutex_enter(&cp->cache_lock);
1797 
1798 	if ((cp->cache_flags & KMF_HASH) &&
1799 	    (cp->cache_buftotal > (cp->cache_hash_mask << 1) ||
1800 	    (cp->cache_buftotal < (cp->cache_hash_mask >> 1) &&
1801 	    cp->cache_hash_mask > KMEM_HASH_INITIAL)))
1802 		need_hash_rescale = 1;
1803 
1804 	mutex_exit(&cp->cache_lock);
1805 
1806 	/*
1807 	 * Update the depot working set statistics.
1808 	 */
1809 	kmem_depot_ws_update(cp);
1810 
1811 	/*
1812 	 * If there's a lot of contention in the depot,
1813 	 * increase the magazine size.
1814 	 */
1815 	mutex_enter(&cp->cache_depot_lock);
1816 
1817 	if (cp->cache_chunksize < cp->cache_magtype->mt_maxbuf &&
1818 	    (int)(cp->cache_depot_contention -
1819 	    cp->cache_depot_contention_prev) > kmem_depot_contention)
1820 		need_magazine_resize = 1;
1821 
1822 	cp->cache_depot_contention_prev = cp->cache_depot_contention;
1823 
1824 	mutex_exit(&cp->cache_depot_lock);
1825 
1826 	if (need_hash_rescale)
1827 		(void) taskq_dispatch(kmem_taskq,
1828 		    (task_func_t *)kmem_hash_rescale, cp, TQ_NOSLEEP);
1829 
1830 	if (need_magazine_resize)
1831 		(void) taskq_dispatch(kmem_taskq,
1832 		    (task_func_t *)kmem_cache_magazine_resize, cp, TQ_NOSLEEP);
1833 }
1834 
1835 static void
1836 kmem_update_timeout(void *dummy)
1837 {
1838 	static void kmem_update(void *);
1839 
1840 	(void) timeout(kmem_update, dummy, kmem_reap_interval);
1841 }
1842 
1843 static void
1844 kmem_update(void *dummy)
1845 {
1846 	kmem_cache_applyall(kmem_cache_update, NULL, TQ_NOSLEEP);
1847 
1848 	/*
1849 	 * We use taskq_dispatch() to reschedule the timeout so that
1850 	 * kmem_update() becomes self-throttling: it won't schedule
1851 	 * new tasks until all previous tasks have completed.
1852 	 */
1853 	if (!taskq_dispatch(kmem_taskq, kmem_update_timeout, dummy, TQ_NOSLEEP))
1854 		kmem_update_timeout(NULL);
1855 }
1856 
1857 static int
1858 kmem_cache_kstat_update(kstat_t *ksp, int rw)
1859 {
1860 	struct kmem_cache_kstat *kmcp = &kmem_cache_kstat;
1861 	kmem_cache_t *cp = ksp->ks_private;
1862 	kmem_slab_t *sp;
1863 	uint64_t cpu_buf_avail;
1864 	uint64_t buf_avail = 0;
1865 	int cpu_seqid;
1866 
1867 	ASSERT(MUTEX_HELD(&kmem_cache_kstat_lock));
1868 
1869 	if (rw == KSTAT_WRITE)
1870 		return (EACCES);
1871 
1872 	mutex_enter(&cp->cache_lock);
1873 
1874 	kmcp->kmc_alloc_fail.value.ui64		= cp->cache_alloc_fail;
1875 	kmcp->kmc_alloc.value.ui64		= cp->cache_slab_alloc;
1876 	kmcp->kmc_free.value.ui64		= cp->cache_slab_free;
1877 	kmcp->kmc_slab_alloc.value.ui64		= cp->cache_slab_alloc;
1878 	kmcp->kmc_slab_free.value.ui64		= cp->cache_slab_free;
1879 
1880 	for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) {
1881 		kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid];
1882 
1883 		mutex_enter(&ccp->cc_lock);
1884 
1885 		cpu_buf_avail = 0;
1886 		if (ccp->cc_rounds > 0)
1887 			cpu_buf_avail += ccp->cc_rounds;
1888 		if (ccp->cc_prounds > 0)
1889 			cpu_buf_avail += ccp->cc_prounds;
1890 
1891 		kmcp->kmc_alloc.value.ui64	+= ccp->cc_alloc;
1892 		kmcp->kmc_free.value.ui64	+= ccp->cc_free;
1893 		buf_avail			+= cpu_buf_avail;
1894 
1895 		mutex_exit(&ccp->cc_lock);
1896 	}
1897 
1898 	mutex_enter(&cp->cache_depot_lock);
1899 
1900 	kmcp->kmc_depot_alloc.value.ui64	= cp->cache_full.ml_alloc;
1901 	kmcp->kmc_depot_free.value.ui64		= cp->cache_empty.ml_alloc;
1902 	kmcp->kmc_depot_contention.value.ui64	= cp->cache_depot_contention;
1903 	kmcp->kmc_full_magazines.value.ui64	= cp->cache_full.ml_total;
1904 	kmcp->kmc_empty_magazines.value.ui64	= cp->cache_empty.ml_total;
1905 	kmcp->kmc_magazine_size.value.ui64	=
1906 	    (cp->cache_flags & KMF_NOMAGAZINE) ?
1907 	    0 : cp->cache_magtype->mt_magsize;
1908 
1909 	kmcp->kmc_alloc.value.ui64		+= cp->cache_full.ml_alloc;
1910 	kmcp->kmc_free.value.ui64		+= cp->cache_empty.ml_alloc;
1911 	buf_avail += cp->cache_full.ml_total * cp->cache_magtype->mt_magsize;
1912 
1913 	mutex_exit(&cp->cache_depot_lock);
1914 
1915 	kmcp->kmc_buf_size.value.ui64	= cp->cache_bufsize;
1916 	kmcp->kmc_align.value.ui64	= cp->cache_align;
1917 	kmcp->kmc_chunk_size.value.ui64	= cp->cache_chunksize;
1918 	kmcp->kmc_slab_size.value.ui64	= cp->cache_slabsize;
1919 	kmcp->kmc_buf_constructed.value.ui64 = buf_avail;
1920 	for (sp = cp->cache_freelist; sp != &cp->cache_nullslab;
1921 	    sp = sp->slab_next)
1922 		buf_avail += sp->slab_chunks - sp->slab_refcnt;
1923 	kmcp->kmc_buf_avail.value.ui64	= buf_avail;
1924 	kmcp->kmc_buf_inuse.value.ui64	= cp->cache_buftotal - buf_avail;
1925 	kmcp->kmc_buf_total.value.ui64	= cp->cache_buftotal;
1926 	kmcp->kmc_buf_max.value.ui64	= cp->cache_bufmax;
1927 	kmcp->kmc_slab_create.value.ui64	= cp->cache_slab_create;
1928 	kmcp->kmc_slab_destroy.value.ui64	= cp->cache_slab_destroy;
1929 	kmcp->kmc_hash_size.value.ui64	= (cp->cache_flags & KMF_HASH) ?
1930 	    cp->cache_hash_mask + 1 : 0;
1931 	kmcp->kmc_hash_lookup_depth.value.ui64	= cp->cache_lookup_depth;
1932 	kmcp->kmc_hash_rescale.value.ui64	= cp->cache_rescale;
1933 	kmcp->kmc_vmem_source.value.ui64	= cp->cache_arena->vm_id;
1934 
1935 	mutex_exit(&cp->cache_lock);
1936 	return (0);
1937 }
1938 
1939 /*
1940  * Return a named statistic about a particular cache.
1941  * This shouldn't be called very often, so it's currently designed for
1942  * simplicity (leverages existing kstat support) rather than efficiency.
1943  */
1944 uint64_t
1945 kmem_cache_stat(kmem_cache_t *cp, char *name)
1946 {
1947 	int i;
1948 	kstat_t *ksp = cp->cache_kstat;
1949 	kstat_named_t *knp = (kstat_named_t *)&kmem_cache_kstat;
1950 	uint64_t value = 0;
1951 
1952 	if (ksp != NULL) {
1953 		mutex_enter(&kmem_cache_kstat_lock);
1954 		(void) kmem_cache_kstat_update(ksp, KSTAT_READ);
1955 		for (i = 0; i < ksp->ks_ndata; i++) {
1956 			if (strcmp(knp[i].name, name) == 0) {
1957 				value = knp[i].value.ui64;
1958 				break;
1959 			}
1960 		}
1961 		mutex_exit(&kmem_cache_kstat_lock);
1962 	}
1963 	return (value);
1964 }
1965 
1966 /*
1967  * Return an estimate of currently available kernel heap memory.
1968  * On 32-bit systems, physical memory may exceed virtual memory,
1969  * we just truncate the result at 1GB.
1970  */
1971 size_t
1972 kmem_avail(void)
1973 {
1974 	spgcnt_t rmem = availrmem - tune.t_minarmem;
1975 	spgcnt_t fmem = freemem - minfree;
1976 
1977 	return ((size_t)ptob(MIN(MAX(MIN(rmem, fmem), 0),
1978 	    1 << (30 - PAGESHIFT))));
1979 }
1980 
1981 /*
1982  * Return the maximum amount of memory that is (in theory) allocatable
1983  * from the heap. This may be used as an estimate only since there
1984  * is no guarentee this space will still be available when an allocation
1985  * request is made, nor that the space may be allocated in one big request
1986  * due to kernel heap fragmentation.
1987  */
1988 size_t
1989 kmem_maxavail(void)
1990 {
1991 	spgcnt_t pmem = availrmem - tune.t_minarmem;
1992 	spgcnt_t vmem = btop(vmem_size(heap_arena, VMEM_FREE));
1993 
1994 	return ((size_t)ptob(MAX(MIN(pmem, vmem), 0)));
1995 }
1996 
1997 /*
1998  * Indicate whether memory-intensive kmem debugging is enabled.
1999  */
2000 int
2001 kmem_debugging(void)
2002 {
2003 	return (kmem_flags & (KMF_AUDIT | KMF_REDZONE));
2004 }
2005 
2006 kmem_cache_t *
2007 kmem_cache_create(
2008 	char *name,		/* descriptive name for this cache */
2009 	size_t bufsize,		/* size of the objects it manages */
2010 	size_t align,		/* required object alignment */
2011 	int (*constructor)(void *, void *, int), /* object constructor */
2012 	void (*destructor)(void *, void *),	/* object destructor */
2013 	void (*reclaim)(void *), /* memory reclaim callback */
2014 	void *private,		/* pass-thru arg for constr/destr/reclaim */
2015 	vmem_t *vmp,		/* vmem source for slab allocation */
2016 	int cflags)		/* cache creation flags */
2017 {
2018 	int cpu_seqid;
2019 	size_t chunksize;
2020 	kmem_cache_t *cp, *cnext, *cprev;
2021 	kmem_magtype_t *mtp;
2022 	size_t csize = KMEM_CACHE_SIZE(max_ncpus);
2023 
2024 #ifdef	DEBUG
2025 	/*
2026 	 * Cache names should conform to the rules for valid C identifiers
2027 	 */
2028 	if (!strident_valid(name)) {
2029 		cmn_err(CE_CONT,
2030 		    "kmem_cache_create: '%s' is an invalid cache name\n"
2031 		    "cache names must conform to the rules for "
2032 		    "C identifiers\n", name);
2033 	}
2034 #endif	/* DEBUG */
2035 
2036 	if (vmp == NULL)
2037 		vmp = kmem_default_arena;
2038 
2039 	/*
2040 	 * If this kmem cache has an identifier vmem arena as its source, mark
2041 	 * it such to allow kmem_reap_idspace().
2042 	 */
2043 	ASSERT(!(cflags & KMC_IDENTIFIER));   /* consumer should not set this */
2044 	if (vmp->vm_cflags & VMC_IDENTIFIER)
2045 		cflags |= KMC_IDENTIFIER;
2046 
2047 	/*
2048 	 * Get a kmem_cache structure.  We arrange that cp->cache_cpu[]
2049 	 * is aligned on a KMEM_CPU_CACHE_SIZE boundary to prevent
2050 	 * false sharing of per-CPU data.
2051 	 */
2052 	cp = vmem_xalloc(kmem_cache_arena, csize, KMEM_CPU_CACHE_SIZE,
2053 	    P2NPHASE(csize, KMEM_CPU_CACHE_SIZE), 0, NULL, NULL, VM_SLEEP);
2054 	bzero(cp, csize);
2055 
2056 	if (align == 0)
2057 		align = KMEM_ALIGN;
2058 
2059 	/*
2060 	 * If we're not at least KMEM_ALIGN aligned, we can't use free
2061 	 * memory to hold bufctl information (because we can't safely
2062 	 * perform word loads and stores on it).
2063 	 */
2064 	if (align < KMEM_ALIGN)
2065 		cflags |= KMC_NOTOUCH;
2066 
2067 	if ((align & (align - 1)) != 0 || align > vmp->vm_quantum)
2068 		panic("kmem_cache_create: bad alignment %lu", align);
2069 
2070 	mutex_enter(&kmem_flags_lock);
2071 	if (kmem_flags & KMF_RANDOMIZE)
2072 		kmem_flags = (((kmem_flags | ~KMF_RANDOM) + 1) & KMF_RANDOM) |
2073 		    KMF_RANDOMIZE;
2074 	cp->cache_flags = (kmem_flags | cflags) & KMF_DEBUG;
2075 	mutex_exit(&kmem_flags_lock);
2076 
2077 	/*
2078 	 * Make sure all the various flags are reasonable.
2079 	 */
2080 	ASSERT(!(cflags & KMC_NOHASH) || !(cflags & KMC_NOTOUCH));
2081 
2082 	if (cp->cache_flags & KMF_LITE) {
2083 		if (bufsize >= kmem_lite_minsize &&
2084 		    align <= kmem_lite_maxalign &&
2085 		    P2PHASE(bufsize, kmem_lite_maxalign) != 0) {
2086 			cp->cache_flags |= KMF_BUFTAG;
2087 			cp->cache_flags &= ~(KMF_AUDIT | KMF_FIREWALL);
2088 		} else {
2089 			cp->cache_flags &= ~KMF_DEBUG;
2090 		}
2091 	}
2092 
2093 	if (cp->cache_flags & KMF_DEADBEEF)
2094 		cp->cache_flags |= KMF_REDZONE;
2095 
2096 	if ((cflags & KMC_QCACHE) && (cp->cache_flags & KMF_AUDIT))
2097 		cp->cache_flags |= KMF_NOMAGAZINE;
2098 
2099 	if (cflags & KMC_NODEBUG)
2100 		cp->cache_flags &= ~KMF_DEBUG;
2101 
2102 	if (cflags & KMC_NOTOUCH)
2103 		cp->cache_flags &= ~KMF_TOUCH;
2104 
2105 	if (cflags & KMC_NOHASH)
2106 		cp->cache_flags &= ~(KMF_AUDIT | KMF_FIREWALL);
2107 
2108 	if (cflags & KMC_NOMAGAZINE)
2109 		cp->cache_flags |= KMF_NOMAGAZINE;
2110 
2111 	if ((cp->cache_flags & KMF_AUDIT) && !(cflags & KMC_NOTOUCH))
2112 		cp->cache_flags |= KMF_REDZONE;
2113 
2114 	if (!(cp->cache_flags & KMF_AUDIT))
2115 		cp->cache_flags &= ~KMF_CONTENTS;
2116 
2117 	if ((cp->cache_flags & KMF_BUFTAG) && bufsize >= kmem_minfirewall &&
2118 	    !(cp->cache_flags & KMF_LITE) && !(cflags & KMC_NOHASH))
2119 		cp->cache_flags |= KMF_FIREWALL;
2120 
2121 	if (vmp != kmem_default_arena || kmem_firewall_arena == NULL)
2122 		cp->cache_flags &= ~KMF_FIREWALL;
2123 
2124 	if (cp->cache_flags & KMF_FIREWALL) {
2125 		cp->cache_flags &= ~KMF_BUFTAG;
2126 		cp->cache_flags |= KMF_NOMAGAZINE;
2127 		ASSERT(vmp == kmem_default_arena);
2128 		vmp = kmem_firewall_arena;
2129 	}
2130 
2131 	/*
2132 	 * Set cache properties.
2133 	 */
2134 	(void) strncpy(cp->cache_name, name, KMEM_CACHE_NAMELEN);
2135 	strident_canon(cp->cache_name, KMEM_CACHE_NAMELEN);
2136 	cp->cache_bufsize = bufsize;
2137 	cp->cache_align = align;
2138 	cp->cache_constructor = constructor;
2139 	cp->cache_destructor = destructor;
2140 	cp->cache_reclaim = reclaim;
2141 	cp->cache_private = private;
2142 	cp->cache_arena = vmp;
2143 	cp->cache_cflags = cflags;
2144 
2145 	/*
2146 	 * Determine the chunk size.
2147 	 */
2148 	chunksize = bufsize;
2149 
2150 	if (align >= KMEM_ALIGN) {
2151 		chunksize = P2ROUNDUP(chunksize, KMEM_ALIGN);
2152 		cp->cache_bufctl = chunksize - KMEM_ALIGN;
2153 	}
2154 
2155 	if (cp->cache_flags & KMF_BUFTAG) {
2156 		cp->cache_bufctl = chunksize;
2157 		cp->cache_buftag = chunksize;
2158 		if (cp->cache_flags & KMF_LITE)
2159 			chunksize += KMEM_BUFTAG_LITE_SIZE(kmem_lite_count);
2160 		else
2161 			chunksize += sizeof (kmem_buftag_t);
2162 	}
2163 
2164 	if (cp->cache_flags & KMF_DEADBEEF) {
2165 		cp->cache_verify = MIN(cp->cache_buftag, kmem_maxverify);
2166 		if (cp->cache_flags & KMF_LITE)
2167 			cp->cache_verify = sizeof (uint64_t);
2168 	}
2169 
2170 	cp->cache_contents = MIN(cp->cache_bufctl, kmem_content_maxsave);
2171 
2172 	cp->cache_chunksize = chunksize = P2ROUNDUP(chunksize, align);
2173 
2174 	/*
2175 	 * Now that we know the chunk size, determine the optimal slab size.
2176 	 */
2177 	if (vmp == kmem_firewall_arena) {
2178 		cp->cache_slabsize = P2ROUNDUP(chunksize, vmp->vm_quantum);
2179 		cp->cache_mincolor = cp->cache_slabsize - chunksize;
2180 		cp->cache_maxcolor = cp->cache_mincolor;
2181 		cp->cache_flags |= KMF_HASH;
2182 		ASSERT(!(cp->cache_flags & KMF_BUFTAG));
2183 	} else if ((cflags & KMC_NOHASH) || (!(cflags & KMC_NOTOUCH) &&
2184 	    !(cp->cache_flags & KMF_AUDIT) &&
2185 	    chunksize < vmp->vm_quantum / KMEM_VOID_FRACTION)) {
2186 		cp->cache_slabsize = vmp->vm_quantum;
2187 		cp->cache_mincolor = 0;
2188 		cp->cache_maxcolor =
2189 		    (cp->cache_slabsize - sizeof (kmem_slab_t)) % chunksize;
2190 		ASSERT(chunksize + sizeof (kmem_slab_t) <= cp->cache_slabsize);
2191 		ASSERT(!(cp->cache_flags & KMF_AUDIT));
2192 	} else {
2193 		size_t chunks, bestfit, waste, slabsize;
2194 		size_t minwaste = LONG_MAX;
2195 
2196 		for (chunks = 1; chunks <= KMEM_VOID_FRACTION; chunks++) {
2197 			slabsize = P2ROUNDUP(chunksize * chunks,
2198 			    vmp->vm_quantum);
2199 			chunks = slabsize / chunksize;
2200 			waste = (slabsize % chunksize) / chunks;
2201 			if (waste < minwaste) {
2202 				minwaste = waste;
2203 				bestfit = slabsize;
2204 			}
2205 		}
2206 		if (cflags & KMC_QCACHE)
2207 			bestfit = VMEM_QCACHE_SLABSIZE(vmp->vm_qcache_max);
2208 		cp->cache_slabsize = bestfit;
2209 		cp->cache_mincolor = 0;
2210 		cp->cache_maxcolor = bestfit % chunksize;
2211 		cp->cache_flags |= KMF_HASH;
2212 	}
2213 
2214 	if (cp->cache_flags & KMF_HASH) {
2215 		ASSERT(!(cflags & KMC_NOHASH));
2216 		cp->cache_bufctl_cache = (cp->cache_flags & KMF_AUDIT) ?
2217 		    kmem_bufctl_audit_cache : kmem_bufctl_cache;
2218 	}
2219 
2220 	if (cp->cache_maxcolor >= vmp->vm_quantum)
2221 		cp->cache_maxcolor = vmp->vm_quantum - 1;
2222 
2223 	cp->cache_color = cp->cache_mincolor;
2224 
2225 	/*
2226 	 * Initialize the rest of the slab layer.
2227 	 */
2228 	mutex_init(&cp->cache_lock, NULL, MUTEX_DEFAULT, NULL);
2229 
2230 	cp->cache_freelist = &cp->cache_nullslab;
2231 	cp->cache_nullslab.slab_cache = cp;
2232 	cp->cache_nullslab.slab_refcnt = -1;
2233 	cp->cache_nullslab.slab_next = &cp->cache_nullslab;
2234 	cp->cache_nullslab.slab_prev = &cp->cache_nullslab;
2235 
2236 	if (cp->cache_flags & KMF_HASH) {
2237 		cp->cache_hash_table = vmem_alloc(kmem_hash_arena,
2238 		    KMEM_HASH_INITIAL * sizeof (void *), VM_SLEEP);
2239 		bzero(cp->cache_hash_table,
2240 		    KMEM_HASH_INITIAL * sizeof (void *));
2241 		cp->cache_hash_mask = KMEM_HASH_INITIAL - 1;
2242 		cp->cache_hash_shift = highbit((ulong_t)chunksize) - 1;
2243 	}
2244 
2245 	/*
2246 	 * Initialize the depot.
2247 	 */
2248 	mutex_init(&cp->cache_depot_lock, NULL, MUTEX_DEFAULT, NULL);
2249 
2250 	for (mtp = kmem_magtype; chunksize <= mtp->mt_minbuf; mtp++)
2251 		continue;
2252 
2253 	cp->cache_magtype = mtp;
2254 
2255 	/*
2256 	 * Initialize the CPU layer.
2257 	 */
2258 	for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) {
2259 		kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid];
2260 		mutex_init(&ccp->cc_lock, NULL, MUTEX_DEFAULT, NULL);
2261 		ccp->cc_flags = cp->cache_flags;
2262 		ccp->cc_rounds = -1;
2263 		ccp->cc_prounds = -1;
2264 	}
2265 
2266 	/*
2267 	 * Create the cache's kstats.
2268 	 */
2269 	if ((cp->cache_kstat = kstat_create("unix", 0, cp->cache_name,
2270 	    "kmem_cache", KSTAT_TYPE_NAMED,
2271 	    sizeof (kmem_cache_kstat) / sizeof (kstat_named_t),
2272 	    KSTAT_FLAG_VIRTUAL)) != NULL) {
2273 		cp->cache_kstat->ks_data = &kmem_cache_kstat;
2274 		cp->cache_kstat->ks_update = kmem_cache_kstat_update;
2275 		cp->cache_kstat->ks_private = cp;
2276 		cp->cache_kstat->ks_lock = &kmem_cache_kstat_lock;
2277 		kstat_install(cp->cache_kstat);
2278 	}
2279 
2280 	/*
2281 	 * Add the cache to the global list.  This makes it visible
2282 	 * to kmem_update(), so the cache must be ready for business.
2283 	 */
2284 	mutex_enter(&kmem_cache_lock);
2285 	cp->cache_next = cnext = &kmem_null_cache;
2286 	cp->cache_prev = cprev = kmem_null_cache.cache_prev;
2287 	cnext->cache_prev = cp;
2288 	cprev->cache_next = cp;
2289 	mutex_exit(&kmem_cache_lock);
2290 
2291 	if (kmem_ready)
2292 		kmem_cache_magazine_enable(cp);
2293 
2294 	return (cp);
2295 }
2296 
2297 void
2298 kmem_cache_destroy(kmem_cache_t *cp)
2299 {
2300 	int cpu_seqid;
2301 
2302 	/*
2303 	 * Remove the cache from the global cache list so that no one else
2304 	 * can schedule tasks on its behalf, wait for any pending tasks to
2305 	 * complete, purge the cache, and then destroy it.
2306 	 */
2307 	mutex_enter(&kmem_cache_lock);
2308 	cp->cache_prev->cache_next = cp->cache_next;
2309 	cp->cache_next->cache_prev = cp->cache_prev;
2310 	cp->cache_prev = cp->cache_next = NULL;
2311 	mutex_exit(&kmem_cache_lock);
2312 
2313 	if (kmem_taskq != NULL)
2314 		taskq_wait(kmem_taskq);
2315 
2316 	kmem_cache_magazine_purge(cp);
2317 
2318 	mutex_enter(&cp->cache_lock);
2319 	if (cp->cache_buftotal != 0)
2320 		cmn_err(CE_WARN, "kmem_cache_destroy: '%s' (%p) not empty",
2321 		    cp->cache_name, (void *)cp);
2322 	cp->cache_reclaim = NULL;
2323 	/*
2324 	 * The cache is now dead.  There should be no further activity.
2325 	 * We enforce this by setting land mines in the constructor and
2326 	 * destructor routines that induce a kernel text fault if invoked.
2327 	 */
2328 	cp->cache_constructor = (int (*)(void *, void *, int))1;
2329 	cp->cache_destructor = (void (*)(void *, void *))2;
2330 	mutex_exit(&cp->cache_lock);
2331 
2332 	kstat_delete(cp->cache_kstat);
2333 
2334 	if (cp->cache_hash_table != NULL)
2335 		vmem_free(kmem_hash_arena, cp->cache_hash_table,
2336 		    (cp->cache_hash_mask + 1) * sizeof (void *));
2337 
2338 	for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++)
2339 		mutex_destroy(&cp->cache_cpu[cpu_seqid].cc_lock);
2340 
2341 	mutex_destroy(&cp->cache_depot_lock);
2342 	mutex_destroy(&cp->cache_lock);
2343 
2344 	vmem_free(kmem_cache_arena, cp, KMEM_CACHE_SIZE(max_ncpus));
2345 }
2346 
2347 /*ARGSUSED*/
2348 static int
2349 kmem_cpu_setup(cpu_setup_t what, int id, void *arg)
2350 {
2351 	ASSERT(MUTEX_HELD(&cpu_lock));
2352 	if (what == CPU_UNCONFIG) {
2353 		kmem_cache_applyall(kmem_cache_magazine_purge,
2354 		    kmem_taskq, TQ_SLEEP);
2355 		kmem_cache_applyall(kmem_cache_magazine_enable,
2356 		    kmem_taskq, TQ_SLEEP);
2357 	}
2358 	return (0);
2359 }
2360 
2361 static void
2362 kmem_cache_init(int pass, int use_large_pages)
2363 {
2364 	int i;
2365 	size_t size;
2366 	kmem_cache_t *cp;
2367 	kmem_magtype_t *mtp;
2368 	char name[KMEM_CACHE_NAMELEN + 1];
2369 
2370 	for (i = 0; i < sizeof (kmem_magtype) / sizeof (*mtp); i++) {
2371 		mtp = &kmem_magtype[i];
2372 		(void) sprintf(name, "kmem_magazine_%d", mtp->mt_magsize);
2373 		mtp->mt_cache = kmem_cache_create(name,
2374 		    (mtp->mt_magsize + 1) * sizeof (void *),
2375 		    mtp->mt_align, NULL, NULL, NULL, NULL,
2376 		    kmem_msb_arena, KMC_NOHASH);
2377 	}
2378 
2379 	kmem_slab_cache = kmem_cache_create("kmem_slab_cache",
2380 	    sizeof (kmem_slab_t), 0, NULL, NULL, NULL, NULL,
2381 	    kmem_msb_arena, KMC_NOHASH);
2382 
2383 	kmem_bufctl_cache = kmem_cache_create("kmem_bufctl_cache",
2384 	    sizeof (kmem_bufctl_t), 0, NULL, NULL, NULL, NULL,
2385 	    kmem_msb_arena, KMC_NOHASH);
2386 
2387 	kmem_bufctl_audit_cache = kmem_cache_create("kmem_bufctl_audit_cache",
2388 	    sizeof (kmem_bufctl_audit_t), 0, NULL, NULL, NULL, NULL,
2389 	    kmem_msb_arena, KMC_NOHASH);
2390 
2391 	if (pass == 2) {
2392 		kmem_va_arena = vmem_create("kmem_va",
2393 		    NULL, 0, PAGESIZE,
2394 		    vmem_alloc, vmem_free, heap_arena,
2395 		    8 * PAGESIZE, VM_SLEEP);
2396 
2397 		if (use_large_pages) {
2398 			kmem_default_arena = vmem_xcreate("kmem_default",
2399 			    NULL, 0, PAGESIZE,
2400 			    segkmem_alloc_lp, segkmem_free_lp, kmem_va_arena,
2401 			    0, VM_SLEEP);
2402 		} else {
2403 			kmem_default_arena = vmem_create("kmem_default",
2404 			    NULL, 0, PAGESIZE,
2405 			    segkmem_alloc, segkmem_free, kmem_va_arena,
2406 			    0, VM_SLEEP);
2407 		}
2408 	} else {
2409 		/*
2410 		 * During the first pass, the kmem_alloc_* caches
2411 		 * are treated as metadata.
2412 		 */
2413 		kmem_default_arena = kmem_msb_arena;
2414 	}
2415 
2416 	/*
2417 	 * Set up the default caches to back kmem_alloc()
2418 	 */
2419 	size = KMEM_ALIGN;
2420 	for (i = 0; i < sizeof (kmem_alloc_sizes) / sizeof (int); i++) {
2421 		size_t align = KMEM_ALIGN;
2422 		size_t cache_size = kmem_alloc_sizes[i];
2423 		/*
2424 		 * If they allocate a multiple of the coherency granularity,
2425 		 * they get a coherency-granularity-aligned address.
2426 		 */
2427 		if (IS_P2ALIGNED(cache_size, 64))
2428 			align = 64;
2429 		if (IS_P2ALIGNED(cache_size, PAGESIZE))
2430 			align = PAGESIZE;
2431 		(void) sprintf(name, "kmem_alloc_%lu", cache_size);
2432 		cp = kmem_cache_create(name, cache_size, align,
2433 		    NULL, NULL, NULL, NULL, NULL, KMC_KMEM_ALLOC);
2434 		while (size <= cache_size) {
2435 			kmem_alloc_table[(size - 1) >> KMEM_ALIGN_SHIFT] = cp;
2436 			size += KMEM_ALIGN;
2437 		}
2438 	}
2439 }
2440 
2441 void
2442 kmem_init(void)
2443 {
2444 	kmem_cache_t *cp;
2445 	int old_kmem_flags = kmem_flags;
2446 	int use_large_pages = 0;
2447 	size_t maxverify, minfirewall;
2448 
2449 	kstat_init();
2450 
2451 	/*
2452 	 * Small-memory systems (< 24 MB) can't handle kmem_flags overhead.
2453 	 */
2454 	if (physmem < btop(24 << 20) && !(old_kmem_flags & KMF_STICKY))
2455 		kmem_flags = 0;
2456 
2457 	/*
2458 	 * Don't do firewalled allocations if the heap is less than 1TB
2459 	 * (i.e. on a 32-bit kernel)
2460 	 * The resulting VM_NEXTFIT allocations would create too much
2461 	 * fragmentation in a small heap.
2462 	 */
2463 #if defined(_LP64)
2464 	maxverify = minfirewall = PAGESIZE / 2;
2465 #else
2466 	maxverify = minfirewall = ULONG_MAX;
2467 #endif
2468 
2469 	/* LINTED */
2470 	ASSERT(sizeof (kmem_cpu_cache_t) == KMEM_CPU_CACHE_SIZE);
2471 
2472 	kmem_null_cache.cache_next = &kmem_null_cache;
2473 	kmem_null_cache.cache_prev = &kmem_null_cache;
2474 
2475 	kmem_metadata_arena = vmem_create("kmem_metadata", NULL, 0, PAGESIZE,
2476 	    vmem_alloc, vmem_free, heap_arena, 8 * PAGESIZE,
2477 	    VM_SLEEP | VMC_NO_QCACHE);
2478 
2479 	kmem_msb_arena = vmem_create("kmem_msb", NULL, 0,
2480 	    PAGESIZE, segkmem_alloc, segkmem_free, kmem_metadata_arena, 0,
2481 	    VM_SLEEP);
2482 
2483 	kmem_cache_arena = vmem_create("kmem_cache", NULL, 0, KMEM_ALIGN,
2484 	    segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, VM_SLEEP);
2485 
2486 	kmem_hash_arena = vmem_create("kmem_hash", NULL, 0, KMEM_ALIGN,
2487 	    segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, VM_SLEEP);
2488 
2489 	kmem_log_arena = vmem_create("kmem_log", NULL, 0, KMEM_ALIGN,
2490 	    segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP);
2491 
2492 	kmem_firewall_va_arena = vmem_create("kmem_firewall_va",
2493 	    NULL, 0, PAGESIZE,
2494 	    kmem_firewall_va_alloc, kmem_firewall_va_free, heap_arena,
2495 	    0, VM_SLEEP);
2496 
2497 	kmem_firewall_arena = vmem_create("kmem_firewall", NULL, 0, PAGESIZE,
2498 	    segkmem_alloc, segkmem_free, kmem_firewall_va_arena, 0, VM_SLEEP);
2499 
2500 	/* temporary oversize arena for mod_read_system_file */
2501 	kmem_oversize_arena = vmem_create("kmem_oversize", NULL, 0, PAGESIZE,
2502 	    segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP);
2503 
2504 	kmem_null_cache.cache_next = &kmem_null_cache;
2505 	kmem_null_cache.cache_prev = &kmem_null_cache;
2506 
2507 	kmem_reap_interval = 15 * hz;
2508 
2509 	/*
2510 	 * Read /etc/system.  This is a chicken-and-egg problem because
2511 	 * kmem_flags may be set in /etc/system, but mod_read_system_file()
2512 	 * needs to use the allocator.  The simplest solution is to create
2513 	 * all the standard kmem caches, read /etc/system, destroy all the
2514 	 * caches we just created, and then create them all again in light
2515 	 * of the (possibly) new kmem_flags and other kmem tunables.
2516 	 */
2517 	kmem_cache_init(1, 0);
2518 
2519 	mod_read_system_file(boothowto & RB_ASKNAME);
2520 
2521 	while ((cp = kmem_null_cache.cache_prev) != &kmem_null_cache)
2522 		kmem_cache_destroy(cp);
2523 
2524 	vmem_destroy(kmem_oversize_arena);
2525 
2526 	if (old_kmem_flags & KMF_STICKY)
2527 		kmem_flags = old_kmem_flags;
2528 
2529 	if (!(kmem_flags & KMF_AUDIT))
2530 		vmem_seg_size = offsetof(vmem_seg_t, vs_thread);
2531 
2532 	if (kmem_maxverify == 0)
2533 		kmem_maxverify = maxverify;
2534 
2535 	if (kmem_minfirewall == 0)
2536 		kmem_minfirewall = minfirewall;
2537 
2538 	/*
2539 	 * give segkmem a chance to figure out if we are using large pages
2540 	 * for the kernel heap
2541 	 */
2542 	use_large_pages = segkmem_lpsetup();
2543 
2544 	/*
2545 	 * To protect against corruption, we keep the actual number of callers
2546 	 * KMF_LITE records seperate from the tunable.  We arbitrarily clamp
2547 	 * to 16, since the overhead for small buffers quickly gets out of
2548 	 * hand.
2549 	 *
2550 	 * The real limit would depend on the needs of the largest KMC_NOHASH
2551 	 * cache.
2552 	 */
2553 	kmem_lite_count = MIN(MAX(0, kmem_lite_pcs), 16);
2554 	kmem_lite_pcs = kmem_lite_count;
2555 
2556 	/*
2557 	 * Normally, we firewall oversized allocations when possible, but
2558 	 * if we are using large pages for kernel memory, and we don't have
2559 	 * any non-LITE debugging flags set, we want to allocate oversized
2560 	 * buffers from large pages, and so skip the firewalling.
2561 	 */
2562 	if (use_large_pages &&
2563 	    ((kmem_flags & KMF_LITE) || !(kmem_flags & KMF_DEBUG))) {
2564 		kmem_oversize_arena = vmem_xcreate("kmem_oversize", NULL, 0,
2565 		    PAGESIZE, segkmem_alloc_lp, segkmem_free_lp, heap_arena,
2566 		    0, VM_SLEEP);
2567 	} else {
2568 		kmem_oversize_arena = vmem_create("kmem_oversize",
2569 		    NULL, 0, PAGESIZE,
2570 		    segkmem_alloc, segkmem_free, kmem_minfirewall < ULONG_MAX?
2571 		    kmem_firewall_va_arena : heap_arena, 0, VM_SLEEP);
2572 	}
2573 
2574 	kmem_cache_init(2, use_large_pages);
2575 
2576 	if (kmem_flags & (KMF_AUDIT | KMF_RANDOMIZE)) {
2577 		if (kmem_transaction_log_size == 0)
2578 			kmem_transaction_log_size = kmem_maxavail() / 50;
2579 		kmem_transaction_log = kmem_log_init(kmem_transaction_log_size);
2580 	}
2581 
2582 	if (kmem_flags & (KMF_CONTENTS | KMF_RANDOMIZE)) {
2583 		if (kmem_content_log_size == 0)
2584 			kmem_content_log_size = kmem_maxavail() / 50;
2585 		kmem_content_log = kmem_log_init(kmem_content_log_size);
2586 	}
2587 
2588 	kmem_failure_log = kmem_log_init(kmem_failure_log_size);
2589 
2590 	kmem_slab_log = kmem_log_init(kmem_slab_log_size);
2591 
2592 	/*
2593 	 * Initialize STREAMS message caches so allocb() is available.
2594 	 * This allows us to initialize the logging framework (cmn_err(9F),
2595 	 * strlog(9F), etc) so we can start recording messages.
2596 	 */
2597 	streams_msg_init();
2598 
2599 	/*
2600 	 * Initialize the ZSD framework in Zones so modules loaded henceforth
2601 	 * can register their callbacks.
2602 	 */
2603 	zone_zsd_init();
2604 	log_init();
2605 	taskq_init();
2606 
2607 	/*
2608 	 * Warn about invalid or dangerous values of kmem_flags.
2609 	 * Always warn about unsupported values.
2610 	 */
2611 	if (((kmem_flags & ~(KMF_AUDIT | KMF_DEADBEEF | KMF_REDZONE |
2612 	    KMF_CONTENTS | KMF_LITE)) != 0) ||
2613 	    ((kmem_flags & KMF_LITE) && kmem_flags != KMF_LITE))
2614 		cmn_err(CE_WARN, "kmem_flags set to unsupported value 0x%x. "
2615 		    "See the Solaris Tunable Parameters Reference Manual.",
2616 		    kmem_flags);
2617 
2618 #ifdef DEBUG
2619 	if ((kmem_flags & KMF_DEBUG) == 0)
2620 		cmn_err(CE_NOTE, "kmem debugging disabled.");
2621 #else
2622 	/*
2623 	 * For non-debug kernels, the only "normal" flags are 0, KMF_LITE,
2624 	 * KMF_REDZONE, and KMF_CONTENTS (the last because it is only enabled
2625 	 * if KMF_AUDIT is set). We should warn the user about the performance
2626 	 * penalty of KMF_AUDIT or KMF_DEADBEEF if they are set and KMF_LITE
2627 	 * isn't set (since that disables AUDIT).
2628 	 */
2629 	if (!(kmem_flags & KMF_LITE) &&
2630 	    (kmem_flags & (KMF_AUDIT | KMF_DEADBEEF)) != 0)
2631 		cmn_err(CE_WARN, "High-overhead kmem debugging features "
2632 		    "enabled (kmem_flags = 0x%x).  Performance degradation "
2633 		    "and large memory overhead possible. See the Solaris "
2634 		    "Tunable Parameters Reference Manual.", kmem_flags);
2635 #endif /* not DEBUG */
2636 
2637 	kmem_cache_applyall(kmem_cache_magazine_enable, NULL, TQ_SLEEP);
2638 
2639 	kmem_ready = 1;
2640 
2641 	/*
2642 	 * Initialize the platform-specific aligned/DMA memory allocator.
2643 	 */
2644 	ka_init();
2645 
2646 	/*
2647 	 * Initialize 32-bit ID cache.
2648 	 */
2649 	id32_init();
2650 }
2651 
2652 void
2653 kmem_thread_init(void)
2654 {
2655 	kmem_taskq = taskq_create_instance("kmem_taskq", 0, 1, minclsyspri,
2656 	    300, INT_MAX, TASKQ_PREPOPULATE);
2657 }
2658 
2659 void
2660 kmem_mp_init(void)
2661 {
2662 	mutex_enter(&cpu_lock);
2663 	register_cpu_setup_func(kmem_cpu_setup, NULL);
2664 	mutex_exit(&cpu_lock);
2665 
2666 	kmem_update_timeout(NULL);
2667 }
2668