xref: /illumos-gate/usr/src/uts/common/sys/kmem_impl.h (revision dfec2ecf)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright 2018 Joyent, Inc.
25  */
26 
27 #ifndef _SYS_KMEM_IMPL_H
28 #define	_SYS_KMEM_IMPL_H
29 
30 #include <sys/kmem.h>
31 #include <sys/vmem.h>
32 #include <sys/thread.h>
33 #include <sys/t_lock.h>
34 #include <sys/time.h>
35 #include <sys/kstat.h>
36 #include <sys/cpuvar.h>
37 #include <sys/systm.h>
38 #include <vm/page.h>
39 #include <sys/avl.h>
40 #include <sys/list.h>
41 
42 #ifdef	__cplusplus
43 extern "C" {
44 #endif
45 
46 /*
47  * kernel memory allocator: implementation-private data structures
48  *
49  * Lock order:
50  * 1. cache_lock
51  * 2. cc_lock in order by CPU ID
52  * 3. cache_depot_lock
53  *
54  * Do not call kmem_cache_alloc() or taskq_dispatch() while holding any of the
55  * above locks.
56  */
57 
58 #define	KMF_AUDIT	0x00000001	/* transaction auditing */
59 #define	KMF_DEADBEEF	0x00000002	/* deadbeef checking */
60 #define	KMF_REDZONE	0x00000004	/* redzone checking */
61 #define	KMF_CONTENTS	0x00000008	/* freed-buffer content logging */
62 #define	KMF_STICKY	0x00000010	/* if set, override /etc/system */
63 #define	KMF_NOMAGAZINE	0x00000020	/* disable per-cpu magazines */
64 #define	KMF_FIREWALL	0x00000040	/* put all bufs before unmapped pages */
65 #define	KMF_LITE	0x00000100	/* lightweight debugging */
66 
67 #define	KMF_HASH	0x00000200	/* cache has hash table */
68 #define	KMF_RANDOMIZE	0x00000400	/* randomize other kmem_flags */
69 
70 #define	KMF_DUMPDIVERT	0x00001000	/* use alternate memory at dump time */
71 #define	KMF_DUMPUNSAFE	0x00002000	/* flag caches used at dump time */
72 #define	KMF_PREFILL	0x00004000	/* Prefill the slab when created. */
73 
74 #define	KMF_BUFTAG	(KMF_DEADBEEF | KMF_REDZONE)
75 #define	KMF_TOUCH	(KMF_BUFTAG | KMF_LITE | KMF_CONTENTS)
76 #define	KMF_RANDOM	(KMF_TOUCH | KMF_AUDIT | KMF_NOMAGAZINE)
77 #define	KMF_DEBUG	(KMF_RANDOM | KMF_FIREWALL)
78 
79 #define	KMEM_STACK_DEPTH	15
80 
81 #define	KMEM_FREE_PATTERN		0xdeadbeefdeadbeefULL
82 #define	KMEM_UNINITIALIZED_PATTERN	0xbaddcafebaddcafeULL
83 #define	KMEM_REDZONE_PATTERN		0xfeedfacefeedfaceULL
84 #define	KMEM_REDZONE_BYTE		0xbb
85 
86 /*
87  * Redzone size encodings for kmem_alloc() / kmem_free().  We encode the
88  * allocation size, rather than storing it directly, so that kmem_free()
89  * can distinguish frees of the wrong size from redzone violations.
90  *
91  * A size of zero is never valid.
92  */
93 #define	KMEM_SIZE_ENCODE(x)	(251 * (x) + 1)
94 #define	KMEM_SIZE_DECODE(x)	((x) / 251)
95 #define	KMEM_SIZE_VALID(x)	((x) % 251 == 1 && (x) != 1)
96 
97 
98 #define	KMEM_ALIGN		8	/* min guaranteed alignment */
99 #define	KMEM_ALIGN_SHIFT	3	/* log2(KMEM_ALIGN) */
100 #define	KMEM_VOID_FRACTION	8	/* never waste more than 1/8 of slab */
101 
102 #define	KMEM_SLAB_IS_PARTIAL(sp)		\
103 	((sp)->slab_refcnt > 0 && (sp)->slab_refcnt < (sp)->slab_chunks)
104 #define	KMEM_SLAB_IS_ALL_USED(sp)		\
105 	((sp)->slab_refcnt == (sp)->slab_chunks)
106 
107 /*
108  * The bufctl (buffer control) structure keeps some minimal information
109  * about each buffer: its address, its slab, and its current linkage,
110  * which is either on the slab's freelist (if the buffer is free), or
111  * on the cache's buf-to-bufctl hash table (if the buffer is allocated).
112  * In the case of non-hashed, or "raw", caches (the common case), only
113  * the freelist linkage is necessary: the buffer address is at a fixed
114  * offset from the bufctl address, and the slab is at the end of the page.
115  *
116  * NOTE: bc_next must be the first field; raw buffers have linkage only.
117  */
118 typedef struct kmem_bufctl {
119 	struct kmem_bufctl	*bc_next;	/* next bufctl struct */
120 	void			*bc_addr;	/* address of buffer */
121 	struct kmem_slab	*bc_slab;	/* controlling slab */
122 } kmem_bufctl_t;
123 
124 /*
125  * The KMF_AUDIT version of the bufctl structure.  The beginning of this
126  * structure must be identical to the normal bufctl structure so that
127  * pointers are interchangeable.
128  */
129 typedef struct kmem_bufctl_audit {
130 	struct kmem_bufctl	*bc_next;	/* next bufctl struct */
131 	void			*bc_addr;	/* address of buffer */
132 	struct kmem_slab	*bc_slab;	/* controlling slab */
133 	kmem_cache_t		*bc_cache;	/* controlling cache */
134 	hrtime_t		bc_timestamp;	/* transaction time */
135 	kthread_t		*bc_thread;	/* thread doing transaction */
136 	struct kmem_bufctl	*bc_lastlog;	/* last log entry */
137 	void			*bc_contents;	/* contents at last free */
138 	int			bc_depth;	/* stack depth */
139 	pc_t			bc_stack[KMEM_STACK_DEPTH];	/* pc stack */
140 } kmem_bufctl_audit_t;
141 
142 /*
143  * A kmem_buftag structure is appended to each buffer whenever any of the
144  * KMF_BUFTAG flags (KMF_DEADBEEF, KMF_REDZONE, KMF_VERIFY) are set.
145  */
146 typedef struct kmem_buftag {
147 	uint64_t		bt_redzone;	/* 64-bit redzone pattern */
148 	kmem_bufctl_t		*bt_bufctl;	/* bufctl */
149 	intptr_t		bt_bxstat;	/* bufctl ^ (alloc/free) */
150 } kmem_buftag_t;
151 
152 /*
153  * A variant of the kmem_buftag structure used for KMF_LITE caches.
154  * Previous callers are stored in reverse chronological order. (i.e. most
155  * recent first)
156  */
157 typedef struct kmem_buftag_lite {
158 	kmem_buftag_t		bt_buftag;	/* a normal buftag */
159 	pc_t			bt_history[1];	/* zero or more callers */
160 } kmem_buftag_lite_t;
161 
162 #define	KMEM_BUFTAG_LITE_SIZE(f)	\
163 	(offsetof(kmem_buftag_lite_t, bt_history[f]))
164 
165 #define	KMEM_BUFTAG(cp, buf)		\
166 	((kmem_buftag_t *)((char *)(buf) + (cp)->cache_buftag))
167 
168 #define	KMEM_BUFCTL(cp, buf)		\
169 	((kmem_bufctl_t *)((char *)(buf) + (cp)->cache_bufctl))
170 
171 #define	KMEM_BUF(cp, bcp)		\
172 	((void *)((char *)(bcp) - (cp)->cache_bufctl))
173 
174 #define	KMEM_SLAB(cp, buf)		\
175 	((kmem_slab_t *)P2END((uintptr_t)(buf), (cp)->cache_slabsize) - 1)
176 
177 /*
178  * Test for using alternate memory at dump time.
179  */
180 #define	KMEM_DUMP(cp)		((cp)->cache_flags & KMF_DUMPDIVERT)
181 #define	KMEM_DUMPCC(ccp)	((ccp)->cc_flags & KMF_DUMPDIVERT)
182 
183 /*
184  * The "CPU" macro loads a cpu_t that refers to the cpu that the current
185  * thread is running on at the time the macro is executed.  A context switch
186  * may occur immediately after loading this data structure, leaving this
187  * thread pointing at the cpu_t for the previous cpu.  This is not a problem;
188  * we'd just end up checking the previous cpu's per-cpu cache, and then check
189  * the other layers of the kmem cache if need be.
190  *
191  * It's not even a problem if the old cpu gets DR'ed out during the context
192  * switch.  The cpu-remove DR operation bzero()s the cpu_t, but doesn't free
193  * it.  So the cpu_t's cpu_cache_offset would read as 0, causing us to use
194  * cpu 0's per-cpu cache.
195  *
196  * So, there is no need to disable kernel preemption while using the CPU macro
197  * below since if we have been context switched, there will not be any
198  * correctness problem, just a momentary use of a different per-cpu cache.
199  */
200 
201 #define	KMEM_CPU_CACHE(cp)						\
202 	((kmem_cpu_cache_t *)((char *)(&cp->cache_cpu) + CPU->cpu_cache_offset))
203 
204 #define	KMEM_MAGAZINE_VALID(cp, mp)	\
205 	(((kmem_slab_t *)P2END((uintptr_t)(mp), PAGESIZE) - 1)->slab_cache == \
206 	    (cp)->cache_magtype->mt_cache)
207 
208 #define	KMEM_SLAB_OFFSET(sp, buf)	\
209 	((size_t)((uintptr_t)(buf) - (uintptr_t)((sp)->slab_base)))
210 
211 #define	KMEM_SLAB_MEMBER(sp, buf)	\
212 	(KMEM_SLAB_OFFSET(sp, buf) < (sp)->slab_cache->cache_slabsize)
213 
214 #define	KMEM_BUFTAG_ALLOC	0xa110c8edUL
215 #define	KMEM_BUFTAG_FREE	0xf4eef4eeUL
216 
217 /* slab_later_count thresholds */
218 #define	KMEM_DISBELIEF		3
219 
220 /* slab_flags */
221 #define	KMEM_SLAB_NOMOVE	0x1
222 #define	KMEM_SLAB_MOVE_PENDING	0x2
223 
224 typedef struct kmem_slab {
225 	struct kmem_cache	*slab_cache;	/* controlling cache */
226 	void			*slab_base;	/* base of allocated memory */
227 	avl_node_t		slab_link;	/* slab linkage */
228 	struct kmem_bufctl	*slab_head;	/* first free buffer */
229 	long			slab_refcnt;	/* outstanding allocations */
230 	long			slab_chunks;	/* chunks (bufs) in this slab */
231 	uint32_t		slab_stuck_offset; /* unmoved buffer offset */
232 	uint16_t		slab_later_count; /* cf KMEM_CBRC_LATER */
233 	uint16_t		slab_flags;	/* bits to mark the slab */
234 } kmem_slab_t;
235 
236 #define	KMEM_HASH_INITIAL	64
237 
238 #define	KMEM_HASH(cp, buf)	\
239 	((cp)->cache_hash_table +	\
240 	(((uintptr_t)(buf) >> (cp)->cache_hash_shift) & (cp)->cache_hash_mask))
241 
242 typedef struct kmem_magazine {
243 	void	*mag_next;
244 	void	*mag_round[1];		/* one or more rounds */
245 } kmem_magazine_t;
246 
247 /*
248  * The magazine types for fast per-cpu allocation
249  */
250 typedef struct kmem_magtype {
251 	short		mt_magsize;	/* magazine size (number of rounds) */
252 	int		mt_align;	/* magazine alignment */
253 	size_t		mt_minbuf;	/* all smaller buffers qualify */
254 	size_t		mt_maxbuf;	/* no larger buffers qualify */
255 	kmem_cache_t	*mt_cache;	/* magazine cache */
256 } kmem_magtype_t;
257 
258 #define	KMEM_CPU_CACHE_SIZE	64	/* must be power of 2 */
259 #define	KMEM_CPU_PAD		(KMEM_CPU_CACHE_SIZE - sizeof (kmutex_t) - \
260 	2 * sizeof (uint64_t) - 2 * sizeof (void *) - sizeof (int) - \
261 	5 * sizeof (short))
262 #define	KMEM_CACHE_SIZE(ncpus)	\
263 	((size_t)(&((kmem_cache_t *)0)->cache_cpu[ncpus]))
264 
265 /* Offset from kmem_cache->cache_cpu for per cpu caches */
266 #define	KMEM_CPU_CACHE_OFFSET(cpuid)					\
267 	((size_t)(&((kmem_cache_t *)0)->cache_cpu[cpuid]) -		\
268 	(size_t)(&((kmem_cache_t *)0)->cache_cpu))
269 
270 typedef struct kmem_cpu_cache {
271 	kmutex_t	cc_lock;	/* protects this cpu's local cache */
272 	uint64_t	cc_alloc;	/* allocations from this cpu */
273 	uint64_t	cc_free;	/* frees to this cpu */
274 	kmem_magazine_t	*cc_loaded;	/* the currently loaded magazine */
275 	kmem_magazine_t	*cc_ploaded;	/* the previously loaded magazine */
276 	int		cc_flags;	/* CPU-local copy of cache_flags */
277 	short		cc_rounds;	/* number of objects in loaded mag */
278 	short		cc_prounds;	/* number of objects in previous mag */
279 	short		cc_magsize;	/* number of rounds in a full mag */
280 	short		cc_dump_rounds;	/* dump time copy of cc_rounds */
281 	short		cc_dump_prounds; /* dump time copy of cc_prounds */
282 	char		cc_pad[KMEM_CPU_PAD]; /* for nice alignment */
283 } kmem_cpu_cache_t;
284 
285 /*
286  * The magazine lists used in the depot.
287  */
288 typedef struct kmem_maglist {
289 	kmem_magazine_t	*ml_list;	/* magazine list */
290 	long		ml_total;	/* number of magazines */
291 	long		ml_min;		/* min since last update */
292 	long		ml_reaplimit;	/* max reapable magazines */
293 	uint64_t	ml_alloc;	/* allocations from this list */
294 } kmem_maglist_t;
295 
296 typedef struct kmem_defrag {
297 	/*
298 	 * Statistics
299 	 */
300 	uint64_t	kmd_callbacks;		/* move callbacks */
301 	uint64_t	kmd_yes;		/* KMEM_CBRC_YES responses */
302 	uint64_t	kmd_no;			/* NO responses */
303 	uint64_t	kmd_later;		/* LATER responses */
304 	uint64_t	kmd_dont_need;		/* DONT_NEED responses */
305 	uint64_t	kmd_dont_know;		/* DONT_KNOW responses */
306 	uint64_t	kmd_slabs_freed;	/* slabs freed by moves */
307 	uint64_t	kmd_defrags;		/* kmem_cache_defrag() */
308 	uint64_t	kmd_scans;		/* kmem_cache_scan() */
309 
310 	/*
311 	 * Consolidator fields
312 	 */
313 	avl_tree_t	kmd_moves_pending;	/* buffer moves pending */
314 	list_t		kmd_deadlist;		/* deferred slab frees */
315 	size_t		kmd_deadcount;		/* # of slabs in kmd_deadlist */
316 	uint8_t		kmd_reclaim_numer;	/* slab usage threshold */
317 	uint8_t		kmd_pad1;		/* compiler padding */
318 	uint16_t	kmd_consolidate;	/* triggers consolidator */
319 	uint32_t	kmd_pad2;		/* compiler padding */
320 	size_t		kmd_slabs_sought;	/* reclaimable slabs sought */
321 	size_t		kmd_slabs_found;	/* reclaimable slabs found */
322 	size_t		kmd_tries;		/* nth scan interval counter */
323 	/*
324 	 * Fields used to ASSERT that the client does not kmem_cache_free()
325 	 * objects passed to the move callback.
326 	 */
327 	void		*kmd_from_buf;		/* object to move */
328 	void		*kmd_to_buf;		/* move destination */
329 	kthread_t	*kmd_thread;		/* thread calling move */
330 } kmem_defrag_t;
331 
332 typedef struct kmem_dump {
333 	void		*kd_freelist;		/* heap during crash dump */
334 	uint_t		kd_alloc_fails;		/* # of allocation failures */
335 	uint_t		kd_unsafe;		/* cache was used, but unsafe */
336 } kmem_dump_t;
337 
338 #define	KMEM_CACHE_NAMELEN	31
339 
340 struct kmem_cache {
341 	/*
342 	 * Statistics
343 	 */
344 	uint64_t	cache_slab_create;	/* slab creates */
345 	uint64_t	cache_slab_destroy;	/* slab destroys */
346 	uint64_t	cache_slab_alloc;	/* slab layer allocations */
347 	uint64_t	cache_slab_free;	/* slab layer frees */
348 	uint64_t	cache_alloc_fail;	/* total failed allocations */
349 	uint64_t	cache_buftotal;		/* total buffers */
350 	uint64_t	cache_bufmax;		/* max buffers ever */
351 	uint64_t	cache_bufslab;		/* buffers free in slab layer */
352 	uint64_t	cache_reap;		/* cache reaps */
353 	uint64_t	cache_rescale;		/* hash table rescales */
354 	uint64_t	cache_lookup_depth;	/* hash lookup depth */
355 	uint64_t	cache_depot_contention;	/* mutex contention count */
356 	uint64_t	cache_depot_contention_prev; /* previous snapshot */
357 
358 	/*
359 	 * Cache properties
360 	 */
361 	char		cache_name[KMEM_CACHE_NAMELEN + 1];
362 	size_t		cache_bufsize;		/* object size */
363 	size_t		cache_align;		/* object alignment */
364 	int		(*cache_constructor)(void *, void *, int);
365 	void		(*cache_destructor)(void *, void *);
366 	void		(*cache_reclaim)(void *);
367 	kmem_cbrc_t	(*cache_move)(void *, void *, size_t, void *);
368 	void		*cache_private;		/* opaque arg to callbacks */
369 	vmem_t		*cache_arena;		/* vmem source for slabs */
370 	int		cache_cflags;		/* cache creation flags */
371 	int		cache_flags;		/* various cache state info */
372 	uint32_t	cache_mtbf;		/* induced alloc failure rate */
373 	uint32_t	cache_pad1;		/* compiler padding */
374 	kstat_t		*cache_kstat;		/* exported statistics */
375 	list_node_t	cache_link;		/* cache linkage */
376 
377 	/*
378 	 * Slab layer
379 	 */
380 	kmutex_t	cache_lock;		/* protects slab layer */
381 	size_t		cache_chunksize;	/* buf + alignment [+ debug] */
382 	size_t		cache_slabsize;		/* size of a slab */
383 	size_t		cache_maxchunks;	/* max buffers per slab */
384 	size_t		cache_bufctl;		/* buf-to-bufctl distance */
385 	size_t		cache_buftag;		/* buf-to-buftag distance */
386 	size_t		cache_verify;		/* bytes to verify */
387 	size_t		cache_contents;		/* bytes of saved content */
388 	size_t		cache_color;		/* next slab color */
389 	size_t		cache_mincolor;		/* maximum slab color */
390 	size_t		cache_maxcolor;		/* maximum slab color */
391 	size_t		cache_hash_shift;	/* get to interesting bits */
392 	size_t		cache_hash_mask;	/* hash table mask */
393 	list_t		cache_complete_slabs;	/* completely allocated slabs */
394 	size_t		cache_complete_slab_count;
395 	avl_tree_t	cache_partial_slabs;	/* partial slab freelist */
396 	size_t		cache_partial_binshift;	/* for AVL sort bins */
397 	kmem_cache_t	*cache_bufctl_cache;	/* source of bufctls */
398 	kmem_bufctl_t	**cache_hash_table;	/* hash table base */
399 	kmem_defrag_t	*cache_defrag;		/* slab consolidator fields */
400 
401 	/*
402 	 * Depot layer
403 	 */
404 	kmutex_t	cache_depot_lock;	/* protects depot */
405 	kmem_magtype_t	*cache_magtype;		/* magazine type */
406 	kmem_maglist_t	cache_full;		/* full magazines */
407 	kmem_maglist_t	cache_empty;		/* empty magazines */
408 	kmem_dump_t	cache_dump;		/* used during crash dump */
409 
410 	/*
411 	 * Per-CPU layer
412 	 */
413 	kmem_cpu_cache_t cache_cpu[1];		/* max_ncpus actual elements */
414 };
415 
416 typedef struct kmem_cpu_log_header {
417 	kmutex_t	clh_lock;
418 	char		*clh_current;
419 	size_t		clh_avail;
420 	int		clh_chunk;
421 	int		clh_hits;
422 	char		clh_pad[64 - sizeof (kmutex_t) - sizeof (char *) -
423 				sizeof (size_t) - 2 * sizeof (int)];
424 } kmem_cpu_log_header_t;
425 
426 typedef struct kmem_log_header {
427 	kmutex_t	lh_lock;
428 	char		*lh_base;
429 	int		*lh_free;
430 	size_t		lh_chunksize;
431 	int		lh_nchunks;
432 	int		lh_head;
433 	int		lh_tail;
434 	int		lh_hits;
435 	kmem_cpu_log_header_t lh_cpu[1];	/* ncpus actually allocated */
436 } kmem_log_header_t;
437 
438 /* kmem_move kmm_flags */
439 #define	KMM_DESPERATE		0x1
440 #define	KMM_NOTIFY		0x2
441 #define	KMM_DEBUG		0x4
442 
443 typedef struct kmem_move {
444 	kmem_slab_t	*kmm_from_slab;
445 	void		*kmm_from_buf;
446 	void		*kmm_to_buf;
447 	avl_node_t	kmm_entry;
448 	int		kmm_flags;
449 } kmem_move_t;
450 
451 /*
452  * In order to consolidate partial slabs, it must be possible for the cache to
453  * have partial slabs.
454  */
455 #define	KMEM_IS_MOVABLE(cp)						\
456 	(((cp)->cache_chunksize * 2) <= (cp)->cache_slabsize)
457 
458 #ifdef	__cplusplus
459 }
460 #endif
461 
462 #endif	/* _SYS_KMEM_IMPL_H */
463