1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2018, Joyent, Inc.
24 * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
25 * Copyright (c) 2014 by Saso Kiselkov. All rights reserved.
26 * Copyright 2017 Nexenta Systems, Inc.  All rights reserved.
27 */
28
29/*
30 * DVA-based Adjustable Replacement Cache
31 *
32 * While much of the theory of operation used here is
33 * based on the self-tuning, low overhead replacement cache
34 * presented by Megiddo and Modha at FAST 2003, there are some
35 * significant differences:
36 *
37 * 1. The Megiddo and Modha model assumes any page is evictable.
38 * Pages in its cache cannot be "locked" into memory.  This makes
39 * the eviction algorithm simple: evict the last page in the list.
40 * This also make the performance characteristics easy to reason
41 * about.  Our cache is not so simple.  At any given moment, some
42 * subset of the blocks in the cache are un-evictable because we
43 * have handed out a reference to them.  Blocks are only evictable
44 * when there are no external references active.  This makes
45 * eviction far more problematic:  we choose to evict the evictable
46 * blocks that are the "lowest" in the list.
47 *
48 * There are times when it is not possible to evict the requested
49 * space.  In these circumstances we are unable to adjust the cache
50 * size.  To prevent the cache growing unbounded at these times we
51 * implement a "cache throttle" that slows the flow of new data
52 * into the cache until we can make space available.
53 *
54 * 2. The Megiddo and Modha model assumes a fixed cache size.
55 * Pages are evicted when the cache is full and there is a cache
56 * miss.  Our model has a variable sized cache.  It grows with
57 * high use, but also tries to react to memory pressure from the
58 * operating system: decreasing its size when system memory is
59 * tight.
60 *
61 * 3. The Megiddo and Modha model assumes a fixed page size. All
62 * elements of the cache are therefore exactly the same size.  So
63 * when adjusting the cache size following a cache miss, its simply
64 * a matter of choosing a single page to evict.  In our model, we
65 * have variable sized cache blocks (rangeing from 512 bytes to
66 * 128K bytes).  We therefore choose a set of blocks to evict to make
67 * space for a cache miss that approximates as closely as possible
68 * the space used by the new block.
69 *
70 * See also:  "ARC: A Self-Tuning, Low Overhead Replacement Cache"
71 * by N. Megiddo & D. Modha, FAST 2003
72 */
73
74/*
75 * The locking model:
76 *
77 * A new reference to a cache buffer can be obtained in two
78 * ways: 1) via a hash table lookup using the DVA as a key,
79 * or 2) via one of the ARC lists.  The arc_read() interface
80 * uses method 1, while the internal ARC algorithms for
81 * adjusting the cache use method 2.  We therefore provide two
82 * types of locks: 1) the hash table lock array, and 2) the
83 * ARC list locks.
84 *
85 * Buffers do not have their own mutexes, rather they rely on the
86 * hash table mutexes for the bulk of their protection (i.e. most
87 * fields in the arc_buf_hdr_t are protected by these mutexes).
88 *
89 * buf_hash_find() returns the appropriate mutex (held) when it
90 * locates the requested buffer in the hash table.  It returns
91 * NULL for the mutex if the buffer was not in the table.
92 *
93 * buf_hash_remove() expects the appropriate hash mutex to be
94 * already held before it is invoked.
95 *
96 * Each ARC state also has a mutex which is used to protect the
97 * buffer list associated with the state.  When attempting to
98 * obtain a hash table lock while holding an ARC list lock you
99 * must use: mutex_tryenter() to avoid deadlock.  Also note that
100 * the active state mutex must be held before the ghost state mutex.
101 *
102 * It as also possible to register a callback which is run when the
103 * arc_meta_limit is reached and no buffers can be safely evicted.  In
104 * this case the arc user should drop a reference on some arc buffers so
105 * they can be reclaimed and the arc_meta_limit honored.  For example,
106 * when using the ZPL each dentry holds a references on a znode.  These
107 * dentries must be pruned before the arc buffer holding the znode can
108 * be safely evicted.
109 *
110 * Note that the majority of the performance stats are manipulated
111 * with atomic operations.
112 *
113 * The L2ARC uses the l2ad_mtx on each vdev for the following:
114 *
115 *	- L2ARC buflist creation
116 *	- L2ARC buflist eviction
117 *	- L2ARC write completion, which walks L2ARC buflists
118 *	- ARC header destruction, as it removes from L2ARC buflists
119 *	- ARC header release, as it removes from L2ARC buflists
120 */
121
122/*
123 * ARC operation:
124 *
125 * Every block that is in the ARC is tracked by an arc_buf_hdr_t structure.
126 * This structure can point either to a block that is still in the cache or to
127 * one that is only accessible in an L2 ARC device, or it can provide
128 * information about a block that was recently evicted. If a block is
129 * only accessible in the L2ARC, then the arc_buf_hdr_t only has enough
130 * information to retrieve it from the L2ARC device. This information is
131 * stored in the l2arc_buf_hdr_t sub-structure of the arc_buf_hdr_t. A block
132 * that is in this state cannot access the data directly.
133 *
134 * Blocks that are actively being referenced or have not been evicted
135 * are cached in the L1ARC. The L1ARC (l1arc_buf_hdr_t) is a structure within
136 * the arc_buf_hdr_t that will point to the data block in memory. A block can
137 * only be read by a consumer if it has an l1arc_buf_hdr_t. The L1ARC
138 * caches data in two ways -- in a list of ARC buffers (arc_buf_t) and
139 * also in the arc_buf_hdr_t's private physical data block pointer (b_pabd).
140 *
141 * The L1ARC's data pointer may or may not be uncompressed. The ARC has the
142 * ability to store the physical data (b_pabd) associated with the DVA of the
143 * arc_buf_hdr_t. Since the b_pabd is a copy of the on-disk physical block,
144 * it will match its on-disk compression characteristics. This behavior can be
145 * disabled by setting 'zfs_compressed_arc_enabled' to B_FALSE. When the
146 * compressed ARC functionality is disabled, the b_pabd will point to an
147 * uncompressed version of the on-disk data.
148 *
149 * Data in the L1ARC is not accessed by consumers of the ARC directly. Each
150 * arc_buf_hdr_t can have multiple ARC buffers (arc_buf_t) which reference it.
151 * Each ARC buffer (arc_buf_t) is being actively accessed by a specific ARC
152 * consumer. The ARC will provide references to this data and will keep it
153 * cached until it is no longer in use. The ARC caches only the L1ARC's physical
154 * data block and will evict any arc_buf_t that is no longer referenced. The
155 * amount of memory consumed by the arc_buf_ts' data buffers can be seen via the
156 * "overhead_size" kstat.
157 *
158 * Depending on the consumer, an arc_buf_t can be requested in uncompressed or
159 * compressed form. The typical case is that consumers will want uncompressed
160 * data, and when that happens a new data buffer is allocated where the data is
161 * decompressed for them to use. Currently the only consumer who wants
162 * compressed arc_buf_t's is "zfs send", when it streams data exactly as it
163 * exists on disk. When this happens, the arc_buf_t's data buffer is shared
164 * with the arc_buf_hdr_t.
165 *
166 * Here is a diagram showing an arc_buf_hdr_t referenced by two arc_buf_t's. The
167 * first one is owned by a compressed send consumer (and therefore references
168 * the same compressed data buffer as the arc_buf_hdr_t) and the second could be
169 * used by any other consumer (and has its own uncompressed copy of the data
170 * buffer).
171 *
172 *   arc_buf_hdr_t
173 *   +-----------+
174 *   | fields    |
175 *   | common to |
176 *   | L1- and   |
177 *   | L2ARC     |
178 *   +-----------+
179 *   | l2arc_buf_hdr_t
180 *   |           |
181 *   +-----------+
182 *   | l1arc_buf_hdr_t
183 *   |           |              arc_buf_t
184 *   | b_buf     +------------>+-----------+      arc_buf_t
185 *   | b_pabd    +-+           |b_next     +---->+-----------+
186 *   +-----------+ |           |-----------|     |b_next     +-->NULL
187 *                 |           |b_comp = T |     +-----------+
188 *                 |           |b_data     +-+   |b_comp = F |
189 *                 |           +-----------+ |   |b_data     +-+
190 *                 +->+------+               |   +-----------+ |
191 *        compressed  |      |               |                 |
192 *           data     |      |<--------------+                 | uncompressed
193 *                    +------+          compressed,            |     data
194 *                                        shared               +-->+------+
195 *                                         data                    |      |
196 *                                                                 |      |
197 *                                                                 +------+
198 *
199 * When a consumer reads a block, the ARC must first look to see if the
200 * arc_buf_hdr_t is cached. If the hdr is cached then the ARC allocates a new
201 * arc_buf_t and either copies uncompressed data into a new data buffer from an
202 * existing uncompressed arc_buf_t, decompresses the hdr's b_pabd buffer into a
203 * new data buffer, or shares the hdr's b_pabd buffer, depending on whether the
204 * hdr is compressed and the desired compression characteristics of the
205 * arc_buf_t consumer. If the arc_buf_t ends up sharing data with the
206 * arc_buf_hdr_t and both of them are uncompressed then the arc_buf_t must be
207 * the last buffer in the hdr's b_buf list, however a shared compressed buf can
208 * be anywhere in the hdr's list.
209 *
210 * The diagram below shows an example of an uncompressed ARC hdr that is
211 * sharing its data with an arc_buf_t (note that the shared uncompressed buf is
212 * the last element in the buf list):
213 *
214 *                arc_buf_hdr_t
215 *                +-----------+
216 *                |           |
217 *                |           |
218 *                |           |
219 *                +-----------+
220 * l2arc_buf_hdr_t|           |
221 *                |           |
222 *                +-----------+
223 * l1arc_buf_hdr_t|           |
224 *                |           |                 arc_buf_t    (shared)
225 *                |    b_buf  +------------>+---------+      arc_buf_t
226 *                |           |             |b_next   +---->+---------+
227 *                |  b_pabd   +-+           |---------|     |b_next   +-->NULL
228 *                +-----------+ |           |         |     +---------+
229 *                              |           |b_data   +-+   |         |
230 *                              |           +---------+ |   |b_data   +-+
231 *                              +->+------+             |   +---------+ |
232 *                                 |      |             |               |
233 *                   uncompressed  |      |             |               |
234 *                        data     +------+             |               |
235 *                                    ^                 +->+------+     |
236 *                                    |       uncompressed |      |     |
237 *                                    |           data     |      |     |
238 *                                    |                    +------+     |
239 *                                    +---------------------------------+
240 *
241 * Writing to the ARC requires that the ARC first discard the hdr's b_pabd
242 * since the physical block is about to be rewritten. The new data contents
243 * will be contained in the arc_buf_t. As the I/O pipeline performs the write,
244 * it may compress the data before writing it to disk. The ARC will be called
245 * with the transformed data and will bcopy the transformed on-disk block into
246 * a newly allocated b_pabd. Writes are always done into buffers which have
247 * either been loaned (and hence are new and don't have other readers) or
248 * buffers which have been released (and hence have their own hdr, if there
249 * were originally other readers of the buf's original hdr). This ensures that
250 * the ARC only needs to update a single buf and its hdr after a write occurs.
251 *
252 * When the L2ARC is in use, it will also take advantage of the b_pabd. The
253 * L2ARC will always write the contents of b_pabd to the L2ARC. This means
254 * that when compressed ARC is enabled that the L2ARC blocks are identical
255 * to the on-disk block in the main data pool. This provides a significant
256 * advantage since the ARC can leverage the bp's checksum when reading from the
257 * L2ARC to determine if the contents are valid. However, if the compressed
258 * ARC is disabled, then the L2ARC's block must be transformed to look
259 * like the physical block in the main data pool before comparing the
260 * checksum and determining its validity.
261 */
262
263#include <sys/spa.h>
264#include <sys/zio.h>
265#include <sys/spa_impl.h>
266#include <sys/zio_compress.h>
267#include <sys/zio_checksum.h>
268#include <sys/zfs_context.h>
269#include <sys/arc.h>
270#include <sys/refcount.h>
271#include <sys/vdev.h>
272#include <sys/vdev_impl.h>
273#include <sys/dsl_pool.h>
274#include <sys/zio_checksum.h>
275#include <sys/multilist.h>
276#include <sys/abd.h>
277#ifdef _KERNEL
278#include <sys/dnlc.h>
279#include <sys/racct.h>
280#endif
281#include <sys/callb.h>
282#include <sys/kstat.h>
283#include <sys/trim_map.h>
284#include <sys/zthr.h>
285#include <zfs_fletcher.h>
286#include <sys/sdt.h>
287#include <sys/aggsum.h>
288#include <sys/cityhash.h>
289
290#include <machine/vmparam.h>
291
292#ifdef illumos
293#ifndef _KERNEL
294/* set with ZFS_DEBUG=watch, to enable watchpoints on frozen buffers */
295boolean_t arc_watch = B_FALSE;
296int arc_procfd;
297#endif
298#endif /* illumos */
299
300/*
301 * This thread's job is to keep enough free memory in the system, by
302 * calling arc_kmem_reap_now() plus arc_shrink(), which improves
303 * arc_available_memory().
304 */
305static zthr_t		*arc_reap_zthr;
306
307/*
308 * This thread's job is to keep arc_size under arc_c, by calling
309 * arc_adjust(), which improves arc_is_overflowing().
310 */
311static zthr_t		*arc_adjust_zthr;
312
313static kmutex_t		arc_adjust_lock;
314static kcondvar_t	arc_adjust_waiters_cv;
315static boolean_t	arc_adjust_needed = B_FALSE;
316
317static kmutex_t		arc_dnlc_evicts_lock;
318static kcondvar_t	arc_dnlc_evicts_cv;
319static boolean_t	arc_dnlc_evicts_thread_exit;
320
321uint_t arc_reduce_dnlc_percent = 3;
322
323/*
324 * The number of headers to evict in arc_evict_state_impl() before
325 * dropping the sublist lock and evicting from another sublist. A lower
326 * value means we're more likely to evict the "correct" header (i.e. the
327 * oldest header in the arc state), but comes with higher overhead
328 * (i.e. more invocations of arc_evict_state_impl()).
329 */
330int zfs_arc_evict_batch_limit = 10;
331
332/* number of seconds before growing cache again */
333int arc_grow_retry = 60;
334
335/*
336 * Minimum time between calls to arc_kmem_reap_soon().  Note that this will
337 * be converted to ticks, so with the default hz=100, a setting of 15 ms
338 * will actually wait 2 ticks, or 20ms.
339 */
340int arc_kmem_cache_reap_retry_ms = 1000;
341
342/* shift of arc_c for calculating overflow limit in arc_get_data_impl */
343int zfs_arc_overflow_shift = 8;
344
345/* shift of arc_c for calculating both min and max arc_p */
346int arc_p_min_shift = 4;
347
348/* log2(fraction of arc to reclaim) */
349int arc_shrink_shift = 7;
350
351/*
352 * log2(fraction of ARC which must be free to allow growing).
353 * I.e. If there is less than arc_c >> arc_no_grow_shift free memory,
354 * when reading a new block into the ARC, we will evict an equal-sized block
355 * from the ARC.
356 *
357 * This must be less than arc_shrink_shift, so that when we shrink the ARC,
358 * we will still not allow it to grow.
359 */
360int			arc_no_grow_shift = 5;
361
362
363/*
364 * minimum lifespan of a prefetch block in clock ticks
365 * (initialized in arc_init())
366 */
367static int		zfs_arc_min_prefetch_ms = 1;
368static int		zfs_arc_min_prescient_prefetch_ms = 6;
369
370/*
371 * If this percent of memory is free, don't throttle.
372 */
373int arc_lotsfree_percent = 10;
374
375static boolean_t arc_initialized;
376extern boolean_t zfs_prefetch_disable;
377
378/*
379 * The arc has filled available memory and has now warmed up.
380 */
381static boolean_t arc_warm;
382
383/*
384 * log2 fraction of the zio arena to keep free.
385 */
386int arc_zio_arena_free_shift = 2;
387
388/*
389 * These tunables are for performance analysis.
390 */
391uint64_t zfs_arc_max;
392uint64_t zfs_arc_min;
393uint64_t zfs_arc_meta_limit = 0;
394uint64_t zfs_arc_meta_min = 0;
395uint64_t zfs_arc_dnode_limit = 0;
396uint64_t zfs_arc_dnode_reduce_percent = 10;
397int zfs_arc_grow_retry = 0;
398int zfs_arc_shrink_shift = 0;
399int zfs_arc_no_grow_shift = 0;
400int zfs_arc_p_min_shift = 0;
401uint64_t zfs_arc_average_blocksize = 8 * 1024; /* 8KB */
402u_int zfs_arc_free_target = 0;
403
404/* Absolute min for arc min / max is 16MB. */
405static uint64_t arc_abs_min = 16 << 20;
406
407/*
408 * ARC dirty data constraints for arc_tempreserve_space() throttle
409 */
410uint_t zfs_arc_dirty_limit_percent = 50;	/* total dirty data limit */
411uint_t zfs_arc_anon_limit_percent = 25;		/* anon block dirty limit */
412uint_t zfs_arc_pool_dirty_percent = 20;		/* each pool's anon allowance */
413
414boolean_t zfs_compressed_arc_enabled = B_TRUE;
415
416static int sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS);
417static int sysctl_vfs_zfs_arc_meta_limit(SYSCTL_HANDLER_ARGS);
418static int sysctl_vfs_zfs_arc_max(SYSCTL_HANDLER_ARGS);
419static int sysctl_vfs_zfs_arc_min(SYSCTL_HANDLER_ARGS);
420static int sysctl_vfs_zfs_arc_no_grow_shift(SYSCTL_HANDLER_ARGS);
421
422#if defined(__FreeBSD__) && defined(_KERNEL)
423static void
424arc_free_target_init(void *unused __unused)
425{
426
427	zfs_arc_free_target = vm_cnt.v_free_target;
428}
429SYSINIT(arc_free_target_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_ANY,
430    arc_free_target_init, NULL);
431
432TUNABLE_QUAD("vfs.zfs.arc_meta_limit", &zfs_arc_meta_limit);
433TUNABLE_QUAD("vfs.zfs.arc_meta_min", &zfs_arc_meta_min);
434TUNABLE_INT("vfs.zfs.arc_shrink_shift", &zfs_arc_shrink_shift);
435TUNABLE_INT("vfs.zfs.arc_grow_retry", &zfs_arc_grow_retry);
436TUNABLE_INT("vfs.zfs.arc_no_grow_shift", &zfs_arc_no_grow_shift);
437SYSCTL_DECL(_vfs_zfs);
438SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_max,
439    CTLTYPE_U64 | CTLFLAG_MPSAFE | CTLFLAG_RWTUN,
440    0, sizeof(uint64_t), sysctl_vfs_zfs_arc_max, "QU", "Maximum ARC size");
441SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_min,
442    CTLTYPE_U64 | CTLFLAG_MPSAFE | CTLFLAG_RWTUN,
443    0, sizeof(uint64_t), sysctl_vfs_zfs_arc_min, "QU", "Minimum ARC size");
444SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_no_grow_shift,
445    CTLTYPE_U32 | CTLFLAG_MPSAFE | CTLFLAG_RWTUN,
446    0, sizeof(uint32_t), sysctl_vfs_zfs_arc_no_grow_shift, "U",
447    "log2(fraction of ARC which must be free to allow growing)");
448SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, arc_average_blocksize, CTLFLAG_RDTUN,
449    &zfs_arc_average_blocksize, 0,
450    "ARC average blocksize");
451SYSCTL_INT(_vfs_zfs, OID_AUTO, arc_shrink_shift, CTLFLAG_RW,
452    &arc_shrink_shift, 0,
453    "log2(fraction of arc to reclaim)");
454SYSCTL_INT(_vfs_zfs, OID_AUTO, arc_grow_retry, CTLFLAG_RW,
455    &arc_grow_retry, 0,
456    "Wait in seconds before considering growing ARC");
457SYSCTL_INT(_vfs_zfs, OID_AUTO, compressed_arc_enabled, CTLFLAG_RDTUN,
458    &zfs_compressed_arc_enabled, 0,
459    "Enable compressed ARC");
460SYSCTL_INT(_vfs_zfs, OID_AUTO, arc_kmem_cache_reap_retry_ms, CTLFLAG_RWTUN,
461    &arc_kmem_cache_reap_retry_ms, 0,
462    "Interval between ARC kmem_cache reapings");
463
464/*
465 * We don't have a tunable for arc_free_target due to the dependency on
466 * pagedaemon initialisation.
467 */
468SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_free_target,
469    CTLTYPE_UINT | CTLFLAG_MPSAFE | CTLFLAG_RW, 0, sizeof(u_int),
470    sysctl_vfs_zfs_arc_free_target, "IU",
471    "Desired number of free pages below which ARC triggers reclaim");
472
473static int
474sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS)
475{
476	u_int val;
477	int err;
478
479	val = zfs_arc_free_target;
480	err = sysctl_handle_int(oidp, &val, 0, req);
481	if (err != 0 || req->newptr == NULL)
482		return (err);
483
484	if (val < minfree)
485		return (EINVAL);
486	if (val > vm_cnt.v_page_count)
487		return (EINVAL);
488
489	zfs_arc_free_target = val;
490
491	return (0);
492}
493
494/*
495 * Must be declared here, before the definition of corresponding kstat
496 * macro which uses the same names will confuse the compiler.
497 */
498SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_meta_limit,
499    CTLTYPE_U64 | CTLFLAG_MPSAFE | CTLFLAG_RW, 0, sizeof(uint64_t),
500    sysctl_vfs_zfs_arc_meta_limit, "QU",
501    "ARC metadata limit");
502#endif
503
504/*
505 * Note that buffers can be in one of 6 states:
506 *	ARC_anon	- anonymous (discussed below)
507 *	ARC_mru		- recently used, currently cached
508 *	ARC_mru_ghost	- recentely used, no longer in cache
509 *	ARC_mfu		- frequently used, currently cached
510 *	ARC_mfu_ghost	- frequently used, no longer in cache
511 *	ARC_l2c_only	- exists in L2ARC but not other states
512 * When there are no active references to the buffer, they are
513 * are linked onto a list in one of these arc states.  These are
514 * the only buffers that can be evicted or deleted.  Within each
515 * state there are multiple lists, one for meta-data and one for
516 * non-meta-data.  Meta-data (indirect blocks, blocks of dnodes,
517 * etc.) is tracked separately so that it can be managed more
518 * explicitly: favored over data, limited explicitly.
519 *
520 * Anonymous buffers are buffers that are not associated with
521 * a DVA.  These are buffers that hold dirty block copies
522 * before they are written to stable storage.  By definition,
523 * they are "ref'd" and are considered part of arc_mru
524 * that cannot be freed.  Generally, they will aquire a DVA
525 * as they are written and migrate onto the arc_mru list.
526 *
527 * The ARC_l2c_only state is for buffers that are in the second
528 * level ARC but no longer in any of the ARC_m* lists.  The second
529 * level ARC itself may also contain buffers that are in any of
530 * the ARC_m* states - meaning that a buffer can exist in two
531 * places.  The reason for the ARC_l2c_only state is to keep the
532 * buffer header in the hash table, so that reads that hit the
533 * second level ARC benefit from these fast lookups.
534 */
535
536typedef struct arc_state {
537	/*
538	 * list of evictable buffers
539	 */
540	multilist_t *arcs_list[ARC_BUFC_NUMTYPES];
541	/*
542	 * total amount of evictable data in this state
543	 */
544	zfs_refcount_t arcs_esize[ARC_BUFC_NUMTYPES];
545	/*
546	 * total amount of data in this state; this includes: evictable,
547	 * non-evictable, ARC_BUFC_DATA, and ARC_BUFC_METADATA.
548	 */
549	zfs_refcount_t arcs_size;
550	/*
551	 * supports the "dbufs" kstat
552	 */
553	arc_state_type_t arcs_state;
554} arc_state_t;
555
556/*
557 * Percentage that can be consumed by dnodes of ARC meta buffers.
558 */
559int zfs_arc_meta_prune = 10000;
560unsigned long zfs_arc_dnode_limit_percent = 10;
561int zfs_arc_meta_strategy = ARC_STRATEGY_META_ONLY;
562int zfs_arc_meta_adjust_restarts = 4096;
563
564SYSCTL_INT(_vfs_zfs, OID_AUTO, arc_meta_strategy, CTLFLAG_RWTUN,
565    &zfs_arc_meta_strategy, 0,
566    "ARC metadata reclamation strategy "
567    "(0 = metadata only, 1 = balance data and metadata)");
568
569/* The 6 states: */
570static arc_state_t ARC_anon;
571static arc_state_t ARC_mru;
572static arc_state_t ARC_mru_ghost;
573static arc_state_t ARC_mfu;
574static arc_state_t ARC_mfu_ghost;
575static arc_state_t ARC_l2c_only;
576
577typedef struct arc_stats {
578	kstat_named_t arcstat_hits;
579	kstat_named_t arcstat_misses;
580	kstat_named_t arcstat_demand_data_hits;
581	kstat_named_t arcstat_demand_data_misses;
582	kstat_named_t arcstat_demand_metadata_hits;
583	kstat_named_t arcstat_demand_metadata_misses;
584	kstat_named_t arcstat_prefetch_data_hits;
585	kstat_named_t arcstat_prefetch_data_misses;
586	kstat_named_t arcstat_prefetch_metadata_hits;
587	kstat_named_t arcstat_prefetch_metadata_misses;
588	kstat_named_t arcstat_mru_hits;
589	kstat_named_t arcstat_mru_ghost_hits;
590	kstat_named_t arcstat_mfu_hits;
591	kstat_named_t arcstat_mfu_ghost_hits;
592	kstat_named_t arcstat_allocated;
593	kstat_named_t arcstat_deleted;
594	/*
595	 * Number of buffers that could not be evicted because the hash lock
596	 * was held by another thread.  The lock may not necessarily be held
597	 * by something using the same buffer, since hash locks are shared
598	 * by multiple buffers.
599	 */
600	kstat_named_t arcstat_mutex_miss;
601	/*
602	 * Number of buffers skipped when updating the access state due to the
603	 * header having already been released after acquiring the hash lock.
604	 */
605	kstat_named_t arcstat_access_skip;
606	/*
607	 * Number of buffers skipped because they have I/O in progress, are
608	 * indirect prefetch buffers that have not lived long enough, or are
609	 * not from the spa we're trying to evict from.
610	 */
611	kstat_named_t arcstat_evict_skip;
612	/*
613	 * Number of times arc_evict_state() was unable to evict enough
614	 * buffers to reach it's target amount.
615	 */
616	kstat_named_t arcstat_evict_not_enough;
617	kstat_named_t arcstat_evict_l2_cached;
618	kstat_named_t arcstat_evict_l2_eligible;
619	kstat_named_t arcstat_evict_l2_ineligible;
620	kstat_named_t arcstat_evict_l2_skip;
621	kstat_named_t arcstat_hash_elements;
622	kstat_named_t arcstat_hash_elements_max;
623	kstat_named_t arcstat_hash_collisions;
624	kstat_named_t arcstat_hash_chains;
625	kstat_named_t arcstat_hash_chain_max;
626	kstat_named_t arcstat_p;
627	kstat_named_t arcstat_c;
628	kstat_named_t arcstat_c_min;
629	kstat_named_t arcstat_c_max;
630	/* Not updated directly; only synced in arc_kstat_update. */
631	kstat_named_t arcstat_size;
632	/*
633	 * Number of compressed bytes stored in the arc_buf_hdr_t's b_pabd.
634	 * Note that the compressed bytes may match the uncompressed bytes
635	 * if the block is either not compressed or compressed arc is disabled.
636	 */
637	kstat_named_t arcstat_compressed_size;
638	/*
639	 * Uncompressed size of the data stored in b_pabd. If compressed
640	 * arc is disabled then this value will be identical to the stat
641	 * above.
642	 */
643	kstat_named_t arcstat_uncompressed_size;
644	/*
645	 * Number of bytes stored in all the arc_buf_t's. This is classified
646	 * as "overhead" since this data is typically short-lived and will
647	 * be evicted from the arc when it becomes unreferenced unless the
648	 * zfs_keep_uncompressed_metadata or zfs_keep_uncompressed_level
649	 * values have been set (see comment in dbuf.c for more information).
650	 */
651	kstat_named_t arcstat_overhead_size;
652	/*
653	 * Number of bytes consumed by internal ARC structures necessary
654	 * for tracking purposes; these structures are not actually
655	 * backed by ARC buffers. This includes arc_buf_hdr_t structures
656	 * (allocated via arc_buf_hdr_t_full and arc_buf_hdr_t_l2only
657	 * caches), and arc_buf_t structures (allocated via arc_buf_t
658	 * cache).
659	 * Not updated directly; only synced in arc_kstat_update.
660	 */
661	kstat_named_t arcstat_hdr_size;
662	/*
663	 * Number of bytes consumed by ARC buffers of type equal to
664	 * ARC_BUFC_DATA. This is generally consumed by buffers backing
665	 * on disk user data (e.g. plain file contents).
666	 * Not updated directly; only synced in arc_kstat_update.
667	 */
668	kstat_named_t arcstat_data_size;
669	/*
670	 * Number of bytes consumed by ARC buffers of type equal to
671	 * ARC_BUFC_METADATA. This is generally consumed by buffers
672	 * backing on disk data that is used for internal ZFS
673	 * structures (e.g. ZAP, dnode, indirect blocks, etc).
674	 * Not updated directly; only synced in arc_kstat_update.
675	 */
676	kstat_named_t arcstat_metadata_size;
677	/*
678	 * Number of bytes consumed by dmu_buf_impl_t objects.
679	 */
680	kstat_named_t arcstat_dbuf_size;
681	/*
682	 * Number of bytes consumed by dnode_t objects.
683	 */
684	kstat_named_t arcstat_dnode_size;
685	/*
686	 * Number of bytes consumed by bonus buffers.
687	 */
688	kstat_named_t arcstat_bonus_size;
689#if defined(__FreeBSD__) && defined(COMPAT_FREEBSD11)
690	/*
691	 * Sum of the previous three counters, provided for compatibility.
692	 */
693	kstat_named_t arcstat_other_size;
694#endif
695	/*
696	 * Total number of bytes consumed by ARC buffers residing in the
697	 * arc_anon state. This includes *all* buffers in the arc_anon
698	 * state; e.g. data, metadata, evictable, and unevictable buffers
699	 * are all included in this value.
700	 * Not updated directly; only synced in arc_kstat_update.
701	 */
702	kstat_named_t arcstat_anon_size;
703	/*
704	 * Number of bytes consumed by ARC buffers that meet the
705	 * following criteria: backing buffers of type ARC_BUFC_DATA,
706	 * residing in the arc_anon state, and are eligible for eviction
707	 * (e.g. have no outstanding holds on the buffer).
708	 * Not updated directly; only synced in arc_kstat_update.
709	 */
710	kstat_named_t arcstat_anon_evictable_data;
711	/*
712	 * Number of bytes consumed by ARC buffers that meet the
713	 * following criteria: backing buffers of type ARC_BUFC_METADATA,
714	 * residing in the arc_anon state, and are eligible for eviction
715	 * (e.g. have no outstanding holds on the buffer).
716	 * Not updated directly; only synced in arc_kstat_update.
717	 */
718	kstat_named_t arcstat_anon_evictable_metadata;
719	/*
720	 * Total number of bytes consumed by ARC buffers residing in the
721	 * arc_mru state. This includes *all* buffers in the arc_mru
722	 * state; e.g. data, metadata, evictable, and unevictable buffers
723	 * are all included in this value.
724	 * Not updated directly; only synced in arc_kstat_update.
725	 */
726	kstat_named_t arcstat_mru_size;
727	/*
728	 * Number of bytes consumed by ARC buffers that meet the
729	 * following criteria: backing buffers of type ARC_BUFC_DATA,
730	 * residing in the arc_mru state, and are eligible for eviction
731	 * (e.g. have no outstanding holds on the buffer).
732	 * Not updated directly; only synced in arc_kstat_update.
733	 */
734	kstat_named_t arcstat_mru_evictable_data;
735	/*
736	 * Number of bytes consumed by ARC buffers that meet the
737	 * following criteria: backing buffers of type ARC_BUFC_METADATA,
738	 * residing in the arc_mru state, and are eligible for eviction
739	 * (e.g. have no outstanding holds on the buffer).
740	 * Not updated directly; only synced in arc_kstat_update.
741	 */
742	kstat_named_t arcstat_mru_evictable_metadata;
743	/*
744	 * Total number of bytes that *would have been* consumed by ARC
745	 * buffers in the arc_mru_ghost state. The key thing to note
746	 * here, is the fact that this size doesn't actually indicate
747	 * RAM consumption. The ghost lists only consist of headers and
748	 * don't actually have ARC buffers linked off of these headers.
749	 * Thus, *if* the headers had associated ARC buffers, these
750	 * buffers *would have* consumed this number of bytes.
751	 * Not updated directly; only synced in arc_kstat_update.
752	 */
753	kstat_named_t arcstat_mru_ghost_size;
754	/*
755	 * Number of bytes that *would have been* consumed by ARC
756	 * buffers that are eligible for eviction, of type
757	 * ARC_BUFC_DATA, and linked off the arc_mru_ghost state.
758	 * Not updated directly; only synced in arc_kstat_update.
759	 */
760	kstat_named_t arcstat_mru_ghost_evictable_data;
761	/*
762	 * Number of bytes that *would have been* consumed by ARC
763	 * buffers that are eligible for eviction, of type
764	 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state.
765	 * Not updated directly; only synced in arc_kstat_update.
766	 */
767	kstat_named_t arcstat_mru_ghost_evictable_metadata;
768	/*
769	 * Total number of bytes consumed by ARC buffers residing in the
770	 * arc_mfu state. This includes *all* buffers in the arc_mfu
771	 * state; e.g. data, metadata, evictable, and unevictable buffers
772	 * are all included in this value.
773	 * Not updated directly; only synced in arc_kstat_update.
774	 */
775	kstat_named_t arcstat_mfu_size;
776	/*
777	 * Number of bytes consumed by ARC buffers that are eligible for
778	 * eviction, of type ARC_BUFC_DATA, and reside in the arc_mfu
779	 * state.
780	 * Not updated directly; only synced in arc_kstat_update.
781	 */
782	kstat_named_t arcstat_mfu_evictable_data;
783	/*
784	 * Number of bytes consumed by ARC buffers that are eligible for
785	 * eviction, of type ARC_BUFC_METADATA, and reside in the
786	 * arc_mfu state.
787	 * Not updated directly; only synced in arc_kstat_update.
788	 */
789	kstat_named_t arcstat_mfu_evictable_metadata;
790	/*
791	 * Total number of bytes that *would have been* consumed by ARC
792	 * buffers in the arc_mfu_ghost state. See the comment above
793	 * arcstat_mru_ghost_size for more details.
794	 * Not updated directly; only synced in arc_kstat_update.
795	 */
796	kstat_named_t arcstat_mfu_ghost_size;
797	/*
798	 * Number of bytes that *would have been* consumed by ARC
799	 * buffers that are eligible for eviction, of type
800	 * ARC_BUFC_DATA, and linked off the arc_mfu_ghost state.
801	 * Not updated directly; only synced in arc_kstat_update.
802	 */
803	kstat_named_t arcstat_mfu_ghost_evictable_data;
804	/*
805	 * Number of bytes that *would have been* consumed by ARC
806	 * buffers that are eligible for eviction, of type
807	 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state.
808	 * Not updated directly; only synced in arc_kstat_update.
809	 */
810	kstat_named_t arcstat_mfu_ghost_evictable_metadata;
811	kstat_named_t arcstat_l2_hits;
812	kstat_named_t arcstat_l2_misses;
813	kstat_named_t arcstat_l2_feeds;
814	kstat_named_t arcstat_l2_rw_clash;
815	kstat_named_t arcstat_l2_read_bytes;
816	kstat_named_t arcstat_l2_write_bytes;
817	kstat_named_t arcstat_l2_writes_sent;
818	kstat_named_t arcstat_l2_writes_done;
819	kstat_named_t arcstat_l2_writes_error;
820	kstat_named_t arcstat_l2_writes_lock_retry;
821	kstat_named_t arcstat_l2_evict_lock_retry;
822	kstat_named_t arcstat_l2_evict_reading;
823	kstat_named_t arcstat_l2_evict_l1cached;
824	kstat_named_t arcstat_l2_free_on_write;
825	kstat_named_t arcstat_l2_abort_lowmem;
826	kstat_named_t arcstat_l2_cksum_bad;
827	kstat_named_t arcstat_l2_io_error;
828	kstat_named_t arcstat_l2_lsize;
829	kstat_named_t arcstat_l2_psize;
830	/* Not updated directly; only synced in arc_kstat_update. */
831	kstat_named_t arcstat_l2_hdr_size;
832	kstat_named_t arcstat_l2_write_trylock_fail;
833	kstat_named_t arcstat_l2_write_passed_headroom;
834	kstat_named_t arcstat_l2_write_spa_mismatch;
835	kstat_named_t arcstat_l2_write_in_l2;
836	kstat_named_t arcstat_l2_write_hdr_io_in_progress;
837	kstat_named_t arcstat_l2_write_not_cacheable;
838	kstat_named_t arcstat_l2_write_full;
839	kstat_named_t arcstat_l2_write_buffer_iter;
840	kstat_named_t arcstat_l2_write_pios;
841	kstat_named_t arcstat_l2_write_buffer_bytes_scanned;
842	kstat_named_t arcstat_l2_write_buffer_list_iter;
843	kstat_named_t arcstat_l2_write_buffer_list_null_iter;
844	kstat_named_t arcstat_memory_throttle_count;
845	kstat_named_t arcstat_memory_direct_count;
846	kstat_named_t arcstat_memory_indirect_count;
847	kstat_named_t arcstat_memory_all_bytes;
848	kstat_named_t arcstat_memory_free_bytes;
849	kstat_named_t arcstat_memory_available_bytes;
850	kstat_named_t arcstat_no_grow;
851	kstat_named_t arcstat_tempreserve;
852	kstat_named_t arcstat_loaned_bytes;
853	kstat_named_t arcstat_prune;
854	/* Not updated directly; only synced in arc_kstat_update. */
855	kstat_named_t arcstat_meta_used;
856	kstat_named_t arcstat_meta_limit;
857	kstat_named_t arcstat_dnode_limit;
858	kstat_named_t arcstat_meta_max;
859	kstat_named_t arcstat_meta_min;
860	kstat_named_t arcstat_async_upgrade_sync;
861	kstat_named_t arcstat_demand_hit_predictive_prefetch;
862	kstat_named_t arcstat_demand_hit_prescient_prefetch;
863} arc_stats_t;
864
865static arc_stats_t arc_stats = {
866	{ "hits",			KSTAT_DATA_UINT64 },
867	{ "misses",			KSTAT_DATA_UINT64 },
868	{ "demand_data_hits",		KSTAT_DATA_UINT64 },
869	{ "demand_data_misses",		KSTAT_DATA_UINT64 },
870	{ "demand_metadata_hits",	KSTAT_DATA_UINT64 },
871	{ "demand_metadata_misses",	KSTAT_DATA_UINT64 },
872	{ "prefetch_data_hits",		KSTAT_DATA_UINT64 },
873	{ "prefetch_data_misses",	KSTAT_DATA_UINT64 },
874	{ "prefetch_metadata_hits",	KSTAT_DATA_UINT64 },
875	{ "prefetch_metadata_misses",	KSTAT_DATA_UINT64 },
876	{ "mru_hits",			KSTAT_DATA_UINT64 },
877	{ "mru_ghost_hits",		KSTAT_DATA_UINT64 },
878	{ "mfu_hits",			KSTAT_DATA_UINT64 },
879	{ "mfu_ghost_hits",		KSTAT_DATA_UINT64 },
880	{ "allocated",			KSTAT_DATA_UINT64 },
881	{ "deleted",			KSTAT_DATA_UINT64 },
882	{ "mutex_miss",			KSTAT_DATA_UINT64 },
883	{ "access_skip",		KSTAT_DATA_UINT64 },
884	{ "evict_skip",			KSTAT_DATA_UINT64 },
885	{ "evict_not_enough",		KSTAT_DATA_UINT64 },
886	{ "evict_l2_cached",		KSTAT_DATA_UINT64 },
887	{ "evict_l2_eligible",		KSTAT_DATA_UINT64 },
888	{ "evict_l2_ineligible",	KSTAT_DATA_UINT64 },
889	{ "evict_l2_skip",		KSTAT_DATA_UINT64 },
890	{ "hash_elements",		KSTAT_DATA_UINT64 },
891	{ "hash_elements_max",		KSTAT_DATA_UINT64 },
892	{ "hash_collisions",		KSTAT_DATA_UINT64 },
893	{ "hash_chains",		KSTAT_DATA_UINT64 },
894	{ "hash_chain_max",		KSTAT_DATA_UINT64 },
895	{ "p",				KSTAT_DATA_UINT64 },
896	{ "c",				KSTAT_DATA_UINT64 },
897	{ "c_min",			KSTAT_DATA_UINT64 },
898	{ "c_max",			KSTAT_DATA_UINT64 },
899	{ "size",			KSTAT_DATA_UINT64 },
900	{ "compressed_size",		KSTAT_DATA_UINT64 },
901	{ "uncompressed_size",		KSTAT_DATA_UINT64 },
902	{ "overhead_size",		KSTAT_DATA_UINT64 },
903	{ "hdr_size",			KSTAT_DATA_UINT64 },
904	{ "data_size",			KSTAT_DATA_UINT64 },
905	{ "metadata_size",		KSTAT_DATA_UINT64 },
906	{ "dbuf_size",			KSTAT_DATA_UINT64 },
907	{ "dnode_size",			KSTAT_DATA_UINT64 },
908	{ "bonus_size",			KSTAT_DATA_UINT64 },
909#if defined(__FreeBSD__) && defined(COMPAT_FREEBSD11)
910	{ "other_size",			KSTAT_DATA_UINT64 },
911#endif
912	{ "anon_size",			KSTAT_DATA_UINT64 },
913	{ "anon_evictable_data",	KSTAT_DATA_UINT64 },
914	{ "anon_evictable_metadata",	KSTAT_DATA_UINT64 },
915	{ "mru_size",			KSTAT_DATA_UINT64 },
916	{ "mru_evictable_data",		KSTAT_DATA_UINT64 },
917	{ "mru_evictable_metadata",	KSTAT_DATA_UINT64 },
918	{ "mru_ghost_size",		KSTAT_DATA_UINT64 },
919	{ "mru_ghost_evictable_data",	KSTAT_DATA_UINT64 },
920	{ "mru_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
921	{ "mfu_size",			KSTAT_DATA_UINT64 },
922	{ "mfu_evictable_data",		KSTAT_DATA_UINT64 },
923	{ "mfu_evictable_metadata",	KSTAT_DATA_UINT64 },
924	{ "mfu_ghost_size",		KSTAT_DATA_UINT64 },
925	{ "mfu_ghost_evictable_data",	KSTAT_DATA_UINT64 },
926	{ "mfu_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
927	{ "l2_hits",			KSTAT_DATA_UINT64 },
928	{ "l2_misses",			KSTAT_DATA_UINT64 },
929	{ "l2_feeds",			KSTAT_DATA_UINT64 },
930	{ "l2_rw_clash",		KSTAT_DATA_UINT64 },
931	{ "l2_read_bytes",		KSTAT_DATA_UINT64 },
932	{ "l2_write_bytes",		KSTAT_DATA_UINT64 },
933	{ "l2_writes_sent",		KSTAT_DATA_UINT64 },
934	{ "l2_writes_done",		KSTAT_DATA_UINT64 },
935	{ "l2_writes_error",		KSTAT_DATA_UINT64 },
936	{ "l2_writes_lock_retry",	KSTAT_DATA_UINT64 },
937	{ "l2_evict_lock_retry",	KSTAT_DATA_UINT64 },
938	{ "l2_evict_reading",		KSTAT_DATA_UINT64 },
939	{ "l2_evict_l1cached",		KSTAT_DATA_UINT64 },
940	{ "l2_free_on_write",		KSTAT_DATA_UINT64 },
941	{ "l2_abort_lowmem",		KSTAT_DATA_UINT64 },
942	{ "l2_cksum_bad",		KSTAT_DATA_UINT64 },
943	{ "l2_io_error",		KSTAT_DATA_UINT64 },
944	{ "l2_size",			KSTAT_DATA_UINT64 },
945	{ "l2_asize",			KSTAT_DATA_UINT64 },
946	{ "l2_hdr_size",		KSTAT_DATA_UINT64 },
947	{ "l2_write_trylock_fail",	KSTAT_DATA_UINT64 },
948	{ "l2_write_passed_headroom",	KSTAT_DATA_UINT64 },
949	{ "l2_write_spa_mismatch",	KSTAT_DATA_UINT64 },
950	{ "l2_write_in_l2",		KSTAT_DATA_UINT64 },
951	{ "l2_write_io_in_progress",	KSTAT_DATA_UINT64 },
952	{ "l2_write_not_cacheable",	KSTAT_DATA_UINT64 },
953	{ "l2_write_full",		KSTAT_DATA_UINT64 },
954	{ "l2_write_buffer_iter",	KSTAT_DATA_UINT64 },
955	{ "l2_write_pios",		KSTAT_DATA_UINT64 },
956	{ "l2_write_buffer_bytes_scanned", KSTAT_DATA_UINT64 },
957	{ "l2_write_buffer_list_iter",	KSTAT_DATA_UINT64 },
958	{ "l2_write_buffer_list_null_iter", KSTAT_DATA_UINT64 },
959	{ "memory_throttle_count",	KSTAT_DATA_UINT64 },
960	{ "memory_direct_count",	KSTAT_DATA_UINT64 },
961	{ "memory_indirect_count",	KSTAT_DATA_UINT64 },
962	{ "memory_all_bytes",		KSTAT_DATA_UINT64 },
963	{ "memory_free_bytes",		KSTAT_DATA_UINT64 },
964	{ "memory_available_bytes",	KSTAT_DATA_UINT64 },
965	{ "arc_no_grow",		KSTAT_DATA_UINT64 },
966	{ "arc_tempreserve",		KSTAT_DATA_UINT64 },
967	{ "arc_loaned_bytes",		KSTAT_DATA_UINT64 },
968	{ "arc_prune",			KSTAT_DATA_UINT64 },
969	{ "arc_meta_used",		KSTAT_DATA_UINT64 },
970	{ "arc_meta_limit",		KSTAT_DATA_UINT64 },
971	{ "arc_dnode_limit",		KSTAT_DATA_UINT64 },
972	{ "arc_meta_max",		KSTAT_DATA_UINT64 },
973	{ "arc_meta_min",		KSTAT_DATA_UINT64 },
974	{ "async_upgrade_sync",		KSTAT_DATA_UINT64 },
975	{ "demand_hit_predictive_prefetch", KSTAT_DATA_UINT64 },
976	{ "demand_hit_prescient_prefetch", KSTAT_DATA_UINT64 },
977};
978
979#define	ARCSTAT(stat)	(arc_stats.stat.value.ui64)
980
981#define	ARCSTAT_INCR(stat, val) \
982	atomic_add_64(&arc_stats.stat.value.ui64, (val))
983
984#define	ARCSTAT_BUMP(stat)	ARCSTAT_INCR(stat, 1)
985#define	ARCSTAT_BUMPDOWN(stat)	ARCSTAT_INCR(stat, -1)
986
987#define	ARCSTAT_MAX(stat, val) {					\
988	uint64_t m;							\
989	while ((val) > (m = arc_stats.stat.value.ui64) &&		\
990	    (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val))))	\
991		continue;						\
992}
993
994#define	ARCSTAT_MAXSTAT(stat) \
995	ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64)
996
997/*
998 * We define a macro to allow ARC hits/misses to be easily broken down by
999 * two separate conditions, giving a total of four different subtypes for
1000 * each of hits and misses (so eight statistics total).
1001 */
1002#define	ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \
1003	if (cond1) {							\
1004		if (cond2) {						\
1005			ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \
1006		} else {						\
1007			ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \
1008		}							\
1009	} else {							\
1010		if (cond2) {						\
1011			ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \
1012		} else {						\
1013			ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\
1014		}							\
1015	}
1016
1017kstat_t			*arc_ksp;
1018static arc_state_t	*arc_anon;
1019static arc_state_t	*arc_mru;
1020static arc_state_t	*arc_mru_ghost;
1021static arc_state_t	*arc_mfu;
1022static arc_state_t	*arc_mfu_ghost;
1023static arc_state_t	*arc_l2c_only;
1024
1025/*
1026 * There are several ARC variables that are critical to export as kstats --
1027 * but we don't want to have to grovel around in the kstat whenever we wish to
1028 * manipulate them.  For these variables, we therefore define them to be in
1029 * terms of the statistic variable.  This assures that we are not introducing
1030 * the possibility of inconsistency by having shadow copies of the variables,
1031 * while still allowing the code to be readable.
1032 */
1033#define	arc_p		ARCSTAT(arcstat_p)	/* target size of MRU */
1034#define	arc_c		ARCSTAT(arcstat_c)	/* target size of cache */
1035#define	arc_c_min	ARCSTAT(arcstat_c_min)	/* min target cache size */
1036#define	arc_c_max	ARCSTAT(arcstat_c_max)	/* max target cache size */
1037#define	arc_meta_limit	ARCSTAT(arcstat_meta_limit) /* max size for metadata */
1038#define	arc_dnode_limit	ARCSTAT(arcstat_dnode_limit) /* max size for dnodes */
1039#define	arc_meta_min	ARCSTAT(arcstat_meta_min) /* min size for metadata */
1040#define	arc_meta_max	ARCSTAT(arcstat_meta_max) /* max size of metadata */
1041#define	arc_dbuf_size	ARCSTAT(arcstat_dbuf_size) /* dbuf metadata */
1042#define	arc_dnode_size	ARCSTAT(arcstat_dnode_size) /* dnode metadata */
1043#define	arc_bonus_size	ARCSTAT(arcstat_bonus_size) /* bonus buffer metadata */
1044
1045/* compressed size of entire arc */
1046#define	arc_compressed_size	ARCSTAT(arcstat_compressed_size)
1047/* uncompressed size of entire arc */
1048#define	arc_uncompressed_size	ARCSTAT(arcstat_uncompressed_size)
1049/* number of bytes in the arc from arc_buf_t's */
1050#define	arc_overhead_size	ARCSTAT(arcstat_overhead_size)
1051
1052/*
1053 * There are also some ARC variables that we want to export, but that are
1054 * updated so often that having the canonical representation be the statistic
1055 * variable causes a performance bottleneck. We want to use aggsum_t's for these
1056 * instead, but still be able to export the kstat in the same way as before.
1057 * The solution is to always use the aggsum version, except in the kstat update
1058 * callback.
1059 */
1060aggsum_t arc_size;
1061aggsum_t arc_meta_used;
1062aggsum_t astat_data_size;
1063aggsum_t astat_metadata_size;
1064aggsum_t astat_hdr_size;
1065aggsum_t astat_bonus_size;
1066aggsum_t astat_dnode_size;
1067aggsum_t astat_dbuf_size;
1068aggsum_t astat_l2_hdr_size;
1069
1070static list_t arc_prune_list;
1071static kmutex_t arc_prune_mtx;
1072static taskq_t *arc_prune_taskq;
1073
1074static int		arc_no_grow;	/* Don't try to grow cache size */
1075static hrtime_t		arc_growtime;
1076static uint64_t		arc_tempreserve;
1077static uint64_t		arc_loaned_bytes;
1078
1079typedef struct arc_callback arc_callback_t;
1080
1081struct arc_callback {
1082	void			*acb_private;
1083	arc_read_done_func_t	*acb_done;
1084	arc_buf_t		*acb_buf;
1085	boolean_t		acb_compressed;
1086	zio_t			*acb_zio_dummy;
1087	zio_t			*acb_zio_head;
1088	arc_callback_t		*acb_next;
1089};
1090
1091typedef struct arc_write_callback arc_write_callback_t;
1092
1093struct arc_write_callback {
1094	void			*awcb_private;
1095	arc_write_done_func_t	*awcb_ready;
1096	arc_write_done_func_t	*awcb_children_ready;
1097	arc_write_done_func_t	*awcb_physdone;
1098	arc_write_done_func_t	*awcb_done;
1099	arc_buf_t		*awcb_buf;
1100};
1101
1102/*
1103 * ARC buffers are separated into multiple structs as a memory saving measure:
1104 *   - Common fields struct, always defined, and embedded within it:
1105 *       - L2-only fields, always allocated but undefined when not in L2ARC
1106 *       - L1-only fields, only allocated when in L1ARC
1107 *
1108 *           Buffer in L1                     Buffer only in L2
1109 *    +------------------------+          +------------------------+
1110 *    | arc_buf_hdr_t          |          | arc_buf_hdr_t          |
1111 *    |                        |          |                        |
1112 *    |                        |          |                        |
1113 *    |                        |          |                        |
1114 *    +------------------------+          +------------------------+
1115 *    | l2arc_buf_hdr_t        |          | l2arc_buf_hdr_t        |
1116 *    | (undefined if L1-only) |          |                        |
1117 *    +------------------------+          +------------------------+
1118 *    | l1arc_buf_hdr_t        |
1119 *    |                        |
1120 *    |                        |
1121 *    |                        |
1122 *    |                        |
1123 *    +------------------------+
1124 *
1125 * Because it's possible for the L2ARC to become extremely large, we can wind
1126 * up eating a lot of memory in L2ARC buffer headers, so the size of a header
1127 * is minimized by only allocating the fields necessary for an L1-cached buffer
1128 * when a header is actually in the L1 cache. The sub-headers (l1arc_buf_hdr and
1129 * l2arc_buf_hdr) are embedded rather than allocated separately to save a couple
1130 * words in pointers. arc_hdr_realloc() is used to switch a header between
1131 * these two allocation states.
1132 */
1133typedef struct l1arc_buf_hdr {
1134	kmutex_t		b_freeze_lock;
1135	zio_cksum_t		*b_freeze_cksum;
1136#ifdef ZFS_DEBUG
1137	/*
1138	 * Used for debugging with kmem_flags - by allocating and freeing
1139	 * b_thawed when the buffer is thawed, we get a record of the stack
1140	 * trace that thawed it.
1141	 */
1142	void			*b_thawed;
1143#endif
1144
1145	arc_buf_t		*b_buf;
1146	uint32_t		b_bufcnt;
1147	/* for waiting on writes to complete */
1148	kcondvar_t		b_cv;
1149	uint8_t			b_byteswap;
1150
1151	/* protected by arc state mutex */
1152	arc_state_t		*b_state;
1153	multilist_node_t	b_arc_node;
1154
1155	/* updated atomically */
1156	clock_t			b_arc_access;
1157	uint32_t		b_mru_hits;
1158	uint32_t		b_mru_ghost_hits;
1159	uint32_t		b_mfu_hits;
1160	uint32_t		b_mfu_ghost_hits;
1161	uint32_t		b_l2_hits;
1162
1163	/* self protecting */
1164	zfs_refcount_t		b_refcnt;
1165
1166	arc_callback_t		*b_acb;
1167	abd_t			*b_pabd;
1168} l1arc_buf_hdr_t;
1169
1170typedef struct l2arc_dev l2arc_dev_t;
1171
1172typedef struct l2arc_buf_hdr {
1173	/* protected by arc_buf_hdr mutex */
1174	l2arc_dev_t		*b_dev;		/* L2ARC device */
1175	uint64_t		b_daddr;	/* disk address, offset byte */
1176	uint32_t		b_hits;
1177
1178	list_node_t		b_l2node;
1179} l2arc_buf_hdr_t;
1180
1181struct arc_buf_hdr {
1182	/* protected by hash lock */
1183	dva_t			b_dva;
1184	uint64_t		b_birth;
1185
1186	arc_buf_contents_t	b_type;
1187	arc_buf_hdr_t		*b_hash_next;
1188	arc_flags_t		b_flags;
1189
1190	/*
1191	 * This field stores the size of the data buffer after
1192	 * compression, and is set in the arc's zio completion handlers.
1193	 * It is in units of SPA_MINBLOCKSIZE (e.g. 1 == 512 bytes).
1194	 *
1195	 * While the block pointers can store up to 32MB in their psize
1196	 * field, we can only store up to 32MB minus 512B. This is due
1197	 * to the bp using a bias of 1, whereas we use a bias of 0 (i.e.
1198	 * a field of zeros represents 512B in the bp). We can't use a
1199	 * bias of 1 since we need to reserve a psize of zero, here, to
1200	 * represent holes and embedded blocks.
1201	 *
1202	 * This isn't a problem in practice, since the maximum size of a
1203	 * buffer is limited to 16MB, so we never need to store 32MB in
1204	 * this field. Even in the upstream illumos code base, the
1205	 * maximum size of a buffer is limited to 16MB.
1206	 */
1207	uint16_t		b_psize;
1208
1209	/*
1210	 * This field stores the size of the data buffer before
1211	 * compression, and cannot change once set. It is in units
1212	 * of SPA_MINBLOCKSIZE (e.g. 2 == 1024 bytes)
1213	 */
1214	uint16_t		b_lsize;	/* immutable */
1215	uint64_t		b_spa;		/* immutable */
1216
1217	/* L2ARC fields. Undefined when not in L2ARC. */
1218	l2arc_buf_hdr_t		b_l2hdr;
1219	/* L1ARC fields. Undefined when in l2arc_only state */
1220	l1arc_buf_hdr_t		b_l1hdr;
1221};
1222
1223#if defined(__FreeBSD__) && defined(_KERNEL)
1224static int
1225sysctl_vfs_zfs_arc_meta_limit(SYSCTL_HANDLER_ARGS)
1226{
1227	uint64_t val;
1228	int err;
1229
1230	val = arc_meta_limit;
1231	err = sysctl_handle_64(oidp, &val, 0, req);
1232	if (err != 0 || req->newptr == NULL)
1233		return (err);
1234
1235        if (val <= 0 || val > arc_c_max)
1236		return (EINVAL);
1237
1238	arc_meta_limit = val;
1239
1240	mutex_enter(&arc_adjust_lock);
1241	arc_adjust_needed = B_TRUE;
1242	mutex_exit(&arc_adjust_lock);
1243	zthr_wakeup(arc_adjust_zthr);
1244
1245	return (0);
1246}
1247
1248static int
1249sysctl_vfs_zfs_arc_no_grow_shift(SYSCTL_HANDLER_ARGS)
1250{
1251	uint32_t val;
1252	int err;
1253
1254	val = arc_no_grow_shift;
1255	err = sysctl_handle_32(oidp, &val, 0, req);
1256	if (err != 0 || req->newptr == NULL)
1257		return (err);
1258
1259        if (val >= arc_shrink_shift)
1260		return (EINVAL);
1261
1262	arc_no_grow_shift = val;
1263	return (0);
1264}
1265
1266static int
1267sysctl_vfs_zfs_arc_max(SYSCTL_HANDLER_ARGS)
1268{
1269	uint64_t val;
1270	int err;
1271
1272	val = zfs_arc_max;
1273	err = sysctl_handle_64(oidp, &val, 0, req);
1274	if (err != 0 || req->newptr == NULL)
1275		return (err);
1276
1277	if (zfs_arc_max == 0) {
1278		/* Loader tunable so blindly set */
1279		zfs_arc_max = val;
1280		return (0);
1281	}
1282
1283	if (val < arc_abs_min || val > kmem_size())
1284		return (EINVAL);
1285	if (val < arc_c_min)
1286		return (EINVAL);
1287	if (zfs_arc_meta_limit > 0 && val < zfs_arc_meta_limit)
1288		return (EINVAL);
1289
1290	arc_c_max = val;
1291
1292	arc_c = arc_c_max;
1293        arc_p = (arc_c >> 1);
1294
1295	if (zfs_arc_meta_limit == 0) {
1296		/* limit meta-data to 1/4 of the arc capacity */
1297		arc_meta_limit = arc_c_max / 4;
1298	}
1299
1300	/* if kmem_flags are set, lets try to use less memory */
1301	if (kmem_debugging())
1302		arc_c = arc_c / 2;
1303
1304	zfs_arc_max = arc_c;
1305
1306	mutex_enter(&arc_adjust_lock);
1307	arc_adjust_needed = B_TRUE;
1308	mutex_exit(&arc_adjust_lock);
1309	zthr_wakeup(arc_adjust_zthr);
1310
1311	return (0);
1312}
1313
1314static int
1315sysctl_vfs_zfs_arc_min(SYSCTL_HANDLER_ARGS)
1316{
1317	uint64_t val;
1318	int err;
1319
1320	val = zfs_arc_min;
1321	err = sysctl_handle_64(oidp, &val, 0, req);
1322	if (err != 0 || req->newptr == NULL)
1323		return (err);
1324
1325	if (zfs_arc_min == 0) {
1326		/* Loader tunable so blindly set */
1327		zfs_arc_min = val;
1328		return (0);
1329	}
1330
1331	if (val < arc_abs_min || val > arc_c_max)
1332		return (EINVAL);
1333
1334	arc_c_min = val;
1335
1336	if (zfs_arc_meta_min == 0)
1337                arc_meta_min = arc_c_min / 2;
1338
1339	if (arc_c < arc_c_min)
1340                arc_c = arc_c_min;
1341
1342	zfs_arc_min = arc_c_min;
1343
1344	return (0);
1345}
1346#endif
1347
1348#define	GHOST_STATE(state)	\
1349	((state) == arc_mru_ghost || (state) == arc_mfu_ghost ||	\
1350	(state) == arc_l2c_only)
1351
1352#define	HDR_IN_HASH_TABLE(hdr)	((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE)
1353#define	HDR_IO_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS)
1354#define	HDR_IO_ERROR(hdr)	((hdr)->b_flags & ARC_FLAG_IO_ERROR)
1355#define	HDR_PREFETCH(hdr)	((hdr)->b_flags & ARC_FLAG_PREFETCH)
1356#define	HDR_PRESCIENT_PREFETCH(hdr)	\
1357	((hdr)->b_flags & ARC_FLAG_PRESCIENT_PREFETCH)
1358#define	HDR_COMPRESSION_ENABLED(hdr)	\
1359	((hdr)->b_flags & ARC_FLAG_COMPRESSED_ARC)
1360
1361#define	HDR_L2CACHE(hdr)	((hdr)->b_flags & ARC_FLAG_L2CACHE)
1362#define	HDR_L2_READING(hdr)	\
1363	(((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) &&	\
1364	((hdr)->b_flags & ARC_FLAG_HAS_L2HDR))
1365#define	HDR_L2_WRITING(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITING)
1366#define	HDR_L2_EVICTED(hdr)	((hdr)->b_flags & ARC_FLAG_L2_EVICTED)
1367#define	HDR_L2_WRITE_HEAD(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD)
1368#define	HDR_SHARED_DATA(hdr)	((hdr)->b_flags & ARC_FLAG_SHARED_DATA)
1369
1370#define	HDR_ISTYPE_METADATA(hdr)	\
1371	((hdr)->b_flags & ARC_FLAG_BUFC_METADATA)
1372#define	HDR_ISTYPE_DATA(hdr)	(!HDR_ISTYPE_METADATA(hdr))
1373
1374#define	HDR_HAS_L1HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L1HDR)
1375#define	HDR_HAS_L2HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)
1376
1377/* For storing compression mode in b_flags */
1378#define	HDR_COMPRESS_OFFSET	(highbit64(ARC_FLAG_COMPRESS_0) - 1)
1379
1380#define	HDR_GET_COMPRESS(hdr)	((enum zio_compress)BF32_GET((hdr)->b_flags, \
1381	HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS))
1382#define	HDR_SET_COMPRESS(hdr, cmp) BF32_SET((hdr)->b_flags, \
1383	HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS, (cmp));
1384
1385#define	ARC_BUF_LAST(buf)	((buf)->b_next == NULL)
1386#define	ARC_BUF_SHARED(buf)	((buf)->b_flags & ARC_BUF_FLAG_SHARED)
1387#define	ARC_BUF_COMPRESSED(buf)	((buf)->b_flags & ARC_BUF_FLAG_COMPRESSED)
1388
1389/*
1390 * Other sizes
1391 */
1392
1393#define	HDR_FULL_SIZE ((int64_t)sizeof (arc_buf_hdr_t))
1394#define	HDR_L2ONLY_SIZE ((int64_t)offsetof(arc_buf_hdr_t, b_l1hdr))
1395
1396/*
1397 * Hash table routines
1398 */
1399
1400#define	HT_LOCK_PAD	CACHE_LINE_SIZE
1401
1402struct ht_lock {
1403	kmutex_t	ht_lock;
1404#ifdef _KERNEL
1405	unsigned char	pad[(HT_LOCK_PAD - sizeof (kmutex_t))];
1406#endif
1407};
1408
1409#define	BUF_LOCKS 256
1410typedef struct buf_hash_table {
1411	uint64_t ht_mask;
1412	arc_buf_hdr_t **ht_table;
1413	struct ht_lock ht_locks[BUF_LOCKS] __aligned(CACHE_LINE_SIZE);
1414} buf_hash_table_t;
1415
1416static buf_hash_table_t buf_hash_table;
1417
1418#define	BUF_HASH_INDEX(spa, dva, birth) \
1419	(buf_hash(spa, dva, birth) & buf_hash_table.ht_mask)
1420#define	BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)])
1421#define	BUF_HASH_LOCK(idx)	(&(BUF_HASH_LOCK_NTRY(idx).ht_lock))
1422#define	HDR_LOCK(hdr) \
1423	(BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
1424
1425uint64_t zfs_crc64_table[256];
1426
1427/*
1428 * Level 2 ARC
1429 */
1430
1431#define	L2ARC_WRITE_SIZE	(8 * 1024 * 1024)	/* initial write max */
1432#define	L2ARC_HEADROOM		2			/* num of writes */
1433/*
1434 * If we discover during ARC scan any buffers to be compressed, we boost
1435 * our headroom for the next scanning cycle by this percentage multiple.
1436 */
1437#define	L2ARC_HEADROOM_BOOST	200
1438#define	L2ARC_FEED_SECS		1		/* caching interval secs */
1439#define	L2ARC_FEED_MIN_MS	200		/* min caching interval ms */
1440
1441#define	l2arc_writes_sent	ARCSTAT(arcstat_l2_writes_sent)
1442#define	l2arc_writes_done	ARCSTAT(arcstat_l2_writes_done)
1443
1444/* L2ARC Performance Tunables */
1445uint64_t l2arc_write_max = L2ARC_WRITE_SIZE;	/* default max write size */
1446uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE;	/* extra write during warmup */
1447uint64_t l2arc_headroom = L2ARC_HEADROOM;	/* number of dev writes */
1448uint64_t l2arc_headroom_boost = L2ARC_HEADROOM_BOOST;
1449uint64_t l2arc_feed_secs = L2ARC_FEED_SECS;	/* interval seconds */
1450uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS;	/* min interval milliseconds */
1451boolean_t l2arc_noprefetch = B_TRUE;		/* don't cache prefetch bufs */
1452boolean_t l2arc_feed_again = B_TRUE;		/* turbo warmup */
1453boolean_t l2arc_norw = B_TRUE;			/* no reads during writes */
1454
1455SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_write_max, CTLFLAG_RWTUN,
1456    &l2arc_write_max, 0, "max write size");
1457SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_write_boost, CTLFLAG_RWTUN,
1458    &l2arc_write_boost, 0, "extra write during warmup");
1459SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_headroom, CTLFLAG_RWTUN,
1460    &l2arc_headroom, 0, "number of dev writes");
1461SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_feed_secs, CTLFLAG_RWTUN,
1462    &l2arc_feed_secs, 0, "interval seconds");
1463SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_feed_min_ms, CTLFLAG_RWTUN,
1464    &l2arc_feed_min_ms, 0, "min interval milliseconds");
1465
1466SYSCTL_INT(_vfs_zfs, OID_AUTO, l2arc_noprefetch, CTLFLAG_RWTUN,
1467    &l2arc_noprefetch, 0, "don't cache prefetch bufs");
1468SYSCTL_INT(_vfs_zfs, OID_AUTO, l2arc_feed_again, CTLFLAG_RWTUN,
1469    &l2arc_feed_again, 0, "turbo warmup");
1470SYSCTL_INT(_vfs_zfs, OID_AUTO, l2arc_norw, CTLFLAG_RWTUN,
1471    &l2arc_norw, 0, "no reads during writes");
1472
1473SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, anon_size, CTLFLAG_RD,
1474    &ARC_anon.arcs_size.rc_count, 0, "size of anonymous state");
1475SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, anon_metadata_esize, CTLFLAG_RD,
1476    &ARC_anon.arcs_esize[ARC_BUFC_METADATA].rc_count, 0,
1477    "size of anonymous state");
1478SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, anon_data_esize, CTLFLAG_RD,
1479    &ARC_anon.arcs_esize[ARC_BUFC_DATA].rc_count, 0,
1480    "size of anonymous state");
1481
1482SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_size, CTLFLAG_RD,
1483    &ARC_mru.arcs_size.rc_count, 0, "size of mru state");
1484SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_metadata_esize, CTLFLAG_RD,
1485    &ARC_mru.arcs_esize[ARC_BUFC_METADATA].rc_count, 0,
1486    "size of metadata in mru state");
1487SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_data_esize, CTLFLAG_RD,
1488    &ARC_mru.arcs_esize[ARC_BUFC_DATA].rc_count, 0,
1489    "size of data in mru state");
1490
1491SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_ghost_size, CTLFLAG_RD,
1492    &ARC_mru_ghost.arcs_size.rc_count, 0, "size of mru ghost state");
1493SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_ghost_metadata_esize, CTLFLAG_RD,
1494    &ARC_mru_ghost.arcs_esize[ARC_BUFC_METADATA].rc_count, 0,
1495    "size of metadata in mru ghost state");
1496SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_ghost_data_esize, CTLFLAG_RD,
1497    &ARC_mru_ghost.arcs_esize[ARC_BUFC_DATA].rc_count, 0,
1498    "size of data in mru ghost state");
1499
1500SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_size, CTLFLAG_RD,
1501    &ARC_mfu.arcs_size.rc_count, 0, "size of mfu state");
1502SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_metadata_esize, CTLFLAG_RD,
1503    &ARC_mfu.arcs_esize[ARC_BUFC_METADATA].rc_count, 0,
1504    "size of metadata in mfu state");
1505SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_data_esize, CTLFLAG_RD,
1506    &ARC_mfu.arcs_esize[ARC_BUFC_DATA].rc_count, 0,
1507    "size of data in mfu state");
1508
1509SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_ghost_size, CTLFLAG_RD,
1510    &ARC_mfu_ghost.arcs_size.rc_count, 0, "size of mfu ghost state");
1511SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_ghost_metadata_esize, CTLFLAG_RD,
1512    &ARC_mfu_ghost.arcs_esize[ARC_BUFC_METADATA].rc_count, 0,
1513    "size of metadata in mfu ghost state");
1514SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_ghost_data_esize, CTLFLAG_RD,
1515    &ARC_mfu_ghost.arcs_esize[ARC_BUFC_DATA].rc_count, 0,
1516    "size of data in mfu ghost state");
1517
1518SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2c_only_size, CTLFLAG_RD,
1519    &ARC_l2c_only.arcs_size.rc_count, 0, "size of mru state");
1520
1521SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_min_prefetch_ms, CTLFLAG_RW,
1522    &zfs_arc_min_prefetch_ms, 0, "Min life of prefetch block in ms");
1523SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_min_prescient_prefetch_ms, CTLFLAG_RW,
1524    &zfs_arc_min_prescient_prefetch_ms, 0, "Min life of prescient prefetched block in ms");
1525
1526/*
1527 * L2ARC Internals
1528 */
1529struct l2arc_dev {
1530	vdev_t			*l2ad_vdev;	/* vdev */
1531	spa_t			*l2ad_spa;	/* spa */
1532	uint64_t		l2ad_hand;	/* next write location */
1533	uint64_t		l2ad_start;	/* first addr on device */
1534	uint64_t		l2ad_end;	/* last addr on device */
1535	boolean_t		l2ad_first;	/* first sweep through */
1536	boolean_t		l2ad_writing;	/* currently writing */
1537	kmutex_t		l2ad_mtx;	/* lock for buffer list */
1538	list_t			l2ad_buflist;	/* buffer list */
1539	list_node_t		l2ad_node;	/* device list node */
1540	zfs_refcount_t		l2ad_alloc;	/* allocated bytes */
1541};
1542
1543static list_t L2ARC_dev_list;			/* device list */
1544static list_t *l2arc_dev_list;			/* device list pointer */
1545static kmutex_t l2arc_dev_mtx;			/* device list mutex */
1546static l2arc_dev_t *l2arc_dev_last;		/* last device used */
1547static list_t L2ARC_free_on_write;		/* free after write buf list */
1548static list_t *l2arc_free_on_write;		/* free after write list ptr */
1549static kmutex_t l2arc_free_on_write_mtx;	/* mutex for list */
1550static uint64_t l2arc_ndev;			/* number of devices */
1551
1552typedef struct l2arc_read_callback {
1553	arc_buf_hdr_t		*l2rcb_hdr;		/* read header */
1554	blkptr_t		l2rcb_bp;		/* original blkptr */
1555	zbookmark_phys_t	l2rcb_zb;		/* original bookmark */
1556	int			l2rcb_flags;		/* original flags */
1557	abd_t			*l2rcb_abd;		/* temporary buffer */
1558} l2arc_read_callback_t;
1559
1560typedef struct l2arc_write_callback {
1561	l2arc_dev_t	*l2wcb_dev;		/* device info */
1562	arc_buf_hdr_t	*l2wcb_head;		/* head of write buflist */
1563} l2arc_write_callback_t;
1564
1565typedef struct l2arc_data_free {
1566	/* protected by l2arc_free_on_write_mtx */
1567	abd_t		*l2df_abd;
1568	size_t		l2df_size;
1569	arc_buf_contents_t l2df_type;
1570	list_node_t	l2df_list_node;
1571} l2arc_data_free_t;
1572
1573static kmutex_t l2arc_feed_thr_lock;
1574static kcondvar_t l2arc_feed_thr_cv;
1575static uint8_t l2arc_thread_exit;
1576
1577static abd_t *arc_get_data_abd(arc_buf_hdr_t *, uint64_t, void *, boolean_t);
1578static void *arc_get_data_buf(arc_buf_hdr_t *, uint64_t, void *);
1579static void arc_get_data_impl(arc_buf_hdr_t *, uint64_t, void *, boolean_t);
1580static void arc_free_data_abd(arc_buf_hdr_t *, abd_t *, uint64_t, void *);
1581static void arc_free_data_buf(arc_buf_hdr_t *, void *, uint64_t, void *);
1582static void arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag);
1583static void arc_hdr_free_pabd(arc_buf_hdr_t *);
1584static void arc_hdr_alloc_pabd(arc_buf_hdr_t *, boolean_t);
1585static void arc_access(arc_buf_hdr_t *, kmutex_t *);
1586static boolean_t arc_is_overflowing();
1587static void arc_buf_watch(arc_buf_t *);
1588static void arc_prune_async(int64_t);
1589
1590static arc_buf_contents_t arc_buf_type(arc_buf_hdr_t *);
1591static uint32_t arc_bufc_to_flags(arc_buf_contents_t);
1592static inline void arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
1593static inline void arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
1594
1595static boolean_t l2arc_write_eligible(uint64_t, arc_buf_hdr_t *);
1596static void l2arc_read_done(zio_t *);
1597
1598static void
1599l2arc_trim(const arc_buf_hdr_t *hdr)
1600{
1601	l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
1602
1603	ASSERT(HDR_HAS_L2HDR(hdr));
1604	ASSERT(MUTEX_HELD(&dev->l2ad_mtx));
1605
1606	if (HDR_GET_PSIZE(hdr) != 0) {
1607		trim_map_free(dev->l2ad_vdev, hdr->b_l2hdr.b_daddr,
1608		    HDR_GET_PSIZE(hdr), 0);
1609	}
1610}
1611
1612/*
1613 * We use Cityhash for this. It's fast, and has good hash properties without
1614 * requiring any large static buffers.
1615 */
1616static uint64_t
1617buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth)
1618{
1619	return (cityhash4(spa, dva->dva_word[0], dva->dva_word[1], birth));
1620}
1621
1622#define	HDR_EMPTY(hdr)						\
1623	((hdr)->b_dva.dva_word[0] == 0 &&			\
1624	(hdr)->b_dva.dva_word[1] == 0)
1625
1626#define	HDR_EQUAL(spa, dva, birth, hdr)				\
1627	((hdr)->b_dva.dva_word[0] == (dva)->dva_word[0]) &&	\
1628	((hdr)->b_dva.dva_word[1] == (dva)->dva_word[1]) &&	\
1629	((hdr)->b_birth == birth) && ((hdr)->b_spa == spa)
1630
1631static void
1632buf_discard_identity(arc_buf_hdr_t *hdr)
1633{
1634	hdr->b_dva.dva_word[0] = 0;
1635	hdr->b_dva.dva_word[1] = 0;
1636	hdr->b_birth = 0;
1637}
1638
1639static arc_buf_hdr_t *
1640buf_hash_find(uint64_t spa, const blkptr_t *bp, kmutex_t **lockp)
1641{
1642	const dva_t *dva = BP_IDENTITY(bp);
1643	uint64_t birth = BP_PHYSICAL_BIRTH(bp);
1644	uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
1645	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
1646	arc_buf_hdr_t *hdr;
1647
1648	mutex_enter(hash_lock);
1649	for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL;
1650	    hdr = hdr->b_hash_next) {
1651		if (HDR_EQUAL(spa, dva, birth, hdr)) {
1652			*lockp = hash_lock;
1653			return (hdr);
1654		}
1655	}
1656	mutex_exit(hash_lock);
1657	*lockp = NULL;
1658	return (NULL);
1659}
1660
1661/*
1662 * Insert an entry into the hash table.  If there is already an element
1663 * equal to elem in the hash table, then the already existing element
1664 * will be returned and the new element will not be inserted.
1665 * Otherwise returns NULL.
1666 * If lockp == NULL, the caller is assumed to already hold the hash lock.
1667 */
1668static arc_buf_hdr_t *
1669buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp)
1670{
1671	uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1672	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
1673	arc_buf_hdr_t *fhdr;
1674	uint32_t i;
1675
1676	ASSERT(!DVA_IS_EMPTY(&hdr->b_dva));
1677	ASSERT(hdr->b_birth != 0);
1678	ASSERT(!HDR_IN_HASH_TABLE(hdr));
1679
1680	if (lockp != NULL) {
1681		*lockp = hash_lock;
1682		mutex_enter(hash_lock);
1683	} else {
1684		ASSERT(MUTEX_HELD(hash_lock));
1685	}
1686
1687	for (fhdr = buf_hash_table.ht_table[idx], i = 0; fhdr != NULL;
1688	    fhdr = fhdr->b_hash_next, i++) {
1689		if (HDR_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr))
1690			return (fhdr);
1691	}
1692
1693	hdr->b_hash_next = buf_hash_table.ht_table[idx];
1694	buf_hash_table.ht_table[idx] = hdr;
1695	arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
1696
1697	/* collect some hash table performance data */
1698	if (i > 0) {
1699		ARCSTAT_BUMP(arcstat_hash_collisions);
1700		if (i == 1)
1701			ARCSTAT_BUMP(arcstat_hash_chains);
1702
1703		ARCSTAT_MAX(arcstat_hash_chain_max, i);
1704	}
1705
1706	ARCSTAT_BUMP(arcstat_hash_elements);
1707	ARCSTAT_MAXSTAT(arcstat_hash_elements);
1708
1709	return (NULL);
1710}
1711
1712static void
1713buf_hash_remove(arc_buf_hdr_t *hdr)
1714{
1715	arc_buf_hdr_t *fhdr, **hdrp;
1716	uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1717
1718	ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx)));
1719	ASSERT(HDR_IN_HASH_TABLE(hdr));
1720
1721	hdrp = &buf_hash_table.ht_table[idx];
1722	while ((fhdr = *hdrp) != hdr) {
1723		ASSERT3P(fhdr, !=, NULL);
1724		hdrp = &fhdr->b_hash_next;
1725	}
1726	*hdrp = hdr->b_hash_next;
1727	hdr->b_hash_next = NULL;
1728	arc_hdr_clear_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
1729
1730	/* collect some hash table performance data */
1731	ARCSTAT_BUMPDOWN(arcstat_hash_elements);
1732
1733	if (buf_hash_table.ht_table[idx] &&
1734	    buf_hash_table.ht_table[idx]->b_hash_next == NULL)
1735		ARCSTAT_BUMPDOWN(arcstat_hash_chains);
1736}
1737
1738/*
1739 * Global data structures and functions for the buf kmem cache.
1740 */
1741static kmem_cache_t *hdr_full_cache;
1742static kmem_cache_t *hdr_l2only_cache;
1743static kmem_cache_t *buf_cache;
1744
1745static void
1746buf_fini(void)
1747{
1748	int i;
1749
1750	kmem_free(buf_hash_table.ht_table,
1751	    (buf_hash_table.ht_mask + 1) * sizeof (void *));
1752	for (i = 0; i < BUF_LOCKS; i++)
1753		mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock);
1754	kmem_cache_destroy(hdr_full_cache);
1755	kmem_cache_destroy(hdr_l2only_cache);
1756	kmem_cache_destroy(buf_cache);
1757}
1758
1759/*
1760 * Constructor callback - called when the cache is empty
1761 * and a new buf is requested.
1762 */
1763/* ARGSUSED */
1764static int
1765hdr_full_cons(void *vbuf, void *unused, int kmflag)
1766{
1767	arc_buf_hdr_t *hdr = vbuf;
1768
1769	bzero(hdr, HDR_FULL_SIZE);
1770	cv_init(&hdr->b_l1hdr.b_cv, NULL, CV_DEFAULT, NULL);
1771	zfs_refcount_create(&hdr->b_l1hdr.b_refcnt);
1772	mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
1773	multilist_link_init(&hdr->b_l1hdr.b_arc_node);
1774	arc_space_consume(HDR_FULL_SIZE, ARC_SPACE_HDRS);
1775
1776	return (0);
1777}
1778
1779/* ARGSUSED */
1780static int
1781hdr_l2only_cons(void *vbuf, void *unused, int kmflag)
1782{
1783	arc_buf_hdr_t *hdr = vbuf;
1784
1785	bzero(hdr, HDR_L2ONLY_SIZE);
1786	arc_space_consume(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
1787
1788	return (0);
1789}
1790
1791/* ARGSUSED */
1792static int
1793buf_cons(void *vbuf, void *unused, int kmflag)
1794{
1795	arc_buf_t *buf = vbuf;
1796
1797	bzero(buf, sizeof (arc_buf_t));
1798	mutex_init(&buf->b_evict_lock, NULL, MUTEX_DEFAULT, NULL);
1799	arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS);
1800
1801	return (0);
1802}
1803
1804/*
1805 * Destructor callback - called when a cached buf is
1806 * no longer required.
1807 */
1808/* ARGSUSED */
1809static void
1810hdr_full_dest(void *vbuf, void *unused)
1811{
1812	arc_buf_hdr_t *hdr = vbuf;
1813
1814	ASSERT(HDR_EMPTY(hdr));
1815	cv_destroy(&hdr->b_l1hdr.b_cv);
1816	zfs_refcount_destroy(&hdr->b_l1hdr.b_refcnt);
1817	mutex_destroy(&hdr->b_l1hdr.b_freeze_lock);
1818	ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
1819	arc_space_return(HDR_FULL_SIZE, ARC_SPACE_HDRS);
1820}
1821
1822/* ARGSUSED */
1823static void
1824hdr_l2only_dest(void *vbuf, void *unused)
1825{
1826	arc_buf_hdr_t *hdr = vbuf;
1827
1828	ASSERT(HDR_EMPTY(hdr));
1829	arc_space_return(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
1830}
1831
1832/* ARGSUSED */
1833static void
1834buf_dest(void *vbuf, void *unused)
1835{
1836	arc_buf_t *buf = vbuf;
1837
1838	mutex_destroy(&buf->b_evict_lock);
1839	arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS);
1840}
1841
1842/*
1843 * Reclaim callback -- invoked when memory is low.
1844 */
1845/* ARGSUSED */
1846static void
1847hdr_recl(void *unused)
1848{
1849	dprintf("hdr_recl called\n");
1850	/*
1851	 * umem calls the reclaim func when we destroy the buf cache,
1852	 * which is after we do arc_fini().
1853	 */
1854	if (arc_initialized)
1855		zthr_wakeup(arc_reap_zthr);
1856}
1857
1858static void
1859buf_init(void)
1860{
1861	uint64_t *ct;
1862	uint64_t hsize = 1ULL << 12;
1863	int i, j;
1864
1865	/*
1866	 * The hash table is big enough to fill all of physical memory
1867	 * with an average block size of zfs_arc_average_blocksize (default 8K).
1868	 * By default, the table will take up
1869	 * totalmem * sizeof(void*) / 8K (1MB per GB with 8-byte pointers).
1870	 */
1871	while (hsize * zfs_arc_average_blocksize < (uint64_t)physmem * PAGESIZE)
1872		hsize <<= 1;
1873retry:
1874	buf_hash_table.ht_mask = hsize - 1;
1875	buf_hash_table.ht_table =
1876	    kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP);
1877	if (buf_hash_table.ht_table == NULL) {
1878		ASSERT(hsize > (1ULL << 8));
1879		hsize >>= 1;
1880		goto retry;
1881	}
1882
1883	hdr_full_cache = kmem_cache_create("arc_buf_hdr_t_full", HDR_FULL_SIZE,
1884	    0, hdr_full_cons, hdr_full_dest, hdr_recl, NULL, NULL, 0);
1885	hdr_l2only_cache = kmem_cache_create("arc_buf_hdr_t_l2only",
1886	    HDR_L2ONLY_SIZE, 0, hdr_l2only_cons, hdr_l2only_dest, hdr_recl,
1887	    NULL, NULL, 0);
1888	buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t),
1889	    0, buf_cons, buf_dest, NULL, NULL, NULL, 0);
1890
1891	for (i = 0; i < 256; i++)
1892		for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--)
1893			*ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY);
1894
1895	for (i = 0; i < BUF_LOCKS; i++) {
1896		mutex_init(&buf_hash_table.ht_locks[i].ht_lock,
1897		    NULL, MUTEX_DEFAULT, NULL);
1898	}
1899}
1900
1901/*
1902 * This is the size that the buf occupies in memory. If the buf is compressed,
1903 * it will correspond to the compressed size. You should use this method of
1904 * getting the buf size unless you explicitly need the logical size.
1905 */
1906int32_t
1907arc_buf_size(arc_buf_t *buf)
1908{
1909	return (ARC_BUF_COMPRESSED(buf) ?
1910	    HDR_GET_PSIZE(buf->b_hdr) : HDR_GET_LSIZE(buf->b_hdr));
1911}
1912
1913int32_t
1914arc_buf_lsize(arc_buf_t *buf)
1915{
1916	return (HDR_GET_LSIZE(buf->b_hdr));
1917}
1918
1919enum zio_compress
1920arc_get_compression(arc_buf_t *buf)
1921{
1922	return (ARC_BUF_COMPRESSED(buf) ?
1923	    HDR_GET_COMPRESS(buf->b_hdr) : ZIO_COMPRESS_OFF);
1924}
1925
1926#define	ARC_MINTIME	(hz>>4) /* 62 ms */
1927
1928static inline boolean_t
1929arc_buf_is_shared(arc_buf_t *buf)
1930{
1931	boolean_t shared = (buf->b_data != NULL &&
1932	    buf->b_hdr->b_l1hdr.b_pabd != NULL &&
1933	    abd_is_linear(buf->b_hdr->b_l1hdr.b_pabd) &&
1934	    buf->b_data == abd_to_buf(buf->b_hdr->b_l1hdr.b_pabd));
1935	IMPLY(shared, HDR_SHARED_DATA(buf->b_hdr));
1936	IMPLY(shared, ARC_BUF_SHARED(buf));
1937	IMPLY(shared, ARC_BUF_COMPRESSED(buf) || ARC_BUF_LAST(buf));
1938
1939	/*
1940	 * It would be nice to assert arc_can_share() too, but the "hdr isn't
1941	 * already being shared" requirement prevents us from doing that.
1942	 */
1943
1944	return (shared);
1945}
1946
1947/*
1948 * Free the checksum associated with this header. If there is no checksum, this
1949 * is a no-op.
1950 */
1951static inline void
1952arc_cksum_free(arc_buf_hdr_t *hdr)
1953{
1954	ASSERT(HDR_HAS_L1HDR(hdr));
1955	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1956	if (hdr->b_l1hdr.b_freeze_cksum != NULL) {
1957		kmem_free(hdr->b_l1hdr.b_freeze_cksum, sizeof (zio_cksum_t));
1958		hdr->b_l1hdr.b_freeze_cksum = NULL;
1959	}
1960	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1961}
1962
1963/*
1964 * Return true iff at least one of the bufs on hdr is not compressed.
1965 */
1966static boolean_t
1967arc_hdr_has_uncompressed_buf(arc_buf_hdr_t *hdr)
1968{
1969	for (arc_buf_t *b = hdr->b_l1hdr.b_buf; b != NULL; b = b->b_next) {
1970		if (!ARC_BUF_COMPRESSED(b)) {
1971			return (B_TRUE);
1972		}
1973	}
1974	return (B_FALSE);
1975}
1976
1977/*
1978 * If we've turned on the ZFS_DEBUG_MODIFY flag, verify that the buf's data
1979 * matches the checksum that is stored in the hdr. If there is no checksum,
1980 * or if the buf is compressed, this is a no-op.
1981 */
1982static void
1983arc_cksum_verify(arc_buf_t *buf)
1984{
1985	arc_buf_hdr_t *hdr = buf->b_hdr;
1986	zio_cksum_t zc;
1987
1988	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1989		return;
1990
1991	if (ARC_BUF_COMPRESSED(buf)) {
1992		ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL ||
1993		    arc_hdr_has_uncompressed_buf(hdr));
1994		return;
1995	}
1996
1997	ASSERT(HDR_HAS_L1HDR(hdr));
1998
1999	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
2000	if (hdr->b_l1hdr.b_freeze_cksum == NULL || HDR_IO_ERROR(hdr)) {
2001		mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
2002		return;
2003	}
2004
2005	fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL, &zc);
2006	if (!ZIO_CHECKSUM_EQUAL(*hdr->b_l1hdr.b_freeze_cksum, zc))
2007		panic("buffer modified while frozen!");
2008	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
2009}
2010
2011static boolean_t
2012arc_cksum_is_equal(arc_buf_hdr_t *hdr, zio_t *zio)
2013{
2014	enum zio_compress compress = BP_GET_COMPRESS(zio->io_bp);
2015	boolean_t valid_cksum;
2016
2017	ASSERT(!BP_IS_EMBEDDED(zio->io_bp));
2018	VERIFY3U(BP_GET_PSIZE(zio->io_bp), ==, HDR_GET_PSIZE(hdr));
2019
2020	/*
2021	 * We rely on the blkptr's checksum to determine if the block
2022	 * is valid or not. When compressed arc is enabled, the l2arc
2023	 * writes the block to the l2arc just as it appears in the pool.
2024	 * This allows us to use the blkptr's checksum to validate the
2025	 * data that we just read off of the l2arc without having to store
2026	 * a separate checksum in the arc_buf_hdr_t. However, if compressed
2027	 * arc is disabled, then the data written to the l2arc is always
2028	 * uncompressed and won't match the block as it exists in the main
2029	 * pool. When this is the case, we must first compress it if it is
2030	 * compressed on the main pool before we can validate the checksum.
2031	 */
2032	if (!HDR_COMPRESSION_ENABLED(hdr) && compress != ZIO_COMPRESS_OFF) {
2033		ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
2034		uint64_t lsize = HDR_GET_LSIZE(hdr);
2035		uint64_t csize;
2036
2037		abd_t *cdata = abd_alloc_linear(HDR_GET_PSIZE(hdr), B_TRUE);
2038		csize = zio_compress_data(compress, zio->io_abd,
2039		    abd_to_buf(cdata), lsize);
2040
2041		ASSERT3U(csize, <=, HDR_GET_PSIZE(hdr));
2042		if (csize < HDR_GET_PSIZE(hdr)) {
2043			/*
2044			 * Compressed blocks are always a multiple of the
2045			 * smallest ashift in the pool. Ideally, we would
2046			 * like to round up the csize to the next
2047			 * spa_min_ashift but that value may have changed
2048			 * since the block was last written. Instead,
2049			 * we rely on the fact that the hdr's psize
2050			 * was set to the psize of the block when it was
2051			 * last written. We set the csize to that value
2052			 * and zero out any part that should not contain
2053			 * data.
2054			 */
2055			abd_zero_off(cdata, csize, HDR_GET_PSIZE(hdr) - csize);
2056			csize = HDR_GET_PSIZE(hdr);
2057		}
2058		zio_push_transform(zio, cdata, csize, HDR_GET_PSIZE(hdr), NULL);
2059	}
2060
2061	/*
2062	 * Block pointers always store the checksum for the logical data.
2063	 * If the block pointer has the gang bit set, then the checksum
2064	 * it represents is for the reconstituted data and not for an
2065	 * individual gang member. The zio pipeline, however, must be able to
2066	 * determine the checksum of each of the gang constituents so it
2067	 * treats the checksum comparison differently than what we need
2068	 * for l2arc blocks. This prevents us from using the
2069	 * zio_checksum_error() interface directly. Instead we must call the
2070	 * zio_checksum_error_impl() so that we can ensure the checksum is
2071	 * generated using the correct checksum algorithm and accounts for the
2072	 * logical I/O size and not just a gang fragment.
2073	 */
2074	valid_cksum = (zio_checksum_error_impl(zio->io_spa, zio->io_bp,
2075	    BP_GET_CHECKSUM(zio->io_bp), zio->io_abd, zio->io_size,
2076	    zio->io_offset, NULL) == 0);
2077	zio_pop_transforms(zio);
2078	return (valid_cksum);
2079}
2080
2081/*
2082 * Given a buf full of data, if ZFS_DEBUG_MODIFY is enabled this computes a
2083 * checksum and attaches it to the buf's hdr so that we can ensure that the buf
2084 * isn't modified later on. If buf is compressed or there is already a checksum
2085 * on the hdr, this is a no-op (we only checksum uncompressed bufs).
2086 */
2087static void
2088arc_cksum_compute(arc_buf_t *buf)
2089{
2090	arc_buf_hdr_t *hdr = buf->b_hdr;
2091
2092	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
2093		return;
2094
2095	ASSERT(HDR_HAS_L1HDR(hdr));
2096
2097	mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
2098	if (hdr->b_l1hdr.b_freeze_cksum != NULL) {
2099		ASSERT(arc_hdr_has_uncompressed_buf(hdr));
2100		mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
2101		return;
2102	} else if (ARC_BUF_COMPRESSED(buf)) {
2103		mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
2104		return;
2105	}
2106
2107	ASSERT(!ARC_BUF_COMPRESSED(buf));
2108	hdr->b_l1hdr.b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t),
2109	    KM_SLEEP);
2110	fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL,
2111	    hdr->b_l1hdr.b_freeze_cksum);
2112	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
2113#ifdef illumos
2114	arc_buf_watch(buf);
2115#endif
2116}
2117
2118#ifdef illumos
2119#ifndef _KERNEL
2120typedef struct procctl {
2121	long cmd;
2122	prwatch_t prwatch;
2123} procctl_t;
2124#endif
2125
2126/* ARGSUSED */
2127static void
2128arc_buf_unwatch(arc_buf_t *buf)
2129{
2130#ifndef _KERNEL
2131	if (arc_watch) {
2132		int result;
2133		procctl_t ctl;
2134		ctl.cmd = PCWATCH;
2135		ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
2136		ctl.prwatch.pr_size = 0;
2137		ctl.prwatch.pr_wflags = 0;
2138		result = write(arc_procfd, &ctl, sizeof (ctl));
2139		ASSERT3U(result, ==, sizeof (ctl));
2140	}
2141#endif
2142}
2143
2144/* ARGSUSED */
2145static void
2146arc_buf_watch(arc_buf_t *buf)
2147{
2148#ifndef _KERNEL
2149	if (arc_watch) {
2150		int result;
2151		procctl_t ctl;
2152		ctl.cmd = PCWATCH;
2153		ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
2154		ctl.prwatch.pr_size = arc_buf_size(buf);
2155		ctl.prwatch.pr_wflags = WA_WRITE;
2156		result = write(arc_procfd, &ctl, sizeof (ctl));
2157		ASSERT3U(result, ==, sizeof (ctl));
2158	}
2159#endif
2160}
2161#endif /* illumos */
2162
2163static arc_buf_contents_t
2164arc_buf_type(arc_buf_hdr_t *hdr)
2165{
2166	arc_buf_contents_t type;
2167	if (HDR_ISTYPE_METADATA(hdr)) {
2168		type = ARC_BUFC_METADATA;
2169	} else {
2170		type = ARC_BUFC_DATA;
2171	}
2172	VERIFY3U(hdr->b_type, ==, type);
2173	return (type);
2174}
2175
2176boolean_t
2177arc_is_metadata(arc_buf_t *buf)
2178{
2179	return (HDR_ISTYPE_METADATA(buf->b_hdr) != 0);
2180}
2181
2182static uint32_t
2183arc_bufc_to_flags(arc_buf_contents_t type)
2184{
2185	switch (type) {
2186	case ARC_BUFC_DATA:
2187		/* metadata field is 0 if buffer contains normal data */
2188		return (0);
2189	case ARC_BUFC_METADATA:
2190		return (ARC_FLAG_BUFC_METADATA);
2191	default:
2192		break;
2193	}
2194	panic("undefined ARC buffer type!");
2195	return ((uint32_t)-1);
2196}
2197
2198void
2199arc_buf_thaw(arc_buf_t *buf)
2200{
2201	arc_buf_hdr_t *hdr = buf->b_hdr;
2202
2203	ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
2204	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
2205
2206	arc_cksum_verify(buf);
2207
2208	/*
2209	 * Compressed buffers do not manipulate the b_freeze_cksum or
2210	 * allocate b_thawed.
2211	 */
2212	if (ARC_BUF_COMPRESSED(buf)) {
2213		ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL ||
2214		    arc_hdr_has_uncompressed_buf(hdr));
2215		return;
2216	}
2217
2218	ASSERT(HDR_HAS_L1HDR(hdr));
2219	arc_cksum_free(hdr);
2220
2221	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
2222#ifdef ZFS_DEBUG
2223	if (zfs_flags & ZFS_DEBUG_MODIFY) {
2224		if (hdr->b_l1hdr.b_thawed != NULL)
2225			kmem_free(hdr->b_l1hdr.b_thawed, 1);
2226		hdr->b_l1hdr.b_thawed = kmem_alloc(1, KM_SLEEP);
2227	}
2228#endif
2229
2230	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
2231
2232#ifdef illumos
2233	arc_buf_unwatch(buf);
2234#endif
2235}
2236
2237void
2238arc_buf_freeze(arc_buf_t *buf)
2239{
2240	arc_buf_hdr_t *hdr = buf->b_hdr;
2241	kmutex_t *hash_lock;
2242
2243	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
2244		return;
2245
2246	if (ARC_BUF_COMPRESSED(buf)) {
2247		ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL ||
2248		    arc_hdr_has_uncompressed_buf(hdr));
2249		return;
2250	}
2251
2252	hash_lock = HDR_LOCK(hdr);
2253	mutex_enter(hash_lock);
2254
2255	ASSERT(HDR_HAS_L1HDR(hdr));
2256	ASSERT(hdr->b_l1hdr.b_freeze_cksum != NULL ||
2257	    hdr->b_l1hdr.b_state == arc_anon);
2258	arc_cksum_compute(buf);
2259	mutex_exit(hash_lock);
2260}
2261
2262/*
2263 * The arc_buf_hdr_t's b_flags should never be modified directly. Instead,
2264 * the following functions should be used to ensure that the flags are
2265 * updated in a thread-safe way. When manipulating the flags either
2266 * the hash_lock must be held or the hdr must be undiscoverable. This
2267 * ensures that we're not racing with any other threads when updating
2268 * the flags.
2269 */
2270static inline void
2271arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)
2272{
2273	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2274	hdr->b_flags |= flags;
2275}
2276
2277static inline void
2278arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)
2279{
2280	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2281	hdr->b_flags &= ~flags;
2282}
2283
2284/*
2285 * Setting the compression bits in the arc_buf_hdr_t's b_flags is
2286 * done in a special way since we have to clear and set bits
2287 * at the same time. Consumers that wish to set the compression bits
2288 * must use this function to ensure that the flags are updated in
2289 * thread-safe manner.
2290 */
2291static void
2292arc_hdr_set_compress(arc_buf_hdr_t *hdr, enum zio_compress cmp)
2293{
2294	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2295
2296	/*
2297	 * Holes and embedded blocks will always have a psize = 0 so
2298	 * we ignore the compression of the blkptr and set the
2299	 * arc_buf_hdr_t's compression to ZIO_COMPRESS_OFF.
2300	 * Holes and embedded blocks remain anonymous so we don't
2301	 * want to uncompress them. Mark them as uncompressed.
2302	 */
2303	if (!zfs_compressed_arc_enabled || HDR_GET_PSIZE(hdr) == 0) {
2304		arc_hdr_clear_flags(hdr, ARC_FLAG_COMPRESSED_ARC);
2305		HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_OFF);
2306		ASSERT(!HDR_COMPRESSION_ENABLED(hdr));
2307		ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
2308	} else {
2309		arc_hdr_set_flags(hdr, ARC_FLAG_COMPRESSED_ARC);
2310		HDR_SET_COMPRESS(hdr, cmp);
2311		ASSERT3U(HDR_GET_COMPRESS(hdr), ==, cmp);
2312		ASSERT(HDR_COMPRESSION_ENABLED(hdr));
2313	}
2314}
2315
2316/*
2317 * Looks for another buf on the same hdr which has the data decompressed, copies
2318 * from it, and returns true. If no such buf exists, returns false.
2319 */
2320static boolean_t
2321arc_buf_try_copy_decompressed_data(arc_buf_t *buf)
2322{
2323	arc_buf_hdr_t *hdr = buf->b_hdr;
2324	boolean_t copied = B_FALSE;
2325
2326	ASSERT(HDR_HAS_L1HDR(hdr));
2327	ASSERT3P(buf->b_data, !=, NULL);
2328	ASSERT(!ARC_BUF_COMPRESSED(buf));
2329
2330	for (arc_buf_t *from = hdr->b_l1hdr.b_buf; from != NULL;
2331	    from = from->b_next) {
2332		/* can't use our own data buffer */
2333		if (from == buf) {
2334			continue;
2335		}
2336
2337		if (!ARC_BUF_COMPRESSED(from)) {
2338			bcopy(from->b_data, buf->b_data, arc_buf_size(buf));
2339			copied = B_TRUE;
2340			break;
2341		}
2342	}
2343
2344	/*
2345	 * There were no decompressed bufs, so there should not be a
2346	 * checksum on the hdr either.
2347	 */
2348	EQUIV(!copied, hdr->b_l1hdr.b_freeze_cksum == NULL);
2349
2350	return (copied);
2351}
2352
2353/*
2354 * Given a buf that has a data buffer attached to it, this function will
2355 * efficiently fill the buf with data of the specified compression setting from
2356 * the hdr and update the hdr's b_freeze_cksum if necessary. If the buf and hdr
2357 * are already sharing a data buf, no copy is performed.
2358 *
2359 * If the buf is marked as compressed but uncompressed data was requested, this
2360 * will allocate a new data buffer for the buf, remove that flag, and fill the
2361 * buf with uncompressed data. You can't request a compressed buf on a hdr with
2362 * uncompressed data, and (since we haven't added support for it yet) if you
2363 * want compressed data your buf must already be marked as compressed and have
2364 * the correct-sized data buffer.
2365 */
2366static int
2367arc_buf_fill(arc_buf_t *buf, boolean_t compressed)
2368{
2369	arc_buf_hdr_t *hdr = buf->b_hdr;
2370	boolean_t hdr_compressed = (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF);
2371	dmu_object_byteswap_t bswap = hdr->b_l1hdr.b_byteswap;
2372
2373	ASSERT3P(buf->b_data, !=, NULL);
2374	IMPLY(compressed, hdr_compressed);
2375	IMPLY(compressed, ARC_BUF_COMPRESSED(buf));
2376
2377	if (hdr_compressed == compressed) {
2378		if (!arc_buf_is_shared(buf)) {
2379			abd_copy_to_buf(buf->b_data, hdr->b_l1hdr.b_pabd,
2380			    arc_buf_size(buf));
2381		}
2382	} else {
2383		ASSERT(hdr_compressed);
2384		ASSERT(!compressed);
2385		ASSERT3U(HDR_GET_LSIZE(hdr), !=, HDR_GET_PSIZE(hdr));
2386
2387		/*
2388		 * If the buf is sharing its data with the hdr, unlink it and
2389		 * allocate a new data buffer for the buf.
2390		 */
2391		if (arc_buf_is_shared(buf)) {
2392			ASSERT(ARC_BUF_COMPRESSED(buf));
2393
2394			/* We need to give the buf it's own b_data */
2395			buf->b_flags &= ~ARC_BUF_FLAG_SHARED;
2396			buf->b_data =
2397			    arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);
2398			arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
2399
2400			/* Previously overhead was 0; just add new overhead */
2401			ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr));
2402		} else if (ARC_BUF_COMPRESSED(buf)) {
2403			/* We need to reallocate the buf's b_data */
2404			arc_free_data_buf(hdr, buf->b_data, HDR_GET_PSIZE(hdr),
2405			    buf);
2406			buf->b_data =
2407			    arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);
2408
2409			/* We increased the size of b_data; update overhead */
2410			ARCSTAT_INCR(arcstat_overhead_size,
2411			    HDR_GET_LSIZE(hdr) - HDR_GET_PSIZE(hdr));
2412		}
2413
2414		/*
2415		 * Regardless of the buf's previous compression settings, it
2416		 * should not be compressed at the end of this function.
2417		 */
2418		buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
2419
2420		/*
2421		 * Try copying the data from another buf which already has a
2422		 * decompressed version. If that's not possible, it's time to
2423		 * bite the bullet and decompress the data from the hdr.
2424		 */
2425		if (arc_buf_try_copy_decompressed_data(buf)) {
2426			/* Skip byteswapping and checksumming (already done) */
2427			ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, !=, NULL);
2428			return (0);
2429		} else {
2430			int error = zio_decompress_data(HDR_GET_COMPRESS(hdr),
2431			    hdr->b_l1hdr.b_pabd, buf->b_data,
2432			    HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr));
2433
2434			/*
2435			 * Absent hardware errors or software bugs, this should
2436			 * be impossible, but log it anyway so we can debug it.
2437			 */
2438			if (error != 0) {
2439				zfs_dbgmsg(
2440				    "hdr %p, compress %d, psize %d, lsize %d",
2441				    hdr, HDR_GET_COMPRESS(hdr),
2442				    HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr));
2443				return (SET_ERROR(EIO));
2444			}
2445		}
2446	}
2447
2448	/* Byteswap the buf's data if necessary */
2449	if (bswap != DMU_BSWAP_NUMFUNCS) {
2450		ASSERT(!HDR_SHARED_DATA(hdr));
2451		ASSERT3U(bswap, <, DMU_BSWAP_NUMFUNCS);
2452		dmu_ot_byteswap[bswap].ob_func(buf->b_data, HDR_GET_LSIZE(hdr));
2453	}
2454
2455	/* Compute the hdr's checksum if necessary */
2456	arc_cksum_compute(buf);
2457
2458	return (0);
2459}
2460
2461int
2462arc_decompress(arc_buf_t *buf)
2463{
2464	return (arc_buf_fill(buf, B_FALSE));
2465}
2466
2467/*
2468 * Return the size of the block, b_pabd, that is stored in the arc_buf_hdr_t.
2469 */
2470static uint64_t
2471arc_hdr_size(arc_buf_hdr_t *hdr)
2472{
2473	uint64_t size;
2474
2475	if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
2476	    HDR_GET_PSIZE(hdr) > 0) {
2477		size = HDR_GET_PSIZE(hdr);
2478	} else {
2479		ASSERT3U(HDR_GET_LSIZE(hdr), !=, 0);
2480		size = HDR_GET_LSIZE(hdr);
2481	}
2482	return (size);
2483}
2484
2485/*
2486 * Increment the amount of evictable space in the arc_state_t's refcount.
2487 * We account for the space used by the hdr and the arc buf individually
2488 * so that we can add and remove them from the refcount individually.
2489 */
2490static void
2491arc_evictable_space_increment(arc_buf_hdr_t *hdr, arc_state_t *state)
2492{
2493	arc_buf_contents_t type = arc_buf_type(hdr);
2494
2495	ASSERT(HDR_HAS_L1HDR(hdr));
2496
2497	if (GHOST_STATE(state)) {
2498		ASSERT0(hdr->b_l1hdr.b_bufcnt);
2499		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2500		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2501		(void) zfs_refcount_add_many(&state->arcs_esize[type],
2502		    HDR_GET_LSIZE(hdr), hdr);
2503		return;
2504	}
2505
2506	ASSERT(!GHOST_STATE(state));
2507	if (hdr->b_l1hdr.b_pabd != NULL) {
2508		(void) zfs_refcount_add_many(&state->arcs_esize[type],
2509		    arc_hdr_size(hdr), hdr);
2510	}
2511	for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2512	    buf = buf->b_next) {
2513		if (arc_buf_is_shared(buf))
2514			continue;
2515		(void) zfs_refcount_add_many(&state->arcs_esize[type],
2516		    arc_buf_size(buf), buf);
2517	}
2518}
2519
2520/*
2521 * Decrement the amount of evictable space in the arc_state_t's refcount.
2522 * We account for the space used by the hdr and the arc buf individually
2523 * so that we can add and remove them from the refcount individually.
2524 */
2525static void
2526arc_evictable_space_decrement(arc_buf_hdr_t *hdr, arc_state_t *state)
2527{
2528	arc_buf_contents_t type = arc_buf_type(hdr);
2529
2530	ASSERT(HDR_HAS_L1HDR(hdr));
2531
2532	if (GHOST_STATE(state)) {
2533		ASSERT0(hdr->b_l1hdr.b_bufcnt);
2534		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2535		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2536		(void) zfs_refcount_remove_many(&state->arcs_esize[type],
2537		    HDR_GET_LSIZE(hdr), hdr);
2538		return;
2539	}
2540
2541	ASSERT(!GHOST_STATE(state));
2542	if (hdr->b_l1hdr.b_pabd != NULL) {
2543		(void) zfs_refcount_remove_many(&state->arcs_esize[type],
2544		    arc_hdr_size(hdr), hdr);
2545	}
2546	for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2547	    buf = buf->b_next) {
2548		if (arc_buf_is_shared(buf))
2549			continue;
2550		(void) zfs_refcount_remove_many(&state->arcs_esize[type],
2551		    arc_buf_size(buf), buf);
2552	}
2553}
2554
2555/*
2556 * Add a reference to this hdr indicating that someone is actively
2557 * referencing that memory. When the refcount transitions from 0 to 1,
2558 * we remove it from the respective arc_state_t list to indicate that
2559 * it is not evictable.
2560 */
2561static void
2562add_reference(arc_buf_hdr_t *hdr, void *tag)
2563{
2564	ASSERT(HDR_HAS_L1HDR(hdr));
2565	if (!MUTEX_HELD(HDR_LOCK(hdr))) {
2566		ASSERT(hdr->b_l1hdr.b_state == arc_anon);
2567		ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2568		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2569	}
2570
2571	arc_state_t *state = hdr->b_l1hdr.b_state;
2572
2573	if ((zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) &&
2574	    (state != arc_anon)) {
2575		/* We don't use the L2-only state list. */
2576		if (state != arc_l2c_only) {
2577			multilist_remove(state->arcs_list[arc_buf_type(hdr)],
2578			    hdr);
2579			arc_evictable_space_decrement(hdr, state);
2580		}
2581		/* remove the prefetch flag if we get a reference */
2582		arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH);
2583	}
2584}
2585
2586/*
2587 * Remove a reference from this hdr. When the reference transitions from
2588 * 1 to 0 and we're not anonymous, then we add this hdr to the arc_state_t's
2589 * list making it eligible for eviction.
2590 */
2591static int
2592remove_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
2593{
2594	int cnt;
2595	arc_state_t *state = hdr->b_l1hdr.b_state;
2596
2597	ASSERT(HDR_HAS_L1HDR(hdr));
2598	ASSERT(state == arc_anon || MUTEX_HELD(hash_lock));
2599	ASSERT(!GHOST_STATE(state));
2600
2601	/*
2602	 * arc_l2c_only counts as a ghost state so we don't need to explicitly
2603	 * check to prevent usage of the arc_l2c_only list.
2604	 */
2605	if (((cnt = zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) &&
2606	    (state != arc_anon)) {
2607		multilist_insert(state->arcs_list[arc_buf_type(hdr)], hdr);
2608		ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0);
2609		arc_evictable_space_increment(hdr, state);
2610	}
2611	return (cnt);
2612}
2613
2614/*
2615 * Returns detailed information about a specific arc buffer.  When the
2616 * state_index argument is set the function will calculate the arc header
2617 * list position for its arc state.  Since this requires a linear traversal
2618 * callers are strongly encourage not to do this.  However, it can be helpful
2619 * for targeted analysis so the functionality is provided.
2620 */
2621void
2622arc_buf_info(arc_buf_t *ab, arc_buf_info_t *abi, int state_index)
2623{
2624	arc_buf_hdr_t *hdr = ab->b_hdr;
2625	l1arc_buf_hdr_t *l1hdr = NULL;
2626	l2arc_buf_hdr_t *l2hdr = NULL;
2627	arc_state_t *state = NULL;
2628
2629	memset(abi, 0, sizeof (arc_buf_info_t));
2630
2631	if (hdr == NULL)
2632		return;
2633
2634	abi->abi_flags = hdr->b_flags;
2635
2636	if (HDR_HAS_L1HDR(hdr)) {
2637		l1hdr = &hdr->b_l1hdr;
2638		state = l1hdr->b_state;
2639	}
2640	if (HDR_HAS_L2HDR(hdr))
2641		l2hdr = &hdr->b_l2hdr;
2642
2643	if (l1hdr) {
2644		abi->abi_bufcnt = l1hdr->b_bufcnt;
2645		abi->abi_access = l1hdr->b_arc_access;
2646		abi->abi_mru_hits = l1hdr->b_mru_hits;
2647		abi->abi_mru_ghost_hits = l1hdr->b_mru_ghost_hits;
2648		abi->abi_mfu_hits = l1hdr->b_mfu_hits;
2649		abi->abi_mfu_ghost_hits = l1hdr->b_mfu_ghost_hits;
2650		abi->abi_holds = zfs_refcount_count(&l1hdr->b_refcnt);
2651	}
2652
2653	if (l2hdr) {
2654		abi->abi_l2arc_dattr = l2hdr->b_daddr;
2655		abi->abi_l2arc_hits = l2hdr->b_hits;
2656	}
2657
2658	abi->abi_state_type = state ? state->arcs_state : ARC_STATE_ANON;
2659	abi->abi_state_contents = arc_buf_type(hdr);
2660	abi->abi_size = arc_hdr_size(hdr);
2661}
2662
2663/*
2664 * Move the supplied buffer to the indicated state. The hash lock
2665 * for the buffer must be held by the caller.
2666 */
2667static void
2668arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
2669    kmutex_t *hash_lock)
2670{
2671	arc_state_t *old_state;
2672	int64_t refcnt;
2673	uint32_t bufcnt;
2674	boolean_t update_old, update_new;
2675	arc_buf_contents_t buftype = arc_buf_type(hdr);
2676
2677	/*
2678	 * We almost always have an L1 hdr here, since we call arc_hdr_realloc()
2679	 * in arc_read() when bringing a buffer out of the L2ARC.  However, the
2680	 * L1 hdr doesn't always exist when we change state to arc_anon before
2681	 * destroying a header, in which case reallocating to add the L1 hdr is
2682	 * pointless.
2683	 */
2684	if (HDR_HAS_L1HDR(hdr)) {
2685		old_state = hdr->b_l1hdr.b_state;
2686		refcnt = zfs_refcount_count(&hdr->b_l1hdr.b_refcnt);
2687		bufcnt = hdr->b_l1hdr.b_bufcnt;
2688		update_old = (bufcnt > 0 || hdr->b_l1hdr.b_pabd != NULL);
2689	} else {
2690		old_state = arc_l2c_only;
2691		refcnt = 0;
2692		bufcnt = 0;
2693		update_old = B_FALSE;
2694	}
2695	update_new = update_old;
2696
2697	ASSERT(MUTEX_HELD(hash_lock));
2698	ASSERT3P(new_state, !=, old_state);
2699	ASSERT(!GHOST_STATE(new_state) || bufcnt == 0);
2700	ASSERT(old_state != arc_anon || bufcnt <= 1);
2701
2702	/*
2703	 * If this buffer is evictable, transfer it from the
2704	 * old state list to the new state list.
2705	 */
2706	if (refcnt == 0) {
2707		if (old_state != arc_anon && old_state != arc_l2c_only) {
2708			ASSERT(HDR_HAS_L1HDR(hdr));
2709			multilist_remove(old_state->arcs_list[buftype], hdr);
2710
2711			if (GHOST_STATE(old_state)) {
2712				ASSERT0(bufcnt);
2713				ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2714				update_old = B_TRUE;
2715			}
2716			arc_evictable_space_decrement(hdr, old_state);
2717		}
2718		if (new_state != arc_anon && new_state != arc_l2c_only) {
2719
2720			/*
2721			 * An L1 header always exists here, since if we're
2722			 * moving to some L1-cached state (i.e. not l2c_only or
2723			 * anonymous), we realloc the header to add an L1hdr
2724			 * beforehand.
2725			 */
2726			ASSERT(HDR_HAS_L1HDR(hdr));
2727			multilist_insert(new_state->arcs_list[buftype], hdr);
2728
2729			if (GHOST_STATE(new_state)) {
2730				ASSERT0(bufcnt);
2731				ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2732				update_new = B_TRUE;
2733			}
2734			arc_evictable_space_increment(hdr, new_state);
2735		}
2736	}
2737
2738	ASSERT(!HDR_EMPTY(hdr));
2739	if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr))
2740		buf_hash_remove(hdr);
2741
2742	/* adjust state sizes (ignore arc_l2c_only) */
2743
2744	if (update_new && new_state != arc_l2c_only) {
2745		ASSERT(HDR_HAS_L1HDR(hdr));
2746		if (GHOST_STATE(new_state)) {
2747			ASSERT0(bufcnt);
2748
2749			/*
2750			 * When moving a header to a ghost state, we first
2751			 * remove all arc buffers. Thus, we'll have a
2752			 * bufcnt of zero, and no arc buffer to use for
2753			 * the reference. As a result, we use the arc
2754			 * header pointer for the reference.
2755			 */
2756			(void) zfs_refcount_add_many(&new_state->arcs_size,
2757			    HDR_GET_LSIZE(hdr), hdr);
2758			ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2759		} else {
2760			uint32_t buffers = 0;
2761
2762			/*
2763			 * Each individual buffer holds a unique reference,
2764			 * thus we must remove each of these references one
2765			 * at a time.
2766			 */
2767			for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2768			    buf = buf->b_next) {
2769				ASSERT3U(bufcnt, !=, 0);
2770				buffers++;
2771
2772				/*
2773				 * When the arc_buf_t is sharing the data
2774				 * block with the hdr, the owner of the
2775				 * reference belongs to the hdr. Only
2776				 * add to the refcount if the arc_buf_t is
2777				 * not shared.
2778				 */
2779				if (arc_buf_is_shared(buf))
2780					continue;
2781
2782				(void) zfs_refcount_add_many(
2783				    &new_state->arcs_size,
2784				    arc_buf_size(buf), buf);
2785			}
2786			ASSERT3U(bufcnt, ==, buffers);
2787
2788			if (hdr->b_l1hdr.b_pabd != NULL) {
2789				(void) zfs_refcount_add_many(
2790				    &new_state->arcs_size,
2791				    arc_hdr_size(hdr), hdr);
2792			} else {
2793				ASSERT(GHOST_STATE(old_state));
2794			}
2795		}
2796	}
2797
2798	if (update_old && old_state != arc_l2c_only) {
2799		ASSERT(HDR_HAS_L1HDR(hdr));
2800		if (GHOST_STATE(old_state)) {
2801			ASSERT0(bufcnt);
2802			ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2803
2804			/*
2805			 * When moving a header off of a ghost state,
2806			 * the header will not contain any arc buffers.
2807			 * We use the arc header pointer for the reference
2808			 * which is exactly what we did when we put the
2809			 * header on the ghost state.
2810			 */
2811
2812			(void) zfs_refcount_remove_many(&old_state->arcs_size,
2813			    HDR_GET_LSIZE(hdr), hdr);
2814		} else {
2815			uint32_t buffers = 0;
2816
2817			/*
2818			 * Each individual buffer holds a unique reference,
2819			 * thus we must remove each of these references one
2820			 * at a time.
2821			 */
2822			for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2823			    buf = buf->b_next) {
2824				ASSERT3U(bufcnt, !=, 0);
2825				buffers++;
2826
2827				/*
2828				 * When the arc_buf_t is sharing the data
2829				 * block with the hdr, the owner of the
2830				 * reference belongs to the hdr. Only
2831				 * add to the refcount if the arc_buf_t is
2832				 * not shared.
2833				 */
2834				if (arc_buf_is_shared(buf))
2835					continue;
2836
2837				(void) zfs_refcount_remove_many(
2838				    &old_state->arcs_size, arc_buf_size(buf),
2839				    buf);
2840			}
2841			ASSERT3U(bufcnt, ==, buffers);
2842			ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
2843			(void) zfs_refcount_remove_many(
2844			    &old_state->arcs_size, arc_hdr_size(hdr), hdr);
2845		}
2846	}
2847
2848	if (HDR_HAS_L1HDR(hdr))
2849		hdr->b_l1hdr.b_state = new_state;
2850
2851	/*
2852	 * L2 headers should never be on the L2 state list since they don't
2853	 * have L1 headers allocated.
2854	 */
2855	ASSERT(multilist_is_empty(arc_l2c_only->arcs_list[ARC_BUFC_DATA]) &&
2856	    multilist_is_empty(arc_l2c_only->arcs_list[ARC_BUFC_METADATA]));
2857}
2858
2859void
2860arc_space_consume(uint64_t space, arc_space_type_t type)
2861{
2862	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
2863
2864	switch (type) {
2865	case ARC_SPACE_DATA:
2866		aggsum_add(&astat_data_size, space);
2867		break;
2868	case ARC_SPACE_META:
2869		aggsum_add(&astat_metadata_size, space);
2870		break;
2871	case ARC_SPACE_BONUS:
2872		aggsum_add(&astat_bonus_size, space);
2873		break;
2874	case ARC_SPACE_DNODE:
2875		aggsum_add(&astat_dnode_size, space);
2876		break;
2877	case ARC_SPACE_DBUF:
2878		aggsum_add(&astat_dbuf_size, space);
2879		break;
2880	case ARC_SPACE_HDRS:
2881		aggsum_add(&astat_hdr_size, space);
2882		break;
2883	case ARC_SPACE_L2HDRS:
2884		aggsum_add(&astat_l2_hdr_size, space);
2885		break;
2886	}
2887
2888	if (type != ARC_SPACE_DATA)
2889		aggsum_add(&arc_meta_used, space);
2890
2891	aggsum_add(&arc_size, space);
2892}
2893
2894void
2895arc_space_return(uint64_t space, arc_space_type_t type)
2896{
2897	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
2898
2899	switch (type) {
2900	case ARC_SPACE_DATA:
2901		aggsum_add(&astat_data_size, -space);
2902		break;
2903	case ARC_SPACE_META:
2904		aggsum_add(&astat_metadata_size, -space);
2905		break;
2906	case ARC_SPACE_BONUS:
2907		aggsum_add(&astat_bonus_size, -space);
2908		break;
2909	case ARC_SPACE_DNODE:
2910		aggsum_add(&astat_dnode_size, -space);
2911		break;
2912	case ARC_SPACE_DBUF:
2913		aggsum_add(&astat_dbuf_size, -space);
2914		break;
2915	case ARC_SPACE_HDRS:
2916		aggsum_add(&astat_hdr_size, -space);
2917		break;
2918	case ARC_SPACE_L2HDRS:
2919		aggsum_add(&astat_l2_hdr_size, -space);
2920		break;
2921	}
2922
2923	if (type != ARC_SPACE_DATA) {
2924		ASSERT(aggsum_compare(&arc_meta_used, space) >= 0);
2925		/*
2926		 * We use the upper bound here rather than the precise value
2927		 * because the arc_meta_max value doesn't need to be
2928		 * precise. It's only consumed by humans via arcstats.
2929		 */
2930		if (arc_meta_max < aggsum_upper_bound(&arc_meta_used))
2931			arc_meta_max = aggsum_upper_bound(&arc_meta_used);
2932		aggsum_add(&arc_meta_used, -space);
2933	}
2934
2935	ASSERT(aggsum_compare(&arc_size, space) >= 0);
2936	aggsum_add(&arc_size, -space);
2937}
2938
2939/*
2940 * Given a hdr and a buf, returns whether that buf can share its b_data buffer
2941 * with the hdr's b_pabd.
2942 */
2943static boolean_t
2944arc_can_share(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2945{
2946	/*
2947	 * The criteria for sharing a hdr's data are:
2948	 * 1. the hdr's compression matches the buf's compression
2949	 * 2. the hdr doesn't need to be byteswapped
2950	 * 3. the hdr isn't already being shared
2951	 * 4. the buf is either compressed or it is the last buf in the hdr list
2952	 *
2953	 * Criterion #4 maintains the invariant that shared uncompressed
2954	 * bufs must be the final buf in the hdr's b_buf list. Reading this, you
2955	 * might ask, "if a compressed buf is allocated first, won't that be the
2956	 * last thing in the list?", but in that case it's impossible to create
2957	 * a shared uncompressed buf anyway (because the hdr must be compressed
2958	 * to have the compressed buf). You might also think that #3 is
2959	 * sufficient to make this guarantee, however it's possible
2960	 * (specifically in the rare L2ARC write race mentioned in
2961	 * arc_buf_alloc_impl()) there will be an existing uncompressed buf that
2962	 * is sharable, but wasn't at the time of its allocation. Rather than
2963	 * allow a new shared uncompressed buf to be created and then shuffle
2964	 * the list around to make it the last element, this simply disallows
2965	 * sharing if the new buf isn't the first to be added.
2966	 */
2967	ASSERT3P(buf->b_hdr, ==, hdr);
2968	boolean_t hdr_compressed = HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF;
2969	boolean_t buf_compressed = ARC_BUF_COMPRESSED(buf) != 0;
2970	return (buf_compressed == hdr_compressed &&
2971	    hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS &&
2972	    !HDR_SHARED_DATA(hdr) &&
2973	    (ARC_BUF_LAST(buf) || ARC_BUF_COMPRESSED(buf)));
2974}
2975
2976/*
2977 * Allocate a buf for this hdr. If you care about the data that's in the hdr,
2978 * or if you want a compressed buffer, pass those flags in. Returns 0 if the
2979 * copy was made successfully, or an error code otherwise.
2980 */
2981static int
2982arc_buf_alloc_impl(arc_buf_hdr_t *hdr, void *tag, boolean_t compressed,
2983    boolean_t fill, arc_buf_t **ret)
2984{
2985	arc_buf_t *buf;
2986
2987	ASSERT(HDR_HAS_L1HDR(hdr));
2988	ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);
2989	VERIFY(hdr->b_type == ARC_BUFC_DATA ||
2990	    hdr->b_type == ARC_BUFC_METADATA);
2991	ASSERT3P(ret, !=, NULL);
2992	ASSERT3P(*ret, ==, NULL);
2993
2994	buf = *ret = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
2995	buf->b_hdr = hdr;
2996	buf->b_data = NULL;
2997	buf->b_next = hdr->b_l1hdr.b_buf;
2998	buf->b_flags = 0;
2999
3000	add_reference(hdr, tag);
3001
3002	/*
3003	 * We're about to change the hdr's b_flags. We must either
3004	 * hold the hash_lock or be undiscoverable.
3005	 */
3006	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
3007
3008	/*
3009	 * Only honor requests for compressed bufs if the hdr is actually
3010	 * compressed.
3011	 */
3012	if (compressed && HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF)
3013		buf->b_flags |= ARC_BUF_FLAG_COMPRESSED;
3014
3015	/*
3016	 * If the hdr's data can be shared then we share the data buffer and
3017	 * set the appropriate bit in the hdr's b_flags to indicate the hdr is
3018	 * sharing it's b_pabd with the arc_buf_t. Otherwise, we allocate a new
3019	 * buffer to store the buf's data.
3020	 *
3021	 * There are two additional restrictions here because we're sharing
3022	 * hdr -> buf instead of the usual buf -> hdr. First, the hdr can't be
3023	 * actively involved in an L2ARC write, because if this buf is used by
3024	 * an arc_write() then the hdr's data buffer will be released when the
3025	 * write completes, even though the L2ARC write might still be using it.
3026	 * Second, the hdr's ABD must be linear so that the buf's user doesn't
3027	 * need to be ABD-aware.
3028	 */
3029	boolean_t can_share = arc_can_share(hdr, buf) && !HDR_L2_WRITING(hdr) &&
3030	    abd_is_linear(hdr->b_l1hdr.b_pabd);
3031
3032	/* Set up b_data and sharing */
3033	if (can_share) {
3034		buf->b_data = abd_to_buf(hdr->b_l1hdr.b_pabd);
3035		buf->b_flags |= ARC_BUF_FLAG_SHARED;
3036		arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);
3037	} else {
3038		buf->b_data =
3039		    arc_get_data_buf(hdr, arc_buf_size(buf), buf);
3040		ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf));
3041	}
3042	VERIFY3P(buf->b_data, !=, NULL);
3043
3044	hdr->b_l1hdr.b_buf = buf;
3045	hdr->b_l1hdr.b_bufcnt += 1;
3046
3047	/*
3048	 * If the user wants the data from the hdr, we need to either copy or
3049	 * decompress the data.
3050	 */
3051	if (fill) {
3052		return (arc_buf_fill(buf, ARC_BUF_COMPRESSED(buf) != 0));
3053	}
3054
3055	return (0);
3056}
3057
3058static char *arc_onloan_tag = "onloan";
3059
3060static inline void
3061arc_loaned_bytes_update(int64_t delta)
3062{
3063	atomic_add_64(&arc_loaned_bytes, delta);
3064
3065	/* assert that it did not wrap around */
3066	ASSERT3S(atomic_add_64_nv(&arc_loaned_bytes, 0), >=, 0);
3067}
3068
3069/*
3070 * Loan out an anonymous arc buffer. Loaned buffers are not counted as in
3071 * flight data by arc_tempreserve_space() until they are "returned". Loaned
3072 * buffers must be returned to the arc before they can be used by the DMU or
3073 * freed.
3074 */
3075arc_buf_t *
3076arc_loan_buf(spa_t *spa, boolean_t is_metadata, int size)
3077{
3078	arc_buf_t *buf = arc_alloc_buf(spa, arc_onloan_tag,
3079	    is_metadata ? ARC_BUFC_METADATA : ARC_BUFC_DATA, size);
3080
3081	arc_loaned_bytes_update(arc_buf_size(buf));
3082
3083	return (buf);
3084}
3085
3086arc_buf_t *
3087arc_loan_compressed_buf(spa_t *spa, uint64_t psize, uint64_t lsize,
3088    enum zio_compress compression_type)
3089{
3090	arc_buf_t *buf = arc_alloc_compressed_buf(spa, arc_onloan_tag,
3091	    psize, lsize, compression_type);
3092
3093	arc_loaned_bytes_update(arc_buf_size(buf));
3094
3095	return (buf);
3096}
3097
3098
3099/*
3100 * Return a loaned arc buffer to the arc.
3101 */
3102void
3103arc_return_buf(arc_buf_t *buf, void *tag)
3104{
3105	arc_buf_hdr_t *hdr = buf->b_hdr;
3106
3107	ASSERT3P(buf->b_data, !=, NULL);
3108	ASSERT(HDR_HAS_L1HDR(hdr));
3109	(void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
3110	(void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
3111
3112	arc_loaned_bytes_update(-arc_buf_size(buf));
3113}
3114
3115/* Detach an arc_buf from a dbuf (tag) */
3116void
3117arc_loan_inuse_buf(arc_buf_t *buf, void *tag)
3118{
3119	arc_buf_hdr_t *hdr = buf->b_hdr;
3120
3121	ASSERT3P(buf->b_data, !=, NULL);
3122	ASSERT(HDR_HAS_L1HDR(hdr));
3123	(void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
3124	(void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag);
3125
3126	arc_loaned_bytes_update(arc_buf_size(buf));
3127}
3128
3129static void
3130l2arc_free_abd_on_write(abd_t *abd, size_t size, arc_buf_contents_t type)
3131{
3132	l2arc_data_free_t *df = kmem_alloc(sizeof (*df), KM_SLEEP);
3133
3134	df->l2df_abd = abd;
3135	df->l2df_size = size;
3136	df->l2df_type = type;
3137	mutex_enter(&l2arc_free_on_write_mtx);
3138	list_insert_head(l2arc_free_on_write, df);
3139	mutex_exit(&l2arc_free_on_write_mtx);
3140}
3141
3142static void
3143arc_hdr_free_on_write(arc_buf_hdr_t *hdr)
3144{
3145	arc_state_t *state = hdr->b_l1hdr.b_state;
3146	arc_buf_contents_t type = arc_buf_type(hdr);
3147	uint64_t size = arc_hdr_size(hdr);
3148
3149	/* protected by hash lock, if in the hash table */
3150	if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
3151		ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3152		ASSERT(state != arc_anon && state != arc_l2c_only);
3153
3154		(void) zfs_refcount_remove_many(&state->arcs_esize[type],
3155		    size, hdr);
3156	}
3157	(void) zfs_refcount_remove_many(&state->arcs_size, size, hdr);
3158	if (type == ARC_BUFC_METADATA) {
3159		arc_space_return(size, ARC_SPACE_META);
3160	} else {
3161		ASSERT(type == ARC_BUFC_DATA);
3162		arc_space_return(size, ARC_SPACE_DATA);
3163	}
3164
3165	l2arc_free_abd_on_write(hdr->b_l1hdr.b_pabd, size, type);
3166}
3167
3168/*
3169 * Share the arc_buf_t's data with the hdr. Whenever we are sharing the
3170 * data buffer, we transfer the refcount ownership to the hdr and update
3171 * the appropriate kstats.
3172 */
3173static void
3174arc_share_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)
3175{
3176	arc_state_t *state = hdr->b_l1hdr.b_state;
3177
3178	ASSERT(arc_can_share(hdr, buf));
3179	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3180	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
3181
3182	/*
3183	 * Start sharing the data buffer. We transfer the
3184	 * refcount ownership to the hdr since it always owns
3185	 * the refcount whenever an arc_buf_t is shared.
3186	 */
3187	zfs_refcount_transfer_ownership(&state->arcs_size, buf, hdr);
3188	hdr->b_l1hdr.b_pabd = abd_get_from_buf(buf->b_data, arc_buf_size(buf));
3189	abd_take_ownership_of_buf(hdr->b_l1hdr.b_pabd,
3190	    HDR_ISTYPE_METADATA(hdr));
3191	arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);
3192	buf->b_flags |= ARC_BUF_FLAG_SHARED;
3193
3194	/*
3195	 * Since we've transferred ownership to the hdr we need
3196	 * to increment its compressed and uncompressed kstats and
3197	 * decrement the overhead size.
3198	 */
3199	ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr));
3200	ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));
3201	ARCSTAT_INCR(arcstat_overhead_size, -arc_buf_size(buf));
3202}
3203
3204static void
3205arc_unshare_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)
3206{
3207	arc_state_t *state = hdr->b_l1hdr.b_state;
3208
3209	ASSERT(arc_buf_is_shared(buf));
3210	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3211	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
3212
3213	/*
3214	 * We are no longer sharing this buffer so we need
3215	 * to transfer its ownership to the rightful owner.
3216	 */
3217	zfs_refcount_transfer_ownership(&state->arcs_size, hdr, buf);
3218	arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
3219	abd_release_ownership_of_buf(hdr->b_l1hdr.b_pabd);
3220	abd_put(hdr->b_l1hdr.b_pabd);
3221	hdr->b_l1hdr.b_pabd = NULL;
3222	buf->b_flags &= ~ARC_BUF_FLAG_SHARED;
3223
3224	/*
3225	 * Since the buffer is no longer shared between
3226	 * the arc buf and the hdr, count it as overhead.
3227	 */
3228	ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr));
3229	ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
3230	ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf));
3231}
3232
3233/*
3234 * Remove an arc_buf_t from the hdr's buf list and return the last
3235 * arc_buf_t on the list. If no buffers remain on the list then return
3236 * NULL.
3237 */
3238static arc_buf_t *
3239arc_buf_remove(arc_buf_hdr_t *hdr, arc_buf_t *buf)
3240{
3241	ASSERT(HDR_HAS_L1HDR(hdr));
3242	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
3243
3244	arc_buf_t **bufp = &hdr->b_l1hdr.b_buf;
3245	arc_buf_t *lastbuf = NULL;
3246
3247	/*
3248	 * Remove the buf from the hdr list and locate the last
3249	 * remaining buffer on the list.
3250	 */
3251	while (*bufp != NULL) {
3252		if (*bufp == buf)
3253			*bufp = buf->b_next;
3254
3255		/*
3256		 * If we've removed a buffer in the middle of
3257		 * the list then update the lastbuf and update
3258		 * bufp.
3259		 */
3260		if (*bufp != NULL) {
3261			lastbuf = *bufp;
3262			bufp = &(*bufp)->b_next;
3263		}
3264	}
3265	buf->b_next = NULL;
3266	ASSERT3P(lastbuf, !=, buf);
3267	IMPLY(hdr->b_l1hdr.b_bufcnt > 0, lastbuf != NULL);
3268	IMPLY(hdr->b_l1hdr.b_bufcnt > 0, hdr->b_l1hdr.b_buf != NULL);
3269	IMPLY(lastbuf != NULL, ARC_BUF_LAST(lastbuf));
3270
3271	return (lastbuf);
3272}
3273
3274/*
3275 * Free up buf->b_data and pull the arc_buf_t off of the the arc_buf_hdr_t's
3276 * list and free it.
3277 */
3278static void
3279arc_buf_destroy_impl(arc_buf_t *buf)
3280{
3281	arc_buf_hdr_t *hdr = buf->b_hdr;
3282
3283	/*
3284	 * Free up the data associated with the buf but only if we're not
3285	 * sharing this with the hdr. If we are sharing it with the hdr, the
3286	 * hdr is responsible for doing the free.
3287	 */
3288	if (buf->b_data != NULL) {
3289		/*
3290		 * We're about to change the hdr's b_flags. We must either
3291		 * hold the hash_lock or be undiscoverable.
3292		 */
3293		ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
3294
3295		arc_cksum_verify(buf);
3296#ifdef illumos
3297		arc_buf_unwatch(buf);
3298#endif
3299
3300		if (arc_buf_is_shared(buf)) {
3301			arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
3302		} else {
3303			uint64_t size = arc_buf_size(buf);
3304			arc_free_data_buf(hdr, buf->b_data, size, buf);
3305			ARCSTAT_INCR(arcstat_overhead_size, -size);
3306		}
3307		buf->b_data = NULL;
3308
3309		ASSERT(hdr->b_l1hdr.b_bufcnt > 0);
3310		hdr->b_l1hdr.b_bufcnt -= 1;
3311	}
3312
3313	arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);
3314
3315	if (ARC_BUF_SHARED(buf) && !ARC_BUF_COMPRESSED(buf)) {
3316		/*
3317		 * If the current arc_buf_t is sharing its data buffer with the
3318		 * hdr, then reassign the hdr's b_pabd to share it with the new
3319		 * buffer at the end of the list. The shared buffer is always
3320		 * the last one on the hdr's buffer list.
3321		 *
3322		 * There is an equivalent case for compressed bufs, but since
3323		 * they aren't guaranteed to be the last buf in the list and
3324		 * that is an exceedingly rare case, we just allow that space be
3325		 * wasted temporarily.
3326		 */
3327		if (lastbuf != NULL) {
3328			/* Only one buf can be shared at once */
3329			VERIFY(!arc_buf_is_shared(lastbuf));
3330			/* hdr is uncompressed so can't have compressed buf */
3331			VERIFY(!ARC_BUF_COMPRESSED(lastbuf));
3332
3333			ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3334			arc_hdr_free_pabd(hdr);
3335
3336			/*
3337			 * We must setup a new shared block between the
3338			 * last buffer and the hdr. The data would have
3339			 * been allocated by the arc buf so we need to transfer
3340			 * ownership to the hdr since it's now being shared.
3341			 */
3342			arc_share_buf(hdr, lastbuf);
3343		}
3344	} else if (HDR_SHARED_DATA(hdr)) {
3345		/*
3346		 * Uncompressed shared buffers are always at the end
3347		 * of the list. Compressed buffers don't have the
3348		 * same requirements. This makes it hard to
3349		 * simply assert that the lastbuf is shared so
3350		 * we rely on the hdr's compression flags to determine
3351		 * if we have a compressed, shared buffer.
3352		 */
3353		ASSERT3P(lastbuf, !=, NULL);
3354		ASSERT(arc_buf_is_shared(lastbuf) ||
3355		    HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF);
3356	}
3357
3358	/*
3359	 * Free the checksum if we're removing the last uncompressed buf from
3360	 * this hdr.
3361	 */
3362	if (!arc_hdr_has_uncompressed_buf(hdr)) {
3363		arc_cksum_free(hdr);
3364	}
3365
3366	/* clean up the buf */
3367	buf->b_hdr = NULL;
3368	kmem_cache_free(buf_cache, buf);
3369}
3370
3371static void
3372arc_hdr_alloc_pabd(arc_buf_hdr_t *hdr, boolean_t do_adapt)
3373{
3374	ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);
3375	ASSERT(HDR_HAS_L1HDR(hdr));
3376	ASSERT(!HDR_SHARED_DATA(hdr));
3377
3378	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3379	hdr->b_l1hdr.b_pabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr, do_adapt);
3380	hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
3381	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3382
3383	ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr));
3384	ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));
3385}
3386
3387static void
3388arc_hdr_free_pabd(arc_buf_hdr_t *hdr)
3389{
3390	ASSERT(HDR_HAS_L1HDR(hdr));
3391	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3392
3393	/*
3394	 * If the hdr is currently being written to the l2arc then
3395	 * we defer freeing the data by adding it to the l2arc_free_on_write
3396	 * list. The l2arc will free the data once it's finished
3397	 * writing it to the l2arc device.
3398	 */
3399	if (HDR_L2_WRITING(hdr)) {
3400		arc_hdr_free_on_write(hdr);
3401		ARCSTAT_BUMP(arcstat_l2_free_on_write);
3402	} else {
3403		arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
3404		    arc_hdr_size(hdr), hdr);
3405	}
3406	hdr->b_l1hdr.b_pabd = NULL;
3407	hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
3408
3409	ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr));
3410	ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
3411}
3412
3413static arc_buf_hdr_t *
3414arc_hdr_alloc(uint64_t spa, int32_t psize, int32_t lsize,
3415    enum zio_compress compression_type, arc_buf_contents_t type)
3416{
3417	arc_buf_hdr_t *hdr;
3418
3419	VERIFY(type == ARC_BUFC_DATA || type == ARC_BUFC_METADATA);
3420
3421	hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);
3422	ASSERT(HDR_EMPTY(hdr));
3423	ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3424	ASSERT3P(hdr->b_l1hdr.b_thawed, ==, NULL);
3425	HDR_SET_PSIZE(hdr, psize);
3426	HDR_SET_LSIZE(hdr, lsize);
3427	hdr->b_spa = spa;
3428	hdr->b_type = type;
3429	hdr->b_flags = 0;
3430	arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L1HDR);
3431	arc_hdr_set_compress(hdr, compression_type);
3432
3433	hdr->b_l1hdr.b_state = arc_anon;
3434	hdr->b_l1hdr.b_arc_access = 0;
3435	hdr->b_l1hdr.b_bufcnt = 0;
3436	hdr->b_l1hdr.b_buf = NULL;
3437
3438	/*
3439	 * Allocate the hdr's buffer. This will contain either
3440	 * the compressed or uncompressed data depending on the block
3441	 * it references and compressed arc enablement.
3442	 */
3443	arc_hdr_alloc_pabd(hdr, B_TRUE);
3444	ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3445
3446	return (hdr);
3447}
3448
3449/*
3450 * Transition between the two allocation states for the arc_buf_hdr struct.
3451 * The arc_buf_hdr struct can be allocated with (hdr_full_cache) or without
3452 * (hdr_l2only_cache) the fields necessary for the L1 cache - the smaller
3453 * version is used when a cache buffer is only in the L2ARC in order to reduce
3454 * memory usage.
3455 */
3456static arc_buf_hdr_t *
3457arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
3458{
3459	ASSERT(HDR_HAS_L2HDR(hdr));
3460
3461	arc_buf_hdr_t *nhdr;
3462	l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
3463
3464	ASSERT((old == hdr_full_cache && new == hdr_l2only_cache) ||
3465	    (old == hdr_l2only_cache && new == hdr_full_cache));
3466
3467	nhdr = kmem_cache_alloc(new, KM_PUSHPAGE);
3468
3469	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
3470	buf_hash_remove(hdr);
3471
3472	bcopy(hdr, nhdr, HDR_L2ONLY_SIZE);
3473
3474	if (new == hdr_full_cache) {
3475		arc_hdr_set_flags(nhdr, ARC_FLAG_HAS_L1HDR);
3476		/*
3477		 * arc_access and arc_change_state need to be aware that a
3478		 * header has just come out of L2ARC, so we set its state to
3479		 * l2c_only even though it's about to change.
3480		 */
3481		nhdr->b_l1hdr.b_state = arc_l2c_only;
3482
3483		/* Verify previous threads set to NULL before freeing */
3484		ASSERT3P(nhdr->b_l1hdr.b_pabd, ==, NULL);
3485	} else {
3486		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
3487		ASSERT0(hdr->b_l1hdr.b_bufcnt);
3488		ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3489
3490		/*
3491		 * If we've reached here, We must have been called from
3492		 * arc_evict_hdr(), as such we should have already been
3493		 * removed from any ghost list we were previously on
3494		 * (which protects us from racing with arc_evict_state),
3495		 * thus no locking is needed during this check.
3496		 */
3497		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3498
3499		/*
3500		 * A buffer must not be moved into the arc_l2c_only
3501		 * state if it's not finished being written out to the
3502		 * l2arc device. Otherwise, the b_l1hdr.b_pabd field
3503		 * might try to be accessed, even though it was removed.
3504		 */
3505		VERIFY(!HDR_L2_WRITING(hdr));
3506		VERIFY3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3507
3508#ifdef ZFS_DEBUG
3509		if (hdr->b_l1hdr.b_thawed != NULL) {
3510			kmem_free(hdr->b_l1hdr.b_thawed, 1);
3511			hdr->b_l1hdr.b_thawed = NULL;
3512		}
3513#endif
3514
3515		arc_hdr_clear_flags(nhdr, ARC_FLAG_HAS_L1HDR);
3516	}
3517	/*
3518	 * The header has been reallocated so we need to re-insert it into any
3519	 * lists it was on.
3520	 */
3521	(void) buf_hash_insert(nhdr, NULL);
3522
3523	ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node));
3524
3525	mutex_enter(&dev->l2ad_mtx);
3526
3527	/*
3528	 * We must place the realloc'ed header back into the list at
3529	 * the same spot. Otherwise, if it's placed earlier in the list,
3530	 * l2arc_write_buffers() could find it during the function's
3531	 * write phase, and try to write it out to the l2arc.
3532	 */
3533	list_insert_after(&dev->l2ad_buflist, hdr, nhdr);
3534	list_remove(&dev->l2ad_buflist, hdr);
3535
3536	mutex_exit(&dev->l2ad_mtx);
3537
3538	/*
3539	 * Since we're using the pointer address as the tag when
3540	 * incrementing and decrementing the l2ad_alloc refcount, we
3541	 * must remove the old pointer (that we're about to destroy) and
3542	 * add the new pointer to the refcount. Otherwise we'd remove
3543	 * the wrong pointer address when calling arc_hdr_destroy() later.
3544	 */
3545
3546	(void) zfs_refcount_remove_many(&dev->l2ad_alloc, arc_hdr_size(hdr),
3547	    hdr);
3548	(void) zfs_refcount_add_many(&dev->l2ad_alloc, arc_hdr_size(nhdr),
3549	    nhdr);
3550
3551	buf_discard_identity(hdr);
3552	kmem_cache_free(old, hdr);
3553
3554	return (nhdr);
3555}
3556
3557/*
3558 * Allocate a new arc_buf_hdr_t and arc_buf_t and return the buf to the caller.
3559 * The buf is returned thawed since we expect the consumer to modify it.
3560 */
3561arc_buf_t *
3562arc_alloc_buf(spa_t *spa, void *tag, arc_buf_contents_t type, int32_t size)
3563{
3564	arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), size, size,
3565	    ZIO_COMPRESS_OFF, type);
3566	ASSERT(!MUTEX_HELD(HDR_LOCK(hdr)));
3567
3568	arc_buf_t *buf = NULL;
3569	VERIFY0(arc_buf_alloc_impl(hdr, tag, B_FALSE, B_FALSE, &buf));
3570	arc_buf_thaw(buf);
3571
3572	return (buf);
3573}
3574
3575/*
3576 * Allocate a compressed buf in the same manner as arc_alloc_buf. Don't use this
3577 * for bufs containing metadata.
3578 */
3579arc_buf_t *
3580arc_alloc_compressed_buf(spa_t *spa, void *tag, uint64_t psize, uint64_t lsize,
3581    enum zio_compress compression_type)
3582{
3583	ASSERT3U(lsize, >, 0);
3584	ASSERT3U(lsize, >=, psize);
3585	ASSERT(compression_type > ZIO_COMPRESS_OFF);
3586	ASSERT(compression_type < ZIO_COMPRESS_FUNCTIONS);
3587
3588	arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize,
3589	    compression_type, ARC_BUFC_DATA);
3590	ASSERT(!MUTEX_HELD(HDR_LOCK(hdr)));
3591
3592	arc_buf_t *buf = NULL;
3593	VERIFY0(arc_buf_alloc_impl(hdr, tag, B_TRUE, B_FALSE, &buf));
3594	arc_buf_thaw(buf);
3595	ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3596
3597	if (!arc_buf_is_shared(buf)) {
3598		/*
3599		 * To ensure that the hdr has the correct data in it if we call
3600		 * arc_decompress() on this buf before it's been written to
3601		 * disk, it's easiest if we just set up sharing between the
3602		 * buf and the hdr.
3603		 */
3604		ASSERT(!abd_is_linear(hdr->b_l1hdr.b_pabd));
3605		arc_hdr_free_pabd(hdr);
3606		arc_share_buf(hdr, buf);
3607	}
3608
3609	return (buf);
3610}
3611
3612static void
3613arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr)
3614{
3615	l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
3616	l2arc_dev_t *dev = l2hdr->b_dev;
3617	uint64_t psize = arc_hdr_size(hdr);
3618
3619	ASSERT(MUTEX_HELD(&dev->l2ad_mtx));
3620	ASSERT(HDR_HAS_L2HDR(hdr));
3621
3622	list_remove(&dev->l2ad_buflist, hdr);
3623
3624	ARCSTAT_INCR(arcstat_l2_psize, -psize);
3625	ARCSTAT_INCR(arcstat_l2_lsize, -HDR_GET_LSIZE(hdr));
3626
3627	vdev_space_update(dev->l2ad_vdev, -psize, 0, 0);
3628
3629	(void) zfs_refcount_remove_many(&dev->l2ad_alloc, psize, hdr);
3630	arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);
3631}
3632
3633static void
3634arc_hdr_destroy(arc_buf_hdr_t *hdr)
3635{
3636	if (HDR_HAS_L1HDR(hdr)) {
3637		ASSERT(hdr->b_l1hdr.b_buf == NULL ||
3638		    hdr->b_l1hdr.b_bufcnt > 0);
3639		ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3640		ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
3641	}
3642	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3643	ASSERT(!HDR_IN_HASH_TABLE(hdr));
3644
3645	if (!HDR_EMPTY(hdr))
3646		buf_discard_identity(hdr);
3647
3648	if (HDR_HAS_L2HDR(hdr)) {
3649		l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
3650		boolean_t buflist_held = MUTEX_HELD(&dev->l2ad_mtx);
3651
3652		if (!buflist_held)
3653			mutex_enter(&dev->l2ad_mtx);
3654
3655		/*
3656		 * Even though we checked this conditional above, we
3657		 * need to check this again now that we have the
3658		 * l2ad_mtx. This is because we could be racing with
3659		 * another thread calling l2arc_evict() which might have
3660		 * destroyed this header's L2 portion as we were waiting
3661		 * to acquire the l2ad_mtx. If that happens, we don't
3662		 * want to re-destroy the header's L2 portion.
3663		 */
3664		if (HDR_HAS_L2HDR(hdr)) {
3665			l2arc_trim(hdr);
3666			arc_hdr_l2hdr_destroy(hdr);
3667		}
3668
3669		if (!buflist_held)
3670			mutex_exit(&dev->l2ad_mtx);
3671	}
3672
3673	if (HDR_HAS_L1HDR(hdr)) {
3674		arc_cksum_free(hdr);
3675
3676		while (hdr->b_l1hdr.b_buf != NULL)
3677			arc_buf_destroy_impl(hdr->b_l1hdr.b_buf);
3678
3679#ifdef ZFS_DEBUG
3680		if (hdr->b_l1hdr.b_thawed != NULL) {
3681			kmem_free(hdr->b_l1hdr.b_thawed, 1);
3682			hdr->b_l1hdr.b_thawed = NULL;
3683		}
3684#endif
3685
3686		if (hdr->b_l1hdr.b_pabd != NULL) {
3687			arc_hdr_free_pabd(hdr);
3688		}
3689	}
3690
3691	ASSERT3P(hdr->b_hash_next, ==, NULL);
3692	if (HDR_HAS_L1HDR(hdr)) {
3693		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3694		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
3695		kmem_cache_free(hdr_full_cache, hdr);
3696	} else {
3697		kmem_cache_free(hdr_l2only_cache, hdr);
3698	}
3699}
3700
3701void
3702arc_buf_destroy(arc_buf_t *buf, void* tag)
3703{
3704	arc_buf_hdr_t *hdr = buf->b_hdr;
3705	kmutex_t *hash_lock = HDR_LOCK(hdr);
3706
3707	if (hdr->b_l1hdr.b_state == arc_anon) {
3708		ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1);
3709		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3710		VERIFY0(remove_reference(hdr, NULL, tag));
3711		arc_hdr_destroy(hdr);
3712		return;
3713	}
3714
3715	mutex_enter(hash_lock);
3716	ASSERT3P(hdr, ==, buf->b_hdr);
3717	ASSERT(hdr->b_l1hdr.b_bufcnt > 0);
3718	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
3719	ASSERT3P(hdr->b_l1hdr.b_state, !=, arc_anon);
3720	ASSERT3P(buf->b_data, !=, NULL);
3721
3722	(void) remove_reference(hdr, hash_lock, tag);
3723	arc_buf_destroy_impl(buf);
3724	mutex_exit(hash_lock);
3725}
3726
3727/*
3728 * Evict the arc_buf_hdr that is provided as a parameter. The resultant
3729 * state of the header is dependent on its state prior to entering this
3730 * function. The following transitions are possible:
3731 *
3732 *    - arc_mru -> arc_mru_ghost
3733 *    - arc_mfu -> arc_mfu_ghost
3734 *    - arc_mru_ghost -> arc_l2c_only
3735 *    - arc_mru_ghost -> deleted
3736 *    - arc_mfu_ghost -> arc_l2c_only
3737 *    - arc_mfu_ghost -> deleted
3738 */
3739static int64_t
3740arc_evict_hdr(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
3741{
3742	arc_state_t *evicted_state, *state;
3743	int64_t bytes_evicted = 0;
3744	int min_lifetime = HDR_PRESCIENT_PREFETCH(hdr) ?
3745	    zfs_arc_min_prescient_prefetch_ms : zfs_arc_min_prefetch_ms;
3746
3747	ASSERT(MUTEX_HELD(hash_lock));
3748	ASSERT(HDR_HAS_L1HDR(hdr));
3749
3750	state = hdr->b_l1hdr.b_state;
3751	if (GHOST_STATE(state)) {
3752		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3753		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
3754
3755		/*
3756		 * l2arc_write_buffers() relies on a header's L1 portion
3757		 * (i.e. its b_pabd field) during it's write phase.
3758		 * Thus, we cannot push a header onto the arc_l2c_only
3759		 * state (removing it's L1 piece) until the header is
3760		 * done being written to the l2arc.
3761		 */
3762		if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) {
3763			ARCSTAT_BUMP(arcstat_evict_l2_skip);
3764			return (bytes_evicted);
3765		}
3766
3767		ARCSTAT_BUMP(arcstat_deleted);
3768		bytes_evicted += HDR_GET_LSIZE(hdr);
3769
3770		DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr);
3771
3772		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3773		if (HDR_HAS_L2HDR(hdr)) {
3774			/*
3775			 * This buffer is cached on the 2nd Level ARC;
3776			 * don't destroy the header.
3777			 */
3778			arc_change_state(arc_l2c_only, hdr, hash_lock);
3779			/*
3780			 * dropping from L1+L2 cached to L2-only,
3781			 * realloc to remove the L1 header.
3782			 */
3783			hdr = arc_hdr_realloc(hdr, hdr_full_cache,
3784			    hdr_l2only_cache);
3785		} else {
3786			arc_change_state(arc_anon, hdr, hash_lock);
3787			arc_hdr_destroy(hdr);
3788		}
3789		return (bytes_evicted);
3790	}
3791
3792	ASSERT(state == arc_mru || state == arc_mfu);
3793	evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
3794
3795	/* prefetch buffers have a minimum lifespan */
3796	if (HDR_IO_IN_PROGRESS(hdr) ||
3797	    ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) &&
3798	    ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access < min_lifetime * hz)) {
3799		ARCSTAT_BUMP(arcstat_evict_skip);
3800		return (bytes_evicted);
3801	}
3802
3803	ASSERT0(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt));
3804	while (hdr->b_l1hdr.b_buf) {
3805		arc_buf_t *buf = hdr->b_l1hdr.b_buf;
3806		if (!mutex_tryenter(&buf->b_evict_lock)) {
3807			ARCSTAT_BUMP(arcstat_mutex_miss);
3808			break;
3809		}
3810		if (buf->b_data != NULL)
3811			bytes_evicted += HDR_GET_LSIZE(hdr);
3812		mutex_exit(&buf->b_evict_lock);
3813		arc_buf_destroy_impl(buf);
3814	}
3815
3816	if (HDR_HAS_L2HDR(hdr)) {
3817		ARCSTAT_INCR(arcstat_evict_l2_cached, HDR_GET_LSIZE(hdr));
3818	} else {
3819		if (l2arc_write_eligible(hdr->b_spa, hdr)) {
3820			ARCSTAT_INCR(arcstat_evict_l2_eligible,
3821			    HDR_GET_LSIZE(hdr));
3822		} else {
3823			ARCSTAT_INCR(arcstat_evict_l2_ineligible,
3824			    HDR_GET_LSIZE(hdr));
3825		}
3826	}
3827
3828	if (hdr->b_l1hdr.b_bufcnt == 0) {
3829		arc_cksum_free(hdr);
3830
3831		bytes_evicted += arc_hdr_size(hdr);
3832
3833		/*
3834		 * If this hdr is being evicted and has a compressed
3835		 * buffer then we discard it here before we change states.
3836		 * This ensures that the accounting is updated correctly
3837		 * in arc_free_data_impl().
3838		 */
3839		arc_hdr_free_pabd(hdr);
3840
3841		arc_change_state(evicted_state, hdr, hash_lock);
3842		ASSERT(HDR_IN_HASH_TABLE(hdr));
3843		arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
3844		DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr);
3845	}
3846
3847	return (bytes_evicted);
3848}
3849
3850static uint64_t
3851arc_evict_state_impl(multilist_t *ml, int idx, arc_buf_hdr_t *marker,
3852    uint64_t spa, int64_t bytes)
3853{
3854	multilist_sublist_t *mls;
3855	uint64_t bytes_evicted = 0;
3856	arc_buf_hdr_t *hdr;
3857	kmutex_t *hash_lock;
3858	int evict_count = 0;
3859
3860	ASSERT3P(marker, !=, NULL);
3861	IMPLY(bytes < 0, bytes == ARC_EVICT_ALL);
3862
3863	mls = multilist_sublist_lock(ml, idx);
3864
3865	for (hdr = multilist_sublist_prev(mls, marker); hdr != NULL;
3866	    hdr = multilist_sublist_prev(mls, marker)) {
3867		if ((bytes != ARC_EVICT_ALL && bytes_evicted >= bytes) ||
3868		    (evict_count >= zfs_arc_evict_batch_limit))
3869			break;
3870
3871		/*
3872		 * To keep our iteration location, move the marker
3873		 * forward. Since we're not holding hdr's hash lock, we
3874		 * must be very careful and not remove 'hdr' from the
3875		 * sublist. Otherwise, other consumers might mistake the
3876		 * 'hdr' as not being on a sublist when they call the
3877		 * multilist_link_active() function (they all rely on
3878		 * the hash lock protecting concurrent insertions and
3879		 * removals). multilist_sublist_move_forward() was
3880		 * specifically implemented to ensure this is the case
3881		 * (only 'marker' will be removed and re-inserted).
3882		 */
3883		multilist_sublist_move_forward(mls, marker);
3884
3885		/*
3886		 * The only case where the b_spa field should ever be
3887		 * zero, is the marker headers inserted by
3888		 * arc_evict_state(). It's possible for multiple threads
3889		 * to be calling arc_evict_state() concurrently (e.g.
3890		 * dsl_pool_close() and zio_inject_fault()), so we must
3891		 * skip any markers we see from these other threads.
3892		 */
3893		if (hdr->b_spa == 0)
3894			continue;
3895
3896		/* we're only interested in evicting buffers of a certain spa */
3897		if (spa != 0 && hdr->b_spa != spa) {
3898			ARCSTAT_BUMP(arcstat_evict_skip);
3899			continue;
3900		}
3901
3902		hash_lock = HDR_LOCK(hdr);
3903
3904		/*
3905		 * We aren't calling this function from any code path
3906		 * that would already be holding a hash lock, so we're
3907		 * asserting on this assumption to be defensive in case
3908		 * this ever changes. Without this check, it would be
3909		 * possible to incorrectly increment arcstat_mutex_miss
3910		 * below (e.g. if the code changed such that we called
3911		 * this function with a hash lock held).
3912		 */
3913		ASSERT(!MUTEX_HELD(hash_lock));
3914
3915		if (mutex_tryenter(hash_lock)) {
3916			uint64_t evicted = arc_evict_hdr(hdr, hash_lock);
3917			mutex_exit(hash_lock);
3918
3919			bytes_evicted += evicted;
3920
3921			/*
3922			 * If evicted is zero, arc_evict_hdr() must have
3923			 * decided to skip this header, don't increment
3924			 * evict_count in this case.
3925			 */
3926			if (evicted != 0)
3927				evict_count++;
3928
3929			/*
3930			 * If arc_size isn't overflowing, signal any
3931			 * threads that might happen to be waiting.
3932			 *
3933			 * For each header evicted, we wake up a single
3934			 * thread. If we used cv_broadcast, we could
3935			 * wake up "too many" threads causing arc_size
3936			 * to significantly overflow arc_c; since
3937			 * arc_get_data_impl() doesn't check for overflow
3938			 * when it's woken up (it doesn't because it's
3939			 * possible for the ARC to be overflowing while
3940			 * full of un-evictable buffers, and the
3941			 * function should proceed in this case).
3942			 *
3943			 * If threads are left sleeping, due to not
3944			 * using cv_broadcast here, they will be woken
3945			 * up via cv_broadcast in arc_adjust_cb() just
3946			 * before arc_adjust_zthr sleeps.
3947			 */
3948			mutex_enter(&arc_adjust_lock);
3949			if (!arc_is_overflowing())
3950				cv_signal(&arc_adjust_waiters_cv);
3951			mutex_exit(&arc_adjust_lock);
3952		} else {
3953			ARCSTAT_BUMP(arcstat_mutex_miss);
3954		}
3955	}
3956
3957	multilist_sublist_unlock(mls);
3958
3959	return (bytes_evicted);
3960}
3961
3962/*
3963 * Evict buffers from the given arc state, until we've removed the
3964 * specified number of bytes. Move the removed buffers to the
3965 * appropriate evict state.
3966 *
3967 * This function makes a "best effort". It skips over any buffers
3968 * it can't get a hash_lock on, and so, may not catch all candidates.
3969 * It may also return without evicting as much space as requested.
3970 *
3971 * If bytes is specified using the special value ARC_EVICT_ALL, this
3972 * will evict all available (i.e. unlocked and evictable) buffers from
3973 * the given arc state; which is used by arc_flush().
3974 */
3975static uint64_t
3976arc_evict_state(arc_state_t *state, uint64_t spa, int64_t bytes,
3977    arc_buf_contents_t type)
3978{
3979	uint64_t total_evicted = 0;
3980	multilist_t *ml = state->arcs_list[type];
3981	int num_sublists;
3982	arc_buf_hdr_t **markers;
3983
3984	IMPLY(bytes < 0, bytes == ARC_EVICT_ALL);
3985
3986	num_sublists = multilist_get_num_sublists(ml);
3987
3988	/*
3989	 * If we've tried to evict from each sublist, made some
3990	 * progress, but still have not hit the target number of bytes
3991	 * to evict, we want to keep trying. The markers allow us to
3992	 * pick up where we left off for each individual sublist, rather
3993	 * than starting from the tail each time.
3994	 */
3995	markers = kmem_zalloc(sizeof (*markers) * num_sublists, KM_SLEEP);
3996	for (int i = 0; i < num_sublists; i++) {
3997		markers[i] = kmem_cache_alloc(hdr_full_cache, KM_SLEEP);
3998
3999		/*
4000		 * A b_spa of 0 is used to indicate that this header is
4001		 * a marker. This fact is used in arc_adjust_type() and
4002		 * arc_evict_state_impl().
4003		 */
4004		markers[i]->b_spa = 0;
4005
4006		multilist_sublist_t *mls = multilist_sublist_lock(ml, i);
4007		multilist_sublist_insert_tail(mls, markers[i]);
4008		multilist_sublist_unlock(mls);
4009	}
4010
4011	/*
4012	 * While we haven't hit our target number of bytes to evict, or
4013	 * we're evicting all available buffers.
4014	 */
4015	while (total_evicted < bytes || bytes == ARC_EVICT_ALL) {
4016		int sublist_idx = multilist_get_random_index(ml);
4017		uint64_t scan_evicted = 0;
4018
4019		/*
4020		 * Try to reduce pinned dnodes with a floor of arc_dnode_limit.
4021		 * Request that 10% of the LRUs be scanned by the superblock
4022		 * shrinker.
4023		 */
4024		if (type == ARC_BUFC_DATA && aggsum_compare(&astat_dnode_size,
4025		    arc_dnode_limit) > 0) {
4026			arc_prune_async((aggsum_upper_bound(&astat_dnode_size) -
4027			    arc_dnode_limit) / sizeof (dnode_t) /
4028			    zfs_arc_dnode_reduce_percent);
4029		}
4030
4031		/*
4032		 * Start eviction using a randomly selected sublist,
4033		 * this is to try and evenly balance eviction across all
4034		 * sublists. Always starting at the same sublist
4035		 * (e.g. index 0) would cause evictions to favor certain
4036		 * sublists over others.
4037		 */
4038		for (int i = 0; i < num_sublists; i++) {
4039			uint64_t bytes_remaining;
4040			uint64_t bytes_evicted;
4041
4042			if (bytes == ARC_EVICT_ALL)
4043				bytes_remaining = ARC_EVICT_ALL;
4044			else if (total_evicted < bytes)
4045				bytes_remaining = bytes - total_evicted;
4046			else
4047				break;
4048
4049			bytes_evicted = arc_evict_state_impl(ml, sublist_idx,
4050			    markers[sublist_idx], spa, bytes_remaining);
4051
4052			scan_evicted += bytes_evicted;
4053			total_evicted += bytes_evicted;
4054
4055			/* we've reached the end, wrap to the beginning */
4056			if (++sublist_idx >= num_sublists)
4057				sublist_idx = 0;
4058		}
4059
4060		/*
4061		 * If we didn't evict anything during this scan, we have
4062		 * no reason to believe we'll evict more during another
4063		 * scan, so break the loop.
4064		 */
4065		if (scan_evicted == 0) {
4066			/* This isn't possible, let's make that obvious */
4067			ASSERT3S(bytes, !=, 0);
4068
4069			/*
4070			 * When bytes is ARC_EVICT_ALL, the only way to
4071			 * break the loop is when scan_evicted is zero.
4072			 * In that case, we actually have evicted enough,
4073			 * so we don't want to increment the kstat.
4074			 */
4075			if (bytes != ARC_EVICT_ALL) {
4076				ASSERT3S(total_evicted, <, bytes);
4077				ARCSTAT_BUMP(arcstat_evict_not_enough);
4078			}
4079
4080			break;
4081		}
4082	}
4083
4084	for (int i = 0; i < num_sublists; i++) {
4085		multilist_sublist_t *mls = multilist_sublist_lock(ml, i);
4086		multilist_sublist_remove(mls, markers[i]);
4087		multilist_sublist_unlock(mls);
4088
4089		kmem_cache_free(hdr_full_cache, markers[i]);
4090	}
4091	kmem_free(markers, sizeof (*markers) * num_sublists);
4092
4093	return (total_evicted);
4094}
4095
4096/*
4097 * Flush all "evictable" data of the given type from the arc state
4098 * specified. This will not evict any "active" buffers (i.e. referenced).
4099 *
4100 * When 'retry' is set to B_FALSE, the function will make a single pass
4101 * over the state and evict any buffers that it can. Since it doesn't
4102 * continually retry the eviction, it might end up leaving some buffers
4103 * in the ARC due to lock misses.
4104 *
4105 * When 'retry' is set to B_TRUE, the function will continually retry the
4106 * eviction until *all* evictable buffers have been removed from the
4107 * state. As a result, if concurrent insertions into the state are
4108 * allowed (e.g. if the ARC isn't shutting down), this function might
4109 * wind up in an infinite loop, continually trying to evict buffers.
4110 */
4111static uint64_t
4112arc_flush_state(arc_state_t *state, uint64_t spa, arc_buf_contents_t type,
4113    boolean_t retry)
4114{
4115	uint64_t evicted = 0;
4116
4117	while (zfs_refcount_count(&state->arcs_esize[type]) != 0) {
4118		evicted += arc_evict_state(state, spa, ARC_EVICT_ALL, type);
4119
4120		if (!retry)
4121			break;
4122	}
4123
4124	return (evicted);
4125}
4126
4127/*
4128 * Helper function for arc_prune_async() it is responsible for safely
4129 * handling the execution of a registered arc_prune_func_t.
4130 */
4131static void
4132arc_prune_task(void *ptr)
4133{
4134	arc_prune_t *ap = (arc_prune_t *)ptr;
4135	arc_prune_func_t *func = ap->p_pfunc;
4136
4137	if (func != NULL)
4138		func(ap->p_adjust, ap->p_private);
4139
4140	zfs_refcount_remove(&ap->p_refcnt, func);
4141}
4142
4143/*
4144 * Notify registered consumers they must drop holds on a portion of the ARC
4145 * buffered they reference.  This provides a mechanism to ensure the ARC can
4146 * honor the arc_meta_limit and reclaim otherwise pinned ARC buffers.  This
4147 * is analogous to dnlc_reduce_cache() but more generic.
4148 *
4149 * This operation is performed asynchronously so it may be safely called
4150 * in the context of the arc_reclaim_thread().  A reference is taken here
4151 * for each registered arc_prune_t and the arc_prune_task() is responsible
4152 * for releasing it once the registered arc_prune_func_t has completed.
4153 */
4154static void
4155arc_prune_async(int64_t adjust)
4156{
4157	arc_prune_t *ap;
4158
4159	mutex_enter(&arc_prune_mtx);
4160	for (ap = list_head(&arc_prune_list); ap != NULL;
4161	    ap = list_next(&arc_prune_list, ap)) {
4162
4163		if (zfs_refcount_count(&ap->p_refcnt) >= 2)
4164			continue;
4165
4166		zfs_refcount_add(&ap->p_refcnt, ap->p_pfunc);
4167		ap->p_adjust = adjust;
4168		if (taskq_dispatch(arc_prune_taskq, arc_prune_task,
4169		    ap, TQ_SLEEP) == TASKQID_INVALID) {
4170			zfs_refcount_remove(&ap->p_refcnt, ap->p_pfunc);
4171			continue;
4172		}
4173		ARCSTAT_BUMP(arcstat_prune);
4174	}
4175	mutex_exit(&arc_prune_mtx);
4176}
4177
4178/*
4179 * Evict the specified number of bytes from the state specified,
4180 * restricting eviction to the spa and type given. This function
4181 * prevents us from trying to evict more from a state's list than
4182 * is "evictable", and to skip evicting altogether when passed a
4183 * negative value for "bytes". In contrast, arc_evict_state() will
4184 * evict everything it can, when passed a negative value for "bytes".
4185 */
4186static uint64_t
4187arc_adjust_impl(arc_state_t *state, uint64_t spa, int64_t bytes,
4188    arc_buf_contents_t type)
4189{
4190	int64_t delta;
4191
4192	if (bytes > 0 && zfs_refcount_count(&state->arcs_esize[type]) > 0) {
4193		delta = MIN(zfs_refcount_count(&state->arcs_esize[type]),
4194		    bytes);
4195		return (arc_evict_state(state, spa, delta, type));
4196	}
4197
4198	return (0);
4199}
4200
4201/*
4202 * The goal of this function is to evict enough meta data buffers from the
4203 * ARC in order to enforce the arc_meta_limit.  Achieving this is slightly
4204 * more complicated than it appears because it is common for data buffers
4205 * to have holds on meta data buffers.  In addition, dnode meta data buffers
4206 * will be held by the dnodes in the block preventing them from being freed.
4207 * This means we can't simply traverse the ARC and expect to always find
4208 * enough unheld meta data buffer to release.
4209 *
4210 * Therefore, this function has been updated to make alternating passes
4211 * over the ARC releasing data buffers and then newly unheld meta data
4212 * buffers.  This ensures forward progress is maintained and meta_used
4213 * will decrease.  Normally this is sufficient, but if required the ARC
4214 * will call the registered prune callbacks causing dentry and inodes to
4215 * be dropped from the VFS cache.  This will make dnode meta data buffers
4216 * available for reclaim.
4217 */
4218static uint64_t
4219arc_adjust_meta_balanced(uint64_t meta_used)
4220{
4221	int64_t delta, prune = 0, adjustmnt;
4222	uint64_t total_evicted = 0;
4223	arc_buf_contents_t type = ARC_BUFC_DATA;
4224	int restarts = MAX(zfs_arc_meta_adjust_restarts, 0);
4225
4226restart:
4227	/*
4228	 * This slightly differs than the way we evict from the mru in
4229	 * arc_adjust because we don't have a "target" value (i.e. no
4230	 * "meta" arc_p). As a result, I think we can completely
4231	 * cannibalize the metadata in the MRU before we evict the
4232	 * metadata from the MFU. I think we probably need to implement a
4233	 * "metadata arc_p" value to do this properly.
4234	 */
4235	adjustmnt = meta_used - arc_meta_limit;
4236
4237	if (adjustmnt > 0 &&
4238	    zfs_refcount_count(&arc_mru->arcs_esize[type]) > 0) {
4239		delta = MIN(zfs_refcount_count(&arc_mru->arcs_esize[type]),
4240		    adjustmnt);
4241		total_evicted += arc_adjust_impl(arc_mru, 0, delta, type);
4242		adjustmnt -= delta;
4243	}
4244
4245	/*
4246	 * We can't afford to recalculate adjustmnt here. If we do,
4247	 * new metadata buffers can sneak into the MRU or ANON lists,
4248	 * thus penalize the MFU metadata. Although the fudge factor is
4249	 * small, it has been empirically shown to be significant for
4250	 * certain workloads (e.g. creating many empty directories). As
4251	 * such, we use the original calculation for adjustmnt, and
4252	 * simply decrement the amount of data evicted from the MRU.
4253	 */
4254
4255	if (adjustmnt > 0 &&
4256	    zfs_refcount_count(&arc_mfu->arcs_esize[type]) > 0) {
4257		delta = MIN(zfs_refcount_count(&arc_mfu->arcs_esize[type]),
4258		    adjustmnt);
4259		total_evicted += arc_adjust_impl(arc_mfu, 0, delta, type);
4260	}
4261
4262	adjustmnt = meta_used - arc_meta_limit;
4263
4264	if (adjustmnt > 0 &&
4265	    zfs_refcount_count(&arc_mru_ghost->arcs_esize[type]) > 0) {
4266		delta = MIN(adjustmnt,
4267		    zfs_refcount_count(&arc_mru_ghost->arcs_esize[type]));
4268		total_evicted += arc_adjust_impl(arc_mru_ghost, 0, delta, type);
4269		adjustmnt -= delta;
4270	}
4271
4272	if (adjustmnt > 0 &&
4273	    zfs_refcount_count(&arc_mfu_ghost->arcs_esize[type]) > 0) {
4274		delta = MIN(adjustmnt,
4275		    zfs_refcount_count(&arc_mfu_ghost->arcs_esize[type]));
4276		total_evicted += arc_adjust_impl(arc_mfu_ghost, 0, delta, type);
4277	}
4278
4279	/*
4280	 * If after attempting to make the requested adjustment to the ARC
4281	 * the meta limit is still being exceeded then request that the
4282	 * higher layers drop some cached objects which have holds on ARC
4283	 * meta buffers.  Requests to the upper layers will be made with
4284	 * increasingly large scan sizes until the ARC is below the limit.
4285	 */
4286	if (meta_used > arc_meta_limit) {
4287		if (type == ARC_BUFC_DATA) {
4288			type = ARC_BUFC_METADATA;
4289		} else {
4290			type = ARC_BUFC_DATA;
4291
4292			if (zfs_arc_meta_prune) {
4293				prune += zfs_arc_meta_prune;
4294				arc_prune_async(prune);
4295			}
4296		}
4297
4298		if (restarts > 0) {
4299			restarts--;
4300			goto restart;
4301		}
4302	}
4303	return (total_evicted);
4304}
4305
4306/*
4307 * Evict metadata buffers from the cache, such that arc_meta_used is
4308 * capped by the arc_meta_limit tunable.
4309 */
4310static uint64_t
4311arc_adjust_meta_only(uint64_t meta_used)
4312{
4313	uint64_t total_evicted = 0;
4314	int64_t target;
4315
4316	/*
4317	 * If we're over the meta limit, we want to evict enough
4318	 * metadata to get back under the meta limit. We don't want to
4319	 * evict so much that we drop the MRU below arc_p, though. If
4320	 * we're over the meta limit more than we're over arc_p, we
4321	 * evict some from the MRU here, and some from the MFU below.
4322	 */
4323	target = MIN((int64_t)(meta_used - arc_meta_limit),
4324	    (int64_t)(zfs_refcount_count(&arc_anon->arcs_size) +
4325	    zfs_refcount_count(&arc_mru->arcs_size) - arc_p));
4326
4327	total_evicted += arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
4328
4329	/*
4330	 * Similar to the above, we want to evict enough bytes to get us
4331	 * below the meta limit, but not so much as to drop us below the
4332	 * space allotted to the MFU (which is defined as arc_c - arc_p).
4333	 */
4334	target = MIN((int64_t)(meta_used - arc_meta_limit),
4335	    (int64_t)(zfs_refcount_count(&arc_mfu->arcs_size) -
4336	    (arc_c - arc_p)));
4337
4338	total_evicted += arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
4339
4340	return (total_evicted);
4341}
4342
4343static uint64_t
4344arc_adjust_meta(uint64_t meta_used)
4345{
4346	if (zfs_arc_meta_strategy == ARC_STRATEGY_META_ONLY)
4347		return (arc_adjust_meta_only(meta_used));
4348	else
4349		return (arc_adjust_meta_balanced(meta_used));
4350}
4351
4352/*
4353 * Return the type of the oldest buffer in the given arc state
4354 *
4355 * This function will select a random sublist of type ARC_BUFC_DATA and
4356 * a random sublist of type ARC_BUFC_METADATA. The tail of each sublist
4357 * is compared, and the type which contains the "older" buffer will be
4358 * returned.
4359 */
4360static arc_buf_contents_t
4361arc_adjust_type(arc_state_t *state)
4362{
4363	multilist_t *data_ml = state->arcs_list[ARC_BUFC_DATA];
4364	multilist_t *meta_ml = state->arcs_list[ARC_BUFC_METADATA];
4365	int data_idx = multilist_get_random_index(data_ml);
4366	int meta_idx = multilist_get_random_index(meta_ml);
4367	multilist_sublist_t *data_mls;
4368	multilist_sublist_t *meta_mls;
4369	arc_buf_contents_t type;
4370	arc_buf_hdr_t *data_hdr;
4371	arc_buf_hdr_t *meta_hdr;
4372
4373	/*
4374	 * We keep the sublist lock until we're finished, to prevent
4375	 * the headers from being destroyed via arc_evict_state().
4376	 */
4377	data_mls = multilist_sublist_lock(data_ml, data_idx);
4378	meta_mls = multilist_sublist_lock(meta_ml, meta_idx);
4379
4380	/*
4381	 * These two loops are to ensure we skip any markers that
4382	 * might be at the tail of the lists due to arc_evict_state().
4383	 */
4384
4385	for (data_hdr = multilist_sublist_tail(data_mls); data_hdr != NULL;
4386	    data_hdr = multilist_sublist_prev(data_mls, data_hdr)) {
4387		if (data_hdr->b_spa != 0)
4388			break;
4389	}
4390
4391	for (meta_hdr = multilist_sublist_tail(meta_mls); meta_hdr != NULL;
4392	    meta_hdr = multilist_sublist_prev(meta_mls, meta_hdr)) {
4393		if (meta_hdr->b_spa != 0)
4394			break;
4395	}
4396
4397	if (data_hdr == NULL && meta_hdr == NULL) {
4398		type = ARC_BUFC_DATA;
4399	} else if (data_hdr == NULL) {
4400		ASSERT3P(meta_hdr, !=, NULL);
4401		type = ARC_BUFC_METADATA;
4402	} else if (meta_hdr == NULL) {
4403		ASSERT3P(data_hdr, !=, NULL);
4404		type = ARC_BUFC_DATA;
4405	} else {
4406		ASSERT3P(data_hdr, !=, NULL);
4407		ASSERT3P(meta_hdr, !=, NULL);
4408
4409		/* The headers can't be on the sublist without an L1 header */
4410		ASSERT(HDR_HAS_L1HDR(data_hdr));
4411		ASSERT(HDR_HAS_L1HDR(meta_hdr));
4412
4413		if (data_hdr->b_l1hdr.b_arc_access <
4414		    meta_hdr->b_l1hdr.b_arc_access) {
4415			type = ARC_BUFC_DATA;
4416		} else {
4417			type = ARC_BUFC_METADATA;
4418		}
4419	}
4420
4421	multilist_sublist_unlock(meta_mls);
4422	multilist_sublist_unlock(data_mls);
4423
4424	return (type);
4425}
4426
4427/*
4428 * Evict buffers from the cache, such that arc_size is capped by arc_c.
4429 */
4430static uint64_t
4431arc_adjust(void)
4432{
4433	uint64_t total_evicted = 0;
4434	uint64_t bytes;
4435	int64_t target;
4436	uint64_t asize = aggsum_value(&arc_size);
4437	uint64_t ameta = aggsum_value(&arc_meta_used);
4438
4439	/*
4440	 * If we're over arc_meta_limit, we want to correct that before
4441	 * potentially evicting data buffers below.
4442	 */
4443	total_evicted += arc_adjust_meta(ameta);
4444
4445	/*
4446	 * Adjust MRU size
4447	 *
4448	 * If we're over the target cache size, we want to evict enough
4449	 * from the list to get back to our target size. We don't want
4450	 * to evict too much from the MRU, such that it drops below
4451	 * arc_p. So, if we're over our target cache size more than
4452	 * the MRU is over arc_p, we'll evict enough to get back to
4453	 * arc_p here, and then evict more from the MFU below.
4454	 */
4455	target = MIN((int64_t)(asize - arc_c),
4456	    (int64_t)(zfs_refcount_count(&arc_anon->arcs_size) +
4457	    zfs_refcount_count(&arc_mru->arcs_size) + ameta - arc_p));
4458
4459	/*
4460	 * If we're below arc_meta_min, always prefer to evict data.
4461	 * Otherwise, try to satisfy the requested number of bytes to
4462	 * evict from the type which contains older buffers; in an
4463	 * effort to keep newer buffers in the cache regardless of their
4464	 * type. If we cannot satisfy the number of bytes from this
4465	 * type, spill over into the next type.
4466	 */
4467	if (arc_adjust_type(arc_mru) == ARC_BUFC_METADATA &&
4468	    ameta > arc_meta_min) {
4469		bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
4470		total_evicted += bytes;
4471
4472		/*
4473		 * If we couldn't evict our target number of bytes from
4474		 * metadata, we try to get the rest from data.
4475		 */
4476		target -= bytes;
4477
4478		total_evicted +=
4479		    arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA);
4480	} else {
4481		bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA);
4482		total_evicted += bytes;
4483
4484		/*
4485		 * If we couldn't evict our target number of bytes from
4486		 * data, we try to get the rest from metadata.
4487		 */
4488		target -= bytes;
4489
4490		total_evicted +=
4491		    arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
4492	}
4493
4494	/*
4495	 * Re-sum ARC stats after the first round of evictions.
4496	 */
4497	asize = aggsum_value(&arc_size);
4498	ameta = aggsum_value(&arc_meta_used);
4499
4500	/*
4501	 * Adjust MFU size
4502	 *
4503	 * Now that we've tried to evict enough from the MRU to get its
4504	 * size back to arc_p, if we're still above the target cache
4505	 * size, we evict the rest from the MFU.
4506	 */
4507	target = asize - arc_c;
4508
4509	if (arc_adjust_type(arc_mfu) == ARC_BUFC_METADATA &&
4510	    ameta > arc_meta_min) {
4511		bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
4512		total_evicted += bytes;
4513
4514		/*
4515		 * If we couldn't evict our target number of bytes from
4516		 * metadata, we try to get the rest from data.
4517		 */
4518		target -= bytes;
4519
4520		total_evicted +=
4521		    arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA);
4522	} else {
4523		bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA);
4524		total_evicted += bytes;
4525
4526		/*
4527		 * If we couldn't evict our target number of bytes from
4528		 * data, we try to get the rest from data.
4529		 */
4530		target -= bytes;
4531
4532		total_evicted +=
4533		    arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
4534	}
4535
4536	/*
4537	 * Adjust ghost lists
4538	 *
4539	 * In addition to the above, the ARC also defines target values
4540	 * for the ghost lists. The sum of the mru list and mru ghost
4541	 * list should never exceed the target size of the cache, and
4542	 * the sum of the mru list, mfu list, mru ghost list, and mfu
4543	 * ghost list should never exceed twice the target size of the
4544	 * cache. The following logic enforces these limits on the ghost
4545	 * caches, and evicts from them as needed.
4546	 */
4547	target = zfs_refcount_count(&arc_mru->arcs_size) +
4548	    zfs_refcount_count(&arc_mru_ghost->arcs_size) - arc_c;
4549
4550	bytes = arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_DATA);
4551	total_evicted += bytes;
4552
4553	target -= bytes;
4554
4555	total_evicted +=
4556	    arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_METADATA);
4557
4558	/*
4559	 * We assume the sum of the mru list and mfu list is less than
4560	 * or equal to arc_c (we enforced this above), which means we
4561	 * can use the simpler of the two equations below:
4562	 *
4563	 *	mru + mfu + mru ghost + mfu ghost <= 2 * arc_c
4564	 *		    mru ghost + mfu ghost <= arc_c
4565	 */
4566	target = zfs_refcount_count(&arc_mru_ghost->arcs_size) +
4567	    zfs_refcount_count(&arc_mfu_ghost->arcs_size) - arc_c;
4568
4569	bytes = arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_DATA);
4570	total_evicted += bytes;
4571
4572	target -= bytes;
4573
4574	total_evicted +=
4575	    arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_METADATA);
4576
4577	return (total_evicted);
4578}
4579
4580void
4581arc_flush(spa_t *spa, boolean_t retry)
4582{
4583	uint64_t guid = 0;
4584
4585	/*
4586	 * If retry is B_TRUE, a spa must not be specified since we have
4587	 * no good way to determine if all of a spa's buffers have been
4588	 * evicted from an arc state.
4589	 */
4590	ASSERT(!retry || spa == 0);
4591
4592	if (spa != NULL)
4593		guid = spa_load_guid(spa);
4594
4595	(void) arc_flush_state(arc_mru, guid, ARC_BUFC_DATA, retry);
4596	(void) arc_flush_state(arc_mru, guid, ARC_BUFC_METADATA, retry);
4597
4598	(void) arc_flush_state(arc_mfu, guid, ARC_BUFC_DATA, retry);
4599	(void) arc_flush_state(arc_mfu, guid, ARC_BUFC_METADATA, retry);
4600
4601	(void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_DATA, retry);
4602	(void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_METADATA, retry);
4603
4604	(void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_DATA, retry);
4605	(void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_METADATA, retry);
4606}
4607
4608static void
4609arc_reduce_target_size(int64_t to_free)
4610{
4611	uint64_t asize = aggsum_value(&arc_size);
4612	if (arc_c > arc_c_min) {
4613		DTRACE_PROBE4(arc__shrink, uint64_t, arc_c, uint64_t,
4614			arc_c_min, uint64_t, arc_p, uint64_t, to_free);
4615		if (arc_c > arc_c_min + to_free)
4616			atomic_add_64(&arc_c, -to_free);
4617		else
4618			arc_c = arc_c_min;
4619
4620		atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift));
4621		if (asize < arc_c)
4622			arc_c = MAX(asize, arc_c_min);
4623		if (arc_p > arc_c)
4624			arc_p = (arc_c >> 1);
4625
4626		DTRACE_PROBE2(arc__shrunk, uint64_t, arc_c, uint64_t,
4627			arc_p);
4628
4629		ASSERT(arc_c >= arc_c_min);
4630		ASSERT((int64_t)arc_p >= 0);
4631	}
4632
4633	if (asize > arc_c) {
4634		DTRACE_PROBE2(arc__shrink_adjust, uint64_t, asize,
4635			uint64_t, arc_c);
4636		/* See comment in arc_adjust_cb_check() on why lock+flag */
4637		mutex_enter(&arc_adjust_lock);
4638		arc_adjust_needed = B_TRUE;
4639		mutex_exit(&arc_adjust_lock);
4640		zthr_wakeup(arc_adjust_zthr);
4641	}
4642}
4643
4644typedef enum free_memory_reason_t {
4645	FMR_UNKNOWN,
4646	FMR_NEEDFREE,
4647	FMR_LOTSFREE,
4648	FMR_SWAPFS_MINFREE,
4649	FMR_PAGES_PP_MAXIMUM,
4650	FMR_HEAP_ARENA,
4651	FMR_ZIO_ARENA,
4652} free_memory_reason_t;
4653
4654int64_t last_free_memory;
4655free_memory_reason_t last_free_reason;
4656
4657/*
4658 * Additional reserve of pages for pp_reserve.
4659 */
4660int64_t arc_pages_pp_reserve = 64;
4661
4662/*
4663 * Additional reserve of pages for swapfs.
4664 */
4665int64_t arc_swapfs_reserve = 64;
4666
4667/*
4668 * Return the amount of memory that can be consumed before reclaim will be
4669 * needed.  Positive if there is sufficient free memory, negative indicates
4670 * the amount of memory that needs to be freed up.
4671 */
4672static int64_t
4673arc_available_memory(void)
4674{
4675	int64_t lowest = INT64_MAX;
4676	int64_t n;
4677	free_memory_reason_t r = FMR_UNKNOWN;
4678
4679#ifdef _KERNEL
4680#ifdef __FreeBSD__
4681	/*
4682	 * Cooperate with pagedaemon when it's time for it to scan
4683	 * and reclaim some pages.
4684	 */
4685	n = PAGESIZE * ((int64_t)freemem - zfs_arc_free_target);
4686	if (n < lowest) {
4687		lowest = n;
4688		r = FMR_LOTSFREE;
4689	}
4690
4691#else
4692	if (needfree > 0) {
4693		n = PAGESIZE * (-needfree);
4694		if (n < lowest) {
4695			lowest = n;
4696			r = FMR_NEEDFREE;
4697		}
4698	}
4699
4700	/*
4701	 * check that we're out of range of the pageout scanner.  It starts to
4702	 * schedule paging if freemem is less than lotsfree and needfree.
4703	 * lotsfree is the high-water mark for pageout, and needfree is the
4704	 * number of needed free pages.  We add extra pages here to make sure
4705	 * the scanner doesn't start up while we're freeing memory.
4706	 */
4707	n = PAGESIZE * (freemem - lotsfree - needfree - desfree);
4708	if (n < lowest) {
4709		lowest = n;
4710		r = FMR_LOTSFREE;
4711	}
4712
4713	/*
4714	 * check to make sure that swapfs has enough space so that anon
4715	 * reservations can still succeed. anon_resvmem() checks that the
4716	 * availrmem is greater than swapfs_minfree, and the number of reserved
4717	 * swap pages.  We also add a bit of extra here just to prevent
4718	 * circumstances from getting really dire.
4719	 */
4720	n = PAGESIZE * (availrmem - swapfs_minfree - swapfs_reserve -
4721	    desfree - arc_swapfs_reserve);
4722	if (n < lowest) {
4723		lowest = n;
4724		r = FMR_SWAPFS_MINFREE;
4725	}
4726
4727
4728	/*
4729	 * Check that we have enough availrmem that memory locking (e.g., via
4730	 * mlock(3C) or memcntl(2)) can still succeed.  (pages_pp_maximum
4731	 * stores the number of pages that cannot be locked; when availrmem
4732	 * drops below pages_pp_maximum, page locking mechanisms such as
4733	 * page_pp_lock() will fail.)
4734	 */
4735	n = PAGESIZE * (availrmem - pages_pp_maximum -
4736	    arc_pages_pp_reserve);
4737	if (n < lowest) {
4738		lowest = n;
4739		r = FMR_PAGES_PP_MAXIMUM;
4740	}
4741
4742#endif	/* __FreeBSD__ */
4743#if defined(__i386) || !defined(UMA_MD_SMALL_ALLOC)
4744	/*
4745	 * If we're on an i386 platform, it's possible that we'll exhaust the
4746	 * kernel heap space before we ever run out of available physical
4747	 * memory.  Most checks of the size of the heap_area compare against
4748	 * tune.t_minarmem, which is the minimum available real memory that we
4749	 * can have in the system.  However, this is generally fixed at 25 pages
4750	 * which is so low that it's useless.  In this comparison, we seek to
4751	 * calculate the total heap-size, and reclaim if more than 3/4ths of the
4752	 * heap is allocated.  (Or, in the calculation, if less than 1/4th is
4753	 * free)
4754	 */
4755	n = uma_avail() - (long)(uma_limit() / 4);
4756	if (n < lowest) {
4757		lowest = n;
4758		r = FMR_HEAP_ARENA;
4759	}
4760#endif
4761
4762	/*
4763	 * If zio data pages are being allocated out of a separate heap segment,
4764	 * then enforce that the size of available vmem for this arena remains
4765	 * above about 1/4th (1/(2^arc_zio_arena_free_shift)) free.
4766	 *
4767	 * Note that reducing the arc_zio_arena_free_shift keeps more virtual
4768	 * memory (in the zio_arena) free, which can avoid memory
4769	 * fragmentation issues.
4770	 */
4771	if (zio_arena != NULL) {
4772		n = (int64_t)vmem_size(zio_arena, VMEM_FREE) -
4773		    (vmem_size(zio_arena, VMEM_ALLOC) >>
4774		    arc_zio_arena_free_shift);
4775		if (n < lowest) {
4776			lowest = n;
4777			r = FMR_ZIO_ARENA;
4778		}
4779	}
4780
4781#else	/* _KERNEL */
4782	/* Every 100 calls, free a small amount */
4783	if (spa_get_random(100) == 0)
4784		lowest = -1024;
4785#endif	/* _KERNEL */
4786
4787	last_free_memory = lowest;
4788	last_free_reason = r;
4789	DTRACE_PROBE2(arc__available_memory, int64_t, lowest, int, r);
4790	return (lowest);
4791}
4792
4793
4794/*
4795 * Determine if the system is under memory pressure and is asking
4796 * to reclaim memory. A return value of B_TRUE indicates that the system
4797 * is under memory pressure and that the arc should adjust accordingly.
4798 */
4799static boolean_t
4800arc_reclaim_needed(void)
4801{
4802	return (arc_available_memory() < 0);
4803}
4804
4805extern kmem_cache_t	*zio_buf_cache[];
4806extern kmem_cache_t	*zio_data_buf_cache[];
4807extern kmem_cache_t	*range_seg_cache;
4808extern kmem_cache_t	*abd_chunk_cache;
4809
4810static __noinline void
4811arc_kmem_reap_soon(void)
4812{
4813	size_t			i;
4814	kmem_cache_t		*prev_cache = NULL;
4815	kmem_cache_t		*prev_data_cache = NULL;
4816
4817	DTRACE_PROBE(arc__kmem_reap_start);
4818#ifdef _KERNEL
4819	if (aggsum_compare(&arc_meta_used, arc_meta_limit) >= 0) {
4820		/*
4821		 * We are exceeding our meta-data cache limit.
4822		 * Purge some DNLC entries to release holds on meta-data.
4823		 */
4824		dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent);
4825	}
4826#if defined(__i386)
4827	/*
4828	 * Reclaim unused memory from all kmem caches.
4829	 */
4830	kmem_reap();
4831#endif
4832#endif
4833
4834	for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
4835		if (zio_buf_cache[i] != prev_cache) {
4836			prev_cache = zio_buf_cache[i];
4837			kmem_cache_reap_soon(zio_buf_cache[i]);
4838		}
4839		if (zio_data_buf_cache[i] != prev_data_cache) {
4840			prev_data_cache = zio_data_buf_cache[i];
4841			kmem_cache_reap_soon(zio_data_buf_cache[i]);
4842		}
4843	}
4844	kmem_cache_reap_soon(abd_chunk_cache);
4845	kmem_cache_reap_soon(buf_cache);
4846	kmem_cache_reap_soon(hdr_full_cache);
4847	kmem_cache_reap_soon(hdr_l2only_cache);
4848	kmem_cache_reap_soon(range_seg_cache);
4849
4850#ifdef illumos
4851	if (zio_arena != NULL) {
4852		/*
4853		 * Ask the vmem arena to reclaim unused memory from its
4854		 * quantum caches.
4855		 */
4856		vmem_qcache_reap(zio_arena);
4857	}
4858#endif
4859	DTRACE_PROBE(arc__kmem_reap_end);
4860}
4861
4862/* ARGSUSED */
4863static boolean_t
4864arc_adjust_cb_check(void *arg, zthr_t *zthr)
4865{
4866	/*
4867	 * This is necessary in order for the mdb ::arc dcmd to
4868	 * show up to date information. Since the ::arc command
4869	 * does not call the kstat's update function, without
4870	 * this call, the command may show stale stats for the
4871	 * anon, mru, mru_ghost, mfu, and mfu_ghost lists. Even
4872	 * with this change, the data might be up to 1 second
4873	 * out of date(the arc_adjust_zthr has a maximum sleep
4874	 * time of 1 second); but that should suffice.  The
4875	 * arc_state_t structures can be queried directly if more
4876	 * accurate information is needed.
4877	 */
4878	if (arc_ksp != NULL)
4879		arc_ksp->ks_update(arc_ksp, KSTAT_READ);
4880
4881	/*
4882	 * We have to rely on arc_get_data_impl() to tell us when to adjust,
4883	 * rather than checking if we are overflowing here, so that we are
4884	 * sure to not leave arc_get_data_impl() waiting on
4885	 * arc_adjust_waiters_cv.  If we have become "not overflowing" since
4886	 * arc_get_data_impl() checked, we need to wake it up.  We could
4887	 * broadcast the CV here, but arc_get_data_impl() may have not yet
4888	 * gone to sleep.  We would need to use a mutex to ensure that this
4889	 * function doesn't broadcast until arc_get_data_impl() has gone to
4890	 * sleep (e.g. the arc_adjust_lock).  However, the lock ordering of
4891	 * such a lock would necessarily be incorrect with respect to the
4892	 * zthr_lock, which is held before this function is called, and is
4893	 * held by arc_get_data_impl() when it calls zthr_wakeup().
4894	 */
4895	return (arc_adjust_needed);
4896}
4897
4898/*
4899 * Keep arc_size under arc_c by running arc_adjust which evicts data
4900 * from the ARC. */
4901/* ARGSUSED */
4902static void
4903arc_adjust_cb(void *arg, zthr_t *zthr)
4904{
4905	uint64_t evicted = 0;
4906
4907	/* Evict from cache */
4908	evicted = arc_adjust();
4909
4910	/*
4911	 * If evicted is zero, we couldn't evict anything
4912	 * via arc_adjust(). This could be due to hash lock
4913	 * collisions, but more likely due to the majority of
4914	 * arc buffers being unevictable. Therefore, even if
4915	 * arc_size is above arc_c, another pass is unlikely to
4916	 * be helpful and could potentially cause us to enter an
4917	 * infinite loop.  Additionally, zthr_iscancelled() is
4918	 * checked here so that if the arc is shutting down, the
4919	 * broadcast will wake any remaining arc adjust waiters.
4920	 */
4921	mutex_enter(&arc_adjust_lock);
4922	arc_adjust_needed = !zthr_iscancelled(arc_adjust_zthr) &&
4923	    evicted > 0 && aggsum_compare(&arc_size, arc_c) > 0;
4924	if (!arc_adjust_needed) {
4925		/*
4926		 * We're either no longer overflowing, or we
4927		 * can't evict anything more, so we should wake
4928		 * up any waiters.
4929		 */
4930		cv_broadcast(&arc_adjust_waiters_cv);
4931	}
4932	mutex_exit(&arc_adjust_lock);
4933}
4934
4935/* ARGSUSED */
4936static boolean_t
4937arc_reap_cb_check(void *arg, zthr_t *zthr)
4938{
4939	int64_t free_memory = arc_available_memory();
4940
4941	/*
4942	 * If a kmem reap is already active, don't schedule more.  We must
4943	 * check for this because kmem_cache_reap_soon() won't actually
4944	 * block on the cache being reaped (this is to prevent callers from
4945	 * becoming implicitly blocked by a system-wide kmem reap -- which,
4946	 * on a system with many, many full magazines, can take minutes).
4947	 */
4948	if (!kmem_cache_reap_active() &&
4949	    free_memory < 0) {
4950		arc_no_grow = B_TRUE;
4951		arc_warm = B_TRUE;
4952		/*
4953		 * Wait at least zfs_grow_retry (default 60) seconds
4954		 * before considering growing.
4955		 */
4956		arc_growtime = gethrtime() + SEC2NSEC(arc_grow_retry);
4957		return (B_TRUE);
4958	} else if (free_memory < arc_c >> arc_no_grow_shift) {
4959		arc_no_grow = B_TRUE;
4960	} else if (gethrtime() >= arc_growtime) {
4961		arc_no_grow = B_FALSE;
4962	}
4963
4964	return (B_FALSE);
4965}
4966
4967/*
4968 * Keep enough free memory in the system by reaping the ARC's kmem
4969 * caches.  To cause more slabs to be reapable, we may reduce the
4970 * target size of the cache (arc_c), causing the arc_adjust_cb()
4971 * to free more buffers.
4972 */
4973/* ARGSUSED */
4974static void
4975arc_reap_cb(void *arg, zthr_t *zthr)
4976{
4977	int64_t free_memory;
4978
4979	/*
4980	 * Kick off asynchronous kmem_reap()'s of all our caches.
4981	 */
4982	arc_kmem_reap_soon();
4983
4984	/*
4985	 * Wait at least arc_kmem_cache_reap_retry_ms between
4986	 * arc_kmem_reap_soon() calls. Without this check it is possible to
4987	 * end up in a situation where we spend lots of time reaping
4988	 * caches, while we're near arc_c_min.  Waiting here also gives the
4989	 * subsequent free memory check a chance of finding that the
4990	 * asynchronous reap has already freed enough memory, and we don't
4991	 * need to call arc_reduce_target_size().
4992	 */
4993	delay((hz * arc_kmem_cache_reap_retry_ms + 999) / 1000);
4994
4995	/*
4996	 * Reduce the target size as needed to maintain the amount of free
4997	 * memory in the system at a fraction of the arc_size (1/128th by
4998	 * default).  If oversubscribed (free_memory < 0) then reduce the
4999	 * target arc_size by the deficit amount plus the fractional
5000	 * amount.  If free memory is positive but less then the fractional
5001	 * amount, reduce by what is needed to hit the fractional amount.
5002	 */
5003	free_memory = arc_available_memory();
5004
5005	int64_t to_free =
5006	    (arc_c >> arc_shrink_shift) - free_memory;
5007	if (to_free > 0) {
5008#ifdef _KERNEL
5009#ifdef illumos
5010		to_free = MAX(to_free, ptob(needfree));
5011#endif
5012#endif
5013		arc_reduce_target_size(to_free);
5014	}
5015}
5016
5017static u_int arc_dnlc_evicts_arg;
5018extern struct vfsops zfs_vfsops;
5019
5020static void
5021arc_dnlc_evicts_thread(void *dummy __unused)
5022{
5023	callb_cpr_t cpr;
5024	u_int percent;
5025
5026	CALLB_CPR_INIT(&cpr, &arc_dnlc_evicts_lock, callb_generic_cpr, FTAG);
5027
5028	mutex_enter(&arc_dnlc_evicts_lock);
5029	while (!arc_dnlc_evicts_thread_exit) {
5030		CALLB_CPR_SAFE_BEGIN(&cpr);
5031		(void) cv_wait(&arc_dnlc_evicts_cv, &arc_dnlc_evicts_lock);
5032		CALLB_CPR_SAFE_END(&cpr, &arc_dnlc_evicts_lock);
5033		if (arc_dnlc_evicts_arg != 0) {
5034			percent = arc_dnlc_evicts_arg;
5035			mutex_exit(&arc_dnlc_evicts_lock);
5036#ifdef _KERNEL
5037			vnlru_free(desiredvnodes * percent / 100, &zfs_vfsops);
5038#endif
5039			mutex_enter(&arc_dnlc_evicts_lock);
5040			/*
5041			 * Clear our token only after vnlru_free()
5042			 * pass is done, to avoid false queueing of
5043			 * the requests.
5044			 */
5045			arc_dnlc_evicts_arg = 0;
5046		}
5047	}
5048	arc_dnlc_evicts_thread_exit = FALSE;
5049	cv_broadcast(&arc_dnlc_evicts_cv);
5050	CALLB_CPR_EXIT(&cpr);
5051	thread_exit();
5052}
5053
5054void
5055dnlc_reduce_cache(void *arg)
5056{
5057	u_int percent;
5058
5059	percent = (u_int)(uintptr_t)arg;
5060	mutex_enter(&arc_dnlc_evicts_lock);
5061	if (arc_dnlc_evicts_arg == 0) {
5062		arc_dnlc_evicts_arg = percent;
5063		cv_broadcast(&arc_dnlc_evicts_cv);
5064	}
5065	mutex_exit(&arc_dnlc_evicts_lock);
5066}
5067
5068/*
5069 * Adapt arc info given the number of bytes we are trying to add and
5070 * the state that we are comming from.  This function is only called
5071 * when we are adding new content to the cache.
5072 */
5073static void
5074arc_adapt(int bytes, arc_state_t *state)
5075{
5076	int mult;
5077	uint64_t arc_p_min = (arc_c >> arc_p_min_shift);
5078	int64_t mrug_size = zfs_refcount_count(&arc_mru_ghost->arcs_size);
5079	int64_t mfug_size = zfs_refcount_count(&arc_mfu_ghost->arcs_size);
5080
5081	if (state == arc_l2c_only)
5082		return;
5083
5084	ASSERT(bytes > 0);
5085	/*
5086	 * Adapt the target size of the MRU list:
5087	 *	- if we just hit in the MRU ghost list, then increase
5088	 *	  the target size of the MRU list.
5089	 *	- if we just hit in the MFU ghost list, then increase
5090	 *	  the target size of the MFU list by decreasing the
5091	 *	  target size of the MRU list.
5092	 */
5093	if (state == arc_mru_ghost) {
5094		mult = (mrug_size >= mfug_size) ? 1 : (mfug_size / mrug_size);
5095		mult = MIN(mult, 10); /* avoid wild arc_p adjustment */
5096
5097		arc_p = MIN(arc_c - arc_p_min, arc_p + bytes * mult);
5098	} else if (state == arc_mfu_ghost) {
5099		uint64_t delta;
5100
5101		mult = (mfug_size >= mrug_size) ? 1 : (mrug_size / mfug_size);
5102		mult = MIN(mult, 10);
5103
5104		delta = MIN(bytes * mult, arc_p);
5105		arc_p = MAX(arc_p_min, arc_p - delta);
5106	}
5107	ASSERT((int64_t)arc_p >= 0);
5108
5109	/*
5110	 * Wake reap thread if we do not have any available memory
5111	 */
5112	if (arc_reclaim_needed()) {
5113		zthr_wakeup(arc_reap_zthr);
5114		return;
5115	}
5116
5117	if (arc_no_grow)
5118		return;
5119
5120	if (arc_c >= arc_c_max)
5121		return;
5122
5123	/*
5124	 * If we're within (2 * maxblocksize) bytes of the target
5125	 * cache size, increment the target cache size
5126	 */
5127	if (aggsum_compare(&arc_size, arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) >
5128	    0) {
5129		DTRACE_PROBE1(arc__inc_adapt, int, bytes);
5130		atomic_add_64(&arc_c, (int64_t)bytes);
5131		if (arc_c > arc_c_max)
5132			arc_c = arc_c_max;
5133		else if (state == arc_anon)
5134			atomic_add_64(&arc_p, (int64_t)bytes);
5135		if (arc_p > arc_c)
5136			arc_p = arc_c;
5137	}
5138	ASSERT((int64_t)arc_p >= 0);
5139}
5140
5141/*
5142 * Check if arc_size has grown past our upper threshold, determined by
5143 * zfs_arc_overflow_shift.
5144 */
5145static boolean_t
5146arc_is_overflowing(void)
5147{
5148	/* Always allow at least one block of overflow */
5149	int64_t overflow = MAX(SPA_MAXBLOCKSIZE,
5150	    arc_c >> zfs_arc_overflow_shift);
5151
5152	/*
5153	 * We just compare the lower bound here for performance reasons. Our
5154	 * primary goals are to make sure that the arc never grows without
5155	 * bound, and that it can reach its maximum size. This check
5156	 * accomplishes both goals. The maximum amount we could run over by is
5157	 * 2 * aggsum_borrow_multiplier * NUM_CPUS * the average size of a block
5158	 * in the ARC. In practice, that's in the tens of MB, which is low
5159	 * enough to be safe.
5160	 */
5161	return (aggsum_lower_bound(&arc_size) >= (int64_t)arc_c + overflow);
5162}
5163
5164static abd_t *
5165arc_get_data_abd(arc_buf_hdr_t *hdr, uint64_t size, void *tag, boolean_t do_adapt)
5166{
5167	arc_buf_contents_t type = arc_buf_type(hdr);
5168
5169	arc_get_data_impl(hdr, size, tag, do_adapt);
5170	if (type == ARC_BUFC_METADATA) {
5171		return (abd_alloc(size, B_TRUE));
5172	} else {
5173		ASSERT(type == ARC_BUFC_DATA);
5174		return (abd_alloc(size, B_FALSE));
5175	}
5176}
5177
5178static void *
5179arc_get_data_buf(arc_buf_hdr_t *hdr, uint64_t size, void *tag)
5180{
5181	arc_buf_contents_t type = arc_buf_type(hdr);
5182
5183	arc_get_data_impl(hdr, size, tag, B_TRUE);
5184	if (type == ARC_BUFC_METADATA) {
5185		return (zio_buf_alloc(size));
5186	} else {
5187		ASSERT(type == ARC_BUFC_DATA);
5188		return (zio_data_buf_alloc(size));
5189	}
5190}
5191
5192/*
5193 * Allocate a block and return it to the caller. If we are hitting the
5194 * hard limit for the cache size, we must sleep, waiting for the eviction
5195 * thread to catch up. If we're past the target size but below the hard
5196 * limit, we'll only signal the reclaim thread and continue on.
5197 */
5198static void
5199arc_get_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag, boolean_t do_adapt)
5200{
5201	arc_state_t *state = hdr->b_l1hdr.b_state;
5202	arc_buf_contents_t type = arc_buf_type(hdr);
5203
5204	if (do_adapt)
5205		arc_adapt(size, state);
5206
5207	/*
5208	 * If arc_size is currently overflowing, and has grown past our
5209	 * upper limit, we must be adding data faster than the evict
5210	 * thread can evict. Thus, to ensure we don't compound the
5211	 * problem by adding more data and forcing arc_size to grow even
5212	 * further past it's target size, we halt and wait for the
5213	 * eviction thread to catch up.
5214	 *
5215	 * It's also possible that the reclaim thread is unable to evict
5216	 * enough buffers to get arc_size below the overflow limit (e.g.
5217	 * due to buffers being un-evictable, or hash lock collisions).
5218	 * In this case, we want to proceed regardless if we're
5219	 * overflowing; thus we don't use a while loop here.
5220	 */
5221	if (arc_is_overflowing()) {
5222		mutex_enter(&arc_adjust_lock);
5223
5224		/*
5225		 * Now that we've acquired the lock, we may no longer be
5226		 * over the overflow limit, lets check.
5227		 *
5228		 * We're ignoring the case of spurious wake ups. If that
5229		 * were to happen, it'd let this thread consume an ARC
5230		 * buffer before it should have (i.e. before we're under
5231		 * the overflow limit and were signalled by the reclaim
5232		 * thread). As long as that is a rare occurrence, it
5233		 * shouldn't cause any harm.
5234		 */
5235		if (arc_is_overflowing()) {
5236			arc_adjust_needed = B_TRUE;
5237			zthr_wakeup(arc_adjust_zthr);
5238			(void) cv_wait(&arc_adjust_waiters_cv,
5239			    &arc_adjust_lock);
5240		}
5241		mutex_exit(&arc_adjust_lock);
5242	}
5243
5244	VERIFY3U(hdr->b_type, ==, type);
5245	if (type == ARC_BUFC_METADATA) {
5246		arc_space_consume(size, ARC_SPACE_META);
5247	} else {
5248		arc_space_consume(size, ARC_SPACE_DATA);
5249	}
5250
5251	/*
5252	 * Update the state size.  Note that ghost states have a
5253	 * "ghost size" and so don't need to be updated.
5254	 */
5255	if (!GHOST_STATE(state)) {
5256
5257		(void) zfs_refcount_add_many(&state->arcs_size, size, tag);
5258
5259		/*
5260		 * If this is reached via arc_read, the link is
5261		 * protected by the hash lock. If reached via
5262		 * arc_buf_alloc, the header should not be accessed by
5263		 * any other thread. And, if reached via arc_read_done,
5264		 * the hash lock will protect it if it's found in the
5265		 * hash table; otherwise no other thread should be
5266		 * trying to [add|remove]_reference it.
5267		 */
5268		if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
5269			ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
5270			(void) zfs_refcount_add_many(&state->arcs_esize[type],
5271			    size, tag);
5272		}
5273
5274		/*
5275		 * If we are growing the cache, and we are adding anonymous
5276		 * data, and we have outgrown arc_p, update arc_p
5277		 */
5278		if (aggsum_upper_bound(&arc_size) < arc_c &&
5279		    hdr->b_l1hdr.b_state == arc_anon &&
5280		    (zfs_refcount_count(&arc_anon->arcs_size) +
5281		    zfs_refcount_count(&arc_mru->arcs_size) > arc_p))
5282			arc_p = MIN(arc_c, arc_p + size);
5283	}
5284	ARCSTAT_BUMP(arcstat_allocated);
5285}
5286
5287static void
5288arc_free_data_abd(arc_buf_hdr_t *hdr, abd_t *abd, uint64_t size, void *tag)
5289{
5290	arc_free_data_impl(hdr, size, tag);
5291	abd_free(abd);
5292}
5293
5294static void
5295arc_free_data_buf(arc_buf_hdr_t *hdr, void *buf, uint64_t size, void *tag)
5296{
5297	arc_buf_contents_t type = arc_buf_type(hdr);
5298
5299	arc_free_data_impl(hdr, size, tag);
5300	if (type == ARC_BUFC_METADATA) {
5301		zio_buf_free(buf, size);
5302	} else {
5303		ASSERT(type == ARC_BUFC_DATA);
5304		zio_data_buf_free(buf, size);
5305	}
5306}
5307
5308/*
5309 * Free the arc data buffer.
5310 */
5311static void
5312arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag)
5313{
5314	arc_state_t *state = hdr->b_l1hdr.b_state;
5315	arc_buf_contents_t type = arc_buf_type(hdr);
5316
5317	/* protected by hash lock, if in the hash table */
5318	if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
5319		ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
5320		ASSERT(state != arc_anon && state != arc_l2c_only);
5321
5322		(void) zfs_refcount_remove_many(&state->arcs_esize[type],
5323		    size, tag);
5324	}
5325	(void) zfs_refcount_remove_many(&state->arcs_size, size, tag);
5326
5327	VERIFY3U(hdr->b_type, ==, type);
5328	if (type == ARC_BUFC_METADATA) {
5329		arc_space_return(size, ARC_SPACE_META);
5330	} else {
5331		ASSERT(type == ARC_BUFC_DATA);
5332		arc_space_return(size, ARC_SPACE_DATA);
5333	}
5334}
5335
5336/*
5337 * This routine is called whenever a buffer is accessed.
5338 * NOTE: the hash lock is dropped in this function.
5339 */
5340static void
5341arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
5342{
5343	clock_t now;
5344
5345	ASSERT(MUTEX_HELD(hash_lock));
5346	ASSERT(HDR_HAS_L1HDR(hdr));
5347
5348	if (hdr->b_l1hdr.b_state == arc_anon) {
5349		/*
5350		 * This buffer is not in the cache, and does not
5351		 * appear in our "ghost" list.  Add the new buffer
5352		 * to the MRU state.
5353		 */
5354
5355		ASSERT0(hdr->b_l1hdr.b_arc_access);
5356		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
5357		DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
5358		arc_change_state(arc_mru, hdr, hash_lock);
5359
5360	} else if (hdr->b_l1hdr.b_state == arc_mru) {
5361		now = ddi_get_lbolt();
5362
5363		/*
5364		 * If this buffer is here because of a prefetch, then either:
5365		 * - clear the flag if this is a "referencing" read
5366		 *   (any subsequent access will bump this into the MFU state).
5367		 * or
5368		 * - move the buffer to the head of the list if this is
5369		 *   another prefetch (to make it less likely to be evicted).
5370		 */
5371		if (HDR_PREFETCH(hdr) || HDR_PRESCIENT_PREFETCH(hdr)) {
5372			if (zfs_refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
5373				/* link protected by hash lock */
5374				ASSERT(multilist_link_active(
5375				    &hdr->b_l1hdr.b_arc_node));
5376			} else {
5377				arc_hdr_clear_flags(hdr,
5378				    ARC_FLAG_PREFETCH |
5379				    ARC_FLAG_PRESCIENT_PREFETCH);
5380				ARCSTAT_BUMP(arcstat_mru_hits);
5381			}
5382			hdr->b_l1hdr.b_arc_access = now;
5383			return;
5384		}
5385
5386		/*
5387		 * This buffer has been "accessed" only once so far,
5388		 * but it is still in the cache. Move it to the MFU
5389		 * state.
5390		 */
5391		if (now > hdr->b_l1hdr.b_arc_access + ARC_MINTIME) {
5392			/*
5393			 * More than 125ms have passed since we
5394			 * instantiated this buffer.  Move it to the
5395			 * most frequently used state.
5396			 */
5397			hdr->b_l1hdr.b_arc_access = now;
5398			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
5399			arc_change_state(arc_mfu, hdr, hash_lock);
5400		}
5401		atomic_inc_32(&hdr->b_l1hdr.b_mru_hits);
5402		ARCSTAT_BUMP(arcstat_mru_hits);
5403	} else if (hdr->b_l1hdr.b_state == arc_mru_ghost) {
5404		arc_state_t	*new_state;
5405		/*
5406		 * This buffer has been "accessed" recently, but
5407		 * was evicted from the cache.  Move it to the
5408		 * MFU state.
5409		 */
5410
5411		if (HDR_PREFETCH(hdr) || HDR_PRESCIENT_PREFETCH(hdr)) {
5412			new_state = arc_mru;
5413			if (zfs_refcount_count(&hdr->b_l1hdr.b_refcnt) > 0) {
5414				arc_hdr_clear_flags(hdr,
5415				    ARC_FLAG_PREFETCH |
5416				    ARC_FLAG_PRESCIENT_PREFETCH);
5417			}
5418			DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
5419		} else {
5420			new_state = arc_mfu;
5421			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
5422		}
5423
5424		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
5425		arc_change_state(new_state, hdr, hash_lock);
5426
5427		atomic_inc_32(&hdr->b_l1hdr.b_mru_ghost_hits);
5428		ARCSTAT_BUMP(arcstat_mru_ghost_hits);
5429	} else if (hdr->b_l1hdr.b_state == arc_mfu) {
5430		/*
5431		 * This buffer has been accessed more than once and is
5432		 * still in the cache.  Keep it in the MFU state.
5433		 *
5434		 * NOTE: an add_reference() that occurred when we did
5435		 * the arc_read() will have kicked this off the list.
5436		 * If it was a prefetch, we will explicitly move it to
5437		 * the head of the list now.
5438		 */
5439
5440		atomic_inc_32(&hdr->b_l1hdr.b_mfu_hits);
5441		ARCSTAT_BUMP(arcstat_mfu_hits);
5442		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
5443	} else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) {
5444		arc_state_t	*new_state = arc_mfu;
5445		/*
5446		 * This buffer has been accessed more than once but has
5447		 * been evicted from the cache.  Move it back to the
5448		 * MFU state.
5449		 */
5450
5451		if (HDR_PREFETCH(hdr) || HDR_PRESCIENT_PREFETCH(hdr)) {
5452			/*
5453			 * This is a prefetch access...
5454			 * move this block back to the MRU state.
5455			 */
5456			new_state = arc_mru;
5457		}
5458
5459		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
5460		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
5461		arc_change_state(new_state, hdr, hash_lock);
5462
5463		atomic_inc_32(&hdr->b_l1hdr.b_mfu_ghost_hits);
5464		ARCSTAT_BUMP(arcstat_mfu_ghost_hits);
5465	} else if (hdr->b_l1hdr.b_state == arc_l2c_only) {
5466		/*
5467		 * This buffer is on the 2nd Level ARC.
5468		 */
5469
5470		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
5471		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
5472		arc_change_state(arc_mfu, hdr, hash_lock);
5473	} else {
5474		ASSERT(!"invalid arc state");
5475	}
5476}
5477
5478/*
5479 * This routine is called by dbuf_hold() to update the arc_access() state
5480 * which otherwise would be skipped for entries in the dbuf cache.
5481 */
5482void
5483arc_buf_access(arc_buf_t *buf)
5484{
5485	mutex_enter(&buf->b_evict_lock);
5486	arc_buf_hdr_t *hdr = buf->b_hdr;
5487
5488	/*
5489	 * Avoid taking the hash_lock when possible as an optimization.
5490	 * The header must be checked again under the hash_lock in order
5491	 * to handle the case where it is concurrently being released.
5492	 */
5493	if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr)) {
5494		mutex_exit(&buf->b_evict_lock);
5495		ARCSTAT_BUMP(arcstat_access_skip);
5496		return;
5497	}
5498
5499	kmutex_t *hash_lock = HDR_LOCK(hdr);
5500	mutex_enter(hash_lock);
5501
5502	if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr)) {
5503		mutex_exit(hash_lock);
5504		mutex_exit(&buf->b_evict_lock);
5505		ARCSTAT_BUMP(arcstat_access_skip);
5506		return;
5507	}
5508
5509	mutex_exit(&buf->b_evict_lock);
5510
5511	ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
5512	    hdr->b_l1hdr.b_state == arc_mfu);
5513
5514	DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
5515	arc_access(hdr, hash_lock);
5516	mutex_exit(hash_lock);
5517
5518	ARCSTAT_BUMP(arcstat_hits);
5519	ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
5520	    demand, prefetch, !HDR_ISTYPE_METADATA(hdr), data, metadata, hits);
5521}
5522
5523/* a generic arc_read_done_func_t which you can use */
5524/* ARGSUSED */
5525void
5526arc_bcopy_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp,
5527    arc_buf_t *buf, void *arg)
5528{
5529	if (buf == NULL)
5530		return;
5531
5532	bcopy(buf->b_data, arg, arc_buf_size(buf));
5533	arc_buf_destroy(buf, arg);
5534}
5535
5536/* a generic arc_read_done_func_t */
5537/* ARGSUSED */
5538void
5539arc_getbuf_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp,
5540    arc_buf_t *buf, void *arg)
5541{
5542	arc_buf_t **bufp = arg;
5543	if (buf == NULL) {
5544		ASSERT(zio == NULL || zio->io_error != 0);
5545		*bufp = NULL;
5546	} else {
5547		ASSERT(zio == NULL || zio->io_error == 0);
5548		*bufp = buf;
5549		ASSERT(buf->b_data != NULL);
5550	}
5551}
5552
5553static void
5554arc_hdr_verify(arc_buf_hdr_t *hdr, blkptr_t *bp)
5555{
5556	if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp)) {
5557		ASSERT3U(HDR_GET_PSIZE(hdr), ==, 0);
5558		ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
5559	} else {
5560		if (HDR_COMPRESSION_ENABLED(hdr)) {
5561			ASSERT3U(HDR_GET_COMPRESS(hdr), ==,
5562			    BP_GET_COMPRESS(bp));
5563		}
5564		ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp));
5565		ASSERT3U(HDR_GET_PSIZE(hdr), ==, BP_GET_PSIZE(bp));
5566	}
5567}
5568
5569static void
5570arc_read_done(zio_t *zio)
5571{
5572	arc_buf_hdr_t	*hdr = zio->io_private;
5573	kmutex_t	*hash_lock = NULL;
5574	arc_callback_t	*callback_list;
5575	arc_callback_t	*acb;
5576	boolean_t	freeable = B_FALSE;
5577	boolean_t	no_zio_error = (zio->io_error == 0);
5578
5579	/*
5580	 * The hdr was inserted into hash-table and removed from lists
5581	 * prior to starting I/O.  We should find this header, since
5582	 * it's in the hash table, and it should be legit since it's
5583	 * not possible to evict it during the I/O.  The only possible
5584	 * reason for it not to be found is if we were freed during the
5585	 * read.
5586	 */
5587	if (HDR_IN_HASH_TABLE(hdr)) {
5588		ASSERT3U(hdr->b_birth, ==, BP_PHYSICAL_BIRTH(zio->io_bp));
5589		ASSERT3U(hdr->b_dva.dva_word[0], ==,
5590		    BP_IDENTITY(zio->io_bp)->dva_word[0]);
5591		ASSERT3U(hdr->b_dva.dva_word[1], ==,
5592		    BP_IDENTITY(zio->io_bp)->dva_word[1]);
5593
5594		arc_buf_hdr_t *found = buf_hash_find(hdr->b_spa, zio->io_bp,
5595		    &hash_lock);
5596
5597		ASSERT((found == hdr &&
5598		    DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
5599		    (found == hdr && HDR_L2_READING(hdr)));
5600		ASSERT3P(hash_lock, !=, NULL);
5601	}
5602
5603	if (no_zio_error) {
5604		/* byteswap if necessary */
5605		if (BP_SHOULD_BYTESWAP(zio->io_bp)) {
5606			if (BP_GET_LEVEL(zio->io_bp) > 0) {
5607				hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;
5608			} else {
5609				hdr->b_l1hdr.b_byteswap =
5610				    DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp));
5611			}
5612		} else {
5613			hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
5614		}
5615	}
5616
5617	arc_hdr_clear_flags(hdr, ARC_FLAG_L2_EVICTED);
5618	if (l2arc_noprefetch && HDR_PREFETCH(hdr))
5619		arc_hdr_clear_flags(hdr, ARC_FLAG_L2CACHE);
5620
5621	callback_list = hdr->b_l1hdr.b_acb;
5622	ASSERT3P(callback_list, !=, NULL);
5623
5624	if (hash_lock && no_zio_error && hdr->b_l1hdr.b_state == arc_anon) {
5625		/*
5626		 * Only call arc_access on anonymous buffers.  This is because
5627		 * if we've issued an I/O for an evicted buffer, we've already
5628		 * called arc_access (to prevent any simultaneous readers from
5629		 * getting confused).
5630		 */
5631		arc_access(hdr, hash_lock);
5632	}
5633
5634	/*
5635	 * If a read request has a callback (i.e. acb_done is not NULL), then we
5636	 * make a buf containing the data according to the parameters which were
5637	 * passed in. The implementation of arc_buf_alloc_impl() ensures that we
5638	 * aren't needlessly decompressing the data multiple times.
5639	 */
5640	int callback_cnt = 0;
5641	for (acb = callback_list; acb != NULL; acb = acb->acb_next) {
5642		if (!acb->acb_done)
5643			continue;
5644
5645		callback_cnt++;
5646
5647		if (no_zio_error) {
5648			int error = arc_buf_alloc_impl(hdr, acb->acb_private,
5649			    acb->acb_compressed, zio->io_error == 0,
5650			    &acb->acb_buf);
5651			if (error != 0) {
5652				/*
5653				 * Decompression failed.  Set io_error
5654				 * so that when we call acb_done (below),
5655				 * we will indicate that the read failed.
5656				 * Note that in the unusual case where one
5657				 * callback is compressed and another
5658				 * uncompressed, we will mark all of them
5659				 * as failed, even though the uncompressed
5660				 * one can't actually fail.  In this case,
5661				 * the hdr will not be anonymous, because
5662				 * if there are multiple callbacks, it's
5663				 * because multiple threads found the same
5664				 * arc buf in the hash table.
5665				 */
5666				zio->io_error = error;
5667			}
5668		}
5669	}
5670	/*
5671	 * If there are multiple callbacks, we must have the hash lock,
5672	 * because the only way for multiple threads to find this hdr is
5673	 * in the hash table.  This ensures that if there are multiple
5674	 * callbacks, the hdr is not anonymous.  If it were anonymous,
5675	 * we couldn't use arc_buf_destroy() in the error case below.
5676	 */
5677	ASSERT(callback_cnt < 2 || hash_lock != NULL);
5678
5679	hdr->b_l1hdr.b_acb = NULL;
5680	arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5681	if (callback_cnt == 0) {
5682		ASSERT(HDR_PREFETCH(hdr));
5683		ASSERT0(hdr->b_l1hdr.b_bufcnt);
5684		ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
5685	}
5686
5687	ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt) ||
5688	    callback_list != NULL);
5689
5690	if (no_zio_error) {
5691		arc_hdr_verify(hdr, zio->io_bp);
5692	} else {
5693		arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
5694		if (hdr->b_l1hdr.b_state != arc_anon)
5695			arc_change_state(arc_anon, hdr, hash_lock);
5696		if (HDR_IN_HASH_TABLE(hdr))
5697			buf_hash_remove(hdr);
5698		freeable = zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
5699	}
5700
5701	/*
5702	 * Broadcast before we drop the hash_lock to avoid the possibility
5703	 * that the hdr (and hence the cv) might be freed before we get to
5704	 * the cv_broadcast().
5705	 */
5706	cv_broadcast(&hdr->b_l1hdr.b_cv);
5707
5708	if (hash_lock != NULL) {
5709		mutex_exit(hash_lock);
5710	} else {
5711		/*
5712		 * This block was freed while we waited for the read to
5713		 * complete.  It has been removed from the hash table and
5714		 * moved to the anonymous state (so that it won't show up
5715		 * in the cache).
5716		 */
5717		ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
5718		freeable = zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
5719	}
5720
5721	/* execute each callback and free its structure */
5722	while ((acb = callback_list) != NULL) {
5723		if (acb->acb_done != NULL) {
5724			if (zio->io_error != 0 && acb->acb_buf != NULL) {
5725				/*
5726				 * If arc_buf_alloc_impl() fails during
5727				 * decompression, the buf will still be
5728				 * allocated, and needs to be freed here.
5729				 */
5730				arc_buf_destroy(acb->acb_buf, acb->acb_private);
5731				acb->acb_buf = NULL;
5732			}
5733			acb->acb_done(zio, &zio->io_bookmark, zio->io_bp,
5734			    acb->acb_buf, acb->acb_private);
5735		}
5736
5737		if (acb->acb_zio_dummy != NULL) {
5738			acb->acb_zio_dummy->io_error = zio->io_error;
5739			zio_nowait(acb->acb_zio_dummy);
5740		}
5741
5742		callback_list = acb->acb_next;
5743		kmem_free(acb, sizeof (arc_callback_t));
5744	}
5745
5746	if (freeable)
5747		arc_hdr_destroy(hdr);
5748}
5749
5750/*
5751 * "Read" the block at the specified DVA (in bp) via the
5752 * cache.  If the block is found in the cache, invoke the provided
5753 * callback immediately and return.  Note that the `zio' parameter
5754 * in the callback will be NULL in this case, since no IO was
5755 * required.  If the block is not in the cache pass the read request
5756 * on to the spa with a substitute callback function, so that the
5757 * requested block will be added to the cache.
5758 *
5759 * If a read request arrives for a block that has a read in-progress,
5760 * either wait for the in-progress read to complete (and return the
5761 * results); or, if this is a read with a "done" func, add a record
5762 * to the read to invoke the "done" func when the read completes,
5763 * and return; or just return.
5764 *
5765 * arc_read_done() will invoke all the requested "done" functions
5766 * for readers of this block.
5767 */
5768int
5769arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_read_done_func_t *done,
5770    void *private, zio_priority_t priority, int zio_flags,
5771    arc_flags_t *arc_flags, const zbookmark_phys_t *zb)
5772{
5773	arc_buf_hdr_t *hdr = NULL;
5774	kmutex_t *hash_lock = NULL;
5775	zio_t *rzio;
5776	uint64_t guid = spa_load_guid(spa);
5777	boolean_t compressed_read = (zio_flags & ZIO_FLAG_RAW) != 0;
5778	int rc = 0;
5779
5780	ASSERT(!BP_IS_EMBEDDED(bp) ||
5781	    BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA);
5782
5783top:
5784	if (!BP_IS_EMBEDDED(bp)) {
5785		/*
5786		 * Embedded BP's have no DVA and require no I/O to "read".
5787		 * Create an anonymous arc buf to back it.
5788		 */
5789		hdr = buf_hash_find(guid, bp, &hash_lock);
5790	}
5791
5792	if (hdr != NULL && HDR_HAS_L1HDR(hdr) && hdr->b_l1hdr.b_pabd != NULL) {
5793		arc_buf_t *buf = NULL;
5794		*arc_flags |= ARC_FLAG_CACHED;
5795
5796		if (HDR_IO_IN_PROGRESS(hdr)) {
5797			zio_t *head_zio = hdr->b_l1hdr.b_acb->acb_zio_head;
5798
5799			ASSERT3P(head_zio, !=, NULL);
5800			if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) &&
5801			    priority == ZIO_PRIORITY_SYNC_READ) {
5802				/*
5803				 * This is a sync read that needs to wait for
5804				 * an in-flight async read. Request that the
5805				 * zio have its priority upgraded.
5806				 */
5807				zio_change_priority(head_zio, priority);
5808				DTRACE_PROBE1(arc__async__upgrade__sync,
5809				    arc_buf_hdr_t *, hdr);
5810				ARCSTAT_BUMP(arcstat_async_upgrade_sync);
5811			}
5812			if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) {
5813				arc_hdr_clear_flags(hdr,
5814				    ARC_FLAG_PREDICTIVE_PREFETCH);
5815			}
5816
5817			if (*arc_flags & ARC_FLAG_WAIT) {
5818				cv_wait(&hdr->b_l1hdr.b_cv, hash_lock);
5819				mutex_exit(hash_lock);
5820				goto top;
5821			}
5822			ASSERT(*arc_flags & ARC_FLAG_NOWAIT);
5823
5824			if (done) {
5825				arc_callback_t *acb = NULL;
5826
5827				acb = kmem_zalloc(sizeof (arc_callback_t),
5828				    KM_SLEEP);
5829				acb->acb_done = done;
5830				acb->acb_private = private;
5831				acb->acb_compressed = compressed_read;
5832				if (pio != NULL)
5833					acb->acb_zio_dummy = zio_null(pio,
5834					    spa, NULL, NULL, NULL, zio_flags);
5835
5836				ASSERT3P(acb->acb_done, !=, NULL);
5837				acb->acb_zio_head = head_zio;
5838				acb->acb_next = hdr->b_l1hdr.b_acb;
5839				hdr->b_l1hdr.b_acb = acb;
5840				mutex_exit(hash_lock);
5841				return (0);
5842			}
5843			mutex_exit(hash_lock);
5844			return (0);
5845		}
5846
5847		ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
5848		    hdr->b_l1hdr.b_state == arc_mfu);
5849
5850		if (done) {
5851			if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) {
5852				/*
5853				 * This is a demand read which does not have to
5854				 * wait for i/o because we did a predictive
5855				 * prefetch i/o for it, which has completed.
5856				 */
5857				DTRACE_PROBE1(
5858				    arc__demand__hit__predictive__prefetch,
5859				    arc_buf_hdr_t *, hdr);
5860				ARCSTAT_BUMP(
5861				    arcstat_demand_hit_predictive_prefetch);
5862				arc_hdr_clear_flags(hdr,
5863				    ARC_FLAG_PREDICTIVE_PREFETCH);
5864			}
5865
5866			if (hdr->b_flags & ARC_FLAG_PRESCIENT_PREFETCH) {
5867				ARCSTAT_BUMP(
5868                                    arcstat_demand_hit_prescient_prefetch);
5869				arc_hdr_clear_flags(hdr,
5870                                    ARC_FLAG_PRESCIENT_PREFETCH);
5871			}
5872
5873			ASSERT(!BP_IS_EMBEDDED(bp) || !BP_IS_HOLE(bp));
5874			/* Get a buf with the desired data in it. */
5875			rc = arc_buf_alloc_impl(hdr, private,
5876			   compressed_read, B_TRUE, &buf);
5877			if (rc != 0) {
5878				arc_buf_destroy(buf, private);
5879				buf = NULL;
5880			}
5881			ASSERT((zio_flags & ZIO_FLAG_SPECULATIVE) ||
5882                            rc == 0 || rc != ENOENT);
5883		} else if (*arc_flags & ARC_FLAG_PREFETCH &&
5884		    zfs_refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
5885			arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
5886		}
5887		DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
5888		arc_access(hdr, hash_lock);
5889		if (*arc_flags & ARC_FLAG_PRESCIENT_PREFETCH)
5890                        arc_hdr_set_flags(hdr, ARC_FLAG_PRESCIENT_PREFETCH);
5891		if (*arc_flags & ARC_FLAG_L2CACHE)
5892			arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
5893		mutex_exit(hash_lock);
5894		ARCSTAT_BUMP(arcstat_hits);
5895		ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
5896		    demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
5897		    data, metadata, hits);
5898
5899		if (done)
5900			done(NULL, zb, bp, buf, private);
5901	} else {
5902		uint64_t lsize = BP_GET_LSIZE(bp);
5903		uint64_t psize = BP_GET_PSIZE(bp);
5904		arc_callback_t *acb;
5905		vdev_t *vd = NULL;
5906		uint64_t addr = 0;
5907		boolean_t devw = B_FALSE;
5908		uint64_t size;
5909
5910		if (hdr == NULL) {
5911			/* this block is not in the cache */
5912			arc_buf_hdr_t *exists = NULL;
5913			arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp);
5914			hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize,
5915			    BP_GET_COMPRESS(bp), type);
5916
5917			if (!BP_IS_EMBEDDED(bp)) {
5918				hdr->b_dva = *BP_IDENTITY(bp);
5919				hdr->b_birth = BP_PHYSICAL_BIRTH(bp);
5920				exists = buf_hash_insert(hdr, &hash_lock);
5921			}
5922			if (exists != NULL) {
5923				/* somebody beat us to the hash insert */
5924				mutex_exit(hash_lock);
5925				buf_discard_identity(hdr);
5926				arc_hdr_destroy(hdr);
5927				goto top; /* restart the IO request */
5928			}
5929		} else {
5930			/*
5931			 * This block is in the ghost cache. If it was L2-only
5932			 * (and thus didn't have an L1 hdr), we realloc the
5933			 * header to add an L1 hdr.
5934			 */
5935			if (!HDR_HAS_L1HDR(hdr)) {
5936				hdr = arc_hdr_realloc(hdr, hdr_l2only_cache,
5937				    hdr_full_cache);
5938			}
5939			ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
5940			ASSERT(GHOST_STATE(hdr->b_l1hdr.b_state));
5941			ASSERT(!HDR_IO_IN_PROGRESS(hdr));
5942			ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
5943			ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
5944			ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
5945
5946			/*
5947			 * This is a delicate dance that we play here.
5948			 * This hdr is in the ghost list so we access it
5949			 * to move it out of the ghost list before we
5950			 * initiate the read. If it's a prefetch then
5951			 * it won't have a callback so we'll remove the
5952			 * reference that arc_buf_alloc_impl() created. We
5953			 * do this after we've called arc_access() to
5954			 * avoid hitting an assert in remove_reference().
5955			 */
5956			arc_adapt(arc_hdr_size(hdr), hdr->b_l1hdr.b_state);
5957			arc_access(hdr, hash_lock);
5958			arc_hdr_alloc_pabd(hdr, B_FALSE);
5959		}
5960		ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
5961		size = arc_hdr_size(hdr);
5962
5963		/*
5964		 * If compression is enabled on the hdr, then will do
5965		 * RAW I/O and will store the compressed data in the hdr's
5966		 * data block. Otherwise, the hdr's data block will contain
5967		 * the uncompressed data.
5968		 */
5969		if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF) {
5970			zio_flags |= ZIO_FLAG_RAW;
5971		}
5972
5973		if (*arc_flags & ARC_FLAG_PREFETCH)
5974			arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
5975		if (*arc_flags & ARC_FLAG_PRESCIENT_PREFETCH)
5976			arc_hdr_set_flags(hdr, ARC_FLAG_PRESCIENT_PREFETCH);
5977
5978		if (*arc_flags & ARC_FLAG_L2CACHE)
5979			arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
5980		if (BP_GET_LEVEL(bp) > 0)
5981			arc_hdr_set_flags(hdr, ARC_FLAG_INDIRECT);
5982		if (*arc_flags & ARC_FLAG_PREDICTIVE_PREFETCH)
5983			arc_hdr_set_flags(hdr, ARC_FLAG_PREDICTIVE_PREFETCH);
5984		ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state));
5985
5986		acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
5987		acb->acb_done = done;
5988		acb->acb_private = private;
5989		acb->acb_compressed = compressed_read;
5990
5991		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
5992		hdr->b_l1hdr.b_acb = acb;
5993		arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5994
5995		if (HDR_HAS_L2HDR(hdr) &&
5996		    (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) {
5997			devw = hdr->b_l2hdr.b_dev->l2ad_writing;
5998			addr = hdr->b_l2hdr.b_daddr;
5999			/*
6000			 * Lock out L2ARC device removal.
6001			 */
6002			if (vdev_is_dead(vd) ||
6003			    !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER))
6004				vd = NULL;
6005		}
6006
6007		/*
6008		 * We count both async reads and scrub IOs as asynchronous so
6009		 * that both can be upgraded in the event of a cache hit while
6010		 * the read IO is still in-flight.
6011		 */
6012		if (priority == ZIO_PRIORITY_ASYNC_READ ||
6013		    priority == ZIO_PRIORITY_SCRUB)
6014			arc_hdr_set_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);
6015		else
6016			arc_hdr_clear_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);
6017
6018		/*
6019		 * At this point, we have a level 1 cache miss.  Try again in
6020		 * L2ARC if possible.
6021		 */
6022		ASSERT3U(HDR_GET_LSIZE(hdr), ==, lsize);
6023
6024		DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp,
6025		    uint64_t, lsize, zbookmark_phys_t *, zb);
6026		ARCSTAT_BUMP(arcstat_misses);
6027		ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
6028		    demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
6029		    data, metadata, misses);
6030#ifdef _KERNEL
6031#ifdef RACCT
6032		if (racct_enable) {
6033			PROC_LOCK(curproc);
6034			racct_add_force(curproc, RACCT_READBPS, size);
6035			racct_add_force(curproc, RACCT_READIOPS, 1);
6036			PROC_UNLOCK(curproc);
6037		}
6038#endif /* RACCT */
6039		curthread->td_ru.ru_inblock++;
6040#endif
6041
6042		if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) {
6043			/*
6044			 * Read from the L2ARC if the following are true:
6045			 * 1. The L2ARC vdev was previously cached.
6046			 * 2. This buffer still has L2ARC metadata.
6047			 * 3. This buffer isn't currently writing to the L2ARC.
6048			 * 4. The L2ARC entry wasn't evicted, which may
6049			 *    also have invalidated the vdev.
6050			 * 5. This isn't prefetch and l2arc_noprefetch is set.
6051			 */
6052			if (HDR_HAS_L2HDR(hdr) &&
6053			    !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) &&
6054			    !(l2arc_noprefetch && HDR_PREFETCH(hdr))) {
6055				l2arc_read_callback_t *cb;
6056				abd_t *abd;
6057				uint64_t asize;
6058
6059				DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);
6060				ARCSTAT_BUMP(arcstat_l2_hits);
6061				atomic_inc_32(&hdr->b_l2hdr.b_hits);
6062
6063				cb = kmem_zalloc(sizeof (l2arc_read_callback_t),
6064				    KM_SLEEP);
6065				cb->l2rcb_hdr = hdr;
6066				cb->l2rcb_bp = *bp;
6067				cb->l2rcb_zb = *zb;
6068				cb->l2rcb_flags = zio_flags;
6069
6070				asize = vdev_psize_to_asize(vd, size);
6071				if (asize != size) {
6072					abd = abd_alloc_for_io(asize,
6073					    HDR_ISTYPE_METADATA(hdr));
6074					cb->l2rcb_abd = abd;
6075				} else {
6076					abd = hdr->b_l1hdr.b_pabd;
6077				}
6078
6079				ASSERT(addr >= VDEV_LABEL_START_SIZE &&
6080				    addr + asize <= vd->vdev_psize -
6081				    VDEV_LABEL_END_SIZE);
6082
6083				/*
6084				 * l2arc read.  The SCL_L2ARC lock will be
6085				 * released by l2arc_read_done().
6086				 * Issue a null zio if the underlying buffer
6087				 * was squashed to zero size by compression.
6088				 */
6089				ASSERT3U(HDR_GET_COMPRESS(hdr), !=,
6090				    ZIO_COMPRESS_EMPTY);
6091				rzio = zio_read_phys(pio, vd, addr,
6092				    asize, abd,
6093				    ZIO_CHECKSUM_OFF,
6094				    l2arc_read_done, cb, priority,
6095				    zio_flags | ZIO_FLAG_DONT_CACHE |
6096				    ZIO_FLAG_CANFAIL |
6097				    ZIO_FLAG_DONT_PROPAGATE |
6098				    ZIO_FLAG_DONT_RETRY, B_FALSE);
6099				acb->acb_zio_head = rzio;
6100
6101				if (hash_lock != NULL)
6102					mutex_exit(hash_lock);
6103
6104				DTRACE_PROBE2(l2arc__read, vdev_t *, vd,
6105				    zio_t *, rzio);
6106				ARCSTAT_INCR(arcstat_l2_read_bytes, size);
6107
6108				if (*arc_flags & ARC_FLAG_NOWAIT) {
6109					zio_nowait(rzio);
6110					return (0);
6111				}
6112
6113				ASSERT(*arc_flags & ARC_FLAG_WAIT);
6114				if (zio_wait(rzio) == 0)
6115					return (0);
6116
6117				/* l2arc read error; goto zio_read() */
6118				if (hash_lock != NULL)
6119					mutex_enter(hash_lock);
6120			} else {
6121				DTRACE_PROBE1(l2arc__miss,
6122				    arc_buf_hdr_t *, hdr);
6123				ARCSTAT_BUMP(arcstat_l2_misses);
6124				if (HDR_L2_WRITING(hdr))
6125					ARCSTAT_BUMP(arcstat_l2_rw_clash);
6126				spa_config_exit(spa, SCL_L2ARC, vd);
6127			}
6128		} else {
6129			if (vd != NULL)
6130				spa_config_exit(spa, SCL_L2ARC, vd);
6131			if (l2arc_ndev != 0) {
6132				DTRACE_PROBE1(l2arc__miss,
6133				    arc_buf_hdr_t *, hdr);
6134				ARCSTAT_BUMP(arcstat_l2_misses);
6135			}
6136		}
6137
6138		rzio = zio_read(pio, spa, bp, hdr->b_l1hdr.b_pabd, size,
6139		    arc_read_done, hdr, priority, zio_flags, zb);
6140		acb->acb_zio_head = rzio;
6141
6142		if (hash_lock != NULL)
6143			mutex_exit(hash_lock);
6144
6145		if (*arc_flags & ARC_FLAG_WAIT)
6146			return (zio_wait(rzio));
6147
6148		ASSERT(*arc_flags & ARC_FLAG_NOWAIT);
6149		zio_nowait(rzio);
6150	}
6151	return (0);
6152}
6153
6154arc_prune_t *
6155arc_add_prune_callback(arc_prune_func_t *func, void *private)
6156{
6157	arc_prune_t *p;
6158
6159	p = kmem_alloc(sizeof (*p), KM_SLEEP);
6160	p->p_pfunc = func;
6161	p->p_private = private;
6162	list_link_init(&p->p_node);
6163	zfs_refcount_create(&p->p_refcnt);
6164
6165	mutex_enter(&arc_prune_mtx);
6166	zfs_refcount_add(&p->p_refcnt, &arc_prune_list);
6167	list_insert_head(&arc_prune_list, p);
6168	mutex_exit(&arc_prune_mtx);
6169
6170	return (p);
6171}
6172
6173void
6174arc_remove_prune_callback(arc_prune_t *p)
6175{
6176	boolean_t wait = B_FALSE;
6177	mutex_enter(&arc_prune_mtx);
6178	list_remove(&arc_prune_list, p);
6179	if (zfs_refcount_remove(&p->p_refcnt, &arc_prune_list) > 0)
6180		wait = B_TRUE;
6181	mutex_exit(&arc_prune_mtx);
6182
6183	/* wait for arc_prune_task to finish */
6184	if (wait)
6185		taskq_wait(arc_prune_taskq);
6186	ASSERT0(zfs_refcount_count(&p->p_refcnt));
6187	zfs_refcount_destroy(&p->p_refcnt);
6188	kmem_free(p, sizeof (*p));
6189}
6190
6191/*
6192 * Notify the arc that a block was freed, and thus will never be used again.
6193 */
6194void
6195arc_freed(spa_t *spa, const blkptr_t *bp)
6196{
6197	arc_buf_hdr_t *hdr;
6198	kmutex_t *hash_lock;
6199	uint64_t guid = spa_load_guid(spa);
6200
6201	ASSERT(!BP_IS_EMBEDDED(bp));
6202
6203	hdr = buf_hash_find(guid, bp, &hash_lock);
6204	if (hdr == NULL)
6205		return;
6206
6207	/*
6208	 * We might be trying to free a block that is still doing I/O
6209	 * (i.e. prefetch) or has a reference (i.e. a dedup-ed,
6210	 * dmu_sync-ed block). If this block is being prefetched, then it
6211	 * would still have the ARC_FLAG_IO_IN_PROGRESS flag set on the hdr
6212	 * until the I/O completes. A block may also have a reference if it is
6213	 * part of a dedup-ed, dmu_synced write. The dmu_sync() function would
6214	 * have written the new block to its final resting place on disk but
6215	 * without the dedup flag set. This would have left the hdr in the MRU
6216	 * state and discoverable. When the txg finally syncs it detects that
6217	 * the block was overridden in open context and issues an override I/O.
6218	 * Since this is a dedup block, the override I/O will determine if the
6219	 * block is already in the DDT. If so, then it will replace the io_bp
6220	 * with the bp from the DDT and allow the I/O to finish. When the I/O
6221	 * reaches the done callback, dbuf_write_override_done, it will
6222	 * check to see if the io_bp and io_bp_override are identical.
6223	 * If they are not, then it indicates that the bp was replaced with
6224	 * the bp in the DDT and the override bp is freed. This allows
6225	 * us to arrive here with a reference on a block that is being
6226	 * freed. So if we have an I/O in progress, or a reference to
6227	 * this hdr, then we don't destroy the hdr.
6228	 */
6229	if (!HDR_HAS_L1HDR(hdr) || (!HDR_IO_IN_PROGRESS(hdr) &&
6230	    zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt))) {
6231		arc_change_state(arc_anon, hdr, hash_lock);
6232		arc_hdr_destroy(hdr);
6233		mutex_exit(hash_lock);
6234	} else {
6235		mutex_exit(hash_lock);
6236	}
6237
6238}
6239
6240/*
6241 * Release this buffer from the cache, making it an anonymous buffer.  This
6242 * must be done after a read and prior to modifying the buffer contents.
6243 * If the buffer has more than one reference, we must make
6244 * a new hdr for the buffer.
6245 */
6246void
6247arc_release(arc_buf_t *buf, void *tag)
6248{
6249	arc_buf_hdr_t *hdr = buf->b_hdr;
6250
6251	/*
6252	 * It would be nice to assert that if it's DMU metadata (level >
6253	 * 0 || it's the dnode file), then it must be syncing context.
6254	 * But we don't know that information at this level.
6255	 */
6256
6257	mutex_enter(&buf->b_evict_lock);
6258
6259	ASSERT(HDR_HAS_L1HDR(hdr));
6260
6261	/*
6262	 * We don't grab the hash lock prior to this check, because if
6263	 * the buffer's header is in the arc_anon state, it won't be
6264	 * linked into the hash table.
6265	 */
6266	if (hdr->b_l1hdr.b_state == arc_anon) {
6267		mutex_exit(&buf->b_evict_lock);
6268		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
6269		ASSERT(!HDR_IN_HASH_TABLE(hdr));
6270		ASSERT(!HDR_HAS_L2HDR(hdr));
6271		ASSERT(HDR_EMPTY(hdr));
6272		ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1);
6273		ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1);
6274		ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
6275
6276		hdr->b_l1hdr.b_arc_access = 0;
6277
6278		/*
6279		 * If the buf is being overridden then it may already
6280		 * have a hdr that is not empty.
6281		 */
6282		buf_discard_identity(hdr);
6283		arc_buf_thaw(buf);
6284
6285		return;
6286	}
6287
6288	kmutex_t *hash_lock = HDR_LOCK(hdr);
6289	mutex_enter(hash_lock);
6290
6291	/*
6292	 * This assignment is only valid as long as the hash_lock is
6293	 * held, we must be careful not to reference state or the
6294	 * b_state field after dropping the lock.
6295	 */
6296	arc_state_t *state = hdr->b_l1hdr.b_state;
6297	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
6298	ASSERT3P(state, !=, arc_anon);
6299
6300	/* this buffer is not on any list */
6301	ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), >, 0);
6302
6303	if (HDR_HAS_L2HDR(hdr)) {
6304		mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx);
6305
6306		/*
6307		 * We have to recheck this conditional again now that
6308		 * we're holding the l2ad_mtx to prevent a race with
6309		 * another thread which might be concurrently calling
6310		 * l2arc_evict(). In that case, l2arc_evict() might have
6311		 * destroyed the header's L2 portion as we were waiting
6312		 * to acquire the l2ad_mtx.
6313		 */
6314		if (HDR_HAS_L2HDR(hdr)) {
6315			l2arc_trim(hdr);
6316			arc_hdr_l2hdr_destroy(hdr);
6317		}
6318
6319		mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx);
6320	}
6321
6322	/*
6323	 * Do we have more than one buf?
6324	 */
6325	if (hdr->b_l1hdr.b_bufcnt > 1) {
6326		arc_buf_hdr_t *nhdr;
6327		uint64_t spa = hdr->b_spa;
6328		uint64_t psize = HDR_GET_PSIZE(hdr);
6329		uint64_t lsize = HDR_GET_LSIZE(hdr);
6330		enum zio_compress compress = HDR_GET_COMPRESS(hdr);
6331		arc_buf_contents_t type = arc_buf_type(hdr);
6332		VERIFY3U(hdr->b_type, ==, type);
6333
6334		ASSERT(hdr->b_l1hdr.b_buf != buf || buf->b_next != NULL);
6335		(void) remove_reference(hdr, hash_lock, tag);
6336
6337		if (arc_buf_is_shared(buf) && !ARC_BUF_COMPRESSED(buf)) {
6338			ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf);
6339			ASSERT(ARC_BUF_LAST(buf));
6340		}
6341
6342		/*
6343		 * Pull the data off of this hdr and attach it to
6344		 * a new anonymous hdr. Also find the last buffer
6345		 * in the hdr's buffer list.
6346		 */
6347		arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);
6348		ASSERT3P(lastbuf, !=, NULL);
6349
6350		/*
6351		 * If the current arc_buf_t and the hdr are sharing their data
6352		 * buffer, then we must stop sharing that block.
6353		 */
6354		if (arc_buf_is_shared(buf)) {
6355			VERIFY(!arc_buf_is_shared(lastbuf));
6356
6357			/*
6358			 * First, sever the block sharing relationship between
6359			 * buf and the arc_buf_hdr_t.
6360			 */
6361			arc_unshare_buf(hdr, buf);
6362
6363			/*
6364			 * Now we need to recreate the hdr's b_pabd. Since we
6365			 * have lastbuf handy, we try to share with it, but if
6366			 * we can't then we allocate a new b_pabd and copy the
6367			 * data from buf into it.
6368			 */
6369			if (arc_can_share(hdr, lastbuf)) {
6370				arc_share_buf(hdr, lastbuf);
6371			} else {
6372				arc_hdr_alloc_pabd(hdr, B_TRUE);
6373				abd_copy_from_buf(hdr->b_l1hdr.b_pabd,
6374				    buf->b_data, psize);
6375			}
6376			VERIFY3P(lastbuf->b_data, !=, NULL);
6377		} else if (HDR_SHARED_DATA(hdr)) {
6378			/*
6379			 * Uncompressed shared buffers are always at the end
6380			 * of the list. Compressed buffers don't have the
6381			 * same requirements. This makes it hard to
6382			 * simply assert that the lastbuf is shared so
6383			 * we rely on the hdr's compression flags to determine
6384			 * if we have a compressed, shared buffer.
6385			 */
6386			ASSERT(arc_buf_is_shared(lastbuf) ||
6387			    HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF);
6388			ASSERT(!ARC_BUF_SHARED(buf));
6389		}
6390		ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
6391		ASSERT3P(state, !=, arc_l2c_only);
6392
6393		(void) zfs_refcount_remove_many(&state->arcs_size,
6394		    arc_buf_size(buf), buf);
6395
6396		if (zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
6397			ASSERT3P(state, !=, arc_l2c_only);
6398			(void) zfs_refcount_remove_many(
6399			    &state->arcs_esize[type],
6400			    arc_buf_size(buf), buf);
6401		}
6402
6403		hdr->b_l1hdr.b_bufcnt -= 1;
6404		arc_cksum_verify(buf);
6405#ifdef illumos
6406		arc_buf_unwatch(buf);
6407#endif
6408
6409		mutex_exit(hash_lock);
6410
6411		/*
6412		 * Allocate a new hdr. The new hdr will contain a b_pabd
6413		 * buffer which will be freed in arc_write().
6414		 */
6415		nhdr = arc_hdr_alloc(spa, psize, lsize, compress, type);
6416		ASSERT3P(nhdr->b_l1hdr.b_buf, ==, NULL);
6417		ASSERT0(nhdr->b_l1hdr.b_bufcnt);
6418		ASSERT0(zfs_refcount_count(&nhdr->b_l1hdr.b_refcnt));
6419		VERIFY3U(nhdr->b_type, ==, type);
6420		ASSERT(!HDR_SHARED_DATA(nhdr));
6421
6422		nhdr->b_l1hdr.b_buf = buf;
6423		nhdr->b_l1hdr.b_bufcnt = 1;
6424		(void) zfs_refcount_add(&nhdr->b_l1hdr.b_refcnt, tag);
6425		buf->b_hdr = nhdr;
6426
6427		mutex_exit(&buf->b_evict_lock);
6428		(void) zfs_refcount_add_many(&arc_anon->arcs_size,
6429		    arc_buf_size(buf), buf);
6430	} else {
6431		mutex_exit(&buf->b_evict_lock);
6432		ASSERT(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt) == 1);
6433		/* protected by hash lock, or hdr is on arc_anon */
6434		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
6435		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
6436		arc_change_state(arc_anon, hdr, hash_lock);
6437		hdr->b_l1hdr.b_arc_access = 0;
6438		mutex_exit(hash_lock);
6439
6440		buf_discard_identity(hdr);
6441		arc_buf_thaw(buf);
6442	}
6443}
6444
6445int
6446arc_released(arc_buf_t *buf)
6447{
6448	int released;
6449
6450	mutex_enter(&buf->b_evict_lock);
6451	released = (buf->b_data != NULL &&
6452	    buf->b_hdr->b_l1hdr.b_state == arc_anon);
6453	mutex_exit(&buf->b_evict_lock);
6454	return (released);
6455}
6456
6457#ifdef ZFS_DEBUG
6458int
6459arc_referenced(arc_buf_t *buf)
6460{
6461	int referenced;
6462
6463	mutex_enter(&buf->b_evict_lock);
6464	referenced = (zfs_refcount_count(&buf->b_hdr->b_l1hdr.b_refcnt));
6465	mutex_exit(&buf->b_evict_lock);
6466	return (referenced);
6467}
6468#endif
6469
6470static void
6471arc_write_ready(zio_t *zio)
6472{
6473	arc_write_callback_t *callback = zio->io_private;
6474	arc_buf_t *buf = callback->awcb_buf;
6475	arc_buf_hdr_t *hdr = buf->b_hdr;
6476	uint64_t psize = BP_IS_HOLE(zio->io_bp) ? 0 : BP_GET_PSIZE(zio->io_bp);
6477
6478	ASSERT(HDR_HAS_L1HDR(hdr));
6479	ASSERT(!zfs_refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt));
6480	ASSERT(hdr->b_l1hdr.b_bufcnt > 0);
6481
6482	/*
6483	 * If we're reexecuting this zio because the pool suspended, then
6484	 * cleanup any state that was previously set the first time the
6485	 * callback was invoked.
6486	 */
6487	if (zio->io_flags & ZIO_FLAG_REEXECUTED) {
6488		arc_cksum_free(hdr);
6489#ifdef illumos
6490		arc_buf_unwatch(buf);
6491#endif
6492		if (hdr->b_l1hdr.b_pabd != NULL) {
6493			if (arc_buf_is_shared(buf)) {
6494				arc_unshare_buf(hdr, buf);
6495			} else {
6496				arc_hdr_free_pabd(hdr);
6497			}
6498		}
6499	}
6500	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
6501	ASSERT(!HDR_SHARED_DATA(hdr));
6502	ASSERT(!arc_buf_is_shared(buf));
6503
6504	callback->awcb_ready(zio, buf, callback->awcb_private);
6505
6506	if (HDR_IO_IN_PROGRESS(hdr))
6507		ASSERT(zio->io_flags & ZIO_FLAG_REEXECUTED);
6508
6509	arc_cksum_compute(buf);
6510	arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
6511
6512	enum zio_compress compress;
6513	if (BP_IS_HOLE(zio->io_bp) || BP_IS_EMBEDDED(zio->io_bp)) {
6514